diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2844f84..4541194 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,7 +16,23 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 22 + # aws/aws-emulator, for the tests that need real ECS shapes + # (test/launcher-ecs.test.js). The package is signed by AWS; check the + # signing key's fingerprint and the signature before installing it. + - name: Install aws-emulator + run: | + base=https://downloads.aws.com/aws-emulator/packages/aws-emulator/LTS/latest + curl -fsSLO "$base/aws-emulator_amd64.deb" + curl -fsSLO "$base/aws-emulator_amd64.deb.sig" + curl -fsSLo aws-publishing.asc https://downloads.aws.com/keys/gpg/current.asc + gpg --import aws-publishing.asc + gpg --fingerprint --with-colons ssm-publishing@amazon.com | grep -q '^fpr:::::::::98BCF336A3BE53B734E967C52E5BF49BD940F624:$' + gpg --verify aws-emulator_amd64.deb.sig aws-emulator_amd64.deb + sudo apt-get install -y ./aws-emulator_amd64.deb + aws-emulator -v # The unit tests and the repo conventions, then the template build, - # which fails at or over the 51,200-byte inline limit. + # which fails at or over the S3 deploy's 1 MB limit. - run: make test + env: + REQUIRE_AWS_EMULATOR: "1" - run: make build diff --git a/CODING_STANDARDS.md b/CODING_STANDARDS.md index 3f2bbc9..6142056 100644 --- a/CODING_STANDARDS.md +++ b/CODING_STANDARDS.md @@ -16,7 +16,7 @@ adoption variable `ZAE_CI_RUNNER`. - **No dependencies.** Each function is one CommonJS file under `functions/`, using only Node.js 22 built-ins and the `@aws-sdk` clients the runtime ships. The build inlines it into the template, so every byte counts against - the 51,200-byte limit. + the 1 MB limit. - **A `createHandler(deps)` factory** holds the logic and takes every side effect (AWS clients, `fetch`, `now`, `env`, `log`) as a dependency, so the tests run the real code with fakes. `exports.handler` only wires the real diff --git a/GLOSSARY.md b/GLOSSARY.md new file mode 100644 index 0000000..e853615 --- /dev/null +++ b/GLOSSARY.md @@ -0,0 +1,43 @@ +# ci-runner + +Single-use self-hosted GitHub Actions runners for the private repos of the `zeroae` org. + +## Language + +**CI runner**: +A self-hosted runner created for one queued job. It runs that job and is then gone. +_Avoid_: agent, worker, pool member + +**Capacity**: +The kind of compute a CI runner runs on: EC2 Spot, EC2 On-Demand, Fargate Spot or Fargate On-Demand. +_Avoid_: pool, warm pool (nothing is started before its job queues) + +**Capacity label**: +The `runs-on` label that picks a job's capacity: `ci-spot`, `ci-ondemand`, `ci-fargate-spot` or `ci-fargate`. + +**Size label**: +The `runs-on` label that picks a CI runner's vCPU count: `ci-cpu-2`, `ci-cpu-4`, `ci-cpu-8` or `ci-cpu-16`, with 4 GB of memory per vCPU. It has no default. +_Avoid_: instance type, xlarge, instance-size + +**Architecture label**: +The `runs-on` label that picks a CI runner's CPU architecture: `x64` or `arm64`. It has no default. + +**Shape**: +A CI runner's capacity, size and architecture together. Runners of the same shape are interchangeable. + +**Label contract**: +The labels a job must carry to be served: `self-hosted`, exactly one capacity label, exactly one size label and exactly one architecture label, plus `linux` if the job wants it. Nothing else is allowed. It holds on every capacity. + +**Swap**: +A CI runner taking a queued job other than the one it was launched for. GitHub gives a job to any idle runner whose labels cover the job's, so swaps between runners of the same shape are normal. + +**Fargate capacity**: +Fargate Spot or Fargate On-Demand, the two capacities with no Docker daemon. A job that picks one promises not to use Docker. +_Avoid_: Fargate pool + +**vCPU budget**: +The most vCPUs that the CI runners of one capacity may hold at once. A job that would go over it waits. +_Avoid_: "cap" in prose (that was the old runner count). The parameters that set the budgets are still named `*VcpuCap` (`SpotVcpuCap`, `OnDemandVcpuCap`, `FargateSpotVcpuCap`, `FargateVcpuCap`). + +**Spot reclaim**: +AWS taking back Spot capacity, EC2 or Fargate, while a job runs. The job is lost and is reported as one on its commit. diff --git a/Makefile b/Makefile index 538ccec..1bab1c6 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,10 @@ STACK ?= ci-runner -PROFILE ?= zeroae-code/AWSPowerUserAccess +PROFILE ?= zeroae-code/ZAEPowerUserAccess REGION ?= us-east-1 ORG ?= zeroae AWS = aws --profile $(PROFILE) --region $(REGION) +# The deploy bucket, made outside the stack (docs/runbook.md). Looked up only when used. +BUCKET ?= $(STACK)-deploy-$(shell $(AWS) sts get-caller-identity --query Account --output text) .PHONY: help test build validate deploy outputs hand-launch runners sweep canary-runs .DEFAULT_GOAL := help @@ -17,12 +19,14 @@ test: ## Run the unit tests (node --test, no npm deps) build: ## Inline functions/*.js into build/template.yaml and report its size node scripts/build-template.js -validate: build ## Ask CloudFormation to validate the built template - $(AWS) cloudformation validate-template --template-body file://build/template.yaml >/dev/null && echo valid +validate: build ## Ask CloudFormation to validate the built template (through the deploy bucket) + $(AWS) s3 cp --only-show-errors build/template.yaml s3://$(BUCKET)/$(STACK)/validate.yaml + $(AWS) cloudformation validate-template \ + --template-url https://$(BUCKET).s3.$(REGION).amazonaws.com/$(STACK)/validate.yaml >/dev/null && echo valid -deploy: test build ## Create or update the stack ($(STACK)) from one command +deploy: test build ## Create or update the stack ($(STACK)) through the deploy bucket $(AWS) cloudformation deploy --stack-name $(STACK) \ - --template-file build/template.yaml \ + --template-file build/template.yaml --s3-bucket $(BUCKET) --s3-prefix $(STACK) \ --capabilities CAPABILITY_NAMED_IAM --no-fail-on-empty-changeset @$(MAKE) --no-print-directory outputs @@ -30,8 +34,8 @@ outputs: ## Print the stack's outputs (setup URL, webhook URL, secret ARN) @$(AWS) cloudformation describe-stacks --stack-name $(STACK) \ --query 'Stacks[0].Outputs[].[OutputKey,OutputValue]' --output text -hand-launch: ## Launch one CI runner by hand for one ci-spot job (needs gh with admin:org) - STACK=$(STACK) PROFILE=$(PROFILE) REGION=$(REGION) scripts/hand-launch.sh +hand-launch: ## Launch one EC2 CI runner by hand of shape CAPACITY (ci-spot or ci-ondemand), SIZE and ARCH, all required, e.g. CAPACITY=ci-spot SIZE=ci-cpu-4 ARCH=x64 (needs gh with admin:org) + STACK=$(STACK) PROFILE=$(PROFILE) REGION=$(REGION) CAPACITY=$(CAPACITY) SIZE=$(SIZE) ARCH=$(ARCH) scripts/hand-launch.sh runners: ## List the stack's CI runner instances that are not terminated @$(AWS) ec2 describe-instances \ @@ -39,7 +43,7 @@ runners: ## List the stack's CI runner instances that are not terminated Name=instance-state-name,Values=pending,running,stopping,stopped,shutting-down \ --query 'Reservations[].Instances[].[InstanceId,State.Name,InstanceType,LaunchTime]' --output text -sweep: ## Run the sweeper now, and print what it terminated and removed +sweep: ## Run the sweeper now, and print what it terminated, stopped and removed @$(AWS) lambda invoke --function-name $(STACK)-sweeper /dev/stdout --query FunctionError --output text canary-runs: ## List the canary repo's recent workflow runs (CI_RUNNER_CANARY_REPO, from .env) diff --git a/README.md b/README.md index f3ea24c..17d041c 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,48 @@ # ci-runner -Single-use EC2 runners for the GitHub Actions jobs of zeroae's private repos. +Single-use EC2 and Fargate runners for the GitHub Actions jobs of zeroae's +private repos. -Each queued job labelled `ci-spot` or `ci-ondemand` gets a fresh `.xlarge` -EC2 instance. The instance runs that one job in `ghcr.io/zeroae/ci-runner:latest` +Each queued job labelled `ci-spot` or `ci-ondemand` gets a fresh EC2 +instance of the size and architecture it asks for. The instance runs that one job in `ghcr.io/zeroae/ci-runner:latest` (GitHub's runner image on Ubuntu 24.04, plus general tools) and then -terminates. The instance has no IAM role, so a job finds no AWS credentials -on it. Short jobs stay on GitHub-hosted runners. +terminates. Each job labelled `ci-fargate-spot` or `ci-fargate` gets a fresh +Fargate task instead, from the same image, which runs that one job and exits. +Neither has an IAM role, so a job finds no AWS credentials on it. Short jobs +stay on GitHub-hosted runners, or go to a Fargate capacity if they need no +Docker. | Label in `runs-on` | Capacity | For | |---|---|---| -| `ci-spot` | EC2 Spot, `.xlarge` (`m7i`/`m6i`/`m6a`/`m7a`) | long jobs: a job that runs well over 2 minutes, which is about how long a CI runner takes to come up | -| `ci-ondemand` | EC2 On-Demand, `.xlarge` | critical jobs: a job that, cut off half-way, leaves state a person must repair (`terraform apply`) | +| `ci-spot` | EC2 Spot (`m8i`/`m8a`/`m7i`/`m6i`/`m6a`/`m7a` for `x64`, `m7g`/`m6g`/`m8g` for `arm64`) | long jobs: a job that runs well over 2 minutes, which is about how long a CI runner takes to come up | +| `ci-ondemand` | EC2 On-Demand (the same families) | critical jobs: a job that, cut off half-way, leaves state a person must repair (`terraform apply`) | +| `ci-fargate-spot` | Fargate Spot, **no Docker** | Docker-free jobs that should start fast: about 30 s from queued to started | +| `ci-fargate` | Fargate On-Demand, **no Docker** | Docker-free jobs that must not be cut off half-way | | (none; `ubuntu-latest`) | GitHub-hosted | everything else | -At most 6 Spot and 2 On-Demand CI runners run at once, across all repos. A -job over the cap waits in a queue until one finishes. If a Spot reclaim kills +A **size label** picks the vCPUs, at 4 GB of memory per vCPU, and an +**architecture label** the CPU: + +| Size label | vCPUs / memory | Instance size | +|---|---|---| +| `ci-cpu-2` | 2 / 8 GB | `.large` | +| `ci-cpu-4` | 4 / 16 GB | `.xlarge` | +| `ci-cpu-8` | 8 / 32 GB | `.2xlarge` | +| `ci-cpu-16` | 16 / 64 GB | `.4xlarge` | + +On a Fargate capacity the size label sets the task's vCPUs and memory the +same way. + +The architecture label is `x64` (Intel and AMD) or `arm64` (Graviton). Both +architectures run the same image, built natively for each. Every job names +one size label and one architecture label: neither has a default, so a job +without them waits for a runner that never comes. A job that used to name only +its capacity got 4 vCPUs and `x64`; add `ci-cpu-4` and `x64` to keep that shape. + +The CI runners hold at most 24 Spot, 8 On-Demand, 48 Fargate Spot and 8 +Fargate On-Demand vCPUs at once, across all repos (six, two, twelve and two +`ci-cpu-4` runners). A job that would go over that waits in +a queue until enough finish. If an EC2 Spot reclaim kills a job, a failed check run named `ci-runner / ` appears on its commit. Nothing retries it automatically: a person re-runs it. @@ -35,18 +62,30 @@ it's in both. ```yaml jobs: tests: - runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON('["self-hosted","ci-spot"]') || 'ubuntu-latest' }} + runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON('["self-hosted","ci-spot","ci-cpu-8","x64"]') || 'ubuntu-latest' }} + build-arm: + runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON('["self-hosted","ci-spot","ci-cpu-4","arm64"]') || 'ubuntu-latest' }} apply: - runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON('["self-hosted","ci-ondemand"]') || 'ubuntu-latest' }} + runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON('["self-hosted","ci-ondemand","ci-cpu-2","x64"]') || 'ubuntu-latest' }} + lint: + runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON('["self-hosted","ci-fargate-spot","ci-cpu-2","arm64"]') || 'ubuntu-latest' }} + docs: + runs-on: ${{ vars.ZAE_CI_RUNNER && fromJSON('["self-hosted","ci-fargate","ci-cpu-4","x64"]') || 'ubuntu-latest' }} ``` + Each `runs-on` names the full shape: capacity, size and architecture. + A job on `ci-fargate-spot` or `ci-fargate` promises not to use Docker + (no `services:`, `container:`, Docker actions or `docker` commands): a + Fargate task has no Docker daemon + ([ADR-0001](docs/adr/0001-fargate-capacities-have-no-docker.md)). + While the variable is unset (the default), every job runs on `ubuntu-latest`, so the change can merge before the repo is switched on. Once the variable holds any value, the moved jobs run on our runners. To roll back, unset it. 3. Make the moved job's first step the repo's own toolset check: a script that names any missing tool and prints each tool's version. 4. Give each moved job a `timeout-minutes` under 120 (110 leaves room). The - sweeper terminates any instance 2 hours after launch, so a job without one + sweeper ends any instance or task 2 hours after launch, so a job without one dies with its machine instead of failing on a timeout that names it. 5. Move one job first, as a canary (usually the test job). Its queued-to-started time is the boot time, and that tells you which other jobs are long enough to @@ -54,10 +93,13 @@ it's in both. 6. Set the variable (`gh variable set ZAE_CI_RUNNER --body 1`) and watch the canary. -**The label contract:** `runs-on` must include `self-hosted` and exactly one of -`ci-spot` or `ci-ondemand`. The only other labels allowed are `linux` and `x64`. -A job with any other label is ignored, and it waits for a runner that never -comes. +**The label contract:** `runs-on` must include `self-hosted`, exactly one of +`ci-spot`, `ci-ondemand`, `ci-fargate-spot` or `ci-fargate`, exactly one size label (`ci-cpu-2`, `ci-cpu-4`, +`ci-cpu-8`, `ci-cpu-16`) and exactly one of `x64` or `arm64`. It may also +include `linux`. None of them has a default. A job missing a size or an +architecture label, or with any other label (`aarch64`, an unknown +`ci-cpu-*`, two size labels, both `x64` and `arm64`), is ignored, and it waits +for a runner that never comes. **What the job gets:** Ubuntu 24.04, the runner as user `runner` (passwordless sudo), Docker through the host's socket, the `docker compose` and `buildx` @@ -73,6 +115,26 @@ a bind mount from any of them works as it does on a GitHub-hosted runner, `:latest`, and the image is rebuilt weekly, for linux/amd64 and linux/arm64, each built and smoke-tested on its own native runner. +**On a Fargate capacity** the job gets the same image, user, tools, paths and +50 GB, with three differences: + +- **No Docker.** There is no daemon and no socket (the `docker` CLI is still + there, and fails). Picking `ci-fargate-spot` or `ci-fargate` is the job's + promise never to use Docker: no `services:`, no `container:`, no Docker + actions, no `docker build`. A job that breaks it fails at that step + ([ADR-0001](docs/adr/0001-fargate-capacities-have-no-docker.md)). +- **Older CPUs.** A Fargate task starts in about 30 s against EC2's 100 s, + but its CPUs are older: a 4-minute Rust build ran about 45% longer. So + Fargate wins for short jobs, and is even at about 4 minutes; for builds + much longer than that, EC2 is faster (the runbook's "Measured numbers"). +- **`nproc` over-reports.** Inside a task, `nproc` can show twice the task's + vCPUs. The runner caps cargo's jobs at the task's vCPUs (in + `~/.cargo/config.toml`, so `Swatinem/rust-cache` shares its cache with EC2) + and sets `MAKEFLAGS=-j` and `CI_RUNNER_VCPUS=`. A tool that sizes + itself by `nproc` some other way should be given `-j$CI_RUNNER_VCPUS`. + +`/tmp`, `$HOME` and `_work` are the container's own paths on its 50 GB disk. + **Deploy jobs** keep their AWS access as before (typically GitHub OIDC into the repo's own role). A trust policy that pins `runner_environment: github-hosted` must drop that condition before a deploy job moves. diff --git a/docs/adr/0001-fargate-capacities-have-no-docker.md b/docs/adr/0001-fargate-capacities-have-no-docker.md new file mode 100644 index 0000000..7ccbaf0 --- /dev/null +++ b/docs/adr/0001-fargate-capacities-have-no-docker.md @@ -0,0 +1,5 @@ +# Fargate capacities have no Docker daemon + +The two Fargate capacities, `ci-fargate` and `ci-fargate-spot`, run a job with no Docker daemon and no socket. Picking one is the job author's promise that the job never uses Docker: no `services:`, no `container:` jobs, no Docker actions, no `docker build`. A job that breaks the promise fails at that step, and nothing detects it beforehand. + +We accepted that because the jobs it serves (Rust builds, for example) need CPU, memory and disk, not containers. Fargate allows no privileged mode, so the only ways to give a job Docker there are rootless daemons or remote builders, which each add a moving part to every job. Jobs that need Docker stay on the EC2 capacities, `ci-spot` and `ci-ondemand`, which keep the host's daemon. diff --git a/docs/adr/0002-push-the-image-to-ecr-from-ci.md b/docs/adr/0002-push-the-image-to-ecr-from-ci.md new file mode 100644 index 0000000..e6dc1c5 --- /dev/null +++ b/docs/adr/0002-push-the-image-to-ecr-from-ci.md @@ -0,0 +1,7 @@ +# Push the runner image to ECR from CI, not through a pull-through cache + +Fargate pulls the whole image (about 820 MB) for every task and caches nothing, so start time depends on lazy loading (SOCI), and that works only from private ECR. The public `image.yml` job therefore pushes the image to an ECR repo that this stack owns, both architectures with a SOCI v2 index, through GitHub OIDC. The OIDC setup follows foresight's `github-deploy-zae.json`: the provider is created only when the account doesn't already have one. EC2 runners keep pulling from GHCR, because an instance with no instance profile can't pull from private ECR. + +## Considered Options + +- **ECR pull-through cache from ghcr.io.** CI wouldn't change at all. We rejected it for three reasons: ghcr.io upstream needs a GitHub token in Secrets Manager even for a public image, so someone must keep rotating it; a cached `:latest` can be up to 24 hours stale; and no AWS document says SOCI works through the cache, while a cached repo can't be pushed to, so we couldn't add an index ourselves. diff --git a/docs/adr/0003-tolerate-swaps-instead-of-run-labels.md b/docs/adr/0003-tolerate-swaps-instead-of-run-labels.md new file mode 100644 index 0000000..8f00c88 --- /dev/null +++ b/docs/adr/0003-tolerate-swaps-instead-of-run-labels.md @@ -0,0 +1,7 @@ +# Tolerate swaps instead of requiring a run label + +GitHub gives a queued job to any idle runner whose labels cover the job's, so a CI runner often runs a job other than the one it was launched for: a swap. We don't prevent swaps. The label contract requires a capacity, a size and an architecture label, so a swap only ever happens between runners of the same shape. The system repairs the bookkeeping instead. When the webhook gets `workflow_job` `in_progress`, it reads `runner_name`. If that runner was launched for another job, the webhook re-queues that other job, so it isn't stranded, and re-tags the runner's instance or task with the job it actually runs, so a Spot reclaim is reported on the right commit. + +## Considered Options + +- **A run label and a per-job label, as CodeBuild-hosted runners do** (`codebuild--${{ github.run_id }}-${{ github.run_attempt }}` plus a unique label per job). We rejected it: every consumer's `runs-on` grows two expressions, matrix jobs need a hand-written label each, and the system would still need to repair the swaps that get through. diff --git a/docs/runbook.md b/docs/runbook.md index 3c1f6bb..4f287c6 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -5,56 +5,147 @@ Lambdas are in [stack.md](stack.md), and the design is in [the spec](superpowers/specs/2026-11-12-ci-runner-design.md). Everything runs in `zeroae-code` / `us-east-1`, under the SSO profile -`zeroae-code/AWSPowerUserAccess`. The commands below use: +`zeroae-code/ZAEPowerUserAccess` (`make deploy`'s default `PROFILE`). Unlike +`AWSPowerUserAccess`, it allows `iam:UpdateAssumeRolePolicy`, which a change +to a role's trust needs. Its roles must be named `PowerUserPB*`, with the +boundary `arn:aws:iam::aws:policy/PowerUserAccess` (the stack's) or +`ZAEPowerUserBoundary`. The commands below use: ```bash -AWS="aws --profile zeroae-code/AWSPowerUserAccess --region us-east-1" +AWS="aws --profile zeroae-code/ZAEPowerUserAccess --region us-east-1" ``` You need `aws`, `gh`, `jq`, `curl` and `node`. For anything that touches the runner group, your `gh` login also needs the `admin:org` scope (`gh auth refresh -h github.com -s admin:org`). -The order the first time is: deploy the stack, create the App, create the -runner group, then turn repos on one at a time. +The order the first time is: create the deploy bucket, deploy the stack, +create the App, create the runner group, then turn repos on one at a time. ## Deploy and update the stack +`make deploy` uploads the template to the **deploy bucket**, +`ci-runner-deploy-`, and deploys from there. The bucket isn't part of the stack, so +it outlives a deleted stack and exists before the first deploy. Make it once +per account, private and encrypted: + +```bash +B=ci-runner-deploy-$($AWS sts get-caller-identity --query Account --output text) +$AWS s3api create-bucket --bucket "$B" +$AWS s3api put-public-access-block --bucket "$B" \ + --public-access-block-configuration BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true +$AWS s3api put-bucket-encryption --bucket "$B" \ + --server-side-encryption-configuration '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"}}]}' +``` + +(`create-bucket` takes no `--create-bucket-configuration` in `us-east-1`; in +any other region it needs `LocationConstraint=`.) Each deploy uploads +the template as `ci-runner/.template`; the objects are small and can be +deleted at any time. A stack named other than `ci-runner` uses +`-deploy-`, or pass `BUCKET=`. + ```bash make deploy # make test, make build, cloudformation deploy, then print the outputs -make outputs # SetupUrl, WebhookUrl, AppSecretArn, LaunchTemplateId, SubnetIds, ... +make outputs # SetupUrl, WebhookUrl, AppSecretArn, LaunchTemplateId, Arm64LaunchTemplateId, SubnetIds, ... ``` The first run creates the stack `ci-runner`, and later runs update it. A run with no changes does nothing. `make deploy` never passes parameters, so a new -stack takes the defaults (`SpotCap` 6, `OnDemandCap` 2, `AppName` -`zeroae-ci-runner`, `RunnerGroup` `ci-runner`, `LogRetentionDays` 30), and an -existing stack keeps whatever values it already has. +stack takes the defaults (`SpotVcpuCap` 24, `OnDemandVcpuCap` 8, +`FargateSpotVcpuCap` 48, `FargateVcpuCap` 8, `AppName` +`zeroae-ci-runner`, `RunnerGroup` `ci-runner`, `LogRetentionDays` 30, +`ExistingOidcProviderArn` empty), and an existing stack keeps whatever values +it already has. In `zeroae-code` GitHub's OIDC provider already exists, so +the first deploy that adds the image push must pass its ARN (see "Push the +image to ECR"); an empty value would try to create a second provider for the +same URL, and fail. To change a parameter, build, then deploy with an override: ```bash make build $AWS cloudformation deploy --stack-name ci-runner \ - --template-file build/template.yaml \ + --template-file build/template.yaml --s3-bucket "$B" --s3-prefix ci-runner \ --capabilities CAPABILITY_NAMED_IAM --no-fail-on-empty-changeset \ - --parameter-overrides SpotCap=6 OnDemandCap=2 + --parameter-overrides SpotVcpuCap=24 OnDemandVcpuCap=8 FargateSpotVcpuCap=48 FargateVcpuCap=8 ``` +`SpotVcpuCap` draws on the EC2 Spot vCPU quota (`L-34B43A08`), +`OnDemandVcpuCap` on the EC2 On-Demand one (`L-1216C47A`), +`FargateSpotVcpuCap` on the Fargate Spot vCPU quota (`L-36FBB829`) and +`FargateVcpuCap` on the Fargate On-Demand one (`L-3032A538`); each covers x64 +and arm64 runners together. + +A budget is **soft**: a burst can go over it. The launcher handles queue +messages in parallel, and each one counts only the runners already running, so +jobs that queue at the same moment all see the same headroom. Seen live on +2026-11-13: a consumer repo queued 8 Spot jobs at once, and all 8 launched, +putting 36 of 24 Spot vCPUs in use. The owner chose to keep it soft. Size the +EC2 quotas for the burst, not the budget. + An override **sticks**: every later `make deploy` keeps it. After lowering a -cap for a test, restore it explicitly, as above. To see the current values: +budget for a test, restore it explicitly, as above. To see the current values: ```bash $AWS cloudformation describe-stacks --stack-name ci-runner \ --query 'Stacks[0].Parameters' --output table ``` -Before raising a cap, check the EC2 quota still covers it (see "Measured +Before raising a budget, check its quota still covers it (see "Measured numbers"). -The built template must stay under 51,200 bytes (`make build` prints its -size and fails at the limit). There's little headroom left (see "Measured -numbers" and the traps table). +The built template must stay under 1 MB, the S3 deploy's limit (`make build` +prints its size and fails at the limit). + +## Push the image to ECR + +`image.yml` pushes the runner image, with a SOCI v2 index, to the stack's +ECR repo on every `main` build (see "The runner image in ECR" in +[stack.md](stack.md)). There is nothing to rotate: the push assumes +`PowerUserPB-ci-runner-image-push` through GitHub OIDC, and no AWS key or +GitHub token is stored anywhere. Set it up once: + +1. Deploy with the account's existing GitHub provider. In `zeroae-code` it + is owned by another stack (see the traps table): + + ```bash + OIDC=$($AWS iam list-open-id-connect-providers --output text \ + --query "OpenIDConnectProviderList[?ends_with(Arn, '/token.actions.githubusercontent.com')].Arn") + echo "$OIDC" # arn:aws:iam:::oidc-provider/token.actions.githubusercontent.com + make build + $AWS cloudformation deploy --stack-name ci-runner \ + --template-file build/template.yaml --s3-bucket "$B" --s3-prefix ci-runner \ + --capabilities CAPABILITY_NAMED_IAM --no-fail-on-empty-changeset \ + --parameter-overrides ExistingOidcProviderArn="$OIDC" + ``` + + The override sticks, so later `make deploy`s keep it. In an account with + no provider, leave it empty and the stack creates one. +2. Give `image.yml` the stack's outputs as repo variables: + + ```bash + out() { $AWS cloudformation describe-stacks --stack-name ci-runner \ + --query "Stacks[0].Outputs[?OutputKey=='$1'].OutputValue" --output text; } + gh variable set ECR_IMAGE --repo zeroae/ci-runner --body "$(out ImageRepositoryUri)" + gh variable set AWS_IMAGE_PUSH_ROLE_ARN --repo zeroae/ci-runner --body "$(out ImagePushRoleArn)" + ``` + + Until both are set, a `main` build skips the ECR push and publishes only + to GHCR. +3. Run a `main` build (`gh workflow run image.yml --repo zeroae/ci-runner --ref main`), + then check that `:latest` is an index with both architectures and a SOCI + index for each: + + ```bash + $AWS ecr batch-get-image --repository-name ci-runner --image-ids imageTag=latest \ + --accepted-media-types application/vnd.oci.image.index.v1+json \ + --query 'images[0].imageManifest' --output text \ + | jq '[.manifests[] | {artifactType, arch: .platform.architecture}]' + ``` + + It should list `amd64` and `arm64` twice each: once with no + `artifactType` (the image) and once with + `application/vnd.amazon.soci.index.v2+json`. ## Create the App and the runner group (the wizard) @@ -160,13 +251,14 @@ A repo is served when all of these hold: ## Read a lost job -When AWS reclaims a Spot instance, GitHub marks the job failed, with *The +When AWS reclaims a Spot instance or a Fargate Spot task, GitHub marks the job failed, with *The runner has received a shutdown signal*, *lost communication with the server*, or (seen in the live FIS test) just *Process completed with exit code 130* on the running step. On the same commit there is a failed check run from the App, named `ci-runner / ` and titled `Spot capacity reclaimed: re-run this job`. -Its summary names the instance and links the run. That is the whole story: -re-run the failed job. Nothing retries it automatically. +Its summary names the instance (or the task's id) and links the run. That is +the whole story: re-run the failed job. Nothing retries it automatically. +One check run per job: the notifier skips a job whose commit has it already. To list those check runs on a commit: @@ -176,9 +268,12 @@ gh api repos/zeroae//commits//check-runs \ ``` A job lost **without** that check run wasn't a Spot reclaim. Look in the -notifier's log for a line `reported : job `. An instance -from `make hand-launch` carries no job tags, so the notifier ignores it and -logs `ignored: is no job instance of this stack`. A job that sits +notifier's log for a line `reported : job ` +(or `already reported ...`). An instance from `make hand-launch`, or a task +run by hand, carries no job tags, so the notifier ignores it and logs +`ignored: is no job instance of this stack` (`... no job task +...` for a task). A task's stop code is in `describe-tasks` (below): +`SpotInterruption` is a reclaim. A job that sits *Waiting for a runner* is not lost: see the traps table, then the launcher's log. @@ -195,8 +290,10 @@ $AWS logs tail /aws/lambda/ci-runner-notifier --since 1d **The webhook** logs one JSON line per accepted job, `{"enqueued": "", "job_id": ..., "label": ...}`, and `{"refused": "", "visibility": ...}` -for a job from any repo but a private `zeroae` one (fence 3). Everything else it -ignores without a log line. +for a job from any repo but a private `zeroae` one (fence 3), and `{"swap": +"", "instance": "i-...", "took": {...}, "requeued": {...}}` (with +`"task": ""` for a Fargate runner) for each swap it repairs: the runner launched for the `requeued` job started the `took` +job instead. Everything else it ignores without a log line. **The launcher** logs one JSON line per job, carrying the job's fields plus one of: @@ -204,7 +301,7 @@ of: | Field | Meaning | |---|---| | `"launched": "i-..."`, `"runner"`, `"label"` | an instance is up for the job | -| `"deferred": "6 of 6 spot"` | at the cap; the job is back on the queue in 60 s | +| `"deferred": "20+8 of 24 spot vCPUs"` | the job's 8 vCPUs would take the capacity's 20 over its budget of 24; the job is back on the queue in 60 s | | `"dropped": "completed"` (or another status) | the job no longer waits (cancelled, or taken by another runner); nothing launched | | `"dropped": "job not found"` | GitHub answered 404 for the job | | `"dropped": "malformed message"` | a message it can't read, consumed | @@ -221,15 +318,30 @@ for q in JobQueueUrl JobDeadLetterQueueUrl; do done ``` +**A Fargate task's runner** logs to `/ecs/ci-runner-runner` (kept 7 days), +one stream per task, `runner/runner/`: `Listening for Jobs`, then +`Running job: `, or the idle exit's `ci-runner: no job within 900s, +stopping`. The launcher's `launched` line names the task's ARN; its last part +is the task id. + +```bash +$AWS logs tail /ecs/ci-runner-runner --since 1h --format short +$AWS ecs list-tasks --cluster ci-runner # the live tasks +$AWS ecs describe-tasks --cluster ci-runner --tasks --include TAGS \ + --query 'tasks[].[lastStatus,capacityProviderName,stopCode,stoppedReason]' +``` + **The sweeper** runs every 10 minutes and logs only what it did: `sweeper: terminated i-...` (an instance launched over 2 hours ago), +`sweeper: stopped arn:aws:ecs:...:task/ci-runner/` (a task launched over +2 hours ago; its `stoppedReason` says `ci-runner sweeper`), `sweeper: removed runner (id )` (an offline registration with no -instance), or that it skipped the registrations: `the GitHub App is not set up -yet`, or `no runner group named ci-runner yet`. +instance or task), or that it skipped the registrations: `the GitHub App is +not set up yet`, or `no runner group named ci-runner yet`. ```bash make runners # the stack's instances that are not terminated -make sweep # run the sweeper now; prints {"terminated": [...], "removed": [...]} +make sweep # run the sweeper now; prints {"terminated": [...], "stopped": [...], "removed": [...]} ``` ## Re-register the App @@ -278,6 +390,37 @@ on GitHub-hosted runners in the meantime. done ``` +## Run a task by hand + +To test the Fargate path without the launcher (the idle exit, say), mint a +JIT config as `scripts/hand-launch.sh` does and run one task of the stack's +task definition. Its labels should match no queued job, so nothing takes it; +`IDLE_SECONDS` in the override shortens its idle exit, for this task only: + +```bash +out() { $AWS cloudformation describe-stacks --stack-name ci-runner \ + --query "Stacks[0].Outputs[?OutputKey=='$1'].OutputValue" --output text; } +GROUP_ID=$(gh api orgs/zeroae/actions/runner-groups --paginate \ + --jq '.runner_groups[] | select(.name == "ci-runner") | .id') +NAME=ci-runner-hand-$(date -u +%Y%m%d%H%M%S) +JIT=$(gh api -X POST orgs/zeroae/actions/runners/generate-jitconfig -f name="$NAME" \ + -F runner_group_id="$GROUP_ID" -f work_folder=_work \ + -f 'labels[]=self-hosted' -f 'labels[]=hand-test' --jq .encoded_jit_config) +NET=$(jq -nc --arg sg "$(out SecurityGroupId)" --arg s "$(out SubnetIds)" \ + '{awsvpcConfiguration: {subnets: ($s | split(",")), securityGroups: [$sg], assignPublicIp: "ENABLED"}}') +$AWS ecs run-task --cluster ci-runner --task-definition "$(out TaskDefinitionArn)" \ + --capacity-provider-strategy capacityProvider=FARGATE_SPOT,weight=1 --network-configuration "$NET" \ + --overrides "$(jq -nc --arg jit "$JIT" '{containerOverrides: [{name: "runner", + command: ["./run.sh", "--jitconfig", $jit], environment: [{name: "IDLE_SECONDS", value: "120"}]}]}')" \ + --tags key=ci-runner:stack,value=ci-runner key=ci-runner:runner,value="$NAME" \ + --query 'tasks[0].taskArn' --output text +``` + +The task stops about 2 minutes after `Listening for Jobs`. Its registration +stays, offline, until the sweeper removes it. A hand-run task that never +stops is stopped by the sweeper 2 hours after launch, as any task in the +cluster is. + ## Measured numbers Measured on the canary repo when the stack first went live (2026-11-12). @@ -288,18 +431,80 @@ types. moving when it runs well over 2 minutes. The first job after a deploy took about 3½ minutes, while the stack's functions were still cold. A hand launch (`make hand-launch`) took about 97 s to pick up its job. -- **EC2 quotas** in `us-east-1`: 256 Spot and 384 On-Demand vCPUs. At 4 vCPUs - per `.xlarge`, the default caps use 24 Spot and 8 On-Demand vCPUs. Other - workloads in the account share the quotas, so check them before raising a - cap: +- **Fargate Spot against EC2 Spot** (the #28 spike, 2026-11-14, with + `scripts/prototype-fargate-spike.sh` on the `prototype/fargate-spot` + branch). One job, `cargo build --locked -p ruff` (ruff 0.17.5, debug, + cold: no cache), at 4 vCPU / 16 GB / 50 GB, the Fargate task running the + ECR image with its SOCI v2 index and `./run.sh --jitconfig` as its command. + The EC2 runners were hand-launched (`hand-launch.sh`, the launcher's + templates and fleet) because other repos' jobs held the stack's Spot budget + at the time: + + | | Fargate x64 | EC2 x64 | Fargate arm64 | EC2 arm64 | + |---|---|---|---|---| + | CPU | Xeon 8259CL (Cascade Lake) | `m6a.xlarge` (EPYC 7R13) | Graviton3 | `m8g.xlarge` (Graviton4) | + | Queued to started | **27 s** | 102 s | **30 s** | 92 s | + | `RunTask` to `pullStartedAt` (provisioning) | 19.3 s | | 20.6 s | | + | `pullStartedAt` to `pullStoppedAt` (820 MB image) | 2.8 s | | 2.6 s | | + | Task started to `Listening for Jobs` | 4 s | | 4 s | | + | Build step | 248 s | 171 s | 168 s | 115 s | + | Queued to completed | 316 s | 306 s | 227 s | 230 s | + | Billed time | 321 s | ~316 s | 234 s | ~234 s | + | Price per hour | $0.0879 | $0.0700 + $0.0055 EBS | $0.0710 | $0.0926 + $0.0055 EBS | + | Cost per job | $0.0078 | $0.0066 | $0.0046 | $0.0064 | + + A first run (ripgrep, a 22 s build) gave Fargate 22 s and 57 s queued to + started: in the 57 s, the arm64 runner was listening 33 s before GitHub + gave it the job. The pull takes about 3 s, so SOCI lazy-loads; the + provisioning before it (13 to 21 s) is the bulk of a Fargate start. A + task reaches `STOPPED` 24 to 29 s after its job completes, and is billed + until then. Fargate's CPUs are + older than the EC2 families' (one x64 task got a Sapphire Rapids 8488C, the + other a Cascade Lake), so the same build ran about 45% longer on both + architectures. Inside 3 of the 4 tasks, `nproc` said 8 and `free` 30 GB: + the task's limits aren't what a job that sizes itself by `nproc` sees. + Fargate prices: 4 vCPU and 16 GB at the Fargate Spot rates, plus 30 GB of + ephemeral storage over the free 20 GB; EC2: the Spot price of that type and + AZ at launch, plus 50 GB of gp3. An EC2 instance's end is estimated at 15 s + after its job, since a self-shutdown leaves no timestamp. + +- **vCPU quotas** in `us-east-1`, read 2026-11-13. Each budget draws on one + quota, shared with every other workload in the account: + + | Quota | Code | Value | AWS default | Budget default | Account peak, 14 days | + |---|---|---|---|---|---| + | EC2 Spot (All Standard Spot Instance Requests) | `L-34B43A08` | 256 | 5 | 24 | 32 | + | EC2 On-Demand (Running On-Demand Standard instances) | `L-1216C47A` | 384 | 5 | 8 | 4 | + | Fargate Spot vCPU resource count | `L-36FBB829` | 140 | 6 | 24 | 0 | + | Fargate On-Demand vCPU resource count | `L-3032A538` | 140 | 6 | 8 | 0 | + + All four cover the default budgets with room to spare, so no increase was + requested. The peak is the `AWS/Usage` `ResourceCount` maximum, and it + includes this stack's own runners. Check the quota, and the peak, before + raising a budget: ```bash - $AWS service-quotas get-service-quota --service-code ec2 --quota-code L-34B43A08 # Spot - $AWS service-quotas get-service-quota --service-code ec2 --quota-code L-1216C47A # On-Demand + $AWS service-quotas get-service-quota --service-code ec2 --quota-code L-34B43A08 # EC2 Spot + $AWS service-quotas get-service-quota --service-code ec2 --quota-code L-1216C47A # EC2 On-Demand + $AWS service-quotas get-service-quota --service-code fargate --quota-code L-36FBB829 # Fargate Spot + $AWS service-quotas get-service-quota --service-code fargate --quota-code L-3032A538 # Fargate On-Demand + $AWS cloudwatch get-metric-statistics --namespace AWS/Usage --metric-name ResourceCount \ + --dimensions Name=Service,Value=Fargate Name=Type,Value=Resource Name=Resource,Value=vCPU \ + Name=Class,Value=Standard/Spot \ + --start-time "$(date -u -d '-14 days' +%FT%TZ)" --end-time "$(date -u +%FT%TZ)" \ + --period 86400 --statistics Maximum --query 'max(Datapoints[].Maximum)' ``` -- **Template size:** 49,908 of 51,200 bytes (97%). A new function, or a big - addition to one, will need the S3-bucket deploy. + For the other three peaks, use `Service` `EC2` and `Class` + `Standard/OnDemand` as fits. + + A Fargate quota at its AWS default of 6 vCPUs is below its budget: with + the default budgets, a job then fails to launch instead of waiting (see the + traps table). + +- **Template size:** 55,871 bytes, 5% of the S3 deploy's 1 MB limit. It's + already over the 51,200 bytes a template can be sent inline, so it deploys + only through the deploy bucket. ## Traps @@ -309,16 +514,36 @@ types. | `permission denied` on `/var/run/docker.sock` | the host's `docker` group isn't gid 123 | | `setup-python` fails, or picks up a broken Python | `AGENT_TOOLSDIRECTORY` isn't `/opt/hostedtoolcache` | | A deploy waits 60 minutes, then fails on the state lock | an `apply` ran on Spot and was reclaimed; it must be `ci-ondemand` | -| A job sits "Waiting for a runner" | the cap is reached, the repo isn't in the group, the App isn't installed on it, or the repo is public. Also check the label contract: `self-hosted`, exactly one of `ci-spot`/`ci-ondemand`, and otherwise only `linux`/`x64`. The launcher's log tells the first case (`deferred`) apart from the rest (no line at all). A job that queued before the repo was turned on stays waiting after the fix: cancel and re-run it | +| A job sits "Waiting for a runner" | its capacity's vCPU budget is full, the repo isn't in the group, the App isn't installed on it, or the repo is public. Also check the label contract: `self-hosted`, exactly one of `ci-spot`/`ci-ondemand`/`ci-fargate-spot`/`ci-fargate`, exactly one size label (`ci-cpu-2`/`-4`/`-8`/`-16`), exactly one of `x64`/`arm64`, and otherwise only `linux`. The launcher's log tells the first case (`deferred`) apart from the rest (no line at all). A job that queued before the repo was turned on stays waiting after the fix: cancel and re-run it | +| A job that ran on our runners before now sits "Waiting for a runner", and the webhook's log shows no `enqueued` line for it | missing size or architecture label: its `runs-on` names only its capacity (`self-hosted`, `ci-spot`). There are no defaults; add a size and an architecture label (`ci-cpu-4` and `x64` keep the old shape), then cancel and re-run the job | +| An `arm64` runner's instance comes up and terminates without taking its job, and its console shows `no matching manifest for linux/arm64` | `ghcr.io/zeroae/ci-runner:latest` has no arm64 entry yet: the image workflow hasn't pushed a multi-arch `:latest` since it began building arm64. Run the image workflow on `main`, then re-run the job | +| A job's runner was launched for another job (`$RUNNER_NAME` ends in another job id), and that other job sits "Waiting for a runner" | a swap. The webhook repairs it on the job's `in_progress` (its log says `swap`), and the launcher then launches a runner for the other job. No `swap` line: check the webhook's log for errors (its role needs `ec2:DescribeInstances` and `ec2:CreateTags`, and for a task `ecs:ListTasks`, `ecs:DescribeTasks` and `ecs:TagResource`); cancel and re-run the waiting job | +| A Fargate Spot job was lost (its task's `stopCode` is `SpotInterruption`) and its commit has no `ci-runner / ` check run | the notifier logged `ignored: is no job task of this stack` for every change of the stop: it found no job tags on the task. ECS keeps a stopped task's tags readable through `DescribeTasks` (measured live), so check the task's tags (`describe-tasks --include TAGS`, within an hour of the stop; `list-tags-for-resource` refuses a stopped task) and the notifier's role. Re-run the job | | A job fails "The runner has received a shutdown signal", "lost communication" or "exit code 130", next to a failed `ci-runner / ` check | Spot reclaim: re-run the job | -| A cap lowered for a test is still low after the next `make deploy` | `cloudformation deploy` keeps a parameter's previous value unless it's overridden; restore it with `--parameter-overrides SpotCap=6 OnDemandCap=2` | +| A budget lowered for a test is still low after the next `make deploy` | `cloudformation deploy` keeps a parameter's previous value unless it's overridden; restore it with `--parameter-overrides SpotVcpuCap=24 OnDemandVcpuCap=8 FargateSpotVcpuCap=48 FargateVcpuCap=8`. Each draws on its own vCPU quota (see "Deploy and update the stack") | +| A job on `ci-fargate-spot` or `ci-fargate` fails at a Docker step: `failed to connect to the docker API at unix:///var/run/docker.sock; check if the path is correct and if the daemon is running`, or a `services:`/`container:` job or Docker action fails to start | Docker used on Fargate. A Fargate capacity has no Docker daemon, by design (ADR-0001): move the job to `ci-spot` or `ci-ondemand`, or drop its Docker use | +| The launcher logs `no task launched:` with a reason about a vCPU limit, and the job's message keeps coming back until the dead-letter queue | a Fargate quota at its default (6 vCPUs, `L-36FBB829` Spot, `L-3032A538` On-Demand) or below the capacity's budget. Request an increase (Service Quotas, service `fargate`) or lower `FargateSpotVcpuCap`/`FargateVcpuCap` under the quota; then re-run the job. The quota counts every Fargate task in the account, not only ours | +| A Fargate job's build is slow, or runs out of memory, with many more compiler processes than the size's vCPUs | `nproc` over-reports inside a task. The job's environment sets `MAKEFLAGS=-j` and `CI_RUNNER_VCPUS=`, and the entrypoint caps cargo with `/home/runner/.cargo/config.toml` (its log says `cargo jobs = `, or why not). A tool that reads `nproc` any other way needs `-j` from `CI_RUNNER_VCPUS` | +| A Fargate job never hits the Rust cache an EC2 job saved (`Swatinem/rust-cache` restores nothing, and the two save under different keys) | an environment variable starting `CARGO`, `CC`, `CFLAGS`, `CXX`, `CMAKE` or `RUST` differs between them: rust-cache hashes them all into its key. The stack sets none on a task (cargo's jobs go in `~/.cargo/config.toml`, not `CARGO_BUILD_JOBS`); if the task has `CARGO_BUILD_JOBS`, the launcher predates that, so deploy. Otherwise look for one the job sets only on Fargate | +| The launcher logs `not authorized to perform: iam:PassRole on resource: ...runner-exec because no permissions boundary allows the iam:PassRole action` for every Fargate job | the launcher's role has the `PowerUserAccess` boundary, which allows no `iam:PassRole`. It must have `ZAEPowerUserBoundary` (see "The Fargate capacities" in stack.md); in an account without that policy, the Fargate capacities can't launch | +| A Fargate task logs `ci-runner: no job within ...s, stopping` but keeps running | the image's idle exit predates its session-wide stop: run the image workflow on `main`. Stop the task by hand meanwhile (`$AWS ecs stop-task --cluster ci-runner --task `) | +| A Fargate task stops at once, its `stoppedReason` naming `ci-runner-idle-exit` (no such file) | the ECR image's `:latest` predates the idle-exit wrapper: run the image workflow on `main`, then re-run the job | +| A Fargate task sits `PROVISIONING`, then stops `TaskFailedToStart` with `CannotPullContainerError` | the execution role can't pull from the stack's ECR repo, or `:latest` isn't there (see "Push the image to ECR") | | The launcher logs `no instance launched`, with an authorization error from `CreateFleet` | `CreateFleet`'s check on the instance, volumes and network interfaces it creates carries no `ec2:LaunchTemplate` key, so a policy that adds that condition to them denies the launch. Grant those to `CreateFleet` without the condition; `RunInstances` keeps it | | The wizard stops with `runner group not created` | Enter at its **y/N** prompt means no. Re-run it, and type `y` at that prompt | | Offline runners pile up in the `ci-runner` group | a runner stopped by the 15-minute idle exit stays registered, offline. The sweeper removes it on its next run, once it's 10 minutes past the stamp in its name (`make sweep` to run it now). If they stay, the sweeper's log says why | -| `make build` fails at or over 51,200 bytes | the largest template `cloudformation deploy` sends without an S3 bucket. Trim the inlined code, or move the deploy to an S3 bucket (`--s3-bucket`, 1 MB limit) | +| `make deploy` or `make validate` fails `NoSuchBucket`, or `AccessDenied` on the upload | the deploy bucket `ci-runner-deploy-` doesn't exist in this account yet; make it (see "Deploy and update the stack") | +| A deploy without `--s3-bucket` fails `Templates with a size greater than 51,200 bytes must be deployed via an S3 Bucket` | the template is over the inline limit; deploy with `make deploy`, or pass `--s3-bucket` as in "Deploy and update the stack" | +| `make build` fails at or over 1,048,576 bytes | the largest template CloudFormation accepts from S3. Trim the inlined code | | The setup URL answers 410 `the GitHub App is already set up` | that's by design once the secret holds a key; see "Re-register the App" | | The setup page fails on GitHub: the App's name is taken | App names are global on GitHub; deploy with another `AppName` | | The webhook answers 503 | the secret holds no App yet (or was emptied): run the wizard | | After re-registering, every delivery gets 401, or the launcher still can't get a token | a warm webhook or launcher still holds the old App's secret or key; restart both (step 5 of "Re-register the App") | | After an update, the App's key is gone from the secret | the template's `GenerateSecretString` changed, so CloudFormation generated a new value. Never edit it; re-register the App | | A deploy job moved to our runners can't assume its AWS role | the role's trust policy pins `runner_environment: github-hosted`; drop that condition | +| A deploy fails creating `GitHubOidcProvider`: the provider already exists | the account already has GitHub's OIDC provider, and an account holds one per URL. Deploy with `ExistingOidcProviderArn=` (see "Push the image to ECR") | +| `image.yml`'s ECR push fails `Not authorized to perform sts:AssumeRoleWithWebIdentity`, after it worked before | in `zeroae-code`, GitHub's OIDC provider is the resource `GitHubOIDCProvider` of the stack **`zael-github-e2e`**, not ours. If that stack was deleted, the provider went with it. Create the provider again once with `zeroae-code/ZAEFederation` (outside any stack; `ZAEPowerUserAccess` can't), then deploy with its ARN as `ExistingOidcProviderArn`. Asking that stack to retain the provider (`DeletionPolicy: Retain`) avoids this | +| `image.yml`'s ECR push fails `Not authorized to perform sts:AssumeRoleWithWebIdentity` on a branch, tag or pull request | by design: the push role trusts only `repo:zeroae@11018406/ci-runner@1455021495:ref:refs/heads/main` (the immutable subject). Branch runs push only to the job's local registry | +| `image.yml`'s ECR push fails `Not authorized to perform sts:AssumeRoleWithWebIdentity` on `main`, the first time | the trust's `sub` doesn't match the token's. This repo uses GitHub's immutable subject (`repo:@/@:...`); compare the role's trust with `gh api repos/zeroae/ci-runner/actions/oidc/customization/sub` | +| `image.yml` on `main` publishes to GHCR but skips both ECR steps | the repo variables `AWS_IMAGE_PUSH_ROLE_ARN` and `ECR_IMAGE` aren't set; set them from the stack's outputs (see "Push the image to ECR") | +| A deploy fails updating `ImagePushRole`'s trust with an access-denied `iam:UpdateAssumeRolePolicy` | it was deployed under `AWSPowerUserAccess`, which lacks that action; deploy under `zeroae-code/ZAEPowerUserAccess`, the default `PROFILE` | diff --git a/docs/stack.md b/docs/stack.md index d28e9d5..9e4690a 100644 --- a/docs/stack.md +++ b/docs/stack.md @@ -37,6 +37,10 @@ scripts/build-template.js template.yaml -> build/template.yaml (git-ignored) - its own role `PowerUserPB-${AWS::StackName}-`, with `PermissionsBoundary: arn:aws:iam::aws:policy/PowerUserAccess` and an inline policy of the same name granting only what that function needs. + The one exception: a role that passes a role (`iam:PassRole`) takes the + account's `ZAEPowerUserBoundary` instead, since `PowerUserAccess` + allows no `iam:PassRole` (see the launcher's boundary in "Launching for + queued jobs"). `test/conventions.test.js` checks both. 4. A Function URL with `AuthType: NONE` needs two permissions: `lambda:InvokeFunctionUrl` (`FunctionUrlAuthType: NONE`) and `lambda:InvokeFunction` (`InvokedViaFunctionUrl: true`). @@ -45,17 +49,22 @@ scripts/build-template.js template.yaml -> build/template.yaml (git-ignored) - `make test`: `node --test test/*.test.js`. - `make build`: writes `build/template.yaml` and prints its size against the - **51,200-byte** limit (the largest template `aws cloudformation deploy` sends - without an S3 bucket). The build fails at or over it, and - `test/build.test.js` checks it too. To save room, the build drops whole-line - `//` comments from the inlined code and whole-line `#` comments from the - template (outside block scalars), so comment the source freely. If it fills - up again, the next step is deploying through an S3 bucket (`--s3-bucket`, - 1 MB limit). + **1 MB** (1,048,576-byte) limit, the largest template CloudFormation accepts + from S3. The build fails at or over it, and `test/build.test.js` checks it + too. The build drops whole-line `//` comments from the inlined code and + whole-line `#` comments from the template (outside block scalars), so + comment the source freely. - `make deploy`: test, build, then `cloudformation deploy` with - `CAPABILITY_NAMED_IAM`, then print the outputs. It creates the stack the - first time and updates it after. Override `STACK`, `PROFILE`, `REGION` on the - command line if needed. + `CAPABILITY_NAMED_IAM` and `--s3-bucket $(BUCKET) --s3-prefix $(STACK)`, then + print the outputs. It uploads the template to the deploy bucket and deploys + from there, since a template sent inline must be under 51,200 bytes. It + creates the stack the first time and updates it after. The bucket is made + outside the stack, once per account (see "Deploy and update the stack" in + the runbook); `BUCKET` defaults to `$(STACK)-deploy-`, looked up + with `sts get-caller-identity`. Override `STACK`, `PROFILE`, `REGION`, + `BUCKET` on the command line if needed. +- `make validate`: build, upload the template to the deploy bucket as + `$(STACK)/validate.yaml`, and validate it from there. ## The setup function @@ -77,18 +86,27 @@ region's first three AZs (`MapPublicIpOnLaunch`), one route table with a default route to an internet gateway, no NAT, no flow logs, no endpoints. The security group `RunnerSecurityGroup` has no inbound rules. -`RunnerLaunchTemplate` holds everything a CI runner instance shares: the AL2023 -image through `resolve:ssm:` (resolved at each launch), 50 GB encrypted gp3 +`RunnerLaunchTemplate` (x64) and `Arm64RunnerLaunchTemplate` (arm64) hold +everything a CI runner instance shares, and differ only in the image: the AL2023 +image for the architecture through `resolve:ssm:` (resolved at each launch: +`al2023-ami-kernel-default-x86_64` or `al2023-ami-kernel-default-arm64`), 50 GB encrypted gp3 deleted on termination, IMDSv2 required with hop limit 1, shutdown behaviour `terminate`, the tag `ci-runner:stack=` on instance and volume. It has **no instance profile, no instance type and no user-data**. -A launch is an EC2 Fleet of type `instant` for one instance: overrides for -`m7i`/`m6i`/`m6a`/`m7a` `.xlarge` in each of the three subnets, Spot -`capacity-optimized` for `ci-spot`. `CreateFleet` can't override user-data, so +A launch is an EC2 Fleet of type `instant` for one instance, from the job's +architecture's launch template: overrides for `m8i`/`m8a`/`m7i`/`m6i`/`m6a`/`m7a` (x64) or +`m7g`/`m6g`/`m8g` (arm64) at the job's size (`.large`, `.xlarge`, `.2xlarge` or +`.4xlarge` for 2, 4, 8 or 16 vCPUs) in each of the three subnets, Spot +`capacity-optimized` for `ci-spot`. Every family is offered at every size in +all three of the stack's AZs (`use1-az1`, `use1-az2`, `use1-az4`; checked +2026-11-13 with `aws ec2 describe-instance-type-offerings --location-type +availability-zone`), so no family/AZ pair is left out. Check again before +adding a family or moving the subnets. `CreateFleet` can't override user-data, so each launch first adds a launch template version (source `$Default`, only `UserData`), launches from it, and deletes that version again. The outputs -`LaunchTemplateId`, `SubnetIds` and `SecurityGroupId` are what a launch needs. +`LaunchTemplateId`, `Arm64LaunchTemplateId`, `SubnetIds` and `SecurityGroupId` +are what a launch needs. ## The instance's user-data @@ -118,9 +136,16 @@ sweeper (or GitHub, for ephemeral runners) removes it. ## Launching one by hand `make hand-launch` (`scripts/hand-launch.sh`) mints an org JIT config for the -`ci-runner` group with labels `self-hosted`, `ci-spot` through the owner's -`gh`, which needs `admin:org`. It then renders the user-data and launches one -instance as above. It prints the instance id. `IDLE_SECONDS=120` shortens the +`ci-runner` group with labels `self-hosted`, `CAPACITY`, `SIZE` and `ARCH` +(all required; none has a default) through the owner's `gh`, which needs +`admin:org`. It then renders the user-data and launches one instance of that +shape as above, with the launcher's fleet options: Spot `capacity-optimized` +for `CAPACITY=ci-spot`, On-Demand `lowest-price` for `ci-ondemand`. It is +tagged with its shape (`make hand-launch CAPACITY=ci-spot SIZE=ci-cpu-8 +ARCH=x64` launches an x64 Spot `.2xlarge`; `ARCH=arm64` a Graviton from the +arm64 launch template). It refuses `ci-fargate-spot` and `ci-fargate`, which +run a task, not an instance: the runbook's "Run a task by hand" does that. +It prints the instance id. `IDLE_SECONDS=120` shortens the idle exit for a test. `make runners` lists the stack's instances that are not terminated. @@ -134,23 +159,31 @@ terminated. 2. Describes the instances tagged `ci-runner:stack=`, and terminates those `pending`, `running`, `stopping` or `stopped` and launched over **2 hours** ago. -3. Removes each runner in the group that is `offline`, not busy, and has no - instance (see below). A GitHub failure doesn't stop step 2, but fails the - run. - -Runners are listed before instances are described, so a runner whose -instance launches between the two is still seen with its instance. +3. Lists the cluster's tasks whose desired status is `RUNNING` (not yet told + to stop), with their tags, and stops (`StopTask`) those created over + **2 hours** ago. Every task in the stack's cluster counts, a hand-run one + too. +4. Removes each runner in the group that is `offline`, not busy, and has no + instance or task (see below). A GitHub failure doesn't stop steps 2 and 3, + but fails the run; an ECS failure fails it after step 2, before any + registration is removed. + +Runners are listed before instances and tasks are, so a runner whose +machine launches in between is still seen with it. The run returns +`{terminated, stopped, removed}`. Its role allows `GetSecretValue` on the secret, `ec2:DescribeInstances`, and `ec2:TerminateInstances` only on instances whose `aws:ResourceTag/ci-runner:stack` is the stack's name. A `TerminateInstances` naming an untagged instance, or -another stack's, is denied. +another stack's, is denied. On ECS it allows `ecs:ListTasks` with the +condition `ecs:cluster` = the cluster, and `ecs:DescribeTasks`/`ecs:StopTask` +only on the cluster's tasks (`task//*`). ### How a runner finds its instance Whatever launches a runner (`scripts/hand-launch.sh`, the launcher) must: -- tag the instance `ci-runner:runner=`, the name it gave +- tag the instance or task `ci-runner:runner=`, the name it gave `generate-jitconfig`. It may also tag `ci-runner:runner-id=`; either tag matching keeps the registration. - put a UTC `YYYYmmddHHMMSS` timestamp of the mint in the name as its own @@ -160,7 +193,10 @@ Whatever launches a runner (`scripts/hand-launch.sh`, the launcher) must: grace. Every instance not yet `terminated` counts, so a runner whose instance is -shutting down is removed on a later run. +shutting down is removed on a later run. A task counts until it is told to +stop: the sweeper lists only tasks whose desired status is `RUNNING`, so a +runner whose task the sweeper stops is removed on the next run. (ECS itself +keeps a stopped task's tags readable through `DescribeTasks`.) ## Launching for queued jobs: webhook, queue, launcher @@ -172,15 +208,57 @@ App's webhook points at), for each delivery: 2. Ignores (`200`) any event but `workflow_job`. 3. **Fence 3:** refuses (`403`, logged) unless `repository.visibility` is `private` and the owner is `GITHUB_ORG`. -4. Ignores (`200`) actions other than `queued`, and jobs outside the label - contract: `self-hosted`, exactly one of `ci-spot`/`ci-ondemand`, and - otherwise only `linux`/`x64` (case-insensitive). -5. Enqueues `{repo, job_id, run_id, head_sha, job_name, label, labels, - installation_id}` (`label` is the capacity label, lower case) and answers - `202`. - -Its role allows only `GetSecretValue` on the secret and `SendMessage` on the -queue. +4. On `in_progress`, repairs a swap (below) and answers. +5. Ignores (`200`) actions other than `queued`, and jobs outside the label + contract (case-insensitive): `self-hosted`, exactly one of + `ci-spot`/`ci-ondemand`/`ci-fargate-spot`/`ci-fargate`, exactly one size label (`ci-cpu-2`, `-4`, `-8`, + `-16`), exactly one of `x64`/`arm64`, and optionally `linux`. None has a + default: a missing size or architecture label, an unknown `ci-cpu-*`, or + both `x64` and `arm64`, are outside it. +6. Enqueues `{repo, job_id, run_id, head_sha, job_name, label, vcpus, arch, + labels, installation_id}` (`label` is the capacity label, lower case; + `vcpus` is 2, 4, 8 or 16; `arch` is `x64` or `arm64`) and answers `202`. + +**Swap repair** (ADR-0003). GitHub gives a queued job to any idle runner whose +labels cover it, so a runner launched for job Y may take job X. Left alone, +X's own message is later dropped as not queued, and Y never gets a runner. +On `workflow_job` `in_progress` the webhook reads `runner_name`: + +- a name that isn't `${stack}-<14-digit stamp>-` (a hand launch, a + GitHub-hosted runner, another stack), or whose job id is the event's, is + ignored (`200`) with no AWS call; +- otherwise it describes the stack's `pending`/`running` instance tagged + `ci-runner:runner=`, and if there is none, the cluster's task tagged + so among those whose desired status is `RUNNING` (`ListTasks`, then + `DescribeTasks` with their tags). No instance or task, or a + `ci-runner:job-id` tag already equal to the event's job (a repeated + delivery), is ignored too; +- otherwise it **re-queues** the machine's job, rebuilt from its tags + (`repo`, `job-id`, `run-id`, `head-sha`, `job-name`, `label`, `vcpus`, + `arch`, `labels`; the event's installation id), so the launcher's "still + queued?" check launches it a runner or drops it. A task's `labels` are + split on spaces, an instance's on commas. A task's `job-name` is the + sanitized one (see "The Fargate capacities"); only tags read it, and the + notifier fetches the real name. An instance launched before those tags + existed re-queues as `ci-cpu-4`/`x64` with `self-hosted` and its capacity + label; +- then **re-tags** the instance (`CreateTags`) or task (`TagResource`) with + the event's job (`ci-runner:repo`, `job-id`, `run-id`, `head-sha`, + `job-name`), so the notifier reports a reclaim on the commit of the job it + really ran. A task's values are sanitized as the launcher's are. It logs + `{"swap": , "instance" or "task", "took": {repo, job_id}, + "requeued": {repo, job_id}}`, answering `202`. + +It queues before it tags: if the tag fails, a redelivery still sees the old +tags and repairs again. Re-queueing a job that another runner already took is +harmless: the launcher drops it. + +Its role allows `GetSecretValue` on the secret, `SendMessage` on the queue, +`ec2:DescribeInstances`, and `ec2:CreateTags` only on instances tagged +`ci-runner:stack=` and only for the five job tag keys. On ECS: +`ecs:ListTasks` with the condition `ecs:cluster` = the cluster, and +`ecs:DescribeTasks` and `ecs:TagResource` only on the cluster's tasks, the +latter only for the five job tag keys (`aws:TagKeys`). **The queue** `${stack}-jobs` has a 120 s visibility timeout. A message that fails 5 receives goes to `${stack}-jobs-dlq` (kept 14 days). The outputs are @@ -189,72 +267,283 @@ fails 5 receives goes to `${stack}-jobs-dlq` (kept 14 days). The outputs are **launcher** (`functions/launcher.js`, fed one message at a time, reporting failures per message), for each job: -1. Drops a malformed message, or one whose repo isn't in `GITHUB_ORG`. +1. Drops a malformed message, or one whose repo isn't in `GITHUB_ORG`, or + whose `vcpus` or `arch` isn't served. A message with neither (enqueued + before the size labels) gets 4 vCPUs and `x64`. 2. Signs an App JWT and gets an installation token for the job's installation. The token is cached until 5 minutes before it expires. 3. `GET /repos/{repo}/actions/jobs/{job_id}`: anything but `queued` (or a `404`) is dropped, and the message is consumed. -4. **The cap:** counts the stack's instances (`ci-runner:stack` tag) that are - `pending`, `running`, `stopping` or `stopped`, split by capacity: an - instance with `InstanceLifecycle: spot` is Spot, any other is On-Demand. - At the job's cap (`SpotCap`, default 6; `OnDemandCap`, default 2) it - sends the message to the queue again with a 60 s delay and consumes the - original, so waiting doesn't count towards the dead-letter queue. Nothing - is minted. It's a soft cap: messages checked at the same moment can - overshoot it. A shutting-down instance no longer counts. +4. **The vCPU budget:** sums the vCPUs of the stack's instances + (`ci-runner:stack` tag) that are `pending`, `running`, `stopping` or + `stopped` on the job's capacity, both architectures together, from their + shape tags (`ci-runner:capacity`, `ci-runner:vcpus`). An instance launched + before those tags counts as 4 vCPUs, Spot if it has + `InstanceLifecycle: spot` and On-Demand otherwise. If the job's vCPUs + would take the sum over the capacity's budget (`SpotVcpuCap`, default 24; + `OnDemandVcpuCap`, default 8), it sends the message to the queue again with + a 60 s delay and consumes the original, so waiting doesn't count towards + the dead-letter queue. Nothing is minted. A job that brings the sum exactly + to the budget launches. It's a soft budget: messages checked at the same + moment can overshoot it. A shutting-down instance no longer counts. + A Fargate job counts the cluster's tasks instead (see "The Fargate + capacities"). 5. Looks up the `RunnerGroup` parameter's group id by name (cached), and mints a JIT config with the job's labels. The runner is named `${stack}--`, as the sweeper requires. -6. Renders the `#cloud-config` and launches one instance as in "Launching one +6. For a Fargate job, runs one task (see "The Fargate capacities"). Otherwise + renders the `#cloud-config` and launches one instance as in "Launching one by hand": Spot `capacity-optimized` for `ci-spot`, On-Demand `lowest-price` - for `ci-ondemand`. Tags: `ci-runner:runner`, `ci-runner:runner-id`, - `ci-runner:label`, `ci-runner:repo`, `ci-runner:job-id`, `ci-runner:run-id`, - `ci-runner:head-sha` and `ci-runner:job-name`, plus the launch template's - `ci-runner:stack`. -7. If no instance launches, it removes the registration it just minted and - fails the message, which comes back 120 s later. + for `ci-ondemand`, at the job's size, from the launch template and the + families of the job's architecture. Tags: `ci-runner:runner`, + `ci-runner:runner-id`, `ci-runner:label`, `ci-runner:repo`, + `ci-runner:job-id`, `ci-runner:run-id`, `ci-runner:head-sha`, + `ci-runner:job-name`, `ci-runner:labels` (the job's labels, comma-joined, + for the swap repair), and the shape: `ci-runner:capacity` (the capacity + label), `ci-runner:vcpus` and `ci-runner:arch`; plus the launch template's + `ci-runner:stack`. The `launched` log line carries `label`, `vcpus` and + `arch`. +7. If no instance or task launches, it removes the registration it just + minted and fails the message, which comes back 120 s later. There is no de-duplication on job id. A duplicate delivery launches a second runner at worst, and the idle exit and the sweeper remove it. `cloudformation deploy` keeps a parameter's previous value unless it's -overridden, so after lowering a cap for a test, restore it explicitly -(`--parameter-overrides SpotCap=6 OnDemandCap=2`). +overridden, so after lowering a budget for a test, restore it explicitly +(`--parameter-overrides SpotVcpuCap=24 OnDemandVcpuCap=8`). The launcher's role allows `GetSecretValue`, the queue's receive, delete and -send, `ec2:DescribeInstances` (for the cap), -`Create`/`DeleteLaunchTemplateVersions` on the launch template, and -`CreateFleet`/`RunInstances` pinned to the launch template, the three subnets +send, `ec2:DescribeInstances` (for the budgets), +`Create`/`DeleteLaunchTemplateVersions` on the two launch templates, and +`CreateFleet`/`RunInstances` pinned to the two launch templates, the three subnets and the security group. It also allows `CreateTags` only while creating (`ec2:CreateAction`), and `ssm:GetParameters` on the public AL2023 parameters, -which the launch template's `resolve:ssm:` image needs. It has no -`iam:PassRole`. `CreateFleet`'s check on the instances, volumes and network +which the launch template's `resolve:ssm:` image needs. For Fargate it +allows `ecs:RunTask` on the two task definitions in the stack's cluster +(`ecs:cluster`), `iam:PassRole` on the execution role only and only to +`ecs-tasks.amazonaws.com`, `ecs:TagResource` only while creating +(`ecs:CreateAction: RunTask`), and `ecs:ListTasks`/`ecs:DescribeTasks` on the +cluster (for the budgets). It has no other `iam:PassRole`. All four conditions +hold live (checked 2026-11-14 with one job on each Fargate capacity). + +Its permissions boundary is the account's `ZAEPowerUserBoundary`, not +`PowerUserAccess` like every other role: `RunTask` passes the task +definition's execution role, and `PowerUserAccess` allows no `iam:PassRole` +at all. `ZAEPowerUserBoundary` is `PowerUserAccess` plus `iam:PassRole` on +`PowerUserPB*` roles; the policy above narrows that to the execution role. +`test/conventions.test.js` requires that boundary on a role that passes one, +and `PowerUserAccess` on every other. `CreateFleet`'s check on the instances, volumes and network interfaces it creates carries no launch-template key, so those resources are granted to `CreateFleet` without one. `RunInstances` keeps the condition. -## Reporting a Spot reclaim: the notifier +## The Fargate capacities + +`ci-fargate-spot` (Fargate Spot) and `ci-fargate` (Fargate On-Demand) run a +job in a Fargate task: no instance to boot, no host to clean up, and **no +Docker daemon** (ADR-0001). The template's Fargate resources: + +- **`RunnerCluster`**, named after the stack, with the `FARGATE` and + `FARGATE_SPOT` capacity providers and no default strategy. +- **`RunnerTaskDefinition`** (x64, family `${stack}-runner`) and + **`Arm64RunnerTaskDefinition`** (arm64, `${stack}-runner-arm64`): two, + because `RunTask` can't override the platform. They differ only in + `CpuArchitecture`: `awsvpc`, 4 vCPU / 16 GB (each launch overrides both), + 50 GB of ephemeral storage, one essential container `runner` from + `${ImageRepositoryUri}:latest` (ECR, lazy-loaded through its SOCI index), + entrypoint `/usr/local/bin/ci-runner-idle-exit`, `InitProcessEnabled`, and + `awslogs` to `RunnerTaskLogGroup` with the stream prefix `runner`. They have + **no task role**, so a job finds no AWS credentials: no + `AWS_CONTAINER_CREDENTIALS_RELATIVE_URI`, and the instance metadata service + isn't reachable from Fargate. The task metadata endpoint + (`ECS_CONTAINER_METADATA_URI_V4`) is there, and holds no credentials. +- **`RunnerExecutionRole`**, `PowerUserPB-${stack}-runner-exec`, which ECS + uses to start a task and the job never sees: `ecr:GetAuthorizationToken`, + the pull calls on `ImageRepository` only, and writing to the log group. +- **`RunnerTaskLogGroup`**, `/ecs/${stack}-runner`, kept 7 days: the runner's + output (`Listening for Jobs`, `Running job`), one stream per task + (`runner/runner/`). + +The outputs `ClusterName`, `TaskDefinitionArn` and `Arm64TaskDefinitionArn` +are what a hand-run task needs. + +**The launch.** For a Fargate job the launcher calls `RunTask` with: + +- the capacity provider (`FARGATE_SPOT` or `FARGATE`), weight 1, and the + task definition of the job's architecture; +- task-level `cpu` and `memory` for the size (`ci-cpu-8`: `8192` and + `32768`, so 4 GB per vCPU as on EC2); +- a container override: the command `./run.sh --jitconfig `, and the + environment below. The JIT config is readable to `ecs:DescribeTasks`, the + same exposure as an instance's user-data; +- `assignPublicIp: ENABLED` in the stack's subnets and security group. arm64 + leaves out any subnet in `use1-az3`, where Fargate has no arm64 (the + template passes each subnet's AZ id as `SUBNET_AZ_IDS`; the stack's subnets + are in `use1-az1`, `-az2` and `-az4` today, so nothing is left out); +- the instances' `ci-runner:*` tags, plus `ci-runner:stack`. ECS allows only + letters, digits, spaces and `_ . : / = + - @` in a tag, so on a task + `ci-runner:labels` is joined by spaces, not commas, and any other character + (a job name's brackets or commas) becomes `_`. The webhook's re-tag + sanitizes the same way, with a copy of the launcher's `ECS_TAG_CHARS` and + `ecsTag` (bundling one costs more bytes than the copy); + `test/conventions.test.js` fails if the copies differ. + +The launch is logged as `launched: `. A `RunTask` that returns no +task fails the message with `no task launched: : `. + +**The environment.** Inside a task, `nproc` can report the host's CPUs (8 +on a 4-vCPU task in 3 of the spike's 4 tasks) and there is no cgroup +`cpu.max`, so tools that size their parallelism by it oversubscribe. The +container override sets `MAKEFLAGS=-j` (which also reaches CMake's Makefile +generator) and `CI_RUNNER_VCPUS=` to the task's vCPUs; the runner's +environment reaches every step. From `CI_RUNNER_VCPUS` the image's entrypoint +writes `/home/runner/.cargo/config.toml` with `[build] jobs = ` (an existing +file is left alone, and logged). Cargo reads `.cargo/config.toml` from every +ancestor of its working directory, so it applies to every checkout under +`/home/runner/_work`, whatever `CARGO_HOME` a job sets. + +No `CARGO_*` or `CMAKE_*` variable is set, on purpose: `Swatinem/rust-cache` +hashes every variable starting `CARGO`, `CC`, `CFLAGS`, `CXX`, `CMAKE` or +`RUST` into its cache key +([`src/config.ts`](https://github.com/Swatinem/rust-cache/blob/c8037a86dde8214a8310e9f54a8fc61ab362aa31/src/config.ts#L111)), +so a `CARGO_BUILD_JOBS` (as the first version set) gave Fargate jobs their own +key, and they never hit the cache an EC2 job saved. Its hash of +`.cargo/config.toml` files looks only inside the workspace, so the home one +doesn't change the key. `IDLE_SECONDS` is passed on too, when the launcher has +it (a test). + +**The idle exit.** `image/ci-runner-idle-exit` runs its arguments in their own +session, copying their output to a file, and stops them if that output has no +`Running job` line after `IDLE_SECONDS` (default 900). A stop is a TERM to every +process in the session (`pkill -s`), then a KILL after `IDLE_GRACE_SECONDS` +(default 30, as `docker stop -t 30` gives an EC2 runner). The session matters: +`run-helper.sh` and `Runner.Listener` run in their own process group, so a TERM +to `run.sh`'s group alone misses them and the task never stops (seen live). +With the whole session signalled, the runner exits at once, status 143. It passes a TERM or INT on (a `StopTask`), and exits with the +runner's status, so the essential container exits and the task stops. The +image sets no `ENTRYPOINT`, so EC2's host script, which runs `./run.sh` with +its own idle exit, is unchanged; only the task definitions use the wrapper. +`image/smoke.sh` checks it with fake runners and a 2 s timeout. For a test, +`IDLE_SECONDS` can be set per task in a container override, as in the +runbook's "Run a task by hand". + +**The budgets.** `FargateSpotVcpuCap` (default 48) and `FargateVcpuCap` +(default 8) budget the two capacities, both architectures together, as the +EC2 budgets do. Fargate Spot's default is 48, double the others: at 24, one +consumer's 9-job matrix at `ci-cpu-8` ran in waves and spent most of its +queued time on budget waits, and the Fargate Spot quota (140) has room. The launcher lists the cluster's tasks not yet `STOPPED`: +those whose desired status is `RUNNING` (provisioning, pending or running), +and those told to stop (desired `STOPPED`) whose last status isn't `STOPPED` +yet, since a deprovisioning task still holds its vCPUs. It sums their +`ci-runner:vcpus` on the job's capacity (`ci-runner:capacity`). A task with no +vCPU tag counts as 4. EC2 instances and tasks never count towards each +other's budgets. + +**A task's lifecycle** is an instance's: the sweeper stops it 2 hours after +launch and keeps its runner's registration while it lives, the swap repair +re-tags it, and the notifier reports a Fargate Spot reclaim (below). -`functions/notifier.js` runs on every **EC2 Spot Instance Interruption -Warning** in the region (the rule `${stack}-notifier`; an event pattern can't -match on tags). For each warning it: +## Reporting a Spot reclaim: the notifier -1. Describes the instance, and ignores it (logged) unless it carries +`functions/notifier.js` runs on two rules: + +- `${stack}-notifier`: every **EC2 Spot Instance Interruption Warning** in + the region (an event pattern can't match on tags); +- `${stack}-notifier-tasks`: every **ECS Task State Change** of the stack's + cluster with `stopCode: SpotInterruption`. A reclaimed task's changes all + carry it, from the warning (desired `STOPPED`, still `RUNNING`, about 2 + minutes before the stop) to `STOPPED`. The notifier reads the tags with + `DescribeTasks`, which still returns them after the stop: measured live on + 2026-11-14, a task's tags stayed readable from `StopTask` through + `STOPPED` and minutes after, while `ListTagsForResource` refused the task + (`The specified task is stopped`) from the moment it was told to stop. + So every change can be read, and the check-run lookup keeps it to one + report per job (aws-emulator, unlike AWS, drops the tags at the stop). + +For each event it: + +1. Describes the instance, or the task (`DescribeTasks` with its tags; the + event carries none), and ignores it (logged) unless it carries `ci-runner:stack=`, `ci-runner:head-sha`, `ci-runner:job-id` and a `ci-runner:repo` in `GITHUB_ORG`: the launcher's tags. An instance from - `make hand-launch` has no job tags and is ignored. + `make hand-launch` or a task run by hand has no job tags and is ignored, + as is a task whose tags are gone. 2. Signs an App JWT, finds the repo's installation (`GET /repos/{repo}/installation`) and gets its token. -3. Creates a check run on the head sha: name `ci-runner / ` (or +3. For a task, reads the job's real name from + `GET /repos/{repo}/actions/jobs/{job_id}` (the tag has `_` for brackets and + commas), falling back to the tag. +4. Skips the job if the commit already has its reclaim check run (same name, + `external_id` the job id, the title below), so a task's later changes and a + repeated event report nothing more. +5. Creates a check run on the head sha: name `ci-runner / ` (or `job ` with no name), `completed`, conclusion `failure`, title `Spot capacity reclaimed: re-run this job`, a summary naming the job and - the instance and linking the run, and `details_url` the job's page. + the instance or task and linking the run, and `details_url` the job's page. + An EC2 reclaim and a Fargate one give the same check run, but for the + summary's machine. A GitHub failure fails the invocation, and EventBridge's async invoke retries -it. A duplicate event creates a second, identical check run. +it. Two events handled at the same moment can still report twice. The launcher, the notifier and the sweeper each carry their own copy of `appJwt()` and `gh()`, because bundling a shared module costs more template bytes than the copies. `test/conventions.test.js` fails if the copies differ, so change all three at once. -Its role allows `GetSecretValue` on the secret and `ec2:DescribeInstances`. +Its role allows `GetSecretValue` on the secret, `ec2:DescribeInstances`, and +`ecs:DescribeTasks` only on the cluster's tasks. + +## The runner image in ECR + +Fargate lazy-loads an image only from private ECR, through a SOCI index +(ADR-0002), so `image.yml` pushes the image to the stack's own ECR repo as +well as to GHCR. EC2 runners keep pulling `ghcr.io/zeroae/ci-runner:latest`: +an instance has no instance profile, so it can't pull from private ECR. + +- **`ImageRepository`**, named after the stack (`ci-runner`). Its lifecycle + rule keeps the last 4 **tagged** images. Each build tags one image index, + under `:latest` and `:-`, so that's the last 4 builds. The + index's children (each architecture's image manifest and SOCI index) are + untagged, and no rule touches them: an untagged rule could expire the + children of an index still in use. `EmptyOnDelete` lets a stack delete + remove it, images and all. +- **`GitHubOidcProvider`**, GitHub's OIDC provider, created only when the + `ExistingOidcProviderArn` parameter is empty (the `CreateProvider` + condition; the pattern is zeroae/foresight's `github-deploy-zae`). An + account holds one provider per URL, so in an account that already has one, + pass its ARN instead. +- **`ImagePushRole`**, `PowerUserPB-${stack}-image-push`, trusted only for + `sts:AssumeRoleWithWebIdentity` from that provider with `aud` + `sts.amazonaws.com` and `sub` exactly + `repo:zeroae@11018406/ci-runner@1455021495:ref:refs/heads/main`. The repo + uses GitHub's immutable subject, so the `sub` names the owner and repo by id + (`gh api repos/zeroae/ci-runner/actions/oidc/customization/sub` shows the + prefix); the name-only form `repo:zeroae/ci-runner:...` never matches. Any other branch, a pull + request, a tag or another repo carries a different `sub`, and can't assume + it. It allows `ecr:GetAuthorizationToken` (which has no resource-level + permissions) and, on the repo only, the layer upload calls, `PutImage`, + `BatchGetImage` and `GetDownloadUrlForLayer`. + +The outputs `ImageRepositoryUri` and `ImagePushRoleArn` are the repo +variables `ECR_IMAGE` and `AWS_IMAGE_PUSH_ROLE_ARN` that `image.yml` reads. +There is nothing to rotate: the push uses a short-lived OIDC token, and no +AWS key or GitHub token is stored anywhere. + +`image.yml`'s `soci` job, on every branch: + +1. downloads each architecture's build as an OCI layout and merges both into + one OCI image index, each manifest with the platform its config names; +2. runs `soci convert --standalone --all-platforms` (the `soci` CLI from + awslabs/soci-snapshotter, release `SOCI_VERSION`, sha256-checked). It + needs no containerd. The result is a SOCI-enabled image (SOCI index + manifest v2): an index holding both image manifests and a SOCI index + (`artifactType: application/vnd.amazon.soci.index.v2+json`) for each, and + the job fails unless both are there; +3. pushes it with `skopeo copy --all --preserve-digests` to a local + `registry:3` service and reads it back, so a branch exercises the push. + +On `main`, with both variables set, it then assumes the push role and +pushes the same index to ECR under `:-` and `:latest`. Without +the variables (before the stack has the repo) those two steps are skipped +and the GHCR publish is unaffected. diff --git a/functions/launcher.js b/functions/launcher.js index 0143751..ddecfaf 100644 --- a/functions/launcher.js +++ b/functions/launcher.js @@ -1,14 +1,29 @@ 'use strict'; // launcher: consumes the queue the webhook fills. For each job: an App JWT, an // installation token, a check that the job is still queued, a JIT config in -// the runner group, the #cloud-config, and one instance from the launch -// template. Inlined into the template; the build bundles ./cloud-config.js. +// the runner group, and one machine: on EC2 the #cloud-config and one instance +// from the launch template, on Fargate one task of the runner's task +// definition. Inlined into the template; the build bundles ./cloud-config.js. const crypto = require('node:crypto'); const { renderCloudConfig } = require('./cloud-config.js'); const API = 'https://api.github.com'; -const TYPES = ['m7i.xlarge', 'm6i.xlarge', 'm6a.xlarge', 'm7a.xlarge']; -const CAPACITY = { 'ci-spot': 'spot', 'ci-ondemand': 'on-demand' }; +// A size label's vCPUs -> the instance size, at 4 GB per vCPU, per family. +const SIZES = { 2: 'large', 4: 'xlarge', 8: '2xlarge', 16: '4xlarge' }; +// An architecture -> its families, offered at every size in the stack's AZs. +const FAMILIES = { x64: ['m8i', 'm8a', 'm7i', 'm6i', 'm6a', 'm7a'], arm64: ['m7g', 'm6g', 'm8g'] }; +const CAPACITY = { 'ci-spot': 'spot', 'ci-ondemand': 'on-demand', 'ci-fargate-spot': 'fargate-spot', 'ci-fargate': 'fargate' }; +// A Fargate capacity's capacity provider. +const PROVIDER = { 'ci-fargate-spot': 'FARGATE_SPOT', 'ci-fargate': 'FARGATE' }; +// Fargate offers no arm64 in this AZ, so arm64 tasks skip a subnet there. +const NO_ARM64_AZ = 'use1-az3'; +// ECS allows only these characters in a tag (EC2 allows any). The webhook's +// re-tag carries a copy. +const ECS_TAG_CHARS = /[^\p{L}\p{Z}\p{N}_.:/=+\-@]/gu; +const ecsTag = ([key, v]) => ({ key, value: String(v).replace(ECS_TAG_CHARS, '_').slice(0, 256) }); + +// An EC2 (Key, Value) or ECS (key, value) tag list as an object. +const tagsOf = (tags) => Object.fromEntries((tags || []).map((x) => [x.Key ?? x.key, x.Value ?? x.value])); const b64u = (s) => Buffer.from(s).toString('base64url'); @@ -18,15 +33,22 @@ function appJwt(appId, pem, nowMs) { return `${data}.${crypto.sign('sha256', Buffer.from(data), pem).toString('base64url')}`; } +// The shape of a message enqueued before the size labels (no vcpus or arch), +// and of an instance or task launched before the shape tags. +const DEFAULT_SHAPE = { vcpus: 4, arch: 'x64' }; +const shapeOf = (m) => ({ vcpus: m.vcpus === undefined ? DEFAULT_SHAPE.vcpus : m.vcpus, arch: m.arch === undefined ? DEFAULT_SHAPE.arch : m.arch }); + function valid(m, org) { return m && typeof m.repo === 'string' && /^[\w.-]+\/[\w.-]+$/.test(m.repo) && m.repo.split('/')[0] === org && Number.isInteger(m.job_id) && Number.isInteger(m.installation_id) && m.label in CAPACITY - && Array.isArray(m.labels); + && Array.isArray(m.labels) && Number.isInteger(shapeOf(m).vcpus) && Object.hasOwn(SIZES, shapeOf(m).vcpus) + && Object.hasOwn(FAMILIES, shapeOf(m).arch); } -function createHandler({ secret, fetch, ec2, sqs, now, env, log = console.log }) { +function createHandler({ secret, fetch, ec2, ecs, sqs, now, env, log = console.log }) { const org = env.GITHUB_ORG; const subnets = env.SUBNET_IDS.split(','); + const budgets = { spot: env.SPOT_VCPU_CAP, 'on-demand': env.ONDEMAND_VCPU_CAP, 'fargate-spot': env.FARGATE_SPOT_VCPU_CAP, fargate: env.FARGATE_VCPU_CAP }; const tokens = new Map(); // installation id -> { token, until } let app; let groupId; @@ -64,9 +86,11 @@ function createHandler({ secret, fetch, ec2, sqs, now, env, log = console.log }) } } - // Live instances of this stack by capacity. On-Demand ones have no - // InstanceLifecycle, so anything without the Spot marker counts as On-Demand. - async function count(type) { + // vCPUs of this stack's live instances on one capacity, from the shape tags. + // An instance launched before them has no tags: it has the default shape's + // vCPUs, and its capacity is Spot only with the Spot marker (On-Demand has + // no InstanceLifecycle). + async function vcpusIn(type) { let n = 0; let NextToken; do { @@ -77,14 +101,92 @@ function createHandler({ secret, fetch, ec2, sqs, now, env, log = console.log }) ], NextToken, }); - for (const res of r.Reservations || []) for (const i of res.Instances || []) n += (i.InstanceLifecycle === 'spot') === (type === 'spot'); + for (const res of r.Reservations || []) { + for (const i of res.Instances || []) { + const t = tagsOf(i.Tags); + const capacity = CAPACITY[t['ci-runner:capacity']] || (i.InstanceLifecycle === 'spot' ? 'spot' : 'on-demand'); + if (capacity === type) n += Number(t['ci-runner:vcpus']) || DEFAULT_SHAPE.vcpus; + } + } NextToken = r.NextToken; } while (NextToken); return n; } + // The tags a runner's instance or task carries: the job, for the swap + // repair and the notifier, and the shape, for the budgets. + function jobTags(m, name, runnerId, labelsSep) { + const { vcpus, arch } = shapeOf(m); + return [ + ['ci-runner:runner', name], ['ci-runner:runner-id', runnerId], ['ci-runner:label', m.label], ['ci-runner:repo', m.repo], + ['ci-runner:job-id', m.job_id], ['ci-runner:run-id', m.run_id], ['ci-runner:head-sha', m.head_sha], + ['ci-runner:job-name', m.job_name], ['ci-runner:labels', m.labels.join(labelsSep)], + ['ci-runner:capacity', m.label], ['ci-runner:vcpus', vcpus], ['ci-runner:arch', arch], + ]; + } + + // One Fargate task. The image's entrypoint (ci-runner-idle-exit, named by + // the task definition) runs the command and gives it the idle exit. + async function launchTask(m, name, runnerId, jitConfig) { + const { vcpus, arch } = shapeOf(m); + const azs = env.SUBNET_AZ_IDS.split(','); + // Inside a task nproc can report the host's CPUs, not the task's, so + // tools that size their parallelism from it get the task's vCPUs here. + // The runner's environment reaches every step. No CARGO* or CMAKE* + // variable: Swatinem/rust-cache hashes them into its key, and a Fargate + // job would miss the cache an EC2 job saved. The image's entrypoint turns + // CI_RUNNER_VCPUS into cargo's jobs in ~/.cargo/config.toml instead. + const environment = [['CI_RUNNER_VCPUS', vcpus], ['MAKEFLAGS', `-j${vcpus}`], + ...(env.IDLE_SECONDS ? [['IDLE_SECONDS', env.IDLE_SECONDS]] : [])].map(([n, v]) => ({ name: n, value: String(v) })); + const r = await ecs.runTask({ + cluster: env.CLUSTER, count: 1, + taskDefinition: arch === 'arm64' ? env.ARM64_TASK_DEFINITION : env.TASK_DEFINITION, + capacityProviderStrategy: [{ capacityProvider: PROVIDER[m.label], weight: 1 }], + networkConfiguration: { awsvpcConfiguration: { + subnets: subnets.filter((s, i) => arch !== 'arm64' || azs[i] !== NO_ARM64_AZ), + securityGroups: [env.SECURITY_GROUP_ID], assignPublicIp: 'ENABLED', + } }, + overrides: { + cpu: String(vcpus * 1024), memory: String(vcpus * 4096), + containerOverrides: [{ name: 'runner', command: ['./run.sh', '--jitconfig', jitConfig], environment }], + }, + // Labels joined by spaces: ECS refuses a comma in a tag. + tags: [['ci-runner:stack', env.STACK_NAME], ...jobTags(m, name, runnerId, ' ')].map(ecsTag), + }); + const arn = r.tasks && r.tasks[0] && r.tasks[0].taskArn; + if (!arn) { + const errs = [...new Set((r.failures || []).map((f) => `${f.reason}${f.detail ? `: ${f.detail}` : ''}`))]; + throw new Error(`no task launched: ${errs.join('; ') || 'no failure given'}`); + } + return arn; + } + + // vCPUs of the cluster's tasks on one Fargate capacity, from the shape + // tags. A task counts until it is STOPPED: one told to stop (desired + // STOPPED) still holds its vCPUs while it deprovisions. + async function taskVcpusIn(type) { + let n = 0; + for (const desiredStatus of ['RUNNING', 'STOPPED']) { + let nextToken; + do { + const r = await ecs.listTasks({ cluster: env.CLUSTER, desiredStatus, nextToken }); + if ((r.taskArns || []).length) { + const d = await ecs.describeTasks({ cluster: env.CLUSTER, tasks: r.taskArns, include: ['TAGS'] }); + for (const task of d.tasks || []) { + const t = tagsOf(task.tags); + if (task.lastStatus !== 'STOPPED' && CAPACITY[t['ci-runner:capacity']] === type) n += Number(t['ci-runner:vcpus']) || DEFAULT_SHAPE.vcpus; + } + } + nextToken = r.nextToken; + } while (nextToken); + } + return n; + } + async function launchInstance(m, name, runnerId, jitConfig) { - const lt = env.LAUNCH_TEMPLATE_ID; + const { vcpus, arch } = shapeOf(m); + // Each architecture has its own launch template, for its own AL2023 image. + const lt = arch === 'arm64' ? env.ARM64_LAUNCH_TEMPLATE_ID : env.LAUNCH_TEMPLATE_ID; const userData = renderCloudConfig({ jitConfig, ...(env.IDLE_SECONDS ? { idleSeconds: Number(env.IDLE_SECONDS) } : {}) }); // CreateFleet can't override user-data: a version per launch carries it. const v = await ec2.createLaunchTemplateVersion({ @@ -101,15 +203,11 @@ function createHandler({ secret, fetch, ec2, sqs, now, env, log = console.log }) ...(type === 'spot' ? { SpotOptions: { AllocationStrategy: 'capacity-optimized' } } : { OnDemandOptions: { AllocationStrategy: 'lowest-price' } }), LaunchTemplateConfigs: [{ LaunchTemplateSpecification: { LaunchTemplateId: lt, Version: version }, - Overrides: TYPES.flatMap((InstanceType) => subnets.map((SubnetId) => ({ InstanceType, SubnetId }))), + Overrides: FAMILIES[arch].flatMap((f) => subnets.map((SubnetId) => ({ InstanceType: `${f}.${SIZES[vcpus]}`, SubnetId }))), }], TagSpecifications: [{ ResourceType: 'instance', - Tags: [ - tag('ci-runner:runner', name), tag('ci-runner:runner-id', runnerId), tag('ci-runner:label', m.label), tag('ci-runner:repo', m.repo), - tag('ci-runner:job-id', m.job_id), tag('ci-runner:run-id', m.run_id), tag('ci-runner:head-sha', m.head_sha), - tag('ci-runner:job-name', m.job_name), - ], + Tags: jobTags(m, name, runnerId, ',').map(([k, v]) => tag(k, v)), }], }); const id = fleet.Instances && fleet.Instances[0] && fleet.Instances[0].InstanceIds[0]; @@ -130,15 +228,16 @@ function createHandler({ secret, fetch, ec2, sqs, now, env, log = console.log }) if (!j) return log(JSON.stringify({ ...job, dropped: 'job not found' })); if (j.status !== 'queued') return log(JSON.stringify({ ...job, dropped: j.status })); - // A soft cap: two launches at once may overshoot it by one. At the cap the - // job goes back on the queue as a new message, so retries don't pile up - // towards the dead-letter queue. + // A soft budget: launches checked at the same moment may overshoot it. A + // job that would go over it goes back on the queue as a new message, so + // retries don't pile up towards the dead-letter queue. const type = CAPACITY[m.label]; - const cap = Number(type === 'spot' ? env.SPOT_CAP : env.ONDEMAND_CAP); - const running = await count(type); - if (running >= cap) { + const budget = Number(budgets[type]); + const held = await (PROVIDER[m.label] ? taskVcpusIn(type) : vcpusIn(type)); + const { vcpus } = shapeOf(m); + if (held + vcpus > budget) { await sqs.sendMessage({ QueueUrl: env.QUEUE_URL, MessageBody: JSON.stringify(m), DelaySeconds: 60 }); - return log(JSON.stringify({ ...job, deferred: `${running} of ${cap} ${type}` })); + return log(JSON.stringify({ ...job, deferred: `${held}+${vcpus} of ${budget} ${type} vCPUs` })); } // The sweeper reads the UTC mint stamp in the name as the registration's @@ -150,8 +249,9 @@ function createHandler({ secret, fetch, ec2, sqs, now, env, log = console.log }) }); try { - const instance = await launchInstance(m, name, r.runner.id, r.encoded_jit_config); - log(JSON.stringify({ ...job, launched: instance, runner: name, label: m.label })); + const launch = PROVIDER[m.label] ? launchTask : launchInstance; + const machine = await launch(m, name, r.runner.id, r.encoded_jit_config); + log(JSON.stringify({ ...job, launched: machine, runner: name, label: m.label, ...shapeOf(m) })); } catch (e) { // Unregister now rather than leave it to the sweeper; the retry mints anew. await gh('DELETE', `/orgs/${org}/actions/runners/${r.runner.id}`, token).catch(() => {}); @@ -184,7 +284,9 @@ module.exports.handler = async (event) => { const sm = require('@aws-sdk/client-secrets-manager'); const EC2 = require('@aws-sdk/client-ec2'); const SQS = require('@aws-sdk/client-sqs'); + const ECS = require('@aws-sdk/client-ecs'); const sqsc = new SQS.SQSClient({}); + const ecsc = new ECS.ECSClient({}); const smc = new sm.SecretsManagerClient({}); const ec2c = new EC2.EC2Client({}); live = createHandler({ @@ -196,6 +298,11 @@ module.exports.handler = async (event) => { deleteLaunchTemplateVersions: (i) => ec2c.send(new EC2.DeleteLaunchTemplateVersionsCommand(i)), describeInstances: (i) => ec2c.send(new EC2.DescribeInstancesCommand(i)), }, + ecs: { + runTask: (i) => ecsc.send(new ECS.RunTaskCommand(i)), + listTasks: (i) => ecsc.send(new ECS.ListTasksCommand(i)), + describeTasks: (i) => ecsc.send(new ECS.DescribeTasksCommand(i)), + }, sqs: { sendMessage: (i) => sqsc.send(new SQS.SendMessageCommand(i)) }, now: Date.now, env: process.env, diff --git a/functions/notifier.js b/functions/notifier.js index 424f67e..39caf79 100644 --- a/functions/notifier.js +++ b/functions/notifier.js @@ -1,7 +1,8 @@ 'use strict'; -// notifier: EventBridge, on an EC2 Spot Instance Interruption Warning. If the -// instance is this stack's and carries a job's launch tags (see launcher.js), -// it creates a failed check run on the job's commit through the App. +// notifier: EventBridge, on an EC2 Spot Instance Interruption Warning or a +// Fargate Spot task's stop (ECS Task State Change, stopCode SpotInterruption). +// If the instance or task is this stack's and carries a job's launch tags (see +// launcher.js), it creates a failed check run on the job's commit through the App. // The App JWT is a copy of the launcher's: bundling a shared module would cost // more template bytes than the copy does. const crypto = require('node:crypto'); @@ -15,7 +16,11 @@ function appJwt(appId, pem, nowMs) { return `${data}.${crypto.sign('sha256', Buffer.from(data), pem).toString('base64url')}`; } -function createHandler({ secret, fetch, ec2, now, env, log = console.log }) { +// An EC2 (Key, Value) or ECS (key, value) tag list as an object, +// keyed without the ci-runner: prefix. +const tagsOf = (tags) => Object.fromEntries((tags || []).map((x) => [(x.Key ?? x.key).replace(/^ci-runner:/, ''), x.Value ?? x.value])); + +function createHandler({ secret, fetch, ec2, ecs, now, env, log = console.log }) { async function gh(method, path, token, body) { const res = await fetch(API + path, { method, body: body && JSON.stringify(body), @@ -27,14 +32,24 @@ function createHandler({ secret, fetch, ec2, now, env, log = console.log }) { } return async (event) => { - const id = event.detail && event.detail['instance-id']; - const t = {}; - const inst = id && (await ec2.describe(id)).Reservations?.[0]?.Instances?.[0]; - for (const { Key, Value } of inst?.Tags || []) t[Key.replace('ci-runner:', '')] = Value; + const d = event.detail || {}; + let id, machine, t; + if (d.taskArn) { + // The event carries no tags; a stopped task is still described for an hour. + id = d.taskArn.split('/').pop(); + machine = `task (${id})`; + const task = (await ecs.describeTasks({ cluster: d.clusterArn, tasks: [d.taskArn], include: ['TAGS'] })).tasks?.[0]; + t = tagsOf(task?.tags); + } else { + id = d['instance-id']; + machine = `instance (${id})`; + const inst = id && (await ec2.describe(id)).Reservations?.[0]?.Instances?.[0]; + t = tagsOf(inst?.Tags); + } const repo = t.repo || ''; const jobId = t['job-id']; if (t.stack !== env.STACK_NAME || !t['head-sha'] || !jobId || repo.split('/')[0] !== env.GITHUB_ORG) { - return log(`ignored: ${id} is no job instance of this stack`); + return log(`ignored: ${id} is no job ${d.taskArn ? 'task' : 'instance'} of this stack`); } const s = await secret.get(); @@ -43,15 +58,24 @@ function createHandler({ secret, fetch, ec2, now, env, log = console.log }) { const { id: installation } = await gh('GET', `/repos/${repo}/installation`, jwt); const { token } = await gh('POST', `/app/installations/${installation}/access_tokens`, jwt); - const job = t['job-name'] || `job ${jobId}`; + // A task's job-name tag has `_` for what ECS refuses (brackets, commas): + // the check run carries the real name, from GitHub. + const ghJob = d.taskArn && (await gh('GET', `/repos/${repo}/actions/jobs/${jobId}`, token).catch(() => null)); + const job = (ghJob && ghJob.name) || t['job-name'] || `job ${jobId}`; const run = `https://github.com/${repo}/actions/runs/${t['run-id']}`; + const name = `ci-runner / ${job}`; + const title = 'Spot capacity reclaimed: re-run this job'; + // A task's reclaim matches on each of its changes, and EventBridge may + // repeat an event: one check run per job is enough. + const { check_runs: existing } = await gh('GET', `/repos/${repo}/commits/${t['head-sha']}/check-runs?filter=all&check_name=${encodeURIComponent(name)}`, token); + if (existing.some((c) => c.external_id === jobId && c.output && c.output.title === title)) return log(`already reported ${id}: ${repo} job ${jobId}`); await gh('POST', `/repos/${repo}/check-runs`, token, { - name: `ci-runner / ${job}`, head_sha: t['head-sha'], external_id: jobId, + name, head_sha: t['head-sha'], external_id: jobId, status: 'completed', conclusion: 'failure', completed_at: new Date(now()).toISOString(), details_url: `${run}/job/${jobId}`, output: { - title: 'Spot capacity reclaimed: re-run this job', - summary: `AWS reclaimed the Spot instance (${id}) running \`${job}\`. Re-run the job: ${run}`, + title, + summary: `AWS reclaimed the Spot ${machine} running \`${job}\`. Re-run the job: ${run}`, }, }); log(`reported ${id}: ${repo} job ${jobId}`); @@ -66,11 +90,14 @@ module.exports.handler = async (event) => { const sm = require('@aws-sdk/client-secrets-manager'); const EC2 = require('@aws-sdk/client-ec2'); const smc = new sm.SecretsManagerClient({}); + const ECS = require('@aws-sdk/client-ecs'); const ec2c = new EC2.EC2Client({}); + const ecsc = new ECS.ECSClient({}); live = createHandler({ secret: { get: async () => JSON.parse((await smc.send(new sm.GetSecretValueCommand({ SecretId: process.env.SECRET_ARN }))).SecretString) }, fetch, ec2: { describe: (id) => ec2c.send(new EC2.DescribeInstancesCommand({ InstanceIds: [id] })) }, + ecs: { describeTasks: (i) => ecsc.send(new ECS.DescribeTasksCommand(i)) }, now: Date.now, env: process.env, }); diff --git a/functions/sweeper.js b/functions/sweeper.js index da02c4c..4c3629d 100644 --- a/functions/sweeper.js +++ b/functions/sweeper.js @@ -1,11 +1,13 @@ 'use strict'; -// sweeper: scheduled. Terminates the stack's CI runner instances older than -// MAX_AGE, and removes the runner group's registrations that are offline and -// have no instance. Inlined into the template by scripts/build-template.js; -// keep it one dependency-free CommonJS file. +// sweeper: scheduled. Terminates the stack's CI runner instances and stops +// the cluster's tasks older than MAX_AGE, and removes the runner group's +// registrations that are offline and have no instance or task. Inlined into +// the template by scripts/build-template.js; keep it one dependency-free +// CommonJS file. const crypto = require('node:crypto'); const MAX_AGE_MS = 2 * 60 * 60 * 1000; +const MAX_AGE = `${MAX_AGE_MS / 3600e3} hours`; const STACK_TAG = 'ci-runner:stack'; // The launcher tags each instance with its runner's name (and, when it has // it, the runner's id); that is how a registration finds its instance. @@ -26,8 +28,9 @@ function appJwt(appId, pem, nowMs) { } const tagOf = (inst, key) => ((inst.Tags || []).find((t) => t.Key === key) || {}).Value; +const taskTag = (task, key) => ((task.tags || []).find((t) => t.key === key) || {}).value; -function createHandler({ ec2, secret, fetch, now, env, log = console.log }) { +function createHandler({ ec2, ecs, secret, fetch, now, env, log = console.log }) { const org = env.GITHUB_ORG; // Instances that can still be (or become) a runner: everything not terminated. const alive = (i) => i.State.Name !== 'terminated'; @@ -46,6 +49,29 @@ function createHandler({ ec2, secret, fetch, now, env, log = console.log }) { return strays; } + // The cluster's tasks not yet told to stop, with their tags. A stopping + // task no longer counts: its runner is already on its way out. + async function listTasks() { + const out = []; + let nextToken; + do { + const r = await ecs.listTasks({ cluster: env.CLUSTER, desiredStatus: 'RUNNING', nextToken }); + if ((r.taskArns || []).length) out.push(...(await ecs.describeTasks({ cluster: env.CLUSTER, tasks: r.taskArns, include: ['TAGS'] })).tasks || []); + nextToken = r.nextToken; + } while (nextToken); + return out; + } + + // Every task in the stack's cluster is the stack's, a hand-run one too. + async function sweepTasks(tasks) { + const strays = tasks.filter((t) => now() - new Date(t.createdAt).getTime() > MAX_AGE_MS).map((t) => t.taskArn); + for (const task of strays) { + await ecs.stopTask({ cluster: env.CLUSTER, task, reason: `ci-runner sweeper: launched over ${MAX_AGE} ago` }); + log(`sweeper: stopped ${task}`); + } + return strays; + } + async function gh(method, path, token, body) { const res = await fetch(API + path, { method, body: body && JSON.stringify(body), @@ -82,10 +108,10 @@ function createHandler({ ec2, secret, fetch, now, env, log = console.log }) { return m ? Date.UTC(m[1], m[2] - 1, m[3], m[4], m[5], m[6]) : undefined; } - async function sweepRunners(token, runners, instances) { + async function sweepRunners(token, runners, instances, tasks) { const present = instances.filter(alive); - const names = new Set(present.map((i) => tagOf(i, RUNNER_TAG)).filter(Boolean)); - const ids = new Set(present.map((i) => tagOf(i, RUNNER_ID_TAG)).filter(Boolean)); + const names = new Set([...present.map((i) => tagOf(i, RUNNER_TAG)), ...tasks.map((t) => taskTag(t, RUNNER_TAG))].filter(Boolean)); + const ids = new Set([...present.map((i) => tagOf(i, RUNNER_ID_TAG)), ...tasks.map((t) => taskTag(t, RUNNER_ID_TAG))].filter(Boolean)); const young = (r) => { const t = minted(r.name); return t !== undefined && now() - t < GRACE_MS; }; const orphans = runners.filter((r) => r.status === 'offline' && !r.busy && !names.has(r.name) && !ids.has(String(r.id)) && !young(r)); @@ -97,8 +123,8 @@ function createHandler({ ec2, secret, fetch, now, env, log = console.log }) { } return async function handler() { - // Registrations are listed before instances are described, so a runner - // whose instance launches in between is still seen with its instance. + // Registrations are listed before instances and tasks are, so a runner + // whose machine launches in between is still seen with it. let token, runners, failure; try { const creds = await secret.get(); @@ -112,9 +138,11 @@ function createHandler({ ec2, secret, fetch, now, env, log = console.log }) { const instances = (await ec2.describe()).filter((i) => tagOf(i, STACK_TAG) === env.STACK_NAME); const terminated = await sweepInstances(instances); + const tasks = await listTasks(); + const stopped = await sweepTasks(tasks); if (failure) throw failure; - const removed = runners ? await sweepRunners(token, runners, instances) : []; - return { terminated, removed }; + const removed = runners ? await sweepRunners(token, runners, instances, tasks) : []; + return { terminated, stopped, removed }; }; } @@ -127,8 +155,10 @@ exports.handler = async (event) => { if (!live) { const sm = require('@aws-sdk/client-secrets-manager'); const ec2sdk = require('@aws-sdk/client-ec2'); + const ECS = require('@aws-sdk/client-ecs'); const smc = new sm.SecretsManagerClient({}); const ec2c = new ec2sdk.EC2Client({}); + const ecsc = new ECS.ECSClient({}); const SecretId = process.env.SECRET_ARN; live = createHandler({ secret: { @@ -144,6 +174,11 @@ exports.handler = async (event) => { }, terminate: async (InstanceIds) => { await ec2c.send(new ec2sdk.TerminateInstancesCommand({ InstanceIds })); }, }, + ecs: { + listTasks: (i) => ecsc.send(new ECS.ListTasksCommand(i)), + describeTasks: (i) => ecsc.send(new ECS.DescribeTasksCommand(i)), + stopTask: (i) => ecsc.send(new ECS.StopTaskCommand(i)), + }, fetch: globalThis.fetch, now: Date.now, env: process.env, diff --git a/functions/webhook.js b/functions/webhook.js index 37caf8f..745ff96 100644 --- a/functions/webhook.js +++ b/functions/webhook.js @@ -1,12 +1,15 @@ 'use strict'; // webhook: GitHub workflow_job receiver behind a Function URL. Verifies the // delivery, applies the label contract and fence 3, and enqueues the job for -// the launcher. It launches nothing itself. Inlined into the template; keep it -// one dependency-free CommonJS file. +// the launcher. It launches nothing itself, but repairs swaps (ADR-0003). +// Inlined into the template; keep it one dependency-free CommonJS file. const crypto = require('node:crypto'); -const CAPACITY = ['ci-spot', 'ci-ondemand']; -const DEFAULT_LABELS = ['linux', 'x64']; +const CAPACITY = ['ci-spot', 'ci-ondemand', 'ci-fargate-spot', 'ci-fargate']; +const SIZE = { 'ci-cpu-2': 2, 'ci-cpu-4': 4, 'ci-cpu-8': 8, 'ci-cpu-16': 16 }; +const ARCH = ['x64', 'arm64']; +// The shape of an instance launched before the shape tags. +const DEFAULT_SHAPE = { vcpus: 4, arch: 'x64' }; function reply(statusCode, text) { return { statusCode, headers: { 'content-type': 'text/plain; charset=utf-8' }, body: text + '\n' }; @@ -25,20 +28,94 @@ function signatureOk(key, raw, sig) { return got.length === want.length && crypto.timingSafeEqual(got, want); } -// The label contract: `self-hosted`, exactly one capacity label, and nothing -// else but the runner's default labels. Returns the capacity label or null. -function capacityLabel(labels) { +// The label contract: `self-hosted`, exactly one capacity label, exactly one +// size label, exactly one architecture label, and optionally `linux`. None has +// a default, so a swap only happens between runners of the same shape. +// Returns the job's shape, or null outside it. +function shapeOf(labels) { const ls = (labels || []).map((l) => String(l).toLowerCase()); - if (!ls.includes('self-hosted')) return null; - const caps = ls.filter((l) => CAPACITY.includes(l)); - if (caps.length !== 1) return null; - const rest = ls.filter((l) => l !== 'self-hosted' && !CAPACITY.includes(l)); - return rest.every((l) => DEFAULT_LABELS.includes(l)) ? caps[0] : null; + const of = (set) => ls.filter((l) => set.includes(l)); + const capacities = of(CAPACITY); + const sizes = of(Object.keys(SIZE)); + const archs = of(ARCH); + if (!ls.includes('self-hosted') || capacities.length !== 1 || sizes.length !== 1 || archs.length !== 1) return null; + const known = ['self-hosted', 'linux', ...CAPACITY, ...Object.keys(SIZE), ...ARCH]; + if (!ls.every((l) => known.includes(l))) return null; + return { capacity: capacities[0], vcpus: SIZE[sizes[0]], arch: archs[0] }; } -function createHandler({ secret, queue, env }) { +// The job fields the instance or task carries, as tags. +const JOB_TAGS = { repo: 'repo', job_id: 'job-id', run_id: 'run-id', head_sha: 'head-sha', job_name: 'job-name' }; +// ECS allows only these characters in a tag (EC2 allows any). A copy of the +// launcher's, so a re-tag reads as the launch did. +const ECS_TAG_CHARS = /[^\p{L}\p{Z}\p{N}_.:/=+\-@]/gu; +const ecsTag = ([key, v]) => ({ key, value: String(v).replace(ECS_TAG_CHARS, '_').slice(0, 256) }); + +// An EC2 (Key, Value) or ECS (key, value) tag list as an object, +// keyed without the ci-runner: prefix. +const tagsOf = (tags) => Object.fromEntries((tags || []).map((x) => [(x.Key ?? x.key).replace(/^ci-runner:/, ''), x.Value ?? x.value])); + +function createHandler({ secret, queue, ec2, ecs, env, log = console.log }) { + // The runner's task in the cluster, not yet told to stop, with its tags. + async function findTask(name) { + let nextToken; + do { + const r = await ecs.listTasks({ cluster: env.CLUSTER, desiredStatus: 'RUNNING', nextToken }); + if ((r.taskArns || []).length) { + const d = await ecs.describeTasks({ cluster: env.CLUSTER, tasks: r.taskArns, include: ['TAGS'] }); + const t = (d.tasks || []).find((x) => (x.tags || []).some((g) => g.key === 'ci-runner:runner' && g.value === name)); + if (t) return t; + } + nextToken = r.nextToken; + } while (nextToken); + return undefined; + } + let key; // the webhook secret, cached once the App exists + // A swap: a runner of this stack, named `${stack}--`, started + // a job other than the one it was launched for. Re-queue that one, so the + // launcher's "still queued?" check launches it a runner or drops it, and + // re-tag the instance or task so the notifier names the job it runs. + async function repairSwap(p, repo, job) { + const name = String(job.runner_name || ''); + const m = name.match(/^(.+)-\d{14}-(\d+)$/); + if (!m || m[1] !== env.STACK_NAME || Number(m[2]) === job.id) return reply(200, 'ignored: no swap'); + const r = await ec2.describeInstances({ + Filters: [ + { Name: 'tag:ci-runner:stack', Values: [env.STACK_NAME] }, + { Name: 'tag:ci-runner:runner', Values: [name] }, + { Name: 'instance-state-name', Values: ['pending', 'running'] }, + ], + }); + const i = (r.Reservations || []).flatMap((x) => x.Instances || [])[0]; + const task = !i && (await findTask(name)); + if (!i && !task) return reply(200, 'ignored: no instance or task'); + const t = tagsOf(i ? i.Tags : task.tags); + // Already re-tagged for this job: a duplicate delivery. + if (!t['job-id'] || Number(t['job-id']) === job.id) return reply(200, 'ignored: no swap'); + + const orig = { + repo: t.repo, job_id: Number(t['job-id']), run_id: Number(t['run-id']), head_sha: t['head-sha'], + job_name: t['job-name'], label: t.label, vcpus: Number(t.vcpus) || DEFAULT_SHAPE.vcpus, arch: t.arch || DEFAULT_SHAPE.arch, + // A task's labels are space-joined: ECS refuses a comma in a tag. + labels: t.labels ? t.labels.split(i ? ',' : ' ') : ['self-hosted', t.label], + installation_id: p.installation && p.installation.id, + }; + // Queue first: if the re-tag then fails, a redelivery still finds the + // old tags and repairs again, rather than skip a job never re-queued. + await queue.send(orig); + const ran = { repo: repo.full_name, job_id: job.id, run_id: job.run_id, head_sha: job.head_sha, job_name: job.name }; + const tags = Object.entries(JOB_TAGS).map(([f, k]) => [`ci-runner:${k}`, ran[f]]); + if (i) await ec2.createTags({ Resources: [i.InstanceId], Tags: tags.map(([Key, v]) => ({ Key, Value: String(v).slice(0, 256) })) }); + else await ecs.tagResource({ resourceArn: task.taskArn, tags: tags.map(ecsTag) }); + log(JSON.stringify({ + swap: name, ...(i ? { instance: i.InstanceId } : { task: task.taskArn }), + took: { repo: ran.repo, job_id: ran.job_id }, requeued: { repo: orig.repo, job_id: orig.job_id }, + })); + return reply(202, 'swap repaired'); + } + return async (event) => { if (!key) key = (await secret.get()).webhook_secret; if (!key) return reply(503, 'not set up'); @@ -54,19 +131,22 @@ function createHandler({ secret, queue, env }) { // Fence 3: whatever the App installation and the runner group say. if (repo.visibility !== 'private' || !repo.owner || repo.owner.login !== env.GITHUB_ORG) { - console.log(JSON.stringify({ refused: repo.full_name, visibility: repo.visibility, job_id: job.id })); + log(JSON.stringify({ refused: repo.full_name, visibility: repo.visibility, job_id: job.id })); return reply(403, 'refused: only private repos of ' + env.GITHUB_ORG); } + if (p.action === 'in_progress') return repairSwap(p, repo, job); if (p.action !== 'queued') return reply(200, 'ignored: action'); - const label = capacityLabel(job.labels); - if (!label) return reply(200, 'ignored: labels'); + const shape = shapeOf(job.labels); + if (!shape) return reply(200, 'ignored: labels'); + // The message's `label` is the capacity label, as queued messages carry it. + const wire = { label: shape.capacity, vcpus: shape.vcpus, arch: shape.arch }; const msg = { repo: repo.full_name, job_id: job.id, run_id: job.run_id, head_sha: job.head_sha, - job_name: job.name, label, labels: job.labels, installation_id: p.installation && p.installation.id, + job_name: job.name, ...wire, labels: job.labels, installation_id: p.installation && p.installation.id, }; await queue.send(msg); - console.log(JSON.stringify({ enqueued: msg.repo, job_id: msg.job_id, label })); + log(JSON.stringify({ enqueued: msg.repo, job_id: msg.job_id, ...wire })); return reply(202, 'enqueued'); }; } @@ -78,11 +158,24 @@ module.exports.handler = async (event) => { if (!live) { const sm = require('@aws-sdk/client-secrets-manager'); const sqs = require('@aws-sdk/client-sqs'); + const EC2 = require('@aws-sdk/client-ec2'); const smc = new sm.SecretsManagerClient({}); const sqsc = new sqs.SQSClient({}); + const ECS = require('@aws-sdk/client-ecs'); + const ec2c = new EC2.EC2Client({}); + const ecsc = new ECS.ECSClient({}); live = createHandler({ secret: { get: async () => JSON.parse((await smc.send(new sm.GetSecretValueCommand({ SecretId: process.env.SECRET_ARN }))).SecretString) }, queue: { send: async (m) => { await sqsc.send(new sqs.SendMessageCommand({ QueueUrl: process.env.QUEUE_URL, MessageBody: JSON.stringify(m) })); } }, + ec2: { + describeInstances: (i) => ec2c.send(new EC2.DescribeInstancesCommand(i)), + createTags: (i) => ec2c.send(new EC2.CreateTagsCommand(i)), + }, + ecs: { + listTasks: (i) => ecsc.send(new ECS.ListTasksCommand(i)), + describeTasks: (i) => ecsc.send(new ECS.DescribeTasksCommand(i)), + tagResource: (i) => ecsc.send(new ECS.TagResourceCommand(i)), + }, env: process.env, }); } diff --git a/image/ci-runner-idle-exit b/image/ci-runner-idle-exit index bef408f..06b34d3 100755 --- a/image/ci-runner-idle-exit +++ b/image/ci-runner-idle-exit @@ -13,6 +13,15 @@ # IDLE_GRACE_SECONDS (default 30, as docker stop gives an EC2 runner): the # runner's helper and listener sit in their own process group, and a TERM # alone doesn't end them. +# +# With CI_RUNNER_VCPUS (the task's vCPUs, set by the launcher), it first writes +# /home/runner/.cargo/config.toml with `[build] jobs = `: nproc over-reports +# in a task, and a CARGO_BUILD_JOBS in the environment would change +# Swatinem/rust-cache's key (it hashes every CARGO*, CC*, CFLAGS*, CXX*, CMAKE* +# and RUST* variable), so a Fargate job would miss the cache an EC2 job saved. +# Cargo reads .cargo/config.toml in every ancestor of its working directory, +# so the file reaches every checkout under /home/runner/_work, whatever +# CARGO_HOME a job sets. A file already there is left alone. set -u idle=${IDLE_SECONDS:-900} @@ -26,6 +35,21 @@ if [[ ! $grace =~ ^[0-9]+$ ]]; then exit 2 fi +cargo_jobs() { + local n=${CI_RUNNER_VCPUS:-} dir=/home/runner/.cargo + [[ -n $n ]] || return 0 + if [[ ! $n =~ ^[1-9][0-9]*$ ]]; then + echo "ci-runner: CI_RUNNER_VCPUS must be a positive integer, not '$n'; no cargo jobs set" >&2 + elif [[ -e $dir/config.toml ]]; then + echo "ci-runner: $dir/config.toml exists, leaving it; cargo's jobs not set to $n" >&2 + elif mkdir -p "$dir" && printf '[build]\njobs = %s\n' "$n" > "$dir/config.toml"; then + echo "ci-runner: cargo jobs = $n ($dir/config.toml)" + else + echo "ci-runner: couldn't write $dir/config.toml; cargo's jobs not set" >&2 + fi +} +cargo_jobs + log=$(mktemp /tmp/ci-runner-idle-exit.XXXXXX) # Its own session (its id is $pid), so a stop reaches run.sh, its helper and # the listener, whatever their process groups. diff --git a/image/smoke.sh b/image/smoke.sh index d723c7c..428b382 100755 --- a/image/smoke.sh +++ b/image/smoke.sh @@ -100,5 +100,25 @@ expect idle-exit-keeps "3 finished" "$rc $(grep -x finished <<<"$out")" out=$(IDLE_SECONDS=0 timeout 10 "$idle" true 2>&1) expect idle-exit-refuses "2" "$?" +echo "== cargo jobs" +# With CI_RUNNER_VCPUS the entrypoint caps cargo's jobs in a config file, not +# in a CARGO_* variable rust-cache would hash. cargo isn't in the image, so +# the file is checked, not a build. +cfg=/home/runner/.cargo/config.toml +rm -f "$cfg" +IDLE_SECONDS=2 timeout 10 "$idle" true >/dev/null 2>&1 +expect cargo-none no "$([[ -e $cfg ]] && echo yes || echo no)" +out=$(CI_RUNNER_VCPUS=4 IDLE_SECONDS=2 timeout 10 "$idle" true 2>&1) +expect cargo-config '[build] jobs = 4' "$(paste -sd' ' "$cfg" 2>&1)" +expect cargo-owner runner "$(stat -c %U "$cfg" 2>&1)" +expect cargo-says yes "$(grep -q 'cargo jobs = 4' <<<"$out" && echo yes || echo no)" +printf '[net]\ngit-fetch-with-cli = true\n' > "$cfg" +out=$(CI_RUNNER_VCPUS=8 IDLE_SECONDS=2 timeout 10 "$idle" true 2>&1) +expect cargo-keeps '[net] git-fetch-with-cli = true' "$(paste -sd' ' "$cfg" 2>&1)" +expect cargo-keeps-says yes "$(grep -q 'exists, leaving it' <<<"$out" && echo yes || echo no)" +rm -f "$cfg" +out=$(CI_RUNNER_VCPUS=four IDLE_SECONDS=2 timeout 10 "$idle" true 2>&1) +expect cargo-bad-value "0 no" "$? $([[ -e $cfg ]] && echo yes || echo no)" + if (( fail )); then echo "SMOKE FAILED"; exit 1; fi echo "SMOKE PASSED" diff --git a/scripts/build-template.js b/scripts/build-template.js index 799a060..6ba99d6 100644 --- a/scripts/build-template.js +++ b/scripts/build-template.js @@ -9,13 +9,12 @@ // replaced by x.js itself, wrapped as a module, so one Lambda can share // another file (the launcher bundles cloud-config.js), and whole-line `//` // comments are dropped, as are the template's own whole-line `#` comments. -// Fails if the result is -// not under LIMIT bytes, the largest template `aws cloudformation deploy` -// sends inline. +// Fails if the result is not under LIMIT bytes, the largest template +// CloudFormation accepts from S3 (`make deploy` uploads it to a bucket first). const fs = require('node:fs'); const path = require('node:path'); -const LIMIT = 51200; +const LIMIT = 1048576; const LOCAL_REQUIRE = /require\('\.\/([\w.-]+\.js)'\)/; // Whole-line `//` comments go, except inside a template literal (tracked by diff --git a/scripts/hand-launch.sh b/scripts/hand-launch.sh index b8f9114..84d0f9b 100755 --- a/scripts/hand-launch.sh +++ b/scripts/hand-launch.sh @@ -1,22 +1,48 @@ #!/usr/bin/env bash # Launches one CI runner by hand: mints an org JIT config for the runner group # (with the owner's gh, which needs admin:org), renders its #cloud-config and -# launches one Spot .xlarge from the stack's launch template. The instance -# takes one job labelled [self-hosted, LABEL] and terminates; given no job, it +# launches one instance of capacity CAPACITY (ci-spot: Spot; ci-ondemand: +# On-Demand) and size SIZE from the stack's launch template for ARCH, with the +# launcher's fleet options. The instance takes one job labelled +# [self-hosted, CAPACITY, SIZE, ARCH] and terminates; given no job, it # terminates after the idle exit (15 minutes; IDLE_SECONDS overrides it). +# CAPACITY, SIZE and ARCH are required, as in the label contract: none has a +# default. A Fargate capacity has no instance: see the runbook's "Run a task +# by hand". # -# scripts/hand-launch.sh # or: make hand-launch +# CAPACITY=ci-spot SIZE=ci-cpu-4 ARCH=x64 scripts/hand-launch.sh +# make hand-launch CAPACITY=ci-ondemand SIZE=ci-cpu-8 ARCH=arm64 # -# Prints the instance id. Overrides: STACK PROFILE REGION ORG GROUP LABEL IDLE_SECONDS +# Prints the instance id. Overrides: STACK PROFILE REGION ORG GROUP IDLE_SECONDS set -euo pipefail STACK=${STACK:-ci-runner} -PROFILE=${PROFILE:-zeroae-code/AWSPowerUserAccess} +PROFILE=${PROFILE:-zeroae-code/ZAEPowerUserAccess} REGION=${REGION:-us-east-1} ORG=${ORG:-zeroae} GROUP=${GROUP:-ci-runner} -LABEL=${LABEL:-ci-spot} -TYPES=(m7i.xlarge m6i.xlarge m6a.xlarge m7a.xlarge) +CAPACITY=${CAPACITY:?CAPACITY is required: ci-spot or ci-ondemand} +SIZE=${SIZE:?SIZE is required: one of ci-cpu-2, ci-cpu-4, ci-cpu-8, ci-cpu-16} +ARCH=${ARCH:?ARCH is required: one of x64, arm64} +# The capacity's fleet options, as the launcher sets them. +case $CAPACITY in + ci-spot) market=spot; options='"SpotOptions": {"AllocationStrategy": "capacity-optimized"}' ;; + ci-ondemand) market=on-demand; options='"OnDemandOptions": {"AllocationStrategy": "lowest-price"}' ;; + ci-fargate-spot | ci-fargate) + echo "$CAPACITY runs a Fargate task, not an instance: see \"Run a task by hand\" in docs/runbook.md" >&2; exit 1 ;; + *) echo "unknown capacity $CAPACITY: ci-spot or ci-ondemand" >&2; exit 1 ;; +esac +# The size label's instance size, at 4 GB per vCPU, as the launcher maps it. +case $SIZE in + ci-cpu-2) size=large ;; ci-cpu-4) size=xlarge ;; ci-cpu-8) size=2xlarge ;; ci-cpu-16) size=4xlarge ;; + *) echo "unknown size $SIZE: one of ci-cpu-2, ci-cpu-4, ci-cpu-8, ci-cpu-16" >&2; exit 1 ;; +esac +# The architecture's launch template and families, as the launcher picks them. +case $ARCH in + x64) lt_output=LaunchTemplateId; TYPES=(m8i.$size m8a.$size m7i.$size m6i.$size m6a.$size m7a.$size) ;; + arm64) lt_output=Arm64LaunchTemplateId; TYPES=(m7g.$size m6g.$size m8g.$size) ;; + *) echo "unknown architecture $ARCH: one of x64, arm64" >&2; exit 1 ;; +esac aws() { command aws --profile "$PROFILE" --region "$REGION" "$@"; } here=$(cd "$(dirname "$0")" && pwd) @@ -24,7 +50,7 @@ output() { aws cloudformation describe-stacks --stack-name "$STACK" \ --query "Stacks[0].Outputs[?OutputKey=='$1'].OutputValue" --output text } -lt=$(output LaunchTemplateId) +lt=$(output $lt_output) IFS=, read -r -a subnets <<<"$(output SubnetIds)" group_id=$(gh api "orgs/$ORG/actions/runner-groups" --paginate \ @@ -38,7 +64,7 @@ chmod 700 "$work" name="$STACK-hand-$(date -u +%Y%m%d%H%M%S)" gh api -X POST "orgs/$ORG/actions/runners/generate-jitconfig" \ -f name="$name" -F runner_group_id="$group_id" -f work_folder=_work \ - -f 'labels[]=self-hosted' -f "labels[]=$LABEL" \ + -f 'labels[]=self-hosted' -f "labels[]=$CAPACITY" -f "labels[]=$SIZE" -f "labels[]=$ARCH" \ --jq .encoded_jit_config > "$work/jit" node "$here/render-cloud-config.js" < "$work/jit" | base64 -w0 > "$work/user-data" @@ -54,15 +80,18 @@ overrides=$(for t in "${TYPES[@]}"; do for s in "${subnets[@]}"; do cat > "$work/fleet.json" <&2 exit 1 fi -echo "$instance runner $name label $LABEL" >&2 +echo "$instance runner $name labels $CAPACITY $SIZE $ARCH" >&2 echo "$instance" diff --git a/scripts/setup-github.sh b/scripts/setup-github.sh index 7309d34..2197a47 100755 --- a/scripts/setup-github.sh +++ b/scripts/setup-github.sh @@ -205,7 +205,7 @@ finish() { ORG=zeroae GROUP=ci-runner STACK="${STACK:-ci-runner}" -PROFILE="${PROFILE:-zeroae-code/AWSPowerUserAccess}" +PROFILE="${PROFILE:-zeroae-code/ZAEPowerUserAccess}" REGION="${REGION:-us-east-1}" aws_() { aws --profile "$PROFILE" --region "$REGION" "$@"; } die() { warn "$1"; exit 1; } diff --git a/template.yaml b/template.yaml index a7c5af4..1fe35db 100644 --- a/template.yaml +++ b/template.yaml @@ -1,7 +1,8 @@ AWSTemplateFormatVersion: "2010-09-09" Description: >- - ci-runner: single-use EC2 self-hosted GitHub Actions runners for zeroae's - private repos. Source template; `make build` inlines functions/*.js. + ci-runner: single-use EC2 and Fargate self-hosted GitHub Actions runners + for zeroae's private repos. Source template; `make build` inlines + functions/*.js. # Layout rules (see docs/stack.md): # - every role and policy is PowerUserPB-${AWS::StackName}- with the @@ -23,14 +24,29 @@ Parameters: Type: String Default: ci-runner Description: The org runner group every CI runner registers into (created after the stack). - SpotCap: + SpotVcpuCap: Type: Number - Default: 6 - Description: Most Spot CI runners at once (soft). - OnDemandCap: + Default: 24 + Description: Most vCPUs the Spot CI runners hold at once, both architectures (soft). Draws on the EC2 Spot vCPU quota. + OnDemandVcpuCap: Type: Number - Default: 2 - Description: Most On-Demand CI runners at once (soft). + Default: 8 + Description: Most vCPUs the On-Demand CI runners hold at once, both architectures (soft). Draws on the EC2 On-Demand vCPU quota. + FargateSpotVcpuCap: + Type: Number + Default: 48 + Description: Most vCPUs the Fargate Spot CI runners hold at once, both architectures (soft). Draws on the Fargate Spot vCPU quota. + FargateVcpuCap: + Type: Number + Default: 8 + Description: Most vCPUs the Fargate On-Demand CI runners hold at once, both architectures (soft). Draws on the Fargate On-Demand vCPU quota. + ExistingOidcProviderArn: + Type: String + Default: "" + Description: Leave empty to create GitHub's OIDC provider. An account holds one provider per URL, so if `aws iam list-open-id-connect-providers` already shows token.actions.githubusercontent.com, give its ARN here. + +Conditions: + CreateProvider: !Equals [!Ref ExistingOidcProviderArn, ""] Resources: # App id, private key and webhook secret. Created holding only `state_key` @@ -144,7 +160,7 @@ Resources: MessageRetentionPeriod: 1209600 SqsManagedSseEnabled: true - # --- webhook: verify, label contract, fence 3, enqueue ---------------------- + # --- webhook: verify, label contract, fence 3, enqueue, swap repair --------- WebhookLogGroup: Type: AWS::Logs::LogGroup Properties: @@ -176,6 +192,36 @@ Resources: - Effect: Allow Action: sqs:SendMessage Resource: !GetAtt JobQueue.Arn + # Swap repair: find a runner's instance, and re-tag it with the + # job it really runs. Only the job tags, only the stack's instances. + - Effect: Allow + Action: ec2:DescribeInstances + Resource: "*" + - Effect: Allow + Action: ec2:CreateTags + Resource: !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:instance/*" + Condition: + StringEquals: + aws:ResourceTag/ci-runner:stack: !Ref AWS::StackName + ForAllValues:StringEquals: + aws:TagKeys: [ci-runner:repo, ci-runner:job-id, ci-runner:run-id, ci-runner:head-sha, ci-runner:job-name] + # The same for a runner's task: only the cluster's tasks, only + # the job tags. + - Effect: Allow + Action: ecs:ListTasks + Resource: "*" + Condition: + ArnEquals: + ecs:cluster: !GetAtt RunnerCluster.Arn + - Effect: Allow + Action: ecs:DescribeTasks + Resource: !Sub "arn:${AWS::Partition}:ecs:${AWS::Region}:${AWS::AccountId}:task/${RunnerCluster}/*" + - Effect: Allow + Action: ecs:TagResource + Resource: !Sub "arn:${AWS::Partition}:ecs:${AWS::Region}:${AWS::AccountId}:task/${RunnerCluster}/*" + Condition: + ForAllValues:StringEquals: + aws:TagKeys: [ci-runner:repo, ci-runner:job-id, ci-runner:run-id, ci-runner:head-sha, ci-runner:job-name] WebhookFunction: Type: AWS::Lambda::Function @@ -194,6 +240,8 @@ Resources: SECRET_ARN: !Ref AppSecret QUEUE_URL: !Ref JobQueue GITHUB_ORG: !Ref GitHubOrg + STACK_NAME: !Ref AWS::StackName + CLUSTER: !Ref RunnerCluster Code: ZipFile: | //@inline functions/webhook.js @@ -231,7 +279,11 @@ Resources: Type: AWS::IAM::Role Properties: RoleName: !Sub "PowerUserPB-${AWS::StackName}-launcher" - PermissionsBoundary: arn:aws:iam::aws:policy/PowerUserAccess + # RunTask passes the execution role, and PowerUserAccess allows no + # iam:PassRole. The account's ZAEPowerUserBoundary is PowerUserAccess + # plus PassRole on PowerUserPB* roles; the policy below narrows that to + # the execution role. + PermissionsBoundary: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/ZAEPowerUserBoundary" AssumeRolePolicyDocument: Version: "2012-10-17" Statement: @@ -252,21 +304,24 @@ Resources: - Effect: Allow Action: [sqs:ReceiveMessage, sqs:DeleteMessage, sqs:GetQueueAttributes, sqs:ChangeMessageVisibility, sqs:SendMessage] Resource: !GetAtt JobQueue.Arn - # The cap counts instances; no resource-level permissions. + # The budgets count instances' vCPUs; no resource-level permissions. - Effect: Allow Action: ec2:DescribeInstances Resource: "*" # A version per launch carries the user-data. - Effect: Allow Action: [ec2:CreateLaunchTemplateVersion, ec2:DeleteLaunchTemplateVersions] - Resource: !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:launch-template/${RunnerLaunchTemplate}" - # The fleet, from this launch template, into this stack's subnets + Resource: + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:launch-template/${RunnerLaunchTemplate}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:launch-template/${Arm64RunnerLaunchTemplate}" + # The fleet, from these launch templates, into this stack's subnets # and security group only. No iam:PassRole: there is no role. - Effect: Allow Action: [ec2:CreateFleet, ec2:RunInstances] Resource: - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:fleet/*" - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:launch-template/${RunnerLaunchTemplate}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:launch-template/${Arm64RunnerLaunchTemplate}" - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:subnet/${Subnet0}" - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:subnet/${Subnet1}" - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:subnet/${Subnet2}" @@ -291,7 +346,9 @@ Resources: - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:spot-instances-request/*" Condition: ArnEquals: - ec2:LaunchTemplate: !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:launch-template/${RunnerLaunchTemplate}" + ec2:LaunchTemplate: + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:launch-template/${RunnerLaunchTemplate}" + - !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:launch-template/${Arm64RunnerLaunchTemplate}" - Effect: Allow Action: ec2:CreateTags Resource: !Sub "arn:${AWS::Partition}:ec2:${AWS::Region}:${AWS::AccountId}:*/*" @@ -302,12 +359,42 @@ Resources: - Effect: Allow Action: ssm:GetParameters Resource: !Sub "arn:${AWS::Partition}:ssm:${AWS::Region}::parameter/aws/service/ami-amazon-linux-latest/*" + # Fargate: the two task definitions, in the stack's cluster only. + - Effect: Allow + Action: ecs:RunTask + Resource: [!Ref RunnerTaskDefinition, !Ref Arm64RunnerTaskDefinition] + Condition: + ArnEquals: + ecs:cluster: !GetAtt RunnerCluster.Arn + # The only role a task gets: its execution role (no task role). + - Effect: Allow + Action: iam:PassRole + Resource: !GetAtt RunnerExecutionRole.Arn + Condition: + StringEquals: + iam:PassedToService: ecs-tasks.amazonaws.com + - Effect: Allow + Action: ecs:TagResource + Resource: !Sub "arn:${AWS::Partition}:ecs:${AWS::Region}:${AWS::AccountId}:task/${RunnerCluster}/*" + Condition: + StringEquals: + ecs:CreateAction: RunTask + # The Fargate budgets count the cluster's tasks. + - Effect: Allow + Action: ecs:ListTasks + Resource: "*" + Condition: + ArnEquals: + ecs:cluster: !GetAtt RunnerCluster.Arn + - Effect: Allow + Action: ecs:DescribeTasks + Resource: !Sub "arn:${AWS::Partition}:ecs:${AWS::Region}:${AWS::AccountId}:task/${RunnerCluster}/*" LauncherFunction: Type: AWS::Lambda::Function Properties: FunctionName: !Sub "${AWS::StackName}-launcher" - Description: Launches one CI runner instance per still-queued job from the queue. + Description: Launches one CI runner instance or task per still-queued job from the queue. Runtime: nodejs22.x Handler: index.handler Role: !GetAtt LauncherRole.Arn @@ -322,10 +409,19 @@ Resources: RUNNER_GROUP: !Ref RunnerGroup STACK_NAME: !Ref AWS::StackName LAUNCH_TEMPLATE_ID: !Ref RunnerLaunchTemplate + ARM64_LAUNCH_TEMPLATE_ID: !Ref Arm64RunnerLaunchTemplate SUBNET_IDS: !Join [",", [!Ref Subnet0, !Ref Subnet1, !Ref Subnet2]] QUEUE_URL: !Ref JobQueue - SPOT_CAP: !Ref SpotCap - ONDEMAND_CAP: !Ref OnDemandCap + SPOT_VCPU_CAP: !Ref SpotVcpuCap + ONDEMAND_VCPU_CAP: !Ref OnDemandVcpuCap + FARGATE_SPOT_VCPU_CAP: !Ref FargateSpotVcpuCap + FARGATE_VCPU_CAP: !Ref FargateVcpuCap + CLUSTER: !Ref RunnerCluster + TASK_DEFINITION: !Ref RunnerTaskDefinition + ARM64_TASK_DEFINITION: !Ref Arm64RunnerTaskDefinition + SECURITY_GROUP_ID: !Ref RunnerSecurityGroup + # In SUBNET_IDS' order: arm64 tasks skip a subnet in use1-az3. + SUBNET_AZ_IDS: !Join [",", [!GetAtt Subnet0.AvailabilityZoneId, !GetAtt Subnet1.AvailabilityZoneId, !GetAtt Subnet2.AvailabilityZoneId]] Code: ZipFile: | //@inline functions/launcher.js @@ -379,12 +475,22 @@ Resources: Condition: StringEquals: aws:ResourceTag/ci-runner:stack: !Ref AWS::StackName + # The cluster's tasks: every one of them is the stack's. + - Effect: Allow + Action: ecs:ListTasks + Resource: "*" + Condition: + ArnEquals: + ecs:cluster: !GetAtt RunnerCluster.Arn + - Effect: Allow + Action: [ecs:DescribeTasks, ecs:StopTask] + Resource: !Sub "arn:${AWS::Partition}:ecs:${AWS::Region}:${AWS::AccountId}:task/${RunnerCluster}/*" SweeperFunction: Type: AWS::Lambda::Function Properties: FunctionName: !Sub "${AWS::StackName}-sweeper" - Description: Terminates CI runner instances over 2 hours old; removes offline runner registrations with no instance. + Description: Terminates CI runner instances and stops tasks over 2 hours old; removes offline runner registrations with neither. Runtime: nodejs22.x Handler: index.handler Role: !GetAtt SweeperRole.Arn @@ -398,6 +504,7 @@ Resources: STACK_NAME: !Ref AWS::StackName GITHUB_ORG: !Ref GitHubOrg RUNNER_GROUP: !Ref RunnerGroup + CLUSTER: !Ref RunnerCluster Code: ZipFile: | //@inline functions/sweeper.js @@ -451,12 +558,15 @@ Resources: - Effect: Allow Action: ec2:DescribeInstances Resource: "*" + - Effect: Allow + Action: ecs:DescribeTasks + Resource: !Sub "arn:${AWS::Partition}:ecs:${AWS::Region}:${AWS::AccountId}:task/${RunnerCluster}/*" NotifierFunction: Type: AWS::Lambda::Function Properties: FunctionName: !Sub "${AWS::StackName}-notifier" - Description: Reports a Spot reclaim as a failed check run. + Description: Reports a Spot reclaim, of an instance or a task, as a failed check run. Runtime: nodejs22.x Handler: index.handler Role: !GetAtt NotifierRole.Arn @@ -493,6 +603,32 @@ Resources: Principal: events.amazonaws.com SourceArn: !GetAtt NotifierRule.Arn + # A Fargate Spot reclaim: every change of the task from the warning on + # carries the stop code. ECS keeps a stopping or stopped task's tags in + # DescribeTasks (aws-emulator drops them), so every change can be read; + # the notifier reports one check run per job. + NotifierTaskRule: + Type: AWS::Events::Rule + Properties: + Name: !Sub "${AWS::StackName}-notifier-tasks" + EventPattern: + source: [aws.ecs] + detail-type: [ECS Task State Change] + detail: + clusterArn: [!GetAtt RunnerCluster.Arn] + stopCode: [SpotInterruption] + Targets: + - Id: notifier + Arn: !GetAtt NotifierFunction.Arn + + NotifierTaskRulePermission: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !Ref NotifierFunction + Action: lambda:InvokeFunction + Principal: events.amazonaws.com + SourceArn: !GetAtt NotifierTaskRule.Arn + # --- network: a VPC of its own, public subnets, no NAT -------------------- Vpc: Type: AWS::EC2::VPC @@ -612,6 +748,212 @@ Resources: Tags: - { Key: ci-runner:stack, Value: !Ref AWS::StackName } + # The arm64 counterpart, for the Graviton families: only the image differs. + Arm64RunnerLaunchTemplate: + Type: AWS::EC2::LaunchTemplate + Properties: + LaunchTemplateName: !Sub "${AWS::StackName}-runner-arm64" + LaunchTemplateData: + # Resolved by EC2 at every launch: always the latest AL2023. + ImageId: resolve:ssm:/aws/service/ami-amazon-linux-latest/al2023-ami-kernel-default-arm64 + InstanceInitiatedShutdownBehavior: terminate + SecurityGroupIds: [!Ref RunnerSecurityGroup] + BlockDeviceMappings: + - DeviceName: /dev/xvda + Ebs: { VolumeSize: 50, VolumeType: gp3, DeleteOnTermination: true, Encrypted: true } + # IMDSv2 only, one hop: a job's containers can't reach the metadata service. + MetadataOptions: + HttpEndpoint: enabled + HttpTokens: required + HttpPutResponseHopLimit: 1 + TagSpecifications: + - ResourceType: instance + Tags: + - { Key: ci-runner:stack, Value: !Ref AWS::StackName } + - { Key: Name, Value: !Sub "${AWS::StackName}-runner" } + - ResourceType: volume + Tags: + - { Key: ci-runner:stack, Value: !Ref AWS::StackName } + + # --- Fargate: the cluster and task definitions of the Fargate capacities -- + # No Docker daemon (ADR-0001) and no task role: a job finds no AWS + # credentials, as on EC2. The launcher runs one task per job. + RunnerCluster: + Type: AWS::ECS::Cluster + Properties: + ClusterName: !Ref AWS::StackName + CapacityProviders: [FARGATE, FARGATE_SPOT] + Tags: + - { Key: ci-runner:stack, Value: !Ref AWS::StackName } + + # The runner's output (Listening for Jobs, Running job), per task. + RunnerTaskLogGroup: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: !Sub "/ecs/${AWS::StackName}-runner" + RetentionInDays: 7 + + # Used by ECS to start a task, never by the job: pull the image, write the log. + RunnerExecutionRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Sub "PowerUserPB-${AWS::StackName}-runner-exec" + PermissionsBoundary: arn:aws:iam::aws:policy/PowerUserAccess + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: { Service: ecs-tasks.amazonaws.com } + Action: sts:AssumeRole + Condition: + StringEquals: + aws:SourceAccount: !Ref AWS::AccountId + Policies: + - PolicyName: !Sub "PowerUserPB-${AWS::StackName}-runner-exec" + PolicyDocument: + Version: "2012-10-17" + Statement: + # The registry login supports no resource-level permissions. + - Effect: Allow + Action: ecr:GetAuthorizationToken + Resource: "*" + - Effect: Allow + Action: [ecr:BatchGetImage, ecr:GetDownloadUrlForLayer, ecr:BatchCheckLayerAvailability] + Resource: !GetAtt ImageRepository.Arn + - Effect: Allow + Action: [logs:CreateLogStream, logs:PutLogEvents] + Resource: !GetAtt RunnerTaskLogGroup.Arn + + # x64 and arm64 are two task definitions: RunTask can't override the platform. + RunnerTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub "${AWS::StackName}-runner" + RequiresCompatibilities: [FARGATE] + NetworkMode: awsvpc + # The 4-vCPU shape; each RunTask overrides cpu and memory for its size. + Cpu: "4096" + Memory: "16384" + EphemeralStorage: { SizeInGiB: 50 } + RuntimePlatform: { OperatingSystemFamily: LINUX, CpuArchitecture: X86_64 } + ExecutionRoleArn: !GetAtt RunnerExecutionRole.Arn + ContainerDefinitions: + - Name: runner + Image: !Sub "${ImageRepository.RepositoryUri}:latest" + Essential: true + # The image's idle exit; the launcher's command is ./run.sh --jitconfig. + EntryPoint: [/usr/local/bin/ci-runner-idle-exit] + Command: [./run.sh] + LinuxParameters: { InitProcessEnabled: true } + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref RunnerTaskLogGroup + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: runner + Tags: + - { Key: ci-runner:stack, Value: !Ref AWS::StackName } + + # The arm64 counterpart: only the architecture differs. + Arm64RunnerTaskDefinition: + Type: AWS::ECS::TaskDefinition + Properties: + Family: !Sub "${AWS::StackName}-runner-arm64" + RequiresCompatibilities: [FARGATE] + NetworkMode: awsvpc + # The 4-vCPU shape; each RunTask overrides cpu and memory for its size. + Cpu: "4096" + Memory: "16384" + EphemeralStorage: { SizeInGiB: 50 } + RuntimePlatform: { OperatingSystemFamily: LINUX, CpuArchitecture: ARM64 } + ExecutionRoleArn: !GetAtt RunnerExecutionRole.Arn + ContainerDefinitions: + - Name: runner + Image: !Sub "${ImageRepository.RepositoryUri}:latest" + Essential: true + # The image's idle exit; the launcher's command is ./run.sh --jitconfig. + EntryPoint: [/usr/local/bin/ci-runner-idle-exit] + Command: [./run.sh] + LinuxParameters: { InitProcessEnabled: true } + LogConfiguration: + LogDriver: awslogs + Options: + awslogs-group: !Ref RunnerTaskLogGroup + awslogs-region: !Ref AWS::Region + awslogs-stream-prefix: runner + Tags: + - { Key: ci-runner:stack, Value: !Ref AWS::StackName } + + # --- image: the runner image in ECR, pushed by image.yml on main ---------- + # Fargate lazy-loads only from private ECR (ADR-0002). image.yml assumes + # the push role through GitHub OIDC: no stored key or token anywhere. + GitHubOidcProvider: + Type: AWS::IAM::OIDCProvider + Condition: CreateProvider + Properties: + Url: https://token.actions.githubusercontent.com + ClientIdList: [sts.amazonaws.com] + + ImagePushRole: + Type: AWS::IAM::Role + Properties: + RoleName: !Sub "PowerUserPB-${AWS::StackName}-image-push" + PermissionsBoundary: arn:aws:iam::aws:policy/PowerUserAccess + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + # Only image.yml on this repo's main (a push, the schedule or a + # dispatch there). Any other branch, a pull request, a tag or + # another repo has a different sub. + - Effect: Allow + Principal: + Federated: !If [CreateProvider, !Ref GitHubOidcProvider, !Ref ExistingOidcProviderArn] + Action: sts:AssumeRoleWithWebIdentity + # zeroae/ci-runner uses GitHub's immutable subject, so its token's + # sub names the owner and repo by id (owner 11018406, repo + # 1455021495), not by name. A rename or a re-created repo with the + # same name doesn't match. Read it with + # `gh api repos/zeroae/ci-runner/actions/oidc/customization/sub`. + Condition: + StringEquals: + token.actions.githubusercontent.com:aud: sts.amazonaws.com + token.actions.githubusercontent.com:sub: repo:zeroae@11018406/ci-runner@1455021495:ref:refs/heads/main + Policies: + - PolicyName: !Sub "PowerUserPB-${AWS::StackName}-image-push" + PolicyDocument: + Version: "2012-10-17" + Statement: + # The registry login supports no resource-level permissions. + - Effect: Allow + Action: ecr:GetAuthorizationToken + Resource: "*" + # Push, and read back what it pushed, on this repo only. + - Effect: Allow + Action: + - ecr:BatchCheckLayerAvailability + - ecr:InitiateLayerUpload + - ecr:UploadLayerPart + - ecr:CompleteLayerUpload + - ecr:PutImage + - ecr:BatchGetImage + - ecr:GetDownloadUrlForLayer + Resource: !GetAtt ImageRepository.Arn + + # Each build tags one image index (:latest and :-), and the + # rule keeps the last 4 of those. It counts only tagged images: the index's + # children (each architecture's manifest and SOCI index) are untagged, and + # an untagged rule could expire the children of an index still in use. + ImageRepository: + Type: AWS::ECR::Repository + Properties: + RepositoryName: !Ref AWS::StackName + EmptyOnDelete: true + LifecyclePolicy: + LifecyclePolicyText: | + {"rules": [{"rulePriority": 1, "description": "keep the last 4 builds", + "selection": {"tagStatus": "tagged", "tagPatternList": ["*"], "countType": "imageCountMoreThan", "countNumber": 4}, + "action": {"type": "expire"}}]} + Outputs: SetupUrl: Description: Open in a browser to create the GitHub App (refuses everything once done). @@ -622,6 +964,8 @@ Outputs: Value: !Ref AppSecret LaunchTemplateId: Value: !Ref RunnerLaunchTemplate + Arm64LaunchTemplateId: + Value: !Ref Arm64RunnerLaunchTemplate SubnetIds: Value: !Join [",", [!Ref Subnet0, !Ref Subnet1, !Ref Subnet2]] SecurityGroupId: @@ -630,3 +974,15 @@ Outputs: Value: !Ref JobQueue JobDeadLetterQueueUrl: Value: !Ref JobDeadLetterQueue + ImageRepositoryUri: + Description: The repo variable ECR_IMAGE, which image.yml pushes to on main. + Value: !GetAtt ImageRepository.RepositoryUri + ImagePushRoleArn: + Description: The repo variable AWS_IMAGE_PUSH_ROLE_ARN, which image.yml assumes on main. + Value: !GetAtt ImagePushRole.Arn + ClusterName: + Value: !Ref RunnerCluster + TaskDefinitionArn: + Value: !Ref RunnerTaskDefinition + Arm64TaskDefinitionArn: + Value: !Ref Arm64RunnerTaskDefinition diff --git a/test/aws-emulator.js b/test/aws-emulator.js new file mode 100644 index 0000000..533cd50 --- /dev/null +++ b/test/aws-emulator.js @@ -0,0 +1,103 @@ +'use strict'; +// aws/aws-emulator (https://github.com/aws/aws-emulator), for the tests that +// need AWS's real API shapes: it validates a request as the service does and +// answers as it would. Not a test file itself (`make test` runs *.test.js). +// A test is skipped when `aws-emulator` isn't installed, unless +// REQUIRE_AWS_EMULATOR is set (CI sets it). +const { spawn, spawnSync } = require('node:child_process'); +const fs = require('node:fs'); +const https = require('node:https'); +const readline = require('node:readline'); + +const installed = spawnSync('aws-emulator', ['-v']).status === 0; +if (!installed && process.env.REQUIRE_AWS_EMULATOR) throw new Error('aws-emulator is not installed'); +const skip = !installed && 'aws-emulator is not installed'; + +const auth = (service) => `AWS4-HMAC-SHA256 Credential=test/20261112/us-east-1/${service}/aws4_request, SignedHeaders=host, Signature=0`; + +// Starts one emulator; `stop()` it after the tests. +async function startEmulator() { + const proc = spawn('aws-emulator', ['-p', '0', '-a', 'off', '-t', 'off', '--idle-timeout', '300'], { stdio: ['ignore', 'pipe', 'inherit'] }); + let endpoint; + let ca; + // The startup line names the endpoint and its CA. + for await (const line of readline.createInterface({ input: proc.stdout })) { + const m = line.startsWith('{') && JSON.parse(line).fields; + if (m && m.endpoint_url) { endpoint = m.endpoint_url; ca = fs.readFileSync(m.ca_file); break; } + } + proc.stdout.resume(); + + // One request; `headers` pick the protocol. + function call(body, headers) { + const url = new URL(endpoint); + return new Promise((resolve, reject) => { + const req = https.request({ + host: url.hostname, port: url.port, path: '/', method: 'POST', ca, + headers: { ...headers, 'content-length': Buffer.byteLength(body) }, + }, (res) => { + const chunks = []; + res.on('data', (c) => chunks.push(c)); + res.on('end', () => resolve({ status: res.statusCode, text: Buffer.concat(chunks).toString() })); + }); + req.on('error', reject); + req.end(body); + }); + } + + // A JSON-protocol service, as the SDK's clients send it. + const json = (service, target, version) => async (op, input) => { + const r = await call(JSON.stringify(input), { + 'content-type': `application/x-amz-json-${version}`, 'x-amz-target': `${target}.${op}`, authorization: auth(service), + }); + const out = JSON.parse(r.text || '{}'); + if (r.status >= 300) throw Object.assign(new Error(`${out.__type}: ${out.message || out.Message}`), { name: out.__type }); + return out; + }; + + // EC2 speaks the query protocol; it answers XML. + async function ec2(params) { + const r = await call(new URLSearchParams({ Version: '2016-11-15', ...params }).toString(), { + 'content-type': 'application/x-www-form-urlencoded', authorization: auth('ec2'), + }); + if (r.status >= 300) throw new Error(r.text); + return r.text; + } + + return { + ecs: json('ecs', 'AmazonEC2ContainerServiceV20141113', '1.1'), + events: json('events', 'AWSEvents', '1.1'), + sqs: json('sqs', 'AmazonSQS', '1.0'), + ec2, + stop: () => proc.kill('SIGTERM'), + }; +} + +const idIn = (xml, tag) => xml.match(new RegExp(`<${tag}>([^<]+)`))[1]; + +// The stack's network as template.yaml makes it, for a task's awsvpc +// configuration: three subnets in these AZs and a security group. +async function network(aws, azs) { + const vpc = idIn(await aws.ec2({ Action: 'CreateVpc', CidrBlock: '10.77.0.0/16' }), 'vpcId'); + const subnets = []; + for (const [i, az] of azs.entries()) { + subnets.push(idIn(await aws.ec2({ Action: 'CreateSubnet', VpcId: vpc, CidrBlock: `10.77.${i}.0/24`, AvailabilityZoneId: az }), 'subnetId')); + } + const sg = idIn(await aws.ec2({ Action: 'CreateSecurityGroup', VpcId: vpc, GroupName: 'ci-runner', GroupDescription: 'ci-runner' }), 'groupId'); + return { subnets, sg }; +} + +// The runner's task definition as template.yaml makes it. +async function taskDefinition(aws, family, cpuArchitecture) { + const r = await aws.ecs('RegisterTaskDefinition', { + family, networkMode: 'awsvpc', requiresCompatibilities: ['FARGATE'], cpu: '4096', memory: '16384', + ephemeralStorage: { sizeInGiB: 50 }, runtimePlatform: { operatingSystemFamily: 'LINUX', cpuArchitecture }, + executionRoleArn: 'arn:aws:iam::000000000000:role/PowerUserPB-ci-runner-runner-exec', + containerDefinitions: [{ + name: 'runner', image: '000000000000.dkr.ecr.us-east-1.amazonaws.com/ci-runner:latest', essential: true, + entryPoint: ['/usr/local/bin/ci-runner-idle-exit'], command: ['./run.sh'], linuxParameters: { initProcessEnabled: true }, + }], + }); + return r.taskDefinition.taskDefinitionArn; +} + +module.exports = { skip, startEmulator, network, taskDefinition }; diff --git a/test/build.test.js b/test/build.test.js index 964f893..fed8351 100644 --- a/test/build.test.js +++ b/test/build.test.js @@ -57,7 +57,7 @@ test('whole-line YAML comments are dropped, but not inside a block scalar', () = 'Description: >-\n text\n # kept: block content\nA:\n B: 1 # trailing stays\n'); }); -test('the real template embeds every function, each runnable alone, under the inline size limit', () => { +test('the real template embeds every function, each runnable alone, under the S3 deploy\'s 1 MB limit', () => { const out = build(path.join(root, 'template.yaml')); assert.doesNotMatch(out, /^ *\/\/@inline /m); const zips = zipFiles(out); @@ -69,6 +69,6 @@ test('the real template embeds every function, each runnable alone, under the in assert.ok(zip, `${fn} not inlined`); assert.equal(typeof load(zip.code).handler, 'function', `${fn} has no handler when inlined`); } - assert.equal(LIMIT, 51200); + assert.equal(LIMIT, 1048576); assert.ok(Buffer.byteLength(out) < LIMIT); }); diff --git a/test/conventions.test.js b/test/conventions.test.js index 9c63c25..10bf20c 100644 --- a/test/conventions.test.js +++ b/test/conventions.test.js @@ -17,7 +17,8 @@ const resources = template.split(/^Resources:$/m)[1].split(/^\w/m)[0] .map((b) => ({ name: b.match(/^ {2}(\w+):/)[1], type: b.match(/Type: (\S+)/)[1], body: b })); test('every IAM role and policy is PowerUserPB-${AWS::StackName}-*, and every role has the boundary', () => { - const iam = resources.filter((r) => r.type.startsWith('AWS::IAM::')); + // GitHub's OIDC provider is named by its URL, and has no policy. + const iam = resources.filter((r) => r.type.startsWith('AWS::IAM::') && r.type !== 'AWS::IAM::OIDCProvider'); assert.ok(iam.length > 0); for (const r of iam) { assert.match(r.type, /^AWS::IAM::(Role|ManagedPolicy)$/, `${r.name}: inline the policy in its role instead`); @@ -25,7 +26,13 @@ test('every IAM role and policy is PowerUserPB-${AWS::StackName}-*, and every ro assert.ok(names.length > 0, `${r.name} has no explicit name`); for (const n of names) assert.match(n, /^!Sub "PowerUserPB-\$\{AWS::StackName\}-[\w-]+"$/, `${r.name}: ${n}`); if (r.type === 'AWS::IAM::Role') { - assert.match(r.body, /PermissionsBoundary: arn:aws:iam::aws:policy\/PowerUserAccess\n/, `${r.name} has no boundary`); + // PowerUserAccess allows no iam:PassRole, so a role that passes one has + // the account's ZAEPowerUserBoundary: PowerUserAccess plus PassRole on + // PowerUserPB* roles only. + const boundary = /Action: iam:PassRole\n/.test(r.body) + ? /PermissionsBoundary: !Sub "arn:\$\{AWS::Partition\}:iam::\$\{AWS::AccountId\}:policy\/ZAEPowerUserBoundary"\n/ + : /PermissionsBoundary: arn:aws:iam::aws:policy\/PowerUserAccess\n/; + assert.match(r.body, boundary, `${r.name} has no boundary, or the wrong one`); } } assert.match(read('Makefile'), /--capabilities CAPABILITY_NAMED_IAM/); @@ -39,10 +46,10 @@ test('every env var a function reads is set on that function in the template', ( assert.equal(fns.length, 5); for (const fn of fns) { const file = fn.body.match(/\/\/@inline (\S+)/)[1]; - const set = new Set([...(fn.body.split(/Variables:\n/)[1] || '').matchAll(/^ {10}([A-Z_]+):/gm)].map((m) => m[1])); + const set = new Set([...(fn.body.split(/Variables:\n/)[1] || '').matchAll(/^ {10}([A-Z0-9_]+):/gm)].map((m) => m[1])); let src = read(file); for (const [, dep] of src.matchAll(/require\('\.\/([\w.-]+\.js)'\)/g)) src += read(path.join(path.dirname(file), dep)); - const used = new Set([...src.matchAll(/\benv\.([A-Z_]+)/g)].map((m) => m[1])); + const used = new Set([...src.matchAll(/\benv\.([A-Z0-9_]+)/g)].map((m) => m[1])); for (const v of used) if (!OPTIONAL_ENV.has(v)) assert.ok(set.has(v), `${file} reads ${v}, which ${fn.name} doesn't set`); for (const v of set) assert.ok(used.has(v), `${fn.name} sets ${v}, which ${file} never reads`); } @@ -63,3 +70,13 @@ test('the launcher, notifier and sweeper carry identical copies of the GitHub he for (const o of rest) assert.equal(o.code, first.code, `${o.f}'s ${name} differs from ${first.f}'s`); } }); + +test('the launcher and the webhook sanitize an ECS tag the same way', () => { + // The webhook re-tags a task the launcher tagged; bundling one rule would + // cost more template bytes than the two-line copy. + for (const re of [/^const ECS_TAG_CHARS = .*$/m, /^const ecsTag = .*$/m]) { + const [launcher, webhook] = ['launcher', 'webhook'].map((f) => (read(`functions/${f}.js`).match(re) || [])[0]); + assert.ok(launcher, `the launcher has no ${re}`); + assert.equal(webhook, launcher, `the webhook's ${re} differs from the launcher's`); + } +}); diff --git a/test/launcher-ecs.test.js b/test/launcher-ecs.test.js new file mode 100644 index 0000000..2b7d290 --- /dev/null +++ b/test/launcher-ecs.test.js @@ -0,0 +1,125 @@ +'use strict'; +// The launcher's RunTask against real ECS shapes, from aws/aws-emulator +// (./aws-emulator.js): the emulator validates the request as ECS does +// (Fargate cpu/memory pairs, capacity providers on the cluster, subnets that +// exist, the characters a tag may carry) and answers with a real Task. +const { test, before, after } = require('node:test'); +const assert = require('node:assert/strict'); +const { skip, startEmulator, network, taskDefinition } = require('./aws-emulator.js'); +const { createHandler } = require('../functions/launcher.js'); + +let aws; +const ecsCall = (op, input) => aws.ecs(op, input); +const ecs = { + runTask: (i) => ecsCall('RunTask', i), + listTasks: (i) => ecsCall('ListTasks', i), + describeTasks: (i) => ecsCall('DescribeTasks', i), +}; + +// The stack's shape, as template.yaml makes it: three subnets (one in +// use1-az3, to see arm64 leave it out), a security group, the cluster with +// both capacity providers, and the two task definitions. +const env = { + GITHUB_ORG: 'zeroae', RUNNER_GROUP: 'ci-runner', STACK_NAME: 'ci-runner', CLUSTER: 'ci-runner', + QUEUE_URL: 'https://sqs.example/jobs', SPOT_VCPU_CAP: '24', ONDEMAND_VCPU_CAP: '8', + FARGATE_SPOT_VCPU_CAP: '64', FARGATE_VCPU_CAP: '64', SUBNET_AZ_IDS: 'use1-az1,use1-az2,use1-az3', +}; +const subnetAz = {}; + +before(async () => { + if (skip) return; + aws = await startEmulator(); + const azs = env.SUBNET_AZ_IDS.split(','); + const net = await network(aws, azs); + net.subnets.forEach((id, i) => { subnetAz[id] = azs[i]; }); + env.SUBNET_IDS = net.subnets.join(','); + env.SECURITY_GROUP_ID = net.sg; + await ecsCall('CreateCluster', { clusterName: 'ci-runner', capacityProviders: ['FARGATE', 'FARGATE_SPOT'] }); + env.TASK_DEFINITION = await taskDefinition(aws, 'ci-runner-runner', 'X86_64'); + env.ARM64_TASK_DEFINITION = await taskDefinition(aws, 'ci-runner-runner-arm64', 'ARM64'); +}); + +after(() => { if (aws) aws.stop(); }); + +// GitHub, faked: the job is queued, the group exists, the JIT config mints. +function fakeFetch() { + const calls = []; + const fetch = async (url, init = {}) => { + const key = `${init.method || 'GET'} ${new URL(url).pathname}`; + calls.push(key); + const res = (status, json) => ({ ok: status < 300, status, json: async () => json }); + if (key.endsWith('/access_tokens')) return res(201, { token: 't', expires_at: new Date(Date.now() + 3600e3).toISOString() }); + if (key.includes('/actions/jobs/')) return res(200, { status: 'queued' }); + if (key.endsWith('/runner-groups')) return res(200, { runner_groups: [{ id: 3, name: 'ci-runner' }] }); + if (key.endsWith('/generate-jitconfig')) return res(201, { runner: { id: 77 }, encoded_jit_config: 'ZW5jb2RlZGppdA==' }); + if (key.startsWith('DELETE ')) return res(204, null); + throw new Error('unexpected ' + key); + }; + return { fetch, calls }; +} + +const { privateKey } = require('node:crypto').generateKeyPairSync('rsa', { modulusLength: 2048 }); +function launcher(gh) { + const logs = []; + const handler = createHandler({ + secret: { get: async () => ({ app_id: 42, private_key: privateKey.export({ type: 'pkcs1', format: 'pem' }) }) }, + fetch: gh.fetch, ecs, ec2: {}, sqs: { sendMessage: async () => ({}) }, now: Date.now, env, log: (l) => logs.push(l), + }); + return { handler, logs }; +} +const job = (label, vcpus, arch, extra = {}) => ({ Records: [{ messageId: 'm0', body: JSON.stringify({ + repo: 'zeroae/app', job_id: 555, run_id: 444, head_sha: 'abc123', job_name: 'build (x64, release)', + label, vcpus, arch, labels: ['self-hosted', label, `ci-cpu-${vcpus}`, arch], installation_id: 181, ...extra, +}) }] }); + +test('ECS accepts the launcher\'s RunTask for every Fargate capacity, size and architecture', { skip }, async () => { + for (const [label, provider] of [['ci-fargate-spot', 'FARGATE_SPOT'], ['ci-fargate', 'FARGATE']]) { + for (const vcpus of [2, 4, 8, 16]) { + for (const arch of ['x64', 'arm64']) { + const what = `${label} ${vcpus} ${arch}`; + const { handler, logs } = launcher(fakeFetch()); + assert.deepEqual(await handler(job(label, vcpus, arch)), { batchItemFailures: [] }, `${what}: ${logs.join('\n')}`); + const arn = JSON.parse(logs.find((l) => l.includes('"launched"'))).launched; + const [task] = (await ecs.describeTasks({ cluster: 'ci-runner', tasks: [arn], include: ['TAGS'] })).tasks; + assert.equal(task.capacityProviderName, provider, what); + assert.equal(task.taskDefinitionArn, arch === 'arm64' ? env.ARM64_TASK_DEFINITION : env.TASK_DEFINITION, what); + assert.deepEqual([task.overrides.cpu, task.overrides.memory], [String(vcpus * 1024), String(vcpus * 4096)], what); + const tags = Object.fromEntries(task.tags.map((t) => [t.key, t.value])); + assert.equal(tags['ci-runner:stack'], 'ci-runner', what); + assert.equal(tags['ci-runner:vcpus'], String(vcpus), what); + const subnet = task.attachments[0].details.find((d) => d.name === 'subnetId').value; + if (arch === 'arm64') assert.notEqual(subnetAz[subnet], 'use1-az3', what); + } + } + } +}); + +test('ECS refuses a size Fargate doesn\'t offer, and the launcher unregisters the runner', { skip }, async () => { + // A 32-vCPU message can't come from the webhook; it stands in for any + // RunTask that ECS refuses. + const gh = fakeFetch(); + const { handler: h } = launcher(gh); + const refused = await ecs.runTask({ + cluster: 'ci-runner', taskDefinition: env.TASK_DEFINITION, capacityProviderStrategy: [{ capacityProvider: 'FARGATE', weight: 1 }], + networkConfiguration: { awsvpcConfiguration: { subnets: env.SUBNET_IDS.split(','), securityGroups: [env.SECURITY_GROUP_ID] } }, + overrides: { cpu: '2048', memory: '3000' }, + }).then(() => null, (e) => e); + assert.match(String(refused), /InvalidParameterException/); + // A capacity provider the cluster lacks fails the launcher's own call the same way. + const saved = env.CLUSTER; + await ecsCall('CreateCluster', { clusterName: 'no-providers' }); + env.CLUSTER = 'no-providers'; + try { + assert.deepEqual(await h(job('ci-fargate-spot', 4, 'x64')), { batchItemFailures: [{ itemIdentifier: 'm0' }] }); + } finally { env.CLUSTER = saved; } + assert.ok(gh.calls.includes('DELETE /orgs/zeroae/actions/runners/77')); +}); + +test('ECS refuses a comma in a tag, which the launcher\'s task tags never carry', { skip }, async () => { + const err = await ecs.runTask({ + cluster: 'ci-runner', taskDefinition: env.TASK_DEFINITION, capacityProviderStrategy: [{ capacityProvider: 'FARGATE', weight: 1 }], + networkConfiguration: { awsvpcConfiguration: { subnets: env.SUBNET_IDS.split(','), securityGroups: [env.SECURITY_GROUP_ID] } }, + tags: [{ key: 'ci-runner:labels', value: 'self-hosted,ci-fargate' }], + }).then(() => null, (e) => e); + assert.match(String(err), /invalid characters/); +}); diff --git a/test/launcher.test.js b/test/launcher.test.js index fe51bc2..92a105b 100644 --- a/test/launcher.test.js +++ b/test/launcher.test.js @@ -9,14 +9,18 @@ const { privateKey, publicKey } = crypto.generateKeyPairSync('rsa', { modulusLen const PEM = privateKey.export({ type: 'pkcs1', format: 'pem' }); const ENV = { GITHUB_ORG: 'zeroae', RUNNER_GROUP: 'ci-runner', STACK_NAME: 'ci-runner', - LAUNCH_TEMPLATE_ID: 'lt-0abc', SUBNET_IDS: 'subnet-a,subnet-b,subnet-c', - QUEUE_URL: 'https://sqs.example/jobs', SPOT_CAP: '6', ONDEMAND_CAP: '2', + LAUNCH_TEMPLATE_ID: 'lt-0abc', ARM64_LAUNCH_TEMPLATE_ID: 'lt-0arm', SUBNET_IDS: 'subnet-a,subnet-b,subnet-c', + QUEUE_URL: 'https://sqs.example/jobs', SPOT_VCPU_CAP: '24', ONDEMAND_VCPU_CAP: '8', + CLUSTER: 'ci-runner', TASK_DEFINITION: 'arn:aws:ecs:us-east-1:1:task-definition/ci-runner-runner:3', + ARM64_TASK_DEFINITION: 'arn:aws:ecs:us-east-1:1:task-definition/ci-runner-runner-arm64:3', + SECURITY_GROUP_ID: 'sg-0run', SUBNET_AZ_IDS: 'use1-az1,use1-az2,use1-az4', + FARGATE_SPOT_VCPU_CAP: '24', FARGATE_VCPU_CAP: '8', }; const JIT = 'ZW5jb2RlZGppdA=='; const MSG = { repo: 'zeroae/app', job_id: 555, run_id: 444, head_sha: 'abc123', job_name: 'tests', - label: 'ci-spot', labels: ['self-hosted', 'ci-spot'], installation_id: 181, + label: 'ci-spot', vcpus: 4, arch: 'x64', labels: ['self-hosted', 'ci-spot'], installation_id: 181, }; function verifyJwt(auth) { @@ -52,14 +56,17 @@ function fakeGitHub({ jobStatus = 'queued', jobHttp = 200, jit = { status: 201 } return { fetch, calls, keys: () => calls.map((c) => c.key) }; } -// `running` lists the stack's instances: { lifecycle: 'spot' | undefined, state }. +// `running` lists the stack's instances: { lifecycle: 'spot' | undefined, state, +// and the shape tags capacity, vcpus, arch, each left off when undefined }. function fakeEc2({ fleet = { Instances: [{ InstanceIds: ['i-0123'] }], Errors: [] }, running = [] } = {}) { const calls = []; return { calls, describeInstances: async (input) => { calls.push(['describe', input]); - const all = running.map((r, i) => ({ InstanceId: `i-r${i}`, State: { Name: r.state || 'running' }, ...(r.lifecycle ? { InstanceLifecycle: r.lifecycle } : {}) })); + const all = running.map((r, i) => ({ InstanceId: `i-r${i}`, State: { Name: r.state || 'running' }, ...(r.lifecycle ? { InstanceLifecycle: r.lifecycle } : {}), + Tags: [['ci-runner:stack', 'ci-runner'], ['ci-runner:capacity', r.capacity], ['ci-runner:vcpus', r.vcpus], ['ci-runner:arch', r.arch]] + .filter(([, v]) => v !== undefined).map(([Key, Value]) => ({ Key, Value: String(Value) })) })); // Two pages, to check NextToken is followed. if (!input.NextToken) return { Reservations: [{ Instances: all.slice(0, 1) }], NextToken: all.length > 1 ? 'p2' : undefined }; return { Reservations: [{ Instances: all.slice(1) }] }; @@ -70,15 +77,48 @@ function fakeEc2({ fleet = { Instances: [{ InstanceIds: ['i-0123'] }], Errors: [ }; } -function setup({ gh = fakeGitHub(), ec2 = fakeEc2(), env = ENV } = {}) { +// `running` lists the cluster's tasks, as { capacity, vcpus, desired, last } with +// the launcher's shape tags (left off when undefined). `run` is RunTask's +// answer; by default one task. +function fakeEcs({ running = [], run } = {}) { + const calls = []; + const all = running.map((t, i) => ({ + taskArn: `arn:aws:ecs:us-east-1:1:task/ci-runner/t${i}`, desiredStatus: t.desired || 'RUNNING', lastStatus: t.last || 'RUNNING', + tags: [['ci-runner:stack', 'ci-runner'], ['ci-runner:capacity', t.capacity], ['ci-runner:vcpus', t.vcpus]] + .filter(([, v]) => v !== undefined).map(([key, value]) => ({ key, value: String(value) })), + })); + return { + calls, + runTask: async (input) => { + calls.push(['runTask', input]); + if (run instanceof Error) throw run; + return run || { tasks: [{ taskArn: 'arn:aws:ecs:us-east-1:1:task/ci-runner/new1', lastStatus: 'PROVISIONING' }], failures: [] }; + }, + // Pages of one, to check nextToken is followed. + listTasks: async (input) => { + calls.push(['listTasks', input]); + const want = all.filter((t) => t.desiredStatus === (input.desiredStatus || 'RUNNING')); + const i = input.nextToken ? Number(input.nextToken) : 0; + return { taskArns: want.slice(i, i + 1).map((t) => t.taskArn), ...(i + 1 < want.length ? { nextToken: String(i + 1) } : {}) }; + }, + describeTasks: async (input) => { + calls.push(['describeTasks', input]); + const tasks = all.filter((t) => input.tasks.includes(t.taskArn)) + .map((t) => (input.include || []).includes('TAGS') ? t : { ...t, tags: undefined }); + return { tasks, failures: [] }; + }, + }; +} + +function setup({ gh = fakeGitHub(), ec2 = fakeEc2(), ecs = fakeEcs(), env = ENV } = {}) { const logs = []; const sent = []; const sqs = { sendMessage: async (input) => { sent.push(input); return {}; } }; const handler = createHandler({ secret: { get: async () => ({ app_id: 42, private_key: PEM, webhook_secret: 'w' }) }, - fetch: gh.fetch, ec2, sqs, now: () => T0, env, log: (l) => logs.push(l), + fetch: gh.fetch, ec2, ecs, sqs, now: () => T0, env, log: (l) => logs.push(l), }); - return { handler, gh, ec2, sqs: sent, logs }; + return { handler, gh, ec2, ecs, sqs: sent, logs }; } const sqsEvent = (...msgs) => ({ Records: msgs.map((m, i) => ({ messageId: `m${i}`, body: JSON.stringify(m) })) }); @@ -130,12 +170,14 @@ test('a queued job launches one Spot instance from the launch template, tagged w assert.deepEqual(fleet.SpotOptions, { AllocationStrategy: 'capacity-optimized' }); const cfg = fleet.LaunchTemplateConfigs[0]; assert.deepEqual(cfg.LaunchTemplateSpecification, { LaunchTemplateId: 'lt-0abc', Version: '9' }); - assert.equal(cfg.Overrides.length, 12); - assert.deepEqual(new Set(cfg.Overrides.map((o) => o.InstanceType)), new Set(['m7i.xlarge', 'm6i.xlarge', 'm6a.xlarge', 'm7a.xlarge'])); + assert.equal(cfg.Overrides.length, 18); + assert.deepEqual(new Set(cfg.Overrides.map((o) => o.InstanceType)), new Set(['m8i.xlarge', 'm8a.xlarge', 'm7i.xlarge', 'm6i.xlarge', 'm6a.xlarge', 'm7a.xlarge'])); assert.deepEqual(new Set(cfg.Overrides.map((o) => o.SubnetId)), new Set(['subnet-a', 'subnet-b', 'subnet-c'])); assert.deepEqual(tags(fleet), { 'ci-runner:runner': 'ci-runner-20261112120000-555', 'ci-runner:runner-id': '77', 'ci-runner:label': 'ci-spot', 'ci-runner:repo': 'zeroae/app', 'ci-runner:job-id': '555', 'ci-runner:run-id': '444', 'ci-runner:head-sha': 'abc123', 'ci-runner:job-name': 'tests', + 'ci-runner:labels': 'self-hosted,ci-spot', + 'ci-runner:capacity': 'ci-spot', 'ci-runner:vcpus': '4', 'ci-runner:arch': 'x64', }); assert.deepEqual(ec2.calls[3][1], { LaunchTemplateId: 'lt-0abc', Versions: ['9'] }); @@ -143,6 +185,82 @@ test('a queued job launches one Spot instance from the launch template, tagged w assert.ok(!logs.some((l) => l.includes(JIT) || l.includes('ghs_install')), 'no secrets in the logs'); }); +test('each size launches its instance size across the four x64 families in the three subnets', async () => { + const sizes = { 2: 'large', 4: 'xlarge', 8: '2xlarge', 16: '4xlarge' }; + for (const [vcpus, size] of Object.entries(sizes)) { + const { handler, ec2 } = setup(); + await handler(sqsEvent({ ...MSG, vcpus: Number(vcpus), labels: ['self-hosted', 'ci-spot', `ci-cpu-${vcpus}`, 'x64'] })); + const fleet = ec2.calls.find((c) => c[0] === 'createFleet')[1]; + const overrides = fleet.LaunchTemplateConfigs[0].Overrides; + const want = ['m8i', 'm8a', 'm7i', 'm6i', 'm6a', 'm7a'].flatMap((f) => ['subnet-a', 'subnet-b', 'subnet-c'].map((s) => `${f}.${size} ${s}`)); + assert.deepEqual(overrides.map((o) => `${o.InstanceType} ${o.SubnetId}`).sort(), want.sort(), `ci-cpu-${vcpus}`); + assert.deepEqual( + [tags(fleet)['ci-runner:capacity'], tags(fleet)['ci-runner:vcpus'], tags(fleet)['ci-runner:arch']], + ['ci-spot', vcpus, 'x64'], + ); + } +}); + +test('an arm64 job launches from the arm64 launch template, across the three Graviton families at its size', async () => { + const sizes = { 2: 'large', 4: 'xlarge', 8: '2xlarge', 16: '4xlarge' }; + for (const label of ['ci-spot', 'ci-ondemand']) { + for (const [vcpus, size] of Object.entries(sizes)) { + // Budgets wide enough for a 16-vCPU On-Demand job. + const { handler, ec2, logs } = setup({ env: { ...ENV, SPOT_VCPU_CAP: '64', ONDEMAND_VCPU_CAP: '64' } }); + const m = { ...MSG, label, vcpus: Number(vcpus), arch: 'arm64', labels: ['self-hosted', label, `ci-cpu-${vcpus}`, 'arm64'] }; + assert.deepEqual(await handler(sqsEvent(m)), { batchItemFailures: [] }); + const [version, fleet, deleted] = ['createVersion', 'createFleet', 'deleteVersions'].map((k) => ec2.calls.find((c) => c[0] === k)[1]); + assert.equal(version.LaunchTemplateId, 'lt-0arm'); + assert.deepEqual(fleet.LaunchTemplateConfigs[0].LaunchTemplateSpecification, { LaunchTemplateId: 'lt-0arm', Version: '9' }); + assert.deepEqual(deleted, { LaunchTemplateId: 'lt-0arm', Versions: ['9'] }); + const want = ['m7g', 'm6g', 'm8g'].flatMap((f) => ['subnet-a', 'subnet-b', 'subnet-c'].map((s) => `${f}.${size} ${s}`)); + const overrides = fleet.LaunchTemplateConfigs[0].Overrides; + assert.deepEqual(overrides.map((o) => `${o.InstanceType} ${o.SubnetId}`).sort(), want.sort(), `${label} ci-cpu-${vcpus}`); + assert.equal(tags(fleet)['ci-runner:arch'], 'arm64'); + assert.equal(JSON.parse(logs.find((l) => l.includes('"launched"'))).arch, 'arm64'); + } + } +}); + +test('an x64 job never touches the arm64 launch template', async () => { + const { handler, ec2 } = setup(); + await handler(sqsEvent(MSG)); + assert.ok(!JSON.stringify(ec2.calls).includes('lt-0arm')); +}); + +test('an On-Demand job is tagged with its capacity', async () => { + const { handler, ec2 } = setup(); + await handler(sqsEvent({ ...MSG, label: 'ci-ondemand', vcpus: 8, labels: ['self-hosted', 'ci-ondemand', 'ci-cpu-8'] })); + const t = tags(ec2.calls.find((c) => c[0] === 'createFleet')[1]); + assert.deepEqual([t['ci-runner:capacity'], t['ci-runner:vcpus'], t['ci-runner:arch']], ['ci-ondemand', '8', 'x64']); +}); + +test('a message from before the size labels (no vcpus, no arch) launches a 4-vCPU x64 .xlarge', async () => { + const { vcpus, arch, ...old } = MSG; + const { handler, ec2 } = setup(); + assert.deepEqual(await handler(sqsEvent(old)), { batchItemFailures: [] }); + const fleet = ec2.calls.find((c) => c[0] === 'createFleet')[1]; + assert.deepEqual(new Set(fleet.LaunchTemplateConfigs[0].Overrides.map((o) => o.InstanceType)), new Set(['m8i.xlarge', 'm8a.xlarge', 'm7i.xlarge', 'm6i.xlarge', 'm6a.xlarge', 'm7a.xlarge'])); + assert.equal(tags(fleet)['ci-runner:vcpus'], '4'); + assert.equal(tags(fleet)['ci-runner:arch'], 'x64'); +}); + +test('a message with an unknown size or architecture is dropped as malformed', async () => { + for (const bad of [{ vcpus: 3 }, { vcpus: 32 }, { vcpus: '8' }, { arch: 'X64' }, { arch: 'ARM64' }, { arch: 'aarch64' }]) { + const { handler, gh, logs } = setup(); + assert.deepEqual(await handler(sqsEvent({ ...MSG, ...bad })), { batchItemFailures: [] }, JSON.stringify(bad)); + assert.equal(gh.calls.length, 0, JSON.stringify(bad)); + assert.match(logs.join('\n'), /malformed/); + } +}); + +test('the launch is logged with the job\'s shape', async () => { + const { handler, logs } = setup(); + await handler(sqsEvent({ ...MSG, vcpus: 8 })); + const line = JSON.parse(logs.find((l) => l.includes('"launched"'))); + assert.deepEqual([line.label, line.vcpus, line.arch], ['ci-spot', 8, 'x64']); +}); + test('a ci-ondemand job launches On-Demand capacity', async () => { const { handler, ec2 } = setup(); await handler(sqsEvent({ ...MSG, label: 'ci-ondemand', labels: ['self-hosted', 'ci-ondemand'] })); @@ -180,12 +298,16 @@ test('a malformed message is dropped, not retried', async () => { assert.equal(gh.calls.length, 0); }); -const SPOT = { lifecycle: 'spot' }; -const OD = {}; +// Live instances as the launcher tags them since #22, and one from before +// (no shape tags: 4 vCPUs, its capacity from InstanceLifecycle). +const spot = (vcpus, arch = 'x64') => ({ lifecycle: 'spot', capacity: 'ci-spot', vcpus, arch }); +const od = (vcpus, arch = 'x64') => ({ capacity: 'ci-ondemand', vcpus, arch }); +const OLD_SPOT = { lifecycle: 'spot' }; +const OLD_OD = {}; const OD_MSG = { ...MSG, label: 'ci-ondemand', labels: ['self-hosted', 'ci-ondemand'] }; const launched = (ec2) => ec2.calls.some((c) => c[0] === 'createFleet'); -test('the cap counts the stack\'s live instances by tag', async () => { +test('the budget counts the stack\'s live instances by tag', async () => { const { handler, ec2 } = setup(); await handler(sqsEvent(MSG)); const [, input] = ec2.calls[0]; @@ -195,49 +317,258 @@ test('the cap counts the stack\'s live instances by tag', async () => { ]); }); -test('at the Spot cap, a Spot job goes back on the queue with a delay and nothing launches', async () => { - const ec2 = fakeEc2({ running: Array(6).fill(SPOT) }); - const { handler, gh, sqs, logs } = setup({ ec2 }); - assert.deepEqual(await handler(sqsEvent(MSG)), { batchItemFailures: [] }, 'the original is consumed'); - assert.deepEqual(sqs, [{ QueueUrl: 'https://sqs.example/jobs', MessageBody: JSON.stringify(MSG), DelaySeconds: 60 }]); - assert.ok(!launched(ec2)); - assert.ok(!gh.keys().some((k) => k.includes('generate-jitconfig')), 'nothing is minted'); - assert.ok(logs.some((l) => /"deferred"/.test(l))); +test('under the Spot budget, mixed sizes of both architectures leave room: the job launches', async () => { + // 2 + 4 + 8 (arm64) = 14 Spot vCPUs; an 8 brings it to 22 of 24. + const ec2 = fakeEc2({ running: [spot(2), spot(4), spot(8, 'arm64'), od(8)] }); + const { handler, sqs } = setup({ ec2 }); + await handler(sqsEvent({ ...MSG, vcpus: 8 })); + assert.ok(launched(ec2)); + assert.deepEqual(sqs, []); }); -test('below the cap, a job launches and nothing is re-enqueued', async () => { - const ec2 = fakeEc2({ running: [...Array(5).fill(SPOT), OD] }); +test('a job that brings its capacity exactly to the budget launches', async () => { + // 4 + 16 (arm64) = 20; a 4 makes 24 of 24. + const ec2 = fakeEc2({ running: [spot(4), spot(16, 'arm64')] }); const { handler, sqs } = setup({ ec2 }); await handler(sqsEvent(MSG)); assert.ok(launched(ec2)); assert.deepEqual(sqs, []); }); -test('Spot and On-Demand are counted separately', async () => { - // Six Spot: On-Demand still has room. - let s = setup({ ec2: fakeEc2({ running: Array(6).fill(SPOT) }) }); - await s.handler(sqsEvent(OD_MSG)); - assert.ok(launched(s.ec2)); +test('a job that would go over the Spot budget goes back on the queue with a delay, logged in vCPUs', async () => { + // 4 + 16 (arm64) = 20; an 8 would make 28 of 24. + const msg = { ...MSG, vcpus: 8 }; + const ec2 = fakeEc2({ running: [spot(4), spot(16, 'arm64')] }); + const { handler, gh, sqs, logs } = setup({ ec2 }); + assert.deepEqual(await handler(sqsEvent(msg)), { batchItemFailures: [] }, 'the original is consumed'); + assert.deepEqual(sqs, [{ QueueUrl: 'https://sqs.example/jobs', MessageBody: JSON.stringify(msg), DelaySeconds: 60 }]); + assert.ok(!launched(ec2)); + assert.ok(!gh.keys().some((k) => k.includes('generate-jitconfig')), 'nothing is minted'); + assert.equal(JSON.parse(logs.find((l) => l.includes('"deferred"'))).deferred, '20+8 of 24 spot vCPUs'); +}); - // Two On-Demand (an instance with no InstanceLifecycle is On-Demand): Spot has room, On-Demand doesn't. - s = setup({ ec2: fakeEc2({ running: [OD, OD] }) }); +test('an instance from before the shape tags counts as 4 vCPUs, on the capacity its lifecycle says', async () => { + // 5 old Spot = 20, plus a tagged 4 = 24: a 2 is over. + let s = setup({ ec2: fakeEc2({ running: [...Array(5).fill(OLD_SPOT), spot(4)] }) }); + await s.handler(sqsEvent({ ...MSG, vcpus: 2 })); + assert.ok(!launched(s.ec2)); + assert.equal(JSON.parse(s.logs.find((l) => l.includes('"deferred"'))).deferred, '24+2 of 24 spot vCPUs'); + + // 1 old On-Demand = 4, plus a tagged 4 = 8 of 8: On-Demand is full, Spot isn't. + s = setup({ ec2: fakeEc2({ running: [OLD_OD, od(4, 'arm64')] }) }); + await s.handler(sqsEvent({ ...OD_MSG, vcpus: 2 })); + assert.ok(!launched(s.ec2)); + assert.equal(JSON.parse(s.logs.find((l) => l.includes('"deferred"'))).deferred, '8+2 of 8 on-demand vCPUs'); + s = setup({ ec2: fakeEc2({ running: [OLD_OD, od(4, 'arm64')] }) }); await s.handler(sqsEvent(MSG)); assert.ok(launched(s.ec2)); - s = setup({ ec2: fakeEc2({ running: [OD, OD] }) }); +}); + +test('Spot and On-Demand have separate budgets, each across both architectures', async () => { + // 24 Spot vCPUs: On-Demand still has room. + let s = setup({ ec2: fakeEc2({ running: [spot(16), spot(8, 'arm64')] }) }); + await s.handler(sqsEvent({ ...OD_MSG, vcpus: 8 })); + assert.ok(launched(s.ec2)); + + // 2 x64 + 4 arm64 On-Demand = 6: a 2 fits, a 4 doesn't, and Spot has room. + const running = [od(2), od(4, 'arm64')]; + s = setup({ ec2: fakeEc2({ running }) }); + await s.handler(sqsEvent({ ...OD_MSG, vcpus: 2 })); + assert.ok(launched(s.ec2)); + s = setup({ ec2: fakeEc2({ running }) }); await s.handler(sqsEvent(OD_MSG)); assert.ok(!launched(s.ec2)); - assert.equal(s.sqs.length, 1); + assert.equal(JSON.parse(s.logs.find((l) => l.includes('"deferred"'))).deferred, '6+4 of 8 on-demand vCPUs'); + s = setup({ ec2: fakeEc2({ running }) }); + await s.handler(sqsEvent({ ...MSG, vcpus: 16 })); + assert.ok(launched(s.ec2)); }); -test('the caps come from the environment', async () => { - const s = setup({ ec2: fakeEc2({ running: [SPOT] }), env: { ...ENV, SPOT_CAP: '1' } }); +test('the budgets come from the environment', async () => { + let s = setup({ ec2: fakeEc2({ running: [spot(2)] }), env: { ...ENV, SPOT_VCPU_CAP: '4' } }); await s.handler(sqsEvent(MSG)); assert.ok(!launched(s.ec2)); assert.equal(s.sqs.length, 1); + s = setup({ env: { ...ENV, ONDEMAND_VCPU_CAP: '16' } }); + await s.handler(sqsEvent({ ...OD_MSG, vcpus: 16 })); + assert.ok(launched(s.ec2)); }); -test('a job no longer queued is dropped before the cap is checked', async () => { - const s = setup({ gh: fakeGitHub({ jobStatus: 'completed' }), ec2: fakeEc2({ running: Array(6).fill(SPOT) }) }); +test('a job no longer queued is dropped before the budget is checked', async () => { + const s = setup({ gh: fakeGitHub({ jobStatus: 'completed' }), ec2: fakeEc2({ running: [spot(16), spot(8)] }) }); await s.handler(sqsEvent(MSG)); assert.deepEqual(s.sqs, []); }); + +// --- the Fargate capacities (#29) ------------------------------------------ +const FG_MSG = { ...MSG, label: 'ci-fargate-spot', labels: ['self-hosted', 'ci-fargate-spot', 'ci-cpu-4', 'x64'] }; +const runTaskInput = (ecs) => (ecs.calls.find((c) => c[0] === 'runTask') || [])[1]; +const taskTags = (input) => Object.fromEntries(input.tags.map((t) => [t.key, t.value])); +const envOf = (input) => Object.fromEntries(input.overrides.containerOverrides[0].environment.map((e) => [e.name, e.value])); + +test('a ci-fargate-spot job runs one Fargate Spot task with the runner as its command, and no instance', async () => { + const { handler, ec2, ecs, logs } = setup(); + assert.deepEqual(await handler(sqsEvent(FG_MSG)), { batchItemFailures: [] }); + assert.ok(!ec2.calls.some((c) => c[0] !== 'describe'), 'no EC2 launch'); + const input = runTaskInput(ecs); + assert.equal(input.cluster, 'ci-runner'); + assert.equal(input.taskDefinition, ENV.TASK_DEFINITION); + assert.equal(input.count, 1); + assert.deepEqual(input.capacityProviderStrategy, [{ capacityProvider: 'FARGATE_SPOT', weight: 1 }]); + assert.equal(input.launchType, undefined, 'a capacity provider strategy excludes a launch type'); + assert.deepEqual(input.networkConfiguration, { awsvpcConfiguration: { + subnets: ['subnet-a', 'subnet-b', 'subnet-c'], securityGroups: ['sg-0run'], assignPublicIp: 'ENABLED', + } }); + assert.equal(input.overrides.cpu, '4096'); + assert.equal(input.overrides.memory, '16384'); + const [c] = input.overrides.containerOverrides; + assert.equal(c.name, 'runner'); + assert.deepEqual(c.command, ['./run.sh', '--jitconfig', JIT]); + // nproc over-reports on Fargate: the job's tools get the task's vCPUs, but + // in no CARGO*/CMAKE* variable, which Swatinem/rust-cache hashes into its key. + assert.deepEqual(envOf(input), { CI_RUNNER_VCPUS: '4', MAKEFLAGS: '-j4' }); + assert.deepEqual(taskTags(input), { + 'ci-runner:stack': 'ci-runner', + 'ci-runner:runner': 'ci-runner-20261112120000-555', 'ci-runner:runner-id': '77', 'ci-runner:label': 'ci-fargate-spot', + 'ci-runner:repo': 'zeroae/app', 'ci-runner:job-id': '555', 'ci-runner:run-id': '444', 'ci-runner:head-sha': 'abc123', + 'ci-runner:job-name': 'tests', 'ci-runner:labels': 'self-hosted ci-fargate-spot ci-cpu-4 x64', + 'ci-runner:capacity': 'ci-fargate-spot', 'ci-runner:vcpus': '4', 'ci-runner:arch': 'x64', + }); + const line = JSON.parse(logs.find((l) => l.includes('"launched"'))); + assert.equal(line.launched, 'arn:aws:ecs:us-east-1:1:task/ci-runner/new1'); + assert.deepEqual([line.label, line.vcpus, line.arch], ['ci-fargate-spot', 4, 'x64']); + assert.ok(!logs.some((l) => l.includes(JIT)), 'no JIT config in the logs'); +}); + +test('each Fargate capacity, size and architecture: its capacity provider, task definition, cpu, memory and parallelism', async () => { + const mem = { 2: '8192', 4: '16384', 8: '32768', 16: '65536' }; + for (const [label, provider] of [['ci-fargate-spot', 'FARGATE_SPOT'], ['ci-fargate', 'FARGATE']]) { + for (const vcpus of [2, 4, 8, 16]) { + for (const [arch, td] of [['x64', ENV.TASK_DEFINITION], ['arm64', ENV.ARM64_TASK_DEFINITION]]) { + const what = `${label} ${vcpus} ${arch}`; + const { handler, ecs } = setup({ env: { ...ENV, FARGATE_SPOT_VCPU_CAP: '64', FARGATE_VCPU_CAP: '64' } }); + const m = { ...MSG, label, vcpus, arch, labels: ['self-hosted', label, `ci-cpu-${vcpus}`, arch] }; + assert.deepEqual(await handler(sqsEvent(m)), { batchItemFailures: [] }, what); + const input = runTaskInput(ecs); + assert.deepEqual(input.capacityProviderStrategy, [{ capacityProvider: provider, weight: 1 }], what); + assert.equal(input.taskDefinition, td, what); + assert.deepEqual([input.overrides.cpu, input.overrides.memory], [String(vcpus * 1024), mem[vcpus]], what); + assert.deepEqual(envOf(input), { CI_RUNNER_VCPUS: `${vcpus}`, MAKEFLAGS: `-j${vcpus}` }, what); + const t = taskTags(input); + assert.deepEqual([t['ci-runner:capacity'], t['ci-runner:vcpus'], t['ci-runner:arch']], [label, String(vcpus), arch], what); + } + } + } +}); + +test('an arm64 task leaves out a subnet in use1-az3, where Fargate has no arm64; an x64 task keeps it', async () => { + const env = { ...ENV, SUBNET_AZ_IDS: 'use1-az1,use1-az3,use1-az4' }; + let s = setup({ env }); + await s.handler(sqsEvent({ ...FG_MSG, arch: 'arm64' })); + assert.deepEqual(runTaskInput(s.ecs).networkConfiguration.awsvpcConfiguration.subnets, ['subnet-a', 'subnet-c']); + s = setup({ env }); + await s.handler(sqsEvent(FG_MSG)); + assert.deepEqual(runTaskInput(s.ecs).networkConfiguration.awsvpcConfiguration.subnets, ['subnet-a', 'subnet-b', 'subnet-c']); +}); + +test('a task\'s tags carry only characters ECS allows', async () => { + const { handler, ecs } = setup(); + await handler(sqsEvent({ ...FG_MSG, job_name: 'build (x64, release) #1' })); + const t = taskTags(runTaskInput(ecs)); + assert.equal(t['ci-runner:job-name'], 'build _x64_ release_ _1'); + for (const { key, value } of runTaskInput(ecs).tags) assert.match(key + value, /^[\p{L}\p{Z}\p{N}_.:/=+\-@]*$/u); +}); + +test('IDLE_SECONDS on the launcher reaches the task, for a test', async () => { + const { handler, ecs } = setup({ env: { ...ENV, IDLE_SECONDS: '120' } }); + await handler(sqsEvent(FG_MSG)); + assert.equal(envOf(runTaskInput(ecs)).IDLE_SECONDS, '120'); +}); + +const fgSpot = (vcpus) => ({ capacity: 'ci-fargate-spot', vcpus }); +const fg = (vcpus) => ({ capacity: 'ci-fargate', vcpus }); +const FGOD_MSG = { ...FG_MSG, label: 'ci-fargate', labels: ['self-hosted', 'ci-fargate', 'ci-cpu-4', 'x64'] }; +const ran = (ecs) => ecs.calls.some((c) => c[0] === 'runTask'); + +test('the Fargate budget lists the cluster\'s tasks, running and told to stop, and reads their tags', async () => { + const ecs = fakeEcs({ running: [fgSpot(4), fgSpot(2), { ...fgSpot(2), desired: 'STOPPED' }] }); + const { handler } = setup({ ecs }); + await handler(sqsEvent(FG_MSG)); + const lists = ecs.calls.filter((c) => c[0] === 'listTasks').map((c) => c[1]); + assert.deepEqual(lists, [ + { cluster: 'ci-runner', desiredStatus: 'RUNNING', nextToken: undefined }, + { cluster: 'ci-runner', desiredStatus: 'RUNNING', nextToken: '1' }, + { cluster: 'ci-runner', desiredStatus: 'STOPPED', nextToken: undefined }, + ]); + for (const [, d] of ecs.calls.filter((c) => c[0] === 'describeTasks')) { + assert.equal(d.cluster, 'ci-runner'); + assert.deepEqual(d.include, ['TAGS']); + } +}); + +test('a task told to stop counts towards the Fargate budget until it is STOPPED', async () => { + // 16 running + 8 told to stop but still deprovisioning = 24 of 24; the 8 + // already STOPPED no longer counts. + const running = [fgSpot(16), { ...fgSpot(8), desired: 'STOPPED', last: 'DEPROVISIONING' }, + { ...fgSpot(8), desired: 'STOPPED', last: 'STOPPED' }]; + const s = setup({ ecs: fakeEcs({ running }) }); + await s.handler(sqsEvent({ ...FG_MSG, vcpus: 2 })); + assert.ok(!ran(s.ecs)); + assert.equal(JSON.parse(s.logs.find((l) => l.includes('"deferred"'))).deferred, '24+2 of 24 fargate-spot vCPUs'); +}); + +test('Fargate budgets sum mixed sizes across tasks; a job over the budget waits, logged in vCPUs', async () => { + // 2 + 4 + 8 + 8 = 22 Fargate Spot vCPUs: a 2 fits exactly, a 4 doesn't. + const running = [fgSpot(2), fgSpot(4), fgSpot(8), fgSpot(8), fg(8)]; + let s = setup({ ecs: fakeEcs({ running }) }); + await s.handler(sqsEvent({ ...FG_MSG, vcpus: 2 })); + assert.ok(ran(s.ecs)); + s = setup({ ecs: fakeEcs({ running }) }); + assert.deepEqual(await s.handler(sqsEvent(FG_MSG)), { batchItemFailures: [] }); + assert.ok(!ran(s.ecs)); + assert.equal(s.sqs.length, 1); + assert.equal(s.sqs[0].DelaySeconds, 60); + assert.ok(!s.gh.keys().some((k) => k.includes('generate-jitconfig')), 'nothing is minted'); + assert.equal(JSON.parse(s.logs.find((l) => l.includes('"deferred"'))).deferred, '22+4 of 24 fargate-spot vCPUs'); +}); + +test('each Fargate capacity has its own budget, and EC2 instances count towards neither', async () => { + // Fargate On-Demand holds 8 of 8; Fargate Spot and EC2 Spot are free. + const ecs = () => fakeEcs({ running: [fg(4), fg(4), fgSpot(16)] }); + let s = setup({ ecs: ecs() }); + await s.handler(sqsEvent({ ...FGOD_MSG, vcpus: 2 })); + assert.ok(!ran(s.ecs)); + assert.equal(JSON.parse(s.logs.find((l) => l.includes('"deferred"'))).deferred, '8+2 of 8 fargate vCPUs'); + s = setup({ ecs: ecs() }); + await s.handler(sqsEvent({ ...FG_MSG, vcpus: 8 })); + assert.ok(ran(s.ecs), '16+8 of 24 Fargate Spot'); + // 24 EC2 Spot vCPUs held: Fargate Spot doesn't see them, and EC2 doesn't see tasks. + s = setup({ ec2: fakeEc2({ running: [spot(16), spot(8)] }) }); + await s.handler(sqsEvent(FG_MSG)); + assert.ok(ran(s.ecs)); + assert.deepEqual(s.ec2.calls, [], 'a Fargate job describes no instances'); + s = setup({ ecs: fakeEcs({ running: [fgSpot(16), fgSpot(8)] }) }); + await s.handler(sqsEvent(MSG)); + assert.ok(launched(s.ec2)); + assert.deepEqual(s.ecs.calls, [], 'an EC2 job lists no tasks'); +}); + +test('the Fargate budgets come from the environment', async () => { + let s = setup({ ecs: fakeEcs({ running: [fgSpot(2)] }), env: { ...ENV, FARGATE_SPOT_VCPU_CAP: '4' } }); + await s.handler(sqsEvent(FG_MSG)); + assert.ok(!ran(s.ecs)); + s = setup({ env: { ...ENV, FARGATE_VCPU_CAP: '16' } }); + await s.handler(sqsEvent({ ...FGOD_MSG, vcpus: 16 })); + assert.ok(ran(s.ecs)); +}); + +test('a failed RunTask removes the registration and fails the message, as a failed fleet does', async () => { + const failed = { tasks: [], failures: [{ arn: 'x', reason: 'Capacity is unavailable at this time', detail: 'FARGATE_SPOT' }] }; + for (const run of [failed, Object.assign(new Error('AccessDeniedException: no'), { name: 'AccessDeniedException' })]) { + const { handler, gh, logs } = setup({ ecs: fakeEcs({ run }) }); + assert.deepEqual(await handler(sqsEvent(FG_MSG)), { batchItemFailures: [{ itemIdentifier: 'm0' }] }); + assert.ok(gh.keys().includes('DELETE /orgs/zeroae/actions/runners/77'), 'the registration is removed'); + const err = JSON.parse(logs.find((l) => l.includes('"error"'))).error; + assert.match(err, run === failed ? /^no task launched: Capacity is unavailable at this time: FARGATE_SPOT$/ : /AccessDenied/); + } +}); diff --git a/test/notifier.test.js b/test/notifier.test.js index 88ffb2f..1b696fb 100644 --- a/test/notifier.test.js +++ b/test/notifier.test.js @@ -31,7 +31,8 @@ function verifyJwt(auth) { return JSON.parse(Buffer.from(p, 'base64url')); } -function fakeGitHub({ checkStatus = 201 } = {}) { +// existing: the check runs already on the commit. +function fakeGitHub({ checkStatus = 201, jobStatus = 200, existing = [] } = {}) { const calls = []; const fetch = async (url, init = {}) => { const u = new URL(url); @@ -49,19 +50,31 @@ function fakeGitHub({ checkStatus = 201 } = {}) { } assert.equal(init.headers.authorization, 'Bearer ghs_install', key); if (key === 'POST /repos/zeroae/app/check-runs') return res(checkStatus, { id: 9 }); + if (key === 'GET /repos/zeroae/app/commits/abc123/check-runs') { + const name = u.searchParams.get('check_name'); + return res(200, { check_runs: existing.filter((c) => c.name === name) }); + } + if (key === 'GET /repos/zeroae/app/actions/jobs/555') return res(jobStatus, { id: 555, name: 'build (x64, a)' }); throw new Error('unexpected ' + key); }; return { fetch, calls, keys: () => calls.map((c) => c.key) }; } // ec2.describe returns DescribeInstances' output as EC2 gives it. -function setup({ inst = instance(TAGS), raw = { Reservations: inst ? [{ Instances: [inst] }] : [] }, gh = fakeGitHub(), creds = { app_id: 42, private_key: PEM } } = {}) { +// ecs.describeTasks returns DescribeTasks' output as ECS gives it. +function setup({ inst = instance(TAGS), raw = { Reservations: inst ? [{ Instances: [inst] }] : [] }, tasks = [], gh = fakeGitHub(), creds = { app_id: 42, private_key: PEM } } = {}) { const described = []; const logs = []; const handler = createHandler({ secret: { get: async () => creds }, fetch: gh.fetch, ec2: { describe: async (id) => { described.push(id); return raw; } }, + ecs: { + describeTasks: async (i) => { + described.push(i); + return { tasks: tasks.filter((t) => i.tasks.includes(t.taskArn)), failures: [] }; + }, + }, now: () => T0, env: ENV, log: (l) => logs.push(l), @@ -73,8 +86,11 @@ test('an interruption of one of our instances creates a failed check run on the const { handler, gh, described } = setup(); await handler(EVENT); assert.deepEqual(described, ['i-0123']); - assert.deepEqual(gh.keys(), ['GET /repos/zeroae/app/installation', 'POST /app/installations/181/access_tokens', 'POST /repos/zeroae/app/check-runs']); - const { body } = gh.calls[2]; + assert.deepEqual(gh.keys(), [ + 'GET /repos/zeroae/app/installation', 'POST /app/installations/181/access_tokens', + 'GET /repos/zeroae/app/commits/abc123/check-runs', 'POST /repos/zeroae/app/check-runs', + ]); + const { body } = gh.calls[3]; const runUrl = 'https://github.com/zeroae/app/actions/runs/444'; assert.equal(body.name, 'ci-runner / tests'); assert.equal(body.head_sha, 'abc123'); @@ -96,7 +112,7 @@ test('an interruption of one of our instances creates a failed check run on the test('a job name falls back to the job id when the tag is empty', async () => { const { handler, gh } = setup({ inst: instance({ ...TAGS, 'ci-runner:job-name': '' }) }); await handler(EVENT); - assert.equal(gh.calls[2].body.name, 'ci-runner / job 555'); + assert.equal(gh.calls.find((c) => c.key === 'POST /repos/zeroae/app/check-runs').body.name, 'ci-runner / job 555'); }); for (const [why, inst] of [ @@ -130,6 +146,23 @@ test('an event with no instance id is ignored without describing anything', asyn assert.deepEqual(gh.keys(), []); }); +test('a reclaim already reported on the commit is not reported again', async () => { + const title = 'Spot capacity reclaimed: re-run this job'; + const gh = fakeGitHub({ existing: [ + { name: 'ci-runner / tests', external_id: '555', output: { title } }, + ] }); + const { handler, logs } = setup({ gh }); + await handler(EVENT); + assert.ok(!gh.keys().includes('POST /repos/zeroae/app/check-runs')); + assert.match(logs.join('\n'), /already reported i-0123/); + // Another job's check, or a check of that name that isn't ours, doesn't count. + for (const other of [{ name: 'ci-runner / tests', external_id: '554', output: { title } }, { name: 'ci-runner / tests', external_id: '555', output: { title: 'Lint' } }]) { + const g = fakeGitHub({ existing: [other] }); + await setup({ gh: g }).handler(EVENT); + assert.ok(g.keys().includes('POST /repos/zeroae/app/check-runs'), JSON.stringify(other)); + } +}); + test('a GitHub failure fails the invocation, so EventBridge retries it', async () => { const { handler } = setup({ gh: fakeGitHub({ checkStatus: 422 }) }); await assert.rejects(handler(EVENT), /check-runs.*422/); @@ -139,3 +172,63 @@ test('before the App exists the invocation fails', async () => { const { handler } = setup({ creds: { state_key: 'x' } }); await assert.rejects(handler(EVENT), /not set up/); }); + +// --- Fargate Spot ------------------------------------------------------------- +const CLUSTER_ARN = 'arn:aws:ecs:us-east-1:123456789012:cluster/ci-runner'; +const TASK_ARN = 'arn:aws:ecs:us-east-1:123456789012:task/ci-runner/0f1e2d3c4b5a69788796a5b4c3d2e1f0'; + +// The first ECS Task State Change the rule matches: the reclaimed task, told +// to stop. The detail carries no tags. The shape is ECS's, as aws-emulator +// publishes it (see test/tasks-ecs.test.js). +const TASK_EVENT = { + version: '0', id: 'e-2', 'detail-type': 'ECS Task State Change', source: 'aws.ecs', + account: '123456789012', time: '2026-11-12T12:00:00Z', region: 'us-east-1', resources: [TASK_ARN], + detail: { + taskArn: TASK_ARN, clusterArn: CLUSTER_ARN, lastStatus: 'RUNNING', desiredStatus: 'STOPPED', + capacityProviderName: 'FARGATE_SPOT', stopCode: 'SpotInterruption', stoppedReason: 'Your Spot Task was interrupted.', + }, +}; + +// A task's tags, as the launcher sets them: ECS refuses brackets and commas, +// so the job name's are `_` and the labels are space-joined. +const TASK_TAGS = { + ...TAGS, 'ci-runner:label': 'ci-fargate-spot', 'ci-runner:job-name': 'build _x64_ a_', + 'ci-runner:labels': 'self-hosted ci-fargate-spot ci-cpu-4 x64', +}; +const task = (tags) => ({ taskArn: TASK_ARN, lastStatus: 'RUNNING', tags: Object.entries(tags).map(([key, value]) => ({ key, value })) }); + +test('a Fargate Spot reclaim creates the same check run as an EC2 one, with the job\'s real name', async () => { + const ec2 = setup({ inst: instance({ ...TAGS, 'ci-runner:job-name': 'build (x64, a)' }) }); + await ec2.handler(EVENT); + const fargate = setup({ inst: null, tasks: [task(TASK_TAGS)] }); + await fargate.handler(TASK_EVENT); + + assert.deepEqual(fargate.described, [{ cluster: CLUSTER_ARN, tasks: [TASK_ARN], include: ['TAGS'] }]); + const check = (s) => s.gh.calls.find((c) => c.key === 'POST /repos/zeroae/app/check-runs').body; + const a = check(ec2); + const b = check(fargate); + assert.equal(b.name, 'ci-runner / build (x64, a)'); + assert.match(b.output.summary, /task \(0f1e2d3c4b5a69788796a5b4c3d2e1f0\)/); + const machine = (c) => ({ ...c, output: { ...c.output, summary: c.output.summary.replace(/instance \(i-0123\)|task \(\w+\)/, '') } }); + assert.deepEqual(machine(b), machine(a)); + assert.match(fargate.logs.join('\n'), /reported 0f1e2d3c4b5a69788796a5b4c3d2e1f0: zeroae\/app job 555/); +}); + +test('a task\'s job name falls back to its tag when GitHub can\'t say', async () => { + const { handler, gh } = setup({ inst: null, tasks: [task(TASK_TAGS)], gh: fakeGitHub({ jobStatus: 404 }) }); + await handler(TASK_EVENT); + assert.equal(gh.calls.find((c) => c.key === 'POST /repos/zeroae/app/check-runs').body.name, 'ci-runner / build _x64_ a_'); +}); + +for (const [why, tasks] of [ + ['a task without job tags (run by hand)', [task({ 'ci-runner:stack': 'ci-runner', 'ci-runner:runner': 'ci-runner-hand-20261112115500' })]], + ['another stack\'s task', [task({ ...TASK_TAGS, 'ci-runner:stack': 'other' })]], + ['a task ECS no longer returns', []], +]) { + test(`${why} is ignored`, async () => { + const { handler, gh, logs } = setup({ inst: null, tasks }); + await handler(TASK_EVENT); + assert.deepEqual(gh.keys(), []); + assert.match(logs.join('\n'), /ignored/); + }); +} diff --git a/test/sweeper.test.js b/test/sweeper.test.js index f304148..c3e9718 100644 --- a/test/sweeper.test.js +++ b/test/sweeper.test.js @@ -1,11 +1,48 @@ 'use strict'; const { test } = require('node:test'); const assert = require('node:assert/strict'); -const { createHandler } = require('../functions/sweeper.js'); +const { createHandler: create } = require('../functions/sweeper.js'); const NOW = Date.parse('2026-11-12T12:00:00Z'); const ago = (min) => new Date(NOW - min * 60 * 1000); -const env = { STACK_NAME: 'ci-runner', GITHUB_ORG: 'zeroae', RUNNER_GROUP: 'ci-runner' }; +const env = { STACK_NAME: 'ci-runner', GITHUB_ORG: 'zeroae', RUNNER_GROUP: 'ci-runner', CLUSTER: 'ci-runner' }; + +// The cluster's tasks, as ECS answers ListTasks and DescribeTasks: ListTasks +// returns at most 100 ARNs a page and filters on the desired status. +function fakeEcs(tasks) { + const stopped = []; + const calls = []; + return { + stopped, + calls, + listTasks: async (i) => { + calls.push(['listTasks', i]); + assert.equal(i.cluster, 'ci-runner'); + const arns = tasks.filter((t) => t.desiredStatus === (i.desiredStatus || 'RUNNING')).map((t) => t.taskArn); + const from = Number(i.nextToken || 0); + return { taskArns: arns.slice(from, from + 100), ...(from + 100 < arns.length && { nextToken: String(from + 100) }) }; + }, + describeTasks: async (i) => { + calls.push(['describeTasks', i]); + assert.equal(i.cluster, 'ci-runner'); + assert.ok(i.tasks.length <= 100); + assert.deepEqual(i.include, ['TAGS']); + return { tasks: tasks.filter((t) => i.tasks.includes(t.taskArn)), failures: [] }; + }, + stopTask: async (i) => { calls.push(['stopTask', i]); stopped.push(i.task); return { task: {} }; }, + }; +} + +function task(id, minutesOld, { lastStatus = 'RUNNING', desiredStatus = 'RUNNING', tags = { 'ci-runner:stack': 'ci-runner' } } = {}) { + return { + taskArn: `arn:aws:ecs:us-east-1:123456789012:task/ci-runner/${id}`, + createdAt: ago(minutesOld), lastStatus, desiredStatus, + tags: Object.entries(tags).map(([key, value]) => ({ key, value })), + }; +} + +// Every test runs with an empty cluster unless it brings its own. +const createHandler = (deps) => create({ ecs: fakeEcs([]), ...deps }); function instance(id, minutesOld, { state = 'running', tags = { 'ci-runner:stack': 'ci-runner' } } = {}) { return { @@ -44,6 +81,31 @@ test('terminates only instances carrying the stack tag that are older than 2 hou assert.deepEqual(out.terminated.sort(), ['i-old', 'i-old-stopped']); }); +test('stops the cluster\'s tasks launched over 2 hours ago', async () => { + const ecs = fakeEcs([ + task('t-old', 121), + task('t-young', 119), + // A task launched by hand, without our tags, is still in the stack's cluster. + task('t-old-hand', 300, { tags: {} }), + // Already told to stop: ListTasks for RUNNING doesn't return it. + task('t-old-stopping', 300, { desiredStatus: 'STOPPED', lastStatus: 'DEACTIVATING' }), + ]); + const out = await createHandler({ ec2: fakeEc2([]), ecs, secret: noApp, fetch: assert.fail, now: () => NOW, env })({}); + const arn = (id) => `arn:aws:ecs:us-east-1:123456789012:task/ci-runner/${id}`; + assert.deepEqual(ecs.stopped.sort(), [arn('t-old'), arn('t-old-hand')]); + assert.deepEqual(out.stopped.sort(), [arn('t-old'), arn('t-old-hand')]); + for (const [op, i] of ecs.calls.filter(([op]) => op === 'stopTask')) { + assert.equal(i.cluster, 'ci-runner', op); + assert.match(i.reason, /2 hours/); + } +}); + +test('pages through a cluster with more than 100 tasks', async () => { + const ecs = fakeEcs(Array.from({ length: 150 }, (_, i) => task(`t${i}`, 200))); + await createHandler({ ec2: fakeEc2([]), ecs, secret: noApp, fetch: assert.fail, now: () => NOW, env })({}); + assert.equal(ecs.stopped.length, 150); +}); + test('terminates nothing when nothing is stray', async () => { const ec2 = fakeEc2([instance('i-young', 5)]); const calls = []; @@ -133,6 +195,41 @@ test('removes only offline registrations in the group that have no instance', as assert.equal(gh.jwts.length, 2); }); +test('a registration is kept while its task lives and removed once the task is gone, next to instances', async () => { + const tags = (more) => ({ 'ci-runner:stack': 'ci-runner', ...more }); + const ec2 = fakeEc2([ + instance('i-1', 10, { tags: tags({ 'ci-runner:runner': 'ci-runner-on-ec2' }) }), + ]); + const ecs = fakeEcs([ + task('t-1', 10, { tags: tags({ 'ci-runner:runner': 'ci-runner-on-fargate' }) }), + task('t-2', 10, { lastStatus: 'PROVISIONING', tags: tags({ 'ci-runner:runner-id': '53' }) }), + // Told to stop (stopped, or on its way): its runner no longer has a machine. + task('t-3', 10, { desiredStatus: 'STOPPED', lastStatus: 'STOPPED', tags: tags({ 'ci-runner:runner': 'ci-runner-task-gone' }) }), + task('t-4', 10, { desiredStatus: 'STOPPED', lastStatus: 'DEPROVISIONING', tags: tags({ 'ci-runner:runner': 'ci-runner-task-stopping' }) }), + ]); + const gh = fakeGitHub([ + runner(50, 'ci-runner-on-ec2'), // its instance is running: kept + runner(51, 'ci-runner-on-fargate'), // its task is running: kept + runner(52, 'ci-runner-task-gone'), // its task stopped: removed + runner(53, 'renamed'), // its task carries its id: kept + runner(54, 'ci-runner-task-stopping'), // its task is stopping: removed + runner(55, 'ci-runner-nowhere'), // neither: removed + ]); + const out = await createHandler({ ec2, ecs, secret: app, fetch: gh.fetch, now: () => NOW, env })({}); + assert.deepEqual(gh.deleted.sort(), [52, 54, 55]); + assert.deepEqual(out, { terminated: [], stopped: [], removed: [52, 54, 55] }); +}); + +test('an ECS failure fails the run after the terminations, and removes no registration', async () => { + const ec2 = fakeEc2([instance('i-old', 200)]); + const ecs = fakeEcs([]); + ecs.listTasks = async () => { throw new Error('ThrottlingException'); }; + const gh = fakeGitHub([runner(60, 'ci-runner-orphan')]); + await assert.rejects(createHandler({ ec2, ecs, secret: app, fetch: gh.fetch, now: () => NOW, env })({}), /Throttling/); + assert.deepEqual(ec2.terminated, ['i-old']); + assert.deepEqual(gh.deleted, []); +}); + test('a registration minted in the last 10 minutes is left alone, even with no instance yet', async () => { // The launcher mints the JIT config before the instance exists. A name // carrying a UTC stamp tells the sweeper its age. @@ -225,7 +322,17 @@ test('the sweeper may terminate only instances carrying the stack tag', () => { const logs = all.filter((s) => s.some((l) => l.includes('logs:'))); assert.equal(logs.length, 1); - assert.equal(all.length, 4, 'nothing else is granted'); + assert.equal(all.length, 6, 'nothing else is granted'); +}); + +test('the sweeper may list, describe and stop only the tasks of the stack\'s cluster', () => { + const ecs = statements().filter((s) => s.some((l) => l.includes('ecs:'))); + const tasks = 'Resource: !Sub "arn:${AWS::Partition}:ecs:${AWS::Region}:${AWS::AccountId}:task/${RunnerCluster}/*"'; + assert.deepEqual(ecs, [ + ['Effect: Allow', 'Action: ecs:ListTasks', 'Resource: "*"', 'Condition:', 'ArnEquals:', 'ecs:cluster: !GetAtt RunnerCluster.Arn'], + ['Effect: Allow', 'Action: [ecs:DescribeTasks, ecs:StopTask]', tasks], + ]); + assert.match(resource('SweeperFunction'), /CLUSTER: !Ref RunnerCluster\n/); }); test('the sweeper runs every 10 minutes', () => { diff --git a/test/tasks-ecs.test.js b/test/tasks-ecs.test.js new file mode 100644 index 0000000..7395327 --- /dev/null +++ b/test/tasks-ecs.test.js @@ -0,0 +1,244 @@ +'use strict'; +// A task's lifecycle against real ECS shapes, from aws/aws-emulator +// (./aws-emulator.js): the launcher runs a task, the webhook repairs a swap +// on it (TagResource validates the characters, as ECS does), the sweeper +// stops it, and the notifier reports a reclaim from the ECS Task State Change +// the emulator publishes to EventBridge, matched by template.yaml's own rule. +// The emulator drops a task's tags once it is told to stop (AWS keeps them +// readable through DescribeTasks, measured live); the tests rely on that only +// where they say so. +const { test, before, after } = require('node:test'); +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const path = require('node:path'); +const { skip, startEmulator, network, taskDefinition } = require('./aws-emulator.js'); +const launcher = require('../functions/launcher.js'); +const webhook = require('../functions/webhook.js'); +const sweeper = require('../functions/sweeper.js'); +const notifier = require('../functions/notifier.js'); + +let aws; +let clusterArn; +let queueUrl; +const env = { + GITHUB_ORG: 'zeroae', RUNNER_GROUP: 'ci-runner', STACK_NAME: 'ci-runner', CLUSTER: 'ci-runner', + QUEUE_URL: 'https://sqs.example/jobs', SPOT_VCPU_CAP: '24', ONDEMAND_VCPU_CAP: '8', + FARGATE_SPOT_VCPU_CAP: '64', FARGATE_VCPU_CAP: '64', SUBNET_AZ_IDS: 'use1-az1,use1-az2', +}; + +// The ECS adapters, as each function's exports.handler builds them. The SDK +// reads a timestamp as a Date; the wire carries epoch seconds. +const op = (name) => (i) => aws.ecs(name, i); +const ecs = { + runTask: op('RunTask'), + listTasks: op('ListTasks'), + describeTasks: async (i) => { + const r = await aws.ecs('DescribeTasks', i); + for (const t of r.tasks || []) if (typeof t.createdAt === 'number') t.createdAt = new Date(t.createdAt * 1000); + return r; + }, + stopTask: op('StopTask'), + tagResource: op('TagResource'), +}; + +// template.yaml's NotifierTaskRule pattern, with the cluster's ARN for its +// !GetAtt. The pattern is flow lists (`key: [a, b]`) and nested keys only. +function notifierTaskPattern() { + const yaml = fs.readFileSync(path.join(__dirname, '..', 'template.yaml'), 'utf8'); + const block = yaml.split(/\n {2}NotifierTaskRule:\n/)[1].split('EventPattern:\n')[1].split(/\n(?! {8})/)[0]; + const root = {}; + const stack = [{ indent: 6, obj: root }]; + for (const line of block.split('\n')) { + const [, sp, key, list] = line.match(/^( *)([\w-]+):(?: \[(.*)\])?$/); + while (sp.length <= stack[stack.length - 1].indent) stack.pop(); + const parent = stack[stack.length - 1].obj; + if (list === undefined) stack.push({ indent: sp.length, obj: (parent[key] = {}) }); + else parent[key] = list.split(', ').map((v) => (v === '!GetAtt RunnerCluster.Arn' ? clusterArn : v)); + } + return root; +} + +before(async () => { + if (skip) return; + aws = await startEmulator(); + const net = await network(aws, env.SUBNET_AZ_IDS.split(',')); + env.SUBNET_IDS = net.subnets.join(','); + env.SECURITY_GROUP_ID = net.sg; + clusterArn = (await aws.ecs('CreateCluster', { clusterName: 'ci-runner', capacityProviders: ['FARGATE', 'FARGATE_SPOT'] })).cluster.clusterArn; + env.TASK_DEFINITION = await taskDefinition(aws, 'ci-runner-runner', 'X86_64'); + env.ARM64_TASK_DEFINITION = await taskDefinition(aws, 'ci-runner-runner-arm64', 'ARM64'); + + // Every task state change of the cluster, to an SQS queue, to read the + // events as EventBridge delivers them. + queueUrl = (await aws.sqs('CreateQueue', { QueueName: 'task-events' })).QueueUrl; + const queueArn = (await aws.sqs('GetQueueAttributes', { QueueUrl: queueUrl, AttributeNames: ['QueueArn'] })).Attributes.QueueArn; + await aws.sqs('SetQueueAttributes', { QueueUrl: queueUrl, Attributes: { Policy: JSON.stringify({ + Version: '2012-10-17', + Statement: [{ Effect: 'Allow', Principal: { Service: 'events.amazonaws.com' }, Action: 'sqs:SendMessage', Resource: queueArn }], + }) } }); + await aws.events('PutRule', { Name: 'all-task-changes', EventPattern: JSON.stringify({ source: ['aws.ecs'], 'detail-type': ['ECS Task State Change'], detail: { clusterArn: [clusterArn] } }) }); + await aws.events('PutTargets', { Rule: 'all-task-changes', Targets: [{ Id: 'queue', Arn: queueArn }] }); +}); + +after(() => { if (aws) aws.stop(); }); + +const { privateKey } = crypto.generateKeyPairSync('rsa', { modulusLength: 2048 }); +const secret = { get: async () => ({ app_id: 42, private_key: privateKey.export({ type: 'pkcs1', format: 'pem' }), webhook_secret: 'whsec' }) }; + +// GitHub, faked: every job is queued and keeps its real name, the group +// exists, JIT configs mint, and runners and check runs are recorded (and a +// commit's check runs listed). +const JOB_NAMES = { 999: 'build (x64, a)', 555: 'test (x64, b)' }; +function fakeGitHub(runners = []) { + const calls = []; + const fetch = async (url, init = {}) => { + const { pathname } = new URL(url); + const key = `${init.method || 'GET'} ${pathname}`; + calls.push({ key, body: init.body && JSON.parse(init.body) }); + const res = (status, json) => ({ ok: status < 300, status, json: async () => json }); + if (key.endsWith('/installation')) return res(200, { id: 181 }); + if (key.endsWith('/access_tokens')) return res(201, { token: 't', expires_at: new Date(Date.now() + 3600e3).toISOString() }); + const job = pathname.match(/\/actions\/jobs\/(\d+)$/); + if (job) return res(200, { id: Number(job[1]), status: 'queued', name: JOB_NAMES[job[1]] }); + if (key.endsWith('/actions/runner-groups')) return res(200, { runner_groups: [{ id: 3, name: 'ci-runner' }] }); + if (key.endsWith('/runner-groups/3/runners')) return res(200, { total_count: runners.length, runners }); + if (key.endsWith('/generate-jitconfig')) return res(201, { runner: { id: 77 }, encoded_jit_config: 'ZW5jb2RlZGppdA==' }); + if (key.startsWith('GET ') && key.endsWith('/check-runs')) { + const name = new URL(url).searchParams.get('check_name'); + return res(200, { check_runs: calls.filter((c) => c.key.startsWith('POST ') && c.key.endsWith('/check-runs') && c.body.name === name).map((c) => c.body) }); + } + if (key.startsWith('DELETE ') || key.endsWith('/check-runs')) return res(201, {}); + throw new Error('unexpected ' + key); + }; + return { fetch, calls }; +} + +// The launcher runs one task for job 999; returns its ARN and runner name. +async function launch() { + const logs = []; + const h = launcher.createHandler({ + secret, fetch: fakeGitHub().fetch, ecs, ec2: {}, sqs: {}, now: Date.now, env, log: (l) => logs.push(l), + }); + const body = JSON.stringify({ + repo: 'zeroae/lib', job_id: 999, run_id: 888, head_sha: 'fff000', job_name: JOB_NAMES[999], + label: 'ci-fargate-spot', vcpus: 4, arch: 'x64', labels: ['self-hosted', 'ci-fargate-spot', 'ci-cpu-4', 'x64', 'linux'], installation_id: 181, + }); + assert.deepEqual(await h({ Records: [{ messageId: 'm', body }] }), { batchItemFailures: [] }, logs.join('\n')); + const l = JSON.parse(logs.find((x) => x.includes('"launched"'))); + return { arn: l.launched, runner: l.runner }; +} + +const tagsOf = async (arn) => Object.fromEntries((await ecs.describeTasks({ cluster: 'ci-runner', tasks: [arn], include: ['TAGS'] })) + .tasks[0].tags.map((t) => [t.key, t.value])); + +test('a swap on a task: the job is re-queued from the task\'s tags, and ECS takes the re-tag', { skip }, async () => { + const { arn, runner } = await launch(); + const sent = []; + const logs = []; + const h = webhook.createHandler({ + secret, queue: { send: async (m) => sent.push(m) }, + ec2: { describeInstances: async () => ({ Reservations: [] }), createTags: assert.fail }, + ecs, env, log: (l) => logs.push(l), + }); + const raw = JSON.stringify({ + action: 'in_progress', + workflow_job: { id: 555, run_id: 444, head_sha: 'abc123', name: JOB_NAMES[555], labels: ['self-hosted', 'ci-fargate-spot', 'ci-cpu-4', 'x64'], runner_name: runner }, + repository: { full_name: 'zeroae/app', visibility: 'private', owner: { login: 'zeroae' } }, + installation: { id: 181 }, + }); + const sig = 'sha256=' + crypto.createHmac('sha256', 'whsec').update(raw).digest('hex'); + const res = await h({ headers: { 'x-github-event': 'workflow_job', 'x-hub-signature-256': sig }, body: raw }); + assert.equal(res.statusCode, 202, logs.join('\n')); + + assert.equal(sent.length, 1); + assert.deepEqual( + { ...sent[0], job_name: undefined }, + { repo: 'zeroae/lib', job_id: 999, run_id: 888, head_sha: 'fff000', job_name: undefined, label: 'ci-fargate-spot', + vcpus: 4, arch: 'x64', labels: ['self-hosted', 'ci-fargate-spot', 'ci-cpu-4', 'x64', 'linux'], installation_id: 181 }, + ); + const tags = await tagsOf(arn); + assert.deepEqual( + ['repo', 'job-id', 'run-id', 'head-sha', 'job-name'].map((k) => tags[`ci-runner:${k}`]), + ['zeroae/app', '555', '444', 'abc123', 'test _x64_ b_'], + ); + assert.equal(tags['ci-runner:runner'], runner, 'the other tags stay'); + await ecs.stopTask({ cluster: 'ci-runner', task: arn }); +}); + +test('the sweeper keeps a live task\'s registration, stops the task after 2 hours, then removes the registration', { skip }, async () => { + const { arn, runner } = await launch(); + const later = Date.now() + 3 * 3600e3; + const sweep = (now, runners) => { + const gh = fakeGitHub(runners); + const h = sweeper.createHandler({ ec2: { describe: async () => [], terminate: assert.fail }, ecs, secret, fetch: gh.fetch, now: () => now, env, log: () => {} }); + return h({}).then((out) => ({ out, deleted: gh.calls.filter((c) => c.key.startsWith('DELETE ')).map((c) => c.key) })); + }; + const runners = [{ id: 77, name: runner, status: 'offline', busy: false }]; + + // 3 hours on: the task is stopped, but listed this run, so its runner stays. + let r = await sweep(later, runners); + assert.deepEqual(r.out.stopped, [arn]); + assert.deepEqual(r.deleted, []); + const [task] = (await ecs.describeTasks({ cluster: 'ci-runner', tasks: [arn] })).tasks; + assert.equal(task.desiredStatus, 'STOPPED'); + + // The next run: the task is gone, so the offline registration goes. + r = await sweep(later, runners); + assert.deepEqual(r.out.stopped, []); + assert.deepEqual(r.deleted, ['DELETE /orgs/zeroae/actions/runners/77']); +}); + +// The task's state changes as EventBridge delivers them, until `done` holds +// (each receive waits up to 1 s for one). +async function changesOf(arn, done) { + const out = []; + for (let i = 0; i < 10 && !done(out); i++) { + const m = await aws.sqs('ReceiveMessage', { QueueUrl: queueUrl, MaxNumberOfMessages: 10, WaitTimeSeconds: 1 }); + for (const x of m.Messages || []) { + const e = JSON.parse(x.Body); + if (e.detail.taskArn === arn) out.push(e); + } + } + return out; +} + +test('a Fargate Spot reclaim: template.yaml\'s rule matches each change of the stop, and the notifier reports it once', { skip }, async () => { + const { arn } = await launch(); + const pattern = JSON.stringify(notifierTaskPattern()); + const matches = async (e) => (await aws.events('TestEventPattern', { EventPattern: pattern, Event: JSON.stringify(e) })).Result; + // AWS stops a reclaimed task with SpotInterruption, the emulator only with + // UserInitiated, so the code is swapped in. The warning: the task's change, + // as ECS publishes it, once it is told to stop and still runs. + const [first] = await changesOf(arn, (c) => c.length > 0); + assert.equal(await matches(first), false, 'a task starting is no reclaim'); + const warning = { ...first, detail: { ...first.detail, desiredStatus: 'STOPPED', stopCode: 'SpotInterruption' } }; + assert.equal(await matches(warning), true); + + const gh = fakeGitHub(); + const logs = []; + const h = notifier.createHandler({ secret, fetch: gh.fetch, ec2: { describe: assert.fail }, ecs, now: Date.now, env, log: (l) => logs.push(l) }); + await h(warning); + const check = gh.calls.find((c) => c.key === 'POST /repos/zeroae/lib/check-runs'); + assert.ok(check, logs.join('\n')); + assert.equal(check.body.name, 'ci-runner / build (x64, a)'); + assert.equal(check.body.head_sha, 'fff000'); + assert.equal(check.body.output.title, 'Spot capacity reclaimed: re-run this job'); + assert.ok(check.body.output.summary.includes(`task (${arn.split('/').pop()})`)); + + // The stop's later changes match too. The emulator has dropped the task's tags by + // then, so the notifier finds no job on it, and reports nothing more. + await ecs.stopTask({ cluster: 'ci-runner', task: arn, reason: 'test' }); + const stopping = await changesOf(arn, (c) => c.some((e) => e.detail.lastStatus === 'STOPPED')); + const user = stopping.find((e) => e.detail.lastStatus === 'STOPPED'); + assert.equal(user.detail.stopCode, 'UserInitiated'); + assert.equal(await matches(user), false, 'a UserInitiated stop is no reclaim'); + const reclaimed = stopping.filter((e) => e.detail.stopCode).map((e) => ({ ...e, detail: { ...e.detail, stopCode: 'SpotInterruption' } })); + assert.ok(reclaimed.length >= 2, 'a stopping change and the stopped one'); + for (const e of reclaimed) { + assert.equal(await matches(e), true, e.detail.lastStatus); + await h(e); + } + assert.equal(gh.calls.filter((c) => c.key === 'POST /repos/zeroae/lib/check-runs').length, 1); + assert.deepEqual((await ecs.describeTasks({ cluster: 'ci-runner', tasks: [arn], include: ['TAGS'] })).tasks[0].tags, []); +}); diff --git a/test/template.test.js b/test/template.test.js new file mode 100644 index 0000000..11e188f --- /dev/null +++ b/test/template.test.js @@ -0,0 +1,190 @@ +'use strict'; +// The template's own promises, read from template.yaml as text. +const { test } = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); + +const template = fs.readFileSync(path.join(__dirname, '..', 'template.yaml'), 'utf8'); +// A top-level resource's block, from its name to the next blank line. +const resource = (name) => template.split(new RegExp(`\\n ${name}:\\n`))[1].split(/\n\n/)[0] + '\n'; +const launchTemplates = (s) => new Set([...s.matchAll(/launch-template\/\$\{(\w+)\}/g)].map((m) => m[1])); + +test('the arm64 launch template is the x64 one with the arm64 AL2023 image', () => { + const x64 = resource('RunnerLaunchTemplate'); + const arm = resource('Arm64RunnerLaunchTemplate'); + assert.match(x64, /ImageId: resolve:ssm:\/aws\/service\/ami-amazon-linux-latest\/al2023-ami-kernel-default-x86_64\n/); + assert.match(arm, /ImageId: resolve:ssm:\/aws\/service\/ami-amazon-linux-latest\/al2023-ami-kernel-default-arm64\n/); + const same = (s) => s.split('LaunchTemplateData:')[1].replace(/ImageId: .*\n/, ''); + assert.equal(same(arm), same(x64), 'disk, IMDSv2, security group, shutdown and tags'); + assert.doesNotMatch(arm, /IamInstanceProfile|UserData|InstanceType/); + assert.match(arm, /LaunchTemplateName: !Sub "\$\{AWS::StackName\}-runner-arm64"/); +}); + +test('the launcher\'s role launches from both launch templates and no other', () => { + const role = resource('LauncherRole'); + assert.deepEqual(launchTemplates(role), new Set(['RunnerLaunchTemplate', 'Arm64RunnerLaunchTemplate'])); + assert.doesNotMatch(role, /launch-template\/\*/); + // Every statement that names one launch template names both. + for (const stmt of role.split(/\n {14}- Effect: /).slice(1)) { + const lts = launchTemplates(stmt); + if (lts.size) assert.equal(lts.size, 2, stmt); + } + assert.match(resource('LauncherFunction'), /ARM64_LAUNCH_TEMPLATE_ID: !Ref Arm64RunnerLaunchTemplate\n/); +}); + +// The image push (ADR-0002). +test('GitHub\'s OIDC provider is created only when ExistingOidcProviderArn is empty', () => { + assert.match(template, /^ {2}ExistingOidcProviderArn:\n {4}Type: String\n {4}Default: ""\n/m); + assert.match(template, /^Conditions:\n {2}CreateProvider: !Equals \[!Ref ExistingOidcProviderArn, ""\]\n/m); + const p = resource('GitHubOidcProvider'); + assert.match(p, /^ {4}Type: AWS::IAM::OIDCProvider\n {4}Condition: CreateProvider\n/); + assert.match(p, /Url: https:\/\/token\.actions\.githubusercontent\.com\n/); + assert.match(p, /ClientIdList: \[sts\.amazonaws\.com\]\n/); +}); + +test('the image push role trusts only zeroae/ci-runner\'s main, through whichever provider', () => { + const r = resource('ImagePushRole'); + assert.match(r, /RoleName: !Sub "PowerUserPB-\$\{AWS::StackName\}-image-push"/); + const trust = r.split('AssumeRolePolicyDocument:')[1].split(/\n {6}\w/)[0]; + assert.equal((trust.match(/- Effect: Allow/g) || []).length, 1, 'one trust statement'); + assert.match(trust, /Federated: !If \[CreateProvider, !Ref GitHubOidcProvider, !Ref ExistingOidcProviderArn\]/); + assert.match(trust, /Action: sts:AssumeRoleWithWebIdentity/); + assert.match(trust, /StringEquals:\n\s+token\.actions\.githubusercontent\.com:aud: sts\.amazonaws\.com\n\s+token\.actions\.githubusercontent\.com:sub: repo:zeroae@11018406\/ci-runner@1455021495:ref:refs\/heads\/main$/); + assert.doesNotMatch(trust, /StringLike|\*/, 'no wildcard in the trust'); +}); + +test('the image push role may push only to the stack\'s image repo', () => { + const policy = resource('ImagePushRole').split('PolicyDocument:')[2]; + const statements = policy.split(/\n {14}- Effect: /).slice(1); + assert.equal(statements.length, 2); + assert.match(statements[0], /^Allow\n {16}Action: ecr:GetAuthorizationToken\n {16}Resource: "\*"/); + assert.match(statements[1], /Resource: !GetAtt ImageRepository\.Arn\n$/); + assert.doesNotMatch(statements[1], /ecr:\*|Delete|SetRepositoryPolicy|PutLifecyclePolicy/); + assert.match(statements[1], /ecr:PutImage/); +}); + +test('the image repo keeps the last 4 tagged images', () => { + const r = resource('ImageRepository'); + assert.match(r, /^ {4}Type: AWS::ECR::Repository\n/); + const text = r.split(/LifecyclePolicyText: \|\n/)[1]; + assert.ok(text, 'no LifecyclePolicyText block'); + const { rules } = JSON.parse(text); + assert.equal(rules.length, 1); + assert.deepEqual(rules[0].selection, { + tagStatus: 'tagged', tagPatternList: ['*'], countType: 'imageCountMoreThan', countNumber: 4, + }); + assert.deepEqual(rules[0].action, { type: 'expire' }); +}); + +// The Fargate capacities (#29, ADR-0001). +const statements = (policy) => policy.split(/\n {14}- Effect: /).slice(1); + +test('the cluster offers both Fargate capacity providers', () => { + const c = resource('RunnerCluster'); + assert.match(c, /^ {4}Type: AWS::ECS::Cluster\n/); + assert.match(c, /CapacityProviders: \[FARGATE, FARGATE_SPOT\]\n/); + assert.match(c, /ClusterName: !Ref AWS::StackName\n/); +}); + +test('the two task definitions differ only in architecture and family: ECR image, wrapper, 50 GB, no task role', () => { + const x64 = resource('RunnerTaskDefinition'); + const arm = resource('Arm64RunnerTaskDefinition'); + assert.match(x64, /CpuArchitecture: X86_64 \}/); + assert.match(arm, /CpuArchitecture: ARM64 \}/); + assert.match(x64, /Family: !Sub "\$\{AWS::StackName\}-runner"\n/); + assert.match(arm, /Family: !Sub "\$\{AWS::StackName\}-runner-arm64"\n/); + const same = (s) => s.split('Properties:')[1].replace(/Family: .*\n/, '').replace(/CpuArchitecture: \w+/, ''); + assert.equal(same(arm), same(x64)); + assert.match(x64, /^ {4}Type: AWS::ECS::TaskDefinition\n/); + assert.match(x64, /RequiresCompatibilities: \[FARGATE\]\n/); + assert.match(x64, /NetworkMode: awsvpc\n/); + assert.match(x64, /EphemeralStorage: \{ SizeInGiB: 50 \}\n/); + assert.match(x64, /ExecutionRoleArn: !GetAtt RunnerExecutionRole\.Arn\n/); + assert.doesNotMatch(x64, /TaskRoleArn/, 'no task role: a job finds no AWS credentials'); + assert.match(x64, /Image: !Sub "\$\{ImageRepository\.RepositoryUri\}:latest"\n/); + assert.match(x64, /- Name: runner\n/); + assert.match(x64, /EntryPoint: \[\/usr\/local\/bin\/ci-runner-idle-exit\]\n/); + assert.match(x64, /awslogs-group: !Ref RunnerTaskLogGroup\n/); + assert.doesNotMatch(x64, /Privileged|docker\.sock/, 'no Docker on Fargate'); +}); + +test('the image itself sets no entrypoint, so EC2 runs it as before', () => { + const dockerfile = fs.readFileSync(path.join(__dirname, '..', 'image', 'Dockerfile'), 'utf8'); + assert.doesNotMatch(dockerfile, /^\s*(ENTRYPOINT|CMD)\b/m); + assert.match(dockerfile, /^COPY --chmod=0755 ci-runner-idle-exit \/usr\/local\/bin\/ci-runner-idle-exit$/m); +}); + +test('the tasks log to /ecs/-runner, kept 7 days', () => { + const g = resource('RunnerTaskLogGroup'); + assert.match(g, /LogGroupName: !Sub "\/ecs\/\$\{AWS::StackName\}-runner"\n/); + assert.match(g, /RetentionInDays: 7\n/); +}); + +test('the execution role may only pull the stack\'s image and write the task log group', () => { + const r = resource('RunnerExecutionRole'); + assert.match(r, /RoleName: !Sub "PowerUserPB-\$\{AWS::StackName\}-runner-exec"/); + assert.match(r, /Principal: \{ Service: ecs-tasks\.amazonaws\.com \}/); + const st = statements(r.split('PolicyDocument:')[2]); + assert.equal(st.length, 3); + assert.match(st[0], /^Allow\n {16}Action: ecr:GetAuthorizationToken\n {16}Resource: "\*"\n?$/); + assert.match(st[1], /^Allow\n {16}Action: \[ecr:BatchGetImage, ecr:GetDownloadUrlForLayer, ecr:BatchCheckLayerAvailability\]\n {16}Resource: !GetAtt ImageRepository\.Arn\n?$/); + assert.match(st[2], /^Allow\n {16}Action: \[logs:CreateLogStream, logs:PutLogEvents\]\n {16}Resource: !GetAtt RunnerTaskLogGroup\.Arn\n?$/); +}); + +test('the launcher\'s role runs only the two task definitions in the cluster, and passes only the execution role', () => { + const st = statements(resource('LauncherRole')); + const run = st.find((s) => /Action: ecs:RunTask\n/.test(s)); + assert.ok(run, 'no ecs:RunTask statement'); + assert.match(run, /Resource: \[!Ref RunnerTaskDefinition, !Ref Arm64RunnerTaskDefinition\]\n/); + assert.match(run, /ArnEquals:\n\s+ecs:cluster: !GetAtt RunnerCluster\.Arn/); + const pass = st.filter((s) => /Action: iam:PassRole/.test(s)); + assert.equal(pass.length, 1); + assert.match(pass[0], /Resource: !GetAtt RunnerExecutionRole\.Arn\n/); + assert.match(pass[0], /iam:PassedToService: ecs-tasks\.amazonaws\.com/); + const tag = st.find((s) => /Action: ecs:TagResource\n/.test(s)); + assert.match(tag, /ecs:CreateAction: RunTask/); + // The budget reads the cluster's tasks; nothing else on ECS. + for (const s of st.filter((x) => /ecs:/.test(x))) { + assert.doesNotMatch(s, /Action: .*(ecs:\*|StopTask|ecs:(Create|Delete|Update|Register))/, s); + assert.match(s, /RunnerCluster|RunnerTaskDefinition/, `${s}: not limited to the cluster`); + } +}); + +// Tasks get the instances' lifecycle (#30). +test('a second notifier rule matches every change of a Fargate Spot reclaim in the cluster', () => { + const rule = resource('NotifierTaskRule'); + assert.match(rule, /^ {4}Type: AWS::Events::Rule\n/); + assert.match(rule, /Name: !Sub "\$\{AWS::StackName\}-notifier-tasks"\n/); + assert.equal(rule.split('EventPattern:\n')[1].split(/\n {6}\w/)[0], [ + ' source: [aws.ecs]', + ' detail-type: [ECS Task State Change]', + ' detail:', + ' clusterArn: [!GetAtt RunnerCluster.Arn]', + ' stopCode: [SpotInterruption]', + ].join('\n')); + assert.match(rule, /Arn: !GetAtt NotifierFunction\.Arn\n/); + const perm = resource('NotifierTaskRulePermission'); + assert.match(perm, /Principal: events\.amazonaws\.com\n/); + assert.match(perm, /SourceArn: !GetAtt NotifierTaskRule\.Arn\n/); +}); + +test('the notifier may describe only the cluster\'s tasks, and nothing else on ECS', () => { + const ecs = statements(resource('NotifierRole')).filter((s) => /ecs:/.test(s)); + assert.deepEqual(ecs, [ + 'Allow\n Action: ecs:DescribeTasks\n Resource: !Sub "arn:${AWS::Partition}:ecs:${AWS::Region}:${AWS::AccountId}:task/${RunnerCluster}/*"\n', + ]); +}); + +test('the webhook may find and re-tag only the cluster\'s tasks, and only with the job tags', () => { + const ecs = statements(resource('WebhookRole')).filter((s) => /ecs:/.test(s)); + const tasks = 'Resource: !Sub "arn:${AWS::Partition}:ecs:${AWS::Region}:${AWS::AccountId}:task/${RunnerCluster}/*"'; + const at = (n) => '\n' + ' '.repeat(n); + assert.deepEqual(ecs, [ + `Allow${at(16)}Action: ecs:ListTasks${at(16)}Resource: "*"${at(16)}Condition:${at(18)}ArnEquals:${at(20)}ecs:cluster: !GetAtt RunnerCluster.Arn`, + `Allow${at(16)}Action: ecs:DescribeTasks${at(16)}${tasks}`, + `Allow${at(16)}Action: ecs:TagResource${at(16)}${tasks}${at(16)}Condition:${at(18)}ForAllValues:StringEquals:${at(20)}` + + 'aws:TagKeys: [ci-runner:repo, ci-runner:job-id, ci-runner:run-id, ci-runner:head-sha, ci-runner:job-name]\n', + ]); + assert.match(resource('WebhookFunction'), /CLUSTER: !Ref RunnerCluster\n/); +}); diff --git a/test/webhook.test.js b/test/webhook.test.js index c419dcb..e910f95 100644 --- a/test/webhook.test.js +++ b/test/webhook.test.js @@ -6,23 +6,61 @@ const { createHandler } = require('../functions/webhook.js'); const SECRET = 'whsec-test'; -function setup({ secretValue = { webhook_secret: SECRET } } = {}) { +// instances: what DescribeInstances knows, as { InstanceId, Tags: { key: value } }. +// The fake applies the tag filters the way EC2 does. +// tasks: the cluster's tasks, as { taskArn, desiredStatus, tags: { key: value } }. +function setup({ secretValue = { webhook_secret: SECRET }, instances = [], tasks = [] } = {}) { const sent = []; + const described = []; + const tagged = []; + const taskTagged = []; + const logs = []; let reads = 0; const handler = createHandler({ secret: { get: async () => { reads++; return secretValue; } }, queue: { send: async (msg) => { sent.push(msg); } }, - env: { GITHUB_ORG: 'zeroae' }, + ec2: { + describeInstances: async (input) => { + described.push(input); + const match = (i) => input.Filters.every((f) => !f.Name.startsWith('tag:') || f.Values.includes(i.Tags[f.Name.slice(4)])); + const Instances = instances.filter(match).map((i) => ({ + InstanceId: i.InstanceId, State: { Name: 'running' }, + Tags: Object.entries(i.Tags).map(([Key, Value]) => ({ Key, Value })), + })); + return { Reservations: Instances.length ? [{ Instances }] : [] }; + }, + createTags: async (input) => { tagged.push(input); }, + }, + ecs: { + listTasks: async (input) => { + described.push(input); + assert.equal(input.cluster, 'ci-runner'); + return { taskArns: tasks.filter((t) => t.desiredStatus === (input.desiredStatus || 'RUNNING')).map((t) => t.taskArn) }; + }, + describeTasks: async (input) => { + assert.deepEqual([input.cluster, input.include], ['ci-runner', ['TAGS']]); + return { + tasks: tasks.filter((t) => input.tasks.includes(t.taskArn)).map((t) => ({ + taskArn: t.taskArn, desiredStatus: t.desiredStatus, lastStatus: 'RUNNING', + tags: Object.entries(t.tags).map(([key, value]) => ({ key, value })), + })), + }; + }, + tagResource: async (input) => { taskTagged.push(input); return {}; }, + }, + env: { GITHUB_ORG: 'zeroae', STACK_NAME: 'ci-runner', CLUSTER: 'ci-runner' }, + log: (line) => logs.push(JSON.parse(line)), }); - return { handler, sent, reads: () => reads }; + return { handler, sent, described, tagged, taskTagged, logs, reads: () => reads }; } // A workflow_job payload as GitHub sends it (only the fields that matter). -function job({ action = 'queued', labels = ['self-hosted', 'ci-spot'], visibility = 'private', owner = 'zeroae' } = {}) { +function job({ action = 'queued', labels = ['self-hosted', 'ci-spot', 'ci-cpu-4', 'x64'], visibility = 'private', owner = 'zeroae', runner } = {}) { return { action, workflow_job: { id: 555, run_id: 444, head_sha: 'abc123', name: 'tests', labels, + ...(runner && { runner_name: runner, runner_id: 77 }), }, repository: { full_name: `${owner}/app`, name: 'app', visibility, private: visibility !== 'public', owner: { login: owner } }, installation: { id: 181 }, @@ -46,10 +84,45 @@ test('an accepted ci-spot job is enqueued with every field', async () => { assert.equal(res.statusCode, 202); assert.deepEqual(sent, [{ repo: 'zeroae/app', job_id: 555, run_id: 444, head_sha: 'abc123', job_name: 'tests', - label: 'ci-spot', labels: ['self-hosted', 'ci-spot'], installation_id: 181, + label: 'ci-spot', vcpus: 4, arch: 'x64', labels: ['self-hosted', 'ci-spot', 'ci-cpu-4', 'x64'], installation_id: 181, }]); }); +test('every size label is enqueued with its vCPUs and the architecture', async () => { + for (const [size, vcpus] of [['ci-cpu-2', 2], ['ci-cpu-4', 4], ['ci-cpu-8', 8], ['ci-cpu-16', 16]]) { + for (const cap of ['ci-spot', 'ci-ondemand']) { + const labels = ['self-hosted', cap, size, 'x64']; + const { handler, sent } = setup(); + assert.equal((await handler(delivery(job({ labels })))).statusCode, 202, labels.join(',')); + assert.deepEqual( + { label: sent[0].label, vcpus: sent[0].vcpus, arch: sent[0].arch, labels: sent[0].labels }, + { label: cap, vcpus, arch: 'x64', labels }, + ); + } + } +}); + +test('a job missing its size label or its architecture label is outside the contract', async () => { + for (const labels of [ + ['self-hosted', 'ci-spot', 'x64'], + ['self-hosted', 'ci-ondemand', 'arm64'], + ['self-hosted', 'ci-spot', 'ci-cpu-4'], + ['self-hosted', 'linux', 'ci-ondemand', 'ci-cpu-8'], + ['self-hosted', 'ci-spot'], + ['self-hosted', 'linux', 'ci-ondemand'], + ]) { + const { handler, sent } = setup(); + assert.equal((await handler(delivery(job({ labels })))).statusCode, 200, labels.join(',')); + assert.equal(sent.length, 0, labels.join(',')); + } +}); + +test('the size label is read case-insensitively', async () => { + const { handler, sent } = setup(); + await handler(delivery(job({ labels: ['Self-Hosted', 'CI-Spot', 'CI-CPU-16', 'X64'] }))); + assert.equal(sent[0].vcpus, 16); +}); + test('a base64-encoded body is verified on its decoded bytes', async () => { const { handler, sent } = setup(); assert.equal((await handler(delivery(job(), { base64: true }))).statusCode, 202); @@ -82,19 +155,45 @@ test('the webhook secret is read once and cached', async () => { assert.equal(reads(), 1); }); -test('the label contract: self-hosted plus exactly one capacity label, otherwise only linux/x64', async () => { +test('an arm64 job is enqueued with arch arm64, on either capacity and at every size', async () => { + for (const cap of ['ci-spot', 'ci-ondemand']) { + for (const size of ['ci-cpu-2', 'ci-cpu-4', 'ci-cpu-8', 'ci-cpu-16']) { + const labels = ['self-hosted', cap, size, 'arm64']; + const { handler, sent } = setup(); + assert.equal((await handler(delivery(job({ labels })))).statusCode, 202, labels.join(',')); + assert.equal(sent[0].arch, 'arm64', labels.join(',')); + } + } + const { handler, sent } = setup(); + await handler(delivery(job({ labels: ['Self-Hosted', 'CI-Spot', 'CI-CPU-4', 'ARM64'] }))); + assert.deepEqual([sent[0].vcpus, sent[0].arch], [4, 'arm64']); +}); + +test('the label contract: self-hosted, exactly one capacity, size and architecture label, and linux', async () => { const accepted = [ - ['self-hosted', 'ci-spot'], - ['self-hosted', 'ci-ondemand'], - ['self-hosted', 'linux', 'x64', 'ci-spot'], - ['Self-Hosted', 'CI-Spot'], + ['self-hosted', 'ci-spot', 'ci-cpu-4', 'x64'], + ['self-hosted', 'ci-ondemand', 'ci-cpu-4', 'x64'], + ['self-hosted', 'linux', 'x64', 'ci-cpu-2', 'ci-spot'], + ['Self-Hosted', 'CI-Spot', 'CI-CPU-4', 'X64'], + ['self-hosted', 'ci-spot', 'ci-cpu-8', 'x64'], + ['self-hosted', 'linux', 'ci-ondemand', 'ci-cpu-2', 'arm64'], + ['self-hosted', 'ci-spot', 'ci-cpu-16', 'arm64'], + ['self-hosted', 'linux', 'ci-spot', 'ci-cpu-8', 'arm64'], ]; const ignored = [ ['ci-spot'], ['self-hosted'], - ['self-hosted', 'ci-spot', 'ci-ondemand'], - ['self-hosted', 'ci-spot', 'gpu'], - ['self-hosted', 'ci-spot', 'arm64'], + ['self-hosted', 'ci-spot', 'ci-ondemand', 'ci-cpu-4', 'x64'], + ['self-hosted', 'ci-spot', 'ci-cpu-4', 'x64', 'gpu'], + ['self-hosted', 'ci-spot', 'ci-cpu-4', 'ci-cpu-8', 'x64'], + ['self-hosted', 'ci-spot', 'ci-cpu-8', 'ci-cpu-8', 'x64'], + ['self-hosted', 'ci-spot', 'ci-cpu-3', 'x64'], + ['self-hosted', 'ci-spot', 'ci-cpu-32', 'x64'], + ['self-hosted', 'ci-spot', 'ci-cpu-4', 'x64', 'arm64'], + ['self-hosted', 'ci-spot', 'ci-cpu-8', 'ARM64', 'x64'], + ['self-hosted', 'ci-spot', 'ci-cpu-4', 'arm64', 'arm64'], + ['self-hosted', 'ci-spot', 'ci-cpu-4', 'aarch64'], + ['self-hosted', 'ci-spot', 'ci-cpu-4', 'x64', 'x64'], ['ubuntu-latest'], [], ]; @@ -112,7 +211,7 @@ test('the label contract: self-hosted plus exactly one capacity label, otherwise test('the capacity label is reported in lower case', async () => { const { handler, sent } = setup(); - await handler(delivery(job({ labels: ['self-hosted', 'CI-OnDemand'] }))); + await handler(delivery(job({ labels: ['self-hosted', 'CI-OnDemand', 'ci-cpu-4', 'x64'] }))); assert.equal(sent[0].label, 'ci-ondemand'); }); @@ -131,7 +230,6 @@ test('fence 3: public or internal repos, and owners other than zeroae, are refus test('other actions and other events are ignored', async () => { for (const d of [ - delivery(job({ action: 'in_progress' })), delivery(job({ action: 'completed' })), delivery({ zen: 'hi' }, { eventName: 'ping' }), ]) { @@ -140,3 +238,178 @@ test('other actions and other events are ignored', async () => { assert.equal(sent.length, 0); } }); + +// A runner the launcher started for job 999 of run 888, and its instance. +const RUNNER = 'ci-runner-20261113120000-999'; +const instance = (tags = {}) => ({ + InstanceId: 'i-0abc', + Tags: { + 'ci-runner:stack': 'ci-runner', 'ci-runner:runner': RUNNER, 'ci-runner:runner-id': '77', + 'ci-runner:label': 'ci-spot', 'ci-runner:labels': 'self-hosted,ci-spot,ci-cpu-8,arm64,linux', + 'ci-runner:capacity': 'ci-spot', 'ci-runner:vcpus': '8', 'ci-runner:arch': 'arm64', + 'ci-runner:repo': 'zeroae/lib', 'ci-runner:job-id': '999', 'ci-runner:run-id': '888', + 'ci-runner:head-sha': 'fff000', 'ci-runner:job-name': 'build', + ...tags, + }, +}); + +test('a swap: the runner\'s own job is re-queued from its tags, and the instance is re-tagged with the job it runs', async () => { + const { handler, sent, tagged, logs } = setup({ instances: [instance()] }); + const res = await handler(delivery(job({ action: 'in_progress', runner: RUNNER }))); + assert.equal(res.statusCode, 202); + assert.deepEqual(sent, [{ + repo: 'zeroae/lib', job_id: 999, run_id: 888, head_sha: 'fff000', job_name: 'build', + label: 'ci-spot', vcpus: 8, arch: 'arm64', labels: ['self-hosted', 'ci-spot', 'ci-cpu-8', 'arm64', 'linux'], installation_id: 181, + }]); + assert.deepEqual(tagged, [{ + Resources: ['i-0abc'], + Tags: [ + { Key: 'ci-runner:repo', Value: 'zeroae/app' }, { Key: 'ci-runner:job-id', Value: '555' }, + { Key: 'ci-runner:run-id', Value: '444' }, { Key: 'ci-runner:head-sha', Value: 'abc123' }, + { Key: 'ci-runner:job-name', Value: 'tests' }, + ], + }]); + assert.deepEqual(logs, [{ + swap: RUNNER, instance: 'i-0abc', took: { repo: 'zeroae/app', job_id: 555 }, requeued: { repo: 'zeroae/lib', job_id: 999 }, + }]); +}); + +test('in_progress on a runner running its own job does nothing', async () => { + const own = 'ci-runner-20261113120000-555'; + const { handler, sent, described, tagged, logs } = setup({ instances: [instance({ 'ci-runner:runner': own, 'ci-runner:job-id': '555' })] }); + assert.equal((await handler(delivery(job({ action: 'in_progress', runner: own })))).statusCode, 200); + assert.deepEqual([sent, described, tagged, logs], [[], [], [], []]); +}); + +test('in_progress on a runner that isn\'t the stack\'s is ignored', async () => { + const others = [ + 'GitHub Actions 2', 'other-stack-20261113120000-999', 'ci-runner-hand-20261113120000', + 'ci-runner-2026111312-999', undefined, + ]; + for (const runner of others) { + const { handler, sent, tagged } = setup({ instances: [instance({ 'ci-runner:runner': runner })] }); + assert.equal((await handler(delivery(job({ action: 'in_progress', runner })))).statusCode, 200, String(runner)); + assert.deepEqual([sent, tagged], [[], []], String(runner)); + } + // The stack's name, but no instance of the stack carries it. + const { handler, sent, tagged } = setup({ instances: [instance({ 'ci-runner:stack': 'other' })] }); + assert.equal((await handler(delivery(job({ action: 'in_progress', runner: RUNNER })))).statusCode, 200); + assert.deepEqual([sent, tagged], [[], []]); +}); + +test('a repeated in_progress after the re-tag does nothing more', async () => { + const { handler, sent, tagged } = setup({ instances: [instance({ 'ci-runner:job-id': '555' })] }); + assert.equal((await handler(delivery(job({ action: 'in_progress', runner: RUNNER })))).statusCode, 200); + assert.deepEqual([sent, tagged], [[], []]); +}); + +test('a swap on an instance launched before the labels and shape tags re-queues a ci-cpu-4 x64 job with self-hosted and its capacity label', async () => { + const old = instance(); + for (const k of ['labels', 'capacity', 'vcpus', 'arch']) delete old.Tags[`ci-runner:${k}`]; + const { handler, sent } = setup({ instances: [old] }); + await handler(delivery(job({ action: 'in_progress', runner: RUNNER }))); + assert.deepEqual([sent[0].labels, sent[0].vcpus, sent[0].arch], [['self-hosted', 'ci-spot'], 4, 'x64']); +}); + +test('in_progress from outside the fence is refused before any lookup', async () => { + const { handler, sent, described } = setup({ instances: [instance()] }); + assert.equal((await handler(delivery(job({ action: 'in_progress', runner: RUNNER, visibility: 'public' })))).statusCode, 403); + assert.deepEqual([sent, described], [[], []]); +}); + +// The Fargate capacities (#29): Docker-free jobs, ADR-0001. +test('both Fargate capacities are enqueued with every size and architecture', async () => { + for (const cap of ['ci-fargate-spot', 'ci-fargate']) { + for (const [size, vcpus] of [['ci-cpu-2', 2], ['ci-cpu-4', 4], ['ci-cpu-8', 8], ['ci-cpu-16', 16]]) { + for (const arch of ['x64', 'arm64']) { + const labels = ['self-hosted', cap, size, arch]; + const { handler, sent } = setup(); + assert.equal((await handler(delivery(job({ labels })))).statusCode, 202, labels.join(',')); + assert.deepEqual( + { label: sent[0].label, vcpus: sent[0].vcpus, arch: sent[0].arch, labels: sent[0].labels }, + { label: cap, vcpus, arch, labels }, + ); + } + } + } + const { handler, sent } = setup(); + await handler(delivery(job({ labels: ['Self-Hosted', 'CI-Fargate-Spot', 'ci-cpu-4', 'ARM64', 'Linux'] }))); + assert.equal(sent[0].label, 'ci-fargate-spot'); +}); + +test('a Fargate job follows the same contract: no default size or architecture, one capacity', async () => { + for (const labels of [ + ['self-hosted', 'ci-fargate-spot'], + ['self-hosted', 'ci-fargate', 'x64'], + ['self-hosted', 'ci-fargate-spot', 'ci-cpu-4'], + ['self-hosted', 'ci-fargate', 'ci-cpu-4', 'ci-cpu-8', 'x64'], + ['self-hosted', 'ci-fargate-spot', 'ci-cpu-4', 'x64', 'arm64'], + ['self-hosted', 'ci-fargate', 'ci-fargate-spot', 'ci-cpu-4', 'x64'], + ['self-hosted', 'ci-fargate-spot', 'ci-spot', 'ci-cpu-4', 'x64'], + ['self-hosted', 'ci-fargate', 'ci-ondemand', 'ci-cpu-4', 'x64'], + ]) { + const { handler, sent } = setup(); + assert.equal((await handler(delivery(job({ labels })))).statusCode, 200, labels.join(',')); + assert.equal(sent.length, 0); + } +}); + +// A runner the launcher started on Fargate Spot for job 999, and its task. ECS +// refuses commas and brackets in a tag: the labels are space-joined and the +// job name's brackets and commas are `_`. +const TASK_ARN = 'arn:aws:ecs:us-east-1:123456789012:task/ci-runner/0f1e2d3c4b5a69788796a5b4c3d2e1f0'; +const fargateTask = (tags = {}, desiredStatus = 'RUNNING') => ({ + taskArn: TASK_ARN, desiredStatus, + tags: { + ...instance().Tags, 'ci-runner:label': 'ci-fargate-spot', 'ci-runner:capacity': 'ci-fargate-spot', + 'ci-runner:labels': 'self-hosted ci-fargate-spot ci-cpu-8 arm64 linux', 'ci-runner:job-name': 'build _x64_ a_', + ...tags, + }, +}); +const fargateJob = () => { + const p = job({ action: 'in_progress', runner: RUNNER, labels: ['self-hosted', 'ci-fargate-spot', 'ci-cpu-8', 'arm64', 'linux'] }); + p.workflow_job.name = 'test (arm64, b)'; + return p; +}; + +test('a swap on a task: the runner\'s own job is re-queued from the task\'s tags, and the task is re-tagged with the job it runs', async () => { + const { handler, sent, tagged, taskTagged, logs } = setup({ tasks: [fargateTask()] }); + const res = await handler(delivery(fargateJob())); + assert.equal(res.statusCode, 202); + assert.deepEqual(sent, [{ + repo: 'zeroae/lib', job_id: 999, run_id: 888, head_sha: 'fff000', job_name: 'build _x64_ a_', + label: 'ci-fargate-spot', vcpus: 8, arch: 'arm64', labels: ['self-hosted', 'ci-fargate-spot', 'ci-cpu-8', 'arm64', 'linux'], installation_id: 181, + }]); + assert.deepEqual(tagged, []); + assert.deepEqual(taskTagged, [{ + resourceArn: TASK_ARN, + tags: [ + { key: 'ci-runner:repo', value: 'zeroae/app' }, { key: 'ci-runner:job-id', value: '555' }, + { key: 'ci-runner:run-id', value: '444' }, { key: 'ci-runner:head-sha', value: 'abc123' }, + { key: 'ci-runner:job-name', value: 'test _arm64_ b_' }, + ], + }]); + assert.deepEqual(logs, [{ + swap: RUNNER, task: TASK_ARN, took: { repo: 'zeroae/app', job_id: 555 }, requeued: { repo: 'zeroae/lib', job_id: 999 }, + }]); +}); + +test('a repeated in_progress after a task\'s re-tag does nothing more', async () => { + const { handler, sent, taskTagged } = setup({ tasks: [fargateTask({ 'ci-runner:job-id': '555' })] }); + assert.equal((await handler(delivery(fargateJob()))).statusCode, 200); + assert.deepEqual([sent, taskTagged], [[], []]); +}); + +test('a task told to stop, or another runner\'s task, is no swap to repair', async () => { + for (const t of [fargateTask({}, 'STOPPED'), fargateTask({ 'ci-runner:runner': 'ci-runner-20261113120000-998' })]) { + const { handler, sent, taskTagged } = setup({ tasks: [t] }); + assert.equal((await handler(delivery(fargateJob()))).statusCode, 200); + assert.deepEqual([sent, taskTagged], [[], []]); + } +}); + +test('a swap on an instance looks up no task', async () => { + const { handler, described } = setup({ instances: [instance()], tasks: [fargateTask()] }); + assert.equal((await handler(delivery(job({ action: 'in_progress', runner: RUNNER })))).statusCode, 202); + assert.ok(described.every((d) => d.Filters), 'only DescribeInstances'); +});