diff --git a/.gitignore b/.gitignore index d5e60da..0822159 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ build/ .env +.claude/worktrees/ diff --git a/AGENTS.md b/AGENTS.md index c8a2f30..6ee6233 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -2,7 +2,9 @@ - Changing the stack, a function or the build: read `docs/stack.md`. - Operating the live stack (deploy, the wizard, logs, a lost job): read `docs/runbook.md`. -- Reviewing a change: apply `CODING_STANDARDS.md`. +- Proving a change live (deploying a branch to the shared stack): read "Proving a change live" in `docs/runbook.md` first. +- Implementing a ticket as a subagent: follow `docs/agents/implementer.md`. +- Reviewing or merging a change: apply `CODING_STANDARDS.md`, and gate the merge per `docs/agents/review-gate.md`. ## Agent skills diff --git a/CODING_STANDARDS.md b/CODING_STANDARDS.md index 6142056..d7852bf 100644 --- a/CODING_STANDARDS.md +++ b/CODING_STANDARDS.md @@ -31,4 +31,4 @@ adoption variable `ZAE_CI_RUNNER`. Grant the least the function does, scoped by the stack's tag (`ci-runner:stack`) or its own resources, never `*` on a write action without a condition. A new condition is checked live: some EC2 calls don't -carry the keys their docs suggest (see the traps table in `docs/runbook.md`). +carry the keys their docs suggest (see `docs/traps.md`). diff --git a/Makefile b/Makefile index 1bab1c6..112d976 100644 --- a/Makefile +++ b/Makefile @@ -6,12 +6,12 @@ AWS = aws --profile $(PROFILE) --region $(REGION) # The deploy bucket, made outside the stack (docs/runbook.md). Looked up only when used. BUCKET ?= $(STACK)-deploy-$(shell $(AWS) sts get-caller-identity --query Account --output text) -.PHONY: help test build validate deploy outputs hand-launch runners sweep canary-runs +.PHONY: help test build validate deploy outputs hand-launch runners sweep canary-runs consumers .DEFAULT_GOAL := help help: ## List the targets @grep -E '^[a-z-]+:.*## ' $(MAKEFILE_LIST) | awk -F':.*## ' '{printf " make %-12s %s\n", $$1, $$2}' - @echo " Layout and conventions: docs/stack.md; procedures and traps: docs/runbook.md" + @echo " Layout and conventions: docs/stack.md; procedures: docs/runbook.md; traps: docs/traps.md" test: ## Run the unit tests (node --test, no npm deps) node --test test/*.test.js @@ -49,3 +49,6 @@ sweep: ## Run the sweeper now, and print what it terminated, stopped and removed canary-runs: ## List the canary repo's recent workflow runs (CI_RUNNER_CANARY_REPO, from .env) @test -f .env || { echo "no .env: run scripts/setup-github.sh first" >&2; exit 1; } @. ./.env && gh run list --repo $(ORG)/$$CI_RUNNER_CANARY_REPO --limit 10 + +consumers: ## List every job, in the runner group's repos, whose runs-on names a capacity (needs gh with admin:org) + @ORG=$(ORG) scripts/consumers.sh diff --git a/README.md b/README.md index 17d041c..cfade50 100644 --- a/README.md +++ b/README.md @@ -99,7 +99,8 @@ it's in both. include `linux`. None of them has a default. A job missing a size or an architecture label, or with any other label (`aarch64`, an unknown `ci-cpu-*`, two size labels, both `x64` and `arm64`), is ignored, and it waits -for a runner that never comes. +for a runner that never comes. Before changing the contract, `make consumers` +lists every job it affects. **What the job gets:** Ubuntu 24.04, the runner as user `runner` (passwordless sudo), Docker through the host's socket, the `docker compose` and `buildx` @@ -142,7 +143,8 @@ must drop that condition before a deploy job moves. ## Docs - [docs/runbook.md](docs/runbook.md): deploy, create the App, turn repos on and - off, read a lost job or the logs, and the traps table. + off, read a lost job or the logs, prove a change live. +- [docs/traps.md](docs/traps.md): symptoms and their causes. - [docs/stack.md](docs/stack.md): the stack's layout, the Lambdas, and how to change them. - [The design spec](docs/superpowers/specs/2026-11-12-ci-runner-design.md). diff --git a/docs/agents/implementer.md b/docs/agents/implementer.md new file mode 100644 index 0000000..f4ac05f --- /dev/null +++ b/docs/agents/implementer.md @@ -0,0 +1,25 @@ +# Implementer + +What every implementer subagent working a ticket here does. The brief names +the ticket and the **integration branch**; it overrides this file where they +differ. + +1. **Base** your branch on the integration branch: `git fetch origin`, then + `git merge-base --is-ancestor origin/ HEAD`; if that fails, + `git reset --hard origin/`. +2. **Build** test-first with the Skill `tdd`, at the `createHandler(deps)` + seams (`docs/stack.md` "Adding a Lambda"). Where a test needs AWS's real + request and response shapes (ECS, EC2), use aws/aws-emulator through + `test/aws-emulator.js`. +3. **AWS** runs under the profile `zeroae-code/ZAEPowerUserAccess`. Only the + owner runs `aws sso login`; when the session expires, report blocked. +4. **Deploy** only as `docs/runbook.md` "Proving a change live" says. +5. **Commit** with the `Co-Authored-By` trailer the harness gives. +6. **Before reporting**: merge the integration branch's tip, then + `make test`, `make build`, and push your branch. +7. **Report** once, when done or blocked: the branch, its head SHA, and what + is unverified. + +Merging into the integration branch, opening PRs and closing issues belong +to the controller, unless the brief hands them to you. Waiting on commands +follows the subagent rules in the user's global `CLAUDE.md`. diff --git a/docs/agents/review-gate.md b/docs/agents/review-gate.md new file mode 100644 index 0000000..db3b61f --- /dev/null +++ b/docs/agents/review-gate.md @@ -0,0 +1,20 @@ +# Review gate + +A change merges to `main` only after a `/code-review` over everything since +the last reviewed commit (the **fixed point**), applying +`CODING_STANDARDS.md`. + +The review ends by posting a commit status on the head it reviewed: + +```bash +gh api repos/zeroae/ci-runner/statuses/ -f state=success \ + -f context=code-review -f description=".." +``` + +`main`'s ruleset requires the `code-review` status on the head being merged. +The status belongs to one SHA, so any later push (a fix, a merge of `main`, +a small landing PR) needs a new review of the delta: the fixed point becomes +the last reviewed SHA, and the new status goes on the new head. + +The gate guards against forgetting the review. It is not a security control: +anyone with push access can post the status. diff --git a/docs/runbook.md b/docs/runbook.md index 4f287c6..dc0c167 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -97,6 +97,36 @@ numbers"). The built template must stay under 1 MB, the S3 deploy's limit (`make build` prints its size and fails at the limit). +## Proving a change live + +The live stack runs whatever branch was deployed last, for every consumer +repo. Whoever deploys a branch owns the stack until they report back. + +- **One prover at a time.** Agree who holds the stack before deploying. +- **Deploy on top of the latest work.** Merge the integration branch's tip + (or `main`) into your branch first, so your deploy carries everyone's + changes, not an older stack. +- **Review twice.** Deploy only code that has had its `/code-review`, and + read the CloudFormation change set before executing it: in-place updates + only, nothing replaced or deleted. +- **Keep shared budgets.** Consumer jobs draw on the same budgets, so prove a + deferral on On-Demand at a low budget, not by lowering Spot. Restore every + parameter you override from the same script, with a `trap`, so a failure + restores it too: + + ```bash + trap '$AWS cloudformation deploy --stack-name ci-runner \ + --template-file build/template.yaml --s3-bucket "$B" --s3-prefix ci-runner \ + --capabilities CAPABILITY_NAMED_IAM --no-fail-on-empty-changeset \ + --parameter-overrides OnDemandVcpuCap=8' EXIT + ``` + +- **Image changes need `main`.** A change under `image/` reaches the runners + only once `image.yml` publishes it from `main` (see "Image changes land on + `main` first" in [stack.md](stack.md)); a branch deploy can't prove it. +- **Clean up** what the proof made: throwaway canary branches, runner + registrations and tasks made by hand. + ## Push the image to ECR `image.yml` pushes the runner image, with a SOCI v2 index, to the stack's @@ -106,7 +136,7 @@ ECR repo on every `main` build (see "The runner image in ECR" in GitHub token is stored anywhere. Set it up once: 1. Deploy with the account's existing GitHub provider. In `zeroae-code` it - is owned by another stack (see the traps table): + is owned by another stack (see [traps.md](traps.md)): ```bash OIDC=$($AWS iam list-open-id-connect-providers --output text \ @@ -249,6 +279,11 @@ A repo is served when all of these hold: gh api -X DELETE orgs/zeroae/actions/runner-groups/$GROUP_ID/repositories/$REPO_ID ``` +**Who a label-contract change affects:** `make consumers` (`scripts/consumers.sh`) +prints `repo workflow job runs-on` for every job, in the group's repos, +whose `runs-on` names a capacity. It reads each repo's workflows directly, +since GitHub's code search misses private repos. + ## Read a lost job When AWS reclaims a Spot instance or a Fargate Spot task, GitHub marks the job failed, with *The @@ -274,7 +309,7 @@ run by hand, carries no job tags, so the notifier ignores it and logs `ignored: is no job instance of this stack` (`... no job task ...` for a task). A task's stop code is in `describe-tasks` (below): `SpotInterruption` is a reclaim. A job that sits -*Waiting for a runner* is not lost: see the traps table, then the launcher's +*Waiting for a runner* is not lost: see [traps.md](traps.md), then the launcher's log. ## Read the logs @@ -499,8 +534,8 @@ types. `Standard/OnDemand` as fits. A Fargate quota at its AWS default of 6 vCPUs is below its budget: with - the default budgets, a job then fails to launch instead of waiting (see the - traps table). + the default budgets, a job then fails to launch instead of waiting (see + [traps.md](traps.md)). - **Template size:** 55,871 bytes, 5% of the S3 deploy's 1 MB limit. It's already over the 51,200 bytes a template can be sent inline, so it deploys @@ -508,42 +543,5 @@ types. ## Traps -| Symptom | Cause | -|---|---| -| A job's `docker run -v ...` sees an empty directory, fails `error mounting "...": not a directory`, or hits `read-only file system` inside a mounted config directory, only on our runners | the source isn't under a path shared with the host, so Docker mounts an empty host path (for a missing file, a directory it creates). Only `/tmp` and `/home/runner` are shared, at the same paths; mount from one of those. If `/tmp` or `$HOME` itself fails, the start script's mounts (`functions/cloud-config.js`) changed | -| `permission denied` on `/var/run/docker.sock` | the host's `docker` group isn't gid 123 | -| `setup-python` fails, or picks up a broken Python | `AGENT_TOOLSDIRECTORY` isn't `/opt/hostedtoolcache` | -| A deploy waits 60 minutes, then fails on the state lock | an `apply` ran on Spot and was reclaimed; it must be `ci-ondemand` | -| A job sits "Waiting for a runner" | its capacity's vCPU budget is full, the repo isn't in the group, the App isn't installed on it, or the repo is public. Also check the label contract: `self-hosted`, exactly one of `ci-spot`/`ci-ondemand`/`ci-fargate-spot`/`ci-fargate`, exactly one size label (`ci-cpu-2`/`-4`/`-8`/`-16`), exactly one of `x64`/`arm64`, and otherwise only `linux`. The launcher's log tells the first case (`deferred`) apart from the rest (no line at all). A job that queued before the repo was turned on stays waiting after the fix: cancel and re-run it | -| A job that ran on our runners before now sits "Waiting for a runner", and the webhook's log shows no `enqueued` line for it | missing size or architecture label: its `runs-on` names only its capacity (`self-hosted`, `ci-spot`). There are no defaults; add a size and an architecture label (`ci-cpu-4` and `x64` keep the old shape), then cancel and re-run the job | -| An `arm64` runner's instance comes up and terminates without taking its job, and its console shows `no matching manifest for linux/arm64` | `ghcr.io/zeroae/ci-runner:latest` has no arm64 entry yet: the image workflow hasn't pushed a multi-arch `:latest` since it began building arm64. Run the image workflow on `main`, then re-run the job | -| A job's runner was launched for another job (`$RUNNER_NAME` ends in another job id), and that other job sits "Waiting for a runner" | a swap. The webhook repairs it on the job's `in_progress` (its log says `swap`), and the launcher then launches a runner for the other job. No `swap` line: check the webhook's log for errors (its role needs `ec2:DescribeInstances` and `ec2:CreateTags`, and for a task `ecs:ListTasks`, `ecs:DescribeTasks` and `ecs:TagResource`); cancel and re-run the waiting job | -| A Fargate Spot job was lost (its task's `stopCode` is `SpotInterruption`) and its commit has no `ci-runner / ` check run | the notifier logged `ignored: is no job task of this stack` for every change of the stop: it found no job tags on the task. ECS keeps a stopped task's tags readable through `DescribeTasks` (measured live), so check the task's tags (`describe-tasks --include TAGS`, within an hour of the stop; `list-tags-for-resource` refuses a stopped task) and the notifier's role. Re-run the job | -| A job fails "The runner has received a shutdown signal", "lost communication" or "exit code 130", next to a failed `ci-runner / ` check | Spot reclaim: re-run the job | -| A budget lowered for a test is still low after the next `make deploy` | `cloudformation deploy` keeps a parameter's previous value unless it's overridden; restore it with `--parameter-overrides SpotVcpuCap=24 OnDemandVcpuCap=8 FargateSpotVcpuCap=48 FargateVcpuCap=8`. Each draws on its own vCPU quota (see "Deploy and update the stack") | -| A job on `ci-fargate-spot` or `ci-fargate` fails at a Docker step: `failed to connect to the docker API at unix:///var/run/docker.sock; check if the path is correct and if the daemon is running`, or a `services:`/`container:` job or Docker action fails to start | Docker used on Fargate. A Fargate capacity has no Docker daemon, by design (ADR-0001): move the job to `ci-spot` or `ci-ondemand`, or drop its Docker use | -| The launcher logs `no task launched:` with a reason about a vCPU limit, and the job's message keeps coming back until the dead-letter queue | a Fargate quota at its default (6 vCPUs, `L-36FBB829` Spot, `L-3032A538` On-Demand) or below the capacity's budget. Request an increase (Service Quotas, service `fargate`) or lower `FargateSpotVcpuCap`/`FargateVcpuCap` under the quota; then re-run the job. The quota counts every Fargate task in the account, not only ours | -| A Fargate job's build is slow, or runs out of memory, with many more compiler processes than the size's vCPUs | `nproc` over-reports inside a task. The job's environment sets `MAKEFLAGS=-j` and `CI_RUNNER_VCPUS=`, and the entrypoint caps cargo with `/home/runner/.cargo/config.toml` (its log says `cargo jobs = `, or why not). A tool that reads `nproc` any other way needs `-j` from `CI_RUNNER_VCPUS` | -| A Fargate job never hits the Rust cache an EC2 job saved (`Swatinem/rust-cache` restores nothing, and the two save under different keys) | an environment variable starting `CARGO`, `CC`, `CFLAGS`, `CXX`, `CMAKE` or `RUST` differs between them: rust-cache hashes them all into its key. The stack sets none on a task (cargo's jobs go in `~/.cargo/config.toml`, not `CARGO_BUILD_JOBS`); if the task has `CARGO_BUILD_JOBS`, the launcher predates that, so deploy. Otherwise look for one the job sets only on Fargate | -| The launcher logs `not authorized to perform: iam:PassRole on resource: ...runner-exec because no permissions boundary allows the iam:PassRole action` for every Fargate job | the launcher's role has the `PowerUserAccess` boundary, which allows no `iam:PassRole`. It must have `ZAEPowerUserBoundary` (see "The Fargate capacities" in stack.md); in an account without that policy, the Fargate capacities can't launch | -| A Fargate task logs `ci-runner: no job within ...s, stopping` but keeps running | the image's idle exit predates its session-wide stop: run the image workflow on `main`. Stop the task by hand meanwhile (`$AWS ecs stop-task --cluster ci-runner --task `) | -| A Fargate task stops at once, its `stoppedReason` naming `ci-runner-idle-exit` (no such file) | the ECR image's `:latest` predates the idle-exit wrapper: run the image workflow on `main`, then re-run the job | -| A Fargate task sits `PROVISIONING`, then stops `TaskFailedToStart` with `CannotPullContainerError` | the execution role can't pull from the stack's ECR repo, or `:latest` isn't there (see "Push the image to ECR") | -| The launcher logs `no instance launched`, with an authorization error from `CreateFleet` | `CreateFleet`'s check on the instance, volumes and network interfaces it creates carries no `ec2:LaunchTemplate` key, so a policy that adds that condition to them denies the launch. Grant those to `CreateFleet` without the condition; `RunInstances` keeps it | -| The wizard stops with `runner group not created` | Enter at its **y/N** prompt means no. Re-run it, and type `y` at that prompt | -| Offline runners pile up in the `ci-runner` group | a runner stopped by the 15-minute idle exit stays registered, offline. The sweeper removes it on its next run, once it's 10 minutes past the stamp in its name (`make sweep` to run it now). If they stay, the sweeper's log says why | -| `make deploy` or `make validate` fails `NoSuchBucket`, or `AccessDenied` on the upload | the deploy bucket `ci-runner-deploy-` doesn't exist in this account yet; make it (see "Deploy and update the stack") | -| A deploy without `--s3-bucket` fails `Templates with a size greater than 51,200 bytes must be deployed via an S3 Bucket` | the template is over the inline limit; deploy with `make deploy`, or pass `--s3-bucket` as in "Deploy and update the stack" | -| `make build` fails at or over 1,048,576 bytes | the largest template CloudFormation accepts from S3. Trim the inlined code | -| The setup URL answers 410 `the GitHub App is already set up` | that's by design once the secret holds a key; see "Re-register the App" | -| The setup page fails on GitHub: the App's name is taken | App names are global on GitHub; deploy with another `AppName` | -| The webhook answers 503 | the secret holds no App yet (or was emptied): run the wizard | -| After re-registering, every delivery gets 401, or the launcher still can't get a token | a warm webhook or launcher still holds the old App's secret or key; restart both (step 5 of "Re-register the App") | -| After an update, the App's key is gone from the secret | the template's `GenerateSecretString` changed, so CloudFormation generated a new value. Never edit it; re-register the App | -| A deploy job moved to our runners can't assume its AWS role | the role's trust policy pins `runner_environment: github-hosted`; drop that condition | -| A deploy fails creating `GitHubOidcProvider`: the provider already exists | the account already has GitHub's OIDC provider, and an account holds one per URL. Deploy with `ExistingOidcProviderArn=` (see "Push the image to ECR") | -| `image.yml`'s ECR push fails `Not authorized to perform sts:AssumeRoleWithWebIdentity`, after it worked before | in `zeroae-code`, GitHub's OIDC provider is the resource `GitHubOIDCProvider` of the stack **`zael-github-e2e`**, not ours. If that stack was deleted, the provider went with it. Create the provider again once with `zeroae-code/ZAEFederation` (outside any stack; `ZAEPowerUserAccess` can't), then deploy with its ARN as `ExistingOidcProviderArn`. Asking that stack to retain the provider (`DeletionPolicy: Retain`) avoids this | -| `image.yml`'s ECR push fails `Not authorized to perform sts:AssumeRoleWithWebIdentity` on a branch, tag or pull request | by design: the push role trusts only `repo:zeroae@11018406/ci-runner@1455021495:ref:refs/heads/main` (the immutable subject). Branch runs push only to the job's local registry | -| `image.yml`'s ECR push fails `Not authorized to perform sts:AssumeRoleWithWebIdentity` on `main`, the first time | the trust's `sub` doesn't match the token's. This repo uses GitHub's immutable subject (`repo:@/@:...`); compare the role's trust with `gh api repos/zeroae/ci-runner/actions/oidc/customization/sub` | -| `image.yml` on `main` publishes to GHCR but skips both ECR steps | the repo variables `AWS_IMAGE_PUSH_ROLE_ARN` and `ECR_IMAGE` aren't set; set them from the stack's outputs (see "Push the image to ECR") | -| A deploy fails updating `ImagePushRole`'s trust with an access-denied `iam:UpdateAssumeRolePolicy` | it was deployed under `AWSPowerUserAccess`, which lacks that action; deploy under `zeroae-code/ZAEPowerUserAccess`, the default `PROFILE` | +A symptom to look up (a job that waits, a failed deploy or push, a task that +won't stop): see [traps.md](traps.md). diff --git a/docs/stack.md b/docs/stack.md index 9e4690a..20f9ebd 100644 --- a/docs/stack.md +++ b/docs/stack.md @@ -1,8 +1,8 @@ # The stack: layout and conventions One CloudFormation stack, `ci-runner`, in `zeroae-code` / `us-east-1`. -`make help` lists the targets. The owner's procedures and the traps table are in -[runbook.md](runbook.md). +`make help` lists the targets. The owner's procedures are in +[runbook.md](runbook.md), and the traps in [traps.md](traps.md). ``` template.yaml source template (valid YAML, with inline markers) @@ -547,3 +547,12 @@ On `main`, with both variables set, it then assumes the push role and pushes the same index to ECR under `:-` and `:latest`. Without the variables (before the stack has the repo) those two steps are skipped and the GHCR publish is unaffected. + +### Image changes land on `main` first + +`image.yml` publishes, to GHCR and to ECR, only from `main`, and the push +role trusts only `main`, so a change under `image/` reaches no runner until +it is on `main`. Make an `image/` change its own commit, and land it on +`main` before anything that needs it live. An integration branch that +carries such commits (cherry-picked to `main`, then merged back) lands with +a merge commit: a rebase would re-resolve every one of those conflicts. diff --git a/docs/traps.md b/docs/traps.md new file mode 100644 index 0000000..339d48e --- /dev/null +++ b/docs/traps.md @@ -0,0 +1,44 @@ +# Traps + +Symptoms seen on the stack, and their causes. A quoted section name is in +[runbook.md](runbook.md) unless the row names another file. + +| Symptom | Cause | +|---|---| +| A job's `docker run -v ...` sees an empty directory, fails `error mounting "...": not a directory`, or hits `read-only file system` inside a mounted config directory, only on our runners | the source isn't under a path shared with the host, so Docker mounts an empty host path (for a missing file, a directory it creates). Only `/tmp` and `/home/runner` are shared, at the same paths; mount from one of those. If `/tmp` or `$HOME` itself fails, the start script's mounts (`functions/cloud-config.js`) changed | +| `permission denied` on `/var/run/docker.sock` | the host's `docker` group isn't gid 123 | +| `setup-python` fails, or picks up a broken Python | `AGENT_TOOLSDIRECTORY` isn't `/opt/hostedtoolcache` | +| A deploy waits 60 minutes, then fails on the state lock | an `apply` ran on Spot and was reclaimed; it must be `ci-ondemand` | +| A job sits "Waiting for a runner" | its capacity's vCPU budget is full, the repo isn't in the group, the App isn't installed on it, or the repo is public. Also check the label contract: `self-hosted`, exactly one of `ci-spot`/`ci-ondemand`/`ci-fargate-spot`/`ci-fargate`, exactly one size label (`ci-cpu-2`/`-4`/`-8`/`-16`), exactly one of `x64`/`arm64`, and otherwise only `linux`. The launcher's log tells the first case (`deferred`) apart from the rest (no line at all). A job that queued before the repo was turned on stays waiting after the fix: cancel and re-run it | +| A job that ran on our runners before now sits "Waiting for a runner", and the webhook's log shows no `enqueued` line for it | missing size or architecture label: its `runs-on` names only its capacity (`self-hosted`, `ci-spot`). There are no defaults; add a size and an architecture label (`ci-cpu-4` and `x64` keep the old shape), then cancel and re-run the job | +| An `arm64` runner's instance comes up and terminates without taking its job, and its console shows `no matching manifest for linux/arm64` | `ghcr.io/zeroae/ci-runner:latest` has no arm64 entry yet: the image workflow hasn't pushed a multi-arch `:latest` since it began building arm64. Run the image workflow on `main`, then re-run the job | +| A job's runner was launched for another job (`$RUNNER_NAME` ends in another job id), and that other job sits "Waiting for a runner" | a swap. The webhook repairs it on the job's `in_progress` (its log says `swap`), and the launcher then launches a runner for the other job. No `swap` line: check the webhook's log for errors (its role needs `ec2:DescribeInstances` and `ec2:CreateTags`, and for a task `ecs:ListTasks`, `ecs:DescribeTasks` and `ecs:TagResource`); cancel and re-run the waiting job | +| A Fargate Spot job was lost (its task's `stopCode` is `SpotInterruption`) and its commit has no `ci-runner / ` check run | the notifier logged `ignored: is no job task of this stack` for every change of the stop: it found no job tags on the task. ECS keeps a stopped task's tags readable through `DescribeTasks` (measured live), so check the task's tags (`describe-tasks --include TAGS`, within an hour of the stop; `list-tags-for-resource` refuses a stopped task) and the notifier's role. Re-run the job | +| A job fails "The runner has received a shutdown signal", "lost communication" or "exit code 130", next to a failed `ci-runner / ` check | Spot reclaim: re-run the job | +| A budget lowered for a test is still low after the next `make deploy` | `cloudformation deploy` keeps a parameter's previous value unless it's overridden; restore it with `--parameter-overrides SpotVcpuCap=24 OnDemandVcpuCap=8 FargateSpotVcpuCap=48 FargateVcpuCap=8`. Each draws on its own vCPU quota (see "Deploy and update the stack") | +| A job on `ci-fargate-spot` or `ci-fargate` fails at a Docker step: `failed to connect to the docker API at unix:///var/run/docker.sock; check if the path is correct and if the daemon is running`, or a `services:`/`container:` job or Docker action fails to start | Docker used on Fargate. A Fargate capacity has no Docker daemon, by design (ADR-0001): move the job to `ci-spot` or `ci-ondemand`, or drop its Docker use | +| The launcher logs `no task launched:` with a reason about a vCPU limit, and the job's message keeps coming back until the dead-letter queue | a Fargate quota at its default (6 vCPUs, `L-36FBB829` Spot, `L-3032A538` On-Demand) or below the capacity's budget. Request an increase (Service Quotas, service `fargate`) or lower `FargateSpotVcpuCap`/`FargateVcpuCap` under the quota; then re-run the job. The quota counts every Fargate task in the account, not only ours | +| A Fargate job's build is slow, or runs out of memory, with many more compiler processes than the size's vCPUs | `nproc` over-reports inside a task. The job's environment sets `MAKEFLAGS=-j` and `CI_RUNNER_VCPUS=`, and the entrypoint caps cargo with `/home/runner/.cargo/config.toml` (its log says `cargo jobs = `, or why not). A tool that reads `nproc` any other way needs `-j` from `CI_RUNNER_VCPUS` | +| A Fargate job never hits the Rust cache an EC2 job saved (`Swatinem/rust-cache` restores nothing, and the two save under different keys) | an environment variable starting `CARGO`, `CC`, `CFLAGS`, `CXX`, `CMAKE` or `RUST` differs between them: rust-cache hashes them all into its key. The stack sets none on a task (cargo's jobs go in `~/.cargo/config.toml`, not `CARGO_BUILD_JOBS`); if the task has `CARGO_BUILD_JOBS`, the launcher predates that, so deploy. Otherwise look for one the job sets only on Fargate | +| The launcher logs `not authorized to perform: iam:PassRole on resource: ...runner-exec because no permissions boundary allows the iam:PassRole action` for every Fargate job | the launcher's role has the `PowerUserAccess` boundary, which allows no `iam:PassRole`. It must have `ZAEPowerUserBoundary` (see "The Fargate capacities" in stack.md); in an account without that policy, the Fargate capacities can't launch | +| A Fargate task logs `ci-runner: no job within ...s, stopping` but keeps running | the image's idle exit predates its session-wide stop: run the image workflow on `main`. Stop the task by hand meanwhile (`$AWS ecs stop-task --cluster ci-runner --task `) | +| A Fargate task stops at once, its `stoppedReason` naming `ci-runner-idle-exit` (no such file) | the ECR image's `:latest` predates the idle-exit wrapper: run the image workflow on `main`, then re-run the job | +| A Fargate task sits `PROVISIONING`, then stops `TaskFailedToStart` with `CannotPullContainerError` | the execution role can't pull from the stack's ECR repo, or `:latest` isn't there (see "Push the image to ECR") | +| The launcher logs `no instance launched`, with an authorization error from `CreateFleet` | `CreateFleet`'s check on the instance, volumes and network interfaces it creates carries no `ec2:LaunchTemplate` key, so a policy that adds that condition to them denies the launch. Grant those to `CreateFleet` without the condition; `RunInstances` keeps it | +| The wizard stops with `runner group not created` | Enter at its **y/N** prompt means no. Re-run it, and type `y` at that prompt | +| Offline runners pile up in the `ci-runner` group | a runner stopped by the 15-minute idle exit stays registered, offline. The sweeper removes it on its next run, once it's 10 minutes past the stamp in its name (`make sweep` to run it now). If they stay, the sweeper's log says why | +| `make deploy` or `make validate` fails `NoSuchBucket`, or `AccessDenied` on the upload | the deploy bucket `ci-runner-deploy-` doesn't exist in this account yet; make it (see "Deploy and update the stack") | +| A deploy without `--s3-bucket` fails `Templates with a size greater than 51,200 bytes must be deployed via an S3 Bucket` | the template is over the inline limit; deploy with `make deploy`, or pass `--s3-bucket` as in "Deploy and update the stack" | +| `make build` fails at or over 1,048,576 bytes | the largest template CloudFormation accepts from S3. Trim the inlined code | +| The setup URL answers 410 `the GitHub App is already set up` | that's by design once the secret holds a key; see "Re-register the App" | +| The setup page fails on GitHub: the App's name is taken | App names are global on GitHub; deploy with another `AppName` | +| The webhook answers 503 | the secret holds no App yet (or was emptied): run the wizard | +| After re-registering, every delivery gets 401, or the launcher still can't get a token | a warm webhook or launcher still holds the old App's secret or key; restart both (step 5 of "Re-register the App") | +| After an update, the App's key is gone from the secret | the template's `GenerateSecretString` changed, so CloudFormation generated a new value. Never edit it; re-register the App | +| A deploy job moved to our runners can't assume its AWS role | the role's trust policy pins `runner_environment: github-hosted`; drop that condition | +| A deploy fails creating `GitHubOidcProvider`: the provider already exists | the account already has GitHub's OIDC provider, and an account holds one per URL. Deploy with `ExistingOidcProviderArn=` (see "Push the image to ECR") | +| `image.yml`'s ECR push fails `Not authorized to perform sts:AssumeRoleWithWebIdentity`, after it worked before | in `zeroae-code`, GitHub's OIDC provider is the resource `GitHubOIDCProvider` of the stack **`zael-github-e2e`**, not ours. If that stack was deleted, the provider went with it. Create the provider again once with `zeroae-code/ZAEFederation` (outside any stack; `ZAEPowerUserAccess` can't), then deploy with its ARN as `ExistingOidcProviderArn`. Asking that stack to retain the provider (`DeletionPolicy: Retain`) avoids this | +| `image.yml`'s ECR push fails `Not authorized to perform sts:AssumeRoleWithWebIdentity` on a branch, tag or pull request | by design: the push role trusts only `repo:zeroae@11018406/ci-runner@1455021495:ref:refs/heads/main` (the immutable subject). Branch runs push only to the job's local registry | +| `image.yml`'s ECR push fails `Not authorized to perform sts:AssumeRoleWithWebIdentity` on `main`, the first time | the trust's `sub` doesn't match the token's. This repo uses GitHub's immutable subject (`repo:@/@:...`); compare the role's trust with `gh api repos/zeroae/ci-runner/actions/oidc/customization/sub` | +| `image.yml` on `main` publishes to GHCR but skips both ECR steps | the repo variables `AWS_IMAGE_PUSH_ROLE_ARN` and `ECR_IMAGE` aren't set; set them from the stack's outputs (see "Push the image to ECR") | +| A deploy fails updating `ImagePushRole`'s trust with an access-denied `iam:UpdateAssumeRolePolicy` | it was deployed under `AWSPowerUserAccess`, which lacks that action; deploy under `zeroae-code/ZAEPowerUserAccess`, the default `PROFILE` | diff --git a/scripts/consumers.sh b/scripts/consumers.sh new file mode 100755 index 0000000..a4bda0e --- /dev/null +++ b/scripts/consumers.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash +# Lists every job that runs on our runners: who a label-contract change +# affects. GitHub's code search misses private repos, and every consumer is +# private, so this reads each repo in the runner group directly: every +# workflow under .github/workflows on its default branch, through the +# contents API (needs gh with admin:org, for the group's repos). +# +# scripts/consumers.sh +# make consumers +# +# Prints one tab-separated line per job: repo, workflow, job, runs-on (a list +# joined with commas), for each runs-on naming ci-spot, ci-ondemand, +# ci-fargate-spot or ci-fargate, plain or in the vars.ZAE_CI_RUNNER && +# fromJSON(...) form. A workflow that names a capacity outside any runs-on (a +# matrix, a reusable workflow's input) prints one line with job "-". +# Overrides: ORG GROUP +set -euo pipefail + +ORG=${ORG:-zeroae} +GROUP=${GROUP:-ci-runner} +CAPACITY='ci-(spot|ondemand|fargate-spot|fargate)([^a-z0-9-]|$)' + +group_id=$(gh api "orgs/$ORG/actions/runner-groups" --paginate \ + --jq ".runner_groups[] | select(.name == \"$GROUP\") | .id") +[[ -n "$group_id" ]] || { echo "no runner group named $GROUP in $ORG" >&2; exit 1; } + +# The jobs of one workflow file (on stdin) whose runs-on names a capacity. +jobs_on_capacity() { + awk -v repo="$1" -v wf="$2" -v re="$CAPACITY" ' + function indent(s) { match(s, /^ */); return RLENGTH } + function flush() { + if (job != "" && runs != "" && runs ~ re) { print repo "\t" wf "\t" job "\t" runs; hit = 1 } + runs = ""; collecting = 0 + } + { sub(/\r$/, ""); line = $0; sub(/[ \t]+#.*$/, "", line) } + line ~ /^[ \t]*$/ { next } + line ~ re { named = 1 } + { i = indent(line); text = substr(line, i + 1) } + collecting && i > runs_indent { + sub(/^- */, "", text); runs = runs (runs == "" ? "" : ",") text; next + } + collecting { flush() } + i == 0 { flush(); in_jobs = (text ~ /^jobs:/); job = ""; job_indent = -1; next } + in_jobs && (job_indent < 0 || i == job_indent) && text ~ /^[^ ]+:/ { + flush(); job_indent = i; job = text; sub(/:.*/, "", job); next + } + in_jobs && job != "" && text ~ /^runs-on:/ { + flush(); runs = text; sub(/^runs-on:[ \t]*/, "", runs) + if (runs == "") { collecting = 1; runs_indent = i } else flush() + } + END { + flush() + if (!hit && named) print repo "\t" wf "\t-\t(a capacity named outside runs-on)" + } + ' +} + +gh api "orgs/$ORG/actions/runner-groups/$group_id/repositories" --paginate \ + --jq '.repositories[].name' | sort | while read -r repo; do + files=$(gh api "repos/$ORG/$repo/contents/.github/workflows" \ + --jq '.[] | select(.type == "file") | select(.name | test("\\.ya?ml$")) | .path' 2>/dev/null) || continue + for path in $files; do + gh api -H 'Accept: application/vnd.github.raw' "repos/$ORG/$repo/contents/$path" \ + | jobs_on_capacity "$repo" "${path##*/}" + done +done