diff --git a/functions/launcher.js b/functions/launcher.js index ddecfaf..47f3a2b 100644 --- a/functions/launcher.js +++ b/functions/launcher.js @@ -22,8 +22,9 @@ const NO_ARM64_AZ = 'use1-az3'; const ECS_TAG_CHARS = /[^\p{L}\p{Z}\p{N}_.:/=+\-@]/gu; const ecsTag = ([key, v]) => ({ key, value: String(v).replace(ECS_TAG_CHARS, '_').slice(0, 256) }); -// An EC2 (Key, Value) or ECS (key, value) tag list as an object. -const tagsOf = (tags) => Object.fromEntries((tags || []).map((x) => [x.Key ?? x.key, x.Value ?? x.value])); +// An EC2 (Key, Value) or ECS (key, value) tag list as an object, keyed +// without the ci-runner: prefix. The webhook and the notifier carry copies. +const tagsOf = (tags) => Object.fromEntries((tags || []).map((x) => [(x.Key ?? x.key).replace(/^ci-runner:/, ''), x.Value ?? x.value])); const b64u = (s) => Buffer.from(s).toString('base64url'); @@ -104,8 +105,8 @@ function createHandler({ secret, fetch, ec2, ecs, sqs, now, env, log = console.l for (const res of r.Reservations || []) { for (const i of res.Instances || []) { const t = tagsOf(i.Tags); - const capacity = CAPACITY[t['ci-runner:capacity']] || (i.InstanceLifecycle === 'spot' ? 'spot' : 'on-demand'); - if (capacity === type) n += Number(t['ci-runner:vcpus']) || DEFAULT_SHAPE.vcpus; + const capacity = CAPACITY[t.capacity] || (i.InstanceLifecycle === 'spot' ? 'spot' : 'on-demand'); + if (capacity === type) n += Number(t.vcpus) || DEFAULT_SHAPE.vcpus; } } NextToken = r.NextToken; @@ -174,7 +175,7 @@ function createHandler({ secret, fetch, ec2, ecs, sqs, now, env, log = console.l const d = await ecs.describeTasks({ cluster: env.CLUSTER, tasks: r.taskArns, include: ['TAGS'] }); for (const task of d.tasks || []) { const t = tagsOf(task.tags); - if (task.lastStatus !== 'STOPPED' && CAPACITY[t['ci-runner:capacity']] === type) n += Number(t['ci-runner:vcpus']) || DEFAULT_SHAPE.vcpus; + if (task.lastStatus !== 'STOPPED' && CAPACITY[t.capacity] === type) n += Number(t.vcpus) || DEFAULT_SHAPE.vcpus; } } nextToken = r.nextToken; diff --git a/functions/webhook.js b/functions/webhook.js index 745ff96..61033fd 100644 --- a/functions/webhook.js +++ b/functions/webhook.js @@ -8,7 +8,8 @@ const crypto = require('node:crypto'); const CAPACITY = ['ci-spot', 'ci-ondemand', 'ci-fargate-spot', 'ci-fargate']; const SIZE = { 'ci-cpu-2': 2, 'ci-cpu-4': 4, 'ci-cpu-8': 8, 'ci-cpu-16': 16 }; const ARCH = ['x64', 'arm64']; -// The shape of an instance launched before the shape tags. +// The shape of an instance or task launched before the shape tags, which +// swap repair re-queues with it. The launcher carries a copy. const DEFAULT_SHAPE = { vcpus: 4, arch: 'x64' }; function reply(statusCode, text) { @@ -139,14 +140,15 @@ function createHandler({ secret, queue, ec2, ecs, env, log = console.log }) { const shape = shapeOf(job.labels); if (!shape) return reply(200, 'ignored: labels'); - // The message's `label` is the capacity label, as queued messages carry it. - const wire = { label: shape.capacity, vcpus: shape.vcpus, arch: shape.arch }; + // The shape as the queue message carries it. The message's `label` is the + // capacity label, as queued messages carry it. + const queuedShape = { label: shape.capacity, vcpus: shape.vcpus, arch: shape.arch }; const msg = { repo: repo.full_name, job_id: job.id, run_id: job.run_id, head_sha: job.head_sha, - job_name: job.name, ...wire, labels: job.labels, installation_id: p.installation && p.installation.id, + job_name: job.name, ...queuedShape, labels: job.labels, installation_id: p.installation && p.installation.id, }; await queue.send(msg); - log(JSON.stringify({ enqueued: msg.repo, job_id: msg.job_id, ...wire })); + log(JSON.stringify({ enqueued: msg.repo, job_id: msg.job_id, ...queuedShape })); return reply(202, 'enqueued'); }; } diff --git a/test/conventions.test.js b/test/conventions.test.js index 10bf20c..02f72e1 100644 --- a/test/conventions.test.js +++ b/test/conventions.test.js @@ -80,3 +80,20 @@ test('the launcher and the webhook sanitize an ECS tag the same way', () => { assert.equal(webhook, launcher, `the webhook's ${re} differs from the launcher's`); } }); + +test('the launcher, webhook and notifier read tags the same way', () => { + // Each strips the ci-runner: prefix, so t.capacity means the same everywhere. + const re = /^const tagsOf = .*$/m; + const [launcher, ...rest] = ['launcher', 'webhook', 'notifier'].map((f) => ({ f, code: (read(`functions/${f}.js`).match(re) || [])[0] })); + assert.ok(launcher.code, 'the launcher has no tagsOf'); + for (const o of rest) assert.equal(o.code, launcher.code, `${o.f}'s tagsOf differs from the launcher's`); +}); + +test('the launcher and the webhook carry the same DEFAULT_SHAPE', () => { + // Swap repair re-queues a runner launched before the shape tags with the + // shape the launcher gave it, so the two defaults must agree. + const re = /^const DEFAULT_SHAPE = .*$/m; + const [launcher, webhook] = ['launcher', 'webhook'].map((f) => (read(`functions/${f}.js`).match(re) || [])[0]); + assert.ok(launcher, 'the launcher has no DEFAULT_SHAPE'); + assert.equal(webhook, launcher, "the webhook's DEFAULT_SHAPE differs from the launcher's"); +}); diff --git a/test/launcher.test.js b/test/launcher.test.js index 92a105b..8dfbb4c 100644 --- a/test/launcher.test.js +++ b/test/launcher.test.js @@ -77,12 +77,12 @@ function fakeEc2({ fleet = { Instances: [{ InstanceIds: ['i-0123'] }], Errors: [ }; } -// `running` lists the cluster's tasks, as { capacity, vcpus, desired, last } with +// `tasks` lists the cluster's tasks, as { capacity, vcpus, desired, last } with // the launcher's shape tags (left off when undefined). `run` is RunTask's // answer; by default one task. -function fakeEcs({ running = [], run } = {}) { +function fakeEcs({ tasks = [], run } = {}) { const calls = []; - const all = running.map((t, i) => ({ + const all = tasks.map((t, i) => ({ taskArn: `arn:aws:ecs:us-east-1:1:task/ci-runner/t${i}`, desiredStatus: t.desired || 'RUNNING', lastStatus: t.last || 'RUNNING', tags: [['ci-runner:stack', 'ci-runner'], ['ci-runner:capacity', t.capacity], ['ci-runner:vcpus', t.vcpus]] .filter(([, v]) => v !== undefined).map(([key, value]) => ({ key, value: String(value) })), @@ -103,9 +103,9 @@ function fakeEcs({ running = [], run } = {}) { }, describeTasks: async (input) => { calls.push(['describeTasks', input]); - const tasks = all.filter((t) => input.tasks.includes(t.taskArn)) + const described = all.filter((t) => input.tasks.includes(t.taskArn)) .map((t) => (input.include || []).includes('TAGS') ? t : { ...t, tags: undefined }); - return { tasks, failures: [] }; + return { tasks: described, failures: [] }; }, }; } @@ -491,7 +491,7 @@ const FGOD_MSG = { ...FG_MSG, label: 'ci-fargate', labels: ['self-hosted', 'ci-f const ran = (ecs) => ecs.calls.some((c) => c[0] === 'runTask'); test('the Fargate budget lists the cluster\'s tasks, running and told to stop, and reads their tags', async () => { - const ecs = fakeEcs({ running: [fgSpot(4), fgSpot(2), { ...fgSpot(2), desired: 'STOPPED' }] }); + const ecs = fakeEcs({ tasks: [fgSpot(4), fgSpot(2), { ...fgSpot(2), desired: 'STOPPED' }] }); const { handler } = setup({ ecs }); await handler(sqsEvent(FG_MSG)); const lists = ecs.calls.filter((c) => c[0] === 'listTasks').map((c) => c[1]); @@ -509,9 +509,9 @@ test('the Fargate budget lists the cluster\'s tasks, running and told to stop, a test('a task told to stop counts towards the Fargate budget until it is STOPPED', async () => { // 16 running + 8 told to stop but still deprovisioning = 24 of 24; the 8 // already STOPPED no longer counts. - const running = [fgSpot(16), { ...fgSpot(8), desired: 'STOPPED', last: 'DEPROVISIONING' }, + const tasks = [fgSpot(16), { ...fgSpot(8), desired: 'STOPPED', last: 'DEPROVISIONING' }, { ...fgSpot(8), desired: 'STOPPED', last: 'STOPPED' }]; - const s = setup({ ecs: fakeEcs({ running }) }); + const s = setup({ ecs: fakeEcs({ tasks }) }); await s.handler(sqsEvent({ ...FG_MSG, vcpus: 2 })); assert.ok(!ran(s.ecs)); assert.equal(JSON.parse(s.logs.find((l) => l.includes('"deferred"'))).deferred, '24+2 of 24 fargate-spot vCPUs'); @@ -519,11 +519,11 @@ test('a task told to stop counts towards the Fargate budget until it is STOPPED' test('Fargate budgets sum mixed sizes across tasks; a job over the budget waits, logged in vCPUs', async () => { // 2 + 4 + 8 + 8 = 22 Fargate Spot vCPUs: a 2 fits exactly, a 4 doesn't. - const running = [fgSpot(2), fgSpot(4), fgSpot(8), fgSpot(8), fg(8)]; - let s = setup({ ecs: fakeEcs({ running }) }); + const tasks = [fgSpot(2), fgSpot(4), fgSpot(8), fgSpot(8), fg(8)]; + let s = setup({ ecs: fakeEcs({ tasks }) }); await s.handler(sqsEvent({ ...FG_MSG, vcpus: 2 })); assert.ok(ran(s.ecs)); - s = setup({ ecs: fakeEcs({ running }) }); + s = setup({ ecs: fakeEcs({ tasks }) }); assert.deepEqual(await s.handler(sqsEvent(FG_MSG)), { batchItemFailures: [] }); assert.ok(!ran(s.ecs)); assert.equal(s.sqs.length, 1); @@ -534,7 +534,7 @@ test('Fargate budgets sum mixed sizes across tasks; a job over the budget waits, test('each Fargate capacity has its own budget, and EC2 instances count towards neither', async () => { // Fargate On-Demand holds 8 of 8; Fargate Spot and EC2 Spot are free. - const ecs = () => fakeEcs({ running: [fg(4), fg(4), fgSpot(16)] }); + const ecs = () => fakeEcs({ tasks: [fg(4), fg(4), fgSpot(16)] }); let s = setup({ ecs: ecs() }); await s.handler(sqsEvent({ ...FGOD_MSG, vcpus: 2 })); assert.ok(!ran(s.ecs)); @@ -547,14 +547,14 @@ test('each Fargate capacity has its own budget, and EC2 instances count towards await s.handler(sqsEvent(FG_MSG)); assert.ok(ran(s.ecs)); assert.deepEqual(s.ec2.calls, [], 'a Fargate job describes no instances'); - s = setup({ ecs: fakeEcs({ running: [fgSpot(16), fgSpot(8)] }) }); + s = setup({ ecs: fakeEcs({ tasks: [fgSpot(16), fgSpot(8)] }) }); await s.handler(sqsEvent(MSG)); assert.ok(launched(s.ec2)); assert.deepEqual(s.ecs.calls, [], 'an EC2 job lists no tasks'); }); test('the Fargate budgets come from the environment', async () => { - let s = setup({ ecs: fakeEcs({ running: [fgSpot(2)] }), env: { ...ENV, FARGATE_SPOT_VCPU_CAP: '4' } }); + let s = setup({ ecs: fakeEcs({ tasks: [fgSpot(2)] }), env: { ...ENV, FARGATE_SPOT_VCPU_CAP: '4' } }); await s.handler(sqsEvent(FG_MSG)); assert.ok(!ran(s.ecs)); s = setup({ env: { ...ENV, FARGATE_VCPU_CAP: '16' } });