Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions functions/launcher.js
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,9 @@ const NO_ARM64_AZ = 'use1-az3';
const ECS_TAG_CHARS = /[^\p{L}\p{Z}\p{N}_.:/=+\-@]/gu;
const ecsTag = ([key, v]) => ({ key, value: String(v).replace(ECS_TAG_CHARS, '_').slice(0, 256) });

// An EC2 (Key, Value) or ECS (key, value) tag list as an object.
const tagsOf = (tags) => Object.fromEntries((tags || []).map((x) => [x.Key ?? x.key, x.Value ?? x.value]));
// An EC2 (Key, Value) or ECS (key, value) tag list as an object, keyed
// without the ci-runner: prefix. The webhook and the notifier carry copies.
const tagsOf = (tags) => Object.fromEntries((tags || []).map((x) => [(x.Key ?? x.key).replace(/^ci-runner:/, ''), x.Value ?? x.value]));

const b64u = (s) => Buffer.from(s).toString('base64url');

Expand Down Expand Up @@ -104,8 +105,8 @@ function createHandler({ secret, fetch, ec2, ecs, sqs, now, env, log = console.l
for (const res of r.Reservations || []) {
for (const i of res.Instances || []) {
const t = tagsOf(i.Tags);
const capacity = CAPACITY[t['ci-runner:capacity']] || (i.InstanceLifecycle === 'spot' ? 'spot' : 'on-demand');
if (capacity === type) n += Number(t['ci-runner:vcpus']) || DEFAULT_SHAPE.vcpus;
const capacity = CAPACITY[t.capacity] || (i.InstanceLifecycle === 'spot' ? 'spot' : 'on-demand');
if (capacity === type) n += Number(t.vcpus) || DEFAULT_SHAPE.vcpus;
}
}
NextToken = r.NextToken;
Expand Down Expand Up @@ -174,7 +175,7 @@ function createHandler({ secret, fetch, ec2, ecs, sqs, now, env, log = console.l
const d = await ecs.describeTasks({ cluster: env.CLUSTER, tasks: r.taskArns, include: ['TAGS'] });
for (const task of d.tasks || []) {
const t = tagsOf(task.tags);
if (task.lastStatus !== 'STOPPED' && CAPACITY[t['ci-runner:capacity']] === type) n += Number(t['ci-runner:vcpus']) || DEFAULT_SHAPE.vcpus;
if (task.lastStatus !== 'STOPPED' && CAPACITY[t.capacity] === type) n += Number(t.vcpus) || DEFAULT_SHAPE.vcpus;
}
}
nextToken = r.nextToken;
Expand Down
12 changes: 7 additions & 5 deletions functions/webhook.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,8 @@ const crypto = require('node:crypto');
const CAPACITY = ['ci-spot', 'ci-ondemand', 'ci-fargate-spot', 'ci-fargate'];
const SIZE = { 'ci-cpu-2': 2, 'ci-cpu-4': 4, 'ci-cpu-8': 8, 'ci-cpu-16': 16 };
const ARCH = ['x64', 'arm64'];
// The shape of an instance launched before the shape tags.
// The shape of an instance or task launched before the shape tags, which
// swap repair re-queues with it. The launcher carries a copy.
const DEFAULT_SHAPE = { vcpus: 4, arch: 'x64' };

function reply(statusCode, text) {
Expand Down Expand Up @@ -139,14 +140,15 @@ function createHandler({ secret, queue, ec2, ecs, env, log = console.log }) {
const shape = shapeOf(job.labels);
if (!shape) return reply(200, 'ignored: labels');

// The message's `label` is the capacity label, as queued messages carry it.
const wire = { label: shape.capacity, vcpus: shape.vcpus, arch: shape.arch };
// The shape as the queue message carries it. The message's `label` is the
// capacity label, as queued messages carry it.
const queuedShape = { label: shape.capacity, vcpus: shape.vcpus, arch: shape.arch };
const msg = {
repo: repo.full_name, job_id: job.id, run_id: job.run_id, head_sha: job.head_sha,
job_name: job.name, ...wire, labels: job.labels, installation_id: p.installation && p.installation.id,
job_name: job.name, ...queuedShape, labels: job.labels, installation_id: p.installation && p.installation.id,
};
await queue.send(msg);
log(JSON.stringify({ enqueued: msg.repo, job_id: msg.job_id, ...wire }));
log(JSON.stringify({ enqueued: msg.repo, job_id: msg.job_id, ...queuedShape }));
return reply(202, 'enqueued');
};
}
Expand Down
17 changes: 17 additions & 0 deletions test/conventions.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -80,3 +80,20 @@ test('the launcher and the webhook sanitize an ECS tag the same way', () => {
assert.equal(webhook, launcher, `the webhook's ${re} differs from the launcher's`);
}
});

test('the launcher, webhook and notifier read tags the same way', () => {
// Each strips the ci-runner: prefix, so t.capacity means the same everywhere.
const re = /^const tagsOf = .*$/m;
const [launcher, ...rest] = ['launcher', 'webhook', 'notifier'].map((f) => ({ f, code: (read(`functions/${f}.js`).match(re) || [])[0] }));
assert.ok(launcher.code, 'the launcher has no tagsOf');
for (const o of rest) assert.equal(o.code, launcher.code, `${o.f}'s tagsOf differs from the launcher's`);
});

test('the launcher and the webhook carry the same DEFAULT_SHAPE', () => {
// Swap repair re-queues a runner launched before the shape tags with the
// shape the launcher gave it, so the two defaults must agree.
const re = /^const DEFAULT_SHAPE = .*$/m;
const [launcher, webhook] = ['launcher', 'webhook'].map((f) => (read(`functions/${f}.js`).match(re) || [])[0]);
assert.ok(launcher, 'the launcher has no DEFAULT_SHAPE');
assert.equal(webhook, launcher, "the webhook's DEFAULT_SHAPE differs from the launcher's");
});
28 changes: 14 additions & 14 deletions test/launcher.test.js
Original file line number Diff line number Diff line change
Expand Up @@ -77,12 +77,12 @@ function fakeEc2({ fleet = { Instances: [{ InstanceIds: ['i-0123'] }], Errors: [
};
}

// `running` lists the cluster's tasks, as { capacity, vcpus, desired, last } with
// `tasks` lists the cluster's tasks, as { capacity, vcpus, desired, last } with
// the launcher's shape tags (left off when undefined). `run` is RunTask's
// answer; by default one task.
function fakeEcs({ running = [], run } = {}) {
function fakeEcs({ tasks = [], run } = {}) {
const calls = [];
const all = running.map((t, i) => ({
const all = tasks.map((t, i) => ({
taskArn: `arn:aws:ecs:us-east-1:1:task/ci-runner/t${i}`, desiredStatus: t.desired || 'RUNNING', lastStatus: t.last || 'RUNNING',
tags: [['ci-runner:stack', 'ci-runner'], ['ci-runner:capacity', t.capacity], ['ci-runner:vcpus', t.vcpus]]
.filter(([, v]) => v !== undefined).map(([key, value]) => ({ key, value: String(value) })),
Expand All @@ -103,9 +103,9 @@ function fakeEcs({ running = [], run } = {}) {
},
describeTasks: async (input) => {
calls.push(['describeTasks', input]);
const tasks = all.filter((t) => input.tasks.includes(t.taskArn))
const described = all.filter((t) => input.tasks.includes(t.taskArn))
.map((t) => (input.include || []).includes('TAGS') ? t : { ...t, tags: undefined });
return { tasks, failures: [] };
return { tasks: described, failures: [] };
},
};
}
Expand Down Expand Up @@ -491,7 +491,7 @@ const FGOD_MSG = { ...FG_MSG, label: 'ci-fargate', labels: ['self-hosted', 'ci-f
const ran = (ecs) => ecs.calls.some((c) => c[0] === 'runTask');

test('the Fargate budget lists the cluster\'s tasks, running and told to stop, and reads their tags', async () => {
const ecs = fakeEcs({ running: [fgSpot(4), fgSpot(2), { ...fgSpot(2), desired: 'STOPPED' }] });
const ecs = fakeEcs({ tasks: [fgSpot(4), fgSpot(2), { ...fgSpot(2), desired: 'STOPPED' }] });
const { handler } = setup({ ecs });
await handler(sqsEvent(FG_MSG));
const lists = ecs.calls.filter((c) => c[0] === 'listTasks').map((c) => c[1]);
Expand All @@ -509,21 +509,21 @@ test('the Fargate budget lists the cluster\'s tasks, running and told to stop, a
test('a task told to stop counts towards the Fargate budget until it is STOPPED', async () => {
// 16 running + 8 told to stop but still deprovisioning = 24 of 24; the 8
// already STOPPED no longer counts.
const running = [fgSpot(16), { ...fgSpot(8), desired: 'STOPPED', last: 'DEPROVISIONING' },
const tasks = [fgSpot(16), { ...fgSpot(8), desired: 'STOPPED', last: 'DEPROVISIONING' },
{ ...fgSpot(8), desired: 'STOPPED', last: 'STOPPED' }];
const s = setup({ ecs: fakeEcs({ running }) });
const s = setup({ ecs: fakeEcs({ tasks }) });
await s.handler(sqsEvent({ ...FG_MSG, vcpus: 2 }));
assert.ok(!ran(s.ecs));
assert.equal(JSON.parse(s.logs.find((l) => l.includes('"deferred"'))).deferred, '24+2 of 24 fargate-spot vCPUs');
});

test('Fargate budgets sum mixed sizes across tasks; a job over the budget waits, logged in vCPUs', async () => {
// 2 + 4 + 8 + 8 = 22 Fargate Spot vCPUs: a 2 fits exactly, a 4 doesn't.
const running = [fgSpot(2), fgSpot(4), fgSpot(8), fgSpot(8), fg(8)];
let s = setup({ ecs: fakeEcs({ running }) });
const tasks = [fgSpot(2), fgSpot(4), fgSpot(8), fgSpot(8), fg(8)];
let s = setup({ ecs: fakeEcs({ tasks }) });
await s.handler(sqsEvent({ ...FG_MSG, vcpus: 2 }));
assert.ok(ran(s.ecs));
s = setup({ ecs: fakeEcs({ running }) });
s = setup({ ecs: fakeEcs({ tasks }) });
assert.deepEqual(await s.handler(sqsEvent(FG_MSG)), { batchItemFailures: [] });
assert.ok(!ran(s.ecs));
assert.equal(s.sqs.length, 1);
Expand All @@ -534,7 +534,7 @@ test('Fargate budgets sum mixed sizes across tasks; a job over the budget waits,

test('each Fargate capacity has its own budget, and EC2 instances count towards neither', async () => {
// Fargate On-Demand holds 8 of 8; Fargate Spot and EC2 Spot are free.
const ecs = () => fakeEcs({ running: [fg(4), fg(4), fgSpot(16)] });
const ecs = () => fakeEcs({ tasks: [fg(4), fg(4), fgSpot(16)] });
let s = setup({ ecs: ecs() });
await s.handler(sqsEvent({ ...FGOD_MSG, vcpus: 2 }));
assert.ok(!ran(s.ecs));
Expand All @@ -547,14 +547,14 @@ test('each Fargate capacity has its own budget, and EC2 instances count towards
await s.handler(sqsEvent(FG_MSG));
assert.ok(ran(s.ecs));
assert.deepEqual(s.ec2.calls, [], 'a Fargate job describes no instances');
s = setup({ ecs: fakeEcs({ running: [fgSpot(16), fgSpot(8)] }) });
s = setup({ ecs: fakeEcs({ tasks: [fgSpot(16), fgSpot(8)] }) });
await s.handler(sqsEvent(MSG));
assert.ok(launched(s.ec2));
assert.deepEqual(s.ecs.calls, [], 'an EC2 job lists no tasks');
});

test('the Fargate budgets come from the environment', async () => {
let s = setup({ ecs: fakeEcs({ running: [fgSpot(2)] }), env: { ...ENV, FARGATE_SPOT_VCPU_CAP: '4' } });
let s = setup({ ecs: fakeEcs({ tasks: [fgSpot(2)] }), env: { ...ENV, FARGATE_SPOT_VCPU_CAP: '4' } });
await s.handler(sqsEvent(FG_MSG));
assert.ok(!ran(s.ecs));
s = setup({ env: { ...ENV, FARGATE_VCPU_CAP: '16' } });
Expand Down
Loading