diff --git a/AGENTS.md b/AGENTS.md index b4a8b1fc..2cb65211 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -202,6 +202,9 @@ their reasons in `oxlint.config.ts`. - **Card specs read the card through `tests/_card.ts`, fetcher specs their requests through `tests/_fetch-mock.ts`** ([#288]); extend them rather than copy them. - **A param's validation is tested once, in `apiParams.spec.ts`.** +- **`apiFuzz.spec.ts` feeds every handler generated queries** through fast-check, its fetchers + answering `_cards.ts`'s samples: a handler joins `HANDLERS`, and a free-text param with a format + of its own gets a `FORMATTED` entry, or the mixed property never reaches the renderer. - **`test:coverage` fails under a package's floor in `vitest.config.coverage.ts`**: raise a floor after adding tests, and lower one only with the owner. Every source file counts, so one no test imports reads as 0% rather than vanishing. diff --git a/packages/core/package.json b/packages/core/package.json index d7165003..80709653 100644 --- a/packages/core/package.json +++ b/packages/core/package.json @@ -93,6 +93,7 @@ "@testing-library/jest-dom": "7.0.1", "@types/node": "26.6.6", "@uppercod/css-to-object": "1.1.1", + "fast-check": "4.10.2", "graphql": "17.1.0", "js-yaml": "5.5.0", "jsdom": "catalog:default", diff --git a/packages/core/tests/_cards.ts b/packages/core/tests/_cards.ts index ffa0994c..84a98b2c 100644 --- a/packages/core/tests/_cards.ts +++ b/packages/core/tests/_cards.ts @@ -161,4 +161,14 @@ const cards = { organization: (locale?: Locale): string => renderOrganizationCard(organizationData, { locale }), }; -export { cards, repoData, statsData }; +export { + cards, + contributedToData, + gistData, + orgActivityData, + orgCalendarData, + organizationData, + repoData, + statsData, + topLangs, +}; diff --git a/packages/core/tests/apiFuzz.spec.ts b/packages/core/tests/apiFuzz.spec.ts new file mode 100644 index 00000000..19c2248b --- /dev/null +++ b/packages/core/tests/apiFuzz.spec.ts @@ -0,0 +1,230 @@ +/** + * @file Every handler, fed query strings no one wrote by hand. + * + * The fetchers answer the shared samples, so a query the boundary lets through reaches the renderer: + * a render-time throw surfaces as `upstream`, which no query string may cause. + */ + +import * as fc from 'fast-check'; +import { describe, expect, it, vi } from 'vitest'; + +import type { ApiResult } from '../src/api/api-result.ts'; +import { + contributedTo, + gist, + org, + orgActivity, + orgCalendar, + pin, + stats, + topLangs, + wakatime, +} from '../src/api/index.ts'; +import type { ParamShape } from '../src/api/params.ts'; +import type { CardConfig } from '../src/common/config.ts'; +import { ORG_CALENDAR_MAX_WEEKS } from '../src/common/constants.ts'; + +import { testConfig } from './_config.ts'; + +// hoisted above the imports, so each factory imports its own sample +vi.mock(import('../src/fetchers/contributed-to/index.ts'), async (original) => { + const actual = await original(); + const { contributedToData: sample } = await import('./_cards.ts'); + return { + ...actual, + fetchContributedTo: vi.fn().mockResolvedValue(sample), + }; +}); +vi.mock(import('../src/fetchers/gist/index.ts'), async (original) => { + const actual = await original(); + const { gistData: sample } = await import('./_cards.ts'); + return { ...actual, fetchGist: vi.fn().mockResolvedValue(sample) }; +}); +vi.mock(import('../src/fetchers/org-activity/index.ts'), async (original) => { + const actual = await original(); + const { orgActivityData: sample } = await import('./_cards.ts'); + return { + ...actual, + fetchOrgActivity: vi.fn().mockResolvedValue(sample), + }; +}); +vi.mock(import('../src/fetchers/org-calendar/index.ts'), async (original) => { + const actual = await original(); + const { orgCalendarData: sample } = await import('./_cards.ts'); + return { + ...actual, + fetchOrgCalendar: vi.fn().mockResolvedValue(sample), + }; +}); +vi.mock(import('../src/fetchers/organization/index.ts'), async (original) => { + const actual = await original(); + const { organizationData: sample } = await import('./_cards.ts'); + return { + ...actual, + fetchOrganization: vi.fn().mockResolvedValue(sample), + }; +}); +vi.mock(import('../src/fetchers/repo/index.ts'), async (original) => { + const actual = await original(); + const { repoData: sample } = await import('./_cards.ts'); + return { ...actual, fetchRepo: vi.fn().mockResolvedValue(sample) }; +}); +vi.mock(import('../src/fetchers/stats/index.ts'), async (original) => { + const actual = await original(); + const { statsData: sample } = await import('./_cards.ts'); + return { ...actual, fetchStats: vi.fn().mockResolvedValue(sample) }; +}); +vi.mock(import('../src/fetchers/top-languages/index.ts'), async (original) => { + const actual = await original(); + const { topLangs: sample } = await import('./_cards.ts'); + return { + ...actual, + fetchTopLanguages: vi.fn().mockResolvedValue(sample), + }; +}); +vi.mock(import('../src/fetchers/wakatime/index.ts'), async (original) => { + const actual = await original(); + const { wakaTimeData } = await import('./_wakatime.ts'); + const sample = wakaTimeData.data; + return { + ...actual, + fetchWakatimeStats: vi.fn().mockResolvedValue(sample), + }; +}); + +type Handler = ((query: Record, config: CardConfig) => Promise) & { + PARAMS: Readonly>; +}; + +/** Each handler, and a query that identifies a card it can draw. */ +const HANDLERS: Array<[string, Handler, Record]> = [ + ['stats', stats, { username: 'anuraghazra' }], + ['top-langs', topLangs, { username: 'anuraghazra' }], + ['pin', pin, { username: 'anuraghazra', repo: 'convoychat' }], + ['contributed-to', contributedTo, { username: 'anuraghazra' }], + ['org', org, { org: 'vitest-dev' }], + ['org-activity', orgActivity, { org: 'vitest-dev' }], + ['org-calendar', orgCalendar, { org: 'vitest-dev' }], + ['gist', gist, { id: 'bbfce31e0217a3689c8d961a356cb10d' }], + ['wakatime', wakatime, { username: 'anuraghazra' }], +]; + +/** Strings a hand-written query gets wrong: empty, signed, scientific, padded, cased, unicode. */ +const NEAR_MISSES = [ + '', + ' ', + '-1', + '0', + '1e3', + '5px', + '1.5', + 'NaN', + 'TRUE', + 'yes', + ',', + '%00', + 'é', +]; + +/** Free-text params with a format of their own, which their kind cannot tell. */ +const FORMATTED: Record> = { + from: fc.integer({ min: 2015, max: 2025 }).map(String), + to: fc.integer({ min: 2015, max: 2025 }).map(String), + scope: fc.constantFrom('anuraghazra/convoychat', 'anuraghazra/*'), + wakatime_host: fc.constantFrom('wakatime.com', 'waka.example.com'), + weeks: fc.integer({ min: 1, max: ORG_CALENDAR_MAX_WEEKS }).map(String), +}; + +/** @returns What the param accepts: the values a form would offer. */ +const wellFormed = (name: string, { kind, choices = [] }: ParamShape): fc.Arbitrary => { + const formatted = FORMATTED[name]; + if (formatted !== undefined) { + return formatted; + } + if (choices.length > 0 && kind === 'list') { + return fc.subarray([...choices], { minLength: 1 }).map((list) => list.join(',')); + } + if (choices.length > 0) { + return fc.constantFrom(...choices); + } + if (kind === 'boolean') { + return fc.constantFrom('true', 'false', '1', '0', 'on', 'off'); + } + if (kind === 'integer' || kind === 'number') { + return fc.integer({ min: 1, max: 2000 }).map(String); + } + if (kind === 'list') { + return fc + .array(fc.stringMatching(/^[a-z][\w-]{0,9}$/), { minLength: 1, maxLength: 3 }) + .map((list) => list.join(',')); + } + return fc.string({ maxLength: 40 }); +}; + +/** @returns Anything at all for one param, weighted towards what a hand-written query gets wrong. */ +const anyValue = (name: string, shape: ParamShape): fc.Arbitrary => + fc.oneof( + wellFormed(name, shape), + fc.constantFrom(...NEAR_MISSES), + fc.integer({ min: -1000, max: 100_000 }).map(String), + fc.string(), + ); + +/** Only what the query string can be blamed for: a parse failure, never a crash. */ +const QUERY_FAULTS = new Set(['invalid_param', 'missing_param']); + +/** Fails unless the result is a card, or a refusal of the query naming a param it was sent. */ +const expectDrawnOrRefused = (result: ApiResult, sent: ReadonlyArray): void => { + expect(result.content.startsWith(' { + const params = Object.entries(handler.PARAMS); + + it('draws any mix of well-formed options, or refuses one by name', async () => { + const queries = fc + .subarray(params) + .chain((chosen) => + fc.tuple( + ...chosen.map(([name, shape]) => + wellFormed(name, shape).map((value): [string, string] => [name, value]), + ), + ), + ) + .map((entries) => Object.fromEntries(entries)); + // fast-check rejects with the shrunk counterexample + await expect( + fc.assert( + fc.asyncProperty(queries, async (fuzzed) => { + const query = { ...fuzzed, ...identity }; + const result = await handler(query, testConfig); + + expectDrawnOrRefused(result, Object.keys(query)); + }), + { numRuns: 200 }, + ), + ).resolves.toBeUndefined(); + }); + + it('draws, or names the param, whatever one param holds', async () => { + const queries = fc + .constantFrom(...params) + .chain(([name, shape]) => anyValue(name, shape).map((value) => ({ [name]: value }))); + await expect( + fc.assert( + fc.asyncProperty(queries, async (fuzzed) => { + const result = await handler({ ...identity, ...fuzzed }, testConfig); + + expectDrawnOrRefused(result, Object.keys(fuzzed)); + }), + { numRuns: 300 }, + ), + ).resolves.toBeUndefined(); + }); +}); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 437c3fed..87899e36 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -365,6 +365,9 @@ importers: '@uppercod/css-to-object': specifier: 1.1.1 version: 1.1.1 + fast-check: + specifier: 4.10.2 + version: 4.10.2 graphql: specifier: 17.1.0 version: 17.1.0 @@ -2849,6 +2852,10 @@ packages: extend@3.0.2: resolution: {integrity: sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==} + fast-check@4.10.2: + resolution: {integrity: sha512-iK2f+YrcmoeGqk6fA0ea2bptcu/itMIm4NfEozq6N25+aG6h7s5HZbB/k1aV7b5w5sFLMCbbtRUsTVR+BgC3xw==} + engines: {node: '>=12.17.0'} + fast-string-truncated-width@3.0.3: resolution: {integrity: sha512-0jjjIEL6+0jag3l2XWWizO64/aZVtpiGE3t0Zgqxv0DPuxiMjvB3M24fCyhZUO4KomJQPj3LTSUnDP3GpdwC0g==} @@ -3720,6 +3727,9 @@ packages: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} + pure-rand@8.4.2: + resolution: {integrity: sha512-vvuOGgcuPJAirlHvuQw1TrOiw7ptaIXXmIbNuiNOY6lNGJJH49PQ1Kj4nd783nPdQhQdicgOjVI2yI/9BD6/Ng==} + radix3@1.1.2: resolution: {integrity: sha512-b484I/7b8rDEdSDKckSSBA8knMpcdsXudlE/LNL639wFoHKwLbEkQFZHWEYwDC0wa0FKUcCY+GAF73Z7wxNVFA==} @@ -6515,6 +6525,10 @@ snapshots: extend@3.0.2: {} + fast-check@4.10.2: + dependencies: + pure-rand: 8.4.2 + fast-string-truncated-width@3.0.3: {} fast-string-width@3.0.2: @@ -7814,6 +7828,8 @@ snapshots: punycode@2.3.1: {} + pure-rand@8.4.2: {} + radix3@1.1.2: {} react-is@17.0.2: {}