diff --git a/scripts/__tests__/check-doc-snippet-types.test.ts b/scripts/__tests__/check-doc-snippet-types.test.ts index 571f20a73a..1b25d28aca 100644 --- a/scripts/__tests__/check-doc-snippet-types.test.ts +++ b/scripts/__tests__/check-doc-snippet-types.test.ts @@ -7,10 +7,12 @@ import { fileURLToPath } from 'node:url'; // Plain-JS CI helper; its types are inferred from the .mjs source by // `tsconfig.scripts.json` (`allowJs`), so no `@ts-expect-error` here. import { + EXIT_CODES, FRAGMENT_MARKER_EXAMPLES, UNDECLARED_CONTROL_PACKAGE, UNGATED_DOCS, analyze, + blockingPreconditions, deriveDeclaredDependencyPaths, derivePackageTypePaths, findInstalledCopy, @@ -45,6 +47,11 @@ import { * therefore pins both directions — a declared dependency IS mapped, and an * installed-but-undeclared one is NOT — plus the two preconditions the * UNDECLARED control needs in order to mean anything. + * 7. **The exit path tells "I could not run" from "I ran and found errors"** + * (objectui#5465). A run that resolved against nothing produced no verdict + * about any document; leaving through the same code as a real snippet + * failure makes neither actionable, and leaving through 0 would be zero + * information wearing a green tick. * * Fixtures are throwaway trees, never `content/docs`: a committed fixture page * would have to contain a deliberately broken snippet, and this very gate scans @@ -363,6 +370,58 @@ describe('third-party resolution reaches exactly as far as the imported packages }); }); +describe('the exit path — "I could not run" is not "I ran and found errors" (objectui#5465)', () => { + /** A workspace package that DECLARES built types, with nothing built. */ + const unbuiltTree = (types: string): string => + tempTree({ + 'content/docs/a.mdx': [`${FENCE}ts`, "import '@fixture/pkg-a';", FENCE].join('\n'), + 'packages/pkg-a/package.json': JSON.stringify({ name: '@fixture/pkg-a', types }), + }); + + it('gives the two failure modes different codes, and neither of them is 0', () => { + expect(EXIT_CODES.verified).toBe(0); + expect(EXIT_CODES.documentsFailed).not.toBe(0); + expect( + EXIT_CODES.couldNotRun, + 'exit 0 with nothing run reads as coverage — the failure shape this gate family exists to prevent', + ).not.toBe(0); + expect( + EXIT_CODES.couldNotRun, + 'an unbuilt tree and a broken snippet are different facts; a caller must be able to tell them apart', + ).not.toBe(EXIT_CODES.documentsFailed); + }); + + it('reads an unbuilt package as a precondition, never as a documentation defect', () => { + const findings = analyze({ root: unbuiltTree('./dist/index.d.ts'), ungated: {} }) + .findings as Finding[]; + expect(findings.map((f) => f.reason)).toContain('unbuilt-package'); + expect(blockingPreconditions(findings).length).toBeGreaterThan(0); + }); + + it('reads a source-typed package the same way — it too judges nothing', () => { + const findings = analyze({ root: unbuiltTree('./src/index.ts'), ungated: {} }) + .findings as Finding[]; + expect(findings.map((f) => f.reason)).toContain('source-typed-package'); + expect(blockingPreconditions(findings).length).toBeGreaterThan(0); + }); + + it('leaves ledger findings OUT of the preconditions — those ARE verdicts, and they exit 1', () => { + const findings: Finding[] = [ + { reason: 'stale-ungated-entry', site: 'content/docs/gone.mdx' }, + { reason: 'unexplained-fragment', site: 'content/docs/a.mdx:3' }, + { reason: 'stale-fragment-marker', site: 'content/docs/a.mdx:7' }, + ]; + expect(blockingPreconditions(findings)).toEqual([]); + }); + + it('states all three codes in its own header, so the contract cannot drift out of the source', () => { + const source = fs.readFileSync(path.join(repoRoot, SCRIPT), 'utf8'); + const header = source.slice(0, source.indexOf('## What this gate answers')); + expect(header).toContain('1 = THE GATE RAN AND FOUND ERRORS'); + expect(header).toContain('2 = THE GATE COULD NOT RUN'); + }); +}); + describe('wiring — a script nothing runs is not a gate', () => { const workflowDir = path.join(repoRoot, '.github/workflows'); const workflowPath = path.join(workflowDir, 'doc-snippet-types.yml'); @@ -407,6 +466,18 @@ describe('wiring — a script nothing runs is not a gate', () => { ); }); + it('builds BEFORE it invokes the gate, so a precondition exit is not a state CI can reach', () => { + const yaml = yamlOf('doc-snippet-types.yml'); + const build = yaml.indexOf('turbo run build'); + const invoke = yaml.search(new RegExp(`run: node ${SCRIPT.replace(/[.\\/]/g, '\\$&')}\\s*$`, 'm')); + expect(build, 'the workflow must build the packages the covered snippets import').toBeGreaterThan(-1); + expect(invoke, 'the workflow must invoke the gate itself').toBeGreaterThan(-1); + expect( + build, + 'invoked before its own build, the gate would red a healthy pull request on a precondition', + ).toBeLessThan(invoke); + }); + it('is reachable by name from the workspace root', () => { const pkg = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8')); expect(pkg.scripts['check:doc-snippets']).toBe(`node ${SCRIPT}`); diff --git a/scripts/check-doc-snippet-types.mjs b/scripts/check-doc-snippet-types.mjs index 888db0738c..ec68e7be71 100644 --- a/scripts/check-doc-snippet-types.mjs +++ b/scripts/check-doc-snippet-types.mjs @@ -8,7 +8,36 @@ * node scripts/check-doc-snippet-types.mjs --build-filter (turbo filter args) * Exit: 0 = every covered snippet parses and type-checks, the harness proved * itself on its own controls, and the coverage ledger is exact. - * 1 = a snippet failed, a control failed, or the ledger is stale. + * 1 = THE GATE RAN AND FOUND ERRORS. A snippet failed to parse or to + * type-check, or the coverage ledger is stale. Everything printed above + * the summary is a verdict about a document. + * 2 = THE GATE COULD NOT RUN, so nothing printed above is a verdict about + * any document: the packages the covered snippets import are not built (or + * are typed from source), or one of the harness's own controls failed. + * Fix the tree and re-run. Never read this as a documentation defect, and + * never as a pass. + * + * ## Why "could not run" is its own exit code (objectui#5465) + * + * Both 1 and 2 are non-zero, so no caller's pass/fail changes: this gate is + * invoked from exactly one workflow step (`doc-snippet-types.yml`), which fails + * on any non-zero, and which builds this gate's own `--build-filter` closure + * BEFORE invoking it — an unbuilt tree is not a state CI can reach. The + * distinction exists for the reader. "I could not run" and "I ran and found + * errors" are different facts, and while they shared one exit code they were + * indistinguishable at the exit-code level: three separate agents in one evening + * (objectui#6171, #6186, #5259) each had to notice the printed message and + * rebuild before their exit code meant anything, and #6171 wrote it down as + * "an unbuilt-tree exit is indistinguishable from a real failure at the + * exit-code level". `scripts/check-eager-closure-budget.mjs` had already drawn + * this same line for this same reason — 1 for "over budget" (a verdict about the + * bundle), 2 for "the gauge produced nothing" (a verdict about the gauge) — so + * this is the repository's convention, not a new one. + * + * ⛔ What is NOT an option here: printing the reason and exiting 0. Zero with + * nothing run reads as coverage, which is the exact failure shape this whole + * gate family exists to prevent (objectui#4846: an unexaminable package must not + * read as a clean one). * * ## What this gate answers, and the three things it does NOT (read this first) * @@ -1025,6 +1054,34 @@ function formatDiagnostic(diagnostic, block) { return `${where} TS${diagnostic.code}: ${message}`; } +/** + * The gate's exit codes, named so that callers and tests can talk about them. + * `couldNotRun` is deliberately distinct from `documentsFailed`: see the "Why + * 'could not run' is its own exit code" section in this file's header. + */ +export const EXIT_CODES = { + /** Every covered snippet compiled, the controls held, the ledger is exact. */ + verified: 0, + /** The gate RAN. A snippet or the ledger is at fault — a verdict was read. */ + documentsFailed: 1, + /** The gate COULD NOT RUN. Nothing it printed is a verdict about a document. */ + couldNotRun: 2, +}; + +/** + * The findings that stop the snippet program from being built at all, so no + * verdict about any document can be read from the run. Kept separate from the + * findings that ARE verdicts (a stale ledger entry, an unexplained fragment) + * because the two leave through different exit codes. + * + * @param {{ reason: string }[]} findings + */ +export function blockingPreconditions(findings) { + return findings.filter( + (f) => f.reason === 'unbuilt-package' || f.reason === 'source-typed-package', + ); +} + function main() { const argv = process.argv.slice(2); const state = analyze({}); @@ -1032,20 +1089,33 @@ function main() { if (argv.includes('--build-filter')) { // Turbo filter arguments for exactly the packages the covered snippets // import. Coverage grows -> the build grows, and nothing else does. + // ⛔ This query answers from an UNBUILT tree by design and must keep exiting + // 0 there: it is what the workflow runs to learn what to build, one step + // BEFORE the build. Making it share the precondition exit would deadlock the + // gate against its own build step. process.stdout.write([...state.neededPackages].sort().map((n) => `--filter=${n}`).join(' ')); process.stdout.write('\n'); return 0; } - const blocking = state.findings.filter( - (f) => f.reason === 'unbuilt-package' || f.reason === 'source-typed-package', - ); + const blocking = blockingPreconditions(state.findings); if (blocking.length > 0) { for (const f of state.findings) console.error(` ${f.site} [${f.reason}] ${f.detail}`); console.error( - '\nThe snippet program was NOT run: the packages it resolves against are not built, or are typed from source.', + '\nPRECONDITION NOT MET (exit ' + + EXIT_CODES.couldNotRun + + ') — The snippet program was NOT run: the packages it resolves against are not built, or are typed from source.', ); - return 1; + console.error( + `This is "I could not run", NOT "I ran and found errors" (exit ${EXIT_CODES.documentsFailed}). ` + + 'No line above is a verdict about any document, and this run says nothing about whether the ' + + 'documentation compiles. Build what the gate needs, then re-run:', + ); + console.error( + ' pnpm exec turbo run build $(node scripts/check-doc-snippet-types.mjs --build-filter) --concurrency=2\n' + + ' pnpm check:doc-snippets', + ); + return EXIT_CODES.couldNotRun; } const run = compileSnippets({ @@ -1152,13 +1222,19 @@ function main() { if (controlFailures.length > 0) { console.error('\nHARNESS CONTROL FAILED — no verdict about the documents can be read from this run:'); for (const c of controlFailures) console.error(` - ${c}`); + console.error( + `\nThe gate COULD NOT RUN (exit ${EXIT_CODES.couldNotRun}). The sentence above is this run's own ` + + 'wording, and the exit code now agrees with it: a broken harness is a verdict about the ' + + 'harness, never about the documents.', + ); + return EXIT_CODES.couldNotRun; } if (failed) { console.error('\nDocumentation snippets must compile against the built types. See the header of this script.'); - return 1; + return EXIT_CODES.documentsFailed; } console.log('\nEvery covered documentation snippet compiles against the built types.'); - return 0; + return EXIT_CODES.verified; } if (isEntrypoint(import.meta.url)) {