fix(scripts): doc-snippet gate exits 2 when it could not run, 1 when it found errors

scripts/__tests__/check-doc-snippet-types.test.ts

@@ -7,10 +7,12 @@ import { fileURLToPath } from 'node:url';
 // Plain-JS CI helper; its types are inferred from the .mjs source by
 // `tsconfig.scripts.json` (`allowJs`), so no `@ts-expect-error` here.
 import {
+ EXIT_CODES,
  FRAGMENT_MARKER_EXAMPLES,
  UNDECLARED_CONTROL_PACKAGE,
  UNGATED_DOCS,
  analyze,
+ blockingPreconditions,
  deriveDeclaredDependencyPaths,
  derivePackageTypePaths,
  findInstalledCopy,
@@ -45,6 +47,11 @@ import {
  * therefore pins both directions — a declared dependency IS mapped, and an
  * installed-but-undeclared one is NOT — plus the two preconditions the
  * UNDECLARED control needs in order to mean anything.
+ * 7. **The exit path tells "I could not run" from "I ran and found errors"**
+ * (objectui#5465). A run that resolved against nothing produced no verdict
+ * about any document; leaving through the same code as a real snippet
+ * failure makes neither actionable, and leaving through 0 would be zero
+ * information wearing a green tick.
  *
  * Fixtures are throwaway trees, never `content/docs`: a committed fixture page
  * would have to contain a deliberately broken snippet, and this very gate scans
@@ -363,6 +370,58 @@ describe('third-party resolution reaches exactly as far as the imported packages
  });
 });
 
+describe('the exit path — "I could not run" is not "I ran and found errors" (objectui#5465)', () => {
+ /** A workspace package that DECLARES built types, with nothing built. */
+ const unbuiltTree = (types: string): string =>
+ tempTree({
+ 'content/docs/a.mdx': [`${FENCE}ts`, "import '@fixture/pkg-a';", FENCE].join('\n'),
+ 'packages/pkg-a/package.json': JSON.stringify({ name: '@fixture/pkg-a', types }),
+ });
+
+ it('gives the two failure modes different codes, and neither of them is 0', () => {
+ expect(EXIT_CODES.verified).toBe(0);
+ expect(EXIT_CODES.documentsFailed).not.toBe(0);
+ expect(
+ EXIT_CODES.couldNotRun,
+ 'exit 0 with nothing run reads as coverage — the failure shape this gate family exists to prevent',
+ ).not.toBe(0);
+ expect(
+ EXIT_CODES.couldNotRun,
+ 'an unbuilt tree and a broken snippet are different facts; a caller must be able to tell them apart',
+ ).not.toBe(EXIT_CODES.documentsFailed);
+ });
+
+ it('reads an unbuilt package as a precondition, never as a documentation defect', () => {
+ const findings = analyze({ root: unbuiltTree('./dist/index.d.ts'), ungated: {} })
+ .findings as Finding[];
+ expect(findings.map((f) => f.reason)).toContain('unbuilt-package');
+ expect(blockingPreconditions(findings).length).toBeGreaterThan(0);
+ });
+
+ it('reads a source-typed package the same way — it too judges nothing', () => {
+ const findings = analyze({ root: unbuiltTree('./src/index.ts'), ungated: {} })
+ .findings as Finding[];
+ expect(findings.map((f) => f.reason)).toContain('source-typed-package');
+ expect(blockingPreconditions(findings).length).toBeGreaterThan(0);
+ });
+
+ it('leaves ledger findings OUT of the preconditions — those ARE verdicts, and they exit 1', () => {
+ const findings: Finding[] = [
+ { reason: 'stale-ungated-entry', site: 'content/docs/gone.mdx' },
+ { reason: 'unexplained-fragment', site: 'content/docs/a.mdx:3' },
+ { reason: 'stale-fragment-marker', site: 'content/docs/a.mdx:7' },
+ ];
+ expect(blockingPreconditions(findings)).toEqual([]);
+ });
+
+ it('states all three codes in its own header, so the contract cannot drift out of the source', () => {
+ const source = fs.readFileSync(path.join(repoRoot, SCRIPT), 'utf8');
+ const header = source.slice(0, source.indexOf('## What this gate answers'));
+ expect(header).toContain('1 = THE GATE RAN AND FOUND ERRORS');
+ expect(header).toContain('2 = THE GATE COULD NOT RUN');
+ });
+});
+
 describe('wiring — a script nothing runs is not a gate', () => {
  const workflowDir = path.join(repoRoot, '.github/workflows');
  const workflowPath = path.join(workflowDir, 'doc-snippet-types.yml');
@@ -407,6 +466,18 @@ describe('wiring — a script nothing runs is not a gate', () => {
  );
  });
 
+ it('builds BEFORE it invokes the gate, so a precondition exit is not a state CI can reach', () => {
+ const yaml = yamlOf('doc-snippet-types.yml');
+ const build = yaml.indexOf('turbo run build');
+ const invoke = yaml.search(new RegExp(`run: node ${SCRIPT.replace(/[.\\/]/g, '\\$&')}\\s*$`, 'm'));
+ expect(build, 'the workflow must build the packages the covered snippets import').toBeGreaterThan(-1);
+ expect(invoke, 'the workflow must invoke the gate itself').toBeGreaterThan(-1);
+ expect(
+ build,
+ 'invoked before its own build, the gate would red a healthy pull request on a precondition',
+ ).toBeLessThan(invoke);
+ });
+
  it('is reachable by name from the workspace root', () => {
  const pkg = JSON.parse(fs.readFileSync(path.join(repoRoot, 'package.json'), 'utf8'));
  expect(pkg.scripts['check:doc-snippets']).toBe(`node ${SCRIPT}`);

scripts/check-doc-snippet-types.mjs

@@ -8,7 +8,36 @@
  * node scripts/check-doc-snippet-types.mjs --build-filter (turbo filter args)
  * Exit: 0 = every covered snippet parses and type-checks, the harness proved
  * itself on its own controls, and the coverage ledger is exact.
- * 1 = a snippet failed, a control failed, or the ledger is stale.
+ * 1 = THE GATE RAN AND FOUND ERRORS. A snippet failed to parse or to
+ * type-check, or the coverage ledger is stale. Everything printed above
+ * the summary is a verdict about a document.
+ * 2 = THE GATE COULD NOT RUN, so nothing printed above is a verdict about
+ * any document: the packages the covered snippets import are not built (or
+ * are typed from source), or one of the harness's own controls failed.
+ * Fix the tree and re-run. Never read this as a documentation defect, and
+ * never as a pass.
+ *
+ * ## Why "could not run" is its own exit code (objectui#5465)
+ *
+ * Both 1 and 2 are non-zero, so no caller's pass/fail changes: this gate is
+ * invoked from exactly one workflow step (`doc-snippet-types.yml`), which fails
+ * on any non-zero, and which builds this gate's own `--build-filter` closure
+ * BEFORE invoking it — an unbuilt tree is not a state CI can reach. The
+ * distinction exists for the reader. "I could not run" and "I ran and found
+ * errors" are different facts, and while they shared one exit code they were
+ * indistinguishable at the exit-code level: three separate agents in one evening
+ * (objectui#6171, #6186, #5259) each had to notice the printed message and
+ * rebuild before their exit code meant anything, and #6171 wrote it down as
+ * "an unbuilt-tree exit is indistinguishable from a real failure at the
+ * exit-code level". `scripts/check-eager-closure-budget.mjs` had already drawn
+ * this same line for this same reason — 1 for "over budget" (a verdict about the
+ * bundle), 2 for "the gauge produced nothing" (a verdict about the gauge) — so
+ * this is the repository's convention, not a new one.
+ *
+ * ⛔ What is NOT an option here: printing the reason and exiting 0. Zero with
+ * nothing run reads as coverage, which is the exact failure shape this whole
+ * gate family exists to prevent (objectui#4846: an unexaminable package must not
+ * read as a clean one).
  *
  * ## What this gate answers, and the three things it does NOT (read this first)
  *
@@ -1025,27 +1054,68 @@ function formatDiagnostic(diagnostic, block) {
  return `${where} TS${diagnostic.code}: ${message}`;
 }
 
+/**
+ * The gate's exit codes, named so that callers and tests can talk about them.
+ * `couldNotRun` is deliberately distinct from `documentsFailed`: see the "Why
+ * 'could not run' is its own exit code" section in this file's header.
+ */
+export const EXIT_CODES = {
+ /** Every covered snippet compiled, the controls held, the ledger is exact. */
+ verified: 0,
+ /** The gate RAN. A snippet or the ledger is at fault — a verdict was read. */
+ documentsFailed: 1,
+ /** The gate COULD NOT RUN. Nothing it printed is a verdict about a document. */
+ couldNotRun: 2,
+};
+
+/**
+ * The findings that stop the snippet program from being built at all, so no
+ * verdict about any document can be read from the run. Kept separate from the
+ * findings that ARE verdicts (a stale ledger entry, an unexplained fragment)
+ * because the two leave through different exit codes.
+ *
+ * @param {{ reason: string }[]} findings
+ */
+export function blockingPreconditions(findings) {
+ return findings.filter(
+ (f) => f.reason === 'unbuilt-package' || f.reason === 'source-typed-package',
+ );
+}
+
 function main() {
  const argv = process.argv.slice(2);
  const state = analyze({});
 
  if (argv.includes('--build-filter')) {
  // Turbo filter arguments for exactly the packages the covered snippets
  // import. Coverage grows -> the build grows, and nothing else does.
+ // ⛔ This query answers from an UNBUILT tree by design and must keep exiting
+ // 0 there: it is what the workflow runs to learn what to build, one step
+ // BEFORE the build. Making it share the precondition exit would deadlock the
+ // gate against its own build step.
  process.stdout.write([...state.neededPackages].sort().map((n) => `--filter=${n}`).join(' '));
  process.stdout.write('\n');
  return 0;
  }
 
- const blocking = state.findings.filter(
- (f) => f.reason === 'unbuilt-package' || f.reason === 'source-typed-package',
- );
+ const blocking = blockingPreconditions(state.findings);
  if (blocking.length > 0) {
  for (const f of state.findings) console.error(` ${f.site} [${f.reason}] ${f.detail}`);
  console.error(
- '\nThe snippet program was NOT run: the packages it resolves against are not built, or are typed from source.',
+ '\nPRECONDITION NOT MET (exit ' +
+ EXIT_CODES.couldNotRun +
+ ') — The snippet program was NOT run: the packages it resolves against are not built, or are typed from source.',
  );
- return 1;
+ console.error(
+ `This is "I could not run", NOT "I ran and found errors" (exit ${EXIT_CODES.documentsFailed}). ` +
+ 'No line above is a verdict about any document, and this run says nothing about whether the ' +
+ 'documentation compiles. Build what the gate needs, then re-run:',
+ );
+ console.error(
+ ' pnpm exec turbo run build $(node scripts/check-doc-snippet-types.mjs --build-filter) --concurrency=2\n' +
+ ' pnpm check:doc-snippets',
+ );
+ return EXIT_CODES.couldNotRun;
  }
 
  const run = compileSnippets({
@@ -1152,13 +1222,19 @@ function main() {
  if (controlFailures.length > 0) {
  console.error('\nHARNESS CONTROL FAILED — no verdict about the documents can be read from this run:');
  for (const c of controlFailures) console.error(` - ${c}`);
+ console.error(
+ `\nThe gate COULD NOT RUN (exit ${EXIT_CODES.couldNotRun}). The sentence above is this run's own ` +
+ 'wording, and the exit code now agrees with it: a broken harness is a verdict about the ' +
+ 'harness, never about the documents.',
+ );
+ return EXIT_CODES.couldNotRun;
  }
  if (failed) {
  console.error('\nDocumentation snippets must compile against the built types. See the header of this script.');
- return 1;
+ return EXIT_CODES.documentsFailed;
  }
  console.log('\nEvery covered documentation snippet compiles against the built types.');
- return 0;
+ return EXIT_CODES.verified;
 }
 
 if (isEntrypoint(import.meta.url)) {