From 2124f03a07ab972ab34aff46122a2954b6072738 Mon Sep 17 00:00:00 2001 From: Claude Date: Fri, 28 Aug 2026 10:24:36 +0000 Subject: [PATCH] fix(scripts): pin the type-check re-measure to CI's own heap ceiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `check:type-check-debt --re-measure` ran tsc under V8's default old-space size, which V8 derives from the physical memory of the box the process starts on. So the gate measured a different world on every machine while only CI's verdict counts: three devs ran it on the agent container against a tree CI had already OOM'd on, and all three read green. A local pass was never a claim about CI, and nothing said so. The ceiling now lives in the gate's own harness, so every caller is CI-shaped without having to remember a flag. The number is read off CI, not off this container: run 33136681083, job `Type Check · debt ledger`, where V8 gave up at `Mark-Compact 4040.3 (4143.8) -> 4029.5 (4147.5) MB` -- bracketing that runner's old space into [4040, 4148] MB, a window whose only V8 default is 4096. The rule is a minimum, never a raise: min(CI's ceiling, this process's own limit, a caller's tighter NODE_OPTIONS cap). A box smaller than CI keeps its own lower ceiling, because promising V8 memory the box lacks converts a recoverable heap error into an exit-137 SIGKILL that carries no diagnostic (the lesson `packages/spec/tsup.config.ts` records from the build side). A caller's ROOMIER cap is refused: it would hand back the exact green-here-red-there reading this ceiling abolishes. The other direction is the one nothing else can catch, so the runner measures it on itself: if CI's own default ever drops BELOW the pinned constant, the pin has stopped describing CI, every local run is silently roomier than CI again, and `--re-measure` refuses on CI naming the reading to re-pin from. On every green run the same line prints the runner's own limit into the log, so the constant stays re-derivable from any CI log of this step instead of from archaeology through a failed job's GC trace. 14 new `--self-test` cases pin all of it, including the two controls that make the family able to fail at all: the roomier-caller refusal, and the same small-box reading OFF ci, which is an ordinary laptop and not a stale pin. Part of #12856 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01CPrUz21stTFhJRUirdc4yw --- scripts/check-type-check-coverage.mjs | 287 +++++++++++++++++++++++++- 1 file changed, 286 insertions(+), 1 deletion(-) diff --git a/scripts/check-type-check-coverage.mjs b/scripts/check-type-check-coverage.mjs index 117846ccc1..7b47f85065 100644 --- a/scripts/check-type-check-coverage.mjs +++ b/scripts/check-type-check-coverage.mjs @@ -447,6 +447,7 @@ import { spawnSync } from 'node:child_process'; import { existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, posix, resolve } from 'node:path'; +import { getHeapStatistics } from 'node:v8'; import { selfTest as workspaceEnumeratorSelfTest, workspacePackageDirs, @@ -2716,6 +2717,163 @@ function countTscErrors(output, { dropRootDirDiagnostics = false } = {}) { return n; } +// --------------------------------------------------------------------------- +// The heap ceiling `--re-measure` runs tsc under (#12856). +// +// ## The defect this closes +// +// tsc's ceiling is V8's default old-space size, and V8 derives that from the +// PHYSICAL MEMORY of the box the process starts on. Nothing here said so, so +// this gate measured a different world on every machine -- while the only +// verdict that counts is CI's. Three devs ran `--re-measure` on the agent +// container against the tree below, all three read green, and CI OOM'd on that +// same tree. ⚠️ The asymmetry IS the defect: a local pass was never a claim +// about CI, and nothing said so out loud. +// +// ## Where the number comes from -- CI, never this box +// +// Read off the CI runner itself: run 33136681083, job `Type Check · debt +// ledger`, at 6d097a604, Node v22.23.2. The `packages/qa/http-conformance` +// TEST_DEBT program died there, and the last GC before it says how much heap it +// was allowed: +// +// [5950] 65768 ms: Mark-Compact 4040.3 (4143.8) -> 4029.5 (4147.5) MB, +// ... allocation failure; GC in old space requested +// FATAL ERROR: Ineffective mark-compacts near heap limit +// Allocation failed - JavaScript heap out of memory +// +// V8 gave up with 4040.3 MB live and 4147.5 MB of heap committed, which +// brackets that runner's old-space limit into [4040, 4148] MB. 4096 is the only +// V8 default that lands in the window, and the two sides agree on the offset: +// `getHeapStatistics().heap_size_limit` reports the old space plus a fixed +// ~48 MB of other spaces (8240 reported for 8192 and 560 for 512, both measured +// with `NODE_OPTIONS` on a box under this file's own eyes), so a 4096 old space +// reports 4144 and commits the 4147.5 above it. +// +// ⚠️ If 4096 is wrong, it is wrong DOWNWARD -- the only safe direction. This +// number's entire job is to be no HIGHER than CI's ceiling. A pin ABOVE CI's is +// worse than no pin at all: it makes local runs pass where CI still OOMs, which +// is exactly this defect with extra confidence attached. +// +// ⛔ Do not raise this to make a local measurement complete. `--re-measure` +// OOMing under this ceiling is the gate WORKING -- it is CI's failure, +// reproduced on your box before you push. What grew is the type graph, not the +// memory CI has. (`packages/spec/tsup.config.ts` carries the other half of this +// lesson from the build side: a ceiling above the box's real memory does not +// buy a bigger run, it converts a recoverable heap error into an exit-137 +// SIGKILL that carries no diagnostic at all.) +const CI_TSC_HEAP_CEILING_MB = 4096; + +/** + * The last `--max-old-space-size` in a `NODE_OPTIONS` string, in MB, or null. + * + * LAST, not first, because that is what V8 does with a repeated flag -- measured + * both ways: `--max-old-space-size=8192 --max-old-space-size=512` reports a + * 560 MB limit and the reverse order reports 8240. That is the whole reason + * `heapCappedEnv` below can APPEND rather than having to rewrite what the + * caller set, and the reason this parser has to agree with V8 about which + * occurrence wins: reading the first one would let a caller's stale flag decide + * a ceiling V8 has already discarded. + * + * Only the `=` spelling exists to parse. `NODE_OPTIONS="--max-old-space-size 512"` + * is not a lower ceiling, it is a node startup error (measured: the space-split + * form makes node reject an unrelated option and exit), so a run that reaches + * this gate at all never carries one. Both the dashed and the underscored flag + * names are accepted, because V8 accepts both. + * + * @param {string | undefined} nodeOptions + * @returns {number | null} + */ +function maxOldSpaceMb(nodeOptions) { + let mb = null; + for (const m of String(nodeOptions ?? '').matchAll(/--max[-_]old[-_]space[-_]size=(\d+)/g)) { + mb = Number(m[1]); + } + return mb; +} + +/** + * The ceiling this run will hand tsc, and the honest name of where it came from. + * + * The rule is a MINIMUM over three numbers, and each one is there for a failure + * that has actually happened somewhere in this repo: + * + * the CI ceiling the point of the exercise -- a roomier box must not + * measure a roomier world than the box whose verdict + * counts. + * this process's own never RAISE a ceiling. On a box smaller than CI, + * promising V8 memory the box does not have does not buy + * a bigger run: the kernel kills the process at the + * container limit long before V8 reaches the ceiling, and + * exit 137 carries no diagnostic (`packages/spec`'s DTS + * pass was killed on every docs deploy for two days that + * way). Lower than CI is the safe direction anyway: heap + * headroom is monotone, so a program that fits under a + * smaller ceiling fits under CI's. + * the caller's an explicit `NODE_OPTIONS` cap is honoured when it is + * TIGHTER, and refused when it is roomier. A caller who + * could hand this gate more heap than CI has could hand + * back the exact green-here-red-there reading this + * ceiling exists to abolish. + * + * `stale` is the other direction, and it is the one nothing else can catch. If + * the runner's OWN default is below the pinned constant, then the constant is + * no longer a description of CI: every local run is roomier than CI again, + * silently, and this file's green is back to meaning nothing. That is only + * measurable on the runner itself, so it is measured there and refused there -- + * an advisory would be a declaration nobody reads, which is the shape this + * repo's ledgers exist to stop. + * + * @param {{heapLimitMb: number, nodeOptions?: string, onCi?: boolean}} where + * @returns {{mb: number, from: string, machineMb: number, stale: string | null}} + */ +function remeasureHeapCeiling({ heapLimitMb, nodeOptions, onCi = false }) { + const caller = maxOldSpaceMb(nodeOptions); + const candidates = [ + { mb: CI_TSC_HEAP_CEILING_MB, from: `the CI-shaped ceiling pinned by ${SELF}` }, + { mb: heapLimitMb, from: "this machine's own default, which is BELOW CI's ceiling" }, + ...(caller === null ? [] : [{ mb: caller, from: "the caller's NODE_OPTIONS, which is tighter" }]), + ]; + // Ties keep the earlier candidate, so the CI ceiling keeps its name on the + // machine that IS CI -- where all three numbers agree and the label is the + // only thing left to read. + const chosen = candidates.reduce((a, b) => (b.mb < a.mb ? b : a)); + const stale = onCi && heapLimitMb < CI_TSC_HEAP_CEILING_MB + ? `${SELF} pins a CI heap ceiling of ${CI_TSC_HEAP_CEILING_MB} MB, but THIS CI runner's own default is ` + + `${heapLimitMb} MB -- the pin is now ABOVE the ceiling it claims to describe, so every local run of ` + + `this gate is roomier than CI again and its green says nothing about this job. Re-pin ` + + `CI_TSC_HEAP_CEILING_MB from this reading (the runner shrank; the remedy is one constant), and ` + + `⛔ do not delete the pin instead -- an unpinned run is the defect #12856 closed.` + : null; + return { mb: chosen.mb, from: chosen.from, machineMb: heapLimitMb, stale }; +} + +/** + * `env` with the ceiling appended to `NODE_OPTIONS`. + * + * APPENDED, never substituted: V8 takes the last occurrence (see + * `maxOldSpaceMb`), so this wins over whatever the caller set without this + * function having to understand the rest of their `NODE_OPTIONS` -- and the + * caller's own flags, which may be the reason their run works at all, survive. + * + * @param {NodeJS.ProcessEnv} env + * @param {number} mb + * @returns {NodeJS.ProcessEnv} + */ +function heapCappedEnv(env, mb) { + return { ...env, NODE_OPTIONS: `${(env.NODE_OPTIONS ?? '').trim()} --max-old-space-size=${mb}`.trim() }; +} + +// Read once, at the ceiling this process actually got rather than at a number +// about the box: `heap_size_limit` already accounts for a caller's flags, a +// cgroup, and whatever V8 decided from physical memory, which is three ways of +// being wrong that this file then does not have to model. +const REMEASURE_HEAP = remeasureHeapCeiling({ + heapLimitMb: Math.floor(getHeapStatistics().heap_size_limit / (1024 * 1024)), + nodeOptions: process.env.NODE_OPTIONS, + onCi: process.env.GITHUB_ACTIONS === 'true', +}); + /** * Run the repo's own tsc over one project and return its raw error count. * `--pretty false` so the count does not depend on whether a TTY is attached; @@ -2736,6 +2894,11 @@ function tscErrorCount(project, options = {}) { cwd: ROOT, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, + // The CI-shaped ceiling (#12856). Every tsc this gate runs gets it, not + // just the generated TEST_DEBT program that happened to OOM first: a DEBT + // entry measured under a roomier ceiling than CI's is the same reading + // dressed as a different one. + env: heapCappedEnv(process.env, REMEASURE_HEAP.mb), }); if (run.error) throw new Error(`tsc could not be run for ${project}: ${run.error.message}`); const output = `${run.stdout ?? ''}${run.stderr ?? ''}`; @@ -4887,6 +5050,112 @@ function selfTest() { if (got !== c.expect) failures.push(`TSC_SETUP_ERROR — ${c.label}: expected ${c.expect}, got ${got}`); } + // THE CI-SHAPED HEAP CEILING (#12856). The only instrument this rule can + // have. Its production reading is a ceiling that is applied and a run that + // then passes -- and a run that passes is precisely what the UNPINNED gate + // also produced on every machine that had the memory to spare. So the + // production verdict cannot tell a correct ceiling from no ceiling at all, + // and the adversarial inputs below are the whole difference. + const ceilingCases = [ + { + label: 'a roomier box than CI is capped to the CI ceiling -- the defect this pin closes', + where: { heapLimitMb: 8240 }, + expect: { mb: CI_TSC_HEAP_CEILING_MB, stale: false }, + }, + { + label: 'on a box shaped like CI the ceiling is a no-op that still names itself', + where: { heapLimitMb: CI_TSC_HEAP_CEILING_MB + 48, onCi: true }, + expect: { mb: CI_TSC_HEAP_CEILING_MB, stale: false }, + }, + { + // Never RAISE. Promising V8 memory the box does not have trades a + // recoverable heap error for a kernel SIGKILL that says nothing. + label: 'a box SMALLER than CI keeps its own lower ceiling', + where: { heapLimitMb: 2096 }, + expect: { mb: 2096, stale: false }, + }, + { + label: "a caller's TIGHTER NODE_OPTIONS cap is honoured", + where: { heapLimitMb: 8240, nodeOptions: '--max-old-space-size=1024' }, + expect: { mb: 1024, stale: false }, + }, + { + // The hole a "respect the caller" rule would leave: a roomier explicit + // cap is the green-here-red-there reading, handed back by request. + label: "a caller's ROOMIER NODE_OPTIONS cap is refused, not respected", + where: { heapLimitMb: 12288, nodeOptions: '--max-old-space-size=12288' }, + expect: { mb: CI_TSC_HEAP_CEILING_MB, stale: false }, + }, + { + // The direction nothing else can catch: the runner shrank, so the pin is + // now ABOVE the ceiling it describes and every local run is roomier than + // CI again -- silently, and with the pin's own confidence attached. + label: 'a CI runner whose own default is BELOW the pin is refused, loudly', + where: { heapLimitMb: 2096, onCi: true }, + expect: { mb: 2096, stale: true }, + }, + { + // THE CONTROL for the case above. Off CI the same numbers are an + // ordinary small box, not evidence about CI -- reading them as a stale + // pin would refuse on every laptop with 4 GB in it. + label: 'the same reading OFF ci is a small box, not a stale pin', + where: { heapLimitMb: 2096, onCi: false }, + expect: { mb: 2096, stale: false }, + }, + ]; + for (const c of ceilingCases) { + const got = remeasureHeapCeiling(c.where); + if (got.mb !== c.expect.mb || (got.stale !== null) !== c.expect.stale) { + failures.push( + `remeasureHeapCeiling — ${c.label}: expected ${c.expect.mb} MB and stale=${c.expect.stale}, ` + + `got ${got.mb} MB and stale=${got.stale === null ? 'false' : JSON.stringify(got.stale)}`, + ); + } + if (got.machineMb !== c.where.heapLimitMb) { + failures.push( + `remeasureHeapCeiling — ${c.label}: reported machineMb ${got.machineMb} for a box of ` + + `${c.where.heapLimitMb} MB. That figure is what a CI log carries forward as the runner's own ` + + `reading, so a wrong one re-pins the constant wrong.`, + ); + } + } + + // The two mechanical halves of the ceiling: which occurrence V8 obeys, and + // that appending is therefore enough. Both measured against node itself + // before they were written down -- `--max-old-space-size=8192 + // --max-old-space-size=512` reports a 560 MB limit, the reverse 8240. + const heapEnvCases = [ + { label: 'no NODE_OPTIONS is no caller cap', options: undefined, expect: null }, + { label: 'an unrelated flag is not a cap', options: '--enable-source-maps', expect: null }, + { label: 'the flag without a value is not a cap', options: '--max-old-space-size', expect: null }, + { label: 'the dashed spelling parses', options: '--max-old-space-size=4096', expect: 4096 }, + { label: 'the underscored spelling V8 also accepts parses', options: '--max_old_space_size=512', expect: 512 }, + { label: 'a repeated flag reads the LAST, as V8 does', options: '--max-old-space-size=8192 --max-old-space-size=512', expect: 512 }, + { label: 'the cap is found among other flags', options: '--enable-source-maps --max-old-space-size=2048 --no-warnings', expect: 2048 }, + ]; + for (const c of heapEnvCases) { + const got = maxOldSpaceMb(c.options); + if (got !== c.expect) failures.push(`maxOldSpaceMb — ${c.label}: expected ${c.expect}, got ${got}`); + // The appended env must be the thing V8 then obeys -- i.e. OUR flag has to + // be the last one in the string, whatever the caller put there. + const appended = heapCappedEnv({ PATH: '/usr/bin', NODE_OPTIONS: c.options }, 777); + if (maxOldSpaceMb(appended.NODE_OPTIONS) !== 777) { + failures.push( + `heapCappedEnv — ${c.label}: the appended ceiling does not win; V8 would obey ` + + `${maxOldSpaceMb(appended.NODE_OPTIONS)} from ${JSON.stringify(appended.NODE_OPTIONS)}`, + ); + } + if (appended.PATH !== '/usr/bin') { + failures.push(`heapCappedEnv — ${c.label}: dropped the rest of the environment`); + } + if (c.options !== undefined && !appended.NODE_OPTIONS.startsWith(c.options)) { + failures.push( + `heapCappedEnv — ${c.label}: dropped the caller's own NODE_OPTIONS (${JSON.stringify(c.options)}), ` + + `which may be the reason their run works at all`, + ); + } + } + // AUTO-LOWERING (#6376). What it refuses matters more than what it writes. const planCases = [ { @@ -5082,7 +5351,8 @@ function selfTest() { `${TYPECHECK_CONFIGS_CASES + coverCases.length + unreadCases.length + accountedCases.length + derivedCases.length + sourceCandidateCases.length + includeRootCases.length + chainCases.length + generatorCases.length + layerCases.length} observation case(s) + ` + - `${driftCases.length + countCases.length + projectCases.length + setupErrorCases.length} re-measure case(s) + ` + + `${driftCases.length + countCases.length + projectCases.length + setupErrorCases.length + + ceilingCases.length + heapEnvCases.length} re-measure case(s) + ` + `${typeEntryCases.length + closureCases.length + staleCases.length + sourceFileCases.length} ` + `built-closure case(s) + ` + `${planCases.length + rewriteCases.length + roundTripCases.length} auto-lowering case(s) hold.`, @@ -5155,6 +5425,21 @@ console.log( // measure, and a wall of tsc output would bury the real failure. Reported after // the summary so the two verdicts read in the order they were reached. if (process.argv.includes('--re-measure')) { + // The ceiling FIRST, before the four minutes of tsc it shapes (#12856). Two + // jobs, and the second is the one that keeps the constant honest: on CI this + // line prints the RUNNER's own default into the run log, so the number + // `CI_TSC_HEAP_CEILING_MB` claims is re-derivable from any CI log of this + // step rather than from archaeology through a failed job's GC trace. + if (REMEASURE_HEAP.stale) { + if (process.env.GITHUB_ACTIONS === 'true') console.log(`::error::${REMEASURE_HEAP.stale}`); + console.error(`\ncheck-type-check-coverage --re-measure: ${REMEASURE_HEAP.stale}`); + process.exit(1); + } + console.log( + ` heap: tsc runs under --max-old-space-size=${REMEASURE_HEAP.mb} MB -- ${REMEASURE_HEAP.from}; ` + + `this process's own limit is ${REMEASURE_HEAP.machineMb} MB. A measurement is only as portable as ` + + `the ceiling it ran under (#12856).`, + ); const started = Date.now(); const measurements = measureLedgers(packages, root.name, state); const { problems: drift, notes, surplus, surplusEntries } = evaluateMeasurements(measurements);