From d415f4fa40bf5b730b7464e3eac20567be3fa7c9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 02:46:40 +0000 Subject: [PATCH 1/4] chore(tooling): TEMPORARY runner-memory probe for the tsc heap ceiling (#14569) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reverted before this PR's final diff. `CI_TSC_HEAP_CEILING_MB` may only move on a reading taken where the verdict is taken -- the `Type Check · debt ledger` job on `ubuntu-latest` -- and this container cannot download job logs. Check-run ANNOTATIONS are readable over REST, so the probe emits its readings as `::notice` workflow commands from that job: - the runner's MemTotal/MemAvailable/Swap, image, nproc, and the gate process's own V8 `heap_size_limit` (the runner's default old space); - what else is resident at the point the re-measure starts (`ps` RSS census); - the `packages/qa/http-conformance` TEST_DEBT program -- the same generated project `measureTestDebt` writes -- run with `--extendedDiagnostics` under `--max-old-space-size=4096` and under `6144`, reporting tsc's own "Memory used", peak RSS, and the machine's minimum MemAvailable during each run. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WLJQhde67SeTccsmnBVarV --- .github/workflows/lint.yml | 6 ++ scripts/check-type-check-coverage.mjs | 124 ++++++++++++++++++++++++++ 2 files changed, 130 insertions(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index e463691b5e..54fe814816 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -4737,6 +4737,12 @@ jobs: - name: Build the ledgered packages' dependencies run: pnpm exec turbo run build --filter='./packages/*' --filter='./packages/*/*' + # TEMPORARY measurement probe (#14569) — reverted before this PR's final + # diff. Prints this runner's memory readings as workflow-command + # annotations, which are readable through the check-run annotations API. + - name: Runner memory probe (TEMPORARY, #14569) + run: node scripts/check-type-check-coverage.mjs --runner-reading + - name: Re-measure the type-check DEBT / TEST_DEBT ledger run: pnpm check:type-check-debt diff --git a/scripts/check-type-check-coverage.mjs b/scripts/check-type-check-coverage.mjs index 7a070e6fdd..02c3b4bca6 100644 --- a/scripts/check-type-check-coverage.mjs +++ b/scripts/check-type-check-coverage.mjs @@ -5591,6 +5591,130 @@ console.log( // clean: a ledger entry naming a package that no longer exists has nothing to // measure, and a wall of tsc output would bury the real failure. Reported after // the summary so the two verdicts read in the order they were reached. +// ── TEMPORARY runner-memory probe (#14569) ────────────────────────────────── +// Emits this job's memory readings as workflow-command annotations so they can +// be read back through the check-run annotations API from an agent container +// that cannot download job logs. ⛔ NOT part of the shipped change: this block +// and the workflow step that calls it are reverted before the PR's final diff. +if (process.argv.includes('--runner-reading')) { + const PROBE_PKG = '@objectstack/http-conformance'; + const notice = (title, body) => { + const text = String(body).trim(); + console.log(`::notice title=${title}::${text.replace(/\r?\n/g, '%0A').slice(0, 3800)}`); + console.log(`[probe] ${title}\n${text}\n`); + }; + const meminfoKb = (key) => { + const m = readFileSync('/proc/meminfo', 'utf8').match(new RegExp(`^${key}:\\s+(\\d+) kB`, 'm')); + return m === null ? null : Number(m[1]); + }; + const psSnapshot = () => { + const run = spawnSync('ps', ['-eo', 'rss=,comm='], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }); + const rows = String(run.stdout ?? '') + .trim() + .split('\n') + .map((l) => l.trim().split(/\s+/)) + .map(([rss, ...c]) => ({ rss: Number(rss), comm: c.join(' ') })) + .filter((r) => Number.isFinite(r.rss)); + rows.sort((a, b) => b.rss - a.rss); + return { total: rows.reduce((s, r) => s + r.rss, 0), rows: rows.slice(0, 12), count: rows.length }; + }; + const childHeapLimit = (mb) => { + const run = spawnSync(process.execPath, ['-e', 'console.log(require("node:v8").getHeapStatistics().heap_size_limit)'], { + encoding: 'utf8', + env: { ...process.env, NODE_OPTIONS: `--max-old-space-size=${mb}` }, + }); + return Math.floor(Number(String(run.stdout ?? '0').trim()) / (1024 * 1024)); + }; + + const snap = psSnapshot(); + notice('probe-runner-env', [ + `date=${new Date().toISOString()}`, + `runner os=${process.env.RUNNER_OS} arch=${process.env.RUNNER_ARCH} image=${process.env.ImageOS} ${process.env.ImageVersion}`, + `node=${process.version} nproc=${String(spawnSync('nproc', { encoding: 'utf8' }).stdout ?? '').trim()}`, + `MemTotal=${meminfoKb('MemTotal')} kB MemAvailable=${meminfoKb('MemAvailable')} kB SwapTotal=${meminfoKb('SwapTotal')} kB SwapFree=${meminfoKb('SwapFree')} kB`, + `gate process heap_size_limit=${Math.floor(getHeapStatistics().heap_size_limit / (1024 * 1024))} MB (runner V8 default)`, + `child heap_size_limit under --max-old-space-size=6144 = ${childHeapLimit(6144)} MB`, + `child heap_size_limit under --max-old-space-size=4096 = ${childHeapLimit(4096)} MB`, + `NODE_OPTIONS=${JSON.stringify(process.env.NODE_OPTIONS ?? '')}`, + `REMEASURE_HEAP=${JSON.stringify(REMEASURE_HEAP)}`, + ].join('\n')); + notice('probe-consumers', [ + `processes=${snap.count} total_rss=${snap.total} kB (${(snap.total / 1024).toFixed(0)} MB) at the point the re-measure starts`, + ...snap.rows.map((r) => `${String(r.rss).padStart(9)} kB ${r.comm}`), + ].join('\n')); + + // The generated TEST_DEBT program for the probe package -- the same project + // `measureTestDebt` writes, run directly so the heap cap can be varied. + const probePkg = packages.find((p) => p.name === PROBE_PKG); + const probeDir = probePkg.dir; + const probeRootAbs = ROOT.replaceAll('\\', '/').replace(/\/$/, ''); + const probePkgAbs = join(ROOT, probeDir).replaceAll('\\', '/').replace(/\/$/, ''); + const probeParsed = JSON.parse( + readFileSync(join(ROOT, probeDir, 'tsconfig.json'), 'utf8').replace(/^\s*\/\/.*$/gm, ''), + ); + const probeRoots = readTsconfig(probeDir, 'tsconfig.json').roots; + const probeHidden = probePkg.hiddenTests ?? []; + const probeUnreachable = probeHidden.filter( + (rel) => !probeRoots.some((r) => r === '' || rel === r || rel.startsWith(`${r}/`)), + ); + const probeProject = remeasureProject({ + pkgAbs: probePkgAbs, + rootAbs: probeRootAbs, + parsed: probeParsed, + unreachable: probeUnreachable, + chain: tsconfigChainFacts(probeDir), + }); + const probeHolder = mkdtempSync(join(tmpdir(), 'objectstack-probe-')); + const probeConfig = join(probeHolder, REMEASURE_CONFIG); + writeFileSync(probeConfig, `${JSON.stringify(probeProject, null, 2)}\n`); + const probeTsc = join(ROOT, 'node_modules', '.bin', 'tsc'); + const hasTime = existsSync('/usr/bin/time'); + + for (const capMb of [4096, 6144]) { + const out = join(probeHolder, `tsc-${capMb}.out`); + const err = join(probeHolder, `tsc-${capMb}.err`); + const sample = join(probeHolder, `mem-${capMb}.txt`); + const shell = [ + 'set -u', + `( while :; do awk '/^MemAvailable:/{print $2}' /proc/meminfo; sleep 0.5; done > ${sample} ) &`, + 'SAMPLER=$!', + `${hasTime ? '/usr/bin/time -v ' : ''}${probeTsc} --noEmit --pretty false --extendedDiagnostics -p ${probeConfig} > ${out} 2> ${err}`, + 'STATUS=$?', + 'kill "$SAMPLER" 2>/dev/null || true', + 'wait "$SAMPLER" 2>/dev/null || true', + 'echo "status=$STATUS"', + ].join('\n'); + const started = Date.now(); + const run = spawnSync('bash', ['-c', shell], { + cwd: ROOT, + encoding: 'utf8', + maxBuffer: 64 * 1024 * 1024, + env: { ...process.env, NODE_OPTIONS: `--max-old-space-size=${capMb}` }, + }); + const wall = ((Date.now() - started) / 1000).toFixed(1); + const stdout = existsSync(out) ? readFileSync(out, 'utf8') : ''; + const stderr = existsSync(err) ? readFileSync(err, 'utf8') : ''; + const pick = (re) => ((stdout.match(re) ?? stderr.match(re) ?? [null, null])[1]); + const samples = existsSync(sample) + ? readFileSync(sample, 'utf8').trim().split('\n').map(Number).filter(Number.isFinite) + : []; + const errs = stdout.match(/error TS\d+/g) ?? []; + notice(`probe-tsc-${capMb}`, [ + `cap=--max-old-space-size=${capMb} wall=${wall}s ${String(run.stdout ?? '').trim()}`, + `Files=${pick(/^Files:\s+(\d+)/m)} LinesOfDefinitions=${pick(/^Lines of Definitions:\s+(\d+)/m)}`, + `Types=${pick(/^Types:\s+(\d+)/m)} Instantiations=${pick(/^Instantiations:\s+(\d+)/m)}`, + `MemoryUsed=${pick(/^Memory used:\s+([\d,]+K)/m)}`, + `CheckTime=${pick(/^Check time:\s+([\d.]+)s/m)}s TotalTime=${pick(/^Total time:\s+([\d.]+)s/m)}s`, + `MaximumRSS=${pick(/Maximum resident set size \(kbytes\):\s+(\d+)/m)} kB`, + `MemAvailable during run: min=${samples.length ? Math.min(...samples) : 'n/a'} kB max=${samples.length ? Math.max(...samples) : 'n/a'} kB samples=${samples.length}`, + `heapOOM=${/JavaScript heap out of memory/.test(`${stdout}${stderr}`)} errCount=${errs.length}`, + `stderrTail=${stderr.trim().split('\n').slice(-3).join(' | ').slice(0, 400)}`, + ].join('\n')); + } + rmSync(probeHolder, { force: true, recursive: true }); + process.exit(0); +} + if (process.argv.includes('--re-measure')) { // The ceiling FIRST, before the four minutes of tsc it shapes (#12856). Two // jobs, and the second is the one that keeps the constant honest: on CI this From 994eb21ae52480fb04a718f7e639a36c8af97065 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 02:54:31 +0000 Subject: [PATCH 2/4] Revert "chore(tooling): TEMPORARY runner-memory probe for the tsc heap ceiling (#14569)" This reverts commit d415f4fa40bf5b730b7464e3eac20567be3fa7c9. --- .github/workflows/lint.yml | 6 -- scripts/check-type-check-coverage.mjs | 124 -------------------------- 2 files changed, 130 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 54fe814816..e463691b5e 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -4737,12 +4737,6 @@ jobs: - name: Build the ledgered packages' dependencies run: pnpm exec turbo run build --filter='./packages/*' --filter='./packages/*/*' - # TEMPORARY measurement probe (#14569) — reverted before this PR's final - # diff. Prints this runner's memory readings as workflow-command - # annotations, which are readable through the check-run annotations API. - - name: Runner memory probe (TEMPORARY, #14569) - run: node scripts/check-type-check-coverage.mjs --runner-reading - - name: Re-measure the type-check DEBT / TEST_DEBT ledger run: pnpm check:type-check-debt diff --git a/scripts/check-type-check-coverage.mjs b/scripts/check-type-check-coverage.mjs index 02c3b4bca6..7a070e6fdd 100644 --- a/scripts/check-type-check-coverage.mjs +++ b/scripts/check-type-check-coverage.mjs @@ -5591,130 +5591,6 @@ console.log( // clean: a ledger entry naming a package that no longer exists has nothing to // measure, and a wall of tsc output would bury the real failure. Reported after // the summary so the two verdicts read in the order they were reached. -// ── TEMPORARY runner-memory probe (#14569) ────────────────────────────────── -// Emits this job's memory readings as workflow-command annotations so they can -// be read back through the check-run annotations API from an agent container -// that cannot download job logs. ⛔ NOT part of the shipped change: this block -// and the workflow step that calls it are reverted before the PR's final diff. -if (process.argv.includes('--runner-reading')) { - const PROBE_PKG = '@objectstack/http-conformance'; - const notice = (title, body) => { - const text = String(body).trim(); - console.log(`::notice title=${title}::${text.replace(/\r?\n/g, '%0A').slice(0, 3800)}`); - console.log(`[probe] ${title}\n${text}\n`); - }; - const meminfoKb = (key) => { - const m = readFileSync('/proc/meminfo', 'utf8').match(new RegExp(`^${key}:\\s+(\\d+) kB`, 'm')); - return m === null ? null : Number(m[1]); - }; - const psSnapshot = () => { - const run = spawnSync('ps', ['-eo', 'rss=,comm='], { encoding: 'utf8', maxBuffer: 32 * 1024 * 1024 }); - const rows = String(run.stdout ?? '') - .trim() - .split('\n') - .map((l) => l.trim().split(/\s+/)) - .map(([rss, ...c]) => ({ rss: Number(rss), comm: c.join(' ') })) - .filter((r) => Number.isFinite(r.rss)); - rows.sort((a, b) => b.rss - a.rss); - return { total: rows.reduce((s, r) => s + r.rss, 0), rows: rows.slice(0, 12), count: rows.length }; - }; - const childHeapLimit = (mb) => { - const run = spawnSync(process.execPath, ['-e', 'console.log(require("node:v8").getHeapStatistics().heap_size_limit)'], { - encoding: 'utf8', - env: { ...process.env, NODE_OPTIONS: `--max-old-space-size=${mb}` }, - }); - return Math.floor(Number(String(run.stdout ?? '0').trim()) / (1024 * 1024)); - }; - - const snap = psSnapshot(); - notice('probe-runner-env', [ - `date=${new Date().toISOString()}`, - `runner os=${process.env.RUNNER_OS} arch=${process.env.RUNNER_ARCH} image=${process.env.ImageOS} ${process.env.ImageVersion}`, - `node=${process.version} nproc=${String(spawnSync('nproc', { encoding: 'utf8' }).stdout ?? '').trim()}`, - `MemTotal=${meminfoKb('MemTotal')} kB MemAvailable=${meminfoKb('MemAvailable')} kB SwapTotal=${meminfoKb('SwapTotal')} kB SwapFree=${meminfoKb('SwapFree')} kB`, - `gate process heap_size_limit=${Math.floor(getHeapStatistics().heap_size_limit / (1024 * 1024))} MB (runner V8 default)`, - `child heap_size_limit under --max-old-space-size=6144 = ${childHeapLimit(6144)} MB`, - `child heap_size_limit under --max-old-space-size=4096 = ${childHeapLimit(4096)} MB`, - `NODE_OPTIONS=${JSON.stringify(process.env.NODE_OPTIONS ?? '')}`, - `REMEASURE_HEAP=${JSON.stringify(REMEASURE_HEAP)}`, - ].join('\n')); - notice('probe-consumers', [ - `processes=${snap.count} total_rss=${snap.total} kB (${(snap.total / 1024).toFixed(0)} MB) at the point the re-measure starts`, - ...snap.rows.map((r) => `${String(r.rss).padStart(9)} kB ${r.comm}`), - ].join('\n')); - - // The generated TEST_DEBT program for the probe package -- the same project - // `measureTestDebt` writes, run directly so the heap cap can be varied. - const probePkg = packages.find((p) => p.name === PROBE_PKG); - const probeDir = probePkg.dir; - const probeRootAbs = ROOT.replaceAll('\\', '/').replace(/\/$/, ''); - const probePkgAbs = join(ROOT, probeDir).replaceAll('\\', '/').replace(/\/$/, ''); - const probeParsed = JSON.parse( - readFileSync(join(ROOT, probeDir, 'tsconfig.json'), 'utf8').replace(/^\s*\/\/.*$/gm, ''), - ); - const probeRoots = readTsconfig(probeDir, 'tsconfig.json').roots; - const probeHidden = probePkg.hiddenTests ?? []; - const probeUnreachable = probeHidden.filter( - (rel) => !probeRoots.some((r) => r === '' || rel === r || rel.startsWith(`${r}/`)), - ); - const probeProject = remeasureProject({ - pkgAbs: probePkgAbs, - rootAbs: probeRootAbs, - parsed: probeParsed, - unreachable: probeUnreachable, - chain: tsconfigChainFacts(probeDir), - }); - const probeHolder = mkdtempSync(join(tmpdir(), 'objectstack-probe-')); - const probeConfig = join(probeHolder, REMEASURE_CONFIG); - writeFileSync(probeConfig, `${JSON.stringify(probeProject, null, 2)}\n`); - const probeTsc = join(ROOT, 'node_modules', '.bin', 'tsc'); - const hasTime = existsSync('/usr/bin/time'); - - for (const capMb of [4096, 6144]) { - const out = join(probeHolder, `tsc-${capMb}.out`); - const err = join(probeHolder, `tsc-${capMb}.err`); - const sample = join(probeHolder, `mem-${capMb}.txt`); - const shell = [ - 'set -u', - `( while :; do awk '/^MemAvailable:/{print $2}' /proc/meminfo; sleep 0.5; done > ${sample} ) &`, - 'SAMPLER=$!', - `${hasTime ? '/usr/bin/time -v ' : ''}${probeTsc} --noEmit --pretty false --extendedDiagnostics -p ${probeConfig} > ${out} 2> ${err}`, - 'STATUS=$?', - 'kill "$SAMPLER" 2>/dev/null || true', - 'wait "$SAMPLER" 2>/dev/null || true', - 'echo "status=$STATUS"', - ].join('\n'); - const started = Date.now(); - const run = spawnSync('bash', ['-c', shell], { - cwd: ROOT, - encoding: 'utf8', - maxBuffer: 64 * 1024 * 1024, - env: { ...process.env, NODE_OPTIONS: `--max-old-space-size=${capMb}` }, - }); - const wall = ((Date.now() - started) / 1000).toFixed(1); - const stdout = existsSync(out) ? readFileSync(out, 'utf8') : ''; - const stderr = existsSync(err) ? readFileSync(err, 'utf8') : ''; - const pick = (re) => ((stdout.match(re) ?? stderr.match(re) ?? [null, null])[1]); - const samples = existsSync(sample) - ? readFileSync(sample, 'utf8').trim().split('\n').map(Number).filter(Number.isFinite) - : []; - const errs = stdout.match(/error TS\d+/g) ?? []; - notice(`probe-tsc-${capMb}`, [ - `cap=--max-old-space-size=${capMb} wall=${wall}s ${String(run.stdout ?? '').trim()}`, - `Files=${pick(/^Files:\s+(\d+)/m)} LinesOfDefinitions=${pick(/^Lines of Definitions:\s+(\d+)/m)}`, - `Types=${pick(/^Types:\s+(\d+)/m)} Instantiations=${pick(/^Instantiations:\s+(\d+)/m)}`, - `MemoryUsed=${pick(/^Memory used:\s+([\d,]+K)/m)}`, - `CheckTime=${pick(/^Check time:\s+([\d.]+)s/m)}s TotalTime=${pick(/^Total time:\s+([\d.]+)s/m)}s`, - `MaximumRSS=${pick(/Maximum resident set size \(kbytes\):\s+(\d+)/m)} kB`, - `MemAvailable during run: min=${samples.length ? Math.min(...samples) : 'n/a'} kB max=${samples.length ? Math.max(...samples) : 'n/a'} kB samples=${samples.length}`, - `heapOOM=${/JavaScript heap out of memory/.test(`${stdout}${stderr}`)} errCount=${errs.length}`, - `stderrTail=${stderr.trim().split('\n').slice(-3).join(' | ').slice(0, 400)}`, - ].join('\n')); - } - rmSync(probeHolder, { force: true, recursive: true }); - process.exit(0); -} - if (process.argv.includes('--re-measure')) { // The ceiling FIRST, before the four minutes of tsc it shapes (#12856). Two // jobs, and the second is the one that keeps the constant honest: on CI this From bfcc67b2f071ceb2e78b33db2e6983bf5e0ce457 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 02:55:56 +0000 Subject: [PATCH 3/4] docs(tooling): record the runner measurement beside CI_TSC_HEAP_CEILING_MB MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pin's provenance was archaeology through a failed job's GC trace, which bracketed the runner's old space into [4040, 4148] MB. It is now a first-hand reading, taken where the verdict is taken -- inside the `Type Check · debt ledger` job, by a temporary probe step (reverted in the previous commit) that emitted its numbers as `::notice` annotations: runner ubuntu24 20260831.293.1, 4 vCPU, MemTotal 16,373,452 kB (~15.6 GiB) -- not the 7 GB the finding assumed consumers 153 processes / 940,316 kB (~918 MB); the job's steps are sequential, so nothing runs beside the re-measure this gate heap_size_limit 4144 MB with NODE_OPTIONS unset -- the runner's V8 default, confirming the 4096 MB old space directly heaviest qa/http-conformance's TEST_DEBT program under two caps: program 4096 -> 4,077,718K used, 4,212,904 kB peak RSS, 26.84s check 6144 -> 4,420,706K used, 4,545,500 kB peak RSS, 21.90s check The pair is the headroom reading the finding asked for: 343 MB more heap keeps 343 MB more live and finishes ~5s sooner, so under 4096 the program is paying GC pressure to fit. The constant does NOT move on it, and the measurement is why: the scarce resource is V8's default old space (4096 MB), not the runner's memory, and this number describes that default exactly. The comment also records what the measurement made mechanically visible -- raising the pin alone cannot deliver a roomier run. `remeasureHeapCeiling` minimises over the pin and the running process's own limit, so a 6144 pin under the runner's default still chooses 4144, and the `stale` arm then refuses the run outright: `--re-measure` exits 1 before the first tsc. Reproduced against a 4144 MB process. Delivering a raise needs the gate PROCESS given the memory first, which is a workflow decision and is escalated on #14569. The self-test row for "a box shaped like CI" gains a note that its `+ 48` is now the measured runner rather than a construction. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01WLJQhde67SeTccsmnBVarV --- scripts/check-type-check-coverage.mjs | 63 ++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/scripts/check-type-check-coverage.mjs b/scripts/check-type-check-coverage.mjs index 7a070e6fdd..2c8e511984 100644 --- a/scripts/check-type-check-coverage.mjs +++ b/scripts/check-type-check-coverage.mjs @@ -2799,10 +2799,54 @@ function countTscErrors(output, { dropRootDirDiagnostics = false } = {}) { // with `NODE_OPTIONS` on a box under this file's own eyes), so a 4096 old space // reports 4144 and commits the 4147.5 above it. // +// ## Re-measured FIRST-HAND on the runner, 2026-09-03 (#14569) +// +// The bracket above is archaeology through a failed job's GC trace. #14569 +// asked for a raise to 6144 to be taken on a measurement rather than on a +// typed number, so the reading was taken where the verdict is taken: inside +// the `Type Check · debt ledger` job itself, by a temporary probe step that +// emitted its numbers as `::notice` annotations (run 33708954003, job +// 100504131338, image `ubuntu24 20260831.293.1`, Node v22.23.2, 4 vCPU). +// +// the runner MemTotal 16,373,452 kB (~15.6 GiB) plus 3,145,724 kB of +// swap -- NOT the 7 GB #14569 assumed. MemAvailable at the +// point the re-measure starts: 14,329,064 kB. +// other consumers 153 processes holding 940,316 kB (~918 MB) altogether: +// Runner.Worker 144 MB, Runner.Listener 98 MB, provjobd +// 96 MB, dockerd 73 MB, containerd 43 MB. The job's steps +// are sequential, so nothing in it runs BESIDE the +// re-measure -- the ledger's tsc has the box to itself. +// this gate's own `heap_size_limit` 4144 MB with `NODE_OPTIONS` unset -- +// ceiling the runner's V8 default, read directly rather than +// inferred. It confirms the 4096 MB old space the GC trace +// above could only bracket. +// the heaviest `packages/qa/http-conformance`'s TEST_DEBT program (906 +// program files, 692,003 lines of definitions, 7,328,937 +// instantiations) under `--extendedDiagnostics`, twice: +// +// cap 4096 Memory used 4,077,718K peak RSS 4,212,904 kB +// check 26.84s +// cap 6144 Memory used 4,420,706K peak RSS 4,545,500 kB +// check 21.90s +// +// Neither OOMs, and the pair IS the headroom finding +// #14569 asked for: handed 343 MB more heap the same +// program keeps 343 MB more live and finishes ~5s sooner, +// so under 4096 it is paying GC pressure to fit rather +// than fitting. Lowest MemAvailable seen at any point +// during either run: 10,562,192 kB. +// +// The scarce resource is therefore NOT the runner's memory -- 15.6 GiB with +// ~918 MB of it spoken for -- but V8's DEFAULT old space on that runner, which +// is 4096 MB. This constant describes that default, and as of 2026-09-03 it +// still describes it exactly. That is why the re-measure leaves it here. +// // ⚠️ If 4096 is wrong, it is wrong DOWNWARD -- the only safe direction. This // number's entire job is to be no HIGHER than CI's ceiling. A pin ABOVE CI's is // worse than no pin at all: it makes local runs pass where CI still OOMs, which -// is exactly this defect with extra confidence attached. +// is exactly this defect with extra confidence attached. The 2026-09-03 +// reading above is the first taken with the runner in hand rather than +// inferred from a crash, and it lands on the same 4096 from the other side. // // ⛔ Do not raise this to make a local measurement complete. `--re-measure` // OOMing under this ceiling is the gate WORKING -- it is CI's failure, @@ -2811,6 +2855,18 @@ function countTscErrors(output, { dropRootDirDiagnostics = false } = {}) { // lesson from the build side: a ceiling above the box's real memory does not // buy a bigger run, it converts a recoverable heap error into an exit-137 // SIGKILL that carries no diagnostic at all.) +// +// ⛔ And raising it ALONE cannot buy the ledger a roomier run -- measured on +// 2026-09-03, not reasoned. `remeasureHeapCeiling` below takes the MINIMUM of +// this pin and the limit the running process actually has, so with the pin at +// 6144 and the gate started under the runner's own default the chosen ceiling +// is still 4144 -- and the `stale` arm below then refuses the run outright: +// `--re-measure` exits 1 before the first tsc ("the pin is now ABOVE the +// ceiling it claims to describe"), reproduced against a 4144 MB process. A +// raise has to hand the gate PROCESS the memory first -- a `NODE_OPTIONS` on +// the job's re-measure step -- so the pin keeps describing what the process +// really has. That is a workflow decision, not one this constant can take on +// its own; #14569 carries it. const CI_TSC_HEAP_CEILING_MB = 4096; /** @@ -5156,6 +5212,11 @@ function selfTest() { expect: { mb: CI_TSC_HEAP_CEILING_MB, stale: false }, }, { + // `+ 48` is the RUNNER, not a construction: the `Type Check · debt + // ledger` job reports a `heap_size_limit` of 4144 MB for its 4096 MB old + // space (measured there 2026-09-03, #14569), so this row is the shape of + // the machine whose verdict the pin exists to describe -- and the row + // above it is every box that is roomier than that one. label: 'on a box shaped like CI the ceiling is a no-op that still names itself', where: { heapLimitMb: CI_TSC_HEAP_CEILING_MB + 48, onCi: true }, expect: { mb: CI_TSC_HEAP_CEILING_MB, stale: false }, From a68aed7ecebd12aa0dc43045c9af2ea5bb699912 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 3 Sep 2026 23:30:45 +0000 Subject: [PATCH 4/4] fix(tooling): pin the re-measure heap ceiling at 6144 and give the job the memory MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ruling A1 (#14569, 2026-09-03). The 02:25Z ruling A — raise on a recorded measurement — is completed the only way the measurement allows: the pin and the process's actual old space move together. - .github/workflows/lint.yml: the `typecheck-debt` job's re-measure step now runs under `NODE_OPTIONS: --max-old-space-size=6144`, so the process running tsc really has the old space the pin describes. V8's default there is 4096 MB, measured on the runner. - CI_TSC_HEAP_CEILING_MB: 4096 -> 6144, with the runner measurement already written beside it kept as the evidence. - Two new `remeasureHeapCeiling` self-test rows pin both directions of the pairing: the runner as the workflow now starts it (6192 reported, caller cap tying the pin, chosen ceiling 6144 named as the CI pin), and the same runner WITHOUT the workflow line (its measured 4144 MB default, refused). The `stale` arm is untouched and still refuses any pin above the process's own limit — that refusal is what caught the bare constant raise, and it is what keeps the two halves inseparable. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012zGPuVVX3deAx9LdjK8jCk --- .github/workflows/lint.yml | 26 ++++++ scripts/check-type-check-coverage.mjs | 115 +++++++++++++++++++------- 2 files changed, 113 insertions(+), 28 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9ba5a50d82..2391c1133a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -4808,7 +4808,33 @@ jobs: - name: Build the ledgered packages' dependencies run: pnpm exec turbo run build --filter='./packages/*' --filter='./packages/*/*' + # ⚠️ `NODE_OPTIONS` here is HALF of the heap ceiling this gate runs tsc + # under, and the two halves ship together or not at all (#14569, ruled + # 2026-09-03). The other half is `CI_TSC_HEAP_CEILING_MB` in + # scripts/check-type-check-coverage.mjs, which is a DESCRIPTION of the old + # space the process running tsc really has -- and the gate chooses the + # MINIMUM of that pin and this process's actual limit. So without this + # line the pin buys nothing (V8's default old space on this runner is + # 4096 MB, measured) and the gate's `stale` arm refuses the run outright, + # before the first tsc, on every PR and on `main`. That refusal is the + # pairing's enforcement, and it is deliberate. + # + # Why 6144 and not more, measured on this runner rather than reasoned + # (run 33708954003, job 100504131338, 2026-09-03): the heaviest ledger + # program keeps 4,077,718K live under a 4096 cap and 4,420,706K under + # 6144 while finishing ~5s sooner -- i.e. under 4096 it was paying GC + # pressure to fit -- and 10,562,192 kB of the box's 16,373,452 kB stayed + # available at the tightest moment of either run. The raise is headroom + # the box really has, not a promise it cannot keep; a ceiling above the + # box's real memory buys nothing and converts a recoverable heap error + # into an exit-137 SIGKILL with no diagnostic. + # + # ⛔ Do not change this number without changing the constant, or the + # reverse. The constant's comment carries the full reading and both + # directions are pinned in that file's `--self-test`. - name: Re-measure the type-check DEBT / TEST_DEBT ledger + env: + NODE_OPTIONS: --max-old-space-size=6144 run: pnpm check:type-check-debt # Lane 4 of 4 behind the required `TypeScript Type Check` context. The diff --git a/scripts/check-type-check-coverage.mjs b/scripts/check-type-check-coverage.mjs index 89615330a9..a3f7431aa9 100644 --- a/scripts/check-type-check-coverage.mjs +++ b/scripts/check-type-check-coverage.mjs @@ -2792,7 +2792,7 @@ function countTscErrors(output, { dropRootDirDiagnostics = false } = {}) { // same tree. ⚠️ The asymmetry IS the defect: a local pass was never a claim // about CI, and nothing said so out loud. // -// ## Where the number comes from -- CI, never this box +// ## Where the runner's DEFAULT old space comes from -- CI, never this box // // Read off the CI runner itself: run 33136681083, job `Type Check · debt // ledger`, at 6d097a604, Node v22.23.2. The `packages/qa/http-conformance` @@ -2851,15 +2851,44 @@ function countTscErrors(output, { dropRootDirDiagnostics = false } = {}) { // // The scarce resource is therefore NOT the runner's memory -- 15.6 GiB with // ~918 MB of it spoken for -- but V8's DEFAULT old space on that runner, which -// is 4096 MB. This constant describes that default, and as of 2026-09-03 it -// still describes it exactly. That is why the re-measure leaves it here. +// the reading above pins at 4096 MB from two directions. // -// ⚠️ If 4096 is wrong, it is wrong DOWNWARD -- the only safe direction. This -// number's entire job is to be no HIGHER than CI's ceiling. A pin ABOVE CI's is -// worse than no pin at all: it makes local runs pass where CI still OOMs, which -// is exactly this defect with extra confidence attached. The 2026-09-03 -// reading above is the first taken with the runner in hand rather than -// inferred from a crash, and it lands on the same 4096 from the other side. +// ## The raise, on that measurement (#14569, ruled A then A1, 2026-09-03) +// +// A default is not a budget. The ledger's heaviest program was paying GC +// pressure to fit inside 4096 rather than fitting, and the tripwire (spec +// declaration growth) is a weekly event, so the ruling raises the ceiling -- +// on the measurement above, never on a typed number. What that raise is NOT +// is a bigger promise about the box. It is the pair below, and ⛔ neither +// half is shippable alone: +// +// the workflow `.github/workflows/lint.yml`, job `typecheck-debt`, step +// "Re-measure the type-check DEBT / TEST_DEBT ledger", now +// runs under `NODE_OPTIONS: --max-old-space-size=6144`. That +// is the half that actually hands the process the old space: +// V8's default there is 4096 and no constant in this file can +// move it. +// this constant 6144 -- a description of the old space that step now +// really has, exactly as 4096 described the default before it. +// +// ⛔ Raising this constant ALONE cannot buy the ledger a roomier run -- +// measured on 2026-09-03, not reasoned. `remeasureHeapCeiling` below takes the +// MINIMUM of this pin and the limit the running process actually has, so with +// the pin at 6144 and the gate started under the runner's DEFAULT the chosen +// ceiling is still 4144 -- and the `stale` arm below then refuses the run +// outright: `--re-measure` exits 1 before the first tsc ("the pin is now ABOVE +// the ceiling it claims to describe"), on every PR and on `main`. That +// refusal is the pairing's enforcement -- it is what caught the bare raise +// when it was attempted -- and both directions are pinned as self-test rows +// below ("the runner as the workflow now starts it" and "the same runner +// WITHOUT it"). Delete the `NODE_OPTIONS` line and the lane says so, loudly, +// on the runner. +// +// ⚠️ If 6144 is wrong, it is wrong DOWNWARD -- the only safe direction. This +// number's entire job is to be no HIGHER than the ceiling the process running +// tsc on CI really has. A pin ABOVE it is worse than no pin at all: it makes +// local runs pass where CI still OOMs, which is exactly this defect with extra +// confidence attached. // // ⛔ Do not raise this to make a local measurement complete. `--re-measure` // OOMing under this ceiling is the gate WORKING -- it is CI's failure, @@ -2867,20 +2896,21 @@ function countTscErrors(output, { dropRootDirDiagnostics = false } = {}) { // memory CI has. (`packages/spec/tsup.config.ts` carries the other half of this // lesson from the build side: a ceiling above the box's real memory does not // buy a bigger run, it converts a recoverable heap error into an exit-137 -// SIGKILL that carries no diagnostic at all.) -// -// ⛔ And raising it ALONE cannot buy the ledger a roomier run -- measured on -// 2026-09-03, not reasoned. `remeasureHeapCeiling` below takes the MINIMUM of -// this pin and the limit the running process actually has, so with the pin at -// 6144 and the gate started under the runner's own default the chosen ceiling -// is still 4144 -- and the `stale` arm below then refuses the run outright: -// `--re-measure` exits 1 before the first tsc ("the pin is now ABOVE the -// ceiling it claims to describe"), reproduced against a 4144 MB process. A -// raise has to hand the gate PROCESS the memory first -- a `NODE_OPTIONS` on -// the job's re-measure step -- so the pin keeps describing what the process -// really has. That is a workflow decision, not one this constant can take on -// its own; #14569 carries it. -const CI_TSC_HEAP_CEILING_MB = 4096; +// SIGKILL that carries no diagnostic at all.) The 6144 is not an exception to +// that rule, it is an application of it: the runner was MEASURED to carry the +// heaviest program under a 6144 cap (4,420,706K used, 4,545,500 kB peak RSS, +// 10,562,192 kB still available at the tightest moment) before it was pinned. +// +// ⚠️ One protection the pair costs, recorded here so nobody rediscovers it as +// a surprise. With `NODE_OPTIONS` set explicitly on that step, +// `heap_size_limit` there reads 6192 whatever the runner's physical memory +// does -- so on THAT job the `stale` arm can no longer notice the runner +// shrinking; it now only notices a pin above a DEFAULTED process. The margin +// is what makes that acceptable: the pin asks for 6144 MB where the +// measurement found 10,562,192 kB available at the heaviest moment, ~1.7x. If +// that margin is ever in doubt the answer is a fresh runner measurement and a +// smaller number in BOTH places, ⛔ never a bigger one here. +const CI_TSC_HEAP_CEILING_MB = 6144; /** * The last `--max-old-space-size` in a `NODE_OPTIONS` string, in MB, or null. @@ -5225,15 +5255,44 @@ function selfTest() { expect: { mb: CI_TSC_HEAP_CEILING_MB, stale: false }, }, { - // `+ 48` is the RUNNER, not a construction: the `Type Check · debt - // ledger` job reports a `heap_size_limit` of 4144 MB for its 4096 MB old - // space (measured there 2026-09-03, #14569), so this row is the shape of - // the machine whose verdict the pin exists to describe -- and the row - // above it is every box that is roomier than that one. + // `+ 48` is the RUNNER's offset, not a construction: V8 reports the old + // space plus a fixed ~48 MB of other spaces, measured on the `Type Check + // · debt ledger` job itself (4144 for a 4096 old space, 2026-09-03, + // #14569). So this row is the machine whose limit EQUALS the pin with no + // caller flag in play -- and the row above it is every box roomier than + // that one. label: 'on a box shaped like CI the ceiling is a no-op that still names itself', where: { heapLimitMb: CI_TSC_HEAP_CEILING_MB + 48, onCi: true }, expect: { mb: CI_TSC_HEAP_CEILING_MB, stale: false }, }, + { + // THE RUNNER AS THE WORKFLOW NOW STARTS IT (#14569). `lint.yml`'s + // re-measure step sets `NODE_OPTIONS: --max-old-space-size=6144`, so the + // gate process reports 6192 AND carries a caller cap EQUAL to the pin. + // Both candidates tie, the tie-break keeps the CI ceiling's name, and + // that name is what the job's log then prints. Pinned because an + // off-by-one in either direction here reads as a caller cap overriding + // the pin on the one machine whose verdict counts. + label: "the workflow's own NODE_OPTIONS ties the pin and is not read as a tighter caller cap", + where: { + heapLimitMb: CI_TSC_HEAP_CEILING_MB + 48, + nodeOptions: `--max-old-space-size=${CI_TSC_HEAP_CEILING_MB}`, + onCi: true, + }, + expect: { mb: CI_TSC_HEAP_CEILING_MB, stale: false }, + }, + { + // THE OTHER HALF OF THE PAIR, and the row that keeps the two halves + // inseparable. 4144 is the runner's DEFAULT `heap_size_limit`, measured + // on that job 2026-09-03. Take the `NODE_OPTIONS` line back out of + // `lint.yml` and this is the reading the gate gets: refused outright, + // before the first tsc, on every PR and on `main`. A bare raise of the + // constant was attempted and this is what caught it, so the pin above + // cannot quietly outlive the workflow line that pays for it. + label: 'the same runner WITHOUT the workflow NODE_OPTIONS -- its 4144 MB default -- is refused', + where: { heapLimitMb: 4144, onCi: true }, + expect: { mb: 4144, stale: true }, + }, { // Never RAISE. Promising V8 memory the box does not have trades a // recoverable heap error for a kernel SIGKILL that says nothing.