diff --git a/.changeset/5924-live-headroom-invariant.md b/.changeset/5924-live-headroom-invariant.md new file mode 100644 index 0000000000..11faf78eb2 --- /dev/null +++ b/.changeset/5924-live-headroom-invariant.md @@ -0,0 +1,31 @@ +--- +--- + +Build tooling and CI only — `scripts/check-eager-closure-budget.mjs`, its unit +test, and a comment in `.github/workflows/performance-budget.yml`. Nothing ships +from this change. + +The console eager-closure gate stated a binding constraint on its own +sensitivity — the headroom above the measured payload must stay SMALLER than the +89 KiB regression the gate exists to catch — and then checked it between two +constants frozen in the same module. That assertion is an arithmetic fact about +the file, true regardless of what the console weighs. The closure shrank 706,013 +gzipped bytes below the pinned baseline without the ceiling following it down, +the live headroom reached 8.6x the regression size, and the check that was +supposed to notice stayed green throughout: a demonstrated +158,006-byte eager +regression, 1.7x the incident this gate was built for, passed with a green tick. + +`evaluateHeadroomSensitivity` now derives that headroom from the report the gate +just read, for the aggregate ceiling and for each of the three per-chunk +ceilings, and treats a ceiling that has drifted more than one regression above +its own measurement as an ERROR (exit 2) rather than a size failure — it is a +verdict about the gauge, the same asymmetry a budgeted chunk absent from the +report already carried. The constant-vs-constant assertions stay as a secondary +guard. + +The aggregate ceiling is re-baselined downward as the decision this records: +`MAX_EAGER_CLOSURE_GZIP_BYTES` 4,086,000 to 3,345,000 over a `BASELINE` moving +4,005,911 (`4c1623c0c`) to 3,299,898 (`48e53814e`). Headroom goes from 8.63x the +regression size to 0.49x. Lowering a ceiling toward reality is a tightening: no +build that passed before and measures under the new line fails after it. The +floor is unchanged — a ceiling is never put below a measured figure. diff --git a/.github/workflows/performance-budget.yml b/.github/workflows/performance-budget.yml index ee94b4d1d8..330b52b235 100644 --- a/.github/workflows/performance-budget.yml +++ b/.github/workflows/performance-budget.yml @@ -88,9 +88,14 @@ jobs: # `scripts/check-eager-closure-budget.mjs` next to the reasoning that # produced it, so a re-baseline is one edit against a documented # argument rather than a number in YAML: today's measured payload plus - # ~2% of headroom, chosen to pass on today's `main` while staying - # NARROWER than the 89 KiB regression the gate exists to catch. It is a - # truthful current-state ceiling, not a statement that 3.79 MB is fine. + # half the 89 KiB regression the gate exists to catch, which keeps the + # headroom NARROWER than that regression while leaving as much room for + # the payload to shrink as to grow. Since objectui#5924 the checker + # enforces that itself, against the report it just read rather than + # against a frozen constant, for the aggregate ceiling and each + # per-chunk one — a ceiling that has drifted out of range of the + # regression it must catch is exit 2, a verdict about the gauge. It is + # a truthful current-state ceiling, not a statement that 3.15 MB is fine. # # Both measurements run before either may fail the step: a run that # reports one number and hides the other teaches readers to distrust diff --git a/scripts/__tests__/check-eager-closure-budget.test.ts b/scripts/__tests__/check-eager-closure-budget.test.ts index da6ccf8126..06278ef6b5 100644 --- a/scripts/__tests__/check-eager-closure-budget.test.ts +++ b/scripts/__tests__/check-eager-closure-budget.test.ts @@ -15,6 +15,7 @@ import { REGRESSION_THIS_GATE_MUST_CATCH_BYTES, SUPPORTED_REPORT_VERSION, evaluateClosureBudget, + evaluateHeadroomSensitivity, evaluatePerChunkBudgets, main, measureChunksByName, @@ -89,6 +90,17 @@ describe('the ceiling itself', () => { expect(evaluateClosureBudget({ report: closureOf(BASELINE.gzipBytes) }).status).toBe('pass'); }); + /** + * ⚠️ This pair of assertions is the SECONDARY guard, and objectui#5924 is the + * record of what it cannot do. Both operands are literals frozen in the + * checker, so it is true regardless of what the console weighs: it stayed + * green while the closure fell ~706 KB below the pinned baseline and the live + * headroom reached 8.6x the regression size. It is kept because it still + * catches the one thing it can — an edit that raises a ceiling past the + * regression size, with no build in sight — and it is no longer the only + * check of this invariant. The live one is `evaluateHeadroomSensitivity`, + * exercised further down. + */ it('would have failed on the regression it exists to catch', () => { const headroom = MAX_EAGER_CLOSURE_GZIP_BYTES - BASELINE.gzipBytes; expect(headroom).toBeLessThan(REGRESSION_THIS_GATE_MUST_CATCH_BYTES); @@ -356,6 +368,185 @@ describe('per-chunk ceilings', () => { }); }); +/** + * objectui#5924 — the headroom invariant, checked against the report the gate + * just read instead of against two literals frozen beside it. + * + * The invariant itself is old and stated in the checker's header: the headroom + * above the measurement must stay SMALLER than the regression the gate exists + * to catch, or a repeat of that regression fits inside it and passes. What was + * new in objectui#5924 is where it was checked. `MAX_EAGER_CLOSURE_GZIP_BYTES - + * BASELINE.gzipBytes < REGRESSION_...` is an arithmetic fact about the module, + * true forever once written, and it stayed true while the console shrank ~706 + * KB underneath it — leaving a demonstrated +158,006-byte eager regression + * green, 1.7x the incident the gate was built for. + * + * These tests are about the four ceilings this file now ships (the aggregate + * plus the three per-chunk lines objectui#5490 added), each weighed against its + * own measurement in the report. + */ +describe('ceiling sensitivity, judged live (objectui#5924)', () => { + /** + * A v2 report totalling exactly `totalGzipBytes`, carrying the budgeted + * chunks at their measured sizes and the remainder of the closure as one + * filler chunk — the shape a real report has, where the budgeted names are a + * minority of the total. + */ + function sensitivityReport(totalGzipBytes: number, sizes: Record = {}) { + const measured: Record = { ...PER_CHUNK_BASELINE, ...sizes }; + const named = [ + { fileName: 'assets/index-A.js', name: 'index', bytes: 0, gzipBytes: 25_910 }, + ...Object.entries(measured).map(([name, gzipBytes]) => ({ + fileName: `assets/${name}-hash.js`, + name, + bytes: 0, + gzipBytes, + })), + ]; + const files = [ + ...named, + { + fileName: 'assets/rest-of-closure.js', + name: 'rest-of-closure', + bytes: 0, + gzipBytes: totalGzipBytes - named.reduce((n, f) => n + f.gzipBytes, 0), + }, + ]; + return report({ + files, + eagerChunkCount: files.length, + eagerGzipBytes: totalGzipBytes, + eagerRawBytes: 0, + }); + } + + /** The ceiling this file shipped before objectui#5924 re-baselined it. */ + const CEILING_BEFORE_5924 = 4_086_000; + + it('reds on the drift objectui#5924 recorded: 8.6x the regression above the live payload', () => { + const result = evaluateHeadroomSensitivity({ + report: sensitivityReport(BASELINE.gzipBytes), + budgetBytes: CEILING_BEFORE_5924, + }); + expect(result.status).toBe('error'); + expect(result.blind).toEqual(['aggregate']); + expect(result.message).toContain('DRIFTED'); + // The multiple, so the failure states HOW blind rather than merely that it is. + expect(result.message).toContain('8.63x'); + // ...and the constant to lower, so the fix is one named edit. + expect(result.message).toContain('MAX_EAGER_CLOSURE_GZIP_BYTES'); + }); + + it('is the check the frozen-constant assertion structurally could not be', () => { + // Same moment, same payload, the assertion that was supposed to guard it: + // the two constants of the day satisfied it comfortably, which is why the + // suite was green through the run above. + expect(CEILING_BEFORE_5924 - 4_005_911).toBeLessThan(REGRESSION_THIS_GATE_MUST_CATCH_BYTES); + }); + + it('passes on the constants and the measurement this file ships today', () => { + const result = evaluateHeadroomSensitivity({ report: sensitivityReport(BASELINE.gzipBytes) }); + expect(result.status).toBe('pass'); + expect(result.blind).toEqual([]); + // Every ceiling in the file is weighed, not just the aggregate one: the + // population objectui#5490 grew to four is the population judged here. + expect(result.sites.map((site) => site.key)).toEqual([ + 'aggregate', + ...Object.keys(PER_CHUNK_GZIP_CEILINGS), + ]); + // A passing run still prints every measurement, so a reader watching a + // ceiling drift upward sees it coming rather than the day it reds. + expect(result.message).toContain('3222.6'); + }); + + it('is exactly one regression wide, from either side of the line', () => { + const atTheLine = MAX_EAGER_CLOSURE_GZIP_BYTES - REGRESSION_THIS_GATE_MUST_CATCH_BYTES; + expect(evaluateHeadroomSensitivity({ report: sensitivityReport(atTheLine + 1) }).status).toBe( + 'pass', + ); + expect(evaluateHeadroomSensitivity({ report: sensitivityReport(atTheLine) }).status).toBe( + 'error', + ); + }); + + it('judges the per-chunk ceilings too — the population is four ceilings, not one', () => { + const result = evaluateHeadroomSensitivity({ + report: sensitivityReport(BASELINE.gzipBytes, { + framework: PER_CHUNK_GZIP_CEILINGS.framework - REGRESSION_THIS_GATE_MUST_CATCH_BYTES, + }), + }); + expect(result.status).toBe('error'); + expect(result.blind).toEqual(['framework']); + expect(result.message).toContain("PER_CHUNK_GZIP_CEILINGS['framework']"); + }); + + it('leaves a ceiling BELOW the payload to the size verdict, and says so', () => { + // Negative headroom is an over-budget bundle. Reporting it here as well + // would turn one regression into an error and teach a reader that exit 2 + // does not mean what the file says it means. + const result = evaluateHeadroomSensitivity({ + report: sensitivityReport(MAX_EAGER_CLOSURE_GZIP_BYTES + 500_000), + }); + expect(result.status).toBe('pass'); + expect(result.message).toContain('the size verdict owns this row'); + }); + + it('errors when there is no report — a ceiling with no measurement is not sensitive', () => { + const result = evaluateHeadroomSensitivity({ report: null }); + expect(result.status).toBe('error'); + expect(result.message).toContain('broken gauge'); + }); + + it('errors on a report it cannot trust rather than judging drift from a bad number', () => { + const result = evaluateHeadroomSensitivity({ + report: { ...sensitivityReport(BASELINE.gzipBytes), reportVersion: 1 }, + }); + expect(result.status).toBe('error'); + expect(result.blind).toEqual([]); + }); + + it('refuses to judge a ceiling whose chunk is absent, instead of reading it as drifted', () => { + // The wrong-reason trap: a budgeted chunk that is not in the report weighs + // zero, so its whole ceiling would look like headroom — "drifted", the + // right exit code for the wrong reason, on a run the per-chunk half already + // explains correctly. + const base = sensitivityReport(BASELINE.gzipBytes); + const files = base.files.filter((f) => f.name !== 'ui-components'); + const result = evaluateHeadroomSensitivity({ + report: report({ + files, + eagerChunkCount: files.length, + eagerGzipBytes: files.reduce((n, f) => n + f.gzipBytes, 0), + eagerRawBytes: 0, + }), + }); + expect(result.status).toBe('error'); + expect(result.blind).toEqual([]); + expect(result.message).toContain('ui-components'); + expect(result.message).toContain('absent'); + }); + + /** + * The acceptance test of objectui#5924, pinned so it cannot quietly come + * undone. Measured on `48e53814e`: an eager `@objectstack/spec/cloud` + * namespace import into `apps/console/src/main.tsx` — a use the bundler + * cannot fold away — put 158,006 gzipped bytes into the eager closure, and + * removing it returned the measurement to 3,299,898 exactly, so the movement + * is the injection and not build noise. + */ + it('the demonstrated regression: green under the old ceiling, red under the new one', () => { + const INJECTED = BASELINE.gzipBytes + 158_006; + // 1.7x the incident this gate was built to catch. + expect(INJECTED - BASELINE.gzipBytes).toBeGreaterThan(REGRESSION_THIS_GATE_MUST_CATCH_BYTES); + + const injected = sensitivityReport(INJECTED); + expect( + evaluateClosureBudget({ report: injected, budgetBytes: CEILING_BEFORE_5924 }).status, + ).toBe('pass'); + expect(evaluateClosureBudget({ report: injected }).status).toBe('fail'); + }); +}); + describe('renderTopChunks', () => { it('names the biggest eager chunks so a failure has suspects', () => { const lines = renderTopChunks(report(), 2).split('\n'); @@ -401,8 +592,8 @@ describe('main', () => { const { code, outputs } = run(budgeted()); expect(code).toBe(0); expect(outputs.closure_status).toBe('pass'); - expect(outputs.closure_chunks).toBe('4'); - expect(outputs.closure_gzip_kb).toBe('1814.3'); + expect(outputs.closure_chunks).toBe('5'); + expect(outputs.closure_gzip_kb).toBe('3222.6'); }); it('exits 1 — a verdict about the BUNDLE — when over budget', () => { @@ -438,10 +629,21 @@ describe('main', () => { expect(outputs.closure_gzip_kb).not.toBe(''); }); - /** A v2 report at the measured per-chunk sizes, well inside the aggregate. */ - function budgeted(sizes: Record = {}) { + /** + * A v2 report at the measured per-chunk sizes, totalling `BASELINE.gzipBytes` + * plus `totalDelta`. + * + * ⚠️ The filler chunk is not padding. Before objectui#5924 this fixture + * carried only the budgeted names, so its total was ~1.8 MB against a 3.3 MB + * ceiling — a shape `main` now (correctly) calls a BLIND ceiling and exits 2 + * on. A report whose total sits far below the aggregate line is not a + * within-budget bundle to be asserted `pass`; it is the defect this card + * fixed. So the fixture carries the rest of the closure, as a real report + * does, and `totalDelta` is how a test moves the total on purpose. + */ + function budgeted(sizes: Record = {}, totalDelta = 0) { const measured: Record = { ...PER_CHUNK_BASELINE, ...sizes }; - const files = [ + const named = [ { fileName: 'assets/index-A.js', name: 'index', bytes: 0, gzipBytes: 25_910 }, ...Object.entries(measured).map(([name, gzipBytes]) => ({ fileName: `assets/${name}-hash.js`, @@ -450,6 +652,19 @@ describe('main', () => { gzipBytes, })), ]; + // Derived from the BASELINE sizes, not from `named`, so an override in + // `sizes` moves the total the way a real chunk growing would. + const baselineNamed = + 25_910 + Object.values(PER_CHUNK_BASELINE).reduce((n, bytes) => n + bytes, 0); + const files = [ + ...named, + { + fileName: 'assets/rest-of-closure.js', + name: 'rest-of-closure', + bytes: 0, + gzipBytes: BASELINE.gzipBytes - baselineNamed + totalDelta, + }, + ]; return report({ files, eagerChunkCount: files.length, @@ -497,6 +712,35 @@ describe('main', () => { expect(outputs.closure_chunk_status).toBe('error'); }); + /** + * objectui#5924: the run that used to be the file's blind spot. Both size + * halves are delighted — nothing is over any line — and the gate still has to + * stop, because neither of those green ticks means anything at this distance. + */ + it('exits 2 when a ceiling has drifted out of range of the regression it must catch', () => { + const { code, outputs } = run(budgeted({}, -REGRESSION_THIS_GATE_MUST_CATCH_BYTES)); + expect(code).toBe(2); + expect(outputs.closure_status).toBe('pass'); + expect(outputs.closure_chunk_status).toBe('pass'); + expect(outputs.closure_headroom_status).toBe('error'); + }); + + /** + * The ordering objectui#5490 established over two halves, held over three: + * `error` outranks `fail`, whichever half noticed. `performance-budget.yml` + * maps exit 2 to `budget_status=error` and any other non-zero to `fail`, so + * collapsing this to 1 would report a gauge that cannot be trusted as a size + * regression. + */ + it('reports the GAUGE verdict when one ceiling is blind and another is over', () => { + const { code, outputs } = run( + budgeted({ 'vendor-objectstack': PER_CHUNK_GZIP_CEILINGS['vendor-objectstack'] + 1 }, -200_000), + ); + expect(code).toBe(2); + expect(outputs.closure_chunk_status).toBe('fail'); + expect(outputs.closure_headroom_status).toBe('error'); + }); + it('exits 2 on a report from a build that predates per-chunk names', () => { const base = budgeted(); const { code, outputs } = run({ ...base, reportVersion: 1 }); diff --git a/scripts/check-eager-closure-budget.mjs b/scripts/check-eager-closure-budget.mjs index 484c0ea05a..43704042ba 100644 --- a/scripts/check-eager-closure-budget.mjs +++ b/scripts/check-eager-closure-budget.mjs @@ -41,19 +41,66 @@ * * ## The ceiling, and why it is this number * - * `MAX_EAGER_CLOSURE_GZIP_BYTES` is today's measurement plus ~2% of headroom. - * Two constraints pin it from both sides: + * `MAX_EAGER_CLOSURE_GZIP_BYTES` is today's measurement plus headroom of half + * {@link REGRESSION_THIS_GATE_MUST_CATCH_BYTES}. Two constraints pin it from + * both sides, and since objectui#5924 BOTH are checked against the report this + * gate just read, not against a literal frozen next to them: * * - It must PASS on today's `main`. A gate that lands red is a gate someone * disables, and this one replaced a gate nobody could fail. Headroom above - * the current 4,005,911 bytes: 80,089 (2.00%). + * the current 3,299,898 bytes: 45,102 (1.37%). * - The headroom must stay SMALLER than the regression the gate exists to - * catch. objectui#5266 was 89 KiB = 91,136 bytes; 80,089 < 91,136, so this + * catch. objectui#5266 was 89 KiB = 91,136 bytes; 45,102 < 91,136, so this * ceiling would have failed on that change. Widening the headroom past ~89 * KiB would leave the gate green through a repeat of its own motivating * incident. * - * ## Why this number moved once (objectui#5328, maintainer ruling on #5531) + * Half the regression size, rather than the ~2% this line used to carry, is a + * deliberate choice that only became necessary once the second constraint + * started being enforced live (next section). Headroom H buys H bytes of growth + * before the gate reds for being over budget, and costs REGRESSION - H bytes of + * SHRINK before it reds for going blind. H = REGRESSION / 2 is the only value + * equidistant from the two, and it is the value that maximises the smaller of + * the two distances: ~45 KB of room in each direction rather than 66 KB one way + * and 25 KB the other. + * + * ## Why the headroom is checked LIVE (objectui#5924) + * + * The second constraint above used to be asserted only in the unit test, as + * + * MAX_EAGER_CLOSURE_GZIP_BYTES - BASELINE.gzipBytes < REGRESSION_... + * + * Both operands are frozen literals from this module, so that assertion was + * true regardless of what the console actually weighed, and it stayed true + * while the closure got ~706 KB SMALLER than the pinned baseline. The invariant + * was STATED about the live bundle and CHECKED about two constants; it would + * have stayed green if the closure halved again. + * + * What that cost, demonstrated rather than inferred (objectui#5924): with the + * ceiling at 4,086,000 over a live 3.3 MB payload, an eager + * `@objectstack/spec/cloud` namespace import into `apps/console/src/main.tsx` + * added 158,006 gzipped bytes to the closure — 1.7x the incident this gate was + * built to catch — and the aggregate half printed a green tick with "headroom: + * 613.4 KB" underneath it. + * + * {@link evaluateHeadroomSensitivity} now derives that headroom from the report, + * for the aggregate ceiling AND for every per-chunk ceiling — four ceilings as + * of objectui#5490 — and calls a ceiling that sits more than one regression + * above its own measurement an ERROR (exit 2). That verdict is about the GAUGE, + * not the bundle, which is why it is an error and not a size failure: a green + * tick that cannot distinguish "no regression" from "the motivating incident, + * twice over" carries no information. It is the same shape as a budget keyed on + * a chunk that is not there, and the same rule applies — measuring nothing must + * be LOUDER than measuring something over the line, never quieter. + * + * The consequence is deliberate: drift in the SHRINKING direction is no longer + * free. A PR that takes more than ~45 KB out of the closure now has to re-pin + * the ceiling it just made decorative, in the same commit, instead of leaving a + * decision that silently comes due and is never taken. The constant-vs-constant + * assertions stay in the unit test as a secondary guard: they still catch an + * edit that raises a ceiling past the regression size without any build. + * + * ## Why this number has moved (objectui#5328 up, objectui#5924 down) * * It was 3,960,000 over a 3,881,609 baseline measured on `77f846a8b`. Pinning * `@objectstack/spec` and its three siblings to 17.1.0 put the closure 41,689 @@ -70,16 +117,33 @@ * the ceiling alone leaves headroom at ~200 KB and fails the test below, which * is the guard working, not an obstacle to route around. * + * objectui#5924 then lowered it to 3,345,000 over 3,299,898 bytes measured on + * `48e53814e`: 706,013 bytes BELOW the `4c1623c0c` baseline the previous ceiling + * was derived from. Nothing was cleaved to earn that — the closure shrank on its + * own while the ceiling stayed put, which is exactly the drift that opened the + * blind band above. Lowering a ceiling TOWARD reality is a tightening, not a + * weakening: no build that passed before the change and measures under 3,345,000 + * fails after it, and the gate's sensitivity is once again larger than the + * headroom it guards. This was taken as a card's stated decision (objectui#5924, + * triage disposition 3) rather than silently, which is what the "Raising it" + * note below asks of a re-baseline in either direction. + * + * ⛔ The floor is unchanged and applies to a LOWERING too: never put a ceiling + * below a measured figure to express an aspiration. That is not a tighter + * ratchet, it is a gate that lands red on `main`, which is how a budget gets + * switched off rather than met. + * * What did NOT move: {@link REGRESSION_THIS_GATE_MUST_CATCH_BYTES}. That is the * gate's sensitivity, the ruling did not touch it, and re-baselining must never * become an excuse to widen it — a ceiling that rises while the sensitivity * relaxes is a gate quietly retiring itself. * - * This is a truthful CURRENT-STATE ceiling, not a target. 3.8 MB gzipped before - * first render is a bad payload, and the honest long-term line is far below it — - * but lowering the line is a separate decision with its own work behind it - * (objectui#5324 names the candidates). Nothing here should be read as a - * finding that 3.82 MB is acceptable. + * This is a truthful CURRENT-STATE ceiling, not a target. 3.15 MB gzipped + * before first render is a bad payload, and the honest long-term line is far + * below it — but lowering the line to a TARGET is a separate decision with its + * own work behind it (objectui#5324 names the candidates), and re-baselining + * onto a fresh measurement is not that. Nothing here should be read as a + * finding that 3.19 MB is acceptable. * * ## Per-chunk ceilings (objectui#5490) * @@ -105,9 +169,14 @@ import { isEntrypoint } from './invoked-as.mjs'; /** * Ceiling for the console eager closure, in gzipped bytes. See the header for - * how this number was chosen; measured 4,005,911 on `4c1623c0c`. + * how this number was chosen; measured 3,299,898 on `48e53814e`. + * + * Re-baselined DOWNWARD by objectui#5924 from 4,086,000 (derived from the + * 4,005,911 reading on `4c1623c0c`, which the payload had since fallen 706,013 + * bytes below). Headroom is now 45,102 bytes — 0.49x + * {@link REGRESSION_THIS_GATE_MUST_CATCH_BYTES}, where it had drifted to 8.6x. */ -export const MAX_EAGER_CLOSURE_GZIP_BYTES = 4_086_000; +export const MAX_EAGER_CLOSURE_GZIP_BYTES = 3_345_000; /** * The measurement the ceiling above was derived from. Exported so the two @@ -118,10 +187,10 @@ export const MAX_EAGER_CLOSURE_GZIP_BYTES = 4_086_000; */ export const BASELINE = Object.freeze({ /** `emitEagerClosureReport`'s `eagerGzipBytes` on this commit. */ - gzipBytes: 4_005_911, + gzipBytes: 3_299_898, chunks: 52, totalChunks: 508, - commit: '4c1623c0c', + commit: '48e53814e', }); /** @@ -569,6 +638,181 @@ export function evaluatePerChunkBudgets({ }; } +/** + * Weigh every ceiling against the payload it governs, and refuse a ceiling that + * has drifted out of range of the regression it exists to catch (objectui#5924). + * + * ## The failure this exists for + * + * Both other halves answer "is the bundle under its line?". Neither can answer + * "is that line still close enough to the bundle to mean anything?", and that + * question has a silent wrong answer: a ceiling far above the payload passes + * everything, prints a green tick with a large `headroom:` figure beside it, and + * reads as a healthy bundle. The header records the measurement — the aggregate + * ceiling was 8.6x the regression size above the live payload, and a +154 KB + * eager regression went green through it. + * + * The invariant was not missing, it was checked in the wrong place: the unit + * test compared {@link MAX_EAGER_CLOSURE_GZIP_BYTES} with + * {@link BASELINE}.gzipBytes, two literals frozen in this module, so it was true + * no matter what the console weighed. This function computes the same quantity + * from the report the gate just read, so drift reds the moment it opens instead + * of the day someone re-measures by hand. + * + * ## Why `error` and not `fail` + * + * `fail` is a verdict about the BUNDLE — it grew past a line. Nothing has grown + * here; the ceiling has stopped being a measurement of anything. That is a + * verdict about the GAUGE, which is what exit 2 means in this file, and it is + * the same asymmetry {@link evaluatePerChunkBudgets} applies to a budgeted chunk + * that is absent: a check that passes by measuring nothing must be LOUDER than + * one that fails by measuring something, never quieter. + * + * ## What it deliberately does not do + * + * It does not treat a NEGATIVE headroom — a ceiling under the payload — as its + * business. That is an over-budget bundle, the other two halves own it, and + * reporting it here as well would turn one regression into an error and teach a + * reader to distrust the exit code. Over-budget rows are still printed, marked + * as such, so the table is a complete picture of every ceiling. + * + * @param {object} input + * @param {unknown} input.report + * @param {number} [input.budgetBytes] the aggregate ceiling + * @param {Record} [input.ceilings] the per-chunk ceilings + * @param {number} [input.regressionBytes] the size this gate must stay able to catch + * @param {string} [input.reportPath] + * @returns {{ status: 'pass' | 'fail' | 'error', message: string, + * sites: { key: string, label: string, constant: string, measuredBytes: number, + * ceilingBytes: number, headroomBytes: number, multiple: number }[], + * blind: string[] }} + */ +export function evaluateHeadroomSensitivity({ + report, + budgetBytes = MAX_EAGER_CLOSURE_GZIP_BYTES, + ceilings = PER_CHUNK_GZIP_CEILINGS, + regressionBytes = REGRESSION_THIS_GATE_MUST_CATCH_BYTES, + reportPath = DEFAULT_REPORT_PATH, +} = {}) { + const base = { sites: [], blind: [] }; + + if (report === null || report === undefined) { + return { + ...base, + status: 'error', + message: + `No eager-closure report at ${reportPath}, so no ceiling could be weighed against the ` + + `payload it governs. Sensitivity is a property of the ceiling AND the measurement — ` + + `with only one of them there is nothing to check. This is a broken gauge, not a ` + + `sensitive gate.`, + }; + } + + const problems = validateReport(report); + if (problems.length > 0) { + return { + ...base, + status: 'error', + message: `Ceiling sensitivity cannot be judged from ${reportPath}:\n - ${problems.join('\n - ')}`, + }; + } + + const measured = measureChunksByName(report); + const sites = [ + { + key: 'aggregate', + label: 'aggregate closure', + constant: 'MAX_EAGER_CLOSURE_GZIP_BYTES', + measuredBytes: /** @type {{ eagerGzipBytes: number }} */ (report).eagerGzipBytes, + ceilingBytes: budgetBytes, + }, + ]; + const absent = []; + for (const [name, ceilingBytes] of Object.entries(ceilings)) { + const entry = measured.get(name); + // A budgeted chunk with nothing to weigh has no headroom to judge. It is + // already an ERROR one level up, and inventing a verdict for it here (0 + // bytes measured, so "drifted") would be a second wrong reason for the + // right exit code. Refuse the whole judgement instead of guessing part of it. + if (entry === undefined) { + absent.push(name); + continue; + } + sites.push({ + key: name, + label: `chunk \`${name}\``, + constant: `PER_CHUNK_GZIP_CEILINGS['${name}']`, + measuredBytes: entry.gzipBytes, + ceilingBytes, + }); + } + + if (absent.length > 0) { + return { + ...base, + status: 'error', + message: + `Cannot judge ceiling sensitivity: budgeted chunk${absent.length === 1 ? '' : 's'} ` + + `${absent.map((n) => `\`${n}\``).join(', ')} ${absent.length === 1 ? 'is' : 'are'} absent ` + + `from ${reportPath}, so ${absent.length === 1 ? 'its ceiling governs' : 'their ceilings govern'} ` + + `nothing measurable. See the per-chunk verdict for what to do about it.`, + }; + } + + const rows = sites.map((site) => { + const headroomBytes = site.ceilingBytes - site.measuredBytes; + return { ...site, headroomBytes, multiple: headroomBytes / regressionBytes }; + }); + const blind = rows.filter((row) => row.headroomBytes >= regressionBytes); + + const table = rows + .map((row) => { + const band = + row.headroomBytes < 0 + ? `OVER by ${kb(-row.headroomBytes)} KB — the size verdict owns this row, not this one` + : `headroom ${kb(row.headroomBytes)} KB = ${row.multiple.toFixed(2)}x the ` + + `${kb(regressionBytes)} KB regression`; + return ( + ` ${row.headroomBytes >= regressionBytes ? '❌' : '✅'} ${row.label.padEnd(28)} ` + + `${kb(row.measuredBytes).padStart(9)} KB measured / ${kb(row.ceilingBytes)} KB ceiling ` + + `(${band}) [${row.constant}]` + ); + }) + .join('\n'); + + if (blind.length > 0) { + return { + sites: rows, + blind: blind.map((row) => row.key), + status: 'error', + message: + `${blind.length} ceiling${blind.length === 1 ? '' : 's'} ` + + `${blind.length === 1 ? 'has' : 'have'} DRIFTED more than one ` + + `${kb(regressionBytes)} KB regression above the payload ` + + `${blind.length === 1 ? 'it governs' : 'they govern'}:\n${table}\n` + + `A ceiling that far above today's measurement cannot tell "no regression" from a repeat ` + + `of objectui#5266 — its green tick carries no information, which makes this a verdict ` + + `about the GAUGE and not about the bundle (objectui#5924: an aggregate ceiling at 8.6x ` + + `passed a demonstrated +154 KB eager regression).\n` + + `The payload almost certainly SHRANK, which is good news — and good news is RE-PINNED ` + + `deliberately, never inferred: lower the named constant, move its baseline with it in ` + + `the same commit, and say in the PR what the new headroom is.\n` + + `⛔ Never lower a ceiling BELOW the measured figure to express an aspiration. A ceiling ` + + `under today's reality lands red on \`main\`, which is how a budget gets switched off ` + + `rather than met.`, + }; + } + + return { + sites: rows, + blind: [], + status: 'pass', + message: + `Ceiling sensitivity (${rows.length} ceilings, each weighed against the report just read):\n` + + `${table}`, + }; +} + /** * The biggest eager chunks, so a failure names suspects instead of a total. * @param {{ files?: { fileName: string, gzipBytes: number }[] }} report @@ -598,16 +842,18 @@ function writeGithubOutput(entries, outputPath = process.env.GITHUB_OUTPUT) { /** * Exit codes: `0` within budget, `1` over budget — the aggregate ceiling or any * per-chunk ceiling — and `2` no trustworthy measurement (report missing, - * stale-shaped, internally inconsistent, or missing a budgeted chunk). + * stale-shaped, internally inconsistent, missing a budgeted chunk, or governed + * by a ceiling that has drifted out of range of the regression it must catch). * * `2` covers the unbuilt tree, and deliberately so: with no * `apps/console/dist/eager-closure.json` this check reports a BROKEN GAUGE and * the workflow fails the step. It never prints a verdict about a bundle nobody * weighed, and it never exits 0 having measured nothing. * - * Both halves are evaluated and printed before either decides the code: a run - * that reports the total and hides which chunk moved (or the reverse) teaches - * readers to ignore the half they cannot see. + * All three halves are evaluated and printed before any of them decides the + * code: a run that reports the total and hides which chunk moved (or hides + * whether either line still means anything) teaches readers to ignore the half + * they cannot see. */ export function main(argv = process.argv.slice(2)) { const flagIndex = argv.indexOf('--report'); @@ -616,6 +862,7 @@ export function main(argv = process.argv.slice(2)) { const report = readReport(resolved); const result = evaluateClosureBudget({ report, reportPath }); const perChunk = evaluatePerChunkBudgets({ report, reportPath }); + const sensitivity = evaluateHeadroomSensitivity({ report, reportPath }); if (result.status === 'pass') { console.log(`✅ ${result.message}`); @@ -627,6 +874,11 @@ export function main(argv = process.argv.slice(2)) { } else { console.error(`❌ ${perChunk.message}`); } + if (sensitivity.status === 'pass') { + console.log(`✅ ${sensitivity.message}`); + } else { + console.error(`❌ ${sensitivity.message}`); + } if (report?.files?.length) { console.log(''); console.log('Largest eagerly loaded chunks (gzipped):'); @@ -639,6 +891,7 @@ export function main(argv = process.argv.slice(2)) { closure_budget_kb: kb(result.budgetBytes), closure_chunks: result.chunkCount === null ? '' : String(result.chunkCount), closure_chunk_status: perChunk.status, + closure_headroom_status: sensitivity.status, }); // Distinct codes so the workflow can tell "over budget" (a real verdict about @@ -647,10 +900,13 @@ export function main(argv = process.argv.slice(2)) { // regression, and a size regression reported as a broken report — each of // which teaches readers to ignore the other. // - // `error` outranks `fail` across BOTH halves for the same reason it does - // within one: a report that cannot be trusted makes its own size verdict - // meaningless, whichever half noticed first. - const statuses = [result.status, perChunk.status]; + // `error` outranks `fail` across ALL THREE halves for the same reason it does + // within one: a report that cannot be trusted — or a ceiling that no longer + // measures the thing it names — makes its own size verdict meaningless, + // whichever half noticed first. objectui#5490 established that ordering over + // two halves; objectui#5924 adds the third under the same rule rather than + // giving sensitivity a code of its own. + const statuses = [result.status, perChunk.status, sensitivity.status]; if (statuses.includes('error')) return 2; return statuses.includes('fail') ? 1 : 0; }