diff --git a/scripts/bench/runtime-publish-gate.bench.mts b/scripts/bench/runtime-publish-gate.bench.mts index b2966e7778..3e2a08efa8 100644 --- a/scripts/bench/runtime-publish-gate.bench.mts +++ b/scripts/bench/runtime-publish-gate.bench.mts @@ -3,6 +3,22 @@ * Runtime publish-gate benchmark (#9851) — what one `active` metadata publish * costs at the shipped gate, as a function of the tenant's stored objects. * + * ── The three modes ───────────────────────────────────────────────────────── + * + * --mode total (default) the whole-gate cost table. #9851's reading. + * --mode per-rule (#9905) the same total, ATTRIBUTED across the rules the + * door dispatches. + * --mode closure (#9905) the whole-gate cost when the `objects` collection + * is narrowed to the written item's reference closure. + * + * ⛔ `--mode closure` measures a HYPOTHETICAL. It narrows what the rules are + * handed HERE, in this script, and changes nothing about the shipped gate — + * which always receives the whole collection. Its closure deriver is a + * deliberately generous over-approximation written to bound the saving, NOT a + * proposed implementation: how (or whether) the shipped gate should scope its + * input is an open maintainer decision (#9612 / #9613), and one that this + * measurement exists to inform rather than to pre-empt. + * * ── Why this file exists ──────────────────────────────────────────────────── * * Two published measurements of THIS operation disagreed by 16-25x with a @@ -81,7 +97,11 @@ * Quote the ref you measured at. */ -import { runRuntimeAuthoringRules } from '../../packages/lint/dist/runtime.js'; +import { + runRuntimeAuthoringRules, + runtimeAuthoringRulesFor, + buildRuntimeWriteSnapshots, +} from '../../packages/lint/dist/runtime.js'; import * as showcaseObjects from '../../examples/app-showcase/src/data/objects/index.js'; import { allFlows } from '../../examples/app-showcase/src/automation/flows/index.js'; @@ -114,6 +134,7 @@ const WRITE_TYPE = arg('type', 'flow'); const SIZES = arg('objects', '21,420').split(',').map((s) => Number(s.trim())).filter(Boolean); const ITERATIONS = Number(arg('iterations', '30')); const WARMUP = Number(arg('warmup', '5')); +const MODE = arg('mode', 'total'); /** The shipped showcase object declarations — the `real` seed's source corpus. */ const SHOWCASE_OBJECTS = (Object.values(showcaseObjects) as AnyRec[]) @@ -204,27 +225,155 @@ if (dispatched.length === 0) { console.log(` ⚠️ no rule gates '${WRITE_TYPE}' at the runtime surface — every number below is gate overhead only`); } -console.log(''); -console.log(' shape N objects median min max'); -const table: Record> = {}; -for (const shape of Object.keys(SHAPES)) { - table[shape] = {}; +/** Median wall-clock of `fn`, same warmup/iteration policy as {@link measure}. */ +function timeMedian(fn: () => void): number { + for (let i = 0; i < WARMUP; i++) fn(); + const samples: number[] = []; + for (let i = 0; i < ITERATIONS; i++) { + const t0 = ms(); + fn(); + samples.push(ms() - t0); + } + return median(samples); +} + +/** #9851's reading: the whole-gate cost table, both shapes, every N. */ +function totalMode() { + console.log(''); + console.log(' shape N objects median min max'); + const table: Record> = {}; + for (const shape of Object.keys(SHAPES)) { + table[shape] = {}; + for (const n of SIZES) { + const r = measure(SHAPES[shape]!(n), item); + table[shape]![n] = r.median; + console.log( + ` ${shape.padEnd(6)} ${String(n).padStart(9)} ${r.median.toFixed(2).padStart(7)} ms ` + + `${r.min.toFixed(2).padStart(7)} ${r.max.toFixed(2).padStart(7)}`, + ); + } + } + + console.log(''); for (const n of SIZES) { - const r = measure(SHAPES[shape]!(n), item); - table[shape]![n] = r.median; - console.log( - ` ${shape.padEnd(6)} ${String(n).padStart(9)} ${r.median.toFixed(2).padStart(7)} ms ` - + `${r.min.toFixed(2).padStart(7)} ${r.max.toFixed(2).padStart(7)}`, - ); + const real = table.real![n]!; + const stub = table.stub![n]!; + console.log(` N=${n}: real / stub = ${(real / stub).toFixed(1)}x — same gate, same N, different authored surface`); } + console.log(''); + console.log(' ⇒ per-publish cost tracks authored object SURFACE, not object COUNT.'); + console.log(' Quote the stack shape with any number taken from this gate (#9851).'); } -console.log(''); -for (const n of SIZES) { - const real = table.real![n]!; - const stub = table.stub![n]!; - console.log(` N=${n}: real / stub = ${(real / stub).toFixed(1)}x — same gate, same N, different authored surface`); +/** + * #9905 deliverable 2 — the same total, attributed across the dispatched rules. + * + * Each rule is timed over BOTH gate passes (baseline + candidate), because that + * is what one publish actually costs it, and the snapshots come from the gate's + * own exported {@link buildRuntimeWriteSnapshots} rather than a mirror — so the + * shares are attributed against the real construction. + * + * ⭐ The UNATTRIBUTED line is the anti-vacuity check and is printed even when it + * is uninteresting: per-rule shares that do not add up to the measured total + * mean the attribution is wrong, and that is the finding. Confirm the top share + * independently by ablating that rule off the door (`runtimeTypes`), rebuilding, + * and checking the total moves by roughly its share — `rules dispatched` above + * changes with it, so a stale `dist/` cannot masquerade as a saving. + */ +function perRuleMode() { + const rules = runtimeAuthoringRulesFor(WRITE_TYPE); + const ctx = { sduiManifest: undefined }; + for (const shape of Object.keys(SHAPES)) { + for (const n of SIZES) { + const objects = SHAPES[shape]!(n); + const snapshots = buildRuntimeWriteSnapshots({ type: WRITE_TYPE, item, context: { objects } }); + if (!snapshots) continue; + const total = timeMedian(() => { run(objects, item); }); + const per = rules + .map((r) => ({ + name: r.name, + ms: timeMedian(() => { + try { r.run(snapshots.baseline, ctx); } catch { /* the gate reports throws as findings */ } + try { r.run(snapshots.candidate, ctx); } catch { /* idem */ } + }), + })) + .sort((a, b) => b.ms - a.ms); + const sum = per.reduce((acc, p) => acc + p.ms, 0); + const pct = (x: number) => `${((x / total) * 100).toFixed(1).padStart(5)}%`; + console.log(''); + console.log(` shape=${shape} N=${n} — whole-gate total ${total.toFixed(2)} ms`); + for (const p of per) console.log(` ${p.name.padEnd(32)} ${p.ms.toFixed(2).padStart(8)} ms ${pct(p.ms)}`); + console.log(` ${'Σ attributed'.padEnd(32)} ${sum.toFixed(2).padStart(8)} ms ${pct(sum)}`); + console.log(` ${'unattributed (snapshot + diff)'.padEnd(32)} ${(total - sum).toFixed(2).padStart(8)} ms ${pct(total - sum)}`); + } + } } -console.log(''); -console.log(' ⇒ per-publish cost tracks authored object SURFACE, not object COUNT.'); -console.log(' Quote the stack shape with any number taken from this gate (#9851).'); + +/** Every string anywhere in a value — the closure seed scan, deliberately broad. */ +function allStrings(value: unknown, out: string[] = []): string[] { + if (typeof value === 'string') out.push(value); + else if (Array.isArray(value)) for (const v of value) allStrings(v, out); + else if (value && typeof value === 'object') for (const v of Object.values(value)) allStrings(v, out); + return out; +} + +/** + * The transitive reference closure of the written item over `objects`: every + * object NAMED anywhere in the item, expanded through the included objects' + * declared relational edges (`fields[].reference`) to a fixed point. + * + * Over-approximating on purpose — it seeds from any string that happens to equal + * an object name, so it can only ever be too LARGE. A saving measured against it + * is therefore a lower bound on what an exact closure would buy, which is the + * safe direction for a number that argues FOR narrowing. + */ +function referenceClosure(written: AnyRec, objects: AnyRec[]): AnyRec[] { + const byName = new Map(); + for (const o of objects) if (typeof o.name === 'string') byName.set(o.name, o); + const reached = new Set(allStrings(written).filter((s) => byName.has(s))); + const frontier = [...reached]; + while (frontier.length > 0) { + const owner = byName.get(frontier.pop()!)!; + for (const field of Object.values((owner.fields ?? {}) as AnyRec)) { + const ref = (field as AnyRec)?.reference; + for (const target of Array.isArray(ref) ? ref : [ref]) { + if (typeof target === 'string' && byName.has(target) && !reached.has(target)) { + reached.add(target); + frontier.push(target); + } + } + } + } + return [...reached].map((name) => byName.get(name)!); +} + +/** + * #9905 deliverable 1 — what NARROWING the objects collection would buy, timed. + * + * Prints `|closure| / N` (the size ratio, which is NOT the saving) next to the + * measured saving (which is), and re-runs the gate on the narrowed collection to + * confirm the differential VERDICT is unchanged — a saving that changes the + * verdict is not a saving, it is PR #7886's phantom-findings failure. + */ +function closureMode() { + const stable = (r: { errors: AnyRec[]; advisories: AnyRec[] }) => + [...r.errors, ...r.advisories].map((f) => `${f.rule}|${f.where}|${f.path}|${f.message}`).sort().join('\n'); + for (const shape of Object.keys(SHAPES)) { + for (const n of SIZES) { + const objects = SHAPES[shape]!(n); + const closure = referenceClosure(item as AnyRec, objects); + const full = timeMedian(() => { run(objects, item); }); + const scoped = timeMedian(() => { run(closure, item); }); + const agree = stable(run(objects, item)) === stable(run(closure, item)); + console.log(''); + console.log(` shape=${shape} N=${n}: |closure| = ${closure.length} (closure/N = ${((closure.length / n) * 100).toFixed(1)}%)`); + console.log(` members: ${closure.map((o) => String(o.name)).join(', ') || ''}`); + console.log(` whole gate: full ${full.toFixed(2)} ms → closure ${scoped.toFixed(2)} ms — saving ${(((full - scoped) / full) * 100).toFixed(1)}%`); + console.log(` differential verdict unchanged: ${agree ? 'YES' : 'NO — narrowing changed the answer'}`); + } + } +} + +if (MODE === 'per-rule') perRuleMode(); +else if (MODE === 'closure') closureMode(); +else totalMode();