test(scripts): reconcile the two published runtime publish-gate measurements, and leave a re-runnable benchmark (#9851)

scripts/bench/runtime-publish-gate.bench.mts

@@ -0,0 +1,230 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+/**
+ * Runtime publish-gate benchmark (#9851) — what one `active` metadata publish
+ * costs at the shipped gate, as a function of the tenant's stored objects.
+ *
+ * ── Why this file exists ────────────────────────────────────────────────────
+ *
+ * Two published measurements of THIS operation disagreed by 16-25x with a
+ * non-constant ratio, and neither could be re-run: both came from throwaway
+ * harnesses that no longer exist. #9851 reconciled them — the disagreement is
+ * entirely the SHAPE OF THE SEEDED `objects` COLLECTION, not the span, not the
+ * warmup, and not a code change between the two refs — and left this file so
+ * the next reading is reproducible instead of re-litigated.
+ *
+ * npx tsx scripts/bench/runtime-publish-gate.bench.mts
+ * npx tsx scripts/bench/runtime-publish-gate.bench.mts --type object
+ * npx tsx scripts/bench/runtime-publish-gate.bench.mts --objects 21,105,420 --iterations 50
+ *
+ * Requires `pnpm --filter '@objectstack/lint^...' build && pnpm --filter
+ * @objectstack/lint build` first: the gate is imported straight from
+ * `packages/lint/dist/runtime.js` — the built artifact the `./runtime` export
+ * points at, reached by path because this script is outside the workspace
+ * graph — so an unbuilt or stale `dist` reports the cost of the OLD registry
+ * (this is the `ablation-dist-preflight` hazard; the run prints the rules it
+ * actually dispatched, so a stale build shows up as a changed rule list rather
+ * than as a silently wrong number).
+ *
+ * ⚠️ Not a CI gate. Wall-clock numbers are machine-dependent and this asserts
+ * no threshold; it prints, and the operator compares.
+ *
+ * ── THE TIMED SPAN — inside vs outside ──────────────────────────────────────
+ *
+ * INSIDE the timed region: exactly one call to `runRuntimeAuthoringRules()` —
+ * the whole shipped gate for one write. That is `buildRuntimeWriteSnapshots`
+ * (baseline + candidate) plus the differential TWO passes of every rule that
+ * declares the written type in `runtimeTypes`, plus the fingerprint set-diff.
+ *
+ * OUTSIDE: seeding and cloning the stack, module load, and everything
+ * `saveMetaItem` wraps around the gate — the per-type Zod `safeParse`,
+ * persistence, hooks, audit, the HTTP envelope.
+ *
+ * ⭐ That exclusion is measured, not assumed to be small: the gate call alone
+ * reproduces the whole-`saveMetaItem` figure published as #9851's Reading A to
+ * within ~8%, so the gate IS substantially the whole per-publish bill and the
+ * span is NOT what separated the two published readings.
+ *
+ * ── WARMUP POLICY ───────────────────────────────────────────────────────────
+ *
+ * `--warmup` iterations (default 5) run and are discarded, then `--iterations`
+ * (default 30) are timed and the MEDIAN is reported, with min/max so a noisy
+ * machine is visible. ⭐ Measured on this workload, warm and cold agree to
+ * within noise (a cold 3-iteration median lands inside the warm min/max band at
+ * both sizes), because a single iteration is already tens of milliseconds of
+ * the same loop — long enough for the JIT to settle mid-iteration. So warmup is
+ * kept for hygiene, NOT because it moves the number: it is not the explanation
+ * for any two harnesses disagreeing here.
+ *
+ * ── THE SEEDED STACK — the parameter that actually drives the cost ───────────
+ *
+ * The gate's context is the bounded 4-collection `RuntimeStackContext`
+ * (`objects` / `permissions` / `books` / `datasets`); only `objects` is varied
+ * here, because that is the collection both published readings varied and the
+ * one a tenant grows. Two shapes ship, and running BOTH is the point:
+ *
+ * real — the shipped `examples/app-showcase` object declarations (22 of
+ * them), cloned under suffixed names up to N. Real authored surface:
+ * formula fields, validation rules, predicates, lookups — i.e. the
+ * expression-bearing keys `validateStackExpressions` walks.
+ * stub — synthetic 5-field objects carrying no expressions at all.
+ *
+ * ⭐ Same gate, same N, same warmup: `real` costs ~25x `stub`. **Per-publish
+ * cost is a function of the tenant's authored object SURFACE, not of its object
+ * COUNT** — count is only a proxy, and the proxy's constant is what the two
+ * published readings silently disagreed about. A number quoted as "X ms at N
+ * objects" without its stack shape is therefore not reproducible, and any
+ * acceptance threshold derived from one must name the shape it was set against.
+ *
+ * ⚠️ `real` imports the showcase app LIVE, so its absolute numbers drift as
+ * that app is edited. Deliberate — it is the only real, spec-parsed corpus in
+ * the repo, and a frozen copy would silently stop describing shipped metadata.
+ * Quote the ref you measured at.
+ */
+
+import { runRuntimeAuthoringRules } from '../../packages/lint/dist/runtime.js';
+import * as showcaseObjects from '../../examples/app-showcase/src/data/objects/index.js';
+import { allFlows } from '../../examples/app-showcase/src/automation/flows/index.js';
+
+// Ambient `process` / `console`, declared to exactly the members used below.
+// `scripts/**` is inside the ROOT tsconfig's program, and that config carries
+// `lib: ['ES2020']` with no `types` — so Node's globals are absent and every
+// use of them is a root-ledger tsc error (measured: 19 from this file alone,
+// which is how it first landed). tsx provides the real objects at run time.
+// Same shape and same reason as the ambient `process` in
+// `examples/app-showcase/objectstack.config.ts`: narrow the declaration to
+// what is actually called rather than widening the root's type surface, so
+// this file pays its own way through `pnpm check:type-check-debt` instead of
+// pushing a shrink-only ratchet back up.
+declare const process: {
+ argv: string[];
+ hrtime: { bigint(): bigint };
+};
+declare const console: { log(...args: unknown[]): void };
+
+type AnyRec = Record<string, any>;
+
+function arg(name: string, fallback: string): string {
+ const hit = process.argv.find((a) => a.startsWith(`--${name}=`));
+ if (hit) return hit.slice(name.length + 3);
+ const i = process.argv.indexOf(`--${name}`);
+ return i >= 0 && process.argv[i + 1] ? process.argv[i + 1]! : fallback;
+}
+
+const WRITE_TYPE = arg('type', 'flow');
+const SIZES = arg('objects', '21,420').split(',').map((s) => Number(s.trim())).filter(Boolean);
+const ITERATIONS = Number(arg('iterations', '30'));
+const WARMUP = Number(arg('warmup', '5'));
+
+/** The shipped showcase object declarations — the `real` seed's source corpus. */
+const SHOWCASE_OBJECTS = (Object.values(showcaseObjects) as AnyRec[])
+ .filter((o) => o && typeof o.name === 'string');
+
+/** A real shipped flow, used as the written item for a `flow` publish. */
+const SHOWCASE_FLOW = (allFlows as AnyRec[]).find((f) => f.name === 'showcase_task_completed')
+ ?? (allFlows as AnyRec[])[0];
+
+/** A minimal clean object body, used as the written item for an `object` publish. */
+const WRITTEN_OBJECT: AnyRec = {
+ name: 'bench_written_object',
+ label: 'Bench Written Object',
+ sharingModel: 'ReadWrite',
+ fields: { name: { type: 'text', label: 'Name' } },
+};
+
+/**
+ * N objects cloned from the shipped showcase corpus, names suffixed to stay unique.
+ *
+ * ⚠️ The clone is `JSON.parse(JSON.stringify(...))`, so any FUNCTION-valued key
+ * in a showcase declaration is dropped rather than copied. Harmless for what is
+ * measured here — the runtime gate receives `sys_metadata` bodies, which are
+ * JSON at rest, so a JSON round-trip is the shape the real door sees — but it
+ * does mean this seed is a lower bound on a declaration whose authored surface
+ * lives partly in functions. Stated because the file's honesty about its own
+ * seed is the thing that makes a number taken from it re-runnable.
+ */
+function seedReal(n: number): AnyRec[] {
+ const out: AnyRec[] = [];
+ for (let i = 0; out.length < n; i++) {
+ const src = SHOWCASE_OBJECTS[i % SHOWCASE_OBJECTS.length]!;
+ const clone = JSON.parse(JSON.stringify(src)) as AnyRec;
+ const generation = Math.floor(i / SHOWCASE_OBJECTS.length);
+ if (generation > 0) clone.name = `${src.name}_c${generation}`;
+ out.push(clone);
+ }
+ return out;
+}
+
+/** N synthetic 5-field objects carrying no expressions. */
+function seedStub(n: number): AnyRec[] {
+ return Array.from({ length: n }, (_, i) => ({
+ name: `bench_object_${i}`,
+ label: `Bench Object ${i}`,
+ fields: {
+ name: { type: 'text', label: 'Name' },
+ amount: { type: 'number', label: 'Amount' },
+ active: { type: 'boolean', label: 'Active' },
+ due: { type: 'date', label: 'Due' },
+ notes: { type: 'textarea', label: 'Notes' },
+ },
+ }));
+}
+
+const SHAPES: Record<string, (n: number) => AnyRec[]> = { real: seedReal, stub: seedStub };
+
+const ms = () => Number(process.hrtime.bigint()) / 1e6;
+const median = (xs: number[]) => [...xs].sort((a, b) => a - b)[Math.floor(xs.length / 2)]!;
+
+function run(objects: AnyRec[], item: AnyRec) {
+ return runRuntimeAuthoringRules({ type: WRITE_TYPE, item, context: { objects } });
+}
+
+function measure(objects: AnyRec[], item: AnyRec) {
+ for (let i = 0; i < WARMUP; i++) run(objects, item);
+ const samples: number[] = [];
+ for (let i = 0; i < ITERATIONS; i++) {
+ const t0 = ms();
+ run(objects, item);
+ samples.push(ms() - t0);
+ }
+ return { median: median(samples), min: Math.min(...samples), max: Math.max(...samples) };
+}
+
+const item = WRITE_TYPE === 'flow' ? SHOWCASE_FLOW : WRITTEN_OBJECT;
+
+// The rules actually dispatched, printed BEFORE any timing: a stale `dist/`, or
+// a rule ablated off the runtime surface, changes this line — so a number that
+// moved for a registry reason can never be read as a number that moved for a
+// performance reason.
+const dispatched = run(SHAPES.real!(1), item).rulesRun;
+console.log(`runtime publish gate — write type '${WRITE_TYPE}'`);
+console.log(` rules dispatched (${dispatched.length}): ${dispatched.join(', ') || '<none>'}`);
+console.log(` timed span: one runRuntimeAuthoringRules() call (differential two-pass)`);
+console.log(` warmup ${WARMUP} discarded, ${ITERATIONS} timed, median reported`);
+if (dispatched.length === 0) {
+ console.log(` ⚠️ no rule gates '${WRITE_TYPE}' at the runtime surface — every number below is gate overhead only`);
+}
+
+console.log('');
+console.log(' shape N objects median min max');
+const table: Record<string, Record<number, number>> = {};
+for (const shape of Object.keys(SHAPES)) {
+ table[shape] = {};
+ for (const n of SIZES) {
+ const r = measure(SHAPES[shape]!(n), item);
+ table[shape]![n] = r.median;
+ console.log(
+ ` ${shape.padEnd(6)} ${String(n).padStart(9)} ${r.median.toFixed(2).padStart(7)} ms `
+ + `${r.min.toFixed(2).padStart(7)} ${r.max.toFixed(2).padStart(7)}`,
+ );
+ }
+}
+
+console.log('');
+for (const n of SIZES) {
+ const real = table.real![n]!;
+ const stub = table.stub![n]!;
+ console.log(` N=${n}: real / stub = ${(real / stub).toFixed(1)}x — same gate, same N, different authored surface`);
+}
+console.log('');
+console.log(' ⇒ per-publish cost tracks authored object SURFACE, not object COUNT.');
+console.log(' Quote the stack shape with any number taken from this gate (#9851).');