diff --git a/scripts/pm/dispatch-gates.mjs b/scripts/pm/dispatch-gates.mjs index 0642e324f1..7d11d5dc5a 100644 --- a/scripts/pm/dispatch-gates.mjs +++ b/scripts/pm/dispatch-gates.mjs @@ -381,6 +381,71 @@ export { isExtractConfigPath, isMetadataFormModulePath }; const ROOT = new URL('../..', import.meta.url).pathname; +// ── The source maskers are memoised, because discovery masks each file ~12x ── +// +// PROFILED, not guessed. `discoverFamilies` hands the SAME source string to six +// analysers in one pass of its per-family loop — `extractWatchHints`, +// `readProgramTargetsInSource`, `payloadEnvDependence`, `firstPartyImportTargets`, +// `spawnedProgramTargets` and `packageManifestTargets` — and every one of them +// re-derives the masked body from scratch, two of them twice (they mask, then +// hand the masked text to `anchoredReadTargets`, which masks again). One source +// therefore pays `maskComments` about seven times and `maskSelfTests` about five, +// per discovery, for bytes that cannot have changed in between. +// +// The cost that buys: a V8 CPU profile of ONE `discoverFamilies()` call on this +// tree — 201 families, 196 distinct gate sources, 11.8 MB of them — spent 14.3 s, +// of which `maskSelfTests` was 4.5 s of self time (31.7%) and the `maskComments` +// inside those six analysers most of another 5.2 s. That is the largest single +// entry in the profile, and all of it above the first pass is repetition. +// +// So the maskers are memoised on their INPUT STRING. Both are pure functions of +// that string, and JavaScript strings are immutable, so a memo is +// observationally identical to calling through: same bytes in, same bytes out, +// and no caller can edit the shared result under another. ⛔ This changes +// nothing about WHAT is masked, scanned or discovered — it is the same +// derivation run once instead of a dozen times, which is the only kind of +// speed-up this tool may take. +// +// The bound is in BYTES rather than entries because the corpora these run over +// differ by three orders of magnitude: the gate set is ~12 MB and fits whole, so +// repeated discoveries in one process reuse it, while a tracked-corpus sweep +// would otherwise grow the cache without limit. Eviction is oldest-first, and a +// miss after eviction is a recomputation — never a different answer. +const MASK_MEMO_BYTE_BUDGET = 32 * 1024 * 1024; + +function memoiseMask(compute) { + const cache = new Map(); + let bytes = 0; + return (source) => { + // A non-string argument is passed straight through: today's behaviour is + // whatever the masker does with it, and a memo must not be the thing that + // decides otherwise. + if (typeof source !== 'string') return compute(source); + const hit = cache.get(source); + if (hit !== undefined) return hit; + const value = compute(source); + cache.set(source, value); + bytes += source.length + value.length; + // `Map` iterates in insertion order, so the first key is the oldest. + while (bytes > MASK_MEMO_BYTE_BUDGET && cache.size > 1) { + const oldest = cache.keys().next().value; + bytes -= oldest.length + cache.get(oldest).length; + cache.delete(oldest); + } + return value; + }; +} + +/** `maskComments`, memoised — see the block above. */ +const maskedComments = memoiseMask((source) => maskComments(source)); + +/** + * `maskSelfTests(maskComments(source))`, memoised — see the block above. It + * composes through `maskedComments` rather than calling `maskComments` again, so + * the comment mask is derived once for the callers that want each half. + */ +const maskedModuleBody = memoiseMask((source) => maskSelfTests(maskedComments(source))); + // ── What a gate that IMPORTS this module inherits (#11556) ───────────────── // // This module is importable and is NOT a discovered gate file — `check:pm-dispatch-gates` @@ -1538,7 +1603,7 @@ const PAYLOAD_ENV_ACCESS = new RegExp( * the difference between classifying a gate and classifying its docblock. */ export function payloadEnvDependence(scriptSource) { - const body = maskSelfTests(maskComments(String(scriptSource))); + const body = maskedModuleBody(String(scriptSource)); return PAYLOAD_ENV_ACCESS.test(body) ? WORKFLOW_PAYLOAD_ENV : null; } @@ -2366,7 +2431,7 @@ export const COMPOUND_ANCHOR_KEYS = new Map( */ export function compoundAnchorDecls(source) { const scan = scanSource(source); - const decommented = maskComments(source); + const decommented = maskedComments(source); const out = []; for (const m of decommented.matchAll(SELF_TEST_DECL)) { if (scan.comment[m.index] || scan.literal[m.index]) continue; @@ -2959,7 +3024,7 @@ export function packageRootAnchoredHint(hint, base, tree, files) { * have to remember to do it. */ export function extractWatchHints(scriptSource, scriptPath = null, { tree = null } = {}) { - const moduleBody = maskSelfTests(maskComments(scriptSource)); + const moduleBody = maskedModuleBody(scriptSource); const hints = new Set(); for (const m of moduleBody.matchAll(/['"`]([^'"`\n]{2,120})['"`]/g)) { const raw = m[1]; @@ -3175,7 +3240,7 @@ export function firstPartyImportTargets(scriptPath, source, { root = ROOT } = {} // The same masking hint extraction uses, for the same reason: an import // written out in a docblock, or one inside a self-test fixture, is a // specifier this script NAMES rather than one it loads. - const body = maskSelfTests(maskComments(String(source))); + const body = maskedModuleBody(String(source)); const specifiers = new Set(); for (const m of body.matchAll(IMPORT_FROM_SPECIFIER)) specifiers.add(m[2]); for (const m of body.matchAll(SIDE_EFFECT_IMPORT)) specifiers.add(m[2]); @@ -5234,7 +5299,7 @@ function combineReadings(readings, name) { * own repo-relative path — the anchor spellings resolve against it. */ export function scratchDirSitesInSource(rel, source) { - const masked = maskComments(String(source)); + const masked = maskedComments(String(source)); // A call spelled inside a STRING is a fixture, not a call — this module's own // self-test plants fixture sources as string literals, and read as code they // reported four sites in a file that creates none of them. Comments are @@ -5394,7 +5459,7 @@ export function readProgramTargetsInSource(rel, source, isTracked) { /** Every TRACKED file the source opens at a path anchored to its own location. */ export function anchoredReadTargets(rel, source, isTracked) { - const masked = maskComments(String(source)); + const masked = maskedComments(String(source)); const { literal } = scanSource(masked); const ctx = { fileSegs: rel.split('/'), @@ -5533,7 +5598,7 @@ export function spawnedProgramTargets(rel, source, isTracked) { // follow inherits a POPULATION, and a spawn written inside a self-test body // is a fixture the self-test drives rather than the gate's work. The read // scan next door wants the opposite from the same bytes, and says so. - const masked = maskSelfTests(maskComments(String(source))); + const masked = maskedModuleBody(String(source)); const { literal } = scanSource(masked); const ctx = { fileSegs: rel.split('/'), @@ -5717,7 +5782,7 @@ const PACKAGE_MANIFEST_TARGET = /(?:^|\/)package\.json$/; const MANIFEST_EXPORTS_READ = /(? PACKAGE_MANIFEST_TARGET.test(t)); } @@ -6635,7 +6700,7 @@ export function stampsAnErrorCodeLiteral(path, readSource = readTrackedSource) { // Comments are masked for the reason the gate masks them: a code DISCUSSED in // prose is not a code stamped in source. This narrows nothing the gate would // have reported, so it costs no recall in the expensive direction. - const masked = maskComments(source); + const masked = maskedComments(source); return CODE_STAMP_POSITION.test(masked) || CODE_CONSTANT_BINDING.test(masked); }