diff --git a/scripts/check-dispatcher-error-vocabulary.mjs b/scripts/check-dispatcher-error-vocabulary.mjs index 4c3a0f1876..abfdd9ff2f 100644 --- a/scripts/check-dispatcher-error-vocabulary.mjs +++ b/scripts/check-dispatcher-error-vocabulary.mjs @@ -108,6 +108,7 @@ */ import { readFileSync, readdirSync, statSync, existsSync } from 'node:fs'; +import { maskComments } from './js-comment-mask.mjs'; import { join, relative, dirname, resolve } from 'node:path'; const ROOT = resolve(new URL('..', import.meta.url).pathname); @@ -147,10 +148,7 @@ export function parseStandardCodes(source) { // The scan // --------------------------------------------------------------------------- -/** Comments stripped: prose naming a retired code is not a producer. */ -export function stripComments(source) { - return source.replace(/\/\*[\s\S]*?\*\//g, '').replace(/\/\/[^\n]*/g, ''); -} + /** * The recognised ways this repo stamps a semantic code onto a value. @@ -283,7 +281,7 @@ export function resolveConstant(name, fileSource, fileRel, readFile, ctx = {}) { for (const cand of [`${base}.ts`, `${base}.mts`, `${base}/index.ts`]) { const abs = join(ROOT, cand); if (!existsSync(abs)) continue; - const target = stripComments(readFile(abs)); + const target = maskComments(readFile(abs)); const hit = new RegExp( `\\bexport\\s+const\\s+${name}\\s*(?::[^=]+)?=\\s*'([A-Za-z][A-Za-z0-9_]*)'`, ).exec(target); @@ -354,7 +352,7 @@ export function deriveSites({ registered, files, readFile, packageDirs = new Map const unresolved = []; // Stripped once: the workspace resolver reads these same sources, and a // comment naming a constant must not resolve one either. - const scanned = files.map(({ rel, source }) => ({ rel, stripped: stripComments(source) })); + const scanned = files.map(({ rel, source }) => ({ rel, stripped: maskComments(source) })); const ctx = { scanned, packageDirs }; for (const { rel, stripped } of scanned) { for (const shape of SHAPES) { @@ -402,7 +400,7 @@ export function deriveSites({ registered, files, readFile, packageDirs = new Map export function parseDeclaration(source) { const start = source.indexOf('export const UNREGISTERED_CODE_SITES'); if (start < 0) throw new Error(`${DECLARATION}: UNREGISTERED_CODE_SITES not found — the anchor moved.`); - const body = stripComments(source.slice(start)); + const body = maskComments(source).slice(start); const end = body.indexOf('\n];'); if (end < 0) throw new Error(`${DECLARATION}: UNREGISTERED_CODE_SITES terminator not found — the anchor moved.`); const rows = []; @@ -579,7 +577,7 @@ export function checkDoorTyping({ doorSource, files }) { `Point REST_DOOR_FILE at the file that now owns the REST error doors.`); return findings; } - const stripped = stripComments(doorSource); + const stripped = maskComments(doorSource); // ① the author-side door narrows to the closed vocabulary const decl = /export\s+function\s+sendDeclaredFault\s*\(([\s\S]*?)\)\s*:\s*void/.exec(stripped); @@ -604,7 +602,7 @@ export function checkDoorTyping({ doorSource, files }) { // ③ decided refusals do not travel through the `any` door for (const { rel, source } of files) { if (!rel.startsWith('packages/rest/')) continue; - const s = stripComments(source); + const s = maskComments(source); DECIDED_THROUGH_THROWN_RE.lastIndex = 0; for (const m of s.matchAll(DECIDED_THROUGH_THROWN_RE)) { add(`${rel}: hands the author-declared code '${m[1]}' to \`sendThrownError\`, whose \`error\` ` + diff --git a/scripts/check-error-code-casing.mjs b/scripts/check-error-code-casing.mjs index 69f5a9623e..142e465d9c 100644 --- a/scripts/check-error-code-casing.mjs +++ b/scripts/check-error-code-casing.mjs @@ -51,6 +51,7 @@ */ import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { maskComments } from './js-comment-mask.mjs'; import { join, relative, sep } from 'node:path'; const ROOT = new URL('..', import.meta.url).pathname.replace(/\/$/, ''); @@ -124,15 +125,10 @@ function walk(dir, out = []) { return out; } -/** Strip line and block comments so a docblock naming an old code is not a hit. */ -function stripComments(src) { - return src - .replace(/\/\*[\s\S]*?\*\//g, (m) => m.replace(/[^\n]/g, ' ')) - .replace(/(^|[^:])\/\/[^\n]*/g, (m, p) => p + m.slice(p.length).replace(/./g, ' ')); -} + export function findViolations(src, file) { - const text = stripComments(src); + const text = maskComments(src); const hits = []; for (const { name, re } of CODE_POSITION_PATTERNS) { re.lastIndex = 0; diff --git a/scripts/check-error-status-conformance.mjs b/scripts/check-error-status-conformance.mjs index a362c0ee81..4715e2c481 100644 --- a/scripts/check-error-status-conformance.mjs +++ b/scripts/check-error-status-conformance.mjs @@ -110,6 +110,7 @@ // `scripts/error-status-unpinned-baseline.json`; a NEW one fails the gate, and a // row that becomes pinned fails it too (ratchet down with `--update`). import { readdirSync, readFileSync, writeFileSync, statSync, existsSync } from 'node:fs'; +import { maskComments } from './js-comment-mask.mjs'; import { join, relative } from 'node:path'; const SCAN_ROOT = 'packages'; @@ -176,7 +177,7 @@ export function buildConstantIndex(sources) { // error family and reporting three declarations it could have resolved. const COMPUTED_ENTRY = /\[\s*([A-Za-z_$][\w$]*(?:\.[A-Za-z_$][\w$]*)?)\s*\]\s*:\s*(?:'([^'\n]*)'|"([^"\n]*)"|(\d{3}))/g; - const clean = [...sources.values()].map(stripComments); + const clean = [...sources.values()].map((src) => maskComments(src)); const objectBodies = []; for (const src of clean) { for (const m of src.matchAll(SCALAR)) put(m[1], m[2] ?? m[3] ?? Number(m[4])); @@ -231,54 +232,23 @@ function lookup(e, index) { // ─────────────────────────────────────────────────────────────────────────── /** - * Blank out comments (keeping byte offsets, so reported line numbers stay - * true), because this repo DOCUMENTS envelopes in prose. + * Comment masking, shared. See `scripts/js-comment-mask.mjs` for the scanner. * - * Not a tidiness measure — it is load-bearing. Two of this gate's first findings - * on `main` were docblocks: `quickjs-runner.ts` narrates the bug it fixed - * ("the action surface answered `{ code: 'RECORD_NOT_FOUND', httpStatus: 400 }`") - * and `protocol.ts` names a shape it exists to PREVENT ("would mint incoherent - * rows — `{ code: 'INTERNAL_ERROR', httpStatus: 409 }`"). Read as producers, - * both manufacture a status disagreement out of a sentence saying the opposite. - * A deriver that mines prose is not a deriver. + * Not a tidiness measure -- it is load-bearing here. Two of this gate's first + * findings on `main` were docblocks: `quickjs-runner.ts` narrates the bug it + * fixed ("the action surface answered `{ code: 'RECORD_NOT_FOUND', httpStatus: + * 400 }`") and `protocol.ts` names a shape it exists to PREVENT ("would mint + * incoherent rows -- `{ code: 'INTERNAL_ERROR', httpStatus: 409 }`"). Read as + * producers, both manufacture a status disagreement out of a sentence saying + * the opposite. A deriver that mines prose is not a deriver. * - * Strings and template literals are tracked only so a `//` or `/*` inside one - * cannot open a phantom comment; their contents are left intact. + * The private scanner this replaces tracked strings and templates but read a + * `/` as division, so a regex literal holding a quote character opened a + * phantom string -- and because a scanner SKIPS string spans, every comment + * inside one went unmasked, which is the FABRICATING direction, not the + * blinding one. Byte offsets still survive the mask, so reported line numbers + * stay true. */ -export function stripComments(src) { - const out = src.split(''); - let i = 0; - const n = src.length; - while (i < n) { - const c = src[i]; - const d = src[i + 1]; - if (c === '/' && d === '/') { - while (i < n && src[i] !== '\n') { out[i] = ' '; i++; } - continue; - } - if (c === '/' && d === '*') { - while (i < n && !(src[i] === '*' && src[i + 1] === '/')) { if (src[i] !== '\n') out[i] = ' '; i++; } - if (i < n) { out[i] = ' '; out[i + 1] = ' '; i += 2; } - continue; - } - if (c === "'" || c === '"' || c === '`') { - const quote = c; - // A `'` or `"` never spans a line in JS, so a quote character mis-read - // out of a regex character class (`/["'`]/`) recovers at end of line - // instead of swallowing the rest of the file. - const bounded = quote !== '`'; - i++; - while (i < n && src[i] !== quote && !(bounded && src[i] === '\n')) { - if (src[i] === '\\') i++; - i++; - } - i++; - continue; - } - i++; - } - return out.join(''); -} /** Brace-matched class bodies, so a property read never escapes its class. */ function classBodies(src) { @@ -324,7 +294,7 @@ export function deriveRuntimeStatuses(sources, index) { unresolved.push(`${where}: code=${String(code).trim()} status=${String(status).trim()}`); for (const [path, raw] of sources) { - const src = stripComments(raw); + const src = maskComments(raw); // R1 — error classes declaring their own code + status/statusCode. for (const { name, body } of classBodies(src)) { const codeM = /^[ \t]*(?:public\s+|protected\s+|private\s+)?readonly\s+code\s*(?::[^=\n]+)?=\s*([^;\n]+);/m.exec(body); diff --git a/scripts/check-examples-live-imports.mjs b/scripts/check-examples-live-imports.mjs index c902127049..7a32c1454f 100644 --- a/scripts/check-examples-live-imports.mjs +++ b/scripts/check-examples-live-imports.mjs @@ -106,6 +106,7 @@ // that has one AND mentions `examples/` in its text, under `unresolved`. import { readFileSync, readdirSync, statSync, existsSync } from 'node:fs'; +import { maskComments } from './js-comment-mask.mjs'; import { join, resolve, relative, dirname, sep, posix } from 'node:path'; import { fileURLToPath } from 'node:url'; import process from 'node:process'; @@ -163,54 +164,20 @@ const TEST_DIR_RE = /(?:^|\/)(?:test|tests|__tests__)(?:\/|$)/; const rel = (abs) => relative(REPO_ROOT, abs).split(sep).join('/'); /** - * Strip line and block comments while preserving string and template contents. + * Comment masking, shared. See `scripts/js-comment-mask.mjs` for the scanner. * * A comment mentioning `examples/app-showcase` is NOT a coupling -- that is the * false-positive class a bare grep produces -- but the specifiers we DO want * are themselves string literals, so a blunt "drop everything quoted" pass * would erase the signal. Hence a real scanner rather than a regex. + * + * The private copy this replaces was string-aware but read a `/` as division, + * so a regex literal holding a quote character opened a phantom string that ran + * to the next matching quote -- to end of file for a backtick. Comments inside + * that span were never removed, so the gate read commented-out imports as live + * ones. Measured on this gate's own corpus, the two worst files were showcase + * tests carrying exactly that shape. */ -function stripComments(src) { - let out = ''; - let i = 0; - const n = src.length; - while (i < n) { - const c = src[i]; - const d = src[i + 1]; - if (c === '/' && d === '/') { - while (i < n && src[i] !== '\n') i++; - continue; - } - if (c === '/' && d === '*') { - i += 2; - while (i < n && !(src[i] === '*' && src[i + 1] === '/')) i++; - i += 2; - continue; - } - if (c === '"' || c === "'" || c === '`') { - const quote = c; - out += c; - i++; - while (i < n) { - if (src[i] === '\\') { - out += src[i] + (src[i + 1] ?? ''); - i += 2; - continue; - } - out += src[i]; - if (src[i] === quote) { - i++; - break; - } - i++; - } - continue; - } - out += c; - i++; - } - return out; -} /** Every string-literal specifier reachable as a live module reference. */ function moduleSpecifiers(code) { @@ -441,7 +408,7 @@ function collect() { continue; } if (!raw.includes('example')) continue; - const code = stripComments(raw); + const code = maskComments(raw); const refs = [...moduleSpecifiers(code), ...readPathLiterals(code)]; const couplings = []; @@ -705,7 +672,7 @@ function selfTest() { const fakeAbs = join(REPO_ROOT, 'packages/cli/test/x.test.ts'); const detects = (src) => { - const code = stripComments(src); + const code = maskComments(src); const refs = [...moduleSpecifiers(code), ...readPathLiterals(code)]; return refs.some(({ spec }) => couplingTarget(spec, fakeAbs, apps) !== null); }; diff --git a/scripts/check-platform-checklist.mjs b/scripts/check-platform-checklist.mjs index b5bacb0412..f0133165f4 100644 --- a/scripts/check-platform-checklist.mjs +++ b/scripts/check-platform-checklist.mjs @@ -39,6 +39,7 @@ // Usage: node scripts/check-platform-checklist.mjs (pnpm check:platform-checklist) import { readdirSync, readFileSync, existsSync } from 'node:fs'; +import { maskComments } from './js-comment-mask.mjs'; import { join, basename } from 'node:path'; const ROOT = new URL('..', import.meta.url).pathname; @@ -167,7 +168,12 @@ for (const file of files) { // exists to force. Extractor rot is loud, not fail-open: a missing file or // export is an error, never a silent skip. function extractEnumMembers(absFile, exportName) { - const src = readFileSync(absFile, 'utf8'); + // Masked ONCE, up front, rather than per-segment: comment spans are blanked + // in place, so every offset below still indexes the real file, and the + // bracket walk can no longer be closed early by a `]` that lives in a + // comment. Masking a SLICE would be the same defect one level down -- a + // fragment starting mid-file has no literal context to scan from. + const src = maskComments(readFileSync(absFile, 'utf8')); const decl = src.match(new RegExp(`(?:export\\s+)?const\\s+${exportName.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b[^=]*=`)); if (!decl) return null; const start = src.indexOf('[', decl.index + decl[0].length); @@ -178,9 +184,7 @@ function extractEnumMembers(absFile, exportName) { else if (src[j] === ']') { depth--; if (depth === 0) { - const seg = src.slice(start, j + 1) - .replace(/\/\/[^\n]*/g, '') - .replace(/\/\*[\s\S]*?\*\//g, ''); + const seg = src.slice(start, j + 1); const seen = new Set(); for (const m of seg.matchAll(/'([a-zA-Z0-9_\-]+)'/g)) seen.add(m[1]); return [...seen]; diff --git a/scripts/check-test-source-alias.mjs b/scripts/check-test-source-alias.mjs index 0b0db41a92..17d1c2fe15 100644 --- a/scripts/check-test-source-alias.mjs +++ b/scripts/check-test-source-alias.mjs @@ -172,6 +172,7 @@ // node scripts/check-test-source-alias.mjs --self-test import { readFileSync, readdirSync, statSync, existsSync, mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { stripComments } from './js-comment-mask.mjs'; import { join, resolve, relative, dirname } from 'node:path'; import { fileURLToPath } from 'node:url'; import { tmpdir } from 'node:os'; @@ -672,66 +673,23 @@ function findVitestConfig(dir) { return null; } -const REGEX_CAN_START_AFTER = new Set(['(', ',', '=', ':', '[', '!', '&', '|', '?', '{', '}', ';', '+', '-', '*', '%', '~', '^', '<', '>', '']); - /** - * Remove comments, leaving strings, template literals and regex literals - * intact. Needed because these configs carry long rationale comments that name - * the very specifiers being matched — reading one as an alias would report a - * package as safe on the strength of a paragraph about why it is not. + * Comment stripping, shared. See `scripts/js-comment-mask.mjs` for the scanner + * and for why this gate takes the deleting projection rather than the blanking + * one (its import regex is lazy, and blanking is quadratic over what it leaves). + * + * Needed because these configs carry long rationale comments that name the very + * specifiers being matched -- reading one as an alias would report a package as + * safe on the strength of a paragraph about why it is not. + * + * The private copy this replaces knew about strings AND regex literals, but + * decided "is this `/` a regex?" from the preceding CHARACTER alone. That misses + * the keyword forms -- `return /["`]/.test(s)`, `case /['`]/.test(x)` -- where a + * value character precedes and only the keyword tells regex from division. The + * shared scanner carries the keyword set. Latent on today's corpus (34 vitest + * configs, none carrying the shape), which is why it is pinned by shape in the + * shared module's self-test rather than by this gate's corpus. */ -function stripComments(src) { - let out = ''; - let previous = ''; - let i = 0; - while (i < src.length) { - const c = src[i]; - const next = src[i + 1]; - if (c === '/' && next === '/') { - while (i < src.length && src[i] !== '\n') i++; - continue; - } - if (c === '/' && next === '*') { - i += 2; - while (i < src.length && !(src[i] === '*' && src[i + 1] === '/')) i++; - i += 2; - continue; - } - if (c === '"' || c === "'" || c === '`') { - out += c; - i++; - while (i < src.length) { - if (src[i] === '\\') { - out += src[i] + (src[i + 1] ?? ''); - i += 2; - continue; - } - out += src[i]; - if (src[i] === c) { - i++; - break; - } - i++; - } - previous = c; - continue; - } - if (c === '/' && REGEX_CAN_START_AFTER.has(previous)) { - const end = scanRegexLiteral(src, i); - if (end > 0) { - out += src.slice(i, end); - i = end; - previous = '/'; - continue; - } - } - out += c; - if (!/\s/.test(c)) previous = c; - i++; - } - return out; -} - /** End index (exclusive) of the regex literal starting at `start`, or -1. */ function scanRegexLiteral(src, start) { let i = start + 1; diff --git a/scripts/js-comment-mask.mjs b/scripts/js-comment-mask.mjs new file mode 100644 index 0000000000..26f45ffca8 --- /dev/null +++ b/scripts/js-comment-mask.mjs @@ -0,0 +1,310 @@ +#!/usr/bin/env node +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * js-comment-mask -- the ONE answer to "is this span a comment, or code?" + * + * node scripts/js-comment-mask.mjs --self-test + * + * Every source-scanning gate in this tree has to separate code from prose + * before it decides anything: a docblock naming a retired error code is not a + * producer of that code, and a paragraph explaining why an alias is wrong is + * not an alias. Each gate used to answer that question with its own private + * `stripComments`, and the copies had drifted into two families with two + * different failure modes -- both silent, and in opposite directions. + * + * ## The two families, and why neither was safe + * + * **Naive regex** -- `src.replace(/\/\*[\s\S]*?\*\//g, '')` and a `//`-to-end- + * of-line rule. A regex has no idea what a string literal is, so any source + * carrying a block-comment OPENER inside a string opens a PHANTOM comment that + * runs to the next real terminator, usually a docblock far below, deleting every + * line of real code in between. The trigger shapes are ordinary: a glob, a route wildcard, a URL + * (the `//` rule), a `/*` mentioned inside a template. The gate then reports + * clean over code it never looked at -- the failure direction AGENTS.md names + * as worse than no verifier at all, because it reports success. + * + * **String-aware scanner, regex-blind** -- tracks strings and templates, but + * treats a `/` as division. A regex literal whose character class holds a quote + * character (`/["'`]/` -- and this tree really writes those) opens a phantom + * STRING instead. Because a scanner SKIPS string spans, every comment inside + * the phantom span is never blanked, and the gate reads genuinely commented-out + * text as live code: it FABRICATES a hit rather than missing one. A backtick is + * the worst of them, because a template is not line-bounded and the phantom + * span runs to the next backtick, or to end of file. + * + * So the two questions are one question, and a scanner has to know about all + * three literal forms to answer it. That is what lives here, once. + * + * ## Blank, never delete + * + * Comment spans are replaced with spaces, newlines kept, so every byte offset + * and every line number survives the mask. A gate that deletes comments reports + * findings against line numbers that no longer exist in the file it read, and + * the drift is invisible until someone opens the file at the reported line. + * + * ## The direction it fails in + * + * A shape this scan gets wrong fails toward masking MORE than it should, which + * costs recall (a real finding dropped) and cannot fabricate a lead. That is + * the deliberate direction: a gate that over-masks under-reports loudly the + * next time someone re-derives its scope, while a gate that under-masks + * manufactures findings out of prose and burns a reader's afternoon proving + * the sentence it quoted meant the opposite. + */ + +/** A character that can end an identifier -- i.e. a value, so `/` is division. */ +const IDENT_CHAR = /[\w$]/; + +/** + * Keywords after which a `/` opens a REGEX, not a division. `return /x/` reads + * as a value character followed by a slash, and only the keyword tells them + * apart. Measured cost of omitting this: a gate whose corpus contained + * `return /["`]/.test(s)` fabricated hits out of every comment below it. + */ +const REGEX_AFTER_KEYWORD = new Set([ + 'return', 'typeof', 'instanceof', 'in', 'of', 'case', 'delete', 'void', + 'yield', 'await', 'new', 'do', 'else', 'throw', +]); + +/** + * One left-to-right pass over a JS source, flagging every character as COMMENT + * content and/or LITERAL content (inside a string, template or regex). Both + * come back as same-length byte arrays, so a caller can blank a span without + * moving any other offset. + * + * The literal flag covers a literal's CONTENT, not its delimiters, so a caller + * blanking comments still sees every string intact. Template interiors are + * treated as literal through `${...}` as well: an interpolation's braces are + * balanced by construction, so ignoring them is right for depth counting, and + * a caller reading raw characters sees them either way. + * + * @param {string} source + * @returns {{ comment: Uint8Array, literal: Uint8Array }} + */ +export function scanSource(source) { + const n = source.length; + const comment = new Uint8Array(n); + const literal = new Uint8Array(n); + let i = 0; + let prev = ''; // last significant CODE character + let word = ''; // ...and the identifier it is the tail of, if any + + // A shebang is a comment to node; it is also the one line whose slashes are + // neither division nor a regex. + if (source.startsWith('#!')) { + while (i < n && source[i] !== '\n') comment[i++] = 1; + } + + while (i < n) { + const c = source[i]; + const next = source[i + 1]; + + if (c === '/' && next === '/') { + while (i < n && source[i] !== '\n') comment[i++] = 1; + continue; + } + if (c === '/' && next === '*') { + comment[i++] = 1; + comment[i++] = 1; + while (i < n && !(source[i] === '*' && source[i + 1] === '/')) comment[i++] = 1; + if (i < n) comment[i++] = 1; + if (i < n) comment[i++] = 1; + continue; + } + if (c === "'" || c === '"') { + i++; // the opening quote is code, so a caller can still pair it + while (i < n && source[i] !== c && source[i] !== '\n') { + literal[i] = 1; + if (source[i] === '\\' && i + 1 < n) literal[++i] = 1; + i++; + } + if (i < n && source[i] === c) i++; + prev = 'x'; // a value just ended + word = ''; + continue; + } + if (c === '`') { + i++; + while (i < n && source[i] !== '`') { + literal[i] = 1; + if (source[i] === '\\' && i + 1 < n) literal[++i] = 1; + i++; + } + if (i < n) i++; + prev = 'x'; + word = ''; + continue; + } + if (c === '/' && !(IDENT_CHAR.test(prev) || prev === ')' || prev === ']')) { + i++; // regex literal: `/` after anything that is not a value + let inClass = false; + while (i < n && source[i] !== '\n') { + const ch = source[i]; + if (ch === '\\' && i + 1 < n) { + literal[i] = 1; + literal[++i] = 1; + i++; + continue; + } + if (ch === '[') inClass = true; + else if (ch === ']') inClass = false; + else if (ch === '/' && !inClass) break; + literal[i] = 1; + i++; + } + if (i < n && source[i] === '/') i++; + prev = 'x'; + word = ''; + continue; + } + if (c === '/' && REGEX_AFTER_KEYWORD.has(word)) { + // `return /x/` -- a value character precedes, but it is a keyword. + prev = ''; + word = ''; + continue; // re-read this `/` with prev cleared, as a regex + } + if (!/\s/.test(c)) { + prev = c; + word = IDENT_CHAR.test(c) ? word + c : ''; + } + i++; + } + return { comment, literal }; +} + +/** Replace every flagged character with a space, keeping newlines and offsets. */ +export function blank(source, flags) { + const out = source.split(''); + for (let k = 0; k < out.length; k++) if (flags[k] && out[k] !== '\n') out[k] = ' '; + return out.join(''); +} + +/** + * The source with its COMMENT characters REMOVED but every newline kept. + * + * The same scanner as `maskComments`, projected differently: line NUMBERS + * survive (a comment line becomes an empty line) but byte offsets do not, and + * the text gets much shorter. + * + * ## Why both projections exist, measured + * + * Blanking is the safer default and the only one that can carry a byte offset. + * But a caller that scans the result with a lazy regex pays for every byte the + * mask leaves behind: `check-test-source-alias.mjs` matches imports with + * `(?:import|export)\s+([\s\S]*?)\s*from` , and a lazy `[\s\S]*?` walked + * across the whitespace runs blanking leaves is quadratic in the comment bytes. + * Converting that gate to `maskComments` alone took its runtime from 6.4s to + * 5m27s on this tree -- same verdict, 51x the cost. Deleting the comment + * characters instead restores it, and that gate reports package-level findings, + * so it never needed the offsets. + * + * Pick by what the caller does with the result: reports a LINE or an offset + * into the original text -> `maskComments`; feeds a scanner and reports neither + * -> `stripComments`. + */ +export function stripComments(source) { + const { comment } = scanSource(source); + let out = ''; + for (let k = 0; k < source.length; k++) { + if (!comment[k] || source[k] === '\n') out += source[k]; + } + return out; +} + +/** + * The source with its COMMENT spans blanked -- line, block and shebang. + * + * Strings, templates and regex literals are left INTACT: a gate's signal is + * usually itself a string literal, so "drop everything quoted" would erase the + * thing being looked for. Only prose goes. + */ +export function maskComments(source) { + return blank(source, scanSource(source).comment); +} + +// --------------------------------------------------------------------------- +// Self-test -- the shapes, not the corpus +// --------------------------------------------------------------------------- + +/** + * A green run over today's tree proves only that today's tree lacks the shape. + * These cases ARE the contract, and each one is valid JavaScript, so the + * expected answer is the one the language gives rather than the one a + * particular implementation happens to produce. + * + * `REAL` marks source that must SURVIVE the mask (dropping it BLINDS the gate); + * `GHOST` marks genuinely commented-out text that must NOT survive (keeping it + * makes the gate FABRICATE a finding out of prose). + */ +export function selfTest() { + const BT = String.fromCharCode(96); // backtick, kept out of the literal below + const cases = [ + ['string containing a block-comment opener', + ["const AUTH = '/api/v1/auth/*';", "err.code = 'REAL';", '/** docblock far below */'].join('\n')], + ['URL inside a string', + "const DOCS = 'https://objectstack.ai/docs'; err.code = 'REAL';"], + ['bare // inside a string', + "const GLOB = 'packages//src'; err.code = 'REAL';"], + ['block-comment opener inside a template literal', + ['const HINT = ' + BT + 'use /* to open a block comment' + BT + ';', "err.code = 'REAL';", '/** doc */'].join('\n')], + ['regex character class holding a double quote', + ['const Q = /["\'' + BT + ']/g;', "/* err.code = 'GHOST' */", "err.code = 'REAL';"].join('\n')], + ['regex character class holding a BACKTICK first', + ['const Q = /[' + BT + '\'"]/g;', "/* err.code = 'GHOST' */", "err.code = 'REAL';"].join('\n')], + ['markdown regex carrying a backtick', + ['const H = /^(#{1,6})\\s(.*)$|^(' + BT + '{3,})/gm;', "/* err.code = 'GHOST' */", "err.code = 'REAL';"].join('\n')], + ['regex literal containing an escaped //', + "const P = /https:\\/\\//; err.code = 'REAL';"], + ['line comment immediately after a colon', + ["const m = { a:// err.code = 'GHOST'", " 1 };", "err.code = 'REAL';"].join('\n')], + ['regex literal after the `return` keyword', + ['function f(s) { return /["' + BT + ']/.test(s); }', "/* err.code = 'GHOST' */", "err.code = 'REAL';"].join('\n')], + ['regex literal after the `case` keyword', + ["switch (true) { case /['" + BT + "]/.test(x): break; }", "/* err.code = 'GHOST' */", "err.code = 'REAL';"].join('\n')], + ['shebang is a comment', + ['#!/usr/bin/env node', "/* err.code = 'GHOST' */", "err.code = 'REAL';"].join('\n')], + ['division after a paren, then a quote-bearing regex', + ['const r = (a) / b;', 'const q = /["' + BT + ']/g;', "/* err.code = 'GHOST' */", "err.code = 'REAL';"].join('\n')], + ['a genuine docblock is still removed', + ['/** Retired: err.code = ' + "'GHOST'" + ' must never come back. */', "err.code = 'REAL';"].join('\n')], + ['a genuine line comment is still removed', + ["// err.code = 'GHOST'", "err.code = 'REAL';"].join('\n')], + ]; + + // Both projections are driven, on every shape: they share one scanner, so a + // shape either family gets wrong is a scanner bug, and a disagreement between + // them about what IS a comment is the thing that must never ship. + let failed = 0; + for (const [name, src] of cases) { + const masked = maskComments(src); + const stripped = stripComments(src); + const problems = []; + for (const [proj, out] of [['mask', masked], ['strip', stripped]]) { + if (/REAL/.test(src) && !/REAL/.test(out)) problems.push(`${proj}: BLINDS (real code removed)`); + if (/GHOST/.test(src) && /GHOST/.test(out)) problems.push(`${proj}: FABRICATES (comment text survived)`); + if (src.split('\n').length !== out.split('\n').length) problems.push(`${proj}: line count changed`); + } + if (masked.length !== src.length) problems.push(`mask: offset drift (${src.length} -> ${masked.length})`); + if (stripped.length > src.length) problems.push('strip: grew'); + if (problems.length) failed++; + console.log(` ${problems.length ? '\u2717' : '\u2713'} ${name}${problems.length ? ' -- ' + problems.join('; ') : ''}`); + } + + if (failed) { + console.error(`\u2717 js-comment-mask self-test: ${failed} of ${cases.length} case(s) failed.`); + process.exit(1); + } + console.log(`\u2713 js-comment-mask self-test: ${cases.length} cases pass.`); +} + +// Executed only as a CLI. Importing this module must have NO side effect: the +// gates below it are the callers, and a shared module that exits on import is +// a shared module nobody can share. +if (process.argv[1] && import.meta.url === new URL(`file://${process.argv[1]}`).href) { + if (process.argv.includes('--self-test')) selfTest(); + else { + console.error('usage: node scripts/js-comment-mask.mjs --self-test'); + process.exit(2); + } +} diff --git a/scripts/pm/dispatch-gates.mjs b/scripts/pm/dispatch-gates.mjs index 68a01c954a..ce70d250b0 100644 --- a/scripts/pm/dispatch-gates.mjs +++ b/scripts/pm/dispatch-gates.mjs @@ -110,6 +110,7 @@ import { isExtractConfigPath, isMetadataFormModulePath, } from '../i18n-bundle-surface.mjs'; +import { blank, maskComments, scanSource } from '../js-comment-mask.mjs'; // Re-exported so this tool's self-test drives the SAME predicates the gate // runs, not copies of them. They used to be written twice — see the shared @@ -459,167 +460,26 @@ export function resolveCheckToFiles(checkName, scriptsMap) { } /** - * Identifier characters, for the regex-vs-division decision in `scanSource`. - */ -const IDENT_CHAR = /[\w$]/; - -/** - * The keywords after which a `/` opens a REGEX rather than dividing. Every - * other case is decided by the preceding character: a `/` that follows a value - * (identifier, number, `)`, `]`, or a closed literal) divides; anything else - * opens a regex. - */ -const REGEX_AFTER_KEYWORD = new Set([ - 'return', 'typeof', 'instanceof', 'in', 'of', 'case', 'delete', 'void', - 'yield', 'await', 'new', 'do', 'else', 'throw', -]); - -/** - * One left-to-right pass over a JS source, flagging every character as COMMENT - * content and/or LITERAL content (inside a string, template or regex). Both - * come back as same-length byte arrays, so a caller can blank a span without - * moving any other offset. - * - * ## Why a scan, and why it has to know about regex literals - * - * The two questions the callers below ask — "is this `//` a comment or the - * middle of a URL?" and "is this `}` the end of a function or a character in a - * fixture?" — are precisely the ones a regex over raw text cannot answer. - * Measured on this tree, both traps are real: `release-github-releases.mjs` - * carries `'https://github.com'` in its module body (a `//` a line-comment rule - * would swallow the rest of the line for), and its own markdown regex - * `/^(#{1,6})\s(.*)$|^(`{3,})/gm` contains a BACKTICK — a scanner that skipped - * regex literals would open a template literal there and treat everything to - * the next backtick, hundreds of lines later, as string content. This file's - * hint regex below puts all three quote characters inside a regex literal for - * the same reason. Regex literals also carry unbalanced `{`/`}` (`{1,6}`), so - * the brace counting in `maskSelfTests` needs them flagged too. - * - * The literal flag covers a literal's CONTENT, not its delimiters, so a caller - * blanking comments still sees every string intact. Template interiors are - * treated as literal through `${…}` as well: an interpolation's braces are - * balanced by construction, so ignoring them is right for depth counting, and - * the hint scan reads the raw characters either way. - * - * A shape this scan gets wrong fails toward masking MORE than it should, which - * costs recall (a real hint dropped) and cannot fabricate a lead — the - * direction this whole file's "22 leads is the same as none" note asks for. - */ -function scanSource(source) { - const n = source.length; - const comment = new Uint8Array(n); - const literal = new Uint8Array(n); - let i = 0; - let prev = ''; // last significant CODE character - let word = ''; // …and the identifier it is the tail of, if any - - // A shebang is a comment to node; it is also the one line whose slashes are - // neither division nor a regex. - if (source.startsWith('#!')) { - while (i < n && source[i] !== '\n') comment[i++] = 1; - } - - while (i < n) { - const c = source[i]; - const next = source[i + 1]; - - if (c === '/' && next === '/') { - while (i < n && source[i] !== '\n') comment[i++] = 1; - continue; - } - if (c === '/' && next === '*') { - comment[i++] = 1; - comment[i++] = 1; - while (i < n && !(source[i] === '*' && source[i + 1] === '/')) comment[i++] = 1; - if (i < n) comment[i++] = 1; - if (i < n) comment[i++] = 1; - continue; - } - if (c === "'" || c === '"') { - i++; // the opening quote is code, so the hint scan can still pair it - while (i < n && source[i] !== c && source[i] !== '\n') { - literal[i] = 1; - if (source[i] === '\\' && i + 1 < n) literal[++i] = 1; - i++; - } - if (i < n && source[i] === c) i++; - prev = 'x'; // a value just ended - word = ''; - continue; - } - if (c === '`') { - i++; - while (i < n && source[i] !== '`') { - literal[i] = 1; - if (source[i] === '\\' && i + 1 < n) literal[++i] = 1; - i++; - } - if (i < n) i++; - prev = 'x'; - word = ''; - continue; - } - if (c === '/' && !(IDENT_CHAR.test(prev) || prev === ')' || prev === ']')) { - i++; // regex literal: `/` after anything that is not a value - let inClass = false; - while (i < n && source[i] !== '\n') { - const ch = source[i]; - if (ch === '\\' && i + 1 < n) { - literal[i] = 1; - literal[++i] = 1; - i++; - continue; - } - if (ch === '[') inClass = true; - else if (ch === ']') inClass = false; - else if (ch === '/' && !inClass) break; - literal[i] = 1; - i++; - } - if (i < n && source[i] === '/') i++; - prev = 'x'; - word = ''; - continue; - } - if (c === '/' && REGEX_AFTER_KEYWORD.has(word)) { - // `return /x/` — a value character precedes, but it is a keyword. - prev = ''; - word = ''; - continue; // re-read this `/` with prev cleared, as a regex - } - if (!/\s/.test(c)) { - prev = c; - word = IDENT_CHAR.test(c) ? word + c : ''; - } - i++; - } - return { comment, literal }; -} - -/** Replace every flagged character with a space, keeping newlines and offsets. */ -function blank(source, flags) { - const out = source.split(''); - for (let k = 0; k < out.length; k++) if (flags[k] && out[k] !== '\n') out[k] = ' '; - return out.join(''); -} - -/** - * The source with its COMMENT spans blanked — line, block and shebang. + * The comment/literal scanner used to live here. It now lives in + * `scripts/js-comment-mask.mjs`, because five source-scanning GATES needed the + * same judgment and each had grown a private copy that got it wrong in one of + * two silent ways -- see that module's header for the two failure families and + * the shapes its self-test pins. * - * ## Why comments must not contribute hints + * ## Why comments must not contribute hints (the reason it was written here) * * A gate's header discusses the tree at length, and the hint scan accepts * backticks, so every backticked path in a header used to be read as a path the * gate operates on. Measured, and self-inflicted: the first draft of * `scripts/pm/check-dispatch-gates.mjs` explained this very pollution with each - * path in backticks, and that header alone produced ten hints — reproducing, + * path in backticks, and that header alone produced ten hints -- reproducing, * from the file documenting the problem, the exact false MATCHED leads it was * written to avoid. It ships today with its paths deliberately unquoted, a - * workaround this function retires: naming a path is not reading it. - */ -export function maskComments(source) { - return blank(source, scanSource(source).comment); -} + * workaround `maskComments` retires: naming a path is not reading it. + * + * Re-exported because this tool's self-test drives the SAME masker the gates + * run, not a copy of it. +export { maskComments }; /** * A top-level self-test function DECLARATION. The anchor is structural, not a