diff --git a/packages/cli/test/vitest-tiers-partition.test.ts b/packages/cli/test/vitest-tiers-partition.test.ts new file mode 100644 index 0000000000..6d39c92d73 --- /dev/null +++ b/packages/cli/test/vitest-tiers-partition.test.ts @@ -0,0 +1,219 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * The two tiers of this package's suite stay a PARTITION, and the integration + * list stays equal to what the files DO (#13504). + * + * `vitest.config.ts` splits the suite into two named projects — `unit` (the + * local default) and `integration` (spawns the real CLI or boots a real + * kernel/driver; CI-mandatory, local on demand). Two things can rot under a + * split like that, and both rot silently, which is why this pin exists: + * + * 1. A test file that matches NO project is not run by `vitest run` at all — + * not by the fast tier AND not by `pnpm test` in CI, because with + * `projects` configured the root run IS the union of the projects. A file + * matching BOTH runs twice and reports twice. So the first two cases hold + * `unit ⊎ integration = every test file on disk`, read from vitest's own + * resolution (`vitest list --filesOnly`, with and without `--project`) + * against a filesystem walk — the config's spelling is judged by what + * vitest actually collects, never by re-reading the config. + * + * 2. `INTEGRATION_FILES` is an explicit list, and the `*.e2e.test.ts` NAME is + * not the predicate (the ACCEPT on #13504 measured 18 of 220 files where + * name and behaviour disagree). So the third case re-derives the tier of + * every file from its comment-masked SOURCE and fails when the list and + * the derivation disagree — a new spawner cannot land in the fast tier + * unnoticed, and a stale entry cannot linger. The predicate, in code + * position (comments masked by `scripts/js-comment-mask.mjs`): + * + * SPAWN = calls `runServe(` (the helper in `test/helpers/serve-process.ts` + * whose body spawns the source entry), OR value-imports + * `node:child_process` AND (names an entry basename — the + * `run-dev` / `run` scripts under `bin/` — OR imports `CLI` / + * `TSX` from that helper OR names the `tsx` binary under + * `node_modules/.bin`); + * KERNEL = value-imports `bootSchemaStack` from `schema-migrate`, OR + * value-imports `better-sqlite3`, OR value-imports any + * `@objectstack/driver-*` package, OR constructs `new ObjectQL(`. + * INTEGRATION = SPAWN ∨ KERNEL. + * + * Value imports only: `import type { … } from '@objectstack/driver-sql'` + * loads nothing, a spelling list that SAYS `'better-sqlite3'` opens no + * database, and `expect(deps).toContain('better-sqlite3')` boots nothing — + * every one of those was a false positive of the text-match census this + * predicate replaced. An import statement is one `import … from ''` + * span containing neither `;` nor another `from` (every import in this + * package's tests ends in `;`, measured on 00ff228fe0). + * + * The fourth case classifies THIS file: it imports `node:child_process` (to + * ask vitest for its file lists) and must still read as `unit`, which is the + * predicate's own regression test against matching its own source. + * + * Runs in the `unit` tier and needs no built `dist/`: it spawns `vitest list`, + * which only globs, and reads sources. + */ + +import { execFileSync } from 'node:child_process'; +import { readdirSync, readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; +import { dirname, join, relative, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { describe, expect, it } from 'vitest'; +import { maskComments } from '../../../scripts/js-comment-mask.mjs'; +import { childEnv } from './helpers/serve-process.js'; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const PKG = resolve(HERE, '..'); +const THIS_FILE = relative(PKG, fileURLToPath(import.meta.url)); +const require = createRequire(import.meta.url); +const VITEST_ENTRY = resolve(dirname(require.resolve('vitest/package.json')), 'vitest.mjs'); + +// --------------------------------------------------------------------------- +// The predicate +// --------------------------------------------------------------------------- + +interface ValueImport { + clause: string; + spec: string; +} + +/** One `import … from ''` statement; `import type` is skipped. */ +const IMPORT_RE = /\bimport\s+(?!type\b)((?:(?!\bfrom\b)[^;])*?)\bfrom\s+['"]([^'"]+)['"]/g; + +function valueImports(code: string): ValueImport[] { + const out: ValueImport[] = []; + for (const m of code.matchAll(IMPORT_RE)) out.push({ clause: m[1], spec: m[2] }); + return out; +} + +/** Inline `type X` specifiers do not make a value import of `X`. */ +function importsValue(imports: ValueImport[], spec: RegExp, name?: RegExp): boolean { + return imports.some((i) => spec.test(i.spec) && (!name || name.test(i.clause.replace(/\btype\s+\w+/g, '')))); +} + +export interface TierSignals { + runServe: boolean; + childProcess: boolean; + entryBasename: boolean; + helperCliOrTsx: boolean; + tsxBin: boolean; + bootSchemaStack: boolean; + betterSqlite3: boolean; + driverPackage: boolean; + objectQLCtor: boolean; +} + +export function tierSignals(maskedCode: string): TierSignals { + const imports = valueImports(maskedCode); + return { + runServe: /\brunServe\s*[(]/.test(maskedCode), + childProcess: importsValue(imports, /^(?:node:)?child_process$/), + entryBasename: /\brun(?:-dev)?[.]js\b/.test(maskedCode), + helperCliOrTsx: importsValue(imports, /helpers\/serve-process(?:\.js)?$/, /\b(?:CLI|TSX)\b/), + tsxBin: /[.]bin[/]tsx\b/.test(maskedCode), + bootSchemaStack: importsValue(imports, /schema-migrate(?:\.js)?$/, /\bbootSchemaStack\b/), + betterSqlite3: importsValue(imports, /^better-sqlite3$/) || /require[(]\s*['"]better-sqlite3['"]\s*[)]/.test(maskedCode), + driverPackage: importsValue(imports, /^@objectstack\/driver-/), + objectQLCtor: /new\s+ObjectQL\s*[(]/.test(maskedCode), + }; +} + +export function isIntegration(s: TierSignals): boolean { + const spawn = s.runServe || (s.childProcess && (s.entryBasename || s.helperCliOrTsx || s.tsxBin)); + const kernel = s.bootSchemaStack || s.betterSqlite3 || s.driverPackage || s.objectQLCtor; + return spawn || kernel; +} + +function firedSignals(s: TierSignals): string { + return (Object.keys(s) as Array).filter((k) => s[k]).join(', ') || 'none'; +} + +// --------------------------------------------------------------------------- +// The two readings: the filesystem, and vitest's own resolution +// --------------------------------------------------------------------------- + +const TEST_FILE_RE = /\.(?:test|spec)\.[cm]?[jt]sx?$/; +const SKIP_DIRS = new Set(['node_modules', 'dist', '.git', '.turbo', 'coverage']); + +function walk(dir: string, out: string[] = []): string[] { + for (const entry of readdirSync(dir, { withFileTypes: true })) { + if (SKIP_DIRS.has(entry.name)) continue; + const abs = join(dir, entry.name); + if (entry.isDirectory()) walk(abs, out); + else if (TEST_FILE_RE.test(entry.name)) out.push(relative(PKG, abs)); + } + return out; +} + +/** `vitest list --filesOnly [--project NAME]`, one relative path per line. */ +function vitestFiles(project?: string): string[] { + const args = [VITEST_ENTRY, 'list', '--filesOnly', ...(project ? ['--project', project] : [])]; + const out = execFileSync(process.execPath, args, { + cwd: PKG, + env: childEnv(), + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'], + }); + return out + .split('\n') + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => line.replace(/^\[[^\]]+\]\s+/, '')); +} + +/** The explicit list as `vitest.config.ts` declares it — read as text, not imported. */ +function declaredIntegrationFiles(): string[] { + const masked = maskComments(readFileSync(join(PKG, 'vitest.config.ts'), 'utf8')); + const block = /INTEGRATION_FILES\s*=\s*\[([^\]]*)\]/.exec(masked); + if (!block) throw new Error('vitest.config.ts no longer declares `INTEGRATION_FILES = [ … ]`'); + return Array.from(block[1].matchAll(/['"]([^'"]+)['"]/g), (m) => m[1]); +} + +const sorted = (xs: Iterable): string[] => [...xs].sort(); + +describe('the two tiers of packages/cli (#13504)', () => { + const onDisk = sorted(walk(PKG)); + const all = sorted(vitestFiles()); + const unit = sorted(vitestFiles('unit')); + const integration = sorted(vitestFiles('integration')); + + it('every test file on disk is one vitest collects with no --project (what `pnpm test` runs)', () => { + expect(onDisk.length).toBeGreaterThan(100); + expect(all, 'vitest run collects a different population than the filesystem holds').toEqual(onDisk); + }); + + it('unit and integration partition that population — no file in both, none in neither', () => { + const inBoth = unit.filter((f) => integration.includes(f)); + expect(inBoth, 'files matched by BOTH projects (they would run and report twice)').toEqual([]); + const union = sorted([...unit, ...integration]); + expect(union, 'files matched by NEITHER project fall out of every tier, including CI').toEqual(all); + }); + + it('INTEGRATION_FILES equals the behavioural predicate over every file on disk', () => { + const declared = declaredIntegrationFiles(); + expect(sorted(new Set(declared)), 'INTEGRATION_FILES carries a duplicate').toEqual(sorted(declared)); + expect(sorted(declared), 'an INTEGRATION_FILES entry names no file vitest can find').toEqual(integration); + + const missing: string[] = []; + const stale: string[] = []; + for (const file of onDisk) { + const signals = tierSignals(maskComments(readFileSync(join(PKG, file), 'utf8'))); + const predicted = isIntegration(signals); + const listed = integration.includes(file); + if (predicted && !listed) missing.push(`${file} [${firedSignals(signals)}]`); + if (!predicted && listed) stale.push(file); + } + expect( + missing, + 'files that spawn the CLI or boot a kernel/driver but are NOT in INTEGRATION_FILES (add them)', + ).toEqual([]); + expect(stale, 'INTEGRATION_FILES entries that neither spawn nor boot (remove them)').toEqual([]); + }); + + it('this pin is itself unit-tier: importing child_process to ask vitest is not spawning the CLI', () => { + const signals = tierSignals(maskComments(readFileSync(join(PKG, THIS_FILE), 'utf8'))); + expect(signals.childProcess).toBe(true); + expect(isIntegration(signals), `fired: ${firedSignals(signals)}`).toBe(false); + expect(unit).toContain(THIS_FILE); + }); +}); diff --git a/packages/cli/vitest.config.ts b/packages/cli/vitest.config.ts index 3d4e6f8929..63212a7d2d 100644 --- a/packages/cli/vitest.config.ts +++ b/packages/cli/vitest.config.ts @@ -492,9 +492,163 @@ // // Before adding a `test` block for speed, re-measure: if `tests` is still the // dominant term, the block is not the lever. -import { defineConfig } from 'vitest/config'; +// +// ## THE TWO TIERS (#13504) — `unit` and `integration`, population on 3b5f8168b5 +// +// Maintainer ruling (2026-09-01): split this suite into two NAMED tiers — a +// unit-fast tier that is the local default and does not monopolise the shared +// verify lock, and a real-kernel integration tier that is CI-mandatory and run +// locally on demand. Nothing is skipped, weakened, deleted or doubled: every +// file below still runs under `pnpm test`, because `vitest run` with no +// `--project` runs every project. The tiers only change what a NARROWED local +// run selects. +// +// pnpm --filter @objectstack/cli exec vitest run --project unit # fast, local default +// pnpm --filter @objectstack/cli exec vitest run --project integration # the real thing, on demand +// pnpm --filter @objectstack/cli test # both — what CI runs +// +// ⛔ THE PREDICATE IS WHAT A FILE DOES, NOT WHAT IT IS CALLED. The ACCEPT on +// #13504 fixed that the `*.e2e.test.ts` name disagrees with behaviour, so a +// tier keyed on the name routes coverage to the wrong place. `INTEGRATION_FILES` +// below is an explicit list, and `test/vitest-tiers-partition.test.ts` (unit +// tier) re-derives it from the comment-masked SOURCE of every test file and +// fails when the two disagree — so a new file that spawns the CLI or boots a +// driver cannot land in the fast tier silently, and a stale entry cannot +// linger. A file is `integration` when, in code position, it either +// +// SPAWNS the real CLI (or this package's TypeScript source in a cold tsx +// child): it calls `runServe(` from `test/helpers/serve-process.ts`, or it +// value-imports `node:child_process` AND names an entry basename +// (`run-dev.js` / `run.js`), or imports `CLI` / `TSX` from that helper, or +// names the `.bin/tsx` binary; +// +// or BOOTS a real kernel or driver in-process: a value import of +// `bootSchemaStack` from `schema-migrate`, of `better-sqlite3`, or of any +// `@objectstack/driver-*` package, or a `new ObjectQL(` construction. +// +// Type-only imports, spelling lists, fixture config objects that merely SAY +// `client: 'better-sqlite3'`, and prose do not count — the pin's own header +// carries the regexes and the false positives they were tuned against. +// +// Population on 3b5f8168b5 (merge of origin/main 2a26536196): 230 files = +// 158 unit + 72 integration (49 spawn the CLI, 24 boot a kernel/driver, 1 +// both). The list moved twice between the first cut (00ff228fe0: 228 = 158 + +// 70) and this one, and the pin caught both in the merge queue: one NEW +// spawner file and one EXISTING file that started constructing `new ObjectQL(` +// — the second is the shape a name-based tier can never see. Reconciled +// against the #13872 census (f532630d02, 220 files, 35 spawners / 29 +// kernel-booters / 1 both): +// on that same tree this predicate finds 46 spawners and 22 kernel-booters. +// The spawner side GROWS by 11 files the basename census could not see — six +// that spawn only through `runServe()` and five through the helper's exported +// `CLI` path constant — and SHRINKS by three that name an entry basename in an +// assertion without importing `child_process` at all. The kernel side shrinks +// because the census counted text matches: two `CONTRACT_ONLY_SPELLINGS` +// lists, a banner fixture, a connection-display formatter, a scaffold +// dependency assertion and two `import type { … } from '@objectstack/driver-*'` +// are not boots. With the behavioural predicate the name-vs-behaviour +// disagreement is 5 files (4 spawn without the `.e2e` name, 1 carries the name +// and spawns plain node), down from the census's 18. +// +// WHAT THE SPLIT COSTS AND BUYS, priced from the #13872 attribution above: the +// 70 integration files hold the 73.4% of test-body time that belongs to the +// spawners plus the 3.4% of the kernel-booters; the unit tier is the remaining +// ~23% of test-body time plus the per-file import floor, which is the part the +// split cannot move. The unit tier's measured wall on this box is recorded in +// the PR that landed this section; re-measure it when the population moves, +// and print the commit here. +// +// ⚠️ INLINE PROJECTS INHERIT NOTHING BY DEFAULT — `extends: true` is what +// carries this file's `resolve.alias` table and `test.server.deps.external` +// into each project (vitest 4.1.10: an inline project without it gets a fresh +// Vite config, so the source aliases the gate above guards would be declared +// here and enforced nowhere). `disableConsoleIntercept: true` is repeated +// inside every project because `check:console-intercept-disarm` measured the +// root-level setting inert under projects. `exclude` for the unit tier spreads +// `configDefaults.exclude` first: an `exclude` that names only the integration +// files would drop the `node_modules` exclusion and start collecting +// dependencies' own test files. +import { configDefaults, defineConfig } from 'vitest/config'; import path from 'path'; +// The integration tier, by MEASURED behaviour (see the section above; the pin +// test `test/vitest-tiers-partition.test.ts` keeps this list equal to what the +// files do). Relative to this package root; each entry is an exact path. +export const INTEGRATION_FILES = [ + 'src/adr-0048-app-split.test.ts', + 'src/commands/meta/delete-reset-carriers.test.ts', + 'src/commands/migrate/duplicates.contract.test.ts', + 'src/commands/migrate/duplicates.created-at-canonical.test.ts', + 'src/commands/migrate/duplicates.integration.test.ts', + 'src/commands/migrate/duplicates.null-seam.test.ts', + 'src/commands/migrate/duplicates.pre-repair.test.ts', + 'src/commands/migrate/meta.stored-flow-resolution.integration.test.ts', + 'src/commands/migrate/multi-value-columns.dialect-probe.test.ts', + 'src/commands/migrate/multi-value-columns.dry-run.test.ts', + 'src/commands/secret/orphans.guards.test.ts', + 'src/commands/validate-json-strict-exit.e2e.test.ts', + 'src/utils/artifact-boot-migration.report-only-drift.test.ts', + 'src/utils/platform-migrations-arming.integration.test.ts', + 'src/utils/schema-migrate.deferred-ddl.integration.test.ts', + 'src/utils/schema-migrate.host-composition.integration.test.ts', + 'src/utils/schema-migrate.integration.test.ts', + 'src/utils/schema-migrate.readonly-probe.integration.test.ts', + 'src/utils/schema-migrate.teardown.integration.test.ts', + 'src/utils/schema-migration-plugins.declaration-boot-write-guard.test.ts', + 'src/utils/secret-reference-union.test.ts', + 'src/utils/sqlite-occupancy.test.ts', + 'src/utils/sys-secret-orphan-sweep.test.ts', + 'src/utils/unmanaged-tables.integration.test.ts', + 'test/artifact-pinned-boot.e2e.test.ts', + 'test/authoring-rule-command-parity.test.ts', + 'test/build-json-advisory-parity.e2e.test.ts', + 'test/build-json-failure-conversions.e2e.test.ts', + 'test/build-json-failure-warnings.e2e.test.ts', + 'test/build-json-undeclared-key-parity.e2e.test.ts', + 'test/build-multi-package-artifact.e2e.test.ts', + 'test/cloud-login-json-ndjson.e2e.test.ts', + 'test/compile-artifact-packages.e2e.test.ts', + 'test/emit-json-pipe.test.ts', + 'test/format-zod-union.test.ts', + 'test/generate-agent-retired.e2e.test.ts', + 'test/generate-skill.e2e.test.ts', + 'test/hook-body-build-reach.e2e.test.ts', + 'test/init-created-files-summary.e2e.test.ts', + 'test/invocation-loudness.e2e.test.ts', + 'test/json-stdout-purity.e2e.test.ts', + 'test/lint-conversion-notices.e2e.test.ts', + 'test/login-json-ndjson.e2e.test.ts', + 'test/login-json-noninteractive.e2e.test.ts', + 'test/metadata-type-schema-gate.test.ts', + 'test/migrate-apply-refuses-before-ddl.e2e.test.ts', + 'test/migrate-exit-code.e2e.test.ts', + 'test/migrate-meta.e2e.test.ts', + 'test/migrate-plan-exits.e2e.test.ts', + 'test/migrate-unloadable-host-config-exit.e2e.test.ts', + 'test/qa-empty-glob-exit-code.e2e.test.ts', + 'test/run-dev-unbuilt-workspace.e2e.test.ts', + 'test/serve-app-anchored-optional-import.e2e.test.ts', + 'test/serve-app-runtime-hooks.e2e.test.ts', + 'test/serve-boot-diagnostics.e2e.test.ts', + 'test/serve-host-fallback-base.e2e.test.ts', + 'test/serve-mcp-capability-collision.e2e.test.ts', + 'test/serve-mcp-stdio-answers.e2e.test.ts', + 'test/serve-no-artifact.e2e.test.ts', + 'test/serve-node-env-production-default.e2e.test.ts', + 'test/serve-organizations-host-resolution.e2e.test.ts', + 'test/serve-organizations-mount-failure.e2e.test.ts', + 'test/serve-port-drift-notice.e2e.test.ts', + 'test/serve-port-readback.e2e.test.ts', + 'test/serve-process-child-env.e2e.test.ts', + 'test/serve-publishes-bound-port.e2e.test.ts', + 'test/serve-stdio-stdout-purity.e2e.test.ts', + 'test/start-port-banner-agreement.e2e.test.ts', + 'test/validate-json-failure-conversions.e2e.test.ts', + 'test/validate-json-failure-warnings.e2e.test.ts', + 'test/validate-json-warning-parity.e2e.test.ts', + 'test/validate-top-level-strict.e2e.test.ts', +]; + export default defineConfig({ resolve: { // Array form with an ANCHORED pattern, per the trap the gate documents: @@ -555,5 +709,27 @@ export default defineConfig({ external: [/packages[\/]types[\/]dist/], }, }, + // The two tiers (#13504) — see the header section of the same name. Both + // `extends: true` so each project inherits the `resolve.alias` table and + // the `server.deps.external` entry above; each repeats the console-intercept + // disarm because the root-level one is inert under projects. + projects: [ + { + extends: true, + test: { + name: 'unit', + disableConsoleIntercept: true, + exclude: [...configDefaults.exclude, ...INTEGRATION_FILES], + }, + }, + { + extends: true, + test: { + name: 'integration', + disableConsoleIntercept: true, + include: INTEGRATION_FILES, + }, + }, + ], }, });