src: make compile cache zstd dictionary reproducible

tools/train_compile_cache_dict.mjs

@@ -0,0 +1,168 @@
+#!/usr/bin/env node
+// =============================================================================
+// train_compile_cache_dict.mjs
+//
+// Maintainer tool that regenerates src/compile_cache_zstd.dict, the zstd
+// dictionary embedded in the node binary and used to shrink compile-cache
+// entries on disk (see src/compile_cache.cc).
+//
+// -----------------------------------------------------------------------------
+// What it does
+// -----------------------------------------------------------------------------
+// The node compile cache stores V8 code caches (the bytecode/metadata blob V8
+// produces when it compiles a module) so later runs can skip recompilation.
+// These blobs share a lot of structure across files, so we zstd-compress them
+// with a trained dictionary to cut the cache's on-disk footprint.
+//
+// This script builds that dictionary end to end:
+//
+// 1. Walks a fixed in-tree corpus (CORPUS below) in sorted order.
+// 2. For each .js file, harvests a V8 code cache via vm.compileFunction with
+// produceCachedData — the same shape the CommonJS loader produces at
+// runtime — and writes each blob to a temp directory.
+// 3. Feeds all the harvested blobs to `zstd --train`, capping the result at
+// MAXDICT (16 KiB) bytes.
+// 4. Overwrites src/compile_cache_zstd.dict with the trained dictionary.
+//
+// The shipped src/compile_cache_zstd.dict is the source of truth; this script
+// exists to document and reproduce exactly how that file was made, so a future
+// maintainer can regenerate it (e.g. after a V8/corpus change) and review the
+// resulting diff.
+//
+// -----------------------------------------------------------------------------
+// Usage
+// -----------------------------------------------------------------------------
+// node tools/train_compile_cache_dict.mjs
+//
+// Run from anywhere (paths are resolved relative to this file). The output is
+// written in place to src/compile_cache_zstd.dict; inspect the diff afterwards
+// and commit it if it looks right. Progress is printed to stderr.
+//
+// Prerequisites:
+// * the `zstd` CLI on PATH, version REQUIRED_ZSTD (matching deps/zstd).
+// * a built node on PATH (this is the node you invoke it with).
+//
+// Note: --predictable is added automatically (see below) — you do not need to
+// pass it yourself.
+//
+// -----------------------------------------------------------------------------
+// Reproducibility
+// -----------------------------------------------------------------------------
+// The output is byte-for-byte stable only if all of the following are pinned:
+//
+// * node is run with --predictable. V8 otherwise randomizes the string hash
+// seed per process, and that seed leaks into vm.compileFunction cachedData,
+// so every harvest (and therefore every trained dictionary) would differ
+// run-to-run even on the same machine. This script re-executes itself with
+// --predictable if needed.
+// * the corpus and its order are fixed (CORPUS below, walked sorted).
+// * the node build is fixed: cachedData embeds the V8 version and build
+// flags, so a different node produces a different (still valid) dictionary.
+// * the zstd CLI matches deps/zstd (currently 1.5.7). ZDICT training defaults
+// change between zstd releases; this script refuses to run on a mismatch.
+//
+// Training on --predictable caches is fine even though the runtime consumes
+// non-predictable caches: the dictionary only supplies shared substrings, and
+// Persist() keeps min(plain, dict) per entry, so a less-than-ideal dictionary
+// can never make any entry larger.
+// =============================================================================
+
+import { spawnSync } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdtempSync, readdirSync,
+ rmSync } from 'node:fs';
+import { join, relative, dirname } from 'node:path';
+import { tmpdir } from 'node:os';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
+const OUT = join(ROOT, 'src', 'compile_cache_zstd.dict');
+const MAXDICT = 16384;
+const REQUIRED_ZSTD = '1.5.7'; // keep in sync with deps/zstd/lib/zstd.h
+
+// Fixed corpus, relative to the repo root. Chosen to be diverse, always
+// present in a checkout, and disjoint from the held-out measurement corpora
+// (e.g. test/parallel) used to report size numbers.
+const CORPUS = ['lib', 'tools', 'deps/npm/node_modules'];
+
+// V8 randomizes the hash seed per process; re-exec under --predictable so the
+// harvested caches — and the trained dictionary — are deterministic.
+if (!process.execArgv.includes('--predictable')) {
+ const r = spawnSync(process.execPath,
+ ['--predictable', fileURLToPath(import.meta.url), ...process.argv.slice(2)],
+ { stdio: 'inherit' });
+ process.exit(r.status ?? 1);
+}
+
+function checkZstd() {
+ const r = spawnSync('zstd', ['--version'], { encoding: 'utf8' });
+ if (r.status !== 0) {
+ console.error('error: zstd CLI not found on PATH'); process.exit(1);
+ }
+ const m = r.stdout.match(/v(\d+\.\d+\.\d+)/);
+ if (!m || m[1] !== REQUIRED_ZSTD) {
+ console.error(`error: zstd ${REQUIRED_ZSTD} required (matching deps/zstd), ` +
+ `found ${m ? m[1] : 'unknown'}`);
+ process.exit(1);
+ }
+}
+
+const PARAMS = ['exports', 'require', 'module', '__filename', '__dirname'];
+
+function* walk(dir) {
+ let ents;
+ try { ents = readdirSync(dir, { withFileTypes: true }); } catch { return; }
+ for (const e of ents.sort((a, b) => (a.name < b.name ? -1 : 1))) {
+ const p = join(dir, e.name);
+ if (e.isDirectory()) yield* walk(p);
+ else if (e.isFile() && p.endsWith('.js')) yield p;
+ }
+}
+
+async function main() {
+ checkZstd();
+ const { default: vm } = await import('node:vm');
+
+ const files = [];
+ for (const root of CORPUS) for (const f of walk(join(ROOT, root))) files.push(f);
+ files.sort();
+
+ const samples = mkdtempSync(join(tmpdir(), 'cc-dict-'));
+ const sampleFiles = [];
+ let ok = 0;
+ let r;
+ try {
+ for (const f of files) {
+ let code;
+ try { code = readFileSync(f, 'utf8'); } catch { continue; }
+ try {
+ const fn = vm.compileFunction(code, PARAMS,
+ { filename: f, produceCachedData: true });
+ const cd = fn.cachedData;
+ if (cd && cd.length > 0) {
+ const name = relative(ROOT, f).replace(/[\\/]/g, '_') + '.cache';
+ const out = join(samples, name);
+ writeFileSync(out, cd);
+ sampleFiles.push(out);
+ ok++;
+ }
+ } catch { /* skip modules V8 can't compile standalone */ }
+ }
+ sampleFiles.sort();
+ console.error(`harvested ${ok}/${files.length} code caches from ` +
+ `${CORPUS.join(', ')}`);
+
+ // One sample path per file is spread on argv; the fixed corpus (~1.4k
+ // files, a few hundred KB of paths) stays well under ARG_MAX.
+ r = spawnSync('zstd',
+ ['--train', ...sampleFiles, `--maxdict=${MAXDICT}`, '-f', '-o', OUT],
+ { stdio: ['ignore', 'ignore', 'inherit'] });
+ } finally {
+ rmSync(samples, { recursive: true, force: true });
+ }
+ if (r.status !== 0) { console.error('error: zstd --train failed'); process.exit(1); }
+
+ const size = readFileSync(OUT).length;
+ console.error(`wrote ${relative(ROOT, OUT)} (${size} bytes)`);
+}
+
+main().catch((e) => { console.error(e); process.exit(1); });

tools/train_compile_cache_dict.mjs

@@ -0,0 +1,168 @@
+#!/usr/bin/env node
+// =============================================================================
+// train_compile_cache_dict.mjs
+//
+// Maintainer tool that regenerates src/compile_cache_zstd.dict, the zstd
+// dictionary embedded in the node binary and used to shrink compile-cache
+// entries on disk (see src/compile_cache.cc).
+//
+// -----------------------------------------------------------------------------
+// What it does
+// -----------------------------------------------------------------------------
+// The node compile cache stores V8 code caches (the bytecode/metadata blob V8
+// produces when it compiles a module) so later runs can skip recompilation.
+// These blobs share a lot of structure across files, so we zstd-compress them
+// with a trained dictionary to cut the cache's on-disk footprint.
+//
+// This script builds that dictionary end to end:
+//
+// 1. Walks a fixed in-tree corpus (CORPUS below) in sorted order.
+// 2. For each .js file, harvests a V8 code cache via vm.compileFunction with
+// produceCachedData — the same shape the CommonJS loader produces at
+// runtime — and writes each blob to a temp directory.
+// 3. Feeds all the harvested blobs to `zstd --train`, capping the result at
+// MAXDICT (16 KiB) bytes.
+// 4. Overwrites src/compile_cache_zstd.dict with the trained dictionary.
+//
+// The shipped src/compile_cache_zstd.dict is the source of truth; this script
+// exists to document and reproduce exactly how that file was made, so a future
+// maintainer can regenerate it (e.g. after a V8/corpus change) and review the
+// resulting diff.
+//
+// -----------------------------------------------------------------------------
+// Usage
+// -----------------------------------------------------------------------------
+// node tools/train_compile_cache_dict.mjs
+//
+// Run from anywhere (paths are resolved relative to this file). The output is
+// written in place to src/compile_cache_zstd.dict; inspect the diff afterwards
+// and commit it if it looks right. Progress is printed to stderr.
+//
+// Prerequisites:
+// * the `zstd` CLI on PATH, version REQUIRED_ZSTD (matching deps/zstd).
+// * a built node on PATH (this is the node you invoke it with).
+//
+// Note: --predictable is added automatically (see below) — you do not need to
+// pass it yourself.
+//
+// -----------------------------------------------------------------------------
+// Reproducibility
+// -----------------------------------------------------------------------------
+// The output is byte-for-byte stable only if all of the following are pinned:
+//
+// * node is run with --predictable. V8 otherwise randomizes the string hash
+// seed per process, and that seed leaks into vm.compileFunction cachedData,
+// so every harvest (and therefore every trained dictionary) would differ
+// run-to-run even on the same machine. This script re-executes itself with
+// --predictable if needed.
+// * the corpus and its order are fixed (CORPUS below, walked sorted).
+// * the node build is fixed: cachedData embeds the V8 version and build
+// flags, so a different node produces a different (still valid) dictionary.
+// * the zstd CLI matches deps/zstd (currently 1.5.7). ZDICT training defaults
+// change between zstd releases; this script refuses to run on a mismatch.
+//
+// Training on --predictable caches is fine even though the runtime consumes
+// non-predictable caches: the dictionary only supplies shared substrings, and
+// Persist() keeps min(plain, dict) per entry, so a less-than-ideal dictionary
+// can never make any entry larger.
+// =============================================================================
+
+import { spawnSync } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdtempSync, readdirSync,
+ rmSync } from 'node:fs';
+import { join, relative, dirname } from 'node:path';
+import { tmpdir } from 'node:os';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
+const OUT = join(ROOT, 'src', 'compile_cache_zstd.dict');
+const MAXDICT = 16384;
+const REQUIRED_ZSTD = '1.5.7'; // keep in sync with deps/zstd/lib/zstd.h
+
+// Fixed corpus, relative to the repo root. Chosen to be diverse, always
+// present in a checkout, and disjoint from the held-out measurement corpora
+// (e.g. test/parallel) used to report size numbers.
+const CORPUS = ['lib', 'tools', 'deps/npm/node_modules'];
+
+// V8 randomizes the hash seed per process; re-exec under --predictable so the
+// harvested caches — and the trained dictionary — are deterministic.
+if (!process.execArgv.includes('--predictable')) {
+ const r = spawnSync(process.execPath,
+ ['--predictable', fileURLToPath(import.meta.url), ...process.argv.slice(2)],
+ { stdio: 'inherit' });
+ process.exit(r.status ?? 1);
+}
+
+function checkZstd() {
+ const r = spawnSync('zstd', ['--version'], { encoding: 'utf8' });
+ if (r.status !== 0) {
+ console.error('error: zstd CLI not found on PATH'); process.exit(1);
+ }
+ const m = r.stdout.match(/v(\d+\.\d+\.\d+)/);
+ if (!m || m[1] !== REQUIRED_ZSTD) {
+ console.error(`error: zstd ${REQUIRED_ZSTD} required (matching deps/zstd), ` +
+ `found ${m ? m[1] : 'unknown'}`);
+ process.exit(1);
+ }
+}
+
+const PARAMS = ['exports', 'require', 'module', '__filename', '__dirname'];
+
+function* walk(dir) {
+ let ents;
+ try { ents = readdirSync(dir, { withFileTypes: true }); } catch { return; }
+ for (const e of ents.sort((a, b) => (a.name < b.name ? -1 : 1))) {
+ const p = join(dir, e.name);
+ if (e.isDirectory()) yield* walk(p);
+ else if (e.isFile() && p.endsWith('.js')) yield p;
+ }
+}
+
+async function main() {
+ checkZstd();
+ const { default: vm } = await import('node:vm');
+
+ const files = [];
+ for (const root of CORPUS) for (const f of walk(join(ROOT, root))) files.push(f);
+ files.sort();
+
+ const samples = mkdtempSync(join(tmpdir(), 'cc-dict-'));
+ const sampleFiles = [];
+ let ok = 0;
+ let r;
+ try {
+ for (const f of files) {
+ let code;
+ try { code = readFileSync(f, 'utf8'); } catch { continue; }
+ try {
+ const fn = vm.compileFunction(code, PARAMS,
+ { filename: f, produceCachedData: true });
+ const cd = fn.cachedData;
+ if (cd && cd.length > 0) {
+ const name = relative(ROOT, f).replace(/[\\/]/g, '_') + '.cache';
+ const out = join(samples, name);
+ writeFileSync(out, cd);
+ sampleFiles.push(out);
+ ok++;
+ }
+ } catch { /* skip modules V8 can't compile standalone */ }
+ }
+ sampleFiles.sort();
+ console.error(`harvested ${ok}/${files.length} code caches from ` +
+ `${CORPUS.join(', ')}`);
+
+ // One sample path per file is spread on argv; the fixed corpus (~1.4k
+ // files, a few hundred KB of paths) stays well under ARG_MAX.
+ r = spawnSync('zstd',
+ ['--train', ...sampleFiles, `--maxdict=${MAXDICT}`, '-f', '-o', OUT],
+ { stdio: ['ignore', 'ignore', 'inherit'] });
+ } finally {
+ rmSync(samples, { recursive: true, force: true });
+ }
+ if (r.status !== 0) { console.error('error: zstd --train failed'); process.exit(1); }
+
+ const size = readFileSync(OUT).length;
+ console.error(`wrote ${relative(ROOT, OUT)} (${size} bytes)`);
+}
+
+main().catch((e) => { console.error(e); process.exit(1); });

tools/train_compile_cache_dict.mjs

@@ -0,0 +1,168 @@
+#!/usr/bin/env node
+// =============================================================================
+// train_compile_cache_dict.mjs
+//
+// Maintainer tool that regenerates src/compile_cache_zstd.dict, the zstd
+// dictionary embedded in the node binary and used to shrink compile-cache
+// entries on disk (see src/compile_cache.cc).
+//
+// -----------------------------------------------------------------------------
+// What it does
+// -----------------------------------------------------------------------------
+// The node compile cache stores V8 code caches (the bytecode/metadata blob V8
+// produces when it compiles a module) so later runs can skip recompilation.
+// These blobs share a lot of structure across files, so we zstd-compress them
+// with a trained dictionary to cut the cache's on-disk footprint.
+//
+// This script builds that dictionary end to end:
+//
+// 1. Walks a fixed in-tree corpus (CORPUS below) in sorted order.
+// 2. For each .js file, harvests a V8 code cache via vm.compileFunction with
+// produceCachedData — the same shape the CommonJS loader produces at
+// runtime — and writes each blob to a temp directory.
+// 3. Feeds all the harvested blobs to `zstd --train`, capping the result at
+// MAXDICT (16 KiB) bytes.
+// 4. Overwrites src/compile_cache_zstd.dict with the trained dictionary.
+//
+// The shipped src/compile_cache_zstd.dict is the source of truth; this script
+// exists to document and reproduce exactly how that file was made, so a future
+// maintainer can regenerate it (e.g. after a V8/corpus change) and review the
+// resulting diff.
+//
+// -----------------------------------------------------------------------------
+// Usage
+// -----------------------------------------------------------------------------
+// node tools/train_compile_cache_dict.mjs
+//
+// Run from anywhere (paths are resolved relative to this file). The output is
+// written in place to src/compile_cache_zstd.dict; inspect the diff afterwards
+// and commit it if it looks right. Progress is printed to stderr.
+//
+// Prerequisites:
+// * the `zstd` CLI on PATH, version REQUIRED_ZSTD (matching deps/zstd).
+// * a built node on PATH (this is the node you invoke it with).
+//
+// Note: --predictable is added automatically (see below) — you do not need to
+// pass it yourself.
+//
+// -----------------------------------------------------------------------------
+// Reproducibility
+// -----------------------------------------------------------------------------
+// The output is byte-for-byte stable only if all of the following are pinned:
+//
+// * node is run with --predictable. V8 otherwise randomizes the string hash
+// seed per process, and that seed leaks into vm.compileFunction cachedData,
+// so every harvest (and therefore every trained dictionary) would differ
+// run-to-run even on the same machine. This script re-executes itself with
+// --predictable if needed.
+// * the corpus and its order are fixed (CORPUS below, walked sorted).
+// * the node build is fixed: cachedData embeds the V8 version and build
+// flags, so a different node produces a different (still valid) dictionary.
+// * the zstd CLI matches deps/zstd (currently 1.5.7). ZDICT training defaults
+// change between zstd releases; this script refuses to run on a mismatch.
+//
+// Training on --predictable caches is fine even though the runtime consumes
+// non-predictable caches: the dictionary only supplies shared substrings, and
+// Persist() keeps min(plain, dict) per entry, so a less-than-ideal dictionary
+// can never make any entry larger.
+// =============================================================================
+
+import { spawnSync } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdtempSync, readdirSync,
+ rmSync } from 'node:fs';
+import { join, relative, dirname } from 'node:path';
+import { tmpdir } from 'node:os';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
+const OUT = join(ROOT, 'src', 'compile_cache_zstd.dict');
+const MAXDICT = 16384;
+const REQUIRED_ZSTD = '1.5.7'; // keep in sync with deps/zstd/lib/zstd.h
+
+// Fixed corpus, relative to the repo root. Chosen to be diverse, always
+// present in a checkout, and disjoint from the held-out measurement corpora
+// (e.g. test/parallel) used to report size numbers.
+const CORPUS = ['lib', 'tools', 'deps/npm/node_modules'];
+
+// V8 randomizes the hash seed per process; re-exec under --predictable so the
+// harvested caches — and the trained dictionary — are deterministic.
+if (!process.execArgv.includes('--predictable')) {
+ const r = spawnSync(process.execPath,
+ ['--predictable', fileURLToPath(import.meta.url), ...process.argv.slice(2)],
+ { stdio: 'inherit' });
+ process.exit(r.status ?? 1);
+}
+
+function checkZstd() {
+ const r = spawnSync('zstd', ['--version'], { encoding: 'utf8' });
+ if (r.status !== 0) {
+ console.error('error: zstd CLI not found on PATH'); process.exit(1);
+ }
+ const m = r.stdout.match(/v(\d+\.\d+\.\d+)/);
+ if (!m || m[1] !== REQUIRED_ZSTD) {
+ console.error(`error: zstd ${REQUIRED_ZSTD} required (matching deps/zstd), ` +
+ `found ${m ? m[1] : 'unknown'}`);
+ process.exit(1);
+ }
+}
+
+const PARAMS = ['exports', 'require', 'module', '__filename', '__dirname'];
+
+function* walk(dir) {
+ let ents;
+ try { ents = readdirSync(dir, { withFileTypes: true }); } catch { return; }
+ for (const e of ents.sort((a, b) => (a.name < b.name ? -1 : 1))) {
+ const p = join(dir, e.name);
+ if (e.isDirectory()) yield* walk(p);
+ else if (e.isFile() && p.endsWith('.js')) yield p;
+ }
+}
+
+async function main() {
+ checkZstd();
+ const { default: vm } = await import('node:vm');
+
+ const files = [];
+ for (const root of CORPUS) for (const f of walk(join(ROOT, root))) files.push(f);
+ files.sort();
+
+ const samples = mkdtempSync(join(tmpdir(), 'cc-dict-'));
+ const sampleFiles = [];
+ let ok = 0;
+ let r;
+ try {
+ for (const f of files) {
+ let code;
+ try { code = readFileSync(f, 'utf8'); } catch { continue; }
+ try {
+ const fn = vm.compileFunction(code, PARAMS,
+ { filename: f, produceCachedData: true });
+ const cd = fn.cachedData;
+ if (cd && cd.length > 0) {
+ const name = relative(ROOT, f).replace(/[\\/]/g, '_') + '.cache';
+ const out = join(samples, name);
+ writeFileSync(out, cd);
+ sampleFiles.push(out);
+ ok++;
+ }
+ } catch { /* skip modules V8 can't compile standalone */ }
+ }
+ sampleFiles.sort();
+ console.error(`harvested ${ok}/${files.length} code caches from ` +
+ `${CORPUS.join(', ')}`);
+
+ // One sample path per file is spread on argv; the fixed corpus (~1.4k
+ // files, a few hundred KB of paths) stays well under ARG_MAX.
+ r = spawnSync('zstd',
+ ['--train', ...sampleFiles, `--maxdict=${MAXDICT}`, '-f', '-o', OUT],
+ { stdio: ['ignore', 'ignore', 'inherit'] });
+ } finally {
+ rmSync(samples, { recursive: true, force: true });
+ }
+ if (r.status !== 0) { console.error('error: zstd --train failed'); process.exit(1); }
+
+ const size = readFileSync(OUT).length;
+ console.error(`wrote ${relative(ROOT, OUT)} (${size} bytes)`);
+}
+
+main().catch((e) => { console.error(e); process.exit(1); });

tools/train_compile_cache_dict.mjs

@@ -0,0 +1,168 @@
+#!/usr/bin/env node
+// =============================================================================
+// train_compile_cache_dict.mjs
+//
+// Maintainer tool that regenerates src/compile_cache_zstd.dict, the zstd
+// dictionary embedded in the node binary and used to shrink compile-cache
+// entries on disk (see src/compile_cache.cc).
+//
+// -----------------------------------------------------------------------------
+// What it does
+// -----------------------------------------------------------------------------
+// The node compile cache stores V8 code caches (the bytecode/metadata blob V8
+// produces when it compiles a module) so later runs can skip recompilation.
+// These blobs share a lot of structure across files, so we zstd-compress them
+// with a trained dictionary to cut the cache's on-disk footprint.
+//
+// This script builds that dictionary end to end:
+//
+// 1. Walks a fixed in-tree corpus (CORPUS below) in sorted order.
+// 2. For each .js file, harvests a V8 code cache via vm.compileFunction with
+// produceCachedData — the same shape the CommonJS loader produces at
+// runtime — and writes each blob to a temp directory.
+// 3. Feeds all the harvested blobs to `zstd --train`, capping the result at
+// MAXDICT (16 KiB) bytes.
+// 4. Overwrites src/compile_cache_zstd.dict with the trained dictionary.
+//
+// The shipped src/compile_cache_zstd.dict is the source of truth; this script
+// exists to document and reproduce exactly how that file was made, so a future
+// maintainer can regenerate it (e.g. after a V8/corpus change) and review the
+// resulting diff.
+//
+// -----------------------------------------------------------------------------
+// Usage
+// -----------------------------------------------------------------------------
+// node tools/train_compile_cache_dict.mjs
+//
+// Run from anywhere (paths are resolved relative to this file). The output is
+// written in place to src/compile_cache_zstd.dict; inspect the diff afterwards
+// and commit it if it looks right. Progress is printed to stderr.
+//
+// Prerequisites:
+// * the `zstd` CLI on PATH, version REQUIRED_ZSTD (matching deps/zstd).
+// * a built node on PATH (this is the node you invoke it with).
+//
+// Note: --predictable is added automatically (see below) — you do not need to
+// pass it yourself.
+//
+// -----------------------------------------------------------------------------
+// Reproducibility
+// -----------------------------------------------------------------------------
+// The output is byte-for-byte stable only if all of the following are pinned:
+//
+// * node is run with --predictable. V8 otherwise randomizes the string hash
+// seed per process, and that seed leaks into vm.compileFunction cachedData,
+// so every harvest (and therefore every trained dictionary) would differ
+// run-to-run even on the same machine. This script re-executes itself with
+// --predictable if needed.
+// * the corpus and its order are fixed (CORPUS below, walked sorted).
+// * the node build is fixed: cachedData embeds the V8 version and build
+// flags, so a different node produces a different (still valid) dictionary.
+// * the zstd CLI matches deps/zstd (currently 1.5.7). ZDICT training defaults
+// change between zstd releases; this script refuses to run on a mismatch.
+//
+// Training on --predictable caches is fine even though the runtime consumes
+// non-predictable caches: the dictionary only supplies shared substrings, and
+// Persist() keeps min(plain, dict) per entry, so a less-than-ideal dictionary
+// can never make any entry larger.
+// =============================================================================
+
+import { spawnSync } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdtempSync, readdirSync,
+ rmSync } from 'node:fs';
+import { join, relative, dirname } from 'node:path';
+import { tmpdir } from 'node:os';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
+const OUT = join(ROOT, 'src', 'compile_cache_zstd.dict');
+const MAXDICT = 16384;
+const REQUIRED_ZSTD = '1.5.7'; // keep in sync with deps/zstd/lib/zstd.h
+
+// Fixed corpus, relative to the repo root. Chosen to be diverse, always
+// present in a checkout, and disjoint from the held-out measurement corpora
+// (e.g. test/parallel) used to report size numbers.
+const CORPUS = ['lib', 'tools', 'deps/npm/node_modules'];
+
+// V8 randomizes the hash seed per process; re-exec under --predictable so the
+// harvested caches — and the trained dictionary — are deterministic.
+if (!process.execArgv.includes('--predictable')) {
+ const r = spawnSync(process.execPath,
+ ['--predictable', fileURLToPath(import.meta.url), ...process.argv.slice(2)],
+ { stdio: 'inherit' });
+ process.exit(r.status ?? 1);
+}
+
+function checkZstd() {
+ const r = spawnSync('zstd', ['--version'], { encoding: 'utf8' });
+ if (r.status !== 0) {
+ console.error('error: zstd CLI not found on PATH'); process.exit(1);
+ }
+ const m = r.stdout.match(/v(\d+\.\d+\.\d+)/);
+ if (!m || m[1] !== REQUIRED_ZSTD) {
+ console.error(`error: zstd ${REQUIRED_ZSTD} required (matching deps/zstd), ` +
+ `found ${m ? m[1] : 'unknown'}`);
+ process.exit(1);
+ }
+}
+
+const PARAMS = ['exports', 'require', 'module', '__filename', '__dirname'];
+
+function* walk(dir) {
+ let ents;
+ try { ents = readdirSync(dir, { withFileTypes: true }); } catch { return; }
+ for (const e of ents.sort((a, b) => (a.name < b.name ? -1 : 1))) {
+ const p = join(dir, e.name);
+ if (e.isDirectory()) yield* walk(p);
+ else if (e.isFile() && p.endsWith('.js')) yield p;
+ }
+}
+
+async function main() {
+ checkZstd();
+ const { default: vm } = await import('node:vm');
+
+ const files = [];
+ for (const root of CORPUS) for (const f of walk(join(ROOT, root))) files.push(f);
+ files.sort();
+
+ const samples = mkdtempSync(join(tmpdir(), 'cc-dict-'));
+ const sampleFiles = [];
+ let ok = 0;
+ let r;
+ try {
+ for (const f of files) {
+ let code;
+ try { code = readFileSync(f, 'utf8'); } catch { continue; }
+ try {
+ const fn = vm.compileFunction(code, PARAMS,
+ { filename: f, produceCachedData: true });
+ const cd = fn.cachedData;
+ if (cd && cd.length > 0) {
+ const name = relative(ROOT, f).replace(/[\\/]/g, '_') + '.cache';
+ const out = join(samples, name);
+ writeFileSync(out, cd);
+ sampleFiles.push(out);
+ ok++;
+ }
+ } catch { /* skip modules V8 can't compile standalone */ }
+ }
+ sampleFiles.sort();
+ console.error(`harvested ${ok}/${files.length} code caches from ` +
+ `${CORPUS.join(', ')}`);
+
+ // One sample path per file is spread on argv; the fixed corpus (~1.4k
+ // files, a few hundred KB of paths) stays well under ARG_MAX.
+ r = spawnSync('zstd',
+ ['--train', ...sampleFiles, `--maxdict=${MAXDICT}`, '-f', '-o', OUT],
+ { stdio: ['ignore', 'ignore', 'inherit'] });
+ } finally {
+ rmSync(samples, { recursive: true, force: true });
+ }
+ if (r.status !== 0) { console.error('error: zstd --train failed'); process.exit(1); }
+
+ const size = readFileSync(OUT).length;
+ console.error(`wrote ${relative(ROOT, OUT)} (${size} bytes)`);
+}
+
+main().catch((e) => { console.error(e); process.exit(1); });

tools/train_compile_cache_dict.mjs

@@ -0,0 +1,168 @@
+#!/usr/bin/env node
+// =============================================================================
+// train_compile_cache_dict.mjs
+//
+// Maintainer tool that regenerates src/compile_cache_zstd.dict, the zstd
+// dictionary embedded in the node binary and used to shrink compile-cache
+// entries on disk (see src/compile_cache.cc).
+//
+// -----------------------------------------------------------------------------
+// What it does
+// -----------------------------------------------------------------------------
+// The node compile cache stores V8 code caches (the bytecode/metadata blob V8
+// produces when it compiles a module) so later runs can skip recompilation.
+// These blobs share a lot of structure across files, so we zstd-compress them
+// with a trained dictionary to cut the cache's on-disk footprint.
+//
+// This script builds that dictionary end to end:
+//
+// 1. Walks a fixed in-tree corpus (CORPUS below) in sorted order.
+// 2. For each .js file, harvests a V8 code cache via vm.compileFunction with
+// produceCachedData — the same shape the CommonJS loader produces at
+// runtime — and writes each blob to a temp directory.
+// 3. Feeds all the harvested blobs to `zstd --train`, capping the result at
+// MAXDICT (16 KiB) bytes.
+// 4. Overwrites src/compile_cache_zstd.dict with the trained dictionary.
+//
+// The shipped src/compile_cache_zstd.dict is the source of truth; this script
+// exists to document and reproduce exactly how that file was made, so a future
+// maintainer can regenerate it (e.g. after a V8/corpus change) and review the
+// resulting diff.
+//
+// -----------------------------------------------------------------------------
+// Usage
+// -----------------------------------------------------------------------------
+// node tools/train_compile_cache_dict.mjs
+//
+// Run from anywhere (paths are resolved relative to this file). The output is
+// written in place to src/compile_cache_zstd.dict; inspect the diff afterwards
+// and commit it if it looks right. Progress is printed to stderr.
+//
+// Prerequisites:
+// * the `zstd` CLI on PATH, version REQUIRED_ZSTD (matching deps/zstd).
+// * a built node on PATH (this is the node you invoke it with).
+//
+// Note: --predictable is added automatically (see below) — you do not need to
+// pass it yourself.
+//
+// -----------------------------------------------------------------------------
+// Reproducibility
+// -----------------------------------------------------------------------------
+// The output is byte-for-byte stable only if all of the following are pinned:
+//
+// * node is run with --predictable. V8 otherwise randomizes the string hash
+// seed per process, and that seed leaks into vm.compileFunction cachedData,
+// so every harvest (and therefore every trained dictionary) would differ
+// run-to-run even on the same machine. This script re-executes itself with
+// --predictable if needed.
+// * the corpus and its order are fixed (CORPUS below, walked sorted).
+// * the node build is fixed: cachedData embeds the V8 version and build
+// flags, so a different node produces a different (still valid) dictionary.
+// * the zstd CLI matches deps/zstd (currently 1.5.7). ZDICT training defaults
+// change between zstd releases; this script refuses to run on a mismatch.
+//
+// Training on --predictable caches is fine even though the runtime consumes
+// non-predictable caches: the dictionary only supplies shared substrings, and
+// Persist() keeps min(plain, dict) per entry, so a less-than-ideal dictionary
+// can never make any entry larger.
+// =============================================================================
+
+import { spawnSync } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdtempSync, readdirSync,
+ rmSync } from 'node:fs';
+import { join, relative, dirname } from 'node:path';
+import { tmpdir } from 'node:os';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
+const OUT = join(ROOT, 'src', 'compile_cache_zstd.dict');
+const MAXDICT = 16384;
+const REQUIRED_ZSTD = '1.5.7'; // keep in sync with deps/zstd/lib/zstd.h
+
+// Fixed corpus, relative to the repo root. Chosen to be diverse, always
+// present in a checkout, and disjoint from the held-out measurement corpora
+// (e.g. test/parallel) used to report size numbers.
+const CORPUS = ['lib', 'tools', 'deps/npm/node_modules'];
+
+// V8 randomizes the hash seed per process; re-exec under --predictable so the
+// harvested caches — and the trained dictionary — are deterministic.
+if (!process.execArgv.includes('--predictable')) {
+ const r = spawnSync(process.execPath,
+ ['--predictable', fileURLToPath(import.meta.url), ...process.argv.slice(2)],
+ { stdio: 'inherit' });
+ process.exit(r.status ?? 1);
+}
+
+function checkZstd() {
+ const r = spawnSync('zstd', ['--version'], { encoding: 'utf8' });
+ if (r.status !== 0) {
+ console.error('error: zstd CLI not found on PATH'); process.exit(1);
+ }
+ const m = r.stdout.match(/v(\d+\.\d+\.\d+)/);
+ if (!m || m[1] !== REQUIRED_ZSTD) {
+ console.error(`error: zstd ${REQUIRED_ZSTD} required (matching deps/zstd), ` +
+ `found ${m ? m[1] : 'unknown'}`);
+ process.exit(1);
+ }
+}
+
+const PARAMS = ['exports', 'require', 'module', '__filename', '__dirname'];
+
+function* walk(dir) {
+ let ents;
+ try { ents = readdirSync(dir, { withFileTypes: true }); } catch { return; }
+ for (const e of ents.sort((a, b) => (a.name < b.name ? -1 : 1))) {
+ const p = join(dir, e.name);
+ if (e.isDirectory()) yield* walk(p);
+ else if (e.isFile() && p.endsWith('.js')) yield p;
+ }
+}
+
+async function main() {
+ checkZstd();
+ const { default: vm } = await import('node:vm');
+
+ const files = [];
+ for (const root of CORPUS) for (const f of walk(join(ROOT, root))) files.push(f);
+ files.sort();
+
+ const samples = mkdtempSync(join(tmpdir(), 'cc-dict-'));
+ const sampleFiles = [];
+ let ok = 0;
+ let r;
+ try {
+ for (const f of files) {
+ let code;
+ try { code = readFileSync(f, 'utf8'); } catch { continue; }
+ try {
+ const fn = vm.compileFunction(code, PARAMS,
+ { filename: f, produceCachedData: true });
+ const cd = fn.cachedData;
+ if (cd && cd.length > 0) {
+ const name = relative(ROOT, f).replace(/[\\/]/g, '_') + '.cache';
+ const out = join(samples, name);
+ writeFileSync(out, cd);
+ sampleFiles.push(out);
+ ok++;
+ }
+ } catch { /* skip modules V8 can't compile standalone */ }
+ }
+ sampleFiles.sort();
+ console.error(`harvested ${ok}/${files.length} code caches from ` +
+ `${CORPUS.join(', ')}`);
+
+ // One sample path per file is spread on argv; the fixed corpus (~1.4k
+ // files, a few hundred KB of paths) stays well under ARG_MAX.
+ r = spawnSync('zstd',
+ ['--train', ...sampleFiles, `--maxdict=${MAXDICT}`, '-f', '-o', OUT],
+ { stdio: ['ignore', 'ignore', 'inherit'] });
+ } finally {
+ rmSync(samples, { recursive: true, force: true });
+ }
+ if (r.status !== 0) { console.error('error: zstd --train failed'); process.exit(1); }
+
+ const size = readFileSync(OUT).length;
+ console.error(`wrote ${relative(ROOT, OUT)} (${size} bytes)`);
+}
+
+main().catch((e) => { console.error(e); process.exit(1); });

tools/train_compile_cache_dict.mjs

@@ -0,0 +1,168 @@
+#!/usr/bin/env node
+// =============================================================================
+// train_compile_cache_dict.mjs
+//
+// Maintainer tool that regenerates src/compile_cache_zstd.dict, the zstd
+// dictionary embedded in the node binary and used to shrink compile-cache
+// entries on disk (see src/compile_cache.cc).
+//
+// -----------------------------------------------------------------------------
+// What it does
+// -----------------------------------------------------------------------------
+// The node compile cache stores V8 code caches (the bytecode/metadata blob V8
+// produces when it compiles a module) so later runs can skip recompilation.
+// These blobs share a lot of structure across files, so we zstd-compress them
+// with a trained dictionary to cut the cache's on-disk footprint.
+//
+// This script builds that dictionary end to end:
+//
+// 1. Walks a fixed in-tree corpus (CORPUS below) in sorted order.
+// 2. For each .js file, harvests a V8 code cache via vm.compileFunction with
+// produceCachedData — the same shape the CommonJS loader produces at
+// runtime — and writes each blob to a temp directory.
+// 3. Feeds all the harvested blobs to `zstd --train`, capping the result at
+// MAXDICT (16 KiB) bytes.
+// 4. Overwrites src/compile_cache_zstd.dict with the trained dictionary.
+//
+// The shipped src/compile_cache_zstd.dict is the source of truth; this script
+// exists to document and reproduce exactly how that file was made, so a future
+// maintainer can regenerate it (e.g. after a V8/corpus change) and review the
+// resulting diff.
+//
+// -----------------------------------------------------------------------------
+// Usage
+// -----------------------------------------------------------------------------
+// node tools/train_compile_cache_dict.mjs
+//
+// Run from anywhere (paths are resolved relative to this file). The output is
+// written in place to src/compile_cache_zstd.dict; inspect the diff afterwards
+// and commit it if it looks right. Progress is printed to stderr.
+//
+// Prerequisites:
+// * the `zstd` CLI on PATH, version REQUIRED_ZSTD (matching deps/zstd).
+// * a built node on PATH (this is the node you invoke it with).
+//
+// Note: --predictable is added automatically (see below) — you do not need to
+// pass it yourself.
+//
+// -----------------------------------------------------------------------------
+// Reproducibility
+// -----------------------------------------------------------------------------
+// The output is byte-for-byte stable only if all of the following are pinned:
+//
+// * node is run with --predictable. V8 otherwise randomizes the string hash
+// seed per process, and that seed leaks into vm.compileFunction cachedData,
+// so every harvest (and therefore every trained dictionary) would differ
+// run-to-run even on the same machine. This script re-executes itself with
+// --predictable if needed.
+// * the corpus and its order are fixed (CORPUS below, walked sorted).
+// * the node build is fixed: cachedData embeds the V8 version and build
+// flags, so a different node produces a different (still valid) dictionary.
+// * the zstd CLI matches deps/zstd (currently 1.5.7). ZDICT training defaults
+// change between zstd releases; this script refuses to run on a mismatch.
+//
+// Training on --predictable caches is fine even though the runtime consumes
+// non-predictable caches: the dictionary only supplies shared substrings, and
+// Persist() keeps min(plain, dict) per entry, so a less-than-ideal dictionary
+// can never make any entry larger.
+// =============================================================================
+
+import { spawnSync } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdtempSync, readdirSync,
+ rmSync } from 'node:fs';
+import { join, relative, dirname } from 'node:path';
+import { tmpdir } from 'node:os';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
+const OUT = join(ROOT, 'src', 'compile_cache_zstd.dict');
+const MAXDICT = 16384;
+const REQUIRED_ZSTD = '1.5.7'; // keep in sync with deps/zstd/lib/zstd.h
+
+// Fixed corpus, relative to the repo root. Chosen to be diverse, always
+// present in a checkout, and disjoint from the held-out measurement corpora
+// (e.g. test/parallel) used to report size numbers.
+const CORPUS = ['lib', 'tools', 'deps/npm/node_modules'];
+
+// V8 randomizes the hash seed per process; re-exec under --predictable so the
+// harvested caches — and the trained dictionary — are deterministic.
+if (!process.execArgv.includes('--predictable')) {
+ const r = spawnSync(process.execPath,
+ ['--predictable', fileURLToPath(import.meta.url), ...process.argv.slice(2)],
+ { stdio: 'inherit' });
+ process.exit(r.status ?? 1);
+}
+
+function checkZstd() {
+ const r = spawnSync('zstd', ['--version'], { encoding: 'utf8' });
+ if (r.status !== 0) {
+ console.error('error: zstd CLI not found on PATH'); process.exit(1);
+ }
+ const m = r.stdout.match(/v(\d+\.\d+\.\d+)/);
+ if (!m || m[1] !== REQUIRED_ZSTD) {
+ console.error(`error: zstd ${REQUIRED_ZSTD} required (matching deps/zstd), ` +
+ `found ${m ? m[1] : 'unknown'}`);
+ process.exit(1);
+ }
+}
+
+const PARAMS = ['exports', 'require', 'module', '__filename', '__dirname'];
+
+function* walk(dir) {
+ let ents;
+ try { ents = readdirSync(dir, { withFileTypes: true }); } catch { return; }
+ for (const e of ents.sort((a, b) => (a.name < b.name ? -1 : 1))) {
+ const p = join(dir, e.name);
+ if (e.isDirectory()) yield* walk(p);
+ else if (e.isFile() && p.endsWith('.js')) yield p;
+ }
+}
+
+async function main() {
+ checkZstd();
+ const { default: vm } = await import('node:vm');
+
+ const files = [];
+ for (const root of CORPUS) for (const f of walk(join(ROOT, root))) files.push(f);
+ files.sort();
+
+ const samples = mkdtempSync(join(tmpdir(), 'cc-dict-'));
+ const sampleFiles = [];
+ let ok = 0;
+ let r;
+ try {
+ for (const f of files) {
+ let code;
+ try { code = readFileSync(f, 'utf8'); } catch { continue; }
+ try {
+ const fn = vm.compileFunction(code, PARAMS,
+ { filename: f, produceCachedData: true });
+ const cd = fn.cachedData;
+ if (cd && cd.length > 0) {
+ const name = relative(ROOT, f).replace(/[\\/]/g, '_') + '.cache';
+ const out = join(samples, name);
+ writeFileSync(out, cd);
+ sampleFiles.push(out);
+ ok++;
+ }
+ } catch { /* skip modules V8 can't compile standalone */ }
+ }
+ sampleFiles.sort();
+ console.error(`harvested ${ok}/${files.length} code caches from ` +
+ `${CORPUS.join(', ')}`);
+
+ // One sample path per file is spread on argv; the fixed corpus (~1.4k
+ // files, a few hundred KB of paths) stays well under ARG_MAX.
+ r = spawnSync('zstd',
+ ['--train', ...sampleFiles, `--maxdict=${MAXDICT}`, '-f', '-o', OUT],
+ { stdio: ['ignore', 'ignore', 'inherit'] });
+ } finally {
+ rmSync(samples, { recursive: true, force: true });
+ }
+ if (r.status !== 0) { console.error('error: zstd --train failed'); process.exit(1); }
+
+ const size = readFileSync(OUT).length;
+ console.error(`wrote ${relative(ROOT, OUT)} (${size} bytes)`);
+}
+
+main().catch((e) => { console.error(e); process.exit(1); });

tools/train_compile_cache_dict.mjs

@@ -0,0 +1,168 @@
+#!/usr/bin/env node
+// =============================================================================
+// train_compile_cache_dict.mjs
+//
+// Maintainer tool that regenerates src/compile_cache_zstd.dict, the zstd
+// dictionary embedded in the node binary and used to shrink compile-cache
+// entries on disk (see src/compile_cache.cc).
+//
+// -----------------------------------------------------------------------------
+// What it does
+// -----------------------------------------------------------------------------
+// The node compile cache stores V8 code caches (the bytecode/metadata blob V8
+// produces when it compiles a module) so later runs can skip recompilation.
+// These blobs share a lot of structure across files, so we zstd-compress them
+// with a trained dictionary to cut the cache's on-disk footprint.
+//
+// This script builds that dictionary end to end:
+//
+// 1. Walks a fixed in-tree corpus (CORPUS below) in sorted order.
+// 2. For each .js file, harvests a V8 code cache via vm.compileFunction with
+// produceCachedData — the same shape the CommonJS loader produces at
+// runtime — and writes each blob to a temp directory.
+// 3. Feeds all the harvested blobs to `zstd --train`, capping the result at
+// MAXDICT (16 KiB) bytes.
+// 4. Overwrites src/compile_cache_zstd.dict with the trained dictionary.
+//
+// The shipped src/compile_cache_zstd.dict is the source of truth; this script
+// exists to document and reproduce exactly how that file was made, so a future
+// maintainer can regenerate it (e.g. after a V8/corpus change) and review the
+// resulting diff.
+//
+// -----------------------------------------------------------------------------
+// Usage
+// -----------------------------------------------------------------------------
+// node tools/train_compile_cache_dict.mjs
+//
+// Run from anywhere (paths are resolved relative to this file). The output is
+// written in place to src/compile_cache_zstd.dict; inspect the diff afterwards
+// and commit it if it looks right. Progress is printed to stderr.
+//
+// Prerequisites:
+// * the `zstd` CLI on PATH, version REQUIRED_ZSTD (matching deps/zstd).
+// * a built node on PATH (this is the node you invoke it with).
+//
+// Note: --predictable is added automatically (see below) — you do not need to
+// pass it yourself.
+//
+// -----------------------------------------------------------------------------
+// Reproducibility
+// -----------------------------------------------------------------------------
+// The output is byte-for-byte stable only if all of the following are pinned:
+//
+// * node is run with --predictable. V8 otherwise randomizes the string hash
+// seed per process, and that seed leaks into vm.compileFunction cachedData,
+// so every harvest (and therefore every trained dictionary) would differ
+// run-to-run even on the same machine. This script re-executes itself with
+// --predictable if needed.
+// * the corpus and its order are fixed (CORPUS below, walked sorted).
+// * the node build is fixed: cachedData embeds the V8 version and build
+// flags, so a different node produces a different (still valid) dictionary.
+// * the zstd CLI matches deps/zstd (currently 1.5.7). ZDICT training defaults
+// change between zstd releases; this script refuses to run on a mismatch.
+//
+// Training on --predictable caches is fine even though the runtime consumes
+// non-predictable caches: the dictionary only supplies shared substrings, and
+// Persist() keeps min(plain, dict) per entry, so a less-than-ideal dictionary
+// can never make any entry larger.
+// =============================================================================
+
+import { spawnSync } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdtempSync, readdirSync,
+ rmSync } from 'node:fs';
+import { join, relative, dirname } from 'node:path';
+import { tmpdir } from 'node:os';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
+const OUT = join(ROOT, 'src', 'compile_cache_zstd.dict');
+const MAXDICT = 16384;
+const REQUIRED_ZSTD = '1.5.7'; // keep in sync with deps/zstd/lib/zstd.h
+
+// Fixed corpus, relative to the repo root. Chosen to be diverse, always
+// present in a checkout, and disjoint from the held-out measurement corpora
+// (e.g. test/parallel) used to report size numbers.
+const CORPUS = ['lib', 'tools', 'deps/npm/node_modules'];
+
+// V8 randomizes the hash seed per process; re-exec under --predictable so the
+// harvested caches — and the trained dictionary — are deterministic.
+if (!process.execArgv.includes('--predictable')) {
+ const r = spawnSync(process.execPath,
+ ['--predictable', fileURLToPath(import.meta.url), ...process.argv.slice(2)],
+ { stdio: 'inherit' });
+ process.exit(r.status ?? 1);
+}
+
+function checkZstd() {
+ const r = spawnSync('zstd', ['--version'], { encoding: 'utf8' });
+ if (r.status !== 0) {
+ console.error('error: zstd CLI not found on PATH'); process.exit(1);
+ }
+ const m = r.stdout.match(/v(\d+\.\d+\.\d+)/);
+ if (!m || m[1] !== REQUIRED_ZSTD) {
+ console.error(`error: zstd ${REQUIRED_ZSTD} required (matching deps/zstd), ` +
+ `found ${m ? m[1] : 'unknown'}`);
+ process.exit(1);
+ }
+}
+
+const PARAMS = ['exports', 'require', 'module', '__filename', '__dirname'];
+
+function* walk(dir) {
+ let ents;
+ try { ents = readdirSync(dir, { withFileTypes: true }); } catch { return; }
+ for (const e of ents.sort((a, b) => (a.name < b.name ? -1 : 1))) {
+ const p = join(dir, e.name);
+ if (e.isDirectory()) yield* walk(p);
+ else if (e.isFile() && p.endsWith('.js')) yield p;
+ }
+}
+
+async function main() {
+ checkZstd();
+ const { default: vm } = await import('node:vm');
+
+ const files = [];
+ for (const root of CORPUS) for (const f of walk(join(ROOT, root))) files.push(f);
+ files.sort();
+
+ const samples = mkdtempSync(join(tmpdir(), 'cc-dict-'));
+ const sampleFiles = [];
+ let ok = 0;
+ let r;
+ try {
+ for (const f of files) {
+ let code;
+ try { code = readFileSync(f, 'utf8'); } catch { continue; }
+ try {
+ const fn = vm.compileFunction(code, PARAMS,
+ { filename: f, produceCachedData: true });
+ const cd = fn.cachedData;
+ if (cd && cd.length > 0) {
+ const name = relative(ROOT, f).replace(/[\\/]/g, '_') + '.cache';
+ const out = join(samples, name);
+ writeFileSync(out, cd);
+ sampleFiles.push(out);
+ ok++;
+ }
+ } catch { /* skip modules V8 can't compile standalone */ }
+ }
+ sampleFiles.sort();
+ console.error(`harvested ${ok}/${files.length} code caches from ` +
+ `${CORPUS.join(', ')}`);
+
+ // One sample path per file is spread on argv; the fixed corpus (~1.4k
+ // files, a few hundred KB of paths) stays well under ARG_MAX.
+ r = spawnSync('zstd',
+ ['--train', ...sampleFiles, `--maxdict=${MAXDICT}`, '-f', '-o', OUT],
+ { stdio: ['ignore', 'ignore', 'inherit'] });
+ } finally {
+ rmSync(samples, { recursive: true, force: true });
+ }
+ if (r.status !== 0) { console.error('error: zstd --train failed'); process.exit(1); }
+
+ const size = readFileSync(OUT).length;
+ console.error(`wrote ${relative(ROOT, OUT)} (${size} bytes)`);
+}
+
+main().catch((e) => { console.error(e); process.exit(1); });

tools/train_compile_cache_dict.mjs

@@ -0,0 +1,168 @@
+#!/usr/bin/env node
+// =============================================================================
+// train_compile_cache_dict.mjs
+//
+// Maintainer tool that regenerates src/compile_cache_zstd.dict, the zstd
+// dictionary embedded in the node binary and used to shrink compile-cache
+// entries on disk (see src/compile_cache.cc).
+//
+// -----------------------------------------------------------------------------
+// What it does
+// -----------------------------------------------------------------------------
+// The node compile cache stores V8 code caches (the bytecode/metadata blob V8
+// produces when it compiles a module) so later runs can skip recompilation.
+// These blobs share a lot of structure across files, so we zstd-compress them
+// with a trained dictionary to cut the cache's on-disk footprint.
+//
+// This script builds that dictionary end to end:
+//
+// 1. Walks a fixed in-tree corpus (CORPUS below) in sorted order.
+// 2. For each .js file, harvests a V8 code cache via vm.compileFunction with
+// produceCachedData — the same shape the CommonJS loader produces at
+// runtime — and writes each blob to a temp directory.
+// 3. Feeds all the harvested blobs to `zstd --train`, capping the result at
+// MAXDICT (16 KiB) bytes.
+// 4. Overwrites src/compile_cache_zstd.dict with the trained dictionary.
+//
+// The shipped src/compile_cache_zstd.dict is the source of truth; this script
+// exists to document and reproduce exactly how that file was made, so a future
+// maintainer can regenerate it (e.g. after a V8/corpus change) and review the
+// resulting diff.
+//
+// -----------------------------------------------------------------------------
+// Usage
+// -----------------------------------------------------------------------------
+// node tools/train_compile_cache_dict.mjs
+//
+// Run from anywhere (paths are resolved relative to this file). The output is
+// written in place to src/compile_cache_zstd.dict; inspect the diff afterwards
+// and commit it if it looks right. Progress is printed to stderr.
+//
+// Prerequisites:
+// * the `zstd` CLI on PATH, version REQUIRED_ZSTD (matching deps/zstd).
+// * a built node on PATH (this is the node you invoke it with).
+//
+// Note: --predictable is added automatically (see below) — you do not need to
+// pass it yourself.
+//
+// -----------------------------------------------------------------------------
+// Reproducibility
+// -----------------------------------------------------------------------------
+// The output is byte-for-byte stable only if all of the following are pinned:
+//
+// * node is run with --predictable. V8 otherwise randomizes the string hash
+// seed per process, and that seed leaks into vm.compileFunction cachedData,
+// so every harvest (and therefore every trained dictionary) would differ
+// run-to-run even on the same machine. This script re-executes itself with
+// --predictable if needed.
+// * the corpus and its order are fixed (CORPUS below, walked sorted).
+// * the node build is fixed: cachedData embeds the V8 version and build
+// flags, so a different node produces a different (still valid) dictionary.
+// * the zstd CLI matches deps/zstd (currently 1.5.7). ZDICT training defaults
+// change between zstd releases; this script refuses to run on a mismatch.
+//
+// Training on --predictable caches is fine even though the runtime consumes
+// non-predictable caches: the dictionary only supplies shared substrings, and
+// Persist() keeps min(plain, dict) per entry, so a less-than-ideal dictionary
+// can never make any entry larger.
+// =============================================================================
+
+import { spawnSync } from 'node:child_process';
+import { readFileSync, writeFileSync, mkdtempSync, readdirSync,
+ rmSync } from 'node:fs';
+import { join, relative, dirname } from 'node:path';
+import { tmpdir } from 'node:os';
+import { fileURLToPath } from 'node:url';
+
+const ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
+const OUT = join(ROOT, 'src', 'compile_cache_zstd.dict');
+const MAXDICT = 16384;
+const REQUIRED_ZSTD = '1.5.7'; // keep in sync with deps/zstd/lib/zstd.h
+
+// Fixed corpus, relative to the repo root. Chosen to be diverse, always
+// present in a checkout, and disjoint from the held-out measurement corpora
+// (e.g. test/parallel) used to report size numbers.
+const CORPUS = ['lib', 'tools', 'deps/npm/node_modules'];
+
+// V8 randomizes the hash seed per process; re-exec under --predictable so the
+// harvested caches — and the trained dictionary — are deterministic.
+if (!process.execArgv.includes('--predictable')) {
+ const r = spawnSync(process.execPath,
+ ['--predictable', fileURLToPath(import.meta.url), ...process.argv.slice(2)],
+ { stdio: 'inherit' });
+ process.exit(r.status ?? 1);
+}
+
+function checkZstd() {
+ const r = spawnSync('zstd', ['--version'], { encoding: 'utf8' });
+ if (r.status !== 0) {
+ console.error('error: zstd CLI not found on PATH'); process.exit(1);
+ }
+ const m = r.stdout.match(/v(\d+\.\d+\.\d+)/);
+ if (!m || m[1] !== REQUIRED_ZSTD) {
+ console.error(`error: zstd ${REQUIRED_ZSTD} required (matching deps/zstd), ` +
+ `found ${m ? m[1] : 'unknown'}`);
+ process.exit(1);
+ }
+}
+
+const PARAMS = ['exports', 'require', 'module', '__filename', '__dirname'];
+
+function* walk(dir) {
+ let ents;
+ try { ents = readdirSync(dir, { withFileTypes: true }); } catch { return; }
+ for (const e of ents.sort((a, b) => (a.name < b.name ? -1 : 1))) {
+ const p = join(dir, e.name);
+ if (e.isDirectory()) yield* walk(p);
+ else if (e.isFile() && p.endsWith('.js')) yield p;
+ }
+}
+
+async function main() {
+ checkZstd();
+ const { default: vm } = await import('node:vm');
+
+ const files = [];
+ for (const root of CORPUS) for (const f of walk(join(ROOT, root))) files.push(f);
+ files.sort();
+
+ const samples = mkdtempSync(join(tmpdir(), 'cc-dict-'));
+ const sampleFiles = [];
+ let ok = 0;
+ let r;
+ try {
+ for (const f of files) {
+ let code;
+ try { code = readFileSync(f, 'utf8'); } catch { continue; }
+ try {
+ const fn = vm.compileFunction(code, PARAMS,
+ { filename: f, produceCachedData: true });
+ const cd = fn.cachedData;
+ if (cd && cd.length > 0) {
+ const name = relative(ROOT, f).replace(/[\\/]/g, '_') + '.cache';
+ const out = join(samples, name);
+ writeFileSync(out, cd);
+ sampleFiles.push(out);
+ ok++;
+ }
+ } catch { /* skip modules V8 can't compile standalone */ }
+ }
+ sampleFiles.sort();
+ console.error(`harvested ${ok}/${files.length} code caches from ` +
+ `${CORPUS.join(', ')}`);
+
+ // One sample path per file is spread on argv; the fixed corpus (~1.4k
+ // files, a few hundred KB of paths) stays well under ARG_MAX.
+ r = spawnSync('zstd',
+ ['--train', ...sampleFiles, `--maxdict=${MAXDICT}`, '-f', '-o', OUT],
+ { stdio: ['ignore', 'ignore', 'inherit'] });
+ } finally {
+ rmSync(samples, { recursive: true, force: true });
+ }
+ if (r.status !== 0) { console.error('error: zstd --train failed'); process.exit(1); }
+
+ const size = readFileSync(OUT).length;
+ console.error(`wrote ${relative(ROOT, OUT)} (${size} bytes)`);
+}
+
+main().catch((e) => { console.error(e); process.exit(1); });