fix: make audit runs side-effect free

.gitattributes

@@ -0,0 +1,3 @@
+audit/*.txt text eol=lf
+audit/matrix.json text eol=lf
+audit/*.md text eol=lf

.gitignore

@@ -1,3 +1,4 @@
 node_modules
 dist
 *.local
+audit/run

audit/harness.ts

@@ -1,4 +1,4 @@
-import { readFileSync, existsSync } from 'node:fs';
+import { appendFileSync, existsSync, mkdirSync, readFileSync } from 'node:fs';
 import { resolve } from 'node:path';
 import { cleanAndVerify, inspectFile } from '../src/core/pipeline';
 import { InspectionReport, VerificationResult } from '../src/core/types';
@@ -109,3 +109,15 @@ export const row = (o: Outcome) =>
     o.downloadable ? 'YES' : 'no',
     o.error ?? o.blocked ?? '',
   ].join(' | ').replace(/ \| $/, '');   // no dangling separator when there is no message
+
+/**
+ * How long a run took and how much heap it cost is a fact about the machine that ran it, not
+ * about FilePass. Recording it next to the evidence made the suite rewrite a tracked file on
+ * every run, which is exactly the kind of unexplained change the evidence chain exists to
+ * catch. Measurements go to an untracked run directory instead; the tracked notes keep the
+ * outcome, which is reproducible.
+ */
+export function measurement(line: string): void {
+  mkdirSync('audit/run', { recursive: true });
+  appendFileSync('audit/run/measurements.txt', line + '\n');
+}

audit/resource.test.ts

@@ -1,10 +1,9 @@
 import { describe, expect, it } from 'vitest';
-import { appendFileSync } from 'node:fs';
 import zlib from 'node:zlib';
 import { inspectFile } from '../src/core/pipeline';
 import { readChunks } from '../src/core/png';
 import { MalformedFileError } from '../src/core/types';
-import { fixture, runBytes as run } from './harness';
+import { fixture, measurement, runBytes as run } from './harness';
 
 const crc = (buf: Buffer) => {
   let c = ~0;
@@ -73,9 +72,11 @@ describe('phase 12: what a file may ask FilePass to unpack', () => {
     const before = process.memoryUsage().heapUsed;
     let outcome = 'parsed';
     try { await inspectFile(hostile); } catch (error) { outcome = (error as Error).constructor.name; }
-    appendFileSync('audit/sabotage-notes.txt',
-      `bomb x10 (~200 MB declared): ${outcome} after ${Date.now() - started} ms, `
-      + `heap delta ${((process.memoryUsage().heapUsed - before) / 1e6).toFixed(0)} MB, file ${(hostile.length / 1024).toFixed(0)} KB\n`);
+    // Not a tracked note either: this file is owned by sabotage.test.ts, and appending to a
+    // file another test truncates makes the result depend on which of the two ran last.
+    measurement(`bomb x10 (~200 MB declared): ${outcome} after ${Date.now() - started} ms, `
+      + `heap delta ${((process.memoryUsage().heapUsed - before) / 1e6).toFixed(0)} MB, `
+      + `file ${(hostile.length / 1024).toFixed(0)} KB`);
     expect(outcome).toBe('MalformedFileError');
   }, 120000);
 });

audit/sabotage-notes.txt

@@ -16,4 +16,4 @@ sniff: ZIP (docx/odt): refused (FilePass v0 supports JPEG, PNG and PDF files onl
 size: limit - 1: MalformedFileError - This JPEG file is damaged and cannot be read safely.
 size: limit: MalformedFileError - This JPEG file is damaged and cannot be read safely.
 size: limit + 1: FileTooLargeError - This file is larger than 50 MB. FilePass keeps everything in
-bomb: refused after 21 ms: The text in this image unpacks to more than FilePass will read, so it will not vouch for it.
+bomb: refused: The text in this image unpacks to more than FilePass will read, so it will not vouch for it.

audit/sabotage.test.ts

@@ -4,7 +4,7 @@ import zlib from 'node:zlib';
 import { inspectFile, verifyClean, cleanAndVerify } from '../src/core/pipeline';
 import { MAX_BYTES, sniffFormat } from '../src/core/sniff';
 import { FileTooLargeError, MalformedFileError, UnsupportedFileError } from '../src/core/types';
-import { fixture } from './harness';
+import { fixture, measurement } from './harness';
 
 const notes: string[] = [];
 const note = (line: string) => notes.push(line);
@@ -209,11 +209,14 @@ describe('phase 12: size and resource limits', () => {
     try {
       const report = await inspectFile(hostile);
       const finding = report.findings.find((f) => f.key.includes('bomb'));
-      result = `parsed in ${Date.now() - started} ms, finding value length ${finding?.value.length}, file ${hostile.length} bytes`;
+      result = `parsed, finding value length ${finding?.value.length}, file ${hostile.length} bytes`;
     } catch (error) {
-      result = `refused after ${Date.now() - started} ms: ${(error as Error).message}`;
+      result = `refused: ${(error as Error).message}`;
     }
     appendFileSync('audit/sabotage-notes.txt', `bomb: ${result}\n`);
+    // How long it took is a fact about this machine, not about FilePass, so it stays out of
+    // the tracked note: a number that changes on every run makes the evidence file change too.
+    measurement(`bomb (40 MB declared): ${result.split(':')[0]} after ${Date.now() - started} ms`);
     expect(result.length).toBeGreaterThan(0);
   }, 120000);
 });