From 5f4369caee19202e8fad2482c51d397fbea3430a Mon Sep 17 00:00:00 2001 From: jackwener Date: Mon, 3 Aug 2026 18:58:51 +0800 Subject: [PATCH 01/10] refactor(storage): make SQLite the sole operational authority --- .../automation-persistence-e2e.test.ts | 27 +- .../credential-store-migration.test.ts | 80 - .../daily-review-archive-store.test.ts | 142 +- .../src/main/__tests__/e2e-fixture.test.ts | 300 +- .../__tests__/execution-store-wiring.test.ts | 156 - .../__tests__/new-session-project.test.ts | 5 +- .../project-startup-migration.test.ts | 44 - .../__tests__/quote-companion-cleanup.test.ts | 19 +- .../__tests__/session-read-error-copy.test.ts | 8 +- apps/desktop/src/main/app-lifecycle.ts | 76 +- apps/desktop/src/main/automation-wiring.ts | 14 +- apps/desktop/src/main/boot.ts | 2 +- apps/desktop/src/main/credential-store.ts | 48 - .../src/main/daily-review-archive-store.ts | 218 +- apps/desktop/src/main/e2e-fixture.ts | 11 +- .../main/e2e-fixture/scenarios-artifacts.ts | 17 +- .../src/main/e2e-fixture/scenarios-chat.ts | 101 +- .../e2e-fixture/scenarios-deep-research.ts | 195 +- .../main/e2e-fixture/scenarios-sessions.ts | 42 +- .../main/e2e-fixture/scenarios-settings.ts | 119 +- .../src/main/e2e-fixture/seed-helpers.ts | 50 +- .../main/oauth/shared-credential-bridge.ts | 185 +- .../src/main/project-startup-migration.ts | 22 - .../src/main/quote-companion-cleanup.ts | 116 +- .../cli/src/__tests__/inspect-command.test.ts | 13 +- .../src/__tests__/runtime-bootstrap.test.ts | 4 +- packages/cli/src/runtime-bootstrap.ts | 18 +- packages/headless/harbor/maka_trajectory.py | 5 - packages/headless/src/__tests__/cli.test.ts | 8 +- .../src/__tests__/harbor-adapter.test.ts | 25 - .../src/__tests__/task-run-inspect.test.ts | 13 +- .../src/__tests__/task-run-store.test.ts | 171 +- packages/headless/src/task-run-store.ts | 377 +-- .../__tests__/execution-host-recovery.test.ts | 138 +- .../fixtures/execution-host-suite.ts | 12 - .../__tests__/root-admission-owner.test.ts | 6 +- .../session-retirement-coordinator.test.ts | 18 +- .../server/session-retirement-coordinator.ts | 2 - .../agent-run-steering-recovery.test.ts | 38 +- .../src/__tests__/context-diagnostics.test.ts | 6 +- .../src/__tests__/conversation-copy.test.ts | 28 +- .../src/__tests__/deep-research-tools.test.ts | 10 +- .../src/__tests__/execution-inspect.test.ts | 13 +- .../runtime-continuation-crash.test.ts | 8 +- ...untime-read-model-persisted-compat.test.ts | 102 - .../__tests__/runtime-resume-crash.test.ts | 11 +- .../sandbox-boundary-restart-recovery.test.ts | 18 +- .../src/__tests__/shell-run-manager.test.ts | 40 +- .../stream-graph-coordinator.test.ts | 19 +- packages/runtime/src/ai-sdk-backend.ts | 2 +- packages/runtime/src/fake-backend.ts | 2 +- packages/runtime/src/materializer.ts | 8 +- .../runtime/src/runtime-event-adapters.ts | 6 +- packages/runtime/src/session-manager.ts | 2 +- packages/runtime/src/tool-runtime.ts | 2 +- packages/storage/package.json | 2 - ...nt-graph-supervisor-root-admission.test.ts | 8 +- .../src/__tests__/agent-run-store.test.ts | 1883 ------------ .../src/__tests__/artifact-store.test.ts | 137 +- .../__tests__/artifact-writer-lock.test.ts | 21 +- .../__tests__/automation-authority.test.ts | 75 +- .../src/__tests__/automation-store.test.ts | 137 - ...claimed-agent-graph-root-admission.test.ts | 12 +- .../__tests__/credential-migration.test.ts | 179 -- .../src/__tests__/deep-research-store.test.ts | 362 --- .../src/__tests__/execution-stores.test.ts | 1967 ------------- .../src/__tests__/interaction-store.test.ts | 822 ------ .../legacy-execution-public-surface.test.ts | 20 - .../legacy-storage-public-surface.test.ts | 46 - .../__tests__/message-receipt-store.test.ts | 59 - .../operational-state-backup.test.ts | 857 +----- .../__tests__/operational-state-store.test.ts | 300 +- .../src/__tests__/plan-reminder-store.test.ts | 490 ---- .../storage/src/__tests__/plan-store.test.ts | 237 -- .../src/__tests__/pricing-store.test.ts | 271 -- .../src/__tests__/project-catalog.test.ts | 48 +- .../project-session-migration.test.ts | 170 -- .../__tests__/runtime-event-transfer.test.ts | 407 --- .../__tests__/session-bundle-policy.test.ts | 1115 +------- .../session-metadata-maintenance.test.ts | 165 -- .../session-metadata-transfer.test.ts | 424 --- .../src/__tests__/session-store.test.ts | 1770 +----------- .../__tests__/settings-store-usage.test.ts | 151 +- .../src/__tests__/shell-run-store.test.ts | 509 ---- .../__tests__/sqlite-artifact-store.test.ts | 292 -- .../sqlite-core-execution-store.test.ts | 316 +-- .../sqlite-session-metadata-store.test.ts | 107 +- .../__tests__/sqlite-session-store.test.ts | 1275 --------- .../src/__tests__/sqlite-usage-store.test.ts | 160 -- .../__tests__/sqlite-workflow-store.test.ts | 252 +- .../__tests__/task-ledger-authority.test.ts | 71 +- .../src/__tests__/task-ledger-store.test.ts | 1338 --------- .../src/__tests__/telemetry-repo.test.ts | 855 ------ ...pace-version-authority-persistence.test.ts | 27 +- .../storage/src/agent-graph-control-store.ts | 5 +- packages/storage/src/agent-run-store.ts | 2519 ++--------------- .../storage/src/artifact-metadata-codec.ts | 35 +- .../storage/src/artifact-storage-layout.ts | 14 +- packages/storage/src/artifact-store.ts | 53 +- packages/storage/src/automation-authority.ts | 137 +- packages/storage/src/automation-store.ts | 140 +- packages/storage/src/credential-store.ts | 135 +- packages/storage/src/deep-research-store.ts | 257 +- packages/storage/src/execution-stores.ts | 4 +- packages/storage/src/index.ts | 34 +- .../storage/src/interaction-store-public.ts | 5 +- packages/storage/src/interaction-store.ts | 1001 +------ packages/storage/src/jsonl-append.ts | 128 - .../src/legacy-execution-test-support.ts | 13 - .../src/legacy-storage-test-support.ts | 16 - packages/storage/src/message-receipt-store.ts | 322 +-- .../storage/src/operational-state-backup.ts | 1363 ++------- .../storage/src/operational-state-store.ts | 381 +-- packages/storage/src/plan-reminder-store.ts | 143 +- packages/storage/src/plan-store.ts | 254 +- packages/storage/src/pricing-store.ts | 311 -- packages/storage/src/project-catalog.ts | 24 - .../storage/src/project-session-migration.ts | 37 - .../storage/src/runtime-event-persistence.ts | 73 + .../storage/src/runtime-event-transfer.ts | 317 --- packages/storage/src/session-bundle-policy.ts | 1465 ++-------- .../src/session-metadata-maintenance.ts | 223 -- .../storage/src/session-metadata-transfer.ts | 284 -- packages/storage/src/session-store.ts | 1277 +-------- packages/storage/src/session-transcript.ts | 38 - packages/storage/src/shell-run-store.ts | 379 +-- .../storage/src/sqlite-artifact-metadata.ts | 111 +- packages/storage/src/sqlite-runtime-schema.ts | 20 +- packages/storage/src/sqlite-runtime-store.ts | 34 +- .../src/sqlite-session-metadata-schema.ts | 30 +- .../src/sqlite-session-metadata-store.ts | 249 +- packages/storage/src/sqlite-usage-store.ts | 180 +- .../storage/src/sqlite-workflow-schema.ts | 22 +- packages/storage/src/task-ledger-store.ts | 701 +---- packages/storage/src/telemetry-file-schema.ts | 142 +- packages/storage/src/telemetry-repo.ts | 466 --- packages/storage/src/usage-stats-store.ts | 72 +- packages/storage/src/usage-stores.ts | 5 +- 138 files changed, 2649 insertions(+), 31229 deletions(-) delete mode 100644 apps/desktop/src/main/__tests__/credential-store-migration.test.ts delete mode 100644 apps/desktop/src/main/__tests__/execution-store-wiring.test.ts delete mode 100644 apps/desktop/src/main/__tests__/project-startup-migration.test.ts delete mode 100644 apps/desktop/src/main/project-startup-migration.ts delete mode 100644 packages/runtime/src/__tests__/runtime-read-model-persisted-compat.test.ts delete mode 100644 packages/storage/src/__tests__/agent-run-store.test.ts delete mode 100644 packages/storage/src/__tests__/automation-store.test.ts delete mode 100644 packages/storage/src/__tests__/credential-migration.test.ts delete mode 100644 packages/storage/src/__tests__/deep-research-store.test.ts delete mode 100644 packages/storage/src/__tests__/execution-stores.test.ts delete mode 100644 packages/storage/src/__tests__/interaction-store.test.ts delete mode 100644 packages/storage/src/__tests__/legacy-execution-public-surface.test.ts delete mode 100644 packages/storage/src/__tests__/legacy-storage-public-surface.test.ts delete mode 100644 packages/storage/src/__tests__/message-receipt-store.test.ts delete mode 100644 packages/storage/src/__tests__/plan-reminder-store.test.ts delete mode 100644 packages/storage/src/__tests__/plan-store.test.ts delete mode 100644 packages/storage/src/__tests__/pricing-store.test.ts delete mode 100644 packages/storage/src/__tests__/project-session-migration.test.ts delete mode 100644 packages/storage/src/__tests__/runtime-event-transfer.test.ts delete mode 100644 packages/storage/src/__tests__/session-metadata-maintenance.test.ts delete mode 100644 packages/storage/src/__tests__/session-metadata-transfer.test.ts delete mode 100644 packages/storage/src/__tests__/shell-run-store.test.ts delete mode 100644 packages/storage/src/__tests__/sqlite-artifact-store.test.ts delete mode 100644 packages/storage/src/__tests__/sqlite-session-store.test.ts delete mode 100644 packages/storage/src/__tests__/sqlite-usage-store.test.ts delete mode 100644 packages/storage/src/__tests__/task-ledger-store.test.ts delete mode 100644 packages/storage/src/__tests__/telemetry-repo.test.ts delete mode 100644 packages/storage/src/jsonl-append.ts delete mode 100644 packages/storage/src/legacy-execution-test-support.ts delete mode 100644 packages/storage/src/legacy-storage-test-support.ts delete mode 100644 packages/storage/src/project-session-migration.ts create mode 100644 packages/storage/src/runtime-event-persistence.ts delete mode 100644 packages/storage/src/runtime-event-transfer.ts delete mode 100644 packages/storage/src/session-metadata-maintenance.ts delete mode 100644 packages/storage/src/session-metadata-transfer.ts delete mode 100644 packages/storage/src/session-transcript.ts diff --git a/apps/desktop/src/main/__tests__/automation-persistence-e2e.test.ts b/apps/desktop/src/main/__tests__/automation-persistence-e2e.test.ts index f17fb28783..d0ef6df773 100644 --- a/apps/desktop/src/main/__tests__/automation-persistence-e2e.test.ts +++ b/apps/desktop/src/main/__tests__/automation-persistence-e2e.test.ts @@ -1,13 +1,11 @@ /** * End-to-end: durable cron persistence + cross-session query/management. * - * Exercises the REAL host wiring (createMainAutomationWiring) against a REAL - * FileAutomationStore on a real temp workspace, simulating an app restart: + * Exercises the real host wiring against the operational SQLite authority, + * simulating an app restart: * - * session A creates a durable cron ──sync──► /automations.json - * │ - * (restart: a fresh wiring loads it) ◄──loadAll────────┘ - * │ + * session A creates a durable cron ──sync──► runtime.sqlite + * (restart: a fresh wiring loads it) ◄──loadAll───────┘ * session B (never saw it) lists / pauses / resumes / deletes it * * This is the query-and-persistence loop the reviewer asked for: a persisted @@ -18,10 +16,11 @@ import { strict as assert } from 'node:assert'; import { describe, it, before, after } from 'node:test'; -import { mkdtemp, rm, readFile } from 'node:fs/promises'; +import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import type { MakaToolContext, MakaTool } from '@maka/runtime'; +import { createAutomationStore } from '@maka/storage'; import { createMainAutomationWiring } from '../automation-wiring.js'; function ctx(sessionId: string): MakaToolContext { @@ -60,15 +59,13 @@ function automationTool(wiring: ReturnType): MakaTool { } async function readStore(workspaceRoot: string): Promise> { - try { - const raw = await readFile(join(workspaceRoot, 'automations.json'), 'utf8'); - return (JSON.parse(raw) as { automations: Array<{ id: string; name: string }> }).automations; - } catch { - return []; - } + return (await createAutomationStore(workspaceRoot).loadAll()).map(({ id, name }) => ({ + id, + name, + })); } -/** The store sync is fire-and-forget; poll the file until it settles. */ +/** Store sync is fire-and-forget; poll the authority until it settles. */ async function waitForStore( workspaceRoot: string, predicate: (rows: Array<{ id: string; name: string }>) => boolean, @@ -175,7 +172,7 @@ describe('E2E: durable cron persistence + cross-session query/management', () => }); describe('E2E: a cron-disabled host (CLI) sharing the workspace never clobbers durable crons', () => { - it('a heartbeat-only wiring neither loads nor overwrites the owner\'s automations.json', async () => { + it('a heartbeat-only wiring neither loads nor overwrites durable Automations', async () => { const ws = await mkdtemp(join(tmpdir(), 'maka-automation-clobber-')); try { // ── owner (cron-enabled, desktop) creates a durable cron ────────────── diff --git a/apps/desktop/src/main/__tests__/credential-store-migration.test.ts b/apps/desktop/src/main/__tests__/credential-store-migration.test.ts deleted file mode 100644 index 31201b6b3d..0000000000 --- a/apps/desktop/src/main/__tests__/credential-store-migration.test.ts +++ /dev/null @@ -1,80 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, it } from 'node:test'; -import { migrateLegacyCredentials, type SafeStorageLike } from '../credential-store.js'; - -// The migration ORCHESTRATION — shared lock, version gate, fail-closed aborts, -// atomic 0600 rewrite, idempotency, malformed/missing handling — is tested in -// @maka/storage against migrateLegacyCredentialFile. This file covers only the -// desktop GLUE: that safeStorage is wired through correctly (base64-decode then -// decryptString) and that its availability is propagated. - -/** Fake safeStorage: "decrypt" strips an `enc:` prefix so every value is proven - * to round-trip through decryptString. */ -function fakeSafeStorage(available: boolean): SafeStorageLike { - return { - isEncryptionAvailable: () => available, - decryptString: (buf) => buf.toString('utf8').replace(/^enc:/, ''), - }; -} - -/** The legacy on-disk encoding: base64(safeStorage.encryptString(value)). */ -function encrypted(value: string): string { - return Buffer.from(`enc:${value}`).toString('base64'); -} - -async function withWorkspace(fn: (root: string, path: string) => Promise): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-cred-mig-')); - try { - return await fn(root, join(root, 'credentials.json')); - } finally { - await rm(root, { recursive: true, force: true }); - } -} - -describe('migrateLegacyCredentials (desktop safeStorage glue)', () => { - it('base64-decodes and safeStorage-decrypts every kind to v1 plaintext-0600', async () => { - await withWorkspace(async (root, path) => { - // Full scope: API key + bot token + proxy password all migrate. - await writeFile( - path, - JSON.stringify({ - values: { - 'openai:apiKey': encrypted('sk-1'), - 'settings:bot:telegram:botToken': encrypted('tok-2'), - 'settings:network-proxy:proxyPassword': encrypted('pw-3'), - }, - }), - 'utf8', - ); - - await migrateLegacyCredentials(root, fakeSafeStorage(true)); - - const after = JSON.parse(await readFile(path, 'utf8')) as { - version: number; - values: Record; - }; - assert.equal(after.version, 1); - assert.deepEqual(after.values, { - 'openai:apiKey': 'sk-1', - 'settings:bot:telegram:botToken': 'tok-2', - 'settings:network-proxy:proxyPassword': 'pw-3', - }); - if (process.platform !== 'win32') { - assert.equal((await stat(path)).mode & 0o777, 0o600); // owner-only at rest - } - }); - }); - - it('propagates safeStorage unavailability: aborts and leaves the file intact', async () => { - await withWorkspace(async (root, path) => { - const original = JSON.stringify({ values: { 'openai:apiKey': encrypted('sk-1') } }); - await writeFile(path, original, 'utf8'); - - await assert.rejects(migrateLegacyCredentials(root, fakeSafeStorage(false)), /unavailable/); - assert.equal(await readFile(path, 'utf8'), original); // untouched — no data loss - }); - }); -}); diff --git a/apps/desktop/src/main/__tests__/daily-review-archive-store.test.ts b/apps/desktop/src/main/__tests__/daily-review-archive-store.test.ts index c1caa1e4e6..b274a23924 100644 --- a/apps/desktop/src/main/__tests__/daily-review-archive-store.test.ts +++ b/apps/desktop/src/main/__tests__/daily-review-archive-store.test.ts @@ -1,10 +1,9 @@ import { strict as assert } from 'node:assert'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, describe, it } from 'node:test'; import type { DailyReviewArchive } from '@maka/core'; - import { createDailyReviewArchiveStore } from '../daily-review-archive-store.js'; const roots: string[] = []; @@ -17,57 +16,10 @@ afterEach(async () => { await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true }))); }); -async function aRoot(): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-daily-review-store-')); - roots.push(root); - return root; -} - -function anArchive(id = '2026-08-03-1d'): DailyReviewArchive { - return { - id, - day: DAY, - range: 1, - status: 'ok', - generatedAt: 3, - trigger: 'manual', - modelKey: '', - sections: { summary: 'healthy' }, - totals: { sessionCount: 1, requestCount: 2, totalTokens: 3, costUsd: 4, errorCount: 0 }, - }; -} - -describe('Daily Review archive migration', () => { - it('reads a legacy deep archive through the range contract', async () => { - const root = await aRoot(); - const archiveRoot = join(root, 'daily-reviews', 'archive'); - await mkdir(archiveRoot, { recursive: true }); - await writeFile( - join(archiveRoot, '2026-08-03-deep.json'), - JSON.stringify({ - id: '2026-08-03-deep', - day: DAY, - mode: 'deep', - status: 'ok', - generatedAt: 3, - trigger: 'manual', - modelKey: '', - sections: { summary: 'legacy' }, - totals: { sessionCount: 1, requestCount: 2, totalTokens: 3, costUsd: 4, errorCount: 0 }, - }), - ); - - const store = createDailyReviewArchiveStore(root); - const archive = await store.getArchive('2026-08-03-deep'); - assert.equal(archive?.range, 7); - assert.equal(archive?.sections.summary, 'legacy'); - assert.deepEqual((await store.listArchives()).map((item) => item.range), [7]); - }); - - it('persists only the three canonical settings', async () => { - const root = await aRoot(); - const store = createDailyReviewArchiveStore(root); - const next = await store.setConfig({ +describe('Daily Review SQLite archive store', () => { + it('persists canonical config across store instances', async () => { + const root = await workspace(); + const next = await createDailyReviewArchiveStore(root).setConfig({ enabled: true, executeTime: '07:45', modelKey: 'openai::gpt-5', @@ -78,49 +30,63 @@ describe('Daily Review archive migration', () => { executeTime: '07:45', modelKey: 'openai::gpt-5', }); - assert.deepEqual( - JSON.parse(await readFile(join(root, 'daily-reviews', 'config.json'), 'utf8')), - next, - ); + assert.deepEqual(await createDailyReviewArchiveStore(root).getConfig(), next); }); -}); -describe('Daily Review archive corruption recovery', () => { - it('keeps healthy archives available when a sibling file is corrupt', async () => { - const root = await aRoot(); + it('stores, replaces, and deletes one canonical archive', async () => { + const root = await workspace(); const store = createDailyReviewArchiveStore(root); - await store.putArchive(anArchive()); - await writeFile( - join(root, 'daily-reviews', 'archive', '2026-08-02-1d.json'), - '{"id":"2026-08-02-1d"', - ); + await store.putArchive(archive('2026-08-03-1d', 3)); + await store.putArchive({ + ...archive('2026-08-03-1d', 4), + sections: { summary: 'updated' }, + }); + + assert.equal((await store.getArchive('2026-08-03-1d'))?.sections.summary, 'updated'); + assert.deepEqual((await store.listArchives()).map(({ id }) => id), ['2026-08-03-1d']); - assert.deepEqual((await store.listArchives()).map((archive) => archive.id), [ - '2026-08-03-1d', - ]); + await store.deleteArchive('2026-08-03-1d'); + assert.equal(await store.getArchive('2026-08-03-1d'), null); }); - it('keeps healthy archives available when a sibling has an invalid schema', async () => { - const root = await aRoot(); + it('orders newest first and prunes older archives in SQLite', async () => { + const root = await workspace(); const store = createDailyReviewArchiveStore(root); - await store.putArchive(anArchive()); - await writeFile( - join(root, 'daily-reviews', 'archive', '2026-08-02-1d.json'), - JSON.stringify({ id: '2026-08-02-1d', range: 1 }), - ); + await store.putArchive(archive('2026-08-01-1d', 1)); + await store.putArchive(archive('2026-08-02-1d', 2)); + await store.putArchive(archive('2026-08-03-1d', 3)); - assert.deepEqual((await store.listArchives()).map((archive) => archive.id), [ - '2026-08-03-1d', - ]); - }); - - it('treats a corrupt canonical archive as missing so generation can replace it', async () => { - const root = await aRoot(); - const archiveRoot = join(root, 'daily-reviews', 'archive'); - await mkdir(archiveRoot, { recursive: true }); - await writeFile(join(archiveRoot, '2026-08-03-1d.json'), '{'); + await store.prune(2); - const store = createDailyReviewArchiveStore(root); - assert.equal(await store.getArchive('2026-08-03-1d'), null); + assert.deepEqual( + (await store.listArchives()).map(({ id }) => id), + ['2026-08-03-1d', '2026-08-02-1d'], + ); }); }); + +async function workspace(): Promise { + const root = await mkdtemp(join(tmpdir(), 'maka-daily-review-store-')); + roots.push(root); + return root; +} + +function archive(id: string, generatedAt: number): DailyReviewArchive { + return { + id, + day: DAY, + range: 1, + status: 'ok', + generatedAt, + trigger: 'manual', + modelKey: '', + sections: { summary: 'healthy' }, + totals: { + sessionCount: 1, + requestCount: 2, + totalTokens: 3, + costUsd: 4, + errorCount: 0, + }, + }; +} diff --git a/apps/desktop/src/main/__tests__/e2e-fixture.test.ts b/apps/desktop/src/main/__tests__/e2e-fixture.test.ts index 21fb1cdcf6..1993b0249b 100644 --- a/apps/desktop/src/main/__tests__/e2e-fixture.test.ts +++ b/apps/desktop/src/main/__tests__/e2e-fixture.test.ts @@ -1,8 +1,10 @@ import { strict as assert } from 'node:assert'; import { mkdtemp, readFile, rm } from 'node:fs/promises'; import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; import { describe, it } from 'node:test'; import { tmpdir } from 'node:os'; +import type { SessionHeader, StoredMessage } from '@maka/core'; import { discoverMarkedStorageRoot } from '@maka/storage/root-authority'; import { getE2eFixtureState, @@ -12,6 +14,19 @@ import { seedE2eFixture, } from '../e2e-fixture.js'; +function readArtifactMetadata(workspaceRoot: string): string[] { + const database = new DatabaseSync(join(workspaceRoot, 'runtime.sqlite'), { readOnly: true }); + try { + return ( + database + .prepare('SELECT record_json FROM artifact_records ORDER BY created_at, storage_key') + .all() as Array<{ record_json: string }> + ).map((row) => row.record_json); + } finally { + database.close(); + } +} + describe('e2e-fixture mode', () => { it('stays fully disabled when MAKA_E2E_FIXTURE is unset', () => { const fixture = resolveE2eFixture(undefined, false); @@ -348,13 +363,29 @@ describe('e2e-fixture mode', () => { }); assert.equal((await discoverMarkedStorageRoot({ path: workspaceRoot })).kind, 'interactive'); assert.equal(getE2eFixtureState(fixture)?.activeSessionId, 'e2e-fixture-turn'); - const tasks = JSON.parse(await readFile( - join(workspaceRoot, 'sessions', 'e2e-fixture-turn', 'tasks.json'), - 'utf8', - )) as Array<{ key: string; parentId?: string; status: string; owner?: { actor: string } }>; + const tasks = withRuntimeDatabase(workspaceRoot, (database) => { + const row = database + .prepare(` + SELECT record_json AS recordJson + FROM workflow_task_ledger_projections + WHERE session_id = ? + `) + .get('e2e-fixture-turn') as { recordJson?: unknown } | undefined; + if (typeof row?.recordJson !== 'string') throw new Error('Task projection not found'); + return JSON.parse(row.recordJson) as Array<{ + id: string; + key: string; + parentId?: string; + status: string; + owner?: { actor: string }; + }>; + }); assert.deepEqual(tasks.map((task) => task.key), ['T1', 'T1.1', 'T1.2', 'T1.2.1', 'T2', 'T3']); assert.equal(tasks.find((task) => task.key === 'T1.2')?.owner?.actor, 'child_agent'); - assert.equal(tasks.find((task) => task.key === 'T1.2.1')?.parentId, 'task-child-ui'); + assert.equal( + tasks.find((task) => task.key === 'T1.2.1')?.parentId, + tasks.find((task) => task.key === 'T1.2')?.id, + ); } finally { await rm(workspaceRoot, { recursive: true, force: true }); } @@ -372,20 +403,20 @@ describe('e2e-fixture mode', () => { now: 1_700_000_000_000, }); assert.equal(getE2eFixtureState(fixture)?.activeSessionId, 'e2e-fixture-deep-research'); - const ledger = await readFile( - join( - workspaceRoot, - 'sessions', - 'e2e-fixture-deep-research', - 'deep-research', - 'events.jsonl', + const events = withRuntimeDatabase(workspaceRoot, (database) => + ( + database + .prepare(` + SELECT record_json AS recordJson + FROM workflow_deep_research_events + WHERE session_id = ? + ORDER BY sequence + `) + .all('e2e-fixture-deep-research') as Array<{ recordJson: string }> + ).map((row) => + JSON.parse(row.recordJson) as { type: string; artifact?: { role?: string } }, ), - 'utf8', ); - const events = ledger.trim().split('\n').map((line) => JSON.parse(line) as { - type: string; - artifact?: { role?: string }; - }); assert.equal(events.at(-1)?.type, 'research_completed'); assert.equal( events.filter((event) => event.artifact?.role === 'report_section').length, @@ -509,33 +540,20 @@ describe('e2e-fixture mode', () => { // the "active + stale → pill still visible" invariant is exercised. assert.equal(state?.activeSessionId, 'e2e-fixture-stale-fake'); - // Connection list MUST NOT contain `fake` / `fake-claude` slugs — - // those are what makes the seeded sessions stale. + // The missing `fake` connection makes the active Session unavailable. const connections = JSON.parse( await readFile(join(workspaceRoot, 'llm-connections.json'), 'utf8'), ) as { defaultSlug: string; connections: Array<{ slug: string }> }; const slugs = new Set(connections.connections.map((c) => c.slug)); assert.equal(slugs.has('fake'), false, 'fake slug must not be a real connection'); - assert.equal(slugs.has('fake-claude'), false, 'fake-claude slug must not be a real connection'); assert.equal(slugs.has('zai-live'), true, 'zai-live must be in the connection list (healthy session uses it)'); - // Three session.jsonl files: one for each session. - const sessionDirs = await Promise.all( - ['e2e-fixture-stale-fake', 'e2e-fixture-stale-legacy', 'e2e-fixture-healthy'].map(async (id) => { - const file = await readFile(join(workspaceRoot, 'sessions', id, 'session.jsonl'), 'utf8'); - return JSON.parse(file.split('\n')[0]!) as { - backend: string; - llmConnectionSlug: string; - model: string; - }; - }), - ); - assert.equal(sessionDirs[0]?.backend, 'fake'); - assert.equal(sessionDirs[0]?.llmConnectionSlug, 'fake'); - assert.equal(sessionDirs[1]?.backend, 'claude'); - assert.equal(sessionDirs[1]?.llmConnectionSlug, 'fake-claude'); - assert.equal(sessionDirs[2]?.backend, 'ai-sdk'); - assert.equal(sessionDirs[2]?.llmConnectionSlug, 'zai-live'); + const stale = await readSessionHeader(workspaceRoot, 'e2e-fixture-stale-fake'); + const healthy = await readSessionHeader(workspaceRoot, 'e2e-fixture-healthy'); + assert.equal(stale.backend, 'fake'); + assert.equal(stale.llmConnectionSlug, 'fake'); + assert.equal(healthy.backend, 'ai-sdk'); + assert.equal(healthy.llmConnectionSlug, 'zai-live'); } finally { await rm(workspaceRoot, { recursive: true, force: true }); } @@ -571,12 +589,7 @@ describe('e2e-fixture mode', () => { ]; for (const expected of expectedSessions) { - const file = await readFile(join(workspaceRoot, 'sessions', expected.id, 'session.jsonl'), 'utf8'); - const header = JSON.parse(file.split('\n')[0]!) as { - status: string; - blockedReason?: string; - isArchived: boolean; - }; + const header = await readSessionHeader(workspaceRoot, expected.id); assert.equal(header.status, expected.status, `${expected.id} should be ${expected.status}`); if ('blockedReason' in expected && expected.blockedReason !== undefined) { assert.equal( @@ -618,14 +631,13 @@ describe('e2e-fixture mode', () => { credentialStore: fakeCredentialStore(), now: 1_700_000_000_000, }); - // The on-disk status is `running` so the status gate self-heals like the + // The durable status is `running` so the status gate self-heals like the // real backgrounded-session path; the lone user message is the tail turn // the indicator anchors to. - const file = await readFile(join(workspaceRoot, 'sessions', 'e2e-fixture-processing', 'session.jsonl'), 'utf8'); - const lines = file.split('\n').filter(Boolean); - const header = JSON.parse(lines[0]!) as { status: string }; + const header = await readSessionHeader(workspaceRoot, 'e2e-fixture-processing'); assert.equal(header.status, 'running'); - const userMessages = lines.slice(1).map((l) => JSON.parse(l) as { type: string }).filter((m) => m.type === 'user'); + const userMessages = (await readSessionMessages(workspaceRoot, 'e2e-fixture-processing')) + .filter((message) => message.type === 'user'); assert.equal(userMessages.length, 1, 'a lone user prompt anchors the tail turn'); } finally { await rm(workspaceRoot, { recursive: true, force: true }); @@ -649,32 +661,35 @@ describe('e2e-fixture mode', () => { assert.equal(state?.sidebarCollapsed, false); assert.equal(state?.activeSessionId, 'e2e-fixture-turn'); - const reminders = JSON.parse( - await readFile(join(workspaceRoot, 'plan-reminders.json'), 'utf8'), - ) as Array<{ + const reminders = withRuntimeDatabase(workspaceRoot, (database) => + ( + database + .prepare('SELECT record_json AS recordJson FROM workflow_plan_reminders') + .all() as Array<{ recordJson: string }> + ).map((row) => JSON.parse(row.recordJson)) as Array<{ id: string; title: string; status: string; enabled: boolean; nextRunAt?: number; lastRun?: { status: string; message: string }; - }>; - assert.deepEqual(reminders.map((reminder) => reminder.id), [ - 'visual-plan-reminder-standup', - 'visual-plan-reminder-paused', - 'visual-plan-reminder-weekly-review', - 'visual-plan-reminder-completed', - ]); - assert.equal(reminders[0]?.status, 'scheduled'); - assert.equal(reminders[0]?.enabled, true); - assert.equal(typeof reminders[0]?.nextRunAt, 'number'); - assert.equal(reminders[1]?.status, 'paused'); - assert.equal(reminders[1]?.enabled, false); - assert.equal(reminders[2]?.status, 'scheduled'); - assert.equal(reminders[2]?.enabled, true); - assert.equal(reminders[3]?.status, 'completed'); - assert.equal(reminders[3]?.lastRun?.status, 'triggered'); - assert.match(reminders[3]?.lastRun?.message ?? '', /计划提醒/); + }>, + ); + assert.equal(reminders.length, 4); + const scheduled = reminders.find((reminder) => reminder.title === '同步项目风险'); + const paused = reminders.find((reminder) => reminder.title === '暂停的发布检查'); + const weekly = reminders.find((reminder) => reminder.title === '每周竞品动态追踪'); + const completed = reminders.find((reminder) => reminder.title === '已触发的本地提醒'); + assert.equal(scheduled?.status, 'scheduled'); + assert.equal(scheduled?.enabled, true); + assert.equal(typeof scheduled?.nextRunAt, 'number'); + assert.equal(paused?.status, 'paused'); + assert.equal(paused?.enabled, false); + assert.equal(weekly?.status, 'scheduled'); + assert.equal(weekly?.enabled, true); + assert.equal(completed?.status, 'completed'); + assert.equal(completed?.lastRun?.status, 'triggered'); + assert.match(completed?.lastRun?.message ?? '', /计划提醒/); } finally { await rm(workspaceRoot, { recursive: true, force: true }); } @@ -821,13 +836,9 @@ describe('e2e-fixture mode', () => { assert.equal(state?.sidebarCollapsed, false, 'sidebar row action screenshots must expand the seeded sidebar'); assert.equal(state?.activeSessionId, 'e2e-fixture-sidebar-long-00'); - // Same 60-session seed actually lands on disk so the sidebar + // Same 60-session seed is durable so the sidebar // is fully populated for the actions-visible capture. - const file = await readFile( - join(workspaceRoot, 'sessions', 'e2e-fixture-sidebar-long-00', 'session.jsonl'), - 'utf8', - ); - const header = JSON.parse(file.split('\n')[0]!) as { id: string; status: string }; + const header = await readSessionHeader(workspaceRoot, 'e2e-fixture-sidebar-long-00'); assert.equal(header.id, 'e2e-fixture-sidebar-long-00'); assert.equal(header.status, 'active'); } finally { @@ -862,13 +873,9 @@ describe('e2e-fixture mode', () => { // sidebar behind the modal looks identical to that baseline. assert.equal(state?.activeSessionId, 'e2e-fixture-sidebar-long-00'); - // Same 60-session seed actually lands on disk so the sidebar + // Same 60-session seed is durable so the sidebar // is fully populated behind the modal. - const file = await readFile( - join(workspaceRoot, 'sessions', 'e2e-fixture-sidebar-long-00', 'session.jsonl'), - 'utf8', - ); - const header = JSON.parse(file.split('\n')[0]!) as { id: string; status: string }; + const header = await readSessionHeader(workspaceRoot, 'e2e-fixture-sidebar-long-00'); assert.equal(header.id, 'e2e-fixture-sidebar-long-00'); assert.equal(header.status, 'active'); } finally { @@ -892,11 +899,7 @@ describe('e2e-fixture mode', () => { assert.equal(state?.paletteOpen, true, 'paletteOpen must be true so the renderer auto-opens CommandPalette'); assert.equal(state?.activeSessionId, 'e2e-fixture-sidebar-long-00'); - const file = await readFile( - join(workspaceRoot, 'sessions', 'e2e-fixture-sidebar-long-00', 'session.jsonl'), - 'utf8', - ); - const header = JSON.parse(file.split('\n')[0]!) as { id: string; status: string }; + const header = await readSessionHeader(workspaceRoot, 'e2e-fixture-sidebar-long-00'); assert.equal(header.id, 'e2e-fixture-sidebar-long-00'); assert.equal(header.status, 'active'); } finally { @@ -926,22 +929,18 @@ describe('e2e-fixture mode', () => { assert.equal(state?.activeSessionId, 'e2e-fixture-sidebar-long-00'); assert.equal(state?.sidebarCollapsed, false, 'sidebar scroll screenshots must expand the seeded sidebar'); - // Verify all 60 sessions exist on disk with deterministic IDs + + // Verify all 60 Sessions exist in SQLite with deterministic IDs + // monotonically decreasing lastMessageAt (newest first). let previousLastMessageAt = Infinity; for (let i = 0; i < 60; i++) { const idSuffix = String(i).padStart(2, '0'); const sessionId = 'e2e-fixture-sidebar-long-' + idSuffix; - const file = await readFile(join(workspaceRoot, 'sessions', sessionId, 'session.jsonl'), 'utf8'); - const header = JSON.parse(file.split('\n')[0]!) as { - id: string; - name: string; - status: string; - lastMessageAt: number; - }; + const header = await readSessionHeader(workspaceRoot, sessionId); assert.equal(header.id, sessionId); assert.equal(header.name, '会话 ' + idSuffix); assert.equal(header.status, 'active'); + assert.equal(typeof header.lastMessageAt, 'number'); + if (header.lastMessageAt === undefined) throw new Error('Session activity missing'); assert.ok( header.lastMessageAt < previousLastMessageAt, 'sessions must be in descending lastMessageAt order so the newest sorts to the top of the sidebar', @@ -987,9 +986,7 @@ describe('e2e-fixture mode', () => { const state = getE2eFixtureState(fixture); assert.equal(state?.activeSessionId, 'e2e-fixture-artifact'); - const lines = (await readFile(join(workspaceRoot, 'artifacts', 'metadata.jsonl'), 'utf8')) - .split('\n') - .filter(Boolean); + const lines = readArtifactMetadata(workspaceRoot); assert.equal(lines.length, 1, 'preview-image fixture must seed exactly one artifact'); for (const line of lines) assertNoAbsolutePathInMetadata(line); @@ -1037,9 +1034,7 @@ describe('e2e-fixture mode', () => { now: 1_700_000_000_000, }); - const lines = (await readFile(join(workspaceRoot, 'artifacts', 'metadata.jsonl'), 'utf8')) - .split('\n') - .filter(Boolean); + const lines = readArtifactMetadata(workspaceRoot); assert.equal(lines.length, 1); for (const line of lines) assertNoAbsolutePathInMetadata(line); @@ -1071,9 +1066,7 @@ describe('e2e-fixture mode', () => { now: 1_700_000_000_000, }); - const lines = (await readFile(join(workspaceRoot, 'artifacts', 'metadata.jsonl'), 'utf8')) - .split('\n') - .filter(Boolean); + const lines = readArtifactMetadata(workspaceRoot); assert.equal(lines.length, 1); for (const line of lines) assertNoAbsolutePathInMetadata(line); @@ -1140,9 +1133,7 @@ describe('e2e-fixture mode', () => { const state = getE2eFixtureState(fixture); assert.equal(state?.activeSessionId, 'e2e-fixture-artifact'); - const metadata = (await readFile(join(workspaceRoot, 'artifacts', 'metadata.jsonl'), 'utf8')) - .split('\n') - .filter(Boolean) + const metadata = readArtifactMetadata(workspaceRoot) .map((line) => JSON.parse(line) as { name: string; relativePath: string; kind: string; status: string }); assert.deepEqual(metadata.map((record) => record.name), ['report.html', 'patch.diff', 'notes.md']); assert.deepEqual(metadata.map((record) => record.kind), ['html', 'diff', 'file']); @@ -1189,13 +1180,10 @@ describe('e2e-fixture mode', () => { 'orphan branch points to NON-existent parent', ); - // Negative case: the orphan parent must NOT be written to disk. + // Negative case: the orphan parent must not exist in SQLite. await assert.rejects( - readFile( - join(workspaceRoot, 'sessions', 'e2e-fixture-turn-control-deleted-parent', 'session.jsonl'), - 'utf8', - ), - /ENOENT/, + readSessionHeader(workspaceRoot, 'e2e-fixture-turn-control-deleted-parent'), + /Session not found/, ); } finally { await rm(workspaceRoot, { recursive: true, force: true }); @@ -1261,7 +1249,7 @@ describe('e2e-fixture mode', () => { assert.equal(state?.activeSessionId, 'e2e-fixture-turn-control-branch-orphan'); }); - it('all three turn-control-* scenarios write the same on-disk session set', async () => { + it('all three turn-control-* scenarios write the same durable Session set', async () => { // Locks the @kenji review note: the three scenarios are a single // state family that only differs in active-session selection. A // future change that diverges their on-disk seed must update @@ -1284,18 +1272,15 @@ describe('e2e-fixture mode', () => { now: 1_700_000_000_000, }); - // Every fixture must seed exactly the three turn-control - // sessions (the orphan parent stays unseeded by design). + // Every fixture must seed the three turn-control Sessions while the + // orphan parent stays absent by design. for (const id of expected) { const header = await readSessionHeader(workspaceRoot, id); assert.equal(header.id, id, `${scenario} should seed ${id}`); } await assert.rejects( - readFile( - join(workspaceRoot, 'sessions', 'e2e-fixture-turn-control-deleted-parent', 'session.jsonl'), - 'utf8', - ), - /ENOENT/, + readSessionHeader(workspaceRoot, 'e2e-fixture-turn-control-deleted-parent'), + /Session not found/, `${scenario} must not seed the orphan parent`, ); } finally { @@ -1319,9 +1304,7 @@ describe('e2e-fixture mode', () => { const state = getE2eFixtureState(fixture); assert.equal(state?.activeSessionId, 'e2e-fixture-artifact'); - const metadata = (await readFile(join(workspaceRoot, 'artifacts', 'metadata.jsonl'), 'utf8')) - .split('\n') - .filter(Boolean) + const metadata = readArtifactMetadata(workspaceRoot) .map((line) => JSON.parse(line) as { id: string; name: string; relativePath: string; kind: string; status: string }); assert.deepEqual(metadata.map((record) => record.id), [ 'artifact-report', @@ -1357,7 +1340,7 @@ describe('settings-bots-onboarding fixture (#1233 deferral)', () => { assert.equal(state?.activeSessionId, 'e2e-fixture-turn'); }); - it('reuses the standard turn seed so no bot-specific on-disk seed is needed', async () => { + it('reuses the standard turn seed so no bot-specific durable seed is needed', async () => { const workspaceRoot = await mkdtemp(join(tmpdir(), 'maka-e2e-fixture-bots-onboarding-')); try { const fixture = resolveE2eFixture('settings-bots-onboarding', false); @@ -1368,8 +1351,7 @@ describe('settings-bots-onboarding fixture (#1233 deferral)', () => { credentialStore: fakeCredentialStore(), now: 1_700_000_000_000, }); - const file = await readFile(join(workspaceRoot, 'sessions', 'e2e-fixture-turn', 'session.jsonl'), 'utf8'); - const header = JSON.parse(file.split('\n')[0]!) as { id: string }; + const header = await readSessionHeader(workspaceRoot, 'e2e-fixture-turn'); assert.equal(header.id, 'e2e-fixture-turn'); } finally { await rm(workspaceRoot, { recursive: true, force: true }); @@ -1394,7 +1376,7 @@ describe('browser-empty chrome fixture (#819)', () => { assert.deepEqual(state?.liveBrowserSessionIds, ['e2e-fixture-turn']); }); - it('reuses the always-seeded turn session so no browser-specific on-disk seed is needed', async () => { + it('reuses the always-seeded turn Session so no browser-specific durable seed is needed', async () => { const workspaceRoot = await mkdtemp(join(tmpdir(), 'maka-e2e-fixture-browser-empty-')); try { const fixture = resolveE2eFixture('browser-empty', false); @@ -1405,11 +1387,10 @@ describe('browser-empty chrome fixture (#819)', () => { credentialStore: fakeCredentialStore(), now: 1_700_000_000_000, }); - // The turn session is part of the standard seed (always written), - // so the active browser session has a real on-disk chat behind the + // The turn Session is part of the standard seed, so the active browser + // Session has a real durable chat behind the // panel without a browser-specific seed branch. - const file = await readFile(join(workspaceRoot, 'sessions', 'e2e-fixture-turn', 'session.jsonl'), 'utf8'); - const header = JSON.parse(file.split('\n')[0]!) as { id: string }; + const header = await readSessionHeader(workspaceRoot, 'e2e-fixture-turn'); assert.equal(header.id, 'e2e-fixture-turn'); } finally { await rm(workspaceRoot, { recursive: true, force: true }); @@ -1425,29 +1406,42 @@ function fakeCredentialStore(secrets: string[] = []) { }; } -async function readSessionHeader(workspaceRoot: string, sessionId: string): Promise<{ - id: string; - parentSessionId?: string; - branchOfTurnId?: string; - status: string; -}> { - const file = await readFile(join(workspaceRoot, 'sessions', sessionId, 'session.jsonl'), 'utf8'); - const firstLine = file.split('\n')[0]; - if (!firstLine) throw new Error(`session.jsonl for ${sessionId} is empty`); - return JSON.parse(firstLine) as { - id: string; - parentSessionId?: string; - branchOfTurnId?: string; - status: string; - }; +async function readSessionHeader(workspaceRoot: string, sessionId: string): Promise { + return withRuntimeDatabase(workspaceRoot, (database) => { + const row = database + .prepare('SELECT payload_json AS payloadJson FROM session_metadata WHERE session_id = ?') + .get(sessionId) as { payloadJson?: unknown } | undefined; + if (typeof row?.payloadJson !== 'string') throw new Error(`Session not found: ${sessionId}`); + return JSON.parse(row.payloadJson) as SessionHeader; + }); +} + +async function readSessionMessages( + workspaceRoot: string, + sessionId: string, +): Promise { + return withRuntimeDatabase(workspaceRoot, (database) => + ( + database + .prepare(` + SELECT record_json AS recordJson + FROM session_messages + WHERE session_id = ? + ORDER BY sequence + `) + .all(sessionId) as Array<{ recordJson: string }> + ).map((row) => JSON.parse(row.recordJson) as StoredMessage), + ); } -async function readSessionMessages(workspaceRoot: string, sessionId: string): Promise { - const file = await readFile(join(workspaceRoot, 'sessions', sessionId, 'session.jsonl'), 'utf8'); - // Skip the first line (the SessionHeader); the rest are StoredMessages. - return file - .split('\n') - .slice(1) - .filter(Boolean) - .map((line) => JSON.parse(line) as unknown); +function withRuntimeDatabase( + workspaceRoot: string, + read: (database: DatabaseSync) => T, +): T { + const database = new DatabaseSync(join(workspaceRoot, 'runtime.sqlite'), { readOnly: true }); + try { + return read(database); + } finally { + database.close(); + } } diff --git a/apps/desktop/src/main/__tests__/execution-store-wiring.test.ts b/apps/desktop/src/main/__tests__/execution-store-wiring.test.ts deleted file mode 100644 index b5f63a2cc2..0000000000 --- a/apps/desktop/src/main/__tests__/execution-store-wiring.test.ts +++ /dev/null @@ -1,156 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { test } from 'node:test'; -import type { AgentRunEvent, AgentRunHeader, ShellRunRecord } from '@maka/core'; -import { - createLegacyAgentRunStoreForTest, - createLegacyShellRunStoreForTest, -} from '@maka/storage/legacy-execution-test-support'; -import { openDesktopExecutionStoreWiring } from '../execution-store-wiring.js'; - -test('Desktop cuts legacy execution state over before use and keeps later writes SQLite-only', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-desktop-execution-stores-')); - try { - const legacyRuns = createLegacyAgentRunStoreForTest(root); - const firstRun = runHeader('run-1', 'turn-1'); - await legacyRuns.createRun(firstRun); - await legacyRuns.appendEvent('session-1', 'run-1', runEvent('event-1', 'run-1', 'turn-1', 1)); - const legacyRunEventsPath = join( - root, - 'sessions', - 'session-1', - 'runs', - 'run-1', - 'events.jsonl', - ); - const legacyRunBytes = await readFile(legacyRunEventsPath); - - const legacyShellRuns = createLegacyShellRunStoreForTest(root); - await legacyShellRuns.createShellRun(shellRun('shell-1')); - const legacyShellPath = join( - root, - 'sessions', - 'session-1', - 'shell-runs', - 'shell-1', - 'shell-run.json', - ); - const legacyShellBytes = await readFile(legacyShellPath); - - const wiring = await openDesktopExecutionStoreWiring(root); - assert.equal((await wiring.runStore.readRun('session-1', 'run-1')).runId, 'run-1'); - assert.equal( - (await wiring.shellRunStore.readShellRun('session-1', 'shell-1')).shellRunId, - 'shell-1', - ); - - await wiring.runStore.appendEvent( - 'session-1', - 'run-1', - runEvent('event-2', 'run-1', 'turn-1', 2), - ); - await wiring.runStore.createRun(runHeader('run-2', 'turn-2')); - await wiring.shellRunStore.updateShellRun('session-1', 'shell-1', { - status: 'completed', - completedAt: 2, - exitCode: 0, - updatedAt: 2, - }); - await wiring.shellRunStore.createShellRun(shellRun('shell-2')); - - assert.deepEqual(await readFile(legacyRunEventsPath), legacyRunBytes); - assert.deepEqual(await readFile(legacyShellPath), legacyShellBytes); - await assert.rejects( - () => stat(join(root, 'sessions', 'session-1', 'runs', 'run-2', 'run.json')), - { code: 'ENOENT' }, - ); - await assert.rejects( - () => - stat( - join(root, 'sessions', 'session-1', 'shell-runs', 'shell-2', 'shell-run.json'), - ), - { code: 'ENOENT' }, - ); - wiring.close(); - wiring.close(); - - const reopened = await openDesktopExecutionStoreWiring(root); - try { - assert.deepEqual( - (await reopened.runStore.readEvents('session-1', 'run-1')).map((event) => event.id), - ['event-1', 'event-2'], - ); - assert.equal((await reopened.runStore.readRun('session-1', 'run-2')).turnId, 'turn-2'); - assert.equal( - (await reopened.shellRunStore.readShellRun('session-1', 'shell-1')).status, - 'completed', - ); - assert.equal( - (await reopened.shellRunStore.readShellRun('session-1', 'shell-2')).status, - 'running', - ); - } finally { - reopened.close(); - } - } finally { - await rm(root, { recursive: true, force: true }); - } -}); - -function runHeader(runId: string, turnId: string): AgentRunHeader { - return { - runId, - sessionId: 'session-1', - turnId, - status: 'created', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/tmp/cwd', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 1, - }; -} - -function runEvent( - id: string, - runId: string, - turnId: string, - ts: number, -): AgentRunEvent { - return { - type: 'run_started', - id, - runId, - sessionId: 'session-1', - turnId, - ts, - }; -} - -function shellRun(shellRunId: string): ShellRunRecord { - return { - shellRunId, - sessionId: 'session-1', - sourceRunId: 'run-1', - sourceTurnId: 'turn-1', - sourceToolCallId: `tool-${shellRunId}`, - cwd: '/workspace', - command: 'printf "ok"', - status: 'running', - startedAt: 1, - updatedAt: 1, - revision: 1, - output: { - mode: 'pipes', - stdout: '', - stderr: '', - stdoutTruncated: false, - stderrTruncated: false, - redacted: false, - }, - }; -} diff --git a/apps/desktop/src/main/__tests__/new-session-project.test.ts b/apps/desktop/src/main/__tests__/new-session-project.test.ts index b50199d64c..2bafd38f42 100644 --- a/apps/desktop/src/main/__tests__/new-session-project.test.ts +++ b/apps/desktop/src/main/__tests__/new-session-project.test.ts @@ -166,7 +166,10 @@ test('new sessions resolve a merged project alias to the surviving project id', }); try { - const original = await catalog.importLegacyPath(join(base, 'missing-original')); + const originalPath = join(base, 'original'); + await mkdir(originalPath); + const original = await catalog.register(originalPath); + await rm(originalPath, { recursive: true, force: true }); const duplicate = await catalog.register(cwd); await catalog.relink(original.id, cwd, async () => {}); diff --git a/apps/desktop/src/main/__tests__/project-startup-migration.test.ts b/apps/desktop/src/main/__tests__/project-startup-migration.test.ts deleted file mode 100644 index 746f8b6f5b..0000000000 --- a/apps/desktop/src/main/__tests__/project-startup-migration.test.ts +++ /dev/null @@ -1,44 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdir, mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { test } from 'node:test'; -import { createProjectCatalog, createSessionStore } from '@maka/storage'; -import { runProjectStartupMigration } from '../project-startup-migration.js'; - -test('startup project migration notifies the renderer after changing session associations', async () => { - const base = await mkdtemp(join(tmpdir(), 'maka-project-startup-migration-')); - const cwd = join(base, 'legacy-project'); - const storage = join(base, 'storage'); - await mkdir(cwd); - const sessions = createSessionStore(storage); - const catalog = createProjectCatalog(storage, { - createId: () => 'project-1', - }); - const changes: string[] = []; - - try { - const session = await sessions.create({ - cwd, - backend: 'fake', - llmConnectionSlug: 'fake', - model: 'fake-model', - permissionMode: 'ask', - name: 'Legacy session', - labels: [], - }); - - await runProjectStartupMigration({ - sessions, - catalog, - emitSessionsChanged: (reason) => changes.push(reason), - logError: () => {}, - }); - - assert.deepEqual(changes, ['migrated']); - assert.equal((await sessions.readHeaderSnapshot(session.id)).projectId, 'project-1'); - } finally { - await sessions.close?.(); - await rm(base, { recursive: true, force: true }); - } -}); diff --git a/apps/desktop/src/main/__tests__/quote-companion-cleanup.test.ts b/apps/desktop/src/main/__tests__/quote-companion-cleanup.test.ts index d02fe6c564..289780c6a7 100644 --- a/apps/desktop/src/main/__tests__/quote-companion-cleanup.test.ts +++ b/apps/desktop/src/main/__tests__/quote-companion-cleanup.test.ts @@ -1,8 +1,9 @@ import { strict as assert } from 'node:assert'; -import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { afterEach, describe, it } from 'node:test'; +import { DatabaseSync } from 'node:sqlite'; import { createQuoteCompanionCleanupAuthority } from '../quote-companion-cleanup.js'; const roots: string[] = []; @@ -89,6 +90,18 @@ describe('quote companion cleanup authority', () => { }); async function readPendingIds(workspaceRoot: string): Promise { - const raw = await readFile(join(workspaceRoot, 'quote-companion-cleanup.json'), 'utf8'); - return (JSON.parse(raw) as { pendingSessionIds: string[] }).pendingSessionIds; + const database = new DatabaseSync(join(workspaceRoot, 'runtime.sqlite'), { readOnly: true }); + try { + return ( + database + .prepare(` + SELECT session_id AS sessionId + FROM workflow_quote_companion_cleanup + ORDER BY tracked_at, session_id + `) + .all() as Array<{ sessionId: string }> + ).map(({ sessionId }) => sessionId); + } finally { + database.close(); + } } diff --git a/apps/desktop/src/main/__tests__/session-read-error-copy.test.ts b/apps/desktop/src/main/__tests__/session-read-error-copy.test.ts index aadffe61f7..49dc72485d 100644 --- a/apps/desktop/src/main/__tests__/session-read-error-copy.test.ts +++ b/apps/desktop/src/main/__tests__/session-read-error-copy.test.ts @@ -12,7 +12,7 @@ describe('session read error copy', () => { { message: 'SessionProjectionCache.readMessages failed', details: { - error: 'EPERM: operation not permitted, open C:\\Users\\alice\\AppData\\Roaming\\Maka\\workspaces\\default\\sessions\\s1\\session.jsonl', + error: 'EPERM: operation not permitted, open C:\\Users\\alice\\AppData\\Roaming\\Maka\\workspaces\\default\\runtime.sqlite', }, }, ], @@ -22,7 +22,7 @@ describe('session read error copy', () => { assert.equal(message, `${MARKER}读取进行中的对话缓存失败：本地会话文件暂时不可用，请稍后重试。`); assert.equal(message.includes('C:\\Users'), false); - assert.equal(message.includes('session.jsonl'), false); + assert.equal(message.includes('runtime.sqlite'), false); }); it('classifies durable runtime ledger read failures', () => { @@ -32,7 +32,7 @@ describe('session read error copy', () => { { message: 'RuntimeEventStore.readRuntimeEvents failed', details: { - error: 'Invalid RuntimeEvent JSONL line 2 for run run_1: Unexpected token', + error: 'Invalid SQLite RuntimeEvent at sequence 2 for run run_1', }, }, ], @@ -56,7 +56,7 @@ describe('session read error copy', () => { }); it('does not classify path-shaped text without an explicit code or diagnostic', () => { - const error = new Error('EPERM: operation not permitted, rename C:\\Users\\alice\\session.jsonl.tmp -> session.jsonl'); + const error = new Error('EPERM: operation not permitted, open C:\\Users\\alice\\runtime.sqlite'); assert.equal( sessionReadMessagesFailureMessage(error), diff --git a/apps/desktop/src/main/app-lifecycle.ts b/apps/desktop/src/main/app-lifecycle.ts index 6bf5cb60c1..bb15e9717c 100644 --- a/apps/desktop/src/main/app-lifecycle.ts +++ b/apps/desktop/src/main/app-lifecycle.ts @@ -1,4 +1,4 @@ -import { app, nativeImage, safeStorage } from 'electron'; +import { app, nativeImage } from 'electron'; import { mkdir } from 'node:fs/promises'; import { join } from 'node:path'; import { setActiveProxy } from '@maka/runtime'; @@ -22,11 +22,9 @@ import type { openRuntimeEventPersistence, } from '@maka/storage'; import type { createAgentGraphControlStore } from '@maka/storage/agent-graph-control-store'; -import { migrateLegacyCredentials } from './credential-store.js'; import type { createFileCredentialStore } from './credential-store.js'; import { startConfigFileWatcher, type ConfigFileWatcher } from './config-file-watcher.js'; import { toContractNetworkSettings } from './network-settings-main.js'; -import { importLegacyOAuthTokenFiles } from './oauth/shared-credential-bridge.js'; import type { resolveE2eFixture } from './e2e-fixture.js'; import type { KeepSystemAwakeController } from './keep-system-awake.js'; import type { createPlanReminderMainService } from './plan-reminders-main.js'; @@ -38,7 +36,6 @@ import type { DesktopExecutionStoreWiring } from './execution-store-wiring.js'; import type { assembleDesktopTools } from './tool-assembly.js'; import type { StreamEvents } from './session-stream.js'; import type { SettingsIpcHandle } from './settings-ipc-main.js'; -import { runProjectStartupMigration } from './project-startup-migration.js'; import { createAppQuitCoordinator } from './app-quit-coordinator.js'; import { resolveDockPresentation } from './dock-presentation.js'; import { resumeSafeBoundaryContinuationsOnStartup } from './startup-safe-boundary-resume.js'; @@ -168,15 +165,6 @@ export function wireAppLifecycle(deps: AppLifecycleDeps): void { } } - async function migrateSessionProjectsOnStartup(): Promise { - await runProjectStartupMigration({ - sessions: sessionStore, - catalog: projectCatalog, - emitSessionsChanged, - logError: console.error, - }); - } - async function ensureBootstrapConnection(): Promise { await mkdir(workspaceRoot, { recursive: true }); if ((await connectionStore.list()).length > 0) return; @@ -228,12 +216,8 @@ export function wireAppLifecycle(deps: AppLifecycleDeps): void { } } - // Credential migration is the one startup phase that must finish before an - // interactive window exists: OAuth logout is otherwise able to race the - // one-shot legacy import. The work is local and normally a missing-file - // check; all non-critical startup below still runs behind the first paint. - // The renderer's first - // IPC calls (session enumeration, settings read, connection listing) + // The renderer's first IPC calls (session enumeration, settings read, + // connection listing) // all read from stores that are initialized synchronously at module load, // so they succeed regardless of whether background startup has // settled. Any state that background startup mutates is pushed to the @@ -242,7 +226,6 @@ export function wireAppLifecycle(deps: AppLifecycleDeps): void { // E2E fixture workspaces are wiped and seeded before stores open in // boot.ts. SQLite keeps live file handles, so resetting the workspace // here after store construction would detach the canonical database. - await runCredentialStartup(); const initialWindowSignal = quitCoordinator.getWindowCreationSignal(); if (!initialWindowSignal) return; app.on('second-instance', quitCoordinator.focusOrCreateWindow); @@ -254,62 +237,22 @@ export function wireAppLifecycle(deps: AppLifecycleDeps): void { await backgroundStartup; }); - async function runCredentialStartup(): Promise { - // One-time migration of credentials.json off Electron safeStorage so - // the pure-Node runtime can read it (issue #32). Runs before any - // credential read/write below; failure is non-fatal (legacy file is - // left intact and later credential reads fail closed with guidance). - try { - await migrateLegacyCredentials(workspaceRoot, safeStorage); - } catch (error) { - console.error('[credentials] migration off safeStorage failed; legacy file left intact:', error); - } - // One-shot import of pre-#1125 safeStorage-encrypted OAuth token - // files into the shared CredentialStore, which is the only token - // authority from here on. Best-effort like the migration above: - // files that cannot be decrypted are left intact for a later start. - try { - const userDataDir = app.getPath('userData'); - const reports = await importLegacyOAuthTokenFiles({ - credentialStore, - decryptor: safeStorage, - files: [ - { slug: 'claude-subscription', filePath: join(userDataDir, '.claude_subscription_token') }, - { slug: 'codex-subscription', filePath: join(userDataDir, '.codex_subscription_token') }, - { slug: 'cursor-subscription', filePath: join(userDataDir, '.cursor_subscription_token') }, - { slug: 'antigravity-subscription', filePath: join(userDataDir, '.antigravity_subscription_token') }, - ], - }); - for (const report of reports) { - const log = report.outcome === 'failed' ? console.error : console.log; - log(`[credentials] legacy OAuth token file for ${report.slug}: ${report.outcome}`, report.error ?? ''); - } - } catch (error) { - console.error('[credentials] legacy OAuth token import failed; files left intact:', error); - } - } - /** * Non-critical startup work that must NOT block the first window paint. * * `setActiveProxy` must be applied before any network-bearing step - * (`botRegistry.applySettings`); usage readiness loads - * the embedded telemetry compatibility repo. Everything here is best-effort and logged on - * failure — none of it should prevent the user from seeing and interacting - * with the app shell. + * (`botRegistry.applySettings`); usage readiness opens the operational SQLite + * telemetry authority. Everything here is best-effort and logged on failure — + * none of it should prevent the user from seeing and interacting with the app shell. */ async function runBackgroundStartup(): Promise { // Each step stands alone, because the doc comment above is a promise the // old shape could not keep: this was a run of bare `await`s, so the first // rejection skipped every step after it, silently. // - // Seen for real: one telemetry record written by an older build failed - // `decodePersistedLlmCallRecord`, `ensureUsageReady()` rejected, and - // session recovery, plan reminders, the daily review scheduler, the config - // watcher and the automation scheduler all - // never ran. Nothing said so — the only trace was an unhandled rejection - // warning about `contextBudget`, which names the record and not one of the - // things that stopped working because of it. + // A malformed telemetry record must not prevent session recovery, plan + // reminders, the daily review scheduler, the config watcher, or the + // Automation scheduler from starting. const step = async (name: string, run: () => unknown): Promise => { try { await run(); @@ -340,7 +283,6 @@ export function wireAppLifecycle(deps: AppLifecycleDeps): void { await step('keep-awake', () => keepSystemAwake.apply(resolved.system.keepSystemAwake)); } await step('usage readiness', () => ensureUsageReady()); - await step('project migration', () => migrateSessionProjectsOnStartup()); await step('session recovery', () => recoverInterruptedSessionsOnStartup()); let botRegistryReady = false; if (settings) { diff --git a/apps/desktop/src/main/automation-wiring.ts b/apps/desktop/src/main/automation-wiring.ts index 9603ff744a..6707a40078 100644 --- a/apps/desktop/src/main/automation-wiring.ts +++ b/apps/desktop/src/main/automation-wiring.ts @@ -14,7 +14,7 @@ export interface MainAutomationWiring { manager: AutomationManager; scheduler: AutomationScheduler; tools: MakaTool[]; - /** Load durable automations from disk and register them. Call once at startup. */ + /** Load durable Automations from operational storage. Call once at startup. */ loadDurableAutomations: () => Promise; } @@ -33,14 +33,11 @@ export function createMainAutomationWiring(deps: CreateMainAutomationWiringDeps) now: () => Date.now(), }); - const store = createAutomationStore(deps.workspaceRoot); + const store = createAutomationStore(deps.workspaceRoot); // Durable persistence is tied to cron capability: only a host that can run - // crons (createFreshRun present) owns durable automations and may load/write - // the shared automations.json. A cron-disabled host has no durable state of - // its own and must never overwrite the store (its full-file sync would clobber - // the owning host's crons). The desktop always provides createFreshRun; this - // gate keeps the invariant explicit and symmetric with the CLI. + // crons (createFreshRun present) owns durable Automations. A cron-disabled + // host has no durable state of its own and must not reconcile shared state. const cronEnabled = deps.createFreshRun !== undefined; // If we fail to READ the existing durable store, we must not WRITE over it — a @@ -53,7 +50,7 @@ export function createMainAutomationWiring(deps: CreateMainAutomationWiringDeps) if (!durableStoreReadable) return; const all = manager.listAll().filter(a => a.durable && (a.status === 'active' || a.status === 'paused')); store.sync(all).catch(err => { - console.warn('[automation-wiring] failed to sync durable automations to disk:', err); + console.warn('[automation-wiring] failed to sync durable automations:', err); }); } : (): void => { /* no durable automations to persist on a cron-disabled host */ }; @@ -90,4 +87,3 @@ export function createMainAutomationWiring(deps: CreateMainAutomationWiringDeps) return { manager, scheduler, tools, loadDurableAutomations }; } - diff --git a/apps/desktop/src/main/boot.ts b/apps/desktop/src/main/boot.ts index 6d2196ace7..5fa0e8cb9d 100644 --- a/apps/desktop/src/main/boot.ts +++ b/apps/desktop/src/main/boot.ts @@ -481,7 +481,7 @@ const automationWiring = createMainAutomationWiring({ }, }); -// Load durable automations from disk on startup (fire-and-forget; errors are logged inside). +// Load durable Automations from operational storage on startup. void automationWiring.loadDurableAutomations(); // Goal execution — autonomous turn-boundary continuation with an external diff --git a/apps/desktop/src/main/credential-store.ts b/apps/desktop/src/main/credential-store.ts index cbf29268d6..9ad42c103c 100644 --- a/apps/desktop/src/main/credential-store.ts +++ b/apps/desktop/src/main/credential-store.ts @@ -1,50 +1,2 @@ -import { join } from 'node:path'; -import { createFileCredentialStore, migrateLegacyCredentialFile } from '@maka/storage'; - -/** - * The slice of Electron `safeStorage` the importer needs, injected so the - * migration can be tested without Electron — and so this module no longer - * imports `electron` at load time. main.ts passes the real `safeStorage`. - */ -export interface SafeStorageLike { - isEncryptionAvailable(): boolean; - decryptString(encrypted: Buffer): string; -} - -// The credential store and its migration are pure-Node and live in -// @maka/storage so the headless runtime can read the same file (issue #32). -// Re-exported here so existing `./credential-store.js` importers keep their path. export type { CredentialKind, CredentialStore } from '@maka/storage'; export { CREDENTIAL_SCHEMA_VERSION, createFileCredentialStore } from '@maka/storage'; - -/** - * One-time migration off Electron `safeStorage` (issue #32). Desktop-only glue: - * it supplies the crypto — the legacy file stored each secret as - * `base64(safeStorage.encryptString(value))`, so here we base64-decode and - * `safeStorage.decryptString` each value — while @maka/storage owns the - * orchestration: the shared cross-process lock, the version gate, fail-closed - * aborts, the atomic 0600 rewrite, and the tombstone. A successful run leaves - * the file as shared v1 plaintext-0600, which the headless `FileCredentialStore` - * then reads. - * - * Scope — this migrates EVERY secret in `credentials.json`, not just API keys: - * bot tokens, bot app secrets, the proxy password, and the Tavily key all - * decrypt to plaintext too. That is required, not incidental — - * the desktop abandons `safeStorage` entirely (the live store is now the - * pure-Node `FileCredentialStore`), so any value left encrypted would become - * permanently unreadable. The accepted at-rest posture for all of them is - * plaintext behind 0600 (SECURITY.md / file-first, #32). - * - * If `safeStorage` is unavailable the migration aborts and leaves the encrypted - * file untouched (a later run migrates once it is available). main.ts runs this - * before any credential use, non-fatally, in `whenReady`. - */ -export function migrateLegacyCredentials( - workspaceRoot: string, - safeStorage: SafeStorageLike, -): Promise { - return migrateLegacyCredentialFile(join(workspaceRoot, 'credentials.json'), { - isAvailable: () => safeStorage.isEncryptionAvailable(), - decrypt: (storedValue) => safeStorage.decryptString(Buffer.from(storedValue, 'base64')), - }); -} diff --git a/apps/desktop/src/main/daily-review-archive-store.ts b/apps/desktop/src/main/daily-review-archive-store.ts index 31ca956c34..c8aeb72f36 100644 --- a/apps/desktop/src/main/daily-review-archive-store.ts +++ b/apps/desktop/src/main/daily-review-archive-store.ts @@ -1,6 +1,3 @@ -import { randomUUID } from 'node:crypto'; -import { mkdir, open, readdir, readFile, rename, rm } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; import { DEFAULT_DAILY_REVIEW_CONFIG, dailyReviewArchiveToSummary, @@ -10,9 +7,9 @@ import { type DailyReviewArchiveSummary, type DailyReviewConfig, } from '@maka/core'; +import { acquireOperationalStateDatabase } from '@maka/storage'; const ARCHIVE_ID_PATTERN = /^\d{4}-\d{2}-\d{2}-(1d|7d|30d)$/; -const READABLE_ARCHIVE_ID_PATTERN = /^\d{4}-\d{2}-\d{2}-(1d|7d|30d|daily|deep)$/; export interface DailyReviewArchiveStore { getConfig(): Promise; @@ -25,157 +22,134 @@ export interface DailyReviewArchiveStore { } export function createDailyReviewArchiveStore(workspaceRoot: string): DailyReviewArchiveStore { - return new FileDailyReviewArchiveStore(workspaceRoot); + return new SqliteDailyReviewArchiveStore(workspaceRoot); } -class FileDailyReviewArchiveStore implements DailyReviewArchiveStore { - private readonly root: string; - private readonly archiveRoot: string; - private readonly configPath: string; - private queue: Promise = Promise.resolve(); - - constructor(workspaceRoot: string) { - this.root = join(workspaceRoot, 'daily-reviews'); - this.archiveRoot = join(this.root, 'archive'); - this.configPath = join(this.root, 'config.json'); - } +class SqliteDailyReviewArchiveStore implements DailyReviewArchiveStore { + constructor(private readonly workspaceRoot: string) {} async getConfig(): Promise { - try { - const raw = await readFile(this.configPath, 'utf8'); - return normalizeDailyReviewConfig(JSON.parse(raw) as Partial); - } catch (error) { - if (isNotFound(error)) return DEFAULT_DAILY_REVIEW_CONFIG; - throw error; - } + return this.withDatabase('read', (database) => { + const row = database + .prepare('SELECT config_json AS configJson FROM workflow_daily_review_state WHERE singleton = 1') + .get() as { configJson?: unknown } | undefined; + return typeof row?.configJson === 'string' + ? normalizeDailyReviewConfig(JSON.parse(row.configJson) as Partial) + : DEFAULT_DAILY_REVIEW_CONFIG; + }); } async setConfig(patch: Partial): Promise { - let next = DEFAULT_DAILY_REVIEW_CONFIG; - await this.withQueue(async () => { - const current = await this.getConfig(); - next = normalizeDailyReviewConfig({ - ...current, - ...patch, - }); - await mkdir(this.root, { recursive: true, mode: 0o700 }); - await writeJsonAtomically(this.configPath, next); + return this.withDatabase('write', (database) => { + const row = database + .prepare('SELECT config_json AS configJson FROM workflow_daily_review_state WHERE singleton = 1') + .get() as { configJson?: unknown } | undefined; + const current = + typeof row?.configJson === 'string' + ? normalizeDailyReviewConfig(JSON.parse(row.configJson) as Partial) + : DEFAULT_DAILY_REVIEW_CONFIG; + const next = normalizeDailyReviewConfig({ ...current, ...patch }); + database + .prepare(` + INSERT INTO workflow_daily_review_state(singleton, config_json) + VALUES (1, ?) + ON CONFLICT(singleton) DO UPDATE SET config_json = excluded.config_json + `) + .run(JSON.stringify(next)); + return next; }); - return next; } async putArchive(archive: DailyReviewArchive): Promise { assertArchiveId(archive.id); - await this.withQueue(async () => { - await mkdir(this.archiveRoot, { recursive: true, mode: 0o700 }); - await writeJsonAtomically(this.archivePath(archive.id), archive); + const normalized = normalizeDailyReviewArchive(archive); + if (normalized.id !== archive.id) throw new Error(`Daily Review archive id mismatch: ${archive.id}`); + this.withDatabase('write', (database) => { + database + .prepare(` + INSERT INTO workflow_daily_review_archives( + archive_id, generated_at, day_from_ms, record_json + ) VALUES (?, ?, ?, ?) + ON CONFLICT(archive_id) DO UPDATE SET + generated_at = excluded.generated_at, + day_from_ms = excluded.day_from_ms, + record_json = excluded.record_json + `) + .run(normalized.id, normalized.generatedAt, normalized.day.fromMs, JSON.stringify(normalized)); }); - return archive; + return normalized; } async listArchives(): Promise { - const archives: DailyReviewArchiveSummary[] = []; - for (const id of await this.listArchiveIds()) { - const archive = await this.getArchive(id); - if (archive) archives.push(dailyReviewArchiveToSummary(archive)); - } - archives.sort((a, b) => b.generatedAt - a.generatedAt || b.day.fromMs - a.day.fromMs || a.id.localeCompare(b.id)); - return archives; + return this.withDatabase('read', (database) => + ( + database + .prepare(` + SELECT archive_id AS archiveId, record_json AS recordJson + FROM workflow_daily_review_archives + ORDER BY generated_at DESC, day_from_ms DESC, archive_id + `) + .all() as Array<{ archiveId: string; recordJson: string }> + ).map((row) => dailyReviewArchiveToSummary(decodeArchive(row.archiveId, row.recordJson))), + ); } async getArchive(id: string): Promise { - assertReadableArchiveId(id); - let raw: string; - try { - raw = await readFile(this.archivePath(id), 'utf8'); - } catch (error) { - if (isNotFound(error)) return null; - throw error; - } - try { - const archive = normalizeDailyReviewArchive(JSON.parse(raw)); - if (archive.id !== id) throw new Error(`Daily Review archive id mismatch: ${id}`); - return archive; - } catch { - return null; - } + assertArchiveId(id); + return this.withDatabase('read', (database) => { + const row = database + .prepare(` + SELECT record_json AS recordJson + FROM workflow_daily_review_archives + WHERE archive_id = ? + `) + .get(id) as { recordJson?: unknown } | undefined; + return typeof row?.recordJson === 'string' ? decodeArchive(id, row.recordJson) : null; + }); } async deleteArchive(id: string): Promise { - assertReadableArchiveId(id); - await rm(this.archivePath(id), { force: true }); + assertArchiveId(id); + this.withDatabase('write', (database) => { + database.prepare('DELETE FROM workflow_daily_review_archives WHERE archive_id = ?').run(id); + }); } async prune(maxArchives: number): Promise { const limit = Math.max(0, Math.trunc(maxArchives)); if (limit === 0) return; - const archives = await this.listArchives(); - for (const archive of archives.slice(limit)) { - await this.deleteArchive(archive.id); - } - } - - private async listArchiveIds(): Promise { - try { - const entries = await readdir(this.archiveRoot, { withFileTypes: true }); - return entries - .filter((entry) => entry.isFile() && entry.name.endsWith('.json')) - .map((entry) => entry.name.slice(0, -'.json'.length)) - .filter((id) => READABLE_ARCHIVE_ID_PATTERN.test(id)); - } catch (error) { - if (isNotFound(error)) return []; - throw error; - } - } - - private archivePath(id: string): string { - assertReadableArchiveId(id); - return join(this.archiveRoot, `${id}.json`); + this.withDatabase('write', (database) => { + database + .prepare(` + DELETE FROM workflow_daily_review_archives + WHERE archive_id IN ( + SELECT archive_id + FROM workflow_daily_review_archives + ORDER BY generated_at DESC, day_from_ms DESC, archive_id + LIMIT -1 OFFSET ? + ) + `) + .run(limit); + }); } - private async withQueue(fn: () => Promise): Promise { - const previous = this.queue; - let release!: () => void; - this.queue = new Promise((resolve) => { - release = resolve; - }); - await previous; + private withDatabase( + mode: 'read' | 'write', + operation: (database: import('node:sqlite').DatabaseSync) => T, + ): T { + const lease = acquireOperationalStateDatabase(this.workspaceRoot); try { - return await fn(); + return lease.transaction(mode, () => operation(lease.database)); } finally { - release(); - } - } -} - -async function writeJsonAtomically(path: string, value: unknown): Promise { - const temporaryPath = `${path}.${randomUUID()}.tmp`; - let file: Awaited> | undefined; - try { - file = await open(temporaryPath, 'wx', 0o600); - await file.writeFile(`${JSON.stringify(value, null, 2)}\n`, 'utf8'); - await file.sync(); - await file.close(); - file = undefined; - await rename(temporaryPath, path); - if (process.platform !== 'win32') { - const directory = await open(dirname(path), 'r'); - try { - await directory.sync(); - } finally { - await directory.close(); - } + lease.close(); } - } finally { - await file?.close().catch(() => undefined); - await rm(temporaryPath, { force: true }).catch(() => undefined); } } -function assertReadableArchiveId(id: string): void { - if (!READABLE_ARCHIVE_ID_PATTERN.test(id)) { - throw new Error(`Invalid Daily Review archive id: ${id}`); - } +function decodeArchive(id: string, recordJson: string): DailyReviewArchive { + const archive = normalizeDailyReviewArchive(JSON.parse(recordJson)); + if (archive.id !== id) throw new Error(`Daily Review archive id mismatch: ${id}`); + return archive; } function assertArchiveId(id: string): void { @@ -183,7 +157,3 @@ function assertArchiveId(id: string): void { throw new Error(`Invalid Daily Review archive id: ${id}`); } } - -function isNotFound(error: unknown): boolean { - return Boolean(error && typeof error === 'object' && 'code' in error && error.code === 'ENOENT'); -} diff --git a/apps/desktop/src/main/e2e-fixture.ts b/apps/desktop/src/main/e2e-fixture.ts index 6b7cf4949a..7a280fd2fa 100644 --- a/apps/desktop/src/main/e2e-fixture.ts +++ b/apps/desktop/src/main/e2e-fixture.ts @@ -56,8 +56,6 @@ import { longTranscriptSession, staleFakeMessages, staleFakeSession, - staleLegacyMessages, - staleLegacySession, turnControlSessions, workstationStatusSessions, } from './e2e-fixture/scenarios-sessions.js'; @@ -691,17 +689,14 @@ export async function seedE2eFixture(input: { await writeSession(input.workspaceRoot, errorSession(now), errorMessages(now)); await writeSession(input.workspaceRoot, artifactSession(now), artifactMessages(now)); await writeArtifacts(input.workspaceRoot, now, input.fixture.scenario); - // Stale-session fixture seeds three sessions reproducing the @WAWQAQ + // Stale-session fixture seeds two sessions reproducing the @WAWQAQ // workspace state that triggered the P0: // - one healthy ai-sdk session (zai-live, correct slug) // - one fake backend session (FakeBackend) - // - one legacy backend kind ('claude' with slug 'fake-claude') - // Together with the connection list (no `fake-claude` slug present), - // the renderer must mark the bottom two as stale + leave the first - // alone. + // The renderer must mark the unavailable fake Session as stale and leave + // the healthy Session alone. if (input.fixture.scenario === 'stale-sessions') { await writeSession(input.workspaceRoot, staleFakeSession(now), staleFakeMessages(now)); - await writeSession(input.workspaceRoot, staleLegacySession(now), staleLegacyMessages(now)); await writeSession(input.workspaceRoot, healthySession(now), healthyMessages(now)); } if (input.fixture.scenario === 'workstation-statuses') { diff --git a/apps/desktop/src/main/e2e-fixture/scenarios-artifacts.ts b/apps/desktop/src/main/e2e-fixture/scenarios-artifacts.ts index 3e1ecbc04b..a263b14169 100644 --- a/apps/desktop/src/main/e2e-fixture/scenarios-artifacts.ts +++ b/apps/desktop/src/main/e2e-fixture/scenarios-artifacts.ts @@ -1,6 +1,7 @@ import { mkdir, stat, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import type { ArtifactRecord, SessionHeader, StoredMessage, E2eFixtureScenario } from '@maka/core'; +import { createSqliteArtifactMetadataRepository } from '@maka/storage'; import { ARTIFACT_SESSION_ID, header } from './seed-helpers.js'; export function artifactSession(now: number): SessionHeader { @@ -220,8 +221,8 @@ export async function writeArtifacts(workspaceRoot: string, now: number, scenari /** * Shared writer for an arbitrary artifact spec list. Writes each * spec to disk (unless `skipFile`), captures the real `sizeBytes` - * via `stat` (unless `sizeBytesOverride`), and emits the - * `metadata.jsonl` index. Used by both the canonical + * via `stat` (unless `sizeBytesOverride`), and persists metadata in + * runtime.sqlite. Used by both the canonical * `artifact-pane` / `artifact-errors` scenarios and the * PR-UI-RENDER-3a-smoke preview scenarios. */ @@ -271,10 +272,10 @@ async function writeArtifactSpecs( }); } - await mkdir(root, { recursive: true }); - await writeFile( - join(root, 'metadata.jsonl'), - records.map((record) => JSON.stringify(record)).join('\n') + '\n', - 'utf8', - ); + const metadata = createSqliteArtifactMetadataRepository(dirname(root)); + try { + metadata.replaceAll(records); + } finally { + metadata.close(); + } } diff --git a/apps/desktop/src/main/e2e-fixture/scenarios-chat.ts b/apps/desktop/src/main/e2e-fixture/scenarios-chat.ts index 456bf1bc22..a4afd47ced 100644 --- a/apps/desktop/src/main/e2e-fixture/scenarios-chat.ts +++ b/apps/desktop/src/main/e2e-fixture/scenarios-chat.ts @@ -1,11 +1,10 @@ -import { writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; import type { SandboxBoundaryRequestEvent, SessionHeader, StoredMessage, E2eFixtureState, } from '@maka/core'; +import { createSqliteTaskLedgerStore } from '@maka/storage'; import { ERROR_SESSION_ID, header, @@ -33,45 +32,69 @@ const STREAMING_ANSWER_MARKDOWN = [ ].join('\n'); export async function writeTaskLedgerFixture(workspaceRoot: string, now: number): Promise { - const tasks = [ - { - id: 'task-root-implementation', key: 'T1', subject: '完成会话任务台账升级', status: 'in_progress', - createdAt: now - 50 * 60_000, updatedAt: now - 2 * 60_000, - owner: { actor: 'main_agent', runId: 'run-task-parent', turnId: 'turn-fixture-2' }, - }, - { - id: 'task-child-storage', key: 'T1.1', parentId: 'task-root-implementation', - subject: '验证旧 JSONL 迁移与并发短 key 分配', status: 'completed', - createdAt: now - 45 * 60_000, updatedAt: now - 8 * 60_000, endedAt: now - 8 * 60_000, + void now; + const store = createSqliteTaskLedgerStore(workspaceRoot); + try { + const root = ( + await store.create( + TURN_SESSION_ID, + [{ subject: '完成会话任务台账升级' }], + { actor: 'main_agent', runId: 'run-task-parent', turnId: 'turn-fixture-2' }, + ) + ).created[0]!; + await store.update( + TURN_SESSION_ID, + root.id, + { status: 'in_progress' }, + { actor: 'main_agent', runId: 'run-task-parent', turnId: 'turn-fixture-2' }, + ); + + const storage = ( + await store.create(TURN_SESSION_ID, [ + { subject: '验证 SQLite authority 与并发短 key 分配', parentId: root.id }, + ]) + ).created[0]!; + await store.update(TURN_SESSION_ID, storage.id, { status: 'in_progress' }); + await store.update(TURN_SESSION_ID, storage.id, { + status: 'completed', completionEvidence: 'Core 与 Storage 定向测试全部通过。', - }, - { - id: 'task-child-ui', key: 'T1.2', parentId: 'task-root-implementation', - subject: '检查窄窗口下的任务树布局', status: 'blocked', - createdAt: now - 40 * 60_000, updatedAt: now - 3 * 60_000, + }); + + const ui = ( + await store.create(TURN_SESSION_ID, [ + { subject: '检查窄窗口下的任务树布局', parentId: root.id }, + ]) + ).created[0]!; + await store.claim(TURN_SESSION_ID, ui.id, { + actor: 'child_agent', + agentId: 'local-read', + runId: 'run-task-child', + turnId: 'turn-task-child', + }); + await store.update(TURN_SESSION_ID, ui.id, { + status: 'blocked', blockedReason: '等待视觉回归截图确认 990px 视口没有文字重叠。', - owner: { actor: 'child_agent', agentId: 'local-read', runId: 'run-task-child', turnId: 'turn-task-child' }, - }, - { - id: 'task-grandchild-copy', key: 'T1.2.1', parentId: 'task-child-ui', - subject: '核对深层缩进、超长任务描述、owner 与阻塞原因在窄窗口中仍可完整换行且不遮挡后续内容', - status: 'pending', createdAt: now - 35 * 60_000, updatedAt: now - 3 * 60_000, - }, - { - id: 'task-docs', key: 'T2', subject: '同步生命周期文档与边界说明', status: 'pending', - createdAt: now - 30 * 60_000, updatedAt: now - 5 * 60_000, - }, - { - id: 'task-runtime', key: 'T3', subject: '验证 Goal 一次提醒门禁', status: 'completed', - createdAt: now - 25 * 60_000, updatedAt: now - 6 * 60_000, endedAt: now - 6 * 60_000, - completionEvidence: 'Goal gate 定向测试覆盖空任务、阻塞任务、一次提醒和上限放行。', - }, - ]; - await writeFile( - join(workspaceRoot, 'sessions', TURN_SESSION_ID, 'tasks.json'), - `${JSON.stringify(tasks, null, 2)}\n`, - 'utf8', - ); + }); + await store.create(TURN_SESSION_ID, [ + { + subject: + '核对深层缩进、超长任务描述、owner 与阻塞原因在窄窗口中仍可完整换行且不遮挡后续内容', + parentId: ui.id, + }, + ]); + await store.create(TURN_SESSION_ID, [{ subject: '同步生命周期文档与边界说明' }]); + const runtime = ( + await store.create(TURN_SESSION_ID, [{ subject: '验证 Goal 一次提醒门禁' }]) + ).created[0]!; + await store.update(TURN_SESSION_ID, runtime.id, { status: 'in_progress' }); + await store.update(TURN_SESSION_ID, runtime.id, { + status: 'completed', + completionEvidence: + 'Goal gate 定向测试覆盖空任务、阻塞任务、一次提醒和上限放行。', + }); + } finally { + store.close(); + } } export function turnSession(now: number): SessionHeader { diff --git a/apps/desktop/src/main/e2e-fixture/scenarios-deep-research.ts b/apps/desktop/src/main/e2e-fixture/scenarios-deep-research.ts index 8f7f73a69a..46e97c65a8 100644 --- a/apps/desktop/src/main/e2e-fixture/scenarios-deep-research.ts +++ b/apps/desktop/src/main/e2e-fixture/scenarios-deep-research.ts @@ -1,10 +1,9 @@ -import { mkdir, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; import { DEEP_RESEARCH_DEFAULT_CHECKLIST, DEEP_RESEARCH_REPORT_SECTION_KEYS, } from '@maka/core'; -import type { DeepResearchEvent, SessionHeader, StoredMessage } from '@maka/core'; +import type { SessionHeader, StoredMessage } from '@maka/core'; +import { createSqliteDeepResearchStore } from '@maka/storage'; import { header } from './seed-helpers.js'; export const DEEP_RESEARCH_SESSION_ID = 'e2e-fixture-deep-research'; @@ -46,42 +45,30 @@ export function deepResearchMessages(now: number): StoredMessage[] { } export async function writeDeepResearchLedger(workspaceRoot: string, now: number): Promise { + let eventNumber = 0; + const store = createSqliteDeepResearchStore(workspaceRoot, { + newId: () => `deep-research-event-${++eventNumber}`, + now: () => now, + }); const hash = `sha256:${'a'.repeat(64)}`; const sourceArtifactId = 'source-paper'; - let eventNumber = 0; - const nextEventId = () => `deep-research-event-${++eventNumber}`; - const events: DeepResearchEvent[] = [ - { - eventId: nextEventId(), - type: 'research_started', - sessionId: DEEP_RESEARCH_SESSION_ID, - ts: now - 20 * 60_000, - objective: 'Reproduce a durable filesystem-backed Deep Research workflow in Maka.', - scopeLevel: 'standard', - }, - { - eventId: nextEventId(), - type: 'research_artifact_recorded', - sessionId: DEEP_RESEARCH_SESSION_ID, - ts: now - 19 * 60_000, - artifact: { - artifactId: sourceArtifactId, - role: 'source', - name: 'fs-researcher-paper.md', - createdAt: now - 19 * 60_000, - locator: 'https://arxiv.org/abs/2602.01566', - contentHash: hash, - sourceArtifactIds: [], - }, - }, - ]; - for (const [index, key] of DEEP_RESEARCH_REPORT_SECTION_KEYS.entries()) { - events.push({ - eventId: nextEventId(), - type: 'research_artifact_recorded', - sessionId: DEEP_RESEARCH_SESSION_ID, - ts: now - (14 - index) * 60_000, - artifact: { + try { + await store.start( + DEEP_RESEARCH_SESSION_ID, + 'Reproduce a durable filesystem-backed Deep Research workflow in Maka.', + 'standard', + ); + await store.recordArtifact(DEEP_RESEARCH_SESSION_ID, { + artifactId: sourceArtifactId, + role: 'source', + name: 'fs-researcher-paper.md', + createdAt: now - 19 * 60_000, + locator: 'https://arxiv.org/abs/2602.01566', + contentHash: hash, + sourceArtifactIds: [], + }); + for (const [index, key] of DEEP_RESEARCH_REPORT_SECTION_KEYS.entries()) { + await store.recordArtifact(DEEP_RESEARCH_SESSION_ID, { artifactId: `section-${key}`, role: 'report_section', name: `${key}.md`, @@ -90,16 +77,9 @@ export async function writeDeepResearchLedger(workspaceRoot: string, now: number sourceArtifactIds: [sourceArtifactId], reportSectionKey: key, reportSectionStatus: 'completed', - }, - }); - } - events.push({ - eventId: nextEventId(), - type: 'research_step_recorded', - sessionId: DEEP_RESEARCH_SESSION_ID, - ts: now - 8 * 60_000, - step: { - stepId: 'step-local-architecture', + }); + } + await store.recordStep(DEEP_RESEARCH_SESSION_ID, { kind: 'local_exploration', status: 'completed', objective: 'Trace the durable workspace from runtime tools to Desktop UI.', @@ -116,91 +96,48 @@ export async function writeDeepResearchLedger(workspaceRoot: string, now: number { kind: 'test', locator: 'packages/runtime/src/__tests__/deep-research-tools.test.ts' }, ], workerRunIds: ['worker-paper-review', 'worker-runtime-audit'], - createdAt: now - 8 * 60_000, - }, - }); - for (const item of DEEP_RESEARCH_DEFAULT_CHECKLIST) { - events.push({ - eventId: nextEventId(), - type: 'research_checklist_updated', - sessionId: DEEP_RESEARCH_SESSION_ID, - ts: now - 7 * 60_000, - item: { + }); + for (const item of DEEP_RESEARCH_DEFAULT_CHECKLIST) { + await store.updateChecklist(DEEP_RESEARCH_SESSION_ID, { itemId: item.itemId, - title: item.title, status: 'completed', evidenceArtifactIds: [sourceArtifactId], - updatedAt: now - 7 * 60_000, - }, + }); + } + await store.recordCheckpoint(DEEP_RESEARCH_SESSION_ID, { + round: 2, + stage: 'report_writing', + status: 'active', + summary: 'Evidence is complete and the implementation handoff is ready.', + openQuestions: [], + nextSteps: ['Review the handoff before entering implementation mode.'], + taskIds: [], + artifactIds: [sourceArtifactId], + }); + await store.recordArtifact(DEEP_RESEARCH_SESSION_ID, { + artifactId: 'report-final', + role: 'report', + name: 'deep-research-report.md', + createdAt: now - 5 * 60_000, + contentHash: hash, + sourceArtifactIds: [sourceArtifactId], }); + await store.recordArtifact(DEEP_RESEARCH_SESSION_ID, { + artifactId: 'handoff-final', + role: 'handoff', + name: 'implementation-handoff.md', + createdAt: now - 4 * 60_000, + contentHash: hash, + sourceArtifactIds: [sourceArtifactId], + }); + await store.complete(DEEP_RESEARCH_SESSION_ID, 'report-final', { + artifactId: 'handoff-final', + implementationTasks: ['Harden resume semantics.', 'Ship the visible progress surface.'], + recommendedIssues: ['Track cross-platform durability failures separately.'], + recommendedPullRequests: ['Land the workspace foundation before autonomous search policy.'], + verificationCommands: ['npm run typecheck', 'npm test'], + }); + } finally { + store.close(); } - events.push( - { - eventId: nextEventId(), - type: 'research_checkpoint_recorded', - sessionId: DEEP_RESEARCH_SESSION_ID, - ts: now - 6 * 60_000, - checkpoint: { - checkpointId: 'checkpoint-report-ready', - round: 2, - stage: 'report_writing', - status: 'active', - summary: 'Evidence is complete and the implementation handoff is ready.', - openQuestions: [], - nextSteps: ['Review the handoff before entering implementation mode.'], - taskIds: [], - artifactIds: [sourceArtifactId], - createdAt: now - 6 * 60_000, - }, - }, - { - eventId: nextEventId(), - type: 'research_artifact_recorded', - sessionId: DEEP_RESEARCH_SESSION_ID, - ts: now - 5 * 60_000, - artifact: { - artifactId: 'report-final', - role: 'report', - name: 'deep-research-report.md', - createdAt: now - 5 * 60_000, - contentHash: hash, - sourceArtifactIds: [sourceArtifactId], - }, - }, - { - eventId: nextEventId(), - type: 'research_artifact_recorded', - sessionId: DEEP_RESEARCH_SESSION_ID, - ts: now - 4 * 60_000, - artifact: { - artifactId: 'handoff-final', - role: 'handoff', - name: 'implementation-handoff.md', - createdAt: now - 4 * 60_000, - contentHash: hash, - sourceArtifactIds: [sourceArtifactId], - }, - }, - { - eventId: nextEventId(), - type: 'research_completed', - sessionId: DEEP_RESEARCH_SESSION_ID, - ts: now - 3 * 60_000, - reportArtifactId: 'report-final', - handoff: { - artifactId: 'handoff-final', - implementationTasks: ['Harden resume semantics.', 'Ship the visible progress surface.'], - recommendedIssues: ['Track cross-platform durability failures separately.'], - recommendedPullRequests: ['Land the workspace foundation before autonomous search policy.'], - verificationCommands: ['npm run typecheck', 'npm test'], - }, - }, - ); - const dir = join(workspaceRoot, 'sessions', DEEP_RESEARCH_SESSION_ID, 'deep-research'); - await mkdir(dir, { recursive: true }); - await writeFile( - join(dir, 'events.jsonl'), - events.map((event) => JSON.stringify(event)).join('\n') + '\n', - 'utf8', - ); } diff --git a/apps/desktop/src/main/e2e-fixture/scenarios-sessions.ts b/apps/desktop/src/main/e2e-fixture/scenarios-sessions.ts index 3641de7ea4..fee79f8763 100644 --- a/apps/desktop/src/main/e2e-fixture/scenarios-sessions.ts +++ b/apps/desktop/src/main/e2e-fixture/scenarios-sessions.ts @@ -6,7 +6,6 @@ import { LONG_SIDEBAR_SESSION_PREFIX, LONG_TRANSCRIPT_SESSION_ID, STALE_FAKE_SESSION_ID, - STALE_LEGACY_SESSION_ID, TURN_CONTROL_BRANCH_ORPHAN_SESSION_ID, TURN_CONTROL_BRANCH_VISIBLE_SESSION_ID, TURN_CONTROL_ORPHAN_PARENT_ID, @@ -566,11 +565,8 @@ export function longSidebarSessions(now: number): Array<{ header: SessionHeader; return seeds; } -// Stale-sessions fixture seeds three sessions reproducing the on-disk -// state that triggered the P0 (WAWQAQ workspace had `fake-claude` + -// `backend=fake` sessions sitting next to a healthy `zai-coding-plan` -// one). Locks the @kenji active-stale pill gate (active session is -// intentionally one of the stale ones). +// Seeds an unavailable connection beside a healthy one. The active Session is +// intentionally unavailable so the reconnect affordance remains covered. export function staleFakeSession(now: number): SessionHeader { return header({ id: STALE_FAKE_SESSION_ID, @@ -584,19 +580,6 @@ export function staleFakeSession(now: number): SessionHeader { }); } -export function staleLegacySession(now: number): SessionHeader { - return header({ - id: STALE_LEGACY_SESSION_ID, - name: '旧的 Claude 连接会话', - connection: 'fake-claude', - model: 'claude-3-sonnet', - now, - lastMessageAt: now - 7 * 24 * 3_600_000, - backend: 'claude' as SessionHeader['backend'], - connectionLocked: true, - }); -} - export function healthySession(now: number): SessionHeader { return header({ id: HEALTHY_SESSION_ID, @@ -630,27 +613,6 @@ export function staleFakeMessages(now: number): StoredMessage[] { ]; } -export function staleLegacyMessages(now: number): StoredMessage[] { - const turnId = 'stale-legacy-turn-1'; - return [ - { - type: 'user', - id: 'stale-legacy-msg-1', - turnId, - ts: now - 7 * 24 * 3_600_000, - text: '这是历史 Claude 连接留下的会话。原连接 fake-claude 已不在连接列表里。', - }, - { - type: 'assistant', - id: 'stale-legacy-msg-2', - turnId, - ts: now - 7 * 24 * 3_600_000 + 3_000, - text: '这条历史会话需要切换到当前可用模型后才能继续发送。', - modelId: 'claude-3-sonnet', - }, - ]; -} - export function healthyMessages(now: number): StoredMessage[] { const turnId = 'healthy-turn-1'; return [ diff --git a/apps/desktop/src/main/e2e-fixture/scenarios-settings.ts b/apps/desktop/src/main/e2e-fixture/scenarios-settings.ts index 6833140b63..26c67666bd 100644 --- a/apps/desktop/src/main/e2e-fixture/scenarios-settings.ts +++ b/apps/desktop/src/main/e2e-fixture/scenarios-settings.ts @@ -1,13 +1,13 @@ -import { mkdir } from 'node:fs/promises'; import { join } from 'node:path'; import type { DailyReviewArchive, LlmConnection, - PlanReminder, E2eFixtureScenario, } from '@maka/core'; import { createDefaultSettings } from '@maka/core/settings'; +import { createSqlitePlanReminderStore } from '@maka/storage'; import { writeJson } from './seed-helpers.js'; +import { createDailyReviewArchiveStore } from '../daily-review-archive-store.js'; export async function writeSettings( workspaceRoot: string, @@ -183,92 +183,46 @@ function model( return { id, capabilities, contextWindow }; } -export async function writePlanReminders(workspaceRoot: string, now: number): Promise { +export async function writePlanReminders(workspaceRoot: string, _now: number): Promise { const scheduledRunAt = Date.UTC(2026, 11, 18, 3, 0, 0); const pausedRunAt = Date.UTC(2026, 11, 20, 3, 0, 0); - const reminders: PlanReminder[] = [ - { - id: 'visual-plan-reminder-standup', + const store = createSqlitePlanReminderStore(workspaceRoot); + try { + await store.create({ title: '同步项目风险', note: '提醒我整理 Sidebar gate、搜索接入和计划任务剩余风险。', - schedule: { kind: 'once', runAt: scheduledRunAt }, - delivery: { channel: 'local' }, - status: 'scheduled', - enabled: true, - createdAt: now - 2 * 60 * 60_000, - updatedAt: now - 2 * 60 * 60_000, - nextRunAt: scheduledRunAt, - runs: [], - runCount: 0, - }, - { - id: 'visual-plan-reminder-paused', + runAt: scheduledRunAt, + }); + const paused = await store.create({ title: '暂停的发布检查', note: '用户可以先暂停提醒，恢复后继续按原时间触发。', - schedule: { kind: 'once', runAt: pausedRunAt }, - delivery: { channel: 'local' }, - status: 'paused', - enabled: false, - createdAt: now - 3 * 60 * 60_000, - updatedAt: now - 30 * 60_000, - runs: [], - runCount: 0, - }, - { - id: 'visual-plan-reminder-weekly-review', + runAt: pausedRunAt, + }); + await store.setEnabled(paused.id, false); + const weekly = await store.create({ title: '每周竞品动态追踪', note: '汇总同类 AI 工具的近期产品变化，提醒我复盘可对标的交互。', - schedule: { kind: 'cron', expression: '0 10 * * 1', startAt: now - 3.5 * 60 * 60_000 }, - delivery: { channel: 'local' }, - status: 'scheduled', - enabled: true, - createdAt: now - 3.5 * 60 * 60_000, - updatedAt: now - 35 * 60_000, - nextRunAt: Date.UTC(2026, 11, 21, 2, 0, 0), - lastRun: { - id: 'visual-plan-run-weekly-review', - at: now - 35 * 60_000, - status: 'triggered', - message: '已生成本周竞品动态摘要', - }, - runs: [ - { - id: 'visual-plan-run-weekly-review', - at: now - 35 * 60_000, - status: 'triggered', - message: '已生成本周竞品动态摘要', - }, - ], - runCount: 1, - }, - { - id: 'visual-plan-reminder-completed', + runAt: scheduledRunAt, + recurrence: 'cron', + cronExpression: '0 10 * * 1', + }); + await store.markTriggered(weekly.id, { + at: scheduledRunAt, + status: 'triggered', + message: '已生成本周竞品动态摘要', + }); + const completed = await store.create({ title: '已触发的本地提醒', - note: '', - schedule: { kind: 'once', runAt: now - 45 * 60_000 }, - delivery: { channel: 'local' }, - status: 'completed', - enabled: false, - createdAt: now - 4 * 60 * 60_000, - updatedAt: now - 45 * 60_000, - lastRun: { - id: 'visual-plan-run-completed', - at: now - 45 * 60_000, - status: 'triggered', - message: '计划提醒已触发', - }, - runs: [ - { - id: 'visual-plan-run-completed', - at: now - 45 * 60_000, - status: 'triggered', - message: '计划提醒已触发', - }, - ], - runCount: 1, - }, - ]; - await writeJson(join(workspaceRoot, 'plan-reminders.json'), reminders); + runAt: scheduledRunAt, + }); + await store.markTriggered(completed.id, { + at: scheduledRunAt, + status: 'triggered', + message: '计划提醒已触发', + }); + } finally { + store.close(); + } } export async function writeDailyReviewArchives(workspaceRoot: string, now: number): Promise { @@ -318,8 +272,7 @@ export async function writeDailyReviewArchives(workspaceRoot: string, now: numbe code: '下一步优先建立模块页 PageShell、SettingsActionRow 和 StatusPill primitives，再迁移 Daily Review、权限中心、计划任务和技能页。', }, }; - const archiveDir = join(workspaceRoot, 'daily-reviews', 'archive'); - await mkdir(archiveDir, { recursive: true }); - await writeJson(join(archiveDir, `${daily.id}.json`), daily); - await writeJson(join(archiveDir, `${deep.id}.json`), deep); + const store = createDailyReviewArchiveStore(workspaceRoot); + await store.putArchive(daily); + await store.putArchive(deep); } diff --git a/apps/desktop/src/main/e2e-fixture/seed-helpers.ts b/apps/desktop/src/main/e2e-fixture/seed-helpers.ts index 87d87bb880..13ade4c77b 100644 --- a/apps/desktop/src/main/e2e-fixture/seed-helpers.ts +++ b/apps/desktop/src/main/e2e-fixture/seed-helpers.ts @@ -1,6 +1,12 @@ import { mkdir, writeFile } from 'node:fs/promises'; import { dirname, join } from 'node:path'; import type { SessionHeader, StoredMessage, E2eFixtureScenario } from '@maka/core'; +import { + acquireOperationalStateDatabase, + createSqliteSessionMetadataStore, + OPERATIONAL_STATE_DATABASE_NAME, + projectSessionCatalogMessages, +} from '@maka/storage'; // Fixed clock for the e2e-fixture. All seeded timestamps and // transient fixture state derive from this value unless tests explicitly @@ -26,7 +32,6 @@ export const WORKSTATION_ABORTED_SESSION_ID = 'e2e-fixture-ws-aborted'; export const ERROR_SESSION_ID = 'e2e-fixture-error'; export const ARTIFACT_SESSION_ID = 'e2e-fixture-artifact'; export const STALE_FAKE_SESSION_ID = 'e2e-fixture-stale-fake'; -export const STALE_LEGACY_SESSION_ID = 'e2e-fixture-stale-legacy'; export const HEALTHY_SESSION_ID = 'e2e-fixture-healthy'; // PR109f (g): turn-control-history primary + branch sessions. The // `BRANCH_ORPHAN` session's `parentSessionId` intentionally references @@ -83,13 +88,8 @@ export function header(input: { now: number; lastMessageAt: number; hasUnread?: boolean; - /** - * Override default `backend: 'ai-sdk'`. Used by stale-sessions fixture - * to seed FakeBackend + legacy backend kinds. SessionHeader's BackendKind - * union allows widening via `as unknown` for legacy values like - * 'claude' that no longer exist in the type. - */ - backend?: SessionHeader['backend'] | 'claude'; + /** Override default `backend: 'ai-sdk'` for the unavailable-connection fixture. */ + backend?: SessionHeader['backend']; connectionLocked?: boolean; /** * PR109b workstation-statuses fixture: override default @@ -118,27 +118,37 @@ export function header(input: { statusUpdatedAt: input.lastMessageAt, hasUnread: input.hasUnread ?? false, ...(input.orchestrationMode ? { orchestrationMode: input.orchestrationMode } : {}), - // Legacy backend kinds like 'claude' aren't in the current BackendKind - // union but are needed for the stale-sessions reproduction. Forward - // the value verbatim into the JSONL so the renderer sees exactly what - // a real legacy workspace would have on disk. - backend: (input.backend ?? 'ai-sdk') as SessionHeader['backend'], + backend: input.backend ?? 'ai-sdk', llmConnectionSlug: input.connection, connectionLocked: input.connectionLocked ?? true, model: input.model, permissionMode: 'ask', + collaborationMode: 'agent', + orchestrationMode: input.orchestrationMode ?? 'default', schemaVersion: 1, }; } -export async function writeSession(workspaceRoot: string, session: SessionHeader, messages: StoredMessage[]): Promise { - const dir = join(workspaceRoot, 'sessions', session.id); - await mkdir(dir, { recursive: true }); - await writeFile( - join(dir, 'session.jsonl'), - [session, ...messages].map((entry) => JSON.stringify(entry)).join('\n') + '\n', - 'utf8', +export async function writeSession( + workspaceRoot: string, + session: SessionHeader, + messages: StoredMessage[], +): Promise { + const databaseLease = acquireOperationalStateDatabase(workspaceRoot); + const sessions = createSqliteSessionMetadataStore( + join(workspaceRoot, OPERATIONAL_STATE_DATABASE_NAME), + { databaseLease }, ); + try { + await sessions.create(session); + await sessions.appendMessages( + session.id, + messages, + projectSessionCatalogMessages(messages), + ); + } finally { + sessions.close(); + } } export async function writeJson(path: string, value: unknown): Promise { diff --git a/apps/desktop/src/main/oauth/shared-credential-bridge.ts b/apps/desktop/src/main/oauth/shared-credential-bridge.ts index 963303bb6f..3594dfce55 100644 --- a/apps/desktop/src/main/oauth/shared-credential-bridge.ts +++ b/apps/desktop/src/main/oauth/shared-credential-bridge.ts @@ -1,20 +1,4 @@ -/** - * OAuth token persistence for the desktop subscription services. - * - * The pure-Node `CredentialStore` (workspace `credentials.json`) is the - * single authority for runtime-usable OAuth tokens (#1125): desktop, - * TUI, and headless all read and write the same store, so a desktop - * login is immediately usable from pure-Node surfaces and vice versa. - * Electron `safeStorage` no longer stores tokens; it only decrypts - * legacy per-service token files once via - * `importLegacyOAuthTokenFiles`, after which those files are removed. - * - * Read/write failures are the caller's to surface (`storage_failed`), - * so unlike the historical best-effort export, these helpers do not - * swallow store errors. - */ - -import { promises as fs } from 'node:fs'; +/** OAuth token persistence through the shared CredentialStore authority. */ import { parseOAuthSubscriptionTokens, serializeOAuthSubscriptionTokens, @@ -30,13 +14,8 @@ export type SharedOAuthCredentialStore = Pick< export type SharedOAuthTokensReadResult = | { status: 'ok'; tokens: OAuthSubscriptionTokens } | { status: 'missing' } - /** Entry exists but is not a valid token payload. The entry is kept - * as-is: reads never destroy a secret (it may be readable by a - * newer schema or repairable by hand), and a fresh login simply - * overwrites it — there is no stuck state a delete would unstick. */ | { status: 'corrupt' }; -/** Persist tokens as the authoritative copy. Throws on store failure. */ export async function saveSharedOAuthTokens( store: Pick, slug: string, @@ -45,12 +24,6 @@ export async function saveSharedOAuthTokens( await store.setSecret(slug, 'oauth_token', serializeOAuthSubscriptionTokens(tokens)); } -/** - * Load the authoritative tokens. Store read errors (corrupt file, - * schema mismatch, stale lock) propagate to the caller; an entry that - * exists but does not parse as a token payload is reported as - * `corrupt` and left untouched. - */ export async function loadSharedOAuthTokens( store: SharedOAuthCredentialStore, slug: string, @@ -58,166 +31,12 @@ export async function loadSharedOAuthTokens( const raw = await store.getSecret(slug, 'oauth_token'); if (raw === null) return { status: 'missing' }; const tokens = parseOAuthSubscriptionTokens(raw); - if (!tokens) return { status: 'corrupt' }; - return { status: 'ok', tokens }; + return tokens ? { status: 'ok', tokens } : { status: 'corrupt' }; } -/** Delete the authoritative tokens. Throws on store failure. */ export async function deleteSharedOAuthTokens( store: Pick, slug: string, ): Promise { await store.deleteSecret(slug, 'oauth_token'); } - -// ============================================================= -// One-shot import of legacy safeStorage-encrypted token files. -// ============================================================= - -/** - * Shape-compatible with Electron's `safeStorage` so main.ts can pass it - * straight through; injected so this module never imports `electron` - * and the import stays testable in pure Node. - */ -export interface LegacySafeStorageDecryptor { - isEncryptionAvailable(): boolean; - decryptString(encrypted: Buffer): string; -} - -export interface LegacyOAuthTokenFile { - slug: string; - filePath: string; -} - -export type LegacyOAuthTokenImportOutcome = - /** Decrypted and written to the store; file removed. */ - | 'imported' - /** The store already held a PARSEABLE token for the slug (it is at - * least as fresh — every legacy write dual-wrote the store, and - * pure-Node refreshes write only the store); file removed as a - * stale duplicate. An unparseable store entry does not supersede: - * a valid legacy token must win over stored garbage. */ - | 'superseded' - /** Decryption unavailable or denied; file left intact for a later - * start (never destroy a possibly recoverable secret). */ - | 'left-encrypted' - /** Decrypted fine but the payload is not a token this build can - * parse; file kept — only an explicit logout destroys it. */ - | 'left-unparseable' - /** Unexpected I/O or store error; file left intact. */ - | 'failed'; - -export interface LegacyOAuthTokenImportReport { - slug: string; - filePath: string; - outcome: LegacyOAuthTokenImportOutcome; - error?: unknown; -} - -/** - * Import legacy safeStorage-encrypted token files into the shared - * store, once per file. Idempotent: a missing file is a no-op. The - * The file is removed after a successful import, when superseded by a - * parseable store token, or when a concurrent logout wins the serialized - * check. Every other outcome keeps the file for a later start (or manual - * recovery), because the import must never destroy the last copy without - * an authoritative replacement or deletion. Never throws — desktop startup - * treats migration as best-effort; returns a report per file that existed. - * The startup owner completes this one-shot phase before exposing interactive - * OAuth mutations; CAS protects it from independent shared-store writers. - */ -export async function importLegacyOAuthTokenFiles(input: { - credentialStore: Pick; - decryptor: LegacySafeStorageDecryptor; - files: LegacyOAuthTokenFile[]; -}): Promise { - const reports: LegacyOAuthTokenImportReport[] = []; - for (const { slug, filePath } of input.files) { - const report = (outcome: LegacyOAuthTokenImportOutcome, error?: unknown): void => { - reports.push({ slug, filePath, outcome, ...(error === undefined ? {} : { error }) }); - }; - let encrypted: Buffer; - try { - encrypted = await fs.readFile(filePath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') report('failed', error); - continue; - } - try { - let existing = await input.credentialStore.getSecret(slug, 'oauth_token'); - if (existing !== null && parseOAuthSubscriptionTokens(existing)) { - if (!input.credentialStore.compareAndSetSecret) { - await fs.unlink(filePath); - report('superseded'); - continue; - } - // A same-value CAS turns the pre-lock "already imported" snapshot into - // a serialized decision before the legacy file may be removed. - const checked = await input.credentialStore.compareAndSetSecret( - slug, - 'oauth_token', - existing, - existing, - ); - if (checked.committed || (checked.current !== null && parseOAuthSubscriptionTokens(checked.current))) { - await fs.unlink(filePath); - report('superseded'); - continue; - } - if (checked.current === null) { - // Logout removes the legacy file before deleting the shared token. - // Once that deletion wins the serialized check, the buffered legacy - // value must not become a new basis and resurrect the credential. - await fs.rm(filePath, { force: true }); - report('superseded'); - continue; - } - report('failed', new Error('OAuth credential changed during legacy import.')); - continue; - } - if (!input.decryptor.isEncryptionAvailable()) { - report('left-encrypted'); - continue; - } - let decoded: string; - try { - decoded = input.decryptor.decryptString(encrypted); - } catch (error) { - // Keychain denied / rolled: possibly recoverable on a later - // start, so keep the file. - report('left-encrypted', error); - continue; - } - const tokens = parseOAuthSubscriptionTokens(decoded); - if (!tokens) { - report('left-unparseable'); - continue; - } - const serialized = serializeOAuthSubscriptionTokens(tokens); - if (input.credentialStore.compareAndSetSecret) { - const committed = await input.credentialStore.compareAndSetSecret( - slug, - 'oauth_token', - existing, - serialized, - ); - if (!committed.committed) { - if (committed.current !== null && parseOAuthSubscriptionTokens(committed.current)) { - await fs.unlink(filePath); - report('superseded'); - } else { - report('failed', new Error('OAuth credential changed during legacy import.')); - } - continue; - } - } else { - await input.credentialStore.setSecret(slug, 'oauth_token', serialized); - } - await fs.unlink(filePath); - report('imported'); - } catch (error) { - report('failed', error); - } - } - return reports; -} diff --git a/apps/desktop/src/main/project-startup-migration.ts b/apps/desktop/src/main/project-startup-migration.ts deleted file mode 100644 index cf06080133..0000000000 --- a/apps/desktop/src/main/project-startup-migration.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { migrateSessionProjects } from '@maka/storage'; -import type { ProjectCatalog, SessionStore } from '@maka/storage'; - -export async function runProjectStartupMigration(deps: { - sessions: Pick; - catalog: Pick; - emitSessionsChanged?(reason: 'migrated'): void; - logError(message: string, error?: unknown): void; -}): Promise { - try { - const result = await migrateSessionProjects({ - sessions: deps.sessions, - catalog: deps.catalog, - }); - if (result.failed > 0) { - deps.logError(`[projects] skipped ${result.failed} legacy session migration(s)`); - } - if (result.migrated > 0) deps.emitSessionsChanged?.('migrated'); - } catch (error) { - deps.logError('[projects] legacy session migration failed:', error); - } -} diff --git a/apps/desktop/src/main/quote-companion-cleanup.ts b/apps/desktop/src/main/quote-companion-cleanup.ts index f652bf72ea..0f87ea85b7 100644 --- a/apps/desktop/src/main/quote-companion-cleanup.ts +++ b/apps/desktop/src/main/quote-companion-cleanup.ts @@ -1,13 +1,4 @@ -import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; - -const CLEANUP_FILE_VERSION = 1; -const CLEANUP_FILE_NAME = 'quote-companion-cleanup.json'; - -interface QuoteCompanionCleanupFile { - version: typeof CLEANUP_FILE_VERSION; - pendingSessionIds: string[]; -} +import { acquireOperationalStateDatabase } from '@maka/storage'; interface QuoteCompanionCleanupStore { list(): Promise; @@ -21,12 +12,7 @@ export interface QuoteCompanionCleanupRecovery { } export interface QuoteCompanionCleanupAuthority { - /** - * Record the cleanup intent before attempting removal. The record is cleared - * only after the complete Desktop session-removal path succeeds. - */ cleanup(sessionId: string): Promise; - /** Retry every cleanup intent left by an earlier failed attempt or app exit. */ recover(): Promise; } @@ -34,13 +20,13 @@ export function createQuoteCompanionCleanupAuthority(input: { workspaceRoot: string; removeSession: (sessionId: string) => Promise; }): QuoteCompanionCleanupAuthority { - return new FileQuoteCompanionCleanupAuthority( - new FileQuoteCompanionCleanupStore(input.workspaceRoot), + return new QuoteCompanionCleanupAuthorityImpl( + new SqliteQuoteCompanionCleanupStore(input.workspaceRoot), input.removeSession, ); } -class FileQuoteCompanionCleanupAuthority implements QuoteCompanionCleanupAuthority { +class QuoteCompanionCleanupAuthorityImpl implements QuoteCompanionCleanupAuthority { private readonly inFlight = new Map>(); constructor( @@ -52,8 +38,7 @@ class FileQuoteCompanionCleanupAuthority implements QuoteCompanionCleanupAuthori const normalized = normalizeSessionId(sessionId); const active = this.inFlight.get(normalized); if (active) return active; - - const operation = this.cleanupOnce(normalized, true).finally(() => { + const operation = this.cleanupOnce(normalized).finally(() => { this.inFlight.delete(normalized); }); this.inFlight.set(normalized, operation); @@ -74,81 +59,66 @@ class FileQuoteCompanionCleanupAuthority implements QuoteCompanionCleanupAuthori return { removed, failed }; } - private async cleanupOnce(sessionId: string, track: boolean): Promise { - // This small file is the companion cleanup WAL: once `track` resolves, a - // failed removal can be replayed after panel unmount or Desktop restart. - if (track) await this.store.track(sessionId); + private async cleanupOnce(sessionId: string): Promise { + await this.store.track(sessionId); await this.removeSession(sessionId); await this.store.forget(sessionId); } } -class FileQuoteCompanionCleanupStore implements QuoteCompanionCleanupStore { - private readonly filePath: string; - private queue: Promise = Promise.resolve(); - - constructor(workspaceRoot: string) { - this.filePath = join(workspaceRoot, CLEANUP_FILE_NAME); - } +class SqliteQuoteCompanionCleanupStore implements QuoteCompanionCleanupStore { + constructor(private readonly workspaceRoot: string) {} async list(): Promise { - await this.queue; - return this.read(); - } - - track(sessionId: string): Promise { - return this.mutate((current) => - current.includes(sessionId) ? current : [...current, sessionId], + return this.withDatabase('read', (database) => + ( + database + .prepare(` + SELECT session_id AS sessionId + FROM workflow_quote_companion_cleanup + ORDER BY tracked_at, session_id + `) + .all() as Array<{ sessionId: string }> + ).map((row) => row.sessionId), ); } - forget(sessionId: string): Promise { - return this.mutate((current) => current.filter((id) => id !== sessionId)); + async track(sessionId: string): Promise { + this.withDatabase('write', (database) => { + database + .prepare(` + INSERT OR IGNORE INTO workflow_quote_companion_cleanup(session_id, tracked_at) + VALUES (?, ?) + `) + .run(sessionId, Date.now()); + }); } - private async mutate(update: (current: string[]) => string[]): Promise { - const run = async () => { - const current = await this.read(); - await this.write(update(current)); - }; - const next = this.queue.then(run, run); - this.queue = next.catch(() => {}); - await next; + async forget(sessionId: string): Promise { + this.withDatabase('write', (database) => { + database + .prepare('DELETE FROM workflow_quote_companion_cleanup WHERE session_id = ?') + .run(sessionId); + }); } - private async read(): Promise { + private withDatabase( + mode: 'read' | 'write', + operation: (database: import('node:sqlite').DatabaseSync) => T, + ): T { + const lease = acquireOperationalStateDatabase(this.workspaceRoot); try { - const raw = await readFile(this.filePath, 'utf8'); - const parsed = JSON.parse(raw) as Partial; - if ( - parsed.version !== CLEANUP_FILE_VERSION || - !Array.isArray(parsed.pendingSessionIds) - ) { - throw new Error('Invalid quote companion cleanup file'); - } - return [...new Set(parsed.pendingSessionIds.map(normalizeSessionId))]; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; + return lease.transaction(mode, () => operation(lease.database)); + } finally { + lease.close(); } } - - private async write(pendingSessionIds: string[]): Promise { - await mkdir(dirname(this.filePath), { recursive: true }); - const next: QuoteCompanionCleanupFile = { - version: CLEANUP_FILE_VERSION, - pendingSessionIds, - }; - const tempPath = `${this.filePath}.${process.pid}.${Date.now()}.tmp`; - await writeFile(tempPath, `${JSON.stringify(next, null, 2)}\n`, 'utf8'); - await rename(tempPath, this.filePath); - } } function normalizeSessionId(value: unknown): string { if (typeof value !== 'string') throw new Error('Invalid quote companion session id'); const normalized = value.trim(); - if (!normalized || normalized.length > 256) { + if (!/^[A-Za-z0-9_-]{1,128}$/.test(normalized)) { throw new Error('Invalid quote companion session id'); } return normalized; diff --git a/packages/cli/src/__tests__/inspect-command.test.ts b/packages/cli/src/__tests__/inspect-command.test.ts index aadb3bc5fd..46b7a7d62a 100644 --- a/packages/cli/src/__tests__/inspect-command.test.ts +++ b/packages/cli/src/__tests__/inspect-command.test.ts @@ -8,10 +8,7 @@ import { createInMemoryTaskRunStore, runTaskOnce } from '@maka/headless'; import type { SessionInspectDocument } from '@maka/runtime'; import type { RuntimeHostConnection } from '@maka/runtime-host/client'; import { createSessionStore } from '@maka/storage'; -import { - createLegacyAgentRunStoreForTest, - createLegacyRuntimeEventStoreForTest, -} from '@maka/storage/legacy-execution-test-support'; +import { createSqliteAgentRunStore, createWorkspaceRuntimeStore } from '@maka/storage'; import { openHeadlessExecutionStoresForWrite, openInteractiveExecutionStoresForWrite, @@ -332,8 +329,8 @@ describe('inspect CLI storage authority boundary', () => { type InspectCommandTestStores = { sessionStore: ReturnType; - agentRunStore: ReturnType; - runtimeEventStore: ReturnType; + agentRunStore: ReturnType; + runtimeEventStore: ReturnType; taskRunStore: ReturnType; }; @@ -391,8 +388,8 @@ async function withStores(run: (stores: InspectCommandTestStores) => Promise { name: 'isolated', }); - await access(join(stateRoot, 'sessions', session.id, 'session.jsonl')); - await assert.rejects(access(join(configRoot, 'sessions', session.id, 'session.jsonl'))); + await access(join(stateRoot, 'runtime.sqlite')); + await assert.rejects(access(join(configRoot, 'runtime.sqlite'))); await access(join(configRoot, 'llm-connections.json')); await access(join(configRoot, 'credentials.json')); await assert.rejects(access(join(stateRoot, 'credentials.json'))); diff --git a/packages/cli/src/runtime-bootstrap.ts b/packages/cli/src/runtime-bootstrap.ts index b62eec475a..4bd246ef4f 100644 --- a/packages/cli/src/runtime-bootstrap.ts +++ b/packages/cli/src/runtime-bootstrap.ts @@ -354,20 +354,12 @@ export async function createMakaCliRuntimeContext( generateId: () => randomUUID(), now: () => Date.now(), }); - // Durable persistence is tied to cron capability. A cron-disabled host is - // heartbeat-only, and heartbeats are never durable — so it has NO durable - // automations of its own. Critically, the CLI shares the desktop's workspace - // (resolveMakaWorkspaceRoot reconstructs the Electron userData path), so its - // automations.json IS the desktop's. store.sync() is a full-file overwrite, - // so a heartbeat-only CLI writing its (empty) durable list would erase the - // desktop's crons, and loading+reconciling crons it can't run would mutate - // them. It therefore does neither — it leaves durable state entirely to the - // host that owns it. (Two cron-enabled hosts sharing a store is the separate, - // still-deferred leader-lock concern.) + // A heartbeat-only CLI owns no durable Automations and must not reconcile the + // shared authority. A cron-enabled host persists through the same operational + // SQLite authority as Desktop. const cronEnabled = input.automationCreateFreshRun !== undefined; - const automationStore = createAutomationStore(configRoot); - // If the durable store fails to READ, we must not WRITE over it (a full sync - // would erase unread crons). Disable persistence loudly until restart. + const automationStore = createAutomationStore(stateRoot); + // If the authority cannot be read, do not attempt a later replacement write. let durableStoreReadable = true; const syncAutomations = cronEnabled ? (): void => { diff --git a/packages/headless/harbor/maka_trajectory.py b/packages/headless/harbor/maka_trajectory.py index 67851d33a4..5429d70c1a 100644 --- a/packages/headless/harbor/maka_trajectory.py +++ b/packages/headless/harbor/maka_trajectory.py @@ -2007,11 +2007,6 @@ def _read_artifact_records(artifact_store_root: Path) -> dict[str, dict[str, Any ).fetchone() if schema != (1,): raise ValueError - cutover = connection.execute( - "SELECT state FROM cutover_journal WHERE store_name = 'artifact_metadata'" - ).fetchone() - if cutover != ("completed",): - raise ValueError rows = connection.execute( """ SELECT artifact_id, session_id, created_at, status, relative_path, record_json diff --git a/packages/headless/src/__tests__/cli.test.ts b/packages/headless/src/__tests__/cli.test.ts index efc68ea9c4..1bd5202184 100644 --- a/packages/headless/src/__tests__/cli.test.ts +++ b/packages/headless/src/__tests__/cli.test.ts @@ -5,6 +5,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; import { describe, test } from 'node:test'; +import { createSessionStore } from '@maka/storage'; import { validateHarborCellOutput } from '../cell-output.js'; import { openHeadlessStorageForWrite } from '../headless-storage.js'; import { readResults } from '../results.js'; @@ -413,7 +414,12 @@ describe('maka-headless CLI', () => { const sessionId = inspectDocument.attempts[0]?.agentRuns[0]?.identity?.sessionId; assert.equal(typeof sessionId, 'string'); - await rm(join(outDir, 'runs', 'sessions', sessionId, 'session.jsonl'), { force: true }); + const sessions = createSessionStore(join(outDir, 'runs')); + try { + await sessions.remove(sessionId); + } finally { + await sessions.close?.(); + } const aheExportDir = join(dir, 'ahe-export'); const aheExported = await runCli([ diff --git a/packages/headless/src/__tests__/harbor-adapter.test.ts b/packages/headless/src/__tests__/harbor-adapter.test.ts index 51fa46cb38..6aba974ef1 100644 --- a/packages/headless/src/__tests__/harbor-adapter.test.ts +++ b/packages/headless/src/__tests__/harbor-adapter.test.ts @@ -2764,15 +2764,6 @@ def write_artifact_database(store_root, records, artifact_schema_version=1): version INTEGER NOT NULL, applied_at INTEGER NOT NULL ); - CREATE TABLE cutover_journal ( - store_name TEXT PRIMARY KEY, - source_path TEXT NOT NULL, - source_fingerprint TEXT NOT NULL, - state TEXT NOT NULL, - started_at INTEGER NOT NULL, - completed_at INTEGER, - validation_json TEXT - ); CREATE TABLE artifact_records ( storage_key TEXT PRIMARY KEY, artifact_id TEXT NOT NULL, @@ -2787,9 +2778,6 @@ def write_artifact_database(store_root, records, artifact_schema_version=1): "INSERT INTO operational_schema_migrations(scope, version, applied_at) VALUES ('artifact', ?, 1)", (artifact_schema_version,), ) - connection.execute( - "INSERT INTO cutover_journal(store_name, source_path, source_fingerprint, state, started_at, completed_at, validation_json) VALUES ('artifact_metadata', 'artifacts/metadata.jsonl', 'none', 'completed', 1, 1, '{}')" - ) for record in records: connection.execute( """ @@ -2886,19 +2874,6 @@ with tempfile.TemporaryDirectory() as tmp: assert image_content[0].source.path.endswith(".png"), image_content assert (logs_dir / image_content[0].source.path).is_file(), image_content - database_path.unlink() - metadata_path = artifact_root / "metadata.jsonl" - metadata_path.write_text(json.dumps(image_record) + "\n", encoding="utf-8") - agent._apply_cell_output(context, { - "status": "completed", - "runtimeEventsPath": "/logs/agent/runtime-events.jsonl", - "runtimeRefs": {"invocationId": "inv-1", "runId": "run-1", "sessionId": "session-1", "turnId": "turn-1"}, - }) - legacy_payload = json.loads((logs_dir / "trajectory.json").read_text(encoding="utf-8")) - assert legacy_payload["extra"]["maka_artifact_kind"] == "summary", legacy_payload - assert legacy_payload["extra"]["maka_summary_reason"] == "image_artifact_metadata_missing", legacy_payload - metadata_path.unlink() - write_artifact_database(artifact_root.parent, [image_record], artifact_schema_version=2) agent._apply_cell_output(context, { "status": "completed", diff --git a/packages/headless/src/__tests__/task-run-inspect.test.ts b/packages/headless/src/__tests__/task-run-inspect.test.ts index 6dadc07523..ee52509644 100644 --- a/packages/headless/src/__tests__/task-run-inspect.test.ts +++ b/packages/headless/src/__tests__/task-run-inspect.test.ts @@ -5,10 +5,7 @@ import { join } from 'node:path'; import { describe, test } from 'node:test'; import type { AgentRunEvent, AgentRunHeader, RuntimeEvent } from '@maka/core'; import { buildHistoryCompactCheckpoint } from '@maka/runtime'; -import { - createLegacyAgentRunStoreForTest, - createLegacyRuntimeEventStoreForTest, -} from '@maka/storage/legacy-execution-test-support'; +import { createSqliteAgentRunStore, createWorkspaceRuntimeStore } from '@maka/storage'; import type { HeavyTaskSemanticSelfCheckState, TaskEvent } from '../task-contracts.js'; import { taskAttemptExecutionEvidence } from '../task-execution-lineage.js'; import { createInMemoryTaskRunStore } from '../task-run-store.js'; @@ -353,16 +350,16 @@ const TURN_ID = 'turn-1'; async function withStores( run: (stores: { taskRunStore: ReturnType; - agentRunStore: ReturnType; - runtimeEventStore: ReturnType; + agentRunStore: ReturnType; + runtimeEventStore: ReturnType; }) => Promise, ): Promise { const root = await mkdtemp(join(tmpdir(), 'maka-task-run-inspect-')); try { await run({ taskRunStore: createInMemoryTaskRunStore(), - agentRunStore: createLegacyAgentRunStoreForTest(root), - runtimeEventStore: createLegacyRuntimeEventStoreForTest(root), + agentRunStore: createSqliteAgentRunStore(root), + runtimeEventStore: createWorkspaceRuntimeStore(root), }); } finally { await rm(root, { recursive: true, force: true }); diff --git a/packages/headless/src/__tests__/task-run-store.test.ts b/packages/headless/src/__tests__/task-run-store.test.ts index 89251b0693..949a6fffbe 100644 --- a/packages/headless/src/__tests__/task-run-store.test.ts +++ b/packages/headless/src/__tests__/task-run-store.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { fork, type ChildProcess } from 'node:child_process'; -import { appendFile, lstat, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; +import { lstat, mkdtemp, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -128,7 +128,7 @@ function completedEvents(taskRunId = 'tr-1', taskId = 'task-1', configId = 'cfg- } describe('TaskRunStore', () => { - test('binds file readers and writers to Headless leases', async () => { + test('binds SQLite readers and writers to Headless leases', async () => { const base = await mkdtemp(join(tmpdir(), 'maka-task-run-lease-')); const headlessRoot = join(base, 'headless'); const interactiveRoot = join(base, 'interactive'); @@ -155,7 +155,9 @@ describe('TaskRunStore', () => { (error: unknown) => error instanceof StorageRootAuthorityError && error.code === 'invalid_lease', ); - await assert.rejects(() => lstat(join(interactiveRoot, 'task-runs')), { code: 'ENOENT' }); + await assert.rejects(() => lstat(join(interactiveRoot, 'runtime.sqlite')), { + code: 'ENOENT', + }); } finally { await owner.close(); } @@ -188,16 +190,16 @@ describe('TaskRunStore', () => { assert.deepEqual(await store.readEvents('tr-concurrent'), events); }); - test('file-backed store appends and replays events after restart', async () => { + test('SQLite store appends and replays events after restart', async () => { const storageRoot = await mkdtemp(join(tmpdir(), 'maka-task-run-store-')); try { - const store = await openFileTaskRunWriter(storageRoot); - const events = completedEvents('tr-file'); + const store = await openSqliteTaskRunWriter(storageRoot); + const events = completedEvents('tr-sqlite'); for (const event of events) await store.appendEvent(event.taskRunId, event); - const restarted = await openFileTaskRunReader(storageRoot); - assert.deepEqual(await restarted.readEvents('tr-file'), events); - assert.equal((await restarted.project('tr-file')).status, 'completed'); + const restarted = await openSqliteTaskRunReader(storageRoot); + assert.deepEqual(await restarted.readEvents('tr-sqlite'), events); + assert.equal((await restarted.project('tr-sqlite')).status, 'completed'); } finally { await rm(storageRoot, { recursive: true, force: true }); } @@ -206,8 +208,8 @@ describe('TaskRunStore', () => { test('atomically publishes the first ledger across independent writers', async () => { const storageRoot = await mkdtemp(join(tmpdir(), 'maka-task-run-store-')); try { - const first = await openFileTaskRunWriter(storageRoot); - const second = await openFileTaskRunWriter(storageRoot); + const first = await openSqliteTaskRunWriter(storageRoot); + const second = await openSqliteTaskRunWriter(storageRoot); const taskRunId = 'concurrent-first-write'; const firstEvent = completedEvents(taskRunId)[0]!; const secondEvent: TaskEvent = { @@ -224,7 +226,7 @@ describe('TaskRunStore', () => { second.appendEvent(taskRunId, secondEvent), ]); - const reader = await openFileTaskRunReader(storageRoot); + const reader = await openSqliteTaskRunReader(storageRoot); assert.deepEqual( (await reader.readEvents(taskRunId)).map((event) => event.id).sort(), [firstEvent.id, secondEvent.id].sort(), @@ -235,21 +237,17 @@ describe('TaskRunStore', () => { } }); - test('serializes large ledger appends across independent processes', async () => { + test('serializes appends across independent processes', async () => { const storageRoot = await mkdtemp(join(tmpdir(), 'maka-task-run-store-')); const children: ChildProcess[] = []; try { - const taskRunId = 'concurrent-large-write'; - const writer = await openFileTaskRunWriter(storageRoot); + const taskRunId = 'concurrent-process-write'; + const writer = await openSqliteTaskRunWriter(storageRoot); await writer.appendEvent(taskRunId, completedEvents(taskRunId)[0]!); - const instructions = [ - `first:${'a'.repeat(768 * 1024)}:first`, - `second:${'b'.repeat(768 * 1024)}:second`, - ]; - const events: TaskEvent[] = instructions.map((instruction, index) => ({ + const events: TaskEvent[] = ['first', 'second'].map((instruction, index) => ({ type: 'task_run_queued', - id: `large-${index + 1}`, + id: `process-${index + 1}`, taskRunId, taskId: `task-${index + 1}`, configId: 'cfg-1', @@ -273,7 +271,7 @@ describe('TaskRunStore', () => { await Promise.all(writers.map((prepared) => prepared.ready)); await Promise.all(writers.map((prepared) => prepared.append())); - const reader = await openFileTaskRunReader(storageRoot); + const reader = await openSqliteTaskRunReader(storageRoot); const replayed = await reader.readEvents(taskRunId); assert.deepEqual( replayed @@ -297,52 +295,27 @@ describe('TaskRunStore', () => { } }); - test('keeps distinct task run identities in separate ledgers', async () => { + test('keeps distinct task run identities separate', async () => { const storageRoot = await mkdtemp(join(tmpdir(), 'maka-task-run-store-')); try { - const store = await openFileTaskRunWriter(storageRoot); - const first = completedEvents('run/a', 'slash-task', 'slash-config'); - const second = completedEvents('run?a', 'question-task', 'question-config'); - for (let index = 0; index < first.length; index += 1) { - await store.appendEvent('run/a', first[index]!); - await store.appendEvent('run?a', second[index]!); - } + const store = await openSqliteTaskRunWriter(storageRoot); + const first = completedEvents('run/a', 'slash-task', 'slash-config')[0]!; + const second = completedEvents('run?a', 'question-task', 'question-config')[0]!; + await store.appendEvent('run/a', first); + await store.appendEvent('run?a', second); assert.deepEqual(await store.listTaskRunIds(), ['run/a', 'run?a']); - assert.deepEqual(await store.readEvents('run/a'), first); - assert.deepEqual(await store.readEvents('run?a'), second); - - const firstProjection = await store.project('run/a'); - assert.equal(firstProjection.taskId, 'slash-task'); - assert.deepEqual(firstProjection.events, first); - const secondProjection = await store.project('run?a'); - assert.equal(secondProjection.taskId, 'question-task'); - assert.deepEqual(secondProjection.events, second); - - const taskRunDir = join(storageRoot, 'task-runs'); - const ledgerNames = (await readdir(taskRunDir)) - .filter((name) => name.endsWith('.jsonl')) - .sort(); - assert.equal(ledgerNames.length, 2); - assert.notEqual(ledgerNames[0], ledgerNames[1]); - for (const ledgerName of ledgerNames) { - const identities = new Set( - (await readFile(join(taskRunDir, ledgerName), 'utf8')) - .trim() - .split('\n') - .map((line) => (JSON.parse(line) as TaskEvent).taskRunId), - ); - assert.equal(identities.size, 1); - } + assert.deepEqual(await store.readEvents('run/a'), [first]); + assert.deepEqual(await store.readEvents('run?a'), [second]); } finally { await rm(storageRoot, { recursive: true, force: true }); } }); - test('rejects empty and malformed task run identities before writing a ledger', async () => { + test('rejects invalid task run and event identities before writing', async () => { const storageRoot = await mkdtemp(join(tmpdir(), 'maka-task-run-store-')); try { - const store = await openFileTaskRunWriter(storageRoot); + const store = await openSqliteTaskRunWriter(storageRoot); await assert.rejects( () => store.appendEvent('', completedEvents('')[0]!), /taskRunId must not be empty/, @@ -352,95 +325,23 @@ describe('TaskRunStore', () => { () => store.appendEvent(malformedId, completedEvents(malformedId)[0]!), /taskRunId must be well-formed Unicode/, ); - await assert.rejects(() => lstat(join(storageRoot, 'task-runs')), { code: 'ENOENT' }); - } finally { - await rm(storageRoot, { recursive: true, force: true }); - } - }); - - test('keeps long domain identities independent from filesystem component limits', async () => { - const storageRoot = await mkdtemp(join(tmpdir(), 'maka-task-run-store-')); - try { - const store = await openFileTaskRunWriter(storageRoot); - const taskRunId = 'long-task-run/'.repeat(64); - const event = completedEvents(taskRunId)[0]!; - await store.appendEvent(taskRunId, event); - - assert.deepEqual(await store.listTaskRunIds(), [taskRunId]); - assert.deepEqual(await store.readEvents(taskRunId), [event]); - } finally { - await rm(storageRoot, { recursive: true, force: true }); - } - }); - - test('discovers identity without scanning corrupt history and repairs a partial tail', async () => { - const storageRoot = await mkdtemp(join(tmpdir(), 'maka-task-run-store-')); - try { - const store = await openFileTaskRunWriter(storageRoot); - const events = completedEvents('tr-corrupt'); - await store.appendEvent('tr-corrupt', events[0] as TaskEvent); - const [ledgerName] = (await readdir(join(storageRoot, 'task-runs'))).filter((name) => - name.endsWith('.jsonl'), - ); - assert.ok(ledgerName); - - await appendFile( - join(storageRoot, 'task-runs', ledgerName), - 'not-json\n{"type":"task_run_completed","id":"partial"', - 'utf8', - ); - - assert.deepEqual(await store.listTaskRunIds(), ['tr-corrupt']); - const replayed = await store.readEvents('tr-corrupt'); - assert.equal(replayed.length, 2); - assert.equal(replayed[1]?.type, 'event_corrupt'); - assert.match((replayed[1] as { error?: string }).error ?? '', /Unexpected/); - const records = await store.readEventRecords('tr-corrupt'); - assert.deepEqual( - records.map((record) => record.cursor.sequence), - [0, 1], - ); - assert.equal(records[1]?.cursor.eventId, 'corrupt-2'); - - const completed = completedEvents('tr-corrupt').at(-1)!; - await store.appendEvent('tr-corrupt', completed); - assert.deepEqual( - (await store.readEvents('tr-corrupt')).map((event) => event.id), - [events[0]!.id, 'corrupt-2', completed.id], - ); - } finally { - await rm(storageRoot, { recursive: true, force: true }); - } - }); - - test('rejects a durable event whose identity differs from its ledger', async () => { - const storageRoot = await mkdtemp(join(tmpdir(), 'maka-task-run-store-')); - try { - const store = await openFileTaskRunWriter(storageRoot); - await store.appendEvent('owned-run', completedEvents('owned-run')[0]!); - const [ledgerName] = (await readdir(join(storageRoot, 'task-runs'))).filter((name) => - name.endsWith('.jsonl'), - ); - assert.ok(ledgerName); - await appendFile( - join(storageRoot, 'task-runs', ledgerName), - `${JSON.stringify(completedEvents('other-run')[0])}\n`, - 'utf8', + await assert.rejects( + () => store.appendEvent('owned-run', completedEvents('other-run')[0]!), + /taskRunId mismatch/, ); - - await assert.rejects(() => store.readEvents('owned-run'), /contains event for other-run/); + await assert.rejects(() => lstat(join(storageRoot, 'runtime.sqlite')), { code: 'ENOENT' }); } finally { await rm(storageRoot, { recursive: true, force: true }); } }); }); -async function openFileTaskRunWriter(storageRoot: string) { +async function openSqliteTaskRunWriter(storageRoot: string) { const capability = await resolveStorageRoot({ path: storageRoot, kind: 'headless' }); return openHeadlessTaskRunWriter(createHeadlessRootLease(capability, 'write')); } -async function openFileTaskRunReader(storageRoot: string) { +async function openSqliteTaskRunReader(storageRoot: string) { const capability = await resolveStorageRoot({ path: storageRoot, kind: 'headless' }); return openHeadlessTaskRunReader(createHeadlessRootLease(capability, 'read')); } diff --git a/packages/headless/src/task-run-store.ts b/packages/headless/src/task-run-store.ts index 22845f3dc3..7dbff7488e 100644 --- a/packages/headless/src/task-run-store.ts +++ b/packages/headless/src/task-run-store.ts @@ -1,25 +1,14 @@ -import { Buffer } from 'node:buffer'; -import { - appendFile, - lstat, - mkdir, - open, - readFile, - readdir, - type FileHandle, -} from 'node:fs/promises'; -import { join } from 'node:path'; +import type { DatabaseSync } from 'node:sqlite'; import type { ExecutionLogCursor } from '@maka/core/execution-evidence'; +import { acquireOperationalStateDatabase, type OperationalStateDatabaseLease } from '@maka/storage'; import { assertStorageRootLease, runWithStorageRootLease, type StorageRootLease, } from '@maka/storage/root-authority'; import { chainWrite } from '@maka/storage/write-queue'; -import { unlock, waitForLock } from 'fs-native-extensions'; -import { publishFileExclusively } from './immutable-file.js'; import type { TaskEvent } from './task-contracts.js'; -import { isTaskRunLocator, taskRunLocator } from './task-run-identity.js'; +import { taskRunLocator } from './task-run-identity.js'; import { projectTaskRun, type TaskRunProjection } from './task-run-projection.js'; export interface TaskRunReader { @@ -48,8 +37,7 @@ export async function openHeadlessTaskRunReader( lease: StorageRootLease<'headless', 'read'>, ): Promise { await assertStorageRootLease(lease, 'headless', 'read'); - const store = new FileTaskRunStore(lease.canonicalPath); - return taskRunReaderFacade(store, (operation) => + return taskRunReaderFacade(new SqliteTaskRunStore(lease.canonicalPath), (operation) => runWithStorageRootLease(lease, 'headless', 'read', operation), ); } @@ -58,19 +46,18 @@ export async function openHeadlessTaskRunWriter( lease: StorageRootLease<'headless', 'write'>, ): Promise { await assertStorageRootLease(lease, 'headless', 'write'); - const store = new FileTaskRunStore(lease.canonicalPath); - return taskRunWriterFacade(store, (operation) => + return taskRunWriterFacade(new SqliteTaskRunStore(lease.canonicalPath), (operation) => runWithStorageRootLease(lease, 'headless', 'write', operation), ); } type RunTaskRunOperation = (operation: () => Promise) => Promise; -function taskRunReaderFacade(store: FileTaskRunStore, run: RunTaskRunOperation): TaskRunReader { +function taskRunReaderFacade(store: SqliteTaskRunStore, run: RunTaskRunOperation): TaskRunReader { return Object.freeze(taskRunReaderMethods(store, run)); } -function taskRunReaderMethods(store: FileTaskRunStore, run: RunTaskRunOperation): TaskRunReader { +function taskRunReaderMethods(store: SqliteTaskRunStore, run: RunTaskRunOperation): TaskRunReader { return { listTaskRunIds: () => run(() => store.listTaskRunIds()), readEventRecords: (taskRunId) => run(() => store.readEventRecords(taskRunId)), @@ -79,12 +66,12 @@ function taskRunReaderMethods(store: FileTaskRunStore, run: RunTaskRunOperation) }; } -function taskRunWriterFacade(store: FileTaskRunStore, run: RunTaskRunOperation): TaskRunWriter { - const writer: TaskRunWriter = { +function taskRunWriterFacade(store: SqliteTaskRunStore, run: RunTaskRunOperation): TaskRunWriter { + return Object.freeze({ ...taskRunReaderMethods(store, run), - appendEvent: (taskRunId, event) => run(() => store.appendEvent(taskRunId, event)), - }; - return Object.freeze(writer); + appendEvent: (taskRunId: string, event: TaskEvent) => + run(() => store.appendEvent(taskRunId, event)), + }); } class InMemoryTaskRunStore implements TaskRunWriter { @@ -100,10 +87,7 @@ class InMemoryTaskRunStore implements TaskRunWriter { } async appendEvent(taskRunId: string, event: TaskEvent): Promise { - if (event.taskRunId !== taskRunId) { - throw new Error(`taskRunId mismatch: append target ${taskRunId}, event ${event.taskRunId}`); - } - + assertTaskRunEventIdentity(taskRunId, event); await chainWrite(this.queues, taskRunId, async () => { const events = this.events.get(taskRunId) ?? []; events.push(event); @@ -131,38 +115,50 @@ class InMemoryTaskRunStore implements TaskRunWriter { } } -class FileTaskRunStore implements TaskRunWriter { - private readonly queues = new Map>(); - +class SqliteTaskRunStore implements TaskRunWriter { constructor(private readonly storageRoot: string) {} async appendEvent(taskRunId: string, event: TaskEvent): Promise { - if (event.taskRunId !== taskRunId) { - throw new Error(`taskRunId mismatch: append target ${taskRunId}, event ${event.taskRunId}`); - } - - const locator = taskRunLocator(taskRunId); - await chainWrite(this.queues, locator, () => - this.appendTaskRunEvent(locator, taskRunId, event), + assertTaskRunEventIdentity(taskRunId, event); + this.withDatabase((lease) => + lease.transaction('write', () => { + const row = lease.database + .prepare(` + SELECT COALESCE(MAX(sequence), -1) AS lastSequence + FROM headless_task_run_events + WHERE task_run_id = ? + `) + .get(taskRunId) as { lastSequence?: unknown }; + if ( + typeof row.lastSequence !== 'number' || + !Number.isSafeInteger(row.lastSequence) || + row.lastSequence < -1 + ) { + throw new Error(`Invalid TaskRun sequence for ${taskRunId}`); + } + lease.database + .prepare(` + INSERT INTO headless_task_run_events( + task_run_id, sequence, event_id, record_json + ) VALUES (?, ?, ?, ?) + `) + .run(taskRunId, row.lastSequence + 1, event.id, JSON.stringify(event)); + }), ); } async listTaskRunIds(): Promise { - let entries: string[]; - try { - entries = (await readdir(this.taskRunDir())).filter((name) => name.endsWith('.jsonl')).sort(); - } catch (error) { - if (isNotFound(error)) return []; - throw error; - } - const identities: string[] = []; - for (const entry of entries) { - const locator = taskRunLocatorFromFilename(entry); - const header = await readTaskRunLedgerHeader(join(this.taskRunDir(), entry)); - assertTaskRunLedgerIdentity(header, locator, entry); - identities.push(header.taskRunId); - } - return identities.sort(); + return this.withDatabase((lease) => + ( + lease.database + .prepare(` + SELECT DISTINCT task_run_id AS taskRunId + FROM headless_task_run_events + ORDER BY task_run_id + `) + .all() as Array<{ taskRunId: string }> + ).map((row) => row.taskRunId), + ); } async readEvents(taskRunId: string): Promise { @@ -170,266 +166,65 @@ class FileTaskRunStore implements TaskRunWriter { } async readEventRecords(taskRunId: string): Promise { - const path = this.taskRunPath(taskRunId); - let content: string; - try { - content = await readFile(path, 'utf8'); - } catch (error) { - if (isNotFound(error)) return []; - throw error; - } - - const lines = durableJsonlLines(content); - const header = parseTaskRunLedgerHeader(lines.shift(), path); - assertTaskRunLedgerIdentity(header, taskRunLocator(taskRunId), path); - if (header.taskRunId !== taskRunId) { - throw new Error(`TaskRun ledger identity does not match requested taskRunId ${taskRunId}`); - } - const records: TaskEventLedgerEntry[] = []; - for (let i = 0; i < lines.length; i += 1) { - const line = lines[i]; - if (!line) continue; - let event: TaskEvent; - try { - const parsed = JSON.parse(line) as unknown; - if ( - typeof parsed !== 'object' || - parsed === null || - typeof (parsed as { taskRunId?: unknown }).taskRunId !== 'string' - ) { - throw new Error('record is not a TaskEvent'); + taskRunLocator(taskRunId); + return this.withDatabase((lease) => + readTaskRunRows(lease.database, taskRunId).map((row) => { + const event = decodeTaskEvent(row.recordJson, taskRunId); + if (event.id !== row.eventId) { + throw new Error(`TaskRun ${taskRunId} event identity does not match SQLite metadata`); } - event = parsed as TaskEvent; - } catch (error) { - event = { - type: 'event_corrupt', - id: `corrupt-${i + 1}`, - taskRunId, - ts: 0, - raw: line, - error: errorMessage(error), + return { + event, + cursor: taskEventCursor(taskRunId, row.sequence, event.id), }; - } - if (event.taskRunId !== taskRunId) { - throw new Error(`TaskRun ledger ${path} contains event for ${String(event.taskRunId)}`); - } - records.push({ - event, - cursor: taskEventCursor(taskRunId, records.length, event.id), - }); - } - return records; + }), + ); } async project(taskRunId: string): Promise { return projectTaskRun(await this.readEvents(taskRunId), taskRunId); } - private taskRunDir(): string { - return join(this.storageRoot, 'task-runs'); - } - - private taskRunPath(taskRunId: string): string { - return join(this.taskRunDir(), `${taskRunLocator(taskRunId)}.jsonl`); - } - - private async appendTaskRunEvent( - locator: string, - taskRunId: string, - event: TaskEvent, - ): Promise { - const eventLine = `${JSON.stringify(event)}\n`; - await mkdir(this.taskRunDir(), { recursive: true }); - const path = join(this.taskRunDir(), `${locator}.jsonl`); - await withTaskRunLedgerLock(`${path}.lock`, async () => { - let header: TaskRunLedgerHeader; - try { - header = await readTaskRunLedgerHeader(path); - } catch (error) { - if (!isNotFound(error)) throw error; - const initialContent = `${JSON.stringify(taskRunLedgerHeader(taskRunId))}\n${eventLine}`; - if (await publishFileExclusively(path, initialContent)) return; - header = await readTaskRunLedgerHeader(path); - } - - assertTaskRunLedgerIdentity(header, locator, path); - if (header.taskRunId !== taskRunId) { - throw new Error(`TaskRun identity collision between ${header.taskRunId} and ${taskRunId}`); - } - await truncatePartialTaskRunTail(path); - await appendFile(path, eventLine, 'utf8'); - }); - } -} - -const TASK_RUN_LEDGER_SCHEMA_VERSION = 1 as const; -const TASK_RUN_LEDGER_TYPE = 'task_run_ledger' as const; - -interface TaskRunLedgerHeader { - schemaVersion: typeof TASK_RUN_LEDGER_SCHEMA_VERSION; - type: typeof TASK_RUN_LEDGER_TYPE; - taskRunId: string; -} - -function taskRunLocatorFromFilename(filename: string): string { - const locator = filename.slice(0, -'.jsonl'.length); - if (!isTaskRunLocator(locator)) { - throw new Error(`Invalid TaskRun ledger filename ${filename}`); - } - return locator; -} - -function taskRunLedgerHeader(taskRunId: string): TaskRunLedgerHeader { - taskRunLocator(taskRunId); - return { - schemaVersion: TASK_RUN_LEDGER_SCHEMA_VERSION, - type: TASK_RUN_LEDGER_TYPE, - taskRunId, - }; -} - -function parseTaskRunLedgerHeader(line: string | undefined, source: string): TaskRunLedgerHeader { - if (!line) throw new Error(`TaskRun ledger ${source} has no durable header`); - let record: unknown; - try { - record = JSON.parse(line); - } catch (error) { - throw new Error(`TaskRun ledger ${source} has an invalid header: ${errorMessage(error)}`); - } - if ( - typeof record !== 'object' || - record === null || - (record as { schemaVersion?: unknown }).schemaVersion !== TASK_RUN_LEDGER_SCHEMA_VERSION || - (record as { type?: unknown }).type !== TASK_RUN_LEDGER_TYPE || - typeof (record as { taskRunId?: unknown }).taskRunId !== 'string' - ) { - throw new Error(`TaskRun ledger ${source} has an invalid header`); - } - const header = record as TaskRunLedgerHeader; - taskRunLocator(header.taskRunId); - return header; -} - -function assertTaskRunLedgerIdentity( - header: TaskRunLedgerHeader, - expectedLocator: string, - source: string, -): void { - if (taskRunLocator(header.taskRunId) !== expectedLocator) { - throw new Error(`TaskRun ledger ${source} does not match its identity locator`); - } -} - -async function withTaskRunLedgerLock(lockPath: string, operation: () => Promise): Promise { - const handle = await open(lockPath, 'a+', 0o600); - try { - await assertStableLockArtifact(handle, lockPath); - await handle.chmod(0o600); - await waitForLock(handle.fd); + private withDatabase(operation: (lease: OperationalStateDatabaseLease) => T): T { + const lease = acquireOperationalStateDatabase(this.storageRoot); try { - await assertStableLockArtifact(handle, lockPath); - return await operation(); + return operation(lease); } finally { - releaseLock(handle); + lease.close(); } - } finally { - await handle.close(); } } -async function assertStableLockArtifact(handle: FileHandle, path: string): Promise { - const [handleStats, pathStats] = await Promise.all([ - handle.stat({ bigint: true }), - lstat(path, { bigint: true }), - ]); - if ( - !handleStats.isFile() || - !pathStats.isFile() || - handleStats.dev !== pathStats.dev || - handleStats.ino !== pathStats.ino - ) { - throw new Error(`TaskRun lock path is not one stable regular file: ${path}`); - } -} - -function releaseLock(handle: FileHandle): void { - try { - unlock(handle.fd); - } catch { - // Closing the file handle is the authoritative release path. - } +interface TaskRunEventRow { + sequence: number; + eventId: string; + recordJson: string; } -async function readTaskRunLedgerHeader(path: string): Promise { - return parseTaskRunLedgerHeader(await readFirstDurableLine(path), path); +function readTaskRunRows(database: DatabaseSync, taskRunId: string): TaskRunEventRow[] { + return database + .prepare(` + SELECT sequence, event_id AS eventId, record_json AS recordJson + FROM headless_task_run_events + WHERE task_run_id = ? + ORDER BY sequence + `) + .all(taskRunId) as unknown as TaskRunEventRow[]; } -async function truncatePartialTaskRunTail(path: string): Promise { - const handle = await open(path, 'r+'); - try { - const { size } = await handle.stat(); - let position = size; - while (position > 0) { - const length = Math.min(position, 4096); - position -= length; - const buffer = Buffer.allocUnsafe(length); - let bytesRead = 0; - while (bytesRead < length) { - const read = await handle.read(buffer, bytesRead, length - bytesRead, position + bytesRead); - if (read.bytesRead === 0) { - throw new Error(`TaskRun ledger ${path} changed while repairing its tail`); - } - bytesRead += read.bytesRead; - } - const newline = buffer.lastIndexOf(0x0a); - if (newline === -1) continue; - const durableSize = position + newline + 1; - if (durableSize < size) { - await handle.truncate(durableSize); - await handle.sync(); - } - return; - } - throw new Error(`TaskRun ledger ${path} has no durable record`); - } finally { - await handle.close(); - } +function decodeTaskEvent(recordJson: string, taskRunId: string): TaskEvent { + const event = JSON.parse(recordJson) as TaskEvent; + assertTaskRunEventIdentity(taskRunId, event); + return event; } -async function readFirstDurableLine(path: string): Promise { - const handle = await open(path, 'r'); - const chunks: Buffer[] = []; - let position = 0; - try { - while (true) { - const buffer = Buffer.allocUnsafe(4096); - const { bytesRead } = await handle.read(buffer, 0, buffer.length, position); - if (bytesRead === 0) return undefined; - const bytes = buffer.subarray(0, bytesRead); - const newline = bytes.indexOf(0x0a); - chunks.push(newline === -1 ? bytes : bytes.subarray(0, newline)); - if (newline !== -1) return Buffer.concat(chunks).toString('utf8'); - position += bytesRead; - } - } finally { - await handle.close(); +function assertTaskRunEventIdentity(taskRunId: string, event: TaskEvent): void { + taskRunLocator(taskRunId); + if (event.taskRunId !== taskRunId) { + throw new Error(`taskRunId mismatch: append target ${taskRunId}, event ${event.taskRunId}`); } } -function durableJsonlLines(content: string): string[] { - return content.endsWith('\n') ? content.split('\n') : content.split('\n').slice(0, -1); -} - function taskEventCursor(taskRunId: string, sequence: number, eventId: string): ExecutionLogCursor { return { ledger: 'task_event', streamId: taskRunId, sequence, eventId }; } - -function isNotFound(error: unknown): boolean { - return ( - typeof error === 'object' && error !== null && (error as { code?: string }).code === 'ENOENT' - ); -} - -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); -} diff --git a/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts b/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts index 8c6bb8bf4a..6a95802c33 100644 --- a/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts +++ b/packages/runtime-host/src/__tests__/execution-host-recovery.test.ts @@ -1,20 +1,11 @@ import assert from 'node:assert/strict'; import { fork, type ChildProcess } from 'node:child_process'; import { randomUUID } from 'node:crypto'; -import { - appendFile, - chmod, - mkdir, - mkdtemp, - readFile, - readdir, - rm, - writeFile, -} from 'node:fs/promises'; +import { mkdtemp, readdir, rm } from 'node:fs/promises'; import { createServer, type Server } from 'node:http'; import { connect, type Socket } from 'node:net'; import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; +import { join } from 'node:path'; import { test } from 'node:test'; import { canonicalToolArgsHash, TOOL_BOUNDARY_PROTOCOL_V1 } from '@maka/core'; import type { AgentRunHeader } from '@maka/core/agent-run'; @@ -99,131 +90,6 @@ test('startup recovery rejects claimed graph Run lineage drift', async () => { }); }); -test('startup recovery imports past a truncated legacy RuntimeEvent tail without rewriting it', async () => { - await withExecutionRoot(async (fixture) => { - const turnId = randomUUID(); - const { runId } = await fixture.seedRunWithoutUserMessage( - turnId, - 'recover after a partial RuntimeEvent write', - ); - const runtimeEventsPath = fixture.runtimeEventsPath(runId); - await mkdir(dirname(runtimeEventsPath), { recursive: true }); - await writeFile(runtimeEventsPath, '{"id":"truncated"', 'utf8'); - - const host = await fixture.startHost(); - const client = await connectClient(fixture.root, 'tui'); - const recovered = await client.queryTurn({ - sessionId: fixture.sessionId, - turnId, - }); - assert.equal(recovered.status, 'failed'); - if (recovered.status === 'failed') { - assert.equal(recovered.failureClass, 'app_restarted'); - } - await client.close(); - await fixture.stopHost(host); - - assert.equal(await readFile(runtimeEventsPath, 'utf8'), '{"id":"truncated"'); - const ledger = await fixture.readTurn(turnId); - assert.equal(ledger.terminalEvents.length, 1); - }); -}); - -test('startup recovery fails closed on a complete malformed RuntimeEvent record', async () => { - await withExecutionRoot(async (fixture) => { - const turnId = randomUUID(); - const { runId } = await fixture.seedRunWithoutUserMessage( - turnId, - 'do not recover across durable corruption', - ); - const runtimeEventsPath = fixture.runtimeEventsPath(runId); - const malformed = '{"id":"malformed"\n'; - await mkdir(dirname(runtimeEventsPath), { recursive: true }); - await writeFile(runtimeEventsPath, malformed, 'utf8'); - - await fixture.expectHostStartupFailure(); - assert.equal(await readFile(runtimeEventsPath, 'utf8'), malformed); - await fixture.assertOwnerAvailable(); - }); -}); - -test('startup recovery fails closed on a complete malformed Session record', async () => { - await withExecutionRoot(async (fixture) => { - await fixture.seedRunWithoutUserMessage( - randomUUID(), - 'do not rewrite durable Session corruption', - ); - const sessionPath = fixture.sessionPath(); - const malformed = '{"type":"user"\n'; - await appendFile(sessionPath, malformed, 'utf8'); - const expected = await readFile(sessionPath, 'utf8'); - - await fixture.expectHostStartupFailure(); - assert.equal(await readFile(sessionPath, 'utf8'), expected); - await fixture.assertOwnerAvailable(); - }); -}); - -test('startup recovery fails closed on a complete malformed AgentRun record', async () => { - await withExecutionRoot(async (fixture) => { - const { runId } = await fixture.seedRunWithoutUserMessage( - randomUUID(), - 'do not recover across durable AgentRun corruption', - ); - const eventsPath = fixture.eventsPath(runId); - const malformed = '{"type":"run_started"\n'; - await mkdir(dirname(eventsPath), { recursive: true }); - await writeFile(eventsPath, malformed, 'utf8'); - - await fixture.expectHostStartupFailure(); - assert.equal(await readFile(eventsPath, 'utf8'), malformed); - await fixture.assertOwnerAvailable(); - }); -}); - -test('a pre-start durability failure rejects turn.start and drains the Host', { - skip: - process.platform === 'win32' || process.getuid?.() === 0 ? 'POSIX file-permission gate' : false, -}, async () => { - await withExecutionRoot(async (fixture) => { - const host = await fixture.startHost(); - const client = await connectClient(fixture.root, 'desktop'); - const turnId = randomUUID(); - const sessionPath = fixture.sessionPath(); - await chmod(sessionPath, 0o400); - try { - await assert.rejects( - () => - client.startTurn({ - sessionId: fixture.sessionId, - turnId, - content: { text: 'fail before the durable start barrier' }, - }), - operationError('internal_failure'), - ); - await client.closed; - await fixture.waitForHostExit(host); - } finally { - await chmod(sessionPath, 0o600); - } - - const successor = await fixture.startHost(); - const observer = await connectClient(fixture.root, 'tui'); - const recovered = await observer.queryTurn({ - sessionId: fixture.sessionId, - turnId, - }); - assert.equal(recovered.status, 'failed'); - await observer.close(); - await fixture.stopHost(successor); - - const ledger = await fixture.readTurn(turnId); - assert.equal(ledger.runs.length, 1); - assert.equal(ledger.userMessages.length, 1); - assert.equal(ledger.terminalEvents.length, 1); - }); -}); - test('retry after a discarded turn.start response reuses the durable semantic admission', async () => { await withExecutionRoot(async (fixture) => { const host = await fixture.startHost(); diff --git a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts index 1f6edae898..e34aef20ab 100644 --- a/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts +++ b/packages/runtime-host/src/__tests__/fixtures/execution-host-suite.ts @@ -105,18 +105,6 @@ export class ExecutionFixture { readonly sessionId: string, ) {} - sessionPath(): string { - return join(this.root, 'sessions', this.sessionId, 'session.jsonl'); - } - - runtimeEventsPath(runId: string): string { - return join(this.root, 'sessions', this.sessionId, 'runs', runId, 'runtime-events.jsonl'); - } - - eventsPath(runId: string): string { - return join(this.root, 'sessions', this.sessionId, 'runs', runId, 'events.jsonl'); - } - async seedPendingChildAdmission( kind: | 'linked_child_initial' diff --git a/packages/runtime-host/src/__tests__/root-admission-owner.test.ts b/packages/runtime-host/src/__tests__/root-admission-owner.test.ts index dcae4f91e2..12428b3b28 100644 --- a/packages/runtime-host/src/__tests__/root-admission-owner.test.ts +++ b/packages/runtime-host/src/__tests__/root-admission-owner.test.ts @@ -8,7 +8,7 @@ import { type RootTurnAdmissionStore, type RootTurnSourceMessage, } from '@maka/storage'; -import { createLegacyAgentRunStoreForTest } from '@maka/storage/legacy-execution-test-support'; +import { createSqliteAgentRunStore } from '@maka/storage'; import { RootAdmissionOwner } from '../server/root-admission-owner.js'; import { SessionAdmissionGate } from '../server/session-admission-gate.js'; @@ -379,11 +379,11 @@ function mutableAdmission(): RootTurnAdmission { } async function withStore( - run: (store: ReturnType) => Promise, + run: (store: ReturnType) => Promise, ): Promise { const root = await mkdtemp(join(tmpdir(), 'maka-root-admission-owner-')); try { - await run(createLegacyAgentRunStoreForTest(root)); + await run(createSqliteAgentRunStore(root)); } finally { await rm(root, { recursive: true, force: true }); } diff --git a/packages/runtime-host/src/__tests__/session-retirement-coordinator.test.ts b/packages/runtime-host/src/__tests__/session-retirement-coordinator.test.ts index e36a47afdd..33353a1c9a 100644 --- a/packages/runtime-host/src/__tests__/session-retirement-coordinator.test.ts +++ b/packages/runtime-host/src/__tests__/session-retirement-coordinator.test.ts @@ -172,7 +172,7 @@ describe('Host Session retirement coordinator', () => { }); }); - test('recovers graph operators orphaned by legacy root-only retirement', async () => { + test('recovers graph operators orphaned by an interrupted retirement', async () => { await withHarness(async (harness) => { const childSessionId = await createClosedGraphOperator(harness, harness.rootId, 'a'); const database = new DatabaseSync(join(harness.workspaceRoot, 'runtime.sqlite')); @@ -199,17 +199,13 @@ describe('Host Session retirement coordinator', () => { } finally { database.close(); } - for (const sessionId of harness.familyIds) { - await harness.store.purgeRemovedSessionTranscript(sessionId); - } - await harness.coordinator.recover(); await waitFor( async () => (await harness.store.probeSessionRemoval(childSessionId)).kind === 'removed' && (await harness.store.listPendingSessionRetirementCleanupIds()).length === 0, - 'Legacy Agent Graph retirement did not converge', + 'Agent Graph retirement did not converge', ); const graphId = agentGraphIdForRootSession(harness.rootId); assert.deepEqual(await harness.graphStore.listAgentGraphScheduleUpdates(graphId), []); @@ -218,7 +214,7 @@ describe('Host Session retirement coordinator', () => { }); }); - test('recovers legacy graph sidecars without operator provisions', async () => { + test('recovers graph sidecars without operator provisions', async () => { await withHarness(async (harness) => { const projectionGraphId = agentGraphIdForRootSession(harness.rootId); const finishedGraphId = agentGraphIdForRootSession(harness.revisionId); @@ -274,15 +270,11 @@ describe('Host Session retirement coordinator', () => { } finally { database.close(); } - for (const sessionId of harness.familyIds) { - await harness.store.purgeRemovedSessionTranscript(sessionId); - } - await harness.coordinator.recover(); await waitFor( async () => (await harness.store.listPendingSessionRetirementCleanupIds()).length === 0, - 'Legacy Agent Graph sidecar cleanup did not converge', + 'Agent Graph sidecar cleanup did not converge', ); assert.equal( await harness.graphStore.readAgentGraphClientProjection(projectionGraphId), @@ -745,8 +737,6 @@ async function withHarness( store.reconcileOrphanedAgentGraphRetirements(), listPendingSessionRetirementCleanupIds: (sessionId) => store.listPendingSessionRetirementCleanupIds(sessionId), - purgeRemovedSessionTranscript: (sessionId) => - store.purgeRemovedSessionTranscript(sessionId), completeSessionRetirementCleanup: (sessionId) => store.completeSessionRetirementCleanup(sessionId), setSessionsLifecycleVersioned: (sessions, state) => diff --git a/packages/runtime-host/src/server/session-retirement-coordinator.ts b/packages/runtime-host/src/server/session-retirement-coordinator.ts index eb28f87a76..470fd9c4c7 100644 --- a/packages/runtime-host/src/server/session-retirement-coordinator.ts +++ b/packages/runtime-host/src/server/session-retirement-coordinator.ts @@ -46,7 +46,6 @@ type RetirementStores = Pick< | 'readHeaderRecordSnapshot' | 'reconcileOrphanedAgentGraphRetirements' | 'listPendingSessionRetirementCleanupIds' - | 'purgeRemovedSessionTranscript' | 'completeSessionRetirementCleanup' | 'removeSessionsVersioned' | 'setSessionsLifecycleVersioned' @@ -461,7 +460,6 @@ export class HostSessionRetirementCoordinator { async #cleanupRetiredSession(sessionId: string): Promise { const worktree = this.#retiredWorktrees.get(sessionId); const outcomes = await Promise.allSettled([ - this.#stores.purgeRemovedSessionTranscript(sessionId), purgeSessionSidecars( { artifacts: this.#artifacts, diff --git a/packages/runtime/src/__tests__/agent-run-steering-recovery.test.ts b/packages/runtime/src/__tests__/agent-run-steering-recovery.test.ts index b5fc01b63d..130a1e9ad4 100644 --- a/packages/runtime/src/__tests__/agent-run-steering-recovery.test.ts +++ b/packages/runtime/src/__tests__/agent-run-steering-recovery.test.ts @@ -5,18 +5,18 @@ import assert from 'node:assert/strict'; import { test } from 'node:test'; import type { AgentRunHeader, RuntimeEvent } from '@maka/core'; import type { SessionEvent } from '@maka/core/events'; -import { createLegacySessionStoreForTest } from '@maka/storage/legacy-storage-test-support'; import { - createLegacyAgentRunStoreForTest, - createLegacyRuntimeEventStoreForTest, -} from '@maka/storage/legacy-execution-test-support'; + createSessionStore, + createSqliteAgentRunStore, + createWorkspaceRuntimeStore, +} from '@maka/storage'; import { AgentRun } from '../agent-run.js'; import { buildStatusPatch } from '../session-projection-helpers.js'; test('does not re-append atomically committed tool facts through the generic event lane', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-agent-run-atomic-tool-boundary-')); try { - const store = createLegacySessionStoreForTest(root); + const store = createSessionStore(root); const session = await store.create({ cwd: '/tmp/cwd', backend: 'fake', @@ -24,7 +24,7 @@ test('does not re-append atomically committed tool facts through the generic eve model: 'fake-model', permissionMode: 'ask', }); - const runtimeEventStore = createLegacyRuntimeEventStoreForTest(root); + const runtimeEventStore = createWorkspaceRuntimeStore(root); const runId = 'run-atomic-tool'; const turnId = 'turn-atomic-tool'; const run = new AgentRun({ @@ -91,7 +91,7 @@ test('does not re-append atomically committed tool facts through the generic eve test('acks a steering event whose canonical append preceded proof publication failure', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-agent-run-steering-recovery-')); try { - const store = createLegacySessionStoreForTest(root); + const store = createSessionStore(root); const session = await store.create({ cwd: '/tmp/cwd', backend: 'fake', @@ -99,8 +99,8 @@ test('acks a steering event whose canonical append preceded proof publication fa model: 'fake-model', permissionMode: 'ask', }); - const runStore = createLegacyAgentRunStoreForTest(root); - const runtimeEventStore = createLegacyRuntimeEventStoreForTest(root); + const runStore = createSqliteAgentRunStore(root); + const runtimeEventStore = createWorkspaceRuntimeStore(root); const runId = 'run-1'; const turnId = 'turn-1'; await runStore.createRun(makeRunHeader(session.id, runId, turnId)); @@ -154,7 +154,7 @@ test('acks a steering event whose canonical append preceded proof publication fa await chmod(proofDirectory, 0o700); await rm(proofDirectory, { recursive: true }); - const recovered = createLegacyRuntimeEventStoreForTest(root); + const recovered = createWorkspaceRuntimeStore(root); await recovered.repairImmutableSteeringMessageProofsForRecovery(session.id); assert.deepEqual(await recovered.readImmutableRuntimeEvents(session.id, runId), [runtimeEvent]); assert.deepEqual( @@ -169,7 +169,7 @@ test('acks a steering event whose canonical append preceded proof publication fa test('awaits canonical Run status persistence before accepting an interaction resume', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-agent-run-status-barrier-')); try { - const store = createLegacySessionStoreForTest(root); + const store = createSessionStore(root); const session = await store.create({ cwd: '/tmp/cwd', backend: 'fake', @@ -177,8 +177,8 @@ test('awaits canonical Run status persistence before accepting an interaction re model: 'fake-model', permissionMode: 'ask', }); - const runStore = createLegacyAgentRunStoreForTest(root); - const runtimeEventStore = createLegacyRuntimeEventStoreForTest(root); + const runStore = createSqliteAgentRunStore(root); + const runtimeEventStore = createWorkspaceRuntimeStore(root); const runId = 'run-status-barrier'; const turnId = 'turn-status-barrier'; await store.updateHeader(session.id, buildStatusPatch('waiting_for_user', 1)); @@ -272,7 +272,7 @@ test('awaits canonical Run status persistence before accepting an interaction re test('required interaction resume recovers a failed best-effort Run Store latch through terminal commit', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-agent-run-status-latch-')); try { - const store = createLegacySessionStoreForTest(root); + const store = createSessionStore(root); const session = await store.create({ cwd: '/tmp/cwd', backend: 'fake', @@ -280,8 +280,8 @@ test('required interaction resume recovers a failed best-effort Run Store latch model: 'fake-model', permissionMode: 'ask', }); - const runStore = createLegacyAgentRunStoreForTest(root); - const runtimeEventStore = createLegacyRuntimeEventStoreForTest(root); + const runStore = createSqliteAgentRunStore(root); + const runtimeEventStore = createWorkspaceRuntimeStore(root); const runId = 'run-status-latch'; const turnId = 'turn-status-latch'; await store.updateHeader(session.id, buildStatusPatch('waiting_for_user', 1)); @@ -387,7 +387,7 @@ test('required interaction resume recovers a failed best-effort Run Store latch test('required interaction resume stays fail-closed until a later required write succeeds', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-agent-run-status-latch-failure-')); try { - const store = createLegacySessionStoreForTest(root); + const store = createSessionStore(root); const session = await store.create({ cwd: '/tmp/cwd', backend: 'fake', @@ -395,8 +395,8 @@ test('required interaction resume stays fail-closed until a later required write model: 'fake-model', permissionMode: 'ask', }); - const runStore = createLegacyAgentRunStoreForTest(root); - const runtimeEventStore = createLegacyRuntimeEventStoreForTest(root); + const runStore = createSqliteAgentRunStore(root); + const runtimeEventStore = createWorkspaceRuntimeStore(root); const runId = 'run-status-latch-failure'; const turnId = 'turn-status-latch-failure'; await store.updateHeader(session.id, buildStatusPatch('waiting_for_user', 1)); diff --git a/packages/runtime/src/__tests__/context-diagnostics.test.ts b/packages/runtime/src/__tests__/context-diagnostics.test.ts index e24c8e1b4f..a9cae07406 100644 --- a/packages/runtime/src/__tests__/context-diagnostics.test.ts +++ b/packages/runtime/src/__tests__/context-diagnostics.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; import type { AgentRunEvent, AgentRunHeader, AgentRunStore } from '@maka/core'; -import { createLegacyAgentRunStoreForTest } from '@maka/storage/legacy-execution-test-support'; +import { createSqliteAgentRunStore } from '@maka/storage'; import { readLatestContextDiagnostics } from '../context-diagnostics.js'; test('reads the latest completed provider request instead of a later failed attempt', async () => { @@ -159,7 +159,7 @@ test('reports the latest history compaction that preceded the displayed request' test('reads the same diagnostics after reopening the durable run ledger', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-context-diagnostics-')); try { - const writer = createLegacyAgentRunStoreForTest(root); + const writer = createSqliteAgentRunStore(root); const header = runHeader('run-1', 1); await writer.createRun(header); await writer.appendEvent( @@ -169,7 +169,7 @@ test('reads the same diagnostics after reopening the durable run ledger', async ); const diagnostics = await readLatestContextDiagnostics( - createLegacyAgentRunStoreForTest(root), + createSqliteAgentRunStore(root), 'session-1', ); diff --git a/packages/runtime/src/__tests__/conversation-copy.test.ts b/packages/runtime/src/__tests__/conversation-copy.test.ts index d4da9cbd49..497035d87a 100644 --- a/packages/runtime/src/__tests__/conversation-copy.test.ts +++ b/packages/runtime/src/__tests__/conversation-copy.test.ts @@ -15,11 +15,11 @@ import { decodeCanonicalToolResultContent, isSessionInlineRun, } from '@maka/core'; -import { createSqliteAgentRunStore, createSqliteRuntimeStore } from '@maka/storage'; import { - createLegacyAgentRunStoreForTest, - createLegacyRuntimeEventStoreForTest, -} from '@maka/storage/legacy-execution-test-support'; + createSqliteAgentRunStore, + createSqliteRuntimeStore, + createWorkspaceRuntimeStore, +} from '@maka/storage'; import { archivedToolResultContainsConversationOwnedReferences, cloneConversationRuntimeLedger, @@ -411,8 +411,8 @@ test('conversation copy turn closure includes legacy children but excludes later test('conversation copy rejects a retained AgentRun without RuntimeEvent facts', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-conversation-missing-runtime-copy-')); try { - const runStore = createLegacyAgentRunStoreForTest(root); - const runtimeEventStore = createLegacyRuntimeEventStoreForTest(root); + const runStore = createSqliteAgentRunStore(root); + const runtimeEventStore = createWorkspaceRuntimeStore(root); const rootRun = agentRunHeader({ runId: 'run-root', invocationId: 'invocation-root', @@ -692,8 +692,8 @@ test('conversation copy rewrites a complete tool recovery bundle atomically', as test('conversation copy validates operational events before persisting target ledgers', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-conversation-copy-preflight-')); try { - const runStore = createLegacyAgentRunStoreForTest(root); - const runtimeEventStore = createLegacyRuntimeEventStoreForTest(root); + const runStore = createSqliteAgentRunStore(root); + const runtimeEventStore = createWorkspaceRuntimeStore(root); await runStore.createRun( agentRunHeader({ runId: 'run-source', @@ -762,8 +762,8 @@ test('conversation copy validates operational events before persisting target le test('conversation copy clones one terminal Runtime ledger with new owned identities', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-conversation-runtime-copy-')); try { - const runStore = createLegacyAgentRunStoreForTest(root); - const runtimeEventStore = createLegacyRuntimeEventStoreForTest(root); + const runStore = createSqliteAgentRunStore(root); + const runtimeEventStore = createWorkspaceRuntimeStore(root); const sourceRun: AgentRunHeader = { runId: 'run-source', invocationId: 'invocation-source', @@ -1129,8 +1129,8 @@ test('conversation copy clones one terminal Runtime ledger with new owned identi test('conversation copy rebuilds an inline checkpoint without legacy child events in its prefix', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-conversation-checkpoint-copy-')); try { - const runStore = createLegacyAgentRunStoreForTest(root); - const runtimeEventStore = createLegacyRuntimeEventStoreForTest(root); + const runStore = createSqliteAgentRunStore(root); + const runtimeEventStore = createWorkspaceRuntimeStore(root); const firstRun = agentRunHeader({ runId: 'run-1', invocationId: 'invocation-1', @@ -1305,8 +1305,8 @@ test('conversation copy rebuilds an inline checkpoint without legacy child event test('conversation copy rebuilds a resumed child checkpoint over its child run chain', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-conversation-child-checkpoint-copy-')); try { - const runStore = createLegacyAgentRunStoreForTest(root); - const runtimeEventStore = createLegacyRuntimeEventStoreForTest(root); + const runStore = createSqliteAgentRunStore(root); + const runtimeEventStore = createWorkspaceRuntimeStore(root); const rootRun = agentRunHeader({ runId: 'run-root', invocationId: 'invocation-root', diff --git a/packages/runtime/src/__tests__/deep-research-tools.test.ts b/packages/runtime/src/__tests__/deep-research-tools.test.ts index 96243a664e..3c9a1ff090 100644 --- a/packages/runtime/src/__tests__/deep-research-tools.test.ts +++ b/packages/runtime/src/__tests__/deep-research-tools.test.ts @@ -5,7 +5,7 @@ import { join } from 'node:path'; import { describe, it } from 'node:test'; import { z } from 'zod'; import type { ArtifactRecord, DeepResearchRun } from '@maka/core'; -import { createLegacyDeepResearchStoreForTest } from '@maka/storage/legacy-storage-test-support'; +import { createSqliteDeepResearchStore } from '@maka/storage'; import { DEEP_RESEARCH_CHECKPOINT_TOOL_NAME, DEEP_RESEARCH_COMPLETE_TOOL_NAME, @@ -108,7 +108,7 @@ async function withTempRoot(fn: (root: string) => Promise): Promise describe('Deep Research runtime tools', () => { it('exposes eight Maka-owned local workspace tools', () => { const tools = buildDeepResearchTools({ - store: createLegacyDeepResearchStoreForTest('/tmp/maka-unused-deep-research'), + store: createSqliteDeepResearchStore('/tmp/maka-unused-deep-research'), artifactStore: new FakeArtifactStore(), }); assert.deepEqual( @@ -130,7 +130,7 @@ describe('Deep Research runtime tools', () => { await withTempRoot(async (root) => { const artifactStore = new FakeArtifactStore(); const notifications: string[] = []; - const store = createLegacyDeepResearchStoreForTest(root); + const store = createSqliteDeepResearchStore(root); const tools = buildDeepResearchTools({ store, artifactStore, @@ -450,7 +450,7 @@ describe('Deep Research runtime tools', () => { it('rejects untraceable derived artifacts at the schema boundary', () => { const tools = buildDeepResearchTools({ - store: createLegacyDeepResearchStoreForTest('/tmp/maka-unused-deep-research-2'), + store: createSqliteDeepResearchStore('/tmp/maka-unused-deep-research-2'), artifactStore: new FakeArtifactStore(), }); const save = findTool(tools, DEEP_RESEARCH_SAVE_ARTIFACT_TOOL_NAME); @@ -527,7 +527,7 @@ describe('Deep Research runtime tools', () => { await withTempRoot(async (root) => { const artifactStore = new FakeArtifactStore(); const tools = buildDeepResearchTools({ - store: createLegacyDeepResearchStoreForTest(root), + store: createSqliteDeepResearchStore(root), artifactStore, }); await execute( diff --git a/packages/runtime/src/__tests__/execution-inspect.test.ts b/packages/runtime/src/__tests__/execution-inspect.test.ts index c3b479c638..a9a791643c 100644 --- a/packages/runtime/src/__tests__/execution-inspect.test.ts +++ b/packages/runtime/src/__tests__/execution-inspect.test.ts @@ -5,10 +5,7 @@ import { join } from 'node:path'; import { describe, test } from 'node:test'; import type { AgentRunEvent, AgentRunHeader, RuntimeEvent } from '@maka/core'; import { createSessionStore } from '@maka/storage'; -import { - createLegacyAgentRunStoreForTest, - createLegacyRuntimeEventStoreForTest, -} from '@maka/storage/legacy-execution-test-support'; +import { createSqliteAgentRunStore, createWorkspaceRuntimeStore } from '@maka/storage'; import { inspectAgentRunDocument, inspectSessionDocument, @@ -20,8 +17,8 @@ describe('versioned execution inspect documents', () => { test('reports unknown tool outcomes without copying Runtime payloads', async () => { await withWorkspace(async (root) => { const sessionStore = createSessionStore(root); - const runStore = createLegacyAgentRunStoreForTest(root); - const runtimeStore = createLegacyRuntimeEventStoreForTest(root); + const runStore = createSqliteAgentRunStore(root); + const runtimeStore = createWorkspaceRuntimeStore(root); const session = await sessionStore.create({ cwd: '/tmp/workspace', backend: 'fake', @@ -88,8 +85,8 @@ describe('versioned execution inspect documents', () => { test('projects a Session as bounded AgentRun documents without reading messages', async () => { await withWorkspace(async (root) => { const sessionStore = createSessionStore(root); - const runStore = createLegacyAgentRunStoreForTest(root); - const runtimeStore = createLegacyRuntimeEventStoreForTest(root); + const runStore = createSqliteAgentRunStore(root); + const runtimeStore = createWorkspaceRuntimeStore(root); const session = await sessionStore.create({ cwd: '/tmp/workspace', name: 'Inspectable session', diff --git a/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts b/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts index 5748e4a031..d9ef5c4bfe 100644 --- a/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts +++ b/packages/runtime/src/__tests__/runtime-continuation-crash.test.ts @@ -9,7 +9,7 @@ import { describe, test } from 'node:test'; import type { AgentRunHeader, RuntimeEvent } from '@maka/core'; import { createSessionStore, createSqliteRuntimeStore } from '@maka/storage'; -import { createLegacyAgentRunStoreForTest } from '@maka/storage/legacy-execution-test-support'; +import { createSqliteAgentRunStore } from '@maka/storage'; import { type RuntimeContinuationFailpoint } from '../agent-run.js'; import { BackendRegistry, SessionManager } from '../session-manager.js'; @@ -38,7 +38,7 @@ if (process.env[CRASH_CHILD_ENV] === '1') { await crashContinuationAt(workspaceRoot, failpoint); const store = createSessionStore(workspaceRoot); - const runStore = createLegacyAgentRunStoreForTest(workspaceRoot); + const runStore = createSqliteAgentRunStore(workspaceRoot); const runtimeEventStore = createCrashRuntimeStore(workspaceRoot); const [session] = await store.list(); assert.ok(session, `${failpoint} did not persist a session`); @@ -123,7 +123,7 @@ async function runCrashChild(): Promise { 'MAKA_RUNTIME_CONTINUATION_FAILPOINT', ) as RuntimeContinuationFailpoint; const store = createSessionStore(workspaceRoot); - const runStore = createLegacyAgentRunStoreForTest(workspaceRoot); + const runStore = createSqliteAgentRunStore(workspaceRoot); const runtimeEventStore = createCrashRuntimeStore(workspaceRoot); const backends = new BackendRegistry(); backends.register( @@ -216,7 +216,7 @@ function createManager(workspaceRoot: string): { sessionStore: ReturnType; } { const store = createSessionStore(workspaceRoot); - const runStore = createLegacyAgentRunStoreForTest(workspaceRoot); + const runStore = createSqliteAgentRunStore(workspaceRoot); const runtimeEventStore = createCrashRuntimeStore(workspaceRoot); const backends = new BackendRegistry(); backends.register( diff --git a/packages/runtime/src/__tests__/runtime-read-model-persisted-compat.test.ts b/packages/runtime/src/__tests__/runtime-read-model-persisted-compat.test.ts deleted file mode 100644 index f1cbe0218f..0000000000 --- a/packages/runtime/src/__tests__/runtime-read-model-persisted-compat.test.ts +++ /dev/null @@ -1,102 +0,0 @@ -import assert from 'node:assert/strict'; -import { copyFile, mkdtemp, readFile, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { test } from 'node:test'; -import type { AgentRunHeader } from '@maka/core'; -import { - createLegacyAgentRunStoreForTest, - createLegacyRuntimeEventStoreForTest, -} from '@maka/storage/legacy-execution-test-support'; -import { RuntimeReadModel } from '../runtime-read-model.js'; -import { isHardRuntimeEventReadModelDiagnostic } from '../runtime-event-read-model.js'; - -const sessionId = 'legacy-runtime-event-session'; -const runId = 'legacy-runtime-event-run'; -const turnId = 'legacy-runtime-event-turn'; -const invocationId = 'legacy-runtime-event-invocation'; -const fixtureUrl = new URL( - '../../src/__tests__/fixtures/legacy-runtime-event-subagent-waiting-permission.jsonl', - import.meta.url, -); - -const header: AgentRunHeader = { - runId, - invocationId, - sessionId, - turnId, - status: 'completed', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/tmp/legacy-runtime-event-workspace', - permissionMode: 'ask', - createdAt: 1_800_000_000_000, - updatedAt: 1_800_000_000_003, - completedAt: 1_800_000_000_003, -}; - -test('reopens a persisted legacy subagent RuntimeEvent without rewriting it', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-runtime-persisted-compat-')); - try { - await createLegacyAgentRunStoreForTest(root).createRun(header); - const ledgerPath = join(root, 'sessions', sessionId, 'runs', runId, 'runtime-events.jsonl'); - await copyFile(fixtureUrl, ledgerPath); - const originalBytes = await readFile(ledgerPath); - - const view = await new RuntimeReadModel({ - runStore: createLegacyAgentRunStoreForTest(root), - runtimeEventStore: createLegacyRuntimeEventStoreForTest(root), - }).getSessionView(sessionId); - - const persistedResponse = view.events.find( - (event) => event.content?.kind === 'function_response', - ); - assert.equal( - persistedResponse?.content?.kind === 'function_response' && - typeof persistedResponse.content.result === 'object' && - persistedResponse.content.result !== null - ? (persistedResponse.content.result as { status?: unknown }).status - : undefined, - 'waiting_permission', - ); - - const messageResult = view.messages.find((message) => message.type === 'tool_result'); - assert.equal( - messageResult?.type === 'tool_result' && messageResult.content.kind === 'subagent' - ? messageResult.content.status - : undefined, - 'waiting_for_user', - ); - assert.deepEqual( - view.messages.map((message) => message.type), - ['tool_call', 'tool_result', 'turn_state'], - ); - - const replayResult = view.replayPlan.items.find((item) => item.kind === 'tool_result'); - assert.equal( - replayResult?.kind === 'tool_result' && - typeof replayResult.output === 'object' && - replayResult.output !== null - ? (replayResult.output as { status?: unknown }).status - : undefined, - 'waiting_for_user', - ); - - assert.equal(view.diagnostics.some(isHardRuntimeEventReadModelDiagnostic), false); - const blockingReplayDiagnosticCodes = new Set([ - 'unsupported_role', - 'unsupported_content', - 'tool_id_mismatch', - ]); - assert.equal( - view.replayPlan.diagnostics.some((diagnostic) => - blockingReplayDiagnosticCodes.has(diagnostic.code), - ), - false, - ); - assert.deepEqual(await readFile(ledgerPath), originalBytes); - } finally { - await rm(root, { recursive: true, force: true }); - } -}); diff --git a/packages/runtime/src/__tests__/runtime-resume-crash.test.ts b/packages/runtime/src/__tests__/runtime-resume-crash.test.ts index 8aad3a1cfd..7536a67e18 100644 --- a/packages/runtime/src/__tests__/runtime-resume-crash.test.ts +++ b/packages/runtime/src/__tests__/runtime-resume-crash.test.ts @@ -8,10 +8,7 @@ import { spawn } from 'node:child_process'; import { describe, test } from 'node:test'; import type { RuntimeEvent } from '@maka/core/runtime-event'; -import { - createLegacyAgentRunStoreForTest, - createLegacyRuntimeEventStoreForTest, -} from '@maka/storage/legacy-execution-test-support'; +import { createSqliteAgentRunStore, createWorkspaceRuntimeStore } from '@maka/storage'; import { RUNTIME_RESUME_FAILPOINTS, @@ -44,7 +41,7 @@ if (process.env[CRASH_CHILD_ENV] === '1') { // Production creates the run header before any RuntimeEvent append. Keep the // crash boundary focused on the child event writer while preserving the // storage identity contract used when the ledger is reopened. - await createLegacyAgentRunStoreForTest(workspaceRoot).createRun({ + await createSqliteAgentRunStore(workspaceRoot).createRun({ runId, invocationId: `invocation-${runId}`, sessionId, @@ -72,7 +69,7 @@ if (process.env[CRASH_CHILD_ENV] === '1') { false, `${failpoint.id} unexpectedly ran finally`, ); - const reopened = createLegacyRuntimeEventStoreForTest(workspaceRoot); + const reopened = createWorkspaceRuntimeStore(workspaceRoot); const recoveredEvents = await reopened.readRuntimeEvents(sessionId, runId); assert.deepEqual( recoveredEvents.map((event) => event.id), @@ -105,7 +102,7 @@ async function runCrashChild(): Promise { const events = JSON.parse( Buffer.from(requiredEnv('MAKA_RUNTIME_RESUME_EVENTS'), 'base64').toString('utf8'), ) as RuntimeEvent[]; - const store = createLegacyRuntimeEventStoreForTest(workspaceRoot); + const store = createWorkspaceRuntimeStore(workspaceRoot); try { for (const event of events) { diff --git a/packages/runtime/src/__tests__/sandbox-boundary-restart-recovery.test.ts b/packages/runtime/src/__tests__/sandbox-boundary-restart-recovery.test.ts index fb9845a2c0..fc0a4288cc 100644 --- a/packages/runtime/src/__tests__/sandbox-boundary-restart-recovery.test.ts +++ b/packages/runtime/src/__tests__/sandbox-boundary-restart-recovery.test.ts @@ -16,18 +16,14 @@ import { type DurableRuntimeEventStore, type SessionAuthorityStore, } from '@maka/storage'; -import { - createLegacyAgentRunStoreForTest, - createLegacyRuntimeEventStoreForTest, -} from '@maka/storage/legacy-execution-test-support'; +import { createSqliteAgentRunStore, createWorkspaceRuntimeStore } from '@maka/storage'; import { BackendRegistry, SessionManager } from '../session-manager.js'; /** - * Restart behaviour against the retired JSONL execution stores retained for - * compatibility coverage. Memory stores can model the shapes but not the - * ordering that makes this bug reachable — the request row commits before its - * RuntimeEvent, and a recovery pass can die between settling the row and - * committing the run's terminal fact. + * Restart behaviour against the canonical SQLite stores. Memory stores can + * model the shapes but not the ordering that makes this bug reachable — the + * request row commits before its RuntimeEvent, and a recovery pass can die + * between settling the row and committing the run's terminal fact. */ describe('sandbox boundary restart recovery on durable stores', () => { it('attributes a closure whose RuntimeEvent never reached the ledger', async () => { @@ -164,8 +160,8 @@ async function withStores( body: (stores: DurableStores) => Promise, ): Promise { const sessions = createSessionStore(root); - const runs = createLegacyAgentRunStoreForTest(root); - const runtimeEvents = createLegacyRuntimeEventStoreForTest(root); + const runs = createSqliteAgentRunStore(root); + const runtimeEvents = createWorkspaceRuntimeStore(root); try { return await body({ sessions, runs, runtimeEvents }); } finally { diff --git a/packages/runtime/src/__tests__/shell-run-manager.test.ts b/packages/runtime/src/__tests__/shell-run-manager.test.ts index 99a1e304db..5ac614a8fd 100644 --- a/packages/runtime/src/__tests__/shell-run-manager.test.ts +++ b/packages/runtime/src/__tests__/shell-run-manager.test.ts @@ -15,7 +15,7 @@ import { type ShellRunUpdate, type ToolResultContent, } from '@maka/core'; -import { createLegacyShellRunStoreForTest } from '@maka/storage/legacy-execution-test-support'; +import { createSqliteShellRunStore } from '@maka/storage'; import { ShellRunProcessManager } from '../shell-run-manager.js'; import { ShellRunPtyControlClosedError } from '../shell-run-contract.js'; @@ -34,7 +34,7 @@ after(async () => { describe('ShellRunProcessManager', () => { test('rejects unprojectable provider tool-call identities before durable admission', async () => { const cwd = await workspace(); - const store = createLegacyShellRunStoreForTest(cwd); + const store = createSqliteShellRunStore(cwd); const manager = createManager(store); const completions: boolean[] = []; const maximumMultibyteId = '😀'.repeat(SHELL_RUN_SOURCE_TOOL_CALL_ID_MAX_BYTES / 4); @@ -62,7 +62,7 @@ describe('ShellRunProcessManager', () => { test('keeps the default pipe path separated, durable, redacted, and observed', async () => { const cwd = await workspace(); - const store = createLegacyShellRunStoreForTest(cwd); + const store = createSqliteShellRunStore(cwd); const manager = createManager(store); const result = await manager.runForegroundBash( shellInput({ @@ -133,7 +133,7 @@ describe('ShellRunProcessManager', () => { test('keeps foreground execution bounded and rejects PTY promotion', async () => { const cwd = await workspace(); - const store = createLegacyShellRunStoreForTest(await workspace()); + const store = createSqliteShellRunStore(await workspace()); const manager = createManager(store); const abort = new AbortController(); const running = manager.runForegroundBash( @@ -167,7 +167,7 @@ describe('ShellRunProcessManager', () => { test('hands off a long pipe command without output and publishes monotonic revisions', async () => { const updates: ShellRunUpdate[] = []; - const store = createLegacyShellRunStoreForTest(await workspace()); + const store = createSqliteShellRunStore(await workspace()); const manager = createManager(store, (update) => updates.push(update)); const initial = await manager.runBackgroundBash( shellInput({ @@ -210,7 +210,7 @@ describe('ShellRunProcessManager', () => { }); test('notifies resource owners when foreground and background commands reach terminal state', async () => { - const store = createLegacyShellRunStoreForTest(await workspace()); + const store = createSqliteShellRunStore(await workspace()); const manager = createManager(store); const completions: boolean[] = []; await manager.runForegroundBash( @@ -288,7 +288,7 @@ describe('ShellRunProcessManager', () => { test('commits a durable starting identity before the native pipe process spawns', async () => { const cwd = await workspace(); const marker = join(cwd, 'spawned'); - const backingStore = createLegacyShellRunStoreForTest(cwd); + const backingStore = createSqliteShellRunStore(cwd); const createCommitted = deferred(); const releaseCreate = deferred(); const store: ShellRunStore = { @@ -341,7 +341,7 @@ describe('ShellRunProcessManager', () => { for (const lifecycle of ['session', 'runtime'] as const) { const cwd = await workspace(); const marker = join(cwd, `${lifecycle}-spawned`); - const backingStore = createLegacyShellRunStoreForTest(cwd); + const backingStore = createSqliteShellRunStore(cwd); const createCommitted = deferred(); const releaseCreate = deferred(); const store: ShellRunStore = { @@ -402,7 +402,7 @@ describe('ShellRunProcessManager', () => { test('rechecks the session fence after the durable running commit', async () => { const cwd = await workspace(); - const backingStore = createLegacyShellRunStoreForTest(cwd); + const backingStore = createSqliteShellRunStore(cwd); const runningCommitted = deferred(); const releaseRunning = deferred(); const store: ShellRunStore = { @@ -527,7 +527,7 @@ describe('ShellRunProcessManager', () => { }, async (context) => { const processDiscovery = delayPosixProcessDiscovery(context); const abort = new AbortController(); - const backingStore = createLegacyShellRunStoreForTest(await workspace()); + const backingStore = createSqliteShellRunStore(await workspace()); const runningCommitted = deferred(); const releaseRunning = deferred(); const store: ShellRunStore = { @@ -1009,7 +1009,7 @@ describe('ShellRunProcessManager', () => { }); test('recovers durable starting and running records without live handles as orphaned', async () => { - const store = createLegacyShellRunStoreForTest(await workspace()); + const store = createSqliteShellRunStore(await workspace()); await store.createShellRun(record({ shellRunId: 'orphan-starting', status: 'starting' })); await store.createShellRun(record({ shellRunId: 'orphan-running', status: 'running' })); await store.createShellRun({ @@ -1036,7 +1036,7 @@ describe('ShellRunProcessManager', () => { }); test('concurrent orphan observers converge on the same durable terminal record', async () => { - const backingStore = createLegacyShellRunStoreForTest(await workspace()); + const backingStore = createSqliteShellRunStore(await workspace()); await backingStore.createShellRun( record({ shellRunId: 'concurrent-orphan', status: 'running' }), ); @@ -1076,7 +1076,7 @@ describe('ShellRunProcessManager', () => { }); test('keeps unauthorized refs non-disclosing and rejects malformed selectors before storage', async () => { - const store = createLegacyShellRunStoreForTest(await workspace()); + const store = createSqliteShellRunStore(await workspace()); await store.createShellRun({ ...record({ shellRunId: 'owned-by-another-session', status: 'running' }), sessionId: 'session-2', @@ -1379,7 +1379,7 @@ describe('ShellRunProcessManager', () => { test('keeps concurrent PTY control and Read persistence in parser-cut order', async () => { const updates: ShellRunUpdate[] = []; - const store = createLegacyShellRunStoreForTest(await workspace()); + const store = createSqliteShellRunStore(await workspace()); const manager = createManager(store, (update) => updates.push(update)); const initial = await manager.runBackgroundBash( shellInput({ @@ -1452,7 +1452,7 @@ describe('ShellRunProcessManager', () => { const dsrSeen = join(cwd, 'dsr-seen'); const exitGate = join(cwd, 'exit-gate'); const sizeBeforeExit = join(cwd, 'size-before-exit'); - const store = createLegacyShellRunStoreForTest(await workspace()); + const store = createSqliteShellRunStore(await workspace()); const manager = createManager(store); const initial = await manager.runBackgroundBash( shellInput({ @@ -1599,7 +1599,7 @@ describe('ShellRunProcessManager', () => { test('restores the trailing PTY flush after a queued control aborts before commit', async () => { const cwd = await workspace(); const dirtyWritten = join(cwd, 'dirty-written'); - const store = createLegacyShellRunStoreForTest(await workspace()); + const store = createSqliteShellRunStore(await workspace()); const manager = createManager(store, undefined, { flushIntervalMs: 1_000 }); const initial = await manager.runBackgroundBash( shellInput({ @@ -2097,7 +2097,7 @@ describe('ShellRunProcessManager', () => { const storageRoot = await workspace(); const sessionsPath = join(storageRoot, 'sessions'); await writeFile(sessionsPath, 'blocks durable ShellRun creation', 'utf8'); - const store = createLegacyShellRunStoreForTest(storageRoot); + const store = createSqliteShellRunStore(storageRoot); const manager = createManager(store, undefined, { maxLiveShellRuns: 2, maxLivePtyRuns: 1 }); try { await assert.rejects(() => @@ -2245,11 +2245,7 @@ async function createTestManager( pipeOutputDrainMs?: number; }, ): Promise { - return createManager( - createLegacyShellRunStoreForTest(await workspace()), - onShellRunUpdate, - options, - ); + return createManager(createSqliteShellRunStore(await workspace()), onShellRunUpdate, options); } function shellInput(input: { diff --git a/packages/runtime/src/__tests__/stream-graph-coordinator.test.ts b/packages/runtime/src/__tests__/stream-graph-coordinator.test.ts index 8a12e5740c..eeb1b46eea 100644 --- a/packages/runtime/src/__tests__/stream-graph-coordinator.test.ts +++ b/packages/runtime/src/__tests__/stream-graph-coordinator.test.ts @@ -17,12 +17,9 @@ import { import { createSessionStore, createSqliteSessionMetadataStore, - SQLITE_SESSION_METADATA_DATABASE_NAME, + OPERATIONAL_STATE_DATABASE_NAME, } from '@maka/storage'; -import { - createLegacyAgentRunStoreForTest, - createLegacyRuntimeEventStoreForTest, -} from '@maka/storage/legacy-execution-test-support'; +import { createSqliteAgentRunStore, createWorkspaceRuntimeStore } from '@maka/storage'; import { FakeBackend } from '../fake-backend.js'; import { BackendRegistry, SessionManager } from '../session-manager.js'; import { SessionActivityRegistry } from '../goal-turn-lifecycle.js'; @@ -45,8 +42,8 @@ describe('host-managed agent graph coordinator', () => { test('boots an empty graph from agent work and recovers it without duplicate topology', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-graph-coordinator-')); const sessionStore = createSessionStore(root); - const runStore = createLegacyAgentRunStoreForTest(root); - const runtimeEventStore = createLegacyRuntimeEventStoreForTest(root); + const runStore = createSqliteAgentRunStore(root); + const runtimeEventStore = createWorkspaceRuntimeStore(root); let graphRuntimeHistoryReads = 0; const countedRuntimeEventStore = new Proxy(runtimeEventStore, { get(target, property) { @@ -99,9 +96,7 @@ describe('host-managed agent graph coordinator', () => { ); assert.ok(sourceRun); - controlStore = createSqliteSessionMetadataStore( - join(root, SQLITE_SESSION_METADATA_DATABASE_NAME), - ); + controlStore = createSqliteSessionMetadataStore(join(root, OPERATIONAL_STATE_DATABASE_NAME)); delayedControlStore = createDelayableControlStore(controlStore); const activities = new SessionActivityRegistry(); let supervisorWakeTurnCount = 0; @@ -935,8 +930,8 @@ describe('host-managed agent graph coordinator', () => { function createCoordinator(input: { sessionStore: ReturnType; - runStore: ReturnType; - runtimeEventStore: ReturnType; + runStore: ReturnType; + runtimeEventStore: ReturnType; controlStore: ReturnType; manager: SessionManager; onReconciliation?: AgentGraphCoordinatorInput['onReconciliation']; diff --git a/packages/runtime/src/ai-sdk-backend.ts b/packages/runtime/src/ai-sdk-backend.ts index b629e991c4..1761c88cbb 100644 --- a/packages/runtime/src/ai-sdk-backend.ts +++ b/packages/runtime/src/ai-sdk-backend.ts @@ -4,7 +4,7 @@ * Provides one `streamText` API across Anthropic / OpenAI / Google / DeepSeek / * OpenAI-compatible endpoints, while keeping all of our home-grown * machinery: session sandbox boundaries, materializer, AsyncEventQueue, - * SessionStore JSONL persistence. + * SessionStore SQLite persistence. * * Maka owns the agent loop. Each ModelAdapter call performs exactly one * provider request; returned tool calls settle through ToolRuntime, become diff --git a/packages/runtime/src/fake-backend.ts b/packages/runtime/src/fake-backend.ts index 3279218ab2..ba1d798ecd 100644 --- a/packages/runtime/src/fake-backend.ts +++ b/packages/runtime/src/fake-backend.ts @@ -115,7 +115,7 @@ export class FakeBackend implements AgentBackend { ' click A "javascript:alert(1)"', '```', ].join('\n') - : `Fake backend received: ${input.text}${attLine}\n\nThis proves the session stream, JSONL storage, and renderer loop are connected.`; + : `Fake backend received: ${input.text}${attLine}\n\nThis proves the session stream, SQLite storage, and renderer loop are connected.`; // Every delta must concatenate to text_complete; `.` would silently drop // line terminators and make structured Markdown reflow only at completion. const chunks = text.match(/[\s\S]{1,9}/g) ?? [text]; diff --git a/packages/runtime/src/materializer.ts b/packages/runtime/src/materializer.ts index 3a6333d711..6f748024a7 100644 --- a/packages/runtime/src/materializer.ts +++ b/packages/runtime/src/materializer.ts @@ -1,7 +1,7 @@ /** - * Materializer — converts the raw JSONL message stream into the view-model - * shape the UI renders. Lives in `runtime` (not `storage`) because correlation - * is a semantic operation, not a disk concern. + * Materializer — converts durable messages into the view-model shape the UI + * renders. Lives in `runtime` (not `storage`) because correlation is a semantic + * operation, not a disk concern. * * Runtime/UI materializer for rebuilding chat and tool activity state from * append-only stored messages. @@ -58,7 +58,7 @@ export interface SessionViewModel { // ============================================================================ /** - * Convert StoredMessage[] (raw JSONL) into a ChatItem[] for rendering. + * Convert StoredMessage[] into a ChatItem[] for rendering. * * A ToolCallMessage with no matching ToolResultMessage is read against its * turn: still `running` means the call is in flight, anything else makes it diff --git a/packages/runtime/src/runtime-event-adapters.ts b/packages/runtime/src/runtime-event-adapters.ts index 1aaeb92edd..28a50d900c 100644 --- a/packages/runtime/src/runtime-event-adapters.ts +++ b/packages/runtime/src/runtime-event-adapters.ts @@ -1,6 +1,6 @@ /** - * RuntimeEvent adapters — narrow bridges between the legacy StoredMessage - * JSONL format and the canonical RuntimeEvent fact model. + * RuntimeEvent adapters — narrow bridges between the durable StoredMessage + * model and the canonical RuntimeEvent fact model. * * Architecture: docs/architecture/runtime-core-architecture-draft.md * @@ -13,7 +13,7 @@ * The reverse direction (RuntimeEvent → StoredMessage draft) is provided * only for the straightforward user/model text cases. Tool, permission, * token-usage, and lifecycle events are deliberately NOT forced back into - * legacy storage shapes — those projections are owned by later nodes and + * StoredMessage shapes — those projections are owned by later nodes and * the materializer already covers the UI path. * * NOTE: imports the new `@maka/core/runtime-event` subpath. The steward diff --git a/packages/runtime/src/session-manager.ts b/packages/runtime/src/session-manager.ts index 405ab941d5..ac525f1612 100644 --- a/packages/runtime/src/session-manager.ts +++ b/packages/runtime/src/session-manager.ts @@ -2,7 +2,7 @@ * SessionManager — the public Runtime API. * * Ties together: - * SessionStore (storage) — JSONL persistence + * SessionStore (storage) — SQLite persistence * AgentBackend (AiSdkBackend etc) — SDK adapter * ExecutionBoundary — session sandbox authority * diff --git a/packages/runtime/src/tool-runtime.ts b/packages/runtime/src/tool-runtime.ts index ecf3bd60ee..768ea0c738 100644 --- a/packages/runtime/src/tool-runtime.ts +++ b/packages/runtime/src/tool-runtime.ts @@ -386,7 +386,7 @@ export interface ToolRuntimeInput { getRunTrace?: () => RunTraceLike | null; recordToolInvocation?: ToolTelemetryRecorder; recordToolArtifacts?: ToolArtifactRecorder; - /** Optional Phase 2 T1/T2 commit boundary. Omitted on legacy JSONL hosts. */ + /** Optional Phase 2 T1/T2 commit boundary for hosts that persist RuntimeEvents. */ runtimeCommitSink?: RuntimeCommitSink; } diff --git a/packages/storage/package.json b/packages/storage/package.json index d8a6115fb8..948f9ec98f 100644 --- a/packages/storage/package.json +++ b/packages/storage/package.json @@ -14,8 +14,6 @@ "./agent-graph-control-store": "./dist/agent-graph-control-store.js", "./interaction-store": "./dist/interaction-store-public.js", "./git-worktree-child-executor": "./dist/git-worktree-child-executor.js", - "./legacy-execution-test-support": "./dist/legacy-execution-test-support.js", - "./legacy-storage-test-support": "./dist/legacy-storage-test-support.js", "./memory-bundle-store": "./dist/memory-bundle-store.js", "./long-term-memory-store": "./dist/long-term-memory-store.js", "./root-authority": "./dist/root-authority.js", diff --git a/packages/storage/src/__tests__/agent-graph-supervisor-root-admission.test.ts b/packages/storage/src/__tests__/agent-graph-supervisor-root-admission.test.ts index af2e31b3bf..02344f1063 100644 --- a/packages/storage/src/__tests__/agent-graph-supervisor-root-admission.test.ts +++ b/packages/storage/src/__tests__/agent-graph-supervisor-root-admission.test.ts @@ -4,11 +4,11 @@ import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; import type { RootExecutionDescriptor } from '@maka/core/agent-run'; -import { createAgentRunStore, type AdmitRootTurnInput } from '../agent-run-store.js'; +import { createSqliteAgentRunStore, type AdmitRootTurnInput } from '../agent-run-store.js'; test('Agent Graph supervisor admission durably binds wake identity and Graph orchestration', async () => { await withTempRoot(async (root) => { - const store = createAgentRunStore(root); + const store = createSqliteAgentRunStore(root); const admitted = await store.admitRootTurn(admissionInput()); assert.equal(admitted.kind, 'admitted'); @@ -18,7 +18,7 @@ test('Agent Graph supervisor admission durably binds wake identity and Graph orc source: 'host_api', }); - const reopened = createAgentRunStore(root); + const reopened = createSqliteAgentRunStore(root); assert.deepEqual( await reopened.readRootTurnAdmission('root-session', 'supervisor-turn'), admitted.admission, @@ -28,7 +28,7 @@ test('Agent Graph supervisor admission durably binds wake identity and Graph orc test('Agent Graph supervisor admission rejects malformed identity and non-Graph orchestration', async () => { await withTempRoot(async (root) => { - const store = createAgentRunStore(root); + const store = createSqliteAgentRunStore(root); await assert.rejects( () => store.admitRootTurn( diff --git a/packages/storage/src/__tests__/agent-run-store.test.ts b/packages/storage/src/__tests__/agent-run-store.test.ts deleted file mode 100644 index ec1af83fd2..0000000000 --- a/packages/storage/src/__tests__/agent-run-store.test.ts +++ /dev/null @@ -1,1883 +0,0 @@ -import { chmod, mkdir, mkdtemp, readFile, readdir, rm, writeFile } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; -import { tmpdir } from 'node:os'; -import { describe, it } from 'node:test'; -import { isDeepStrictEqual } from 'node:util'; -import assert from 'node:assert/strict'; -import { createAgentRunStore, createRuntimeEventStore } from '../agent-run-store.js'; -import { - DurableStoreWriteError, - type AgentRunEvent, - type AgentRunHeader, - type RuntimeEvent, -} from '@maka/core'; - -const PARTIAL_STREAM_CHUNK_COUNT = process.env.MAKA_STORAGE_STRESS === '1' ? 10_000 : 100; - -describe('AgentRunStore', () => { - it('creates, reads, updates, and lists runs under a session', async () => { - await withStore(async (store, root) => { - const first = makeHeader({ - runId: 'run-1', - invocationId: 'invocation-1', - createdAt: 1, - updatedAt: 1, - }); - const second = makeHeader({ - runId: 'run-2', - turnId: 'turn-2', - status: 'waiting_for_user', - createdAt: 2, - updatedAt: 2, - }); - - await store.createRun(second); - await store.createRun(first); - await store.updateRun('session-1', 'run-1', { - status: 'completed', - completedAt: 10, - updatedAt: 10, - }); - - const read = await store.readRun('session-1', 'run-1'); - assert.equal(read.status, 'completed'); - assert.equal(read.completedAt, 10); - assert.equal(read.invocationId, 'invocation-1'); - assert.equal((await store.readRun('session-1', 'run-2')).status, 'waiting_for_user'); - assert.deepEqual( - (await store.listSessionRuns('session-1')).map((run) => run.runId), - ['run-1', 'run-2'], - ); - assert.equal( - JSON.parse( - await readFile(join(root, 'sessions', 'session-1', 'runs', 'run-1', 'run.json'), 'utf8'), - ).runId, - 'run-1', - ); - }); - }); - - it('finds a duplicate run identity across sessions with a bounded result', async () => { - await withStore(async (store) => { - await store.createRun(makeHeader({ sessionId: 'session-c', runId: 'shared-run' })); - await store.createRun(makeHeader({ sessionId: 'session-a', runId: 'shared-run' })); - await store.createRun(makeHeader({ sessionId: 'session-b', runId: 'shared-run' })); - await store.createRun(makeHeader({ sessionId: 'session-a', runId: 'other-run' })); - - const bounded = await store.findRunsById('shared-run', 2); - assert.equal(bounded.truncated, true); - assert.deepEqual( - bounded.runs.map((run) => run.sessionId), - ['session-a', 'session-b'], - ); - assert.deepEqual(await store.findRunsById('missing-run', 2), { - runs: [], - truncated: false, - }); - await assert.rejects(() => store.findRunsById('shared-run', 0), RangeError); - }); - }); - - it('bounds Session headers and evidence before returning records', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader({ runId: 'run-2', createdAt: 2 })); - await runStore.createRun(makeHeader({ runId: 'run-1', createdAt: 1 })); - assert.deepEqual(await runStore.listSessionRunsBounded('session-1', 1), { - runs: [makeHeader({ runId: 'run-1', createdAt: 1 })], - truncated: true, - }); - - await runStore.appendEvent('session-1', 'run-1', makeEvent({ id: 'event-1' })); - await runStore.appendEvent('session-1', 'run-1', makeEvent({ id: 'event-2', ts: 2 })); - assert.deepEqual( - await runStore.readEventsBounded('session-1', 'run-1', { - maxRecords: 1, - maxBytes: 1024, - }), - { status: 'limit_exceeded' }, - ); - - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ id: 'runtime-1' }), - ); - const runtimeResult = await runtimeEventStore.readRuntimeEventsBounded('session-1', 'run-1', { - maxRecords: 2, - maxBytes: 1024, - }); - assert.equal(runtimeResult.status, 'complete'); - if (runtimeResult.status !== 'complete') return; - assert.deepEqual( - runtimeResult.records.map((event) => event.id), - ['runtime-1'], - ); - assert.equal(runtimeResult.sourceRecordCount, 1); - assert.ok(runtimeResult.storedBytes > 0); - }); - }); - - it('rejects duplicate run creation without replacing the existing header', async () => { - await withStore(async (store) => { - const original = makeHeader({ invocationId: 'invocation-original' }); - await store.createRun(original); - - await assert.rejects( - () => store.createRun(makeHeader({ invocationId: 'invocation-replacement', updatedAt: 2 })), - /already exists/i, - ); - - assert.deepEqual(await store.readRun('session-1', 'run-1'), original); - }); - }); - - it('rejects updates to immutable admission and continuation identity', async () => { - await withStore(async (store) => { - const original = makeHeader({ - continuationSource: { - sourceInvocationId: 'source-invocation', - sourceRunId: 'source-run', - sourceTurnId: 'source-turn', - sourceRuntimeEventHighWater: 1, - }, - }); - await store.createRun(original); - - await assert.rejects( - store.updateRun('session-1', 'run-1', { modelId: 'different-model' }), - /admission identity is immutable: modelId/, - ); - await assert.rejects( - store.updateRun('session-1', 'run-1', { continuationSource: undefined }), - /admission identity is immutable: continuationSource/, - ); - assert.deepEqual(await store.readRun('session-1', 'run-1'), original); - }); - }); - - it('rejects malformed run headers instead of returning partial records', async () => { - await withStore(async (store, root) => { - await store.createRun(makeHeader()); - const runPath = join(root, 'sessions', 'session-1', 'runs', 'run-1', 'run.json'); - await writeFile( - runPath, - JSON.stringify({ runId: 'run-1', sessionId: 'session-1' }) + '\n', - 'utf8', - ); - - await assert.rejects( - () => store.readRun('session-1', 'run-1'), - /Invalid AgentRun header for run run-1: malformed fields/, - ); - assert.deepEqual(await store.listSessionRuns('session-1'), []); - }); - }); - - it('decodes and lists legacy waiting_permission headers as waiting_for_user', async () => { - await withStore(async (store, root) => { - const runPath = join(root, 'sessions', 'session-1', 'runs', 'run-1', 'run.json'); - await mkdir(dirname(runPath), { recursive: true }); - await writeFile( - runPath, - JSON.stringify({ ...makeHeader(), status: 'waiting_permission' }) + '\n', - 'utf8', - ); - - assert.equal((await store.readRun('session-1', 'run-1')).status, 'waiting_for_user'); - assert.deepEqual( - (await store.listSessionRuns('session-1')).map((run) => [run.runId, run.status]), - [['run-1', 'waiting_for_user']], - ); - assert.deepEqual( - (await store.listSessionRunsForRecovery('session-1')).map((run) => [run.runId, run.status]), - [['run-1', 'waiting_for_user']], - ); - assert.equal( - JSON.parse(await readFile(runPath, 'utf8')).status, - 'waiting_permission', - 'read recovery must not rewrite legacy bytes', - ); - }); - }); - - it('rejects malformed run headers on update without overwriting bytes', async () => { - await withStore(async (store, root) => { - await store.createRun(makeHeader()); - const runPath = join(root, 'sessions', 'session-1', 'runs', 'run-1', 'run.json'); - const invalid = - JSON.stringify( - { - runId: 'run-1', - sessionId: 'session-1', - turnId: 'turn-1', - status: 'running', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/tmp/cwd', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 'soon', - }, - null, - 2, - ) + '\n'; - await writeFile(runPath, invalid, 'utf8'); - - await assert.rejects( - () => - store.updateRun('session-1', 'run-1', { - status: 'completed', - completedAt: 10, - updatedAt: 10, - }), - /Invalid AgentRun header for run run-1: malformed fields/, - ); - assert.equal(await readFile(runPath, 'utf8'), invalid); - }); - }); - - it('rejects malformed optional run header fields', async () => { - await withStore(async (store, root) => { - const runPath = join(root, 'sessions', 'session-1', 'runs', 'run-1', 'run.json'); - for (const patch of [{ automationId: 42 }, { abortSource: false }]) { - await mkdir(dirname(runPath), { recursive: true }); - await writeFile(runPath, JSON.stringify({ ...makeHeader(), ...patch }) + '\n', 'utf8'); - await assert.rejects( - () => store.readRun('session-1', 'run-1'), - /Invalid AgentRun header for run run-1: malformed fields/, - ); - } - }); - }); - - it('serializes same-run event appends', async () => { - await withStore(async (store) => { - await store.createRun(makeHeader()); - - await Promise.all( - Array.from({ length: 20 }, (_, index) => - store.appendEvent('session-1', 'run-1', makeEvent({ id: `event-${index}`, ts: index })), - ), - ); - - const events = await store.readEvents('session-1', 'run-1'); - assert.equal(events.length, 20); - assert.equal(new Set(events.map((event) => event.id)).size, 20); - }); - }); - - it('writes a bounded projection for accepted history compact checkpoints', async () => { - await withStore(async (store) => { - const projectionStore = store as typeof store & { - readEventProjection( - sessionId: string, - type: AgentRunEvent['type'], - ): Promise; - }; - await store.createRun(makeHeader({ runId: 'run-accepted' })); - const checkpointEvent = (runId: string, eventCount: number): AgentRunEvent => - makeEvent({ - type: 'history_compact_checkpoint_recorded', - id: `checkpoint-${runId}`, - runId, - turnId: `turn-${runId}`, - data: { - checkpoint: { - kind: 'maka.history_compact_checkpoint', - version: 2, - checkpointId: `hcheckpoint-${runId}`, - sessionId: 'session-1', - coverage: { eventCount }, - }, - }, - }); - - await store.appendEvent('session-1', 'run-accepted', checkpointEvent('run-accepted', 3)); - - const projected = await projectionStore.readEventProjection( - 'session-1', - 'history_compact_checkpoint_recorded', - ); - assert.equal( - projected?.data?.checkpoint && - (projected.data.checkpoint as { checkpointId?: string }).checkpointId, - 'hcheckpoint-run-accepted', - ); - }); - }); - - it('initializes an empty checkpoint projection for a new session', async () => { - await withStore(async (store) => { - const projectionStore = store as typeof store & { - readEventProjection( - sessionId: string, - type: AgentRunEvent['type'], - ): Promise; - }; - - await store.createRun(makeHeader()); - - assert.equal( - await projectionStore.readEventProjection( - 'session-1', - 'history_compact_checkpoint_recorded', - ), - null, - ); - }); - }); - - it('preserves a missing checkpoint projection when prior runs require recovery', async () => { - await withStore(async (store, root) => { - const projectionStore = store as typeof store & { - readEventProjection( - sessionId: string, - type: AgentRunEvent['type'], - ): Promise; - }; - await store.createRun(makeHeader({ runId: 'run-before-crash' })); - await rm( - join( - root, - 'sessions', - 'session-1', - 'projections', - 'history_compact_checkpoint_recorded.json', - ), - ); - - await store.createRun(makeHeader({ runId: 'run-after-restart' })); - - assert.equal( - await projectionStore.readEventProjection( - 'session-1', - 'history_compact_checkpoint_recorded', - ), - undefined, - ); - }); - }); - - it('does not let a stale repair overwrite a further checkpoint projection', async () => { - await withStore(async (store, root) => { - const projectionStore = store as typeof store & { - readEventProjection( - sessionId: string, - type: AgentRunEvent['type'], - ): Promise; - repairEventProjection( - sessionId: string, - type: AgentRunEvent['type'], - event: AgentRunEvent | null, - ): Promise; - }; - const checkpointEvent = (runId: string, eventCount: number): AgentRunEvent => - makeEvent({ - type: 'history_compact_checkpoint_recorded', - id: `checkpoint-${runId}`, - runId, - turnId: `turn-${runId}`, - data: { - checkpoint: { - kind: 'maka.history_compact_checkpoint', - version: 2, - checkpointId: `hcheckpoint-${runId}`, - sessionId: 'session-1', - coverage: { eventCount }, - }, - }, - }); - const stale = checkpointEvent('run-stale-repair', 1); - const further = checkpointEvent('run-further-write', 2); - await store.createRun(makeHeader({ runId: stale.runId })); - await store.createRun(makeHeader({ runId: further.runId })); - await store.appendEvent('session-1', stale.runId, stale); - await rm( - join( - root, - 'sessions', - 'session-1', - 'projections', - 'history_compact_checkpoint_recorded.json', - ), - ); - - await store.appendEvent('session-1', further.runId, further); - await projectionStore.repairEventProjection( - 'session-1', - 'history_compact_checkpoint_recorded', - stale, - ); - - const projected = await projectionStore.readEventProjection( - 'session-1', - 'history_compact_checkpoint_recorded', - ); - assert.equal(projected?.id, further.id); - }); - }); - - it('does not let a legacy append or repair downgrade a source-bound checkpoint projection', async () => { - await withStore(async (store) => { - const projectionStore = store as typeof store & { - readEventProjection( - sessionId: string, - type: AgentRunEvent['type'], - ): Promise; - repairEventProjection( - sessionId: string, - type: AgentRunEvent['type'], - event: AgentRunEvent | null, - ): Promise; - }; - const sourceBound = makeEvent({ - type: 'history_compact_checkpoint_recorded', - id: 'checkpoint-source-bound', - data: { - checkpoint: { - coverage: { eventCount: 2 }, - source: { kind: 'runtime_event_projection' }, - }, - }, - }); - const legacy = makeEvent({ - type: 'history_compact_checkpoint_recorded', - id: 'checkpoint-legacy', - data: { checkpoint: { coverage: { eventCount: 99 } } }, - }); - await store.createRun(makeHeader()); - await store.appendEvent('session-1', sourceBound.runId, sourceBound); - await store.appendEvent('session-1', legacy.runId, legacy); - - assert.equal( - ( - await projectionStore.readEventProjection( - 'session-1', - 'history_compact_checkpoint_recorded', - ) - )?.id, - sourceBound.id, - ); - - await projectionStore.repairEventProjection( - 'session-1', - 'history_compact_checkpoint_recorded', - legacy, - ); - - const projected = await projectionStore.readEventProjection( - 'session-1', - 'history_compact_checkpoint_recorded', - ); - assert.equal(projected?.id, sourceBound.id); - }); - }); - - it('replaces the same parseable but semantically invalid projection during repair', async () => { - await withStore(async (store, root) => { - const projectionStore = store as typeof store & { - readEventProjection( - sessionId: string, - type: AgentRunEvent['type'], - ): Promise; - repairEventProjection( - sessionId: string, - type: AgentRunEvent['type'], - event: AgentRunEvent | null, - options?: { replaceEventId?: string }, - ): Promise; - }; - await store.createRun(makeHeader()); - const invalidProjection = makeEvent({ - type: 'history_compact_checkpoint_recorded', - id: 'invalid-projection-event', - data: { checkpoint: { coverage: { eventCount: 999 } } }, - }); - const canonicalEvent = makeEvent({ - type: 'history_compact_checkpoint_recorded', - id: 'canonical-projection-event', - data: { - checkpoint: { - kind: 'maka.history_compact_checkpoint', - version: 2, - checkpointId: 'hcheckpoint-canonical', - sessionId: 'session-1', - coverage: { eventCount: 1 }, - }, - }, - }); - await writeFile( - join( - root, - 'sessions', - 'session-1', - 'projections', - 'history_compact_checkpoint_recorded.json', - ), - JSON.stringify({ version: 1, event: invalidProjection }) + '\n', - ); - - await projectionStore.repairEventProjection( - 'session-1', - 'history_compact_checkpoint_recorded', - canonicalEvent, - { replaceEventId: invalidProjection.id }, - ); - - const projected = await projectionStore.readEventProjection( - 'session-1', - 'history_compact_checkpoint_recorded', - ); - assert.equal(projected?.id, canonicalEvent.id); - }); - }); - - it('does not retain a checkpoint projection when the canonical ledger append fails', async () => { - await withStore(async (store, root) => { - const projectionStore = store as typeof store & { - readEventProjection( - sessionId: string, - type: AgentRunEvent['type'], - ): Promise; - }; - await store.createRun(makeHeader()); - const checkpointEvent = (checkpointId: string, eventCount: number): AgentRunEvent => - makeEvent({ - type: 'history_compact_checkpoint_recorded', - data: { - checkpoint: { - kind: 'maka.history_compact_checkpoint', - version: 2, - checkpointId, - sessionId: 'session-1', - coverage: { eventCount }, - }, - }, - }); - await store.appendEvent('session-1', 'run-1', checkpointEvent('hcheckpoint-previous', 1)); - const eventsPath = join(root, 'sessions', 'session-1', 'runs', 'run-1', 'events.jsonl'); - await rm(eventsPath); - await mkdir(eventsPath); - - await assert.rejects(() => - store.appendEvent('session-1', 'run-1', checkpointEvent('hcheckpoint-orphan', 2)), - ); - - assert.equal( - await projectionStore.readEventProjection( - 'session-1', - 'history_compact_checkpoint_recorded', - ), - undefined, - ); - }); - }); - - it('recovers corrupt event lines without hiding later events', async () => { - await withStore(async (store, root) => { - await store.createRun(makeHeader()); - await store.appendEvent('session-1', 'run-1', makeEvent({ id: 'good-1', ts: 1 })); - const eventsPath = join(root, 'sessions', 'session-1', 'runs', 'run-1', 'events.jsonl'); - await writeFile( - eventsPath, - '{"type":"run_started"\n' + JSON.stringify(makeEvent({ id: 'good-2', ts: 2 })) + '\n', - { - flag: 'a', - }, - ); - - const events = await store.readEvents('session-1', 'run-1'); - assert.equal(events[0]?.id, 'good-1'); - assert.equal(events[1]?.type, 'event_corrupt'); - assert.equal(events[2]?.id, 'good-2'); - }); - }); - - it('drops an unterminated corrupt tail event', async () => { - await withStore(async (store, root) => { - await store.createRun(makeHeader()); - const eventsPath = join(root, 'sessions', 'session-1', 'runs', 'run-1', 'events.jsonl'); - await mkdir(join(root, 'sessions', 'session-1', 'runs', 'run-1'), { recursive: true }); - await writeFile( - eventsPath, - JSON.stringify(makeEvent({ id: 'good-1', ts: 1 })) + '\n{"type":"run_started"', - ); - - const events = await store.readEvents('session-1', 'run-1'); - assert.deepEqual( - events.map((event) => event.id), - ['good-1'], - ); - }); - }); - - it('does not mistake an invalid unterminated tail for a crash prefix', async () => { - await withStore(async (store, root) => { - await store.createRun(makeHeader()); - const eventsPath = join(root, 'sessions', 'session-1', 'runs', 'run-1', 'events.jsonl'); - const bytes = JSON.stringify(makeEvent({ id: 'good-1', ts: 1 })) + '\n{"type":]'; - await writeFile(eventsPath, bytes, 'utf8'); - - const events = await store.readEvents('session-1', 'run-1'); - assert.deepEqual( - events.map((event) => event.type), - ['run_started', 'event_corrupt'], - ); - await assert.rejects( - () => store.readEventsForRecovery('session-1', 'run-1'), - /AgentRun run-1 has a corrupt JSONL record at line 2/, - ); - assert.equal(await readFile(eventsPath, 'utf8'), bytes); - }); - }); - - it('keeps newline-terminated corrupt tail events as durable corruption notes', async () => { - await withStore(async (store, root) => { - await store.createRun(makeHeader()); - const eventsPath = join(root, 'sessions', 'session-1', 'runs', 'run-1', 'events.jsonl'); - await mkdir(join(root, 'sessions', 'session-1', 'runs', 'run-1'), { recursive: true }); - await writeFile( - eventsPath, - JSON.stringify(makeEvent({ id: 'good-1', ts: 1 })) + '\n{"type":"run_started"\n', - ); - - const events = await store.readEvents('session-1', 'run-1'); - assert.deepEqual( - events.map((event) => event.type), - ['run_started', 'event_corrupt'], - ); - assert.equal(events[1]?.data?.lineNumber, 2); - }); - }); - - it('rejects complete schema-invalid and identity-mismatched events during recovery', async () => { - await withStore(async (store, root) => { - await store.createRun(makeHeader()); - const eventsPath = join(root, 'sessions', 'session-1', 'runs', 'run-1', 'events.jsonl'); - for (const record of [ - {}, - makeEvent({ sessionId: 'other-session' }), - makeEvent({ runId: 'other-run' }), - makeEvent({ turnId: 'other-turn' }), - ]) { - await writeFile(eventsPath, JSON.stringify(record)); - await assert.rejects( - () => store.readEventsForRecovery('session-1', 'run-1'), - /AgentRun run-1 has a corrupt JSONL record at line 1/, - ); - } - }); - }); - - it('appends and reads runtime events from a separate per-run ledger', async () => { - await withStores(async (runStore, runtimeEventStore, root) => { - await runStore.createRun(makeHeader({ invocationId: 'turn-1' })); - await runStore.appendEvent('session-1', 'run-1', makeEvent({ id: 'operational-event' })); - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ id: 'runtime-1', role: 'user' }), - ); - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ id: 'runtime-2', role: 'model' }), - ); - - const runtimeEvents = await runtimeEventStore.readRuntimeEvents('session-1', 'run-1'); - assert.deepEqual( - runtimeEvents.map((event) => event.id), - ['runtime-1', 'runtime-2'], - ); - assert.deepEqual( - runtimeEvents.map((event) => event.role), - ['user', 'model'], - ); - assert.deepEqual( - (await runStore.readEvents('session-1', 'run-1')).map((event) => event.id), - ['operational-event'], - ); - - const runtimeEventsPath = join( - root, - 'sessions', - 'session-1', - 'runs', - 'run-1', - 'runtime-events.jsonl', - ); - const operationalEventsPath = join( - root, - 'sessions', - 'session-1', - 'runs', - 'run-1', - 'events.jsonl', - ); - assert.match(await readFile(runtimeEventsPath, 'utf8'), /"id":"runtime-1"/); - assert.match(await readFile(operationalEventsPath, 'utf8'), /"id":"operational-event"/); - }); - }); - - it('keeps authoritative tool facts out of the JSONL generic writer', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - const dispatch = makeRuntimeEvent({ - id: 'dispatch-reserved', - role: 'system', - author: 'system', - content: undefined, - actions: { - toolDispatch: { - protocol: 't1_after_preflight_v1', - operationId: 'operation-1', - providerToolCallId: 'call-1', - toolName: 'Write', - canonicalArgsHash: 'sha256:reserved', - recoveryMode: 'reconcile', - }, - }, - refs: { operationId: 'operation-1', toolCallId: 'call-1' }, - }); - const outcome = makeRuntimeEvent({ - id: 'outcome-reserved', - role: 'tool', - author: 'tool', - content: { - kind: 'function_response', - id: 'call-1', - name: 'Write', - result: 'ok', - }, - refs: { operationId: 'operation-1', toolCallId: 'call-1' }, - }); - const recovery = makeRuntimeEvent({ - id: 'recovery-reserved', - role: 'system', - author: 'system', - content: undefined, - actions: { - toolRecovery: { - kind: 'maka.tool.reconcile_result', - version: 1, - payload: { - protocol: 'tool_reconcile_v1', - operationId: 'operation-1', - observation: 'unreadable', - observationSchema: 'state_identity_v1', - observationDigest: - 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', - }, - }, - }, - refs: { operationId: 'operation-1', toolCallId: 'call-1' }, - }); - - for (const event of [dispatch, outcome, recovery]) { - await assert.rejects( - runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', event), - /atomic tool|atomic recovery/, - ); - } - assert.deepEqual( - await runtimeEventStore.readImmutableRuntimeEvents('session-1', 'run-1'), - [], - ); - }); - }); - - it('keeps continuation-start authority facts out of the JSONL generic writer', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - const continuationStart = makeRuntimeEvent({ - id: 'continuation-start-reserved', - role: 'system', - author: 'system', - content: undefined, - actions: { - continuationStart: { - protocol: 'continuation_start_v2', - provenance: 'runtime_admission', - claimId: 'claim-1', - boundaryDigest: - 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', - immediateSource: { - sessionId: 'session-1', - invocationId: 'source-invocation', - runId: 'source-run', - turnId: 'source-turn', - highWater: 1, - prefixDigest: - 'sha256:bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', - }, - replayManifestDigest: - 'sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', - providerProjectionVersion: 1, - providerReplayDigest: - 'sha256:cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc', - }, - }, - }); - - await assert.rejects( - runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', continuationStart), - /continuation authority/i, - ); - assert.deepEqual( - await runtimeEventStore.readImmutableRuntimeEvents('session-1', 'run-1'), - [], - ); - }); - }); - - it('keeps an exact ordinary RuntimeEvent retry idempotent in JSONL', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - const event = makeRuntimeEvent({ id: 'ordinary-event-1' }); - - await runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', event); - await runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', event); - - assert.deepEqual( - (await runtimeEventStore.readImmutableRuntimeEvents('session-1', 'run-1')).map( - (item) => item.id, - ), - ['ordinary-event-1'], - ); - }); - }); - - it('re-establishes stable storage on an exact durable RuntimeEvent retry', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - const event = makeRuntimeEvent({ id: 'ordinary-event-1' }); - await runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', event); - - await runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', event, { - durable: true, - }); - - assert.deepEqual( - (await runtimeEventStore.readImmutableRuntimeEvents('session-1', 'run-1')).map( - (item) => item.id, - ), - ['ordinary-event-1'], - ); - }); - }); - - it('keeps an exact tool call retry idempotent in JSONL', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - const call = makeRuntimeEvent({ - id: 'call-event-1', - role: 'model', - author: 'agent', - content: { - kind: 'function_call', - id: 'provider-call-1', - name: 'Write', - args: { path: 'note.txt', content: 'hello' }, - }, - refs: { toolCallId: 'provider-call-1' }, - }); - - await runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', call); - await runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', call); - - assert.deepEqual( - (await runtimeEventStore.readImmutableRuntimeEvents('session-1', 'run-1')).map( - (item) => item.id, - ), - ['call-event-1'], - ); - }); - }); - - it('rejects a conflicting RuntimeEvent retry without changing JSONL', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - const original = makeRuntimeEvent({ id: 'ordinary-event-1' }); - await runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', original); - - await assert.rejects( - runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', { - ...original, - ts: original.ts + 1, - }), - /RuntimeEvent identity conflict/, - ); - assert.deepEqual(await runtimeEventStore.readImmutableRuntimeEvents('session-1', 'run-1'), [ - original, - ]); - }); - }); - - it('rejects a RuntimeEvent whose identity does not match the target run header', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader({ invocationId: 'turn-1' })); - - for (const event of [ - makeRuntimeEvent({ sessionId: 'other-session' }), - makeRuntimeEvent({ runId: 'other-run' }), - makeRuntimeEvent({ turnId: 'other-turn' }), - makeRuntimeEvent({ invocationId: 'other-invocation' }), - ]) { - await assert.rejects( - runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', event), - /RuntimeEvent identity does not match its run/, - ); - } - assert.deepEqual( - await runtimeEventStore.readImmutableRuntimeEvents('session-1', 'run-1'), - [], - ); - }); - }); - - it('rejects a duplicate provider call identity in the JSONL ledger', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - const call = makeRuntimeEvent({ - id: 'call-event-1', - role: 'model', - author: 'agent', - content: { - kind: 'function_call', - id: 'provider-call-1', - name: 'Write', - args: { path: 'notes.txt', content: 'after' }, - }, - }); - await runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', call); - - await assert.rejects( - runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', { - ...call, - id: 'call-event-duplicate', - }), - /duplicate_call/, - ); - assert.deepEqual( - (await runtimeEventStore.readImmutableRuntimeEvents('session-1', 'run-1')).map( - ({ id }) => id, - ), - ['call-event-1'], - ); - }); - }); - - it('returns an empty runtime event list when the runtime ledger is missing', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - - assert.deepEqual(await runtimeEventStore.readRuntimeEvents('session-1', 'run-1'), []); - }); - }); - - it('coalesces text stream chunks into one recoverable partial snapshot', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - for (const [index, text] of ['hel', 'lo', '!'].entries()) { - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: `runtime-partial-${index}`, - ts: index + 1, - partial: true, - role: 'model', - author: 'agent', - content: { kind: 'text', text }, - refs: { providerEventId: 'message-1' }, - }), - ); - } - - const events = await runtimeEventStore.readRuntimeEvents('session-1', 'run-1'); - const readImmutable = runtimeEventStore.readImmutableRuntimeEvents; - assert.ok(readImmutable); - const immutableEvents = await readImmutable.call(runtimeEventStore, 'session-1', 'run-1'); - - assert.equal(events.length, 1); - assert.equal(events[0]?.partial, true); - assert.deepEqual(events[0]?.content, { kind: 'text', text: 'hello!' }); - assert.deepEqual(immutableEvents, []); - }); - }); - - it('does not merge partial streams across branch lineage', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: 'runtime-branch-a', - ts: 1, - branch: 'agent-a', - partial: true, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'alpha' }, - refs: { providerEventId: 'message-1' }, - }), - ); - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: 'runtime-branch-b', - ts: 2, - branch: 'agent-b', - partial: true, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'beta' }, - refs: { providerEventId: 'message-1' }, - }), - ); - - const events = await runtimeEventStore.readRuntimeEvents('session-1', 'run-1'); - - assert.deepEqual( - events.map((event) => ({ - branch: event.branch, - text: event.content?.kind === 'text' ? event.content.text : undefined, - })), - [ - { branch: 'agent-a', text: 'alpha' }, - { branch: 'agent-b', text: 'beta' }, - ], - ); - }); - }); - - it('recovers the accumulated partial snapshot after reopening the store', async () => { - await withStores(async (runStore, runtimeEventStore, root) => { - await runStore.createRun(makeHeader()); - for (const [index, text] of ['still ', 'working'].entries()) { - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: `runtime-partial-${index}`, - partial: true, - role: 'model', - author: 'agent', - content: { kind: 'text', text }, - refs: { providerEventId: 'message-1' }, - }), - ); - } - - const reopened = createRuntimeEventStore(root); - const events = await reopened.readRuntimeEvents('session-1', 'run-1'); - - assert.equal(events.length, 1); - assert.deepEqual(events[0]?.content, { kind: 'text', text: 'still working' }); - }); - }); - - it('keeps a long text stream bounded to one durable partial snapshot', async () => { - await withStores(async (runStore, runtimeEventStore, root) => { - await runStore.createRun(makeHeader()); - for (let index = 0; index < PARTIAL_STREAM_CHUNK_COUNT; index += 1) { - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: `runtime-partial-${index}`, - partial: true, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'x' }, - refs: { providerEventId: 'message-1' }, - }), - ); - } - - const events = await runtimeEventStore.readRuntimeEvents('session-1', 'run-1'); - const partialFiles = await readdir( - join(root, 'sessions', 'session-1', 'runs', 'run-1', 'runtime-partials'), - ); - - assert.equal(events.length, 1); - assert.equal( - events[0]?.content?.kind === 'text' && events[0].content.text.length, - PARTIAL_STREAM_CHUNK_COUNT, - ); - assert.equal(partialFiles.filter((name) => name.endsWith('.partial')).length, 1); - assert.equal(partialFiles.length, 1); - const immutableLedger = await readFile( - join(root, 'sessions', 'session-1', 'runs', 'run-1', 'runtime-events.jsonl'), - 'utf8', - ).catch((error) => { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return ''; - throw error; - }); - assert.equal(immutableLedger, ''); - }); - }); - - it('coalesces thinking chunks into one recoverable partial snapshot', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - for (const [index, text] of ['reason', 'ing ', 'continues'].entries()) { - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: `runtime-thinking-${index}`, - partial: true, - role: 'model', - author: 'agent', - content: { kind: 'thinking', text }, - refs: { providerEventId: 'message-1' }, - }), - ); - } - - const events = await runtimeEventStore.readRuntimeEvents('session-1', 'run-1'); - - assert.equal(events.length, 1); - assert.deepEqual(events[0]?.content, { kind: 'thinking', text: 'reasoning continues' }); - }); - }); - - it('coalesces tool stream heartbeats until the durable tool result arrives', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: 'runtime-tool-call', - role: 'model', - author: 'agent', - content: { - kind: 'function_call', - id: 'tool-call-1', - name: 'Bash', - args: { command: 'echo done' }, - }, - refs: { toolCallId: 'tool-call-1' }, - }), - ); - for (let index = 0; index < 100; index += 1) { - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: `runtime-tool-progress-${index}`, - partial: true, - role: 'tool', - author: 'tool', - content: undefined, - refs: { toolCallId: 'tool-call-1' }, - }), - ); - } - - assert.equal((await runtimeEventStore.readRuntimeEvents('session-1', 'run-1')).length, 2); - - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: 'runtime-tool-result', - ts: 2, - role: 'tool', - author: 'tool', - content: { - kind: 'function_response', - id: 'tool-call-1', - name: 'Bash', - result: 'done', - }, - refs: { toolCallId: 'tool-call-1' }, - }), - ); - - const events = await runtimeEventStore.readRuntimeEvents('session-1', 'run-1'); - assert.deepEqual( - events.map((event) => event.id), - ['runtime-tool-call', 'runtime-tool-result'], - ); - }); - }); - - it('keeps partial events with lifecycle actions as immutable facts', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - for (let index = 0; index < 2; index += 1) { - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: `runtime-lifecycle-${index}`, - partial: true, - role: 'system', - author: 'system', - content: undefined, - actions: { stateDelta: { progress: index } }, - refs: { toolCallId: 'tool-call-1' }, - }), - ); - } - - const events = await runtimeEventStore.readRuntimeEvents('session-1', 'run-1'); - const readImmutable = runtimeEventStore.readImmutableRuntimeEvents; - assert.ok(readImmutable); - const immutableEvents = await readImmutable.call(runtimeEventStore, 'session-1', 'run-1'); - - assert.deepEqual( - events.map((event) => event.id), - ['runtime-lifecycle-0', 'runtime-lifecycle-1'], - ); - assert.deepEqual( - immutableEvents.map((event) => event.id), - ['runtime-lifecycle-0', 'runtime-lifecycle-1'], - ); - }); - }); - - it('replaces a text partial snapshot with the durable final event', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: 'runtime-partial', - partial: true, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'hello' }, - refs: { providerEventId: 'message-1' }, - }), - ); - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: 'runtime-final', - ts: 2, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'hello world' }, - refs: { providerEventId: 'message-1' }, - }), - ); - - const events = await runtimeEventStore.readRuntimeEvents('session-1', 'run-1'); - - assert.deepEqual( - events.map((event) => event.id), - ['runtime-final'], - ); - assert.deepEqual(events[0]?.content, { kind: 'text', text: 'hello world' }); - }); - }); - - it('cleans a retained partial snapshot on an exact final-event retry', async () => { - await withStores(async (runStore, runtimeEventStore, root) => { - await runStore.createRun(makeHeader()); - const partial = makeRuntimeEvent({ - id: 'runtime-partial', - partial: true, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'hello' }, - refs: { providerEventId: 'message-1' }, - }); - const final = makeRuntimeEvent({ - id: 'runtime-final', - ts: 2, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'hello world' }, - refs: { providerEventId: 'message-1' }, - }); - await runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', partial); - const partialsDirectory = join( - root, - 'sessions', - 'session-1', - 'runs', - 'run-1', - 'runtime-partials', - ); - const [partialFilename] = await readdir(partialsDirectory); - assert.ok(partialFilename); - const partialPath = join(partialsDirectory, partialFilename); - const retainedSnapshot = await readFile(partialPath); - await runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', final); - - // Simulate a crash after the immutable append but before replaceable - // partial cleanup, then drive recovery through the public exact retry. - await writeFile(partialPath, retainedSnapshot); - await runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', final); - - assert.deepEqual(await readdir(partialsDirectory), []); - assert.deepEqual( - (await runtimeEventStore.readImmutableRuntimeEvents('session-1', 'run-1')).map( - (event) => event.id, - ), - ['runtime-final'], - ); - }); - }); - - it('produces equivalent durable replay for differently chunked final output', async () => { - const replayFor = async (chunks: readonly string[]) => { - let replay: RuntimeEvent[] = []; - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - for (const [index, text] of chunks.entries()) { - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: `runtime-partial-${index}`, - ts: index + 1, - partial: true, - role: 'model', - author: 'agent', - content: { kind: 'text', text }, - refs: { providerEventId: 'message-1' }, - }), - ); - } - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: 'runtime-final', - ts: 10, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'same final output' }, - refs: { providerEventId: 'message-1' }, - }), - ); - replay = await runtimeEventStore.readRuntimeEvents('session-1', 'run-1'); - }); - return replay; - }; - - const finelyChunked = await replayFor(['same ', 'final ', 'output']); - const coarselyChunked = await replayFor(['same final output']); - - assert.deepEqual(finelyChunked, coarselyChunked); - assert.deepEqual( - finelyChunked.map((event) => event.id), - ['runtime-final'], - ); - }); - - it('ignores a stale partial snapshot when its final event is already durable', async () => { - await withStores(async (runStore, runtimeEventStore, root) => { - await runStore.createRun(makeHeader()); - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: 'runtime-final', - ts: 2, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'complete' }, - refs: { providerEventId: 'message-1' }, - }), - ); - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: 'runtime-stale-partial', - ts: 1, - partial: true, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'stale' }, - refs: { providerEventId: 'message-1' }, - }), - ); - - const events = await runtimeEventStore.readRuntimeEvents('session-1', 'run-1'); - const partialFiles = await readdir( - join(root, 'sessions', 'session-1', 'runs', 'run-1', 'runtime-partials'), - ).catch((error) => { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - }); - - assert.deepEqual( - events.map((event) => event.id), - ['runtime-final'], - ); - assert.deepEqual(partialFiles, []); - }); - }); - - it('restores a retained partial snapshot before a later terminal event', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader()); - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: 'runtime-user', - ts: 1, - }), - ); - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: 'runtime-partial', - ts: 2, - partial: true, - role: 'model', - author: 'agent', - content: { kind: 'text', text: 'retained output' }, - refs: { providerEventId: 'message-1' }, - }), - ); - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ - id: 'runtime-terminal', - ts: 1, - role: 'system', - author: 'system', - status: 'failed', - content: { kind: 'error', message: 'provider failed' }, - }), - ); - - const events = await runtimeEventStore.readRuntimeEvents('session-1', 'run-1'); - - assert.deepEqual( - events.map((event) => event.id), - ['runtime-user', 'runtime-partial', 'runtime-terminal'], - ); - }); - }); - - it('rejects durable corrupt runtime event lines instead of shortening the canonical ledger', async () => { - await withStores(async (runStore, runtimeEventStore, root) => { - await runStore.createRun(makeHeader()); - const runtimeEventsPath = join( - root, - 'sessions', - 'session-1', - 'runs', - 'run-1', - 'runtime-events.jsonl', - ); - await writeFile( - runtimeEventsPath, - JSON.stringify(makeRuntimeEvent({ id: 'runtime-1' })) + - '\n{"id":"corrupt"\n' + - JSON.stringify(makeRuntimeEvent({ id: 'runtime-2' })) + - '\n', - ); - - await assert.rejects( - () => runtimeEventStore.readRuntimeEvents('session-1', 'run-1'), - /Invalid RuntimeEvent JSONL line 2 for run run-1/, - ); - }); - }); - - it('re-establishes a terminal durability barrier without duplicating the event', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader({ invocationId: 'turn-1' })); - const terminal = makeRuntimeEvent({ - id: 'runtime-terminal', - role: 'system', - author: 'system', - status: 'completed', - content: undefined, - actions: { endInvocation: true }, - }); - await runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', terminal); - - await runtimeEventStore.ensureTerminalRuntimeEventDurable('session-1', 'run-1', terminal); - await runtimeEventStore.ensureTerminalRuntimeEventDurable('session-1', 'run-1', terminal); - - const events = await runtimeEventStore.readImmutableRuntimeEvents('session-1', 'run-1'); - assert.deepEqual( - events.map((event) => event.id), - ['runtime-terminal'], - ); - await assert.rejects( - () => - runtimeEventStore.ensureTerminalRuntimeEventDurable('session-1', 'run-1', { - ...terminal, - ts: terminal.ts + 1, - }), - /does not match the durable ledger record/, - ); - await assert.rejects( - () => - runtimeEventStore.ensureTerminalRuntimeEventDurable('session-1', 'run-1', { - ...terminal, - id: 'runtime-terminal-2', - }), - /already has terminal RuntimeEvent runtime-terminal/, - ); - assert.deepEqual( - (await runtimeEventStore.readImmutableRuntimeEvents('session-1', 'run-1')).map( - (event) => event.id, - ), - ['runtime-terminal'], - ); - }); - }); - - it('rejects a terminal event whose identity does not match the target run', async () => { - for (const mismatch of [ - { sessionId: 'other-session' }, - { runId: 'other-run' }, - { turnId: 'other-turn' }, - { invocationId: 'other-invocation' }, - ] satisfies Array>) { - await withStores(async (runStore, runtimeEventStore, root) => { - await runStore.createRun(makeHeader({ invocationId: 'turn-1' })); - const terminal = makeRuntimeEvent({ - id: 'runtime-terminal', - role: 'system', - author: 'system', - status: 'completed', - content: undefined, - actions: { endInvocation: true }, - ...mismatch, - }); - - await assert.rejects( - runtimeEventStore.ensureTerminalRuntimeEventDurable('session-1', 'run-1', terminal), - /RuntimeEvent identity does not match its run/, - ); - const ledger = await readFile( - join(root, 'sessions', 'session-1', 'runs', 'run-1', 'runtime-events.jsonl'), - 'utf8', - ).catch((error) => { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return ''; - throw error; - }); - assert.equal(ledger, ''); - }); - } - }); - - it('rejects complete schema-invalid and path-mismatched runtime events', async () => { - await withStores(async (runStore, runtimeEventStore, root) => { - await runStore.createRun(makeHeader({ invocationId: 'turn-1' })); - const runtimeEventsPath = join( - root, - 'sessions', - 'session-1', - 'runs', - 'run-1', - 'runtime-events.jsonl', - ); - for (const record of [ - {}, - makeRuntimeEvent({ sessionId: 'other-session' }), - makeRuntimeEvent({ runId: 'other-run' }), - makeRuntimeEvent({ turnId: 'other-turn' }), - makeRuntimeEvent({ invocationId: 'other-invocation' }), - ]) { - await writeFile(runtimeEventsPath, JSON.stringify(record)); - await assert.rejects( - () => runtimeEventStore.readRuntimeEvents('session-1', 'run-1'), - /Invalid RuntimeEvent JSONL line 1 for run run-1/, - ); - } - }); - }); - - it('ignores an unterminated partial runtime event tail', async () => { - await withStores(async (runStore, runtimeEventStore, root) => { - await runStore.createRun(makeHeader()); - const runtimeEventsPath = join( - root, - 'sessions', - 'session-1', - 'runs', - 'run-1', - 'runtime-events.jsonl', - ); - await writeFile( - runtimeEventsPath, - JSON.stringify(makeRuntimeEvent({ id: 'runtime-1' })) + '\n{"id":"partial"', - ); - - const events = await runtimeEventStore.readRuntimeEvents('session-1', 'run-1'); - assert.deepEqual( - events.map((event) => event.id), - ['runtime-1'], - ); - }); - }); - - it('rejects an invalid unterminated runtime event tail without changing it', async () => { - await withStores(async (runStore, runtimeEventStore, root) => { - await runStore.createRun(makeHeader()); - const runtimeEventsPath = join( - root, - 'sessions', - 'session-1', - 'runs', - 'run-1', - 'runtime-events.jsonl', - ); - const bytes = JSON.stringify(makeRuntimeEvent({ id: 'runtime-1' })) + '\n{"id":]'; - await writeFile(runtimeEventsPath, bytes, 'utf8'); - - await assert.rejects( - () => runtimeEventStore.readRuntimeEvents('session-1', 'run-1'), - /Invalid RuntimeEvent JSONL line 2 for run run-1/, - ); - assert.equal(await readFile(runtimeEventsPath, 'utf8'), bytes); - }); - }); - - it('reads session runtime events through RuntimeEventStore in stable chronology', async () => { - await withStores(async (runStore, runtimeEventStore) => { - await runStore.createRun(makeHeader({ runId: 'run-2', turnId: 'turn-2' })); - await runStore.createRun(makeHeader({ runId: 'run-1', turnId: 'turn-1' })); - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-2', - makeRuntimeEvent({ id: 'runtime-2', runId: 'run-2', turnId: 'turn-2', ts: 20 }), - ); - await runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - makeRuntimeEvent({ id: 'runtime-1', runId: 'run-1', turnId: 'turn-1', ts: 10 }), - ); - - const events = await runtimeEventStore.readSessionRuntimeEvents('session-1'); - - assert.deepEqual( - events.map((event) => event.id), - ['runtime-1', 'runtime-2'], - ); - }); - }); - - it('preflights durable steering identity before append and repairs a missing proof', async () => { - await withStores(async (runStore, runtimeEventStore, root) => { - await runStore.createRun(makeHeader()); - await runStore.createRun(makeHeader({ runId: 'run-2', turnId: 'turn-2' })); - const steering = makeRuntimeEvent({ - id: 'runtime-steering', - content: { kind: 'text', text: 'original steering', steering: true }, - refs: { providerEventId: 'message-steering' }, - }); - - await runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', steering); - await runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', steering); - assert.deepEqual(await runtimeEventStore.readImmutableRuntimeEvents('session-1', 'run-1'), [ - steering, - ]); - - await rm( - join(root, 'sessions', 'session-1', 'message-proofs', 'steering', 'message-steering.json'), - ); - const reopened = createRuntimeEventStore(root); - await reopened.appendRuntimeEvent('session-1', 'run-1', steering); - assert.deepEqual( - await reopened.readImmutableSteeringMessageProof('session-1', 'message-steering'), - { event: steering }, - ); - - const conflicting = makeRuntimeEvent({ - ...steering, - id: 'runtime-conflicting-steering', - runId: 'run-2', - turnId: 'turn-2', - content: { kind: 'text', text: 'conflicting steering', steering: true }, - }); - await assert.rejects( - () => reopened.appendRuntimeEvent('session-1', 'run-2', conflicting), - /Immutable steering message identity conflict: message-steering/, - ); - - const recovered = createRuntimeEventStore(root); - await recovered.repairImmutableSteeringMessageProofsForRecovery('session-1'); - assert.deepEqual( - await recovered.readImmutableSteeringMessageProof('session-1', 'message-steering'), - { event: steering }, - ); - assert.deepEqual(await recovered.readImmutableRuntimeEvents('session-1', 'run-1'), [ - steering, - ]); - assert.deepEqual(await recovered.readImmutableRuntimeEvents('session-1', 'run-2'), []); - }); - }); - - it('reports a durable canonical steering event when proof publication fails', { - skip: process.platform === 'win32', - }, async () => { - await withStores(async (runStore, runtimeEventStore, root) => { - await runStore.createRun(makeHeader()); - const steering = makeRuntimeEvent({ - id: 'runtime-steering', - content: { kind: 'text', text: 'persisted before proof', steering: true }, - refs: { providerEventId: 'message-steering' }, - }); - const proofDirectory = join(root, 'sessions', 'session-1', 'message-proofs', 'steering'); - await mkdir(proofDirectory, { recursive: true }); - await chmod(proofDirectory, 0o500); - - await assert.rejects( - () => runtimeEventStore.appendRuntimeEvent('session-1', 'run-1', steering), - (error: unknown) => { - assert.ok(error instanceof Error); - assert.equal(error.name, 'RuntimeEventPostEffectError'); - assert.match(error.message, /RuntimeEvent runtime-steering is durable/); - assert.ok(!(error instanceof DurableStoreWriteError)); - assert.ok(error.cause instanceof DurableStoreWriteError); - return true; - }, - ); - assert.deepEqual(await runtimeEventStore.readImmutableRuntimeEvents('session-1', 'run-1'), [ - steering, - ]); - - await chmod(proofDirectory, 0o700); - await rm(proofDirectory, { recursive: true }); - const recovered = createRuntimeEventStore(root); - await recovered.repairImmutableSteeringMessageProofsForRecovery('session-1'); - - assert.deepEqual( - await recovered.readImmutableSteeringMessageProof('session-1', 'message-steering'), - { event: steering }, - ); - assert.deepEqual(await recovered.readImmutableRuntimeEvents('session-1', 'run-1'), [ - steering, - ]); - }); - }); - - it('linearizes competing steering identities across runs in one session', async () => { - await withStores(async (runStore, runtimeEventStore, root) => { - await runStore.createRun(makeHeader()); - await runStore.createRun(makeHeader({ runId: 'run-2', turnId: 'turn-2' })); - const candidates = [ - makeRuntimeEvent({ - id: 'runtime-steering-1', - content: { kind: 'text', text: 'first candidate', steering: true }, - refs: { providerEventId: 'message-shared' }, - }), - makeRuntimeEvent({ - id: 'runtime-steering-2', - runId: 'run-2', - turnId: 'turn-2', - content: { kind: 'text', text: 'second candidate', steering: true }, - refs: { providerEventId: 'message-shared' }, - }), - ] as const; - - const results = await Promise.allSettled( - candidates.map((event) => - runtimeEventStore.appendRuntimeEvent('session-1', event.runId, event), - ), - ); - assert.equal(results.filter((result) => result.status === 'fulfilled').length, 1); - assert.equal(results.filter((result) => result.status === 'rejected').length, 1); - - const reopened = createRuntimeEventStore(root); - await reopened.repairImmutableSteeringMessageProofsForRecovery('session-1'); - const ledger = ( - await Promise.all( - ['run-1', 'run-2'].map((runId) => - reopened.readImmutableRuntimeEvents('session-1', runId), - ), - ) - ).flat(); - const proof = await reopened.readImmutableSteeringMessageProof('session-1', 'message-shared'); - assert.equal(ledger.length, 1); - assert.deepEqual(proof, { event: ledger[0] }); - assert.ok(candidates.some((candidate) => isDeepStrictEqual(candidate, ledger[0]))); - }); - }); -}); - -async function withStore( - fn: (store: ReturnType, root: string) => Promise, -): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-agent-run-store-')); - try { - await fn(createAgentRunStore(root), root); - } finally { - await rm(root, { recursive: true, force: true }); - } -} - -async function withStores( - fn: ( - runStore: ReturnType, - runtimeEventStore: ReturnType, - root: string, - ) => Promise, -): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-agent-run-store-')); - try { - await fn(createAgentRunStore(root), createRuntimeEventStore(root), root); - } finally { - await rm(root, { recursive: true, force: true }); - } -} - -function makeHeader(overrides: Partial = {}): AgentRunHeader { - return { - runId: 'run-1', - sessionId: 'session-1', - turnId: 'turn-1', - status: 'created', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/tmp/cwd', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 1, - ...overrides, - }; -} - -function makeEvent(overrides: Partial = {}): AgentRunEvent { - return { - type: 'run_started', - id: 'event-1', - runId: 'run-1', - sessionId: 'session-1', - turnId: 'turn-1', - ts: 1, - ...overrides, - }; -} - -function makeRuntimeEvent(overrides: Partial = {}): RuntimeEvent { - return { - id: 'runtime-1', - invocationId: 'turn-1', - runId: 'run-1', - sessionId: 'session-1', - turnId: 'turn-1', - ts: 1, - partial: false, - role: 'user', - author: 'user', - content: { kind: 'text', text: 'hello' }, - ...overrides, - }; -} diff --git a/packages/storage/src/__tests__/artifact-store.test.ts b/packages/storage/src/__tests__/artifact-store.test.ts index 2547763b24..ffd4fb1ec1 100644 --- a/packages/storage/src/__tests__/artifact-store.test.ts +++ b/packages/storage/src/__tests__/artifact-store.test.ts @@ -30,6 +30,7 @@ import { sanitizeArtifactName, } from '../artifact-store.js'; import { withArtifactWriterLock } from '../artifact-writer-lock.js'; +import { createSqliteArtifactMetadataRepository } from '../sqlite-artifact-metadata.js'; describe('SQLite Artifact store', () => { test('creates a missing workspace root before acquiring the writer lock', async () => { @@ -343,59 +344,6 @@ describe('SQLite Artifact store', () => { }); }); - test('reads merge-base names and replays the same legacy inputs without renaming', async () => { - await withWorkspace(async (root) => { - const fixtures = [ - { id: 'legacy-dotfile', inputName: '-.gitignore', storedName: '.gitignore' }, - { - id: 'legacy-leading-hyphen', - inputName: '. -report.txt', - storedName: '-report.txt', - }, - { id: 'legacy-hyphen', inputName: 'report-', storedName: 'report-' }, - { id: 'legacy-dot', inputName: 'report.', storedName: 'report.' }, - { - id: 'legacy-surrogate', - inputName: `${'a'.repeat(119)}😀tail`, - storedName: `${'a'.repeat(119)}\ud83d`, - }, - { - id: 'legacy-trailing-space', - inputName: `${'a'.repeat(119)} tail`, - storedName: `${'a'.repeat(119)} `, - }, - ]; - const records = fixtures.map(({ id, storedName }) => - canonicalRecord({ - id, - sessionId: 'session-1', - name: storedName, - sizeBytes: Buffer.byteLength(id), - }), - ); - for (const record of records) { - const payloadPath = join(root, 'artifacts', record.relativePath); - await mkdir(dirname(payloadPath), { recursive: true }); - await writeFile(payloadPath, record.id, 'utf8'); - } - await writeArtifactMetadata(root, records); - - const reopened = createArtifactStore(root); - for (const [index, fixture] of fixtures.entries()) { - const record = records[index]!; - assert.deepEqual(await reopened.get(record.id), record); - assert.deepEqual(await reopened.readText(record.id), { ok: true, text: record.id }); - assert.deepEqual( - await reopened.create({ - ...artifactInput(fixture.id, fixture.id, 1), - name: fixture.inputName, - }), - record, - ); - } - }); - }); - test('rejects metadata names that neither writer could have produced', async () => { for (const name of ['short ', 'embedded\ttab', 'embedded\nnewline']) { await withWorkspace(async (root) => { @@ -404,7 +352,7 @@ describe('SQLite Artifact store', () => { ]); await assert.rejects( () => createArtifactStore(root).list('session-1', { includeDeleted: true }), - /Invalid artifact metadata line 1/, + /Invalid artifact metadata record 1/, ); }); } @@ -757,17 +705,17 @@ describe('SQLite Artifact store', () => { test('self-managed SQLite mutation adopts an exact stable-id target-only orphan', async () => { await withWorkspace(async (root) => { const input = { - ...artifactInput('legacy-orphan', 'orphan bytes', 1), - name: '-.gitignore', + ...artifactInput('target-orphan', 'orphan bytes', 1), + name: 'report.txt', }; - const orphanPath = join(root, 'artifacts', 'session-1', 'legacy-orphan-.gitignore'); + const orphanPath = join(root, 'artifacts', 'session-1', 'target-orphan-report.txt'); await mkdir(dirname(orphanPath), { recursive: true }); await writeFile(orphanPath, input.content, { flag: 'wx' }); const bare = createArtifactStore(root); const adopted = await bare.create(input); - assert.equal(adopted.name, '.gitignore'); - assert.equal(adopted.relativePath, 'session-1/legacy-orphan-.gitignore'); + assert.equal(adopted.name, 'report.txt'); + assert.equal(adopted.relativePath, 'session-1/target-orphan-report.txt'); assert.deepEqual(await createArtifactStore(root).readText(adopted.id), { ok: true, text: input.content, @@ -830,26 +778,6 @@ describe('SQLite Artifact store', () => { }); }); - await withWorkspace(async (root) => { - const input = { - ...artifactInput('legacy-surrogate-orphan', 'surrogate bytes', 1), - name: `${'a'.repeat(119)}😀tail`, - }; - const legacyName = `${'a'.repeat(119)}\ud83d`; - const orphanPath = join(root, 'artifacts', 'session-1', `${input.id}-${legacyName}`); - await mkdir(dirname(orphanPath), { recursive: true }); - await writeFile(orphanPath, input.content, { flag: 'wx' }); - - const authority = createArtifactStoreWriteAuthority(root); - await authority.recover(); - const adopted = await authority.store.create(input); - assert.equal(adopted.name, legacyName); - assert.deepEqual(await authority.store.readText(adopted.id), { - ok: true, - text: input.content, - }); - }); - await withWorkspace(async (root) => { const input = artifactInput('late-payload', 'same bytes', 1); const authority = createArtifactStoreWriteAuthority(root); @@ -868,19 +796,19 @@ describe('SQLite Artifact store', () => { }); }); - test('write recovery durably removes only canonical root transaction temp files', async () => { + test('write recovery removes only canonical purge-intent temp files', async () => { await withWorkspace(async (root) => { const artifactRoot = join(root, 'artifacts'); await mkdir(artifactRoot, { recursive: true }); const canonicalTemps = [ - join(artifactRoot, 'metadata.jsonl.123.00000000-0000-4000-8000-000000000000.tmp'), - join(artifactRoot, 'metadata.jsonl.123.1700000000000.tmp'), join( artifactRoot, '.artifact-purge-intent.json.123.00000000-0000-4000-8000-000000000000.tmp', ), ]; const unknownFiles = [ + join(artifactRoot, 'metadata.jsonl.123.00000000-0000-4000-8000-000000000000.tmp'), + join(artifactRoot, 'metadata.jsonl.123.1700000000000.tmp'), join(artifactRoot, 'metadata.jsonl.123.not-a-uuid.tmp'), join(artifactRoot, '.artifact-purge-intent.json.123.not-a-uuid.tmp'), join(artifactRoot, 'other.jsonl.123.00000000-0000-4000-8000-000000000000.tmp'), @@ -914,17 +842,7 @@ describe('SQLite Artifact store', () => { }); }); - test('fails closed on malformed metadata and mismatched publication residue', async () => { - await withWorkspace(async (root) => { - const metadataPath = join(root, 'artifacts', 'metadata.jsonl'); - await mkdir(join(root, 'artifacts'), { recursive: true }); - await writeFile(metadataPath, '{"id":"incomplete","status":"live"}\n', 'utf8'); - await assert.rejects( - () => createArtifactStore(root).list('session-1'), - /Invalid artifact metadata line 1/, - ); - }); - + test('fails closed on mismatched publication residue', async () => { await withWorkspace(async (root) => { const residue = await createPublicationResidue(root, 'mismatch', 'file.txt', 'payload'); await rm(residue.targetPath); @@ -952,7 +870,7 @@ describe('SQLite Artifact store', () => { await writeArtifactMetadata(root, [recordWithIdentity(field, value)]); await assert.rejects( () => createArtifactStore(root).list('session-1', { includeDeleted: true }), - /Invalid artifact metadata line 1/, + /Invalid artifact metadata record 1/, ); }); } @@ -984,20 +902,6 @@ describe('SQLite Artifact store', () => { } }); - test('fails loud when the metadata path cannot be read as a file', async () => { - await withWorkspace(async (root) => { - const metadataPath = join(root, 'artifacts', 'metadata.jsonl'); - await mkdir(metadataPath, { recursive: true }); - - await assert.rejects( - () => createArtifactStore(root).list('session-1'), - (error: unknown) => - typeof error === 'object' && error !== null && 'code' in error && error.code === 'EISDIR', - ); - assert.equal((await stat(metadataPath)).isDirectory(), true); - }); - }); - test('soft delete tombstones bytes until idempotent purge', async () => { await withWorkspace(async (root) => { const store = createArtifactStore(root); @@ -1248,7 +1152,7 @@ describe('SQLite Artifact store', () => { test('self-managed SQLite stores recover an interrupted purge before the next write', async () => { await withWorkspace(async (root) => { const store = createArtifactStore(root); - const record = await store.create(artifactInput('legacy-purge', 'remove me', 1)); + const record = await store.create(artifactInput('interrupted-purge', 'remove me', 1)); await rm(join(root, 'artifacts', record.relativePath)); await writeFile( join(root, 'artifacts', '.artifact-purge-intent.json'), @@ -1539,14 +1443,13 @@ async function writeArtifactMetadata( root: string, records: readonly ArtifactRecord[], ): Promise { - const metadataPath = join(root, 'artifacts', 'metadata.jsonl'); - await mkdir(dirname(metadataPath), { recursive: true }); - await writeFile( - metadataPath, - records.map((record) => JSON.stringify(record)).join('\n') + '\n', - 'utf8', - ); - return metadataPath; + const repository = createSqliteArtifactMetadataRepository(root); + try { + repository.replaceAll(records); + } finally { + repository.close(); + } + return join(root, 'runtime.sqlite'); } async function createSymlinkOrSkip( diff --git a/packages/storage/src/__tests__/artifact-writer-lock.test.ts b/packages/storage/src/__tests__/artifact-writer-lock.test.ts index 987057020d..1b300ce1e1 100644 --- a/packages/storage/src/__tests__/artifact-writer-lock.test.ts +++ b/packages/storage/src/__tests__/artifact-writer-lock.test.ts @@ -397,15 +397,22 @@ test('bundle export holds Artifact authority through selected-session projection }); await assertPending(bundleExport, 'bundle export'); - const projectedTranscript = withArtifactWriterLock(stateRoot, () => - readFile(join(destinationRoot, 'sessions', session.id, 'session.jsonl'), 'utf8'), - ); - await assertPending(projectedTranscript, 'writer queued after bundle export'); + const projectedMessages = withArtifactWriterLock(stateRoot, async () => { + const projectedSessions = createSessionStore(destinationRoot); + try { + return await projectedSessions.readMessagesSnapshot(session.id); + } finally { + await projectedSessions.close?.(); + } + }); + await assertPending(projectedMessages, 'writer queued after bundle export'); await releaseHolder(holder); - assert.match( - await withTimeout(projectedTranscript, OPERATION_TIMEOUT_MS, 'selected-session projection'), - /portable transcript/, + assert.equal( + ( + await withTimeout(projectedMessages, OPERATION_TIMEOUT_MS, 'selected-session projection') + ).find((message) => message.type === 'user')?.text, + 'portable transcript', ); await withTimeout(bundleExport, OPERATION_TIMEOUT_MS, 'bundle export'); } finally { diff --git a/packages/storage/src/__tests__/automation-authority.test.ts b/packages/storage/src/__tests__/automation-authority.test.ts index 0d3f80ca84..75dc3dc15f 100644 --- a/packages/storage/src/__tests__/automation-authority.test.ts +++ b/packages/storage/src/__tests__/automation-authority.test.ts @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; @@ -73,68 +73,6 @@ describe('interactive Automation authority', () => { }); }); - test('imports the legacy file once and fails closed if that source changes after cutover', async () => { - await withInteractiveRoot(async ({ root, capability }) => { - await writeLegacy(root, [definition()]); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(owner); - if (!owner) return; - try { - const writer = await openInteractiveAutomationAuthorityForWrite(owner.lease); - const imported = await writer.read(); - assert.equal(imported.revision, 1); - assert.deepEqual( - imported.automations.map((automation) => automation.id), - ['automation-1'], - ); - writer.close(); - - await writeLegacy(root, [{ ...definition(), name: 'changed legacy source' }]); - await assert.rejects( - () => openInteractiveAutomationAuthorityForWrite(owner.lease), - /cutover|fingerprint|source/i, - ); - } finally { - if (!owner.closed) await owner.close(); - } - }); - }); - - test('imports the full legacy text contract and bounds historical failure diagnostics', async () => { - await withInteractiveRoot(async ({ root, capability }) => { - await writeLegacy(root, [ - { - ...definition(), - name: '名'.repeat(100), - prompt: '提'.repeat(2_000), - lastError: '故'.repeat(4_000), - }, - { - ...definition(), - id: 'automation-2', - createdAt: 2, - lastError: '', - }, - ]); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(owner); - if (!owner) return; - try { - const writer = await openInteractiveAutomationAuthorityForWrite(owner.lease); - const automations = (await writer.read()).automations; - const imported = automations[0]; - assert.equal(imported?.name, '名'.repeat(100)); - assert.equal(imported?.prompt, '提'.repeat(2_000)); - assert.equal(imported?.lastError, '故'.repeat(2_000)); - assert.equal(Buffer.byteLength(imported?.lastError ?? '', 'utf8'), 6_000); - assert.equal(automations[1]?.lastError, null); - writer.close(); - } finally { - if (!owner.closed) await owner.close(); - } - }); - }); - test('updates only changed definition rows while advancing a full-snapshot revision', async () => { await withInteractiveRoot(async ({ root, capability }) => { const owner = await tryAcquireInteractiveRootOwner(capability); @@ -311,17 +249,6 @@ function pendingFire(): AutomationPendingFire { }; } -async function writeLegacy( - root: string, - automations: readonly AutomationDefinition[], -): Promise { - await writeFile( - join(root, 'automations.json'), - `${JSON.stringify({ version: 1, automations }, null, 2)}\n`, - 'utf8', - ); -} - async function withWriter( run: (input: { writer: InteractiveAutomationAuthorityWriter }) => Promise, ): Promise { diff --git a/packages/storage/src/__tests__/automation-store.test.ts b/packages/storage/src/__tests__/automation-store.test.ts deleted file mode 100644 index bfd8bd2dbc..0000000000 --- a/packages/storage/src/__tests__/automation-store.test.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { describe, test, beforeEach, afterEach } from 'node:test'; -import assert from 'node:assert/strict'; -import { mkdir, rm, readFile } from 'node:fs/promises'; -import { join } from 'node:path'; -import { tmpdir } from 'node:os'; -import { createAutomationStore } from '../automation-store.js'; - -interface TestRecord { - id: string; - name: string; - status?: string; -} - -const TEST_DIR = join(tmpdir(), `maka-automation-store-test-${process.pid}`); - -describe('AutomationStore', () => { - beforeEach(async () => { - await mkdir(TEST_DIR, { recursive: true }); - }); - - afterEach(async () => { - await rm(TEST_DIR, { recursive: true, force: true }); - }); - - test('loadAll returns empty array when file does not exist', async () => { - const store = createAutomationStore(TEST_DIR); - const result = await store.loadAll(); - assert.deepEqual(result, []); - }); - - test('save persists automation to disk', async () => { - const store = createAutomationStore(TEST_DIR); - await store.save({ id: 'auto-1', name: 'test', status: 'active' }); - - const raw = await readFile(join(TEST_DIR, 'automations.json'), 'utf8'); - const parsed = JSON.parse(raw); - assert.equal(parsed.version, 1); - assert.equal(parsed.automations.length, 1); - assert.equal(parsed.automations[0].id, 'auto-1'); - }); - - test('loadAll reads back saved automations', async () => { - const store = createAutomationStore(TEST_DIR); - await store.save({ id: 'auto-1', name: 'first' }); - await store.save({ id: 'auto-2', name: 'second' }); - - const result = await store.loadAll(); - assert.equal(result.length, 2); - assert.equal(result[0].id, 'auto-1'); - assert.equal(result[1].id, 'auto-2'); - }); - - test('save updates existing automation by id', async () => { - const store = createAutomationStore(TEST_DIR); - await store.save({ id: 'auto-1', name: 'original' }); - await store.save({ id: 'auto-1', name: 'updated' }); - - const result = await store.loadAll(); - assert.equal(result.length, 1); - assert.equal(result[0].name, 'updated'); - }); - - test('remove deletes automation from file', async () => { - const store = createAutomationStore(TEST_DIR); - await store.save({ id: 'auto-1', name: 'a' }); - await store.save({ id: 'auto-2', name: 'b' }); - - await store.remove('auto-1'); - const result = await store.loadAll(); - assert.equal(result.length, 1); - assert.equal(result[0].id, 'auto-2'); - }); - - test('remove with nonexistent id is a no-op', async () => { - const store = createAutomationStore(TEST_DIR); - await store.save({ id: 'auto-1', name: 'a' }); - - await store.remove('nonexistent'); - const result = await store.loadAll(); - assert.equal(result.length, 1); - }); - - test('sync replaces all automations at once', async () => { - const store = createAutomationStore(TEST_DIR); - await store.save({ id: 'old-1', name: 'old' }); - - await store.sync([ - { id: 'new-1', name: 'alpha' }, - { id: 'new-2', name: 'beta' }, - ]); - - const result = await store.loadAll(); - assert.equal(result.length, 2); - assert.equal(result[0].id, 'new-1'); - assert.equal(result[1].id, 'new-2'); - }); - - test('loadAll FAILS LOUD on a corrupt file (never masks unreadable data as empty)', async () => { - const { writeFile } = await import('node:fs/promises'); - await writeFile(join(TEST_DIR, 'automations.json'), 'not valid json{{{', 'utf8'); - - const store = createAutomationStore(TEST_DIR); - // Returning [] here would let a subsequent full-overwrite sync erase real data. - await assert.rejects(() => store.loadAll(), /not valid JSON/); - }); - - test('loadAll FAILS LOUD on an unrecognized version/shape', async () => { - const { writeFile } = await import('node:fs/promises'); - await writeFile( - join(TEST_DIR, 'automations.json'), - JSON.stringify({ version: 99, automations: [] }), - 'utf8', - ); - - const store = createAutomationStore(TEST_DIR); - await assert.rejects(() => store.loadAll(), /unrecognized shape or version/); - }); - - test('atomic write: file is not corrupted on concurrent saves', async () => { - const store = createAutomationStore(TEST_DIR); - await Promise.all([ - store.save({ id: 'a', name: 'alpha' }), - store.save({ id: 'b', name: 'beta' }), - store.save({ id: 'c', name: 'gamma' }), - ]); - - const result = await store.loadAll(); - assert.deepEqual( - [...result].sort((left, right) => left.id.localeCompare(right.id)), - [ - { id: 'a', name: 'alpha' }, - { id: 'b', name: 'beta' }, - { id: 'c', name: 'gamma' }, - ], - ); - }); -}); diff --git a/packages/storage/src/__tests__/claimed-agent-graph-root-admission.test.ts b/packages/storage/src/__tests__/claimed-agent-graph-root-admission.test.ts index c435da4ef1..2a7a8fdaa1 100644 --- a/packages/storage/src/__tests__/claimed-agent-graph-root-admission.test.ts +++ b/packages/storage/src/__tests__/claimed-agent-graph-root-admission.test.ts @@ -5,12 +5,12 @@ import { join } from 'node:path'; import { describe, test } from 'node:test'; import { AGENT_GRAPH_INTENT_CLAIM_SCHEMA_VERSION } from '@maka/core/agent-graph-control'; import type { RootExecutionDescriptor } from '@maka/core/agent-run'; -import { createAgentRunStore, type AdmitRootTurnInput } from '../agent-run-store.js'; +import { createSqliteAgentRunStore, type AdmitRootTurnInput } from '../agent-run-store.js'; describe('claimed agent graph root admission', () => { test('round-trips an exact, deeply frozen durable descriptor', async () => { await withTempRoot(async (root) => { - const store = createAgentRunStore(root); + const store = createSqliteAgentRunStore(root); const result = await store.admitRootTurn(admissionInput()); assert.equal(result.kind, 'admitted'); assert.equal(result.admission.execution.kind, 'claimed_agent_graph_intent'); @@ -21,7 +21,7 @@ describe('claimed agent graph root admission', () => { assert.equal(Object.isFrozen(result.admission.execution), true); assert.equal(Object.isFrozen(result.admission.execution.claim), true); - const reopened = createAgentRunStore(root); + const reopened = createSqliteAgentRunStore(root); const stored = await reopened.readRootTurnAdmission('session-child', 'turn-next'); assert.deepEqual(stored, result.admission); assert.equal(Object.isFrozen(stored?.execution), true); @@ -41,7 +41,7 @@ describe('claimed agent graph root admission', () => { test('rejects descriptor unknown fields and malformed claims', async () => { await withTempRoot(async (root) => { - const store = createAgentRunStore(root); + const store = createSqliteAgentRunStore(root); await assert.rejects( () => store.admitRootTurn( @@ -87,7 +87,7 @@ describe('claimed agent graph root admission', () => { test('rejects every claim target identity drift before writing admission', async () => { await withTempRoot(async (root) => { - const store = createAgentRunStore(root); + const store = createSqliteAgentRunStore(root); for (const [field, value] of [ ['targetSessionId', 'different-session'], ['targetTurnId', 'different-turn'], @@ -127,7 +127,7 @@ describe('claimed agent graph root admission', () => { test('rejects queue sources and a missing canonical UserMessage', async () => { await withTempRoot(async (root) => { - const store = createAgentRunStore(root); + const store = createSqliteAgentRunStore(root); await assert.rejects( () => store.admitRootTurn( diff --git a/packages/storage/src/__tests__/credential-migration.test.ts b/packages/storage/src/__tests__/credential-migration.test.ts deleted file mode 100644 index b35fb32656..0000000000 --- a/packages/storage/src/__tests__/credential-migration.test.ts +++ /dev/null @@ -1,179 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, it } from 'node:test'; -import { - migrateLegacyCredentialFile, - type LegacyCredentialDecryptor, -} from '../credential-store.js'; - -// Behavioral tests over real temp files for the one-time migration. The -// decryptor is injected (the crypto is the caller's concern), so these run -// under plain `node --test`. No fixed sleeps: concurrency is proven by racing -// two real migrations, not by waiting a guessed number of milliseconds. - -/** Fake decryptor: "decrypt" strips an `enc:` prefix so every value is proven - * to round-trip through decrypt(). */ -function fakeDecryptor(available: boolean): LegacyCredentialDecryptor { - return { - isAvailable: () => available, - decrypt: (stored) => stored.replace(/^enc:/, ''), - }; -} - -function legacyFile(values: Record): string { - return JSON.stringify({ values }); -} - -async function withWorkspace(fn: (path: string) => Promise): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-cred-mig-')); - try { - return await fn(join(root, 'credentials.json')); - } finally { - await rm(root, { recursive: true, force: true }); - } -} - -const isPosix = process.platform !== 'win32'; - -describe('migrateLegacyCredentialFile', () => { - it('decrypts ALL secret kinds to v1 plaintext-0600 in place', async () => { - await withWorkspace(async (path) => { - // Full scope: not just API keys — bot/proxy/etc. secrets migrate too. - await writeFile( - path, - legacyFile({ - 'openai:apiKey': 'enc:sk-1', - 'settings:bot:telegram:botToken': 'enc:tok-2', - 'settings:network-proxy:proxyPassword': 'enc:pw-3', - }), - 'utf8', - ); - - await migrateLegacyCredentialFile(path, fakeDecryptor(true)); - - const after = JSON.parse(await readFile(path, 'utf8')) as { - version: number; - values: Record; - }; - assert.equal(after.version, 1); - assert.deepEqual(after.values, { - 'openai:apiKey': 'sk-1', - 'settings:bot:telegram:botToken': 'tok-2', - 'settings:network-proxy:proxyPassword': 'pw-3', - }); - if (isPosix) { - assert.equal((await stat(path)).mode & 0o777, 0o600); // owner-only at rest - } - }); - }); - - it('aborts and leaves the legacy file intact when the decryptor is unavailable', async () => { - await withWorkspace(async (path) => { - const original = legacyFile({ 'openai:apiKey': 'enc:sk-1' }); - await writeFile(path, original, 'utf8'); - - await assert.rejects(migrateLegacyCredentialFile(path, fakeDecryptor(false)), /unavailable/); - assert.equal(await readFile(path, 'utf8'), original); // untouched — no data loss - }); - }); - - it('migrates an EMPTY legacy file even when the decryptor is unavailable', async () => { - await withWorkspace(async (path) => { - // A user who deleted their last secret under the old store left `values: {}`. - // There is nothing to decrypt, so an unavailable decryptor must NOT block - // the stamp — otherwise the v1 store (which refuses unversioned files) - // would be permanently unusable for that workspace. - await writeFile(path, legacyFile({}), 'utf8'); - - await migrateLegacyCredentialFile(path, fakeDecryptor(false)); - - const after = JSON.parse(await readFile(path, 'utf8')) as { - version: number; - values: Record; - }; - assert.equal(after.version, 1); - assert.deepEqual(after.values, {}); - }); - }); - - it('is a no-op on an already-migrated v1 file', async () => { - await withWorkspace(async (path) => { - const v1 = JSON.stringify({ version: 1, values: { 'openai:apiKey': 'sk-1' } }); - await writeFile(path, v1, 'utf8'); - - await migrateLegacyCredentialFile(path, fakeDecryptor(true)); - assert.equal(await readFile(path, 'utf8'), v1); // unchanged - }); - }); - - it('does not wait on a leftover lock when the file is already v1', async () => { - await withWorkspace(async (path) => { - const v1 = JSON.stringify({ version: 1, values: { 'openai:apiKey': 'sk-1' } }); - await writeFile(path, v1, 'utf8'); - await mkdir(`${path}.lock`); - - await migrateLegacyCredentialFile(path, fakeDecryptor(true)); - - assert.equal(await readFile(path, 'utf8'), v1); // unchanged - await stat(`${path}.lock`); // still owned by whoever left it behind - }); - }); - - it('refuses a malformed legacy file rather than tombstone it empty', async () => { - await withWorkspace(async (path) => { - const malformed = JSON.stringify({ foo: 1 }); // no version, no values - await writeFile(path, malformed, 'utf8'); - - await assert.rejects(migrateLegacyCredentialFile(path, fakeDecryptor(true)), /malformed/); - assert.equal(await readFile(path, 'utf8'), malformed); // untouched - }); - }); - - it('refuses a legacy file whose values are not all strings, leaving it untouched', async () => { - // A number, null, or nested object can't be a safeStorage-encrypted string; - // it must fail closed BEFORE reaching the decryptor (mirrors the v1 reader), - // not be fed garbage to decrypt. - for (const badValue of ['123', 'null', '{ "nested": "no" }']) { - await withWorkspace(async (path) => { - const original = `{ "values": { "x:apiKey": ${badValue} } }`; - await writeFile(path, original, 'utf8'); - - await assert.rejects( - migrateLegacyCredentialFile(path, fakeDecryptor(true)), - /not a string/, - ); - assert.equal(await readFile(path, 'utf8'), original); // byte-for-byte untouched - }); - } - }); - - it('is a no-op when there is no credentials file', async () => { - await withWorkspace(async (path) => { - await migrateLegacyCredentialFile(path, fakeDecryptor(true)); - await assert.rejects(stat(path)); // still absent, no file created - }); - }); - - it('serializes two racing migrations: one migrates, the other no-ops on the re-read', async () => { - await withWorkspace(async (path) => { - await writeFile(path, legacyFile({ 'openai:apiKey': 'enc:sk-1' }), 'utf8'); - - // The shared lock serializes the two; the loser re-reads inside the lock, - // sees v1, and no-ops instead of decrypting a stale snapshot again. Both - // resolve cleanly and the result is the single correct v1 file. - await Promise.all([ - migrateLegacyCredentialFile(path, fakeDecryptor(true)), - migrateLegacyCredentialFile(path, fakeDecryptor(true)), - ]); - - const after = JSON.parse(await readFile(path, 'utf8')) as { - version: number; - values: Record; - }; - assert.equal(after.version, 1); - assert.deepEqual(after.values, { 'openai:apiKey': 'sk-1' }); - }); - }); -}); diff --git a/packages/storage/src/__tests__/deep-research-store.test.ts b/packages/storage/src/__tests__/deep-research-store.test.ts deleted file mode 100644 index 107e332a08..0000000000 --- a/packages/storage/src/__tests__/deep-research-store.test.ts +++ /dev/null @@ -1,362 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdtemp, mkdir, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { dirname, join } from 'node:path'; -import { describe, it } from 'node:test'; -import { createDeepResearchStore } from '../deep-research-store.js'; - -const SESSION_ID = 'session-1'; -const HASH = `sha256:${'b'.repeat(64)}`; - -async function withTempRoot(fn: (root: string) => Promise): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-deep-research-')); - try { - await fn(root); - } finally { - await rm(root, { recursive: true, force: true }); - } -} - -describe('DeepResearchStore', () => { - it('persists an append-only run that can resume after store recreation', async () => { - await withTempRoot(async (root) => { - let id = 0; - let now = 100; - const store = createDeepResearchStore(root, { - newId: () => `id-${++id}`, - now: () => ++now, - }); - - await store.start(SESSION_ID, ' Reproduce durable Deep Research in Maka. ', 'deep', { - runId: 'run-1', - turnId: 'turn-1', - toolCallId: 'call-start', - }); - await store.recordArtifact(SESSION_ID, { - artifactId: 'source-1', - role: 'source', - name: 'paper.md', - createdAt: 110, - locator: 'https://arxiv.org/abs/2602.01566', - contentHash: HASH, - sourceArtifactIds: [], - }); - await store.recordCheckpoint(SESSION_ID, { - round: 1, - stage: 'knowledge_base', - status: 'active', - summary: 'Paper and Maka architecture are mapped.', - openQuestions: ['Which slice is smallest?'], - nextSteps: ['Build the event ledger.'], - taskIds: ['T1'], - artifactIds: ['source-1'], - }); - for (const [key, artifactId] of [ - ['conclusion', 'section-conclusion'], - ['source_evidence', 'section-source-evidence'], - ['borrow_diverge_risk_gate', 'section-tradeoffs'], - ['implementation_recommendations', 'section-implementation'], - ['verification', 'section-verification'], - ] as const) { - await store.recordArtifact(SESSION_ID, { - artifactId, - role: 'report_section', - name: `${artifactId}.md`, - createdAt: 115, - contentHash: HASH, - sourceArtifactIds: ['source-1'], - reportSectionKey: key, - reportSectionStatus: 'completed', - }); - } - await store.recordArtifact(SESSION_ID, { - artifactId: 'report-1', - role: 'report', - name: 'report.md', - createdAt: 120, - contentHash: HASH, - sourceArtifactIds: ['source-1'], - }); - await store.recordArtifact(SESSION_ID, { - artifactId: 'handoff-1', - role: 'handoff', - name: 'handoff.md', - createdAt: 121, - contentHash: HASH, - sourceArtifactIds: ['source-1'], - }); - for (const itemId of [ - 'project_entrypoints', - 'core_flow', - 'boundaries', - 'verification_evidence', - ]) { - await store.updateChecklist(SESSION_ID, { - itemId, - status: 'completed', - evidenceArtifactIds: ['source-1'], - }); - } - const completed = await store.complete( - SESSION_ID, - 'report-1', - { - artifactId: 'handoff-1', - implementationTasks: ['Implement the workspace.'], - recommendedIssues: ['Track the UI slice.'], - recommendedPullRequests: [], - verificationCommands: ['npm test'], - }, - { runId: 'run-2', turnId: 'turn-2', toolCallId: 'call-complete' }, - ); - await store.complete( - SESSION_ID, - 'report-1', - { - artifactId: 'handoff-1', - implementationTasks: ['Implement the workspace.'], - recommendedIssues: ['Track the UI slice.'], - recommendedPullRequests: [], - verificationCommands: ['npm test'], - }, - { runId: 'run-2', turnId: 'turn-2', toolCallId: 'call-complete' }, - ); - await assert.rejects( - () => - store.complete( - SESSION_ID, - 'report-1', - { - artifactId: 'handoff-1', - implementationTasks: ['Implement the workspace.'], - recommendedIssues: ['Track the UI slice.'], - recommendedPullRequests: [], - verificationCommands: ['npm run test:fast'], - }, - { runId: 'run-2', turnId: 'turn-2', toolCallId: 'call-complete' }, - ), - /retried with different input/, - ); - - assert.equal(completed.objective, 'Reproduce durable Deep Research in Maka.'); - assert.equal(completed.scopeLevel, 'deep'); - assert.equal(completed.status, 'completed'); - assert.equal(completed.reportArtifactId, 'report-1'); - - const reopened = createDeepResearchStore(root); - const restored = await reopened.read(SESSION_ID); - assert.equal(restored?.status, 'completed'); - assert.equal(restored?.round, 1); - assert.equal(restored?.checkpoints[0]?.taskIds[0], 'T1'); - - const events = await reopened.readEvents(SESSION_ID); - assert.equal(events.length, 15); - assert.equal(events[0]?.type, 'research_started'); - assert.equal(events.at(-1)?.type, 'research_completed'); - assert.equal(events.filter((event) => event.type === 'research_checklist_updated').length, 4); - assert.equal(events[0]?.refs?.toolCallId, 'call-start'); - assert.equal(events.at(-1)?.refs?.toolCallId, 'call-complete'); - - const eventText = await readFile( - join(root, 'sessions', SESSION_ID, 'deep-research', 'events.jsonl'), - 'utf8', - ); - assert.equal(eventText.trim().split('\n').length, 15); - }); - }); - - it('rejects a mutation that would violate source traceability without appending it', async () => { - await withTempRoot(async (root) => { - let id = 0; - const store = createDeepResearchStore(root, { - newId: () => `id-${++id}`, - now: () => 100, - }); - await store.start(SESSION_ID, 'Trace every claim.', 'standard'); - - await assert.rejects( - () => - store.recordArtifact(SESSION_ID, { - artifactId: 'note-1', - role: 'evidence_note', - name: 'note.md', - createdAt: 100, - contentHash: HASH, - sourceArtifactIds: ['missing-source'], - }), - /non-source artifact missing-source/, - ); - - assert.equal((await store.readEvents(SESSION_ID)).length, 1); - }); - }); - - it('accepts exact mutation replays and rejects conflicting input for the same tool call', async () => { - await withTempRoot(async (root) => { - let id = 0; - const store = createDeepResearchStore(root, { - newId: () => `id-${++id}`, - now: () => 100, - }); - const startContext = { turnId: 'turn-1', toolCallId: 'call-start' }; - await store.start(SESSION_ID, 'Trace every decision.', 'standard', startContext); - await store.start(SESSION_ID, 'Trace every decision.', 'standard', startContext); - await assert.rejects( - () => store.start(SESSION_ID, 'A different objective.', 'standard', startContext), - /retried with different input/, - ); - - const source = { - artifactId: 'source-1', - role: 'source' as const, - name: 'source.md', - summary: 'Primary source.', - createdAt: 100, - locator: 'https://example.com/source', - contentHash: HASH, - sourceArtifactIds: [], - }; - const artifactContext = { turnId: 'turn-1', toolCallId: 'call-artifact' }; - await store.recordArtifact(SESSION_ID, source, artifactContext); - await store.recordArtifact(SESSION_ID, source, artifactContext); - await assert.rejects( - () => - store.recordArtifact( - SESSION_ID, - { ...source, locator: 'https://example.com/other' }, - artifactContext, - ), - /retried with different input/, - ); - await assert.rejects( - () => - store.recordArtifact( - SESSION_ID, - { ...source, name: 'renamed-source.md' }, - artifactContext, - ), - /retried with different input/, - ); - await assert.rejects( - () => - store.recordArtifact( - SESSION_ID, - { ...source, summary: 'Different summary.' }, - artifactContext, - ), - /retried with different input/, - ); - await assert.rejects( - () => - store.recordCheckpoint( - SESSION_ID, - { - round: 1, - stage: 'knowledge_base', - status: 'active', - summary: 'Cross-tool replay.', - openQuestions: [], - nextSteps: [], - taskIds: [], - artifactIds: [], - }, - artifactContext, - ), - /already used for research_artifact_recorded/, - ); - - const checklistContext = { turnId: 'turn-1', toolCallId: 'call-checklist' }; - await store.updateChecklist( - SESSION_ID, - { - itemId: 'project_entrypoints', - status: 'in_progress', - evidenceArtifactIds: [], - }, - checklistContext, - ); - await store.updateChecklist( - SESSION_ID, - { - itemId: 'project_entrypoints', - status: 'in_progress', - evidenceArtifactIds: [], - }, - checklistContext, - ); - await assert.rejects( - () => - store.updateChecklist( - SESSION_ID, - { - itemId: 'project_entrypoints', - status: 'completed', - evidenceArtifactIds: ['source-1'], - }, - checklistContext, - ), - /retried with different input/, - ); - - const step = { - kind: 'local_exploration' as const, - status: 'stopped' as const, - objective: 'Inspect the entrypoint.', - summary: 'Stopped at the declared boundary.', - roots: ['packages/core'], - keywords: ['entrypoint'], - ignoredPaths: ['dist'], - stoppingCondition: 'Stop after the exported contract is found.', - expectedEvidence: 'A concrete exported symbol.', - evidenceArtifactIds: [], - inspectedRefs: [{ kind: 'file' as const, locator: 'packages/core/src/index.ts' }], - workerRunIds: [], - }; - const stepContext = { turnId: 'turn-1', toolCallId: 'call-step' }; - await store.recordStep(SESSION_ID, step, stepContext); - await store.recordStep(SESSION_ID, step, stepContext); - await assert.rejects( - () => store.recordStep(SESSION_ID, { ...step, summary: 'Different result.' }, stepContext), - /retried with different input/, - ); - - const checkpoint = { - round: 1, - stage: 'knowledge_base' as const, - status: 'active' as const, - summary: 'Entrypoint inspected.', - openQuestions: [], - nextSteps: ['Continue.'], - taskIds: [], - artifactIds: ['source-1'], - }; - const checkpointContext = { turnId: 'turn-1', toolCallId: 'call-checkpoint' }; - await store.recordCheckpoint(SESSION_ID, checkpoint, checkpointContext); - await store.recordCheckpoint(SESSION_ID, checkpoint, checkpointContext); - await assert.rejects( - () => - store.recordCheckpoint( - SESSION_ID, - { ...checkpoint, summary: 'Conflicting checkpoint.' }, - checkpointContext, - ), - /retried with different input/, - ); - - assert.equal((await store.readEvents(SESSION_ID)).length, 5); - }); - }); - - it('fails closed when the durable JSONL ledger is corrupt', async () => { - await withTempRoot(async (root) => { - const path = join(root, 'sessions', SESSION_ID, 'deep-research', 'events.jsonl'); - await mkdir(dirname(path), { recursive: true }); - await writeFile(path, '{"not":"an event"}\n', 'utf8'); - - await assert.rejects( - () => createDeepResearchStore(root).read(SESSION_ID), - /unexpected event shape/, - ); - }); - }); -}); diff --git a/packages/storage/src/__tests__/execution-stores.test.ts b/packages/storage/src/__tests__/execution-stores.test.ts deleted file mode 100644 index 0ee73f971f..0000000000 --- a/packages/storage/src/__tests__/execution-stores.test.ts +++ /dev/null @@ -1,1967 +0,0 @@ -import assert from 'node:assert/strict'; -import { - appendFile, - mkdir, - mkdtemp, - readFile, - rename, - rm, - stat, - writeFile, -} from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, test } from 'node:test'; -import { - canonicalToolArgsHash, - MAX_ATTACHMENT_BYTES, - MAX_ATTACHMENT_COUNT, - type AgentRunEvent, - type AgentRunHeader, - type AttachmentRef, - type QuoteRef, - type RuntimeEvent, -} from '@maka/core'; -import { - createAgentRunStore, - createRuntimeEventStore, - ROOT_TURN_ADMISSION_MAX_CONTENT_BYTES, - ROOT_TURN_ADMISSION_MAX_RECORD_BYTES, - ROOT_TURN_ADMISSION_SCHEMA_VERSION, -} from '../agent-run-store.js'; -import { - authenticateExecutionStoresReader, - authenticateExecutionStoresWriter, - openHeadlessExecutionStoresForRead, - openHeadlessExecutionStoresForWrite, - openInteractiveExecutionStoresForRead, - openInteractiveExecutionStoresForWrite, -} from '../execution-stores.js'; -import { - createHeadlessRootLease, - resolveStorageRoot, - StorageRootAuthorityError, - tryAcquireInteractiveRootOwner, - tryAcquireInteractiveRootReader, - type StorageRootLease, -} from '../root-authority.js'; -import { createSessionStore } from '../session-store.js'; - -const chartAttachment: AttachmentRef = { - kind: 'image', - name: 'chart.png', - mimeType: 'image/png', - bytes: 128, - ref: { - kind: 'session_file', - sessionId: 'session-files', - relativePath: 'attachments/chart.png', - }, -}; -const notesAttachment: AttachmentRef = { - kind: 'doc', - name: 'notes.txt', - mimeType: 'text/plain', - bytes: 64, - ref: { kind: 'workspace_file', relativePath: 'notes/notes.txt' }, -}; - -describe('execution stores', () => { - test('binds Headless execution readers and writers to Headless leases', async () => { - await withRoot(async ({ base, root }) => { - const capability = await resolveStorageRoot({ - path: root, - kind: 'headless', - }); - const writer = await openHeadlessExecutionStoresForWrite( - createHeadlessRootLease(capability, 'write'), - ); - assert.equal(writer.kind, 'headless'); - const session = await writer.sessionStore.create(sessionInput(root)); - await writer.sessionStore.appendMessage(session.id, { - type: 'user', - id: 'message-1', - turnId: 'turn-1', - ts: 10, - text: 'hello', - }); - - const reader = await openHeadlessExecutionStoresForRead( - createHeadlessRootLease(capability, 'read'), - ); - assert.equal(reader.kind, 'headless'); - assert.equal((await reader.sessionStore.list()).length, 1); - assert.equal((await reader.sessionStore.readMessages(session.id))[0]?.id, 'message-1'); - - const interactive = await resolveStorageRoot({ - path: join(base, 'interactive'), - kind: 'interactive', - }); - const owner = await tryAcquireInteractiveRootOwner(interactive); - assert.ok(owner); - if (!owner) return; - try { - await assert.rejects( - () => - openHeadlessExecutionStoresForWrite( - owner.lease as unknown as StorageRootLease<'headless', 'write'>, - ), - (error: unknown) => - error instanceof StorageRootAuthorityError && error.code === 'invalid_lease', - ); - } finally { - await owner.close(); - } - }); - }); - - test('routes Headless RuntimeEvents through SQLite after importing legacy JSONL once', async () => { - await withRoot(async ({ root }) => { - const capability = await resolveStorageRoot({ - path: root, - kind: 'headless', - }); - const sessionId = 'legacy-session'; - const runId = 'legacy-run'; - await createAgentRunStore(root).createRun(runHeader(sessionId, runId)); - const legacy = createRuntimeEventStore(root); - await legacy.appendRuntimeEvent( - sessionId, - runId, - runtimeEvent(sessionId, runId, 'legacy-event', 1), - ); - const legacyPath = join(root, 'sessions', sessionId, 'runs', runId, 'runtime-events.jsonl'); - const legacyBytes = await readFile(legacyPath); - - const writer = await openHeadlessExecutionStoresForWrite( - createHeadlessRootLease(capability, 'write'), - ); - try { - assert.deepEqual( - (await writer.runtimeEventStore.readRuntimeEvents(sessionId, runId)).map( - (event) => event.id, - ), - ['legacy-event'], - ); - await writer.runtimeEventStore.appendRuntimeEvent( - sessionId, - runId, - runtimeEvent(sessionId, runId, 'sqlite-event', 2), - ); - - const reader = await openHeadlessExecutionStoresForRead( - createHeadlessRootLease(capability, 'read'), - ); - try { - assert.deepEqual( - (await reader.runtimeEventStore.readRuntimeEvents(sessionId, runId)).map( - (event) => event.id, - ), - ['legacy-event', 'sqlite-event'], - ); - } finally { - await reader.sessionStore.close?.(); - } - - assert.ok((await stat(join(root, 'runtime.sqlite'))).isFile()); - assert.deepEqual(await readFile(legacyPath), legacyBytes); - } finally { - await writer.sessionStore.close?.(); - } - }); - }); - - test('purges one incomplete conversation-copy ledger from canonical SQLite', async () => { - await withRoot(async ({ root }) => { - const capability = await resolveStorageRoot({ - path: root, - kind: 'headless', - }); - const writer = await openHeadlessExecutionStoresForWrite( - createHeadlessRootLease(capability, 'write'), - ); - try { - const retained = await writer.sessionStore.create(sessionInput(root)); - const copied = await writer.sessionStore.create(sessionInput(root)); - await writer.agentRunStore.createRun(runHeader(retained.id, 'retained-run')); - await writer.agentRunStore.createRun(runHeader(copied.id, 'copied-run')); - await writer.runtimeEventStore.importConversationCopyRuntimeEvents(retained.id, [ - { - runId: 'retained-run', - events: [runtimeEvent(retained.id, 'retained-run', 'retained-event', 1)], - }, - ]); - await writer.runtimeEventStore.importConversationCopyRuntimeEvents(copied.id, [ - { - runId: 'copied-run', - events: conversationCopyToolLedger(copied.id, 'copied-run'), - }, - ]); - - await writer.purgeConversationOperationalState(copied.id); - - assert.deepEqual( - (await writer.agentRunStore.listSessionRuns(retained.id)).map((run) => run.runId), - ['retained-run'], - ); - assert.deepEqual(await writer.agentRunStore.listSessionRuns(copied.id), []); - assert.deepEqual( - (await writer.runtimeEventStore.readSessionRuntimeEvents(retained.id)).map( - (event) => event.id, - ), - ['retained-event'], - ); - assert.deepEqual(await writer.runtimeEventStore.readSessionRuntimeEvents(copied.id), []); - } finally { - await writer.sessionStore.close?.(); - } - }); - }); - - test('freezes and authenticates execution store facades', async () => { - await withRoot(async ({ base, root }) => { - const capability = await resolveStorageRoot({ - path: root, - kind: 'headless', - }); - const writer = await openHeadlessExecutionStoresForWrite( - createHeadlessRootLease(capability, 'write'), - ); - const session = await writer.sessionStore.create(sessionInput(root)); - const reader = await openHeadlessExecutionStoresForRead( - createHeadlessRootLease(capability, 'read'), - ); - const rawLocalStore = createSessionStore(root); - - assert.equal(Reflect.set(reader, 'sessionStore', rawLocalStore), false); - assert.equal( - Reflect.set( - reader.sessionStore, - 'readHeader', - rawLocalStore.readHeader.bind(rawLocalStore), - ), - false, - ); - await reader.sessionStore.readHeader(session.id); - assert.equal((await rawLocalStore.readHeaderSnapshot(session.id)).connectionLocked, false); - - const otherRoot = join(base, 'other-headless'); - await resolveStorageRoot({ path: otherRoot, kind: 'headless' }); - const rawOtherStore = createSessionStore(otherRoot); - assert.equal(Reflect.set(writer, 'sessionStore', rawOtherStore), false); - assert.equal( - Reflect.set(writer.sessionStore, 'create', rawOtherStore.create.bind(rawOtherStore)), - false, - ); - - const copiedReader = { ...reader, sessionStore: rawLocalStore }; - assert.throws( - () => authenticateExecutionStoresReader(copiedReader, 'headless'), - (error: unknown) => - error instanceof StorageRootAuthorityError && error.code === 'invalid_lease', - ); - const copiedWriter = { ...writer, sessionStore: rawOtherStore }; - assert.throws( - () => authenticateExecutionStoresWriter(copiedWriter, 'headless'), - (error: unknown) => - error instanceof StorageRootAuthorityError && error.code === 'invalid_lease', - ); - }); - }); - - test('round-trips canonical root admission content and retains immutable identity', async () => { - await withRoot(async ({ root }) => { - const capability = await resolveStorageRoot({ - path: root, - kind: 'interactive', - }); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(owner); - if (!owner) return; - try { - const stores = await openInteractiveExecutionStoresForWrite(owner.lease); - assert.equal(stores.kind, 'interactive'); - const session = await stores.sessionStore.create(sessionInput(root)); - const mutableAttachment = { - ...chartAttachment, - ref: { ...chartAttachment.ref }, - }; - const first = await stores.agentRunStore.admitRootTurn({ - sessionId: session.id, - turnId: 'turn-1', - proposedRunId: 'run-1', - proposedUserMessageId: 'message-1', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { - text: 'hello', - displayText: 'hello', - attachments: [mutableAttachment, notesAttachment], - }, - turnOrchestration: { mode: 'swarm', source: 'host_api' }, - sourceMessages: [ - { - messageId: 'source-1', - content: { - text: 'hello', - displayText: 'hello', - attachments: [mutableAttachment, notesAttachment], - }, - placement: 'current_turn', - disposition: 'steering', - }, - ], - admittedAt: 10, - }); - assert.equal(first.kind, 'admitted'); - mutableAttachment.name = 'mutated.png'; - assert.equal(first.admission.normalizedInput.attachments?.[0]?.name, 'chart.png'); - assert.equal(Object.isFrozen(first.admission), true); - assert.deepEqual(first.admission.turnOrchestration, { - mode: 'swarm', - source: 'host_api', - }); - assert.equal(Object.isFrozen(first.admission.turnOrchestration), true); - assert.equal(Object.isFrozen(first.admission.normalizedInput.attachments?.[0]?.ref), true); - assert.deepEqual(await stores.agentRunStore.listSessionRuns(session.id), []); - - const retry = await stores.agentRunStore.admitRootTurn({ - sessionId: session.id, - turnId: 'turn-1', - proposedRunId: 'run-never-used', - proposedUserMessageId: 'message-never-used', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { - text: 'hello', - displayText: 'hello', - attachments: [chartAttachment, notesAttachment], - }, - turnOrchestration: { mode: 'swarm', source: 'host_api' }, - sourceMessages: [ - { - messageId: 'source-1', - content: { - text: 'hello', - displayText: 'hello', - attachments: [chartAttachment, notesAttachment], - }, - placement: 'current_turn', - disposition: 'steering', - }, - ], - admittedAt: 20, - }); - assert.equal(retry.kind, 'existing'); - assert.equal(retry.admission.runId, 'run-1'); - assert.equal(retry.admission.userMessageId, 'message-1'); - assert.equal(retry.admission.admittedAt, 10); - assert.deepEqual(retry.admission.normalizedInput.attachments, [ - chartAttachment, - notesAttachment, - ]); - - const orchestrationConflict = await stores.agentRunStore.admitRootTurn({ - sessionId: session.id, - turnId: 'turn-1', - proposedRunId: 'run-never-used', - proposedUserMessageId: 'message-never-used', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { - text: 'hello', - displayText: 'hello', - attachments: [chartAttachment, notesAttachment], - }, - turnOrchestration: { mode: 'default', source: 'host_api' }, - sourceMessages: [ - { - messageId: 'source-1', - content: { - text: 'hello', - displayText: 'hello', - attachments: [chartAttachment, notesAttachment], - }, - placement: 'current_turn', - disposition: 'steering', - }, - ], - admittedAt: 20, - }); - assert.equal(orchestrationConflict.kind, 'conflict'); - - const stored = await stores.agentRunStore.readRootTurnAdmission(session.id, 'turn-1'); - assert.deepEqual(stored, first.admission); - assert.equal(Object.isFrozen(stored?.sourceMessages[0]?.content), true); - - const receipt = await stores.agentRunStore.readRootTurnSourceMessageReceipt( - session.id, - 'source-1', - ); - assert.equal(receipt?.admission.turnId, 'turn-1'); - assert.deepEqual(receipt?.sourceMessage, first.admission.sourceMessages[0]); - assert.equal(Object.isFrozen(receipt), true); - const rootProofPath = join( - root, - 'sessions', - session.id, - 'message-proofs', - 'root', - 'source-1.json', - ); - await assert.rejects(() => stat(rootProofPath), { code: 'ENOENT' }); - assert.equal( - (await stores.agentRunStore.readRootTurnSourceMessageReceipt(session.id, 'source-1')) - ?.admission.turnId, - 'turn-1', - ); - await stores.agentRunStore.listRootTurnAdmissionsForRecovery(session.id); - assert.equal( - (await stores.agentRunStore.readRootTurnSourceMessageReceipt(session.id, 'source-1')) - ?.admission.turnId, - 'turn-1', - ); - const unrelatedAdmissionEntry = join( - root, - 'sessions', - session.id, - 'turn-admissions', - 'not-an-admission', - ); - await mkdir(join(root, 'sessions', session.id, 'turn-admissions'), { - recursive: true, - }); - await writeFile(unrelatedAdmissionEntry, 'unrelated'); - assert.equal( - (await stores.agentRunStore.readRootTurnSourceMessageReceipt(session.id, 'source-1')) - ?.admission.turnId, - 'turn-1', - ); - await rm(unrelatedAdmissionEntry); - - const conflict = await stores.agentRunStore.admitRootTurn({ - sessionId: session.id, - turnId: 'turn-1', - proposedRunId: 'run-never-used', - proposedUserMessageId: 'message-never-used', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { - text: 'hello', - displayText: 'hello', - attachments: [notesAttachment, chartAttachment], - }, - sourceMessages: [ - { - messageId: 'source-1', - content: { - text: 'hello', - displayText: 'hello', - attachments: [notesAttachment, chartAttachment], - }, - placement: 'current_turn', - disposition: 'steering', - }, - ], - admittedAt: 30, - }); - assert.equal(conflict.kind, 'conflict'); - assert.equal(conflict.admission.runId, 'run-1'); - - const dispositionConflict = await stores.agentRunStore.admitRootTurn({ - sessionId: session.id, - turnId: 'turn-1', - proposedRunId: 'run-never-used', - proposedUserMessageId: 'message-never-used', - execution: { kind: 'linked_child_initial', agentId: 'agent', agentName: 'Agent' }, - previousRootTurnId: null, - normalizedInput: { - text: 'hello', - displayText: 'hello', - attachments: [chartAttachment, notesAttachment], - }, - sourceMessages: [], - admittedAt: 35, - }); - assert.equal(dispositionConflict.kind, 'conflict'); - - const lineageConflict = await stores.agentRunStore.admitRootTurn({ - sessionId: session.id, - turnId: 'turn-1', - proposedRunId: 'run-never-used', - proposedUserMessageId: 'message-never-used', - execution: { kind: 'external_message' }, - previousRootTurnId: 'different-predecessor', - normalizedInput: { - text: 'hello', - displayText: 'hello', - attachments: [chartAttachment, notesAttachment], - }, - sourceMessages: [ - { - messageId: 'source-1', - content: { - text: 'hello', - displayText: 'hello', - attachments: [chartAttachment, notesAttachment], - }, - placement: 'current_turn', - disposition: 'steering', - }, - ], - admittedAt: 40, - }); - assert.equal(lineageConflict.kind, 'conflict'); - assert.equal(lineageConflict.admission.previousRootTurnId, null); - - const header = runHeader(session.id, first.admission.runId); - await stores.agentRunStore.createRun(header); - const legacyRunPath = join( - root, - 'sessions', - session.id, - 'runs', - first.admission.runId, - 'run.json', - ); - await assert.rejects(() => stat(legacyRunPath), { code: 'ENOENT' }); - await assert.rejects( - () => stores.agentRunStore.createRun({ ...header, updatedAt: 99 }), - /Agent run already exists/, - ); - assert.deepEqual(await stores.agentRunStore.readRun(session.id, header.runId), header); - await assert.rejects(() => stat(legacyRunPath), { code: 'ENOENT' }); - } finally { - await owner.close(); - } - }); - }); - - test('durably preserves ordered quote provenance across admission and recovery', async () => { - await withRoot(async ({ root }) => { - const store = createAgentRunStore(root); - const expectedFirstQuote = { - text: 'first excerpt', - label: 'Assistant answer', - sourceTurnId: 'origin-turn-1', - }; - const expectedSecondQuote = { - text: 'second excerpt', - label: 'Tool result', - sourceTurnId: 'origin-turn-2', - }; - const mutableFirstQuote = { ...expectedFirstQuote }; - const mutableSecondQuote = { ...expectedSecondQuote }; - const mutableNormalizedQuotes = [mutableFirstQuote, mutableSecondQuote]; - const mutableFirstSourceQuotes = [mutableFirstQuote]; - const mutableSecondSourceQuotes = [mutableSecondQuote]; - const sourceMessages = (firstQuotes: QuoteRef[], secondQuotes: QuoteRef[]) => [ - { - messageId: 'source-first', - content: { - text: 'first', - displayText: 'First visible', - quotes: firstQuotes, - }, - placement: 'current_turn' as const, - disposition: 'steering' as const, - }, - { - messageId: 'source-second', - content: { - text: 'second', - quotes: secondQuotes, - }, - placement: 'next_turn' as const, - disposition: 'followup' as const, - }, - ]; - - const admitted = await store.admitRootTurn({ - sessionId: 'session-quotes', - turnId: 'turn-1', - proposedRunId: 'run-1', - proposedUserMessageId: 'message-1', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { - text: 'first\n\nsecond', - displayText: 'First visible\n\nsecond', - quotes: mutableNormalizedQuotes, - }, - sourceMessages: sourceMessages(mutableFirstSourceQuotes, mutableSecondSourceQuotes), - admittedAt: 10, - }); - assert.equal(admitted.kind, 'admitted'); - - mutableFirstQuote.text = 'caller-mutated'; - mutableSecondQuote.sourceTurnId = 'caller-mutated'; - mutableNormalizedQuotes.reverse(); - mutableFirstSourceQuotes.push({ - text: 'late excerpt', - label: 'Late', - sourceTurnId: 'origin-turn-late', - }); - mutableSecondSourceQuotes.length = 0; - - assert.deepEqual(admitted.admission.normalizedInput.quotes, [ - expectedFirstQuote, - expectedSecondQuote, - ]); - assert.deepEqual(admitted.admission.sourceMessages[0]?.content.quotes, [expectedFirstQuote]); - assert.deepEqual(admitted.admission.sourceMessages[1]?.content.quotes, [expectedSecondQuote]); - assert.equal(Object.isFrozen(admitted.admission.normalizedInput.quotes), true); - assert.equal(Object.isFrozen(admitted.admission.normalizedInput.quotes?.[0]), true); - assert.equal( - Object.isFrozen(admitted.admission.sourceMessages[0]?.content.quotes?.[0]), - true, - ); - - const stored = await createAgentRunStore(root).readRootTurnAdmission( - 'session-quotes', - 'turn-1', - ); - assert.deepEqual(stored, admitted.admission); - assert.notEqual(stored?.normalizedInput.quotes, admitted.admission.normalizedInput.quotes); - assert.notEqual( - stored?.normalizedInput.quotes?.[0], - admitted.admission.normalizedInput.quotes?.[0], - ); - - await assert.rejects( - () => - store.admitRootTurn({ - sessionId: 'session-reversed-quotes', - turnId: 'turn-1', - proposedRunId: 'run-1', - proposedUserMessageId: 'message-1', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { - text: 'first\n\nsecond', - displayText: 'First visible\n\nsecond', - quotes: [expectedSecondQuote, expectedFirstQuote], - }, - sourceMessages: sourceMessages([expectedFirstQuote], [expectedSecondQuote]), - admittedAt: 10, - }), - /input content does not match source messages/, - ); - - const changedFirstQuote = { - ...expectedFirstQuote, - sourceTurnId: 'origin-turn-changed', - }; - const provenanceConflict = await store.admitRootTurn({ - sessionId: 'session-quotes', - turnId: 'turn-1', - proposedRunId: 'run-never-used', - proposedUserMessageId: 'message-never-used', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { - text: 'first\n\nsecond', - displayText: 'First visible\n\nsecond', - quotes: [changedFirstQuote, expectedSecondQuote], - }, - sourceMessages: sourceMessages([changedFirstQuote], [expectedSecondQuote]), - admittedAt: 20, - }); - assert.equal(provenanceConflict.kind, 'conflict'); - - const proofPath = join( - root, - 'sessions', - 'session-quotes', - 'message-proofs', - 'root', - 'source-first.json', - ); - await rm(proofPath); - const restarted = createAgentRunStore(root); - const recovered = await restarted.listRootTurnAdmissionsForRecovery('session-quotes'); - assert.deepEqual(recovered, [admitted.admission]); - const receipt = await restarted.readRootTurnSourceMessageReceipt( - 'session-quotes', - 'source-first', - ); - assert.deepEqual(receipt?.sourceMessage.content.quotes, [expectedFirstQuote]); - assert.deepEqual(receipt?.admission.normalizedInput.quotes, [ - expectedFirstQuote, - expectedSecondQuote, - ]); - assert.equal(Object.isFrozen(receipt?.sourceMessage.content.quotes?.[0]), true); - }); - }); - - test('rejects invalid root admission contracts before write and when reading disk', async () => { - await withRoot(async ({ root }) => { - const store = createAgentRunStore(root); - const turnStartedSource = { - messageId: 'source-message', - content: { text: 'source' }, - placement: 'current_turn' as const, - disposition: 'turn_started' as const, - }; - const invalidContracts = [ - { - name: 'external-null-message', - userMessageId: null, - execution: { kind: 'external_message' as const }, - sourceMessages: [], - }, - { - name: 'retry-with-message', - userMessageId: 'message-1', - execution: { - kind: 'linked_child_provider_retry' as const, - agentId: 'agent', - agentName: 'Agent', - sourceRunId: 'source-run', - }, - sourceMessages: [], - }, - { - name: 'child-with-source', - userMessageId: 'message-1', - execution: { - kind: 'linked_child_initial' as const, - agentId: 'agent', - agentName: 'Agent', - }, - sourceMessages: [turnStartedSource], - }, - { - name: 'child-with-turn-orchestration', - userMessageId: 'message-1', - execution: { - kind: 'linked_child_initial' as const, - agentId: 'agent', - agentName: 'Agent', - }, - turnOrchestration: { mode: 'swarm' as const, source: 'host_api' as const }, - sourceMessages: [], - }, - { - name: 'resume-self-source', - userMessageId: 'message-1', - execution: { - kind: 'linked_child_resume' as const, - agentId: 'agent', - agentName: 'Agent', - sourceRunId: 'run-1', - }, - sourceMessages: [], - }, - { - name: 'retry-self-source', - userMessageId: null, - execution: { - kind: 'linked_child_provider_retry' as const, - agentId: 'agent', - agentName: 'Agent', - sourceRunId: 'run-1', - }, - sourceMessages: [], - }, - { - name: 'turn-started-owner-mismatch', - userMessageId: 'message-1', - execution: { kind: 'external_message' as const }, - sourceMessages: [turnStartedSource], - }, - ]; - - for (const invalid of invalidContracts) { - const sessionId = `session-${invalid.name}`; - const turnId = 'turn-1'; - const normalizedInput = - invalid.sourceMessages.length > 0 ? { text: 'source' } : { text: 'input' }; - await assert.rejects( - () => - store.admitRootTurn({ - sessionId, - turnId, - proposedRunId: 'run-1', - proposedUserMessageId: invalid.userMessageId, - execution: invalid.execution, - previousRootTurnId: null, - normalizedInput, - ...('turnOrchestration' in invalid - ? { turnOrchestration: invalid.turnOrchestration } - : {}), - sourceMessages: invalid.sourceMessages, - admittedAt: 10, - }), - /Invalid root turn admission contract/, - ); - assert.equal(await store.readRootTurnAdmission(sessionId, turnId), undefined); - - const admissionRoot = join(root, 'sessions', sessionId, 'turn-admissions'); - await mkdir(admissionRoot, { recursive: true }); - await writeFile( - join(admissionRoot, `${turnId}.json`), - `${JSON.stringify({ - schemaVersion: ROOT_TURN_ADMISSION_SCHEMA_VERSION, - sessionId, - turnId, - runId: 'run-1', - userMessageId: invalid.userMessageId, - execution: invalid.execution, - previousRootTurnId: null, - normalizedInput, - ...('turnOrchestration' in invalid - ? { turnOrchestration: invalid.turnOrchestration } - : {}), - sourceMessages: invalid.sourceMessages, - admittedAt: 10, - })}\n`, - ); - await assert.rejects( - () => store.readRootTurnAdmission(sessionId, turnId), - /Invalid root turn admission contract/, - ); - } - }); - }); - - test('treats ordered source messages as identity and fails closed on ambiguous receipts', async () => { - await withRoot(async ({ root }) => { - const store = createAgentRunStore(root); - const sources = [ - { - messageId: 'source-steering', - content: { text: 'steering' }, - placement: 'current_turn' as const, - disposition: 'steering' as const, - }, - { - messageId: 'source-followup', - content: { text: 'followup' }, - placement: 'next_turn' as const, - disposition: 'followup' as const, - }, - ]; - const base = { - sessionId: 'session-source-order', - turnId: 'turn-1', - proposedRunId: 'run-1', - proposedUserMessageId: 'message-1', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - admittedAt: 10, - } as const; - assert.equal( - ( - await store.admitRootTurn({ - ...base, - normalizedInput: { text: 'steering\n\nfollowup' }, - sourceMessages: sources, - }) - ).kind, - 'admitted', - ); - - const reordered = await store.admitRootTurn({ - ...base, - proposedRunId: 'unused-run', - proposedUserMessageId: 'unused-message', - normalizedInput: { text: 'followup\n\nsteering' }, - sourceMessages: [...sources].reverse(), - }); - assert.equal(reordered.kind, 'conflict'); - - await assert.rejects( - () => - store.admitRootTurn({ - ...base, - proposedRunId: 'unused-run', - proposedUserMessageId: 'unused-message', - normalizedInput: { text: 'steering\n\nfollowup' }, - sourceMessages: [{ ...sources[0]! }, { ...sources[1]!, placement: 'current_turn' }], - }), - /Invalid root turn source message/, - ); - - const invalidAdmissionRoot = join( - root, - 'sessions', - 'session-invalid-followup', - 'turn-admissions', - ); - await mkdir(invalidAdmissionRoot, { recursive: true }); - await writeFile( - join(invalidAdmissionRoot, 'turn-1.json'), - `${JSON.stringify({ - schemaVersion: ROOT_TURN_ADMISSION_SCHEMA_VERSION, - sessionId: 'session-invalid-followup', - turnId: 'turn-1', - runId: 'run-1', - userMessageId: 'message-1', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { text: 'followup' }, - sourceMessages: [ - { - messageId: 'source-followup', - content: { text: 'followup' }, - placement: 'current_turn', - disposition: 'followup', - }, - ], - admittedAt: 10, - })}\n`, - ); - await assert.rejects( - () => store.readRootTurnAdmission('session-invalid-followup', 'turn-1'), - /Invalid root turn source message/, - ); - - const startedFromNextPlacement = await store.admitRootTurn({ - sessionId: 'session-turn-started-next', - turnId: 'turn-1', - proposedRunId: 'run-1', - proposedUserMessageId: 'message-1', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { text: 'started' }, - sourceMessages: [ - { - messageId: 'message-1', - content: { text: 'started' }, - placement: 'next_turn', - disposition: 'turn_started', - }, - ], - admittedAt: 10, - }); - assert.equal(startedFromNextPlacement.kind, 'admitted'); - - await assert.rejects( - () => - store.admitRootTurn({ - ...base, - turnId: 'turn-2', - proposedRunId: 'run-2', - proposedUserMessageId: 'message-2', - execution: { kind: 'external_message' }, - previousRootTurnId: 'turn-1', - normalizedInput: { text: 'followup' }, - sourceMessages: [{ ...sources[1]! }], - }), - /Root source message identity belongs to both turn-1 and turn-2/, - ); - }); - }); - - test('rejects a competing source owner without publishing its Turn or partial proofs', async () => { - await withRoot(async ({ root }) => { - const store = createAgentRunStore(root); - const admit = (suffix: 'a' | 'b') => - store.admitRootTurn({ - sessionId: 'session-proof-conflict', - turnId: `turn-${suffix}`, - proposedRunId: `run-${suffix}`, - proposedUserMessageId: `message-${suffix}`, - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { text: `unique-${suffix}\n\nshared` }, - sourceMessages: [ - { - messageId: `source-${suffix}`, - content: { text: `unique-${suffix}` }, - placement: 'current_turn', - disposition: 'steering', - }, - { - messageId: 'source-shared', - content: { text: 'shared' }, - placement: 'next_turn', - disposition: 'followup', - }, - ], - admittedAt: suffix === 'a' ? 10 : 20, - }); - - const settled = await Promise.allSettled([admit('a'), admit('b')]); - const admittedIndex = settled.findIndex( - (result) => result.status === 'fulfilled' && result.value.kind === 'admitted', - ); - const rejectedIndex = settled.findIndex((result) => result.status === 'rejected'); - assert.notEqual(admittedIndex, -1); - assert.notEqual(rejectedIndex, -1); - assert.match( - String( - rejectedIndex === -1 - ? undefined - : (settled[rejectedIndex] as PromiseRejectedResult).reason, - ), - /Root source message identity belongs to both/, - ); - - const winner = admittedIndex === 0 ? 'a' : 'b'; - const loser = winner === 'a' ? 'b' : 'a'; - assert.equal( - await store.readRootTurnAdmission('session-proof-conflict', `turn-${loser}`), - undefined, - ); - assert.equal( - await store.readRootTurnSourceMessageReceipt('session-proof-conflict', `source-${loser}`), - undefined, - ); - assert.equal( - (await store.readRootTurnSourceMessageReceipt('session-proof-conflict', 'source-shared')) - ?.admission.turnId, - `turn-${winner}`, - ); - - await rm( - join( - root, - 'sessions', - 'session-proof-conflict', - 'message-proofs', - 'root', - `source-${winner}.json`, - ), - ); - const restartedStore = createAgentRunStore(root); - const recovered = - await restartedStore.listRootTurnAdmissionsForRecovery('session-proof-conflict'); - assert.deepEqual( - recovered.map((admission) => admission.turnId), - [`turn-${winner}`], - ); - assert.equal( - ( - await restartedStore.readRootTurnSourceMessageReceipt( - 'session-proof-conflict', - `source-${winner}`, - ) - )?.admission.turnId, - `turn-${winner}`, - ); - assert.equal( - await restartedStore.readRootTurnSourceMessageReceipt( - 'session-proof-conflict', - `source-${loser}`, - ), - undefined, - ); - }); - }); - - test('shares root admission serialization across writer facades from one live lease', async () => { - await withRoot(async ({ root }) => { - const capability = await resolveStorageRoot({ path: root, kind: 'interactive' }); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(owner); - if (!owner) return; - - let winner: 'a' | 'b' | undefined; - let loser: 'a' | 'b' | undefined; - try { - const [first, second] = await Promise.all([ - openInteractiveExecutionStoresForWrite(owner.lease), - openInteractiveExecutionStoresForWrite(owner.lease), - ]); - const session = await first.sessionStore.create(sessionInput(root)); - const input = (suffix: 'a' | 'b') => ({ - sessionId: session.id, - turnId: `turn-${suffix}`, - proposedRunId: `run-${suffix}`, - proposedUserMessageId: `message-${suffix}`, - execution: { kind: 'external_message' as const }, - previousRootTurnId: null, - normalizedInput: { text: `unique-${suffix}\n\nshared` }, - sourceMessages: [ - { - messageId: `source-${suffix}`, - content: { text: `unique-${suffix}` }, - placement: 'current_turn' as const, - disposition: 'steering' as const, - }, - { - messageId: 'source-shared', - content: { text: 'shared' }, - placement: 'next_turn' as const, - disposition: 'followup' as const, - }, - ], - admittedAt: suffix === 'a' ? 10 : 20, - }); - - const settled = await Promise.allSettled([ - first.agentRunStore.admitRootTurn(input('a')), - second.agentRunStore.admitRootTurn(input('b')), - ]); - const admittedIndex = settled.findIndex( - (result) => result.status === 'fulfilled' && result.value.kind === 'admitted', - ); - const rejectedIndex = settled.findIndex((result) => result.status === 'rejected'); - assert.notEqual(admittedIndex, -1); - assert.notEqual(rejectedIndex, -1); - assert.match( - String( - rejectedIndex === -1 - ? undefined - : (settled[rejectedIndex] as PromiseRejectedResult).reason, - ), - /Root source message identity belongs to both/, - ); - - winner = admittedIndex === 0 ? 'a' : 'b'; - loser = winner === 'a' ? 'b' : 'a'; - assert.equal( - await first.agentRunStore.readRootTurnAdmission(session.id, `turn-${loser}`), - undefined, - ); - assert.equal( - await first.agentRunStore.readRootTurnSourceMessageReceipt(session.id, `source-${loser}`), - undefined, - ); - assert.equal(first, second); - - await assert.rejects( - () => - stat( - join(root, 'sessions', session.id, 'message-proofs', 'root', `source-${winner}.json`), - ), - { code: 'ENOENT' }, - ); - } finally { - await owner.close(); - } - - assert.ok(winner); - assert.ok(loser); - const successor = await tryAcquireInteractiveRootOwner(capability); - assert.ok(successor); - if (!successor) return; - try { - const reopened = await openInteractiveExecutionStoresForWrite(successor.lease); - const sessions = await reopened.sessionStore.list(); - assert.equal(sessions.length, 1); - const sessionId = sessions[0]!.id; - const recovered = await reopened.agentRunStore.listRootTurnAdmissionsForRecovery(sessionId); - assert.deepEqual( - recovered.map((admission) => admission.turnId), - [`turn-${winner}`], - ); - assert.equal( - ( - await reopened.agentRunStore.readRootTurnSourceMessageReceipt( - sessionId, - `source-${winner}`, - ) - )?.admission.turnId, - `turn-${winner}`, - ); - assert.equal( - await reopened.agentRunStore.readRootTurnSourceMessageReceipt( - sessionId, - `source-${loser}`, - ), - undefined, - ); - } finally { - await successor.close(); - } - }); - }); - - test('round-trips only the final root admission schema v1', async () => { - await withRoot(async ({ root }) => { - const store = createAgentRunStore(root); - const admitted = await store.admitRootTurn({ - sessionId: 'session-schema', - turnId: 'turn-1', - proposedRunId: 'run-1', - proposedUserMessageId: 'message-1', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { text: 'hello' }, - turnOrchestration: { mode: 'graph', source: 'host_api' }, - sourceMessages: [], - admittedAt: 10, - }); - assert.equal(admitted.kind, 'admitted'); - - const path = join(root, 'sessions', 'session-schema', 'turn-admissions', 'turn-1.json'); - const v1 = JSON.parse(await readFile(path, 'utf8')) as Record; - assert.equal(v1.schemaVersion, 1); - assert.deepEqual(v1.turnOrchestration, { mode: 'graph', source: 'host_api' }); - assert.deepEqual( - await createAgentRunStore(root).readRootTurnAdmission('session-schema', 'turn-1'), - admitted.admission, - ); - - await writeFile(path, `${JSON.stringify({ ...v1, schemaVersion: 2 })}\n`, 'utf8'); - const reopened = createAgentRunStore(root); - await assert.rejects( - () => reopened.readRootTurnAdmission('session-schema', 'turn-1'), - /Invalid root turn admission/, - ); - await assert.rejects( - () => reopened.listRootTurnAdmissionsForRecovery('session-schema'), - /Invalid root turn admission/, - ); - }); - }); - - test('keys operation receipts by Host Epoch and discards completed Epochs', async () => { - await withRoot(async ({ root }) => { - const capability = await resolveStorageRoot({ path: root, kind: 'interactive' }); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(owner); - if (!owner) return; - try { - const stores = await openInteractiveExecutionStoresForWrite(owner.lease); - const receipt = { - payload: { - originHostEpoch: 'epoch-1', - sessionId: 'session-1', - retractId: 'retract-1', - }, - result: { queueRevision: 3, retracted: [] }, - }; - await stores.messageReceiptStore.beginHostEpoch('epoch-1'); - await stores.messageReceiptStore.commit( - 'epoch-1', - 'retract', - 'session-1', - 'retract-1', - receipt, - ); - assert.deepEqual( - await stores.messageReceiptStore.read('epoch-1', 'retract', 'session-1', 'retract-1'), - receipt, - ); - await assert.rejects( - () => - stores.messageReceiptStore.commit('epoch-1', 'retract', 'session-1', 'retract-1', { - ...receipt, - result: { queueRevision: 4, retracted: [] }, - }), - /identity conflict/, - ); - - await stores.messageReceiptStore.beginHostEpoch('epoch-2'); - assert.equal( - await stores.messageReceiptStore.read('epoch-1', 'retract', 'session-1', 'retract-1'), - undefined, - ); - } finally { - await owner.close(); - } - }); - }); - - test('rejects malformed and oversized admission content at the Store boundary', async () => { - await withRoot(async ({ root }) => { - const store = createAgentRunStore(root); - const admit = (label: string, attachment: AttachmentRef) => - store.admitRootTurn({ - sessionId: `session-${label}`, - turnId: 'turn-1', - proposedRunId: `run-${label}`, - proposedUserMessageId: `message-${label}`, - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { text: 'content', attachments: [attachment] }, - sourceMessages: [], - admittedAt: 10, - }); - - await assert.rejects( - () => - admit('unsafe-path', { - ...notesAttachment, - ref: { kind: 'workspace_file', relativePath: '../secret' }, - }), - /Invalid root turn normalized input attachment/, - ); - await assert.rejects( - () => - admit('unsafe-session', { - ...chartAttachment, - ref: { - kind: 'session_file', - sessionId: 'not/safe', - relativePath: 'attachments/chart.png', - }, - }), - /Invalid root turn normalized input attachment/, - ); - await assert.rejects( - () => - admit('oversized-attachment', { ...chartAttachment, bytes: MAX_ATTACHMENT_BYTES + 1 }), - /Invalid root turn normalized input attachment/, - ); - await assert.rejects( - () => - store.admitRootTurn({ - sessionId: 'session-too-many', - turnId: 'turn-1', - proposedRunId: 'run-too-many', - proposedUserMessageId: 'message-too-many', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { - text: 'content', - attachments: Array.from({ length: MAX_ATTACHMENT_COUNT + 1 }, () => notesAttachment), - }, - sourceMessages: [], - admittedAt: 10, - }), - /Invalid root turn normalized input/, - ); - await assert.rejects( - () => - store.admitRootTurn({ - sessionId: 'session-large-content', - turnId: 'turn-1', - proposedRunId: 'run-large-content', - proposedUserMessageId: 'message-large-content', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { text: 'x'.repeat(ROOT_TURN_ADMISSION_MAX_CONTENT_BYTES + 1) }, - sourceMessages: [], - admittedAt: 10, - }), - /content exceeds size limit/, - ); - await assert.rejects( - () => - store.admitRootTurn({ - sessionId: 'session-large-quote', - turnId: 'turn-1', - proposedRunId: 'run-large-quote', - proposedUserMessageId: 'message-large-quote', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { - text: 'content', - quotes: [{ text: 'q'.repeat(ROOT_TURN_ADMISSION_MAX_CONTENT_BYTES) }], - }, - sourceMessages: [], - admittedAt: 10, - }), - /content exceeds size limit/, - ); - - const oversizedAdmissionRoot = join( - root, - 'sessions', - 'session-large-record', - 'turn-admissions', - ); - await mkdir(oversizedAdmissionRoot, { recursive: true }); - await writeFile( - join(oversizedAdmissionRoot, 'turn-1.json'), - JSON.stringify({ padding: 'x'.repeat(ROOT_TURN_ADMISSION_MAX_RECORD_BYTES) }), - ); - await assert.rejects( - () => store.readRootTurnAdmission('session-large-record', 'turn-1'), - /record exceeds size limit/, - ); - - await assert.rejects( - () => - store.admitRootTurn({ - sessionId: 'session-invalid-source', - turnId: 'turn-1', - proposedRunId: 'run-invalid-source', - proposedUserMessageId: 'message-invalid-source', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { text: 'content' }, - sourceMessages: [ - { - messageId: 'invalid/source', - content: { text: 'content' }, - placement: 'current_turn', - disposition: 'turn_started', - }, - ], - admittedAt: 10, - }), - /Invalid root turn source message/, - ); - }); - }); - - test('keeps shared execution reads observational', async () => { - await withRoot(async ({ root }) => { - const capability = await resolveStorageRoot({ - path: root, - kind: 'interactive', - }); - const rawSessionStore = createSessionStore(root); - const session = await rawSessionStore.create(sessionInput(root)); - await rawSessionStore.appendMessage(session.id, { - type: 'user', - id: 'message-1', - turnId: 'turn-1', - ts: 10, - text: 'hello', - }); - const rawAgentRunStore = createAgentRunStore(root); - await rawAgentRunStore.admitRootTurn({ - sessionId: session.id, - turnId: 'turn-1', - proposedRunId: 'run-1', - proposedUserMessageId: 'message-1', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { text: 'hello' }, - sourceMessages: [], - admittedAt: 9, - }); - await rawAgentRunStore.createRun(runHeader(session.id, 'run-1')); - const sessionPath = join(root, 'sessions', session.id, 'session.jsonl'); - const before = await readFile(sessionPath, 'utf8'); - - const reader = await tryAcquireInteractiveRootReader(capability); - assert.ok(reader); - if (!reader) return; - try { - const stores = await openInteractiveExecutionStoresForRead(reader.lease); - assert.equal((await stores.sessionStore.list()).length, 1); - assert.equal((await stores.sessionStore.readHeader(session.id)).connectionLocked, false); - assert.equal((await stores.sessionStore.readMessages(session.id)).length, 1); - assert.equal((await stores.sessionStore.listTurns(session.id)).length, 1); - assert.equal((await stores.agentRunStore.listSessionRuns(session.id)).length, 1); - assert.equal((await stores.agentRunStore.readRun(session.id, 'run-1')).turnId, 'turn-1'); - assert.equal((await stores.agentRunStore.readEvents(session.id, 'run-1')).length, 0); - assert.equal( - (await stores.agentRunStore.readRootTurnAdmission(session.id, 'turn-1'))?.runId, - 'run-1', - ); - assert.equal( - (await stores.runtimeEventStore.readRuntimeEvents(session.id, 'run-1')).length, - 0, - ); - assert.equal( - (await stores.runtimeEventStore.readImmutableRuntimeEvents(session.id, 'run-1')).length, - 0, - ); - assert.equal( - (await stores.runtimeEventStore.readSessionRuntimeEvents(session.id)).length, - 0, - ); - } finally { - await reader.close(); - } - - assert.equal(await readFile(sessionPath, 'utf8'), before); - assert.equal((await rawSessionStore.readHeaderSnapshot(session.id)).connectionLocked, false); - }); - }); - - test('keeps AgentRun and RuntimeEvent legacy JSONL read-only after SQLite cutover', async () => { - await withRoot(async ({ root }) => { - const capability = await resolveStorageRoot({ - path: root, - kind: 'interactive', - }); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(owner); - if (!owner) return; - try { - const stores = await openInteractiveExecutionStoresForWrite(owner.lease); - const session = await stores.sessionStore.create(sessionInput(root)); - const header = runHeader(session.id, 'run-1'); - await stores.agentRunStore.createRun(header); - - const sessionPath = join(root, 'sessions', session.id, 'session.jsonl'); - await appendFile(sessionPath, '{"type":"user"', 'utf8'); - await stores.sessionStore.appendMessage(session.id, { - type: 'user', - id: 'message-1', - turnId: 'turn-1', - ts: 11, - text: 'hello', - }); - assert.deepEqual( - (await stores.sessionStore.readMessages(session.id)).map((message) => message.id), - ['message-1'], - ); - - const eventsPath = join(root, 'sessions', session.id, 'runs', header.runId, 'events.jsonl'); - await mkdir(join(root, 'sessions', session.id, 'runs', header.runId), { - recursive: true, - }); - await writeFile( - eventsPath, - JSON.stringify(runEvent(session.id, header.runId, 'event-1', 12)), - 'utf8', - ); - await stores.agentRunStore.appendEvent( - session.id, - header.runId, - runEvent(session.id, header.runId, 'event-2', 13), - ); - await appendFile(eventsPath, '{"type":"run_started"', 'utf8'); - await stores.agentRunStore.appendEvent( - session.id, - header.runId, - runEvent(session.id, header.runId, 'event-3', 14), - ); - assert.deepEqual( - (await stores.agentRunStore.readEvents(session.id, header.runId)).map( - (event) => event.id, - ), - ['event-2', 'event-3'], - ); - assert.equal( - await readFile(eventsPath, 'utf8'), - `${JSON.stringify(runEvent(session.id, header.runId, 'event-1', 12))}{"type":"run_started"`, - ); - - const runtimeEventsPath = join( - root, - 'sessions', - session.id, - 'runs', - header.runId, - 'runtime-events.jsonl', - ); - await writeFile(runtimeEventsPath, '{"id":"truncated"', 'utf8'); - await stores.runtimeEventStore.appendRuntimeEvent( - session.id, - header.runId, - runtimeEvent(session.id, header.runId, 'runtime-1', 15), - ); - assert.deepEqual( - (await stores.runtimeEventStore.readImmutableRuntimeEvents(session.id, header.runId)).map( - (event) => event.id, - ), - ['runtime-1'], - ); - assert.equal(await readFile(runtimeEventsPath, 'utf8'), '{"id":"truncated"'); - const steering = { - ...runtimeEvent(session.id, header.runId, 'runtime-steering', 16), - content: { kind: 'text' as const, text: 'steer', steering: true as const }, - refs: { providerEventId: 'message-steering' }, - }; - await stores.runtimeEventStore.appendRuntimeEvent(session.id, header.runId, steering); - assert.deepEqual( - await stores.runtimeEventStore.readImmutableSteeringMessageProof( - session.id, - 'message-steering', - ), - { event: steering }, - ); - await stores.runtimeEventStore.repairImmutableSteeringMessageProofsForRecovery(session.id); - assert.deepEqual( - await stores.runtimeEventStore.readImmutableSteeringMessageProof( - session.id, - 'message-steering', - ), - { event: steering }, - ); - assert.equal( - await stores.runtimeEventStore.readImmutableSteeringMessageProof( - session.id, - 'message-unknown', - ), - undefined, - ); - - for (const path of [sessionPath]) { - const lines = (await readFile(path, 'utf8')).split('\n').filter(Boolean); - for (const line of lines) assert.doesNotThrow(() => JSON.parse(line)); - } - assert.equal(await readFile(runtimeEventsPath, 'utf8'), '{"id":"truncated"'); - } finally { - await owner.close(); - } - }); - }); - - test('refuses to truncate a syntactically invalid JSONL tail', async () => { - await withRoot(async ({ root }) => { - const capability = await resolveStorageRoot({ - path: root, - kind: 'interactive', - }); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(owner); - if (!owner) return; - try { - const stores = await openInteractiveExecutionStoresForWrite(owner.lease); - const session = await stores.sessionStore.create(sessionInput(root)); - const sessionPath = join(root, 'sessions', session.id, 'session.jsonl'); - await appendFile(sessionPath, '{"type":]', 'utf8'); - const before = await readFile(sessionPath, 'utf8'); - - await assert.rejects( - () => - stores.sessionStore.appendMessage(session.id, { - type: 'user', - id: 'message-1', - turnId: 'turn-1', - ts: 1, - text: 'must not overwrite corruption', - }), - /Cannot append after an invalid JSONL tail record/, - ); - assert.equal(await readFile(sessionPath, 'utf8'), before); - } finally { - await owner.close(); - } - }); - }); - - test('rejects stale writers before a replacement root is mutated', async () => { - await withRoot(async ({ base, root }) => { - const capability = await resolveStorageRoot({ - path: root, - kind: 'interactive', - }); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(owner); - if (!owner) return; - const stores = await openInteractiveExecutionStoresForWrite(owner.lease); - const moved = join(base, 'moved-root'); - await rename(root, moved); - await mkdir(root); - try { - await assert.rejects( - () => stores.sessionStore.create(sessionInput(root)), - (error: unknown) => - error instanceof StorageRootAuthorityError && error.code === 'root_identity_changed', - ); - await assert.rejects(() => stat(join(root, 'sessions')), { - code: 'ENOENT', - }); - } finally { - await owner.close(); - } - }); - }); - - test('SQLite recovery ignores and preserves retired file-store staging artifacts', async () => { - await withRoot(async ({ root }) => { - const capability = await resolveStorageRoot({ - path: root, - kind: 'interactive', - }); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(owner); - if (!owner) return; - try { - const stores = await openInteractiveExecutionStoresForWrite(owner.lease); - const session = await stores.sessionStore.create(sessionInput(root)); - await stores.agentRunStore.admitRootTurn({ - sessionId: session.id, - turnId: 'turn-1', - proposedRunId: 'run-1', - proposedUserMessageId: 'message-1', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { text: 'hello' }, - sourceMessages: [], - admittedAt: 10, - }); - - const suffix = '123.00000000-0000-4000-8000-000000000000.tmp'; - const admissionsRoot = join(root, 'sessions', session.id, 'turn-admissions'); - const admissionTemp = join(admissionsRoot, `turn-1.json.${suffix}`); - await mkdir(admissionsRoot, { recursive: true }); - await writeFile(admissionTemp, 'staging', 'utf8'); - const runDirectory = join(root, 'sessions', session.id, 'runs', 'run-staging'); - await mkdir(runDirectory, { recursive: true }); - await writeFile(join(runDirectory, `run.json.${suffix}`), 'staging', 'utf8'); - - const admissions = await stores.agentRunStore.listRootTurnAdmissionsForRecovery(session.id); - assert.deepEqual( - admissions.map((admission) => admission.turnId), - ['turn-1'], - ); - assert.deepEqual(await stores.agentRunStore.listSessionRunsForRecovery(session.id), []); - assert.ok((await stat(admissionTemp)).isFile()); - assert.ok((await stat(runDirectory)).isDirectory()); - } finally { - await owner.close(); - } - }); - }); - - test('strict recovery orders same-millisecond admissions by predecessor lineage', async () => { - await withRoot(async ({ root }) => { - const store = createAgentRunStore(root); - await store.admitRootTurn({ - sessionId: 'session', - turnId: 'z-root', - proposedRunId: 'run-root', - proposedUserMessageId: 'message-root', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { text: 'root' }, - sourceMessages: [], - admittedAt: 100, - }); - await store.admitRootTurn({ - sessionId: 'session', - turnId: 'a-successor', - proposedRunId: 'run-successor', - proposedUserMessageId: 'message-successor', - execution: { kind: 'external_message' }, - previousRootTurnId: 'z-root', - normalizedInput: { text: 'successor' }, - sourceMessages: [], - admittedAt: 100, - }); - - const chain = await store.listRootTurnAdmissionsForRecovery('session'); - assert.deepEqual( - chain.map((admission) => admission.turnId), - ['z-root', 'a-successor'], - ); - }); - }); - - test('strict recovery rejects malformed predecessor graphs', async () => { - await withRoot(async ({ root }) => { - const store = createAgentRunStore(root); - const admissionsRoot = join(root, 'sessions', 'session', 'turn-admissions'); - const install = async ( - records: readonly ReturnType[], - ): Promise => { - await rm(admissionsRoot, { recursive: true, force: true }); - await mkdir(admissionsRoot, { recursive: true }); - await Promise.all( - records.map((record) => - writeFile( - join(admissionsRoot, `${record.turnId}.json`), - `${JSON.stringify(record)}\n`, - 'utf8', - ), - ), - ); - }; - - await install([rootAdmissionRecord('root', null), rootAdmissionRecord('missing', 'absent')]); - await assert.rejects( - () => store.listRootTurnAdmissionsForRecovery('session'), - /missing predecessor/, - ); - - await install([rootAdmissionRecord('root-a', null), rootAdmissionRecord('root-b', null)]); - await assert.rejects( - () => store.listRootTurnAdmissionsForRecovery('session'), - /exactly one root/, - ); - - await install([ - rootAdmissionRecord('root', null), - rootAdmissionRecord('left', 'root'), - rootAdmissionRecord('right', 'root'), - ]); - await assert.rejects(() => store.listRootTurnAdmissionsForRecovery('session'), /branches/); - - await install([ - rootAdmissionRecord('cycle-a', 'cycle-b'), - rootAdmissionRecord('cycle-b', 'cycle-a'), - ]); - await assert.rejects( - () => store.listRootTurnAdmissionsForRecovery('session'), - /exactly one root/, - ); - }); - }); - - test('SQLite recovery is isolated from post-cutover legacy file corruption', async () => { - await withRoot(async ({ root }) => { - const capability = await resolveStorageRoot({ - path: root, - kind: 'interactive', - }); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(owner); - if (!owner) return; - try { - const stores = await openInteractiveExecutionStoresForWrite(owner.lease); - const session = await stores.sessionStore.create(sessionInput(root)); - await stores.agentRunStore.admitRootTurn({ - sessionId: session.id, - turnId: 'turn-1', - proposedRunId: 'run-1', - proposedUserMessageId: 'message-1', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { text: 'hello' }, - sourceMessages: [], - admittedAt: 10, - }); - const admissionRoot = join(root, 'sessions', session.id, 'turn-admissions'); - await mkdir(admissionRoot, { recursive: true }); - await writeFile(join(admissionRoot, 'turn-1.json'), '{"turnId":"wrong"}\n', 'utf8'); - assert.equal( - (await stores.agentRunStore.listRootTurnAdmissionsForRecovery(session.id))[0]?.turnId, - 'turn-1', - ); - - await stores.agentRunStore.createRun(runHeader(session.id, 'run-1')); - const runRoot = join(root, 'sessions', session.id, 'runs', 'run-1'); - await mkdir(runRoot, { recursive: true }); - await writeFile(join(runRoot, 'run.json'), '{"runId":"wrong"}\n', 'utf8'); - assert.equal( - (await stores.agentRunStore.listSessionRunsForRecovery(session.id))[0]?.runId, - 'run-1', - ); - - await writeFile( - join(root, 'sessions', session.id, 'session.jsonl'), - '{"id":"wrong"}\n', - 'utf8', - ); - await assert.rejects(() => stores.sessionStore.listForRecovery()); - } finally { - await owner.close(); - } - }); - }); -}); - -function rootAdmissionRecord(turnId: string, previousRootTurnId: string | null) { - return { - schemaVersion: ROOT_TURN_ADMISSION_SCHEMA_VERSION, - sessionId: 'session', - turnId, - runId: `run-${turnId}`, - userMessageId: `message-${turnId}`, - execution: { kind: 'external_message' as const }, - previousRootTurnId, - normalizedInput: { text: turnId }, - sourceMessages: [], - admittedAt: 100, - }; -} - -async function withRoot( - run: (paths: { base: string; root: string }) => Promise, -): Promise { - const base = await mkdtemp(join(tmpdir(), 'maka-execution-stores-')); - const root = join(base, 'root'); - try { - await run({ base, root }); - } finally { - await rm(base, { recursive: true, force: true }); - } -} - -function sessionInput(root: string) { - return { - cwd: root, - backend: 'fake' as const, - llmConnectionSlug: 'fake', - model: 'fake-model', - permissionMode: 'ask' as const, - }; -} - -function runHeader(sessionId: string, runId: string): AgentRunHeader { - return { - runId, - invocationId: runId, - sessionId, - turnId: 'turn-1', - status: 'created', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/tmp/cwd', - permissionMode: 'ask', - createdAt: 10, - updatedAt: 10, - }; -} - -function runEvent(sessionId: string, runId: string, id: string, ts: number): AgentRunEvent { - return { - type: 'run_started', - id, - runId, - sessionId, - turnId: 'turn-1', - ts, - }; -} - -function runtimeEvent(sessionId: string, runId: string, id: string, ts: number): RuntimeEvent { - return { - id, - invocationId: runId, - runId, - sessionId, - turnId: 'turn-1', - ts, - partial: false, - role: 'user', - author: 'user', - content: { kind: 'text', text: 'hello' }, - }; -} - -function conversationCopyToolLedger(sessionId: string, runId: string): RuntimeEvent[] { - const operationId = `${runId}-operation`; - const providerToolCallId = `${runId}-provider-call`; - const canonicalArgsHash = canonicalToolArgsHash('Read', { path: 'README.md' }); - const identity = { - invocationId: runId, - runId, - sessionId, - turnId: 'turn-1', - partial: false as const, - }; - return [ - { - ...identity, - id: `${runId}-call`, - ts: 2, - role: 'model', - author: 'agent', - content: { - kind: 'function_call', - id: providerToolCallId, - name: 'Read', - args: { path: 'README.md' }, - }, - }, - { - ...identity, - id: `${runId}-dispatch`, - ts: 3, - role: 'system', - author: 'system', - actions: { - toolDispatch: { - protocol: 't1_after_preflight_v1', - operationId, - providerToolCallId, - toolName: 'Read', - canonicalArgsHash, - recoveryMode: 'replay_safe', - }, - }, - refs: { operationId, toolCallId: providerToolCallId }, - }, - { - ...identity, - id: `${runId}-response`, - ts: 4, - role: 'tool', - author: 'tool', - content: { - kind: 'function_response', - id: providerToolCallId, - name: 'Read', - result: 'contents', - }, - refs: { operationId, toolCallId: providerToolCallId }, - }, - ]; -} diff --git a/packages/storage/src/__tests__/interaction-store.test.ts b/packages/storage/src/__tests__/interaction-store.test.ts deleted file mode 100644 index 59c8aca91d..0000000000 --- a/packages/storage/src/__tests__/interaction-store.test.ts +++ /dev/null @@ -1,822 +0,0 @@ -import assert from 'node:assert/strict'; -import { execFile } from 'node:child_process'; -import { - link, - mkdir, - mkdtemp, - open, - readFile, - readdir, - rm, - symlink, - unlink, - writeFile, -} from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { promisify } from 'node:util'; -import { describe, mock, test } from 'node:test'; -import { - projectInteractionQuestionRequest, - type InteractionCanonicalOutcome, - type InteractionRequest, -} from '@maka/core'; -import { - interactionLocator, - openInteractiveInteractionStoreForRead, - openInteractiveInteractionStoreForWrite, - STORED_INTERACTION_REQUEST_MAX_BYTES, - type InteractiveInteractionStoreWriterFacade, - type StoredInteractionRequest, -} from '../interaction-store.js'; -import { - InteractionLocatorLane, - runInteractionStoreOperation, -} from '../interaction-locator-lane.js'; -import { - resolveStorageRoot, - tryAcquireInteractiveRootReader, - tryAcquireInteractiveRootOwner, - type InteractiveRootOwner, -} from '../root-authority.js'; - -const execFileAsync = promisify(execFile); - -describe('Interaction Store', () => { - test('rejects an open whose owner closes while recovery is blocked without caching a facade', { - skip: process.platform === 'win32', - }, async () => { - const base = await mkdtemp(join(tmpdir(), 'maka-interaction-open-close-race-')); - const root = join(base, 'root'); - await mkdir(root); - const capability = await resolveStorageRoot({ path: root, kind: 'interactive' }); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(owner); - if (!owner) return; - - const probe = await open(root, 'r'); - const fileHandlePrototype = Object.getPrototypeOf(probe) as { - sync: typeof probe.sync; - }; - const originalSync = fileHandlePrototype.sync; - await probe.close(); - const recoveryBlocked = deferred(); - const releaseRecovery = deferred(); - let blocked = false; - const syncMock = mock.method(fileHandlePrototype, 'sync', async function (this: typeof probe) { - if (!blocked) { - blocked = true; - recoveryBlocked.resolve(); - await releaseRecovery.promise; - } - return originalSync.call(this); - }); - - let opening: Promise | undefined; - let closing: Promise | undefined; - try { - opening = openInteractiveInteractionStoreForWrite(owner.lease); - await recoveryBlocked.promise; - closing = owner.close(); - assert.equal(owner.closed, true); - releaseRecovery.resolve(); - - await assert.rejects(opening, { code: 'invalid_lease' }); - await closing; - await assert.rejects(openInteractiveInteractionStoreForWrite(owner.lease), { - code: 'invalid_lease', - }); - } finally { - releaseRecovery.resolve(); - await Promise.allSettled([opening, closing].filter((task) => task !== undefined)); - syncMock.mock.restore(); - if (!owner.closed) await owner.close(); - await rm(owner.controlDirectory, { recursive: true, force: true }); - await rm(base, { recursive: true, force: true }); - } - }); - - test('keeps one immutable winner across concurrent calls on the same lease facade', async () => { - await withStore(async ({ owner, store }) => { - const sameLeaseStore = await openInteractiveInteractionStoreForWrite(owner.lease); - const request = storedQuestion('request_winner', 100); - const established = await Promise.all([ - store.establishRequest(request), - sameLeaseStore.establishRequest({ ...request, runId: 'run_competing' }), - ]); - assert.equal( - established.filter((result) => result.status === 'stable' && result.matches).length, - 1, - ); - assert.equal( - established.filter((result) => result.status === 'stable' && !result.matches).length, - 1, - ); - - const winner = await store.readInteraction(request.requestId); - assert.ok(winner); - const equivalentRetry = await store.establishRequest(winner.request); - assert.equal(equivalentRetry.status, 'stable'); - if (equivalentRetry.status === 'stable') assert.equal(equivalentRetry.matches, true); - const first = questionOutcome('first', 200); - const second = questionOutcome('second', 201); - const outcomes = await Promise.all([ - store.commitOutcome(request.requestId, first), - sameLeaseStore.commitOutcome(request.requestId, second), - ]); - assert.equal( - outcomes.filter((result) => result.status === 'stable' && result.matches).length, - 1, - ); - assert.equal( - outcomes.filter((result) => result.status === 'stable' && !result.matches).length, - 1, - ); - const canonical = await store.readInteraction(request.requestId); - assert.ok(canonical?.outcome); - const equivalent = { - ...canonical.outcome.outcome, - committedAt: canonical.outcome.outcome.committedAt + 1, - } as InteractionCanonicalOutcome; - const retry = await store.commitOutcome(request.requestId, equivalent); - assert.equal(retry.status, 'stable'); - if (retry.status === 'stable') assert.equal(retry.matches, true); - }); - }); - - test('filters pending requests and excludes committed records', async () => { - await withStore(async ({ store }) => { - const first = storedQuestion('request_first', 200); - const second = { - ...storedQuestion('request_second', 100), - turnId: 'turn_2', - }; - await store.establishRequest(first); - await store.establishRequest(second); - assert.deepEqual(await store.listPending({ turnId: 'turn_2', kind: 'question' }), [second]); - assert.deepEqual( - await store.listPending({ sessionId: 'session_1', turnId: 'turn_2', kind: 'question' }), - [second], - ); - await store.commitOutcome(second.requestId, questionOutcome('done', 300)); - assert.deepEqual(await store.listPending(), [first]); - }); - }); - - test('lists one Session without reading unrelated canonical history', async () => { - await withStore(async ({ root, store }) => { - const target = storedQuestion('request_target', 100); - await store.establishRequest(target); - for (let index = 0; index < 64; index += 1) { - await store.establishRequest({ - ...storedQuestion(`request_unrelated_${index}`, index), - sessionId: 'session_unrelated', - }); - } - const damagedLocator = interactionLocator('request_unrelated_31'); - await writeFile(join(root, 'interactions', damagedLocator, 'request.json'), '{broken'); - - assert.deepEqual(await store.listSessionPending(target.sessionId), [target]); - assert.deepEqual(await store.listPending({ sessionId: target.sessionId }), [target]); - }); - }); - - test('recovery repairs missing and stale Session pending markers', async () => { - await withStore(async ({ root, owner, store }) => { - const pending = storedQuestion('request_pending_recovery', 100); - const settled = storedQuestion('request_settled_recovery', 101); - await store.establishRequest(pending); - await store.establishRequest(settled); - await store.commitOutcome(settled.requestId, questionOutcome('done', 200)); - - const sessionPending = join(root, 'interactions', 'pending', interactionLocator('session_1')); - const missingMarker = join(sessionPending, interactionLocator(pending.requestId)); - const staleMarker = join(sessionPending, interactionLocator(settled.requestId)); - await unlink(missingMarker); - await writeFile(staleMarker, ''); - assert.deepEqual(await store.listSessionPending('session_1'), []); - await owner.close(); - - const capability = await resolveStorageRoot({ path: root, kind: 'interactive' }); - const readerOwner = await tryAcquireInteractiveRootReader(capability); - assert.ok(readerOwner); - if (!readerOwner) return; - try { - const reader = await openInteractiveInteractionStoreForRead(readerOwner.lease); - assert.deepEqual(await reader.listSessionPending('session_1'), [pending]); - assert.deepEqual(await reader.listPending({ sessionId: 'session_1' }), [pending]); - } finally { - await readerOwner.close(); - } - - const recoveredOwner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(recoveredOwner); - if (!recoveredOwner) return; - try { - const recovered = await openInteractiveInteractionStoreForWrite(recoveredOwner.lease); - assert.deepEqual(await recovered.listSessionPending('session_1'), [pending]); - assert.equal(await readFile(missingMarker, 'utf8'), ''); - await assert.rejects(readFile(staleMarker), { code: 'ENOENT' }); - } finally { - await recoveredOwner.close(); - } - }); - }); - - test('rejects symlinked pending index directories without writing outside the root', { - skip: process.platform === 'win32', - }, async () => { - const base = await mkdtemp(join(tmpdir(), 'maka-interaction-index-parent-')); - const root = join(base, 'root'); - const outside = join(base, 'outside'); - await mkdir(root); - await mkdir(outside); - const capability = await resolveStorageRoot({ path: root, kind: 'interactive' }); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(owner); - if (!owner) return; - try { - await mkdir(join(root, 'interactions')); - await symlink(outside, join(root, 'interactions', 'pending'), 'dir'); - await assert.rejects(openInteractiveInteractionStoreForWrite(owner.lease), { - code: 'invalid_record', - }); - assert.deepEqual(await readdir(outside), []); - } finally { - await owner.close(); - await rm(owner.controlDirectory, { recursive: true, force: true }); - await rm(base, { recursive: true, force: true }); - } - }); - - test('rejects a symlinked Session index directory without writing outside the root', { - skip: process.platform === 'win32', - }, async () => { - await withStore(async ({ root, store }) => { - const outside = await mkdtemp(join(tmpdir(), 'maka-interaction-index-session-')); - const request = storedQuestion('request_session_symlink', 100); - const sessionDirectory = join( - root, - 'interactions', - 'pending', - interactionLocator(request.sessionId), - ); - try { - await symlink(outside, sessionDirectory, 'dir'); - const result = await store.establishRequest(request); - assert.equal(result.status, 'unresolved'); - if (result.status === 'unresolved') { - assert.equal(result.failure.code, 'invalid_record'); - } - assert.deepEqual(await readdir(outside), []); - } finally { - await rm(outside, { recursive: true, force: true }); - } - }); - }); - - test('does not publish a request through a pre-existing locator symlink', { - skip: process.platform === 'win32', - }, async () => { - await withStore(async ({ root, store }) => { - const outside = await mkdtemp(join(tmpdir(), 'maka-interaction-locator-request-')); - const request = storedQuestion('request_locator_symlink', 100); - const locator = join(root, 'interactions', interactionLocator(request.requestId)); - const sentinel = join(outside, 'sentinel'); - try { - await writeFile(sentinel, 'unchanged'); - await symlink(outside, locator, 'dir'); - - const result = await store.establishRequest(request); - assert.equal(result.status, 'unresolved'); - if (result.status === 'unresolved') assert.equal(result.failure.code, 'invalid_record'); - assert.deepEqual(await readdir(outside), ['sentinel']); - assert.equal(await readFile(sentinel, 'utf8'), 'unchanged'); - } finally { - await rm(outside, { recursive: true, force: true }); - } - }); - }); - - test('does not publish an outcome through a replaced locator symlink', { - skip: process.platform === 'win32', - }, async () => { - await withStore(async ({ root, store }) => { - const outside = await mkdtemp(join(tmpdir(), 'maka-interaction-locator-outcome-')); - const request = storedQuestion('outcome_locator_symlink', 100); - const locator = join(root, 'interactions', interactionLocator(request.requestId)); - const sentinel = join(outside, 'sentinel'); - try { - const established = await store.establishRequest(request); - assert.equal(established.status, 'stable'); - await rm(locator, { recursive: true }); - await writeFile(sentinel, 'unchanged'); - await symlink(outside, locator, 'dir'); - - const result = await store.commitOutcome(request.requestId, questionOutcome('done', 200)); - assert.equal(result.status, 'unresolved'); - if (result.status === 'unresolved') assert.equal(result.failure.code, 'invalid_record'); - assert.deepEqual(await readdir(outside), ['sentinel']); - assert.equal(await readFile(sentinel, 'utf8'), 'unchanged'); - } finally { - await rm(outside, { recursive: true, force: true }); - } - }); - }); - - test('does not read canonical documents through a locator symlink', { - skip: process.platform === 'win32', - }, async () => { - await withStore(async ({ root, store }) => { - const outside = await mkdtemp(join(tmpdir(), 'maka-interaction-locator-read-')); - const request = storedQuestion('read_locator_symlink', 100); - const locator = join(root, 'interactions', interactionLocator(request.requestId)); - const externalRequest = join(outside, 'request.json'); - try { - await writeFile(externalRequest, `${JSON.stringify(request)}\n`); - await symlink(outside, locator, 'dir'); - - await assert.rejects(store.readInteraction(request.requestId), { - code: 'invalid_record', - }); - assert.deepEqual(await readdir(outside), ['request.json']); - assert.equal(await readFile(externalRequest, 'utf8'), `${JSON.stringify(request)}\n`); - } finally { - await rm(outside, { recursive: true, force: true }); - } - }); - }); - - test('does not follow a canonical request symlink during publication', { - skip: process.platform === 'win32', - }, async () => { - await withStore(async ({ root, store }) => { - const outside = await mkdtemp(join(tmpdir(), 'maka-interaction-request-document-')); - const request = storedQuestion('request_document_symlink', 100); - const locator = join(root, 'interactions', interactionLocator(request.requestId)); - const externalRequest = join(outside, 'request.json'); - const externalContents = `${JSON.stringify(request)}\n`; - try { - await mkdir(locator); - await writeFile(externalRequest, externalContents); - await symlink(externalRequest, join(locator, 'request.json')); - - const result = await store.establishRequest(request); - assert.equal(result.status, 'unresolved'); - if (result.status === 'unresolved') assert.equal(result.failure.code, 'invalid_record'); - assert.equal(await readFile(externalRequest, 'utf8'), externalContents); - } finally { - await rm(outside, { recursive: true, force: true }); - } - }); - }); - - test('does not follow a canonical outcome symlink during publication', { - skip: process.platform === 'win32', - }, async () => { - await withStore(async ({ root, store }) => { - const outside = await mkdtemp(join(tmpdir(), 'maka-interaction-outcome-document-')); - const request = storedQuestion('outcome_document_symlink', 100); - const locator = join(root, 'interactions', interactionLocator(request.requestId)); - const externalOutcome = join(outside, 'outcome.json'); - const outcome = questionOutcome('done', 200); - const externalContents = `${JSON.stringify({ ...identity(request), outcome })}\n`; - try { - const established = await store.establishRequest(request); - assert.equal(established.status, 'stable'); - await writeFile(externalOutcome, externalContents); - await symlink(externalOutcome, join(locator, 'outcome.json')); - - const result = await store.commitOutcome(request.requestId, outcome); - assert.equal(result.status, 'unresolved'); - if (result.status === 'unresolved') assert.equal(result.failure.code, 'invalid_record'); - assert.equal(await readFile(externalOutcome, 'utf8'), externalContents); - } finally { - await rm(outside, { recursive: true, force: true }); - } - }); - }); - - test('recovery rejects a locator symlink without reading or cleaning the external directory', { - skip: process.platform === 'win32', - }, async () => { - await withStore(async ({ root, owner }) => { - const outside = await mkdtemp(join(tmpdir(), 'maka-interaction-locator-recovery-')); - const request = storedQuestion('recovery_locator_symlink', 100); - const locator = join(root, 'interactions', interactionLocator(request.requestId)); - const externalRequest = join(outside, 'request.json'); - const externalTemp = join(outside, 'outcome.json.00000000-0000-4000-8000-000000000003.tmp'); - try { - await writeFile(externalRequest, `${JSON.stringify(request)}\n`); - await writeFile(externalTemp, 'must remain'); - await symlink(outside, locator, 'dir'); - await owner.close(); - - const capability = await resolveStorageRoot({ path: root, kind: 'interactive' }); - const recoveredOwner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(recoveredOwner); - if (!recoveredOwner) return; - try { - await assert.rejects(openInteractiveInteractionStoreForWrite(recoveredOwner.lease), { - code: 'invalid_record', - }); - assert.deepEqual((await readdir(outside)).sort(), [ - 'outcome.json.00000000-0000-4000-8000-000000000003.tmp', - 'request.json', - ]); - assert.equal(await readFile(externalRequest, 'utf8'), `${JSON.stringify(request)}\n`); - assert.equal(await readFile(externalTemp, 'utf8'), 'must remain'); - } finally { - await recoveredOwner.close(); - } - } finally { - await rm(outside, { recursive: true, force: true }); - } - }); - }); - - test('recovery rejects a non-directory with a canonical locator name', async () => { - await withStore(async ({ root, owner }) => { - const request = storedQuestion('recovery_locator_file', 100); - const locator = join(root, 'interactions', interactionLocator(request.requestId)); - await writeFile(locator, 'not a directory'); - await owner.close(); - - const capability = await resolveStorageRoot({ path: root, kind: 'interactive' }); - const recoveredOwner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(recoveredOwner); - if (!recoveredOwner) return; - try { - await assert.rejects(openInteractiveInteractionStoreForWrite(recoveredOwner.lease), { - code: 'invalid_record', - }); - assert.equal(await readFile(locator, 'utf8'), 'not a directory'); - } finally { - await recoveredOwner.close(); - } - }); - }); - - test('rejects a malformed marker during outcome cleanup and recovery', async () => { - await withStore(async ({ root, owner, store }) => { - const request = storedQuestion('request_malformed_cleanup', 100); - await store.establishRequest(request); - const marker = join( - root, - 'interactions', - 'pending', - interactionLocator(request.sessionId), - interactionLocator(request.requestId), - ); - await writeFile(marker, 'not-empty'); - - const committed = await store.commitOutcome(request.requestId, questionOutcome('done', 200)); - assert.equal(committed.status, 'unresolved'); - if (committed.status === 'unresolved') { - assert.equal(committed.failure.code, 'invalid_record'); - } - assert.equal(await readFile(marker, 'utf8'), 'not-empty'); - await owner.close(); - - const capability = await resolveStorageRoot({ path: root, kind: 'interactive' }); - const recoveredOwner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(recoveredOwner); - if (!recoveredOwner) return; - try { - await assert.rejects(openInteractiveInteractionStoreForWrite(recoveredOwner.lease), { - code: 'invalid_record', - }); - assert.equal(await readFile(marker, 'utf8'), 'not-empty'); - } finally { - await recoveredOwner.close(); - } - }); - }); - - test('fails a FIFO marker EEXIST retry without blocking', { - skip: process.platform === 'win32', - timeout: 3_000, - }, async () => { - await withStore(async ({ root, store }) => { - const request = storedQuestion('request_fifo_marker', 100); - await store.establishRequest(request); - const marker = join( - root, - 'interactions', - 'pending', - interactionLocator(request.sessionId), - interactionLocator(request.requestId), - ); - await unlink(marker); - await execFileAsync('mkfifo', [marker]); - - const startedAt = Date.now(); - const retry = await store.establishRequest(request); - assert.ok(Date.now() - startedAt < 1_000); - assert.equal(retry.status, 'unresolved'); - if (retry.status === 'unresolved') { - assert.equal(retry.failure.code, 'invalid_record'); - } - }); - }); - - test('recovery fails closed on a non-regular expected pending marker', async () => { - await withStore(async ({ root, owner, store }) => { - const pending = storedQuestion('request_non_regular_marker', 100); - await store.establishRequest(pending); - const marker = join( - root, - 'interactions', - 'pending', - interactionLocator(pending.sessionId), - interactionLocator(pending.requestId), - ); - await unlink(marker); - await mkdir(marker); - const retry = await store.establishRequest(pending); - assert.equal(retry.status, 'unresolved'); - if (retry.status === 'unresolved') { - assert.equal(retry.failure.code, 'invalid_record'); - } - await owner.close(); - - const capability = await resolveStorageRoot({ path: root, kind: 'interactive' }); - const recoveredOwner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(recoveredOwner); - if (!recoveredOwner) return; - try { - await assert.rejects(openInteractiveInteractionStoreForWrite(recoveredOwner.lease), { - code: 'invalid_record', - }); - } finally { - await recoveredOwner.close(); - } - }); - }); - - test('reads back and stabilizes an outcome linked before an ambiguous failure', async () => { - await withStore(async ({ root, store }) => { - const request = storedQuestion('request_torn', 100); - await store.establishRequest(request); - const locator = join(root, 'interactions', interactionLocator(request.requestId)); - const temporary = join(locator, 'outcome.json.00000000-0000-4000-8000-000000000000.tmp'); - const outcome = questionOutcome('durable', 200); - await writeFile(temporary, `${JSON.stringify({ ...identity(request), outcome })}\n`); - await link(temporary, join(locator, 'outcome.json')); - - const result = await store.commitOutcome(request.requestId, outcome); - assert.equal(result.status, 'stable'); - if (result.status === 'stable') assert.equal(result.matches, true); - await assert.rejects(readFile(temporary), { code: 'ENOENT' }); - }); - }); - - test('serializes a same-locator read behind active publication and stabilization', async () => { - await withStore(async ({ root, store }) => { - const request = storedQuestion('request_serialized_read', 100); - const locator = join(root, 'interactions', interactionLocator(request.requestId)); - const temporary = join(locator, 'request.json.00000000-0000-4000-8000-000000000002.tmp'); - await mkdir(locator, { recursive: true }); - await writeFile(temporary, 'interrupted publication'); - - await assert.rejects(store.readInteraction(request.requestId), { code: 'invalid_record' }); - - const publication = store.establishRequest(request); - const concurrentRead = store.readInteraction(request.requestId); - const [established, observed] = await Promise.all([publication, concurrentRead]); - - assert.equal(established.status, 'stable'); - if (established.status === 'stable') assert.equal(established.matches, true); - assert.deepEqual(observed, { request }); - await assert.rejects(readFile(temporary), { code: 'ENOENT' }); - }); - }); - - test('retains a same-locator reservation until its predecessor drains', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-interaction-locator-lane-')); - const firstDirectory = join(root, 'first'); - const otherDirectory = join(root, 'other'); - const temporary = join(firstDirectory, 'request.json.tmp'); - const canonical = join(firstDirectory, 'request.json'); - await mkdir(firstDirectory); - await mkdir(otherDirectory); - await writeFile(temporary, 'interrupted publication'); - - const lane = new InteractionLocatorLane(); - const firstEntered = deferred(); - const releaseFirst = deferred(); - const thirdAuthorized = deferred(); - let thirdEntered = false; - const authorityFailure = new Error('authority failed'); - let first: Promise | undefined; - let third: Promise | undefined; - try { - first = lane.run('same-locator', runInteractionStoreOperation, async () => { - firstEntered.resolve(); - await releaseFirst.promise; - await unlink(temporary); - await writeFile(canonical, 'stable'); - }); - await firstEntered.promise; - - const rejectAuthority = async (_operation: () => Promise): Promise => { - throw authorityFailure; - }; - const failedAuthority = lane.run('same-locator', rejectAuthority, async () => { - throw new Error('authority failure must not execute its operation'); - }); - await assert.rejects(failedAuthority, authorityFailure); - - third = lane.run( - 'same-locator', - async (operation) => { - thirdAuthorized.resolve(); - return operation(); - }, - async () => { - thirdEntered = true; - await assert.rejects(readFile(temporary), { code: 'ENOENT' }); - return readFile(canonical, 'utf8'); - }, - ); - await thirdAuthorized.promise; - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(thirdEntered, false); - - const other = await lane.run('other-locator', runInteractionStoreOperation, async () => { - const path = join(otherDirectory, 'request.json'); - await writeFile(path, 'independent'); - return readFile(path, 'utf8'); - }); - assert.equal(other, 'independent'); - - releaseFirst.resolve(); - await first; - assert.equal(await third, 'stable'); - } finally { - releaseFirst.resolve(); - await Promise.allSettled([first, third].filter((task) => task !== undefined)); - await rm(root, { recursive: true, force: true }); - } - }); - - test('rejects dense and sparse documents beyond the stored byte limit', async () => { - await withStore(async ({ root, store }) => { - const request = storedQuestion('request_oversized', 100); - await store.establishRequest(request); - const requestPath = join( - root, - 'interactions', - interactionLocator(request.requestId), - 'request.json', - ); - - await writeFile(requestPath, Buffer.alloc(STORED_INTERACTION_REQUEST_MAX_BYTES + 1)); - await assert.rejects(store.readInteraction(request.requestId), { code: 'invalid_record' }); - - const sparse = await open(requestPath, 'w'); - try { - await sparse.truncate(STORED_INTERACTION_REQUEST_MAX_BYTES + 1); - } finally { - await sparse.close(); - } - await assert.rejects(store.readInteraction(request.requestId), { code: 'invalid_record' }); - }); - }); - - test('rejects a stored outcome with fields outside the closed schema', async () => { - await withStore(async ({ root, store }) => { - const request = storedQuestion('request_invalid_outcome', 100); - await store.establishRequest(request); - const locator = join(root, 'interactions', interactionLocator(request.requestId)); - await writeFile( - join(locator, 'outcome.json'), - `${JSON.stringify({ - ...identity(request), - outcome: questionOutcome('answer', 200), - unexpected: true, - })}\n`, - ); - - await assert.rejects(store.readInteraction(request.requestId), { code: 'invalid_record' }); - }); - }); - - test('only persists canonical safe question projections', async () => { - await withStore(async ({ store }) => { - const unsafe: StoredInteractionRequest = { - ...storedQuestion('request_unsafe_question', 100), - request: { - kind: 'question', - toolUseId: 'tool_1', - questions: [ - { - question: 'password=store-secret\u202e', - options: [{ label: 'First' }, { label: 'Second' }], - }, - ], - }, - }; - await assert.rejects(store.establishRequest(unsafe), { code: 'invalid_input' }); - assert.equal(await store.readInteraction(unsafe.requestId), undefined); - - const canonical: StoredInteractionRequest = { - ...storedQuestion('request_safe_question', 101), - request: projectInteractionQuestionRequest({ - toolUseId: 'tool_1', - questions: [ - { - question: 'password=store-secret\u202e', - options: [{ label: 'First' }, { label: 'Second' }], - }, - ], - }), - }; - const established = await store.establishRequest(canonical); - assert.equal(established.status, 'stable'); - if (established.status === 'stable') assert.equal(established.matches, true); - assert.deepEqual(await store.readInteraction(canonical.requestId), { request: canonical }); - }); - }); -}); - -function storedQuestion(requestId: string, createdAt: number): StoredInteractionRequest { - return { - sessionId: 'session_1', - turnId: 'turn_1', - runId: 'run_1', - requestId, - createdAt, - request: { - kind: 'question', - toolUseId: 'tool_1', - questions: [ - { - question: 'Choose', - options: [ - { label: 'First', description: 'First' }, - { label: 'Second', description: 'Second' }, - ], - }, - ], - } as InteractionRequest, - }; -} - -function questionOutcome(answer: string, committedAt: number): InteractionCanonicalOutcome { - return { - kind: 'question_answer', - answers: [answer], - committedAt, - } as InteractionCanonicalOutcome; -} - -function identity(request: StoredInteractionRequest) { - return { - sessionId: request.sessionId, - turnId: request.turnId, - runId: request.runId, - requestId: request.requestId, - }; -} - -interface StoreContext { - root: string; - owner: InteractiveRootOwner; - store: InteractiveInteractionStoreWriterFacade; -} - -async function withStore(run: (context: StoreContext) => Promise): Promise { - const base = await mkdtemp(join(tmpdir(), 'maka-interaction-store-')); - const root = join(base, 'root'); - await mkdir(root); - const capability = await resolveStorageRoot({ - path: root, - kind: 'interactive', - }); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(owner); - if (!owner) return; - const [store, sameStore] = await Promise.all([ - openInteractiveInteractionStoreForWrite(owner.lease), - openInteractiveInteractionStoreForWrite(owner.lease), - ]); - assert.strictEqual(store, sameStore); - try { - await run({ root, owner, store }); - } finally { - if (!owner.closed) await owner.close(); - await rm(owner.controlDirectory, { recursive: true, force: true }); - await rm(base, { recursive: true, force: true }); - } -} - -function deferred() { - let resolve!: (value: T | PromiseLike) => void; - let reject!: (reason?: unknown) => void; - const promise = new Promise((resolvePromise, rejectPromise) => { - resolve = resolvePromise; - reject = rejectPromise; - }); - return { promise, resolve, reject }; -} diff --git a/packages/storage/src/__tests__/legacy-execution-public-surface.test.ts b/packages/storage/src/__tests__/legacy-execution-public-surface.test.ts deleted file mode 100644 index 8ff92a4710..0000000000 --- a/packages/storage/src/__tests__/legacy-execution-public-surface.test.ts +++ /dev/null @@ -1,20 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'node:test'; -import * as publicStorage from '../index.js'; -import * as legacyTestSupport from '../legacy-execution-test-support.js'; - -test('retired execution file writers are absent from the production storage entrypoint', () => { - assert.equal('createAgentRunStore' in publicStorage, false); - assert.equal('createRuntimeEventStore' in publicStorage, false); - assert.equal('createShellRunStore' in publicStorage, false); - assert.equal(typeof publicStorage.createSqliteAgentRunStore, 'function'); - assert.equal(typeof publicStorage.createSqliteShellRunStore, 'function'); -}); - -test('legacy execution writers are isolated behind explicit test support names', () => { - assert.deepEqual(Object.keys(legacyTestSupport).sort(), [ - 'createLegacyAgentRunStoreForTest', - 'createLegacyRuntimeEventStoreForTest', - 'createLegacyShellRunStoreForTest', - ]); -}); diff --git a/packages/storage/src/__tests__/legacy-storage-public-surface.test.ts b/packages/storage/src/__tests__/legacy-storage-public-surface.test.ts deleted file mode 100644 index 226f2df5aa..0000000000 --- a/packages/storage/src/__tests__/legacy-storage-public-surface.test.ts +++ /dev/null @@ -1,46 +0,0 @@ -import assert from 'node:assert/strict'; -import { test } from 'node:test'; -import * as publicStorage from '../index.js'; -import * as publicInteractionStore from '../interaction-store-public.js'; -import * as legacyTestSupport from '../legacy-storage-test-support.js'; - -const retiredRootWriters = [ - 'createDeepResearchStore', - 'createLegacyFileSessionStore', - 'createPlanReminderStore', - 'createPlanStore', - 'createTaskLedgerStore', - 'createTelemetryRepo', -] as const; - -test('retired structured file writers are absent from production storage entrypoints', () => { - for (const name of retiredRootWriters) { - assert.equal(name in publicStorage, false, `${name} must not be a production root export`); - } - assert.equal('openInteractiveInteractionStoreForRead' in publicInteractionStore, false); - assert.equal('openInteractiveInteractionStoreForWrite' in publicInteractionStore, false); - - assert.equal(typeof publicStorage.createSqliteDeepResearchStore, 'function'); - assert.equal(typeof publicStorage.createSqlitePlanReminderStore, 'function'); - assert.equal(typeof publicStorage.createSqlitePlanStore, 'function'); - assert.equal(typeof publicStorage.createSqliteTaskLedgerStore, 'function'); - assert.equal(typeof publicStorage.createSqliteTelemetryRepo, 'function'); - assert.equal(typeof publicStorage.createSessionStore, 'function'); - assert.equal( - typeof publicInteractionStore.openSqliteInteractiveInteractionStoreForWrite, - 'function', - ); -}); - -test('remaining legacy writers are isolated behind explicit test support names', () => { - assert.deepEqual(Object.keys(legacyTestSupport).sort(), [ - 'createLegacyDeepResearchStoreForTest', - 'createLegacyPlanReminderStoreForTest', - 'createLegacyPlanStoreForTest', - 'createLegacySessionStoreForTest', - 'createLegacyTaskLedgerStoreForTest', - 'createLegacyTelemetryRepoForTest', - 'openLegacyInteractiveInteractionStoreForReadForTest', - 'openLegacyInteractiveInteractionStoreForWriteForTest', - ]); -}); diff --git a/packages/storage/src/__tests__/message-receipt-store.test.ts b/packages/storage/src/__tests__/message-receipt-store.test.ts deleted file mode 100644 index 530eaa7697..0000000000 --- a/packages/storage/src/__tests__/message-receipt-store.test.ts +++ /dev/null @@ -1,59 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { test } from 'node:test'; -import { createMessageReceiptStore } from '../message-receipt-store.js'; - -test('commit returns the detached canonical JSON snapshot written to disk', async () => { - await withStore(async (store) => { - const receipt = { - payload: { nested: { value: 'before' } }, - result: { queueRevision: 1, retracted: [] }, - }; - - const committing = store.commit('epoch-1', 'retract', 'session-1', 'retract-1', receipt); - receipt.payload.nested.value = 'after'; - receipt.result.queueRevision = 2; - - const committed = await committing; - assert.deepEqual(committed, { - payload: { nested: { value: 'before' } }, - result: { queueRevision: 1, retracted: [] }, - }); - assert.deepEqual(await store.read('epoch-1', 'retract', 'session-1', 'retract-1'), committed); - }); -}); - -test('commit rejects receipts that are not exactly JSON representable', async () => { - await withStore(async (store) => { - const cycle: Record = {}; - cycle.self = cycle; - const invalid = [ - { label: 'undefined', payload: undefined }, - { label: 'non-finite', payload: { value: Number.NaN } }, - { label: 'bigint', payload: { value: 1n } }, - { label: 'cycle', payload: cycle }, - ]; - - for (const candidate of invalid) { - await assert.rejects(() => - store.commit('epoch-1', 'submit', 'session-1', candidate.label, { - payload: candidate.payload, - result: { disposition: 'turn_started', turnId: 'turn-1' }, - }), - ); - } - }); -}); - -async function withStore( - run: (store: ReturnType) => Promise, -): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-message-receipts-')); - try { - await run(createMessageReceiptStore(root)); - } finally { - await rm(root, { recursive: true, force: true }); - } -} diff --git a/packages/storage/src/__tests__/operational-state-backup.test.ts b/packages/storage/src/__tests__/operational-state-backup.test.ts index b7b0fb5acf..95e7ed6743 100644 --- a/packages/storage/src/__tests__/operational-state-backup.test.ts +++ b/packages/storage/src/__tests__/operational-state-backup.test.ts @@ -1,845 +1,54 @@ import assert from 'node:assert/strict'; -import { createHash } from 'node:crypto'; -import { - lstat, - mkdir, - mkdtemp, - readFile, - readdir, - rm, - stat, - symlink, - writeFile, -} from 'node:fs/promises'; +import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; -import { setTimeout as delay } from 'node:timers/promises'; import { test } from 'node:test'; -import { DatabaseSync } from 'node:sqlite'; +import { createSessionStore } from '../session-store.js'; import { createOperationalStateBackup, - OPERATIONAL_BACKUP_MANIFEST_FILE, - OPERATIONAL_BACKUP_SCHEMA_VERSION, - type OperationalBackupError, restoreOperationalStateBackup, validateOperationalStateBackup, } from '../operational-state-backup.js'; -import { - LEGACY_AUTOMATION_FILE, - openInteractiveAutomationAuthorityForWrite, -} from '../automation-authority.js'; -import { createSqliteArtifactStore } from '../artifact-store.js'; -import { resolveStorageRoot, tryAcquireInteractiveRootOwner } from '../root-authority.js'; -import { createSessionStore } from '../session-store.js'; - -test('backs up live WAL state and restores a writable relational closure', async () => { - await withBackupRoots(async ({ stateRoot, backupRoot, restoreRoot }) => { - const sessions = createSessionStore(stateRoot); - const artifacts = createSqliteArtifactStore(stateRoot); - try { - const first = await sessions.create(sessionInput('First')); - const second = await sessions.create(sessionInput('Second')); - await sessions.appendMessage(first.id, { - type: 'user', - id: 'message-1', - turnId: 'turn-1', - ts: 10, - text: 'first transcript', - }); - await sessions.appendMessage(second.id, { - type: 'user', - id: 'message-2', - turnId: 'turn-1', - ts: 11, - text: 'second transcript', - }); - const firstArtifact = await artifacts.create({ - id: 'first-artifact', - sessionId: first.id, - turnId: 'turn-1', - name: 'first.txt', - kind: 'file', - content: 'first payload\n', - now: 10, - }); - const secondArtifact = await artifacts.create({ - id: 'second-artifact', - sessionId: second.id, - turnId: 'turn-1', - name: 'second.txt', - kind: 'file', - content: 'second payload\n', - now: 11, - }); - seedAutomationState(stateRoot, first.id); - await writeFile(join(stateRoot, 'credentials.json'), 'secret-canary\n'); - await writeFile(join(stateRoot, 'settings.json'), 'settings-canary\n'); - - const liveEntries = await readdir(stateRoot); - assert.ok(liveEntries.includes('runtime.sqlite-wal'), liveEntries.join(', ')); - assert.ok(liveEntries.includes('runtime.sqlite-shm'), liveEntries.join(', ')); - - const created = await createOperationalStateBackup({ - stateRoot, - destinationRoot: backupRoot, - now: () => 100, - }); - assert.equal(created.createdAt, 100); - assert.deepEqual( - created.transcripts.map((entry) => entry.sessionId).sort(), - [first.id, second.id].sort(), - ); - assert.deepEqual( - created.artifacts.map((entry) => entry.record.id).sort(), - [firstArtifact.id, secondArtifact.id].sort(), - ); - assert.equal((await validateOperationalStateBackup(backupRoot)).createdAt, 100); - if (process.platform !== 'win32') { - assert.equal((await stat(backupRoot)).mode & 0o777, 0o700); - assert.equal((await stat(join(backupRoot, 'runtime.sqlite'))).mode & 0o777, 0o600); - assert.equal( - (await stat(join(backupRoot, created.artifacts[0]!.path))).mode & 0o777, - 0o600, - ); - } - await assert.rejects(lstat(join(backupRoot, 'credentials.json')), { code: 'ENOENT' }); - await assert.rejects(lstat(join(backupRoot, 'settings.json')), { code: 'ENOENT' }); - await assert.rejects(lstat(join(backupRoot, 'runtime.sqlite-wal')), { code: 'ENOENT' }); - await assert.rejects(lstat(join(backupRoot, 'runtime.sqlite-shm')), { code: 'ENOENT' }); - await assert.rejects(lstat(join(backupRoot, 'artifacts', 'metadata.jsonl')), { - code: 'ENOENT', - }); - - await restoreOperationalStateBackup({ backupRoot, destinationRoot: restoreRoot }); - if (process.platform !== 'win32') { - assert.equal((await stat(restoreRoot)).mode & 0o777, 0o700); - assert.equal((await stat(join(restoreRoot, 'runtime.sqlite'))).mode & 0o777, 0o600); - } - await assert.rejects(lstat(join(restoreRoot, OPERATIONAL_BACKUP_MANIFEST_FILE)), { - code: 'ENOENT', - }); - const restoredSessions = createSessionStore(restoreRoot); - const restoredArtifacts = createSqliteArtifactStore(restoreRoot); - try { - assert.deepEqual(readAutomationState(restoreRoot), { - revision: 1, - automationId: 'backup-automation', - name: 'Backup automation', - pendingFireId: 'backup-fire', - }); - assert.deepEqual( - (await restoredSessions.list()).map((session) => session.id).sort(), - [first.id, second.id].sort(), - ); - assert.match( - await readFile(join(restoreRoot, 'sessions', first.id, 'session.jsonl'), 'utf8'), - /first transcript/, - ); - assert.deepEqual(await restoredArtifacts.list(first.id), [firstArtifact]); - assert.deepEqual(await restoredArtifacts.list(second.id), [secondArtifact]); - assert.deepEqual(await restoredArtifacts.readText(secondArtifact.id), { - ok: true, - text: 'second payload\n', - }); - const continued = await restoredArtifacts.create({ - id: 'continued-write', - sessionId: first.id, - turnId: 'turn-2', - name: 'continued.txt', - kind: 'file', - content: 'continued\n', - now: 12, - }); - assert.equal((await restoredArtifacts.get(continued.id))?.id, continued.id); - await restoredSessions.appendMessage(first.id, { - type: 'user', - id: 'message-3', - turnId: 'turn-2', - ts: 12, - text: 'continued transcript', - }); - } finally { - restoredArtifacts.close?.(); - await restoredSessions.close?.(); - } - } finally { - artifacts.close?.(); - await sessions.close?.(); - } - }); -}); - -test('restores a writable Automation authority after a real legacy cutover', async () => { - await withBackupRoots(async ({ stateRoot, backupRoot, restoreRoot }) => { - const sessionId = await createSession(stateRoot); - const legacyAutomation = backupAutomationDefinition(sessionId); - await writeFile( - join(stateRoot, LEGACY_AUTOMATION_FILE), - `${JSON.stringify({ version: 1, automations: [legacyAutomation] }, null, 2)}\n`, - ); - const sourceCapability = await resolveStorageRoot({ path: stateRoot, kind: 'interactive' }); - const sourceOwner = await tryAcquireInteractiveRootOwner(sourceCapability); - assert.ok(sourceOwner); - if (!sourceOwner) return; - try { - const writer = await openInteractiveAutomationAuthorityForWrite(sourceOwner.lease); - assert.deepEqual( - (await writer.read()).automations.map((automation) => automation.id), - [legacyAutomation.id], - ); - writer.close(); - } finally { - if (!sourceOwner.closed) await sourceOwner.close(); - } - - const manifest = await createOperationalStateBackup({ - stateRoot, - destinationRoot: backupRoot, +test('backs up and restores runtime.sqlite plus artifact bytes', async () => { + const base = await mkdtemp(join(tmpdir(), 'maka-operational-backup-')); + const stateRoot = join(base, 'state'); + const backupRoot = join(base, 'backup'); + const restoreRoot = join(base, 'restore'); + const sessions = createSessionStore(stateRoot); + try { + const session = await sessions.create({ + cwd: '/tmp/cwd', + backend: 'fake', + llmConnectionSlug: 'fake', + model: 'fake-model', + permissionMode: 'ask', + name: 'Backup', + labels: [], }); - assert.equal(manifest.schemaVersion, OPERATIONAL_BACKUP_SCHEMA_VERSION); - assert.equal( - manifest.schemaVersion === OPERATIONAL_BACKUP_SCHEMA_VERSION - ? manifest.legacyAutomation?.path - : undefined, - LEGACY_AUTOMATION_FILE, - ); - await restoreOperationalStateBackup({ backupRoot, destinationRoot: restoreRoot }); - - const restoredCapability = await resolveStorageRoot({ - path: restoreRoot, - kind: 'interactive', + await sessions.appendMessage(session.id, { + type: 'user', + id: 'message-1', + turnId: 'turn-1', + ts: 1, + text: 'durable', }); - const restoredOwner = await tryAcquireInteractiveRootOwner(restoredCapability); - assert.ok(restoredOwner); - if (!restoredOwner) return; - try { - const writer = await openInteractiveAutomationAuthorityForWrite(restoredOwner.lease); - const restored = await writer.read(); - assert.deepEqual(restored.pendingFires, []); - assert.equal(restored.automations[0]?.id, legacyAutomation.id); - const committed = await writer.commit({ - expectedRevision: restored.revision, - automations: [{ ...restored.automations[0]!, name: 'Restored automation' }], - pendingFires: [], - }); - assert.equal(committed.kind, 'committed'); - assert.equal((await writer.read()).automations[0]?.name, 'Restored automation'); - writer.close(); - } finally { - if (!restoredOwner.closed) await restoredOwner.close(); - } - }); -}); - -test('restores a pre-Automation v1 backup into a writable current authority', async () => { - await withBackupRoots(async ({ stateRoot, backupRoot, restoreRoot }) => { - const sessionId = await createSession(stateRoot); - await createOperationalStateBackup({ stateRoot, destinationRoot: backupRoot }); - - const databasePath = join(backupRoot, 'runtime.sqlite'); - const database = new DatabaseSync(databasePath); - try { - database.exec(` - DROP TABLE automation_pending_fires; - DROP TABLE automation_definitions; - DROP TABLE automation_authority_state; - DELETE FROM operational_schema_migrations WHERE scope = 'automation'; - `); - } finally { - database.close(); - } - const manifestPath = join(backupRoot, OPERATIONAL_BACKUP_MANIFEST_FILE); - const manifest = JSON.parse(await readFile(manifestPath, 'utf8')) as Record & { - database: { sizeBytes: number; sha256: string }; - }; - manifest.schemaVersion = 1; - delete manifest.legacyAutomation; - manifest.database.sizeBytes = (await stat(databasePath)).size; - manifest.database.sha256 = await hashFile(databasePath); - await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); + await sessions.close?.(); + await mkdir(join(stateRoot, 'artifacts'), { recursive: true }); + await writeFile(join(stateRoot, 'artifacts', 'note.txt'), 'artifact'); - assert.equal((await validateOperationalStateBackup(backupRoot)).schemaVersion, 1); + await createOperationalStateBackup({ stateRoot, destinationRoot: backupRoot, now: () => 10 }); + assert.equal((await validateOperationalStateBackup(backupRoot)).createdAt, 10); await restoreOperationalStateBackup({ backupRoot, destinationRoot: restoreRoot }); - const capability = await resolveStorageRoot({ path: restoreRoot, kind: 'interactive' }); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(owner); - if (!owner) return; + const restored = createSessionStore(restoreRoot); try { - const writer = await openInteractiveAutomationAuthorityForWrite(owner.lease); - const restored = await writer.read(); - assert.deepEqual(restored, { revision: 0, automations: [], pendingFires: [] }); - const committed = await writer.commit({ - expectedRevision: 0, - automations: [backupAutomationDefinition(sessionId)], - pendingFires: [], - }); - assert.equal(committed.kind, 'committed'); - assert.equal((await writer.read()).automations[0]?.id, 'backup-automation'); - writer.close(); + assert.equal((await restored.readMessages(session.id))[0]?.id, 'message-1'); + assert.equal(await readFile(join(restoreRoot, 'artifacts', 'note.txt'), 'utf8'), 'artifact'); } finally { - if (!owner.closed) await owner.close(); + await restored.close?.(); } - }); -}); - -function backupAutomationDefinition(sessionId: string) { - return { - id: 'backup-automation', - kind: 'heartbeat' as const, - name: 'Backup automation', - status: 'active' as const, - prompt: 'Preserve this Automation.', - sessionId, - schedule: { type: 'interval' as const, seconds: 60 }, - createdAt: 1, - updatedAt: 1, - nextFireAt: null, - lastFireAt: 60_001, - lastRunId: null, - fireCount: 1, - maxFires: null, - expiresAt: 604_800_001, - lastError: null, - consecutiveFailures: 0, - }; -} - -function seedAutomationState(root: string, sessionId: string): void { - const database = new DatabaseSync(join(root, 'runtime.sqlite')); - try { - const automation = backupAutomationDefinition(sessionId); - database - .prepare(` - INSERT INTO automation_definitions( - automation_id, session_id, created_at, status, durable, record_json - ) VALUES (?, ?, ?, ?, ?, ?) - `) - .run( - automation.id, - automation.sessionId, - automation.createdAt, - automation.status, - 0, - JSON.stringify(automation), - ); - const fire = { - id: 'backup-fire', - automationId: automation.id, - automationKind: automation.kind, - automationName: automation.name, - prompt: automation.prompt, - scheduledFor: 60_001, - targetSessionId: sessionId, - turnId: 'backup-turn', - runId: 'backup-run', - userMessageId: 'backup-message', - status: 'admitted', - admittedAt: 60_001, - updatedAt: 60_001, - }; - database - .prepare(` - INSERT INTO automation_pending_fires( - fire_id, automation_id, target_session_id, admitted_at, record_json - ) VALUES (?, ?, ?, ?, ?) - `) - .run(fire.id, fire.automationId, fire.targetSessionId, fire.admittedAt, JSON.stringify(fire)); - database - .prepare('UPDATE automation_authority_state SET revision = 1 WHERE singleton = 1') - .run(); - } finally { - database.close(); - } -} - -function readAutomationState(root: string): { - revision: number; - automationId: string; - name: string; - pendingFireId: string; -} { - const database = new DatabaseSync(join(root, 'runtime.sqlite'), { readOnly: true }); - try { - const state = database - .prepare('SELECT revision FROM automation_authority_state WHERE singleton = 1') - .get() as { revision: number }; - const row = database - .prepare('SELECT automation_id, record_json FROM automation_definitions') - .get() as { automation_id: string; record_json: string }; - const fire = database.prepare('SELECT fire_id FROM automation_pending_fires').get() as { - fire_id: string; - }; - const record = JSON.parse(row.record_json) as { name: string }; - return { - revision: state.revision, - automationId: row.automation_id, - name: record.name, - pendingFireId: fire.fire_id, - }; } finally { - database.close(); - } -} - -test('linearizes an Artifact mutation after the backup snapshot', async () => { - await withBackupRoots(async ({ stateRoot, backupRoot, restoreRoot }) => { - const sessions = createSessionStore(stateRoot); - const artifacts = createSqliteArtifactStore(stateRoot); - let queuedMutation: Promise | undefined; - try { - const session = await sessions.create(sessionInput('Selected')); - await artifacts.create({ - id: 'seed', - sessionId: session.id, - turnId: 'turn-1', - name: 'seed.txt', - kind: 'file', - content: 'seed\n', - now: 1, - }); - let mutationSettled = false; - await createOperationalStateBackup({ - stateRoot, - destinationRoot: backupRoot, - failpoint: async (point) => { - if (point !== 'after_database_snapshot') return; - queuedMutation = artifacts - .create({ - id: 'after-snapshot', - sessionId: session.id, - turnId: 'turn-2', - name: 'after.txt', - kind: 'file', - content: 'after\n', - now: 2, - }) - .finally(() => { - mutationSettled = true; - }); - await delay(50); - assert.equal(mutationSettled, false); - }, - }); - await queuedMutation; - - await restoreOperationalStateBackup({ backupRoot, destinationRoot: restoreRoot }); - const restored = createSqliteArtifactStore(restoreRoot); - try { - assert.deepEqual( - (await restored.list(session.id)).map((record) => record.id), - ['seed'], - ); - } finally { - restored.close?.(); - } - assert.deepEqual( - (await artifacts.list(session.id)).map((record) => record.id), - ['after-snapshot', 'seed'], - ); - } finally { - await queuedMutation?.catch(() => {}); - artifacts.close?.(); - await sessions.close?.(); - } - }); -}); - -test('fails the whole backup when a transcript changes across the SQLite snapshot', async () => { - await withBackupRoots(async ({ stateRoot, backupRoot }) => { - const sessions = createSessionStore(stateRoot); - try { - const session = await sessions.create(sessionInput('Selected')); - await sessions.appendMessage(session.id, { - type: 'user', - id: 'message-1', - turnId: 'turn-1', - ts: 1, - text: 'before snapshot', - }); - - await assertBackupError( - createOperationalStateBackup({ - stateRoot, - destinationRoot: backupRoot, - failpoint: async (point) => { - if (point !== 'after_database_snapshot') return; - await sessions.appendMessage(session.id, { - type: 'user', - id: 'message-2', - turnId: 'turn-2', - ts: 2, - text: 'after snapshot', - }); - }, - }), - 'source_changed', - ); - await assert.rejects(lstat(backupRoot), { code: 'ENOENT' }); - } finally { - await sessions.close?.(); - } - }); -}); - -test('fails the whole backup when a legacy Automation source appears after its snapshot', async () => { - await withBackupRoots(async ({ stateRoot, backupRoot }) => { - const sessionId = await createSession(stateRoot); - - await assertBackupError( - createOperationalStateBackup({ - stateRoot, - destinationRoot: backupRoot, - failpoint: async (point) => { - if (point !== 'after_database_snapshot') return; - await writeFile( - join(stateRoot, LEGACY_AUTOMATION_FILE), - `${JSON.stringify({ - version: 1, - automations: [backupAutomationDefinition(sessionId)], - })}\n`, - ); - }, - }), - 'source_changed', - ); - await assert.rejects(lstat(backupRoot), { code: 'ENOENT' }); - }); -}); - -test('rejects a corrupt source transcript instead of preserving unreadable state', async () => { - await withBackupRoots(async ({ stateRoot, backupRoot }) => { - const sessionId = await createSession(stateRoot); - await writeFile( - join(stateRoot, 'sessions', sessionId, 'session.jsonl'), - '{"type":"session_transcript","sessionId":"wrong","schemaVersion":1}\n', - ); - - await assertBackupError( - createOperationalStateBackup({ stateRoot, destinationRoot: backupRoot }), - 'corrupt_backup', - ); - await assert.rejects(lstat(backupRoot), { code: 'ENOENT' }); - }); -}); - -test('rejects an invalid legacy Automation source instead of preserving unreadable state', async () => { - await withBackupRoots(async ({ stateRoot, backupRoot }) => { - await createSession(stateRoot); - await writeFile(join(stateRoot, LEGACY_AUTOMATION_FILE), '{"version":2,"automations":[]}\n'); - - await assertBackupError( - createOperationalStateBackup({ stateRoot, destinationRoot: backupRoot }), - 'corrupt_backup', - ); - await assert.rejects(lstat(backupRoot), { code: 'ENOENT' }); - }); -}); - -test('rejects a digest-matching invalid legacy Automation backup before restore', async () => { - await withBackupRoots(async ({ stateRoot, backupRoot, restoreRoot }) => { - const sessionId = await createSession(stateRoot); - const legacyPath = join(stateRoot, LEGACY_AUTOMATION_FILE); - await writeFile( - legacyPath, - `${JSON.stringify({ - version: 1, - automations: [backupAutomationDefinition(sessionId)], - })}\n`, - ); - await createOperationalStateBackup({ stateRoot, destinationRoot: backupRoot }); - - const backupLegacyPath = join(backupRoot, LEGACY_AUTOMATION_FILE); - await writeFile(backupLegacyPath, '{"version":2,"automations":[]}\n'); - const manifestPath = join(backupRoot, OPERATIONAL_BACKUP_MANIFEST_FILE); - const manifest = JSON.parse(await readFile(manifestPath, 'utf8')) as { - legacyAutomation: { sizeBytes: number; sha256: string }; - }; - manifest.legacyAutomation.sizeBytes = (await stat(backupLegacyPath)).size; - manifest.legacyAutomation.sha256 = await hashFile(backupLegacyPath); - await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); - - await assertBackupError(validateOperationalStateBackup(backupRoot), 'corrupt_backup'); - await assertBackupError( - restoreOperationalStateBackup({ backupRoot, destinationRoot: restoreRoot }), - 'corrupt_backup', - ); - await assert.rejects(lstat(restoreRoot), { code: 'ENOENT' }); - }); -}); - -test('cleans interrupted backup and restore staging so both operations can retry', async () => { - await withBackupRoots(async ({ root, stateRoot, backupRoot, restoreRoot }) => { - const sessionId = await createSession(stateRoot); - const artifacts = createSqliteArtifactStore(stateRoot); - try { - await artifacts.create({ - id: 'artifact-1', - sessionId, - turnId: 'turn-1', - name: 'output.txt', - kind: 'file', - content: 'output\n', - now: 1, - }); - } finally { - artifacts.close?.(); - } - - await assert.rejects( - createOperationalStateBackup({ - stateRoot, - destinationRoot: backupRoot, - failpoint: (point) => { - if (point === 'after_database_snapshot') throw new Error('backup crash'); - }, - }), - /backup crash/, - ); - await assert.rejects(lstat(backupRoot), { code: 'ENOENT' }); - assert.equal( - (await readdir(root)).some((entry) => entry.startsWith('backup.staging-')), - false, - ); - - await createOperationalStateBackup({ stateRoot, destinationRoot: backupRoot }); - await assert.rejects( - restoreOperationalStateBackup({ - backupRoot, - destinationRoot: restoreRoot, - failpoint: (point) => { - if (point === 'after_restore_copy') throw new Error('restore crash'); - }, - }), - /restore crash/, - ); - await assert.rejects(lstat(restoreRoot), { code: 'ENOENT' }); - assert.equal( - (await readdir(root)).some((entry) => entry.startsWith('restore.staging-')), - false, - ); - await restoreOperationalStateBackup({ backupRoot, destinationRoot: restoreRoot }); - }); -}); - -test('rejects missing, changed, extra, and symlinked backup payloads before restore', async (t) => { - const corruptions: ReadonlyArray<{ - name: string; - apply: (input: { backupRoot: string; artifactPath: string }) => Promise; - }> = [ - { - name: 'missing payload', - apply: async ({ artifactPath }) => rm(artifactPath), - }, - { - name: 'changed payload', - apply: async ({ artifactPath }) => writeFile(artifactPath, 'tampered\n'), - }, - { - name: 'extra payload', - apply: async ({ backupRoot }) => - writeFile(join(backupRoot, 'artifacts', 'extra.txt'), 'extra\n'), - }, - { - name: 'extra empty directory', - apply: async ({ backupRoot }) => mkdir(join(backupRoot, 'unmanifested')), - }, - { - name: 'symlinked payload', - apply: async ({ backupRoot, artifactPath }) => { - const target = join(backupRoot, 'outside.txt'); - await writeFile(target, 'outside\n'); - await rm(artifactPath); - await symlink(target, artifactPath); - }, - }, - ]; - - for (const corruption of corruptions) { - await t.test(corruption.name, async () => { - await withBackupRoots(async ({ stateRoot, backupRoot, restoreRoot }) => { - const sessionId = await createSession(stateRoot); - const artifacts = createSqliteArtifactStore(stateRoot); - let relativePath: string; - try { - relativePath = ( - await artifacts.create({ - id: 'artifact-1', - sessionId, - turnId: 'turn-1', - name: 'output.txt', - kind: 'file', - content: 'output\n', - now: 1, - }) - ).relativePath; - } finally { - artifacts.close?.(); - } - await createOperationalStateBackup({ stateRoot, destinationRoot: backupRoot }); - await corruption.apply({ - backupRoot, - artifactPath: join(backupRoot, 'artifacts', relativePath!), - }); - - await assertBackupError(validateOperationalStateBackup(backupRoot), 'corrupt_backup'); - await assertBackupError( - restoreOperationalStateBackup({ backupRoot, destinationRoot: restoreRoot }), - 'corrupt_backup', - ); - await assert.rejects(lstat(restoreRoot), { code: 'ENOENT' }); - }); - }); + await rm(base, { recursive: true, force: true }); } }); - -test('fails closed when retained legacy Artifact evidence changes after cutover', async () => { - await withBackupRoots(async ({ stateRoot, backupRoot }) => { - const sessionId = await createSession(stateRoot); - const artifacts = createSqliteArtifactStore(stateRoot); - try { - await artifacts.create({ - id: 'sqlite-artifact', - sessionId, - turnId: 'turn-1', - name: 'sqlite.txt', - kind: 'file', - content: 'sqlite\n', - now: 1, - }); - } finally { - artifacts.close?.(); - } - await writeFile( - join(stateRoot, 'artifacts', 'metadata.jsonl'), - `${JSON.stringify({ - id: 'stale-writer-artifact', - sessionId, - turnId: 'turn-2', - createdAt: 2, - name: 'stale.txt', - kind: 'file', - relativePath: `${sessionId}/stale-writer-artifact-stale.txt`, - sizeBytes: 6, - status: 'live', - })}\n`, - ); - - await assert.rejects( - createOperationalStateBackup({ stateRoot, destinationRoot: backupRoot }), - /Legacy artifact_metadata source changed after cutover completed/, - ); - await assert.rejects(lstat(backupRoot), { code: 'ENOENT' }); - }); -}); - -test('rejects a newer SQLite schema even when the rewritten manifest digest matches', async () => { - await withBackupRoots(async ({ stateRoot, backupRoot, restoreRoot }) => { - await createSession(stateRoot); - await createOperationalStateBackup({ stateRoot, destinationRoot: backupRoot }); - const databasePath = join(backupRoot, 'runtime.sqlite'); - const database = new DatabaseSync(databasePath); - try { - database - .prepare(` - UPDATE operational_schema_migrations - SET version = version + 1 - WHERE scope = 'artifact' - `) - .run(); - } finally { - database.close(); - } - const manifestPath = join(backupRoot, OPERATIONAL_BACKUP_MANIFEST_FILE); - const manifest = JSON.parse(await readFile(manifestPath, 'utf8')) as { - database: { sizeBytes: number; sha256: string }; - }; - manifest.database.sizeBytes = (await stat(databasePath)).size; - manifest.database.sha256 = await hashFile(databasePath); - await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`); - - await assertBackupError(validateOperationalStateBackup(backupRoot), 'unsupported_schema'); - await assertBackupError( - restoreOperationalStateBackup({ backupRoot, destinationRoot: restoreRoot }), - 'unsupported_schema', - ); - await assert.rejects(lstat(restoreRoot), { code: 'ENOENT' }); - }); -}); - -test('rejects overlapping roots and an existing restore destination', async () => { - await withBackupRoots(async ({ root, stateRoot, backupRoot, restoreRoot }) => { - await createSession(stateRoot); - await assertBackupError( - createOperationalStateBackup({ - stateRoot, - destinationRoot: join(stateRoot, 'backup'), - }), - 'overlapping_roots', - ); - await createOperationalStateBackup({ stateRoot, destinationRoot: backupRoot }); - await mkdir(restoreRoot); - await assertBackupError( - restoreOperationalStateBackup({ backupRoot, destinationRoot: restoreRoot }), - 'destination_exists', - ); - assert.ok((await readdir(root)).includes('restore')); - }); -}); - -async function assertBackupError( - operation: Promise, - code: OperationalBackupError['code'], -): Promise { - await assert.rejects(operation, (error: unknown) => { - assert.equal((error as OperationalBackupError).code, code); - return true; - }); -} - -async function createSession(stateRoot: string): Promise { - const sessions = createSessionStore(stateRoot); - try { - return (await sessions.create(sessionInput('Selected'))).id; - } finally { - await sessions.close?.(); - } -} - -function sessionInput(name: string) { - return { - cwd: '/repo', - backend: 'fake' as const, - llmConnectionSlug: 'fixture', - model: 'fixture-model', - permissionMode: 'execute' as const, - name, - }; -} - -async function withBackupRoots( - operation: (roots: { - root: string; - stateRoot: string; - backupRoot: string; - restoreRoot: string; - }) => Promise, -): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-operational-backup-')); - const stateRoot = join(root, 'state'); - const backupRoot = join(root, 'backup'); - const restoreRoot = join(root, 'restore'); - await mkdir(stateRoot); - try { - await operation({ root, stateRoot, backupRoot, restoreRoot }); - } finally { - await rm(root, { recursive: true, force: true }); - } -} - -async function hashFile(path: string): Promise { - return `sha256:${createHash('sha256') - .update(await readFile(path)) - .digest('hex')}`; -} diff --git a/packages/storage/src/__tests__/operational-state-store.test.ts b/packages/storage/src/__tests__/operational-state-store.test.ts index 2a0656fda0..3387ae0622 100644 --- a/packages/storage/src/__tests__/operational-state-store.test.ts +++ b/packages/storage/src/__tests__/operational-state-store.test.ts @@ -1,289 +1,59 @@ import assert from 'node:assert/strict'; -import { mkdtemp, rm, stat, writeFile } from 'node:fs/promises'; +import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; -import { describe, test } from 'node:test'; +import { test } from 'node:test'; import type { SessionHeader } from '@maka/core'; -import { - acquireOperationalStateDatabase, - type OperationalStateCutoverFailpoint, -} from '../operational-state-store.js'; -import { - createSqliteRuntimeStore, - SQLITE_RUNTIME_SCHEMA_VERSION, -} from '../sqlite-runtime-store.js'; -import { - createSqliteSessionMetadataStore, - SQLITE_SESSION_METADATA_SCHEMA_VERSION, -} from '../sqlite-session-metadata-store.js'; -import { SQLITE_USAGE_SCHEMA_VERSION } from '../sqlite-usage-schema.js'; +import { acquireOperationalStateDatabase } from '../operational-state-store.js'; +import { createSqliteSessionMetadataStore } from '../sqlite-session-metadata-store.js'; -describe('operational state database cutover', () => { - test('keeps the owner alive until an admitted online backup completes', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-operational-backup-owner-')); - const backupPath = join(root, 'backup.sqlite'); - try { - const lease = acquireOperationalStateDatabase(root); - const metadata = createSqliteSessionMetadataStore(join(root, 'runtime.sqlite'), { - databaseLease: lease, - }); - await metadata.create(sessionHeader({ name: 'Backup session' })); - const backup = lease.backup(backupPath); - metadata.close(); - assert.ok((await backup) > 0); - - const reopened = new DatabaseSync(backupPath, { readOnly: true }); - try { - assert.equal( - ( - reopened.prepare('SELECT COUNT(*) AS count FROM session_metadata').get() as { - count: number; - } - ).count, - 1, - ); - } finally { - reopened.close(); - } - } finally { - await rm(root, { recursive: true, force: true }); - } - }); - - test('imports sessions.sqlite into runtime.sqlite once and rejects later legacy changes', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-operational-cutover-')); - const legacyPath = join(root, 'sessions.sqlite'); - const runtimePath = join(root, 'runtime.sqlite'); - try { - const legacy = createSqliteSessionMetadataStore(legacyPath, { now: () => 10 }); - await legacy.create(sessionHeader()); - legacy.close(); - - const lease = acquireOperationalStateDatabase(root, { now: () => 20 }); - const secondLease = acquireOperationalStateDatabase(root); - assert.equal(secondLease.database, lease.database); - secondLease.close(); - const metadata = createSqliteSessionMetadataStore(runtimePath, { - databaseLease: lease, - now: () => 30, - }); - assert.equal((await metadata.read('session-1')).header.name, 'Legacy session'); - assert.deepEqual( - lease.database - .prepare(`SELECT scope, version FROM operational_schema_migrations ORDER BY scope`) - .all() - .map((row) => ({ ...row })), - [ - { scope: 'artifact', version: 1 }, - { scope: 'automation', version: 1 }, - { scope: 'core_execution', version: 1 }, - { scope: 'operational', version: 1 }, - { scope: 'runtime', version: SQLITE_RUNTIME_SCHEMA_VERSION }, - { scope: 'session_metadata', version: SQLITE_SESSION_METADATA_SCHEMA_VERSION }, - { scope: 'usage', version: SQLITE_USAGE_SCHEMA_VERSION }, - { scope: 'workflow', version: 1 }, - ], - ); - assert.deepEqual( - { - ...(lease.database - .prepare(` - SELECT state, source_path, validation_json - FROM cutover_journal - WHERE store_name = 'session_metadata' - `) - .get() as Record), - }, - { - state: 'completed', - source_path: legacyPath, - validation_json: JSON.stringify({ - session_metadata: 1, - session_metadata_labels: 1, - session_metadata_import_sources: 0, - session_metadata_tombstones: 0, - subagent_spawns: 0, - agent_graph_intent_claims: 0, - agent_graph_schedule_updates: 0, - agent_graph_operator_provisions: 0, - agent_graph_client_projections: 0, - agent_graph_client_operator_projections: 0, - agent_graph_client_terminal_activity: 0, - agent_graph_client_applied_records: 0, - agent_graph_supervisor_wakes: 0, - agent_graph_supervisor_wake_attempts: 0, - sandbox_boundary_log: 1, - }), - }, - ); - metadata.close(); - assert.ok((await stat(legacyPath)).isFile(), 'legacy database remains as cutover evidence'); - - const reopenedLease = acquireOperationalStateDatabase(root); - reopenedLease.close(); - - const changedLegacy = createSqliteSessionMetadataStore(legacyPath); - await changedLegacy.update('session-1', { name: 'Changed by an old binary' }); - changedLegacy.close(); - assert.throws( - () => acquireOperationalStateDatabase(root), - /changed after session metadata cutover completed/, - ); - } finally { - await rm(root, { recursive: true, force: true }); - } - }); - - test('fails closed instead of merging conflicting canonical and legacy rows', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-operational-cutover-conflict-')); - const legacyPath = join(root, 'sessions.sqlite'); - const runtimePath = join(root, 'runtime.sqlite'); - try { - const legacy = createSqliteSessionMetadataStore(legacyPath); - await legacy.create(sessionHeader()); - legacy.close(); - - const runtime = createSqliteRuntimeStore(runtimePath); - runtime.close(); - const canonical = createSqliteSessionMetadataStore(runtimePath); - await canonical.create(sessionHeader({ name: 'Canonical session' })); - canonical.close(); +test('shares one operational database and produces an online backup', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-operational-state-')); + const backupPath = join(root, 'backup.sqlite'); + try { + const lease = acquireOperationalStateDatabase(root); + const secondLease = acquireOperationalStateDatabase(root); + assert.equal(secondLease.database, lease.database); + secondLease.close(); - assert.throws( - () => acquireOperationalStateDatabase(root), - /Session metadata cutover conflict in table session_metadata/, - ); - const verified = createSqliteSessionMetadataStore(runtimePath); - try { - assert.equal((await verified.read('session-1')).header.name, 'Canonical session'); - } finally { - verified.close(); - } - } finally { - await rm(root, { recursive: true, force: true }); - } - }); + const metadata = createSqliteSessionMetadataStore(join(root, 'runtime.sqlite'), { + databaseLease: lease, + }); + await metadata.create(sessionHeader()); + const backup = lease.backup(backupPath); + metadata.close(); + assert.ok((await backup) > 0); - test('rejects an empty legacy database instead of recording an empty cutover', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-operational-cutover-empty-')); + const reopened = new DatabaseSync(backupPath, { readOnly: true }); try { - await writeFile(join(root, 'sessions.sqlite'), ''); - assert.throws( - () => acquireOperationalStateDatabase(root), - /not a non-empty regular database file/, + assert.equal( + ( + reopened.prepare('SELECT COUNT(*) AS count FROM session_metadata').get() as { + count: number; + } + ).count, + 1, ); - const target = new DatabaseSync(join(root, 'runtime.sqlite')); - try { - assert.equal( - ( - target.prepare(`SELECT COUNT(*) AS count FROM cutover_journal`).get() as { - count: number; - } - ).count, - 0, - ); - } finally { - target.close(); - } } finally { - await rm(root, { recursive: true, force: true }); + reopened.close(); } - }); - - for (const failpoint of [ - 'after_cutover_started', - 'after_cutover_rows_copied', - 'after_cutover_validated', - ] satisfies OperationalStateCutoverFailpoint[]) { - test(`resumes atomically after ${failpoint}`, async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-operational-cutover-crash-')); - const legacyPath = join(root, 'sessions.sqlite'); - const runtimePath = join(root, 'runtime.sqlite'); - try { - const legacy = createSqliteSessionMetadataStore(legacyPath); - await legacy.create(sessionHeader()); - legacy.close(); - - assert.throws( - () => - acquireOperationalStateDatabase(root, { - failpoint: (point) => { - if (point === failpoint) throw new Error(`failpoint:${point}`); - }, - }), - new RegExp(`failpoint:${failpoint}`), - ); - - const interrupted = new DatabaseSync(runtimePath); - try { - assert.deepEqual( - { - ...(interrupted - .prepare(` - SELECT state, completed_at - FROM cutover_journal - WHERE store_name = 'session_metadata' - `) - .get() as Record), - }, - { state: 'started', completed_at: null }, - ); - assert.equal( - ( - interrupted.prepare(`SELECT COUNT(*) AS count FROM session_metadata`).get() as { - count: number; - } - ).count, - 0, - ); - } finally { - interrupted.close(); - } - - const resumed = acquireOperationalStateDatabase(root); - try { - assert.equal( - ( - resumed.database.prepare(`SELECT COUNT(*) AS count FROM session_metadata`).get() as { - count: number; - } - ).count, - 1, - ); - assert.equal( - ( - resumed.database - .prepare(` - SELECT state - FROM cutover_journal - WHERE store_name = 'session_metadata' - `) - .get() as { state: string } - ).state, - 'completed', - ); - } finally { - resumed.close(); - } - } finally { - await rm(root, { recursive: true, force: true }); - } - }); + } finally { + await rm(root, { recursive: true, force: true }); } }); -function sessionHeader(overrides: Partial = {}): SessionHeader { +function sessionHeader(): SessionHeader { return { id: 'session-1', workspaceRoot: '/workspace', cwd: '/workspace', createdAt: 1, lastUsedAt: 2, - name: 'Legacy session', + name: 'Session', titleIsManual: true, isFlagged: false, - labels: ['legacy'], + labels: [], isArchived: false, status: 'active', hasUnread: false, @@ -291,11 +61,9 @@ function sessionHeader(overrides: Partial = {}): SessionHeader { llmConnectionSlug: 'test', connectionLocked: true, model: 'test-model', - thinkingLevel: 'medium', permissionMode: 'ask', collaborationMode: 'agent', - orchestrationMode: 'swarm', + orchestrationMode: 'default', schemaVersion: 1, - ...overrides, }; } diff --git a/packages/storage/src/__tests__/plan-reminder-store.test.ts b/packages/storage/src/__tests__/plan-reminder-store.test.ts deleted file mode 100644 index 4fd3df504e..0000000000 --- a/packages/storage/src/__tests__/plan-reminder-store.test.ts +++ /dev/null @@ -1,490 +0,0 @@ -import { describe, it } from 'node:test'; -import assert from 'node:assert/strict'; -import { mkdtemp, readFile, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; -import { tmpdir } from 'node:os'; -import { createPlanReminderStore } from '../plan-reminder-store.js'; - -describe('PlanReminderStore', () => { - it('persists reminders and exposes due reminders', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const store = createPlanReminderStore(root); - const runAt = Date.now() + 60_000; - - const reminder = await store.create({ - title: ' 站会提醒 ', - note: '准备昨天的 blocker', - runAt, - }); - assert.equal(reminder.title, '站会提醒'); - assert.equal(reminder.enabled, true); - assert.equal(reminder.nextRunAt, runAt); - assert.deepEqual(reminder.delivery, { channel: 'local' }); - assert.deepEqual(reminder.runs, []); - - const reloaded = createPlanReminderStore(root); - assert.equal((await reloaded.list()).length, 1); - assert.equal((await reloaded.listDue(runAt - 1)).length, 0); - assert.equal((await reloaded.listDue(runAt)).length, 1); - - const raw = JSON.parse(await readFile(join(root, 'plan-reminders.json'), 'utf8')) as unknown[]; - assert.equal(raw.length, 1); - }); - - it('persists bot delivery and defaults legacy records to local delivery', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const store = createPlanReminderStore(root); - const runAt = Date.now() + 60_000; - - const reminder = await store.create({ - title: '投递到 Telegram', - runAt, - delivery: { channel: 'bot', platform: 'telegram', chatId: ' 12345 ' }, - }); - assert.deepEqual(reminder.delivery, { channel: 'bot', platform: 'telegram', chatId: '12345' }); - - const raw = JSON.parse(await readFile(join(root, 'plan-reminders.json'), 'utf8')) as unknown[]; - assert.deepEqual((raw[0] as { delivery?: unknown }).delivery, { - channel: 'bot', - platform: 'telegram', - chatId: '12345', - }); - - await writeFile( - join(root, 'plan-reminders.json'), - JSON.stringify([ - { - id: 'legacy', - title: '旧提醒', - note: '', - schedule: { kind: 'once', runAt }, - status: 'scheduled', - enabled: true, - createdAt: runAt - 1000, - updatedAt: runAt - 1000, - nextRunAt: runAt, - runs: [], - runCount: 0, - }, - ]), - 'utf8', - ); - - const reloaded = await createPlanReminderStore(root).list(); - assert.deepEqual(reloaded[0]?.delivery, { channel: 'local' }); - }); - - it('keeps recurring reminders active after a trigger', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const store = createPlanReminderStore(root); - const runAt = Date.now() + 60_000; - - const reminder = await store.create({ title: '每日复盘', runAt, recurrence: 'daily' }); - assert.equal(reminder.schedule.kind, 'recurring'); - assert.equal(reminder.nextRunAt, runAt); - - const triggered = await store.markTriggered(reminder.id, { - at: runAt, - status: 'triggered', - message: '提醒已触发', - }); - assert.equal(triggered.status, 'scheduled'); - assert.equal(triggered.enabled, true); - assert.equal(triggered.nextRunAt, runAt + 24 * 60 * 60 * 1000); - assert.deepEqual( - triggered.runs.map((run) => run.status), - ['triggered'], - ); - assert.equal((await store.listDue(runAt + 1)).length, 0); - assert.equal((await store.listDue(runAt + 24 * 60 * 60 * 1000)).length, 1); - }); - - it('keeps cron reminders active and persists their expression', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const store = createPlanReminderStore(root); - const runAt = Date.now() + 60_000; - - const reminder = await store.create({ - title: '工作日早报', - runAt, - recurrence: 'cron', - cronExpression: '30 9 * * 1-5', - }); - assert.deepEqual(reminder.schedule, { - kind: 'cron', - startAt: runAt, - expression: '30 9 * * 1-5', - }); - assert.equal(typeof reminder.nextRunAt, 'number'); - - const triggered = await store.markTriggered(reminder.id, { - at: reminder.nextRunAt!, - status: 'triggered', - message: '提醒已触发', - }); - assert.equal(triggered.status, 'scheduled'); - assert.equal(triggered.enabled, true); - assert.equal(triggered.schedule.kind, 'cron'); - assert.equal(typeof triggered.nextRunAt, 'number'); - assert.ok(triggered.nextRunAt! > reminder.nextRunAt!); - }); - - it('supports pause, resume, delete, and triggered run records', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const store = createPlanReminderStore(root); - const runAt = Date.now() + 60_000; - const reminder = await store.create({ title: '复盘', runAt }); - - const paused = await store.setEnabled(reminder.id, false); - assert.equal(paused.status, 'paused'); - assert.equal(paused.nextRunAt, undefined); - - const resumed = await store.setEnabled(reminder.id, true); - assert.equal(resumed.status, 'scheduled'); - assert.equal(resumed.nextRunAt, runAt); - - const triggered = await store.markTriggered(reminder.id, { - at: runAt, - status: 'triggered', - message: '提醒已触发', - }); - assert.equal(triggered.status, 'completed'); - assert.equal(triggered.lastRun?.status, 'triggered'); - assert.deepEqual( - triggered.runs.map((run) => run.id), - [triggered.lastRun?.id], - ); - assert.equal(triggered.runCount, 1); - - await store.remove(reminder.id); - assert.equal((await store.list()).length, 0); - }); - - it('snoozes scheduled reminders without changing the recurrence contract', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const store = createPlanReminderStore(root); - const runAt = Date.now() + 60_000; - const reminder = await store.create({ - title: '工作日早报', - runAt, - recurrence: 'cron', - cronExpression: '0 9 * * 1-5', - }); - - const snoozed = await store.snooze(reminder.id, 10 * 60 * 1000, runAt - 30_000); - assert.equal(snoozed.status, 'scheduled'); - assert.equal(snoozed.enabled, true); - assert.equal(snoozed.schedule.kind, 'cron'); - assert.equal(snoozed.nextRunAt, Math.max(runAt - 30_000, reminder.nextRunAt!) + 10 * 60 * 1000); - assert.equal(snoozed.runs.length, 0); - }); - - it('updates title, schedule, recurrence, delivery, and note through the edit path', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const store = createPlanReminderStore(root); - const runAt = Date.now() + 60_000; - const reminder = await store.create({ title: '旧标题', note: '旧备注', runAt }); - const nextRunAt = runAt + 60_000; - - const updated = await store.update(reminder.id, { - title: '新标题', - note: '', - runAt: nextRunAt, - recurrence: 'weekly', - delivery: { channel: 'bot', platform: 'telegram', chatId: ' 42 ' }, - }); - - assert.equal(updated.title, '新标题'); - assert.equal(updated.note, ''); - assert.deepEqual(updated.schedule, { - kind: 'recurring', - startAt: nextRunAt, - recurrence: 'weekly', - }); - assert.deepEqual(updated.delivery, { channel: 'bot', platform: 'telegram', chatId: '42' }); - assert.equal(updated.status, 'scheduled'); - assert.equal(updated.nextRunAt, nextRunAt); - }); - - it('rejects impossible cron schedule edits even when the reminder is paused', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const store = createPlanReminderStore(root); - const runAt = Date.now() + 60_000; - const reminder = await store.create({ title: '每周同步', runAt, recurrence: 'weekly' }); - await store.setEnabled(reminder.id, false); - - await assert.rejects( - () => - store.update(reminder.id, { - recurrence: 'cron', - cronExpression: '0 9 31 2 *', - }), - /schedule has no run within one year/, - ); - - const persisted = (await store.list()).find((entry) => entry.id === reminder.id); - assert.equal(persisted?.schedule.kind, 'recurring'); - assert.equal(persisted?.status, 'paused'); - }); - - it('rejects enabling legacy paused cron reminders that have no future run', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const runAt = Date.now() + 60_000; - await writeFile( - join(root, 'plan-reminders.json'), - JSON.stringify([ - { - id: 'legacy-impossible-cron', - title: '坏 cron', - note: '', - schedule: { kind: 'cron', startAt: runAt, expression: '0 9 31 2 *' }, - delivery: { channel: 'local' }, - status: 'paused', - enabled: false, - createdAt: runAt - 1000, - updatedAt: runAt - 1000, - runs: [], - runCount: 0, - }, - ]), - 'utf8', - ); - - const store = createPlanReminderStore(root); - await assert.rejects( - () => store.setEnabled('legacy-impossible-cron', true), - /schedule has no run within one year/, - ); - - const persisted = (await store.list()).find((entry) => entry.id === 'legacy-impossible-cron'); - assert.equal(persisted?.status, 'paused'); - assert.equal(persisted?.enabled, false); - assert.equal(persisted?.nextRunAt, undefined); - }); - - it('resumes paused recurring reminders at the next future occurrence', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const store = createPlanReminderStore(root); - const runAt = Date.now() + 60_000; - const reminder = await store.create({ title: '每周同步', runAt, recurrence: 'weekly' }); - await store.setEnabled(reminder.id, false); - - const resumed = await store.setEnabled(reminder.id, true); - assert.equal(resumed.status, 'scheduled'); - assert.equal(resumed.schedule.kind, 'recurring'); - assert.equal(resumed.nextRunAt, runAt); - }); - - it('lists active reminders before paused reminders and completed history', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const store = createPlanReminderStore(root); - const base = Date.now() + 60_000; - - const completed = await store.create({ title: '已触发', runAt: base + 30_000 }); - const paused = await store.create({ title: '暂停中', runAt: base + 20_000 }); - const scheduled = await store.create({ title: '待触发', runAt: base + 10_000 }); - - await store.markTriggered(completed.id, { - at: base + 30_000, - status: 'triggered', - message: '提醒已触发', - }); - await store.setEnabled(paused.id, false); - - assert.deepEqual( - (await store.list()).map((reminder) => reminder.title), - ['待触发', '暂停中', '已触发'], - ); - }); - - it('keeps recurring run history in newest-first order', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const store = createPlanReminderStore(root); - const runAt = Date.now() + 60_000; - const reminder = await store.create({ title: '每日复盘', runAt, recurrence: 'daily' }); - - const first = await store.markTriggered(reminder.id, { - id: 'run-1', - at: runAt, - status: 'triggered', - message: '第一次触发', - }); - const second = await store.markBlocked(reminder.id, { - id: 'run-2', - at: runAt + 24 * 60 * 60 * 1000, - message: '隐私模式已开启', - blockReason: 'incognito_active', - }); - - assert.deepEqual( - first.runs.map((run) => run.id), - ['run-1'], - ); - assert.deepEqual( - second.runs.map((run) => run.id), - ['run-2', 'run-1'], - ); - assert.equal(second.lastRun?.id, 'run-2'); - assert.equal(second.runCount, 2); - }); - - it('clears run history without deleting the reminder or resetting run count', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const store = createPlanReminderStore(root); - const runAt = Date.now() + 60_000; - const reminder = await store.create({ title: '每日复盘', runAt, recurrence: 'daily' }); - - const triggered = await store.markTriggered(reminder.id, { - id: 'run-1', - at: runAt, - status: 'triggered', - message: '第一次触发', - }); - assert.equal(triggered.runs.length, 1); - assert.equal(triggered.runCount, 1); - - const cleared = await store.clearRunHistory(reminder.id); - assert.equal(cleared.id, reminder.id); - assert.equal(cleared.title, '每日复盘'); - assert.equal(cleared.status, 'scheduled'); - assert.deepEqual(cleared.runs, []); - assert.equal(cleared.lastRun, undefined); - assert.equal(cleared.runCount, 1); - assert.equal((await store.list()).find((entry) => entry.id === reminder.id)?.runs.length, 0); - }); - - it('rejects clearing completed one-shot history so completed rows do not vanish silently', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const store = createPlanReminderStore(root); - const runAt = Date.now() + 60_000; - const reminder = await store.create({ title: '一次性提醒', runAt }); - await store.markTriggered(reminder.id, { - id: 'run-1', - at: runAt, - status: 'triggered', - message: '提醒已触发', - }); - - await assert.rejects( - () => store.clearRunHistory(reminder.id), - /Completed plan reminder history cannot be cleared/, - ); - assert.equal( - (await store.list()).find((entry) => entry.id === reminder.id)?.lastRun?.id, - 'run-1', - ); - }); - - it('rejects wrong top-level reminder files instead of overwriting them as empty', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const filePath = join(root, 'plan-reminders.json'); - const invalid = JSON.stringify({ reminders: [] }, null, 2) + '\n'; - await writeFile(filePath, invalid, 'utf8'); - - const store = createPlanReminderStore(root); - await assert.rejects(() => store.list(), /expected an array/); - await assert.rejects( - () => store.create({ title: '新提醒', runAt: Date.now() + 60_000 }), - /expected an array/, - ); - assert.equal(await readFile(filePath, 'utf8'), invalid); - }); - - it('rejects malformed reminder entries instead of filtering them out on write', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const filePath = join(root, 'plan-reminders.json'); - const runAt = Date.now() + 60_000; - const invalid = - JSON.stringify( - [ - { - id: 'valid', - title: '保留提醒', - note: '', - schedule: { kind: 'once', runAt }, - delivery: { channel: 'local' }, - status: 'scheduled', - enabled: true, - createdAt: runAt - 1000, - updatedAt: runAt - 1000, - nextRunAt: runAt, - runs: [], - runCount: 0, - }, - { - id: 'corrupt', - title: '坏提醒', - note: '', - schedule: { kind: 'once', runAt }, - delivery: { channel: 'local' }, - status: 'scheduled', - enabled: true, - createdAt: runAt - 1000, - updatedAt: runAt - 1000, - nextRunAt: runAt, - }, - ], - null, - 2, - ) + '\n'; - await writeFile(filePath, invalid, 'utf8'); - - const store = createPlanReminderStore(root); - await assert.rejects(() => store.list(), /entry 2 is malformed/); - await assert.rejects( - () => store.create({ title: '新提醒', runAt: runAt + 60_000 }), - /entry 2 is malformed/, - ); - assert.equal(await readFile(filePath, 'utf8'), invalid); - }); - - it('rejects malformed reminder run history instead of dropping bad run records', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const filePath = join(root, 'plan-reminders.json'); - const runAt = Date.now() + 60_000; - const invalid = - JSON.stringify( - [ - { - id: 'run-history', - title: '历史提醒', - note: '', - schedule: { kind: 'recurring', startAt: runAt, recurrence: 'daily' }, - delivery: { channel: 'local' }, - status: 'scheduled', - enabled: true, - createdAt: runAt - 1000, - updatedAt: runAt - 1000, - nextRunAt: runAt, - runs: [ - { id: 'run-1', at: runAt - 1000, status: 'triggered', message: '已触发' }, - { id: 'run-2', at: runAt, status: 'unknown', message: '坏历史' }, - ], - runCount: 2, - }, - ], - null, - 2, - ) + '\n'; - await writeFile(filePath, invalid, 'utf8'); - - const store = createPlanReminderStore(root); - await assert.rejects(() => store.list(), /entry 1 has malformed run record 2/); - await assert.rejects( - () => store.update('run-history', { title: '改名' }), - /entry 1 has malformed run record 2/, - ); - assert.equal(await readFile(filePath, 'utf8'), invalid); - }); - - it('rejects invalid creates before writing', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-reminders-')); - const store = createPlanReminderStore(root); - - await assert.rejects( - () => store.create({ title: '', runAt: Date.now() + 1000 }), - /title cannot be empty/, - ); - assert.equal((await store.list()).length, 0); - }); -}); diff --git a/packages/storage/src/__tests__/plan-store.test.ts b/packages/storage/src/__tests__/plan-store.test.ts deleted file mode 100644 index 884ba454c9..0000000000 --- a/packages/storage/src/__tests__/plan-store.test.ts +++ /dev/null @@ -1,237 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, test } from 'node:test'; - -import { createPlanStore } from '../plan-store.js'; - -describe('FilePlanStore', () => { - test('requires plain-text titles and descriptions for every step', async () => { - await withStore(async (store) => { - await assert.rejects( - store.submitProposal({ - sessionId: 'session-1', - turnId: 'turn-1', - title: 'Plan', - steps: [{ id: 'one', title: '**First step**', description: 'Do the work' }], - }), - /plain text without Markdown formatting/, - ); - await assert.rejects( - store.submitProposal({ - sessionId: 'session-1', - turnId: 'turn-1', - title: 'Plan', - steps: [{ id: 'one', title: 'First step', description: '- Do the work' }], - }), - /plain text without Markdown formatting/, - ); - }); - }); - - test('persists proposal revisions and completes an approved execution', async () => { - await withStore(async (store) => { - const first = await store.submitProposal({ - sessionId: 'session-1', - turnId: 'turn-1', - title: 'First plan', - steps: [{ id: 'inspect', title: 'Inspect code', description: 'Inspect the code' }], - }); - const second = await store.submitProposal({ - sessionId: 'session-1', - turnId: 'turn-2', - title: 'Revised plan', - steps: [ - { id: 'inspect', title: 'Inspect code', description: 'Inspect the code' }, - { id: 'implement', title: 'Implement change', description: 'Implement the change' }, - ], - }); - - assert.equal(first.state.proposals[0]?.status, 'pending_approval'); - assert.equal(second.state.proposals[0]?.status, 'stale'); - assert.equal(second.state.proposals[1]?.revision, 2); - assert.equal(second.state.proposals[1]?.planId, first.state.proposals[0]?.planId); - - const proposal = second.state.proposals[1]!; - const approved = await store.approveProposal({ - sessionId: 'session-1', - proposalId: proposal.proposalId, - expectedRevision: proposal.revision, - expectedStoreVersion: second.state.storeVersion, - }); - assert.equal(approved.state.activeExecutionId, approved.state.executions[0]?.executionId); - - const execution = approved.state.executions[0]!; - const completed = await store.updateExecution({ - sessionId: 'session-1', - executionId: execution.executionId, - steps: execution.steps.map((step) => ({ id: step.id, status: 'completed' })), - }); - assert.equal(completed.event.type, 'plan_execution_completed'); - assert.equal(completed.state.activeExecutionId, undefined); - assert.equal((await store.readState('session-1')).executions[0]?.status, 'completed'); - - const nextPlan = await store.submitProposal({ - sessionId: 'session-1', - turnId: 'turn-3', - title: 'A different plan', - steps: [{ id: 'new', title: 'Start task', description: 'Start a different task' }], - }); - assert.notEqual(nextPlan.state.proposals[2]?.planId, proposal.planId); - assert.equal(nextPlan.state.proposals[2]?.revision, 1); - }); - }); - - test('serializes concurrent approval and returns the same execution id', async () => { - await withStore(async (store) => { - const submitted = await store.submitProposal({ - sessionId: 'session-1', - turnId: 'turn-1', - title: 'Plan', - steps: [{ id: 'one', title: 'First step', description: 'One' }], - }); - const proposal = submitted.state.proposals[0]!; - const input = { - sessionId: 'session-1', - proposalId: proposal.proposalId, - expectedRevision: proposal.revision, - }; - const [left, right] = await Promise.all([ - store.approveProposal(input), - store.approveProposal(input), - ]); - assert.equal(left.state.executions.length, 1); - assert.equal(right.state.executions.length, 1); - assert.equal(left.state.activeExecutionId, right.state.activeExecutionId); - }); - }); - - test('abandons only the latest pending proposal without deleting its history', async () => { - await withStore(async (store) => { - const submitted = await store.submitProposal({ - sessionId: 'session-1', - turnId: 'turn-1', - title: 'Pending plan', - steps: [{ id: 'one', title: 'First step', description: 'One' }], - }); - const proposal = submitted.state.proposals[0]!; - - const abandoned = await store.abandonProposal({ - sessionId: 'session-1', - proposalId: proposal.proposalId, - reason: 'User exited Plan Mode', - }); - - assert.equal(abandoned.event.type, 'plan_abandoned'); - assert.equal(abandoned.state.proposals[0]?.status, 'stale'); - assert.equal(abandoned.state.proposals[0]?.title, 'Pending plan'); - await assert.rejects( - store.approveProposal({ - sessionId: 'session-1', - proposalId: proposal.proposalId, - expectedRevision: proposal.revision, - }), - /latest pending plan proposal/, - ); - }); - }); - - test('replans an interrupted execution and retires it when the revision is approved', async () => { - await withStore(async (store) => { - const submitted = await store.submitProposal({ - sessionId: 'session-1', - turnId: 'turn-1', - title: 'Original plan', - steps: [{ id: 'one', title: 'First step', description: 'One' }], - }); - const originalProposal = submitted.state.proposals[0]!; - const approved = await store.approveProposal({ - sessionId: 'session-1', - proposalId: originalProposal.proposalId, - expectedRevision: originalProposal.revision, - }); - const originalExecution = approved.state.executions[0]!; - await store.interruptActiveExecution('session-1', 'User stopped execution'); - - const replanned = await store.submitProposal({ - sessionId: 'session-1', - turnId: 'turn-2', - title: 'Replanned remaining work', - sourceExecutionId: originalExecution.executionId, - steps: [{ id: 'one', title: 'Finish differently', description: 'Finish one differently' }], - }); - const revision = replanned.state.proposals[1]!; - assert.equal(revision.planId, originalProposal.planId); - assert.equal(revision.revision, 2); - assert.equal(revision.sourceExecutionId, originalExecution.executionId); - - const replacement = await store.approveProposal({ - sessionId: 'session-1', - proposalId: revision.proposalId, - expectedRevision: revision.revision, - }); - assert.equal(replacement.state.executions[0]?.status, 'cancelled'); - assert.match(replacement.state.executions[0]?.cancelReason ?? '', /Replanned by proposal/); - assert.equal(replacement.state.executions[1]?.status, 'active'); - assert.equal( - replacement.state.activeExecutionId, - replacement.state.executions[1]?.executionId, - ); - }); - }); - - test('allows the user to abandon an interrupted execution', async () => { - await withStore(async (store) => { - const submitted = await store.submitProposal({ - sessionId: 'session-1', - turnId: 'turn-1', - title: 'Interrupted plan', - steps: [{ id: 'one', title: 'First step', description: 'One' }], - }); - const proposal = submitted.state.proposals[0]!; - const approved = await store.approveProposal({ - sessionId: 'session-1', - proposalId: proposal.proposalId, - expectedRevision: proposal.revision, - }); - const execution = approved.state.executions[0]!; - await store.interruptActiveExecution('session-1', 'User stopped execution'); - - const cancelled = await store.cancelExecution({ - sessionId: 'session-1', - executionId: execution.executionId, - reason: 'User abandoned the interrupted plan', - }); - - assert.equal(cancelled.event.type, 'plan_execution_cancelled'); - assert.equal(cancelled.state.executions[0]?.status, 'cancelled'); - assert.equal(cancelled.state.activeExecutionId, undefined); - await assert.rejects( - store.cancelExecution({ - sessionId: 'session-1', - executionId: execution.executionId, - reason: 'Cancel twice', - }), - /cannot be cancelled/, - ); - }); - }); - - test('interrupt is a no-op when a queued mutation already ended the execution', async () => { - await withStore(async (store) => { - assert.equal(await store.interruptActiveExecution('session-1', 'shutdown'), null); - }); - }); -}); - -async function withStore( - run: (store: ReturnType) => Promise, -): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-plan-store-')); - try { - await run(createPlanStore(root)); - } finally { - await rm(root, { recursive: true, force: true }); - } -} diff --git a/packages/storage/src/__tests__/pricing-store.test.ts b/packages/storage/src/__tests__/pricing-store.test.ts deleted file mode 100644 index 35addab1eb..0000000000 --- a/packages/storage/src/__tests__/pricing-store.test.ts +++ /dev/null @@ -1,271 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, mock, test } from 'node:test'; -import { - createPricingStore, - PricingCommitUnknownError, - PricingRevisionConflictError, - PricingStorePublicationError, - PricingValidationError, -} from '../pricing-store.js'; - -describe('PricingStore', () => { - test('single-flights concurrent load and allows retry after failure', async () => { - await withRoot(async (root) => { - const path = join(root, 'pricing.json'); - await writeFile(path, '{'); - const store = createPricingStore(root); - - const first = store.load(); - const concurrent = store.load(); - assert.equal(concurrent, first); - await assert.rejects(() => first, SyntaxError); - - await writeFile( - path, - JSON.stringify({ version: 1, revision: 0, overrides: [] }, null, 2) + '\n', - ); - await store.load(); - assert.deepEqual(store.snapshot(), { revision: 0, overrides: [] }); - await store.close(); - }); - }); - - test('close joins a load started in the same tick', async () => { - await withRoot(async (root) => { - const store = createPricingStore(root); - - let loadSettled = false; - const load = store.load().finally(() => { - loadSettled = true; - }); - await store.close(); - assert.equal(loadSettled, true); - await load; - - assert.match(await readFile(join(root, 'pricing.json'), 'utf8'), /"version": 1/); - assert.throws(() => store.snapshot(), /draining or closed/); - }); - }); - - test('serializes revision CAS so only one mutation at a revision commits', async () => { - await withRoot(async (root) => { - const store = createPricingStore(root); - await store.load(); - - const results = await Promise.allSettled([ - store.upsert(0, pricing('openai:gpt-5')), - store.upsert(0, pricing('anthropic:claude')), - ]); - - assert.equal(results.filter((result) => result.status === 'fulfilled').length, 1); - const rejected = results.find((result) => result.status === 'rejected'); - assert(rejected?.status === 'rejected'); - assert(rejected.reason instanceof PricingRevisionConflictError); - assert.equal(rejected.reason.actualRevision, 1); - assert.equal(store.snapshot().revision, 1); - await store.close(); - }); - }); - - test('rejects invalid values without advancing revision', async () => { - await withRoot(async (root) => { - const store = createPricingStore(root); - await store.load(); - - assert.throws( - () => store.upsert(0, { ...pricing('openai:gpt-5'), inputUsdPer1M: -1 }), - PricingValidationError, - ); - assert.equal(store.snapshot().revision, 0); - await store.close(); - }); - }); - - test('persists a stable revision document and reloads it', async () => { - await withRoot(async (root) => { - const first = createPricingStore(root); - await first.load(); - await first.upsert(0, pricing('openai:gpt-5')); - await first.close(); - - const second = createPricingStore(root); - await second.load(); - assert.deepEqual(second.snapshot(), { - revision: 1, - overrides: [pricing('openai:gpt-5')], - }); - assert.match(await readFile(join(root, 'pricing.json'), 'utf8'), /"revision": 1/); - await second.close(); - }); - }); - - test('normalizes mutation input and uses canonical equality for no-op detection', async () => { - await withRoot(async (root) => { - const store = createPricingStore(root); - await store.load(); - const first = await store.upsert(0, { - ...pricing(' openai:gpt-5 '), - ignored: true, - } as ReturnType); - assert.equal(first.changed, true); - - const second = await store.upsert(1, pricing('openai:gpt-5')); - assert.equal(second.changed, false); - assert.equal(second.snapshot.revision, 1); - assert.deepEqual(second.snapshot.overrides, [pricing('openai:gpt-5')]); - assert.doesNotMatch(await readFile(join(root, 'pricing.json'), 'utf8'), /ignored/); - await store.close(); - }); - }); - - test('persists exact-distinct Unicode keys in shared comparator order across reopen', async () => { - await withRoot(async (root) => { - const composed = '\u00e9'; - const decomposed = 'e\u0301'; - const first = createPricingStore(root); - await first.load(); - await first.upsert(0, pricing(composed)); - await first.upsert(1, pricing(decomposed)); - const snapshot = first.snapshot(); - assert.deepEqual( - snapshot.overrides.map((item) => item.modelKey), - [decomposed, composed], - ); - await first.close(); - - const before = await readFile(join(root, 'pricing.json'), 'utf8'); - assert.deepEqual( - (JSON.parse(before) as { overrides: Array<{ modelKey: string }> }).overrides.map( - (item) => item.modelKey, - ), - [decomposed, composed], - ); - const second = createPricingStore(root); - await second.load(); - assert.deepEqual(second.snapshot(), snapshot); - await second.close(); - assert.equal(await readFile(join(root, 'pricing.json'), 'utf8'), before); - }); - }); - - test('rejects noncanonical and unsorted version 1 documents without rewriting them', async () => { - await withRoot(async (root) => { - const documents = [ - { - version: 1, - revision: 0, - overrides: [{ ...pricing('openai:gpt-5'), unknown: true }], - }, - { - version: 1, - revision: 0, - overrides: [pricing(' openai:gpt-5 ')], - }, - { - version: 1, - revision: 0, - overrides: [pricing('z:model'), pricing('a:model')], - }, - ]; - for (const document of documents) { - const bytes = JSON.stringify(document, null, 2) + '\n'; - await writeFile(join(root, 'pricing.json'), bytes); - const store = createPricingStore(root); - await assert.rejects(() => store.load(), PricingValidationError); - assert.equal(await readFile(join(root, 'pricing.json'), 'utf8'), bytes); - } - }); - }); - - test('latches directory preparation failure across mutation, flush, and close', async () => { - await withRoot(async (root) => { - const store = createPricingStore(root); - await store.load(); - await rm(root, { recursive: true }); - await writeFile(root, 'not a directory'); - - const failure = await store.upsert(0, pricing('openai:gpt-5')).then( - () => undefined, - (error: unknown) => error, - ); - assert(failure instanceof PricingStorePublicationError); - await assert.rejects( - () => store.flush(), - (error) => error === failure, - ); - await assert.rejects( - () => store.close(), - (error) => error === failure, - ); - }); - }); - - test('poisons reads after rename succeeds but directory sync fails', { - skip: process.platform === 'win32', - }, async () => { - await withRoot(async (root) => { - const store = createPricingStore(root); - await store.load(); - const fault = new Error('injected pricing directory sync failure'); - const probe = await open(root, 'r'); - const fileHandlePrototype = Object.getPrototypeOf(probe) as { - sync: typeof probe.sync; - }; - const originalSync = fileHandlePrototype.sync; - await probe.close(); - let injected = false; - const syncMock = mock.method( - fileHandlePrototype, - 'sync', - async function (this: typeof probe) { - const metadata = await this.stat(); - if (!injected && metadata.isDirectory()) { - injected = true; - throw fault; - } - return originalSync.call(this); - }, - ); - let failure: unknown; - try { - failure = await store.upsert(0, pricing('openai:gpt-5')).then( - () => undefined, - (error: unknown) => error, - ); - } finally { - syncMock.mock.restore(); - } - - assert(failure instanceof PricingCommitUnknownError); - assert.match(await readFile(join(root, 'pricing.json'), 'utf8'), /"revision": 1/); - assert.throws( - () => store.snapshot(), - (error) => error === failure, - ); - await assert.rejects( - () => store.flush(), - (error) => error === failure, - ); - await assert.rejects( - () => store.close(), - (error) => error === failure, - ); - }); - }); -}); - -function pricing(modelKey: string) { - return { modelKey, inputUsdPer1M: 1.25, outputUsdPer1M: 10 }; -} - -async function withRoot(run: (root: string) => Promise): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-pricing-store-')); - try { - await run(root); - } finally { - await rm(root, { recursive: true, force: true }); - } -} diff --git a/packages/storage/src/__tests__/project-catalog.test.ts b/packages/storage/src/__tests__/project-catalog.test.ts index e4c1ad4994..0f9cf5fa6a 100644 --- a/packages/storage/src/__tests__/project-catalog.test.ts +++ b/packages/storage/src/__tests__/project-catalog.test.ts @@ -39,7 +39,7 @@ test('a Git probe failure cannot persistently downgrade a repository to a folder await mkdir(repository); await execFileAsync('git', ['init', '--quiet'], { cwd: repository }); - await assert.rejects(() => importLegacyProjectWithoutGit(repository, storage)); + await assert.rejects(() => registerProjectWithoutGit(repository, storage)); await assert.rejects(() => readFile(join(storage, 'projects.json')), { code: 'ENOENT' }); } finally { await rm(base, { recursive: true, force: true }); @@ -75,9 +75,9 @@ async function resolveProjectLocationWithoutGit(path: string): Promise { +async function registerProjectWithoutGit(path: string, storage: string): Promise { await runProjectCatalogWithoutGit( - 'const [moduleUrl, path, storage] = process.argv.slice(1); const { createProjectCatalog } = await import(moduleUrl); await createProjectCatalog(storage).importLegacyPath(path);', + 'const [moduleUrl, path, storage] = process.argv.slice(1); const { createProjectCatalog } = await import(moduleUrl); await createProjectCatalog(storage).register(path);', path, storage, ); @@ -227,34 +227,6 @@ test('a missing project directory remains in the catalog as unavailable', async } }); -test('importing a missing legacy path preserves it as an unavailable project', async () => { - const base = await mkdtemp(join(tmpdir(), 'maka-project-legacy-missing-')); - try { - const missingPath = join(base, 'moved-project'); - const catalog = createProjectCatalog(join(base, 'storage'), { - now: () => 1_000, - createId: () => 'project-1', - }); - - const project = await catalog.importLegacyPath(missingPath); - - assert.equal(project.id, 'project-1'); - assert.equal(project.name, 'moved-project'); - assert.equal(project.available, false); - assert.equal(project.preferredPath, undefined); - assert.deepEqual(project.locations, [ - { - path: missingPath, - isWorktree: false, - }, - ]); - assert.equal((await catalog.importLegacyPath(missingPath)).id, project.id); - assert.equal((await catalog.list()).length, 1); - } finally { - await rm(base, { recursive: true, force: true }); - } -}); - test('relinking an unavailable project preserves its id and adopts the new directory', async () => { const base = await mkdtemp(join(tmpdir(), 'maka-project-relink-')); try { @@ -295,7 +267,10 @@ test('conflicting relink waits for a retryable merge before removing the duplica now: () => 1_000, createId: () => `project-${++id}`, }); - const original = await catalog.importLegacyPath(join(base, 'missing-original')); + const originalPath = join(base, 'original'); + await mkdir(originalPath); + const original = await catalog.register(originalPath); + await rm(originalPath, { recursive: true, force: true }); const duplicate = await catalog.register(relocated); const interrupted = new Error('session reassignment interrupted'); @@ -340,7 +315,10 @@ test('conflicting relink preserves every available worktree location from the me now: () => 1_000, createId: () => `project-${++id}`, }); - const original = await catalog.importLegacyPath(join(base, 'missing-original')); + const originalPath = join(base, 'original'); + await mkdir(originalPath); + const original = await catalog.register(originalPath); + await rm(originalPath, { recursive: true, force: true }); await catalog.register(repository); await catalog.register(linkedWorktree); @@ -423,7 +401,9 @@ test('selecting a project returns its most recent available location and rejects createId: () => `project-${++id}`, }); const available = await catalog.register(availablePath); - const missing = await catalog.importLegacyPath(missingPath); + await mkdir(missingPath); + const missing = await catalog.register(missingPath); + await rm(missingPath, { recursive: true, force: true }); now = 2_000; const selected = await catalog.select(available.id); diff --git a/packages/storage/src/__tests__/project-session-migration.test.ts b/packages/storage/src/__tests__/project-session-migration.test.ts deleted file mode 100644 index ec1d662309..0000000000 --- a/packages/storage/src/__tests__/project-session-migration.test.ts +++ /dev/null @@ -1,170 +0,0 @@ -import assert from 'node:assert/strict'; -import { appendFile, mkdir, mkdtemp, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { test } from 'node:test'; -import type { CreateSessionInput } from '@maka/core'; -import { createProjectCatalog } from '../project-catalog.js'; -import { migrateSessionProjects } from '../project-session-migration.js'; -import { createLegacyFileSessionStore, createSessionStore } from '../session-store.js'; -import { createGitRepositoryWithWorktree } from './fixtures/git-repository.js'; - -test('migrates legacy sessions into stable projects once without losing missing paths', async () => { - const base = await mkdtemp(join(tmpdir(), 'maka-project-session-migration-')); - const repository = join(base, 'repository'); - const linkedWorktree = join(base, 'linked'); - const missingPath = join(base, 'moved-project'); - const storage = join(base, 'storage'); - await createGitRepositoryWithWorktree(repository, linkedWorktree, 'migration-linked'); - - const sessions = createLegacyFileSessionStore(storage); - let id = 0; - const catalog = createProjectCatalog(storage, { - now: () => 1_000, - createId: () => `project-${++id}`, - }); - - try { - const main = await sessions.create(makeInput(repository, 'Main checkout')); - const linked = await sessions.create(makeInput(linkedWorktree, 'Linked worktree')); - const missing = await sessions.create(makeInput(missingPath, 'Missing checkout')); - - const first = await migrateSessionProjects({ sessions, catalog }); - - assert.deepEqual(first, { migrated: 3, unchanged: 0, failed: 0 }); - const mainHeader = await sessions.readHeaderSnapshot(main.id); - const linkedHeader = await sessions.readHeaderSnapshot(linked.id); - const missingHeader = await sessions.readHeaderSnapshot(missing.id); - assert.equal(mainHeader.projectId, linkedHeader.projectId); - assert.notEqual(missingHeader.projectId, mainHeader.projectId); - const projectsById = new Map( - (await catalog.list()).map((project) => [ - project.id, - { available: project.available, locations: project.locations.length }, - ]), - ); - assert.deepEqual(projectsById.get(mainHeader.projectId!), { - available: true, - locations: 2, - }); - assert.deepEqual(projectsById.get(missingHeader.projectId!), { - available: false, - locations: 1, - }); - - assert.deepEqual(await migrateSessionProjects({ sessions, catalog }), { - migrated: 0, - unchanged: 3, - failed: 0, - }); - assert.equal((await catalog.list()).length, 2); - } finally { - await sessions.close?.(); - await rm(base, { recursive: true, force: true }); - } -}); - -test('migrates SQLite session headers without reading corrupt transcript bodies', async () => { - const base = await mkdtemp(join(tmpdir(), 'maka-project-session-sqlite-migration-')); - const project = join(base, 'project'); - const storage = join(base, 'storage'); - await mkdir(project); - const sessions = createSessionStore(storage); - const catalog = createProjectCatalog(storage, { - createId: () => 'project-1', - }); - - try { - const session = await sessions.create(makeInput(project, 'Corrupt transcript')); - await appendFile( - join(storage, 'sessions', session.id, 'session.jsonl'), - '{"type":"user","broken"\n', - 'utf8', - ); - - assert.deepEqual(await migrateSessionProjects({ sessions, catalog }), { - migrated: 1, - unchanged: 0, - failed: 0, - }); - assert.equal((await sessions.readHeaderSnapshot(session.id)).projectId, 'project-1'); - } finally { - await sessions.close?.(); - await rm(base, { recursive: true, force: true }); - } -}); - -test('preserves session recency when building the project catalog', async () => { - const base = await mkdtemp(join(tmpdir(), 'maka-project-session-recency-')); - const alpha = join(base, 'alpha'); - const beta = join(base, 'beta'); - const storage = join(base, 'storage'); - await Promise.all([mkdir(alpha), mkdir(beta)]); - const sessions = createSessionStore(storage); - let id = 0; - const catalog = createProjectCatalog(storage, { - now: () => 999, - createId: () => `project-${++id}`, - }); - - try { - const alphaOlder = await sessions.create(makeInput(alpha, 'Alpha older')); - const betaOnly = await sessions.create(makeInput(beta, 'Beta')); - const alphaNewest = await sessions.create(makeInput(alpha, 'Alpha newest')); - await sessions.updateHeader(alphaOlder.id, { lastUsedAt: 100, lastMessageAt: 100 }); - await sessions.updateHeader(betaOnly.id, { lastUsedAt: 300, lastMessageAt: 300 }); - await sessions.updateHeader(alphaNewest.id, { lastUsedAt: 200, lastMessageAt: 200 }); - - await migrateSessionProjects({ sessions, catalog }); - - assert.deepEqual( - (await catalog.list()).map((project) => project.name), - ['beta', 'alpha'], - ); - } finally { - await sessions.close?.(); - await rm(base, { recursive: true, force: true }); - } -}); - -test('does not migrate an explicit no-project session after SQLite reopen', async () => { - const base = await mkdtemp(join(tmpdir(), 'maka-project-session-no-project-')); - const project = join(base, 'project'); - const storage = join(base, 'storage'); - await mkdir(project); - const sessions = createSessionStore(storage); - const created = await sessions.create({ - ...makeInput(project, 'No project'), - projectId: null, - }); - await sessions.close?.(); - - const reopened = createSessionStore(storage); - const catalog = createProjectCatalog(storage, { - createId: () => 'project-1', - }); - try { - assert.deepEqual(await migrateSessionProjects({ sessions: reopened, catalog }), { - migrated: 0, - unchanged: 1, - failed: 0, - }); - assert.equal((await reopened.readHeaderSnapshot(created.id)).projectId, null); - assert.deepEqual(await catalog.list(), []); - } finally { - await reopened.close?.(); - await rm(base, { recursive: true, force: true }); - } -}); - -function makeInput(cwd: string, name: string): CreateSessionInput { - return { - cwd, - backend: 'fake', - llmConnectionSlug: 'fake', - model: 'fake-model', - permissionMode: 'ask', - name, - labels: [], - }; -} diff --git a/packages/storage/src/__tests__/runtime-event-transfer.test.ts b/packages/storage/src/__tests__/runtime-event-transfer.test.ts deleted file mode 100644 index 28b5ba0516..0000000000 --- a/packages/storage/src/__tests__/runtime-event-transfer.test.ts +++ /dev/null @@ -1,407 +0,0 @@ -import assert from 'node:assert/strict'; -import { appendFile, mkdtemp, readFile, rm, stat } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, it } from 'node:test'; -import type { AgentRunHeader, RuntimeEvent } from '@maka/core'; -import { createAgentRunStore, createRuntimeEventStore } from '../agent-run-store.js'; -import { createSqliteRuntimeStore } from '../sqlite-runtime-store.js'; -import { - exportRuntimeEventsToJsonl, - importLegacyRuntimeEventJsonlTree, - importRuntimeEventsFromJsonl, - openRuntimeEventPersistence, - openRuntimeEventReadPersistence, -} from '../runtime-event-transfer.js'; - -describe('runtime event JSONL compatibility transfer', () => { - it('imports a legacy runtime tree idempotently into the SQLite canonical store', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-runtime-import-')); - const legacy = createRuntimeEventStore(root); - const sqlite = createSqliteRuntimeStore(join(root, 'runtime.sqlite')); - try { - const runs = createAgentRunStore(root); - await runs.createRun(runHeader()); - await runs.createRun(runHeader({ runId: 'run-2', invocationId: 'run-2', turnId: 'turn-2' })); - await legacy.appendRuntimeEvent('session-1', 'run-1', runtimeEvent('event-1')); - await legacy.appendRuntimeEvent('session-1', 'run-1', runtimeEvent('event-2', { ts: 2 })); - await legacy.appendRuntimeEvent( - 'session-1', - 'run-2', - runtimeEvent('event-3', { - invocationId: 'run-2', - runId: 'run-2', - turnId: 'turn-2', - ts: 3, - }), - ); - await appendFile( - join(root, 'sessions', 'session-1', 'runs', 'run-1', 'runtime-events.jsonl'), - `${JSON.stringify( - runtimeEvent('event-4', { - ts: 4, - content: undefined, - actions: { - permissionRequest: { - requestId: 'pr-1', - toolUseId: 'tc-1', - toolName: 'Bash', - category: 'shell_unsafe', - reason: 'shell_dangerous', - args: { command: 'rm foo' }, - }, - } as never, - }), - )}\n`, - ); - - const first = await importLegacyRuntimeEventJsonlTree({ - workspaceRoot: root, - destination: sqlite, - }); - const second = await importLegacyRuntimeEventJsonlTree({ - workspaceRoot: root, - destination: sqlite, - }); - - assert.deepEqual(first, { - filesScanned: 2, - eventsRead: 4, - eventsImported: 4, - eventsExisting: 0, - }); - assert.deepEqual(second, { - filesScanned: 0, - eventsRead: 0, - eventsImported: 0, - eventsExisting: 0, - }); - assert.deepEqual( - (await sqlite.readSessionRuntimeEvents('session-1')).map((event) => event.id), - ['event-1', 'event-2', 'event-3', 'event-4'], - ); - const importedRequest = (await sqlite.readRuntimeEvents('session-1', 'run-1')).at(-1)?.actions - ?.permissionRequest; - assert.equal(importedRequest?.kind, 'tool_permission'); - assert.equal(importedRequest?.rememberForTurnAllowed, false); - } finally { - sqlite.close(); - await rm(root, { recursive: true, force: true }); - } - }); - - it('skips legacy stream partial snapshots left in the JSONL log instead of failing', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-runtime-import-partial-')); - const legacy = createRuntimeEventStore(root); - const sqlite = createSqliteRuntimeStore(join(root, 'runtime.sqlite')); - try { - await createAgentRunStore(root).createRun(runHeader()); - await legacy.appendRuntimeEvent('session-1', 'run-1', runtimeEvent('event-1')); - await legacy.appendRuntimeEvent('session-1', 'run-1', runtimeEvent('event-2', { ts: 2 })); - // Older versions wrote stream partial snapshots straight into the JSONL - // log; current code diverts them to .partial files. Simulate the legacy row. - const jsonlPath = join( - root, - 'sessions', - 'session-1', - 'runs', - 'run-1', - 'runtime-events.jsonl', - ); - const legacyStreamPartial = { - ...runtimeEvent('partial-thinking', { - ts: 3, - partial: true, - role: 'model', - author: 'agent', - content: { kind: 'thinking', text: 'interrupted thought' }, - }), - unexpectedField: true, - }; - const partialRowWithStatus = runtimeEvent('partial-terminal', { - ts: 4, - partial: true, - role: 'model', - author: 'agent', - status: 'failed', - actions: { endInvocation: true }, - }); - await appendFile( - jsonlPath, - `${JSON.stringify(legacyStreamPartial)}\n${JSON.stringify(partialRowWithStatus)}\n`, - 'utf8', - ); - - const report = await importLegacyRuntimeEventJsonlTree({ - workspaceRoot: root, - destination: sqlite, - }); - - assert.deepEqual(report, { - filesScanned: 1, - eventsRead: 3, - eventsImported: 3, - eventsExisting: 0, - }); - assert.deepEqual( - (await sqlite.readImmutableRuntimeEvents('session-1', 'run-1')).map((event) => event.id), - ['event-1', 'event-2', 'partial-terminal'], - ); - } finally { - sqlite.close(); - await rm(root, { recursive: true, force: true }); - } - }); - - it('round-trips explicit JSONL export without creating a second live writer', async () => { - const sqlite = createSqliteRuntimeStore(':memory:'); - try { - const events = [runtimeEvent('event-1'), runtimeEvent('event-2', { ts: 2 })]; - for (const event of events) await sqlite.appendRuntimeEvent('session-1', 'run-1', event); - - const jsonl = await exportRuntimeEventsToJsonl(sqlite, 'session-1', 'run-1'); - const destination = createSqliteRuntimeStore(':memory:'); - try { - const report = await importRuntimeEventsFromJsonl({ - jsonl, - sessionId: 'session-1', - runId: 'run-1', - destination, - }); - assert.deepEqual(report, { eventsRead: 2, eventsImported: 2, eventsExisting: 0 }); - assert.deepEqual(await destination.readRuntimeEvents('session-1', 'run-1'), events); - } finally { - destination.close(); - } - } finally { - sqlite.close(); - } - }); - - it('rejects malformed rows and cross-run identity drift', async () => { - const sqlite = createSqliteRuntimeStore(':memory:'); - try { - await assert.rejects( - importRuntimeEventsFromJsonl({ - jsonl: '{not-json}\n', - sessionId: 'session-1', - runId: 'run-1', - destination: sqlite, - }), - /Invalid RuntimeEvent JSONL line 1/, - ); - await assert.rejects( - importRuntimeEventsFromJsonl({ - jsonl: `${JSON.stringify(runtimeEvent('event-1', { runId: 'other-run' }))}\n`, - sessionId: 'session-1', - runId: 'run-1', - destination: sqlite, - }), - /identity mismatch/i, - ); - await assert.rejects( - importRuntimeEventsFromJsonl({ - jsonl: `${JSON.stringify(runtimeEvent('partial-1', { partial: true }))}\n`, - sessionId: 'session-1', - runId: 'run-1', - destination: sqlite, - }), - /partial RuntimeEvent/i, - ); - } finally { - sqlite.close(); - } - }); - - it('imports the stable prefix of a crash-truncated legacy tail without rewriting it', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-runtime-truncated-import-')); - const sqlite = createSqliteRuntimeStore(join(root, 'runtime.sqlite')); - const sourcePath = join(root, 'sessions', 'session-1', 'runs', 'run-1', 'runtime-events.jsonl'); - try { - await createAgentRunStore(root).createRun(runHeader()); - const legacy = createRuntimeEventStore(root); - await legacy.appendRuntimeEvent('session-1', 'run-1', runtimeEvent('event-1')); - await appendFile(sourcePath, '{"id":"truncated"', 'utf8'); - const before = await readFile(sourcePath); - - const report = await importLegacyRuntimeEventJsonlTree({ - workspaceRoot: root, - destination: sqlite, - }); - - assert.deepEqual(report, { - filesScanned: 1, - eventsRead: 1, - eventsImported: 1, - eventsExisting: 0, - }); - assert.deepEqual( - (await sqlite.readImmutableRuntimeEvents('session-1', 'run-1')).map((event) => event.id), - ['event-1'], - ); - assert.deepEqual(await readFile(sourcePath), before); - - await appendFile(sourcePath, '\n', 'utf8'); - await assert.rejects( - importLegacyRuntimeEventJsonlTree({ workspaceRoot: root, destination: sqlite }), - /Invalid legacy RuntimeEvent JSONL line 2/, - ); - } finally { - sqlite.close(); - await rm(root, { recursive: true, force: true }); - } - }); - - it('always selects the SQLite writer, imports first, and leaves legacy JSONL unchanged', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-runtime-persistence-')); - const legacy = createRuntimeEventStore(root); - try { - await createAgentRunStore(root).createRun(runHeader()); - await legacy.appendRuntimeEvent('session-1', 'run-1', runtimeEvent('event-1')); - const legacyPath = join( - root, - 'sessions', - 'session-1', - 'runs', - 'run-1', - 'runtime-events.jsonl', - ); - const legacyBytes = await readFile(legacyPath); - const legacyMtime = (await stat(legacyPath)).mtimeMs; - - const opened = await openRuntimeEventPersistence({ - workspaceRoot: root, - }); - try { - assert.equal(opened.kind, 'sqlite'); - assert.ok(opened.runtimeCommitStore); - assert.strictEqual(opened.runtimeEventStore, opened.runtimeCommitStore); - assert.equal(opened.importReport?.eventsImported, 1); - assert.deepEqual( - (await opened.runtimeEventStore.readRuntimeEvents('session-1', 'run-1')).map( - (event) => event.id, - ), - ['event-1'], - ); - await opened.runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - runtimeEvent('sqlite-only-event', { ts: 2 }), - ); - } finally { - opened.close(); - } - assert.deepEqual(await readFile(legacyPath), legacyBytes); - assert.equal((await stat(legacyPath)).mtimeMs, legacyMtime); - - const reopened = await openRuntimeEventPersistence({ - workspaceRoot: root, - }); - try { - assert.equal(reopened.kind, 'sqlite'); - assert.deepEqual( - (await reopened.runtimeEventStore.readRuntimeEvents('session-1', 'run-1')).map( - (event) => event.id, - ), - ['event-1', 'sqlite-only-event'], - ); - } finally { - reopened.close(); - } - - const fresh = await openRuntimeEventPersistence({ - workspaceRoot: join(root, 'legacy-only'), - }); - assert.equal(fresh.kind, 'sqlite'); - assert.strictEqual(fresh.runtimeEventStore, fresh.runtimeCommitStore); - fresh.close(); - } finally { - await rm(root, { recursive: true, force: true }); - } - }); - - it('keeps legacy-only roots read-only until a writer performs the canonical import', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-runtime-reader-')); - try { - await createAgentRunStore(root).createRun(runHeader()); - const legacy = createRuntimeEventStore(root); - await legacy.appendRuntimeEvent('session-1', 'run-1', runtimeEvent('event-1')); - - const legacyReader = await openRuntimeEventReadPersistence({ workspaceRoot: root }); - try { - assert.equal(legacyReader.kind, 'jsonl'); - assert.deepEqual( - (await legacyReader.runtimeEventStore.readRuntimeEvents('session-1', 'run-1')).map( - (event) => event.id, - ), - ['event-1'], - ); - assert.equal('appendRuntimeEvent' in legacyReader.runtimeEventStore, false); - } finally { - legacyReader.close(); - } - await assert.rejects(stat(join(root, 'runtime.sqlite')), { code: 'ENOENT' }); - - const writer = await openRuntimeEventPersistence({ workspaceRoot: root }); - try { - await writer.runtimeEventStore.appendRuntimeEvent( - 'session-1', - 'run-1', - runtimeEvent('event-2', { ts: 2 }), - ); - } finally { - writer.close(); - } - - const sqliteReader = await openRuntimeEventReadPersistence({ workspaceRoot: root }); - try { - assert.equal(sqliteReader.kind, 'sqlite'); - assert.deepEqual( - (await sqliteReader.runtimeEventStore.readRuntimeEvents('session-1', 'run-1')).map( - (event) => event.id, - ), - ['event-1', 'event-2'], - ); - assert.equal('appendRuntimeEvent' in sqliteReader.runtimeEventStore, false); - } finally { - sqliteReader.close(); - } - } finally { - await rm(root, { recursive: true, force: true }); - } - }); -}); - -function runtimeEvent(id: string, overrides: Partial = {}): RuntimeEvent { - return { - id, - invocationId: 'run-1', - runId: 'run-1', - sessionId: 'session-1', - turnId: 'turn-1', - ts: 1, - partial: false, - role: 'user', - author: 'user', - content: { kind: 'text', text: id }, - ...overrides, - }; -} - -function runHeader(overrides: Partial = {}): AgentRunHeader { - return { - runId: 'run-1', - invocationId: 'run-1', - sessionId: 'session-1', - turnId: 'turn-1', - status: 'created', - backendKind: 'fake', - llmConnectionSlug: 'fake', - modelId: 'fake-model', - cwd: '/workspace', - permissionMode: 'ask', - createdAt: 1, - updatedAt: 1, - ...overrides, - }; -} diff --git a/packages/storage/src/__tests__/session-bundle-policy.test.ts b/packages/storage/src/__tests__/session-bundle-policy.test.ts index ada9e52432..6ada451a50 100644 --- a/packages/storage/src/__tests__/session-bundle-policy.test.ts +++ b/packages/storage/src/__tests__/session-bundle-policy.test.ts @@ -1,117 +1,31 @@ import assert from 'node:assert/strict'; -import { createHash } from 'node:crypto'; -import { - link, - lstat, - mkdtemp, - mkdir, - readFile, - readdir, - realpath, - rm, - symlink, - writeFile, -} from 'node:fs/promises'; +import { mkdir, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; -import { dirname, join, resolve } from 'node:path'; +import { join } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; import { test } from 'node:test'; -import { - MAX_SANDBOX_BOUNDARY_SERIALIZED_BYTES, - type ArtifactRecord, - type RuntimeEvent, -} from '@maka/core'; -import { - assertSessionBundleRootLayout, - exportSessionBundleState, - planSessionBundleExport, - type SessionBundleExportError, -} from '../session-bundle-policy.js'; -import { - createSqliteArtifactStore, - createSqliteArtifactStoreWriteAuthority, -} from '../artifact-store.js'; +import type { CreateSessionInput } from '@maka/core'; import { createSessionStore } from '../session-store.js'; -import { LONG_TERM_MEMORY_DATABASE_NAME } from '../long-term-memory-store.js'; -import { SqliteMemoryItemStore } from '../sqlite-long-term-memory-store.js'; -import { createSqliteRuntimeStore } from '../sqlite-runtime-store.js'; - -const createArtifactStore = createSqliteArtifactStore; -const createArtifactStoreWriteAuthority = createSqliteArtifactStoreWriteAuthority; - -test('exports one session only and excludes credential/config canaries', async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const sessions = createSessionStore(stateRoot); - const selected = await sessions.create(sessionInput('Selected'), { - kind: 'external', - revision: 0, - }); - const other = await sessions.create(sessionInput('Other')); - await sessions.appendMessage(selected.id, { - type: 'user', - id: 'message-1', - turnId: 'turn-1', - ts: 10, - text: 'selected transcript', - }); +import { exportSessionBundleState } from '../session-bundle-policy.js'; + +test('exports one Session as filtered SQLite', async () => { + const base = await mkdtemp(join(tmpdir(), 'maka-session-bundle-')); + const stateRoot = join(base, 'state'); + const configRoot = join(base, 'config'); + const destinationRoot = join(base, 'bundle'); + await mkdir(configRoot, { recursive: true }); + const sessions = createSessionStore(stateRoot); + try { + const selected = await sessions.create(input('Selected')); + const excluded = await sessions.create(input('Excluded')); + await sessions.appendMessage(selected.id, message('selected-message')); + await sessions.appendMessage(excluded.id, message('excluded-message')); await sessions.close?.(); - const selectedSessionRoot = join(stateRoot, 'sessions', selected.id); - const portableSessionEntries = [ - ['runs/run-1/run.json', '{"run":"selected"}\n'], - ['projections/history-compact.json', '{"projection":"selected"}\n'], - ['turn-admissions/turn-1.json', '{"turn":"selected"}\n'], - ['shell-runs/shell-1/shell-run.json', '{"shell":"selected"}\n'], - ['deep-research/events.jsonl', '{"research":"selected"}\n'], - ['tasks.json', '{"tasks":[]}\n'], - ['task-events.jsonl', '{"task":"selected"}\n'], - ['plan-events.jsonl', '{"planEvent":"selected"}\n'], - ['plans.json', '{"plans":[]}\n'], - ] as const; - for (const [relativePath, contents] of portableSessionEntries) { - const path = join(selectedSessionRoot, relativePath); - await mkdir(dirname(path), { recursive: true }); - await writeFile(path, contents); - } - const artifacts = createSqliteArtifactStore(stateRoot); - const selectedArtifact = await artifacts.create({ - id: 'selected-output', - sessionId: selected.id, - turnId: 'turn-1', - name: 'output.txt', - kind: 'file', - content: 'session output\n', - now: 10, - }); - const otherArtifact = await artifacts.create({ - id: 'other-output', - sessionId: other.id, - turnId: 'turn-1', - name: 'other-output.txt', - kind: 'file', - content: 'other output\n', - now: 11, - }); - const runtime = createSqliteRuntimeStore(join(stateRoot, 'runtime.sqlite')); - await runtime.appendRuntimeEvent( - selected.id, - 'run-1', - runtimeEvent('event-1', { sessionId: selected.id }), - ); - await runtime.appendRuntimeEvent( - other.id, - 'run-2', - runtimeEvent('event-2', { - sessionId: other.id, - runId: 'run-2', - invocationId: 'run-2', - }), - ); - runtime.close(); - await writeFile(join(stateRoot, 'credentials.json'), 'super-secret-api-key'); - await writeFile(join(stateRoot, 'llm-connections.json'), 'host-provider-config'); - await writeFile(join(stateRoot, '.maka_cli_claude_device_id'), 'device-identity'); - await mkdir(join(stateRoot, 'task-runs')); - await writeFile(join(stateRoot, 'task-runs', 'headless-ledger.jsonl'), 'headless-only\n'); - await writeFile(join(configRoot, 'credentials.json'), 'config-secret-canary'); + const sourceDatabase = new DatabaseSync(join(stateRoot, 'runtime.sqlite')); + sourceDatabase + .prepare('INSERT INTO usage_pricing_overrides(model_key, record_json) VALUES (?, ?)') + .run('private-model', '{}'); + sourceDatabase.close(); const plan = await exportSessionBundleState({ stateRoot, @@ -119,967 +33,50 @@ test('exports one session only and excludes credential/config canaries', async ( destinationRoot, sessionId: selected.id, }); - - assert.deepEqual(plan.includedEntries, ['artifacts', 'runtime.sqlite', 'sessions']); - assert.deepEqual( - [...plan.excludedEntries].sort(), - [ - '.maka_cli_claude_device_id', - 'credentials.json', - 'llm-connections.json', - `artifacts/${other.id}`, - 'runtime.sqlite-shm', - 'runtime.sqlite-wal', - `sessions/${other.id}`, - 'task-runs', - ].sort(), - ); - const exportedTranscript = await readFile( - join(destinationRoot, 'sessions', selected.id, 'session.jsonl'), - 'utf8', - ); - const [exportedHeader, exportedMessage] = exportedTranscript - .trim() - .split('\n') - .map((line) => JSON.parse(line) as Record); - assert.equal(exportedHeader.id, selected.id); - assert.equal(exportedHeader.name, 'Selected'); - assert.equal(exportedHeader.type, undefined); - assert.equal(exportedMessage.text, 'selected transcript'); - for (const [relativePath, contents] of portableSessionEntries) { + assert.deepEqual(plan.includedEntries, ['runtime.sqlite']); + const database = new DatabaseSync(join(destinationRoot, 'runtime.sqlite'), { readOnly: true }); + try { + const ids = database + .prepare('SELECT session_id FROM session_metadata ORDER BY session_id') + .all() + .map((row) => (row as { session_id: string }).session_id); + assert.deepEqual(ids, [selected.id]); assert.equal( - await readFile(join(destinationRoot, 'sessions', selected.id, relativePath), 'utf8'), - contents, + ( + database.prepare('SELECT COUNT(*) AS count FROM session_messages').get() as { + count: number; + } + ).count, + 1, ); - } - await assert.rejects(readFile(join(destinationRoot, 'sessions', other.id, 'session.jsonl'))); - await assert.rejects(readFile(join(destinationRoot, 'sessions.sqlite'))); - await assert.rejects(readFile(join(destinationRoot, 'artifacts', otherArtifact.relativePath))); - await assert.rejects(lstat(join(destinationRoot, 'artifacts', 'metadata.jsonl')), { - code: 'ENOENT', - }); - await assert.rejects(lstat(join(destinationRoot, 'task-runs')), { code: 'ENOENT' }); - const reopenedArtifacts = createSqliteArtifactStore(destinationRoot); - try { - assert.deepEqual(await reopenedArtifacts.list(selected.id), [selectedArtifact]); - assert.deepEqual(await reopenedArtifacts.list(other.id), []); - assert.deepEqual(await reopenedArtifacts.readText(selectedArtifact.id), { - ok: true, - text: 'session output\n', - }); - const createdAfterRestore = await reopenedArtifacts.create({ - id: 'restored-write', - sessionId: selected.id, - turnId: 'turn-2', - name: 'restored.txt', - kind: 'file', - content: 'restored write\n', - now: 12, - }); - assert.deepEqual(await reopenedArtifacts.readText(createdAfterRestore.id), { - ok: true, - text: 'restored write\n', - }); - } finally { - reopenedArtifacts.close?.(); - artifacts.close?.(); - } - const exportedRuntime = createSqliteRuntimeStore(join(destinationRoot, 'runtime.sqlite')); - try { - assert.equal((await exportedRuntime.readSessionRuntimeEvents(selected.id)).length, 1); - assert.equal((await exportedRuntime.readSessionRuntimeEvents(other.id)).length, 0); - } finally { - exportedRuntime.close(); - } - await assert.rejects(readFile(join(destinationRoot, 'credentials.json'), 'utf8')); - await assert.rejects(readFile(join(destinationRoot, 'llm-connections.json'), 'utf8')); - await assert.rejects(readFile(join(destinationRoot, '.maka_cli_claude_device_id'), 'utf8')); - await assert.rejects(readFile(join(destinationRoot, '.maka-artifact-writer.lock'), 'utf8')); - assert.equal( - await readFile(join(configRoot, 'credentials.json'), 'utf8'), - 'config-secret-canary', - ); - const restoredSessions = createSessionStore(destinationRoot); - try { - assert.equal((await restoredSessions.readExecutionBoundary(selected.id)).kind, 'external'); - } finally { - await restoredSessions.close?.(); - } - await assert.rejects( - readFile(join(destinationRoot, 'sessions', selected.id, 'execution-boundary.json'), 'utf8'), - ); - }); -}); - -test('exports while the operational DB remains open and protects its sidecars', async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const sessions = createSessionStore(stateRoot); - try { - const selected = await sessions.create(sessionInput('Selected')); - await sessions.appendMessage(selected.id, { - type: 'user', - id: 'message-1', - turnId: 'turn-1', - ts: 10, - text: 'selected transcript', - }); - - const sidecars = ['runtime.sqlite-wal', 'runtime.sqlite-shm']; - const liveEntries = await readdir(stateRoot); - for (const sidecar of sidecars) { - assert.ok(liveEntries.includes(sidecar), liveEntries.join(', ')); - } - - const planned = await planSessionBundleExport({ - stateRoot, - configRoot, - destinationRoot, - sessionId: selected.id, - }); - for (const sidecar of sidecars) { - assert.ok(planned.excludedEntries.includes(sidecar), JSON.stringify(planned)); - } - - const exported = await exportSessionBundleState({ - stateRoot, - configRoot, - destinationRoot, - sessionId: selected.id, - }); - for (const sidecar of sidecars) { - assert.ok(exported.excludedEntries.includes(sidecar), JSON.stringify(exported)); - await assert.rejects(readFile(join(destinationRoot, sidecar))); - } - assert.match( - await readFile(join(destinationRoot, 'sessions', selected.id, 'session.jsonl'), 'utf8'), - /selected transcript/, + assert.equal( + ( + database.prepare('SELECT COUNT(*) AS count FROM usage_pricing_overrides').get() as { + count: number; + } + ).count, + 0, ); } finally { - await sessions.close?.(); - } - }); -}); - -test('excludes the long-term memory database and its sidecars from Session Bundles', async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const sessions = createSessionStore(stateRoot); - const selected = await sessions.create(sessionInput('Selected')); - await sessions.close?.(); - - const databasePath = join(stateRoot, LONG_TERM_MEMORY_DATABASE_NAME); - const memory = new SqliteMemoryItemStore(databasePath); - memory.close(); - const protectedEntries = [ - LONG_TERM_MEMORY_DATABASE_NAME, - `${LONG_TERM_MEMORY_DATABASE_NAME}-wal`, - `${LONG_TERM_MEMORY_DATABASE_NAME}-shm`, - `${LONG_TERM_MEMORY_DATABASE_NAME}-journal`, - ]; - for (const entry of protectedEntries.slice(1)) { - await writeFile(join(stateRoot, entry), 'memory-sidecar-canary'); - } - - const plan = await exportSessionBundleState({ - stateRoot, - configRoot, - destinationRoot, - sessionId: selected.id, - }); - - for (const entry of protectedEntries) { - assert.ok(plan.excludedEntries.includes(entry), JSON.stringify(plan)); - await assert.rejects(readFile(join(destinationRoot, entry))); - } - }); -}); - -test('exports legacy Artifact metadata into SQLite regardless of source record order', async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const sessionId = await createSelectedSession(stateRoot); - const artifacts = createArtifactStore(stateRoot); - const later = await artifacts.create({ - id: 'later-artifact', - sessionId, - turnId: 'turn-1', - name: 'later.txt', - kind: 'file', - content: 'later\n', - now: 2, - }); - const earlier = await artifacts.create({ - id: 'earlier-artifact', - sessionId, - turnId: 'turn-1', - name: 'earlier.txt', - kind: 'file', - content: 'earlier\n', - now: 1, - }); - - await exportSessionBundleState({ stateRoot, configRoot, destinationRoot, sessionId }); - - await assert.rejects(lstat(join(destinationRoot, 'artifacts', 'metadata.jsonl')), { - code: 'ENOENT', - }); - const reopened = createSqliteArtifactStore(destinationRoot); - try { - assert.deepEqual(await reopened.list(sessionId), [later, earlier]); - } finally { - reopened.close?.(); - } - }); -}); - -test('fails closed when legacy Artifact evidence changes after SQLite cutover', async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const sessionId = await createSelectedSession(stateRoot); - await seedLegacyArtifact(stateRoot, { - id: 'before-cutover', - sessionId, - turnId: 'turn-1', - name: 'before.txt', - kind: 'file', - content: 'before\n', - now: 1, - }); - const sqlite = createSqliteArtifactStore(stateRoot); - try { - assert.equal((await sqlite.list(sessionId)).length, 1); - } finally { - sqlite.close?.(); - } - - await seedLegacyArtifact(stateRoot, { - id: 'stale-writer', - sessionId, - turnId: 'turn-2', - name: 'stale.txt', - kind: 'file', - content: 'stale\n', - now: 2, - }); - - await assert.rejects( - exportSessionBundleState({ stateRoot, configRoot, destinationRoot, sessionId }), - (error: unknown) => { - const exportError = error as SessionBundleExportError; - assert.equal(exportError.code, 'unsupported_entry'); - assert.match(exportError.message, /Artifact metadata changed during session bundle export/); - return true; - }, - ); - await assert.rejects(readdir(destinationRoot), { code: 'ENOENT' }); - }); -}); - -test('restores the accumulated managed boundary from a session bundle', async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const sessions = createSessionStore(stateRoot); - const selected = await sessions.create(sessionInput('Selected')); - await sessions.createSandboxBoundaryRequest({ - sessionId: selected.id, - requestId: 'request-1', - turnId: 'turn-1', - expansion: { - filesystem: { - entries: [{ path: '/outside/selected.txt', access: 'write', scope: 'exact' }], - }, - }, - justification: 'Write the selected output.', - }); - await sessions.settleSandboxBoundaryRequest({ - sessionId: selected.id, - requestId: 'request-1', - decision: 'allow', - }); - const sourceBoundary = await sessions.readExecutionBoundary(selected.id); - await sessions.close?.(); - - await exportSessionBundleState({ - stateRoot, - configRoot, - destinationRoot, - sessionId: selected.id, - }); - const restoredSessions = createSessionStore(destinationRoot); - try { - const restoredBoundary = await restoredSessions.readExecutionBoundary(selected.id); - assert.equal(sourceBoundary.kind, 'managed'); - assert.equal(restoredBoundary.kind, 'managed'); - if (sourceBoundary.kind !== 'managed' || restoredBoundary.kind !== 'managed') return; - assert.deepEqual(restoredBoundary.profile, sourceBoundary.profile); - assert.equal(restoredBoundary.revision, 0); - } finally { - await restoredSessions.close?.(); - } - }); -}); - -test('round-trips a cumulative boundary larger than one expansion payload', async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const sessions = createSessionStore(stateRoot); - const selected = await sessions.create(sessionInput('Selected')); - for (let request = 0; request < 3; request += 1) { - const requestId = `request-${request}`; - await sessions.createSandboxBoundaryRequest({ - sessionId: selected.id, - requestId, - turnId: 'turn-1', - expansion: { - filesystem: { - entries: Array.from({ length: 32 }, (_, entry) => ({ - path: `/outside/${request}/${entry}-${'x'.repeat(700)}`, - access: 'read' as const, - scope: 'exact' as const, - })), - }, - }, - justification: 'Read generated inputs.', - }); - await sessions.settleSandboxBoundaryRequest({ - sessionId: selected.id, - requestId, - decision: 'allow', - }); - } - const sourceBoundary = await sessions.readExecutionBoundary(selected.id); - await sessions.close?.(); - - await exportSessionBundleState({ - stateRoot, - configRoot, - destinationRoot, - sessionId: selected.id, - }); - const transferPath = join(destinationRoot, 'sessions', selected.id, 'execution-boundary.json'); - assert.ok((await lstat(transferPath)).size > MAX_SANDBOX_BOUNDARY_SERIALIZED_BYTES); - - const restoredSessions = createSessionStore(destinationRoot); - try { - const restoredBoundary = await restoredSessions.readExecutionBoundary(selected.id); - assert.equal(sourceBoundary.kind, 'managed'); - assert.equal(restoredBoundary.kind, 'managed'); - if (sourceBoundary.kind !== 'managed' || restoredBoundary.kind !== 'managed') return; - assert.deepEqual(restoredBoundary.profile, sourceBoundary.profile); - } finally { - await restoredSessions.close?.(); - } - }); -}); - -test('fails closed on unknown top-level state entries', async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - await writeFile(join(stateRoot, 'new-secret-store.json'), 'unknown-secret'); - - await assertExportError( - planSessionBundleExport({ stateRoot, configRoot, destinationRoot, sessionId: 'session-1' }), - 'unknown_entry', - ); - }); -}); - -test('requires Artifact authority recovery for selected-session payloads absent from metadata', async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const sessionId = await createSelectedSession(stateRoot); - const artifacts = createArtifactStore(stateRoot); - await artifacts.create({ - id: 'canonical-artifact', - sessionId, - turnId: 'turn-1', - name: 'canonical.txt', - kind: 'file', - content: 'canonical payload\n', - now: 1, - }); - const orphanRelativePath = `artifacts/${sessionId}/orphan-payload.txt`; - await writeFile(join(stateRoot, orphanRelativePath), 'orphan payload\n'); - - await assertArtifactRecoveryRequired( - exportSessionBundleState({ - stateRoot, - configRoot, - destinationRoot, - sessionId, - }), - orphanRelativePath, - ); - }); -}); - -test('fails closed on unknown directories in the selected Artifact payload root', async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const sessionId = await createSelectedSession(stateRoot); - const artifacts = createArtifactStore(stateRoot); - await artifacts.create({ - id: 'canonical-artifact', - sessionId, - turnId: 'turn-1', - name: 'canonical.txt', - kind: 'file', - content: 'canonical payload\n', - now: 1, - }); - await mkdir(join(stateRoot, 'artifacts', sessionId, 'unclassified-directory')); - - await assertExportError( - planSessionBundleExport({ stateRoot, configRoot, destinationRoot, sessionId }), - 'unknown_entry', - ); - }); -}); - -test('requires Artifact authority recovery for canonical transaction residue', async (t) => { - const residueStates = [ - 'publication staging with linked target', - 'purge intent with live payload and metadata', - 'purge intent temp with live payload and metadata', - 'metadata temp with uncommitted target metadata', - ] as const; - - for (const residueState of residueStates) { - await t.test(residueState, async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const sessionId = await createSelectedSession(stateRoot); - const residuePath = await createArtifactTransactionResidue( - stateRoot, - sessionId, - residueState, - ); - - const expectedResidue = - residueState === 'metadata temp with uncommitted target metadata' - ? undefined - : residuePath; - await assertArtifactRecoveryRequired( - planSessionBundleExport({ stateRoot, configRoot, destinationRoot, sessionId }), - expectedResidue, - ); - await assertArtifactRecoveryRequired( - exportSessionBundleState({ stateRoot, configRoot, destinationRoot, sessionId }), - expectedResidue, - ); - await assert.rejects(readdir(destinationRoot), { code: 'ENOENT' }); - }); - }); - } -}); - -test('authority recovery removes canonical root temps before bundle export', async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const sessionId = await createSelectedSession(stateRoot); - const artifacts = createArtifactStore(stateRoot); - const record = await artifacts.create({ - id: 'canonical-artifact', - sessionId, - turnId: 'turn-1', - name: 'canonical.txt', - kind: 'file', - content: 'canonical payload\n', - now: 1, - }); - const artifactRoot = join(stateRoot, 'artifacts'); - const uuid = '00000000-0000-4000-8000-000000000000'; - const tempPaths = [ - join(artifactRoot, `metadata.jsonl.123.${uuid}.tmp`), - join(artifactRoot, 'metadata.jsonl.123.1700000000000.tmp'), - join(artifactRoot, `.artifact-purge-intent.json.123.${uuid}.tmp`), - ]; - await writeFile(tempPaths[0]!, 'retired metadata writer residue\n'); - await writeFile(tempPaths[1]!, 'retired metadata writer residue\n'); - await writeFile(tempPaths[2]!, JSON.stringify({ schemaVersion: 1, artifactIds: [record.id] })); - - await assertArtifactRecoveryRequired( - planSessionBundleExport({ stateRoot, configRoot, destinationRoot, sessionId }), - ); - const authority = createArtifactStoreWriteAuthority(stateRoot); - await authority.recover(); - for (const tempPath of tempPaths) { - await assert.rejects(lstat(tempPath), { code: 'ENOENT' }); - } - - await exportSessionBundleState({ - stateRoot, - configRoot, - destinationRoot, - sessionId, - }); - const reopened = createSqliteArtifactStore(destinationRoot); - try { - assert.deepEqual(await reopened.list(sessionId), [record]); - assert.deepEqual(await reopened.readText(record.id), { - ok: true, - text: 'canonical payload\n', - }); - } finally { - reopened.close?.(); + database.close(); } - }); -}); - -test('requires Artifact authority recovery for publication residue in another session', async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const selectedSessionId = await createSelectedSession(stateRoot); - const otherSessionId = await createSelectedSession(stateRoot); - const residuePath = await createArtifactTransactionResidue( - stateRoot, - otherSessionId, - 'publication staging with linked target', - ); - - await assertArtifactRecoveryRequired( - planSessionBundleExport({ - stateRoot, - configRoot, - destinationRoot, - sessionId: selectedSessionId, - }), - residuePath, - ); - }); -}); - -test('fails closed on Artifact residue lookalikes with invalid names', async () => { - const invalidEntries = [ - { - relativePath: 'artifacts/.artifact-purge-intent.json.123.not-a-uuid.tmp', - insideSelectedSession: false, - }, - { - relativePath: 'artifacts/metadata.jsonl.123.not-a-uuid.tmp', - insideSelectedSession: false, - }, - { - relativePath: `.artifact-publish.${'a'.repeat(64)}.not-a-uuid.tmp`, - insideSelectedSession: true, - }, - ] as const; - - for (const invalid of invalidEntries) { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const sessionId = await createSelectedSession(stateRoot); - const artifactRoot = join(stateRoot, 'artifacts'); - const relativePath = invalid.insideSelectedSession - ? `artifacts/${sessionId}/${invalid.relativePath}` - : invalid.relativePath; - await mkdir(dirname(join(stateRoot, relativePath)), { recursive: true }); - await writeFile(join(stateRoot, relativePath), 'not valid residue\n'); - - await assertExportError( - planSessionBundleExport({ stateRoot, configRoot, destinationRoot, sessionId }), - 'unknown_entry', - ); - }); - } -}); - -test('rejects every Artifact metadata file that the Artifact store cannot reopen', async (t) => { - const corruptions: ReadonlyArray<{ - name: string; - apply: (records: Array>) => unknown[]; - }> = [ - { - name: 'unknown field in an unselected session', - apply: (records) => [records[0], { ...records[1], hostSecret: 'must-not-export' }], - }, - { - name: 'invalid kind in an unselected session', - apply: (records) => [records[0], { ...records[1], kind: 'archive' }], - }, - { - name: 'path that does not match the record identity', - apply: (records) => [ - { ...records[0], relativePath: `${String(records[0]?.sessionId)}/wrong-name.txt` }, - records[1], - ], - }, - { - name: 'name that the Artifact writer could not persist', - apply: (records) => { - const selected = records[0]!; - return [ - { - ...selected, - name: 'invalid/name.txt', - relativePath: `${String(selected.sessionId)}/${String(selected.id)}-invalid/name.txt`, - }, - records[1], - ]; - }, - }, - { - name: 'duplicate artifact id', - apply: (records) => [records[0], records[1], { ...records[0] }], - }, - ]; - - for (const corruption of corruptions) { - await t.test(corruption.name, async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const selectedSessionId = await createSelectedSession(stateRoot); - const otherSessionId = await createSelectedSession(stateRoot); - const selectedRecord = await seedLegacyArtifact(stateRoot, { - id: 'selected-artifact', - sessionId: selectedSessionId, - turnId: 'turn-1', - name: 'selected.txt', - kind: 'file', - content: 'selected\n', - now: 1, - }); - const otherRecord = await seedLegacyArtifact(stateRoot, { - id: 'other-artifact', - sessionId: otherSessionId, - turnId: 'turn-1', - name: 'other.txt', - kind: 'file', - content: 'other\n', - now: 2, - }); - - const metadataPath = join(stateRoot, 'artifacts', 'metadata.jsonl'); - const records: Array> = [{ ...selectedRecord }, { ...otherRecord }]; - const corrupted = corruption.apply(records); - await writeFile( - metadataPath, - `${corrupted.map((record) => JSON.stringify(record)).join('\n')}\n`, - ); - - await assert.rejects( - () => createArtifactStore(stateRoot).list(selectedSessionId), - /Invalid artifact metadata line/, - ); - await assertArtifactMetadataRejected( - exportSessionBundleState({ - stateRoot, - configRoot, - destinationRoot, - sessionId: selectedSessionId, - }), - ); - }); - }); + } finally { + await rm(base, { recursive: true, force: true }); } }); -test('fails closed on unknown entries inside the selected session', async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const sessionId = await createSelectedSession(stateRoot); - await writeFile( - join(stateRoot, 'sessions', sessionId, 'unclassified-cache.json'), - '{"secret":"must-not-export"}\n', - ); - - await assertExportError( - planSessionBundleExport({ stateRoot, configRoot, destinationRoot, sessionId }), - 'unknown_entry', - ); - }); -}); - -test('fails closed on symlinked state entries and path escape', async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot, root }) => { - const outside = join(root, 'outside-secret.txt'); - await writeFile(outside, 'outside-secret'); - const sessionId = await createSelectedSession(stateRoot); - await symlink(outside, join(stateRoot, 'sessions', sessionId, 'escaped.txt')); - - await assertExportError( - planSessionBundleExport({ stateRoot, configRoot, destinationRoot, sessionId }), - 'symlink', - ); - }); -}); - -test('rejects a missing state root without creating it', async () => { - await withBundleRoots(async ({ root, configRoot, destinationRoot }) => { - const stateRoot = join(root, 'missing-state'); - - await assertExportError( - exportSessionBundleState({ - stateRoot, - configRoot, - destinationRoot, - sessionId: 'missing-session', - }), - 'invalid_root', - ); - await assert.rejects(lstat(stateRoot), { code: 'ENOENT' }); - }); -}); - -test('rejects a symlinked state root without modifying its target', async () => { - await withBundleRoots(async ({ root, stateRoot, configRoot, destinationRoot }) => { - await createSelectedSession(stateRoot); - const linkedRoot = join(root, 'linked-state'); - await symlink(stateRoot, linkedRoot); - - await assertExportError( - exportSessionBundleState({ - stateRoot: linkedRoot, - configRoot, - destinationRoot, - sessionId: 'missing-session', - }), - 'symlink', - ); - await assert.rejects(lstat(join(stateRoot, '.maka-artifact-writer.lock')), { code: 'ENOENT' }); - }); -}); - -test('rejects unsafe root overlap while allowing explicit legacy sharing', async () => { - await withBundleRoots(async ({ stateRoot, configRoot }) => { - await assertSessionBundleRootLayout({ stateRoot, configRoot }); - await assertSessionBundleRootLayout({ - stateRoot, - configRoot: stateRoot, - allowShared: true, - }); - - await assertExportError( - assertSessionBundleRootLayout({ - stateRoot, - configRoot: join(stateRoot, 'config'), - }), - 'overlapping_roots', - ); - }); -}); - -test('rejects a bundle destination nested inside a source root', async () => { - await withBundleRoots(async ({ stateRoot, configRoot }) => { - await assertExportError( - planSessionBundleExport({ - stateRoot, - configRoot, - destinationRoot: join(stateRoot, 'bundle'), - sessionId: 'session-1', - }), - 'overlapping_roots', - ); - }); -}); - -test('preserves the complete suffix for a deeply missing destination path', async () => { - await withBundleRoots(async ({ root, stateRoot, configRoot }) => { - const sessionId = await createSelectedSession(stateRoot); - const destinationRoot = join(root, 'missing', 'deep', 'bundle'); - const plan = await planSessionBundleExport({ - stateRoot, - configRoot, - destinationRoot, - sessionId, - }); - assert.equal(plan.destinationRoot, resolve(await realpath(root), 'missing', 'deep', 'bundle')); - }); -}); - -test('fails closed when the state root contains no sessions', async () => { - await withBundleRoots(async ({ stateRoot, configRoot, destinationRoot }) => { - const sessions = createSessionStore(stateRoot); - await sessions.close?.(); - await assertExportError( - planSessionBundleExport({ - stateRoot, - configRoot, - destinationRoot, - sessionId: 'missing-session', - }), - 'invalid_root', - ); - }); -}); - -async function assertExportError( - operation: Promise, - code: SessionBundleExportError['code'], -): Promise { - await assert.rejects(operation, (error: unknown) => { - assert.equal((error as SessionBundleExportError).code, code); - return true; - }); -} - -async function assertArtifactRecoveryRequired( - operation: Promise, - relativePath?: string, -): Promise { - await assert.rejects(operation, (error: unknown) => { - const exportError = error as SessionBundleExportError; - assert.equal(exportError.code, 'unsupported_entry'); - assert.match(exportError.message, /Artifact write authority recovery is required/); - if (relativePath !== undefined) { - assert.match(exportError.message, new RegExp(escapeRegExp(relativePath))); - } - return true; - }); -} - -async function assertArtifactMetadataRejected(operation: Promise): Promise { - await assert.rejects(operation, (error: unknown) => { - const exportError = error as SessionBundleExportError; - assert.equal(exportError.code, 'unsupported_entry'); - assert.match(exportError.message, /cannot be reopened by the Artifact store/); - assert.match( - String((exportError as Error & { cause?: unknown }).cause), - /Invalid artifact metadata line \d+/, - ); - return true; - }); -} - -async function createArtifactTransactionResidue( - stateRoot: string, - sessionId: string, - residueState: - | 'publication staging with linked target' - | 'purge intent with live payload and metadata' - | 'purge intent temp with live payload and metadata' - | 'metadata temp with uncommitted target metadata', -): Promise { - const artifactRoot = join(stateRoot, 'artifacts'); - const sessionArtifactRoot = join(artifactRoot, sessionId); - const targetName = 'artifact-1-output.txt'; - const targetPath = join(sessionArtifactRoot, targetName); - const relativeTargetPath = `${sessionId}/${targetName}`; - const payload = 'transactional artifact\n'; - const metadata = `${JSON.stringify({ - id: 'artifact-1', - sessionId, - turnId: 'turn-1', - createdAt: 1, - name: 'output.txt', - kind: 'file', - relativePath: relativeTargetPath, - sizeBytes: Buffer.byteLength(payload), - status: 'live', - })}\n`; - const uuid = '00000000-0000-4000-8000-000000000000'; - await mkdir(sessionArtifactRoot, { recursive: true }); - - if (residueState === 'publication staging with linked target') { - const targetHash = createHash('sha256').update(targetName).digest('hex'); - const stagingName = `.artifact-publish.${targetHash}.${uuid}.tmp`; - const stagingPath = join(sessionArtifactRoot, stagingName); - await writeFile(stagingPath, payload); - await link(stagingPath, targetPath); - await writeFile(join(artifactRoot, 'metadata.jsonl'), ''); - return `artifacts/${sessionId}/${stagingName}`; - } - - await writeFile(targetPath, payload); - if (residueState === 'metadata temp with uncommitted target metadata') { - const metadataTempName = `metadata.jsonl.123.${uuid}.tmp`; - await writeFile(join(artifactRoot, 'metadata.jsonl'), ''); - await writeFile(join(artifactRoot, metadataTempName), metadata); - return `artifacts/${metadataTempName}`; - } - - await writeFile(join(artifactRoot, 'metadata.jsonl'), metadata); - const purgeIntent = JSON.stringify({ schemaVersion: 1, artifactIds: ['artifact-1'] }); - if (residueState === 'purge intent with live payload and metadata') { - await writeFile(join(artifactRoot, '.artifact-purge-intent.json'), purgeIntent); - return 'artifacts/.artifact-purge-intent.json'; - } - - const purgeIntentTempName = `.artifact-purge-intent.json.123.${uuid}.tmp`; - await writeFile(join(artifactRoot, purgeIntentTempName), purgeIntent); - return `artifacts/${purgeIntentTempName}`; -} - -async function seedLegacyArtifact( - stateRoot: string, - input: { - id: string; - sessionId: string; - turnId: string; - name: string; - kind: 'file'; - content: string; - now: number; - }, -): Promise { - const relativePath = `${input.sessionId}/${input.id}-${input.name}`; - const payloadPath = join(stateRoot, 'artifacts', relativePath); - const metadataPath = join(stateRoot, 'artifacts', 'metadata.jsonl'); - await mkdir(dirname(payloadPath), { recursive: true }); - await writeFile(payloadPath, input.content); - const record: ArtifactRecord = { - id: input.id, - sessionId: input.sessionId, - turnId: input.turnId, - createdAt: input.now, - name: input.name, - kind: input.kind, - relativePath, - sizeBytes: Buffer.byteLength(input.content), - status: 'live', - }; - let existing = ''; - try { - existing = await readFile(metadataPath, 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - } - await writeFile(metadataPath, `${existing}${JSON.stringify(record)}\n`); - return record; -} - -function escapeRegExp(value: string): string { - return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -} - -function runtimeEvent(id: string, overrides: Partial = {}): RuntimeEvent { +function input(name: string): CreateSessionInput { return { - id, - invocationId: 'run-1', - runId: 'run-1', - sessionId: 'session-1', - turnId: 'turn-1', - ts: 1, - partial: false, - role: 'user', - author: 'user', - content: { kind: 'text', text: id }, - ...overrides, - }; -} - -function sessionInput(name: string) { - return { - cwd: '/repo', - backend: 'fake' as const, - llmConnectionSlug: 'fixture', - model: 'fixture-model', - permissionMode: 'execute' as const, + cwd: '/tmp/cwd', + backend: 'fake', + llmConnectionSlug: 'fake', + model: 'fake-model', + permissionMode: 'ask' as const, name, + labels: [], }; } -async function createSelectedSession(stateRoot: string): Promise { - const sessions = createSessionStore(stateRoot); - try { - return (await sessions.create(sessionInput('Selected'))).id; - } finally { - await sessions.close?.(); - } -} - -async function withBundleRoots( - fn: (roots: { - root: string; - stateRoot: string; - configRoot: string; - destinationRoot: string; - }) => Promise, -): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-session-bundle-')); - const stateRoot = join(root, 'state'); - const configRoot = join(root, 'config'); - const destinationRoot = join(root, 'export'); - await Promise.all([mkdir(stateRoot), mkdir(configRoot)]); - try { - await fn({ root, stateRoot, configRoot, destinationRoot }); - } finally { - await rm(root, { recursive: true, force: true }); - } +function message(id: string) { + return { type: 'user' as const, id, turnId: 'turn-1', ts: 1, text: id }; } diff --git a/packages/storage/src/__tests__/session-metadata-maintenance.test.ts b/packages/storage/src/__tests__/session-metadata-maintenance.test.ts deleted file mode 100644 index 7ae4aa8ba5..0000000000 --- a/packages/storage/src/__tests__/session-metadata-maintenance.test.ts +++ /dev/null @@ -1,165 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdtemp, readFile, rm } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, test } from 'node:test'; -import { createExternalExecutionBoundary, type CreateSessionInput } from '@maka/core'; -import { - backupSessionMetadataDatabase, - exportLegacySessionTree, - exportLegacySessionTreeSnapshot, - SESSION_METADATA_EXPORT_FORMAT, -} from '../session-metadata-maintenance.js'; -import { - createLegacyFileSessionStore, - createSessionStore, - SQLITE_SESSION_METADATA_DATABASE_NAME, -} from '../session-store.js'; -import { createSqliteSessionMetadataStore } from '../sqlite-session-metadata-store.js'; - -describe('session metadata migration maintenance', () => { - test('exports a legacy-compatible tree and creates an online SQLite backup', async () => { - const container = await mkdtemp(join(tmpdir(), 'maka-session-metadata-maintenance-')); - const workspaceRoot = join(container, 'workspace'); - const exportRoot = join(container, 'legacy-export'); - const backupPath = join(container, 'backups', 'sessions.sqlite'); - const store = createSessionStore(workspaceRoot); - try { - const created = await store.create(makeInput(), createExternalExecutionBoundary()); - await store.appendMessage(created.id, { - type: 'user', - id: 'user-1', - turnId: 'turn-1', - ts: 10, - text: 'portable transcript', - }); - await store.rename(created.id, 'Canonical SQLite title'); - - const exported = await exportLegacySessionTree({ - workspaceRoot, - destinationRoot: exportRoot, - now: () => 123, - }); - assert.equal(exported.sessionsExported, 1); - - const legacy = createLegacyFileSessionStore(exportRoot); - assert.equal((await legacy.readHeader(created.id)).name, 'Canonical SQLite title'); - assert.equal((await legacy.readMessages(created.id))[0]?.type, 'user'); - const restoredExport = createSessionStore(exportRoot); - try { - assert.equal((await restoredExport.readExecutionBoundary(created.id)).kind, 'external'); - } finally { - await restoredExport.close?.(); - } - - const manifest = JSON.parse(await readFile(exported.manifestPath, 'utf8')) as { - format: string; - exportedAt: number; - sessions: Array<{ header: { id: string; name: string } }>; - }; - assert.equal(manifest.format, SESSION_METADATA_EXPORT_FORMAT); - assert.equal(manifest.exportedAt, 123); - assert.deepEqual( - manifest.sessions.map((record) => record.header.id), - [created.id], - ); - assert.equal(manifest.sessions[0]?.header.name, 'Canonical SQLite title'); - - const backup = await backupSessionMetadataDatabase({ - workspaceRoot, - destinationPath: backupPath, - }); - assert.equal(backup.destinationPath, backupPath); - assert.ok(backup.pagesCopied > 0); - const restored = createSqliteSessionMetadataStore(backupPath); - try { - assert.equal((await restored.read(created.id)).header.name, 'Canonical SQLite title'); - } finally { - restored.close(); - } - - const liveTranscript = await readFile( - join(workspaceRoot, 'sessions', created.id, 'session.jsonl'), - 'utf8', - ); - assert.equal(JSON.parse(liveTranscript.split('\n')[0]!).type, 'session_transcript'); - await assert.rejects( - () => exportLegacySessionTree({ workspaceRoot, destinationRoot: exportRoot }), - /already exists/, - ); - await assert.rejects( - () => backupSessionMetadataDatabase({ workspaceRoot, destinationPath: backupPath }), - /already exists/, - ); - } finally { - await store.close?.(); - await rm(container, { recursive: true, force: true }); - } - }); - - test('refuses to export over the live workspace', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-metadata-export-overlap-')); - const store = createSessionStore(root); - try { - await store.create(makeInput()); - await assert.rejects( - () => exportLegacySessionTree({ workspaceRoot: root, destinationRoot: root }), - /overlaps/, - ); - await assert.rejects( - () => - backupSessionMetadataDatabase({ - workspaceRoot: root, - destinationPath: join(root, SQLITE_SESSION_METADATA_DATABASE_NAME), - }), - /differ from the source/, - ); - } finally { - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('refuses to publish a snapshot without every selected execution boundary', async () => { - const container = await mkdtemp(join(tmpdir(), 'maka-session-metadata-boundary-export-')); - const workspaceRoot = join(container, 'workspace'); - const destinationRoot = join(container, 'export'); - const store = createSessionStore(workspaceRoot); - try { - const created = await store.create(makeInput()); - const metadata = createSqliteSessionMetadataStore( - join(workspaceRoot, SQLITE_SESSION_METADATA_DATABASE_NAME), - ); - try { - const record = await metadata.read(created.id); - await assert.rejects( - () => - exportLegacySessionTreeSnapshot({ - workspaceRoot, - destinationRoot, - records: [record], - boundaries: new Map(), - }), - /execution boundary.*missing/i, - ); - await assert.rejects(() => readFile(destinationRoot), /ENOENT/); - } finally { - metadata.close(); - } - } finally { - await store.close?.(); - await rm(container, { recursive: true, force: true }); - } - }); -}); - -function makeInput(): CreateSessionInput { - return { - cwd: '/tmp/cwd', - backend: 'fake', - llmConnectionSlug: 'fake', - model: 'fake-model', - permissionMode: 'ask', - name: 'Initial title', - }; -} diff --git a/packages/storage/src/__tests__/session-metadata-transfer.test.ts b/packages/storage/src/__tests__/session-metadata-transfer.test.ts deleted file mode 100644 index ad216f23ce..0000000000 --- a/packages/storage/src/__tests__/session-metadata-transfer.test.ts +++ /dev/null @@ -1,424 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, test } from 'node:test'; -import type { CreateSessionInput } from '@maka/core'; -import { importLegacySessionMetadataTree } from '../session-metadata-transfer.js'; -import { - createLegacyFileSessionStore as createLegacyStore, - createSessionStore, - SQLITE_SESSION_METADATA_DATABASE_NAME, -} from '../session-store.js'; -import { createSqliteSessionMetadataStore } from '../sqlite-session-metadata-store.js'; - -describe('legacy session metadata transfer', () => { - test('imports every legacy line-1 header without reading transcript payloads as metadata', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-transfer-')); - const legacy = createLegacyStore(root); - const sqlite = createSqliteSessionMetadataStore(join(root, 'state.sqlite')); - try { - const first = await legacy.create(makeInput({ name: 'First', labels: ['alpha'] })); - const second = await legacy.create(makeInput({ name: 'Second', labels: ['beta'] })); - await legacy.appendMessage(first.id, { - type: 'user', - id: 'user-1', - turnId: 'turn-1', - ts: 10, - text: 'This transcript row is not session metadata.', - }); - await legacy.updateHeader(second.id, { - status: 'blocked', - blockedReason: 'permission_required', - hasUnread: true, - }); - - const report = await importLegacySessionMetadataTree({ - workspaceRoot: root, - destination: sqlite, - }); - assert.deepEqual(report, { - filesScanned: 2, - headersRead: 2, - headersImported: 2, - headersExisting: 0, - sourcesAlreadyImported: 0, - sourcesTombstoned: 0, - }); - assert.deepEqual((await sqlite.list()).map((record) => record.header.name).sort(), [ - 'First', - 'Second', - ]); - assert.deepEqual( - (await sqlite.read(first.id)).header, - await legacy.readHeaderSnapshot(first.id), - ); - assert.deepEqual( - (await sqlite.read(second.id)).header, - await legacy.readHeaderSnapshot(second.id), - ); - - await legacy.appendMessage(second.id, { - type: 'assistant', - id: 'assistant-1', - turnId: 'turn-1', - ts: 11, - text: 'Appending transcript bytes must not invalidate the imported header.', - modelId: 'fake-model', - }); - await sqlite.update(first.id, { name: 'SQLite is canonical now' }); - const repeated = await importLegacySessionMetadataTree({ - workspaceRoot: root, - destination: sqlite, - }); - assert.deepEqual(repeated, { - filesScanned: 2, - headersRead: 2, - headersImported: 0, - headersExisting: 0, - sourcesAlreadyImported: 2, - sourcesTombstoned: 0, - }); - assert.equal((await sqlite.read(first.id)).header.name, 'SQLite is canonical now'); - } finally { - sqlite.close(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('decodes legacy compatibility defaults through the FileSessionStore codec', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-transfer-legacy-')); - const sqlite = createSqliteSessionMetadataStore(join(root, 'state.sqlite')); - const sessionId = 'legacy-session'; - const path = join(root, 'sessions', sessionId, 'session.jsonl'); - try { - const legacy = { - id: sessionId, - workspaceRoot: root, - cwd: '/workspace', - createdAt: 1, - lastUsedAt: 2, - name: 'New Session', - isFlagged: false, - labels: [], - isArchived: false, - pendingCwdReminder: { - from: '/workspace/old', - to: '/workspace', - }, - hasUnread: false, - backend: 'pi', - llmConnectionSlug: 'legacy', - connectionLocked: false, - schemaVersion: 1, - }; - await mkdir(join(root, 'sessions', sessionId), { recursive: true }); - await writeFile(path, `${JSON.stringify(legacy)}\n`, 'utf8'); - - await importLegacySessionMetadataTree({ workspaceRoot: root, destination: sqlite }); - const header = (await sqlite.read(sessionId)).header; - assert.equal(header.backend, 'pi-agent'); - assert.equal(header.model, 'default'); - assert.equal(header.permissionMode, 'ask'); - assert.equal(header.collaborationMode, 'agent'); - assert.equal(header.orchestrationMode, 'default'); - assert.equal(header.status, 'active'); - assert.equal(header.titleIsManual, false); - assert.equal(header.name, 'New Chat'); - assert.equal(Object.hasOwn(header, 'pendingCwdReminder'), false); - } finally { - sqlite.close(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('skips a malformed header without tombstoning it while importing valid sessions', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-transfer-invalid-')); - const legacy = createLegacyStore(root); - const sqlite = createSqliteSessionMetadataStore(join(root, 'state.sqlite')); - try { - const valid = await legacy.create(makeInput({ name: 'Valid' })); - const invalid = await legacy.create(makeInput({ name: 'Invalid' })); - const invalidPath = join(root, 'sessions', invalid.id, 'session.jsonl'); - const lines = (await readFile(invalidPath, 'utf8')).split('\n'); - lines[0] = JSON.stringify({ ...JSON.parse(lines[0]!), labels: 'not-an-array' }); - await writeFile(invalidPath, lines.join('\n'), 'utf8'); - - const report = await importLegacySessionMetadataTree({ - workspaceRoot: root, - destination: sqlite, - }); - assert.equal(report.filesScanned, 2); - assert.equal(report.headersImported, 1); - // The valid session was imported. - assert.equal((await sqlite.read(valid.id)).header.name, 'Valid'); - // The malformed session was skipped, not imported and not tombstoned. - assert.equal(await sqlite.has(invalid.id), false); - assert.equal(await sqlite.isTombstoned(invalid.id), false); - // Re-importing should skip the malformed session again (not tombstoned). - const repeated = await importLegacySessionMetadataTree({ - workspaceRoot: root, - destination: sqlite, - }); - assert.equal(repeated.filesScanned, 2); - assert.equal(repeated.headersImported, 0); - // Repairing the header should allow it to be imported on the next run. - const repairedLines = (await readFile(invalidPath, 'utf8')).split('\n'); - const repairedHeader = JSON.parse(repairedLines[0]!); - repairedHeader.labels = ['repaired']; - repairedLines[0] = JSON.stringify(repairedHeader); - await writeFile(invalidPath, repairedLines.join('\n'), 'utf8'); - const repaired = await importLegacySessionMetadataTree({ - workspaceRoot: root, - destination: sqlite, - }); - assert.equal(repaired.headersImported, 1); - assert.equal((await sqlite.read(invalid.id)).header.name, 'Invalid'); - } finally { - sqlite.close(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('keeps canonical metadata readable when its optional transcript is missing', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-transfer-missing-transcript-')); - const legacy = createLegacyStore(root); - const sqlite = createSqliteSessionMetadataStore(join(root, 'state.sqlite')); - try { - const created = await legacy.create(makeInput({ name: 'Canonical metadata' })); - await importLegacySessionMetadataTree({ workspaceRoot: root, destination: sqlite }); - await rm(join(root, 'sessions', created.id, 'session.jsonl')); - - const report = await importLegacySessionMetadataTree({ - workspaceRoot: root, - destination: sqlite, - }); - - assert.deepEqual(report, { - filesScanned: 1, - headersRead: 0, - headersImported: 0, - headersExisting: 0, - sourcesAlreadyImported: 0, - sourcesTombstoned: 0, - }); - assert.equal((await sqlite.read(created.id)).header.name, 'Canonical metadata'); - } finally { - sqlite.close(); - await rm(root, { recursive: true, force: true }); - } - }); -}); - -describe('malformed legacy headers at the public SessionStore boundary', () => { - async function setupWorkspaceWithCorruptSession( - corruptContent: (originalHeaderLine: string) => string, - ): Promise<{ - root: string; - validId: string; - corruptId: string; - corruptPath: string; - originalHeaderLine: string; - }> { - const root = await mkdtemp(join(tmpdir(), 'maka-session-boundary-')); - const legacy = createLegacyStore(root); - const valid = await legacy.create(makeInput({ name: 'Valid' })); - const corrupt = await legacy.create(makeInput({ name: 'Corrupt' })); - await legacy.close?.(); - const corruptPath = join(root, 'sessions', corrupt.id, 'session.jsonl'); - const originalHeaderLine = (await readFile(corruptPath, 'utf8')).split('\n')[0]!; - await writeFile(corruptPath, corruptContent(originalHeaderLine), 'utf8'); - return { - root, - validId: valid.id, - corruptId: corrupt.id, - corruptPath, - originalHeaderLine, - }; - } - - function openMetadata(root: string) { - return createSqliteSessionMetadataStore(join(root, SQLITE_SESSION_METADATA_DATABASE_NAME)); - } - - async function assertRepairableMalformedHeader( - corruptContent: (originalHeaderLine: string) => string, - ): Promise { - const { root, validId, corruptId, corruptPath, originalHeaderLine } = - await setupWorkspaceWithCorruptSession(corruptContent); - try { - const store = createSessionStore(root); - try { - assert.deepEqual( - (await store.list()).map((session) => session.id), - [validId], - ); - await assert.rejects(store.readHeader(corruptId), /Session metadata not found/); - } finally { - await store.close?.(); - } - - const meta = openMetadata(root); - try { - assert.equal(await meta.has(corruptId), false); - assert.equal(await meta.isTombstoned(corruptId), false); - } finally { - meta.close(); - } - - await writeFile(corruptPath, `${originalHeaderLine}\n`, 'utf8'); - const reopened = createSessionStore(root); - try { - const repaired = await reopened.list(); - assert.equal(repaired.length, 2); - assert.equal( - repaired.some((session) => session.id === corruptId && session.name === 'Corrupt'), - true, - ); - } finally { - await reopened.close?.(); - } - } finally { - await rm(root, { recursive: true, force: true }); - } - } - - const repairableMalformedHeaders = [ - { - name: 'empty session.jsonl', - content: () => '', - }, - { - name: 'truncated JSON without a newline', - content: () => '{ "id": "truncated"', - }, - { - name: 'complete valid header over 1 MiB', - content: (originalHeaderLine: string) => { - const header = JSON.parse(originalHeaderLine) as Record; - header.name = 'x'.repeat(1024 * 1024); - return `${JSON.stringify(header)}\n`; - }, - }, - { - name: 'turn_state first record', - content: () => - `${JSON.stringify({ - type: 'turn_state', - id: 'state-1', - turnId: 'turn-1', - ts: 1, - status: 'running', - partialOutputRetained: false, - })}\n`, - }, - ]; - - for (const { name, content } of repairableMalformedHeaders) { - test(`${name}: skips only the corrupt session and imports it after repair`, () => - assertRepairableMalformedHeader(content)); - } - - async function assertInvalidCurrentTranscriptMarker(marker: { - sessionId: string; - schemaVersion: number; - }): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-session-boundary-invalid-marker-')); - const legacy = createLegacyStore(root); - try { - const valid = await legacy.create(makeInput({ name: 'Valid' })); - await legacy.close?.(); - const markerSessionId = 'invalid-marker-session'; - const markerDir = join(root, 'sessions', markerSessionId); - await mkdir(markerDir, { recursive: true }); - await writeFile( - join(markerDir, 'session.jsonl'), - `${JSON.stringify({ - type: 'session_transcript', - ...marker, - })}\n`, - 'utf8', - ); - - const store = createSessionStore(root); - try { - await assert.rejects(store.list(), /invalid transcript marker/); - } finally { - await store.close?.(); - } - - const meta = openMetadata(root); - try { - assert.equal(await meta.has(valid.id), false); - assert.equal(await meta.has(markerSessionId), false); - assert.equal(await meta.isTombstoned(markerSessionId), false); - } finally { - meta.close(); - } - } finally { - await rm(root, { recursive: true, force: true }); - } - } - - for (const { name, marker } of [ - { - name: 'wrong session id', - marker: { sessionId: 'wrong-session-id', schemaVersion: 1 }, - }, - { - name: 'unsupported schema version', - marker: { sessionId: 'invalid-marker-session', schemaVersion: 2 }, - }, - ]) { - test(`current transcript marker with ${name} rejects without partial import`, () => - assertInvalidCurrentTranscriptMarker(marker)); - } - - test('EISDIR filesystem failure: store.list() rejects with EISDIR, no metadata or tombstone written', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-boundary-eisdir-')); - const legacy = createLegacyStore(root); - try { - const valid = await legacy.create(makeInput({ name: 'Valid' })); - const unreadable = await legacy.create(makeInput({ name: 'Unreadable' })); - await legacy.close?.(); - const unreadablePath = join(root, 'sessions', unreadable.id, 'session.jsonl'); - // Replace the session file with a directory to provoke EISDIR. - await rm(unreadablePath); - await mkdir(unreadablePath); - - const store = createSessionStore(root); - try { - await assert.rejects( - store.list(), - (error: NodeJS.ErrnoException) => error.code === 'EISDIR', - ); - } finally { - await store.close?.(); - } - - const meta = openMetadata(root); - try { - assert.equal(await meta.has(valid.id), false); - assert.equal(await meta.has(unreadable.id), false); - assert.equal(await meta.isTombstoned(unreadable.id), false); - } finally { - meta.close(); - } - } finally { - await rm(root, { recursive: true, force: true }); - } - }); -}); - -function makeInput(overrides: Partial = {}): CreateSessionInput { - return { - cwd: '/tmp/cwd', - backend: 'fake', - llmConnectionSlug: 'fake', - model: 'fake-model', - permissionMode: 'ask', - name: 'Session', - labels: [], - ...overrides, - }; -} diff --git a/packages/storage/src/__tests__/session-store.test.ts b/packages/storage/src/__tests__/session-store.test.ts index 18c84389d9..b902a51d91 100644 --- a/packages/storage/src/__tests__/session-store.test.ts +++ b/packages/storage/src/__tests__/session-store.test.ts @@ -1,1605 +1,103 @@ import assert from 'node:assert/strict'; -import { mkdir, mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; +import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; +import { join } from 'node:path'; import { describe, test } from 'node:test'; -import { - WORKSPACE_AUTHORITY_SESSION_ID, - type CreateSessionInput, - type SessionHeader, - type StoredMessage, - type SubagentSessionParent, - type SubagentSessionRuntime, - type SubagentSessionSpawn, -} from '@maka/core'; -import { - assertSafeSessionId, - createLegacyFileSessionStore as createSessionStore, - createSessionStore as createSqliteSessionStore, -} from '../session-store.js'; - -describe('FileSessionStore CRUD', () => { - test('reserves the workspace authority control-plane session id', () => { - assert.throws(() => assertSafeSessionId(WORKSPACE_AUTHORITY_SESSION_ID), /Invalid session id/); - }); - test('list on a missing workspace is observational and does not create session storage', async () => { - const workspaceRoot = await mkdtemp(join(tmpdir(), 'maka-session-list-')); +import type { CreateSessionInput } from '@maka/core'; +import { createSessionStore, isSessionNotFoundError } from '../session-store.js'; +import { OPERATIONAL_STATE_DATABASE_NAME } from '../operational-state-store.js'; +import { createSqliteSessionMetadataStore } from '../sqlite-session-metadata-store.js'; + +describe('SQLite SessionStore', () => { + test('persists session metadata and messages in one SQLite authority', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-session-sqlite-')); + const store = createSessionStore(root); try { - const sessionsRoot = join(workspaceRoot, 'sessions'); - assert.deepEqual(await createSessionStore(workspaceRoot).list(), []); - await assert.rejects(() => readFile(sessionsRoot), { code: 'ENOENT' }); - } finally { - await rm(workspaceRoot, { recursive: true, force: true }); - } - }); - - test('archive sets isArchived and archivedAt; unarchive clears them', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Archived me' })); - - await store.archive(header.id); - const archived = await store.readHeader(header.id); - assert.equal(archived.isArchived, true); - assert.equal(archived.status, 'archived'); - assert.equal(typeof archived.archivedAt, 'number'); - - await store.unarchive(header.id); - const restored = await store.readHeader(header.id); - assert.equal(restored.isArchived, false); - assert.equal(restored.status, 'active'); - assert.equal(restored.archivedAt, undefined); - }); - }); - - test('new sessions default to active status and include it in summaries', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Status' })); - - assert.equal(header.status, 'active'); - assert.equal(header.collaborationMode, 'agent'); - assert.equal(header.orchestrationMode, 'default'); - assert.equal(typeof header.statusUpdatedAt, 'number'); - const [summary] = await store.list(); - assert.equal(summary?.status, 'active'); - assert.equal(summary?.statusUpdatedAt, header.statusUpdatedAt); - assert.equal(summary?.model, 'fake-model'); - assert.equal(summary?.cwd, '/tmp/cwd'); - assert.equal(summary?.collaborationMode, 'agent'); - assert.equal(summary?.orchestrationMode, 'default'); - }); - }); - - test('persists a requested orchestration mode in headers and summaries', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ orchestrationMode: 'swarm' })); - assert.equal(header.orchestrationMode, 'swarm'); - assert.equal((await store.list())[0]?.orchestrationMode, 'swarm'); - }); - }); - - test('persists the stable project association in headers and summaries', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ projectId: 'project-1' })); - - assert.equal(header.projectId, 'project-1'); - assert.equal((await store.readHeader(header.id)).projectId, 'project-1'); - assert.equal((await store.list())[0]?.projectId, 'project-1'); - }); - }); - - test('readHeaderSnapshot is observational and does not lock the connection', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Inspect me' })); - await store.appendMessage(header.id, { - type: 'user', - id: 'user-1', - turnId: 'turn-1', - ts: 1, - text: 'hello', - }); - - assert.equal((await store.readHeaderSnapshot(header.id)).connectionLocked, false); - assert.equal((await store.readHeaderSnapshot(header.id)).connectionLocked, false); - assert.equal((await store.readHeader(header.id)).connectionLocked, true); - }); - }); - - test('round-trips frozen inline references with the user message', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Inline references' })); - const inlineReferences = [ - { kind: 'skill' as const, value: '/skill:writer', label: 'Writer', start: 4 }, - { - kind: 'workspace_file' as const, - value: '@docs/my plan.md', - label: 'my plan.md', - start: 21, - }, - ]; - await store.appendMessage(header.id, { + const session = await store.create(makeInput()); + await store.appendMessage(session.id, { type: 'user', - id: 'user-1', + id: 'message-1', turnId: 'turn-1', - ts: 1, - text: 'Use /skill:writer on @docs/my plan.md', - inlineReferences, - }); - - const message = (await store.readMessages(header.id))[0]; - assert.deepEqual( - message?.type === 'user' ? message.inlineReferences : undefined, - inlineReferences, - ); - }); - }); - - test('round-trips an explicit empty inline-reference projection', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Plain current-format message' })); - await store.appendMessage(header.id, { - type: 'user', - id: 'user-empty', - turnId: 'turn-empty', - ts: 1, - text: 'plain', - inlineReferences: [], - }); - const message = (await store.readMessages(header.id))[0]; - assert.deepEqual(message?.type === 'user' ? message.inlineReferences : undefined, []); - }); - }); - - test('list summary carries thinkingLevel when set and omits it when cleared', async () => { - await withStore(async (store) => { - // No level on create: the summary omits the field (UI shows 默认). - const header = await store.create(makeInput({ name: 'Thinking' })); - assert.equal((await store.list())[0]?.thinkingLevel, undefined); - - // Setting a level persists it and the list summary surfaces it — this is - // the projection the renderer's refreshSessions reads, so the model chip - // reflects the chosen level instead of silently dropping it. - await store.updateHeader(header.id, { thinkingLevel: 'high' }); - assert.equal((await store.list())[0]?.thinkingLevel, 'high'); - - // Clearing it back to undefined removes the field from the summary. - await store.updateHeader(header.id, { thinkingLevel: undefined }); - assert.equal((await store.list())[0]?.thinkingLevel, undefined); - }); - }); - - test('reads legacy pending cwd reminders but drops them during normalization', async () => { - await withStore(async (store, workspaceRoot) => { - const header = await store.create(makeInput({ name: 'Moved session' })); - const path = join(workspaceRoot, 'sessions', header.id, 'session.jsonl'); - const legacyHeader = { - ...JSON.parse(await readFile(path, 'utf8')), - pendingCwdReminder: { - from: '/tmp/old-worktree', - to: '/tmp/new-worktree', - }, - }; - await writeFile(path, `${JSON.stringify(legacyHeader)}\n`, 'utf8'); - - const normalized = await store.readHeader(header.id); - assert.equal(Object.hasOwn(normalized, 'pendingCwdReminder'), false); - assert.equal(Object.hasOwn((await store.list())[0]!, 'pendingCwdReminder'), false); - - await store.updateHeader(header.id, { name: 'Normalized after move' }); - const persisted = JSON.parse(await readFile(path, 'utf8')) as Record; - assert.equal(Object.hasOwn(persisted, 'pendingCwdReminder'), false); - }); - }); - - test('create with a thinking level surfaces it in the list summary', async () => { - await withStore(async (store) => { - const header = await store.create( - makeInput({ name: 'Thinking from start', thinkingLevel: 'medium' }), - ); - assert.equal(header.thinkingLevel, 'medium'); - assert.equal((await store.list())[0]?.thinkingLevel, 'medium'); - }); - }); - - test('persists session branch lineage in header and summaries', async () => { - await withStore(async (store) => { - const header = await store.create( - makeInput({ - name: 'Branch', - parentSessionId: 'parent-session', - branchOfTurnId: 'turn-parent', - }), - ); - - assert.equal(header.parentSessionId, 'parent-session'); - assert.equal(header.branchOfTurnId, 'turn-parent'); - const [summary] = await store.list(); - assert.equal(summary?.parentSessionId, 'parent-session'); - assert.equal(summary?.branchOfTurnId, 'turn-parent'); - }); - }); - - test('persists typed subagent lineage but rejects relation scans without SQLite', async () => { - await withStore(async (store) => { - const parent = await store.create(makeInput({ name: 'Parent' })); - const child = await store.create( - makeInput({ - name: 'Child', - subagentParent: makeSubagentParent(parent.id), - }), - ); - await store.create( - makeInput({ - name: 'Other child', - subagentParent: makeSubagentParent('other-parent'), - }), - ); - - assert.deepEqual((await store.readHeader(child.id)).subagentParent, { - ...makeSubagentParent(parent.id), + ts: 10, + text: 'hello from SQLite', }); - assert.equal((await store.readHeader(child.id)).subagentRuntime, undefined); - await assert.rejects( - () => store.list({ subagentParentSessionId: parent.id }), - /require SQLite session metadata/, - ); - assert.equal('createSubagent' in store, false); - }); - }); - - test('keeps subagent lineage distinct from branch and revision lineage', async () => { - await withStore(async (store) => { - await assert.rejects( - () => - store.create( - makeInput({ - parentSessionId: 'branch-parent', - branchOfTurnId: 'branch-turn', - subagentParent: makeSubagentParent(), - }), - ), - /Invalid subagent session lineage/, - ); - await assert.rejects( - () => - store.create( - makeInput({ - revisionRootSessionId: 'root-session', - revisionParentSessionId: 'previous-version', - revisionOfTurnId: 'turn-edited', - revisionIndex: 2, - revisionState: 'preparing', - subagentParent: makeSubagentParent(), - }), - ), - /Invalid subagent session lineage/, - ); - }); - }); - - test('rejects malformed or mutated subagent lineage', async () => { - await withStore(async (store) => { - await assert.rejects( - () => - store.create( - makeInput({ - subagentParent: { - ...makeSubagentParent(), - spawnedBy: { - parentRunId: 'parent-run', - parentTurnId: 'parent-turn', - }, - } as SubagentSessionParent, - }), - ), - /Invalid subagent session lineage/, - ); - - const child = await store.create( - makeInput({ subagentParent: makeSubagentParent('parent-session') }), - ); - await assert.rejects( - () => store.updateHeader(child.id, { subagentParent: undefined }), - /parent relation is immutable/, - ); - await assert.rejects( - () => store.updateHeader(child.id, { subagentRuntime: undefined }), - /runtime snapshot is immutable/, - ); - await assert.rejects( - () => store.updateHeader(child.id, { subagentSpawn: undefined }), - /spawn identity is immutable/, - ); - await assert.rejects( - () => - store.create( - makeInput({ - permissionMode: 'execute', - subagentParent: makeSubagentParent(), - subagentRuntime: makeSubagentRuntime(), - }), - ), - /Invalid subagent session lineage/, - ); - await assert.rejects( - () => - store.create( - makeInput({ - subagentRuntime: makeSubagentRuntime(), - }), - ), - /Invalid subagent session lineage/, - ); - }); - }); - - test('persists edit-and-resend revision lineage separately from branch lineage', async () => { - await withStore(async (store) => { - const header = await store.create( - makeInput({ - name: 'Conversation version', - revisionRootSessionId: 'root-session', - revisionParentSessionId: 'previous-version', - revisionOfTurnId: 'turn-edited', - revisionIndex: 2, - revisionState: 'preparing', - }), - ); - - assert.equal(header.parentSessionId, undefined); - assert.equal(header.revisionRootSessionId, 'root-session'); - assert.equal(header.revisionParentSessionId, 'previous-version'); - assert.equal(header.revisionOfTurnId, 'turn-edited'); - assert.equal(header.revisionIndex, 2); - assert.equal(header.revisionState, 'preparing'); - const [summary] = await store.list(); - assert.equal(summary?.revisionRootSessionId, 'root-session'); - assert.equal(summary?.revisionParentSessionId, 'previous-version'); - assert.equal(summary?.revisionOfTurnId, 'turn-edited'); - assert.equal(summary?.revisionIndex, 2); - assert.equal(summary?.revisionState, 'preparing'); - }); - }); - - test('rejects partial or unsafe edit-and-resend revision lineage', async () => { - await withStore(async (store) => { - await assert.rejects( - () => store.create(makeInput({ revisionRootSessionId: 'root-only' })), - /Invalid session revision lineage/, - ); - await assert.rejects( - () => - store.create( - makeInput({ - revisionRootSessionId: '../escape', - revisionParentSessionId: 'parent', - revisionOfTurnId: 'turn', - revisionIndex: 2, - revisionState: 'preparing', - }), - ), - /Invalid session revision lineage/, - ); - }); - }); - - test('setFlagged toggles the flag without touching other fields', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Pin me' })); - - await store.setFlagged(header.id, true); - const pinned = await store.readHeader(header.id); - assert.equal(pinned.isFlagged, true); - assert.equal(pinned.name, 'Pin me'); - - await store.setFlagged(header.id, false); - const unpinned = await store.readHeader(header.id); - assert.equal(unpinned.isFlagged, false); - }); - }); - - test('markSessionReadThrough clears unread only through the current last message', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Unread' })); - await store.updateHeader(header.id, { hasUnread: true, lastMessageAt: 250 }); - - const unchanged = await store.markSessionReadThrough(header.id, 200); - assert.equal(unchanged.lastMessageAt, 250); - assert.equal(unchanged.hasUnread, true); - assert.equal((await store.readHeader(header.id)).hasUnread, true); - - const cleared = await store.markSessionReadThrough(header.id, 250); - assert.equal(cleared.lastMessageAt, 250); - assert.equal(cleared.hasUnread, false); - assert.equal((await store.readHeader(header.id)).hasUnread, false); - }); - }); - - test('markSessionReadThrough uses visible message timestamps when header lastMessageAt is stale', async () => { - for (const headerLastMessageAt of [100, undefined]) { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Stale unread' })); - await store.appendMessage(header.id, assistantMessageAt(250)); - await store.updateHeader(header.id, { - hasUnread: true, - lastMessageAt: headerLastMessageAt, - }); - - const unchanged = await store.markSessionReadThrough(header.id, 200); - assert.equal(unchanged.hasUnread, true); - assert.equal((await store.list())[0]?.lastMessageAt, 250); - assert.equal((await store.readHeader(header.id)).hasUnread, true); - const cleared = await store.markSessionReadThrough(header.id, 250); - assert.equal(cleared.hasUnread, false); - assert.equal((await store.readHeader(header.id)).hasUnread, false); - }); + assert.equal((await store.readMessages(session.id))[0]?.id, 'message-1'); + const page = await store.listCatalogPage(undefined, undefined, 10); + assert.equal(page.kind, 'page'); + if (page.kind !== 'page') assert.fail('expected a catalog page'); + assert.equal(page.records[0]?.summary.lastMessagePreview, 'hello from SQLite'); + } finally { + await store.close?.(); } - }); - - test('rename trims whitespace, rejects empty strings, and caps absurd lengths', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Old' })); - - await store.rename(header.id, ' Brand new name '); - const renamed = await store.readHeader(header.id); - assert.equal(renamed.name, 'Brand new name'); - assert.equal(renamed.titleIsManual, true); - await assert.rejects(store.rename(header.id, ' '), /name cannot be empty/); - - const overly = 'a'.repeat(200); - await store.rename(header.id, overly); - const bounded = await store.readHeader(header.id); - assert.equal(bounded.name.length, 80); - }); + const reopened = createSessionStore(root); + try { + const [session] = await reopened.listHeaders(); + assert.ok(session); + assert.equal((await reopened.readMessages(session.id))[0]?.id, 'message-1'); + } finally { + await reopened.close?.(); + await rm(root, { recursive: true, force: true }); + } }); - test('generated titles only replace untouched default titles', async () => { - await withStore(async (store) => { - const generatedFirst = await store.create(makeInput({ name: 'New Chat' })); - assert.equal(generatedFirst.titleIsManual, false); - assert.equal( - (await store.setGeneratedTitleIfAbsent(generatedFirst.id, ' Generated title '))?.name, - 'Generated title', + test('commits message and catalog projection atomically', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-session-atomic-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); + const metadata = createSqliteSessionMetadataStore( + join(root, OPERATIONAL_STATE_DATABASE_NAME), ); - await store.rename(generatedFirst.id, 'Manual title'); - assert.equal(await store.setGeneratedTitleIfAbsent(generatedFirst.id, 'Too late'), null); - assert.equal((await store.readHeader(generatedFirst.id)).name, 'Manual title'); - - const manualFirst = await store.create(makeInput({ name: 'New Chat' })); - await store.rename(manualFirst.id, 'Manual wins'); - assert.equal(await store.setGeneratedTitleIfAbsent(manualFirst.id, 'Generated loses'), null); - assert.equal((await store.readHeader(manualFirst.id)).name, 'Manual wins'); - - const unchanged = await store.create(makeInput({ name: 'New Chat' })); - assert.equal(await store.setGeneratedTitleIfAbsent(unchanged.id, ' New Chat '), null); - }); - }); - - test('retries a Windows atomic header replacement while a reader briefly holds the session file open', { - skip: process.platform !== 'win32', - }, async () => { - await withStore(async (store, workspaceRoot) => { - const header = await store.create(makeInput({ name: 'Reader overlap' })); - const sessionPath = join(workspaceRoot, 'sessions', header.id, 'session.jsonl'); - const reader = await open(sessionPath, 'r'); - const releaseReader = setTimeout(() => void reader.close(), 40); - try { - const updated = await store.updateHeader(header.id, { name: 'Write survived' }); - assert.equal(updated.name, 'Write survived'); - assert.equal((await store.readHeader(header.id)).name, 'Write survived'); + await store.appendMessage(session.id, { + type: 'assistant', + id: 'message-1', + turnId: 'turn-1', + ts: 20, + text: 'atomic preview', + modelId: 'fake-model', + }); + assert.equal((await metadata.readMessages(session.id))[0]?.id, 'message-1'); + assert.equal( + (await metadata.listCatalogPage({}, undefined, 10)).records[0]?.lastMessagePreview, + 'atomic preview', + ); } finally { - clearTimeout(releaseReader); - await reader.close().catch(() => {}); + metadata.close(); } - }); - }); - - test('remove deletes the session directory entirely', async () => { - await withStore(async (store, workspaceRoot) => { - const header = await store.create(makeInput({ name: 'Goodbye' })); - const sessionDir = join(workspaceRoot, 'sessions', header.id); - - // sanity: file exists before remove - const before = await readFile(join(sessionDir, 'session.jsonl'), 'utf8'); - assert.match(before, /Goodbye/); - - await store.remove(header.id); - - await assert.rejects(readFile(join(sessionDir, 'session.jsonl'), 'utf8')); - const remaining = await store.list(); - assert.equal( - remaining.find((s) => s.id === header.id), - undefined, - ); - }); - }); - - test('rejects traversal-style session ids before touching the filesystem', async () => { - await withStore(async (store, workspaceRoot) => { - const victim = join(workspaceRoot, 'outside-victim'); - await mkdir(victim, { recursive: true }); - await writeFile(join(victim, 'keep.txt'), 'keep', 'utf8'); - - await assert.rejects(store.readMessages('../outside-victim'), /Invalid session id/); - await assert.rejects(store.remove('../outside-victim'), /Invalid session id/); - - assert.equal(await readFile(join(victim, 'keep.txt'), 'utf8'), 'keep'); - }); - }); - - test('rejects malformed session headers instead of returning partial records', async () => { - await withStore(async (store, workspaceRoot) => { - const sessionId = 'malformed-header'; - const sessionDir = join(workspaceRoot, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - JSON.stringify({ - ...makeRawHeader({ id: sessionId, workspaceRoot, name: 'Broken labels' }), - labels: 'not-an-array', - }) + '\n', - 'utf8', - ); - - await assert.rejects( - () => store.readHeader(sessionId), - /Invalid session header for session malformed-header: malformed fields/, - ); - assert.deepEqual(await store.list(), []); - }); - }); - - test('rejects malformed session headers on write paths without overwriting bytes', async () => { - await withStore(async (store, workspaceRoot) => { - const sessionId = 'malformed-write'; - const sessionDir = join(workspaceRoot, 'sessions', sessionId); - const sessionPath = join(sessionDir, 'session.jsonl'); - const invalid = - JSON.stringify({ - ...makeRawHeader({ id: sessionId, workspaceRoot, name: 'Broken timestamp' }), - lastUsedAt: 'soon', - }) + '\n'; - await mkdir(sessionDir, { recursive: true }); - await writeFile(sessionPath, invalid, 'utf8'); - - await assert.rejects( - () => store.setFlagged(sessionId, true), - /Invalid session header for session malformed-write: malformed fields/, - ); - assert.equal(await readFile(sessionPath, 'utf8'), invalid); - }); - }); - - test('rejects session headers whose id does not match the directory', async () => { - await withStore(async (store, workspaceRoot) => { - const sessionId = 'header-id-mismatch'; - const sessionDir = join(workspaceRoot, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - JSON.stringify(makeRawHeader({ id: 'other-session', workspaceRoot })) + '\n', - 'utf8', - ); - - await assert.rejects( - () => store.readMessages(sessionId), - /Invalid session header for session header-id-mismatch: malformed fields/, - ); - assert.deepEqual(await store.list(), []); - }); - }); - - test('migrates legacy headers without permissionMode to ask', async () => { - await withStore(async (store, workspaceRoot) => { - const sessionId = 'legacy-session'; - const sessionDir = join(workspaceRoot, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - JSON.stringify({ - id: sessionId, - workspaceRoot, - cwd: '/tmp/cwd', - createdAt: 1, - lastUsedAt: 1, - name: 'Legacy', - isFlagged: false, - labels: [], - isArchived: false, - hasUnread: false, - backend: 'claude', - llmConnectionSlug: 'legacy', - connectionLocked: false, - model: 'legacy-model', - schemaVersion: 1, - }) + '\n', - 'utf8', - ); - - const header = await store.readHeader(sessionId); - assert.equal(header.backend, 'ai-sdk'); - assert.equal(header.permissionMode, 'ask'); - assert.equal(header.status, 'active'); - assert.equal(header.titleIsManual, true); - assert.equal(header.orchestrationMode, 'default'); - assert.equal(header.subagentParent, undefined); - const [summary] = await store.list(); - assert.equal(summary?.permissionMode, 'ask'); - assert.equal(summary?.status, 'active'); - assert.equal(summary?.orchestrationMode, 'default'); - assert.equal(summary?.subagentParent, undefined); - }); - }); - - test('migrates legacy default titles as generated-title candidates', async () => { - await withStore(async (store, workspaceRoot) => { - const sessionId = 'legacy-new-chat'; - const sessionDir = join(workspaceRoot, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - const legacy = makeRawHeader({ id: sessionId, workspaceRoot, name: 'New Chat' }); - delete (legacy as Partial).titleIsManual; - await writeFile(join(sessionDir, 'session.jsonl'), JSON.stringify(legacy) + '\n', 'utf8'); - - assert.equal((await store.readHeader(sessionId)).titleIsManual, false); - }); - }); - - test('migrates New Session as the canonical generated-title candidate', async () => { - await withStore(async (store, workspaceRoot) => { - const sessionId = 'legacy-new-session'; - const sessionDir = join(workspaceRoot, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - const legacy = makeRawHeader({ id: sessionId, workspaceRoot, name: 'New Session' }); - delete (legacy as Partial).titleIsManual; - await writeFile(join(sessionDir, 'session.jsonl'), JSON.stringify(legacy) + '\n', 'utf8'); - - const header = await store.readHeader(sessionId); - assert.equal(header.name, 'New Chat'); - assert.equal(header.titleIsManual, false); - }); - }); - - test('migrates legacy headers without model to default and exposes model in summaries', async () => { - await withStore(async (store, workspaceRoot) => { - const sessionId = 'legacy-no-model'; - const sessionDir = join(workspaceRoot, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - JSON.stringify({ - id: sessionId, - workspaceRoot, - cwd: '/tmp/cwd', - createdAt: 1, - lastUsedAt: 1, - name: 'Legacy no model', - isFlagged: false, - labels: [], - isArchived: false, - hasUnread: false, - backend: 'ai-sdk', - llmConnectionSlug: 'anthropic', - connectionLocked: false, - permissionMode: 'ask', - schemaVersion: 1, - }) + '\n', - 'utf8', - ); - - const header = await store.readHeader(sessionId); - assert.equal(header.model, 'default'); - const [summary] = await store.list(); - assert.equal(summary?.model, 'default'); - }); - }); - - test('migrates archived legacy headers to archived status', async () => { - await withStore(async (store, workspaceRoot) => { - const sessionId = 'legacy-archived'; - const sessionDir = join(workspaceRoot, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - JSON.stringify({ - id: sessionId, - workspaceRoot, - cwd: '/tmp/cwd', - createdAt: 1, - lastUsedAt: 2, - name: 'Legacy archived', - isFlagged: false, - labels: [], - isArchived: true, - archivedAt: 3, - hasUnread: false, - backend: 'fake', - llmConnectionSlug: 'fake', - connectionLocked: false, - model: 'fake-model', - permissionMode: 'ask', - schemaVersion: 1, - }) + '\n', - 'utf8', - ); - - const header = await store.readHeader(sessionId); - assert.equal(header.status, 'archived'); - assert.equal(header.statusUpdatedAt, 3); - }); - }); - - test('normalizes exact legacy shell tool results while reading session JSONL', async () => { - await withStore(async (store, workspaceRoot) => { - const header = await store.create(makeInput({ name: 'Legacy shell results' })); - const path = join(workspaceRoot, 'sessions', header.id, 'session.jsonl'); - const existing = await readFile(path, 'utf8'); - const legacyResults = [ - { - type: 'tool_result', - id: 'terminal-result', - turnId: 'turn-1', - ts: 2, - toolUseId: 'terminal-call', - isError: false, - content: { - kind: 'terminal', - cwd: '/workspace', - cmd: 'printf ok', - status: 'completed', - exitCode: 0, - stdout: 'ok', - stderr: '', - stdoutTruncated: false, - stderrTruncated: false, - }, - }, - { - type: 'tool_result', - id: 'shell-result', - turnId: 'turn-2', - ts: 4, - toolUseId: 'shell-call', - isError: false, - content: { - kind: 'shell_run', - ref: 'maka://runtime/background-tasks/shell-1', - status: 'cancelled', - cwd: '/workspace', - cmd: 'sleep 30', - startedAt: 1, - updatedAt: 4, - completedAt: 4, - exitCode: 130, - stdout: 'ready', - stderr: '', - latestOutputStream: 'stdout', - stdoutTruncated: false, - stderrTruncated: false, - observedAt: 4, - cancelled: true, - }, - }, - ]; - const mixedResult = { - type: 'tool_result', - id: 'mixed-terminal-result', - turnId: 'turn-3', - ts: 6, - toolUseId: 'mixed-terminal-call', - isError: false, - content: { - kind: 'terminal', - cwd: '/workspace', - cmd: 'printf bad', - status: 'completed', - exitCode: 0, - stdout: 'bad', - stderr: '', - stdoutTruncated: false, - stderrTruncated: false, - output: { - mode: 'pipes', - stdout: 'bad', - stderr: '', - stdoutTruncated: false, - stderrTruncated: false, - redacted: false, - }, - }, - }; - await writeFile( - path, - existing + - [...legacyResults, mixedResult].map((message) => JSON.stringify(message)).join('\n') + - '\n', - 'utf8', - ); - - const messages = await store.readMessages(header.id); - const terminal = messages.find((message) => message.id === 'terminal-result'); - const shellRun = messages.find((message) => message.id === 'shell-result'); - assert.deepEqual(terminal?.type === 'tool_result' ? terminal.content : undefined, { - kind: 'terminal', - cwd: '/workspace', - cmd: 'printf ok', - status: 'completed', - exitCode: 0, - output: { - mode: 'pipes', - stdout: 'ok', - stderr: '', - stdoutTruncated: false, - stderrTruncated: false, - redacted: false, - }, - }); - assert.deepEqual(shellRun?.type === 'tool_result' ? shellRun.content : undefined, { - kind: 'shell_run', - ref: 'maka://runtime/background-tasks/shell-1', - mode: 'pipes', - status: 'cancelled', - cwd: '/workspace', - cmd: 'sleep 30', - startedAt: 1, - updatedAt: 4, - completedAt: 4, - exitCode: 130, - revision: 1, - output: { - mode: 'pipes', - stdout: 'ready', - stderr: '', - latestStream: 'stdout', - stdoutTruncated: false, - stderrTruncated: false, - redacted: false, - }, - operation: { kind: 'stop', applied: true }, - }); - assert.equal( - messages.some((message) => message.id === 'mixed-terminal-result'), - false, - ); - assert.equal( - messages.some( - (message) => message.type === 'system_note' && message.id.startsWith('jsonl-corrupt-'), - ), - true, - ); - await assert.rejects( - () => store.readMessagesForRecovery(header.id), - /Session .* has a corrupt JSONL record at line 2/, - ); - }); - }); - - test('reads exact legacy subagent results during normal and strict recovery without rewriting', async () => { - const workspaceRoot = await mkdtemp(join(tmpdir(), 'maka-legacy-subagent-result-')); - const store = createSqliteSessionStore(workspaceRoot); - try { - const header = await store.create(makeInput({ name: 'Legacy subagent result' })); - const path = join(workspaceRoot, 'sessions', header.id, 'session.jsonl'); - const legacyRecord = await readFile( - new URL( - '../../src/__tests__/fixtures/legacy-subagent-waiting-permission.jsonl', - import.meta.url, - ), - ); - const originalBytes = Buffer.concat([await readFile(path), legacyRecord]); - await writeFile(path, originalBytes); - - const normalMessages = await store.readMessages(header.id); - assert.equal(subagentStatus(normalMessages), 'waiting_for_user'); - assert.deepEqual(await readFile(path), originalBytes); - - const recoveryMessages = await store.readMessagesForRecovery(header.id); - assert.equal(subagentStatus(recoveryMessages), 'waiting_for_user'); - assert.deepEqual(await readFile(path), originalBytes); - - assert.deepEqual( - (await store.listForRecovery()).map((session) => session.id), - [header.id], - ); - assert.deepEqual(await readFile(path), originalBytes); } finally { await store.close?.(); - await rm(workspaceRoot, { recursive: true, force: true }); + await rm(root, { recursive: true, force: true }); } }); - test('recovers readable messages around a corrupt JSONL message line', async () => { - await withStore(async (store, workspaceRoot) => { - const sessionId = 'corrupt-middle-line'; - const sessionDir = join(workspaceRoot, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - [ - JSON.stringify(makeRawHeader({ id: sessionId, workspaceRoot, name: 'Corrupt middle' })), - JSON.stringify({ type: 'user', id: 'u1', turnId: 't1', ts: 2, text: 'hello' }), - '{"type":"assistant","id":"broken"', - JSON.stringify({ - type: 'assistant', - id: 'a1', - turnId: 't1', - ts: 4, - text: 'recovered answer', - modelId: 'fake', - }), - '', - ].join('\n'), - 'utf8', - ); - - const messages = await store.readMessages(sessionId); - assert.equal(messages.length, 3); - assert.equal(messages[0]?.type, 'user'); - const note = messages[1]; - assert.equal(note?.type, 'system_note'); - if (note?.type !== 'system_note') throw new Error('corruption note missing'); - assert.equal(note.kind, 'error'); - assert.equal((note.data as { code?: unknown }).code, 'jsonl_parse_error'); - assert.equal((note.data as { lineNumber?: unknown }).lineNumber, 3); - assert.equal(typeof (note.data as { message?: unknown }).message, 'string'); - assert.ok(((note.data as { message?: string }).message ?? '').length > 0); - assert.equal(messages[2]?.type, 'assistant'); - - const [summary] = await store.list(); - assert.equal(summary?.id, sessionId); - assert.equal(summary?.lastMessagePreview, 'recovered answer'); - }); - }); - - test('silently drops a truncated tail JSONL message line', async () => { - await withStore(async (store, workspaceRoot) => { - const sessionId = 'truncated-tail-line'; - const sessionDir = join(workspaceRoot, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - [ - JSON.stringify(makeRawHeader({ id: sessionId, workspaceRoot, name: 'Truncated tail' })), - JSON.stringify({ type: 'user', id: 'u1', turnId: 't1', ts: 2, text: 'survives' }), - '{"type":"assistant","id":"partial"', - ].join('\n'), - 'utf8', - ); - - const messages = await store.readMessages(sessionId); - assert.deepEqual( - messages.map((message) => message.type), - ['user'], - ); - - const [summary] = await store.list(); - assert.equal(summary?.id, sessionId); - assert.equal(summary?.lastMessagePreview, 'survives'); - }); - }); - - test('reports an invalid unterminated tail instead of treating it as a crash prefix', async () => { - await withStore(async (store, workspaceRoot) => { - const sessionId = 'invalid-unterminated-tail'; - const sessionDir = join(workspaceRoot, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - const path = join(sessionDir, 'session.jsonl'); - const bytes = [ - JSON.stringify( - makeRawHeader({ - id: sessionId, - workspaceRoot, - name: 'Invalid tail', - connectionLocked: true, - }), - ), - JSON.stringify({ - type: 'user', - id: 'u1', - turnId: 't1', - ts: 2, - text: 'survives', - }), - '{"type":]', - ].join('\n'); - await writeFile(path, bytes, 'utf8'); - - const messages = await store.readMessages(sessionId); - assert.deepEqual( - messages.map((message) => message.type), - ['user', 'system_note'], - ); - await assert.rejects( - () => store.readMessagesForRecovery(sessionId), - /Session invalid-unterminated-tail has a corrupt JSONL record at line 3/, - ); - assert.equal(await readFile(path, 'utf8'), bytes); - }); - }); - - test('reports a corrupt tail JSONL message line when it was newline-terminated', async () => { - await withStore(async (store, workspaceRoot) => { - const sessionId = 'corrupt-terminated-tail-line'; - const sessionDir = join(workspaceRoot, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - [ - JSON.stringify( - makeRawHeader({ id: sessionId, workspaceRoot, name: 'Corrupt terminated tail' }), - ), - JSON.stringify({ type: 'user', id: 'u1', turnId: 't1', ts: 2, text: 'survives' }), - '{"type":"assistant","id":"durably-broken"', - '', - ].join('\n'), - 'utf8', - ); - - const messages = await store.readMessages(sessionId); - assert.equal(messages.length, 2); - assert.equal(messages[0]?.type, 'user'); - const note = messages[1]; - assert.equal(note?.type, 'system_note'); - if (note?.type !== 'system_note') throw new Error('corruption note missing'); - assert.equal(note.kind, 'error'); - assert.equal((note.data as { code?: unknown }).code, 'jsonl_parse_error'); - assert.equal((note.data as { lineNumber?: unknown }).lineNumber, 3); - }); - }); - - test('rejects a complete schema-invalid message during strict recovery', async () => { - await withStore(async (store, workspaceRoot) => { - const sessionId = 'schema-invalid-message'; - const sessionDir = join(workspaceRoot, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - [ - JSON.stringify( - makeRawHeader({ - id: sessionId, - workspaceRoot, - name: 'Invalid message', - }), - ), - JSON.stringify({}), - ].join('\n'), - 'utf8', - ); - - await assert.rejects( - () => store.readMessagesForRecovery(sessionId), - /Session schema-invalid-message has a corrupt JSONL record at line 2/, - ); - }); - }); - - test('recovers a canonical token usage message with nested diagnostics', async () => { - await withStore(async (store) => { - const session = await store.create(makeInput({ name: 'Token usage recovery' })); + test('deletes metadata and messages through the same transaction boundary', async () => { + const root = await mkdtemp(join(tmpdir(), 'maka-session-delete-')); + const store = createSessionStore(root); + try { + const session = await store.create(makeInput()); await store.appendMessage(session.id, { - type: 'token_usage', - id: 'usage-1', - turnId: 'turn-1', - ts: 10, - input: 100, - output: 20, - cacheHitInput: 80, - prefixChangeReason: 'stable', - promptSegments: [{ kind: 'prior_history', chars: 400, estimatedTokens: 100 }], - contextBudget: { - enabled: true, - policyName: 'bounded-history', - estimatedTokensBefore: 120, - estimatedTokensAfter: 100, - keptTurns: 4, - droppedTurns: 1, - keptEvents: 8, - droppedEvents: 2, - semanticCompactMode: 'validate_only', - compactionDecisions: [ - { - stage: 'priorReplay', - sourceKind: 'runtimeEvents', - decision: 'unchanged', - coveredTurns: 4, - }, - ], - }, - }); - - const messages = await store.readMessagesForRecovery(session.id); - assert.equal(messages[0]?.type, 'token_usage'); - assert.equal( - messages[0]?.type === 'token_usage' && messages[0].contextBudget?.policyName, - 'bounded-history', - ); - }); - }); - - test('rejects malformed nested message payloads during strict recovery', async () => { - await withStore(async (store, workspaceRoot) => { - const malformed = [ - { - type: 'token_usage', - id: 'usage-1', - turnId: 'turn-1', - ts: 1, - input: 1, - output: 1, - contextBudget: { - enabled: true, - policyName: 42, - estimatedTokensBefore: 1, - estimatedTokensAfter: 1, - keptTurns: 1, - droppedTurns: 0, - keptEvents: 1, - droppedEvents: 0, - }, - }, - exploreToolResult({ candidateFiles: [null], matches: [] }), - exploreToolResult({ candidateFiles: [], matches: [42] }), - ]; - - for (const [index, message] of malformed.entries()) { - const sessionId = `malformed-nested-${index}`; - const sessionDir = join(workspaceRoot, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - [ - JSON.stringify( - makeRawHeader({ - id: sessionId, - workspaceRoot, - name: 'Malformed nested', - }), - ), - JSON.stringify(message), - ].join('\n'), - 'utf8', - ); - await assert.rejects( - () => store.readMessagesForRecovery(sessionId), - new RegExp(`Session malformed-nested-${index} has a corrupt JSONL record at line 2`), - ); - } - }); - }); - - test('derives lastMessagePreview from visible user and assistant messages', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Preview' })); - - await store.appendMessages(header.id, [ - { - type: 'system_note', - id: 'sys-1', - ts: 1, - kind: 'mode_change', - data: { from: 'ask', to: 'execute' }, - }, - { - type: 'tool_call', - id: 'tool-1', - turnId: 't1', - ts: 2, - toolName: 'Read', - args: { file: 'secret.ts' }, - }, - { - type: 'assistant', - id: 'a1', - turnId: 't1', - ts: 3, - text: 'Here is the latest answer.\nIt spans lines.', - modelId: 'fake', - }, - ]); - - const [summary] = await store.list(); - assert.equal(summary?.lastMessagePreview, 'Here is the latest answer. It spans lines.'); - }); - }); - - test('lastMessagePreview skips internal-only tails, preserves emoji, and falls back for attachments', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Emoji' })); - const longText = `hello ${'🙂'.repeat(120)} tail`; - - await store.appendMessages(header.id, [ - { - type: 'user', - id: 'u1', - turnId: 't1', - ts: 1, - text: longText, - }, - { type: 'system_note', id: 'sys-1', turnId: 't1', ts: 2, kind: 'session_resume' }, - ]); - - const [summary] = await store.list(); - assert.equal(summary?.lastMessagePreview?.endsWith('…'), true); - assert.equal(summary?.lastMessagePreview?.includes('�'), false); - assert.equal(summary?.lastMessagePreview?.startsWith('hello 🙂'), true); - }); - - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Attachment' })); - - await store.appendMessage(header.id, { type: 'user', - id: 'u1', - turnId: 't1', - ts: 1, - text: ' ', - attachments: [ - { - kind: 'image', - name: 'shot.png', - mimeType: 'image/png', - bytes: 10, - ref: { kind: 'session_file', sessionId: header.id, relativePath: 'shot.png' }, - }, - ], - }); - - const [summary] = await store.list(); - assert.equal(summary?.lastMessagePreview, '附件'); - }); - }); - - test('summary lastMessageAt derives from visible messages when header timestamp is missing or stale', async () => { - await withStore(async (store, workspaceRoot) => { - const missingId = 'missing-last-message-at'; - await mkdir(join(workspaceRoot, 'sessions', missingId), { recursive: true }); - await writeFile( - join(workspaceRoot, 'sessions', missingId, 'session.jsonl'), - [ - JSON.stringify( - makeRawHeader({ id: missingId, workspaceRoot, name: 'Missing timestamp' }), - ), - JSON.stringify({ - type: 'user', - id: 'u1', - turnId: 't1', - ts: 20, - text: 'new visible user text', - }), - JSON.stringify({ type: 'system_note', id: 'sys-1', ts: 30, kind: 'session_resume' }), - '', - ].join('\n'), - 'utf8', - ); - - const staleId = 'stale-last-message-at'; - await mkdir(join(workspaceRoot, 'sessions', staleId), { recursive: true }); - await writeFile( - join(workspaceRoot, 'sessions', staleId, 'session.jsonl'), - [ - JSON.stringify( - makeRawHeader({ - id: staleId, - workspaceRoot, - name: 'Stale timestamp', - lastMessageAt: 5, - }), - ), - JSON.stringify({ - type: 'assistant', - id: 'a1', - turnId: 't1', - ts: 40, - text: 'new visible assistant text', - modelId: 'fake', - }), - '', - ].join('\n'), - 'utf8', - ); - - const summaries = await store.list(); - const missing = summaries.find((summary) => summary.id === missingId); - const stale = summaries.find((summary) => summary.id === staleId); - - assert.equal(missing?.lastMessageAt, 20); - assert.equal(missing?.lastMessagePreview, 'new visible user text'); - assert.equal(stale?.lastMessageAt, 40); - assert.equal(stale?.lastMessagePreview, 'new visible assistant text'); - assert.deepEqual( - summaries.slice(0, 2).map((summary) => summary.id), - [staleId, missingId], - ); - }); - }); - - test('list derives previews for sessions outside the first three without full detail reads', async () => { - await withStore(async (store, workspaceRoot) => { - for (let index = 0; index < 5; index += 1) { - const sessionId = `preview-tail-${index}`; - await mkdir(join(workspaceRoot, 'sessions', sessionId), { recursive: true }); - await writeFile( - join(workspaceRoot, 'sessions', sessionId, 'session.jsonl'), - [ - JSON.stringify( - makeRawHeader({ - id: sessionId, - workspaceRoot, - name: `Preview tail ${index}`, - lastMessageAt: 100 - index, - }), - ), - JSON.stringify({ - type: 'assistant', - id: `a-${index}`, - turnId: `t-${index}`, - ts: 100 - index, - text: `tail preview ${index}`, - modelId: 'fake', - }), - '', - ].join('\n'), - 'utf8', - ); - } - - const summaries = await store.list(); - - assert.equal(summaries.length, 5); - assert.deepEqual( - summaries.map((summary) => summary.lastMessagePreview), - ['tail preview 0', 'tail preview 1', 'tail preview 2', 'tail preview 3', 'tail preview 4'], - ); - }); - }); - - test('list accepts unusually large but valid session headers', async () => { - await withStore(async (store, workspaceRoot) => { - const sessionId = 'large-valid-header'; - await mkdir(join(workspaceRoot, 'sessions', sessionId), { recursive: true }); - await writeFile( - join(workspaceRoot, 'sessions', sessionId, 'session.jsonl'), - [ - JSON.stringify( - makeRawHeader({ - id: sessionId, - workspaceRoot, - name: 'Large header', - labels: Array.from({ length: 700 }, (_, index) => `label-${index}`), - lastMessageAt: 10, - }), - ), - JSON.stringify({ - type: 'assistant', - id: 'a1', - turnId: 't1', - ts: 10, - text: 'large header survives', - modelId: 'fake', - }), - '', - ].join('\n'), - 'utf8', - ); - - const [summary] = await store.list(); - - assert.equal(summary?.id, sessionId); - assert.equal(summary?.lastMessagePreview, 'large header survives'); - }); - }); - - test('summary lastMessageAt does not move backwards when copying older visible messages', async () => { - await withStore(async (store, workspaceRoot) => { - const sessionId = 'newer-header-with-old-copy'; - await mkdir(join(workspaceRoot, 'sessions', sessionId), { recursive: true }); - await writeFile( - join(workspaceRoot, 'sessions', sessionId, 'session.jsonl'), - [ - JSON.stringify( - makeRawHeader({ - id: sessionId, - workspaceRoot, - name: 'Newer header', - lastMessageAt: 100, - }), - ), - JSON.stringify({ - type: 'assistant', - id: 'a1', - turnId: 't1', - ts: 40, - text: 'old copied text', - modelId: 'fake', - }), - '', - ].join('\n'), - 'utf8', - ); - - const [summary] = await store.list(); - - assert.equal(summary?.id, sessionId); - assert.equal(summary?.lastMessageAt, 100); - assert.equal(summary?.lastMessagePreview, 'old copied text'); - }); - }); - - test('listTurns derives latest persisted turn states and lineage', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Turns' })); - - await store.appendMessages(header.id, [ - { type: 'user', id: 'u1', turnId: 't1', ts: 1, text: 'hello' }, - { - type: 'turn_state', - id: 'state-1', - turnId: 't1', - ts: 2, - status: 'running', - partialOutputRetained: false, - }, - { type: 'assistant', id: 'a1', turnId: 't1', ts: 3, text: 'partial', modelId: 'fake' }, - { - type: 'turn_state', - id: 'state-2', - turnId: 't1', - ts: 4, - status: 'aborted', - retriedFromTurnId: 't0', - abortedAt: 4, - partialOutputRetained: false, - }, - ]); - - assert.deepEqual(await store.listTurns(header.id), [ - { - turnId: 't1', - status: 'aborted', - statusSource: 'recorded', - retriedFromTurnId: 't0', - abortedAt: 4, - partialOutputRetained: true, - }, - ]); - }); - }); - - test('listTurns projects legacy message-only turns as completed', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Legacy turn' })); - await store.appendMessages(header.id, [ - { type: 'user', id: 'u1', turnId: 'legacy', ts: 1, text: 'hello' }, - { type: 'assistant', id: 'a1', turnId: 'legacy', ts: 2, text: 'world', modelId: 'fake' }, - ]); - - const turns = await store.listTurns(header.id); - assert.equal(turns[0]?.turnId, 'legacy'); - assert.equal(turns[0]?.status, 'completed'); - assert.equal(turns[0]?.partialOutputRetained, true); - }); - }); - - // PR-UI-IPC-2 (@kenji msg 0474c3fe + @xuan msg 88d96a87): - // session-name normalize contract is enforced at the store - // boundary by `normalizeUserSessionName`. These integration - // tests verify that the create + rename + (derived) branch - // paths all converge on the same chokepoint — locking @xuan's - // merge-gate criterion "all write entry points use same helper". - describe('normalizeUserSessionName store-boundary integration (PR-UI-IPC-2)', () => { - test('create with control chars in name → store persists sanitized name', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'multi\nline\tname' })); - const persisted = await store.readHeader(header.id); - assert.equal(persisted.name, 'multi line name'); - }); - }); - - test('create with bidi RLO spoof → spoof char replaced before persistence', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'safe‮evil' })); - const persisted = await store.readHeader(header.id); - assert.ok(!persisted.name.includes('‮'), 'RLO must be stripped at store boundary'); - assert.equal(persisted.name, 'safe evil'); - }); - }); - - test('create with zero-width injection ("ad\\u200Bmin") → ZWSP removed', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'ad​min' })); - const persisted = await store.readHeader(header.id); - assert.equal(persisted.name, 'admin'); - }); - }); - - test('create with undefined name → uses canonical "New Chat" default', async () => { - await withStore(async (store) => { - const input = makeInput(); - delete (input as Partial).name; - const header = await store.create(input); - const persisted = await store.readHeader(header.id); - assert.equal(persisted.name, 'New Chat'); - }); - }); - - test('create with explicit empty string name → REJECT (no silent default fallback)', async () => { - // Per @xuan caller-semantics lock: empty-after-sanitize on - // an EXPLICIT input must reject, not silently use the - // default. Default is reserved for the truly omitted - // (undefined) case. - await withStore(async (store) => { - await assert.rejects(store.create(makeInput({ name: '' })), /cannot be empty/); - await assert.rejects(store.create(makeInput({ name: ' ' })), /cannot be empty/); - await assert.rejects(store.create(makeInput({ name: '\n\n' })), /cannot be empty/); - }); - }); - - test('rename with control chars → sanitized at store boundary (replaces v1 inline trim/cap)', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Old' })); - await store.rename(header.id, 'new\x00name\x1b[31mwith\x7fcontrols'); - const persisted = await store.readHeader(header.id); - assert.ok(!persisted.name.includes('\x00')); - assert.ok(!persisted.name.includes('\x1b')); - assert.ok(!persisted.name.includes('\x7f')); - // Each control replaced with single space, then collapsed: - assert.equal(persisted.name, 'new name [31mwith controls'); - }); - }); - - test('rename with non-string runtime type rejects (TS signature is not enough at IPC boundary)', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Valid' })); - // Intentionally cast around the TS signature to simulate an - // IPC payload that didn't honor the type contract. - await assert.rejects( - store.rename(header.id, null as unknown as string), - /must be a string/, - ); - await assert.rejects(store.rename(header.id, 42 as unknown as string), /must be a string/); - }); - }); - - test('rename with 100-char input → capped to 80 code points', async () => { - await withStore(async (store) => { - const header = await store.create(makeInput({ name: 'Old' })); - await store.rename(header.id, 'a'.repeat(100)); - const persisted = await store.readHeader(header.id); - assert.equal(Array.from(persisted.name).length, 80); + id: 'message-1', + turnId: 'turn-1', + ts: 30, + text: 'delete me', }); - }); - - test('create with emoji at the cap boundary → surrogate pair never cut in half', async () => { - // 79 ASCII + 1 emoji = 80 code points, 81 UTF-16 code units. - // Naive `.slice(0, 80)` would cut the emoji's high-surrogate - // and leave an invalid lone low-surrogate. The helper uses - // code-point iteration to prevent this. - await withStore(async (store) => { - const header = await store.create(makeInput({ name: `${'a'.repeat(79)}🦊` })); - const persisted = await store.readHeader(header.id); - assert.ok(persisted.name.endsWith('🦊'), 'emoji must be intact at cap boundary'); + await store.remove(session.id); + await assert.rejects(store.readHeaderSnapshot(session.id), (error) => { + assert.equal(isSessionNotFoundError(error), true); + return true; }); - }); - - test('branch derived name with control-char parent → sanitized', async () => { - // Simulates the runtime branch path: derived name is - // `${parent} · 分支`. If parent.name has somehow accumulated - // dirty bytes (legacy session, manual file edit), the - // derived name passed to `store.create` still goes through - // the same normalize gate. - await withStore(async (store) => { - const dirtyParent = 'parent\nwith\ttabs'; - // Simulate runtime's `name: input.name ?? '${header.name} · 分支'` - const derived = `${dirtyParent} · 分支`; - const branchHeader = await store.create(makeInput({ name: derived })); - const persisted = await store.readHeader(branchHeader.id); - assert.ok(!persisted.name.includes('\n'), 'newline in derived must be sanitized'); - assert.ok(!persisted.name.includes('\t'), 'tab in derived must be sanitized'); - assert.equal(persisted.name, 'parent with tabs · 分支'); + await assert.rejects(store.readMessages(session.id), (error) => { + assert.equal(isSessionNotFoundError(error), true); + return true; }); - }); + } finally { + await store.close?.(); + await rm(root, { recursive: true, force: true }); + } }); }); @@ -1615,123 +113,3 @@ function makeInput(overrides: Partial = {}): CreateSessionIn ...overrides, }; } - -function makeSubagentParent(parentSessionId = 'parent-session'): SubagentSessionParent { - return { - kind: 'subagent', - parentSessionId, - spawnedBy: { - parentRunId: 'parent-run', - parentTurnId: 'parent-turn', - toolCallId: 'tool-call', - }, - lifecycle: 'foreground', - }; -} - -function makeSubagentRuntime(): SubagentSessionRuntime { - return { - schemaVersion: 1, - definitionVersion: 1, - agentId: 'local-read', - agentName: 'Local Read', - profile: 'local_read', - systemPrompt: 'Read the assigned workspace task.', - toolNames: ['Read', 'Glob', 'Grep'], - categoryPolicy: { read: 'allow' }, - permissionCeiling: 'ask', - }; -} - -function makeSubagentSpawn(): SubagentSessionSpawn { - return { - schemaVersion: 1, - requestFingerprint: 'a'.repeat(64), - initialTurnId: 'child-turn', - initialRunId: 'child-run', - }; -} - -function makeRawHeader(overrides: Partial = {}): SessionHeader { - return { - id: 'raw-session', - workspaceRoot: '/tmp/workspace', - cwd: '/tmp/cwd', - createdAt: 1, - lastUsedAt: 1, - name: 'Raw session', - titleIsManual: true, - isFlagged: false, - labels: [], - isArchived: false, - status: 'active', - statusUpdatedAt: 1, - hasUnread: false, - backend: 'fake', - llmConnectionSlug: 'fake', - connectionLocked: false, - model: 'fake-model', - permissionMode: 'ask', - schemaVersion: 1, - ...overrides, - }; -} - -function exploreToolResult(overrides: { candidateFiles: unknown[]; matches: unknown[] }): unknown { - return { - type: 'tool_result', - id: 'tool-result-1', - turnId: 'turn-1', - ts: 1, - toolUseId: 'tool-1', - isError: false, - content: { - kind: 'explore_agent', - ok: true, - mode: 'read_only', - objective: 'inspect', - roots: ['/tmp'], - queries: ['needle'], - filesInspected: 1, - filesSkipped: 0, - bytesRead: 10, - progress: [], - candidateFiles: overrides.candidateFiles, - matches: overrides.matches, - notes: [], - }, - }; -} - -function assistantMessageAt(ts: number): StoredMessage { - return { - type: 'assistant', - id: `assistant-${ts}`, - turnId: `turn-${ts}`, - ts, - text: 'ok', - modelId: 'fake-model', - }; -} - -function subagentStatus(messages: StoredMessage[]): string | undefined { - const result = messages.find((message) => message.id === 'legacy-subagent-result'); - return result?.type === 'tool_result' && result.content.kind === 'subagent' - ? result.content.status - : undefined; -} - -async function withStore( - fn: (store: ReturnType, workspaceRoot: string) => Promise, -): Promise { - const workspaceRoot = await mkdtemp(join(tmpdir(), 'maka-session-store-')); - const store = createSessionStore(workspaceRoot); - try { - await fn(store, workspaceRoot); - } finally { - await rm(workspaceRoot, { recursive: true, force: true }); - } -} - -// Silence unused-import warnings (kept for type clarity). -type _Header = SessionHeader; diff --git a/packages/storage/src/__tests__/settings-store-usage.test.ts b/packages/storage/src/__tests__/settings-store-usage.test.ts index e9fa39763e..91a5414d22 100644 --- a/packages/storage/src/__tests__/settings-store-usage.test.ts +++ b/packages/storage/src/__tests__/settings-store-usage.test.ts @@ -1,47 +1,28 @@ import { strict as assert } from 'node:assert'; -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, it } from 'node:test'; -import type { SessionHeader, StoredMessage } from '@maka/core/session'; +import type { StoredMessage } from '@maka/core/session'; import { createSessionStore } from '../session-store.js'; import { createSettingsStore } from '../settings-store.js'; -function makeHeader(overrides: Partial = {}): SessionHeader { - return { - id: 'session-1', - workspaceRoot: '/tmp/maka-workspace', - cwd: '/tmp/maka-workspace', - createdAt: 1, - lastUsedAt: 1, - name: 'Usage fixture', - titleIsManual: true, - isFlagged: false, - labels: [], - isArchived: false, - status: 'active', - hasUnread: false, - backend: 'ai-sdk', - llmConnectionSlug: 'anthropic', - connectionLocked: true, - model: 'claude-sonnet-4', - permissionMode: 'ask', - schemaVersion: 1, - ...overrides, - }; -} - -async function seedSession( - workspaceRoot: string, - header: SessionHeader, - messages: StoredMessage[], -) { - const sessionDir = join(workspaceRoot, 'sessions', header.id); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - [header, ...messages].map((entry) => JSON.stringify(entry)).join('\n') + '\n', - ); +async function seedSession(workspaceRoot: string, messages: StoredMessage[]): Promise { + const sessions = createSessionStore(workspaceRoot); + try { + const header = await sessions.create({ + cwd: '/tmp/maka-workspace', + backend: 'ai-sdk', + llmConnectionSlug: 'anthropic', + model: 'claude-sonnet-4', + permissionMode: 'ask', + name: 'Usage fixture', + }); + await sessions.appendMessages(header.id, messages); + return header.id; + } finally { + await sessions.close?.(); + } } describe('SettingsStore.usageStats request logs', () => { @@ -89,7 +70,7 @@ describe('SettingsStore.usageStats request logs', () => { it('includes tool invocation rows without inflating model usage totals', async () => { const workspaceRoot = await mkdtemp(join(tmpdir(), 'maka-settings-usage-')); try { - await seedSession(workspaceRoot, makeHeader(), [ + const sessionId = await seedSession(workspaceRoot, [ { type: 'assistant', id: 'assistant-1', @@ -146,7 +127,7 @@ describe('SettingsStore.usageStats request logs', () => { const modelLog = stats.logs.find((log) => log.kind === 'model'); assert.ok(modelLog); - assert.equal(modelLog.sessionId, 'session-1'); + assert.equal(modelLog.sessionId, sessionId); assert.equal(modelLog.turnId, 'turn-1'); assert.equal(modelLog.model, 'claude-sonnet-4-runtime'); assert.equal(modelLog.inputTokens, 120); @@ -159,7 +140,7 @@ describe('SettingsStore.usageStats request logs', () => { const toolLog = stats.logs.find((log) => log.kind === 'tool'); assert.ok(toolLog); assert.equal(toolLog.id, 'tool:tool-1'); - assert.equal(toolLog.sessionId, 'session-1'); + assert.equal(toolLog.sessionId, sessionId); assert.equal(toolLog.turnId, 'turn-1'); assert.equal(toolLog.provider, 'anthropic'); assert.equal(toolLog.model, 'claude-sonnet-4'); @@ -201,7 +182,7 @@ describe('SettingsStore.usageStats request logs', () => { }, ]; - await seedSession(workspaceRoot, makeHeader({ id: 'session-a' }), [ + await seedSession(workspaceRoot, [ ...bashTurn('session-a', false, 20), { type: 'tool_call', @@ -223,9 +204,7 @@ describe('SettingsStore.usageStats request logs', () => { content: { kind: 'text', text: 'ok' }, }, ]); - await seedSession(workspaceRoot, makeHeader({ id: 'session-b' }), [ - ...bashTurn('session-b', true, 40), - ]); + await seedSession(workspaceRoot, [...bashTurn('session-b', true, 40)]); const stats = await createSettingsStore(workspaceRoot).usageStats('all'); @@ -252,88 +231,4 @@ describe('SettingsStore.usageStats request logs', () => { await rm(workspaceRoot, { recursive: true, force: true }); } }); - - it('keeps valid usage rows when one session message line is corrupt', async () => { - const workspaceRoot = await mkdtemp(join(tmpdir(), 'maka-settings-usage-corrupt-line-')); - try { - const header = makeHeader(); - const sessionDir = join(workspaceRoot, 'sessions', header.id); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - [ - JSON.stringify(header), - JSON.stringify({ - type: 'assistant', - id: 'assistant-1', - turnId: 'turn-1', - ts: 10, - text: 'tracked', - modelId: 'runtime-model', - }), - '{"type":"tool_call"', - JSON.stringify({ - type: 'token_usage', - id: 'usage-1', - turnId: 'turn-1', - ts: 20, - input: 10, - output: 5, - cacheRead: 2, - costUsd: 0.02, - }), - ].join('\n') + '\n', - ); - - const stats = await createSettingsStore(workspaceRoot).usageStats('all'); - - assert.equal(stats.summary.totalRequests, 1); - assert.equal(stats.summary.totalTokens, 15); - assert.equal(stats.summary.cacheRead, 2); - assert.equal(stats.summary.totalCostUsd, 0.02); - assert.equal(stats.logs[0]?.model, 'runtime-model'); - } finally { - await rm(workspaceRoot, { recursive: true, force: true }); - } - }); - - it('ignores malformed usage rows instead of poisoning totals', async () => { - const workspaceRoot = await mkdtemp(join(tmpdir(), 'maka-settings-usage-bad-token-row-')); - try { - const header = makeHeader(); - const sessionDir = join(workspaceRoot, 'sessions', header.id); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - [ - JSON.stringify(header), - JSON.stringify({ - type: 'token_usage', - id: 'bad-usage', - turnId: 'turn-1', - ts: 20, - input: '10', - output: 5, - }), - JSON.stringify({ - type: 'token_usage', - id: 'good-usage', - turnId: 'turn-2', - ts: 30, - input: 7, - output: 3, - }), - ].join('\n') + '\n', - ); - - const stats = await createSettingsStore(workspaceRoot).usageStats('all'); - - assert.equal(stats.summary.totalRequests, 1); - assert.equal(stats.summary.totalTokens, 10); - assert.equal(stats.logs.length, 1); - assert.equal(stats.logs[0]?.id, 'good-usage'); - } finally { - await rm(workspaceRoot, { recursive: true, force: true }); - } - }); }); diff --git a/packages/storage/src/__tests__/shell-run-store.test.ts b/packages/storage/src/__tests__/shell-run-store.test.ts deleted file mode 100644 index 998894532d..0000000000 --- a/packages/storage/src/__tests__/shell-run-store.test.ts +++ /dev/null @@ -1,509 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, it } from 'node:test'; -import { - SHELL_RUN_SOURCE_TOOL_CALL_ID_MAX_BYTES, - type ShellRunPatch, - type ShellRunRecord, -} from '@maka/core'; -import { createShellRunStore } from '../shell-run-store.js'; - -describe('ShellRunStore', () => { - it('enforces the canonical provider tool-call identity bound on create', async () => { - await withStore(async (store) => { - const maximum = '😀'.repeat(SHELL_RUN_SOURCE_TOOL_CALL_ID_MAX_BYTES / 4); - assert.equal(Buffer.byteLength(maximum, 'utf8'), SHELL_RUN_SOURCE_TOOL_CALL_ID_MAX_BYTES); - const created = await store.createShellRun( - record({ shellRunId: 'shell-max-id', sourceToolCallId: maximum }), - ); - assert.equal(created.sourceToolCallId, maximum); - - await assert.rejects( - () => - store.createShellRun( - record({ - shellRunId: 'shell-oversized-id', - sourceToolCallId: `${maximum}x`, - }), - ), - /Invalid ShellRun record for shell-oversized-id: malformed fields/, - ); - }); - }); - - it('creates, updates, reads, and lists ShellRuns under a session', async () => { - await withStore(async (store, root) => { - await store.createShellRun(record({ shellRunId: 'shell-2', startedAt: 2, updatedAt: 2 })); - await store.createShellRun(record({ shellRunId: 'shell-1', startedAt: 1, updatedAt: 1 })); - - const updated = await store.updateShellRun('session-1', 'shell-1', { - status: 'completed', - exitCode: 0, - output: pipeOutput({ stdout: 'done', latestStream: 'stdout' }), - completedAt: 10, - updatedAt: 10, - }); - - assert.equal(updated.status, 'completed'); - assert.equal(updated.output.mode === 'pipes' ? updated.output.stdout : '', 'done'); - assert.equal( - updated.output.mode === 'pipes' ? updated.output.latestStream : undefined, - 'stdout', - ); - assert.equal(updated.revision, 2); - assert.deepEqual( - (await store.listSessionShellRuns('session-1')).map((run) => run.shellRunId), - ['shell-1', 'shell-2'], - ); - assert.equal((await store.readShellRun('session-1', 'shell-1')).completedAt, 10); - assert.equal( - JSON.parse( - await readFile( - join(root, 'sessions', 'session-1', 'shell-runs', 'shell-1', 'shell-run.json'), - 'utf8', - ), - ).shellRunId, - 'shell-1', - ); - }); - }); - - it('rejects duplicate create without overwriting the existing record', async () => { - await withStore(async (store, root) => { - await store.createShellRun(record({ shellRunId: 'shell-1', command: 'first' })); - - await assert.rejects( - () => store.createShellRun(record({ shellRunId: 'shell-1', command: 'second' })), - /ShellRun already exists: shell-1/, - ); - - const raw = await readFile( - join(root, 'sessions', 'session-1', 'shell-runs', 'shell-1', 'shell-run.json'), - 'utf8', - ); - assert.equal(JSON.parse(raw).command, 'first'); - }); - }); - - it('preserves forward-only starting, running, and terminal lifecycle transitions', async () => { - await withStore(async (store) => { - await store.createShellRun(record({ shellRunId: 'shell-1', status: 'starting' })); - const running = await store.updateShellRun('session-1', 'shell-1', { - status: 'running', - updatedAt: 2, - }); - const completed = await store.updateShellRun('session-1', 'shell-1', { - status: 'completed', - exitCode: 0, - completedAt: 3, - updatedAt: 3, - }); - - assert.equal(running.revision, 2); - assert.equal(completed.revision, 3); - await assert.rejects( - () => - store.updateShellRun('session-1', 'shell-1', { - status: 'running', - completedAt: undefined, - exitCode: undefined, - updatedAt: 4, - }), - /Invalid ShellRun status transition: completed -> running/, - ); - }); - }); - - it('terminalizes failed or lost starts and keeps terminal outcomes immutable', async () => { - await withStore(async (store) => { - await store.createShellRun(record({ shellRunId: 'failed-launch', status: 'starting' })); - const failed = await store.updateShellRun('session-1', 'failed-launch', { - status: 'failed', - failureMessage: 'spawn failed', - completedAt: 2, - updatedAt: 2, - }); - assert.equal(failed.status, 'failed'); - - await store.createShellRun(record({ shellRunId: 'lost-launch', status: 'starting' })); - const orphaned = await store.updateShellRun('session-1', 'lost-launch', { - status: 'orphaned', - failureMessage: 'host restarted before launch outcome was known', - completedAt: 2, - updatedAt: 2, - }); - assert.equal(orphaned.status, 'orphaned'); - - await assert.rejects( - () => - store.updateShellRun('session-1', 'failed-launch', { - failureMessage: 'different failure', - updatedAt: 3, - }), - /ShellRun terminal outcome is immutable: failed/, - ); - }); - }); - - it('round-trips sandbox execution and one-shot escalation audit facts', async () => { - await withStore(async (store) => { - const created = await store.createShellRun({ - ...record({ shellRunId: 'shell-escalated' }), - sandboxExecution: { type: 'none', enforced: false }, - sandboxEscalation: { commandHash: 'command-hash', unsandboxed: true }, - }); - - assert.deepEqual(created.sandboxExecution, { type: 'none', enforced: false }); - assert.deepEqual(created.sandboxEscalation, { - commandHash: 'command-hash', - unsandboxed: true, - }); - assert.deepEqual( - (await store.readShellRun('session-1', 'shell-escalated')).sandboxEscalation, - created.sandboxEscalation, - ); - }); - }); - - it('rejects inconsistent sandbox execution audit facts', async () => { - await withStore(async (store) => { - await assert.rejects( - () => - store.createShellRun({ - ...record({ shellRunId: 'shell-invalid-enforcement' }), - sandboxExecution: { type: 'macos-seatbelt', enforced: false }, - }), - /malformed fields/, - ); - await assert.rejects( - () => - store.createShellRun({ - ...record({ shellRunId: 'shell-invalid-escalation' }), - sandboxExecution: { type: 'macos-seatbelt', enforced: true }, - sandboxEscalation: { commandHash: 'command-hash', unsandboxed: true }, - }), - /malformed fields/, - ); - }); - }); - - it('increments revision only when durable state changes', async () => { - await withStore(async (store) => { - await store.createShellRun(record({ shellRunId: 'shell-1' })); - - const unchanged = await store.updateShellRun('session-1', 'shell-1', { - exitCode: undefined, - failureMessage: undefined, - }); - const changed = await store.updateShellRun('session-1', 'shell-1', { - output: pipeOutput({ stdout: 'next' }), - updatedAt: 2, - }); - - assert.equal(unchanged.revision, 1); - assert.equal(changed.revision, 2); - }); - }); - - it('records only the first terminal observation under concurrent updates', async () => { - await withStore(async (store) => { - await store.createShellRun( - record({ - shellRunId: 'shell-1', - status: 'completed', - completedAt: 2, - exitCode: 0, - }), - ); - - const [first, second] = await Promise.all([ - store.updateShellRun('session-1', 'shell-1', { observedAt: 10 }), - store.updateShellRun('session-1', 'shell-1', { observedAt: 20 }), - ]); - - assert.equal(first.observedAt, 10); - assert.equal(second.observedAt, 10); - assert.equal(second.revision, 2); - assert.equal((await store.readShellRun('session-1', 'shell-1')).revision, 2); - }); - }); - - it('keeps launch identity and output mode immutable', async () => { - await withStore(async (store) => { - await store.createShellRun(record({ shellRunId: 'shell-1' })); - - await assert.rejects( - () => - store.updateShellRun('session-1', 'shell-1', { - command: 'replacement', - } as unknown as ShellRunPatch), - /ShellRun field is immutable: command/, - ); - await assert.rejects( - () => - store.updateShellRun('session-1', 'shell-1', { - output: { - mode: 'pty', - screen: '', - scrollback: '', - cols: 80, - rows: 24, - cursor: { x: 0, y: 0, visible: true }, - alternateScreen: false, - truncated: false, - redacted: false, - }, - }), - /ShellRun output mode is immutable: pipes/, - ); - await assert.rejects( - () => - store.createShellRun({ - ...record({ shellRunId: 'shell-with-operation' }), - operation: { kind: 'stop', applied: true }, - } as unknown as ShellRunRecord), - /malformed fields/, - ); - }); - }); - - it('rejects malformed records and ignores malformed folders while listing', async () => { - await withStore(async (store, root) => { - await store.createShellRun(record({ shellRunId: 'shell-good' })); - const badPath = join( - root, - 'sessions', - 'session-1', - 'shell-runs', - 'shell-bad', - 'shell-run.json', - ); - await mkdir(join(root, 'sessions', 'session-1', 'shell-runs', 'shell-bad'), { - recursive: true, - }); - await writeFile( - badPath, - JSON.stringify({ - shellRunId: 'shell-bad', - sessionId: 'session-1', - status: 'mystery', - }) + '\n', - 'utf8', - ); - - await assert.rejects( - () => store.readShellRun('session-1', 'shell-bad'), - /Invalid ShellRun record for shell-bad: malformed fields/, - ); - assert.deepEqual( - (await store.listSessionShellRuns('session-1')).map((run) => run.shellRunId), - ['shell-good'], - ); - }); - }); - - it('normalizes an exact legacy ShellRun record and writes only the current shape on update', async () => { - await withStore(async (store, root) => { - const dir = join(root, 'sessions', 'session-1', 'shell-runs', 'shell-legacy'); - const path = join(dir, 'shell-run.json'); - await mkdir(dir, { recursive: true }); - await writeFile( - path, - JSON.stringify({ - shellRunId: 'shell-legacy', - sessionId: 'session-1', - sourceRunId: 'run-1', - sourceTurnId: 'turn-1', - sourceToolCallId: 'tool-1', - cwd: '/workspace', - command: 'printf ready; sleep 30', - status: 'running', - startedAt: 1, - updatedAt: 1, - timeoutMs: 30_000, - stdoutTail: 'ready', - stderrTail: '', - latestOutputStream: 'stdout', - stdoutTruncated: false, - stderrTruncated: false, - pid: 123, - }) + '\n', - 'utf8', - ); - - const restored = await store.readShellRun('session-1', 'shell-legacy'); - assert.equal(restored.revision, 1); - assert.deepEqual(restored.output, { - mode: 'pipes', - stdout: 'ready', - stderr: '', - latestStream: 'stdout', - stdoutTruncated: false, - stderrTruncated: false, - redacted: false, - }); - - await store.updateShellRun('session-1', 'shell-legacy', { - updatedAt: 2, - output: pipeOutput({ stdout: 'ready\nnext', latestStream: 'stdout' }), - }); - const written = JSON.parse(await readFile(path, 'utf8')) as Record; - assert.equal(written.revision, 2); - assert.equal(Object.hasOwn(written, 'stdoutTail'), false); - assert.equal(Object.hasOwn(written, 'pid'), false); - assert.deepEqual(written.output, { - mode: 'pipes', - stdout: 'ready\nnext', - stderr: '', - latestStream: 'stdout', - stdoutTruncated: false, - stderrTruncated: false, - redacted: false, - }); - }); - }); - - it('rejects legacy ShellRun records that violate the preceding state invariants', async () => { - await withStore(async (store, root) => { - const cases = [ - { - shellRunId: 'legacy-completed-with-orphan-reason', - status: 'completed', - completedAt: 2, - exitCode: 0, - orphanedReason: 'contradictory', - }, - { - shellRunId: 'legacy-failed-without-exit', - status: 'failed', - completedAt: 2, - failureMessage: 'old store required a non-zero exit code', - }, - ] as const; - for (const invalid of cases) { - const dir = join(root, 'sessions', 'session-1', 'shell-runs', invalid.shellRunId); - await mkdir(dir, { recursive: true }); - await writeFile( - join(dir, 'shell-run.json'), - JSON.stringify({ - sessionId: 'session-1', - sourceRunId: 'run-1', - sourceTurnId: 'turn-1', - sourceToolCallId: 'tool-1', - cwd: '/workspace', - command: 'printf ready', - ...invalid, - startedAt: 1, - updatedAt: 2, - stdoutTail: 'ready', - stderrTail: '', - latestOutputStream: 'stdout', - stdoutTruncated: false, - stderrTruncated: false, - }) + '\n', - 'utf8', - ); - await assert.rejects( - () => store.readShellRun('session-1', invalid.shellRunId), - /Invalid ShellRun record/, - ); - } - }); - }); - - it('rejects inconsistent ShellRun state fields', async () => { - await withStore(async (store) => { - await assert.rejects( - () => store.createShellRun(record({ shellRunId: 'bad-completed', status: 'completed' })), - /inconsistent state fields/, - ); - - await store.createShellRun(record({ shellRunId: 'running-1' })); - await assert.rejects( - () => - store.updateShellRun('session-1', 'running-1', { - exitCode: 0, - updatedAt: 2, - }), - /inconsistent state fields/, - ); - await assert.rejects( - () => - store.updateShellRun('session-1', 'running-1', { - status: 'completed', - updatedAt: 3, - }), - /inconsistent state fields/, - ); - }); - }); - - it('rejects unsafe session and shell run ids', async () => { - await withStore(async (store) => { - await assert.rejects( - () => store.createShellRun(record({ sessionId: '../outside', shellRunId: 'shell-1' })), - /Invalid session id/, - ); - await assert.rejects( - () => store.createShellRun(record({ sessionId: 'session-1', shellRunId: '../outside' })), - /Invalid shell run id/, - ); - }); - }); -}); - -async function withStore( - fn: (store: ReturnType, root: string) => Promise, -): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-shell-run-store-')); - try { - await fn(createShellRunStore(root), root); - } finally { - await rm(root, { recursive: true, force: true }); - } -} - -function record(input: { - sessionId?: string; - shellRunId: string; - command?: string; - status?: ShellRunRecord['status']; - startedAt?: number; - updatedAt?: number; - completedAt?: number; - exitCode?: number; - sourceToolCallId?: string; -}): ShellRunRecord { - return { - shellRunId: input.shellRunId, - sessionId: input.sessionId ?? 'session-1', - sourceRunId: 'run-1', - sourceTurnId: 'turn-1', - sourceToolCallId: input.sourceToolCallId ?? 'tool-1', - cwd: '/workspace', - command: input.command ?? 'printf "ok"', - status: input.status ?? 'running', - startedAt: input.startedAt ?? 1, - updatedAt: input.updatedAt ?? 1, - ...(input.completedAt !== undefined ? { completedAt: input.completedAt } : {}), - ...(input.exitCode !== undefined ? { exitCode: input.exitCode } : {}), - revision: 1, - output: pipeOutput(), - }; -} - -function pipeOutput( - input: { stdout?: string; stderr?: string; latestStream?: 'stdout' | 'stderr' } = {}, -): Extract { - return { - mode: 'pipes', - stdout: input.stdout ?? '', - stderr: input.stderr ?? '', - ...(input.latestStream ? { latestStream: input.latestStream } : {}), - stdoutTruncated: false, - stderrTruncated: false, - redacted: false, - }; -} diff --git a/packages/storage/src/__tests__/sqlite-artifact-store.test.ts b/packages/storage/src/__tests__/sqlite-artifact-store.test.ts deleted file mode 100644 index 48888afe9e..0000000000 --- a/packages/storage/src/__tests__/sqlite-artifact-store.test.ts +++ /dev/null @@ -1,292 +0,0 @@ -import assert from 'node:assert/strict'; -import { createHash } from 'node:crypto'; -import { link, mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { basename, dirname, join } from 'node:path'; -import { describe, test } from 'node:test'; -import type { ArtifactRecord } from '@maka/core/artifacts'; -import { - createSqliteArtifactStore, - createSqliteArtifactStoreWriteAuthority, -} from '../artifact-store.js'; -import * as artifactStoreModule from '../artifact-store.js'; -import { withArtifactWriterLock } from '../artifact-writer-lock.js'; - -describe('SQLite artifact metadata', () => { - test('does not expose the retired JSONL Artifact writer', () => { - assert.equal(Reflect.has(artifactStoreModule, 'createArtifactStore'), false); - assert.equal(Reflect.has(artifactStoreModule, 'createArtifactStoreWriteAuthority'), false); - }); - - test('cuts over legacy metadata once and makes SQLite canonical without rewriting JSONL', async () => { - await withWorkspace(async (root) => { - const migrated = await seedLegacyArtifact(root, artifactInput('migrated', 'legacy bytes', 1)); - const metadataPath = join(root, 'artifacts', 'metadata.jsonl'); - const legacyMetadata = await readFile(metadataPath); - - const store = createSqliteArtifactStore(root); - try { - assert.deepEqual(await store.get(migrated.id), migrated); - const created = await store.create(artifactInput('sqlite', 'canonical bytes', 2)); - await store.delete(migrated.id); - - assert.deepEqual(await readFile(metadataPath), legacyMetadata); - assert.ok((await stat(join(root, 'runtime.sqlite'))).isFile()); - assert.deepEqual(await store.readText(created.id), { - ok: true, - text: 'canonical bytes', - }); - } finally { - store.close?.(); - } - - const reopened = createSqliteArtifactStore(root); - try { - assert.equal((await reopened.get(migrated.id))?.status, 'deleted'); - assert.equal((await reopened.get('sqlite'))?.status, 'live'); - } finally { - reopened.close?.(); - } - }); - }); - - test('resumes an interrupted cutover atomically', async () => { - await withWorkspace(async (root) => { - const record = await seedLegacyArtifact(root, artifactInput('legacy', 'durable', 1)); - const interrupted = createSqliteArtifactStore(root, { - failpoint: (point) => { - if (point === 'after_cutover_rows_copied') { - throw new Error('simulated cutover crash'); - } - }, - }); - await assert.rejects(() => interrupted.list('session-1'), /simulated cutover crash/); - interrupted.close?.(); - - const recovered = createSqliteArtifactStore(root); - try { - assert.deepEqual(await recovered.list('session-1'), [record]); - assert.deepEqual(await recovered.readText(record.id), { ok: true, text: 'durable' }); - } finally { - recovered.close?.(); - } - }); - }); - - test('runs the first legacy cutover under the artifact writer lock', async () => { - await withWorkspace(async (root) => { - await seedLegacyArtifact(root, artifactInput('legacy', 'durable', 1)); - const held = await holdArtifactWriterLock(root); - const store = createSqliteArtifactStore(root); - let settled = false; - const opening = store.list('session-1').finally(() => { - settled = true; - }); - await new Promise((resolve) => setImmediate(resolve)); - assert.equal(settled, false); - - held.release(); - await held.finished; - assert.equal((await opening)[0]?.id, 'legacy'); - store.close?.(); - }); - }); - - test('fails closed when an old writer changes JSONL after cutover', async () => { - await withWorkspace(async (root) => { - await seedLegacyArtifact(root, artifactInput('legacy', 'durable', 1)); - const store = createSqliteArtifactStore(root); - await store.list('session-1'); - store.close?.(); - - await seedLegacyArtifact(root, artifactInput('old-writer', 'stale', 2)); - const reopened = createSqliteArtifactStore(root); - try { - await assert.rejects( - () => reopened.list('session-1'), - /Legacy artifact_metadata source changed after cutover completed/, - ); - } finally { - reopened.close?.(); - } - }); - }); - - test('recovery removes a payload whose SQLite metadata transaction did not commit', async () => { - await withWorkspace(async (root) => { - const residue = await createPublicationResidue( - root, - 'uncommitted', - 'uncommitted.txt', - 'not committed', - ); - const authority = createSqliteArtifactStoreWriteAuthority(root); - try { - await authority.recover(); - await assert.rejects(() => stat(residue.stagingPath), { code: 'ENOENT' }); - await assert.rejects(() => stat(residue.targetPath), { code: 'ENOENT' }); - assert.equal(await authority.store.get('uncommitted'), null); - } finally { - authority.close(); - } - }); - }); - - test('recovery keeps a committed payload and removes its publication staging file', async () => { - await withWorkspace(async (root) => { - const initial = createSqliteArtifactStore(root); - const record = await initial.create(artifactInput('committed', 'durable', 1)); - initial.close?.(); - - const targetPath = join(root, 'artifacts', record.relativePath); - const stagingPath = publicationStagingPath(targetPath); - await writeFile(stagingPath, await readFile(targetPath)); - - const authority = createSqliteArtifactStoreWriteAuthority(root); - try { - await authority.recover(); - await assert.rejects(() => stat(stagingPath), { code: 'ENOENT' }); - assert.equal(await readFile(targetPath, 'utf8'), 'durable'); - assert.deepEqual(await authority.store.readText(record.id), { - ok: true, - text: 'durable', - }); - } finally { - authority.close(); - } - }); - }); - - test('recovery completes a purge interrupted before the SQLite metadata commit', async () => { - await withWorkspace(async (root) => { - const authority = createSqliteArtifactStoreWriteAuthority(root); - try { - await authority.recover(); - const record = await authority.store.create(artifactInput('purge-retry', 'remove me', 1)); - const payloadPath = join(root, 'artifacts', record.relativePath); - const purgeIntentPath = join(root, 'artifacts', '.artifact-purge-intent.json'); - await rm(payloadPath); - await writeFile( - purgeIntentPath, - JSON.stringify({ schemaVersion: 1, artifactIds: [record.id] }), - 'utf8', - ); - - await authority.recover(); - assert.equal(await authority.store.get(record.id), null); - await assert.rejects(() => stat(purgeIntentPath), { code: 'ENOENT' }); - } finally { - authority.close(); - } - }); - }); - - test('does not create legacy metadata for a new SQLite-backed store', async () => { - await withWorkspace(async (root) => { - const store = createSqliteArtifactStore(root); - try { - const record = await store.create(artifactInput('sqlite-only', 'bytes', 1)); - assert.equal(await readFile(join(root, 'artifacts', record.relativePath), 'utf8'), 'bytes'); - await assert.rejects(() => stat(join(root, 'artifacts', 'metadata.jsonl')), { - code: 'ENOENT', - }); - } finally { - store.close?.(); - } - }); - }); -}); - -function artifactInput(id: string, content: string, now: number) { - return { - id, - sessionId: 'session-1', - turnId: 'turn-1', - name: `${id}.txt`, - kind: 'file' as const, - content, - source: 'fixture' as const, - now, - }; -} - -async function seedLegacyArtifact( - root: string, - input: ReturnType, -): Promise { - const relativePath = `${input.sessionId}/${input.id}-${input.name}`; - const payloadPath = join(root, 'artifacts', relativePath); - const metadataPath = join(root, 'artifacts', 'metadata.jsonl'); - await mkdir(dirname(payloadPath), { recursive: true }); - await writeFile(payloadPath, input.content); - const record: ArtifactRecord = { - id: input.id, - sessionId: input.sessionId, - turnId: input.turnId, - createdAt: input.now, - name: input.name, - kind: input.kind, - relativePath, - sizeBytes: Buffer.byteLength(input.content), - source: input.source, - status: 'live', - }; - let existing = ''; - try { - existing = await readFile(metadataPath, 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - } - await writeFile(metadataPath, `${existing}${JSON.stringify(record)}\n`); - return record; -} - -async function createPublicationResidue( - root: string, - id: string, - name: string, - content: string, -): Promise<{ stagingPath: string; targetPath: string }> { - const targetPath = join(root, 'artifacts', 'session-1', `${id}-${name}`); - const stagingPath = publicationStagingPath(targetPath); - await mkdir(dirname(targetPath), { recursive: true }); - await writeFile(stagingPath, content, { flag: 'wx' }); - await link(stagingPath, targetPath); - return { stagingPath, targetPath }; -} - -function publicationStagingPath(targetPath: string): string { - const hash = createHash('sha256').update(basename(targetPath)).digest('hex'); - return join( - dirname(targetPath), - `.artifact-publish.${hash}.00000000-0000-4000-8000-000000000000.tmp`, - ); -} - -async function withWorkspace(run: (root: string) => Promise): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-sqlite-artifact-')); - try { - await run(root); - } finally { - await rm(root, { recursive: true, force: true }); - } -} - -async function holdArtifactWriterLock( - root: string, -): Promise<{ release: () => void; finished: Promise }> { - let release!: () => void; - let acquired!: () => void; - const gate = new Promise((resolve) => { - release = resolve; - }); - const entered = new Promise((resolve) => { - acquired = resolve; - }); - const finished = withArtifactWriterLock(root, async () => { - acquired(); - await gate; - }); - await entered; - return { release, finished }; -} diff --git a/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts b/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts index 78a684297a..8b05298279 100644 --- a/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { mkdtemp, readFile, rm, stat } from 'node:fs/promises'; +import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, test } from 'node:test'; @@ -10,281 +10,95 @@ import type { InteractionRequest, ShellRunRecord, } from '@maka/core'; -import { createAgentRunStore, createSqliteAgentRunStore } from '../agent-run-store.js'; +import { createSqliteAgentRunStore } from '../agent-run-store.js'; import { closeSqliteInteractionStoreFacade, - interactionLocator, - openInteractiveInteractionStoreForWrite, openSqliteInteractiveInteractionStoreForWrite, type StoredInteractionRequest, } from '../interaction-store.js'; -import { - createMessageReceiptStore, - createSqliteMessageReceiptStore, -} from '../message-receipt-store.js'; -import { - acquireOperationalStateDatabase, - OPERATIONAL_STATE_DATABASE_NAME, -} from '../operational-state-store.js'; -import { createShellRunStore, createSqliteShellRunStore } from '../shell-run-store.js'; +import { createSqliteMessageReceiptStore } from '../message-receipt-store.js'; import { resolveStorageRoot, tryAcquireInteractiveRootOwner } from '../root-authority.js'; +import { createSqliteShellRunStore } from '../shell-run-store.js'; -describe('SQLite core execution cutover', () => { - test('imports AgentRun state once and never appends to legacy files', async () => { +describe('SQLite core execution stores', () => { + test('persists AgentRun header and events', async () => { await withRoot(async (root) => { - const legacy = createAgentRunStore(root); - const header = runHeader(); - const firstEvent = runEvent('event-1', 1); - await legacy.createRun(header); - await legacy.appendEvent(header.sessionId, header.runId, firstEvent); - await legacy.admitRootTurn({ - sessionId: header.sessionId, - turnId: header.turnId, - proposedRunId: header.runId, - proposedUserMessageId: 'message-1', - execution: { kind: 'external_message' }, - previousRootTurnId: null, - normalizedInput: { text: 'hello' }, - sourceMessages: [], - admittedAt: 1, - }); - const legacyEventsPath = join( - root, - 'sessions', - header.sessionId, - 'runs', - header.runId, - 'events.jsonl', - ); - const legacyBytes = await readFile(legacyEventsPath); - - const sqlite = createSqliteAgentRunStore(root); - await sqlite.ready?.(); - assert.deepEqual(await sqlite.readRun(header.sessionId, header.runId), header); - assert.deepEqual(await sqlite.readEvents(header.sessionId, header.runId), [firstEvent]); - assert.equal( - (await sqlite.listRootTurnAdmissionsForRecovery(header.sessionId))[0]?.turnId, - header.turnId, - ); - await sqlite.appendEvent(header.sessionId, header.runId, runEvent('event-2', 2)); - assert.deepEqual( - (await sqlite.readEvents(header.sessionId, header.runId)).map((event) => event.id), - ['event-1', 'event-2'], - ); - assert.deepEqual(await readFile(legacyEventsPath), legacyBytes); - sqlite.close?.(); + const store = createSqliteAgentRunStore(root); + await store.createRun(runHeader()); + await store.appendEvent('session-1', 'run-1', runEvent()); + store.close?.(); const reopened = createSqliteAgentRunStore(root); - await reopened.ready?.(); - assert.deepEqual( - (await reopened.readEvents(header.sessionId, header.runId)).map((event) => event.id), - ['event-1', 'event-2'], - ); - reopened.close?.(); - }); - }); - - test('rejects SQLite updates to immutable AgentRun admission identity', async () => { - await withRoot(async (root) => { - const store = createSqliteAgentRunStore(root); try { - const original = { - ...runHeader(), - continuationSource: { - sourceInvocationId: 'source-invocation', - sourceRunId: 'source-run', - sourceTurnId: 'source-turn', - sourceRuntimeEventHighWater: 1, - }, - } satisfies AgentRunHeader; - await store.createRun(original); - - await assert.rejects( - store.updateRun(original.sessionId, original.runId, { modelId: 'different-model' }), - /admission identity is immutable: modelId/, - ); - await assert.rejects( - store.updateRun(original.sessionId, original.runId, { continuationSource: undefined }), - /admission identity is immutable: continuationSource/, - ); - assert.deepEqual(await store.readRun(original.sessionId, original.runId), original); + assert.equal((await reopened.readRun('session-1', 'run-1')).runId, 'run-1'); + assert.equal((await reopened.readEvents('session-1', 'run-1'))[0]?.id, 'event-1'); } finally { - store.close?.(); + reopened.close?.(); } }); }); - test('rolls back copied AgentRun rows and resumes a started cutover', async () => { - await withRoot(async (root) => { - const legacy = createAgentRunStore(root); - await legacy.createRun(runHeader()); - await legacy.appendEvent('session-1', 'run-1', runEvent('event-1', 1)); - - const interrupted = createSqliteAgentRunStore(root, { - failpoint: (point) => { - if (point === 'after_cutover_rows_copied') throw new Error('simulated crash'); - }, - }); - assert.ok(interrupted.ready); - await assert.rejects(interrupted.ready(), /simulated crash/); - interrupted.close?.(); - - const inspection = acquireOperationalStateDatabase(root); - const journal = inspection.database - .prepare(` - SELECT state - FROM cutover_journal - WHERE store_name = 'agent_runs' - `) - .get() as { state?: unknown } | undefined; - const count = inspection.database - .prepare('SELECT COUNT(*) AS count FROM core_agent_runs') - .get() as { count?: unknown }; - assert.equal(journal?.state, 'started'); - assert.equal(count.count, 0); - inspection.close(); - - const resumed = createSqliteAgentRunStore(root); - await resumed.ready?.(); - assert.equal((await resumed.readRun('session-1', 'run-1')).runId, 'run-1'); - resumed.close?.(); - }); - }); - - test('imports ShellRun state and keeps payload files unchanged', async () => { + test('persists ShellRun records', async () => { await withRoot(async (root) => { - const legacy = createShellRunStore(root); - await legacy.createShellRun(shellRun()); - const legacyPath = join( - root, - 'sessions', - 'session-1', - 'shell-runs', - 'shell-1', - 'shell-run.json', - ); - const before = await readFile(legacyPath); + const store = createSqliteShellRunStore(root); + await store.createShellRun(shellRun()); + store.close(); - const sqlite = createSqliteShellRunStore(root); - await sqlite.ready(); - assert.equal((await sqlite.readShellRun('session-1', 'shell-1')).status, 'running'); - const updated = await sqlite.updateShellRun('session-1', 'shell-1', { - status: 'completed', - completedAt: 2, - exitCode: 0, - updatedAt: 2, - }); - assert.equal(updated.revision, 2); - assert.deepEqual(await readFile(legacyPath), before); - sqlite.close(); + const reopened = createSqliteShellRunStore(root); + try { + assert.equal((await reopened.readShellRun('session-1', 'shell-1')).command, 'printf "ok"'); + } finally { + reopened.close(); + } }); }); - test('fails closed when retired AgentRun or ShellRun writers change cutover evidence', async () => { + test('persists message receipts', async () => { await withRoot(async (root) => { - const legacyRuns = createAgentRunStore(root); - await legacyRuns.createRun(runHeader()); - const legacyShellRuns = createShellRunStore(root); - await legacyShellRuns.createShellRun(shellRun()); - - const sqliteRuns = createSqliteAgentRunStore(root); - const sqliteShellRuns = createSqliteShellRunStore(root); - await Promise.all([sqliteRuns.ready?.(), sqliteShellRuns.ready()]); - sqliteRuns.close?.(); - sqliteShellRuns.close(); - - await legacyRuns.appendEvent('session-1', 'run-1', runEvent('stale-event', 2)); - await legacyShellRuns.updateShellRun('session-1', 'shell-1', { - status: 'completed', - completedAt: 2, - exitCode: 0, - updatedAt: 2, + const store = createSqliteMessageReceiptStore(root); + await store.beginHostEpoch('epoch-1'); + await store.commit('epoch-1', 'submit', 'session-1', 'operation-1', { + payload: { text: 'hello' }, + result: { disposition: 'turn_started', turnId: 'turn-1' }, }); + store.close(); - const rejectedRuns = createSqliteAgentRunStore(root); - assert.ok(rejectedRuns.ready); - await assert.rejects( - rejectedRuns.ready(), - /Legacy agent_runs source changed after cutover completed/, - ); - rejectedRuns.close?.(); - - const rejectedShellRuns = createSqliteShellRunStore(root); - await assert.rejects( - rejectedShellRuns.ready(), - /Legacy shell_runs source changed after cutover completed/, - ); - rejectedShellRuns.close(); + const reopened = createSqliteMessageReceiptStore(root); + try { + assert.deepEqual( + (await reopened.read('epoch-1', 'submit', 'session-1', 'operation-1'))?.payload, + { text: 'hello' }, + ); + } finally { + reopened.close(); + } }); }); - test('imports receipts and interactions while new writes stay SQLite-only', async () => { + test('persists interaction request and outcome', async () => { await withRoot(async (root) => { - const receipts = createMessageReceiptStore(root); - await receipts.beginHostEpoch('epoch-1'); - await receipts.commit('epoch-1', 'submit', 'session-1', 'operation-1', { - payload: { text: 'hello' }, - result: { accepted: true }, - }); - const capability = await resolveStorageRoot({ path: root, kind: 'interactive' }); - const legacyOwner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(legacyOwner); - if (!legacyOwner) return; - const legacyInteractions = await openInteractiveInteractionStoreForWrite(legacyOwner.lease); - const request = storedQuestion('request-1', 1); - assert.equal((await legacyInteractions.establishRequest(request)).status, 'stable'); - assert.equal( - (await legacyInteractions.commitOutcome('request-1', questionOutcome('First', 2))).status, - 'stable', - ); - await legacyOwner.close(); - - const sqliteReceipts = createSqliteMessageReceiptStore(root); - await sqliteReceipts.ready(); - assert.deepEqual(await sqliteReceipts.read('epoch-1', 'submit', 'session-1', 'operation-1'), { - payload: { text: 'hello' }, - result: { accepted: true }, - }); - await sqliteReceipts.commit('epoch-1', 'interrupt', 'session-1', 'operation-2', { - payload: { reason: 'stop' }, - result: { interrupted: true }, - }); - await assert.rejects( - () => - stat( - join(root, 'message-receipts', 'epoch-1', 'interrupt', 'session-1', 'operation-2.json'), - ), - { code: 'ENOENT' }, - ); - sqliteReceipts.close(); - - const sqliteOwner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(sqliteOwner); - if (!sqliteOwner) return; - const sqliteInteractions = await openSqliteInteractiveInteractionStoreForWrite( - sqliteOwner.lease, - ); - assert.equal( - (await sqliteInteractions.readInteraction('request-1'))?.outcome?.outcome.kind, - 'question_answer', - ); - const sqliteOnly = storedQuestion('request-2', 3); - assert.equal((await sqliteInteractions.establishRequest(sqliteOnly)).status, 'stable'); - await assert.rejects( - () => stat(join(root, 'interactions', interactionLocator(sqliteOnly.requestId))), - { code: 'ENOENT' }, - ); - closeSqliteInteractionStoreFacade(sqliteInteractions); - await sqliteOwner.close(); - - assert.ok((await stat(join(root, OPERATIONAL_STATE_DATABASE_NAME))).isFile()); + const owner = await tryAcquireInteractiveRootOwner(capability); + assert.ok(owner); + if (!owner) return; + const store = await openSqliteInteractiveInteractionStoreForWrite(owner.lease); + try { + await store.establishRequest(storedQuestion()); + await store.commitOutcome('request-1', questionOutcome()); + assert.equal( + (await store.readInteraction('request-1'))?.outcome?.outcome.kind, + 'question_answer', + ); + } finally { + closeSqliteInteractionStoreFacade(store); + await owner.close(); + } }); }); }); async function withRoot(run: (root: string) => Promise): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-sqlite-core-execution-')); + const root = await mkdtemp(join(tmpdir(), 'maka-sqlite-execution-')); try { await run(root); } finally { @@ -308,14 +122,14 @@ function runHeader(): AgentRunHeader { }; } -function runEvent(id: string, ts: number): AgentRunEvent { +function runEvent(): AgentRunEvent { return { type: 'run_started', - id, + id: 'event-1', runId: 'run-1', sessionId: 'session-1', turnId: 'turn-1', - ts, + ts: 2, }; } @@ -343,13 +157,13 @@ function shellRun(): ShellRunRecord { }; } -function storedQuestion(requestId: string, createdAt: number): StoredInteractionRequest { +function storedQuestion(): StoredInteractionRequest { return { sessionId: 'session-1', turnId: 'turn-1', runId: 'run-1', - requestId, - createdAt, + requestId: 'request-1', + createdAt: 1, request: { kind: 'question', toolUseId: 'tool-1', @@ -366,10 +180,10 @@ function storedQuestion(requestId: string, createdAt: number): StoredInteraction }; } -function questionOutcome(answer: string, committedAt: number): InteractionCanonicalOutcome { +function questionOutcome(): InteractionCanonicalOutcome { return { kind: 'question_answer', - answers: [answer], - committedAt, + answers: ['First'], + committedAt: 2, } as InteractionCanonicalOutcome; } diff --git a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts index 12f4ab2476..4f04b89197 100644 --- a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts @@ -606,16 +606,11 @@ describe('SqliteSessionMetadataStore', () => { const store = createSqliteSessionMetadataStore(':memory:', { now: nextNow(212) }); const { name: _name, ...unnamedProfile } = createWorkspaceWritePermissionProfile(); try { - await store.importEntries([ - { - header: fullHeader(), - initialBoundary: { kind: 'managed', profile: unnamedProfile, revision: 0 }, - source: { - path: '/workspace/sessions/session-1/session.jsonl', - fingerprint: 'unnamed-profile', - }, - }, - ]); + await store.create(fullHeader(), { + kind: 'managed', + profile: unnamedProfile, + revision: 0, + }); await store.setExecutionBoundaryKind('session-1', 'bypass'); const restored = await store.setExecutionBoundaryKind('session-1', 'managed'); @@ -652,20 +647,11 @@ describe('SqliteSessionMetadataStore', () => { const store = createSqliteSessionMetadataStore(':memory:', { now: nextNow(220) }); const { name: _name, ...unnamedReadOnlyProfile } = createReadOnlyPermissionProfile(); try { - await store.importEntries([ - { - header: fullHeader({ permissionMode: 'explore' }), - initialBoundary: { - kind: 'managed', - profile: unnamedReadOnlyProfile, - revision: 0, - }, - source: { - path: '/workspace/sessions/session-1/session.jsonl', - fingerprint: 'unnamed-read-only-profile', - }, - }, - ]); + await store.create(fullHeader({ permissionMode: 'explore' }), { + kind: 'managed', + profile: unnamedReadOnlyProfile, + revision: 0, + }); const restored = await store.setExecutionBoundaryKind('session-1', 'managed', { permissionMode: 'ask', @@ -1830,70 +1816,6 @@ describe('SqliteSessionMetadataStore', () => { } }); - test('imports source-marked metadata idempotently and rejects identity drift', async () => { - const store = createSqliteSessionMetadataStore(':memory:'); - const entry = { - header: fullHeader(), - source: { path: '/workspace/sessions/session-1/session.jsonl', fingerprint: '1:1' }, - }; - try { - assert.deepEqual(await store.importEntries([entry]), { - created: [true], - sourcesAlreadyImported: 0, - sourcesTombstoned: 0, - }); - assert.deepEqual(await store.importEntries([entry]), { - created: [], - sourcesAlreadyImported: 1, - sourcesTombstoned: 0, - }); - await assert.rejects( - () => - store.importEntries([ - { - ...entry, - header: fullHeader({ name: 'Changed outside SQLite' }), - source: { ...entry.source, fingerprint: '2:2' }, - }, - ]), - SessionMetadataConflictError, - ); - assert.equal((await store.read('session-1')).header.name, 'Session'); - } finally { - store.close(); - } - }); - - test('rolls back the whole import batch when a later source marker fails', async () => { - let markers = 0; - const store = createSqliteSessionMetadataStore(':memory:', { - failpoint: (point) => { - if (point === 'after_session_import_marker_write' && ++markers === 2) { - throw new Error('second marker failed'); - } - }, - }); - try { - await assert.rejects( - () => - store.importEntries([ - { - header: fullHeader({ id: 'session-1' }), - source: { path: '/session-1.jsonl', fingerprint: '1:1' }, - }, - { - header: fullHeader({ id: 'session-2' }), - source: { path: '/session-2.jsonl', fingerprint: '2:2' }, - }, - ]), - /second marker failed/, - ); - assert.deepEqual(await store.list(), []); - } finally { - store.close(); - } - }); - test('deletes metadata and its label projection atomically', async () => { const store = createSqliteSessionMetadataStore(':memory:'); try { @@ -1903,15 +1825,6 @@ describe('SqliteSessionMetadataStore', () => { assert.equal(await store.has('session-1'), false); assert.equal(await store.isTombstoned('session-1'), true); assert.deepEqual(await store.list({ labelSlug: 'alpha' }), []); - assert.deepEqual( - await store.importEntries([ - { - header: fullHeader(), - source: { path: '/session-1.jsonl', fingerprint: '1:1' }, - }, - ]), - { created: [], sourcesAlreadyImported: 0, sourcesTombstoned: 1 }, - ); await assert.rejects(() => store.create(fullHeader()), /tombstoned/); } finally { store.close(); diff --git a/packages/storage/src/__tests__/sqlite-session-store.test.ts b/packages/storage/src/__tests__/sqlite-session-store.test.ts deleted file mode 100644 index 851525c544..0000000000 --- a/packages/storage/src/__tests__/sqlite-session-store.test.ts +++ /dev/null @@ -1,1275 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { DatabaseSync } from 'node:sqlite'; -import { describe, test } from 'node:test'; -import { - SUBAGENT_WORKSPACE_BINDING_SCHEMA_VERSION, - type CreateSessionInput, - type SessionHeader, - type SessionListFilter, - type SubagentWorkspaceBinding, -} from '@maka/core'; -import { - createLegacyFileSessionStore, - createSessionStore, - isSessionNotFoundError, - SessionReadMarkerMessageNotFoundError, - SessionNotFoundError, - SQLITE_SESSION_METADATA_DATABASE_NAME, - type StableSessionCreateInput, -} from '../session-store.js'; -import { createSessionStoreForTest } from '../session-store-test-support.js'; -import { - createSqliteSessionMetadataStore, - SessionMetadataVersionConflictError, -} from '../sqlite-session-metadata-store.js'; -import { - buildSqliteSessionCatalogPageQuery, - type SqliteSessionCatalogCursor, -} from '../sqlite-session-catalog-query.js'; -import { createSessionTranscriptMarker } from '../session-transcript.js'; - -describe('default SQLite session metadata store', () => { - test('SQLite and legacy File stores expose one typed missing-Session boundary', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-store-not-found-')); - const stores = [ - createSessionStore(join(root, 'sqlite')), - createLegacyFileSessionStore(join(root, 'legacy')), - ]; - try { - for (const store of stores) { - await assert.rejects(store.readHeaderSnapshot('deleted-session'), (error) => { - assert.ok(error instanceof SessionNotFoundError); - assert.equal(isSessionNotFoundError(error), true); - assert.equal(error.code, 'session_not_found'); - assert.equal(error.sessionId, 'deleted-session'); - return true; - }); - } - } finally { - await Promise.all(stores.map((store) => store.close?.())); - await rm(root, { recursive: true, force: true }); - } - }); - - test('waits for in-flight legacy metadata import before closing SQLite', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-store-close-ready-')); - const legacy = createLegacyFileSessionStore(root); - const created = await legacy.create(makeInput({ name: 'Imported before close' })); - const store = createSessionStore(root); - - try { - await store.close?.(); - const metadata = createSqliteSessionMetadataStore( - join(root, SQLITE_SESSION_METADATA_DATABASE_NAME), - ); - try { - assert.equal((await metadata.read(created.id)).header.name, 'Imported before close'); - } finally { - metadata.close(); - } - } finally { - await rm(root, { recursive: true, force: true }); - } - }); - - test('paginates the catalog with a durable keyset revision', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-catalog-page-')); - const store = createSessionStore(root); - try { - for (let index = 0; index < 40; index += 1) { - const header = await store.create( - makeInput({ name: `Session ${String(index).padStart(2, '0')}` }), - ); - await store.updateHeader(header.id, { lastMessageAt: 1_000 + index }); - } - - const first = await store.listCatalogPage(undefined, undefined, 10); - assert.equal(first.kind, 'page'); - if (first.kind !== 'page') assert.fail('Catalog start must return a page'); - assert.equal(first.records.length, 10); - assert.equal(first.hasMore, true); - const firstIds = new Set(first.records.map((record) => record.header.id)); - const last = first.records.at(-1); - assert.ok(last); - if (!last) assert.fail('Catalog page must have a keyset boundary'); - - const second = await store.listCatalogPage( - undefined, - { - activityAt: last.header.lastMessageAt ?? last.header.lastUsedAt ?? last.header.createdAt, - sessionId: last.header.id, - }, - 10, - first.revision, - ); - assert.equal(second.kind, 'page'); - if (second.kind !== 'page') assert.fail('Stable continuation must return a page'); - assert.equal(second.records.length, 10); - assert.equal( - second.records.some((record) => firstIds.has(record.header.id)), - false, - ); - - await store.appendMessage(last.header.id, { - type: 'user', - id: 'catalog-revision-message', - turnId: 'catalog-revision-turn', - ts: 2_000, - text: 'invalidate the catalog snapshot', - }); - const stale = await store.listCatalogPage( - undefined, - { - activityAt: last.header.lastMessageAt ?? last.header.lastUsedAt ?? last.header.createdAt, - sessionId: last.header.id, - }, - 10, - first.revision, - ); - assert.equal(stale.kind, 'revision_changed'); - if (stale.kind === 'revision_changed') { - assert.equal(stale.expectedRevision, first.revision); - assert.notEqual(stale.actualRevision, first.revision); - } - } finally { - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('publishes transcript previews and catalog revisions through one SQLite commit', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-catalog-projection-')); - const store = createSessionStore(root); - try { - const session = await store.create(makeInput()); - await store.appendMessage(session.id, { - type: 'user', - id: 'message-1', - turnId: 'turn-1', - ts: 10, - text: 'first preview', - }); - const first = await store.listCatalogPage(undefined, undefined, 10); - assert.equal(first.kind, 'page'); - if (first.kind !== 'page') assert.fail('Catalog start must return a page'); - assert.equal(first.records[0]?.summary.lastMessagePreview, 'first preview'); - await store.close?.(); - - const metadata = createSqliteSessionMetadataStore( - join(root, SQLITE_SESSION_METADATA_DATABASE_NAME), - ); - const transcript = createLegacyFileSessionStore(root); - try { - const base = await metadata.listCatalogPage({}, undefined, 10); - await metadata.beginCatalogProjectionWrite(); - await transcript.appendMessage(session.id, { - type: 'assistant', - id: 'message-2', - turnId: 'turn-1', - ts: 20, - text: 'second preview', - modelId: 'fake-model', - }); - - const beforeCommit = await metadata.listCatalogPage({}, undefined, 10); - assert.equal(beforeCommit.records[0]?.lastMessagePreview, 'first preview'); - assert.deepEqual(beforeCommit.revision, base.revision); - - await metadata.commitCatalogProjectionWrite(session.id, { - lastMessageAt: 20, - lastMessagePreview: 'second preview', - }); - const afterCommit = await metadata.listCatalogPage({}, undefined, 10); - assert.equal(afterCommit.records[0]?.lastMessagePreview, 'second preview'); - assert.ok(afterCommit.revision.generation > beforeCommit.revision.generation); - } finally { - metadata.close(); - await transcript.close?.(); - } - } finally { - await rm(root, { recursive: true, force: true }); - } - }); - - test('advances the catalog generation once per metadata transaction', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-catalog-generation-')); - const store = createSessionStore(root); - let metadata: ReturnType | undefined; - try { - const session = await store.create(makeInput({ labels: ['one', 'two'] })); - await store.close?.(); - metadata = createSqliteSessionMetadataStore( - join(root, SQLITE_SESSION_METADATA_DATABASE_NAME), - ); - const initial = await metadata.listCatalogPage({}, undefined, 10); - - await metadata.update(session.id, { name: 'Renamed' }); - const renamed = await metadata.listCatalogPage({}, undefined, 10); - assert.equal(renamed.revision.generation, initial.revision.generation + 1); - assert.deepEqual(renamed.records[0]?.header.labels, ['one', 'two']); - - await metadata.update(session.id, { labels: ['three'] }); - const relabeled = await metadata.listCatalogPage({ labelSlug: 'three' }, undefined, 10); - assert.equal(relabeled.revision.generation, renamed.revision.generation + 1); - assert.equal(relabeled.records[0]?.header.id, session.id); - } finally { - if (metadata) metadata.close(); - else await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('repairs a transcript append left pending before its catalog projection commit', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-catalog-recovery-')); - const store = createSessionStore(root); - const session = await store.create(makeInput()); - await store.close?.(); - - const metadata = createSqliteSessionMetadataStore( - join(root, SQLITE_SESSION_METADATA_DATABASE_NAME), - ); - const transcript = createLegacyFileSessionStore(root); - try { - await metadata.beginCatalogProjectionWrite(); - await transcript.appendMessage(session.id, { - type: 'user', - id: 'recovered-message', - turnId: 'recovered-turn', - ts: 30, - text: 'recovered preview', - }); - } finally { - metadata.close(); - await transcript.close?.(); - } - - const reopened = createSessionStore(root); - try { - const page = await reopened.listCatalogPage(undefined, undefined, 10); - assert.equal(page.kind, 'page'); - if (page.kind !== 'page') assert.fail('Recovered catalog must return a page'); - assert.equal(page.records[0]?.summary.lastMessageAt, 30); - assert.equal(page.records[0]?.summary.lastMessagePreview, 'recovered preview'); - } finally { - await reopened.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('repairs a pending catalog projection from a message larger than the preview window', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-catalog-large-recovery-')); - const store = createSessionStore(root); - const session = await store.create(makeInput()); - await store.close?.(); - - const metadata = createSqliteSessionMetadataStore( - join(root, SQLITE_SESSION_METADATA_DATABASE_NAME), - ); - const transcript = createLegacyFileSessionStore(root); - try { - await metadata.beginCatalogProjectionWrite(); - await transcript.appendMessage(session.id, { - type: 'user', - id: 'large-recovered-message', - turnId: 'large-recovered-turn', - ts: 40, - text: `large recovered preview ${'x'.repeat(70 * 1024)}`, - }); - } finally { - metadata.close(); - await transcript.close?.(); - } - - const reopened = createSessionStore(root); - try { - const page = await reopened.listCatalogPage(undefined, undefined, 10); - assert.equal(page.kind, 'page'); - if (page.kind !== 'page') assert.fail('Recovered catalog must return a page'); - assert.equal(page.records[0]?.summary.lastMessageAt, 40); - assert.match(page.records[0]?.summary.lastMessagePreview ?? '', /^large recovered preview/); - } finally { - await reopened.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('repairs a pending catalog projection before the same store serves another page', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-catalog-live-recovery-')); - const store = createSessionStore(root); - const metadata = createSqliteSessionMetadataStore( - join(root, SQLITE_SESSION_METADATA_DATABASE_NAME), - ); - const transcript = createLegacyFileSessionStore(root); - try { - const session = await store.create(makeInput()); - await metadata.beginCatalogProjectionWrite(); - await transcript.appendMessage(session.id, { - type: 'assistant', - id: 'live-recovered-message', - turnId: 'live-recovered-turn', - ts: 50, - text: 'live recovered preview', - modelId: 'fake-model', - }); - - const page = await store.listCatalogPage(undefined, undefined, 10); - assert.equal(page.kind, 'page'); - if (page.kind !== 'page') assert.fail('Recovered catalog must return a page'); - assert.equal(page.records[0]?.summary.lastMessageAt, 50); - assert.equal(page.records[0]?.summary.lastMessagePreview, 'live recovered preview'); - assert.equal(await metadata.hasPendingCatalogProjectionWrites(), false); - } finally { - metadata.close(); - await transcript.close?.(); - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('uses ordered catalog indexes without temporary sorting', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-catalog-index-')); - const store = createSessionStore(root); - try { - const parent = await store.create(makeInput({ name: 'Parent' })); - await store.create( - makeInput({ - name: 'Indexed', - labels: ['indexed'], - subagentParent: { - kind: 'subagent', - parentSessionId: parent.id, - lifecycle: 'foreground', - spawnedBy: { - parentRunId: 'parent-run', - parentTurnId: 'parent-turn', - toolCallId: 'tool-call', - }, - }, - }), - ); - await store.close?.(); - - const database = new DatabaseSync(join(root, SQLITE_SESSION_METADATA_DATABASE_NAME)); - try { - assertCatalogQueryPlan(database, {}, undefined, 'session_catalog_by_activity'); - assertCatalogQueryPlan( - database, - { isArchived: false, isFlagged: false }, - undefined, - 'session_catalog_by_archived_flagged_activity', - ); - assertCatalogQueryPlan( - database, - { labelSlug: 'indexed' }, - undefined, - 'session_catalog_labels_by_label_activity', - ); - assertCatalogQueryPlan( - database, - {}, - { activityAt: Number.MAX_SAFE_INTEGER, sessionId: parent.id }, - 'session_catalog_by_activity', - 'SEARCH projection', - ); - assertCatalogQueryPlan( - database, - { labelSlug: 'indexed' }, - { activityAt: Number.MAX_SAFE_INTEGER, sessionId: parent.id }, - 'session_catalog_labels_by_label_activity', - 'SEARCH selected_label', - ); - } finally { - database.close(); - } - } finally { - await rm(root, { recursive: true, force: true }); - } - }); - - test('uses SQLite as canonical metadata while keeping transcript bodies in JSONL', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-default-session-store-')); - const store = createSessionStore(root); - try { - const created = await store.create(makeInput({ name: 'Initial title' })); - await store.appendMessage(created.id, { - type: 'user', - id: 'user-1', - turnId: 'turn-1', - ts: 10, - text: 'hello', - }); - await store.rename(created.id, 'SQLite title'); - await store.updateHeader(created.id, { - hasUnread: true, - lastMessageAt: 10, - }); - - assert.equal((await store.readHeader(created.id)).name, 'SQLite title'); - assert.equal((await store.list())[0]?.name, 'SQLite title'); - assert.equal((await store.readMessages(created.id))[0]?.type, 'user'); - - const transcriptPath = join(root, 'sessions', created.id, 'session.jsonl'); - const [marker, message] = (await readFile(transcriptPath, 'utf8')) - .trim() - .split('\n') - .map((line) => JSON.parse(line) as Record); - assert.deepEqual(marker, { - type: 'session_transcript', - sessionId: created.id, - schemaVersion: 1, - }); - assert.equal('name' in (marker ?? {}), false); - assert.equal(message?.type, 'user'); - await stat(join(root, SQLITE_SESSION_METADATA_DATABASE_NAME)); - await assert.rejects(() => stat(join(root, 'sessions.sqlite')), { code: 'ENOENT' }); - - await store.close?.(); - const reopened = createSessionStore(root); - try { - assert.equal((await reopened.readHeader(created.id)).name, 'SQLite title'); - assert.equal((await reopened.readMessages(created.id)).length, 1); - } finally { - await reopened.close?.(); - } - } finally { - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('provides stable create retries and message-identity read markers', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-default-session-stable-create-')); - const sessionId = 'stable-session'; - const requestFingerprint = `sha256:${'c'.repeat(64)}`; - const request = { - sessionId, - requestFingerprint, - input: makeInput({ name: 'Original' }), - }; - const store = createSessionStore(root); - try { - const created = await store.createStableSession(request); - assert.equal(created.kind, 'created'); - assert.equal(created.record.header.id, sessionId); - - const retried = await store.createStableSession({ - ...request, - input: makeInput({ name: 'Changed default' }), - }); - assert.equal(retried.kind, 'existing'); - assert.equal(retried.record.header.name, 'Original'); - assert.deepEqual( - await store.createStableSession({ - ...request, - requestFingerprint: `sha256:${'d'.repeat(64)}`, - }), - { kind: 'conflict', reason: 'identity_mismatch' }, - ); - - await store.appendMessages(sessionId, [ - { type: 'user', id: 'message-1', turnId: 'turn-1', ts: 1, text: 'one' }, - { type: 'user', id: 'message-2', turnId: 'turn-2', ts: 2, text: 'two' }, - { type: 'user', id: 'message-3', turnId: 'turn-3', ts: 3, text: 'three' }, - ]); - await store.updateHeader(sessionId, { hasUnread: true, lastMessageAt: 3 }); - - const readMessagesSnapshot = store.readMessagesSnapshot.bind(store); - let interleaved = false; - store.readMessagesSnapshot = async (id) => { - const messages = await readMessagesSnapshot(id); - if (!interleaved) { - interleaved = true; - await store.updateHeader(id, { name: 'Concurrent metadata update' }); - } - return messages; - }; - const partial = await store.markSessionReadThroughMessage(sessionId, 'message-2'); - assert.equal(interleaved, true); - assert.equal(partial.header.lastReadMessageId, 'message-2'); - assert.equal(partial.header.hasUnread, true); - const complete = await store.markSessionReadThroughMessage(sessionId, 'message-3'); - assert.equal(complete.header.lastReadMessageId, 'message-3'); - assert.equal(complete.header.hasUnread, false); - assert.deepEqual(await store.markSessionReadThroughMessage(sessionId, 'message-1'), complete); - await assert.rejects( - store.markSessionReadThroughMessage(sessionId, 'missing-message'), - (error) => { - assert.ok(error instanceof SessionReadMarkerMessageNotFoundError); - assert.equal(error.sessionId, sessionId); - assert.equal(error.messageId, 'missing-message'); - return true; - }, - ); - - await store.appendMessage(sessionId, { - type: 'user', - id: 'message-4', - turnId: 'turn-4', - ts: 4, - text: 'four', - }); - await store.updateHeader(sessionId, { hasUnread: true, lastMessageAt: 4 }); - const updateHeaderVersioned = store.updateHeaderVersioned.bind(store); - let conflictAttempts = 0; - store.updateHeaderVersioned = async (id, patch, expectedRevision) => { - conflictAttempts += 1; - throw new SessionMetadataVersionConflictError( - id, - expectedRevision, - expectedRevision + conflictAttempts, - ); - }; - await assert.rejects( - store.markSessionReadThroughMessage(sessionId, 'message-4'), - SessionMetadataVersionConflictError, - ); - assert.equal(conflictAttempts, 3); - store.updateHeaderVersioned = updateHeaderVersioned; - - await store.remove(sessionId); - } finally { - await store.close?.(); - } - - const reopened = createSessionStore(root); - try { - assert.deepEqual(await reopened.createStableSession(request), { - kind: 'conflict', - reason: 'removed', - }); - } finally { - await reopened.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('recovers stable creation after a claimed marker-only transcript is left behind', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-default-session-stable-recovery-')); - const sessionId = 'stable-recovery'; - const requestFingerprint = `sha256:${'e'.repeat(64)}`; - try { - const metadata = createSqliteSessionMetadataStore( - join(root, SQLITE_SESSION_METADATA_DATABASE_NAME), - ); - try { - assert.deepEqual(await metadata.claimStableSessionCreate(sessionId, requestFingerprint), { - kind: 'absent', - }); - } finally { - metadata.close(); - } - const sessionDir = join(root, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - `${JSON.stringify(createSessionTranscriptMarker(sessionId))}\n`, - ); - - const store = createSessionStore(root); - try { - const request = { - sessionId, - requestFingerprint, - input: makeInput({ name: 'Recovered' }), - }; - const created = await store.createStableSession(request); - assert.equal(created.kind, 'created'); - assert.equal(created.record.header.name, 'Recovered'); - assert.deepEqual(await store.readMessagesSnapshot(sessionId), []); - assert.equal((await store.createStableSession(request)).kind, 'existing'); - } finally { - await store.close?.(); - } - } finally { - await rm(root, { recursive: true, force: true }); - } - }); - - test('repairs empty and recognizable truncated stable-create transcripts', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-default-session-stable-truncated-')); - const requestFingerprint = `sha256:${'f'.repeat(64)}`; - const fixtures = ['stable-empty', 'stable-truncated'].map((sessionId, index) => { - const marker = `${JSON.stringify(createSessionTranscriptMarker(sessionId))}\n`; - return { - sessionId, - marker, - initial: index === 0 ? '' : marker.slice(0, -5), - }; - }); - try { - const metadata = createSqliteSessionMetadataStore( - join(root, SQLITE_SESSION_METADATA_DATABASE_NAME), - ); - try { - for (const fixture of fixtures) { - assert.deepEqual( - await metadata.claimStableSessionCreate(fixture.sessionId, requestFingerprint), - { kind: 'absent' }, - ); - const sessionDir = join(root, 'sessions', fixture.sessionId); - await mkdir(sessionDir, { recursive: true }); - await writeFile(join(sessionDir, 'session.jsonl'), fixture.initial); - } - } finally { - metadata.close(); - } - - const store = createSessionStore(root); - try { - for (const fixture of fixtures) { - const created = await store.createStableSession({ - sessionId: fixture.sessionId, - requestFingerprint, - input: makeInput({ name: `Recovered ${fixture.sessionId}` }), - }); - assert.equal(created.kind, 'created'); - assert.equal( - await readFile(join(root, 'sessions', fixture.sessionId, 'session.jsonl'), 'utf8'), - fixture.marker, - ); - assert.equal( - (await stat(join(root, 'sessions', fixture.sessionId, 'session.jsonl'))).mode & 0o777, - 0o600, - ); - } - } finally { - await store.close?.(); - } - } finally { - await rm(root, { recursive: true, force: true }); - } - }); - - test('hides preparing conversation copies and discards them without tombstoning stable identity', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-copy-publication-')); - const store = createSessionStore(root); - const sessionId = 'stable-copy'; - const requestFingerprint: `sha256:${string}` = `sha256:${'f'.repeat(64)}`; - const input: StableSessionCreateInput = { - ...makeInput({ - parentSessionId: 'source-session', - branchOfTurnId: 'turn-1', - }), - conversationCopy: { - kind: 'branch', - sourceSessionId: 'source-session', - sourceTurnId: 'turn-1', - requestFingerprint, - state: 'preparing', - }, - }; - try { - await assert.rejects( - () => store.create(input), - /Conversation copy metadata requires createStableSession/, - ); - assert.equal( - (await store.createStableSession({ sessionId, requestFingerprint, input })).kind, - 'created', - ); - assert.deepEqual(await store.list(), []); - const page = await store.listCatalogPage(undefined, undefined, 10); - assert.equal(page.kind, 'page'); - if (page.kind !== 'page') assert.fail('Expected a Session catalog page'); - assert.deepEqual(page.records, []); - await assert.rejects(() => store.readCatalogRecord(sessionId), isSessionNotFoundError); - - assert.equal(await store.discardStableConversationCopy(sessionId, requestFingerprint), true); - assert.equal( - (await store.createStableSession({ sessionId, requestFingerprint, input })).kind, - 'created', - ); - await store.updateHeader(sessionId, { - conversationCopy: { ...input.conversationCopy!, state: 'committed' }, - }); - assert.equal((await store.list())[0]?.id, sessionId); - assert.equal((await store.readCatalogRecord(sessionId)).header.id, sessionId); - await assert.rejects( - () => store.updateHeader(sessionId, { conversationCopy: undefined }), - /conversation-copy identity is immutable/, - ); - await assert.rejects( - () => store.discardStableConversationCopy(sessionId, requestFingerprint), - /matching incomplete conversation copy/, - ); - } finally { - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('recovers pending catalog publication after an incomplete copy loses its transcript', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-copy-recovery-')); - const sessionId = 'interrupted-copy'; - const requestFingerprint: `sha256:${string}` = `sha256:${'a'.repeat(64)}`; - const input: StableSessionCreateInput = { - ...makeInput({ - parentSessionId: 'source-session', - branchOfTurnId: 'turn-1', - }), - conversationCopy: { - kind: 'branch', - sourceSessionId: 'source-session', - sourceTurnId: 'turn-1', - requestFingerprint, - state: 'preparing', - }, - }; - const initial = createSessionStore(root); - try { - assert.equal( - ( - await initial.createStableSession({ - sessionId, - requestFingerprint, - input, - }) - ).kind, - 'created', - ); - } finally { - await initial.close?.(); - } - - const metadata = createSqliteSessionMetadataStore( - join(root, SQLITE_SESSION_METADATA_DATABASE_NAME), - ); - try { - await metadata.requireCatalogProjectionRecovery(); - } finally { - metadata.close(); - } - await rm(join(root, 'sessions', sessionId), { recursive: true, force: true }); - - const recovered = createSessionStore(root); - try { - assert.equal((await recovered.listHeaders())[0]?.id, sessionId); - assert.equal( - await recovered.discardStableConversationCopy(sessionId, requestFingerprint), - true, - ); - assert.equal( - ( - await recovered.createStableSession({ - sessionId, - requestFingerprint, - input, - }) - ).kind, - 'created', - ); - } finally { - await recovered.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('persists the stable project association in SQLite metadata and summaries', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-default-session-project-')); - const store = createSessionStore(root); - try { - const created = await store.create(makeInput({ projectId: 'project-1' })); - - assert.equal(created.projectId, 'project-1'); - assert.equal((await store.readHeaderSnapshot(created.id)).projectId, 'project-1'); - assert.equal((await store.list())[0]?.projectId, 'project-1'); - - await store.close?.(); - const reopened = createSessionStore(root); - try { - assert.equal((await reopened.readHeaderSnapshot(created.id)).projectId, 'project-1'); - assert.equal((await reopened.list())[0]?.projectId, 'project-1'); - } finally { - await reopened.close?.(); - } - } finally { - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('persists an explicit no-project association across SQLite reopen', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-default-session-no-project-')); - const store = createSessionStore(root); - try { - const created = await store.create(makeInput({ projectId: null })); - - assert.equal(created.projectId, null); - assert.equal((await store.readHeaderSnapshot(created.id)).projectId, null); - assert.equal((await store.list())[0]?.projectId, null); - - await store.close?.(); - const reopened = createSessionStore(root); - try { - assert.equal((await reopened.readHeaderSnapshot(created.id)).projectId, null); - assert.equal((await reopened.list())[0]?.projectId, null); - } finally { - await reopened.close?.(); - } - } finally { - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('lists linked child sessions through SQLite without conflating ordinary branches', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-default-session-relations-')); - const store = createSessionStore(root); - try { - const parent = await store.create(makeInput({ name: 'Parent' })); - const childInput = makeInput({ - name: 'Child', - permissionMode: 'execute', - subagentParent: { - kind: 'subagent', - parentSessionId: parent.id, - spawnedBy: { - parentRunId: 'parent-run', - parentTurnId: 'parent-turn', - toolCallId: 'tool-call', - }, - lifecycle: 'foreground', - }, - subagentRuntime: { - schemaVersion: 1, - definitionVersion: 1, - agentId: 'local-read', - agentName: 'Local Read', - profile: 'local_read', - systemPrompt: 'Read the assigned workspace task.', - toolNames: ['Read', 'Glob', 'Grep'], - categoryPolicy: { read: 'allow' }, - }, - subagentSpawn: { - schemaVersion: 1, - requestFingerprint: 'a'.repeat(64), - initialTurnId: 'child-turn', - initialRunId: 'child-run', - }, - }); - const { header: child, created } = await store.createSubagent(childInput); - assert.equal(created, true); - const retry = await store.createSubagent(childInput); - assert.equal(retry.created, false); - assert.equal(retry.header.id, child.id); - await assert.rejects( - () => - store.createSubagent({ - ...childInput, - subagentSpawn: { - ...childInput.subagentSpawn!, - requestFingerprint: 'b'.repeat(64), - }, - }), - /reused for different work/, - ); - await store.create( - makeInput({ - name: 'Branch', - parentSessionId: parent.id, - branchOfTurnId: 'parent-turn', - }), - ); - - const children = await store.list({ subagentParentSessionId: parent.id }); - assert.deepEqual( - children.map((session) => session.id), - [child.id], - ); - assert.equal(children[0]?.subagentParent?.parentSessionId, parent.id); - assert.deepEqual(children[0]?.subagentRuntime?.toolNames, ['Read', 'Glob', 'Grep']); - } finally { - store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('persists an immutable child worktree binding across summaries and reopen', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-default-session-worktree-')); - const store = createSessionStore(root); - const workspace = makeSubagentWorkspace(); - let childId = ''; - try { - const parent = await store.create( - makeInput({ name: 'Parent', cwd: '/workspace/repository', projectId: 'project-1' }), - ); - const { header: child } = await store.createSubagent( - makeInput({ - name: 'Implementation child', - cwd: workspace.worktreePath, - projectId: 'project-1', - permissionMode: 'execute', - subagentParent: { - kind: 'subagent', - parentSessionId: parent.id, - spawnedBy: { - parentRunId: 'parent-run', - parentTurnId: 'parent-turn', - toolCallId: 'tool-call', - }, - lifecycle: 'foreground', - }, - subagentRuntime: { - schemaVersion: 1, - definitionVersion: 1, - agentId: 'implementation', - agentName: 'Implementation', - profile: 'implementation', - systemPrompt: 'Implement the assigned task.', - toolNames: ['Read', 'Glob', 'Grep', 'Write', 'Edit', 'Bash'], - categoryPolicy: { - read: 'allow', - file_write: 'allow', - shell_safe: 'allow', - shell_unsafe: 'allow', - }, - permissionCeiling: 'execute', - }, - subagentSpawn: { - schemaVersion: 1, - requestFingerprint: 'c'.repeat(64), - initialTurnId: 'child-turn', - initialRunId: 'child-run', - }, - subagentWorkspace: workspace, - }), - ); - childId = child.id; - - assert.deepEqual(child.subagentWorkspace, workspace); - assert.deepEqual( - (await store.list()).find((session) => session.id === child.id)?.subagentWorkspace, - workspace, - ); - await assert.rejects( - () => - store.updateHeader(child.id, { - subagentWorkspace: { ...workspace, branch: 'maka/subagent/rebound' }, - }), - /workspace binding is immutable/, - ); - } finally { - await store.close?.(); - } - - const reopened = createSessionStore(root); - try { - assert.deepEqual((await reopened.readHeaderSnapshot(childId)).subagentWorkspace, workspace); - } finally { - await reopened.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('imports an existing JSONL catalog once and preserves later SQLite-only updates', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-default-session-migration-')); - const legacy = createLegacyFileSessionStore(root); - const created = await legacy.create(makeInput({ name: 'Legacy title', labels: ['legacy'] })); - await legacy.updateHeader(created.id, { - status: 'blocked', - blockedReason: 'permission_required', - hasUnread: true, - }); - await legacy.appendMessage(created.id, { - type: 'user', - id: 'user-1', - turnId: 'turn-1', - ts: 10, - text: 'legacy message', - }); - - const first = createSessionStore(root); - try { - assert.equal((await first.readHeader(created.id)).name, 'Legacy title'); - await first.rename(created.id, 'SQLite title'); - await first.appendMessage(created.id, { - type: 'assistant', - id: 'assistant-1', - turnId: 'turn-1', - ts: 11, - text: 'new message', - modelId: 'fake-model', - }); - } finally { - await first.close?.(); - } - - const reopened = createSessionStore(root); - try { - assert.equal((await reopened.readHeader(created.id)).name, 'SQLite title'); - assert.equal((await reopened.readMessages(created.id)).length, 2); - assert.deepEqual( - (await reopened.list({ labelSlug: 'legacy' })).map((item) => item.id), - [created.id], - ); - } finally { - await reopened.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('uses tombstones to prevent a deleted legacy transcript from resurrecting', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-default-session-delete-')); - const store = createSessionStore(root); - let header!: SessionHeader; - try { - header = await store.create(makeInput({ name: 'Delete me' })); - await store.remove(header.id); - assert.deepEqual(await store.listPendingSessionRetirementCleanupIds(), [header.id]); - } finally { - await store.close?.(); - } - - const sessionDir = join(root, 'sessions', header.id); - await mkdir(sessionDir, { recursive: true }); - await writeFile(join(sessionDir, 'session.jsonl'), `${JSON.stringify(header)}\n`, 'utf8'); - - const reopened = createSessionStore(root); - try { - assert.deepEqual(await reopened.list(), []); - await assert.rejects(() => reopened.readHeader(header.id), /not found/); - assert.deepEqual(await reopened.listPendingSessionRetirementCleanupIds(), [header.id]); - await reopened.purgeRemovedSessionTranscript(header.id); - await reopened.completeSessionRetirementCleanup(header.id); - await waitForMissingPath(sessionDir); - } finally { - await reopened.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('commits family removal before aggregate retirement cleanup', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-family-cleanup-')); - const store = createSessionStore(root); - try { - const familyRoot = await store.create(makeInput({ name: 'Family root' })); - const familyRevision = await store.create( - makeInput({ - name: 'Family revision', - revisionRootSessionId: familyRoot.id, - revisionParentSessionId: familyRoot.id, - revisionOfTurnId: 'turn-1', - revisionIndex: 2, - revisionState: 'committed', - }), - ); - const familyIds = [familyRoot.id, familyRevision.id]; - const identities = await Promise.all( - familyIds.map(async (sessionId) => { - const snapshot = await store.readHeaderRecordSnapshot(sessionId); - return { sessionId, expectedVersion: snapshot.revision }; - }), - ); - - assert.deepEqual( - new Set(await store.removeSessionsVersioned(identities)), - new Set(familyIds), - ); - assert.deepEqual( - new Set(await store.listPendingSessionRetirementCleanupIds(familyRevision.id)), - new Set(familyIds), - ); - for (const sessionId of familyIds) { - assert.deepEqual(await store.probeSessionRemoval(sessionId), { kind: 'removed' }); - await store.purgeRemovedSessionTranscript(sessionId); - await store.completeSessionRetirementCleanup(sessionId); - await assert.rejects(stat(join(root, 'sessions', sessionId)), { code: 'ENOENT' }); - } - assert.deepEqual(await store.listPendingSessionRetirementCleanupIds(), []); - } finally { - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('keeps aggregate retirement cleanup pending across storage restart', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-session-startup-cleanup-')); - const seed = createSessionStore(root); - let sessionId = ''; - try { - const session = await seed.create(makeInput({ name: 'Startup cleanup' })); - sessionId = session.id; - const snapshot = await seed.readHeaderRecordSnapshot(sessionId); - await seed.removeSessionsVersioned([{ sessionId, expectedVersion: snapshot.revision }]); - } finally { - await seed.close?.(); - } - - const reopened = createSessionStore(root); - try { - assert.deepEqual(await reopened.list(), []); - assert.deepEqual(await reopened.listPendingSessionRetirementCleanupIds(), [sessionId]); - await reopened.purgeRemovedSessionTranscript(sessionId); - await reopened.completeSessionRetirementCleanup(sessionId); - await assert.rejects(stat(join(root, 'sessions', sessionId)), { code: 'ENOENT' }); - assert.deepEqual(await reopened.listPendingSessionRetirementCleanupIds(), []); - } finally { - await reopened.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('skips a malformed header during startup migration while importing valid sessions', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-default-session-invalid-')); - const legacy = createLegacyFileSessionStore(root); - const valid = await legacy.create(makeInput({ name: 'Valid' })); - const invalid = await legacy.create(makeInput({ name: 'Invalid' })); - const invalidPath = join(root, 'sessions', invalid.id, 'session.jsonl'); - const lines = (await readFile(invalidPath, 'utf8')).split('\n'); - lines[0] = JSON.stringify({ ...JSON.parse(lines[0]!), labels: 'invalid' }); - await writeFile(invalidPath, lines.join('\n'), 'utf8'); - - const store = createSessionStore(root); - try { - // Startup import must not fail closed: valid sessions stay available. - assert.deepEqual( - (await store.list()).map((item) => item.id), - [valid.id], - ); - assert.equal((await store.readHeader(valid.id)).name, 'Valid'); - await assert.rejects(() => store.readHeader(invalid.id), /not found/); - } finally { - await store.close?.(); - } - - const metadata = createSqliteSessionMetadataStore( - join(root, SQLITE_SESSION_METADATA_DATABASE_NAME), - ); - try { - assert.equal((await metadata.read(valid.id)).header.name, 'Valid'); - assert.equal(await metadata.has(invalid.id), false); - // Malformed legacy headers are skipped, not tombstoned, so a repaired - // header can be imported on the next startup. - assert.equal(await metadata.isTombstoned(invalid.id), false); - } finally { - metadata.close(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('recovers an exact empty transcript marker left before SQLite admission', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-default-session-orphan-marker-')); - const sessionId = 'orphan-session'; - const sessionDir = join(root, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - `${JSON.stringify({ - type: 'session_transcript', - sessionId, - schemaVersion: 1, - })}\n`, - 'utf8', - ); - - const store = createSessionStore(root); - try { - assert.deepEqual(await store.list(), []); - await assert.rejects( - stat(sessionDir), - (error: NodeJS.ErrnoException) => error.code === 'ENOENT', - ); - } finally { - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('fails closed when an orphan transcript marker contains any additional state', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-default-session-nonempty-orphan-marker-')); - const sessionId = 'nonempty-orphan-session'; - const sessionDir = join(root, 'sessions', sessionId); - await mkdir(sessionDir, { recursive: true }); - await writeFile( - join(sessionDir, 'session.jsonl'), - `${JSON.stringify({ - type: 'session_transcript', - sessionId, - schemaVersion: 1, - })}\n${JSON.stringify({ type: 'user', id: 'message-1', text: 'must survive' })}\n`, - 'utf8', - ); - - const store = createSessionStore(root); - try { - await assert.rejects(() => store.list(), /has no SQLite metadata/); - assert.equal((await stat(join(sessionDir, 'session.jsonl'))).isFile(), true); - } finally { - await store.close?.(); - await rm(root, { recursive: true, force: true }); - } - }); -}); - -function assertCatalogQueryPlan( - database: DatabaseSync, - filter: SessionListFilter, - cursor: SqliteSessionCatalogCursor | undefined, - expectedIndex: string, - expectedAccess?: string, -): void { - const query = buildSqliteSessionCatalogPageQuery(filter, cursor); - const rows = database - .prepare(`EXPLAIN QUERY PLAN ${query.sql}`) - .all(...query.parameters, 33) as unknown as Array<{ detail: string }>; - const details = rows.map((row) => row.detail); - assert.equal( - details.some((detail) => detail.includes('USE TEMP B-TREE')), - false, - details.join('\n'), - ); - assert.equal( - details.some((detail) => detail.includes(expectedIndex)), - true, - details.join('\n'), - ); - if (expectedAccess) { - assert.equal( - details.some((detail) => detail.includes(expectedAccess) && detail.includes('activity_at')), - true, - details.join('\n'), - ); - } -} - -async function waitForMissingPath(path: string): Promise { - const deadline = Date.now() + 5_000; - while (true) { - try { - await stat(path); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; - throw error; - } - if (Date.now() >= deadline) throw new Error(`Timed out waiting for removal: ${path}`); - await new Promise((resolve) => setImmediate(resolve)); - } -} - -function makeInput(overrides: Partial = {}): CreateSessionInput { - return { - cwd: '/tmp/cwd', - backend: 'fake', - llmConnectionSlug: 'fake', - model: 'fake-model', - permissionMode: 'ask', - name: 'Session', - labels: [], - ...overrides, - }; -} - -function makeSubagentWorkspace(): SubagentWorkspaceBinding { - return { - schemaVersion: SUBAGENT_WORKSPACE_BINDING_SCHEMA_VERSION, - kind: 'git_worktree', - leaseId: `subagent_worktree_${'c'.repeat(32)}`, - gitCommonDir: '/workspace/repository/.git', - worktreePath: `/workspace/state/subagent-worktrees/${'c'.repeat(32)}`, - branch: `maka/subagent/${'c'.repeat(32)}`, - baseCommit: 'd'.repeat(40), - }; -} diff --git a/packages/storage/src/__tests__/sqlite-usage-store.test.ts b/packages/storage/src/__tests__/sqlite-usage-store.test.ts deleted file mode 100644 index 867fc892aa..0000000000 --- a/packages/storage/src/__tests__/sqlite-usage-store.test.ts +++ /dev/null @@ -1,160 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; -import { tmpdir } from 'node:os'; -import { join } from 'node:path'; -import { describe, test } from 'node:test'; -import { acquireOperationalStateDatabase } from '../operational-state-store.js'; -import { PricingStorePublicationError } from '../pricing-store.js'; -import { createSqlitePricingStore, createSqliteTelemetryRepo } from '../sqlite-usage-store.js'; - -describe('SQLite usage and pricing stores', () => { - test('resumes a cutover crash after copied rows without exposing partial state', async () => { - await withRoot(async (root) => { - await writeFile( - join(root, 'telemetry.json'), - JSON.stringify({ usageRecords: [llmRecord()], toolInvocations: [] }), - ); - const interrupted = createSqliteTelemetryRepo(root, { - managePricing: false, - failpoint: (point) => { - if (point === 'after_cutover_rows_copied') throw new Error('simulated crash'); - }, - }); - await assert.rejects(() => interrupted.load(), /simulated crash/); - await interrupted.close(); - - const resumed = createSqliteTelemetryRepo(root, { managePricing: false }); - await resumed.load(); - assert.equal(resumed.logs({ range: 'all' }).total, 1); - await resumed.close(); - }); - }); - - test('fails closed when a legacy source changes after completed cutover', async () => { - await withRoot(async (root) => { - const path = join(root, 'telemetry.json'); - await writeFile(path, JSON.stringify({ usageRecords: [llmRecord()], toolInvocations: [] })); - const first = createSqliteTelemetryRepo(root, { managePricing: false }); - await first.load(); - await first.close(); - - await writeFile( - path, - JSON.stringify({ - usageRecords: [llmRecord(), llmRecord({ id: 'usage_2' })], - toolInvocations: [], - }), - ); - const reopened = createSqliteTelemetryRepo(root, { managePricing: false }); - await assert.rejects( - () => reopened.load(), - /Legacy usage_pricing source changed after cutover completed/, - ); - await reopened.close(); - }); - }); - - test('writes only runtime.sqlite after cutover and preserves pricing revisions', async () => { - await withRoot(async (root) => { - const telemetry = createSqliteTelemetryRepo(root, { managePricing: false }); - const pricing = createSqlitePricingStore(root); - await telemetry.load(); - await pricing.load(); - await telemetry.insertLlmCall(llmRecord()); - const changed = await pricing.upsert(0, { - modelKey: 'openai:gpt-5', - inputUsdPer1M: 1.25, - outputUsdPer1M: 10, - }); - assert.equal(changed.snapshot.revision, 1); - await telemetry.close(); - await pricing.close(); - - await assert.rejects( - () => readFile(join(root, 'telemetry.json'), 'utf8'), - (error: NodeJS.ErrnoException) => error.code === 'ENOENT', - ); - await assert.rejects( - () => readFile(join(root, 'pricing.json'), 'utf8'), - (error: NodeJS.ErrnoException) => error.code === 'ENOENT', - ); - assert.ok((await readFile(join(root, 'runtime.sqlite'))).byteLength > 0); - - const reopenedTelemetry = createSqliteTelemetryRepo(root, { managePricing: false }); - const reopenedPricing = createSqlitePricingStore(root); - await reopenedTelemetry.load(); - await reopenedPricing.load(); - assert.equal(reopenedTelemetry.logs({ range: 'all' }).total, 1); - assert.deepEqual(reopenedPricing.snapshot(), changed.snapshot); - await reopenedTelemetry.close(); - await reopenedPricing.close(); - }); - }); - - test('rolls back the whole pricing revision when an override row cannot commit', async () => { - await withRoot(async (root) => { - const pricing = createSqlitePricingStore(root); - await pricing.load(); - await pricing.upsert(0, { - modelKey: 'provider:stable', - inputUsdPer1M: 1, - outputUsdPer1M: 2, - }); - const before = pricing.snapshot(); - const inspection = acquireOperationalStateDatabase(root); - inspection.database.exec(` - CREATE TRIGGER reject_test_pricing_override - BEFORE INSERT ON usage_pricing_overrides - WHEN NEW.model_key = 'provider:reject' - BEGIN - SELECT RAISE(ABORT, 'injected pricing row failure'); - END; - `); - await assert.rejects( - () => - pricing.upsert(before.revision, { - modelKey: 'provider:reject', - inputUsdPer1M: 3, - outputUsdPer1M: 4, - }), - PricingStorePublicationError, - ); - assert.deepEqual(pricing.snapshot(), before); - inspection.database.exec('DROP TRIGGER reject_test_pricing_override'); - inspection.close(); - await pricing.close(); - }); - }); -}); - -async function withRoot(run: (root: string) => Promise): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-sqlite-usage-')); - try { - await run(root); - } finally { - await rm(root, { recursive: true, force: true }); - } -} - -function llmRecord(overrides: Record = {}) { - return { - id: 'usage_1', - providerId: 'openai', - modelId: 'gpt-5', - inputTokens: 10, - outputTokens: 20, - cacheHitInputTokens: 0, - cacheMissInputTokens: 10, - cachedInputTokens: 0, - cacheWriteInputTokens: 0, - reasoningTokens: 0, - totalTokens: 30, - costUsd: 0.001, - latencyMs: 100, - status: 'success', - date: '2026-01-01', - ts: Date.UTC(2026, 0, 1), - startedAt: Date.UTC(2026, 0, 1) - 100, - ...overrides, - } as Parameters['insertLlmCall']>[0]; -} diff --git a/packages/storage/src/__tests__/sqlite-workflow-store.test.ts b/packages/storage/src/__tests__/sqlite-workflow-store.test.ts index 62126d764f..c2a11a07a2 100644 --- a/packages/storage/src/__tests__/sqlite-workflow-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-workflow-store.test.ts @@ -1,210 +1,100 @@ import assert from 'node:assert/strict'; -import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, test } from 'node:test'; -import { createDeepResearchStore, createSqliteDeepResearchStore } from '../deep-research-store.js'; -import { createPlanReminderStore, createSqlitePlanReminderStore } from '../plan-reminder-store.js'; -import { createPlanStore, createSqlitePlanStore } from '../plan-store.js'; -import { createSqliteTaskLedgerStore, createTaskLedgerStore } from '../task-ledger-store.js'; -import { acquireOperationalStateDatabase } from '../operational-state-store.js'; +import { createSqliteDeepResearchStore } from '../deep-research-store.js'; +import { createSqlitePlanReminderStore } from '../plan-reminder-store.js'; +import { createSqlitePlanStore } from '../plan-store.js'; +import { createSqliteTaskLedgerStore } from '../task-ledger-store.js'; const SESSION_ID = 'session-workflow'; -describe('SQLite workflow cutover', () => { - test('imports every workflow ledger once and keeps later writes SQLite-only', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-sqlite-workflow-')); - let planId = 0; - let researchId = 0; - try { - const legacyTasks = createTaskLedgerStore(root); - const { - created: [legacyTask], - } = await legacyTasks.create(SESSION_ID, [{ subject: 'Migrate workflow state' }]); - assert.ok(legacyTask); +describe('SQLite workflow stores', () => { + test('persists Task Ledger events and projections', async () => { + await withRoot(async (root) => { + const store = createSqliteTaskLedgerStore(root); + const { created } = await store.create(SESSION_ID, [{ subject: 'Implement SQLite' }]); + assert.equal(created[0]?.status, 'pending'); + store.close(); - const legacyPlan = createPlanStore(root, { - newId: () => `plan-id-${++planId}`, - now: () => 100 + planId, - }); - const submitted = await legacyPlan.submitProposal({ - sessionId: SESSION_ID, - turnId: 'turn-plan', - title: 'Workflow migration', - steps: [{ id: 'migrate', title: 'Migrate state', description: 'Move state to SQLite' }], - }); - - const legacyResearch = createDeepResearchStore(root, { - newId: () => `research-id-${++researchId}`, - now: () => 200 + researchId, - }); - await legacyResearch.start(SESSION_ID, 'Map the workflow stores', 'deep'); - - const legacyReminders = createPlanReminderStore(root); - const legacyReminder = await legacyReminders.create({ - title: 'Review migration', - runAt: Date.now() + 60_000, - }); - - const legacyPaths = [ - join(root, 'sessions', SESSION_ID, 'task-events.jsonl'), - join(root, 'sessions', SESSION_ID, 'tasks.json'), - join(root, 'sessions', SESSION_ID, 'plan-events.jsonl'), - join(root, 'sessions', SESSION_ID, 'plans.json'), - join(root, 'sessions', SESSION_ID, 'deep-research', 'events.jsonl'), - join(root, 'plan-reminders.json'), - ]; - const before = await Promise.all(legacyPaths.map((path) => readFile(path, 'utf8'))); + const reopened = createSqliteTaskLedgerStore(root); + try { + assert.equal((await reopened.list(SESSION_ID))[0]?.subject, 'Implement SQLite'); + } finally { + reopened.close(); + } + }); + }); - const tasks = createSqliteTaskLedgerStore(root); - const plan = createSqlitePlanStore(root, { - newId: () => `sqlite-plan-${++planId}`, - now: () => 400 + planId, + test('persists Plan events and their projection', async () => { + await withRoot(async (root) => { + const store = createSqlitePlanStore(root, { + newId: (() => { + let id = 0; + return () => `plan-${++id}`; + })(), + now: () => 100, }); - const research = createSqliteDeepResearchStore(root, { - newId: () => `sqlite-research-${++researchId}`, - now: () => 500 + researchId, - }); - const reminders = createSqlitePlanReminderStore(root); - await Promise.all([tasks.ready(), plan.ready(), research.ready(), reminders.ready()]); - - assert.equal((await tasks.get(SESSION_ID, legacyTask.id))?.subject, legacyTask.subject); - assert.equal( - (await plan.readState(SESSION_ID)).latestProposalId, - submitted.state.latestProposalId, - ); - assert.equal((await research.read(SESSION_ID))?.objective, 'Map the workflow stores'); - assert.equal((await reminders.list())[0]?.id, legacyReminder.id); - - await tasks.update(SESSION_ID, legacyTask.id, { status: 'in_progress' }); - await plan.requestRevision({ + const submitted = await store.submitProposal({ sessionId: SESSION_ID, - proposalId: submitted.state.latestProposalId!, + turnId: 'turn-1', + title: 'SQLite plan', + steps: [{ id: 'one', title: 'Persist state', description: 'Write one transaction' }], }); - await research.recordArtifact(SESSION_ID, { - artifactId: 'artifact-1', - role: 'source', - name: 'source.md', - createdAt: 501, - locator: 'https://example.com/source', - contentHash: `sha256:${'a'.repeat(64)}`, - sourceArtifactIds: [], - }); - await reminders.setEnabled(legacyReminder.id, false); - - assert.deepEqual( - await Promise.all(legacyPaths.map((path) => readFile(path, 'utf8'))), - before, - ); + store.close(); - const lease = acquireOperationalStateDatabase(root); + const reopened = createSqlitePlanStore(root); try { - const counts = lease.database - .prepare(` - SELECT - (SELECT COUNT(*) FROM workflow_task_ledger_events) AS tasks, - (SELECT COUNT(*) FROM workflow_plan_events) AS plans, - (SELECT COUNT(*) FROM workflow_deep_research_events) AS research, - (SELECT COUNT(*) FROM workflow_plan_reminders) AS reminders - `) - .get() as Record; - assert.deepEqual( - { ...counts }, - { - tasks: 2, - plans: 2, - research: 2, - reminders: 1, - }, + assert.equal( + (await reopened.readState(SESSION_ID)).latestProposalId, + submitted.state.latestProposalId, ); } finally { - lease.close(); + reopened.close(); } - - tasks.close(); - plan.close(); - research.close(); - reminders.close(); - - const reopenedTasks = createSqliteTaskLedgerStore(root); - const reopenedPlan = createSqlitePlanStore(root); - const reopenedResearch = createSqliteDeepResearchStore(root); - const reopenedReminders = createSqlitePlanReminderStore(root); - await Promise.all([ - reopenedTasks.ready(), - reopenedPlan.ready(), - reopenedResearch.ready(), - reopenedReminders.ready(), - ]); - assert.equal((await reopenedTasks.get(SESSION_ID, legacyTask.id))?.status, 'in_progress'); - assert.equal((await reopenedPlan.readState(SESSION_ID)).proposals[0]?.status, 'stale'); - assert.equal((await reopenedResearch.readEvents(SESSION_ID)).length, 2); - assert.equal( - (await reopenedReminders.list()).find((reminder) => reminder.id === legacyReminder.id) - ?.enabled, - false, - ); - reopenedTasks.close(); - reopenedPlan.close(); - reopenedResearch.close(); - reopenedReminders.close(); - } finally { - await rm(root, { recursive: true, force: true }); - } + }); }); - test('rolls back a copied workflow ledger and resumes the same cutover', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-sqlite-workflow-crash-')); - try { - await createTaskLedgerStore(root).create(SESSION_ID, [{ subject: 'Crash-safe import' }]); - const interrupted = createSqliteTaskLedgerStore(root, { - failpoint: (point) => { - if (point === 'after_cutover_rows_copied') throw new Error('simulated crash'); - }, + test('persists Deep Research events', async () => { + await withRoot(async (root) => { + const store = createSqliteDeepResearchStore(root, { + newId: () => 'research-1', + now: () => 200, }); - await assert.rejects(interrupted.ready(), /simulated crash/); - interrupted.close(); + await store.start(SESSION_ID, 'Map the SQLite authority', 'deep'); + store.close(); - const resumed = createSqliteTaskLedgerStore(root); - await resumed.ready(); - assert.equal((await resumed.list(SESSION_ID)).length, 1); - const lease = acquireOperationalStateDatabase(root); + const reopened = createSqliteDeepResearchStore(root); try { - const row = lease.database - .prepare(` - SELECT state - FROM cutover_journal - WHERE store_name = 'workflow_task_ledger' - `) - .get() as { state?: unknown }; - assert.equal(row.state, 'completed'); + assert.equal((await reopened.read(SESSION_ID))?.objective, 'Map the SQLite authority'); } finally { - lease.close(); + reopened.close(); } - resumed.close(); - } finally { - await rm(root, { recursive: true, force: true }); - } + }); }); - test('fails closed if a legacy workflow ledger changes after cutover', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-sqlite-workflow-fingerprint-')); - try { - const legacy = createTaskLedgerStore(root); - await legacy.create(SESSION_ID, [{ subject: 'Imported task' }]); - const migrated = createSqliteTaskLedgerStore(root); - await migrated.ready(); - migrated.close(); + test('persists Plan Reminders', async () => { + await withRoot(async (root) => { + const store = createSqlitePlanReminderStore(root); + const reminder = await store.create({ title: 'Review SQLite', runAt: Date.now() + 60_000 }); + store.close(); - await legacy.create(SESSION_ID, [{ subject: 'Unexpected legacy write' }]); - const reopened = createSqliteTaskLedgerStore(root); - await assert.rejects( - reopened.ready(), - /Legacy workflow_task_ledger source changed after cutover completed/, - ); - reopened.close(); - } finally { - await rm(root, { recursive: true, force: true }); - } + const reopened = createSqlitePlanReminderStore(root); + try { + assert.equal((await reopened.list())[0]?.id, reminder.id); + } finally { + reopened.close(); + } + }); }); }); + +async function withRoot(run: (root: string) => Promise): Promise { + const root = await mkdtemp(join(tmpdir(), 'maka-sqlite-workflow-')); + try { + await run(root); + } finally { + await rm(root, { recursive: true, force: true }); + } +} diff --git a/packages/storage/src/__tests__/task-ledger-authority.test.ts b/packages/storage/src/__tests__/task-ledger-authority.test.ts index 179671e88a..60187eb379 100644 --- a/packages/storage/src/__tests__/task-ledger-authority.test.ts +++ b/packages/storage/src/__tests__/task-ledger-authority.test.ts @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, test } from 'node:test'; @@ -52,65 +52,6 @@ describe('interactive task ledger authority', () => { }); }); - test('reads a current legacy tasks.json as canonical when no event ledger exists', async () => { - await withInteractiveOwner( - async ({ writer }) => { - const listed = await writer.list(SESSION_ID); - assert.equal(listed.length, 1); - assert.equal(listed[0]?.id, 'legacy-task'); - assert.equal(listed[0]?.key, 'T1'); - assert.equal((await writer.get(SESSION_ID, 'T1'))?.subject, 'legacy canonical task'); - }, - async (root) => { - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - await writeFile( - tasksPath(root), - JSON.stringify([ - { - id: 'legacy-task', - subject: 'legacy canonical task', - status: 'pending', - createdAt: 1, - updatedAt: 1, - }, - ]), - 'utf8', - ); - }, - ); - }); - - test('fails closed on a corrupt event ledger even when a legacy cache exists', async () => { - await withInteractiveRoot(async ({ root, capability }) => { - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - await writeFile( - tasksPath(root), - JSON.stringify([ - { - id: 'cached-task', - subject: 'must not become canonical', - status: 'pending', - createdAt: 1, - updatedAt: 1, - }, - ]), - 'utf8', - ); - await writeFile(eventsPath(root), '{"not":"a task event"}\n', 'utf8'); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert.ok(owner); - if (!owner) return; - try { - await assert.rejects( - () => openInteractiveTaskLedgerStoreForWrite(owner.lease), - /Invalid task event JSONL line/, - ); - } finally { - if (!owner.closed) await owner.close(); - } - }); - }); - test('rejects canonical reads and mutations after the owner releases its lease', async () => { await withInteractiveRoot(async ({ capability }) => { const owner = await tryAcquireInteractiveRootOwner(capability); @@ -161,10 +102,8 @@ describe('interactive task ledger authority', () => { async function withInteractiveOwner( run: (input: { root: string; writer: InteractiveTaskLedgerWriter }) => Promise, - prepare?: (root: string) => Promise, ): Promise { await withInteractiveRoot(async ({ root, capability }) => { - await prepare?.(root); const owner = await tryAcquireInteractiveRootOwner(capability); assert.ok(owner); if (!owner) return; @@ -204,14 +143,6 @@ async function withTempDir(run: (base: string) => Promise): Promise } } -function tasksPath(root: string): string { - return join(root, 'sessions', SESSION_ID, 'tasks.json'); -} - -function eventsPath(root: string): string { - return join(root, 'sessions', SESSION_ID, 'task-events.jsonl'); -} - function isInvalidLease(error: unknown): boolean { return error instanceof StorageRootAuthorityError && error.code === 'invalid_lease'; } diff --git a/packages/storage/src/__tests__/task-ledger-store.test.ts b/packages/storage/src/__tests__/task-ledger-store.test.ts deleted file mode 100644 index 21b544c40b..0000000000 --- a/packages/storage/src/__tests__/task-ledger-store.test.ts +++ /dev/null @@ -1,1338 +0,0 @@ -import { describe, it } from 'node:test'; -import assert from 'node:assert/strict'; -import { mkdtemp, mkdir, readFile, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; -import { tmpdir } from 'node:os'; -import { TASK_LEDGER_MAX_TASKS, TASK_SUBJECT_MAX_CHARS } from '@maka/core/task-ledger'; -import { createTaskLedgerStore } from '../task-ledger-store.js'; - -const SESSION_ID = 'sess-abc'; - -async function tempRoot(): Promise { - return mkdtemp(join(tmpdir(), 'maka-task-ledger-')); -} - -function tasksFilePath(root: string): string { - return join(root, 'sessions', SESSION_ID, 'tasks.json'); -} - -function taskEventsFilePath(root: string): string { - return join(root, 'sessions', SESSION_ID, 'task-events.jsonl'); -} - -describe('TaskLedgerStore', () => { - it('creates tasks with normalized subjects and pending status, returning created tasks and the new total', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - - const { created, total } = await store.create(SESSION_ID, [ - { subject: ' 写测试 ' }, - { subject: '实现功能' }, - ]); - assert.equal(created.length, 2); - assert.equal(created[0]?.subject, '写测试'); - assert.equal(created[0]?.status, 'pending'); - assert.equal(typeof created[0]?.id, 'string'); - assert.equal(created[0]?.createdAt, created[0]?.updatedAt); - assert.equal(total, 2); - - const reloaded = await createTaskLedgerStore(root).list(SESSION_ID); - assert.equal(reloaded.length, 2); - assert.deepEqual( - reloaded.map((t) => t.subject), - ['写测试', '实现功能'], - ); - - const raw = JSON.parse(await readFile(tasksFilePath(root), 'utf8')) as unknown[]; - assert.equal(raw.length, 2); - const eventLines = (await readFile(taskEventsFilePath(root), 'utf8')).trim().split('\n'); - assert.equal(eventLines.length, 2); - const firstEvent = JSON.parse(eventLines[0]!) as { type: string; refs?: unknown }; - assert.equal(firstEvent.type, 'task_created'); - }); - - it('persists mutation context refs in task events', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - - const { - created: [task], - } = await store.create(SESSION_ID, [{ subject: 'context task' }], { - runId: 'run-1', - turnId: 'turn-1', - toolCallId: 'call-1', - source: 'tool', - actor: 'main_agent', - }); - assert.ok(task); - await store.update(SESSION_ID, task.id, { status: 'in_progress' }); - await store.update( - SESSION_ID, - task.id, - { status: 'completed', completionEvidence: 'node --test passed' }, - { - runId: 'run-2', - turnId: 'turn-2', - toolCallId: 'call-2', - source: 'tool', - actor: 'main_agent', - }, - ); - - const events = (await readFile(taskEventsFilePath(root), 'utf8')) - .trim() - .split('\n') - .map( - (line) => - JSON.parse(line) as { - type: string; - refs?: { runId?: string; turnId?: string; toolCallId?: string }; - source?: string; - actor?: string; - }, - ); - assert.equal(events[0]?.type, 'task_created'); - assert.equal(events[0]?.refs?.runId, 'run-1'); - assert.equal(events[0]?.refs?.turnId, 'turn-1'); - assert.equal(events[0]?.refs?.toolCallId, 'call-1'); - assert.equal(events[0]?.source, 'tool'); - assert.equal(events[0]?.actor, 'main_agent'); - assert.equal(events[1]?.type, 'task_started'); - assert.equal(events[2]?.type, 'task_completed'); - assert.equal(events[2]?.refs?.runId, 'run-2'); - assert.equal(events[2]?.refs?.turnId, 'turn-2'); - assert.equal(events[2]?.refs?.toolCallId, 'call-2'); - }); - - it('copies Task Ledger state at a turn boundary and rewrites Session and Run ownership', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [task], - } = await store.create(SESSION_ID, [{ subject: 'copy boundary task' }], { - runId: 'source-run', - turnId: 'turn-retained', - source: 'tool', - actor: 'main_agent', - }); - assert.ok(task); - await store.update( - SESSION_ID, - task.id, - { status: 'in_progress' }, - { - runId: 'later-run', - turnId: 'turn-later', - source: 'tool', - actor: 'main_agent', - }, - ); - - await assert.rejects( - () => - store.copyConversationTaskLedger({ - sourceSessionId: SESSION_ID, - targetSessionId: 'session-copy-missing-run', - turnIds: ['turn-retained'], - runIdMap: [], - }), - /missing AgentRun source-run/, - ); - assert.deepEqual(await store.list('session-copy-missing-run'), []); - - await store.copyConversationTaskLedger({ - sourceSessionId: SESSION_ID, - targetSessionId: 'session-copy', - turnIds: ['turn-retained'], - runIdMap: [{ sourceRunId: 'source-run', targetRunId: 'target-run' }], - }); - - const copied = await store.list('session-copy'); - assert.equal(copied.length, 1); - assert.equal(copied[0]?.status, 'pending'); - assert.equal(copied[0]?.owner?.runId, 'target-run'); - const [event] = ( - await readFile(join(root, 'sessions', 'session-copy', 'task-events.jsonl'), 'utf8') - ) - .trim() - .split('\n') - .map( - (line) => - JSON.parse(line) as { - sessionId: string; - refs?: { runId?: string; turnId?: string }; - }, - ); - assert.equal(event?.sessionId, 'session-copy'); - assert.deepEqual(event?.refs, { - runId: 'target-run', - turnId: 'turn-retained', - }); - await store.purgeConversationTaskLedger('session-copy'); - assert.deepEqual(await store.list('session-copy'), []); - assert.equal((await store.list(SESSION_ID))[0]?.status, 'in_progress'); - }); - - it('clears stale evidence when tasks leave evidence-bearing statuses', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [task], - } = await store.create(SESSION_ID, [{ subject: 'evidence lifecycle' }]); - assert.ok(task); - - await store.update(SESSION_ID, task.id, { status: 'in_progress' }); - const blocked = await store.update(SESSION_ID, task.id, { - status: 'blocked', - blockedReason: 'waiting for user input', - }); - assert.equal(blocked.updated.blockedReason, 'waiting for user input'); - - const resumed = await store.update(SESSION_ID, task.id, { status: 'in_progress' }); - assert.equal(resumed.updated.status, 'in_progress'); - assert.equal(resumed.updated.blockedReason, undefined); - - const completed = await store.update(SESSION_ID, task.id, { - status: 'completed', - completionEvidence: 'node --test passed', - }); - assert.equal(completed.updated.completionEvidence, 'node --test passed'); - - const reopened = await store.update(SESSION_ID, task.id, { - status: 'in_progress', - explicitReopen: true, - }); - assert.equal(reopened.updated.status, 'in_progress'); - assert.equal(reopened.updated.completionEvidence, undefined); - - const listed = await store.list(SESSION_ID); - const reloaded = listed.find((t) => t.id === task.id); - assert.equal(reloaded?.blockedReason, undefined); - assert.equal(reloaded?.completionEvidence, undefined); - }); - - it('requires explicit reopen and records task_reopened events', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [task], - } = await store.create(SESSION_ID, [{ subject: 'reopen task' }]); - assert.ok(task); - - await store.update(SESSION_ID, task.id, { status: 'in_progress' }); - await store.update(SESSION_ID, task.id, { - status: 'completed', - completionEvidence: 'verified', - }); - await assert.rejects( - () => store.update(SESSION_ID, task.id, { status: 'in_progress' }), - /Invalid task status transition/, - ); - - const reopened = await store.update(SESSION_ID, task.id, { - status: 'in_progress', - explicitReopen: true, - }); - assert.equal(reopened.updated.status, 'in_progress'); - assert.equal('explicitReopen' in reopened.updated, false); - - const events = (await readFile(taskEventsFilePath(root), 'utf8')) - .trim() - .split('\n') - .map( - (line) => - JSON.parse(line) as { - type: string; - previousStatus?: string; - nextStatus?: string; - task?: Record; - }, - ); - const reopenEvent = events.at(-1); - assert.equal(reopenEvent?.type, 'task_reopened'); - assert.equal(reopenEvent?.previousStatus, 'completed'); - assert.equal(reopenEvent?.nextStatus, 'in_progress'); - assert.equal(reopenEvent?.task?.completionEvidence, undefined); - }); - - it('lists an empty ledger when the file does not exist', async () => { - const root = await tempRoot(); - assert.deepEqual(await createTaskLedgerStore(root).list(SESSION_ID), []); - }); - - it('gets one task by id and classifies resume trust on recovery reads', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [task], - } = await store.create(SESSION_ID, [{ subject: 'recover me' }]); - assert.ok(task); - await store.update(SESSION_ID, task.id, { status: 'in_progress' }); - - const plain = await store.get(SESSION_ID, task.id); - assert.equal(plain?.resumeTrust, undefined); - const classified = await store.get(SESSION_ID, task.id, { classifyResumeTrust: true }); - assert.equal(classified?.resumeTrust, 'stale'); - assert.equal(await store.get(SESSION_ID, 'missing'), undefined); - }); - - it('updates a task status and subject, returning the updated task and the new total', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [task], - total: afterCreate, - } = await store.create(SESSION_ID, [{ subject: '原始' }, { subject: '其他' }]); - assert.ok(task); - assert.equal(afterCreate, 2); - - const { updated, total } = await store.update(SESSION_ID, task.id, { - status: 'in_progress', - subject: '改过', - }); - assert.equal(updated.status, 'in_progress'); - assert.equal(updated.subject, '改过'); - assert.ok(updated.updatedAt >= task.updatedAt); - assert.equal(updated.createdAt, task.createdAt); - // total is the post-mutation count from inside the write queue; re-read the - // ledger to verify the updated task landed and the file matches it. - assert.equal(total, 2); - const all = await store.list(SESSION_ID); - assert.deepEqual( - all.find((t) => t.id === task.id), - updated, - ); - - const reloaded = await createTaskLedgerStore(root).list(SESSION_ID); - assert.deepEqual(reloaded, all); - }); - - it('rejects an unknown task id, an empty patch, an invalid status, and empty create drafts', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [task], - } = await store.create(SESSION_ID, [{ subject: 'x' }]); - assert.ok(task); - - await assert.rejects( - () => - store.update(SESSION_ID, 'no-such-id', { status: 'completed', completionEvidence: 'done' }), - /No such task/, - ); - await assert.rejects(() => store.update(SESSION_ID, task.id, {}), /at least one/); - await assert.rejects( - () => store.update(SESSION_ID, task.id, { status: 'bogus' }), - /Task status/, - ); - await store.update(SESSION_ID, task.id, { status: 'in_progress' }); - await assert.rejects( - () => store.update(SESSION_ID, task.id, { status: 'completed' }), - /completionEvidence/, - ); - await assert.rejects(() => store.create(SESSION_ID, []), /at least one/); - await assert.rejects(() => store.create(SESSION_ID, [{ subject: ' ' }]), /empty/); - }); - - it('does not rewrite the file when the update target does not exist', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - await store.create(SESSION_ID, [{ subject: 'x' }]); - const before = await readFile(tasksFilePath(root), 'utf8'); - - await assert.rejects( - () => - store.update(SESSION_ID, 'no-such-id', { status: 'completed', completionEvidence: 'done' }), - /No such task/, - ); - - const after = await readFile(tasksFilePath(root), 'utf8'); - assert.equal(after, before); - }); - - it('degrades a corrupt ledger to an empty list on the render path', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - await writeFile(tasksFilePath(root), 'not json at all', 'utf8'); - assert.deepEqual(await createTaskLedgerStore(root).list(SESSION_ID), []); - }); - - it('refuses to mutate over a corrupt ledger and leaves the file untouched', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - const store = createTaskLedgerStore(root); - - for (const corrupt of ['not json at all', '{"not":"an array"}']) { - await writeFile(tasksFilePath(root), corrupt, 'utf8'); - await assert.rejects( - () => store.create(SESSION_ID, [{ subject: '新任务' }]), - /corrupt; refusing to overwrite/, - ); - await assert.rejects( - () => - store.update(SESSION_ID, 'any-id', { status: 'completed', completionEvidence: 'done' }), - /corrupt; refusing to overwrite/, - ); - // The mutation must not have replaced the damaged file with fn([]). - assert.equal(await readFile(tasksFilePath(root), 'utf8'), corrupt); - // The render path still degrades to empty so turns are not wedged. - assert.deepEqual(await store.list(SESSION_ID), []); - } - }); - - it('treats a corrupt task event log as authoritative corruption', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - await store.create(SESSION_ID, [{ subject: 'x' }]); - await writeFile(taskEventsFilePath(root), 'not json\n', 'utf8'); - - const rendered = await store.list(SESSION_ID); - assert.equal(rendered.length, 1); - assert.equal(rendered[0]?.subject, 'x'); - assert.equal(rendered[0]?.resumeTrust, 'untrusted'); - await assert.rejects( - () => store.create(SESSION_ID, [{ subject: 'new' }]), - /task event ledger|Invalid task event/i, - ); - }); - - it('degrades to an empty render ledger when corrupt task events have no readable cache', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - await writeFile(taskEventsFilePath(root), 'not json\n', 'utf8'); - await writeFile(tasksFilePath(root), '{not json', 'utf8'); - - const store = createTaskLedgerStore(root); - assert.deepEqual(await store.list(SESSION_ID), []); - await assert.rejects( - () => store.create(SESSION_ID, [{ subject: 'new' }]), - /task event ledger|Invalid task event/i, - ); - assert.equal(await readFile(taskEventsFilePath(root), 'utf8'), 'not json\n'); - assert.equal(await readFile(tasksFilePath(root), 'utf8'), '{not json'); - }); - - it('drops malformed entries while keeping valid ones', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - await writeFile( - tasksFilePath(root), - JSON.stringify([ - { id: 'good', subject: '有效', status: 'pending', createdAt: 1, updatedAt: 1 }, - { id: 'bad-status', subject: 'x', status: 'nope', createdAt: 1, updatedAt: 1 }, - { subject: 'no id', status: 'pending', createdAt: 1, updatedAt: 1 }, - 'garbage', - ]), - 'utf8', - ); - const tasks = await createTaskLedgerStore(root).list(SESSION_ID); - assert.equal(tasks.length, 1); - assert.equal(tasks[0]?.id, 'good'); - }); - - it('re-applies subject normalization on read: discards overlong/blank/empty subjects and normalizes whitespace', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - await writeFile( - tasksFilePath(root), - JSON.stringify([ - { - id: 'overlong', - subject: 'X'.repeat(TASK_SUBJECT_MAX_CHARS + 1), - status: 'pending', - createdAt: 1, - updatedAt: 1, - }, - { id: 'blank', subject: ' ', status: 'pending', createdAt: 1, updatedAt: 1 }, - { id: 'empty', subject: '', status: 'pending', createdAt: 1, updatedAt: 1 }, - { - id: 'whitespace', - subject: 'a\t\tb\n\nc d', - status: 'pending', - createdAt: 1, - updatedAt: 1, - }, - { id: 'good', subject: '有效', status: 'pending', createdAt: 1, updatedAt: 1 }, - ]), - 'utf8', - ); - const tasks = await createTaskLedgerStore(root).list(SESSION_ID); - // overlong/blank/empty subjects are discarded per-record; good + whitespace survive. - assert.equal( - tasks.length, - 2, - `expected 2 surviving tasks, got ${tasks.length}: ${JSON.stringify(tasks.map((t) => t.id))}`, - ); - const ids = tasks.map((t) => t.id); - assert.ok(ids.includes('good')); - assert.ok(ids.includes('whitespace')); - // whitespace subject is normalized (collapse + trim) on read. - const ws = tasks.find((t) => t.id === 'whitespace'); - assert.equal( - ws?.subject, - 'a b c d', - `expected normalized subject, got ${JSON.stringify(ws?.subject)}`, - ); - }); - - it('treats an over-cap tasks.json as corrupt: list() rejects and mutate stays fail-closed', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - const overcap = Array.from({ length: TASK_LEDGER_MAX_TASKS + 1 }, (_, i) => ({ - id: `cap-${i}`, - subject: `任务${i}`, - status: 'pending', - createdAt: i, - updatedAt: i, - })); - await writeFile(tasksFilePath(root), JSON.stringify(overcap), 'utf8'); - const store = createTaskLedgerStore(root); - // render path degrades to an empty list (readForRender try/catches the over-cap file) - assert.deepEqual(await store.list(SESSION_ID), []); - // mutate path stays fail-closed: a create must not silently truncate-and-overwrite the over-cap file - await assert.rejects( - () => store.create(SESSION_ID, [{ subject: '新任务' }]), - /corrupt|limit|exceed/i, - ); - // the file is left untouched (not truncated) - const raw = await readFile(tasksFilePath(root), 'utf8'); - assert.equal(JSON.parse(raw).length, TASK_LEDGER_MAX_TASKS + 1); - }); - - it('treats a tasks.json with duplicate ids as corrupt: render degrades to empty, mutate stays fail-closed', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - await writeFile( - tasksFilePath(root), - JSON.stringify([ - { id: 'dup-id', subject: 'first', status: 'pending', createdAt: 1, updatedAt: 1 }, - { id: 'dup-id', subject: 'second', status: 'pending', createdAt: 2, updatedAt: 2 }, - { id: 'uniq', subject: 'unique', status: 'pending', createdAt: 3, updatedAt: 3 }, - ]), - 'utf8', - ); - const store = createTaskLedgerStore(root); - // render path degrades to empty: no duplicate id reaches the turn tail - // (two same-id tasks would be indistinguishable to the model). - assert.deepEqual(await store.list(SESSION_ID), []); - // mutate path stays fail-closed: an update must not silently keep both - // dups and rewrite a "half-correct" file (first updated, second stale). - await assert.rejects( - () => store.update(SESSION_ID, 'dup-id', { status: 'completed', completionEvidence: 'done' }), - /corrupt|duplicate|ambiguous/i, - ); - const raw = await readFile(tasksFilePath(root), 'utf8'); - assert.equal(JSON.parse(raw).length, 3, 'file must be left untouched'); - }); - - it('rejects non-finite timestamps (1e999 -> Infinity) so they cannot round-trip to null and vanish', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - // raw JSON with 1e999, which JSON.parse reads as Infinity; JSON.stringify of - // Infinity is null, so writing via JSON.stringify could not reproduce this -- - // only a hand-edited or legacy file carries it. - await writeFile( - tasksFilePath(root), - '[{"id":"good","subject":"ok","status":"pending","createdAt":1,"updatedAt":1},' + - '{"id":"bad-ts","subject":"inf","status":"pending","createdAt":1e999,"updatedAt":1e999}]', - 'utf8', - ); - const store = createTaskLedgerStore(root); - // read path drops the non-finite record (render degrades to the valid one) - assert.deepEqual( - (await store.list(SESSION_ID)).map((t) => t.id), - ['good'], - ); - // mutate path: a create must not round-trip the Infinity to null - await store.create(SESSION_ID, [{ subject: 'after' }]); - const raw = JSON.parse(await readFile(tasksFilePath(root), 'utf8')) as Array<{ - createdAt: unknown; - updatedAt: unknown; - }>; - for (const r of raw) { - assert.equal( - Number.isFinite(r.createdAt), - true, - `createdAt must stay finite after mutate, got ${JSON.stringify(r)}`, - ); - assert.equal( - Number.isFinite(r.updatedAt), - true, - `updatedAt must stay finite after mutate, got ${JSON.stringify(r)}`, - ); - } - }); - - it('rejects records with unsafe ids (newline, overlong, empty, whitespace) on read', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - await writeFile( - tasksFilePath(root), - JSON.stringify([ - { id: 'abc\nINJECTED', subject: '换行id', status: 'pending', createdAt: 1, updatedAt: 1 }, - { id: 'X'.repeat(5000), subject: '超长id', status: 'pending', createdAt: 2, updatedAt: 2 }, - { id: '', subject: '空id', status: 'pending', createdAt: 3, updatedAt: 3 }, - { id: 'has space', subject: '带空格id', status: 'pending', createdAt: 4, updatedAt: 4 }, - { id: 'good-id', subject: '正常', status: 'pending', createdAt: 5, updatedAt: 5 }, - ]), - 'utf8', - ); - const tasks = await createTaskLedgerStore(root).list(SESSION_ID); - assert.equal( - tasks.length, - 1, - `expected only the safe-id record to survive, got ${JSON.stringify(tasks.map((t) => t.id))}`, - ); - assert.equal(tasks[0]?.id, 'good-id'); - }); - - it('rejects ids that are not redaction-stable tokens (tag-like, angle brackets, quotes, parens, secret-shaped); keeps UUID-shaped and simple ids', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - await writeFile( - tasksFilePath(root), - JSON.stringify([ - { - id: 'ab', - subject: 'tag-like', - status: 'pending', - createdAt: 1, - updatedAt: 1, - }, - { id: 'a>b', subject: 'gt', status: 'pending', createdAt: 2, updatedAt: 2 }, - { id: 'a"b', subject: 'quote', status: 'pending', createdAt: 3, updatedAt: 3 }, - { id: 'a(b)', subject: 'paren', status: 'pending', createdAt: 4, updatedAt: 4 }, - { id: 'a=b', subject: 'equals', status: 'pending', createdAt: 5, updatedAt: 5 }, - // secret-shaped stable tokens: pass the charset/length rules but redactSecrets - // would render them as (id: [redacted]), so TaskUpdate on [redacted] would miss. - { - id: 'ghp_abcdefghijklmnopqrstuvwxyz', - subject: 'ghp', - status: 'pending', - createdAt: 6, - updatedAt: 6, - }, - { id: 'sk-abcdefghi', subject: 'sk', status: 'pending', createdAt: 7, updatedAt: 7 }, - { id: 'a'.repeat(40), subject: 'hex40', status: 'pending', createdAt: 8, updatedAt: 8 }, - { - id: 'AIza' + 'X'.repeat(24), - subject: 'aiza', - status: 'pending', - createdAt: 9, - updatedAt: 9, - }, - { - id: '123e4567-e89b-12d3-a456-426614174000', - subject: 'uuid', - status: 'pending', - createdAt: 10, - updatedAt: 10, - }, - { id: 'good-id_1:2', subject: 'simple', status: 'pending', createdAt: 11, updatedAt: 11 }, - ]), - 'utf8', - ); - const tasks = await createTaskLedgerStore(root).list(SESSION_ID); - const ids = tasks.map((t) => t.id); - // Only ids that are stable tokens AND survive redaction (so the rendered id - // equals the stored id) survive; a TaskUpdate on the rendered id then hits. - assert.deepEqual(ids, ['123e4567-e89b-12d3-a456-426614174000', 'good-id_1:2']); - }); - - it('rejects an oversized batch before generating tasks or writing (existing ledger unchanged)', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - await store.create(SESSION_ID, [{ subject: 'seed' }]); - const before = await readFile(tasksFilePath(root), 'utf8'); - // Oversized batch with an invalid draft in the middle: without an early - // batch-size check, normalizeCreateTaskInput runs during `drafts.map` and - // throws the per-draft subject error; with the early check, the batch is - // rejected as a batch before any draft is touched or any id is generated. - const batch = Array.from({ length: TASK_LEDGER_MAX_TASKS + 5 }, (_, i) => - i === 2 ? { subject: '' } : { subject: `任务${i}` }, - ); - await assert.rejects(() => store.create(SESSION_ID, batch), /cap|limit|exceed|batch/i); - const after = await readFile(tasksFilePath(root), 'utf8'); - assert.equal(after, before, 'existing ledger must be unchanged'); - }); - - it('rejects an unsafe session id', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - await assert.rejects(() => store.list('../escape'), /Invalid session id/); - }); - - it('serializes concurrent creates without losing writes', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - await Promise.all([ - store.create(SESSION_ID, [{ subject: 'a' }]), - store.create(SESSION_ID, [{ subject: 'b' }]), - store.create(SESSION_ID, [{ subject: 'c' }]), - ]); - const tasks = await store.list(SESSION_ID); - assert.equal(tasks.length, 3); - assert.deepEqual(new Set(tasks.map((t) => t.subject)), new Set(['a', 'b', 'c'])); - }); - - it('enforces the total-task cap inside the write queue without touching the file', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const fill = Array.from({ length: TASK_LEDGER_MAX_TASKS }, (_, i) => ({ subject: `t${i}` })); - await store.create(SESSION_ID, fill); - - // Over-cap create must reject with a clear total-count message... - await assert.rejects( - () => store.create(SESSION_ID, [{ subject: 'overflow' }]), - new RegExp(`limited to ${TASK_LEDGER_MAX_TASKS} tasks total`), - ); - - // ...and must not have written anything: the ledger is unchanged. - const tasks = await store.list(SESSION_ID); - assert.equal(tasks.length, TASK_LEDGER_MAX_TASKS); - assert.equal( - tasks.some((t) => t.subject === 'overflow'), - false, - ); - - // Completing tasks does not free capacity: the cap is on total count. - const first = tasks[0]; - assert.ok(first); - await store.update(SESSION_ID, first.id, { status: 'in_progress' }); - await store.update(SESSION_ID, first.id, { status: 'completed', completionEvidence: 'done' }); - await assert.rejects( - () => store.create(SESSION_ID, [{ subject: 'still-over' }]), - /hard runaway guard/, - ); - - // A single batch larger than the cap rejects at the front door (per-batch - // cap, before generating ids), so the ledger stays empty. - const freshStore = createTaskLedgerStore(await tempRoot()); - const oversizedBatch = Array.from({ length: TASK_LEDGER_MAX_TASKS + 1 }, (_, i) => ({ - subject: `b${i}`, - })); - await assert.rejects(() => freshStore.create(SESSION_ID, oversizedBatch), /per-batch cap/); - assert.deepEqual(await freshStore.list(SESSION_ID), []); - }); - - it('persists blocked failed completed evidence fields and resumeTrust when present on disk', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [first, second, third], - } = await store.create(SESSION_ID, [ - { subject: 'blocked task' }, - { subject: 'failed task' }, - { subject: 'completed task' }, - ]); - assert.ok(first); - assert.ok(second); - assert.ok(third); - - await store.update(SESSION_ID, first.id, { status: 'in_progress' }); - await store.update(SESSION_ID, second.id, { status: 'in_progress' }); - await store.update(SESSION_ID, third.id, { status: 'in_progress' }); - await store.update(SESSION_ID, first.id, { - status: 'blocked', - blockedReason: 'waiting for user approval', - }); - await store.update(SESSION_ID, second.id, { - status: 'failed', - failureReason: 'test suite cannot pass', - }); - await store.update(SESSION_ID, third.id, { - status: 'completed', - completionEvidence: 'npm test passed', - }); - - const reloaded = await createTaskLedgerStore(root).list(SESSION_ID); - assert.equal( - reloaded.find((t) => t.id === first.id)?.blockedReason, - 'waiting for user approval', - ); - assert.equal(reloaded.find((t) => t.id === second.id)?.failureReason, 'test suite cannot pass'); - assert.equal(reloaded.find((t) => t.id === third.id)?.completionEvidence, 'npm test passed'); - - const events = (await readFile(taskEventsFilePath(root), 'utf8')) - .trim() - .split('\n') - .map((line) => JSON.parse(line) as { type: string }); - assert.deepEqual( - events.map((event) => event.type), - [ - 'task_created', - 'task_created', - 'task_created', - 'task_started', - 'task_started', - 'task_started', - 'task_blocked', - 'task_failed', - 'task_completed', - ], - ); - }); - - it('prefers task event replay over a stale tasks.json cache', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [task], - } = await store.create(SESSION_ID, [{ subject: 'event source' }]); - assert.ok(task); - await writeFile(tasksFilePath(root), JSON.stringify([], null, 2), 'utf8'); - const tasks = await createTaskLedgerStore(root).list(SESSION_ID); - assert.equal(tasks.length, 1); - assert.equal(tasks[0]?.subject, 'event source'); - }); - - it('falls back to legacy tasks.json when no task event log exists', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - await writeFile( - tasksFilePath(root), - JSON.stringify([ - { - id: 'legacy-task', - subject: 'old task', - status: 'pending', - createdAt: 1, - updatedAt: 1, - owner: { - actor: 'child_agent', - sessionId: 'child-session', - agentId: 'local-read', - runId: 'child-run', - turnId: 'child-turn', - }, - }, - ]), - 'utf8', - ); - const tasks = await createTaskLedgerStore(root).list(SESSION_ID); - assert.deepEqual( - tasks.map((t) => t.id), - ['legacy-task'], - ); - assert.deepEqual(tasks[0]?.owner, { - actor: 'child_agent', - sessionId: 'child-session', - agentId: 'local-read', - runId: 'child-run', - turnId: 'child-turn', - }); - }); - - it('imports legacy tasks into the event log before appending the first create', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - await writeFile( - tasksFilePath(root), - JSON.stringify([ - { id: 'legacy-task', subject: 'old task', status: 'pending', createdAt: 1, updatedAt: 1 }, - ]), - 'utf8', - ); - const store = createTaskLedgerStore(root); - - const { - created: [created], - total, - } = await store.create(SESSION_ID, [{ subject: 'new task' }]); - assert.ok(created); - assert.equal(total, 2); - - const reloaded = await createTaskLedgerStore(root).list(SESSION_ID); - assert.deepEqual( - reloaded.map((t) => t.id), - ['legacy-task', created.id], - ); - - const events = (await readFile(taskEventsFilePath(root), 'utf8')) - .trim() - .split('\n') - .map( - (line) => - JSON.parse(line) as { type: string; taskId: string; source?: string; actor?: string }, - ); - assert.deepEqual( - events.map((event) => event.type), - ['task_imported', 'task_created'], - ); - assert.equal(events[0]?.taskId, 'legacy-task'); - assert.equal(events[0]?.source, 'import'); - assert.equal(events[0]?.actor, 'system'); - assert.equal(events[1]?.taskId, created.id); - }); - - it('imports legacy tasks into the event log before appending the first update', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - await writeFile( - tasksFilePath(root), - JSON.stringify([ - { id: 'legacy-task', subject: 'old task', status: 'pending', createdAt: 1, updatedAt: 1 }, - ]), - 'utf8', - ); - const store = createTaskLedgerStore(root); - - const { updated, total } = await store.update(SESSION_ID, 'legacy-task', { - status: 'in_progress', - }); - assert.equal(updated.status, 'in_progress'); - assert.equal(total, 1); - - const reloaded = await createTaskLedgerStore(root).list(SESSION_ID); - assert.deepEqual( - reloaded.map((t) => [t.id, t.status]), - [['legacy-task', 'in_progress']], - ); - - const events = (await readFile(taskEventsFilePath(root), 'utf8')) - .trim() - .split('\n') - .map((line) => JSON.parse(line) as { type: string; taskId: string }); - assert.deepEqual( - events.map((event) => event.type), - ['task_imported', 'task_started'], - ); - assert.equal(events[0]?.taskId, 'legacy-task'); - assert.equal(events[1]?.taskId, 'legacy-task'); - }); - - it('keeps legacy completed and cancelled tasks readable without evidence', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - await writeFile( - tasksFilePath(root), - JSON.stringify([ - { - id: 'legacy-completed', - subject: 'old done', - status: 'completed', - createdAt: 1, - updatedAt: 1, - }, - { - id: 'legacy-cancelled', - subject: 'old cancelled', - status: 'cancelled', - createdAt: 2, - updatedAt: 2, - }, - ]), - 'utf8', - ); - const tasks = await createTaskLedgerStore(root).list(SESSION_ID); - assert.deepEqual( - tasks.map((t) => t.id), - ['legacy-completed', 'legacy-cancelled'], - ); - }); - - it('can retry a failed task as pending without poisoning event replay', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [task], - } = await store.create(SESSION_ID, [{ subject: 'retry me' }]); - assert.ok(task); - await store.update(SESSION_ID, task.id, { status: 'in_progress' }); - await store.update(SESSION_ID, task.id, { status: 'failed', failureReason: 'test failed' }); - - const retried = await store.update(SESSION_ID, task.id, { status: 'pending' }); - assert.equal(retried.updated.status, 'pending'); - assert.equal(retried.updated.failureReason, undefined); - - const reloadedStore = createTaskLedgerStore(root); - assert.equal((await reloadedStore.get(SESSION_ID, task.id))?.status, 'pending'); - const { - created: [afterRetry], - total, - } = await reloadedStore.create(SESSION_ID, [{ subject: 'after retry' }]); - assert.ok(afterRetry); - assert.equal(total, 2); - - const events = (await readFile(taskEventsFilePath(root), 'utf8')) - .trim() - .split('\n') - .map((line) => JSON.parse(line) as { type: string; taskId: string }); - assert.deepEqual( - events.map((event) => event.type), - ['task_created', 'task_started', 'task_failed', 'task_reopened', 'task_created'], - ); - }); - - it('rejects blocked failed and completed updates without required evidence without rewriting', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [task], - } = await store.create(SESSION_ID, [{ subject: 'x' }]); - assert.ok(task); - await store.update(SESSION_ID, task.id, { status: 'in_progress' }); - const before = await readFile(tasksFilePath(root), 'utf8'); - - await assert.rejects( - () => store.update(SESSION_ID, task.id, { status: 'blocked' }), - /blockedReason/, - ); - await assert.rejects( - () => store.update(SESSION_ID, task.id, { status: 'failed' }), - /failureReason/, - ); - await assert.rejects( - () => store.update(SESSION_ID, task.id, { status: 'completed' }), - /completionEvidence/, - ); - assert.equal(await readFile(tasksFilePath(root), 'utf8'), before); - }); - - it('allocates stable hierarchical keys under concurrent creates and resolves key or UUID', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [parent], - } = await store.create(SESSION_ID, [{ subject: 'parent' }]); - assert.ok(parent); - const batches = await Promise.all([ - store.create(SESSION_ID, [{ subject: 'child a', parentId: parent.key }]), - store.create(SESSION_ID, [{ subject: 'child b', parentId: parent.id }]), - store.create(SESSION_ID, [{ subject: 'child c', parentId: parent.key }]), - ]); - const children = batches.flatMap((batch) => batch.created); - assert.deepEqual(new Set(children.map((task) => task.key)), new Set(['T1.1', 'T1.2', 'T1.3'])); - assert.equal( - (await store.get(SESSION_ID, 'T1.2'))?.id, - children.find((task) => task.key === 'T1.2')?.id, - ); - await store.update(SESSION_ID, children[0]!.key, { status: 'in_progress' }); - assert.equal((await store.get(SESSION_ID, children[0]!.id))?.status, 'in_progress'); - assert.deepEqual( - (await createTaskLedgerStore(root).list(SESSION_ID)).map((task) => task.key), - ['T1', 'T1.1', 'T1.2', 'T1.3'], - ); - }); - - it('rejects completing a parent with active descendants', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [parent], - } = await store.create(SESSION_ID, [{ subject: 'parent' }]); - assert.ok(parent); - await store.create(SESSION_ID, [{ subject: 'child', parentId: parent.key }]); - await store.update(SESSION_ID, parent.id, { status: 'in_progress' }); - await assert.rejects( - () => - store.update(SESSION_ID, parent.key, { - status: 'completed', - completionEvidence: 'parent done', - }), - /descendant T1\.1 is pending/, - ); - }); - - it('atomically self-claims only available tasks and limits one task per child turn', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [first, second], - } = await store.create( - SESSION_ID, - [{ subject: 'first shared task' }, { subject: 'second shared task' }], - { actor: 'main_agent', runId: 'lead-run', turnId: 'lead-turn' }, - ); - assert.ok(first && second); - const owner = { - actor: 'child_agent' as const, - agentId: 'local-read', - turnId: 'child-turn', - }; - const scope = { parentRunId: 'lead-run' }; - const claimed = await store.claimAvailable(SESSION_ID, first.id, owner, scope, { - actor: 'child_agent', - source: 'tool', - }); - assert.equal(claimed.updated.status, 'in_progress'); - assert.deepEqual(claimed.updated.owner, owner); - assert.equal( - (await store.claimAvailable(SESSION_ID, first.id, owner, scope)).updated.id, - first.id, - ); - await assert.rejects( - () => store.claimAvailable(SESSION_ID, second.id, owner, scope), - /already owns task T1/, - ); - }); - - it('refuses self-claim of unowned tasks or tasks shared by another lead run', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [unowned], - } = await store.create(SESSION_ID, [{ subject: 'ordinary task' }]); - const { - created: [olderRun], - } = await store.create(SESSION_ID, [{ subject: 'older team task' }], { - actor: 'main_agent', - runId: 'older-lead-run', - turnId: 'older-lead-turn', - }); - assert.ok(unowned && olderRun); - const owner = { actor: 'child_agent' as const, agentId: 'agent-a', turnId: 'turn-a' }; - const scope = { parentRunId: 'lead-run' }; - await assert.rejects( - () => store.claimAvailable(SESSION_ID, unowned.id, owner, scope), - /not shared by parent run/, - ); - await assert.rejects( - () => store.claimAvailable(SESSION_ID, olderRun.id, owner, scope), - /not shared by parent run/, - ); - }); - - it('resolves concurrent self-claim races inside the write queue', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [task], - } = await store.create(SESSION_ID, [{ subject: 'shared' }], { - actor: 'main_agent', - runId: 'lead-run', - turnId: 'lead-turn', - }); - assert.ok(task); - const outcomes = await Promise.allSettled([ - store.claimAvailable( - SESSION_ID, - task.id, - { actor: 'child_agent', agentId: 'agent-a', turnId: 'turn-a' }, - { parentRunId: 'lead-run' }, - ), - store.claimAvailable( - SESSION_ID, - task.id, - { actor: 'child_agent', agentId: 'agent-b', turnId: 'turn-b' }, - { parentRunId: 'lead-run' }, - ), - ]); - assert.equal(outcomes.filter((outcome) => outcome.status === 'fulfilled').length, 1); - assert.equal(outcomes.filter((outcome) => outcome.status === 'rejected').length, 1); - assert.equal((await store.get(SESSION_ID, task.id))?.status, 'in_progress'); - }); - - it('backfills old JSONL fields and persists compatibility events on the next write', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - const legacyTask = { - id: 'legacy-event-task', - subject: 'old event', - status: 'pending', - createdAt: 1, - updatedAt: 1, - }; - await writeFile( - taskEventsFilePath(root), - `${JSON.stringify({ - eventId: 'event-1', - type: 'task_created', - ts: 1, - sessionId: SESSION_ID, - taskId: legacyTask.id, - nextStatus: legacyTask.status, - task: legacyTask, - })}\n`, - 'utf8', - ); - const store = createTaskLedgerStore(root); - assert.equal((await store.get(SESSION_ID, legacyTask.id))?.key, 'T1'); - await store.update(SESSION_ID, 'T1', { status: 'in_progress' }); - const events = (await readFile(taskEventsFilePath(root), 'utf8')) - .trim() - .split('\n') - .map((line) => JSON.parse(line)); - assert.deepEqual( - events.map((item) => item.type), - ['task_created', 'task_updated', 'task_started'], - ); - assert.equal(events[1].task.key, 'T1'); - assert.equal((await createTaskLedgerStore(root).get(SESSION_ID, 'T1'))?.id, legacyTask.id); - }); - - it('fails closed when a persisted child key skips a level under its parent', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - const parent = { - id: 'parent', - key: 'T1', - subject: 'parent', - status: 'pending', - createdAt: 1, - updatedAt: 1, - }; - const child = { - id: 'child', - key: 'T1.1.1', - parentId: parent.id, - subject: 'child', - status: 'pending', - createdAt: 2, - updatedAt: 2, - }; - const events = [parent, child].map((task, index) => ({ - eventId: `event-${index}`, - type: 'task_created', - ts: task.createdAt, - sessionId: SESSION_ID, - taskId: task.id, - nextStatus: task.status, - task, - })); - await writeFile( - taskEventsFilePath(root), - `${events.map((event) => JSON.stringify(event)).join('\n')}\n`, - 'utf8', - ); - const store = createTaskLedgerStore(root); - assert.deepEqual(await store.list(SESSION_ID), []); - await assert.rejects( - () => store.update(SESSION_ID, parent.id, { status: 'in_progress' }), - /projection diagnostics|does not belong under parent key/, - ); - }); - - it('filters archived terminal tasks while preserving compatibility defaults', async () => { - const root = await tempRoot(); - await mkdir(join(root, 'sessions', SESSION_ID), { recursive: true }); - await writeFile( - tasksFilePath(root), - JSON.stringify([ - { - id: 'old-completed', - subject: 'old done', - status: 'completed', - createdAt: 1, - updatedAt: 2, - endedAt: 2, - completionEvidence: 'done', - }, - { id: 'active-task', subject: 'active', status: 'pending', createdAt: 3, updatedAt: 3 }, - ]), - 'utf8', - ); - const store = createTaskLedgerStore(root); - assert.deepEqual( - (await store.list(SESSION_ID)).map((task) => task.key), - ['T1', 'T2'], - ); - assert.deepEqual( - (await store.list(SESSION_ID, { includeArchived: false, now: 10 ** 12 })).map( - (task) => task.key, - ), - ['T2'], - ); - assert.deepEqual( - (await store.list(SESSION_ID, { includeTerminal: false })).map((task) => task.key), - ['T2'], - ); - }); - - it('notifies subscribers and preserves child outcomes without auto-completing success', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const changes: Array<{ sessionId: string; taskIds: string[] }> = []; - const unsubscribe = store.subscribe((event) => changes.push(event)); - const { - created: [task], - } = await store.create(SESSION_ID, [{ subject: 'delegated' }]); - assert.ok(task); - const owner = { actor: 'child_agent' as const, agentId: 'local-read', turnId: 'child-turn' }; - await store.claim(SESSION_ID, task.key, owner); - await store.settleAgentOutcome(SESSION_ID, task.id, { - status: 'completed', - owner: { ...owner, runId: 'child-run' }, - reason: 'child reported success', - }); - const settled = await store.get(SESSION_ID, task.id); - assert.equal(settled?.status, 'in_progress'); - assert.equal(settled?.owner?.runId, 'child-run'); - await assert.rejects( - () => store.claim(SESSION_ID, task.id, { ...owner, turnId: 'other-child-turn' }), - /already claimed/, - ); - unsubscribe(); - assert.equal(changes.length, 3); - assert.equal( - changes.every((event) => event.sessionId === SESSION_ID), - true, - ); - }); - - it('persists failed and cancelled child outcomes with stable owner refs', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [failedTask, cancelledTask], - } = await store.create(SESSION_ID, [{ subject: 'fails' }, { subject: 'cancels' }]); - assert.ok(failedTask && cancelledTask); - const failedOwner = { - actor: 'child_agent' as const, - agentId: 'local-read', - turnId: 'failed-turn', - }; - const cancelledOwner = { - actor: 'child_agent' as const, - agentId: 'local-read', - turnId: 'cancelled-turn', - }; - await store.claim(SESSION_ID, failedTask.id, failedOwner); - await store.claim(SESSION_ID, cancelledTask.id, cancelledOwner); - await store.settleAgentOutcome(SESSION_ID, failedTask.id, { - status: 'failed', - owner: { ...failedOwner, runId: 'failed-run' }, - reason: 'child tests failed', - }); - await store.settleAgentOutcome(SESSION_ID, cancelledTask.id, { - status: 'cancelled', - owner: { ...cancelledOwner, runId: 'cancelled-run' }, - reason: 'parent stopped', - }); - - const reloaded = await createTaskLedgerStore(root).list(SESSION_ID); - const failed = reloaded.find((task) => task.id === failedTask.id); - const cancelled = reloaded.find((task) => task.id === cancelledTask.id); - assert.equal(failed?.status, 'failed'); - assert.equal(failed?.failureReason, 'child tests failed'); - assert.equal(failed?.owner?.runId, 'failed-run'); - assert.equal(typeof failed?.endedAt, 'number'); - assert.equal(cancelled?.status, 'cancelled'); - assert.equal(cancelled?.owner?.runId, 'cancelled-run'); - assert.equal(typeof cancelled?.endedAt, 'number'); - }); - - it('records a generic child user wait without inventing a permission reason', async () => { - const root = await tempRoot(); - const store = createTaskLedgerStore(root); - const { - created: [task], - } = await store.create(SESSION_ID, [{ subject: 'needs input' }]); - assert.ok(task); - const owner = { - actor: 'child_agent' as const, - agentId: 'local-read', - turnId: 'waiting-turn', - }; - await store.claim(SESSION_ID, task.id, owner); - await store.settleAgentOutcome(SESSION_ID, task.id, { - status: 'waiting_for_user', - owner: { ...owner, runId: 'waiting-run' }, - }); - - const waiting = await createTaskLedgerStore(root).get(SESSION_ID, task.id); - assert.equal(waiting?.status, 'blocked'); - assert.equal(waiting?.blockedReason, 'Child agent is waiting for user input'); - assert.equal(waiting?.owner?.runId, 'waiting-run'); - }); -}); diff --git a/packages/storage/src/__tests__/telemetry-repo.test.ts b/packages/storage/src/__tests__/telemetry-repo.test.ts deleted file mode 100644 index 72aaadec2e..0000000000 --- a/packages/storage/src/__tests__/telemetry-repo.test.ts +++ /dev/null @@ -1,855 +0,0 @@ -import assert from 'node:assert/strict'; -import { mkdir, mkdtemp, open, readFile, rm, writeFile } from 'node:fs/promises'; -import { join } from 'node:path'; -import { tmpdir } from 'node:os'; -import { describe, mock, test } from 'node:test'; -import { createTelemetryRepo, TelemetryRepoPublicationError } from '../telemetry-repo.js'; - -describe('FileTelemetryRepo', () => { - test('single-flights concurrent load and allows retry after failure', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-telemetry-load-retry-')); - const path = join(root, 'telemetry.json'); - const repo = createTelemetryRepo(root); - try { - await writeFile(path, '{'); - - const first = repo.load(); - const concurrent = repo.load(); - assert.equal(concurrent, first); - await assert.rejects(() => first, SyntaxError); - - await writeFile( - path, - JSON.stringify({ version: 1, usageRecords: [], toolInvocations: [] }, null, 2) + '\n', - ); - await repo.load(); - assert.equal(repo.summary({ range: 'all' }).totalRequests, 0); - } finally { - await repo.close(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('close joins a load started in the same tick and closes managed pricing', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-telemetry-load-close-')); - const repo = createTelemetryRepo(root); - try { - let loadSettled = false; - const load = repo.load().finally(() => { - loadSettled = true; - }); - await repo.close(); - assert.equal(loadSettled, true); - await load; - - assert.match(await readFile(join(root, 'telemetry.json'), 'utf8'), /"version": 1/); - assert.match(await readFile(join(root, 'pricing.json'), 'utf8'), /"version": 1/); - assert.throws(() => repo.summary({ range: 'all' }), /draining or closed/); - } finally { - await rm(root, { recursive: true, force: true }); - } - }); - - test('retry does not reuse legacy pricing from a failed load attempt', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-telemetry-stale-load-')); - const repo = createTelemetryRepo(root); - try { - await writeFile( - join(root, 'telemetry.json'), - JSON.stringify({ - pricingOverrides: [{ modelKey: 'openai:stale', inputUsdPer1M: 1, outputUsdPer1M: 2 }], - }), - ); - await writeFile(join(root, 'pricing.json'), '{'); - await assert.rejects(() => repo.load(), SyntaxError); - - await Promise.all([rm(join(root, 'telemetry.json')), rm(join(root, 'pricing.json'))]); - await repo.load(); - assert.deepEqual(repo.listPricingOverrides(), []); - assert.deepEqual( - ( - JSON.parse(await readFile(join(root, 'pricing.json'), 'utf8')) as { - overrides: unknown[]; - } - ).overrides, - [], - ); - } finally { - await repo.close(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('upserts LLM calls by id and aggregates the latest record', async () => { - await withRepo(async (repo) => { - await repo.load(); - await repo.insertLlmCall( - llmRecord({ - id: 'usage_turn_1', - inputTokens: 10, - outputTokens: 20, - cacheMissInputTokens: 10, - totalTokens: 30, - }), - ); - await repo.insertLlmCall( - llmRecord({ - id: 'usage_turn_1', - inputTokens: 30, - outputTokens: 40, - cacheMissInputTokens: 30, - totalTokens: 70, - }), - ); - - const summary = repo.summary({ range: 'all' }); - const logs = repo.logs({ range: 'all' }); - - assert.equal(summary.totalRequests, 1); - assert.equal(summary.totalTokens.input, 30); - assert.equal(summary.totalTokens.output, 40); - assert.equal(summary.totalTokens.cacheMiss, 30); - assert.equal(summary.totalTokens.total, 70); - assert.equal(logs.total, 1); - assert.equal(logs.rows[0]?.inputTokens, 30); - assert.equal(logs.rows[0]?.cacheMissTokens, 30); - }); - }); - - test('carries the tool-availability diagnostic and tool-schema change reason through logs()', async () => { - await withRepo(async (repo) => { - await repo.load(); - await repo.insertLlmCall( - llmRecord({ - id: 'usage_diag', - systemPromptHash: 'sys-hash', - toolSchemaChangeReason: 'tool_source_enabled', - toolAvailability: { - mode: 'economy', - enabledSourceIds: ['docs'], - availableSourceIds: ['rive'], - connectorToolName: 'load_tools', - visibleToolNamesBySource: { docs: ['docs_edit'] }, - hiddenToolCount: 1, - }, - }), - ); - - const row = repo.logs({ range: 'all' }).rows[0]; - assert.equal(row?.systemPromptHash, 'sys-hash'); - assert.equal(row?.toolSchemaChangeReason, 'tool_source_enabled'); - assert.equal(row?.toolAvailability?.mode, 'economy'); - assert.deepEqual(row?.toolAvailability?.enabledSourceIds, ['docs']); - assert.equal(row?.toolAvailability?.hiddenToolCount, 1); - }); - }); - - test('strictly admits current token aliases and diagnostic shapes', async () => { - await withRepo(async (repo) => { - await repo.load(); - await repo.insertLlmCall( - llmRecord({ - cachedInputTokens: 4, - cacheHitInputTokens: 4, - rawUsage: { - prompt_tokens: 10, - prompt_tokens_details: { cached_tokens: 4 }, - }, - promptSegments: [{ kind: 'system_prompt', chars: 12, estimatedTokens: 3 }], - contextBudget: { - enabled: true, - estimatedTokensBefore: 10, - estimatedTokensAfter: 8, - keptTurns: 1, - droppedTurns: 0, - keptEvents: 2, - droppedEvents: 0, - }, - }), - ); - - assert.equal(repo.logs({ range: 'all' }).rows[0]?.cacheReadTokens, 4); - }); - }); - - test('keeps a compaction that made the request bigger', async () => { - // `estimatedTokensSaved` is a signed difference, not a count: a compaction - // that grew the request saves a negative number of tokens, and that is the - // outcome most worth being able to see. The producer says so in the field's - // own name (`estimatedTokensSavedSigned`), and the shape validator reads it - // with `isOptionalFiniteNumber`. - // - // A blanket "no negative numbers anywhere under contextBudget" sweep - // contradicted both, and the contradiction was not cheap: decoding throws - // on the first record it cannot read, so eight such decisions made all 1297 - // records in a real telemetry file unreadable. - await withRepo(async (repo, root) => { - await repo.load(); - await repo.insertLlmCall( - llmRecord({ - contextBudget: { - enabled: true, - estimatedTokensBefore: 10, - estimatedTokensAfter: 12, - keptTurns: 1, - droppedTurns: 0, - keptEvents: 2, - droppedEvents: 0, - compactionDecisions: [ - { - stage: 'activeStep', - sourceKind: 'runtimeEvents', - decision: 'replaced', - estimatedTokensSaved: -396, - }, - ], - }, - }), - ); - await repo.flush?.(); - - const reopened = createTelemetryRepo(root); - await reopened.load(); - const rows = reopened.logs({ range: 'all' }).rows; - assert.equal(rows.length, 1, 'the file still decodes'); - const decisions = rows[0]?.contextBudget?.compactionDecisions as - | Array<{ estimatedTokensSaved: number }> - | undefined; - assert.equal( - decisions?.[0]?.estimatedTokensSaved, - -396, - 'and the negative saving survives the round trip', - ); - }); - }); - - test('owns admitted records and returns detached frozen diagnostics', async () => { - await withRepo(async (repo, root) => { - await repo.load(); - const record = llmRecord({ - contextBudget: { - enabled: true, - estimatedTokensBefore: 10, - estimatedTokensAfter: 8, - keptTurns: 1, - droppedTurns: 0, - keptEvents: 2, - droppedEvents: 0, - }, - }); - const inserted = repo.insertLlmCall(record); - const callerContext = record.contextBudget as { keptTurns: number }; - callerContext.keptTurns = 99; - record.inputTokens = 999; - await inserted; - - const first = repo.logs({ range: 'all' }).rows[0]; - assert.equal(repo.summary({ range: 'all' }).totalTokens.input, 10); - assert.equal(first?.contextBudget?.keptTurns, 1); - assert.equal(Object.isFrozen(first?.contextBudget), true); - assert.throws(() => { - (first?.contextBudget as { keptTurns: number }).keptTurns = 77; - }, TypeError); - assert.equal(repo.logs({ range: 'all' }).rows[0]?.contextBudget?.keptTurns, 1); - const bytes = await readFile(join(root, 'telemetry.json'), 'utf8'); - assert.match(bytes, /"inputTokens": 10/); - assert.match(bytes, /"keptTurns": 1/); - }); - }); - - test('rejects unknown fields and invalid diagnostics at writer admission', async () => { - await withRepo(async (repo) => { - await repo.load(); - await assert.rejects( - () => repo.insertLlmCall(llmRecord({ unexpected: true })), - /invalid LLM row keys/, - ); - await assert.rejects( - () => - repo.insertLlmCall( - llmRecord({ - contextBudget: { - enabled: 'yes', - estimatedTokensBefore: 1, - estimatedTokensAfter: 1, - keptTurns: 0, - droppedTurns: 0, - keptEvents: 0, - droppedEvents: 0, - }, - }), - ), - /invalid contextBudget/, - ); - await assert.rejects( - () => - repo.insertLlmCall( - llmRecord({ - contextBudget: { - enabled: true, - estimatedTokensBefore: 10, - estimatedTokensAfter: 12, - keptTurns: 1, - droppedTurns: 0, - keptEvents: 2, - droppedEvents: 0, - compactionDecisions: [ - { - stage: 'activeStep', - sourceKind: 'runtimeEvents', - decision: 'replaced', - estimatedTokensSaved: -2, - coveredTurns: -1, - }, - ], - }, - }), - ), - /invalid contextBudget/, - ); - await assert.rejects( - () => - repo.insertLlmCall( - llmRecord({ - contextBudget: { - enabled: true, - estimatedTokensBefore: 10, - estimatedTokensAfter: 12, - keptTurns: 1, - droppedTurns: 0, - keptEvents: 2, - droppedEvents: 0, - compactionDecisions: [ - { - stage: 'activeStep', - sourceKind: 'runtimeEvents', - decision: 'failedOpen', - validationReasonCounts: { malformed: -1 }, - }, - ], - }, - }), - ), - /invalid contextBudget/, - ); - assert.deepEqual(repo.logs({ range: 'all' }), { rows: [], total: 0 }); - }); - }); - - test('carries auxiliary LLM call identities through logs()', async () => { - await withRepo(async (repo) => { - await repo.load(); - await Promise.all([ - repo.insertLlmCall( - llmRecord({ - id: 'usage_history_compact_turn_1_1_2', - callKind: 'history_compact', - callId: 'history_compact_turn_1_1_2', - ts: 1, - }), - ), - repo.insertLlmCall( - llmRecord({ - id: 'usage_goal_evaluation_session_1_3', - callKind: 'goal_evaluation', - callId: 'goal_evaluation_session_1_3', - ts: 3, - }), - ), - repo.insertLlmCall( - llmRecord({ - id: 'usage_semantic_compact_turn_1_2_3', - callKind: 'semantic_compact', - callId: 'semantic_compact_turn_1_2_3', - ts: 2, - }), - ), - ]); - - const rows = repo.logs({ range: 'all' }).rows; - assert.deepEqual( - rows.map((row) => [row.callKind, row.callId]), - [ - ['goal_evaluation', 'goal_evaluation_session_1_3'], - ['semantic_compact', 'semantic_compact_turn_1_2_3'], - ['history_compact', 'history_compact_turn_1_1_2'], - ], - ); - }); - }); - - test('filters logs by range, status, provider, model, and pagination', async () => { - await withRepo(async (repo) => { - await repo.load(); - await repo.insertLlmCall( - llmRecord({ id: 'old', ts: 1, status: 'success', providerId: 'openai', modelId: 'gpt-4o' }), - ); - await repo.insertLlmCall( - llmRecord({ - id: 'new-success', - ts: 20, - status: 'success', - providerId: 'openai', - modelId: 'gpt-4o', - }), - ); - await repo.insertLlmCall( - llmRecord({ - id: 'middle-success', - ts: 15, - status: 'success', - providerId: 'openai', - modelId: 'gpt-4o', - contextBudget: { - enabled: true, - estimatedTokensBefore: 10, - estimatedTokensAfter: 8, - keptTurns: 1, - droppedTurns: 0, - keptEvents: 2, - droppedEvents: 0, - }, - }), - ); - await repo.insertLlmCall( - llmRecord({ - id: 'new-error', - ts: 30, - status: 'error', - providerId: 'anthropic', - modelId: 'claude', - }), - ); - - const logs = repo.logs( - { range: { from: 10, to: 40 }, status: 'success', providerId: 'openai', modelId: 'gpt-4o' }, - 1, - 1, - ); - - assert.equal(logs.total, 2); - assert.deepEqual( - logs.rows.map((row) => row.id), - ['middle-success'], - ); - assert.equal(Object.isFrozen(logs), true); - assert.equal(Object.isFrozen(logs.rows), true); - assert.equal(Object.isFrozen(logs.rows[0]?.contextBudget), true); - assert.throws(() => { - (logs.rows[0]?.contextBudget as { keptTurns: number }).keptTurns = 99; - }, TypeError); - assert.equal( - repo.logs({ range: 'all' }).rows.find((row) => row.id === 'middle-success')?.contextBudget - ?.keptTurns, - 1, - ); - }); - }); - - test('filters and returns latest LLM runtime probes by connection slug', async () => { - await withRepo(async (repo) => { - await repo.load(); - await repo.insertLlmCall( - llmRecord({ - id: 'conn-a-old', - connectionSlug: 'conn-a', - modelId: 'glm-4.7', - ts: 10, - status: 'success', - }), - ); - await repo.insertLlmCall( - llmRecord({ - id: 'conn-b-new', - connectionSlug: 'conn-b', - modelId: 'glm-4.7', - ts: 50, - status: 'error', - }), - ); - await repo.insertLlmCall( - llmRecord({ - id: 'conn-a-new', - connectionSlug: 'conn-a', - modelId: 'glm-4.7', - ts: 40, - status: 'aborted', - }), - ); - - const logs = repo.logs({ range: 'all', connectionSlug: 'conn-a' }); - const latest = repo.latestLlmRuntimeProbe('conn-a', 'glm-4.7'); - - assert.equal(logs.total, 2); - assert.equal(logs.rows[0]?.id, 'conn-a-new'); - assert.equal(logs.rows[0]?.connectionSlug, 'conn-a'); - assert.equal(latest?.id, 'conn-a-new'); - assert.equal(repo.latestLlmRuntimeProbe('missing'), undefined); - }); - }); - - test('builds provider, model, day, hour, and tool buckets', async () => { - await withRepo(async (repo) => { - await repo.load(); - await repo.insertLlmCall( - llmRecord({ - id: 'usage_1', - providerId: 'openai', - modelId: 'gpt-4o', - ts: Date.UTC(2026, 0, 1, 1), - date: '2026-01-01', - }), - ); - await repo.insertLlmCall( - llmRecord({ - id: 'usage_2', - providerId: 'openai', - modelId: 'gpt-4o-mini', - ts: Date.UTC(2026, 0, 1, 2), - date: '2026-01-01', - }), - ); - await repo.insertToolInvocation( - toolRecord({ id: 'tool_1', toolName: 'Bash', durationMs: 30, status: 'success' }), - ); - await repo.insertToolInvocation( - toolRecord({ id: 'tool_2', toolName: 'Bash', durationMs: 90, status: 'error' }), - ); - - assert.equal(repo.buckets({ range: 'all' }, 'provider')[0]?.key, 'openai'); - assert.equal(repo.buckets({ range: 'all' }, 'model').length, 2); - assert.equal(repo.buckets({ range: 'all' }, 'day')[0]?.key, '2026-01-01'); - assert.equal(repo.buckets({ range: 'all' }, 'hour').length, 2); - - const tool = repo.buckets({ range: 'all' }, 'tool')[0]; - assert.equal(tool?.key, 'Bash'); - assert.equal(tool?.requests, 2); - assert.equal(tool?.avgLatencyMs, 60); - assert.equal(tool?.errorRate, 0.5); - }); - }); - - test('separates LLM and tool logs and rejects inapplicable filters', async () => { - await withRepo(async (repo) => { - await repo.load(); - await repo.insertLlmCall(llmRecord()); - await repo.insertToolInvocation( - toolRecord({ - id: 'bash', - toolName: 'Bash', - resultSummary: { kind: 'shell', completedItemCount: 1 }, - }), - ); - await repo.insertToolInvocation(toolRecord({ id: 'read', toolName: 'Read' })); - - assert.deepEqual( - repo.logs({ range: 'all' }).rows.map((row) => row.id), - ['usage_1'], - ); - assert.deepEqual( - repo.toolLogs({ range: 'all', toolName: 'Bash' }).rows.map((row) => row.id), - ['bash'], - ); - const toolResult = repo.toolLogs({ range: 'all', toolName: 'Bash' }).rows[0]?.resultSummary; - assert.equal(Object.isFrozen(toolResult), true); - assert.throws(() => { - (toolResult as { completedItemCount: number }).completedItemCount = 9; - }, TypeError); - assert.equal( - repo.toolLogs({ range: 'all', toolName: 'Bash' }).rows[0]?.resultSummary - ?.completedItemCount, - 1, - ); - assert.throws( - () => repo.logs({ range: 'all', toolName: 'Bash' }), - /toolName is not applicable to LLM logs/, - ); - assert.throws( - () => - repo.toolLogs({ - range: 'all', - providerId: 'openai', - } as Parameters[0]), - /accept only range, toolName, and status/, - ); - }); - }); - - test('persists pricing overrides and reloads them from disk', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-telemetry-')); - const first = createTelemetryRepo(root); - const second = createTelemetryRepo(root); - try { - await first.load(); - await first.upsertPricing({ - modelKey: 'openai:gpt-4o', - inputUsdPer1M: 2.5, - outputUsdPer1M: 10, - }); - - await second.load(); - - assert.deepEqual(second.listPricingOverrides(), [ - { modelKey: 'openai:gpt-4o', inputUsdPer1M: 2.5, outputUsdPer1M: 10 }, - ]); - } finally { - await Promise.all([first.close(), second.close()]); - await rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }); - } - }); - - test('load creates an empty telemetry file only when missing', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-telemetry-missing-')); - const repo = createTelemetryRepo(root); - try { - await repo.load(); - const raw = await readFile(join(root, 'telemetry.json'), 'utf8'); - - assert.deepEqual(repo.logs({ range: 'all' }), { rows: [], total: 0 }); - assert.match(raw, /"usageRecords": \[\]/); - assert.match(raw, /"toolInvocations": \[\]/); - } finally { - await repo.close(); - await rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }); - } - }); - - test('load accepts legacy telemetry files with only known array sections', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-telemetry-legacy-')); - const repo = createTelemetryRepo(root); - try { - await writeFile( - join(root, 'telemetry.json'), - JSON.stringify({ usageRecords: [] }) + '\n', - 'utf8', - ); - await repo.load(); - - assert.deepEqual(repo.logs({ range: 'all' }), { rows: [], total: 0 }); - assert.deepEqual(repo.listPricingOverrides(), []); - } finally { - await repo.close(); - await rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }); - } - }); - - test('load rejects corrupt telemetry.json without overwriting usage history bytes', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-telemetry-corrupt-')); - const repo = createTelemetryRepo(root); - try { - const corrupt = '{"usageRecords":[{"id":"usage_1"}]'; - await writeFile(join(root, 'telemetry.json'), corrupt, 'utf8'); - await assert.rejects(() => repo.load(), SyntaxError); - assert.equal(await readFile(join(root, 'telemetry.json'), 'utf8'), corrupt); - } finally { - await repo.close(); - await rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }); - } - }); - - test('load rejects wrong telemetry schema without overwriting bytes', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-telemetry-wrong-schema-')); - const repo = createTelemetryRepo(root); - try { - const wrongShape = JSON.stringify({ reminders: [] }, null, 2) + '\n'; - await writeFile(join(root, 'telemetry.json'), wrongShape, 'utf8'); - await assert.rejects(() => repo.load(), /expected known telemetry sections/); - assert.equal(await readFile(join(root, 'telemetry.json'), 'utf8'), wrongShape); - } finally { - await repo.close(); - await rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }); - } - }); - - test('load rejects known telemetry sections with non-array values', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-telemetry-bad-section-')); - const repo = createTelemetryRepo(root); - try { - const wrongShape = JSON.stringify({ usageRecords: {} }, null, 2) + '\n'; - await writeFile(join(root, 'telemetry.json'), wrongShape, 'utf8'); - await assert.rejects(() => repo.load(), /usageRecords must be an array/); - assert.equal(await readFile(join(root, 'telemetry.json'), 'utf8'), wrongShape); - } finally { - await repo.close(); - await rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }); - } - }); - - test('version 1 rejects unknown fields and invalid canonical values without rewriting bytes', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-telemetry-canonical-invalid-')); - const repo = createTelemetryRepo(root); - try { - const invalidCanonical = - JSON.stringify( - { - version: 1, - usageRecords: [{ ...llmRecord(), inputTokens: -1, unexpected: true }], - toolInvocations: [], - }, - null, - 2, - ) + '\n'; - await writeFile(join(root, 'telemetry.json'), invalidCanonical); - - await assert.rejects(() => repo.load(), /invalid LLM row keys/); - assert.equal(await readFile(join(root, 'telemetry.json'), 'utf8'), invalidCanonical); - } finally { - await repo.close(); - await rm(root, { recursive: true, force: true }); - } - }); - - test('surfaces publication failure through mutation, flush, and close', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-telemetry-publication-')); - try { - const repo = createTelemetryRepo(root); - await repo.load(); - await rm(join(root, 'telemetry.json')); - await mkdir(join(root, 'telemetry.json')); - - await assert.rejects(() => repo.insertLlmCall(llmRecord()), /Unable to publish telemetry/); - assert.equal(repo.summary({ range: 'all' }).totalRequests, 0); - await assert.rejects(() => repo.flush(), /Unable to publish telemetry/); - await assert.rejects(() => repo.close(), /Unable to publish telemetry/); - } finally { - await rm(root, { recursive: true, force: true }); - } - }); - - test('poisons reads after rename succeeds but directory sync fails', { - skip: process.platform === 'win32', - }, async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-telemetry-commit-unknown-')); - try { - const repo = createTelemetryRepo(root); - await repo.load(); - const fault = new Error('injected telemetry directory sync failure'); - const probe = await open(root, 'r'); - const fileHandlePrototype = Object.getPrototypeOf(probe) as { - sync: typeof probe.sync; - }; - const originalSync = fileHandlePrototype.sync; - await probe.close(); - let injected = false; - const syncMock = mock.method( - fileHandlePrototype, - 'sync', - async function (this: typeof probe) { - const metadata = await this.stat(); - if (!injected && metadata.isDirectory()) { - injected = true; - throw fault; - } - return originalSync.call(this); - }, - ); - let failure: unknown; - try { - failure = await repo.insertLlmCall(llmRecord()).then( - () => undefined, - (error: unknown) => error, - ); - } finally { - syncMock.mock.restore(); - } - - assert(failure instanceof TelemetryRepoPublicationError); - assert.equal(failure.commitUnknown, true); - assert.match(await readFile(join(root, 'telemetry.json'), 'utf8'), /"usage_1"/); - assert.throws( - () => repo.summary({ range: 'all' }), - (error) => error === failure, - ); - await assert.rejects( - () => repo.flush(), - (error) => error === failure, - ); - await assert.rejects( - () => repo.close(), - (error) => error === failure, - ); - } finally { - await rm(root, { recursive: true, force: true }); - } - }); - - test('close settles and aggregates telemetry and managed pricing failures', async () => { - const root = await mkdtemp(join(tmpdir(), 'maka-telemetry-close-failures-')); - try { - const repo = createTelemetryRepo(root); - await repo.load(); - await Promise.all([rm(join(root, 'telemetry.json')), rm(join(root, 'pricing.json'))]); - await Promise.all([mkdir(join(root, 'telemetry.json')), mkdir(join(root, 'pricing.json'))]); - - const pricingFailure = repo.upsertPricing({ - modelKey: 'openai:gpt-4o', - inputUsdPer1M: 2.5, - outputUsdPer1M: 10, - }); - const pricingRejected = assert.rejects( - () => pricingFailure, - /Unable to publish pricing authority/, - ); - await assert.rejects(() => repo.insertLlmCall(llmRecord()), /Unable to publish telemetry/); - await pricingRejected; - await assert.rejects( - () => repo.close(), - (error) => error instanceof AggregateError && error.errors.length === 2, - ); - } finally { - await rm(root, { recursive: true, force: true }); - } - }); -}); - -async function withRepo( - fn: (repo: ReturnType, root: string) => Promise, -): Promise { - const root = await mkdtemp(join(tmpdir(), 'maka-telemetry-')); - const repo = createTelemetryRepo(root); - try { - await fn(repo, root); - } finally { - await repo.close(); - await rm(root, { recursive: true, force: true, maxRetries: 5, retryDelay: 20 }); - } -} - -function llmRecord(overrides: Record = {}) { - return { - id: 'usage_1', - providerId: 'openai', - modelId: 'gpt-4o', - inputTokens: 10, - outputTokens: 20, - cacheHitInputTokens: 0, - cacheMissInputTokens: 10, - cachedInputTokens: 0, - cacheWriteInputTokens: 0, - reasoningTokens: 0, - totalTokens: 30, - costUsd: 0.001, - latencyMs: 100, - status: 'success', - date: '2026-01-01', - ts: Date.UTC(2026, 0, 1), - startedAt: Date.UTC(2026, 0, 1) - 100, - ...overrides, - } as Parameters['insertLlmCall']>[0]; -} - -function toolRecord(overrides: Record = {}) { - return { - id: 'tool_1', - toolName: 'Bash', - durationMs: 10, - status: 'success', - argsSummary: '', - bytesIn: 0, - bytesOut: 0, - date: '2026-01-01', - ts: Date.UTC(2026, 0, 1), - startedAt: Date.UTC(2026, 0, 1) - 10, - ...overrides, - } as Parameters['insertToolInvocation']>[0]; -} diff --git a/packages/storage/src/__tests__/workspace-version-authority-persistence.test.ts b/packages/storage/src/__tests__/workspace-version-authority-persistence.test.ts index dbee079901..6aaef2424e 100644 --- a/packages/storage/src/__tests__/workspace-version-authority-persistence.test.ts +++ b/packages/storage/src/__tests__/workspace-version-authority-persistence.test.ts @@ -12,7 +12,6 @@ import { type RuntimeEvent, type WorkspaceBaselineAuthorityInput, } from '@maka/core'; -import { createRuntimeEventStore } from '../agent-run-store.js'; import { createConversationOperationalStateStore } from '../conversation-operational-state.js'; import { createSqliteRuntimeStore, @@ -229,7 +228,7 @@ describe('workspace version persistence authority', () => { }); }); - it('rejects workspace facts through SQLite and JSONL generic writers', async () => { + it('rejects workspace facts through generic RuntimeEvent writers', async () => { await withDatabase(async ({ store, root }) => { const { epochOpenedEvent } = buildWorkspaceBaselineAuthorityEvents(baselineInput()); await assert.rejects( @@ -339,30 +338,6 @@ describe('workspace version persistence authority', () => { }), /workspace version authority writer/i, ); - - const jsonl = createRuntimeEventStore(join(root, 'jsonl')); - await assert.rejects( - jsonl.appendRuntimeEvent( - epochOpenedEvent.sessionId, - epochOpenedEvent.runId, - epochOpenedEvent, - ), - /workspace version authority writer/i, - ); - await assert.rejects( - jsonl.ensureTerminalRuntimeEventDurable( - epochOpenedEvent.sessionId, - epochOpenedEvent.runId, - epochOpenedEvent, - ), - /workspace version authority writer/i, - ); - await assert.rejects( - jsonl.importConversationCopyRuntimeEvents(epochOpenedEvent.sessionId, [ - { runId: epochOpenedEvent.runId, events: [epochOpenedEvent] }, - ]), - /workspace version authority writer/i, - ); }); }); diff --git a/packages/storage/src/agent-graph-control-store.ts b/packages/storage/src/agent-graph-control-store.ts index 36da21bc7c..62fd96d69b 100644 --- a/packages/storage/src/agent-graph-control-store.ts +++ b/packages/storage/src/agent-graph-control-store.ts @@ -9,9 +9,8 @@ import { } from './sqlite-session-metadata-store.js'; /** - * Open the metadata repository owned by the operational database. Session - * JSONL remains the transcript-body authority; graph schedule/topology/claim - * relationships are canonical in runtime.sqlite. + * Open the graph-control repository owned by the operational database. + * Session messages and graph state share runtime.sqlite as one authority. */ export function createAgentGraphControlStore(workspaceRoot: string): SqliteSessionMetadataStore { const databaseLease = acquireOperationalStateDatabase(workspaceRoot); diff --git a/packages/storage/src/agent-run-store.ts b/packages/storage/src/agent-run-store.ts index 367df6d679..5faf8e9c03 100644 --- a/packages/storage/src/agent-run-store.ts +++ b/packages/storage/src/agent-run-store.ts @@ -1,37 +1,17 @@ -import { createHash, randomUUID } from 'node:crypto'; -import type { Dirent } from 'node:fs'; -import { - appendFile, - link, - mkdir, - open, - readFile, - readdir, - rename, - rm, - stat, - unlink, - writeFile, -} from 'node:fs/promises'; -import { dirname, join, resolve } from 'node:path'; +import { createHash } from 'node:crypto'; +import { resolve } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import type { DatabaseSync } from 'node:sqlite'; -import { appendJsonl } from './jsonl-append.js'; import { decodeAgentRunEvent, decodeAgentRunHeader, decodeRuntimeEvent, } from './execution-record-codec.js'; -import { classifyJsonRecord } from './json-prefix.js'; import { immutableSteeringMessageId } from './runtime-event-invariants.js'; import { assertNoReservedWorkspaceAuthorityAppend } from './runtime-event-authority.js'; -import { syncDirectory, syncDirectoryChain, syncFile } from './stable-storage.js'; -import { chainWrite } from './write-queue.js'; import { acquireOperationalStateDatabase, - completeOperationalStoreCutover, type OperationalStateDatabaseLease, - type OperationalStoreCutoverFailpoint, } from './operational-state-store.js'; import { assertEvidenceReadBudget, @@ -69,9 +49,6 @@ import { } from '@maka/core/orchestration'; const SAFE_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; -const EXCLUSIVE_TEMP_SUFFIX_PATTERN = - /^\d+\.[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}\.tmp$/; - export const ROOT_TURN_ADMISSION_SCHEMA_VERSION = 1 as const; export const ROOT_TURN_ADMISSION_MAX_SOURCE_MESSAGES = 64; export const ROOT_TURN_ADMISSION_MAX_CONTENT_BYTES = 64 * 1024; @@ -209,38 +186,19 @@ class RuntimeEventPostEffectError extends Error { } } -export function createAgentRunStore(workspaceRoot: string): DurableAgentRunStore { - return new FileAgentRunStore(workspaceRoot); -} - -export interface SqliteAgentRunStoreOptions { - readonly failpoint?: (point: OperationalStoreCutoverFailpoint) => void; -} - -export function createSqliteAgentRunStore( - workspaceRoot: string, - options: SqliteAgentRunStoreOptions = {}, -): DurableAgentRunStore { - return new SqliteAgentRunStore(workspaceRoot, options); -} - -export function createRuntimeEventStore(workspaceRoot: string): DurableRuntimeEventStore { - return new FileRuntimeEventStore(workspaceRoot); +export function createSqliteAgentRunStore(workspaceRoot: string): DurableAgentRunStore { + return new SqliteAgentRunStore(workspaceRoot); } class SqliteAgentRunStore implements DurableAgentRunStore { - readonly #root: string; readonly #lease: OperationalStateDatabaseLease; - readonly #ready: Promise; - constructor(workspaceRoot: string, options: SqliteAgentRunStoreOptions) { - this.#root = resolve(workspaceRoot); - this.#lease = acquireOperationalStateDatabase(this.#root); - this.#ready = importLegacyAgentRuns(this.#root, this.#lease, options); + constructor(workspaceRoot: string) { + this.#lease = acquireOperationalStateDatabase(resolve(workspaceRoot)); } ready(): Promise { - return this.#ready; + return Promise.resolve(); } async createRun( @@ -248,7 +206,6 @@ class SqliteAgentRunStore implements DurableAgentRunStore { _options: { durable?: boolean } = {}, ): Promise { const normalized = normalizeAgentRunHeader(header, header.sessionId, header.runId); - await this.#ready; this.#lease.transaction('write', () => { const inserted = this.#lease.database .prepare(` @@ -296,7 +253,6 @@ class SqliteAgentRunStore implements DurableAgentRunStore { assertMutableRunHeaderPatch(patch); assertSafeId(sessionId, 'Invalid session id'); assertSafeId(runId, 'Invalid run id'); - await this.#ready; return this.#lease.transaction('write', () => { const current = readSqliteAgentRun(this.#lease.database, sessionId, runId); const next = normalizeAgentRunHeader( @@ -319,7 +275,6 @@ class SqliteAgentRunStore implements DurableAgentRunStore { async readRun(sessionId: string, runId: string): Promise { assertSafeId(sessionId, 'Invalid session id'); assertSafeId(runId, 'Invalid run id'); - await this.#ready; return readSqliteAgentRun(this.#lease.database, sessionId, runId); } @@ -330,7 +285,6 @@ class SqliteAgentRunStore implements DurableAgentRunStore { async findRunsById(runId: string, limit: number): Promise { assertSafeId(runId, 'Invalid run id'); assertIdentitySearchLimit(limit); - await this.#ready; const rows = this.#lease.database .prepare(` SELECT session_id, record_json @@ -356,7 +310,6 @@ class SqliteAgentRunStore implements DurableAgentRunStore { ): Promise { assertSafeId(sessionId, 'Invalid session id'); assertIdentitySearchLimit(limit); - await this.#ready; const rows = this.#lease.database .prepare(` SELECT run_id, record_json @@ -378,7 +331,6 @@ class SqliteAgentRunStore implements DurableAgentRunStore { async listSessionRunsForRecovery(sessionId: string): Promise { assertSafeId(sessionId, 'Invalid session id'); - await this.#ready; const rows = this.#lease.database .prepare(` SELECT run_id, record_json @@ -403,7 +355,6 @@ class SqliteAgentRunStore implements DurableAgentRunStore { ): Promise { assertSafeId(sessionId, 'Invalid session id'); assertSafeId(runId, 'Invalid run id'); - await this.#ready; this.#lease.transaction('write', () => { const header = readSqliteAgentRun(this.#lease.database, sessionId, runId); const normalized = decodeAgentRunEvent(JSON.parse(JSON.stringify(event, sanitizeJson)), { @@ -437,7 +388,6 @@ class SqliteAgentRunStore implements DurableAgentRunStore { assertSafeId(sessionId, 'Invalid session id'); assertSafeId(runId, 'Invalid run id'); assertEvidenceReadBudget(budget); - await this.#ready; const rows = this.#lease.database .prepare(` SELECT length(CAST(record_json AS BLOB)) AS stored_bytes @@ -463,14 +413,12 @@ class SqliteAgentRunStore implements DurableAgentRunStore { async readEventsForRecovery(sessionId: string, runId: string): Promise { assertSafeId(sessionId, 'Invalid session id'); assertSafeId(runId, 'Invalid run id'); - await this.#ready; return readSqliteAgentRunEvents(this.#lease.database, sessionId, runId); } async readEventsForEvidence(sessionId: string, runId: string): Promise { assertSafeId(sessionId, 'Invalid session id'); assertSafeId(runId, 'Invalid run id'); - await this.#ready; return readSqliteAgentRunEventsForEvidence(this.#lease.database, sessionId, runId); } @@ -479,7 +427,6 @@ class SqliteAgentRunStore implements DurableAgentRunStore { type: AgentRunEventType, ): Promise { assertSafeId(sessionId, 'Invalid session id'); - await this.#ready; return readSqliteAgentRunProjection(this.#lease.database, sessionId, type); } @@ -493,7 +440,6 @@ class SqliteAgentRunStore implements DurableAgentRunStore { if (event !== null && !isProjectedAgentRunEvent(event, sessionId, type)) { throw new Error(`Invalid AgentRun event projection repair for ${type}`); } - await this.#ready; this.#lease.transaction('write', () => { const current = readSqliteAgentRunProjection(this.#lease.database, sessionId, type); if ( @@ -508,7 +454,6 @@ class SqliteAgentRunStore implements DurableAgentRunStore { async admitRootTurn(input: AdmitRootTurnInput): Promise { const admission = normalizeAdmitRootTurnInput(input); - await this.#ready; return this.#lease.transaction('write', () => { const existing = readSqliteRootTurnAdmission( this.#lease.database, @@ -565,7 +510,6 @@ class SqliteAgentRunStore implements DurableAgentRunStore { ): Promise { assertSafeId(sessionId, 'Invalid session id'); assertSafeId(turnId, 'Invalid turn id'); - await this.#ready; return readSqliteRootTurnAdmission(this.#lease.database, sessionId, turnId); } @@ -575,7 +519,6 @@ class SqliteAgentRunStore implements DurableAgentRunStore { ): Promise { assertSafeId(sessionId, 'Invalid session id'); assertSafeId(sourceMessageId, 'Invalid source message id'); - await this.#ready; const row = this.#lease.database .prepare(` SELECT turn_id @@ -602,7 +545,6 @@ class SqliteAgentRunStore implements DurableAgentRunStore { async listRootTurnAdmissionsForRecovery(sessionId: string): Promise { assertSafeId(sessionId, 'Invalid session id'); - await this.#ready; const rows = this.#lease.database .prepare(` SELECT turn_id, record_json @@ -625,2057 +567,321 @@ class SqliteAgentRunStore implements DurableAgentRunStore { } } -class FileAgentRunStore implements DurableAgentRunStore { - private readonly durabilityRoot: string; - private readonly sessionsRoot: string; - private readonly writeQueues = new Map>(); - private readonly rootTurnAdmissionWriteQueues = new Map>(); - private readonly projectionWriteQueues = new Map>(); - - constructor(workspaceRoot: string) { - this.durabilityRoot = resolve(workspaceRoot); - this.sessionsRoot = join(this.durabilityRoot, 'sessions'); - } - - async createRun( - header: AgentRunHeader, - options: { durable?: boolean } = {}, - ): Promise { - assertSafeId(header.sessionId, 'Invalid session id'); - assertSafeId(header.runId, 'Invalid run id'); - await this.withQueue(header.sessionId, header.runId, async () => { - const created = await writeExclusiveAtomic( - this.runPath(header.sessionId, header.runId), - JSON.stringify(header, sanitizeJson) + '\n', - options, - this.durabilityRoot, - ); - if (!created) throw new Error(`Agent run already exists: ${header.runId}`); - }); - await this.withProjectionQueue( - header.sessionId, - 'history_compact_checkpoint_recorded', - async () => { - await this.initializeEventProjectionUnlocked( - header.sessionId, - header.runId, - 'history_compact_checkpoint_recorded', - ); - }, - ).catch(() => { - // Projection initialization is derived state; recovery can rebuild it from the run ledger. - }); - return header; - } - - async admitRootTurn(input: AdmitRootTurnInput): Promise { - assertSafeId(input.sessionId, 'Invalid session id'); - assertSafeId(input.turnId, 'Invalid turn id'); - assertSafeId(input.proposedRunId, 'Invalid run id'); - if (input.proposedUserMessageId !== null) { - assertSafeId(input.proposedUserMessageId, 'Invalid user message id'); - } - if (input.previousRootTurnId !== null) { - assertSafeId(input.previousRootTurnId, 'Invalid previous root turn id'); - if (input.previousRootTurnId === input.turnId) { - throw new Error('Root turn admission cannot reference itself'); - } - } - const { normalizedInput, sourceMessages } = normalizeRootTurnAdmissionPayload( - input.normalizedInput, - input.sourceMessages, - ); - const turnOrchestration = normalizeTurnOrchestration(input.turnOrchestration); - if (!Number.isSafeInteger(input.admittedAt) || input.admittedAt < 0) { - throw new Error('Invalid root turn admission timestamp'); - } - const admission: RootTurnAdmission = { - schemaVersion: ROOT_TURN_ADMISSION_SCHEMA_VERSION, - sessionId: input.sessionId, - turnId: input.turnId, - runId: input.proposedRunId, - userMessageId: input.proposedUserMessageId, - execution: normalizeRootExecutionDescriptor(input.execution), - previousRootTurnId: input.previousRootTurnId, - normalizedInput, - ...(turnOrchestration ? { turnOrchestration } : {}), - sourceMessages, - admittedAt: input.admittedAt, - }; - assertRootTurnAdmissionContract(admission); - assertRootTurnAdmissionRecordSize(admission); - deepFreezeRootTurnAdmission(admission); - let result: AdmitRootTurnResult | undefined; - await chainWrite(this.rootTurnAdmissionWriteQueues, input.sessionId, async () => { - const path = this.rootTurnAdmissionPath(input.sessionId, input.turnId); - let existing = await this.readRootTurnAdmission(input.sessionId, input.turnId); - if (!existing) { - await this.assertRootSourceMessageProofOwners(admission); - const created = await writeExclusiveAtomic( - path, - JSON.stringify(admission) + '\n', - { durable: true }, - this.durabilityRoot, - ); - if (created) { - await this.ensureRootSourceMessageProofs(admission); - result = { kind: 'admitted', admission }; - return; - } - existing = await this.readRootTurnAdmission(input.sessionId, input.turnId); - if (!existing) throw new Error(`Root turn admission disappeared: ${input.turnId}`); - } - await this.ensureRootSourceMessageProofs(existing); - result = - existing.previousRootTurnId === input.previousRootTurnId && - rootTurnAdmissionPayloadsEqual(existing, admission) - ? { kind: 'existing', admission: existing } - : { kind: 'conflict', admission: existing }; - }); - if (!result) throw new Error(`Root turn admission did not complete: ${input.turnId}`); - return result; - } +function normalizeAgentRunHeader(value: unknown, sessionId: string, runId: string): AgentRunHeader { + assertSafeId(sessionId, 'Invalid session id'); + assertSafeId(runId, 'Invalid run id'); + return decodeAgentRunHeader(JSON.parse(JSON.stringify(value, sanitizeJson)), { + sessionId, + runId, + }); +} - async readRootTurnAdmission( - sessionId: string, - turnId: string, - ): Promise { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(turnId, 'Invalid turn id'); - let raw: string; - try { - raw = await readFile(this.rootTurnAdmissionPath(sessionId, turnId), 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; - throw error; - } - assertRootTurnAdmissionSerializedSize(raw); - return normalizeRootTurnAdmission(JSON.parse(raw), sessionId, turnId); +function readSqliteAgentRun(db: DatabaseSync, sessionId: string, runId: string): AgentRunHeader { + const row = db + .prepare(` + SELECT record_json + FROM core_agent_runs + WHERE session_id = ? AND run_id = ? + `) + .get(sessionId, runId) as { record_json?: unknown } | undefined; + if (!row) { + const error = new Error(`Agent run does not exist: ${runId}`) as NodeJS.ErrnoException; + error.code = 'ENOENT'; + throw error; } + if (typeof row.record_json !== 'string') throw new Error('Invalid SQLite AgentRun row'); + return normalizeAgentRunHeader(JSON.parse(row.record_json), sessionId, runId); +} - async readRootTurnSourceMessageReceipt( - sessionId: string, - sourceMessageId: string, - ): Promise { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(sourceMessageId, 'Invalid source message id'); - let raw: string; - try { - raw = await readFile(this.rootSourceMessageProofPath(sessionId, sourceMessageId), 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; - throw error; +function readSqliteAgentRunEvents( + db: DatabaseSync, + sessionId: string, + runId: string, +): AgentRunEvent[] { + const rows = db + .prepare(` + SELECT record_json + FROM core_agent_run_events + WHERE session_id = ? AND run_id = ? + ORDER BY sequence + `) + .all(sessionId, runId) as Array<{ record_json?: unknown }>; + if (rows.length === 0) return []; + const header = readSqliteAgentRun(db, sessionId, runId); + return rows.map((row) => { + if (typeof row.record_json !== 'string') { + throw new Error('Invalid SQLite AgentRun event row'); } - const pointer = decodeRootSourceMessageProofPointer( - JSON.parse(raw), + return decodeAgentRunEvent(JSON.parse(row.record_json), { sessionId, - sourceMessageId, - ); - const admission = await this.readRootTurnAdmission(sessionId, pointer.turnId); - if (!admission) { - throw new Error(`Root source message proof references missing Turn ${pointer.turnId}`); - } - const matching = admission.sourceMessages.filter( - (source) => source.messageId === sourceMessageId, - ); - if (matching.length !== 1) { - throw new Error( - `Root source message proof does not identify exactly one source: ${sourceMessageId}`, - ); - } - return Object.freeze({ admission, sourceMessage: matching[0]! }); - } - - async listRootTurnAdmissionsForRecovery(sessionId: string): Promise { - assertSafeId(sessionId, 'Invalid session id'); - const admissionsRoot = this.rootTurnAdmissionsRoot(sessionId); - let entries; - try { - entries = await readdir(admissionsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const admissions: RootTurnAdmission[] = []; - let removedStagingFile = false; - for (const entry of entries) { - if (!entry.isFile()) { - throw new Error(`Invalid root turn admission entry: ${entry.name}`); - } - const turnId = turnIdFromAdmissionFile(entry.name); - if (turnId) { - admissions.push((await this.readRootTurnAdmission(sessionId, turnId)) as RootTurnAdmission); - continue; - } - if (isRootTurnAdmissionTemp(entry.name)) { - await rm(join(admissionsRoot, entry.name), { force: true }); - removedStagingFile = true; - continue; - } - throw new Error(`Invalid root turn admission entry: ${entry.name}`); - } - if (removedStagingFile) await syncDirectory(admissionsRoot); - const ordered = orderRootTurnAdmissionChain(sessionId, admissions); - for (const admission of ordered) await this.ensureRootSourceMessageProofs(admission); - return ordered; - } - - async updateRun( - sessionId: string, - runId: string, - patch: Partial, - options: { durable?: boolean } = {}, - ): Promise { - assertMutableRunHeaderPatch(patch); - let next: AgentRunHeader | undefined; - await this.withQueue(sessionId, runId, async () => { - const current = await this.readRunUnlocked(sessionId, runId); - next = decodeAgentRunHeader({ ...current, ...patch, sessionId, runId }, { sessionId, runId }); - await writeAtomic(this.runPath(sessionId, runId), JSON.stringify(next, sanitizeJson) + '\n', { - ...options, - durabilityRoot: this.durabilityRoot, - }); + runId, + turnId: header.turnId, }); - if (!next) throw new Error(`Failed to update run ${runId}`); - return next; - } - - async readRun(sessionId: string, runId: string): Promise { - return this.readRunUnlocked(sessionId, runId); - } + }); +} - async listSessionRuns(sessionId: string): Promise { - assertSafeId(sessionId, 'Invalid session id'); - const runsRoot = this.runsRoot(sessionId); - let entries; +function readSqliteAgentRunEventsForEvidence( + db: DatabaseSync, + sessionId: string, + runId: string, +): AgentRunEvent[] { + const rows = db + .prepare(` + SELECT sequence, record_json + FROM core_agent_run_events + WHERE session_id = ? AND run_id = ? + ORDER BY sequence + `) + .all(sessionId, runId) as Array<{ sequence?: unknown; record_json?: unknown }>; + if (rows.length === 0) return []; + const header = readSqliteAgentRun(db, sessionId, runId); + return rows.map((row) => { + const lineNumber = + typeof row.sequence === 'number' && Number.isSafeInteger(row.sequence) ? row.sequence + 1 : 0; try { - entries = await readdir(runsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const headers: AgentRunHeader[] = []; - for (const entry of entries) { - if (!entry.isDirectory() || !isSafeId(entry.name)) continue; - try { - headers.push(await this.readRunUnlocked(sessionId, entry.name)); - } catch { - // Malformed run folders should not hide the rest of the session. + if (typeof row.record_json !== 'string') { + throw new Error('Invalid SQLite AgentRun event row'); } - } - return headers.sort((a, b) => a.createdAt - b.createdAt || a.runId.localeCompare(b.runId)); - } - - async findRunsById(runId: string, limit: number): Promise { - assertSafeId(runId, 'Invalid run id'); - assertIdentitySearchLimit(limit); - let sessions; - try { - sessions = await readdir(this.sessionsRoot, { withFileTypes: true }); + return decodeAgentRunEvent(JSON.parse(row.record_json), { + sessionId, + runId, + turnId: header.turnId, + }); } catch (error) { - if (isMissingFile(error)) return { runs: [], truncated: false }; - throw error; - } - const runs: AgentRunHeader[] = []; - for (const entry of sessions.sort((a, b) => a.name.localeCompare(b.name))) { - if (!entry.isDirectory() || !isSafeId(entry.name)) continue; - try { - runs.push(await this.readRunUnlocked(entry.name, runId)); - } catch (error) { - if (isMissingFile(error)) continue; - throw error; - } - if (runs.length > limit) { - return { runs: runs.slice(0, limit), truncated: true }; - } + return { + type: 'event_corrupt', + id: `run-event-corrupt-${lineNumber}`, + runId, + sessionId, + turnId: header.turnId, + ts: header.updatedAt, + message: error instanceof Error ? error.message : 'Invalid SQLite AgentRun event row', + data: { lineNumber }, + }; } - return { runs, truncated: false }; - } + }); +} - async listSessionRunsBounded( - sessionId: string, - limit: number, - ): Promise { - assertSafeId(sessionId, 'Invalid session id'); - assertIdentitySearchLimit(limit); - let entries; - try { - entries = await readdir(this.runsRoot(sessionId), { withFileTypes: true }); - } catch (error) { - if (isMissingFile(error)) return { runs: [], truncated: false }; - throw error; - } - const runs: AgentRunHeader[] = []; - for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { - if (!entry.isDirectory() || !isSafeId(entry.name)) continue; - try { - runs.push(await this.readRunUnlocked(sessionId, entry.name)); - } catch { - // Malformed run folders should not hide the rest of the session. - } - if (runs.length > limit) break; - } - const truncated = runs.length > limit; - return { - runs: runs - .slice(0, limit) - .sort( - (left, right) => - left.createdAt - right.createdAt || left.runId.localeCompare(right.runId), - ), - truncated, - }; +function insertAgentRunEvent(db: DatabaseSync, event: AgentRunEvent): void { + const row = db + .prepare(` + SELECT COALESCE(MAX(sequence), -1) + 1 AS sequence + FROM core_agent_run_events + WHERE session_id = ? AND run_id = ? + `) + .get(event.sessionId, event.runId) as { sequence?: unknown }; + if (typeof row.sequence !== 'number' || !Number.isSafeInteger(row.sequence)) { + throw new Error('Invalid next AgentRun event sequence'); } + db.prepare(` + INSERT INTO core_agent_run_events( + session_id, run_id, sequence, event_id, event_type, event_ts, record_json + ) VALUES (?, ?, ?, ?, ?, ?, ?) + `).run( + event.sessionId, + event.runId, + row.sequence, + event.id, + event.type, + event.ts, + JSON.stringify(event, sanitizeJson), + ); +} - async listSessionRunsForRecovery(sessionId: string): Promise { - assertSafeId(sessionId, 'Invalid session id'); - const runsRoot = this.runsRoot(sessionId); - let entries; - try { - entries = await readdir(runsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const headers: AgentRunHeader[] = []; - for (const entry of entries) { - if (!entry.isDirectory() || !isSafeId(entry.name)) { - throw new Error(`Invalid AgentRun entry for session ${sessionId}: ${entry.name}`); - } - try { - headers.push(await this.readRunUnlocked(sessionId, entry.name)); - } catch (error) { - if ( - isMissingFile(error) && - (await this.removeUncommittedRunDirectory(sessionId, entry.name)) - ) { - continue; - } - throw error; - } - } - return headers.sort((a, b) => a.createdAt - b.createdAt || a.runId.localeCompare(b.runId)); +function readSqliteAgentRunProjection( + db: DatabaseSync, + sessionId: string, + type: AgentRunEventType, +): AgentRunEvent | null | undefined { + const row = db + .prepare(` + SELECT event_json + FROM core_agent_run_projections + WHERE session_id = ? AND event_type = ? + `) + .get(sessionId, type) as { event_json?: unknown } | undefined; + if (!row) return undefined; + if (row.event_json === null) return null; + if (typeof row.event_json !== 'string') { + throw new Error(`Invalid AgentRun event projection for ${type}`); } - - async appendEvent( - sessionId: string, - runId: string, - event: AgentRunEvent, - options: { durable?: boolean } = {}, - ): Promise { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - if (event.type === 'history_compact_checkpoint_recorded') { - await this.withProjectionQueue(sessionId, event.type, async () => { - let current: AgentRunEvent | null | undefined; - try { - current = await this.readEventProjectionUnlocked(sessionId, event.type); - } catch { - current = undefined; - } - await rm(this.eventProjectionPath(sessionId, event.type), { - force: true, - }); - await this.appendRunEvent(sessionId, runId, event, options); - const projected = shouldPreserveCheckpointProjectionDuringAppend(current, event) - ? current! - : event; - await this.writeEventProjectionUnlocked(sessionId, event.type, projected).catch(() => { - // The canonical event is durable; a missing derived projection safely replays raw history. - }); - }); - return; - } - await this.appendRunEvent(sessionId, runId, event, options); - } - - private async appendRunEvent( - sessionId: string, - runId: string, - event: AgentRunEvent, - options: { durable?: boolean }, - ): Promise { - await this.withQueue(sessionId, runId, async () => { - await mkdir(this.runDir(sessionId, runId), { recursive: true }); - await appendJsonl( - this.eventsPath(sessionId, runId), - JSON.stringify(event, sanitizeJson) + '\n', - { ...options, durabilityRoot: this.durabilityRoot }, - ); - }); - } - - async readEvents(sessionId: string, runId: string): Promise { - return this.readEventsWithPolicy(sessionId, runId, false); - } - - async readEventsBounded( - sessionId: string, - runId: string, - budget: EvidenceReadBudget, - ): Promise> { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - assertEvidenceReadBudget(budget); - const path = this.eventsPath(sessionId, runId); - let storedBytes = 0; - try { - storedBytes = (await stat(path)).size; - } catch (error) { - if (!isMissingFile(error)) throw error; - } - if (storedBytes > budget.maxBytes) return { status: 'limit_exceeded' }; - const records = await this.readEventsWithPolicy(sessionId, runId, false, true); - if (records.length > budget.maxRecords) return { status: 'limit_exceeded' }; - return { - status: 'complete', - records, - sourceRecordCount: records.length, - storedBytes, - }; - } - - async readEventsForRecovery(sessionId: string, runId: string): Promise { - return this.readEventsWithPolicy(sessionId, runId, true); - } - - async readEventsForEvidence(sessionId: string, runId: string): Promise { - return this.readEventsWithPolicy(sessionId, runId, false, true); - } - - private async readEventsWithPolicy( - sessionId: string, - runId: string, - strict: boolean, - preserveIncompleteTail = false, - ): Promise { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - let text: string; - try { - text = await readFile(this.eventsPath(sessionId, runId), 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const header = await this.readRunUnlocked(sessionId, runId); - const rawLines = text.split('\n'); - const endsWithNewline = text.endsWith('\n'); - const lines = rawLines - .map((line, index) => ({ line, lineNumber: index + 1 })) - .filter((entry) => entry.line.trim().length > 0); - const lastLineNumber = lines.at(-1)?.lineNumber; - const events: AgentRunEvent[] = []; - for (const entry of lines) { - let parsed: unknown; - try { - parsed = JSON.parse(entry.line); - } catch (error) { - const incompleteTail = - !endsWithNewline && - entry.lineNumber === lastLineNumber && - classifyJsonRecord(entry.line) === 'incomplete-prefix'; - if (incompleteTail && !preserveIncompleteTail) continue; - if (strict) { - const detail = error instanceof Error ? error.message : String(error); - throw new Error( - `AgentRun ${runId} has a corrupt JSONL record at line ${entry.lineNumber}: ${detail}`, - ); - } - events.push({ - type: 'event_corrupt', - id: `run-event-corrupt-${entry.lineNumber}`, - runId, - sessionId, - turnId: header.turnId, - ts: header.updatedAt, - message: incompleteTail - ? 'Incomplete AgentRun event JSONL tail' - : error instanceof Error - ? error.message - : 'Invalid AgentRun event JSONL line', - data: { lineNumber: entry.lineNumber }, - }); - continue; - } - try { - events.push( - decodeAgentRunEvent(parsed, { - sessionId, - runId, - turnId: header.turnId, - }), - ); - } catch (error) { - if (strict) { - const detail = error instanceof Error ? error.message : String(error); - throw new Error( - `AgentRun ${runId} has a corrupt JSONL record at line ${entry.lineNumber}: ${detail}`, - ); - } - events.push({ - type: 'event_corrupt', - id: `run-event-corrupt-${entry.lineNumber}`, - runId, - sessionId, - turnId: header.turnId, - ts: header.updatedAt, - message: error instanceof Error ? error.message : 'Invalid AgentRun event JSONL line', - data: { lineNumber: entry.lineNumber }, - }); - } - } - return events; - } - - async readEventProjection( - sessionId: string, - type: AgentRunEventType, - ): Promise { - assertSafeId(sessionId, 'Invalid session id'); - return this.readEventProjectionUnlocked(sessionId, type); - } - - async repairEventProjection( - sessionId: string, - type: AgentRunEventType, - event: AgentRunEvent | null, - options: { replaceEventId?: string } = {}, - ): Promise { - assertSafeId(sessionId, 'Invalid session id'); - if (event !== null && !isProjectedAgentRunEvent(event, sessionId, type)) { - throw new Error(`Invalid AgentRun event projection repair for ${type}`); - } - await this.withProjectionQueue(sessionId, type, async () => { - let current: AgentRunEvent | null | undefined; - try { - current = await this.readEventProjectionUnlocked(sessionId, type); - } catch { - current = undefined; - } - if ( - current?.id !== options.replaceEventId && - shouldPreserveProjectionDuringRepair(current, event, type) - ) - return; - await this.writeEventProjectionUnlocked(sessionId, type, event); - }); - } - - private async readRunUnlocked(sessionId: string, runId: string): Promise { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - return decodeAgentRunHeader( - JSON.parse(await readFile(this.runPath(sessionId, runId), 'utf8')), - { sessionId, runId }, - ); - } - - private runsRoot(sessionId: string): string { - assertSafeId(sessionId, 'Invalid session id'); - return join(this.sessionsRoot, sessionId, 'runs'); - } - - private runDir(sessionId: string, runId: string): string { - assertSafeId(runId, 'Invalid run id'); - return join(this.runsRoot(sessionId), runId); - } - - private runPath(sessionId: string, runId: string): string { - return join(this.runDir(sessionId, runId), 'run.json'); - } - - private eventsPath(sessionId: string, runId: string): string { - return join(this.runDir(sessionId, runId), 'events.jsonl'); - } - - private eventProjectionPath(sessionId: string, type: AgentRunEventType): string { - return join(this.sessionsRoot, sessionId, 'projections', `${type}.json`); - } - - private rootTurnAdmissionPath(sessionId: string, turnId: string): string { - return join(this.rootTurnAdmissionsRoot(sessionId), `${turnId}.json`); - } - - private rootTurnAdmissionsRoot(sessionId: string): string { - return join(this.sessionsRoot, sessionId, 'turn-admissions'); - } - - private rootSourceMessageProofPath(sessionId: string, messageId: string): string { - return join(this.sessionsRoot, sessionId, 'message-proofs', 'root', `${messageId}.json`); - } - - private async assertRootSourceMessageProofOwners(admission: RootTurnAdmission): Promise { - for (const source of admission.sourceMessages) { - const path = this.rootSourceMessageProofPath(admission.sessionId, source.messageId); - let raw: string; - try { - raw = await readFile(path, 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue; - throw error; - } - const existing = decodeRootSourceMessageProofPointer( - JSON.parse(raw), - admission.sessionId, - source.messageId, - ); - if (existing.turnId !== admission.turnId) { - throw new Error( - `Root source message identity belongs to both ${existing.turnId} and ${admission.turnId}`, - ); - } - } - } - - private async ensureRootSourceMessageProofs(admission: RootTurnAdmission): Promise { - for (const source of admission.sourceMessages) { - const pointer = { - schemaVersion: 1, - sessionId: admission.sessionId, - messageId: source.messageId, - turnId: admission.turnId, - }; - const path = this.rootSourceMessageProofPath(admission.sessionId, source.messageId); - const created = await writeExclusiveAtomic( - path, - `${JSON.stringify(pointer)}\n`, - { durable: true }, - this.durabilityRoot, - ); - if (created) continue; - const existing = decodeRootSourceMessageProofPointer( - JSON.parse(await readFile(path, 'utf8')), - admission.sessionId, - source.messageId, - ); - if (existing.turnId !== admission.turnId) { - throw new Error( - `Root source message identity belongs to both ${existing.turnId} and ${admission.turnId}`, - ); - } - } - } - - private async removeUncommittedRunDirectory(sessionId: string, runId: string): Promise { - const directory = this.runDir(sessionId, runId); - const entries = await readdir(directory, { withFileTypes: true }); - if (entries.some((entry) => !entry.isFile() || !isExclusiveWriteTemp(entry.name, 'run.json'))) { - return false; - } - await rm(directory, { recursive: true }); - await syncDirectory(this.runsRoot(sessionId)); - return true; - } - - private withQueue( - sessionId: string, - runId: string, - operation: () => Promise, - ): Promise { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - return chainWrite(this.writeQueues, `${sessionId}:${runId}`, operation); - } - - private withProjectionQueue( - sessionId: string, - type: AgentRunEventType, - operation: () => Promise, - ): Promise { - return chainWrite(this.projectionWriteQueues, `${sessionId}:${type}`, operation); - } - - private async readEventProjectionUnlocked( - sessionId: string, - type: AgentRunEventType, - ): Promise { - let raw: string; - try { - raw = await readFile(this.eventProjectionPath(sessionId, type), 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; - throw error; - } - const parsed = JSON.parse(raw) as { version?: unknown; event?: unknown }; - if (parsed.version !== 1 || !Object.hasOwn(parsed, 'event')) { - throw new Error(`Invalid AgentRun event projection for ${type}`); - } - if (parsed.event === null) return null; - if (!isProjectedAgentRunEvent(parsed.event, sessionId, type)) { - throw new Error(`Invalid AgentRun event projection for ${type}`); - } - return parsed.event; - } - - private async writeEventProjectionUnlocked( - sessionId: string, - type: AgentRunEventType, - event: AgentRunEvent | null, - ): Promise { - await writeAtomic( - this.eventProjectionPath(sessionId, type), - JSON.stringify({ version: 1, event }, sanitizeJson) + '\n', - ); - } - - private async initializeEventProjectionUnlocked( - sessionId: string, - currentRunId: string, - type: AgentRunEventType, - ): Promise { - try { - await readFile(this.eventProjectionPath(sessionId, type), 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - const runs = await readdir(this.runsRoot(sessionId), { - withFileTypes: true, - }); - if ( - runs.some( - (entry) => entry.isDirectory() && isSafeId(entry.name) && entry.name !== currentRunId, - ) - ) { - return; - } - await this.writeEventProjectionUnlocked(sessionId, type, null); - } - } -} - -interface LegacyAgentRunProjection { - readonly sessionId: string; - readonly type: AgentRunEventType; - readonly event: AgentRunEvent | null; -} - -interface LegacyAgentRunSnapshot { - readonly runs: readonly AgentRunHeader[]; - readonly events: readonly { - readonly sessionId: string; - readonly runId: string; - readonly records: readonly AgentRunEvent[]; - }[]; - readonly projections: readonly LegacyAgentRunProjection[]; - readonly admissions: readonly RootTurnAdmission[]; -} - -async function importLegacyAgentRuns( - root: string, - lease: OperationalStateDatabaseLease, - options: SqliteAgentRunStoreOptions, -): Promise { - const snapshot = await readLegacyAgentRunSnapshot(root); - const fingerprint = createHash('sha256') - .update(JSON.stringify(snapshot, sanitizeJson)) - .digest('hex'); - completeOperationalStoreCutover(lease, { - storeName: 'agent_runs', - sourcePath: join(root, 'sessions'), - sourceFingerprint: `sha256:${fingerprint}`, - failpoint: options.failpoint, - importAndValidate: (db) => { - for (const run of snapshot.runs) insertOrValidateAgentRun(db, run); - for (const stream of snapshot.events) { - stream.records.forEach((event, sequence) => { - insertOrValidateAgentRunEvent(db, event, sequence); - }); - } - for (const projection of snapshot.projections) { - insertOrValidateAgentRunProjection(db, projection); - } - for (const admission of snapshot.admissions) { - insertOrValidateRootTurnAdmission(db, admission); - } - return { - agent_runs: snapshot.runs.length, - agent_run_events: snapshot.events.reduce( - (count, stream) => count + stream.records.length, - 0, - ), - agent_run_projections: snapshot.projections.length, - root_turn_admissions: snapshot.admissions.length, - root_source_message_proofs: snapshot.admissions.reduce( - (count, admission) => count + admission.sourceMessages.length, - 0, - ), - }; - }, - }); -} - -async function readLegacyAgentRunSnapshot(root: string): Promise { - const sessionsRoot = join(root, 'sessions'); - let sessions; - try { - sessions = await readdir(sessionsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return { runs: [], events: [], projections: [], admissions: [] }; - } - throw error; - } - const legacy = new FileAgentRunStore(root); - const runs: AgentRunHeader[] = []; - const events: Array<{ - sessionId: string; - runId: string; - records: AgentRunEvent[]; - }> = []; - const projections: LegacyAgentRunProjection[] = []; - const admissions: RootTurnAdmission[] = []; - for (const session of sessions.sort((left, right) => left.name.localeCompare(right.name))) { - if (!session.isDirectory() || !isSafeId(session.name)) continue; - const sessionRuns = await readLegacyAgentRunsForSession(root, legacy, session.name); - runs.push(...sessionRuns.runs); - events.push(...sessionRuns.events); - const sessionAdmissions = await legacy.listRootTurnAdmissionsForRecovery(session.name); - admissions.push(...sessionAdmissions); - await assertLegacyRootProofClosure(root, session.name, sessionAdmissions); - projections.push(...(await readLegacyAgentRunProjections(root, session.name))); - } - return { runs, events, projections, admissions }; -} - -async function readLegacyAgentRunsForSession( - root: string, - legacy: FileAgentRunStore, - sessionId: string, -): Promise<{ - runs: AgentRunHeader[]; - events: Array<{ sessionId: string; runId: string; records: AgentRunEvent[] }>; -}> { - const runsRoot = join(root, 'sessions', sessionId, 'runs'); - let entries; - try { - entries = await readdir(runsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return { runs: [], events: [] }; - throw error; - } - const runs: AgentRunHeader[] = []; - const events: Array<{ sessionId: string; runId: string; records: AgentRunEvent[] }> = []; - for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { - if (!entry.isDirectory() || !isSafeId(entry.name)) { - throw new Error(`Invalid AgentRun entry for session ${sessionId}: ${entry.name}`); - } - const runRoot = join(runsRoot, entry.name); - const runEntries = await readdir(runRoot, { withFileTypes: true }); - const hasHeader = runEntries.some( - (candidate) => candidate.isFile() && candidate.name === 'run.json', - ); - if (!hasHeader) { - const runtimeOnly = runEntries.every( - (candidate) => - (candidate.isFile() && candidate.name === 'runtime-events.jsonl') || - (candidate.isDirectory() && candidate.name === 'runtime-partials') || - (candidate.isFile() && isExclusiveWriteTemp(candidate.name, 'run.json')), - ); - if (runtimeOnly) continue; - throw Object.assign(new Error(`AgentRun ${entry.name} has no durable header`), { - code: 'ENOENT', - }); - } - const run = await legacy.readRun(sessionId, entry.name); - runs.push(run); - events.push({ - sessionId, - runId: run.runId, - records: await legacy.readEventsForRecovery(sessionId, run.runId), - }); - } - return { runs, events }; -} - -async function readLegacyAgentRunProjections( - root: string, - sessionId: string, -): Promise { - const projectionRoot = join(root, 'sessions', sessionId, 'projections'); - let entries; - try { - entries = await readdir(projectionRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const result: LegacyAgentRunProjection[] = []; - for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { - if (!entry.isFile() || !entry.name.endsWith('.json')) { - throw new Error(`Invalid AgentRun projection entry: ${entry.name}`); - } - const type = entry.name.slice(0, -'.json'.length) as AgentRunEventType; - const parsed = JSON.parse(await readFile(join(projectionRoot, entry.name), 'utf8')) as { - version?: unknown; - event?: unknown; - }; - if (parsed.version !== 1 || !Object.hasOwn(parsed, 'event')) { - throw new Error(`Invalid AgentRun event projection for ${type}`); - } - if (parsed.event !== null && !isProjectedAgentRunEvent(parsed.event, sessionId, type)) { - throw new Error(`Invalid AgentRun event projection for ${type}`); - } - result.push({ - sessionId, - type, - event: parsed.event as AgentRunEvent | null, - }); - } - return result; -} - -async function assertLegacyRootProofClosure( - root: string, - sessionId: string, - admissions: readonly RootTurnAdmission[], -): Promise { - const expected = new Map(); - for (const admission of admissions) { - for (const source of admission.sourceMessages) { - expected.set(source.messageId, admission.turnId); - } - } - const proofRoot = join(root, 'sessions', sessionId, 'message-proofs', 'root'); - let entries; - try { - entries = await readdir(proofRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - if (expected.size === 0) return; - throw new Error(`Missing root source message proofs for session ${sessionId}`); - } - throw error; - } - const seen = new Set(); - for (const entry of entries) { - if (!entry.isFile() || !entry.name.endsWith('.json')) { - throw new Error(`Invalid root source message proof entry: ${entry.name}`); - } - const messageId = entry.name.slice(0, -'.json'.length); - assertSafeId(messageId, 'Invalid root source message proof identity'); - const pointer = decodeRootSourceMessageProofPointer( - JSON.parse(await readFile(join(proofRoot, entry.name), 'utf8')), - sessionId, - messageId, - ); - if (expected.get(messageId) !== pointer.turnId) { - throw new Error(`Orphan root source message proof: ${messageId}`); - } - seen.add(messageId); - } - if (seen.size !== expected.size) { - throw new Error(`Missing root source message proofs for session ${sessionId}`); - } -} - -function normalizeAgentRunHeader(value: unknown, sessionId: string, runId: string): AgentRunHeader { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - return decodeAgentRunHeader(JSON.parse(JSON.stringify(value, sanitizeJson)), { - sessionId, - runId, - }); -} - -function readSqliteAgentRun(db: DatabaseSync, sessionId: string, runId: string): AgentRunHeader { - const row = db - .prepare(` - SELECT record_json - FROM core_agent_runs - WHERE session_id = ? AND run_id = ? - `) - .get(sessionId, runId) as { record_json?: unknown } | undefined; - if (!row) { - const error = new Error(`Agent run does not exist: ${runId}`) as NodeJS.ErrnoException; - error.code = 'ENOENT'; - throw error; - } - if (typeof row.record_json !== 'string') throw new Error('Invalid SQLite AgentRun row'); - return normalizeAgentRunHeader(JSON.parse(row.record_json), sessionId, runId); -} - -function readSqliteAgentRunEvents( - db: DatabaseSync, - sessionId: string, - runId: string, -): AgentRunEvent[] { - const rows = db - .prepare(` - SELECT record_json - FROM core_agent_run_events - WHERE session_id = ? AND run_id = ? - ORDER BY sequence - `) - .all(sessionId, runId) as Array<{ record_json?: unknown }>; - if (rows.length === 0) return []; - const header = readSqliteAgentRun(db, sessionId, runId); - return rows.map((row) => { - if (typeof row.record_json !== 'string') { - throw new Error('Invalid SQLite AgentRun event row'); - } - return decodeAgentRunEvent(JSON.parse(row.record_json), { - sessionId, - runId, - turnId: header.turnId, - }); - }); -} - -function readSqliteAgentRunEventsForEvidence( - db: DatabaseSync, - sessionId: string, - runId: string, -): AgentRunEvent[] { - const rows = db - .prepare(` - SELECT sequence, record_json - FROM core_agent_run_events - WHERE session_id = ? AND run_id = ? - ORDER BY sequence - `) - .all(sessionId, runId) as Array<{ sequence?: unknown; record_json?: unknown }>; - if (rows.length === 0) return []; - const header = readSqliteAgentRun(db, sessionId, runId); - return rows.map((row) => { - const lineNumber = - typeof row.sequence === 'number' && Number.isSafeInteger(row.sequence) ? row.sequence + 1 : 0; - try { - if (typeof row.record_json !== 'string') { - throw new Error('Invalid SQLite AgentRun event row'); - } - return decodeAgentRunEvent(JSON.parse(row.record_json), { - sessionId, - runId, - turnId: header.turnId, - }); - } catch (error) { - return { - type: 'event_corrupt', - id: `run-event-corrupt-${lineNumber}`, - runId, - sessionId, - turnId: header.turnId, - ts: header.updatedAt, - message: error instanceof Error ? error.message : 'Invalid SQLite AgentRun event row', - data: { lineNumber }, - }; - } - }); -} - -function insertAgentRunEvent(db: DatabaseSync, event: AgentRunEvent): void { - const row = db - .prepare(` - SELECT COALESCE(MAX(sequence), -1) + 1 AS sequence - FROM core_agent_run_events - WHERE session_id = ? AND run_id = ? - `) - .get(event.sessionId, event.runId) as { sequence?: unknown }; - if (typeof row.sequence !== 'number' || !Number.isSafeInteger(row.sequence)) { - throw new Error('Invalid next AgentRun event sequence'); - } - db.prepare(` - INSERT INTO core_agent_run_events( - session_id, run_id, sequence, event_id, event_type, event_ts, record_json - ) VALUES (?, ?, ?, ?, ?, ?, ?) - `).run( - event.sessionId, - event.runId, - row.sequence, - event.id, - event.type, - event.ts, - JSON.stringify(event, sanitizeJson), - ); -} - -function readSqliteAgentRunProjection( - db: DatabaseSync, - sessionId: string, - type: AgentRunEventType, -): AgentRunEvent | null | undefined { - const row = db - .prepare(` - SELECT event_json - FROM core_agent_run_projections - WHERE session_id = ? AND event_type = ? - `) - .get(sessionId, type) as { event_json?: unknown } | undefined; - if (!row) return undefined; - if (row.event_json === null) return null; - if (typeof row.event_json !== 'string') { - throw new Error(`Invalid AgentRun event projection for ${type}`); - } - const event = JSON.parse(row.event_json); - if (!isProjectedAgentRunEvent(event, sessionId, type)) { - throw new Error(`Invalid AgentRun event projection for ${type}`); - } - return event; -} - -function writeSqliteAgentRunProjection( - db: DatabaseSync, - sessionId: string, - type: AgentRunEventType, - event: AgentRunEvent | null, -): void { - db.prepare(` - INSERT INTO core_agent_run_projections(session_id, event_type, event_json) - VALUES (?, ?, ?) - ON CONFLICT(session_id, event_type) DO UPDATE SET event_json = excluded.event_json - `).run(sessionId, type, event === null ? null : JSON.stringify(event, sanitizeJson)); -} - -function readSqliteRootTurnAdmission( - db: DatabaseSync, - sessionId: string, - turnId: string, -): RootTurnAdmission | undefined { - const row = db - .prepare(` - SELECT record_json - FROM core_root_turn_admissions - WHERE session_id = ? AND turn_id = ? - `) - .get(sessionId, turnId) as { record_json?: unknown } | undefined; - if (!row) return undefined; - if (typeof row.record_json !== 'string') throw new Error('Invalid root turn admission row'); - return normalizeRootTurnAdmission(JSON.parse(row.record_json), sessionId, turnId); -} - -function normalizeAdmitRootTurnInput(input: AdmitRootTurnInput): RootTurnAdmission { - assertSafeId(input.sessionId, 'Invalid session id'); - assertSafeId(input.turnId, 'Invalid turn id'); - assertSafeId(input.proposedRunId, 'Invalid run id'); - if (input.proposedUserMessageId !== null) { - assertSafeId(input.proposedUserMessageId, 'Invalid user message id'); - } - if (input.previousRootTurnId !== null) { - assertSafeId(input.previousRootTurnId, 'Invalid previous root turn id'); - if (input.previousRootTurnId === input.turnId) { - throw new Error('Root turn admission cannot reference itself'); - } - } - if (!Number.isSafeInteger(input.admittedAt) || input.admittedAt < 0) { - throw new Error('Invalid root turn admission timestamp'); - } - const { normalizedInput, sourceMessages } = normalizeRootTurnAdmissionPayload( - input.normalizedInput, - input.sourceMessages, - ); - const turnOrchestration = normalizeTurnOrchestration(input.turnOrchestration); - const admission: RootTurnAdmission = { - schemaVersion: ROOT_TURN_ADMISSION_SCHEMA_VERSION, - sessionId: input.sessionId, - turnId: input.turnId, - runId: input.proposedRunId, - userMessageId: input.proposedUserMessageId, - execution: normalizeRootExecutionDescriptor(input.execution), - previousRootTurnId: input.previousRootTurnId, - normalizedInput, - ...(turnOrchestration ? { turnOrchestration } : {}), - sourceMessages, - admittedAt: input.admittedAt, - }; - assertRootTurnAdmissionContract(admission); - assertRootTurnAdmissionRecordSize(admission); - return deepFreezeRootTurnAdmission(admission); -} - -function insertOrValidateAgentRun(db: DatabaseSync, run: AgentRunHeader): void { - const encoded = JSON.stringify(run, sanitizeJson); - const result = db - .prepare(` - INSERT OR IGNORE INTO core_agent_runs(session_id, run_id, created_at, record_json) - VALUES (?, ?, ?, ?) - `) - .run(run.sessionId, run.runId, run.createdAt, encoded); - if (result.changes !== 0) return; - if (!isDeepStrictEqual(readSqliteAgentRun(db, run.sessionId, run.runId), run)) { - throw new Error(`AgentRun cutover conflict: ${run.runId}`); - } -} - -function insertOrValidateAgentRunEvent( - db: DatabaseSync, - event: AgentRunEvent, - sequence: number, -): void { - const encoded = JSON.stringify(event, sanitizeJson); - const result = db - .prepare(` - INSERT OR IGNORE INTO core_agent_run_events( - session_id, run_id, sequence, event_id, event_type, event_ts, record_json - ) VALUES (?, ?, ?, ?, ?, ?, ?) - `) - .run(event.sessionId, event.runId, sequence, event.id, event.type, event.ts, encoded); - if (result.changes !== 0) return; - const row = db - .prepare(` - SELECT record_json - FROM core_agent_run_events - WHERE session_id = ? AND run_id = ? AND sequence = ? - `) - .get(event.sessionId, event.runId, sequence) as { record_json?: unknown } | undefined; - if (typeof row?.record_json !== 'string' || row.record_json !== encoded) { - throw new Error(`AgentRun event cutover conflict: ${event.runId}:${sequence}`); - } -} - -function insertOrValidateAgentRunProjection( - db: DatabaseSync, - projection: LegacyAgentRunProjection, -): void { - const encoded = projection.event === null ? null : JSON.stringify(projection.event, sanitizeJson); - const result = db - .prepare(` - INSERT OR IGNORE INTO core_agent_run_projections(session_id, event_type, event_json) - VALUES (?, ?, ?) - `) - .run(projection.sessionId, projection.type, encoded); - if (result.changes !== 0) return; - const existing = readSqliteAgentRunProjection(db, projection.sessionId, projection.type); - if (!isDeepStrictEqual(existing, projection.event)) { - throw new Error(`AgentRun projection cutover conflict: ${projection.type}`); - } -} - -function insertOrValidateRootTurnAdmission(db: DatabaseSync, admission: RootTurnAdmission): void { - const result = db - .prepare(` - INSERT OR IGNORE INTO core_root_turn_admissions( - session_id, turn_id, admitted_at, record_json - ) VALUES (?, ?, ?, ?) - `) - .run(admission.sessionId, admission.turnId, admission.admittedAt, JSON.stringify(admission)); - if ( - result.changes === 0 && - !isDeepStrictEqual( - readSqliteRootTurnAdmission(db, admission.sessionId, admission.turnId), - admission, - ) - ) { - throw new Error(`Root turn admission cutover conflict: ${admission.turnId}`); - } - for (const source of admission.sourceMessages) { - const proof = db - .prepare(` - SELECT turn_id - FROM core_root_source_message_proofs - WHERE session_id = ? AND message_id = ? - `) - .get(admission.sessionId, source.messageId) as { turn_id?: unknown } | undefined; - if (proof && proof.turn_id !== admission.turnId) { - throw new Error(`Root source message proof cutover conflict: ${source.messageId}`); - } - if (!proof) { - db.prepare(` - INSERT INTO core_root_source_message_proofs(session_id, message_id, turn_id) - VALUES (?, ?, ?) - `).run(admission.sessionId, source.messageId, admission.turnId); - } - } -} - -const MUTABLE_AGENT_RUN_HEADER_FIELDS = new Set([ - 'status', - 'updatedAt', - 'completedAt', - 'failureClass', - 'failureMessage', - 'abortSource', - 'traceWriteError', -]); - -function assertMutableRunHeaderPatch(patch: Partial): void { - const immutable = Object.keys(patch).filter( - (key) => !MUTABLE_AGENT_RUN_HEADER_FIELDS.has(key as keyof AgentRunHeader), - ); - if (immutable.length > 0) { - throw new Error(`AgentRun admission identity is immutable: ${immutable.sort().join(', ')}`); - } -} - -function shouldPreserveCheckpointProjectionDuringAppend( - current: AgentRunEvent | null | undefined, - candidate: AgentRunEvent, -): boolean { - if (!current) return false; - const currentSourceBound = historyCompactProjectionIsSourceBound(current); - const candidateSourceBound = historyCompactProjectionIsSourceBound(candidate); - if (currentSourceBound !== candidateSourceBound) return currentSourceBound; - const currentCoverage = historyCompactProjectionCoverage(current); - const candidateCoverage = historyCompactProjectionCoverage(candidate); - return ( - currentCoverage !== undefined && - (candidateCoverage === undefined || currentCoverage > candidateCoverage) - ); -} - -function shouldPreserveProjectionDuringRepair( - current: AgentRunEvent | null | undefined, - candidate: AgentRunEvent | null, - type: AgentRunEventType, -): boolean { - if (!current) return false; - if (type !== 'history_compact_checkpoint_recorded') return true; - const currentSourceBound = historyCompactProjectionIsSourceBound(current); - const candidateSourceBound = candidate ? historyCompactProjectionIsSourceBound(candidate) : false; - if (currentSourceBound !== candidateSourceBound) return currentSourceBound; - const currentCoverage = historyCompactProjectionCoverage(current); - const candidateCoverage = candidate && historyCompactProjectionCoverage(candidate); - return ( - currentCoverage !== undefined && - (candidateCoverage === null || - candidateCoverage === undefined || - currentCoverage >= candidateCoverage) - ); -} - -function historyCompactProjectionIsSourceBound(event: AgentRunEvent): boolean { - const checkpoint = event.data?.checkpoint; - if (!checkpoint || typeof checkpoint !== 'object') return false; - const source = (checkpoint as { source?: unknown }).source; - if (!source || typeof source !== 'object') return false; - return (source as { kind?: unknown }).kind === 'runtime_event_projection'; -} - -function assertNoReservedToolLedgerFact(event: RuntimeEvent): void { - assertNoReservedWorkspaceAuthorityAppend(event); - if (event.actions?.continuationStart !== undefined) { - throw new Error('Continuation start facts require SQLite continuation authority'); - } - const validation = validateGenericToolLedgerAppend(event); - if (validation.ok) return; - if (validation.code === 'reserved_recovery_fact') { - throw new Error('Tool recovery facts require the atomic recovery bundle writer'); - } - if (validation.code === 'reserved_tool_boundary_fact') { - throw new Error('Durable tool facts require the atomic tool boundary writer'); - } - throw new Error(`RuntimeEvent ${event.id} violates its semantic lane`); -} - -function canonicalizeRuntimeEventForStorage(event: RuntimeEvent): RuntimeEvent { - return encodeCanonicalRuntimeEvent(event).event; -} - -function isToolLedgerBearingEvent(event: RuntimeEvent): boolean { - return ( - event.content?.kind === 'function_call' || - event.content?.kind === 'function_response' || - event.actions?.toolDispatch !== undefined || - event.actions?.toolRecovery !== undefined - ); -} - -function historyCompactProjectionCoverage(event: AgentRunEvent): number | undefined { - const checkpoint = event.data?.checkpoint; - if (!checkpoint || typeof checkpoint !== 'object') return undefined; - const coverage = (checkpoint as { coverage?: unknown }).coverage; - if (!coverage || typeof coverage !== 'object') return undefined; - const eventCount = (coverage as { eventCount?: unknown }).eventCount; - return typeof eventCount === 'number' && Number.isSafeInteger(eventCount) && eventCount >= 0 - ? eventCount - : undefined; -} - -class FileRuntimeEventStore implements DurableRuntimeEventStore { - private readonly durabilityRoot: string; - private readonly sessionsRoot: string; - private readonly writeQueues = new Map>(); - private readonly immutableSteeringWriteQueues = new Map>(); - - constructor(workspaceRoot: string) { - this.durabilityRoot = resolve(workspaceRoot); - this.sessionsRoot = join(this.durabilityRoot, 'sessions'); - } - - async appendRuntimeEvent( - sessionId: string, - runId: string, - event: RuntimeEvent, - options: { durable?: boolean } = {}, - ): Promise { - const canonicalEvent = canonicalizeRuntimeEventForStorage(event); - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - assertNoReservedToolLedgerFact(canonicalEvent); - const steeringMessageId = immutableSteeringMessageId(canonicalEvent); - if (steeringMessageId) { - await this.withImmutableSteeringQueue(sessionId, async () => { - if (await this.preflightImmutableSteeringMessage(canonicalEvent, steeringMessageId)) return; - await this.appendRuntimeEventForRun(sessionId, runId, canonicalEvent, options); - }); - return; - } - await this.appendRuntimeEventForRun(sessionId, runId, canonicalEvent, options); - } - - async importConversationCopyRuntimeEvents( - sessionId: string, - batches: readonly ConversationCopyRuntimeEventBatch[], - ): Promise { - assertSafeId(sessionId, 'Invalid session id'); - const canonicalBatches = batches.map(({ runId, events }) => { - assertSafeId(runId, 'Invalid run id'); - return { - runId, - events: events.map(canonicalizeRuntimeEventForStorage), - }; - }); - const canonicalEvents = canonicalBatches.flatMap(({ events }) => events); - for (const event of canonicalEvents) assertNoReservedWorkspaceAuthorityAppend(event); - if (canonicalEvents.some((event) => event.partial)) { - throw new Error('Conversation copy cannot import partial RuntimeEvents'); - } - const scan = scanToolLedger(canonicalEvents); - if (scan.hasCorruption) { - throw new Error( - `Conversation copy RuntimeEvent ledger is corrupt: ${scan.issues[0]?.code ?? 'unknown'}`, - ); - } - - for (const { runId, events } of canonicalBatches) { - await this.withQueue(sessionId, runId, async () => { - const header = await this.readRunHeader(sessionId, runId); - for (const event of events) decodeRuntimeEvent(event, header); - const path = this.runtimeEventsPath(sessionId, runId); - const existing = await readRuntimeEventJsonl(path, header); - if (existing.length > 0) { - if (!isDeepStrictEqual(existing, events)) { - throw new Error(`Conversation copy RuntimeEvent identity conflict for run ${runId}`); - } - } else if (events.length > 0) { - await appendJsonl( - path, - `${events.map((event) => encodeCanonicalRuntimeEvent(event).json).join('\n')}\n`, - { durable: true, durabilityRoot: this.durabilityRoot }, - ); - } - for (const event of events) { - await this.settleImmutableRuntimeEventPostEffects({ - sessionId, - runId, - event, - path, - ensureDurability: false, - }); - } - }); - } - } - - private async appendRuntimeEventForRun( - sessionId: string, - runId: string, - event: RuntimeEvent, - options: { durable?: boolean }, - ): Promise { - await this.withQueue(sessionId, runId, async () => { - const header = await this.readRunHeader(sessionId, runId); - decodeRuntimeEvent(event, header); - const partial = partialRuntimeStream(event); - if (partial) { - const partialPath = this.runtimePartialPath(sessionId, runId, partial.key); - try { - await readFile(partialPath, 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - const immutableEvents = await readRuntimeEventJsonl( - this.runtimeEventsPath(sessionId, runId), - header, - ); - if ( - immutableEvents.some((item) => completedPartialRuntimeStreamKey(item) === partial.key) - ) { - return; - } - const metadata: RuntimePartialSnapshot = { - version: 1, - event: partial.snapshot, - ...(immutableEvents.at(-1)?.id ? { afterEventId: immutableEvents.at(-1)!.id } : {}), - }; - await writeAtomic(partialPath, JSON.stringify(metadata, sanitizeJson) + '\n', { - ...options, - durabilityRoot: this.durabilityRoot, - }); - } - if (partial.text) { - if (options.durable) { - await appendFileDurably(partialPath, partial.text, this.durabilityRoot); - } else { - await appendFile(partialPath, partial.text, 'utf8'); - } - } - return; - } - const existingEvents = await readRuntimeEventJsonl( - this.runtimeEventsPath(sessionId, runId), - header, - ); - const matching = existingEvents.filter((item) => item.id === event.id); - if (matching.length > 1) { - throw new Error(`RuntimeEvent ${event.id} appears more than once in run ${runId}`); - } - if (matching.length === 1) { - if (!isDeepStrictEqual(matching[0], event)) { - throw new Error(`RuntimeEvent identity conflict for ${event.id}`); - } - await this.settleImmutableRuntimeEventPostEffects({ - sessionId, - runId, - event, - path: this.runtimeEventsPath(sessionId, runId), - ensureDurability: - options.durable === true || immutableSteeringMessageId(event) !== undefined, - }); - return; - } - if (isToolLedgerBearingEvent(event)) { - const validation = validateToolLedgerTransition({ - existingEvents, - candidateEvents: [event], - expectedTransition: 'generic_append', - }); - if (!validation.ok) { - throw new Error( - `Tool ledger transition rejected: ${validation.code} at ${validation.eventId}`, - ); - } - } - const steeringMessageId = immutableSteeringMessageId(event); - await appendJsonl( - this.runtimeEventsPath(sessionId, runId), - encodeCanonicalRuntimeEvent(event).json + '\n', - { - ...options, - ...(steeringMessageId ? { durable: true } : {}), - durabilityRoot: this.durabilityRoot, - }, - ); - await this.settleImmutableRuntimeEventPostEffects({ - sessionId, - runId, - event, - path: this.runtimeEventsPath(sessionId, runId), - ensureDurability: false, - }); - }); - } - - private async settleImmutableRuntimeEventPostEffects(input: { - sessionId: string; - runId: string; - event: RuntimeEvent; - path: string; - ensureDurability: boolean; - }): Promise { - if (input.ensureDurability) { - try { - await syncFile(input.path); - await syncDirectoryChain(dirname(input.path), this.durabilityRoot); - } catch (error) { - throw new DurableStoreWriteError( - `RuntimeEvent did not reach stable storage: ${input.path}`, - error, - ); - } - } - const steeringMessageId = immutableSteeringMessageId(input.event); - if (steeringMessageId) { - try { - await this.ensureImmutableSteeringMessageProof(input.event, steeringMessageId); - } catch (error) { - if (!(error instanceof DurableStoreWriteError)) throw error; - throw new RuntimeEventPostEffectError( - `RuntimeEvent ${input.event.id} is durable but its steering proof was not published`, - error, - ); - } - } - const completedPartialKey = completedPartialRuntimeStreamKey(input.event); - if (completedPartialKey) { - await rm(this.runtimePartialPath(input.sessionId, input.runId, completedPartialKey), { - force: true, - }).catch(() => { - // The immutable final is already durable. Reads suppress any stale snapshot. - }); - } - } - - async ensureTerminalRuntimeEventDurable( - sessionId: string, - runId: string, - event: RuntimeEvent, - ): Promise { - const canonicalEvent = canonicalizeRuntimeEventForStorage(event); - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - assertNoReservedToolLedgerFact(canonicalEvent); - if (canonicalEvent.partial || !isTerminalRuntimeEvent(canonicalEvent)) { - throw new Error( - 'Only a final terminal RuntimeEvent can cross the terminal durability barrier', - ); - } - await this.withQueue(sessionId, runId, async () => { - const path = this.runtimeEventsPath(sessionId, runId); - const header = await this.readRunHeader(sessionId, runId); - decodeRuntimeEvent(canonicalEvent, header); - const existing = await readRuntimeEventJsonl(path, header); - const matching = existing.filter((candidate) => candidate.id === canonicalEvent.id); - if (matching.length > 1) { - throw new Error(`RuntimeEvent ${canonicalEvent.id} appears more than once in run ${runId}`); - } - if (matching.length === 1) { - if (!isDeepStrictEqual(matching[0], canonicalEvent)) { - throw new Error( - `RuntimeEvent ${canonicalEvent.id} does not match the durable ledger record`, - ); - } - try { - await syncFile(path); - await syncDirectoryChain(dirname(path), this.durabilityRoot); - } catch (error) { - throw new DurableStoreWriteError( - `Terminal RuntimeEvent did not reach stable storage: ${path}`, - error, - ); - } - return; - } - const existingTerminal = existing.find(isTerminalRuntimeEvent); - if (existingTerminal) { - throw new Error(`Run ${runId} already has terminal RuntimeEvent ${existingTerminal.id}`); - } - await appendJsonl(path, encodeCanonicalRuntimeEvent(canonicalEvent).json + '\n', { - durable: true, - durabilityRoot: this.durabilityRoot, - }); - }); - } - - async readRuntimeEvents(sessionId: string, runId: string): Promise { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - const events = await readRuntimeEventJsonl( - this.runtimeEventsPath(sessionId, runId), - await this.readRunHeader(sessionId, runId), - ); - const partials = await this.readRuntimePartials(sessionId, runId); - const completedPartialKeys = new Set( - events - .map(completedPartialRuntimeStreamKey) - .filter((key): key is string => key !== undefined), - ); - const visiblePartials = partials.filter(({ event }) => { - const key = partialRuntimeStream(event)?.key; - return !key || !completedPartialKeys.has(key); - }); - return mergeRuntimePartialSnapshots(events, visiblePartials); - } - - async readRuntimeEventsBounded( - sessionId: string, - runId: string, - budget: EvidenceReadBudget, - ): Promise> { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - assertEvidenceReadBudget(budget); - let storedBytes = await fileSizeOrZero(this.runtimeEventsPath(sessionId, runId)); - let partialEntries: Dirent[]; - try { - partialEntries = await readdir(this.runtimePartialsDir(sessionId, runId), { - withFileTypes: true, - }); - } catch (error) { - if (!isMissingFile(error)) throw error; - partialEntries = []; - } - for (const entry of partialEntries) { - if (!entry.isFile() || !entry.name.endsWith('.partial')) continue; - storedBytes += await fileSizeOrZero( - join(this.runtimePartialsDir(sessionId, runId), entry.name), - ); - if (storedBytes > budget.maxBytes) return { status: 'limit_exceeded' }; - } - const records = await this.readRuntimeEvents(sessionId, runId); - if (records.length > budget.maxRecords) return { status: 'limit_exceeded' }; - return { - status: 'complete', - records, - sourceRecordCount: records.length, - storedBytes, - }; - } - - async readImmutableRuntimeEvents(sessionId: string, runId: string): Promise { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - return readRuntimeEventJsonl( - this.runtimeEventsPath(sessionId, runId), - await this.readRunHeader(sessionId, runId), - ); - } - - async readImmutableSteeringMessageProof( - sessionId: string, - messageId: string, - ): Promise { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(messageId, 'Invalid message id'); - let raw: string; - try { - raw = await readFile(this.immutableSteeringMessageProofPath(sessionId, messageId), 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; - throw error; - } - const parsed: unknown = JSON.parse(raw); - if ( - !isPlainRecord(parsed) || - !hasExactKeys(parsed, ['schemaVersion', 'messageId', 'event']) || - parsed.schemaVersion !== 1 || - parsed.messageId !== messageId || - !isPlainRecord(parsed.event) || - typeof parsed.event.runId !== 'string' || - !isSafeId(parsed.event.runId) - ) { - throw new Error(`Invalid immutable steering message proof: ${messageId}`); - } - const event = decodeRuntimeEvent( - parsed.event, - await this.readRunHeader(sessionId, parsed.event.runId), - ); - if (immutableSteeringMessageId(event) !== messageId) { - throw new Error(`Invalid immutable steering message proof: ${messageId}`); - } - return Object.freeze({ event }); - } - - async repairImmutableSteeringMessageProofsForRecovery(sessionId: string): Promise { - assertSafeId(sessionId, 'Invalid session id'); - await this.withImmutableSteeringQueue(sessionId, async () => { - const events = await this.readSessionRuntimeEvents(sessionId); - for (const event of events) { - const messageId = immutableSteeringMessageId(event); - if (messageId) await this.ensureImmutableSteeringMessageProof(event, messageId); - } - }); - } - - async readSessionRuntimeEvents(sessionId: string): Promise { - assertSafeId(sessionId, 'Invalid session id'); - const runsRoot = this.runsRoot(sessionId); - let entries; - try { - entries = await readdir(runsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const ordered: Array<{ - event: RuntimeEvent; - runId: string; - eventIndex: number; - }> = []; - for (const entry of entries) { - if (!entry.isDirectory() || !isSafeId(entry.name)) continue; - const events = await this.readRuntimeEvents(sessionId, entry.name); - for (let eventIndex = 0; eventIndex < events.length; eventIndex += 1) { - ordered.push({ - event: events[eventIndex]!, - runId: entry.name, - eventIndex, - }); - } - } - ordered.sort( - (a, b) => - a.event.ts - b.event.ts || - a.runId.localeCompare(b.runId) || - a.eventIndex - b.eventIndex || - a.event.id.localeCompare(b.event.id), - ); - return ordered.map((item) => item.event); - } - - private runsRoot(sessionId: string): string { - assertSafeId(sessionId, 'Invalid session id'); - return join(this.sessionsRoot, sessionId, 'runs'); - } - - private runDir(sessionId: string, runId: string): string { - assertSafeId(runId, 'Invalid run id'); - return join(this.runsRoot(sessionId), runId); - } - - private runtimeEventsPath(sessionId: string, runId: string): string { - return join(this.runDir(sessionId, runId), 'runtime-events.jsonl'); - } - - private async readRunHeader(sessionId: string, runId: string): Promise { - return decodeAgentRunHeader( - JSON.parse(await readFile(join(this.runDir(sessionId, runId), 'run.json'), 'utf8')), - { - sessionId, - runId, - }, - ); + const event = JSON.parse(row.event_json); + if (!isProjectedAgentRunEvent(event, sessionId, type)) { + throw new Error(`Invalid AgentRun event projection for ${type}`); } + return event; +} - private runtimePartialsDir(sessionId: string, runId: string): string { - return join(this.runDir(sessionId, runId), 'runtime-partials'); - } +function writeSqliteAgentRunProjection( + db: DatabaseSync, + sessionId: string, + type: AgentRunEventType, + event: AgentRunEvent | null, +): void { + db.prepare(` + INSERT INTO core_agent_run_projections(session_id, event_type, event_json) + VALUES (?, ?, ?) + ON CONFLICT(session_id, event_type) DO UPDATE SET event_json = excluded.event_json + `).run(sessionId, type, event === null ? null : JSON.stringify(event, sanitizeJson)); +} - private runtimePartialPath(sessionId: string, runId: string, key: string): string { - return join(this.runtimePartialsDir(sessionId, runId), `${key}.partial`); - } +function readSqliteRootTurnAdmission( + db: DatabaseSync, + sessionId: string, + turnId: string, +): RootTurnAdmission | undefined { + const row = db + .prepare(` + SELECT record_json + FROM core_root_turn_admissions + WHERE session_id = ? AND turn_id = ? + `) + .get(sessionId, turnId) as { record_json?: unknown } | undefined; + if (!row) return undefined; + if (typeof row.record_json !== 'string') throw new Error('Invalid root turn admission row'); + return normalizeRootTurnAdmission(JSON.parse(row.record_json), sessionId, turnId); +} - private immutableSteeringMessageProofPath(sessionId: string, messageId: string): string { - return join(this.sessionsRoot, sessionId, 'message-proofs', 'steering', `${messageId}.json`); +function normalizeAdmitRootTurnInput(input: AdmitRootTurnInput): RootTurnAdmission { + assertSafeId(input.sessionId, 'Invalid session id'); + assertSafeId(input.turnId, 'Invalid turn id'); + assertSafeId(input.proposedRunId, 'Invalid run id'); + if (input.proposedUserMessageId !== null) { + assertSafeId(input.proposedUserMessageId, 'Invalid user message id'); } - - private async ensureImmutableSteeringMessageProof( - event: RuntimeEvent, - messageId: string, - ): Promise { - const path = this.immutableSteeringMessageProofPath(event.sessionId, messageId); - const stored = { schemaVersion: 1, messageId, event }; - const created = await writeExclusiveAtomic( - path, - `${JSON.stringify(stored, sanitizeJson)}\n`, - { durable: true }, - this.durabilityRoot, - ); - if (created) return; - const existing = await this.readImmutableSteeringMessageProof(event.sessionId, messageId); - if ( - !existing || - !isDeepStrictEqual(existing.event, JSON.parse(JSON.stringify(event, sanitizeJson))) - ) { - throw new Error(`Immutable steering message identity conflict: ${messageId}`); + if (input.previousRootTurnId !== null) { + assertSafeId(input.previousRootTurnId, 'Invalid previous root turn id'); + if (input.previousRootTurnId === input.turnId) { + throw new Error('Root turn admission cannot reference itself'); } } - - private async preflightImmutableSteeringMessage( - event: RuntimeEvent, - messageId: string, - ): Promise { - const canonicalEvent = JSON.parse(JSON.stringify(event, sanitizeJson)) as RuntimeEvent; - const proof = await this.readImmutableSteeringMessageProof(event.sessionId, messageId); - if (!proof) return false; - if (!isDeepStrictEqual(proof.event, canonicalEvent)) { - throw new Error(`Immutable steering message identity conflict: ${messageId}`); - } - return true; + if (!Number.isSafeInteger(input.admittedAt) || input.admittedAt < 0) { + throw new Error('Invalid root turn admission timestamp'); } + const { normalizedInput, sourceMessages } = normalizeRootTurnAdmissionPayload( + input.normalizedInput, + input.sourceMessages, + ); + const turnOrchestration = normalizeTurnOrchestration(input.turnOrchestration); + const admission: RootTurnAdmission = { + schemaVersion: ROOT_TURN_ADMISSION_SCHEMA_VERSION, + sessionId: input.sessionId, + turnId: input.turnId, + runId: input.proposedRunId, + userMessageId: input.proposedUserMessageId, + execution: normalizeRootExecutionDescriptor(input.execution), + previousRootTurnId: input.previousRootTurnId, + normalizedInput, + ...(turnOrchestration ? { turnOrchestration } : {}), + sourceMessages, + admittedAt: input.admittedAt, + }; + assertRootTurnAdmissionContract(admission); + assertRootTurnAdmissionRecordSize(admission); + return deepFreezeRootTurnAdmission(admission); +} - private async readRuntimePartials( - sessionId: string, - runId: string, - ): Promise { - let entries; - try { - entries = await readdir(this.runtimePartialsDir(sessionId, runId), { - withFileTypes: true, - }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const partials: RuntimePartialSnapshot[] = []; - for (const entry of entries) { - if (!entry.isFile() || !entry.name.endsWith('.partial')) continue; - const key = entry.name.slice(0, -'.partial'.length); - try { - const stored = await readFile(this.runtimePartialPath(sessionId, runId, key), 'utf8'); - const headerEnd = stored.indexOf('\n'); - if (headerEnd < 0) continue; - const snapshot = JSON.parse(stored.slice(0, headerEnd)) as RuntimePartialSnapshot; - if (snapshot.version !== 1 || !snapshot.event?.partial) continue; - const event = snapshot.event; - if (event.content?.kind === 'text' || event.content?.kind === 'thinking') { - event.content = { - ...event.content, - text: stored.slice(headerEnd + 1), - }; - } - partials.push({ ...snapshot, event }); - } catch { - // A replaceable partial snapshot must never make the immutable ledger unreadable. - } - } - return partials; - } +const MUTABLE_AGENT_RUN_HEADER_FIELDS = new Set([ + 'status', + 'updatedAt', + 'completedAt', + 'failureClass', + 'failureMessage', + 'abortSource', + 'traceWriteError', +]); - private withQueue( - sessionId: string, - runId: string, - operation: () => Promise, - ): Promise { - assertSafeId(sessionId, 'Invalid session id'); - assertSafeId(runId, 'Invalid run id'); - return chainWrite(this.writeQueues, `${sessionId}:${runId}`, operation); +function assertMutableRunHeaderPatch(patch: Partial): void { + const immutable = Object.keys(patch).filter( + (key) => !MUTABLE_AGENT_RUN_HEADER_FIELDS.has(key as keyof AgentRunHeader), + ); + if (immutable.length > 0) { + throw new Error(`AgentRun admission identity is immutable: ${immutable.sort().join(', ')}`); } +} - private withImmutableSteeringQueue( - sessionId: string, - operation: () => Promise, - ): Promise { - assertSafeId(sessionId, 'Invalid session id'); - return chainWrite(this.immutableSteeringWriteQueues, sessionId, operation); - } +function shouldPreserveCheckpointProjectionDuringAppend( + current: AgentRunEvent | null | undefined, + candidate: AgentRunEvent, +): boolean { + if (!current) return false; + const currentSourceBound = historyCompactProjectionIsSourceBound(current); + const candidateSourceBound = historyCompactProjectionIsSourceBound(candidate); + if (currentSourceBound !== candidateSourceBound) return currentSourceBound; + const currentCoverage = historyCompactProjectionCoverage(current); + const candidateCoverage = historyCompactProjectionCoverage(candidate); + return ( + currentCoverage !== undefined && + (candidateCoverage === undefined || currentCoverage > candidateCoverage) + ); } -function mergeRuntimePartialSnapshots( - immutableEvents: readonly RuntimeEvent[], - snapshots: readonly RuntimePartialSnapshot[], -): RuntimeEvent[] { - const leading: RuntimePartialSnapshot[] = []; - const afterEvent = new Map(); - for (const snapshot of snapshots) { - if (!snapshot.afterEventId) { - leading.push(snapshot); - continue; - } - const grouped = afterEvent.get(snapshot.afterEventId) ?? []; - grouped.push(snapshot); - afterEvent.set(snapshot.afterEventId, grouped); - } - const order = (a: RuntimePartialSnapshot, b: RuntimePartialSnapshot) => - a.event.ts - b.event.ts || a.event.id.localeCompare(b.event.id); - const merged = leading.sort(order).map(({ event }) => event); - for (const event of immutableEvents) { - merged.push(event); - const anchored = afterEvent.get(event.id); - if (!anchored) continue; - merged.push(...anchored.sort(order).map((snapshot) => snapshot.event)); - afterEvent.delete(event.id); - } - for (const orphaned of afterEvent.values()) { - merged.push(...orphaned.sort(order).map((snapshot) => snapshot.event)); - } - return merged; +function shouldPreserveProjectionDuringRepair( + current: AgentRunEvent | null | undefined, + candidate: AgentRunEvent | null, + type: AgentRunEventType, +): boolean { + if (!current) return false; + if (type !== 'history_compact_checkpoint_recorded') return true; + const currentSourceBound = historyCompactProjectionIsSourceBound(current); + const candidateSourceBound = candidate ? historyCompactProjectionIsSourceBound(candidate) : false; + if (currentSourceBound !== candidateSourceBound) return currentSourceBound; + const currentCoverage = historyCompactProjectionCoverage(current); + const candidateCoverage = candidate && historyCompactProjectionCoverage(candidate); + return ( + currentCoverage !== undefined && + (candidateCoverage === null || + candidateCoverage === undefined || + currentCoverage >= candidateCoverage) + ); } -function partialRuntimeStream(event: RuntimeEvent): - | { - key: string; - snapshot: RuntimeEvent; - text: string; - } - | undefined { - if (!event.partial || event.status !== undefined || event.actions) return undefined; - const content = event.content; - let identity: string | undefined; - let text = ''; - if ( - content?.kind === 'text' && - content.attachments === undefined && - event.refs?.providerEventId && - hasOnlyKeys(event.refs, ['providerEventId']) - ) { - identity = `${content.kind}:provider:${event.refs.providerEventId}`; - text = content.text; - } else if ( - content?.kind === 'thinking' && - content.signature === undefined && - event.refs?.providerEventId && - hasOnlyKeys(event.refs, ['providerEventId']) - ) { - identity = `${content.kind}:provider:${event.refs.providerEventId}`; - text = content.text; - } else if (!content && event.refs?.toolCallId && hasOnlyKeys(event.refs, ['toolCallId'])) { - identity = `tool:call:${event.refs.toolCallId}`; - } - if (!identity) return undefined; - const key = runtimePartialStreamKey(identity, event); - const snapshot = - content?.kind === 'text' || content?.kind === 'thinking' - ? { ...event, content: { ...content, text: '' } } - : event; - return { key, snapshot, text }; +function historyCompactProjectionIsSourceBound(event: AgentRunEvent): boolean { + const checkpoint = event.data?.checkpoint; + if (!checkpoint || typeof checkpoint !== 'object') return false; + const source = (checkpoint as { source?: unknown }).source; + if (!source || typeof source !== 'object') return false; + return (source as { kind?: unknown }).kind === 'runtime_event_projection'; } -function completedPartialRuntimeStreamKey(event: RuntimeEvent): string | undefined { - if (event.partial) return undefined; - const content = event.content; - let identity: string | undefined; - if ((content?.kind === 'text' || content?.kind === 'thinking') && event.refs?.providerEventId) { - identity = `${content.kind}:provider:${event.refs.providerEventId}`; - } else if (content?.kind === 'function_response' && event.refs?.toolCallId) { - identity = `tool:call:${event.refs.toolCallId}`; +function assertNoReservedToolLedgerFact(event: RuntimeEvent): void { + assertNoReservedWorkspaceAuthorityAppend(event); + if (event.actions?.continuationStart !== undefined) { + throw new Error('Continuation start facts require SQLite continuation authority'); } - return identity ? runtimePartialStreamKey(identity, event) : undefined; + const validation = validateGenericToolLedgerAppend(event); + if (validation.ok) return; + if (validation.code === 'reserved_recovery_fact') { + throw new Error('Tool recovery facts require the atomic recovery bundle writer'); + } + if (validation.code === 'reserved_tool_boundary_fact') { + throw new Error('Durable tool facts require the atomic tool boundary writer'); + } + throw new Error(`RuntimeEvent ${event.id} violates its semantic lane`); } -function runtimePartialStreamKey(identity: string, event: RuntimeEvent): string { - return createHash('sha256') - .update( - JSON.stringify([ - identity, - event.sessionId, - event.invocationId, - event.runId, - event.turnId, - event.branch ?? null, - event.role, - event.author, - ]), - ) - .digest('hex'); +function canonicalizeRuntimeEventForStorage(event: RuntimeEvent): RuntimeEvent { + return encodeCanonicalRuntimeEvent(event).event; } -function hasOnlyKeys(value: object, allowed: readonly string[]): boolean { - return Object.keys(value).every((key) => allowed.includes(key)); +function isToolLedgerBearingEvent(event: RuntimeEvent): boolean { + return ( + event.content?.kind === 'function_call' || + event.content?.kind === 'function_response' || + event.actions?.toolDispatch !== undefined || + event.actions?.toolRecovery !== undefined + ); } -async function readRuntimeEventJsonl( - path: string, - expected: AgentRunHeader, -): Promise { - let text: string; - try { - text = await readFile(path, 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const rawLines = text.split('\n'); - const endsWithNewline = text.endsWith('\n'); - const lines = rawLines - .map((line, index) => ({ line, lineNumber: index + 1 })) - .filter((entry) => entry.line.trim().length > 0); - const lastLineNumber = lines.at(-1)?.lineNumber; - const events: RuntimeEvent[] = []; - for (const entry of lines) { - let parsed: unknown; - try { - parsed = JSON.parse(entry.line); - } catch (error) { - if ( - !endsWithNewline && - entry.lineNumber === lastLineNumber && - classifyJsonRecord(entry.line) === 'incomplete-prefix' - ) - continue; - const message = error instanceof Error ? error.message : 'Invalid JSON'; - throw new Error( - `Invalid RuntimeEvent JSONL line ${entry.lineNumber} for run ${expected.runId}: ${message}`, - ); - } - try { - events.push(decodeRuntimeEvent(parsed, expected)); - } catch (error) { - const message = error instanceof Error ? error.message : 'Invalid RuntimeEvent'; - throw new Error( - `Invalid RuntimeEvent JSONL line ${entry.lineNumber} for run ${expected.runId}: ${message}`, - ); - } - } - return events; +function historyCompactProjectionCoverage(event: AgentRunEvent): number | undefined { + const checkpoint = event.data?.checkpoint; + if (!checkpoint || typeof checkpoint !== 'object') return undefined; + const coverage = (checkpoint as { coverage?: unknown }).coverage; + if (!coverage || typeof coverage !== 'object') return undefined; + const eventCount = (coverage as { eventCount?: unknown }).eventCount; + return typeof eventCount === 'number' && Number.isSafeInteger(eventCount) && eventCount >= 0 + ? eventCount + : undefined; } function isProjectedAgentRunEvent( @@ -2695,125 +901,6 @@ function isProjectedAgentRunEvent( ); } -interface AtomicWriteOptions { - durable?: boolean; - durabilityRoot?: string; -} - -async function writeAtomic( - path: string, - content: string, - options: AtomicWriteOptions = {}, -): Promise { - try { - await writeAtomicUnchecked(path, content, options); - } catch (error) { - if (!options.durable || error instanceof DurableStoreWriteError) throw error; - throw new DurableStoreWriteError( - `Durable atomic write did not reach stable storage: ${path}`, - error, - ); - } -} - -async function appendFileDurably( - path: string, - content: string, - durabilityRoot: string, -): Promise { - try { - const handle = await open(path, 'a'); - try { - await handle.appendFile(content, 'utf8'); - await handle.sync(); - } finally { - await handle.close(); - } - await syncDirectoryChain(dirname(path), durabilityRoot); - } catch (error) { - if (error instanceof DurableStoreWriteError) throw error; - throw new DurableStoreWriteError(`Durable append did not reach stable storage: ${path}`, error); - } -} - -async function writeAtomicUnchecked( - path: string, - content: string, - options: AtomicWriteOptions, -): Promise { - await mkdir(dirname(path), { recursive: true }); - const tempPath = `${path}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`; - if (options.durable) { - const handle = await open(tempPath, 'wx', 0o600); - try { - await handle.writeFile(content, 'utf8'); - await handle.sync(); - } finally { - await handle.close(); - } - } else { - await writeFile(tempPath, content, 'utf8'); - } - await rename(tempPath, path); - if (options.durable) { - if (!options.durabilityRoot) { - throw new Error('Durable atomic write requires a durability root'); - } - await syncDirectoryChain(dirname(path), options.durabilityRoot); - } -} - -async function writeExclusiveAtomic( - path: string, - content: string, - options: { durable?: boolean }, - durabilityRoot: string, -): Promise { - try { - return await writeExclusiveAtomicUnchecked(path, content, options, durabilityRoot); - } catch (error) { - if (!options.durable || error instanceof DurableStoreWriteError) throw error; - throw new DurableStoreWriteError( - `Durable exclusive write did not reach stable storage: ${path}`, - error, - ); - } -} - -async function writeExclusiveAtomicUnchecked( - path: string, - content: string, - options: { durable?: boolean }, - durabilityRoot: string, -): Promise { - const directory = dirname(path); - await mkdir(directory, { recursive: true }); - const tempPath = `${path}.${process.pid}.${randomUUID()}.tmp`; - const handle = await open(tempPath, 'wx', 0o600); - try { - await handle.writeFile(content, 'utf8'); - if (options.durable) await handle.sync(); - } finally { - await handle.close(); - } - try { - await link(tempPath, path); - await unlink(tempPath); - if (options.durable) await syncDirectoryChain(directory, durabilityRoot); - return true; - } catch (error) { - await unlink(tempPath).catch(() => {}); - if ((error as NodeJS.ErrnoException).code === 'EEXIST') { - if (options.durable) { - await syncFile(path); - await syncDirectoryChain(directory, durabilityRoot); - } - return false; - } - throw error; - } -} - function assertSafeId(value: string, message: string): void { if (!isSafeId(value)) throw new Error(message); } @@ -3352,53 +1439,11 @@ function isPlainRecord(value: unknown): value is Record { return prototype === Object.prototype || prototype === null; } -function turnIdFromAdmissionFile(name: string): string | undefined { - if (!name.endsWith('.json')) return undefined; - const turnId = name.slice(0, -'.json'.length); - return isSafeId(turnId) ? turnId : undefined; -} - -function isRootTurnAdmissionTemp(name: string): boolean { - const marker = '.json.'; - const markerIndex = name.indexOf(marker); - if (markerIndex < 1) return false; - return ( - isSafeId(name.slice(0, markerIndex)) && - EXCLUSIVE_TEMP_SUFFIX_PATTERN.test(name.slice(markerIndex + marker.length)) - ); -} - -function isExclusiveWriteTemp(name: string, targetName: string): boolean { - const prefix = `${targetName}.`; - return name.startsWith(prefix) && EXCLUSIVE_TEMP_SUFFIX_PATTERN.test(name.slice(prefix.length)); -} - function hasExactKeys(record: Record, expected: readonly string[]): boolean { const keys = Object.keys(record); return keys.length === expected.length && expected.every((key) => Object.hasOwn(record, key)); } -async function syncDirectoryIfPresent(path: string): Promise { - try { - await syncDirectory(path); - } catch (error) { - if (!isMissingFile(error)) throw error; - } -} - -async function fileSizeOrZero(path: string): Promise { - try { - return (await stat(path)).size; - } catch (error) { - if (isMissingFile(error)) return 0; - throw error; - } -} - -function isMissingFile(error: unknown): boolean { - return (error as NodeJS.ErrnoException | undefined)?.code === 'ENOENT'; -} - function sanitizeJson(_key: string, value: unknown): unknown { return value === undefined ? undefined : value; } diff --git a/packages/storage/src/artifact-metadata-codec.ts b/packages/storage/src/artifact-metadata-codec.ts index 3621f5fe5f..405f1545d8 100644 --- a/packages/storage/src/artifact-metadata-codec.ts +++ b/packages/storage/src/artifact-metadata-codec.ts @@ -31,23 +31,21 @@ const ARTIFACT_RECORD_KEYS = new Set([ 'status', ]); -export function decodeArtifactMetadata(text: string): ArtifactRecord[] { +export function decodeArtifactRecordJsons(values: readonly unknown[]): ArtifactRecord[] { const records: ArtifactRecord[] = []; const ids = new Set(); - const lines = text.split('\n'); - for (const [index, line] of lines.entries()) { - if (line.length === 0 && index === lines.length - 1) continue; - if (line.trim().length === 0) throw invalidMetadataLine(index + 1); + for (const [index, value] of values.entries()) { try { - const record = decodeArtifactRecord(JSON.parse(line), index + 1); - if (ids.has(record.id)) throw invalidMetadataLine(index + 1); + if (typeof value !== 'string') throw invalidMetadataRecord(index + 1); + const record = decodeArtifactRecord(JSON.parse(value), index + 1); + if (ids.has(record.id)) throw invalidMetadataRecord(index + 1); ids.add(record.id); records.push(record); } catch (error) { - if (error instanceof Error && error.message.startsWith('Invalid artifact metadata line')) { + if (error instanceof Error && error.message.startsWith('Invalid artifact metadata record')) { throw error; } - throw invalidMetadataLine(index + 1, error); + throw invalidMetadataRecord(index + 1, error); } } return records; @@ -74,10 +72,10 @@ export function validateCanonicalArtifactTargetName(targetName: string): void { } } -function decodeArtifactRecord(value: unknown, line: number): ArtifactRecord { - if (!isRecord(value)) throw invalidMetadataLine(line); +function decodeArtifactRecord(value: unknown, index: number): ArtifactRecord { + if (!isRecord(value)) throw invalidMetadataRecord(index); if (Object.keys(value).some((key) => !ARTIFACT_RECORD_KEYS.has(key))) { - throw invalidMetadataLine(line); + throw invalidMetadataRecord(index); } if ( !isCanonicalArtifactEntityId(value.id) || @@ -102,13 +100,13 @@ function decodeArtifactRecord(value: unknown, line: number): ArtifactRecord { (typeof value.source !== 'string' || !ARTIFACT_SOURCE_SET.has(value.source as ArtifactSource))) ) { - throw invalidMetadataLine(line); + throw invalidMetadataRecord(index); } validateRelativeArtifactPath(value.relativePath); validateCanonicalArtifactTargetName(basename(value.relativePath)); - if (!isCompatibleArtifactName(value.name)) throw invalidMetadataLine(line); + if (!isCompatibleArtifactName(value.name)) throw invalidMetadataRecord(index); if (value.relativePath !== `${value.sessionId}/${value.id}-${value.name}`) { - throw invalidMetadataLine(line); + throw invalidMetadataRecord(index); } return value as unknown as ArtifactRecord; } @@ -121,8 +119,11 @@ function isCompatibleArtifactName(name: string): boolean { return true; } -function invalidMetadataLine(line: number, cause?: unknown): Error { - return new Error(`Invalid artifact metadata line ${line}`, cause === undefined ? {} : { cause }); +function invalidMetadataRecord(index: number, cause?: unknown): Error { + return new Error( + `Invalid artifact metadata record ${index}`, + cause === undefined ? {} : { cause }, + ); } function isRecord(value: unknown): value is Record { diff --git a/packages/storage/src/artifact-storage-layout.ts b/packages/storage/src/artifact-storage-layout.ts index 03ab38d354..43100b167f 100644 --- a/packages/storage/src/artifact-storage-layout.ts +++ b/packages/storage/src/artifact-storage-layout.ts @@ -7,20 +7,10 @@ export const ARTIFACT_PURGE_INTENT_TEMP_PATTERN = new RegExp( `^\\.artifact-purge-intent\\.json\\.[0-9]+\\.${UUID_V4_FRAGMENT}\\.tmp$`, ); -export const ARTIFACT_METADATA_TEMP_PATTERN = new RegExp( - `^metadata\\.jsonl\\.[0-9]+\\.${UUID_V4_FRAGMENT}\\.tmp$`, -); - -export const ARTIFACT_LEGACY_METADATA_TEMP_PATTERN = /^metadata\.jsonl\.[0-9]+\.[0-9]+\.tmp$/; - export const ARTIFACT_PUBLICATION_STAGING_PATTERN = new RegExp( `^\\.artifact-publish\\.([a-f0-9]{64})\\.${UUID_V4_FRAGMENT}\\.tmp$`, ); -export function isCanonicalArtifactRecoveryTempName(name: string): boolean { - return ( - ARTIFACT_PURGE_INTENT_TEMP_PATTERN.test(name) || - ARTIFACT_METADATA_TEMP_PATTERN.test(name) || - ARTIFACT_LEGACY_METADATA_TEMP_PATTERN.test(name) - ); +export function isArtifactPurgeRecoveryTempName(name: string): boolean { + return ARTIFACT_PURGE_INTENT_TEMP_PATTERN.test(name); } diff --git a/packages/storage/src/artifact-store.ts b/packages/storage/src/artifact-store.ts index edeb817ba1..60aa5949f4 100644 --- a/packages/storage/src/artifact-store.ts +++ b/packages/storage/src/artifact-store.ts @@ -37,7 +37,7 @@ import { publishMarkerFile, readBoundedMarkerFile } from './marker-file.js'; import { ARTIFACT_PUBLICATION_STAGING_PATTERN, ARTIFACT_PURGE_INTENT_FILE, - isCanonicalArtifactRecoveryTempName, + isArtifactPurgeRecoveryTempName, } from './artifact-storage-layout.js'; import { isSafeRelativeArtifactPath, @@ -51,10 +51,7 @@ import { import type { ArtifactWriterLockAuthority } from './root-authority.js'; import { syncDirectory, syncDirectoryChain, syncFile } from './stable-storage.js'; import type { ArtifactMetadataRepository } from './artifact-metadata-repository.js'; -import { - createSqliteArtifactMetadataRepository, - type CreateSqliteArtifactMetadataOptions, -} from './sqlite-artifact-metadata.js'; +import { createSqliteArtifactMetadataRepository } from './sqlite-artifact-metadata.js'; export { isSafeRelativeArtifactPath } from './artifact-metadata-codec.js'; @@ -205,14 +202,11 @@ export interface ArtifactStoreWriteAuthority { close(): void; } -export function createSqliteArtifactStore( - workspaceRoot: string, - options: CreateSqliteArtifactMetadataOptions = {}, -): ArtifactStore { +export function createSqliteArtifactStore(workspaceRoot: string): ArtifactStore { return new SqliteArtifactStore( workspaceRoot, 'self_managed', - createSqliteArtifactMetadataRepository(workspaceRoot, options), + createSqliteArtifactMetadataRepository(workspaceRoot), undefined, undefined, ); @@ -223,13 +217,12 @@ export function createSqliteArtifactStoreWriteAuthority( options: { assertAuthority?: () => Promise; leaseBoundWriterLockAuthority?: ArtifactWriterLockAuthority; - cutover?: CreateSqliteArtifactMetadataOptions; } = {}, ): ArtifactStoreWriteAuthority { const store = new SqliteArtifactStore( workspaceRoot, 'authority', - createSqliteArtifactMetadataRepository(workspaceRoot, options.cutover), + createSqliteArtifactMetadataRepository(workspaceRoot), options.assertAuthority, options.leaseBoundWriterLockAuthority, ); @@ -511,12 +504,11 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { canonical: { id: string; name: string; relativePath: string }, ): Promise { const expectedBytes = Buffer.from(input.content); - const compatibleNames = compatibleArtifactNames(input.name, canonical.name); if ( existing.id !== canonical.id || existing.sessionId !== input.sessionId || existing.turnId !== input.turnId || - !compatibleNames.has(existing.name) || + existing.name !== canonical.name || existing.kind !== input.kind || existing.relativePath !== `${input.sessionId}/${canonical.id}-${existing.name}` || existing.sizeBytes !== expectedBytes.byteLength || @@ -950,7 +942,7 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { if (isNotFound(error)) return false; throw error; } - if (sessionEntries.some((entry) => isCanonicalArtifactRecoveryTempName(entry.name))) { + if (sessionEntries.some((entry) => isArtifactPurgeRecoveryTempName(entry.name))) { return true; } for (const sessionEntry of sessionEntries) { @@ -981,7 +973,7 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { input: CreateArtifactInput, identity: { id: string; canonicalName: string }, ): Promise { - const names = [...compatibleArtifactNames(input.name, identity.canonicalName)]; + const names = [identity.canonicalName]; const candidates = names .map((name) => ({ name, @@ -1100,7 +1092,7 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { const referencedPaths = new Set( this.records.map((record) => filesystemPathKey(record.relativePath)), ); - for (const name of compatibleArtifactNames(input.name, identity.canonicalName)) { + for (const name of [identity.canonicalName]) { const relativePath = `${input.sessionId}/${identity.id}-${name}`; if (referencedPaths.has(filesystemPathKey(relativePath))) continue; const path = join(this.artifactRoot, relativePath); @@ -1129,9 +1121,7 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { ): Promise { const sessionDirectory = join(this.artifactRoot, input.sessionId); const targetHashes = new Set( - [...compatibleArtifactNames(input.name, identity.canonicalName)].map((name) => - artifactTargetHash(`${identity.id}-${name}`), - ), + [identity.canonicalName].map((name) => artifactTargetHash(`${identity.id}-${name}`)), ); let entries: Dirent[]; try { @@ -1156,7 +1146,7 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { input: Pick, identity: { id: string; canonicalName: string }, ): Promise { - const names = compatibleArtifactNames(input.name, identity.canonicalName); + const names = new Set([identity.canonicalName]); for (const name of names) { const path = join(this.artifactRoot, input.sessionId, `${identity.id}-${name}`); try { @@ -1173,9 +1163,7 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { identity: { id: string; canonicalName: string }, ): Promise { const targetHashes = new Set( - [...compatibleArtifactNames(input.name, identity.canonicalName)].map((name) => - artifactTargetHash(`${identity.id}-${name}`), - ), + [identity.canonicalName].map((name) => artifactTargetHash(`${identity.id}-${name}`)), ); let entries: Dirent[]; try { @@ -1288,7 +1276,7 @@ class SqliteArtifactStore implements ArtifactAuthorityStore { throw error; } for (const entry of entries) { - if (!isCanonicalArtifactRecoveryTempName(entry.name)) continue; + if (!isArtifactPurgeRecoveryTempName(entry.name)) continue; if (!entry.isFile()) { throw new Error(`Artifact recovery temp is not a regular file: ${entry.name}`); } @@ -1589,21 +1577,6 @@ export function sanitizeArtifactName(name: string): string { return truncated || 'artifact'; } -function sanitizeLegacyArtifactName(name: string): string { - const cleaned = name - .trim() - .replace(/[\\/:*?"<>|\0]/g, '-') - .replace(/\s+/g, ' ') - .replace(/^\.+/, '') - .replace(/^-+/, '') - .trim(); - return (cleaned || 'artifact').slice(0, 120); -} - -function compatibleArtifactNames(inputName: string, canonicalName: string): ReadonlySet { - return new Set([canonicalName, sanitizeLegacyArtifactName(inputName)]); -} - function filesystemPathKey(path: string): string { return Buffer.from(path, 'utf8').toString('utf8'); } diff --git a/packages/storage/src/automation-authority.ts b/packages/storage/src/automation-authority.ts index e80416d845..e420814300 100644 --- a/packages/storage/src/automation-authority.ts +++ b/packages/storage/src/automation-authority.ts @@ -1,6 +1,3 @@ -import { createHash } from 'node:crypto'; -import { readFile } from 'node:fs/promises'; -import { join } from 'node:path'; import { isCollaborationMode } from '@maka/core/collaboration'; import { isOrchestrationMode } from '@maka/core/orchestration'; import { isThinkingLevel } from '@maka/core/model-thinking'; @@ -10,7 +7,6 @@ import { AUTOMATION_NAME_LIMIT, AUTOMATION_PROMPT_LIMIT, isAutomationTextWithinLimit, - truncateAutomationText, type AutomationAuthoritySnapshot, type AutomationDefinition, type AutomationExecutionTemplate, @@ -18,10 +14,8 @@ import { type AutomationSchedule, } from '@maka/core/automation'; import { - completeOperationalStoreCutover, acquireOperationalStateDatabase, type OperationalStateDatabaseLease, - type OperationalStoreCutoverFailpoint, } from './operational-state-store.js'; import { assertStorageRootLease, @@ -30,7 +24,6 @@ import { type StorageRootLease, } from './root-authority.js'; -export const LEGACY_AUTOMATION_FILE = 'automations.json'; const MAX_AUTOMATIONS = 10_000; const MAX_PENDING_FIRES = 10_000; const ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; @@ -98,8 +91,17 @@ export interface InteractiveAutomationAuthorityWriter { close(): void; } -export interface OpenAutomationAuthorityOptions { - readonly failpoint?: (point: OperationalStoreCutoverFailpoint) => void; +export interface AutomationAuthorityRepository { + ready(): Promise; + read(): AutomationAuthoritySnapshot; + commit(input: CommitAutomationAuthorityInput): CommitAutomationAuthorityResult; + close(): void; +} + +export function createSqliteAutomationAuthority( + workspaceRoot: string, +): AutomationAuthorityRepository { + return new SqliteAutomationAuthority(workspaceRoot); } export function authenticateInteractiveAutomationAuthorityWriter( @@ -116,7 +118,6 @@ export function authenticateInteractiveAutomationAuthorityWriter( export async function openInteractiveAutomationAuthorityForWrite( lease: StorageRootLease<'interactive', 'write'>, - options: OpenAutomationAuthorityOptions = {}, ): Promise { await assertStorageRootLease(lease, 'interactive', 'write'); const existing = writerByLease.get(lease); @@ -128,7 +129,7 @@ export async function openInteractiveAutomationAuthorityForWrite( let store: SqliteAutomationAuthority | undefined; try { store = await runWithStorageRootLease(lease, 'interactive', 'write', async (root) => { - const opened = new SqliteAutomationAuthority(root, options); + const opened = new SqliteAutomationAuthority(root); await opened.ready(); return opened; }); @@ -188,17 +189,15 @@ function createWriterFacade( return Object.freeze(writer); } -class SqliteAutomationAuthority { +class SqliteAutomationAuthority implements AutomationAuthorityRepository { readonly #lease: OperationalStateDatabaseLease; - readonly #ready: Promise; - constructor(root: string, options: OpenAutomationAuthorityOptions) { + constructor(root: string) { this.#lease = acquireOperationalStateDatabase(root); - this.#ready = importLegacyAutomations(root, this.#lease, options); } ready(): Promise { - return this.#ready; + return Promise.resolve(); } read(): AutomationAuthoritySnapshot { @@ -313,101 +312,6 @@ class SqliteAutomationAuthority { } } -async function importLegacyAutomations( - root: string, - lease: OperationalStateDatabaseLease, - options: OpenAutomationAuthorityOptions, -): Promise { - const sourcePath = join(root, LEGACY_AUTOMATION_FILE); - const source = await readLegacyAutomationFile(sourcePath); - completeOperationalStoreCutover(lease, { - storeName: 'automations', - sourcePath, - sourceFingerprint: source.fingerprint, - ...(options.failpoint ? { failpoint: options.failpoint } : {}), - importAndValidate: (database) => { - const insert = database.prepare(` - INSERT OR IGNORE INTO automation_definitions( - automation_id, session_id, created_at, status, durable, record_json - ) VALUES (?, ?, ?, ?, ?, ?) - `); - for (const automation of source.automations) { - const encoded = JSON.stringify(automation); - const result = insert.run( - automation.id, - automation.sessionId, - automation.createdAt, - automation.status, - automation.durable === true ? 1 : 0, - encoded, - ); - if (result.changes === 0) { - const row = database - .prepare('SELECT record_json FROM automation_definitions WHERE automation_id = ?') - .get(automation.id) as { record_json?: unknown } | undefined; - if (!row || row.record_json !== encoded) { - throw new Error(`Automation cutover conflict: ${automation.id}`); - } - } - } - if (source.automations.length > 0) { - database - .prepare(` - UPDATE automation_authority_state - SET revision = CASE WHEN revision = 0 THEN 1 ELSE revision END - WHERE singleton = 1 - `) - .run(); - } - return { automations: source.automations.length }; - }, - }); -} - -async function readLegacyAutomationFile( - path: string, -): Promise<{ fingerprint: string; automations: readonly AutomationDefinition[] }> { - let contents: string; - try { - contents = await readFile(path, 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return { - fingerprint: `sha256:${createHash('sha256').update('absent').digest('hex')}`, - automations: [], - }; - } - throw error; - } - return { - fingerprint: `sha256:${createHash('sha256').update(contents).digest('hex')}`, - automations: decodeLegacyAutomationFile(contents), - }; -} - -export function decodeLegacyAutomationFile(contents: string): readonly AutomationDefinition[] { - let parsed: unknown; - try { - parsed = JSON.parse(contents); - } catch (error) { - throw new Error('Legacy Automation store is not valid JSON', { cause: error }); - } - if (!isRecord(parsed) || parsed.version !== 1 || !Array.isArray(parsed.automations)) { - throw new Error('Legacy Automation store has an unsupported shape or version'); - } - if (parsed.automations.length > MAX_AUTOMATIONS) { - throw new Error('Legacy Automation store exceeds the definition limit'); - } - const automations = parsed.automations.map((automation) => - normalizeAutomationDefinition(automation, { legacy: true }), - ); - assertUnique( - automations.map((automation) => automation.id), - 'Automation id', - ); - return automations; -} - function normalizeCommitInput( input: CommitAutomationAuthorityInput, ): CommitAutomationAuthorityInput { @@ -466,10 +370,7 @@ function assertSnapshotRelationships( } } -function normalizeAutomationDefinition( - value: unknown, - options: { readonly legacy?: boolean } = {}, -): AutomationDefinition { +function normalizeAutomationDefinition(value: unknown): AutomationDefinition { if (!isRecord(value) || !hasOnlyKeys(value, DEFINITION_KEYS)) { throw new Error('Invalid Automation definition'); } @@ -490,7 +391,7 @@ function normalizeAutomationDefinition( ) { throw new Error('Invalid Automation state'); } - const lastError = normalizeLastError(value.lastError, options.legacy === true); + const lastError = normalizeLastError(value.lastError); if ( !isNonnegativeInteger(value.createdAt) || !isNonnegativeInteger(value.updatedAt) || @@ -730,11 +631,9 @@ function isBoundedString(value: unknown, max: number): value is string { return typeof value === 'string' && value.length > 0 && Buffer.byteLength(value, 'utf8') <= max; } -function normalizeLastError(value: unknown, legacy: boolean): string | null | undefined { +function normalizeLastError(value: unknown): string | null | undefined { if (value === null) return null; - if (legacy && value === '') return null; if (typeof value !== 'string' || value.length === 0) return undefined; - if (legacy) return truncateAutomationText(value, AUTOMATION_LAST_ERROR_LIMIT); return isAutomationTextWithinLimit(value, AUTOMATION_LAST_ERROR_LIMIT) ? value : undefined; } diff --git a/packages/storage/src/automation-store.ts b/packages/storage/src/automation-store.ts index bcbafbcb07..8bdb5bad8c 100644 --- a/packages/storage/src/automation-store.ts +++ b/packages/storage/src/automation-store.ts @@ -1,108 +1,72 @@ -import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; -import { chainWrite } from './write-queue.js'; +import type { AutomationDefinition } from '@maka/core/automation'; +import { + createSqliteAutomationAuthority, + type AutomationAuthorityRepository, +} from './automation-authority.js'; -/** Minimal constraint for records stored by the automation store. */ -export interface AutomationRecord { - id: string; -} - -export interface AutomationStore { - loadAll(): Promise; - save(automation: T): Promise; +export interface AutomationStore { + loadAll(): Promise; + save(automation: AutomationDefinition): Promise; remove(id: string): Promise; - sync(automations: T[]): Promise; + sync(automations: AutomationDefinition[]): Promise; } -interface AutomationFile { - version: 1; - automations: AutomationRecord[]; +export function createAutomationStore(workspaceRoot: string): AutomationStore { + return new SqliteAutomationStore(workspaceRoot); } -export function createAutomationStore( - workspaceRoot: string, -): AutomationStore { - return new FileAutomationStore(workspaceRoot); -} - -class FileAutomationStore implements AutomationStore { - private readonly filePath: string; - private readonly writeQueue = new Map>(); - private static readonly QUEUE_KEY = 'automations'; - - constructor(workspaceRoot: string) { - this.filePath = join(workspaceRoot, 'automations.json'); - } +class SqliteAutomationStore implements AutomationStore { + constructor(private readonly workspaceRoot: string) {} - async loadAll(): Promise { - let text: string; - try { - text = await readFile(this.filePath, 'utf8'); - } catch (error) { - // Absent file → legitimately empty store, safe to start fresh. - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - // A present-but-unreadable file (EMFILE/EBUSY/EACCES/…) must NOT be masked - // as empty: a caller that then full-overwrites the store would erase data - // it never read. Fail loud so the host disables persistence instead. - throw new Error( - `[automation-store] failed to read ${this.filePath}: ${(error as Error).message}`, - ); - } - let parsed: unknown; - try { - parsed = JSON.parse(text); - } catch (error) { - throw new Error( - `[automation-store] ${this.filePath} is not valid JSON: ${(error as Error).message}`, - ); - } - if (!isAutomationFile(parsed)) { - // Present but unrecognized (wrong version/shape) — same danger as a read - // error: treating it as empty and overwriting would drop real data. - throw new Error(`[automation-store] ${this.filePath} has an unrecognized shape or version`); - } - return parsed.automations as T[]; + async loadAll(): Promise { + return this.withAuthority((authority) => + authority.read().automations.map((automation) => ({ ...automation })), + ); } - async save(automation: T): Promise { - await chainWrite(this.writeQueue, FileAutomationStore.QUEUE_KEY, async () => { - const current = await this.loadAll(); - const index = current.findIndex((a) => a.id === automation.id); - if (index >= 0) { - current[index] = automation; - } else { - current.push(automation); - } - await this.writeFile(current); + async save(automation: AutomationDefinition): Promise { + await this.mutate((current) => { + const next = [...current]; + const index = next.findIndex((candidate) => candidate.id === automation.id); + if (index === -1) next.push(automation); + else next[index] = automation; + return next; }); } async remove(id: string): Promise { - await chainWrite(this.writeQueue, FileAutomationStore.QUEUE_KEY, async () => { - const current = await this.loadAll(); - const filtered = current.filter((a) => a.id !== id); - if (filtered.length === current.length) return; - await this.writeFile(filtered); - }); + await this.mutate((current) => current.filter((automation) => automation.id !== id)); } - async sync(automations: T[]): Promise { - await chainWrite(this.writeQueue, FileAutomationStore.QUEUE_KEY, async () => { - await this.writeFile(automations); - }); + async sync(automations: AutomationDefinition[]): Promise { + await this.mutate(() => automations); } - private async writeFile(automations: T[]): Promise { - const data: AutomationFile = { version: 1, automations }; - await mkdir(dirname(this.filePath), { recursive: true }); - const tempPath = `${this.filePath}.${process.pid}.${Date.now()}.tmp`; - await writeFile(tempPath, JSON.stringify(data, null, 2) + '\n', 'utf8'); - await rename(tempPath, this.filePath); + private async mutate( + update: (current: readonly AutomationDefinition[]) => readonly AutomationDefinition[], + ): Promise { + this.withAuthority((authority) => { + for (let attempt = 0; attempt < 4; attempt += 1) { + const snapshot = authority.read(); + const automations = [...update(snapshot.automations)]; + const retained = new Set(automations.map((automation) => automation.id)); + const result = authority.commit({ + expectedRevision: snapshot.revision, + automations, + pendingFires: snapshot.pendingFires.filter((fire) => retained.has(fire.automationId)), + }); + if (result.kind === 'committed') return; + } + throw new Error('Automation authority changed repeatedly while committing'); + }); } -} -function isAutomationFile(value: unknown): value is AutomationFile { - if (typeof value !== 'object' || value === null) return false; - const obj = value as Record; - return obj.version === 1 && Array.isArray(obj.automations); + private withAuthority(operation: (authority: AutomationAuthorityRepository) => T): T { + const authority = createSqliteAutomationAuthority(this.workspaceRoot); + try { + return operation(authority); + } finally { + authority.close(); + } + } } diff --git a/packages/storage/src/credential-store.ts b/packages/storage/src/credential-store.ts index ceac14390d..c6d8bbdf6d 100644 --- a/packages/storage/src/credential-store.ts +++ b/packages/storage/src/credential-store.ts @@ -74,9 +74,7 @@ export interface CredentialStore { /** * Optional compare-and-set write. Persist `value` for `(slug, kind)` only * while the stored entry still equals `expected` — the basis the caller read - * before deciding to write. `expected: null` asserts the entry is absent, for - * a write-if-not-present (e.g. the one-shot legacy import deciding "store - * already has a token" and writing inside one serialized step). + * before deciding to write. `expected: null` asserts the entry is absent. * * The basis check and the write run together under the same cross-process * lock as `setSecret`, so no concurrent writer can slip in between them; the @@ -100,120 +98,6 @@ export function createFileCredentialStore(workspaceRoot: string): CredentialStor return new FileCredentialStore(join(workspaceRoot, 'credentials.json')); } -/** - * Injected decryptor for the one-time legacy migration. The legacy - * credentials.json stored each secret as an opaque, externally-encrypted string - * (Electron `safeStorage`, base64-wrapped); only the desktop main process can - * decrypt it. The migration lives here so it shares the live store's lock and - * atomic writer, but the crypto stays the caller's: desktop passes a decryptor - * backed by `safeStorage`, while a headless caller has none and never runs it. - */ -export interface LegacyCredentialDecryptor { - /** Whether decryption is currently possible. If false, the migration aborts - * and leaves the encrypted file untouched (never destroy unrecoverable - * secrets). */ - isAvailable(): boolean; - /** Decrypt one legacy stored value to plaintext. */ - decrypt(storedValue: string): string; -} - -/** - * One-time migration of a legacy (pre-version, externally-encrypted) - * credentials.json to the shared v1 plaintext-0600 shape, in place. - * - * Idempotent: a no-op when the file is missing or already v1. Missing and - * current-v1 files return before locking so stale legacy locks do not block - * startup. The legacy migration path still runs under the SAME cross-process - * lock as the live store and re-reads inside it, so a racing process that - * already migrated (and a live writer that added a newer secret) is never - * clobbered by a stale snapshot. Fails closed: an unexpected version, a - * malformed `values`, or a decryptor that is unavailable while there are values - * to decrypt throws and leaves the file untouched rather than risk tombstoning - * unrecoverable secrets. - * - * Tombstone, not dual-active: a successful run rewrites every value as - * plaintext, so no decryptable copy survives. - */ -export async function migrateLegacyCredentialFile( - path: string, - decryptor: LegacyCredentialDecryptor, -): Promise { - let snapshot: string; - try { - snapshot = await readFile(path, 'utf8'); - } catch (error) { - if ((error as { code?: string }).code === 'ENOENT') return; // nothing to migrate - throw error; - } - - try { - const parsed = JSON.parse(snapshot) as { version?: number }; - if (parsed.version === CREDENTIAL_SCHEMA_VERSION) return; // already migrated; do not wait on stale legacy locks - } catch { - // Preserve the fail-closed lock path below for malformed files. - } - - await withCredentialFileLock(path, async () => { - let raw: string; - try { - raw = await readFile(path, 'utf8'); - } catch (error) { - if ((error as { code?: string }).code === 'ENOENT') return; // nothing to migrate - throw error; - } - - const parsed = JSON.parse(raw) as { version?: number; values?: Record }; - if (parsed.version === CREDENTIAL_SCHEMA_VERSION) return; // already migrated (possibly by a racing process) - if (parsed.version !== undefined) { - throw new Error( - `Cannot migrate credentials.json: unexpected schema version ${parsed.version}.`, - ); - } - - const legacy = parsed.values; - if (legacy === null || typeof legacy !== 'object' || Array.isArray(legacy)) { - throw new Error( - 'Cannot migrate credentials.json: missing or malformed `values`. Leaving it untouched.', - ); - } - const entries = Object.entries(legacy); - // Every legacy value must be a string we can hand to the decryptor. A - // non-string entry means a corrupt or foreign file, so fail closed and - // leave it untouched rather than feed garbage to decrypt — the same - // per-value guarantee the v1 reader enforces. - for (const [key, storedValue] of entries) { - if (typeof storedValue !== 'string') { - throw new Error( - `Cannot migrate credentials.json: value for "${key}" is not a string. Leaving it untouched.`, - ); - } - } - // Only the actual decryption needs the decryptor. An empty legacy file has - // nothing to decrypt, so it must still migrate to the v1 empty shape even - // when the decryptor is unavailable — otherwise a user who deleted their - // last secret under the old store, on a box where safeStorage is now - // unavailable, would be stuck: the v1 store refuses the unversioned file and - // the migration would refuse to stamp it. - if (entries.length > 0 && !decryptor.isAvailable()) { - throw new Error( - 'Cannot migrate credentials.json: the legacy decryptor is unavailable. Leaving the encrypted file untouched.', - ); - } - - // Decrypt EVERY legacy value to plaintext. Keys are preserved verbatim - // (slugs can contain ':', so we never parse them apart). - const migrated: Record = {}; - for (const [key, storedValue] of entries) { - migrated[key] = decryptor.decrypt(storedValue); - } - - await writeSecretFileAtomic( - path, - JSON.stringify({ version: CREDENTIAL_SCHEMA_VERSION, values: migrated }, null, 2) + '\n', - ); - }); -} - class FileCredentialStore implements CredentialStore { constructor(private readonly path: string) {} @@ -304,16 +188,11 @@ class FileCredentialStore implements CredentialStore { throw error; } const parsed = JSON.parse(raw) as Partial; - // Fail closed on an unknown / pre-migration schema. A legacy file - // (safeStorage-encrypted, no `version`) lands here as `undefined` - // and must be migrated by the desktop importer before use — we do - // not silently start a parallel plaintext store next to it. + // Fail closed on an unknown schema rather than inventing defaults. if (parsed.version !== CREDENTIAL_SCHEMA_VERSION) { throw new Error( `Unsupported credentials.json schema version: ${String(parsed.version)} ` + - `(expected ${CREDENTIAL_SCHEMA_VERSION}). Open the desktop app once to migrate, ` + - `or re-authenticate. If migration keeps failing, a stale lock may be blocking it — ` + - `remove ${this.path}.lock and retry.`, + `(expected ${CREDENTIAL_SCHEMA_VERSION}). Remove the file and re-authenticate.`, ); } // A v1 file must carry a well-formed `values` map. Treat a missing or @@ -353,8 +232,7 @@ async function ensureSecretDir(dir: string): Promise { * Owner-only atomic write for a credentials file: a 0700 dir, an exclusive * 0600 temp ('wx'/O_EXCL so we never follow a pre-planted symlink at a * predictable path), 0600 re-enforced, an atomic rename, and temp cleanup on - * failure. Shared by the live store and the one-time migration so the hardening - * can't drift between the two write paths. + * failure. */ async function writeSecretFileAtomic(path: string, contents: string): Promise { await ensureSecretDir(dirname(path)); @@ -408,13 +286,12 @@ const delay = (ms: number): Promise => new Promise((resolve) => setTimeout * directory behind, and the next writer fails loud until it is removed — an * explicit, one-command recovery, never a silent lost update. A clean exit or a * completed write releases it via the finally. credentials.json is written - * rarely and is local, so this is the right trade for credential data. Used by - * both the live store and the one-time migration so they serialize on one lock. + * rarely and is local, so this is the right trade for credential data. * * `timeoutMs` defaults to LOCK_TIMEOUT_MS; it is a parameter only so a test can * drive the fail-loud path with a small value. Exported for that test — it is * deliberately NOT re-exported from index.ts, so the package's public surface - * stays the typed store + migration and callers can't drive the lock directly. + * stays the typed store and callers can't drive the lock directly. */ export async function withCredentialFileLock( targetPath: string, diff --git a/packages/storage/src/deep-research-store.ts b/packages/storage/src/deep-research-store.ts index b239641beb..17cce19865 100644 --- a/packages/storage/src/deep-research-store.ts +++ b/packages/storage/src/deep-research-store.ts @@ -1,6 +1,5 @@ -import { createHash, randomUUID } from 'node:crypto'; -import { mkdir, readFile, readdir } from 'node:fs/promises'; -import { dirname, join, resolve } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { resolve } from 'node:path'; import type { DatabaseSync } from 'node:sqlite'; import { isDeepResearchEvent, @@ -21,13 +20,10 @@ import { type DeepResearchStore, } from '@maka/core/deep-research-run'; import { assertSafeSessionId } from './session-store.js'; -import { appendJsonl } from './jsonl-append.js'; import { chainWrite } from './write-queue.js'; import { acquireOperationalStateDatabase, - completeOperationalStoreCutover, type OperationalStateDatabaseLease, - type OperationalStoreCutoverFailpoint, } from './operational-state-store.js'; export type { DeepResearchStore } from '@maka/core/deep-research-run'; @@ -37,36 +33,26 @@ export interface CreateDeepResearchStoreOptions { now?: () => number; } -export function createDeepResearchStore( - workspaceRoot: string, - options: CreateDeepResearchStoreOptions = {}, -): DeepResearchStore { - return new FileDeepResearchStore( - workspaceRoot, - options.newId ?? randomUUID, - options.now ?? Date.now, - ); -} - export interface SqliteDeepResearchStore extends DeepResearchStore { ready(): Promise; close(): void; } -export interface CreateSqliteDeepResearchStoreOptions extends CreateDeepResearchStoreOptions { - failpoint?: (point: OperationalStoreCutoverFailpoint) => void; -} +export type CreateSqliteDeepResearchStoreOptions = CreateDeepResearchStoreOptions; export function createSqliteDeepResearchStore( workspaceRoot: string, options: CreateSqliteDeepResearchStoreOptions = {}, ): SqliteDeepResearchStore { - return new SqliteDeepResearchStoreImpl(workspaceRoot, options); + return new SqliteDeepResearchStoreImpl( + workspaceRoot, + options.newId ?? randomUUID, + options.now ?? Date.now, + ); } -class FileDeepResearchStore implements DeepResearchStore { - private readonly sessionsRoot: string; - private readonly durabilityRoot: string; +class SqliteDeepResearchStoreImpl implements SqliteDeepResearchStore { + readonly #lease: OperationalStateDatabaseLease; private readonly writeQueues = new Map>(); private readonly subscribers = new Set<(event: DeepResearchChangedEvent) => void>(); @@ -75,8 +61,15 @@ class FileDeepResearchStore implements DeepResearchStore { private readonly newId: () => string, private readonly now: () => number, ) { - this.sessionsRoot = join(workspaceRoot, 'sessions'); - this.durabilityRoot = workspaceRoot; + this.#lease = acquireOperationalStateDatabase(resolve(workspaceRoot)); + } + + ready(): Promise { + return Promise.resolve(); + } + + close(): void { + this.#lease.close(); } async read(sessionId: string): Promise { @@ -86,38 +79,7 @@ class FileDeepResearchStore implements DeepResearchStore { async readEvents(sessionId: string): Promise { assertSafeSessionId(sessionId); - const canonical = await this.readCanonicalEvents(sessionId); - if (canonical) return canonical; - let text: string; - try { - text = await readFile(this.eventsPath(sessionId), 'utf8'); - } catch (error) { - if (isNodeError(error) && error.code === 'ENOENT') return []; - throw error; - } - const events: DeepResearchEvent[] = []; - const lines = text.split(/\r?\n/); - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index]!.trim(); - if (!line) continue; - let parsed: unknown; - try { - parsed = JSON.parse(line); - } catch (error) { - throw new Error( - `Invalid deep research event JSONL line ${index + 1}: ${ - error instanceof Error ? error.message : String(error) - }`, - ); - } - if (!isDeepResearchEvent(parsed)) { - throw new Error( - `Invalid deep research event JSONL line ${index + 1}: unexpected event shape`, - ); - } - events.push(parsed); - } - return events; + return readSqliteDeepResearchEvents(this.#lease.database, sessionId); } subscribe(listener: (event: DeepResearchChangedEvent) => void): () => void { @@ -377,164 +339,10 @@ class FileDeepResearchStore implements DeepResearchStore { } private async appendEvent(sessionId: string, event: DeepResearchEvent): Promise { - if (await this.appendCanonicalEvent(sessionId, event)) return; - const path = this.eventsPath(sessionId); - await mkdir(dirname(path), { recursive: true }); - await appendJsonl(path, `${JSON.stringify(event)}\n`, { - durable: true, - durabilityRoot: this.durabilityRoot, - }); - } - - protected async readCanonicalEvents( - _sessionId: string, - ): Promise { - return undefined; - } - - protected async appendCanonicalEvent( - _sessionId: string, - _event: DeepResearchEvent, - ): Promise { - return false; - } - - private eventsPath(sessionId: string): string { - return join(this.sessionsRoot, sessionId, 'deep-research', 'events.jsonl'); - } -} - -class SqliteDeepResearchStoreImpl extends FileDeepResearchStore implements SqliteDeepResearchStore { - readonly #lease: OperationalStateDatabaseLease; - readonly #ready: Promise; - - constructor(workspaceRoot: string, options: CreateSqliteDeepResearchStoreOptions) { - super(workspaceRoot, options.newId ?? randomUUID, options.now ?? Date.now); - const root = resolve(workspaceRoot); - this.#lease = acquireOperationalStateDatabase(root); - this.#ready = importLegacyDeepResearchState(root, this.#lease, options.failpoint); - } - - ready(): Promise { - return this.#ready; - } - - close(): void { - this.#lease.close(); - } - - protected override async readCanonicalEvents(sessionId: string): Promise { - await this.#ready; - return readSqliteDeepResearchEvents(this.#lease.database, sessionId); - } - - protected override async appendCanonicalEvent( - sessionId: string, - event: DeepResearchEvent, - ): Promise { - await this.#ready; this.#lease.transaction('write', () => { insertDeepResearchEvent(this.#lease.database, sessionId, event); }); - return true; - } -} - -interface LegacyDeepResearchLedger { - sessionId: string; - events: DeepResearchEvent[]; -} - -async function importLegacyDeepResearchState( - root: string, - lease: OperationalStateDatabaseLease, - failpoint?: (point: OperationalStoreCutoverFailpoint) => void, -): Promise { - const ledgers = await readLegacyDeepResearchLedgers(root); - const fingerprint = `sha256:${createHash('sha256') - .update(JSON.stringify(ledgers)) - .digest('hex')}`; - completeOperationalStoreCutover(lease, { - storeName: 'workflow_deep_research', - sourcePath: join(root, 'sessions'), - sourceFingerprint: fingerprint, - failpoint, - importAndValidate: (database) => { - let eventCount = 0; - for (const ledger of ledgers) { - for (const event of ledger.events) { - insertOrValidateDeepResearchEvent(database, ledger.sessionId, event); - eventCount += 1; - } - } - const persisted = database - .prepare('SELECT COUNT(*) AS count FROM workflow_deep_research_events') - .get() as { count?: unknown }; - if (persisted.count !== eventCount) { - throw new Error('Deep Research cutover row-count validation failed'); - } - return { sessions: ledgers.length, events: eventCount }; - }, - }); -} - -async function readLegacyDeepResearchLedgers(root: string): Promise { - const sessionsRoot = join(root, 'sessions'); - let entries; - try { - entries = await readdir(sessionsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const ledgers: LegacyDeepResearchLedger[] = []; - for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { - if (!entry.isDirectory()) continue; - assertSafeSessionId(entry.name); - let text: string; - try { - text = await readFile( - join(sessionsRoot, entry.name, 'deep-research', 'events.jsonl'), - 'utf8', - ); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue; - throw error; - } - const events = decodeLegacyDeepResearchEvents(text, entry.name); - const projection = projectDeepResearchEvents(events); - if (projection.diagnostics.length > 0) { - throw new Error( - `Deep Research ledger projection failed: ${projection.diagnostics.join('; ')}`, - ); - } - ledgers.push({ sessionId: entry.name, events }); } - return ledgers; -} - -function decodeLegacyDeepResearchEvents(text: string, sessionId: string): DeepResearchEvent[] { - const events: DeepResearchEvent[] = []; - for (const [index, line] of text.split(/\r?\n/).entries()) { - if (!line.trim()) continue; - let parsed: unknown; - try { - parsed = JSON.parse(line); - } catch (error) { - throw new Error( - `Invalid deep research event JSONL line ${index + 1}: ${ - error instanceof Error ? error.message : String(error) - }`, - ); - } - if (!isDeepResearchEvent(parsed) || parsed.sessionId !== sessionId) { - throw new Error( - `Invalid deep research event JSONL line ${index + 1}: unexpected event shape`, - ); - } - events.push(parsed); - } - return events; } function readSqliteDeepResearchEvents( @@ -586,27 +394,6 @@ function insertDeepResearchEvent( .run(sessionId, row.sequence, event.eventId, JSON.stringify(event)); } -function insertOrValidateDeepResearchEvent( - database: DatabaseSync, - sessionId: string, - event: DeepResearchEvent, -): void { - const existing = database - .prepare(` - SELECT record_json - FROM workflow_deep_research_events - WHERE session_id = ? AND event_id = ? - `) - .get(sessionId, event.eventId) as { record_json?: unknown } | undefined; - if (existing) { - if (existing.record_json !== JSON.stringify(event)) { - throw new Error(`Deep Research cutover conflict: ${event.eventId}`); - } - return; - } - insertDeepResearchEvent(database, sessionId, event); -} - function refsFromContext(context: DeepResearchMutationContext): { refs?: DeepResearchEventRefs } { const refs: DeepResearchEventRefs = { ...(context.runId ? { runId: context.runId } : {}), @@ -616,10 +403,6 @@ function refsFromContext(context: DeepResearchMutationContext): { refs?: DeepRes return Object.keys(refs).length > 0 ? { refs } : {}; } -function isNodeError(error: unknown): error is NodeJS.ErrnoException { - return error instanceof Error && 'code' in error; -} - function sameStrings(left: readonly string[], right: readonly string[]): boolean { return left.length === right.length && left.every((value, index) => value === right[index]); } diff --git a/packages/storage/src/execution-stores.ts b/packages/storage/src/execution-stores.ts index 2d439e78dc..49d153dffa 100644 --- a/packages/storage/src/execution-stores.ts +++ b/packages/storage/src/execution-stores.ts @@ -48,7 +48,7 @@ import { import { openRuntimeEventPersistence, openRuntimeEventReadPersistence, -} from './runtime-event-transfer.js'; +} from './runtime-event-persistence.js'; import type { CommitToolOutcomeInput, CommitToolPreparedInput, @@ -379,8 +379,6 @@ async function createExecutionStoresForWrite sessionStore.reconcileOrphanedAgentGraphRetirements()), listPendingSessionRetirementCleanupIds: (sessionId) => run(() => sessionStore.listPendingSessionRetirementCleanupIds(sessionId)), - purgeRemovedSessionTranscript: (sessionId) => - run(() => sessionStore.purgeRemovedSessionTranscript(sessionId)), completeSessionRetirementCleanup: (sessionId) => run(() => sessionStore.completeSessionRetirementCleanup(sessionId)), close: () => diff --git a/packages/storage/src/index.ts b/packages/storage/src/index.ts index 00c7734fe4..42ccd5cbf3 100644 --- a/packages/storage/src/index.ts +++ b/packages/storage/src/index.ts @@ -1,14 +1,13 @@ export { - SQLITE_SESSION_METADATA_DATABASE_NAME, SessionNotFoundError, SessionReadMarkerMessageNotFoundError, assertSafeSessionId, createSessionStore, createUserMessage, - decodeSessionHeader, isSafeSessionId, isSessionNotFoundError, normalizeSessionHeader, + projectSessionCatalogMessages, } from './session-store.js'; export type { CreateStableSessionRequest, @@ -23,10 +22,7 @@ export type { StableSessionCreateInput, UpdateSessionConfigurationRequest, } from './session-store.js'; -export * from './session-transcript.js'; export * from './sqlite-session-metadata-store.js'; -export * from './session-metadata-transfer.js'; -export * from './session-metadata-maintenance.js'; export { ROOT_TURN_ADMISSION_MAX_CONTENT_BYTES, ROOT_TURN_ADMISSION_MAX_RECORD_BYTES, @@ -46,29 +42,14 @@ export type { RootTurnAdmissionStore, RootTurnSourceMessage, RootTurnSourceMessageReceipt, - SqliteAgentRunStoreOptions, } from './agent-run-store.js'; export { createSqliteShellRunStore } from './shell-run-store.js'; -export type { - ClosableShellRunStore, - SqliteShellRunStoreOptions, -} from './shell-run-store.js'; +export type { ClosableShellRunStore } from './shell-run-store.js'; export * from './connection-store.js'; -// Narrow public surface: only the typed store + the one-time migration. The -// file lock and atomic writer stay internal so callers can't bypass the -// CredentialStore contract and drive the low-level lock directly. -export { - CREDENTIAL_SCHEMA_VERSION, - createFileCredentialStore, - migrateLegacyCredentialFile, -} from './credential-store.js'; -export type { - CredentialCasResult, - CredentialKind, - CredentialStore, - LegacyCredentialDecryptor, -} from './credential-store.js'; +export { CREDENTIAL_SCHEMA_VERSION, createFileCredentialStore } from './credential-store.js'; +export type { CredentialCasResult, CredentialKind, CredentialStore } from './credential-store.js'; export * from './settings-store.js'; +export { createSqliteArtifactMetadataRepository } from './sqlite-artifact-metadata.js'; export { TelemetryQueryValidationError, TelemetryRepoClosedError, @@ -105,7 +86,6 @@ export * from './artifact-attachments.js'; export * from './provider-request-capture-artifact.js'; export { createSqlitePlanReminderStore } from './plan-reminder-store.js'; export type { - CreateSqlitePlanReminderStoreOptions, PlanReminderStore, SqlitePlanReminderStore, } from './plan-reminder-store.js'; @@ -118,7 +98,6 @@ export type { export { createSqliteTaskLedgerStore } from './task-ledger-store.js'; export type { ConversationTaskLedgerCopyInput, - CreateSqliteTaskLedgerStoreOptions, SqliteTaskLedgerStore, TaskLedgerAuthorityStore, TaskLedgerStore, @@ -135,7 +114,7 @@ export * from './config-transfer.js'; export * from './automation-store.js'; export * from './automation-authority.js'; export * from './sqlite-runtime-store.js'; -export * from './runtime-event-transfer.js'; +export * from './runtime-event-persistence.js'; export * from './operational-state-store.js'; export * from './operational-state-backup.js'; export * from './mcp-config-store.js'; @@ -145,7 +124,6 @@ export * from './long-term-memory-store.js'; export * from './project-catalog.js'; export * from './git-worktree-child-executor.js'; export * from './git-workspace-service.js'; -export * from './project-session-migration.js'; export * from './session-bundle-policy.js'; export * from './session-bundle-contract.js'; export * from './session-bundle-manifest.js'; diff --git a/packages/storage/src/interaction-store-public.ts b/packages/storage/src/interaction-store-public.ts index 6eabf82a9e..9409c1ed64 100644 --- a/packages/storage/src/interaction-store-public.ts +++ b/packages/storage/src/interaction-store-public.ts @@ -1,8 +1,7 @@ /** * Production interaction-store surface. * - * Legacy file-backed open functions intentionally remain in the implementation - * module for import and compatibility tests, but are not package exports. + * The facade is backed by the canonical operational SQLite database. */ export { STORED_INTERACTION_OUTCOME_MAX_BYTES, @@ -11,7 +10,6 @@ export { authenticateInteractionStoreReader, authenticateInteractionStoreWriter, closeSqliteInteractionStoreFacade, - interactionLocator, openSqliteInteractiveInteractionStoreForRead, openSqliteInteractiveInteractionStoreForWrite, } from './interaction-store.js'; @@ -27,7 +25,6 @@ export type { InteractiveInteractionStoreReaderFacade, InteractiveInteractionStoreWriterFacade, PendingInteractionFilter, - SqliteInteractionStoreOptions, StoredInteractionOutcome, StoredInteractionRequest, } from './interaction-store.js'; diff --git a/packages/storage/src/interaction-store.ts b/packages/storage/src/interaction-store.ts index e3d59c4f37..4394745616 100644 --- a/packages/storage/src/interaction-store.ts +++ b/packages/storage/src/interaction-store.ts @@ -1,7 +1,4 @@ -import { createHash, randomUUID } from 'node:crypto'; -import { constants as fsConstants, type BigIntStats } from 'node:fs'; -import { link, lstat, mkdir, open, readdir, rmdir, unlink } from 'node:fs/promises'; -import { join, resolve } from 'node:path'; +import { resolve } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import { decodeInteractionCanonicalOutcome, @@ -12,30 +9,19 @@ import { type InteractionCanonicalOutcome, type InteractionRequest, } from '@maka/core'; -import { syncDirectory, syncDirectoryChain } from './stable-storage.js'; import { assertStorageRootLease, runWithStorageRootLease, StorageRootAuthorityError, type StorageRootLease, } from './root-authority.js'; -import { readBoundedMarkerFile } from './marker-file.js'; -import { - InteractionLocatorLane, - runInteractionStoreOperation, - type InteractionStoreOperationRunner, -} from './interaction-locator-lane.js'; import { acquireOperationalStateDatabase, - completeOperationalStoreCutover, type OperationalStateDatabaseLease, - type OperationalStoreCutoverFailpoint, } from './operational-state-store.js'; const SAFE_ID = /^[A-Za-z0-9_-]{1,128}$/; const REMEMBER_SCOPE_ID = /^[0-9a-f]{64}$/; -const LOCATOR = /^[0-9a-f]{64}$/; -const TEMP_FILE = /^(request|outcome)\.json\.[0-9a-f-]+\.tmp$/; export const STORED_INTERACTION_REQUEST_MAX_BYTES = 20 * 1024; export const STORED_INTERACTION_OUTCOME_MAX_BYTES = 12 * 1024; @@ -134,11 +120,6 @@ export interface InteractiveInteractionStoreWriterFacade extends InteractionStor readonly access: 'write'; } -const writersByLease = new WeakMap(); -const writerOpeningsByLease = new WeakMap< - object, - Promise ->(); const readers = new WeakSet(); const writers = new WeakSet(); const sqliteWritersByLease = new WeakMap(); @@ -162,78 +143,11 @@ export function authenticateInteractionStoreWriter( return store; } -export function interactionLocator(requestId: string): string { - return createHash('sha256').update(assertId(requestId)).digest('hex'); -} - -export async function openInteractiveInteractionStoreForRead( - lease: StorageRootLease<'interactive', 'read'>, -): Promise { - await assertStorageRootLease(lease, 'interactive', 'read'); - const store = new FileInteractionStore(lease.canonicalPath, false); - const run = (operation: () => Promise) => - runWithStorageRootLease(lease, 'interactive', 'read', operation); - const facade = Object.freeze({ - kind: 'interactive' as const, - access: 'read' as const, - readInteraction: (requestId: string) => store.readInteraction(requestId, run), - listSessionPending: (sessionId: string) => run(() => store.listSessionPending(sessionId)), - listPending: (filter?: PendingInteractionFilter) => run(() => store.listPending(filter)), - }); - readers.add(facade); - return facade; -} - -export async function openInteractiveInteractionStoreForWrite( - lease: StorageRootLease<'interactive', 'write'>, -): Promise { - await assertStorageRootLease(lease, 'interactive', 'write'); - const existing = writersByLease.get(lease); - if (existing) return existing; - const opening = writerOpeningsByLease.get(lease); - if (opening) return opening; - - const pending = Promise.resolve().then(async () => { - const store = new FileInteractionStore(lease.canonicalPath, true); - const run = (operation: () => Promise) => - runWithStorageRootLease(lease, 'interactive', 'write', operation); - await run(() => store.recover()); - return run(async () => { - const recoveredExisting = writersByLease.get(lease); - if (recoveredExisting) return recoveredExisting; - const facade = Object.freeze({ - kind: 'interactive' as const, - access: 'write' as const, - readInteraction: (requestId: string) => store.readInteraction(requestId, run), - listSessionPending: (sessionId: string) => run(() => store.listSessionPending(sessionId)), - listPending: (filter?: PendingInteractionFilter) => run(() => store.listPending(filter)), - establishRequest: (input: StoredInteractionRequest) => store.establishRequest(input, run), - commitOutcome: (requestId: string, outcome: InteractionCanonicalOutcome) => - store.commitOutcome(requestId, outcome, run), - }); - writers.add(facade); - writersByLease.set(lease, facade); - return facade; - }); - }); - writerOpeningsByLease.set(lease, pending); - try { - return await pending; - } finally { - if (writerOpeningsByLease.get(lease) === pending) writerOpeningsByLease.delete(lease); - } -} - -export interface SqliteInteractionStoreOptions { - readonly failpoint?: (point: OperationalStoreCutoverFailpoint) => void; -} - export async function openSqliteInteractiveInteractionStoreForRead( lease: StorageRootLease<'interactive', 'read'>, - options: SqliteInteractionStoreOptions = {}, ): Promise { await assertStorageRootLease(lease, 'interactive', 'read'); - const store = new SqliteInteractionStore(lease.canonicalPath, options); + const store = new SqliteInteractionStore(lease.canonicalPath); await store.ready(); const run = (operation: () => Promise) => runWithStorageRootLease(lease, 'interactive', 'read', operation); @@ -251,7 +165,6 @@ export async function openSqliteInteractiveInteractionStoreForRead( export async function openSqliteInteractiveInteractionStoreForWrite( lease: StorageRootLease<'interactive', 'write'>, - options: SqliteInteractionStoreOptions = {}, ): Promise { await assertStorageRootLease(lease, 'interactive', 'write'); const existing = sqliteWritersByLease.get(lease); @@ -259,7 +172,7 @@ export async function openSqliteInteractiveInteractionStoreForWrite( const opening = sqliteWriterOpeningsByLease.get(lease); if (opening) return opening; const pending = Promise.resolve().then(async () => { - const store = new SqliteInteractionStore(lease.canonicalPath, options); + const store = new SqliteInteractionStore(lease.canonicalPath); await store.ready(); const run = (operation: () => Promise) => runWithStorageRootLease(lease, 'interactive', 'write', operation); @@ -304,18 +217,14 @@ export function closeSqliteInteractionStoreFacade( } class SqliteInteractionStore implements InteractionStoreWriter { - readonly #root: string; readonly #lease: OperationalStateDatabaseLease; - readonly #ready: Promise; - constructor(root: string, options: SqliteInteractionStoreOptions) { - this.#root = resolve(root); - this.#lease = acquireOperationalStateDatabase(this.#root); - this.#ready = importLegacyInteractions(this.#root, this.#lease, options); + constructor(root: string) { + this.#lease = acquireOperationalStateDatabase(resolve(root)); } ready(): Promise { - return this.#ready; + return Promise.resolve(); } async establishRequest( @@ -323,7 +232,6 @@ class SqliteInteractionStore implements InteractionStoreWriter { ): Promise { const candidate = normalizeRequest(input, 'input'); const encoded = encode(candidate, STORED_INTERACTION_REQUEST_MAX_BYTES).toString('utf8').trim(); - await this.#ready; try { return this.#lease.transaction('write', () => { const existing = readSqliteInteraction(this.#lease, candidate.requestId); @@ -368,7 +276,6 @@ class SqliteInteractionStore implements InteractionStoreWriter { outcome: InteractionCanonicalOutcome, ): Promise { assertId(requestId); - await this.#ready; return this.#lease.transaction('write', () => { const record = readSqliteInteraction(this.#lease, requestId); if (!record) { @@ -413,7 +320,6 @@ class SqliteInteractionStore implements InteractionStoreWriter { async readInteraction(requestId: string): Promise { assertId(requestId); - await this.#ready; return readSqliteInteraction(this.#lease, requestId); } @@ -423,7 +329,6 @@ class SqliteInteractionStore implements InteractionStoreWriter { async listPending(filter: PendingInteractionFilter = {}): Promise { normalizeFilter(filter); - await this.#ready; const rows = this.#lease.database .prepare(` SELECT request_id @@ -453,668 +358,6 @@ class SqliteInteractionStore implements InteractionStoreWriter { } } -class FileInteractionStore { - private readonly root: string; - private readonly interactionsRoot: string; - private readonly sessionPendingRoot: string; - private readonly locatorLane = new InteractionLocatorLane(); - - constructor( - root: string, - private readonly useSessionPendingIndex: boolean, - ) { - this.root = resolve(root); - this.interactionsRoot = join(this.root, 'interactions'); - this.sessionPendingRoot = join(this.interactionsRoot, 'pending'); - } - - async recover(): Promise { - const indexRoot = await this.ensureSessionPendingRoot(); - await syncDirectoryChain(this.sessionPendingRoot, this.root); - await sessionPendingDirectoryPolicy.assert(indexRoot); - const expectedPending = new Map>(); - for (const locator of await this.locators()) { - await this.withLocator(locator, runInteractionStoreOperation, async () => { - const bindings = await this.bindExistingLocator(locator); - if (!bindings) invalidInteractionLocator(); - await this.stabilizeBoundLocator(bindings); - const record = await this.readBoundLocatorUnlocked(bindings); - if (!record) { - const directory = bindings.at(-1)?.path; - if (!directory) invalidInteractionLocator(); - await interactionDirectoryPolicy.assert(bindings); - await rmdir(directory); - await syncDirectory(this.interactionsRoot); - return; - } - if (!record.outcome) addPendingLocator(expectedPending, record.request.sessionId, locator); - }); - } - await sessionPendingDirectoryPolicy.assert(indexRoot); - await this.reconcileSessionPendingIndex(expectedPending); - } - - async listRecordsForMigration(): Promise { - let rootEntries; - try { - rootEntries = await readdir(this.interactionsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - for (const entry of rootEntries) { - if (entry.name === 'pending' && entry.isDirectory()) continue; - if (!entry.isDirectory() || !LOCATOR.test(entry.name)) invalidInteractionLocator(); - } - const records: InteractionRecord[] = []; - for (const locator of await this.locators()) { - const bindings = await this.bindExistingLocator(locator); - if (!bindings) invalidInteractionLocator(); - const record = await this.readBoundLocatorUnlocked(bindings); - if (!record) invalidInteractionLocator(); - records.push(record); - } - return records.sort( - (left, right) => - left.request.createdAt - right.request.createdAt || - left.request.requestId.localeCompare(right.request.requestId), - ); - } - - async establishRequest( - input: StoredInteractionRequest, - authorize: InteractionStoreOperationRunner = runInteractionStoreOperation, - ): Promise { - const candidate = normalizeRequest(input, 'input'); - const locator = interactionLocator(candidate.requestId); - return this.withLocator(locator, authorize, async () => { - const attempt = await this.publish( - locator, - 'request.json', - encode(candidate, STORED_INTERACTION_REQUEST_MAX_BYTES), - ); - try { - const bindings = await this.bindExistingLocator(locator); - let record: InteractionRecord | undefined; - if (bindings) { - await this.stabilizeBoundLocator(bindings); - record = await this.readBoundLocatorUnlocked(bindings, candidate.requestId); - } - if (record) { - if (record.outcome) { - await this.removeSessionPendingMarker(record.request.sessionId, locator); - } else { - await this.publishSessionPendingMarker(record.request.sessionId, locator); - } - return { - status: 'stable', - matches: isDeepStrictEqual(record.request, candidate), - record, - }; - } - } catch (error) { - return { - status: 'unresolved', - failure: failure(error, 'Request publication could not be stabilized'), - }; - } - return attempt === 'not_attempted' - ? { - status: 'definitely_not_published', - failure: new InteractionStoreError( - 'io_failed', - 'Request publication did not reach its exclusive link', - ), - } - : { - status: 'unresolved', - failure: new InteractionStoreError( - 'io_failed', - 'Request publication outcome is ambiguous', - ), - }; - }); - } - - async commitOutcome( - requestId: string, - outcome: InteractionCanonicalOutcome, - authorize: InteractionStoreOperationRunner = runInteractionStoreOperation, - ): Promise { - assertId(requestId); - const locator = interactionLocator(requestId); - return this.withLocator(locator, authorize, async () => { - let record: InteractionRecord | undefined; - try { - const bindings = await this.bindExistingLocator(locator); - if (bindings) { - await this.stabilizeBoundLocator(bindings); - record = await this.readBoundLocatorUnlocked(bindings, requestId); - } - } catch (error) { - return { - status: 'unresolved', - failure: failure(error, 'Request could not be read'), - }; - } - if (!record) - throw new InteractionStoreError( - 'request_not_found', - `Interaction request '${requestId}' does not exist`, - ); - let canonical: InteractionCanonicalOutcome; - try { - canonical = decodeInteractionCanonicalOutcome(outcome); - } catch (error) { - decodeFailure('input', 'Invalid Interaction outcome', error); - } - if (!isInteractionCanonicalOutcomeValidForRequest(record.request.request, canonical)) { - throw new InteractionStoreError('invalid_input', 'Outcome is not valid for its request'); - } - const candidate: StoredInteractionOutcome = { - ...identity(record.request), - outcome: canonical, - }; - const attempt = await this.publish( - locator, - 'outcome.json', - encode(candidate, STORED_INTERACTION_OUTCOME_MAX_BYTES), - ); - try { - const bindings = await this.bindExistingLocator(locator); - let settled: InteractionRecord | undefined; - if (bindings) { - await this.stabilizeBoundLocator(bindings); - settled = await this.readBoundLocatorUnlocked(bindings, requestId); - } - if (settled?.outcome) { - await this.removeSessionPendingMarker(settled.request.sessionId, locator); - return { - status: 'stable', - matches: interactionCanonicalOutcomesEquivalent(settled.outcome.outcome, canonical), - record: settled as InteractionRecord & { - outcome: StoredInteractionOutcome; - }, - }; - } - } catch (error) { - return { - status: 'unresolved', - failure: failure(error, 'Outcome publication could not be stabilized'), - }; - } - return attempt === 'not_attempted' - ? { - status: 'definitely_not_published', - failure: new InteractionStoreError( - 'io_failed', - 'Outcome publication did not reach its exclusive link', - ), - } - : { - status: 'unresolved', - failure: new InteractionStoreError( - 'io_failed', - 'Outcome publication outcome is ambiguous', - ), - }; - }); - } - - async readInteraction( - requestId: string, - authorize: InteractionStoreOperationRunner = runInteractionStoreOperation, - ): Promise { - const locator = interactionLocator(requestId); - return this.withLocator(locator, authorize, async () => { - const bindings = await this.bindExistingLocator(locator); - return bindings ? this.readBoundLocatorUnlocked(bindings, requestId) : undefined; - }); - } - - async listSessionPending(sessionId: string): Promise { - const canonicalSessionId = assertId(sessionId); - if (!this.useSessionPendingIndex) { - return this.listCanonicalPending({ sessionId: canonicalSessionId }); - } - const result: StoredInteractionRequest[] = []; - for (const locator of await this.sessionPendingLocators(canonicalSessionId)) { - const record = await this.withLocator(locator, runInteractionStoreOperation, async () => { - const bindings = await this.bindExistingLocator(locator); - return bindings ? this.readBoundLocatorUnlocked(bindings) : undefined; - }); - if (record && !record.outcome && record.request.sessionId === canonicalSessionId) { - result.push(record.request); - } - } - return sortPending(result); - } - - async listPending(filter: PendingInteractionFilter = {}): Promise { - normalizeFilter(filter); - if (this.useSessionPendingIndex && filter.sessionId !== undefined) { - return (await this.listSessionPending(filter.sessionId)).filter((request) => - matches(request, filter), - ); - } - return this.listCanonicalPending(filter); - } - - private async listCanonicalPending( - filter: PendingInteractionFilter, - ): Promise { - const result: StoredInteractionRequest[] = []; - for (const locator of await this.locators()) { - const record = await this.withLocator(locator, runInteractionStoreOperation, async () => { - const bindings = await this.bindExistingLocator(locator); - return bindings ? this.readBoundLocatorUnlocked(bindings) : undefined; - }); - if (record && !record.outcome && matches(record.request, filter)) result.push(record.request); - } - return sortPending(result); - } - - private async publishSessionPendingMarker(sessionId: string, locator: string): Promise { - await this.publishSessionPendingMarkerByLocator(interactionLocator(sessionId), locator); - } - - private async removeSessionPendingMarker(sessionId: string, locator: string): Promise { - const bindings = await this.bindExistingSessionPendingDirectory(interactionLocator(sessionId)); - if (!bindings) return; - const directory = bindings.at(-1)?.path; - if (!directory) throw invalidSessionPendingMarker(); - const marker = join(directory, locator); - try { - await readSessionPendingMarker(marker); - } catch (error) { - if (isNodeError(error, 'ENOENT')) return; - throw error; - } - await sessionPendingDirectoryPolicy.assert(bindings); - try { - await unlink(marker); - } catch (error) { - if (isNodeError(error, 'ENOENT')) return; - throw error; - } - await syncDirectory(directory); - await sessionPendingDirectoryPolicy.assert(bindings); - } - - private async reconcileSessionPendingIndex( - expectedPending: ReadonlyMap>, - ): Promise { - for (const sessionLocator of await this.sessionPendingDirectories()) { - const directory = join(this.sessionPendingRoot, sessionLocator); - const bindings = await sessionPendingDirectoryPolicy.bind([ - this.interactionsRoot, - this.sessionPendingRoot, - directory, - ]); - const expected = expectedPending.get(sessionLocator); - for (const entry of await readdir(directory, { withFileTypes: true })) { - if (!LOCATOR.test(entry.name)) invalidSessionPendingMarker(); - const marker = join(directory, entry.name); - await readSessionPendingMarker(marker); - if (expected?.has(entry.name)) continue; - await sessionPendingDirectoryPolicy.assert(bindings); - await unlink(marker); - } - await syncDirectory(directory); - await sessionPendingDirectoryPolicy.assert(bindings); - } - for (const [sessionLocator, locators] of expectedPending) { - for (const locator of locators) { - await this.publishSessionPendingMarkerByLocator(sessionLocator, locator); - } - } - } - - private async publishSessionPendingMarkerByLocator( - sessionLocator: string, - locator: string, - ): Promise { - const bindings = await this.ensureSessionPendingDirectory(sessionLocator); - const directory = bindings.at(-1)?.path; - if (!directory) throw invalidSessionPendingMarker(); - await syncDirectoryChain(directory, this.root); - await sessionPendingDirectoryPolicy.assert(bindings); - const marker = join(directory, locator); - let handle; - try { - handle = await open(marker, canonicalCreateFlags(), 0o600); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; - await readSessionPendingMarker(marker); - await sessionPendingDirectoryPolicy.assert(bindings); - return; - } - try { - assertSessionPendingMarker(await handle.stat({ bigint: true })); - await handle.sync(); - } finally { - await handle.close(); - } - await sessionPendingDirectoryPolicy.assert(bindings); - await readSessionPendingMarker(marker); - await syncDirectory(directory); - await sessionPendingDirectoryPolicy.assert(bindings); - } - - private async sessionPendingLocators(sessionId: string): Promise { - const bindings = await this.bindExistingSessionPendingDirectory(interactionLocator(sessionId)); - if (!bindings) return []; - const directory = bindings.at(-1)?.path; - if (!directory) throw invalidSessionPendingMarker(); - let entries; - try { - entries = await readdir(directory, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const locators: string[] = []; - for (const entry of entries) { - if (!LOCATOR.test(entry.name)) invalidSessionPendingMarker(); - try { - await readSessionPendingMarker(join(directory, entry.name)); - } catch (error) { - if (isNodeError(error, 'ENOENT')) continue; - throw error; - } - locators.push(entry.name); - } - await sessionPendingDirectoryPolicy.assert(bindings); - return locators.sort(); - } - - private async sessionPendingDirectories(): Promise { - const rootBindings = await sessionPendingDirectoryPolicy.bind([ - this.interactionsRoot, - this.sessionPendingRoot, - ]); - const entries = await readdir(this.sessionPendingRoot, { withFileTypes: true }); - const locators: string[] = []; - for (const entry of entries) { - if (!LOCATOR.test(entry.name)) invalidSessionPendingMarker(); - await sessionPendingDirectoryPolicy.bind([ - ...rootBindings.map((binding) => binding.path), - join(this.sessionPendingRoot, entry.name), - ]); - locators.push(entry.name); - } - await sessionPendingDirectoryPolicy.assert(rootBindings); - return locators.sort(); - } - - private async ensureSessionPendingRoot(): Promise { - await mkdirIfMissing(this.interactionsRoot); - const interactions = await sessionPendingDirectoryPolicy.bind([this.interactionsRoot]); - await mkdirIfMissing(this.sessionPendingRoot); - return sessionPendingDirectoryPolicy.bind([ - ...interactions.map((binding) => binding.path), - this.sessionPendingRoot, - ]); - } - - private async ensureSessionPendingDirectory(sessionLocator: string): Promise { - const rootBindings = await this.ensureSessionPendingRoot(); - await sessionPendingDirectoryPolicy.assert(rootBindings); - const directory = join(this.sessionPendingRoot, sessionLocator); - await mkdirIfMissing(directory); - return sessionPendingDirectoryPolicy.bind([ - ...rootBindings.map((binding) => binding.path), - directory, - ]); - } - - private async bindExistingSessionPendingDirectory( - sessionLocator: string, - ): Promise { - try { - return await sessionPendingDirectoryPolicy.bind([ - this.interactionsRoot, - this.sessionPendingRoot, - join(this.sessionPendingRoot, sessionLocator), - ]); - } catch (error) { - if (isNodeError(error, 'ENOENT')) return undefined; - throw error; - } - } - - private async publish( - locator: string, - name: string, - bytes: Buffer, - ): Promise<'published_or_existing' | 'not_attempted'> { - let bindings: DirectoryBinding[] | undefined; - let directory: string | undefined; - let temp: string | undefined; - let tempIdentity: FileIdentity | undefined; - let linked = false; - let linkAttempted = false; - try { - bindings = await this.ensureLocator(locator); - directory = bindings.at(-1)?.path; - if (!directory) invalidInteractionLocator(); - await syncDirectoryChain(directory, this.root); - await interactionDirectoryPolicy.assert(bindings); - temp = join(directory, `${name}.${randomUUID()}.tmp`); - const handle = await open(temp, canonicalCreateFlags(), 0o600); - try { - const metadata = await handle.stat({ bigint: true }); - assertCanonicalFile(metadata); - tempIdentity = { dev: metadata.dev, ino: metadata.ino }; - await handle.writeFile(bytes); - await handle.sync(); - } finally { - await handle.close(); - } - await interactionDirectoryPolicy.assert(bindings); - if (!tempIdentity) invalidInteractionDocument(); - await assertCanonicalPathIdentity(temp, tempIdentity); - try { - linkAttempted = true; - await link(temp, join(directory, name)); - linked = true; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; - } - await interactionDirectoryPolicy.assert(bindings); - await syncDirectory(directory); - await removeCanonicalTemp(bindings, temp, tempIdentity); - temp = undefined; - await syncDirectory(directory); - await interactionDirectoryPolicy.assert(bindings); - return 'published_or_existing'; - } catch { - if (temp && bindings) { - await removeCanonicalTemp(bindings, temp, tempIdentity).catch(() => undefined); - } - return linked || linkAttempted ? 'published_or_existing' : 'not_attempted'; - } - } - - private async readBoundLocatorUnlocked( - bindings: readonly DirectoryBinding[], - expectedId?: string, - ): Promise { - const directory = bindings.at(-1)?.path; - if (!directory) invalidInteractionLocator(); - let entries; - try { - entries = await readdir(directory, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - await interactionDirectoryPolicy.assert(bindings); - return undefined; - } - throw error; - } - await interactionDirectoryPolicy.assert(bindings); - if (entries.some((entry) => TEMP_FILE.test(entry.name))) - throw new InteractionStoreError( - 'invalid_record', - 'Interaction contains an unresolved temporary artifact', - ); - const requestRaw = await readOptional( - join(directory, 'request.json'), - STORED_INTERACTION_REQUEST_MAX_BYTES, - bindings, - ); - if (requestRaw === undefined) { - if (entries.some((entry) => entry.name === 'outcome.json')) - throw new InteractionStoreError('invalid_record', 'Outcome exists without request'); - return undefined; - } - const request = normalizeRequest(parseJsonRecord(requestRaw, 'request'), 'record'); - if ( - (expectedId && request.requestId !== expectedId) || - join(this.interactionsRoot, interactionLocator(request.requestId)) !== directory - ) - throw new InteractionStoreError('invalid_record', 'Request identity does not match locator'); - const outcomeRaw = await readOptional( - join(directory, 'outcome.json'), - STORED_INTERACTION_OUTCOME_MAX_BYTES, - bindings, - ); - const outcome = - outcomeRaw === undefined - ? undefined - : normalizeOutcome(parseJsonRecord(outcomeRaw, 'outcome'), request); - await interactionDirectoryPolicy.assert(bindings); - return deepFreeze({ request, ...(outcome ? { outcome } : {}) }); - } - - private async stabilizeBoundLocator(bindings: readonly DirectoryBinding[]): Promise { - const directory = bindings.at(-1)?.path; - if (!directory) invalidInteractionLocator(); - const entries = await readdir(directory, { withFileTypes: true }); - for (const entry of entries) { - if (TEMP_FILE.test(entry.name)) { - await removeCanonicalTemp(bindings, join(directory, entry.name)); - } - } - await syncDirectory(directory); - await interactionDirectoryPolicy.assert(bindings); - } - - private async locators(): Promise { - let rootBindings: DirectoryBinding[]; - try { - rootBindings = await interactionDirectoryPolicy.bind([this.interactionsRoot]); - } catch (error) { - if (isNodeError(error, 'ENOENT')) return []; - throw error; - } - let entries; - try { - entries = await readdir(this.interactionsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const locators: string[] = []; - for (const entry of entries) { - if (!LOCATOR.test(entry.name)) continue; - await interactionDirectoryPolicy.bind([ - ...rootBindings.map((binding) => binding.path), - join(this.interactionsRoot, entry.name), - ]); - locators.push(entry.name); - } - await interactionDirectoryPolicy.assert(rootBindings); - return locators.sort(); - } - - private async ensureLocator(locator: string): Promise { - await mkdirIfMissing(this.interactionsRoot); - const rootBindings = await interactionDirectoryPolicy.bind([this.interactionsRoot]); - await interactionDirectoryPolicy.assert(rootBindings); - const directory = join(this.interactionsRoot, locator); - await mkdirIfMissing(directory); - return interactionDirectoryPolicy.bind([ - ...rootBindings.map((binding) => binding.path), - directory, - ]); - } - - private async bindExistingLocator(locator: string): Promise { - try { - return await interactionDirectoryPolicy.bind([ - this.interactionsRoot, - join(this.interactionsRoot, locator), - ]); - } catch (error) { - if (isNodeError(error, 'ENOENT')) return undefined; - throw error; - } - } - - private async withLocator( - locator: string, - authorize: InteractionStoreOperationRunner, - operation: () => Promise, - ): Promise { - return this.locatorLane.run(locator, authorize, operation); - } -} - -async function importLegacyInteractions( - root: string, - lease: OperationalStateDatabaseLease, - options: SqliteInteractionStoreOptions, -): Promise { - const records = await new FileInteractionStore(root, false).listRecordsForMigration(); - const fingerprint = createHash('sha256').update(JSON.stringify(records)).digest('hex'); - completeOperationalStoreCutover(lease, { - storeName: 'interactions', - sourcePath: join(root, 'interactions'), - sourceFingerprint: `sha256:${fingerprint}`, - failpoint: options.failpoint, - importAndValidate: (db) => { - for (const record of records) insertOrValidateInteraction(db, record); - return { - interaction_requests: records.length, - interaction_outcomes: records.filter((record) => record.outcome !== undefined).length, - }; - }, - }); -} - -function insertOrValidateInteraction( - db: import('node:sqlite').DatabaseSync, - record: InteractionRecord, -): void { - const request = normalizeRequest(record.request, 'record'); - db.prepare(` - INSERT OR IGNORE INTO core_interaction_requests( - request_id, session_id, turn_id, run_id, request_kind, created_at, record_json - ) VALUES (?, ?, ?, ?, ?, ?, ?) - `).run( - request.requestId, - request.sessionId, - request.turnId, - request.runId, - request.request.kind, - request.createdAt, - JSON.stringify(request), - ); - if (record.outcome) { - const outcome = normalizeOutcome(record.outcome, request); - db.prepare(` - INSERT OR IGNORE INTO core_interaction_outcomes(request_id, record_json) - VALUES (?, ?) - `).run(request.requestId, JSON.stringify(outcome)); - } - const leaseLike = { database: db } as OperationalStateDatabaseLease; - const existing = readSqliteInteraction(leaseLike, request.requestId); - if (!existing || !isDeepStrictEqual(existing, deepFreeze(record))) { - throw new Error(`Interaction cutover conflict: ${request.requestId}`); - } -} - function readSqliteInteraction( lease: Pick, requestId: string, @@ -1144,244 +387,12 @@ function readSqliteInteraction( return deepFreeze({ request, ...(outcome ? { outcome } : {}) }); } -function addPendingLocator( - pending: Map>, - sessionId: string, - locator: string, -): void { - const sessionLocator = interactionLocator(sessionId); - const locators = pending.get(sessionLocator) ?? new Set(); - locators.add(locator); - pending.set(sessionLocator, locators); -} - function sortPending(requests: StoredInteractionRequest[]): StoredInteractionRequest[] { return requests.sort( (a, b) => a.createdAt - b.createdAt || a.requestId.localeCompare(b.requestId), ); } -interface DirectoryBinding { - readonly path: string; - readonly dev: bigint; - readonly ino: bigint; -} - -interface FileIdentity { - readonly dev: bigint; - readonly ino: bigint; -} - -async function mkdirIfMissing(path: string): Promise { - try { - await mkdir(path, { mode: 0o700 }); - } catch (error) { - if (!isNodeError(error, 'EEXIST')) throw error; - } -} - -function directoryBindingPolicy(invalidDirectory: () => never): { - bind(paths: readonly string[]): Promise; - assert(bindings: readonly DirectoryBinding[]): Promise; -} { - return { - async bind(paths) { - const bindings: DirectoryBinding[] = []; - for (const path of paths) { - const metadata = await lstat(path, { bigint: true }); - if (!metadata.isDirectory()) invalidDirectory(); - bindings.push({ path, dev: metadata.dev, ino: metadata.ino }); - } - return bindings; - }, - async assert(bindings) { - for (const binding of bindings) { - const metadata = await lstat(binding.path, { bigint: true }); - if ( - !metadata.isDirectory() || - metadata.dev !== binding.dev || - metadata.ino !== binding.ino - ) { - invalidDirectory(); - } - } - }, - }; -} - -const sessionPendingDirectoryPolicy = directoryBindingPolicy(invalidSessionPendingMarker); -const interactionDirectoryPolicy = directoryBindingPolicy(invalidInteractionLocator); - -async function removeCanonicalTemp( - bindings: readonly DirectoryBinding[], - path: string, - expected?: FileIdentity, -): Promise { - const bound = await lstat(path, { bigint: true }); - assertCanonicalFile(bound); - if (expected) assertCanonicalFileIdentity(bound, expected); - await interactionDirectoryPolicy.assert(bindings); - await assertCanonicalPathIdentity(path, bound); - await unlink(path); - await interactionDirectoryPolicy.assert(bindings); -} - -async function assertCanonicalPathIdentity(path: string, expected: FileIdentity): Promise { - const metadata = await lstat(path, { bigint: true }); - assertCanonicalFileIdentity(metadata, expected); -} - -async function readSessionPendingMarker(path: string): Promise { - try { - const contents = await readBoundedMarkerFile({ - path, - maxBytes: 0, - invalidFile: sessionPendingMarkerError, - }); - if (contents !== '') invalidSessionPendingMarker(); - } catch (error) { - if (error instanceof InteractionStoreError || isNodeError(error, 'ENOENT')) { - throw error; - } - if ( - isNodeError(error, 'ELOOP') || - isNodeError(error, 'EISDIR') || - isNodeError(error, 'ENXIO') || - isNodeError(error, 'ENODEV') - ) { - throw sessionPendingMarkerError(error); - } - throw error; - } -} - -function assertSessionPendingMarker(metadata: BigIntStats): void { - if (!metadata.isFile() || metadata.size !== 0n) invalidSessionPendingMarker(); -} - -function canonicalCreateFlags(): string | number { - if (process.platform === 'win32') return 'wx'; - return ( - fsConstants.O_WRONLY | - fsConstants.O_CREAT | - fsConstants.O_EXCL | - fsConstants.O_NONBLOCK | - fsConstants.O_NOFOLLOW - ); -} - -function canonicalReadFlags(): string | number { - if (process.platform === 'win32') return fsConstants.O_RDONLY | fsConstants.O_NONBLOCK; - return fsConstants.O_RDONLY | fsConstants.O_NONBLOCK | fsConstants.O_NOFOLLOW; -} - -function sessionPendingMarkerError(cause?: unknown): InteractionStoreError { - return new InteractionStoreError( - 'invalid_record', - 'Session pending index contains an invalid marker', - cause === undefined ? undefined : { cause }, - ); -} - -function invalidSessionPendingMarker(cause?: unknown): never { - throw sessionPendingMarkerError(cause); -} - -function invalidInteractionLocator(cause?: unknown): never { - throw new InteractionStoreError( - 'invalid_record', - 'Interaction locator is not a stable directory', - cause === undefined ? undefined : { cause }, - ); -} - -function invalidInteractionDocument(cause?: unknown): never { - throw new InteractionStoreError( - 'invalid_record', - 'Interaction document is not a stable regular file', - cause === undefined ? undefined : { cause }, - ); -} - -function assertCanonicalFile(metadata: BigIntStats): void { - if (!metadata.isFile()) invalidInteractionDocument(); -} - -function assertCanonicalFileIdentity(metadata: BigIntStats, expected: FileIdentity): void { - if (!metadata.isFile() || metadata.dev !== expected.dev || metadata.ino !== expected.ino) { - invalidInteractionDocument(); - } -} - -function isNodeError(error: unknown, code: string): boolean { - return ( - error instanceof Error && 'code' in error && (error as NodeJS.ErrnoException).code === code - ); -} - -async function readOptional( - path: string, - limit: number, - bindings: readonly DirectoryBinding[], -): Promise { - let pathMetadata: BigIntStats; - try { - pathMetadata = await lstat(path, { bigint: true }); - } catch (error) { - if (isNodeError(error, 'ENOENT')) { - await interactionDirectoryPolicy.assert(bindings); - return undefined; - } - throw error; - } - assertCanonicalFile(pathMetadata); - await interactionDirectoryPolicy.assert(bindings); - let handle; - try { - handle = await open(path, canonicalReadFlags()); - } catch (error) { - if (isNodeError(error, 'ENOENT')) { - await interactionDirectoryPolicy.assert(bindings); - return undefined; - } - if (isNodeError(error, 'ELOOP') || isNodeError(error, 'EISDIR')) { - invalidInteractionDocument(error); - } - throw error; - } - try { - const stat = await handle.stat({ bigint: true }); - assertCanonicalFile(stat); - if (stat.size > BigInt(limit)) - throw new InteractionStoreError('invalid_record', 'Interaction document exceeds size limit'); - assertCanonicalFileIdentity(stat, pathMetadata); - await assertCanonicalPathIdentity(path, stat); - await interactionDirectoryPolicy.assert(bindings); - const bytes = Buffer.alloc(limit + 1); - let total = 0; - while (total < bytes.length) { - const read = await handle.read(bytes, total, bytes.length - total, null); - if (read.bytesRead === 0) break; - total += read.bytesRead; - } - if (total > limit) - throw new InteractionStoreError('invalid_record', 'Interaction document exceeds size limit'); - let value: string; - try { - value = new TextDecoder('utf-8', { fatal: true }).decode(bytes.subarray(0, total)); - } catch (error) { - throw new InteractionStoreError('invalid_record', 'Interaction document is not valid UTF-8', { - cause: error, - }); - } - await assertCanonicalPathIdentity(path, stat); - await interactionDirectoryPolicy.assert(bindings); - return value; - } finally { - await handle.close(); - } -} - type DecodeSource = 'input' | 'record'; function normalizeRequest(value: unknown, source: DecodeSource): StoredInteractionRequest { diff --git a/packages/storage/src/jsonl-append.ts b/packages/storage/src/jsonl-append.ts deleted file mode 100644 index f81582579e..0000000000 --- a/packages/storage/src/jsonl-append.ts +++ /dev/null @@ -1,128 +0,0 @@ -import { constants } from 'node:fs'; -import { open, type FileHandle } from 'node:fs/promises'; -import { dirname } from 'node:path'; -import { DurableStoreWriteError } from '@maka/core'; -import { classifyJsonRecord } from './json-prefix.js'; -import { syncDirectoryChain } from './stable-storage.js'; - -const REVERSE_SCAN_CHUNK_BYTES = 64 * 1024; - -class JsonlAppendPreconditionError extends Error { - readonly name = 'JsonlAppendPreconditionError'; -} - -export interface AppendJsonlOptions { - durable?: boolean; - durabilityRoot?: string; - requireExistingRecord?: boolean; -} - -export async function appendJsonl( - path: string, - payload: string, - options: AppendJsonlOptions = {}, -): Promise { - try { - await appendJsonlUnchecked(path, payload, options); - } catch (error) { - if ( - !options.durable || - error instanceof DurableStoreWriteError || - error instanceof JsonlAppendPreconditionError - ) { - throw error; - } - throw new DurableStoreWriteError( - `Durable JSONL append did not reach stable storage: ${path}`, - error, - ); - } -} - -async function appendJsonlUnchecked( - path: string, - payload: string, - options: AppendJsonlOptions, -): Promise { - if (payload.length === 0 || !payload.endsWith('\n')) { - throw new JsonlAppendPreconditionError('JSONL append payload must end with a newline'); - } - - const flags = - constants.O_RDWR | constants.O_APPEND | (options.requireExistingRecord ? 0 : constants.O_CREAT); - const handle = await open(path, flags, 0o600); - try { - const size = (await handle.stat()).size; - if (size === 0 && options.requireExistingRecord) { - throw new JsonlAppendPreconditionError('Cannot append to an empty JSONL document'); - } - - let separator = ''; - if (size > 0 && !(await endsWithNewline(handle, size))) { - const tailStart = await findTrailingRecordStart(handle, size); - const tail = await readRange(handle, tailStart, size); - const classification = classifyJsonRecord(tail); - if (classification === 'complete') { - separator = '\n'; - } else if (classification === 'incomplete-prefix') { - if (tailStart === 0 && options.requireExistingRecord) { - throw new JsonlAppendPreconditionError('Cannot repair a truncated JSONL document header'); - } - await handle.truncate(tailStart); - await handle.sync(); - } else { - throw new JsonlAppendPreconditionError('Cannot append after an invalid JSONL tail record'); - } - } - - await handle.appendFile(separator + payload, 'utf8'); - if (options.durable) await handle.sync(); - } finally { - await handle.close(); - } - if (options.durable) { - if (!options.durabilityRoot) { - throw new Error('Durable JSONL append requires a durability root'); - } - await syncDirectoryChain(dirname(path), options.durabilityRoot); - } -} - -async function endsWithNewline(handle: FileHandle, size: number): Promise { - const byte = Buffer.allocUnsafe(1); - await readFully(handle, byte, size - 1); - return byte[0] === 0x0a; -} - -async function findTrailingRecordStart(handle: FileHandle, size: number): Promise { - let end = size; - while (end > 0) { - const start = Math.max(0, end - REVERSE_SCAN_CHUNK_BYTES); - const chunk = Buffer.allocUnsafe(end - start); - await readFully(handle, chunk, start); - const newline = chunk.lastIndexOf(0x0a); - if (newline >= 0) return start + newline + 1; - end = start; - } - return 0; -} - -async function readRange(handle: FileHandle, start: number, end: number): Promise { - const bytes = Buffer.allocUnsafe(end - start); - await readFully(handle, bytes, start); - return bytes.toString('utf8'); -} - -async function readFully(handle: FileHandle, buffer: Buffer, position: number): Promise { - let offset = 0; - while (offset < buffer.length) { - const { bytesRead } = await handle.read( - buffer, - offset, - buffer.length - offset, - position + offset, - ); - if (bytesRead === 0) throw new Error('Unexpected end of JSONL document'); - offset += bytesRead; - } -} diff --git a/packages/storage/src/legacy-execution-test-support.ts b/packages/storage/src/legacy-execution-test-support.ts deleted file mode 100644 index 8b26b8c81f..0000000000 --- a/packages/storage/src/legacy-execution-test-support.ts +++ /dev/null @@ -1,13 +0,0 @@ -/** - * Test-only constructors for seeding and exercising retired execution files. - * - * Production callers must use the canonical SQLite factories from - * `@maka/storage`. Keeping these behind an explicit subpath prevents an - * accidental root import from reviving a JSON/JSONL writer while preserving - * legacy import and compatibility coverage. - */ -export { - createAgentRunStore as createLegacyAgentRunStoreForTest, - createRuntimeEventStore as createLegacyRuntimeEventStoreForTest, -} from './agent-run-store.js'; -export { createShellRunStore as createLegacyShellRunStoreForTest } from './shell-run-store.js'; diff --git a/packages/storage/src/legacy-storage-test-support.ts b/packages/storage/src/legacy-storage-test-support.ts deleted file mode 100644 index 1301f5a11c..0000000000 --- a/packages/storage/src/legacy-storage-test-support.ts +++ /dev/null @@ -1,16 +0,0 @@ -/** - * Test-only constructors for seeding and exercising retired structured files. - * - * Production callers must use the canonical SQLite factories and lease-bound - * facades. This subpath exists only for legacy import and compatibility tests. - */ -export { createDeepResearchStore as createLegacyDeepResearchStoreForTest } from './deep-research-store.js'; -export { - openInteractiveInteractionStoreForRead as openLegacyInteractiveInteractionStoreForReadForTest, - openInteractiveInteractionStoreForWrite as openLegacyInteractiveInteractionStoreForWriteForTest, -} from './interaction-store.js'; -export { createPlanReminderStore as createLegacyPlanReminderStoreForTest } from './plan-reminder-store.js'; -export { createPlanStore as createLegacyPlanStoreForTest } from './plan-store.js'; -export { createLegacyFileSessionStore as createLegacySessionStoreForTest } from './session-store.js'; -export { createTaskLedgerStore as createLegacyTaskLedgerStoreForTest } from './task-ledger-store.js'; -export { createTelemetryRepo as createLegacyTelemetryRepoForTest } from './telemetry-repo.js'; diff --git a/packages/storage/src/message-receipt-store.ts b/packages/storage/src/message-receipt-store.ts index 5fb2111520..3c791b7b77 100644 --- a/packages/storage/src/message-receipt-store.ts +++ b/packages/storage/src/message-receipt-store.ts @@ -1,16 +1,10 @@ -import { createHash, randomUUID } from 'node:crypto'; -import { link, mkdir, open, readFile, readdir, rm, unlink } from 'node:fs/promises'; -import { dirname, join, resolve } from 'node:path'; +import { resolve } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import type { DatabaseSync } from 'node:sqlite'; import { acquireOperationalStateDatabase, - completeOperationalStoreCutover, type OperationalStateDatabaseLease, - type OperationalStoreCutoverFailpoint, } from './operational-state-store.js'; -import { syncDirectory, syncDirectoryChain } from './stable-storage.js'; -import { chainWrite } from './write-queue.js'; const SAFE_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; const RECEIPT_SCHEMA_VERSION = 1 as const; @@ -50,14 +44,6 @@ interface StoredMessageOperationReceipt { readonly result: unknown; } -export function createMessageReceiptStore(workspaceRoot: string): MessageReceiptStore { - return new FileMessageReceiptStore(workspaceRoot); -} - -export interface SqliteMessageReceiptStoreOptions { - readonly failpoint?: (point: OperationalStoreCutoverFailpoint) => void; -} - export interface ClosableMessageReceiptStore extends MessageReceiptStore { ready(): Promise; close(): void; @@ -65,29 +51,23 @@ export interface ClosableMessageReceiptStore extends MessageReceiptStore { export function createSqliteMessageReceiptStore( workspaceRoot: string, - options: SqliteMessageReceiptStoreOptions = {}, ): ClosableMessageReceiptStore { - return new SqliteMessageReceiptStore(workspaceRoot, options); + return new SqliteMessageReceiptStore(workspaceRoot); } class SqliteMessageReceiptStore implements ClosableMessageReceiptStore { - readonly #root: string; readonly #lease: OperationalStateDatabaseLease; - readonly #ready: Promise; - constructor(workspaceRoot: string, options: SqliteMessageReceiptStoreOptions) { - this.#root = resolve(workspaceRoot); - this.#lease = acquireOperationalStateDatabase(this.#root); - this.#ready = importLegacyMessageReceipts(this.#root, this.#lease, options); + constructor(workspaceRoot: string) { + this.#lease = acquireOperationalStateDatabase(resolve(workspaceRoot)); } ready(): Promise { - return this.#ready; + return Promise.resolve(); } async beginHostEpoch(hostEpoch: string): Promise { validateHostEpoch(hostEpoch); - await this.#ready; this.#lease.transaction('write', () => { this.#lease.database .prepare('INSERT OR IGNORE INTO core_message_host_epochs(host_epoch) VALUES (?)') @@ -105,7 +85,6 @@ class SqliteMessageReceiptStore implements ClosableMessageReceiptStore { operationId: string, ): Promise { validateIdentity(hostEpoch, operation, sessionId, operationId); - await this.#ready; const row = this.#lease.database .prepare(` SELECT payload_json, result_json @@ -134,7 +113,6 @@ class SqliteMessageReceiptStore implements ClosableMessageReceiptStore { ): Promise { validateIdentity(hostEpoch, operation, sessionId, operationId); const stored = normalizeReceipt(hostEpoch, operation, sessionId, operationId, receipt); - await this.#ready; return this.#lease.transaction('write', () => { this.#lease.database .prepare('INSERT OR IGNORE INTO core_message_host_epochs(host_epoch) VALUES (?)') @@ -175,264 +153,6 @@ class SqliteMessageReceiptStore implements ClosableMessageReceiptStore { } } -class FileMessageReceiptStore implements MessageReceiptStore { - readonly #durabilityRoot: string; - readonly #epochsRoot: string; - readonly #writeQueues = new Map>(); - - constructor(workspaceRoot: string) { - this.#durabilityRoot = resolve(workspaceRoot); - this.#epochsRoot = join(this.#durabilityRoot, 'message-receipts'); - } - - async beginHostEpoch(hostEpoch: string): Promise { - assertSafeId(hostEpoch, 'Invalid Host Epoch'); - await mkdir(join(this.#epochsRoot, hostEpoch), { recursive: true }); - const entries = await readdir(this.#epochsRoot, { withFileTypes: true }); - let removed = false; - for (const entry of entries) { - if (!entry.isDirectory() || !SAFE_ID_PATTERN.test(entry.name)) { - throw new Error(`Invalid message receipt Epoch entry: ${entry.name}`); - } - if (entry.name === hostEpoch) continue; - await rm(join(this.#epochsRoot, entry.name), { recursive: true }); - removed = true; - } - if (removed) await syncDirectory(this.#epochsRoot); - } - - async read( - hostEpoch: string, - operation: MessageReceiptOperation, - sessionId: string, - operationId: string, - ): Promise { - validateIdentity(hostEpoch, operation, sessionId, operationId); - let raw: string; - try { - raw = await readFile(this.#receiptPath(hostEpoch, operation, sessionId, operationId), 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; - throw error; - } - if (Buffer.byteLength(raw, 'utf8') > RECEIPT_MAX_BYTES) { - throw new Error('Message operation receipt exceeds size limit'); - } - const stored = decodeStoredReceipt(JSON.parse(raw), { - hostEpoch, - operation, - sessionId, - operationId, - }); - return Object.freeze({ payload: stored.payload, result: stored.result }); - } - - async commit( - hostEpoch: string, - operation: MessageReceiptOperation, - sessionId: string, - operationId: string, - receipt: MessageOperationReceipt, - ): Promise { - validateIdentity(hostEpoch, operation, sessionId, operationId); - const stored: StoredMessageOperationReceipt = { - schemaVersion: RECEIPT_SCHEMA_VERSION, - hostEpoch, - operation, - sessionId, - operationId, - payload: receipt.payload, - result: receipt.result, - }; - const encoded = JSON.stringify(stored); - const snapshot = decodeStoredReceipt(JSON.parse(encoded), { - hostEpoch, - operation, - sessionId, - operationId, - }); - if (!isDeepStrictEqual(snapshot, stored)) { - throw new Error('Message operation receipt is not exactly JSON representable'); - } - const serialized = `${encoded}\n`; - if (Buffer.byteLength(serialized, 'utf8') > RECEIPT_MAX_BYTES) { - throw new Error('Message operation receipt exceeds size limit'); - } - const key = `${hostEpoch}:${operation}:${sessionId}:${operationId}`; - let committed: MessageOperationReceipt | undefined; - await chainWrite(this.#writeQueues, key, async () => { - const path = this.#receiptPath(hostEpoch, operation, sessionId, operationId); - const created = await writeExclusiveDurable(path, serialized, this.#durabilityRoot); - if (!created) { - const existing = await this.read(hostEpoch, operation, sessionId, operationId); - if (!existing) throw new Error('Message operation receipt disappeared'); - if ( - !isDeepStrictEqual(existing, { - payload: snapshot.payload, - result: snapshot.result, - }) - ) { - throw new Error('Message operation receipt identity conflict'); - } - committed = existing; - return; - } - committed = Object.freeze({ payload: snapshot.payload, result: snapshot.result }); - }); - if (!committed) throw new Error('Message operation receipt commit produced no result'); - return committed; - } - - #receiptPath( - hostEpoch: string, - operation: MessageReceiptOperation, - sessionId: string, - operationId: string, - ): string { - return join(this.#epochsRoot, hostEpoch, operation, sessionId, `${operationId}.json`); - } -} - -interface LegacyMessageReceiptSnapshot { - readonly epochs: readonly string[]; - readonly records: readonly StoredMessageOperationReceipt[]; -} - -async function importLegacyMessageReceipts( - root: string, - lease: OperationalStateDatabaseLease, - options: SqliteMessageReceiptStoreOptions, -): Promise { - const snapshot = await readLegacyMessageReceiptSnapshot(root); - const fingerprint = createHash('sha256').update(JSON.stringify(snapshot)).digest('hex'); - completeOperationalStoreCutover(lease, { - storeName: 'message_receipts', - sourcePath: join(root, 'message-receipts'), - sourceFingerprint: `sha256:${fingerprint}`, - failpoint: options.failpoint, - importAndValidate: (db) => { - for (const epoch of snapshot.epochs) { - db.prepare('INSERT OR IGNORE INTO core_message_host_epochs(host_epoch) VALUES (?)').run( - epoch, - ); - } - for (const record of snapshot.records) { - insertOrValidateReceipt(db, record); - } - return { - host_epochs: snapshot.epochs.length, - message_receipts: snapshot.records.length, - }; - }, - }); -} - -async function readLegacyMessageReceiptSnapshot( - root: string, -): Promise { - const epochsRoot = join(root, 'message-receipts'); - let epochEntries; - try { - epochEntries = await readdir(epochsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return { epochs: [], records: [] }; - } - throw error; - } - const legacy = new FileMessageReceiptStore(root); - const epochs: string[] = []; - const records: StoredMessageOperationReceipt[] = []; - for (const epochEntry of epochEntries.sort((a, b) => a.name.localeCompare(b.name))) { - validateHostEpochEntry(epochEntry.name, epochEntry.isDirectory()); - epochs.push(epochEntry.name); - const epochRoot = join(epochsRoot, epochEntry.name); - for (const operation of ['interrupt', 'retract', 'submit'] as const) { - const operationRoot = join(epochRoot, operation); - let sessionEntries; - try { - sessionEntries = await readdir(operationRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue; - throw error; - } - for (const sessionEntry of sessionEntries.sort((a, b) => a.name.localeCompare(b.name))) { - if (!sessionEntry.isDirectory() || !SAFE_ID_PATTERN.test(sessionEntry.name)) { - throw new Error(`Invalid message receipt Session entry: ${sessionEntry.name}`); - } - const sessionRoot = join(operationRoot, sessionEntry.name); - const receiptEntries = await readdir(sessionRoot, { withFileTypes: true }); - for (const receiptEntry of receiptEntries.sort((a, b) => a.name.localeCompare(b.name))) { - if (!receiptEntry.isFile() || !receiptEntry.name.endsWith('.json')) { - throw new Error(`Invalid message receipt entry: ${receiptEntry.name}`); - } - const operationId = receiptEntry.name.slice(0, -'.json'.length); - validateIdentity(epochEntry.name, operation, sessionEntry.name, operationId); - const receipt = await legacy.read( - epochEntry.name, - operation, - sessionEntry.name, - operationId, - ); - if (!receipt) throw new Error('Legacy message operation receipt disappeared'); - records.push({ - schemaVersion: RECEIPT_SCHEMA_VERSION, - hostEpoch: epochEntry.name, - operation, - sessionId: sessionEntry.name, - operationId, - payload: receipt.payload, - result: receipt.result, - }); - } - } - } - const unexpected = (await readdir(epochRoot, { withFileTypes: true })).filter( - (entry) => - !entry.isDirectory() || - (entry.name !== 'submit' && entry.name !== 'retract' && entry.name !== 'interrupt'), - ); - if (unexpected.length > 0) { - throw new Error(`Invalid message receipt Operation entry: ${unexpected[0]!.name}`); - } - } - return { epochs, records }; -} - -function insertOrValidateReceipt(db: DatabaseSync, record: StoredMessageOperationReceipt): void { - db.prepare('INSERT OR IGNORE INTO core_message_host_epochs(host_epoch) VALUES (?)').run( - record.hostEpoch, - ); - const inserted = db - .prepare(` - INSERT OR IGNORE INTO core_message_receipts( - host_epoch, operation, session_id, operation_id, payload_json, result_json - ) VALUES (?, ?, ?, ?, ?, ?) - `) - .run( - record.hostEpoch, - record.operation, - record.sessionId, - record.operationId, - JSON.stringify(record.payload), - JSON.stringify(record.result), - ); - if (inserted.changes !== 0) return; - const existing = readSqliteReceipt( - db, - record.hostEpoch, - record.operation, - record.sessionId, - record.operationId, - ); - if ( - !existing || - !isDeepStrictEqual(existing, { payload: record.payload, result: record.result }) - ) { - throw new Error('Message receipt cutover conflict'); - } -} - function readSqliteReceipt( db: DatabaseSync, hostEpoch: string, @@ -491,12 +211,6 @@ function validateHostEpoch(hostEpoch: string): void { assertSafeId(hostEpoch, 'Invalid Host Epoch'); } -function validateHostEpochEntry(name: string, directory: boolean): void { - if (!directory || !SAFE_ID_PATTERN.test(name)) { - throw new Error(`Invalid message receipt Epoch entry: ${name}`); - } -} - function validateIdentity( hostEpoch: string, operation: MessageReceiptOperation, @@ -543,32 +257,6 @@ function decodeStoredReceipt( return record as unknown as StoredMessageOperationReceipt; } -async function writeExclusiveDurable( - path: string, - content: string, - durabilityRoot: string, -): Promise { - await mkdir(dirname(path), { recursive: true }); - const tempPath = `${path}.${process.pid}.${randomUUID()}.tmp`; - const handle = await open(tempPath, 'wx', 0o600); - try { - await handle.writeFile(content, 'utf8'); - await handle.sync(); - } finally { - await handle.close(); - } - try { - await link(tempPath, path); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'EEXIST') return false; - throw error; - } finally { - await unlink(tempPath).catch(() => undefined); - } - await syncDirectoryChain(dirname(path), durabilityRoot); - return true; -} - function assertSafeId(value: string, message: string): void { if (!SAFE_ID_PATTERN.test(value)) throw new Error(message); } diff --git a/packages/storage/src/operational-state-backup.ts b/packages/storage/src/operational-state-backup.ts index 18437625f6..52652be8ba 100644 --- a/packages/storage/src/operational-state-backup.ts +++ b/packages/storage/src/operational-state-backup.ts @@ -2,30 +2,18 @@ import { createHash, randomUUID } from 'node:crypto'; import { createReadStream } from 'node:fs'; import { copyFile, - chmod, lstat, mkdir, readFile, readdir, - realpath, rename, rm, stat, writeFile, } from 'node:fs/promises'; -import { createRequire } from 'node:module'; -import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'; -import type { DatabaseSync } from 'node:sqlite'; -import type { ArtifactRecord } from '@maka/core/artifacts'; -import { decodeLegacyAutomationFile, LEGACY_AUTOMATION_FILE } from './automation-authority.js'; -import { decodeArtifactMetadata } from './artifact-metadata-codec.js'; -import { - ARTIFACT_PUBLICATION_STAGING_PATTERN, - ARTIFACT_PURGE_INTENT_FILE, - isCanonicalArtifactRecoveryTempName, -} from './artifact-storage-layout.js'; +import { dirname, relative, resolve } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; import { withArtifactWriterLock } from './artifact-writer-lock.js'; -import { decodeStoredMessageForRecovery } from './execution-record-codec.js'; import { acquireOperationalStateDatabase, OPERATIONAL_STATE_DATABASE_NAME, @@ -33,42 +21,22 @@ import { } from './operational-state-store.js'; import { SQLITE_ARTIFACT_SCHEMA_VERSION } from './sqlite-artifact-schema.js'; import { SQLITE_AUTOMATION_SCHEMA_VERSION } from './sqlite-automation-schema.js'; -import { createSqliteArtifactMetadataRepository } from './sqlite-artifact-metadata.js'; import { SQLITE_CORE_EXECUTION_SCHEMA_VERSION } from './sqlite-core-execution-schema.js'; import { SQLITE_RUNTIME_SCHEMA_VERSION } from './sqlite-runtime-schema.js'; -import { decodeSessionHeader } from './session-store.js'; import { SQLITE_SESSION_METADATA_SCHEMA_VERSION } from './sqlite-session-metadata-schema.js'; -import { decodeSessionTranscriptMarker, isSessionTranscriptMarker } from './session-transcript.js'; import { SQLITE_USAGE_SCHEMA_VERSION } from './sqlite-usage-schema.js'; import { SQLITE_WORKFLOW_SCHEMA_VERSION } from './sqlite-workflow-schema.js'; -import { syncDirectory, syncDirectoryChain, syncFile } from './stable-storage.js'; export const OPERATIONAL_BACKUP_FORMAT = 'maka-operational-backup'; -export const OPERATIONAL_BACKUP_SCHEMA_VERSION = 2 as const; +export const OPERATIONAL_BACKUP_SCHEMA_VERSION = 3 as const; export const OPERATIONAL_BACKUP_MANIFEST_FILE = 'operational-backup.json'; -const MAX_MANIFEST_BYTES = 64 * 1024 * 1024; -const LEGACY_OPERATIONAL_BACKUP_SCHEMA_VERSION = 1 as const; -const require = createRequire(import.meta.url); -const supportedSchemaVersions = new Map([ - ['runtime', SQLITE_RUNTIME_SCHEMA_VERSION], - ['session_metadata', SQLITE_SESSION_METADATA_SCHEMA_VERSION], - ['core_execution', SQLITE_CORE_EXECUTION_SCHEMA_VERSION], - ['workflow', SQLITE_WORKFLOW_SCHEMA_VERSION], - ['usage', SQLITE_USAGE_SCHEMA_VERSION], - ['artifact', SQLITE_ARTIFACT_SCHEMA_VERSION], - ['automation', SQLITE_AUTOMATION_SCHEMA_VERSION], - ['operational', OPERATIONAL_STATE_SCHEMA_VERSION], -]); - export type OperationalBackupErrorCode = | 'invalid_root' | 'overlapping_roots' - | 'destination_exists' - | 'source_changed' - | 'invalid_manifest' - | 'corrupt_backup' - | 'unsupported_schema'; + | 'destination_not_empty' + | 'unsupported_schema' + | 'corrupt_backup'; export class OperationalBackupError extends Error { constructor( @@ -81,259 +49,117 @@ export class OperationalBackupError extends Error { } } -export type OperationalBackupFailpoint = - | 'after_database_snapshot' - | 'after_payload_copy' - | 'after_manifest_write' - | 'after_restore_copy'; - export interface OperationalBackupFile { readonly path: string; - readonly sizeBytes: number; + readonly size: number; readonly sha256: `sha256:${string}`; } -export interface OperationalBackupTranscript extends OperationalBackupFile { - readonly sessionId: string; -} - -export interface OperationalBackupArtifact extends OperationalBackupFile { - readonly record: ArtifactRecord; -} - -interface OperationalBackupManifestBase { +export interface OperationalBackupManifest { readonly format: typeof OPERATIONAL_BACKUP_FORMAT; - readonly createdAt: number; - readonly database: OperationalBackupFile; - readonly transcripts: readonly OperationalBackupTranscript[]; - readonly artifacts: readonly OperationalBackupArtifact[]; -} - -export interface LegacyOperationalBackupManifest extends OperationalBackupManifestBase { - readonly schemaVersion: typeof LEGACY_OPERATIONAL_BACKUP_SCHEMA_VERSION; -} - -export interface CurrentOperationalBackupManifest extends OperationalBackupManifestBase { readonly schemaVersion: typeof OPERATIONAL_BACKUP_SCHEMA_VERSION; - readonly legacyAutomation: OperationalBackupFile | null; + readonly createdAt: number; + readonly files: readonly OperationalBackupFile[]; } -export type OperationalBackupManifest = - | LegacyOperationalBackupManifest - | CurrentOperationalBackupManifest; - export interface CreateOperationalBackupInput { readonly stateRoot: string; readonly destinationRoot: string; readonly now?: () => number; - readonly failpoint?: (point: OperationalBackupFailpoint) => void | Promise; } export interface RestoreOperationalBackupInput { readonly backupRoot: string; readonly destinationRoot: string; - readonly failpoint?: (point: OperationalBackupFailpoint) => void | Promise; -} - -interface SourceFileSnapshot extends OperationalBackupFile { - readonly sourcePath: string; - readonly dev: bigint; - readonly ino: bigint; - readonly mtimeNs: bigint; - readonly ctimeNs: bigint; -} - -interface SourceTranscriptSnapshot extends SourceFileSnapshot { - readonly sessionId: string; -} - -type OptionalSourceFileSnapshot = - | { readonly kind: 'present'; readonly snapshot: SourceFileSnapshot } - | { readonly kind: 'absent'; readonly sourcePath: string; readonly path: string }; - -interface ValidatedDatabaseSnapshot { - readonly artifacts: ArtifactRecord[]; - readonly sessionIds: string[]; } export async function createOperationalStateBackup( input: CreateOperationalBackupInput, ): Promise { - const stateRoot = await canonicalizeExistingDirectory(input.stateRoot, 'state'); - const destinationRoot = await canonicalizeMissingPath(input.destinationRoot, 'destination'); - assertRootsDoNotOverlap(stateRoot, destinationRoot); - await assertPathMissing(destinationRoot, 'Operational backup destination'); - - return withArtifactWriterLock(stateRoot, async (lockedRoot) => { - if (lockedRoot !== stateRoot) { - throw new OperationalBackupError('invalid_root', 'Operational backup state root changed'); - } - const stagingRoot = `${destinationRoot}.staging-${randomUUID()}`; - const metadata = createSqliteArtifactMetadataRepository(stateRoot); - const database = acquireOperationalStateDatabase(stateRoot); + const stateRoot = resolve(input.stateRoot); + const destinationRoot = resolve(input.destinationRoot); + assertSeparateRoots(stateRoot, destinationRoot); + await assertMissing(destinationRoot, 'backup destination'); + return withArtifactWriterLock(stateRoot, async (canonicalStateRoot) => { + assertSeparateRoots(canonicalStateRoot, destinationRoot); + const stagingRoot = `${destinationRoot}.${process.pid}.${randomUUID()}.tmp`; try { - await assertPathMissing(stagingRoot, 'Operational backup staging destination'); - await metadata.ready(); - const artifactRecords = metadata.readAll(); - const transcriptSnapshots = await snapshotSessionTranscripts(stateRoot); - const artifactSnapshots = await snapshotArtifactPayloads(stateRoot, artifactRecords); - const legacyAutomationSnapshot = await snapshotOptionalSourceFile( - stateRoot, - LEGACY_AUTOMATION_FILE, - ); - if (legacyAutomationSnapshot.kind === 'present') { - await validateLegacyAutomationSourceSnapshot(legacyAutomationSnapshot.snapshot); + await mkdir(stagingRoot, { recursive: true }); + const database = acquireOperationalStateDatabase(canonicalStateRoot); + try { + await database.backup(resolve(stagingRoot, OPERATIONAL_STATE_DATABASE_NAME)); + } finally { + database.close(); + } + const artifactRoot = resolve(canonicalStateRoot, 'artifacts'); + if (await pathExists(artifactRoot)) { + await copyRegularTree(artifactRoot, resolve(stagingRoot, 'artifacts'), artifactRoot); } - - await mkdir(stagingRoot, { recursive: true, mode: 0o700 }); - const databasePath = resolve(stagingRoot, OPERATIONAL_STATE_DATABASE_NAME); - await database.backup(databasePath); - normalizeStandaloneSqliteSnapshot(databasePath); - await chmod(databasePath, 0o600); - await syncFile(databasePath); - await input.failpoint?.('after_database_snapshot'); - - const validated = await validateDatabaseSnapshot( - databasePath, - OPERATIONAL_BACKUP_SCHEMA_VERSION, - ); - assertSameArtifactRecords( - validated.artifacts, - artifactRecords, - 'source changed during backup', - 'source_changed', - ); - assertSameStringSet( - validated.sessionIds, - transcriptSnapshots.map((snapshot) => snapshot.sessionId), - 'SQLite sessions do not match transcript files', - 'source_changed', - ); - - const transcripts = await copyTranscriptSnapshots(stagingRoot, transcriptSnapshots); - const artifacts = await copyArtifactSnapshots( - stagingRoot, - artifactSnapshots, - artifactRecords, - ); - const legacyAutomation = await copyOptionalSourceSnapshot( - stagingRoot, - legacyAutomationSnapshot, - ); - assertSameArtifactRecords( - metadata.readAll(), - artifactRecords, - 'source changed during backup', - 'source_changed', - ); - await input.failpoint?.('after_payload_copy'); - const createdAt = (input.now ?? Date.now)(); if (!Number.isSafeInteger(createdAt) || createdAt < 0) { - throw new OperationalBackupError('invalid_manifest', 'Operational backup time is invalid'); + throw new OperationalBackupError('corrupt_backup', 'Backup creation time is invalid'); } - const manifest: CurrentOperationalBackupManifest = { + const manifest: OperationalBackupManifest = { format: OPERATIONAL_BACKUP_FORMAT, schemaVersion: OPERATIONAL_BACKUP_SCHEMA_VERSION, createdAt, - database: await describeBackupFile(databasePath, OPERATIONAL_STATE_DATABASE_NAME), - transcripts, - artifacts, - legacyAutomation, + files: await inventory(stagingRoot), }; - const manifestPath = resolve(stagingRoot, OPERATIONAL_BACKUP_MANIFEST_FILE); - await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { - encoding: 'utf8', - flag: 'wx', - mode: 0o600, - }); - await syncFile(manifestPath); - await input.failpoint?.('after_manifest_write'); - - await validateOperationalStateBackup(stagingRoot); - await syncDirectoryChain(stagingRoot, stagingRoot); + await writeFile( + resolve(stagingRoot, OPERATIONAL_BACKUP_MANIFEST_FILE), + `${JSON.stringify(manifest, null, 2)}\n`, + 'utf8', + ); + await mkdir(dirname(destinationRoot), { recursive: true }); await rename(stagingRoot, destinationRoot); - await syncDirectory(dirname(destinationRoot)); return manifest; } catch (error) { await rm(stagingRoot, { recursive: true, force: true }).catch(() => {}); throw error; - } finally { - database.close(); - metadata.close(); } }); } export async function validateOperationalStateBackup( - backupRootInput: string, + backupRoot: string, ): Promise { - const backupRoot = await canonicalizeExistingDirectory(backupRootInput, 'backup'); - const manifestPath = resolve(backupRoot, OPERATIONAL_BACKUP_MANIFEST_FILE); - const manifest = await readOperationalBackupManifest(manifestPath); - await assertBackupTreeExactlyMatchesManifest(backupRoot, manifest); - await validateManifestFiles(backupRoot, manifest); - const validated = await validateDatabaseSnapshot( - resolve(backupRoot, manifest.database.path), - manifest.schemaVersion, - ); - assertSameArtifactRecords( - validated.artifacts, - manifest.artifacts.map((artifact) => artifact.record), - 'Artifact manifest does not match runtime.sqlite', - 'corrupt_backup', - ); - assertSameStringSet( - validated.sessionIds, - manifest.transcripts.map((transcript) => transcript.sessionId), - 'Transcript manifest does not match runtime.sqlite', - 'corrupt_backup', - ); + const root = resolve(backupRoot); + let parsed: unknown; + try { + parsed = JSON.parse(await readFile(resolve(root, OPERATIONAL_BACKUP_MANIFEST_FILE), 'utf8')); + } catch (error) { + throw new OperationalBackupError('corrupt_backup', 'Backup manifest is missing or invalid', { + cause: error, + }); + } + const manifest = decodeManifest(parsed); + const actual = await inventory(root); + if (JSON.stringify(actual) !== JSON.stringify(manifest.files)) { + throw new OperationalBackupError('corrupt_backup', 'Backup file inventory does not match'); + } + validateSqlite(resolve(root, OPERATIONAL_STATE_DATABASE_NAME)); return manifest; } export async function restoreOperationalStateBackup( input: RestoreOperationalBackupInput, ): Promise { - const backupRoot = await canonicalizeExistingDirectory(input.backupRoot, 'backup'); - const destinationRoot = await canonicalizeMissingPath(input.destinationRoot, 'destination'); - assertRootsDoNotOverlap(backupRoot, destinationRoot); - await assertPathMissing(destinationRoot, 'Operational restore destination'); + const backupRoot = resolve(input.backupRoot); + const destinationRoot = resolve(input.destinationRoot); + assertSeparateRoots(backupRoot, destinationRoot); + await assertMissing(destinationRoot, 'restore destination'); const manifest = await validateOperationalStateBackup(backupRoot); - const stagingRoot = `${destinationRoot}.staging-${randomUUID()}`; + const stagingRoot = `${destinationRoot}.${process.pid}.${randomUUID()}.tmp`; try { - await assertPathMissing(stagingRoot, 'Operational restore staging destination'); - await mkdir(stagingRoot, { recursive: true, mode: 0o700 }); - for (const file of manifestFiles(manifest)) { - const sourcePath = resolveBackupPath(backupRoot, file.path); - const destinationPath = resolveBackupPath(stagingRoot, file.path); - await mkdir(dirname(destinationPath), { recursive: true, mode: 0o700 }); - await copyFile(sourcePath, destinationPath); - await chmod(destinationPath, 0o600); - await syncFile(destinationPath); - await syncDirectoryChain(dirname(destinationPath), stagingRoot); - } - await input.failpoint?.('after_restore_copy'); - await validateManifestFiles(stagingRoot, manifest); - const validated = await validateDatabaseSnapshot( - resolve(stagingRoot, manifest.database.path), - manifest.schemaVersion, - ); - assertSameArtifactRecords( - validated.artifacts, - manifest.artifacts.map((artifact) => artifact.record), - 'Restored Artifact rows do not match the backup manifest', - 'corrupt_backup', - ); - assertSameStringSet( - validated.sessionIds, - manifest.transcripts.map((transcript) => transcript.sessionId), - 'Restored sessions do not match the backup manifest', - 'corrupt_backup', - ); - await syncDirectoryChain(stagingRoot, stagingRoot); + await mkdir(stagingRoot, { recursive: true }); + for (const file of manifest.files) { + const source = resolveInside(backupRoot, file.path); + const destination = resolveInside(stagingRoot, file.path); + await mkdir(dirname(destination), { recursive: true }); + await copyFile(source, destination); + } + await mkdir(dirname(destinationRoot), { recursive: true }); await rename(stagingRoot, destinationRoot); - await syncDirectory(dirname(destinationRoot)); return manifest; } catch (error) { await rm(stagingRoot, { recursive: true, force: true }).catch(() => {}); @@ -341,985 +167,190 @@ export async function restoreOperationalStateBackup( } } -async function snapshotSessionTranscripts(stateRoot: string): Promise { - const sessionsRoot = resolve(stateRoot, 'sessions'); - let entries; - try { - entries = await readdir(sessionsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; +function decodeManifest(value: unknown): OperationalBackupManifest { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new OperationalBackupError('corrupt_backup', 'Backup manifest must be an object'); } - await assertRegularDirectory(sessionsRoot, stateRoot, 'sessions'); - const snapshots: SourceTranscriptSnapshot[] = []; - for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { - if (!isSafeEntityId(entry.name) || !entry.isDirectory() || entry.isSymbolicLink()) { - throw new OperationalBackupError( - 'invalid_root', - `Operational backup found an invalid sessions entry: sessions/${entry.name}`, - ); - } - const sessionRoot = resolve(sessionsRoot, entry.name); - await assertRegularDirectory(sessionRoot, stateRoot, `sessions/${entry.name}`); - const relativePath = `sessions/${entry.name}/session.jsonl`; - const sourcePath = resolve(sessionRoot, 'session.jsonl'); - const snapshot = { - sessionId: entry.name, - ...(await snapshotSourceFile(sourcePath, stateRoot, relativePath)), - }; - await validateTranscriptFile(sourcePath, entry.name); - snapshots.push(snapshot); + const record = value as Record; + if (record.format !== OPERATIONAL_BACKUP_FORMAT) { + throw new OperationalBackupError('corrupt_backup', 'Backup format is invalid'); } - return snapshots; -} - -async function snapshotOptionalSourceFile( - stateRoot: string, - relativePath: string, -): Promise { - const sourcePath = resolve(stateRoot, relativePath); - try { - await lstat(sourcePath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return { kind: 'absent', sourcePath, path: relativePath }; - } - throw error; + if (record.schemaVersion !== OPERATIONAL_BACKUP_SCHEMA_VERSION) { + throw new OperationalBackupError('unsupported_schema', 'Backup schema is unsupported'); } - return { - kind: 'present', - snapshot: await snapshotSourceFile(sourcePath, stateRoot, relativePath), - }; -} - -async function snapshotArtifactPayloads( - stateRoot: string, - records: readonly ArtifactRecord[], -): Promise { - const artifactRoot = resolve(stateRoot, 'artifacts'); - const expectedPaths = new Set(records.map((record) => record.relativePath)); - let rootEntries; - try { - rootEntries = await readdir(artifactRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT' && records.length === 0) return []; - throw new OperationalBackupError( - 'invalid_root', - 'Operational backup Artifact root is missing', - { - cause: error, - }, - ); + if ( + !Number.isSafeInteger(record.createdAt) || + (record.createdAt as number) < 0 || + !Array.isArray(record.files) + ) { + throw new OperationalBackupError('corrupt_backup', 'Backup manifest fields are invalid'); } - await assertRegularDirectory(artifactRoot, stateRoot, 'artifacts'); - const foundPaths = new Set(); - for (const entry of rootEntries.sort((left, right) => left.name.localeCompare(right.name))) { - const relativeRootPath = `artifacts/${entry.name}`; - if (entry.isSymbolicLink()) throw invalidSourceEntry(relativeRootPath); - if (entry.name === 'metadata.jsonl') { - await assertRegularFile(resolve(artifactRoot, entry.name), stateRoot, relativeRootPath); - continue; + const files = record.files.map((value): OperationalBackupFile => { + if (!value || typeof value !== 'object' || Array.isArray(value)) { + throw new OperationalBackupError('corrupt_backup', 'Backup file entry is invalid'); } + const file = value as Record; if ( - entry.name === ARTIFACT_PURGE_INTENT_FILE || - isCanonicalArtifactRecoveryTempName(entry.name) + typeof file.path !== 'string' || + file.path === OPERATIONAL_BACKUP_MANIFEST_FILE || + !Number.isSafeInteger(file.size) || + (file.size as number) < 0 || + typeof file.sha256 !== 'string' || + !/^sha256:[a-f0-9]{64}$/.test(file.sha256) ) { - throw new OperationalBackupError( - 'source_changed', - `Artifact authority recovery is required before backup: ${relativeRootPath}`, - ); - } - if (!entry.isDirectory() || !isSafeEntityId(entry.name)) { - throw invalidSourceEntry(relativeRootPath); - } - const sessionRoot = resolve(artifactRoot, entry.name); - await assertRegularDirectory(sessionRoot, stateRoot, relativeRootPath); - for (const child of await readdir(sessionRoot, { withFileTypes: true })) { - const relativePath = `${entry.name}/${child.name}`; - const backupPath = `artifacts/${relativePath}`; - if (child.isSymbolicLink()) throw invalidSourceEntry(backupPath); - if (ARTIFACT_PUBLICATION_STAGING_PATTERN.test(child.name)) { - throw new OperationalBackupError( - 'source_changed', - `Artifact authority recovery is required before backup: ${backupPath}`, - ); - } - if (!child.isFile() || !expectedPaths.has(relativePath)) { - throw invalidSourceEntry(backupPath); - } - await assertRegularFile(resolve(sessionRoot, child.name), stateRoot, backupPath); - foundPaths.add(relativePath); - } - } - for (const record of records) { - if (!foundPaths.has(record.relativePath)) { - throw new OperationalBackupError( - 'invalid_root', - `Artifact payload is missing: artifacts/${record.relativePath}`, - ); - } - } - const snapshots: SourceFileSnapshot[] = []; - for (const record of records) { - const snapshot = await snapshotSourceFile( - resolve(artifactRoot, record.relativePath), - stateRoot, - `artifacts/${record.relativePath}`, - ); - if (snapshot.sizeBytes !== record.sizeBytes) { - throw new OperationalBackupError( - 'corrupt_backup', - `Artifact payload size does not match metadata: artifacts/${record.relativePath}`, - ); + throw new OperationalBackupError('corrupt_backup', 'Backup file entry is invalid'); } - snapshots.push(snapshot); - } - return snapshots; -} - -async function copyTranscriptSnapshots( - stagingRoot: string, - snapshots: readonly SourceTranscriptSnapshot[], -): Promise { - const copied: OperationalBackupTranscript[] = []; - for (const snapshot of snapshots) { - await copySourceSnapshot(stagingRoot, snapshot); - copied.push({ - sessionId: snapshot.sessionId, - path: snapshot.path, - sizeBytes: snapshot.sizeBytes, - sha256: snapshot.sha256, - }); - } - return copied; -} - -async function copyArtifactSnapshots( - stagingRoot: string, - snapshots: readonly SourceFileSnapshot[], - records: readonly ArtifactRecord[], -): Promise { - const copied: OperationalBackupArtifact[] = []; - for (const [index, snapshot] of snapshots.entries()) { - const record = records[index]; - if (!record || snapshot.path !== `artifacts/${record.relativePath}`) { - throw new OperationalBackupError('source_changed', 'Artifact snapshot order changed'); - } - await copySourceSnapshot(stagingRoot, snapshot); - copied.push({ - record, - path: snapshot.path, - sizeBytes: snapshot.sizeBytes, - sha256: snapshot.sha256, - }); - } - return copied; -} - -async function copyOptionalSourceSnapshot( - stagingRoot: string, - optional: OptionalSourceFileSnapshot, -): Promise { - if (optional.kind === 'absent') { - await assertOptionalSourceStillAbsent(optional); - return null; - } - const { snapshot } = optional; - await copySourceSnapshot(stagingRoot, snapshot); - return { - path: snapshot.path, - sizeBytes: snapshot.sizeBytes, - sha256: snapshot.sha256, - }; -} - -async function assertOptionalSourceStillAbsent( - snapshot: Extract, -): Promise { - try { - await lstat(snapshot.sourcePath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; - throw new OperationalBackupError( - 'source_changed', - `Operational backup source changed during copy: ${snapshot.path}`, - { cause: error }, - ); - } - throw new OperationalBackupError( - 'source_changed', - `Operational backup source changed during copy: ${snapshot.path}`, - ); -} - -async function copySourceSnapshot( - stagingRoot: string, - snapshot: SourceFileSnapshot, -): Promise { - const destinationPath = resolveBackupPath(stagingRoot, snapshot.path); - await mkdir(dirname(destinationPath), { recursive: true, mode: 0o700 }); - await copyFile(snapshot.sourcePath, destinationPath); - await chmod(destinationPath, 0o600); - const current = await snapshotSourceFile( - snapshot.sourcePath, - dirname(snapshot.sourcePath), - basename(snapshot.sourcePath), - ); - if ( - current.dev !== snapshot.dev || - current.ino !== snapshot.ino || - current.mtimeNs !== snapshot.mtimeNs || - current.ctimeNs !== snapshot.ctimeNs || - current.sizeBytes !== snapshot.sizeBytes || - current.sha256 !== snapshot.sha256 - ) { - throw new OperationalBackupError( - 'source_changed', - `Operational backup source changed during copy: ${snapshot.path}`, - ); - } - const destination = await describeBackupFile(destinationPath, snapshot.path); - if (destination.sizeBytes !== snapshot.sizeBytes || destination.sha256 !== snapshot.sha256) { - throw new OperationalBackupError( - 'corrupt_backup', - `Operational backup copy failed validation: ${snapshot.path}`, - ); - } - await syncFile(destinationPath); - await syncDirectoryChain(dirname(destinationPath), stagingRoot); -} - -async function snapshotSourceFile( - path: string, - root: string, - relativePath: string, -): Promise { - await assertRegularFile(path, root, relativePath); - const metadata = await lstat(path, { bigint: true }); - const descriptor = await describeBackupFile(path, relativePath); - return { - ...descriptor, - sourcePath: path, - dev: metadata.dev, - ino: metadata.ino, - mtimeNs: metadata.mtimeNs, - ctimeNs: metadata.ctimeNs, - }; -} - -async function describeBackupFile( - path: string, - relativePath: string, -): Promise { - const metadata = await stat(path); - if (!metadata.isFile() || !Number.isSafeInteger(metadata.size) || metadata.size < 0) { - throw new OperationalBackupError('corrupt_backup', `Backup file is invalid: ${relativePath}`); + resolveInside('/backup-root', file.path); + return file as unknown as OperationalBackupFile; + }); + if (!files.some((file) => file.path === OPERATIONAL_STATE_DATABASE_NAME)) { + throw new OperationalBackupError('corrupt_backup', 'Backup runtime.sqlite is missing'); } return { - path: relativePath, - sizeBytes: metadata.size, - sha256: await hashFile(path), + format: OPERATIONAL_BACKUP_FORMAT, + schemaVersion: OPERATIONAL_BACKUP_SCHEMA_VERSION, + createdAt: record.createdAt as number, + files, }; } -async function hashFile(path: string): Promise<`sha256:${string}`> { - const hash = createHash('sha256'); - for await (const chunk of createReadStream(path)) hash.update(chunk as Buffer); - return `sha256:${hash.digest('hex')}`; -} - -async function validateDatabaseSnapshot( - path: string, - backupSchemaVersion: OperationalBackupManifest['schemaVersion'], -): Promise { - const metadata = await lstat(path).catch((error: unknown) => { - throw new OperationalBackupError('corrupt_backup', 'Backup runtime.sqlite is missing', { - cause: error, - }); - }); - if (!metadata.isFile() || metadata.isSymbolicLink()) { - throw new OperationalBackupError('corrupt_backup', 'Backup runtime.sqlite is not a file'); - } - const Database = loadDatabaseSync(); - let database: DatabaseSync; - try { - database = new Database(path, { readOnly: true }); - } catch (error) { - throw new OperationalBackupError('corrupt_backup', 'Unable to open backup runtime.sqlite', { - cause: error, - }); - } - try { - database.exec('PRAGMA query_only = ON; PRAGMA foreign_keys = ON; BEGIN'); - const integrity = database.prepare('PRAGMA integrity_check').all() as Array<{ - integrity_check?: unknown; - }>; - if (integrity.length !== 1 || integrity[0]?.integrity_check !== 'ok') { - throw new OperationalBackupError('corrupt_backup', 'Backup SQLite integrity check failed'); - } - const foreignKeys = database.prepare('PRAGMA foreign_key_check').all(); - if (foreignKeys.length > 0) { - throw new OperationalBackupError('corrupt_backup', 'Backup SQLite foreign keys are invalid'); - } - validateOperationalSchemaVersions(database, backupSchemaVersion); - if (!sqliteTableExists(database, 'artifact_records')) { - throw new OperationalBackupError('corrupt_backup', 'Backup is missing artifact_records'); - } - if (!sqliteTableExists(database, 'session_metadata')) { - throw new OperationalBackupError('corrupt_backup', 'Backup is missing session_metadata'); - } - const artifactRows = database - .prepare(` - SELECT artifact_id, session_id, created_at, status, relative_path, record_json - FROM artifact_records - ORDER BY created_at, storage_key - `) - .all() as Array<{ - artifact_id?: unknown; - session_id?: unknown; - created_at?: unknown; - status?: unknown; - relative_path?: unknown; - record_json?: unknown; - }>; - if (artifactRows.some((row) => typeof row.record_json !== 'string')) { - throw new OperationalBackupError('corrupt_backup', 'Backup Artifact row is invalid'); - } - let artifacts: ArtifactRecord[]; - try { - const text = artifactRows.map((row) => row.record_json as string).join('\n'); - artifacts = decodeArtifactMetadata(text ? `${text}\n` : ''); - } catch (error) { - throw new OperationalBackupError('corrupt_backup', 'Backup Artifact metadata is invalid', { - cause: error, - }); - } - for (const [index, record] of artifacts.entries()) { - const row = artifactRows[index]; - if ( - row?.artifact_id !== record.id || - row.session_id !== record.sessionId || - row.created_at !== record.createdAt || - row.status !== record.status || - row.relative_path !== record.relativePath - ) { - throw new OperationalBackupError( - 'corrupt_backup', - `Backup Artifact indexes do not match record ${record.id}`, - ); - } - } - const sessionRows = database - .prepare('SELECT session_id FROM session_metadata ORDER BY session_id') - .all() as Array<{ session_id?: unknown }>; - if (sessionRows.some((row) => typeof row.session_id !== 'string')) { - throw new OperationalBackupError('corrupt_backup', 'Backup session metadata is invalid'); - } - return { - artifacts, - sessionIds: sessionRows.map((row) => row.session_id as string), - }; - } catch (error) { - if (error instanceof OperationalBackupError) throw error; - throw new OperationalBackupError('corrupt_backup', 'Unable to validate backup runtime.sqlite', { - cause: error, - }); - } finally { - try { - database.exec('ROLLBACK'); - } catch { - // Preserve the validation error. - } - database.close(); - } -} - -function normalizeStandaloneSqliteSnapshot(path: string): void { - const Database = loadDatabaseSync(); - const database = new Database(path); - try { - const row = database.prepare('PRAGMA journal_mode = DELETE').get() as - | { journal_mode?: unknown } - | undefined; - if (row?.journal_mode !== 'delete') { - throw new OperationalBackupError( - 'corrupt_backup', - 'Unable to make the SQLite backup self-contained', - ); - } - } finally { - database.close(); - } +async function inventory(root: string): Promise { + const result: OperationalBackupFile[] = []; + await walk(root, root, result); + return result.sort((left, right) => left.path.localeCompare(right.path)); } -function validateOperationalSchemaVersions( - database: DatabaseSync, - backupSchemaVersion: OperationalBackupManifest['schemaVersion'], -): void { - if (!sqliteTableExists(database, 'operational_schema_migrations')) { - throw new OperationalBackupError('corrupt_backup', 'Backup is missing schema migrations'); - } - const rows = database - .prepare('SELECT scope, version FROM operational_schema_migrations ORDER BY scope') - .all() as Array<{ scope?: unknown; version?: unknown }>; - const actual = new Map(); - for (const row of rows) { +async function walk(root: string, current: string, result: OperationalBackupFile[]): Promise { + for (const entry of (await readdir(current, { withFileTypes: true })).sort((a, b) => + a.name.localeCompare(b.name), + )) { + if (entry.name === OPERATIONAL_BACKUP_MANIFEST_FILE) continue; if ( - typeof row.scope !== 'string' || - typeof row.version !== 'number' || - !Number.isSafeInteger(row.version) || - row.version < 0 + entry.name === `${OPERATIONAL_STATE_DATABASE_NAME}-shm` || + entry.name === `${OPERATIONAL_STATE_DATABASE_NAME}-wal` ) { - throw new OperationalBackupError('corrupt_backup', 'Backup schema registry is invalid'); - } - actual.set(row.scope, row.version); - } - for (const [scope, supported] of supportedSchemaVersions) { - const version = actual.get(scope); - if (version === undefined) { - if ( - scope === 'automation' && - backupSchemaVersion === LEGACY_OPERATIONAL_BACKUP_SCHEMA_VERSION - ) { - continue; - } - throw new OperationalBackupError('corrupt_backup', `Backup schema is missing ${scope}`); + continue; } - if (version > supported) { - throw new OperationalBackupError( - 'unsupported_schema', - `Backup schema ${scope} version ${version} is newer than supported ${supported}`, - ); + const path = resolve(current, entry.name); + if (entry.isSymbolicLink()) { + throw new OperationalBackupError('corrupt_backup', 'Backup cannot contain symlinks'); } - if (version !== supported) { - throw new OperationalBackupError( - 'unsupported_schema', - `Backup schema ${scope} version ${version} requires migration before restore`, - ); + if (entry.isDirectory()) { + await walk(root, path, result); + continue; } - } - for (const scope of actual.keys()) { - if (!supportedSchemaVersions.has(scope)) { - throw new OperationalBackupError( - 'unsupported_schema', - `Backup contains an unknown schema scope: ${scope}`, - ); + if (!entry.isFile()) { + throw new OperationalBackupError('corrupt_backup', 'Backup contains a non-regular file'); } + result.push(await describeFile(root, path)); } } -async function readOperationalBackupManifest(path: string): Promise { - const metadata = await lstat(path).catch((error: unknown) => { - throw new OperationalBackupError('invalid_manifest', 'Operational backup manifest is missing', { - cause: error, - }); - }); - if (!metadata.isFile() || metadata.isSymbolicLink() || metadata.size > MAX_MANIFEST_BYTES) { - throw new OperationalBackupError('invalid_manifest', 'Operational backup manifest is invalid'); - } - let value: unknown; - try { - value = JSON.parse(await readFile(path, 'utf8')) as unknown; - } catch (error) { - throw new OperationalBackupError('invalid_manifest', 'Operational backup manifest is invalid', { - cause: error, - }); - } - return decodeOperationalBackupManifest(value); -} - -function decodeOperationalBackupManifest(value: unknown): OperationalBackupManifest { - if (!isRecord(value)) throw invalidManifest(); - const legacy = value.schemaVersion === LEGACY_OPERATIONAL_BACKUP_SCHEMA_VERSION; - const current = value.schemaVersion === OPERATIONAL_BACKUP_SCHEMA_VERSION; - const commonKeys = [ - 'format', - 'schemaVersion', - 'createdAt', - 'database', - 'transcripts', - 'artifacts', - ]; - if (!legacy && !current) throw invalidManifest(); - if (!hasExactKeys(value, current ? [...commonKeys, 'legacyAutomation'] : commonKeys)) { - throw invalidManifest(); - } - if ( - value.format !== OPERATIONAL_BACKUP_FORMAT || - typeof value.createdAt !== 'number' || - !Number.isSafeInteger(value.createdAt) || - value.createdAt < 0 || - !Array.isArray(value.transcripts) || - !Array.isArray(value.artifacts) - ) { - throw invalidManifest(); - } - const database = decodeBackupFile(value.database); - if (database.path !== OPERATIONAL_STATE_DATABASE_NAME) throw invalidManifest(); - const transcripts = value.transcripts.map((entry) => decodeTranscript(entry)); - const artifacts = value.artifacts.map((entry) => decodeArtifact(entry)); - const legacyAutomation = current - ? value.legacyAutomation === null - ? null - : decodeBackupFile(value.legacyAutomation) - : undefined; - if (legacyAutomation && legacyAutomation.path !== LEGACY_AUTOMATION_FILE) { - throw invalidManifest(); +async function describeFile(root: string, path: string): Promise { + const hash = createHash('sha256'); + let size = 0; + for await (const chunk of createReadStream(path)) { + const bytes = chunk as Buffer; + size += bytes.byteLength; + hash.update(bytes); } - assertUniqueManifestPaths([ - database, - ...transcripts, - ...artifacts, - ...(legacyAutomation ? [legacyAutomation] : []), - ]); - assertUniqueValues( - transcripts.map((entry) => entry.sessionId), - 'session id', - ); - assertUniqueValues( - artifacts.map((entry) => entry.record.id), - 'Artifact id', - ); - const common: OperationalBackupManifestBase = { - format: OPERATIONAL_BACKUP_FORMAT, - createdAt: value.createdAt, - database, - transcripts, - artifacts, + return { + path: relative(root, path).split('\\').join('/'), + size, + sha256: `sha256:${hash.digest('hex')}`, }; - return current - ? { - ...common, - schemaVersion: OPERATIONAL_BACKUP_SCHEMA_VERSION, - legacyAutomation: legacyAutomation ?? null, - } - : { - ...common, - schemaVersion: LEGACY_OPERATIONAL_BACKUP_SCHEMA_VERSION, - }; -} - -function decodeBackupFile(value: unknown): OperationalBackupFile { - if (!isRecord(value) || !hasExactKeys(value, ['path', 'sizeBytes', 'sha256'])) { - throw invalidManifest(); - } - if ( - typeof value.path !== 'string' || - !isSafeBackupPath(value.path) || - typeof value.sizeBytes !== 'number' || - !Number.isSafeInteger(value.sizeBytes) || - value.sizeBytes < 0 || - typeof value.sha256 !== 'string' || - !/^sha256:[0-9a-f]{64}$/.test(value.sha256) - ) { - throw invalidManifest(); - } - return value as unknown as OperationalBackupFile; -} - -function decodeTranscript(value: unknown): OperationalBackupTranscript { - if (!isRecord(value) || !hasExactKeys(value, ['sessionId', 'path', 'sizeBytes', 'sha256'])) { - throw invalidManifest(); - } - const file = decodeBackupFile({ - path: value.path, - sizeBytes: value.sizeBytes, - sha256: value.sha256, - }); - if ( - typeof value.sessionId !== 'string' || - !isSafeEntityId(value.sessionId) || - file.path !== `sessions/${value.sessionId}/session.jsonl` - ) { - throw invalidManifest(); - } - return { sessionId: value.sessionId, ...file }; -} - -function decodeArtifact(value: unknown): OperationalBackupArtifact { - if (!isRecord(value) || !hasExactKeys(value, ['record', 'path', 'sizeBytes', 'sha256'])) { - throw invalidManifest(); - } - const file = decodeBackupFile({ - path: value.path, - sizeBytes: value.sizeBytes, - sha256: value.sha256, - }); - let records: ArtifactRecord[]; - try { - records = decodeArtifactMetadata(`${JSON.stringify(value.record)}\n`); - } catch (error) { - throw new OperationalBackupError('invalid_manifest', 'Artifact manifest record is invalid', { - cause: error, - }); - } - const record = records[0]; - if ( - !record || - file.path !== `artifacts/${record.relativePath}` || - file.sizeBytes !== record.sizeBytes - ) { - throw invalidManifest(); - } - return { record, ...file }; } -async function assertBackupTreeExactlyMatchesManifest( - backupRoot: string, - manifest: OperationalBackupManifest, -): Promise { - const expected = new Set([ - OPERATIONAL_BACKUP_MANIFEST_FILE, - ...manifestFiles(manifest).map((file) => file.path), - ]); - const expectedDirectories = manifestDirectories(expected); - const actual = await listBackupTree(backupRoot); - if ( - actual.files.length !== expected.size || - actual.files.some((path) => !expected.has(path)) || - actual.directories.length !== expectedDirectories.size || - actual.directories.some((path) => !expectedDirectories.has(path)) - ) { - throw new OperationalBackupError( - 'corrupt_backup', - `Operational backup contains missing or unmanifested entries: ${[ - ...actual.directories, - ...actual.files, - ].join(', ')}`, - ); +async function copyRegularTree(source: string, destination: string, root: string): Promise { + const metadata = await lstat(source); + if (metadata.isSymbolicLink()) { + throw new OperationalBackupError('invalid_root', 'Artifact tree cannot contain symlinks'); } -} - -async function validateManifestFiles( - root: string, - manifest: OperationalBackupManifest, -): Promise { - for (const file of manifestFiles(manifest)) { - const path = resolveBackupPath(root, file.path); - const actual = await describeBackupFile(path, file.path).catch((error: unknown) => { - throw new OperationalBackupError('corrupt_backup', `Backup file is missing: ${file.path}`, { - cause: error, - }); - }); - if (actual.sizeBytes !== file.sizeBytes || actual.sha256 !== file.sha256) { - throw new OperationalBackupError( - 'corrupt_backup', - `Backup file digest does not match manifest: ${file.path}`, - ); + if (metadata.isDirectory()) { + await mkdir(destination, { recursive: true }); + for (const entry of await readdir(source)) { + await copyRegularTree(resolve(source, entry), resolve(destination, entry), root); } + return; } - for (const transcript of manifest.transcripts) { - await validateTranscriptFile(resolveBackupPath(root, transcript.path), transcript.sessionId); - } - if (manifest.schemaVersion === OPERATIONAL_BACKUP_SCHEMA_VERSION && manifest.legacyAutomation) { - await validateLegacyAutomationBackupFile( - resolveBackupPath(root, manifest.legacyAutomation.path), - ); - } -} - -async function validateLegacyAutomationSourceSnapshot(snapshot: SourceFileSnapshot): Promise { - const contents = await readFile(snapshot.sourcePath); - const sha256 = `sha256:${createHash('sha256').update(contents).digest('hex')}`; - if (contents.byteLength !== snapshot.sizeBytes || sha256 !== snapshot.sha256) { - throw new OperationalBackupError( - 'source_changed', - `Operational backup source changed during validation: ${snapshot.path}`, - ); - } - try { - decodeLegacyAutomationFile(contents.toString('utf8')); - } catch (error) { - throw new OperationalBackupError( - 'corrupt_backup', - 'Legacy Automation source cannot be backed up', - { cause: error }, - ); + if (!metadata.isFile()) { + throw new OperationalBackupError('invalid_root', 'Artifact tree contains a non-regular file'); } + resolveInside(root, relative(root, source)); + await mkdir(dirname(destination), { recursive: true }); + await copyFile(source, destination); } -async function validateLegacyAutomationBackupFile(path: string): Promise { +function validateSqlite(path: string): void { try { - decodeLegacyAutomationFile(await readFile(path, 'utf8')); - } catch (error) { - throw new OperationalBackupError( - 'corrupt_backup', - 'Legacy Automation backup payload cannot be restored', - { cause: error }, - ); - } -} - -async function validateTranscriptFile(path: string, sessionId: string): Promise { - const text = await readFile(path, 'utf8'); - const lines = text - .split('\n') - .map((line, index) => ({ line, lineNumber: index + 1 })) - .filter((entry) => entry.line.trim().length > 0); - if (lines.length === 0 || !lines[0]) { - throw new OperationalBackupError('corrupt_backup', `Session ${sessionId} transcript is empty`); - } - try { - const first = JSON.parse(lines[0].line) as unknown; - if (isSessionTranscriptMarker(first)) decodeSessionTranscriptMarker(first, sessionId); - else decodeSessionHeader(first, sessionId); - for (const entry of lines.slice(1)) { - decodeStoredMessageForRecovery(JSON.parse(entry.line) as unknown); - } - } catch (error) { - throw new OperationalBackupError( - 'corrupt_backup', - `Session ${sessionId} transcript cannot be restored`, - { cause: error }, - ); - } -} - -function manifestFiles(manifest: OperationalBackupManifest): OperationalBackupFile[] { - return [ - manifest.database, - ...manifest.transcripts, - ...manifest.artifacts, - ...(manifest.schemaVersion === OPERATIONAL_BACKUP_SCHEMA_VERSION && manifest.legacyAutomation - ? [manifest.legacyAutomation] - : []), - ]; -} - -async function listBackupTree( - root: string, - current = root, -): Promise<{ files: string[]; directories: string[] }> { - const files: string[] = []; - const directories: string[] = []; - for (const entry of await readdir(current, { withFileTypes: true })) { - const path = resolve(current, entry.name); - const relativePath = relative(root, path).split(sep).join('/'); - if (entry.isSymbolicLink()) { - throw new OperationalBackupError( - 'corrupt_backup', - `Backup contains symlink: ${relativePath}`, - ); - } - if (entry.isDirectory()) { - directories.push(relativePath); - const child = await listBackupTree(root, path); - files.push(...child.files); - directories.push(...child.directories); - } else if (entry.isFile()) files.push(relativePath); - else { - throw new OperationalBackupError( - 'corrupt_backup', - `Backup contains special file: ${relativePath}`, - ); - } - } - return { - files: files.sort((left, right) => left.localeCompare(right)), - directories: directories.sort((left, right) => left.localeCompare(right)), - }; -} - -function manifestDirectories(files: ReadonlySet): Set { - const directories = new Set(); - for (const file of files) { - const parts = file.split('/'); - for (let index = 1; index < parts.length; index += 1) { - directories.add(parts.slice(0, index).join('/')); - } - } - return directories; -} - -async function assertRegularFile(path: string, root: string, relativePath: string): Promise { - const metadata = await lstat(path).catch((error: unknown) => { - throw new OperationalBackupError('invalid_root', `Required file is missing: ${relativePath}`, { - cause: error, - }); - }); - if (!metadata.isFile() || metadata.isSymbolicLink()) throw invalidSourceEntry(relativePath); - await assertCanonicalPathInside(path, root, relativePath); -} - -async function assertRegularDirectory( - path: string, - root: string, - relativePath: string, -): Promise { - const metadata = await lstat(path).catch((error: unknown) => { - throw new OperationalBackupError( - 'invalid_root', - `Required directory is missing: ${relativePath}`, - { cause: error }, - ); - }); - if (!metadata.isDirectory() || metadata.isSymbolicLink()) throw invalidSourceEntry(relativePath); - await assertCanonicalPathInside(path, root, relativePath); -} - -async function assertCanonicalPathInside(path: string, root: string, role: string): Promise { - const canonical = await realpath(path); - if (!pathIsAtOrInside(root, canonical)) { - throw new OperationalBackupError('invalid_root', `Path escapes state root: ${role}`); - } -} - -async function canonicalizeExistingDirectory(path: string, role: string): Promise { - const requested = resolve(path); - const metadata = await lstat(requested).catch((error: unknown) => { - throw new OperationalBackupError('invalid_root', `Operational ${role} root does not exist`, { - cause: error, - }); - }); - if (!metadata.isDirectory() || metadata.isSymbolicLink()) { - throw new OperationalBackupError('invalid_root', `Operational ${role} root is invalid`); - } - return realpath(requested); -} - -async function canonicalizeMissingPath(path: string, role: string): Promise { - const requested = resolve(path); - let candidate = requested; - const suffix: string[] = []; - while (true) { + const database = new DatabaseSync(path, { readOnly: true }); try { - const metadata = await lstat(candidate); - if (!metadata.isDirectory() || metadata.isSymbolicLink()) { - throw new OperationalBackupError( - 'invalid_root', - `Operational ${role} parent is invalid: ${candidate}`, - ); - } - return resolve(await realpath(candidate), ...suffix.reverse()); - } catch (error) { - if (error instanceof OperationalBackupError) throw error; - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - const parent = dirname(candidate); - if (parent === candidate) { - throw new OperationalBackupError('invalid_root', `Operational ${role} path is invalid`); + const row = database.prepare('PRAGMA quick_check').get() as { quick_check?: unknown }; + if (row.quick_check !== 'ok') throw new Error('quick_check failed'); + const expected = new Map([ + ['runtime', SQLITE_RUNTIME_SCHEMA_VERSION], + ['session_metadata', SQLITE_SESSION_METADATA_SCHEMA_VERSION], + ['core_execution', SQLITE_CORE_EXECUTION_SCHEMA_VERSION], + ['workflow', SQLITE_WORKFLOW_SCHEMA_VERSION], + ['usage', SQLITE_USAGE_SCHEMA_VERSION], + ['artifact', SQLITE_ARTIFACT_SCHEMA_VERSION], + ['automation', SQLITE_AUTOMATION_SCHEMA_VERSION], + ['operational', OPERATIONAL_STATE_SCHEMA_VERSION], + ]); + const rows = database + .prepare('SELECT scope, version FROM operational_schema_migrations') + .all() as Array<{ scope?: unknown; version?: unknown }>; + if ( + rows.length !== expected.size || + rows.some( + (entry) => typeof entry.scope !== 'string' || expected.get(entry.scope) !== entry.version, + ) + ) { + throw new Error('operational schema versions do not match'); } - suffix.push(basename(candidate)); - candidate = parent; + } finally { + database.close(); } - } -} - -async function assertPathMissing(path: string, role: string): Promise { - try { - await lstat(path); } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; - throw error; + throw new OperationalBackupError('corrupt_backup', 'Backup runtime.sqlite is invalid', { + cause: error, + }); } - throw new OperationalBackupError('destination_exists', `${role} already exists: ${path}`); } -function assertRootsDoNotOverlap(left: string, right: string): void { - if (pathIsAtOrInside(left, right) || pathIsAtOrInside(right, left)) { - throw new OperationalBackupError( - 'overlapping_roots', - `Operational backup roots overlap: ${left} and ${right}`, - ); +function resolveInside(root: string, path: string): string { + const candidate = resolve(root, path); + const rel = relative(root, candidate); + if (rel === '' || rel.startsWith('..') || rel.includes(':')) { + throw new OperationalBackupError('corrupt_backup', `Unsafe backup path: ${path}`); } + return candidate; } -function resolveBackupPath(root: string, relativePath: string): string { - if (!isSafeBackupPath(relativePath)) throw invalidManifest(); - const path = resolve(root, ...relativePath.split('/')); - if (!pathIsAtOrInside(root, path)) throw invalidManifest(); - return path; -} - -function isSafeBackupPath(path: string): boolean { - if (!path || isAbsolute(path) || path.includes('\\') || path.includes('\0')) return false; - const parts = path.split('/'); - return parts.every((part) => part.length > 0 && part !== '.' && part !== '..'); -} - -function pathIsAtOrInside(root: string, candidate: string): boolean { - const path = relative(root, candidate); - return path === '' || (path !== '..' && !path.startsWith(`..${sep}`) && !isAbsolute(path)); -} - -function isSafeEntityId(value: string): boolean { - return /^[A-Za-z0-9_-]{1,128}$/.test(value); -} - -function sqliteTableExists(database: DatabaseSync, table: string): boolean { - return ( - database.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`).get(table) !== - undefined - ); -} - -function assertSameArtifactRecords( - left: readonly ArtifactRecord[], - right: readonly ArtifactRecord[], - message: string, - code: OperationalBackupErrorCode, -): void { - const canonical = (records: readonly ArtifactRecord[]) => - records.map((record) => JSON.stringify(record)).sort((a, b) => a.localeCompare(b)); - const leftRecords = canonical(left); - const rightRecords = canonical(right); +function assertSeparateRoots(left: string, right: string): void { + const leftToRight = relative(left, right); + const rightToLeft = relative(right, left); if ( - leftRecords.length !== rightRecords.length || - leftRecords.some((record, index) => record !== rightRecords[index]) + left === right || + (!leftToRight.startsWith('..') && !leftToRight.includes(':')) || + (!rightToLeft.startsWith('..') && !rightToLeft.includes(':')) ) { - throw new OperationalBackupError(code, message); + throw new OperationalBackupError('overlapping_roots', 'Backup roots must not overlap'); } } -function assertSameStringSet( - left: readonly string[], - right: readonly string[], - message: string, - code: OperationalBackupErrorCode, -): void { - const leftValues = [...left].sort((a, b) => a.localeCompare(b)); - const rightValues = [...right].sort((a, b) => a.localeCompare(b)); - if ( - leftValues.length !== rightValues.length || - leftValues.some((value, index) => value !== rightValues[index]) - ) { - throw new OperationalBackupError(code, message); +async function assertMissing(path: string, label: string): Promise { + if (await pathExists(path)) { + throw new OperationalBackupError('destination_not_empty', `${label} already exists: ${path}`); } } -function assertUniqueManifestPaths(files: readonly OperationalBackupFile[]): void { - assertUniqueValues( - files.map((file) => file.path), - 'file path', - ); -} - -function assertUniqueValues(values: readonly string[], role: string): void { - if (new Set(values).size !== values.length) { - throw new OperationalBackupError('invalid_manifest', `Duplicate backup ${role}`); +async function pathExists(path: string): Promise { + try { + await stat(path); + return true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; + throw error; } } - -function hasExactKeys(value: Record, expected: readonly string[]): boolean { - const keys = Object.keys(value).sort((a, b) => a.localeCompare(b)); - const canonicalExpected = [...expected].sort((a, b) => a.localeCompare(b)); - return ( - keys.length === canonicalExpected.length && - keys.every((key, index) => key === canonicalExpected[index]) - ); -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} - -function invalidManifest(): OperationalBackupError { - return new OperationalBackupError('invalid_manifest', 'Operational backup manifest is invalid'); -} - -function invalidSourceEntry(path: string): OperationalBackupError { - return new OperationalBackupError('invalid_root', `Invalid operational backup source: ${path}`); -} - -function loadDatabaseSync(): typeof import('node:sqlite').DatabaseSync { - return (require('node:sqlite') as typeof import('node:sqlite')).DatabaseSync; -} diff --git a/packages/storage/src/operational-state-store.ts b/packages/storage/src/operational-state-store.ts index efde4d62af..84096f5641 100644 --- a/packages/storage/src/operational-state-store.ts +++ b/packages/storage/src/operational-state-store.ts @@ -1,6 +1,5 @@ -import { createHash } from 'node:crypto'; -import { existsSync, lstatSync, mkdirSync } from 'node:fs'; -import { dirname, join, resolve } from 'node:path'; +import { existsSync, mkdirSync } from 'node:fs'; +import { dirname, resolve } from 'node:path'; import { createRequire } from 'node:module'; import type { DatabaseSync } from 'node:sqlite'; import { @@ -10,7 +9,6 @@ import { } from './sqlite-runtime-schema.js'; import { migrateSqliteSessionMetadataDatabase, - SQLITE_AGENT_GRAPH_CONTROL_TABLES, SQLITE_SESSION_METADATA_SCHEMA_VERSION, } from './sqlite-session-metadata-schema.js'; import { @@ -32,30 +30,13 @@ import { } from './sqlite-automation-schema.js'; export const OPERATIONAL_STATE_DATABASE_NAME = 'runtime.sqlite'; -export const LEGACY_SESSION_METADATA_DATABASE_NAME = 'sessions.sqlite'; export const OPERATIONAL_STATE_SCHEMA_VERSION = 1; -const SESSION_METADATA_TABLES = [ - 'session_metadata', - 'session_metadata_labels', - 'session_metadata_import_sources', - 'session_metadata_tombstones', - 'subagent_spawns', - ...SQLITE_AGENT_GRAPH_CONTROL_TABLES, - 'sandbox_boundary_log', -] as const; - const require = createRequire(import.meta.url); const owners = new Map(); -export type OperationalStateCutoverFailpoint = - | 'after_cutover_started' - | 'after_cutover_rows_copied' - | 'after_cutover_validated'; - export interface OperationalStateDatabaseOptions { now?: () => number; - failpoint?: (point: OperationalStateCutoverFailpoint) => void; } export interface OperationalStateDatabaseLease { @@ -66,30 +47,6 @@ export interface OperationalStateDatabaseLease { close(): void; } -export type OperationalStoreCutoverFailpoint = - | 'after_cutover_started' - | 'after_cutover_rows_copied' - | 'after_cutover_validated'; - -export interface OperationalStoreCutoverInput { - readonly storeName: string; - readonly sourcePath: string; - readonly sourceFingerprint: string; - readonly importAndValidate: (database: DatabaseSync) => Readonly>; - readonly now?: () => number; - readonly failpoint?: (point: OperationalStoreCutoverFailpoint) => void; -} - -interface CutoverJournalRow { - source_fingerprint: string; - state: 'started' | 'completed'; -} - -interface TableColumn { - name: string; - pk: number; -} - /** * Acquire the process-local owner for the operational SQLite authority. * @@ -133,13 +90,6 @@ class OperationalStateDatabaseOwner { migrateSqliteArtifactDatabase(this.database); migrateSqliteAutomationDatabase(this.database); migrateOperationalStateDatabase(this.database, options.now ?? Date.now); - cutoverLegacySessionMetadata({ - destination: this.database, - destinationPath: databasePath, - sourcePath: join(dirname(databasePath), LEGACY_SESSION_METADATA_DATABASE_NAME), - now: options.now ?? Date.now, - failpoint: options.failpoint, - }); } catch (error) { this.database.close(); this.closed = true; @@ -220,16 +170,6 @@ function migrateOperationalStateDatabase(db: DatabaseSync, now: () => number): v version INTEGER NOT NULL CHECK (version >= 0), applied_at INTEGER NOT NULL CHECK (applied_at >= 0) ); - - CREATE TABLE IF NOT EXISTS cutover_journal ( - store_name TEXT PRIMARY KEY, - source_path TEXT NOT NULL, - source_fingerprint TEXT NOT NULL, - state TEXT NOT NULL CHECK (state IN ('started', 'completed')), - started_at INTEGER NOT NULL CHECK (started_at >= 0), - completed_at INTEGER, - validation_json TEXT - ); `); const appliedAt = now(); registerSchema(db, 'runtime', SQLITE_RUNTIME_SCHEMA_VERSION, appliedAt); @@ -247,78 +187,6 @@ function migrateOperationalStateDatabase(db: DatabaseSync, now: () => number): v } } -/** - * Complete one logical legacy-store cutover under the operational owner. - * - * The durable `started` row intentionally commits before the data transaction. - * A crash therefore resumes against the same source fingerprint, while copied - * rows and the completed marker commit atomically. - */ -export function completeOperationalStoreCutover( - lease: OperationalStateDatabaseLease, - input: OperationalStoreCutoverInput, -): void { - const journal = lease.database - .prepare(` - SELECT source_fingerprint, state - FROM cutover_journal - WHERE store_name = ? - `) - .get(input.storeName) as CutoverJournalRow | undefined; - if (journal?.state === 'completed') { - if (journal.source_fingerprint !== input.sourceFingerprint) { - throw new Error(`Legacy ${input.storeName} source changed after cutover completed`); - } - return; - } - if (journal && journal.source_fingerprint !== input.sourceFingerprint) { - throw new Error(`Legacy ${input.storeName} source changed after cutover started`); - } - if (!journal) { - lease.transaction('write', () => { - lease.database - .prepare(` - INSERT INTO cutover_journal( - store_name, - source_path, - source_fingerprint, - state, - started_at - ) VALUES (?, ?, ?, 'started', ?) - `) - .run(input.storeName, input.sourcePath, input.sourceFingerprint, (input.now ?? Date.now)()); - }); - } - input.failpoint?.('after_cutover_started'); - lease.transaction('write', () => { - const validation = input.importAndValidate(lease.database); - input.failpoint?.('after_cutover_rows_copied'); - for (const [name, count] of Object.entries(validation)) { - if (!Number.isSafeInteger(count) || count < 0) { - throw new Error(`Invalid ${input.storeName} cutover validation count for ${name}`); - } - } - input.failpoint?.('after_cutover_validated'); - const result = lease.database - .prepare(` - UPDATE cutover_journal - SET state = 'completed', completed_at = ?, validation_json = ? - WHERE store_name = ? - AND source_fingerprint = ? - AND state = 'started' - `) - .run( - (input.now ?? Date.now)(), - JSON.stringify(validation), - input.storeName, - input.sourceFingerprint, - ); - if (result.changes !== 1) { - throw new Error(`Unable to complete ${input.storeName} cutover journal`); - } - }); -} - function registerSchema(db: DatabaseSync, scope: string, version: number, appliedAt: number): void { const existing = db .prepare('SELECT version FROM operational_schema_migrations WHERE scope = ?') @@ -344,251 +212,6 @@ function registerSchema(db: DatabaseSync, scope: string, version: number, applie `).run(scope, version, appliedAt); } -function cutoverLegacySessionMetadata(input: { - destination: DatabaseSync; - destinationPath: string; - sourcePath: string; - now: () => number; - failpoint?: (point: OperationalStateCutoverFailpoint) => void; -}): void { - if ( - !existsSync(input.sourcePath) || - resolve(input.sourcePath) === resolve(input.destinationPath) - ) { - return; - } - const sourceStat = lstatSync(input.sourcePath); - if (!sourceStat.isFile() || sourceStat.isSymbolicLink() || sourceStat.size === 0) { - throw new Error('Legacy sessions.sqlite is not a non-empty regular database file'); - } - - const Database = loadDatabaseSync(); - const source = new Database(input.sourcePath); - try { - configureSqliteRuntimeDatabase(source); - migrateSqliteSessionMetadataDatabase(source); - source.exec('BEGIN IMMEDIATE'); - try { - const fingerprint = fingerprintSessionMetadata(source); - const journal = input.destination - .prepare(` - SELECT source_fingerprint, state - FROM cutover_journal - WHERE store_name = 'session_metadata' - `) - .get() as CutoverJournalRow | undefined; - if (journal?.state === 'completed') { - if (journal.source_fingerprint !== fingerprint) { - throw new Error( - 'Legacy sessions.sqlite changed after session metadata cutover completed', - ); - } - source.exec('COMMIT'); - return; - } - if (journal && journal.source_fingerprint !== fingerprint) { - throw new Error('Legacy sessions.sqlite changed after session metadata cutover started'); - } - if (!journal) { - input.destination.exec('BEGIN IMMEDIATE'); - try { - input.destination - .prepare(` - INSERT INTO cutover_journal( - store_name, - source_path, - source_fingerprint, - state, - started_at - ) VALUES ('session_metadata', ?, ?, 'started', ?) - `) - .run(input.sourcePath, fingerprint, input.now()); - input.destination.exec('COMMIT'); - } catch (error) { - rollback(input.destination); - throw error; - } - } - input.failpoint?.('after_cutover_started'); - - input.destination.exec('BEGIN IMMEDIATE'); - let attached = false; - try { - input.destination.exec( - `ATTACH DATABASE ${quoteString(input.sourcePath)} AS legacy_sessions`, - ); - attached = true; - const validation: Record = {}; - for (const table of SESSION_METADATA_TABLES) { - const columns = readTableColumns(source, table); - assertNoCutoverConflicts(input.destination, table, columns); - const names = columns.map((column) => quoteIdentifier(column.name)).join(', '); - input.destination.exec(` - INSERT OR IGNORE INTO main.${quoteIdentifier(table)} (${names}) - SELECT ${names} FROM legacy_sessions.${quoteIdentifier(table)} - `); - } - input.failpoint?.('after_cutover_rows_copied'); - for (const table of SESSION_METADATA_TABLES) { - const columns = readTableColumns(source, table); - assertAllSourceRowsCopied(input.destination, table, columns); - validation[table] = readRowCount(source, table); - } - input.failpoint?.('after_cutover_validated'); - input.destination - .prepare(` - UPDATE cutover_journal - SET state = 'completed', completed_at = ?, validation_json = ? - WHERE store_name = 'session_metadata' - AND source_fingerprint = ? - AND state = 'started' - `) - .run(input.now(), JSON.stringify(validation), fingerprint); - input.destination.exec('COMMIT'); - } catch (error) { - rollback(input.destination); - throw error; - } finally { - if (attached) { - input.destination.exec('DETACH DATABASE legacy_sessions'); - } - } - source.exec('COMMIT'); - } catch (error) { - rollback(source); - throw error; - } - } finally { - source.close(); - } -} - -function fingerprintSessionMetadata(db: DatabaseSync): string { - const hash = createHash('sha256'); - hash.update(`session_metadata_schema:${SQLITE_SESSION_METADATA_SCHEMA_VERSION}\n`); - for (const table of SESSION_METADATA_TABLES) { - const columns = readTableColumns(db, table); - const order = primaryKeyColumns(columns); - const rows = db - .prepare( - `SELECT * FROM ${quoteIdentifier(table)} ORDER BY ${order - .map((column) => quoteIdentifier(column)) - .join(', ')}`, - ) - .all() as Record[]; - hash.update(`${table}:${rows.length}\n`); - for (const row of rows) { - hash.update(JSON.stringify(columns.map((column) => row[column.name]))); - hash.update('\n'); - } - } - return `sha256:${hash.digest('hex')}`; -} - -function assertNoCutoverConflicts( - db: DatabaseSync, - table: string, - columns: readonly TableColumn[], -): void { - const primaryKey = primaryKeyColumns(columns); - const identity = primaryKey - .map( - (column) => - `main.${quoteIdentifier(table)}.${quoteIdentifier(column)} IS legacy.${quoteIdentifier(column)}`, - ) - .join(' AND '); - const equality = columns - .map( - (column) => - `main.${quoteIdentifier(table)}.${quoteIdentifier(column.name)} IS legacy.${quoteIdentifier(column.name)}`, - ) - .join(' AND '); - const row = db - .prepare(` - SELECT COUNT(*) AS count - FROM legacy_sessions.${quoteIdentifier(table)} AS legacy - JOIN main.${quoteIdentifier(table)} - ON ${identity} - WHERE NOT (${equality}) - `) - .get() as { count?: unknown } | undefined; - if (row?.count !== 0) { - throw new Error(`Session metadata cutover conflict in table ${table}`); - } -} - -function assertAllSourceRowsCopied( - db: DatabaseSync, - table: string, - columns: readonly TableColumn[], -): void { - const equality = columns - .map( - (column) => - `main.${quoteIdentifier(table)}.${quoteIdentifier(column.name)} IS legacy.${quoteIdentifier(column.name)}`, - ) - .join(' AND '); - const row = db - .prepare(` - SELECT COUNT(*) AS count - FROM legacy_sessions.${quoteIdentifier(table)} AS legacy - WHERE NOT EXISTS ( - SELECT 1 FROM main.${quoteIdentifier(table)} - WHERE ${equality} - ) - `) - .get() as { count?: unknown } | undefined; - if (row?.count !== 0) { - throw new Error(`Session metadata cutover validation failed for table ${table}`); - } -} - -function readTableColumns(db: DatabaseSync, table: string): TableColumn[] { - const rows = db.prepare(`PRAGMA table_info(${quoteIdentifier(table)})`).all() as Array<{ - name?: unknown; - pk?: unknown; - }>; - const columns = rows.map((row) => { - if ( - typeof row.name !== 'string' || - typeof row.pk !== 'number' || - !Number.isSafeInteger(row.pk) - ) { - throw new Error(`Invalid SQLite table metadata for ${table}`); - } - return { name: row.name, pk: row.pk }; - }); - if (columns.length === 0 || columns.every((column) => column.pk === 0)) { - throw new Error(`Session metadata cutover table ${table} has no primary key`); - } - return columns; -} - -function primaryKeyColumns(columns: readonly TableColumn[]): string[] { - return columns - .filter((column) => column.pk > 0) - .sort((left, right) => left.pk - right.pk) - .map((column) => column.name); -} - -function readRowCount(db: DatabaseSync, table: string): number { - const row = db.prepare(`SELECT COUNT(*) AS count FROM ${quoteIdentifier(table)}`).get() as - | { count?: unknown } - | undefined; - if (typeof row?.count !== 'number' || !Number.isSafeInteger(row.count) || row.count < 0) { - throw new Error(`Invalid row count for ${table}`); - } - return row.count; -} - -function quoteIdentifier(value: string): string { - return `"${value.replaceAll('"', '""')}"`; -} - -function quoteString(value: string): string { - return `'${value.replaceAll("'", "''")}'`; -} - function loadDatabaseSync(): typeof import('node:sqlite').DatabaseSync { const emitWarning = process.emitWarning; process.emitWarning = ((warning: string | Error, ...args: unknown[]) => { diff --git a/packages/storage/src/plan-reminder-store.ts b/packages/storage/src/plan-reminder-store.ts index f1cced4576..ad168d34a9 100644 --- a/packages/storage/src/plan-reminder-store.ts +++ b/packages/storage/src/plan-reminder-store.ts @@ -1,6 +1,5 @@ -import { mkdir, readFile, rename, writeFile } from 'node:fs/promises'; -import { dirname, join, resolve } from 'node:path'; -import { createHash, randomUUID } from 'node:crypto'; +import { resolve } from 'node:path'; +import { randomUUID } from 'node:crypto'; import type { DatabaseSync } from 'node:sqlite'; import { createPlanReminderSchedule, @@ -17,9 +16,7 @@ import { } from '@maka/core/plan-reminders'; import { acquireOperationalStateDatabase, - completeOperationalStoreCutover, type OperationalStateDatabaseLease, - type OperationalStoreCutoverFailpoint, } from './operational-state-store.js'; export interface PlanReminderStore { @@ -41,32 +38,29 @@ export interface PlanReminderStore { ): Promise; } -export function createPlanReminderStore(workspaceRoot: string): PlanReminderStore { - return new FilePlanReminderStore(workspaceRoot); -} - export interface SqlitePlanReminderStore extends PlanReminderStore { ready(): Promise; close(): void; } -export interface CreateSqlitePlanReminderStoreOptions { - failpoint?: (point: OperationalStoreCutoverFailpoint) => void; -} - -export function createSqlitePlanReminderStore( - workspaceRoot: string, - options: CreateSqlitePlanReminderStoreOptions = {}, -): SqlitePlanReminderStore { - return new SqlitePlanReminderStoreImpl(workspaceRoot, options); +export function createSqlitePlanReminderStore(workspaceRoot: string): SqlitePlanReminderStore { + return new SqlitePlanReminderStoreImpl(workspaceRoot); } -class FilePlanReminderStore implements PlanReminderStore { - private readonly filePath: string; +class SqlitePlanReminderStoreImpl implements SqlitePlanReminderStore { + readonly #lease: OperationalStateDatabaseLease; private queue: Promise = Promise.resolve(); constructor(workspaceRoot: string) { - this.filePath = join(workspaceRoot, 'plan-reminders.json'); + this.#lease = acquireOperationalStateDatabase(resolve(workspaceRoot)); + } + + ready(): Promise { + return Promise.resolve(); + } + + close(): void { + this.#lease.close(); } async list(): Promise { @@ -299,19 +293,7 @@ class FilePlanReminderStore implements PlanReminderStore { } private async read(): Promise { - const canonical = await this.readCanonical(); - if (canonical) return canonical; - try { - const text = await readFile(this.filePath, 'utf8'); - const parsed = JSON.parse(text) as unknown; - if (!Array.isArray(parsed)) { - throw new Error('Invalid plan reminders file: expected an array'); - } - return parsed.map((value, index) => normalizePersistedPlanReminder(value, index)); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } + return readSqlitePlanReminders(this.#lease.database); } private async mutate(fn: (reminders: PlanReminder[]) => PlanReminder[]): Promise { @@ -325,93 +307,13 @@ class FilePlanReminderStore implements PlanReminderStore { } private async write(reminders: PlanReminder[]): Promise { - if (await this.writeCanonical(reminders)) return; - await mkdir(dirname(this.filePath), { recursive: true }); - const tempPath = `${this.filePath}.${process.pid}.${Date.now()}.tmp`; - await writeFile(tempPath, JSON.stringify(reminders, null, 2) + '\n', 'utf8'); - await rename(tempPath, this.filePath); - } - - protected async readCanonical(): Promise { - return undefined; - } - - protected async writeCanonical(_reminders: PlanReminder[]): Promise { - return false; - } -} - -class SqlitePlanReminderStoreImpl extends FilePlanReminderStore implements SqlitePlanReminderStore { - readonly #lease: OperationalStateDatabaseLease; - readonly #ready: Promise; - - constructor(workspaceRoot: string, options: CreateSqlitePlanReminderStoreOptions) { - super(workspaceRoot); - const root = resolve(workspaceRoot); - this.#lease = acquireOperationalStateDatabase(root); - this.#ready = importLegacyPlanReminders(root, this.#lease, options.failpoint); - } - - ready(): Promise { - return this.#ready; - } - - close(): void { - this.#lease.close(); - } - - protected override async readCanonical(): Promise { - await this.#ready; - return readSqlitePlanReminders(this.#lease.database); - } - - protected override async writeCanonical(reminders: PlanReminder[]): Promise { - await this.#ready; this.#lease.transaction('write', () => { this.#lease.database.prepare('DELETE FROM workflow_plan_reminders').run(); for (const reminder of reminders) insertPlanReminder(this.#lease.database, reminder); }); - return true; } } -async function importLegacyPlanReminders( - root: string, - lease: OperationalStateDatabaseLease, - failpoint?: (point: OperationalStoreCutoverFailpoint) => void, -): Promise { - let reminders: PlanReminder[] = []; - const sourcePath = join(root, 'plan-reminders.json'); - try { - const parsed = JSON.parse(await readFile(sourcePath, 'utf8')) as unknown; - if (!Array.isArray(parsed)) { - throw new Error('Invalid plan reminders file: expected an array'); - } - reminders = parsed.map((value, index) => normalizePersistedPlanReminder(value, index)); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - } - const fingerprint = `sha256:${createHash('sha256') - .update(JSON.stringify(reminders)) - .digest('hex')}`; - completeOperationalStoreCutover(lease, { - storeName: 'workflow_plan_reminders', - sourcePath, - sourceFingerprint: fingerprint, - failpoint, - importAndValidate: (database) => { - for (const reminder of reminders) insertOrValidatePlanReminder(database, reminder); - const persisted = database - .prepare('SELECT COUNT(*) AS count FROM workflow_plan_reminders') - .get() as { count?: unknown }; - if (persisted.count !== reminders.length) { - throw new Error('Plan reminder cutover row-count validation failed'); - } - return { reminders: reminders.length }; - }, - }); -} - function readSqlitePlanReminders(database: DatabaseSync): PlanReminder[] { const rows = database .prepare(` @@ -438,19 +340,6 @@ function insertPlanReminder(database: DatabaseSync, reminder: PlanReminder): voi .run(reminder.id, reminder.createdAt, reminder.updatedAt, JSON.stringify(reminder)); } -function insertOrValidatePlanReminder(database: DatabaseSync, reminder: PlanReminder): void { - const existing = database - .prepare('SELECT record_json FROM workflow_plan_reminders WHERE reminder_id = ?') - .get(reminder.id) as { record_json?: unknown } | undefined; - if (existing) { - if (existing.record_json !== JSON.stringify(reminder)) { - throw new Error(`Plan reminder cutover conflict: ${reminder.id}`); - } - return; - } - insertPlanReminder(database, reminder); -} - function comparePlanRemindersForList(a: PlanReminder, b: PlanReminder): number { const statusDelta = planReminderListPriority(a) - planReminderListPriority(b); if (statusDelta !== 0) return statusDelta; diff --git a/packages/storage/src/plan-store.ts b/packages/storage/src/plan-store.ts index 87199e0973..4d870dbe7b 100644 --- a/packages/storage/src/plan-store.ts +++ b/packages/storage/src/plan-store.ts @@ -1,6 +1,5 @@ -import { createHash, randomUUID } from 'node:crypto'; -import { mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'; -import { dirname, join, resolve } from 'node:path'; +import { randomUUID } from 'node:crypto'; +import { resolve } from 'node:path'; import type { DatabaseSync } from 'node:sqlite'; import { PlanConflictError, @@ -22,14 +21,10 @@ import { type SubmitPlanProposalInput, type UpdatePlanExecutionInput, } from '@maka/core/plan'; -import { appendJsonl } from './jsonl-append.js'; -import { classifyJsonRecord } from './json-prefix.js'; import { chainWrite } from './write-queue.js'; import { acquireOperationalStateDatabase, - completeOperationalStoreCutover, type OperationalStateDatabaseLease, - type OperationalStoreCutoverFailpoint, } from './operational-state-store.js'; const SAFE_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; @@ -39,21 +34,12 @@ export interface CreatePlanStoreOptions { now?: () => number; } -export function createPlanStore( - workspaceRoot: string, - options: CreatePlanStoreOptions = {}, -): PlanStore { - return new FilePlanStore(workspaceRoot, options); -} - export interface SqlitePlanStore extends PlanStore { ready(): Promise; close(): void; } -export interface CreateSqlitePlanStoreOptions extends CreatePlanStoreOptions { - failpoint?: (point: OperationalStoreCutoverFailpoint) => void; -} +export type CreateSqlitePlanStoreOptions = CreatePlanStoreOptions; export function createSqlitePlanStore( workspaceRoot: string, @@ -62,20 +48,26 @@ export function createSqlitePlanStore( return new SqlitePlanStoreImpl(workspaceRoot, options); } -class FilePlanStore implements PlanStore { - private readonly durabilityRoot: string; - private readonly sessionsRoot: string; +class SqlitePlanStoreImpl implements SqlitePlanStore { + readonly #lease: OperationalStateDatabaseLease; private readonly queues = new Map>(); private readonly newId: () => string; private readonly now: () => number; constructor(workspaceRoot: string, options: CreatePlanStoreOptions) { - this.durabilityRoot = resolve(workspaceRoot); - this.sessionsRoot = join(this.durabilityRoot, 'sessions'); + this.#lease = acquireOperationalStateDatabase(resolve(workspaceRoot)); this.newId = options.newId ?? randomUUID; this.now = options.now ?? Date.now; } + ready(): Promise { + return Promise.resolve(); + } + + close(): void { + this.#lease.close(); + } + async readState(sessionId: string): Promise { return (await this.readLedger(sessionId)).state; } @@ -357,216 +349,19 @@ class FilePlanStore implements PlanStore { sessionId: string, ): Promise<{ events: PlanEvent[]; state: PlanSessionState }> { assertSafeId(sessionId); - const canonical = await this.readCanonicalLedger(sessionId); - if (canonical) return canonical; - let text: string; - try { - text = await readFile(this.eventsPath(sessionId), 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return { events: [], state: emptyPlanSessionState(sessionId) }; - } - throw error; - } - const rawLines = text.split('\n'); - const events: PlanEvent[] = []; - for (let index = 0; index < rawLines.length; index += 1) { - const line = rawLines[index]!; - if (!line.trim()) continue; - try { - events.push(decodePlanEvent(JSON.parse(line), sessionId)); - } catch (error) { - const isLast = index === rawLines.length - 1; - if (isLast && !text.endsWith('\n') && classifyJsonRecord(line) === 'incomplete-prefix') { - continue; - } - throw new Error(`Invalid Plan event at line ${index + 1}`, { cause: error }); - } - } - let state = emptyPlanSessionState(sessionId); - for (const event of events) state = applyPlanEvent(state, event); - return { events, state }; - } - - protected async readCanonicalLedger( - _sessionId: string, - ): Promise<{ events: PlanEvent[]; state: PlanSessionState } | undefined> { - return undefined; - } - - protected async appendCanonicalEvent( - sessionId: string, - event: PlanEvent, - state: PlanSessionState, - ): Promise { - await mkdir(this.sessionDir(sessionId), { recursive: true }); - await appendJsonl(this.eventsPath(sessionId), `${JSON.stringify(event)}\n`, { - durable: true, - durabilityRoot: this.durabilityRoot, - }); - await this.writeProjection(sessionId, state).catch(() => { - // Derived cache only. The append-only event ledger remains authoritative. - }); - } - - private async writeProjection(sessionId: string, state: PlanSessionState): Promise { - const path = this.projectionPath(sessionId); - await mkdir(dirname(path), { recursive: true }); - const temp = `${path}.${process.pid}.${this.newId()}.tmp`; - await writeFile(temp, `${JSON.stringify(state)}\n`, { encoding: 'utf8', mode: 0o600 }); - try { - await rename(temp, path); - } finally { - await rm(temp, { force: true }).catch(() => {}); - } - } - - private sessionDir(sessionId: string): string { - return join(this.sessionsRoot, sessionId); - } - - private eventsPath(sessionId: string): string { - return join(this.sessionDir(sessionId), 'plan-events.jsonl'); - } - - private projectionPath(sessionId: string): string { - return join(this.sessionDir(sessionId), 'plans.json'); - } -} - -class SqlitePlanStoreImpl extends FilePlanStore implements SqlitePlanStore { - readonly #root: string; - readonly #lease: OperationalStateDatabaseLease; - readonly #ready: Promise; - - constructor(workspaceRoot: string, options: CreateSqlitePlanStoreOptions) { - super(workspaceRoot, options); - this.#root = resolve(workspaceRoot); - this.#lease = acquireOperationalStateDatabase(this.#root); - this.#ready = importLegacyPlanState(this.#root, this.#lease, options.failpoint); - } - - ready(): Promise { - return this.#ready; - } - - close(): void { - this.#lease.close(); + return readSqlitePlanLedger(this.#lease.database, sessionId); } - protected override async appendCanonicalEvent( + private async appendCanonicalEvent( sessionId: string, event: PlanEvent, state: PlanSessionState, ): Promise { - await this.#ready; this.#lease.transaction('write', () => { insertPlanEvent(this.#lease.database, event); writePlanProjection(this.#lease.database, sessionId, state); }); } - - protected override async readCanonicalLedger( - sessionId: string, - ): Promise<{ events: PlanEvent[]; state: PlanSessionState } | undefined> { - await this.#ready; - return readSqlitePlanLedger(this.#lease.database, sessionId); - } -} - -interface LegacyPlanLedger { - sessionId: string; - events: PlanEvent[]; - state: PlanSessionState; -} - -async function importLegacyPlanState( - root: string, - lease: OperationalStateDatabaseLease, - failpoint?: (point: OperationalStoreCutoverFailpoint) => void, -): Promise { - const ledgers = await readLegacyPlanLedgers(root); - const fingerprint = `sha256:${createHash('sha256') - .update(JSON.stringify(ledgers)) - .digest('hex')}`; - completeOperationalStoreCutover(lease, { - storeName: 'workflow_plan', - sourcePath: join(root, 'sessions'), - sourceFingerprint: fingerprint, - failpoint, - importAndValidate: (database) => { - let eventCount = 0; - for (const ledger of ledgers) { - for (const event of ledger.events) { - insertOrValidatePlanEvent(database, event); - eventCount += 1; - } - writePlanProjection(database, ledger.sessionId, ledger.state); - } - const persisted = database - .prepare('SELECT COUNT(*) AS count FROM workflow_plan_events') - .get() as { count?: unknown }; - if (persisted.count !== eventCount) { - throw new Error('Plan cutover row-count validation failed'); - } - return { sessions: ledgers.length, events: eventCount }; - }, - }); -} - -async function readLegacyPlanLedgers(root: string): Promise { - const sessionsRoot = join(root, 'sessions'); - let entries; - try { - entries = await readdir(sessionsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const result: LegacyPlanLedger[] = []; - for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { - if (!entry.isDirectory() || !SAFE_ID_PATTERN.test(entry.name)) continue; - const eventsPath = join(sessionsRoot, entry.name, 'plan-events.jsonl'); - let text: string | undefined; - try { - text = await readFile(eventsPath, 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - } - const events = text === undefined ? [] : decodeLegacyPlanEventText(text, entry.name); - let state = emptyPlanSessionState(entry.name); - for (const event of events) state = applyPlanEvent(state, event); - const projectionPath = join(sessionsRoot, entry.name, 'plans.json'); - try { - const projection = JSON.parse(await readFile(projectionPath, 'utf8')) as PlanSessionState; - if (JSON.stringify(projection) !== JSON.stringify(state)) { - throw new Error(`Plan projection does not match its event ledger for ${entry.name}`); - } - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - } - if (events.length > 0) result.push({ sessionId: entry.name, events, state }); - } - return result; -} - -function decodeLegacyPlanEventText(text: string, sessionId: string): PlanEvent[] { - const lines = text.split('\n'); - const events: PlanEvent[] = []; - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index]!; - if (!line.trim()) continue; - try { - events.push(decodePlanEvent(JSON.parse(line), sessionId)); - } catch (error) { - const isLast = index === lines.length - 1; - if (isLast && !text.endsWith('\n') && classifyJsonRecord(line) === 'incomplete-prefix') { - continue; - } - throw new Error(`Invalid Plan event at line ${index + 1}`, { cause: error }); - } - } - return events; } function readSqlitePlanLedger( @@ -613,23 +408,6 @@ function insertPlanEvent(database: DatabaseSync, event: PlanEvent): void { .run(event.sessionId, row.sequence, event.id, event.storeVersion, JSON.stringify(event)); } -function insertOrValidatePlanEvent(database: DatabaseSync, event: PlanEvent): void { - const existing = database - .prepare(` - SELECT record_json - FROM workflow_plan_events - WHERE session_id = ? AND store_version = ? - `) - .get(event.sessionId, event.storeVersion) as { record_json?: unknown } | undefined; - if (existing) { - if (existing.record_json !== JSON.stringify(event)) { - throw new Error(`Plan cutover conflict: ${event.sessionId}:${event.storeVersion}`); - } - return; - } - insertPlanEvent(database, event); -} - function writePlanProjection( database: DatabaseSync, sessionId: string, diff --git a/packages/storage/src/pricing-store.ts b/packages/storage/src/pricing-store.ts index ddffe5a9df..08a76417e1 100644 --- a/packages/storage/src/pricing-store.ts +++ b/packages/storage/src/pricing-store.ts @@ -1,24 +1,4 @@ -import { randomUUID } from 'node:crypto'; -import { mkdir, open, readFile, rename, rm } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; -import { - canonicalPricingConfigsEqual, - comparePricingModelKeys, - normalizePricingConfig, - normalizePricingModelKey, - validateCanonicalPricingConfig, -} from '@maka/core/usage-stats/pricing'; import type { PricingConfig } from '@maka/core/usage-stats/types'; -import { throwDeduplicatedFailures } from './failure-utils.js'; -import { syncDirectory } from './stable-storage.js'; - -const PRICING_DOCUMENT_VERSION = 1; - -interface PricingDocument { - readonly version: 1; - readonly revision: number; - readonly overrides: readonly Readonly[]; -} export interface PricingSnapshot { readonly revision: number; @@ -43,7 +23,6 @@ export interface PricingStore { export interface CreatePricingStoreOptions { readonly createIfMissing?: boolean; - readonly initialOverrides?: readonly unknown[]; } export class PricingStoreClosedError extends Error { @@ -94,293 +73,3 @@ export class PricingCommitUnknownError extends Error { this.name = 'PricingCommitUnknownError'; } } - -export function createPricingStore( - workspaceRoot: string, - options: CreatePricingStoreOptions = {}, -): PricingStore { - return new FilePricingStore( - workspaceRoot, - options.createIfMissing ?? true, - options.initialOverrides ?? [], - ); -} - -class FilePricingStore implements PricingStore { - private readonly path: string; - private document: PricingDocument; - private loaded = false; - private queue: Promise = Promise.resolve(); - private failure: PricingCommitUnknownError | PricingStorePublicationError | undefined; - private state: 'open' | 'draining' | 'closed' = 'open'; - private loadPromise: Promise | undefined; - private closePromise: Promise | undefined; - - constructor( - workspaceRoot: string, - private readonly createIfMissing: boolean, - private readonly initialOverrides: readonly unknown[], - ) { - this.path = join(workspaceRoot, 'pricing.json'); - this.document = makeDocument(0, []); - } - - load(): Promise { - if (this.loaded) return Promise.resolve(); - this.assertOpen(); - if (this.loadPromise) return this.loadPromise; - const operation = this.loadFromDisk(); - this.loadPromise = operation; - void operation.catch(() => { - if (this.state === 'open' && this.loadPromise === operation) { - this.loadPromise = undefined; - } - }); - return operation; - } - - private async loadFromDisk(): Promise { - try { - this.document = decodeDocument(JSON.parse(await readFile(this.path, 'utf8'))); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - this.document = makeDocument(0, normalizeLegacyOverrides(this.initialOverrides)); - if (this.createIfMissing) await this.publish(this.document); - } - this.loaded = true; - } - - snapshot(): PricingSnapshot { - this.assertReady(); - if (this.failure instanceof PricingCommitUnknownError) throw this.failure; - return cloneSnapshot(this.document); - } - - upsert(expectedRevision: number, pricing: PricingConfig): Promise { - assertRevision(expectedRevision, 'expectedRevision'); - const normalized = normalizePricingConfig(pricing); - if (!normalized.ok) throw new PricingValidationError(normalized.error); - const admitted = freezePricing(normalized.value); - return this.enqueueMutation(expectedRevision, (current) => { - const existing = current.find((item) => item.modelKey === admitted.modelKey); - if (existing && canonicalPricingConfigsEqual(existing, admitted)) return current; - return [...current.filter((item) => item.modelKey !== admitted.modelKey), admitted].sort( - (left, right) => comparePricingModelKeys(left.modelKey, right.modelKey), - ); - }); - } - - delete(expectedRevision: number, modelKey: string): Promise { - assertRevision(expectedRevision, 'expectedRevision'); - const normalized = normalizePricingModelKey(modelKey); - if (!normalized.ok) throw new PricingValidationError(normalized.error); - return this.enqueueMutation(expectedRevision, (current) => - current.some((item) => item.modelKey === normalized.value) - ? current.filter((item) => item.modelKey !== normalized.value) - : current, - ); - } - - async flush(): Promise { - this.assertLoaded(); - await this.queue; - if (this.failure) throw this.failure; - } - - beginDrain(): Promise { - if (this.state === 'open') this.state = 'draining'; - return this.flush(); - } - - close(): Promise { - if (this.closePromise) return this.closePromise; - this.state = 'draining'; - this.closePromise = Promise.allSettled([ - this.loadPromise ?? Promise.resolve(), - this.queue.then(() => { - if (this.failure) throw this.failure; - }), - ]) - .then((results) => { - throwDeduplicatedFailures( - 'Unable to close pricing store', - results.flatMap((result) => (result.status === 'rejected' ? [result.reason] : [])), - ); - }) - .finally(() => { - this.state = 'closed'; - }); - return this.closePromise; - } - - private enqueueMutation( - expectedRevision: number, - mutate: (current: readonly Readonly[]) => readonly Readonly[], - ): Promise { - this.assertReady(); - const operation = this.queue.then(async () => { - if (this.failure) throw this.failure; - const current = this.document; - if (current.revision !== expectedRevision) { - throw new PricingRevisionConflictError(expectedRevision, current.revision); - } - const overrides = mutate(current.overrides); - if (overrides === current.overrides) { - return { committed: false, changed: false, snapshot: cloneSnapshot(current) }; - } - if (current.revision === Number.MAX_SAFE_INTEGER) { - throw new PricingValidationError('revision cannot advance beyond Number.MAX_SAFE_INTEGER'); - } - const candidate = makeDocument(current.revision + 1, overrides); - try { - await this.publish(candidate); - } catch (error) { - if ( - error instanceof PricingCommitUnknownError || - error instanceof PricingStorePublicationError - ) { - this.failure = error; - } - throw error; - } - this.document = candidate; - return { committed: true, changed: true, snapshot: cloneSnapshot(candidate) }; - }); - this.queue = operation.then( - () => undefined, - () => undefined, - ); - return operation; - } - - private async publish(document: PricingDocument): Promise { - const directory = dirname(this.path); - const temporaryPath = `${this.path}.${randomUUID()}.tmp`; - let handle: Awaited> | undefined; - let published = false; - try { - await mkdir(directory, { recursive: true }); - handle = await open(temporaryPath, 'wx', 0o600); - await handle.writeFile(JSON.stringify(document, null, 2) + '\n', 'utf8'); - await handle.sync(); - await handle.close(); - handle = undefined; - await rename(temporaryPath, this.path); - published = true; - await syncDirectory(directory); - } catch (cause) { - if (published) throw new PricingCommitUnknownError({ cause }); - throw new PricingStorePublicationError({ cause }); - } finally { - await handle?.close().catch(() => undefined); - if (!published) await rm(temporaryPath, { force: true }).catch(() => undefined); - } - } - - private assertLoaded(): void { - if (!this.loaded) throw new PricingStoreNotLoadedError(); - } - - private assertOpen(): void { - if (this.state !== 'open') throw new PricingStoreClosedError(); - } - - private assertReady(): void { - this.assertOpen(); - this.assertLoaded(); - } -} - -function decodeDocument(input: unknown): PricingDocument { - if (!isRecord(input)) throw new PricingValidationError('expected an object'); - const keys = Object.keys(input); - if ( - keys.length !== 3 || - !keys.includes('version') || - !keys.includes('revision') || - !keys.includes('overrides') - ) { - throw new PricingValidationError('document must contain exactly version, revision, overrides'); - } - if (input.version !== PRICING_DOCUMENT_VERSION) { - throw new PricingValidationError(`expected version ${PRICING_DOCUMENT_VERSION}`); - } - assertRevision(input.revision, 'revision'); - if (!Array.isArray(input.overrides)) { - throw new PricingValidationError('overrides must be an array'); - } - return makeDocument(input.revision, decodeCanonicalOverrides(input.overrides)); -} - -function normalizeLegacyOverrides(input: readonly unknown[]): readonly Readonly[] { - const result = input.map((value, index) => { - const normalized = normalizePricingConfig(value); - if (!normalized.ok) { - throw new PricingValidationError(`overrides[${index}]: ${normalized.error}`); - } - return freezePricing(normalized.value); - }); - const keys = new Set(); - for (const item of result) { - if (keys.has(item.modelKey)) { - throw new PricingValidationError(`duplicate modelKey: ${item.modelKey}`); - } - keys.add(item.modelKey); - } - return result.sort((left, right) => comparePricingModelKeys(left.modelKey, right.modelKey)); -} - -function decodeCanonicalOverrides(input: readonly unknown[]): readonly Readonly[] { - const result = input.map((value, index) => { - const canonical = validateCanonicalPricingConfig(value); - if (!canonical.ok) { - throw new PricingValidationError(`overrides[${index}]: ${canonical.error}`); - } - return freezePricing(canonical.value); - }); - for (let index = 1; index < result.length; index += 1) { - const previous = result[index - 1]; - const current = result[index]; - if (!previous || !current) continue; - const order = comparePricingModelKeys(previous.modelKey, current.modelKey); - if (order === 0) { - throw new PricingValidationError(`duplicate modelKey: ${current.modelKey}`); - } - if (order > 0) { - throw new PricingValidationError('overrides must be sorted by modelKey'); - } - } - return result; -} - -function makeDocument( - revision: number, - overrides: readonly Readonly[], -): PricingDocument { - return Object.freeze({ - version: PRICING_DOCUMENT_VERSION, - revision, - overrides: Object.freeze(overrides.map(freezePricing)), - }); -} - -function cloneSnapshot(document: PricingDocument): PricingSnapshot { - return Object.freeze({ - revision: document.revision, - overrides: Object.freeze(document.overrides.map(freezePricing)), - }); -} - -function freezePricing(pricing: PricingConfig): Readonly { - return Object.freeze({ ...pricing }); -} - -function assertRevision(value: unknown, label: string): asserts value is number { - if (!Number.isSafeInteger(value) || (value as number) < 0) { - throw new PricingValidationError(`${label} must be a nonnegative safe integer`); - } -} - -function isRecord(value: unknown): value is Record { - return typeof value === 'object' && value !== null && !Array.isArray(value); -} diff --git a/packages/storage/src/project-catalog.ts b/packages/storage/src/project-catalog.ts index e6d7cfa524..9fe69b247f 100644 --- a/packages/storage/src/project-catalog.ts +++ b/packages/storage/src/project-catalog.ts @@ -38,7 +38,6 @@ export interface ProjectRelinkContext { export interface ProjectCatalog { list(): Promise; register(path: string): Promise; - importLegacyPath(path: string, usedAt?: number): Promise; select(projectId: string): Promise<{ project: ProjectRecord; path: string }>; touch(projectId: string, path?: string): Promise; relink( @@ -112,29 +111,6 @@ class FileProjectCatalog implements ProjectCatalog { return this.upsertResolvedProject(resolved, this.now()); } - async importLegacyPath(path: string, usedAt: number = this.now()): Promise { - let resolved: ResolvedProjectLocation; - try { - resolved = await resolveProjectLocation({ path }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - let pathIsMissing = false; - try { - await stat(path); - } catch (pathError) { - pathIsMissing = (pathError as NodeJS.ErrnoException).code === 'ENOENT'; - } - if (!pathIsMissing) throw error; - const canonicalPath = normalize(resolve(path)); - resolved = { - canonicalPath, - identity: `folder:${canonicalPath}`, - kind: 'folder', - }; - } - return this.upsertResolvedProject(resolved, usedAt); - } - private async upsertResolvedProject( resolved: ResolvedProjectLocation, timestamp: number, diff --git a/packages/storage/src/project-session-migration.ts b/packages/storage/src/project-session-migration.ts deleted file mode 100644 index a9cc9e2c6a..0000000000 --- a/packages/storage/src/project-session-migration.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { SessionStore } from './session-store.js'; -import type { ProjectCatalog } from './project-catalog.js'; - -export interface ProjectSessionMigrationResult { - migrated: number; - unchanged: number; - failed: number; -} - -export async function migrateSessionProjects(input: { - sessions: Pick; - catalog: Pick; -}): Promise { - const headers = await input.sessions.listHeaders(); - let migrated = 0; - let unchanged = 0; - let failed = 0; - - for (const header of headers) { - if (header.projectId !== undefined) { - unchanged++; - continue; - } - try { - const project = await input.catalog.importLegacyPath( - header.cwd, - header.lastMessageAt ?? header.lastUsedAt, - ); - await input.sessions.updateHeader(header.id, { projectId: project.id }); - migrated++; - } catch { - failed++; - } - } - - return { migrated, unchanged, failed }; -} diff --git a/packages/storage/src/runtime-event-persistence.ts b/packages/storage/src/runtime-event-persistence.ts new file mode 100644 index 0000000000..5efb86571b --- /dev/null +++ b/packages/storage/src/runtime-event-persistence.ts @@ -0,0 +1,73 @@ +import { join } from 'node:path'; +import type { RuntimeEvent } from '@maka/core'; +import type { BoundedEvidenceReadResult, EvidenceReadBudget } from './agent-run-store.js'; +import { createSqliteRuntimeStore, type SqliteRuntimeStore } from './sqlite-runtime-store.js'; +import { + acquireOperationalStateDatabase, + OPERATIONAL_STATE_DATABASE_NAME, +} from './operational-state-store.js'; + +export type RuntimeEventPersistence = { + kind: 'sqlite'; + runtimeEventStore: SqliteRuntimeStore; + runtimeCommitStore: SqliteRuntimeStore; + close(): void; +}; + +export type RuntimeEventReadPersistence = { + kind: 'sqlite'; + runtimeEventStore: RuntimeEventReadStore; + close(): void; +}; + +export interface RuntimeEventReadStore { + readRuntimeEvents(sessionId: string, runId: string): Promise; + readRuntimeEventsBounded( + sessionId: string, + runId: string, + budget: EvidenceReadBudget, + ): Promise>; + readImmutableRuntimeEvents(sessionId: string, runId: string): Promise; + readSessionRuntimeEvents(sessionId: string): Promise; +} + +export async function openRuntimeEventPersistence(input: { + workspaceRoot: string; +}): Promise { + const store = createWorkspaceRuntimeStore(input.workspaceRoot); + return { + kind: 'sqlite', + runtimeEventStore: store, + runtimeCommitStore: store, + close: () => store.close(), + }; +} + +export function createWorkspaceRuntimeStore(workspaceRoot: string): SqliteRuntimeStore { + const databaseLease = acquireOperationalStateDatabase(workspaceRoot); + return createSqliteRuntimeStore(join(workspaceRoot, OPERATIONAL_STATE_DATABASE_NAME), { + databaseLease, + }); +} + +export async function openRuntimeEventReadPersistence(input: { + workspaceRoot: string; +}): Promise { + const store = createSqliteRuntimeStore( + join(input.workspaceRoot, OPERATIONAL_STATE_DATABASE_NAME), + { readOnly: true }, + ); + return { + kind: 'sqlite', + runtimeEventStore: Object.freeze({ + readRuntimeEvents: (sessionId: string, runId: string) => + store.readRuntimeEvents(sessionId, runId), + readRuntimeEventsBounded: (sessionId: string, runId: string, budget: EvidenceReadBudget) => + store.readRuntimeEventsBounded(sessionId, runId, budget), + readImmutableRuntimeEvents: (sessionId: string, runId: string) => + store.readImmutableRuntimeEvents(sessionId, runId), + readSessionRuntimeEvents: (sessionId: string) => store.readSessionRuntimeEvents(sessionId), + }), + close: () => store.close(), + }; +} diff --git a/packages/storage/src/runtime-event-transfer.ts b/packages/storage/src/runtime-event-transfer.ts deleted file mode 100644 index 6aaa414f8c..0000000000 --- a/packages/storage/src/runtime-event-transfer.ts +++ /dev/null @@ -1,317 +0,0 @@ -import { readFile, readdir, stat } from 'node:fs/promises'; -import { join } from 'node:path'; -import { decodePersistedRuntimeEvent, type RuntimeEvent } from '@maka/core'; -import { - createRuntimeEventStore, - type BoundedEvidenceReadResult, - type EvidenceReadBudget, -} from './agent-run-store.js'; -import { classifyJsonRecord } from './json-prefix.js'; -import type { SqliteRuntimeStore } from './sqlite-runtime-store.js'; -import { createSqliteRuntimeStore } from './sqlite-runtime-store.js'; -import { - acquireOperationalStateDatabase, - OPERATIONAL_STATE_DATABASE_NAME, -} from './operational-state-store.js'; - -export const SQLITE_RUNTIME_DATABASE_NAME = OPERATIONAL_STATE_DATABASE_NAME; -const SAFE_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; - -export type RuntimeEventPersistence = { - kind: 'sqlite'; - runtimeEventStore: SqliteRuntimeStore; - runtimeCommitStore: SqliteRuntimeStore; - importReport?: LegacyRuntimeEventImportReport; - close(): void; -}; - -export type RuntimeEventReadPersistence = { - kind: 'jsonl' | 'sqlite'; - runtimeEventStore: RuntimeEventReadStore; - close(): void; -}; - -export interface RuntimeEventExportSource { - readRuntimeEvents(sessionId: string, runId: string): Promise; - readImmutableRuntimeEvents?(sessionId: string, runId: string): Promise; -} - -export interface RuntimeEventReadStore extends RuntimeEventExportSource { - readRuntimeEventsBounded( - sessionId: string, - runId: string, - budget: EvidenceReadBudget, - ): Promise>; - readImmutableRuntimeEvents(sessionId: string, runId: string): Promise; - readSessionRuntimeEvents(sessionId: string): Promise; -} - -export interface RuntimeEventImportReport { - eventsRead: number; - eventsImported: number; - eventsExisting: number; -} - -export interface LegacyRuntimeEventImportReport extends RuntimeEventImportReport { - filesScanned: number; -} - -export async function openRuntimeEventPersistence(input: { - workspaceRoot: string; -}): Promise { - const databasePath = join(input.workspaceRoot, SQLITE_RUNTIME_DATABASE_NAME); - const databaseLease = acquireOperationalStateDatabase(input.workspaceRoot); - const store = createSqliteRuntimeStore(databasePath, { databaseLease }); - try { - const importReport = await importLegacyRuntimeEventJsonlTree({ - workspaceRoot: input.workspaceRoot, - destination: store, - }); - return { - kind: 'sqlite', - runtimeEventStore: store, - runtimeCommitStore: store, - importReport, - close: () => store.close(), - }; - } catch (error) { - store.close(); - throw error; - } -} - -/** - * Open the canonical RuntimeEvent read model without mutating the storage root. - * - * A legacy-only root remains readable until a writer performs the one-way - * import. Once runtime.sqlite exists, readers never merge or fall back to - * JSONL, so SQLite-only facts cannot disappear behind a stale file projection. - */ -export async function openRuntimeEventReadPersistence(input: { - workspaceRoot: string; -}): Promise { - const databasePath = join(input.workspaceRoot, SQLITE_RUNTIME_DATABASE_NAME); - if (!(await pathExists(databasePath))) { - return { - kind: 'jsonl', - runtimeEventStore: asRuntimeEventReader(createRuntimeEventStore(input.workspaceRoot)), - close: () => {}, - }; - } - const store = createSqliteRuntimeStore(databasePath, { readOnly: true }); - return { - kind: 'sqlite', - runtimeEventStore: asRuntimeEventReader(store), - close: () => store.close(), - }; -} - -async function pathExists(path: string): Promise { - try { - await stat(path); - return true; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; - throw error; - } -} - -export async function exportRuntimeEventsToJsonl( - source: RuntimeEventExportSource, - sessionId: string, - runId: string, -): Promise { - const events = source.readImmutableRuntimeEvents - ? await source.readImmutableRuntimeEvents(sessionId, runId) - : await source.readRuntimeEvents(sessionId, runId); - return events.length === 0 ? '' : `${events.map((event) => JSON.stringify(event)).join('\n')}\n`; -} - -function asRuntimeEventReader(store: RuntimeEventReadStore): RuntimeEventReadStore { - return Object.freeze({ - readRuntimeEvents: (sessionId: string, runId: string) => - store.readRuntimeEvents(sessionId, runId), - readRuntimeEventsBounded: (sessionId: string, runId: string, budget: EvidenceReadBudget) => - store.readRuntimeEventsBounded(sessionId, runId, budget), - readImmutableRuntimeEvents: (sessionId: string, runId: string) => - store.readImmutableRuntimeEvents(sessionId, runId), - readSessionRuntimeEvents: (sessionId: string) => store.readSessionRuntimeEvents(sessionId), - }); -} - -export async function importRuntimeEventsFromJsonl(input: { - jsonl: string; - sessionId: string; - runId: string; - destination: SqliteRuntimeStore; -}): Promise { - const events = parseRuntimeEventJsonl(input.jsonl, input.sessionId, input.runId); - return importRuntimeEvents(events, input.sessionId, input.runId, input.destination); -} - -export async function importLegacyRuntimeEventJsonlTree(input: { - workspaceRoot: string; - destination: SqliteRuntimeStore; -}): Promise { - const sessionsRoot = join(input.workspaceRoot, 'sessions'); - const report: LegacyRuntimeEventImportReport = { - filesScanned: 0, - eventsRead: 0, - eventsImported: 0, - eventsExisting: 0, - }; - for (const session of await directoryNames(sessionsRoot)) { - const runsRoot = join(sessionsRoot, session, 'runs'); - for (const run of await directoryNames(runsRoot)) { - const sourcePath = join(runsRoot, run, 'runtime-events.jsonl'); - const sourceStat = await stat(sourcePath).catch((error: NodeJS.ErrnoException) => { - if (error.code === 'ENOENT') return undefined; - throw error; - }); - if (!sourceStat) continue; - const fingerprint = `${sourceStat.size}:${sourceStat.mtimeMs}`; - if (await input.destination.isRuntimeImportSourceCurrent(sourcePath, fingerprint)) continue; - const events = parseLegacyRuntimeEventJsonl(await readFile(sourcePath, 'utf8'), session, run); - report.filesScanned += 1; - const imported = await importRuntimeEvents(events, session, run, input.destination, { - path: sourcePath, - fingerprint, - }); - report.eventsRead += imported.eventsRead; - report.eventsImported += imported.eventsImported; - report.eventsExisting += imported.eventsExisting; - } - } - return report; -} - -async function importRuntimeEvents( - events: readonly RuntimeEvent[], - sessionId: string, - runId: string, - destination: SqliteRuntimeStore, - source?: { path: string; fingerprint: string }, -): Promise { - const report: RuntimeEventImportReport = { - eventsRead: events.length, - eventsImported: 0, - eventsExisting: 0, - }; - for (const event of events) { - assertRuntimeEventImportIdentity(event, sessionId, runId); - } - const imported = await destination.importRuntimeEventsBatch({ - sessionId, - runId, - events, - ...(source ? { source } : {}), - }); - for (const created of imported.created) { - if (created) report.eventsImported += 1; - else report.eventsExisting += 1; - } - return report; -} - -function parseRuntimeEventJsonl(jsonl: string, sessionId: string, runId: string): RuntimeEvent[] { - const events: RuntimeEvent[] = []; - const lines = jsonl.split('\n'); - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index]; - if (!line?.trim()) continue; - let event: RuntimeEvent; - try { - event = JSON.parse(line) as RuntimeEvent; - } catch (error) { - throw new Error(`Invalid RuntimeEvent JSONL line ${index + 1} for run ${runId}`, { - cause: error, - }); - } - assertRuntimeEventImportIdentity(event, sessionId, runId); - if (event.partial === true) { - throw new Error(`Partial RuntimeEvent ${event.id} cannot be imported as immutable JSONL`); - } - events.push(event); - } - return events; -} - -function parseLegacyRuntimeEventJsonl( - jsonl: string, - sessionId: string, - runId: string, -): RuntimeEvent[] { - const events: RuntimeEvent[] = []; - const lines = jsonl.split('\n'); - let lastNonEmptyIndex = -1; - for (let index = lines.length - 1; index >= 0; index -= 1) { - if (lines[index]?.trim()) { - lastNonEmptyIndex = index; - break; - } - } - const endsWithNewline = jsonl.endsWith('\n'); - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index]; - if (!line?.trim()) continue; - let event: RuntimeEvent; - try { - const parsed: unknown = JSON.parse(line); - if (isLegacyStreamPartialSnapshot(parsed)) continue; - event = decodePersistedRuntimeEvent(parsed); - } catch (error) { - if ( - !endsWithNewline && - index === lastNonEmptyIndex && - classifyJsonRecord(line) === 'incomplete-prefix' - ) { - continue; - } - throw new Error(`Invalid legacy RuntimeEvent JSONL line ${index + 1} for run ${runId}`, { - cause: error, - }); - } - assertRuntimeEventImportIdentity(event, sessionId, runId); - events.push(event); - } - return events; -} - -function assertRuntimeEventImportIdentity( - event: RuntimeEvent, - sessionId: string, - runId: string, -): void { - if ( - !event || - typeof event !== 'object' || - event.sessionId !== sessionId || - event.runId !== runId - ) { - throw new Error(`RuntimeEvent import identity mismatch for session ${sessionId}, run ${runId}`); - } -} - -// Legacy JSONL logs may physically contain stream partial snapshots written by -// older versions. They are mutable projection state, not immutable facts: a -// completed stream leaves a separate durable final event, and a dangling -// partial is already handled by the replay boundary gates. Legacy tree import -// skips them; the strict importRuntimeEventsFromJsonl API still rejects them. -function isLegacyStreamPartialSnapshot(value: unknown): boolean { - if (!value || typeof value !== 'object') return false; - const event = value as Record; - return event.partial === true && event.status === undefined && event.actions === undefined; -} - -async function directoryNames(root: string): Promise { - try { - const entries = await readdir(root, { withFileTypes: true }); - return entries - .filter((entry) => entry.isDirectory() && SAFE_ID_PATTERN.test(entry.name)) - .map((entry) => entry.name) - .sort(); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } -} diff --git a/packages/storage/src/session-bundle-policy.ts b/packages/storage/src/session-bundle-policy.ts index 063d2af371..e5db31620c 100644 --- a/packages/storage/src/session-bundle-policy.ts +++ b/packages/storage/src/session-bundle-policy.ts @@ -1,105 +1,17 @@ -import { - copyFile, - lstat, - mkdir, - readFile, - readdir, - realpath, - rm, - rename, - writeFile, -} from 'node:fs/promises'; +import { copyFile, lstat, mkdir, readFile, readdir, realpath, rename, rm } from 'node:fs/promises'; import { randomUUID } from 'node:crypto'; -import { createRequire } from 'node:module'; -import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'; -import type { DatabaseSync } from 'node:sqlite'; -import type { RuntimeEvent } from '@maka/core'; +import { dirname, isAbsolute, join, relative, resolve, sep } from 'node:path'; +import { DatabaseSync } from 'node:sqlite'; import type { ArtifactRecord } from '@maka/core/artifacts'; -import { - ARTIFACT_PUBLICATION_STAGING_PATTERN, - ARTIFACT_PURGE_INTENT_FILE, - ARTIFACT_WRITER_LOCK_FILE, - isCanonicalArtifactRecoveryTempName, -} from './artifact-storage-layout.js'; -import { decodeArtifactMetadata } from './artifact-metadata-codec.js'; +import { decodeArtifactRecordJsons } from './artifact-metadata-codec.js'; import { withArtifactWriterLock } from './artifact-writer-lock.js'; -import { LONG_TERM_MEMORY_DATABASE_NAME } from './long-term-memory-store.js'; -import { exportLegacySessionTreeSnapshot } from './session-metadata-maintenance.js'; -import { - encodeExecutionBoundaryTransfer, - EXECUTION_BOUNDARY_TRANSFER_FILE, -} from './session-metadata-transfer.js'; import { - LEGACY_SESSION_METADATA_DATABASE_NAME, + acquireOperationalStateDatabase, OPERATIONAL_STATE_DATABASE_NAME, } from './operational-state-store.js'; -import { createSqliteSessionMetadataStore } from './sqlite-session-metadata-store.js'; -import type { SessionAuthoritySnapshot } from './sqlite-session-metadata-store.js'; -import { createSqliteRuntimeStore } from './sqlite-runtime-store.js'; -import { - artifactMetadataSourceFingerprint, - createSqliteArtifactMetadataRepository, -} from './sqlite-artifact-metadata.js'; - -/** - * The first bundle slice is deliberately an uncompressed state-tree export. - * Compression, manifests, and activation inputs belong to later bundle work; - * this module owns the trust boundary that all of those consumers must use. - */ -export const SESSION_BUNDLE_STATE_ENTRIES = ['sessions', 'artifacts', 'runtime.sqlite'] as const; - -export const SESSION_BUNDLE_PORTABLE_SESSION_DIRECTORIES = [ - 'deep-research', - 'projections', - 'runs', - 'shell-runs', - 'turn-admissions', -] as const; -export const SESSION_BUNDLE_PORTABLE_SESSION_FILES = [ - EXECUTION_BOUNDARY_TRANSFER_FILE, - 'plan-events.jsonl', - 'plans.json', - 'task-events.jsonl', - 'tasks.json', -] as const; - -export const SESSION_BUNDLE_PROTECTED_ENTRIES = [ - '.maka-storage-root.json', - '.maka_cli_claude_device_id', - 'credentials.json', - 'llm-connections.json', - 'settings.json', - 'automations.json', - 'mcp.json', - 'skills', - 'memory', - 'daily-reviews', - 'logs', - 'log', - 'activation.json', - 'activation-input.json', - 'task-runs', - '.maka', - ARTIFACT_WRITER_LOCK_FILE, - LEGACY_SESSION_METADATA_DATABASE_NAME, - `${LEGACY_SESSION_METADATA_DATABASE_NAME}-wal`, - `${LEGACY_SESSION_METADATA_DATABASE_NAME}-shm`, - `${LEGACY_SESSION_METADATA_DATABASE_NAME}-journal`, - LONG_TERM_MEMORY_DATABASE_NAME, - `${LONG_TERM_MEMORY_DATABASE_NAME}-wal`, - `${LONG_TERM_MEMORY_DATABASE_NAME}-shm`, - `${LONG_TERM_MEMORY_DATABASE_NAME}-journal`, - 'runtime.sqlite-wal', - 'runtime.sqlite-shm', - 'runtime.sqlite-journal', -] as const; - -const allowedEntries = new Set(SESSION_BUNDLE_STATE_ENTRIES); -const protectedEntries = new Set(SESSION_BUNDLE_PROTECTED_ENTRIES); -const portableSessionDirectories = new Set(SESSION_BUNDLE_PORTABLE_SESSION_DIRECTORIES); -const portableSessionFiles = new Set(SESSION_BUNDLE_PORTABLE_SESSION_FILES); -const require = createRequire(import.meta.url); +export const SESSION_BUNDLE_STATE_ENTRIES = ['artifacts', OPERATIONAL_STATE_DATABASE_NAME] as const; +export const SESSION_BUNDLE_PROTECTED_ENTRIES = [] as const; export type SessionBundleExportErrorCode = | 'invalid_root' @@ -124,18 +36,13 @@ export class SessionBundleExportError extends Error { export interface SessionBundleRootLayoutInput { stateRoot: string; configRoot: string; - /** Legacy callers intentionally use one shared root. */ allowShared?: boolean; } export interface SessionBundleExportPlanEntry { relativePath: string; kind: 'file' | 'directory'; - source: - | 'copy' - | 'selected_session_metadata' - | 'selected_session_boundary' - | 'filtered_runtime_sqlite'; + source: 'copy' | 'filtered_runtime_sqlite'; } export interface SessionBundleExportPlan { @@ -153,1210 +60,310 @@ export interface SessionBundleExportInput extends SessionBundleRootLayoutInput { sessionId: string; } -interface ArtifactMetadataSnapshot { - artifactRootExists: boolean; - metadataExists: boolean; - canonicalText: string; - selectedRecords: ArtifactRecord[]; -} - -interface PreparedSessionBundleExport { - plan: SessionBundleExportPlan; - artifactMetadata: ArtifactMetadataSnapshot; - requestedDestinationRoot: string; - stateRootIdentity: SessionBundleRootIdentity; -} - -interface SessionBundleRoots { - stateRoot: string; - stateRootIdentity: SessionBundleRootIdentity; - configRoot: string; - destinationRoot: string; -} - -interface SessionBundleRootIdentity { - dev: bigint; - ino: bigint; -} - -/** - * Validate the state/config split. Identical roots are accepted only for the - * legacy compatibility path; nested roots are never safe because one root - * could silently contain the other root's protected material. - */ export async function assertSessionBundleRootLayout( input: SessionBundleRootLayoutInput, ): Promise { - const stateRoot = await canonicalizeExistingOrMissingRoot(input.stateRoot, 'state'); - const configRoot = await canonicalizeExistingOrMissingRoot(input.configRoot, 'config'); - assertRootsDoNotOverlap(stateRoot, configRoot, input.allowShared === true); + const stateRoot = await canonicalRoot(input.stateRoot, 'state'); + const configRoot = await canonicalRoot(input.configRoot, 'config', true); + assertRootsSeparate(stateRoot, configRoot, input.allowShared === true); } -/** - * Build an auditable export plan. Only the three session-owned top-level - * entries are eligible. Known host/config entries are recorded as excluded; - * any new top-level entry fails closed until it is explicitly classified. - */ export async function planSessionBundleExport( input: SessionBundleExportInput, ): Promise { - return (await prepareSessionBundleExport(input)).plan; -} - -async function prepareSessionBundleExport( - input: SessionBundleExportInput, -): Promise { - const { stateRoot, stateRootIdentity, configRoot, destinationRoot } = - await resolveSessionBundleRoots(input); + assertSafeSessionId(input.sessionId); + const stateRoot = await canonicalRoot(input.stateRoot, 'state'); + const configRoot = await canonicalRoot(input.configRoot, 'config', true); + const destinationRoot = resolve(input.destinationRoot); + assertRootsSeparate(stateRoot, configRoot, input.allowShared === true); + assertRootsSeparate(stateRoot, destinationRoot, false); + assertRootsSeparate(configRoot, destinationRoot, false); - const includedEntries: string[] = []; - const excludedEntries: string[] = []; - const entries: SessionBundleExportPlanEntry[] = []; - const artifactMetadata = await readArtifactMetadataSnapshot(stateRoot, input.sessionId); - if (!artifactMetadata.artifactRootExists && artifactMetadata.selectedRecords.length > 0) { - throw new SessionBundleExportError( - 'invalid_root', - `Artifact metadata references a missing artifacts root for session ${input.sessionId}`, - ); - } - let sessionsClassified = false; - let artifactsClassified = false; - const topLevelEntries = await readdir(stateRoot, { withFileTypes: true }); - for (const entry of topLevelEntries.sort((a, b) => a.name.localeCompare(b.name))) { - const sourcePath = resolve(stateRoot, entry.name); - assertNoSymlink(entry.isSymbolicLink(), entry.name); - if (entry.name === 'sessions') { - sessionsClassified = true; - await planSelectedSessionTree( - sourcePath, - input.sessionId, - stateRoot, - excludedEntries, - entries, - ); - includedEntries.push(entry.name); - continue; - } - if (entry.name === 'artifacts') { - artifactsClassified = true; - await planSelectedArtifactTree( - sourcePath, - input.sessionId, - stateRoot, - excludedEntries, - entries, - artifactMetadata, + const databasePath = resolve(stateRoot, OPERATIONAL_STATE_DATABASE_NAME); + await assertRegularFile(databasePath, OPERATIONAL_STATE_DATABASE_NAME); + const database = new DatabaseSync(databasePath, { readOnly: true }); + let artifacts: ArtifactRecord[]; + try { + const session = database + .prepare('SELECT 1 AS present FROM session_metadata WHERE session_id = ?') + .get(input.sessionId); + if (!session) { + throw new SessionBundleExportError( + 'invalid_root', + `Session bundle session does not exist: ${input.sessionId}`, ); - includedEntries.push(entry.name); - continue; } - if (allowedEntries.has(entry.name)) { - includedEntries.push(entry.name); - if (entry.name === 'runtime.sqlite') { - await assertFile(sourcePath, 'runtime SQLite', stateRoot, entry.name); - entries.push({ - relativePath: entry.name, - kind: 'file', - source: 'filtered_runtime_sqlite', - }); - } else { - await inspectTree(sourcePath, entry.name, stateRoot, entries, 'copy'); + const rows = database + .prepare( + 'SELECT record_json FROM artifact_records WHERE session_id = ? ORDER BY created_at, storage_key', + ) + .all(input.sessionId) as Array<{ record_json?: unknown }>; + artifacts = decodeArtifactRecordJsons(rows.map((row) => row.record_json)); + } finally { + database.close(); + } + + const entries: SessionBundleExportPlanEntry[] = [ + { + relativePath: OPERATIONAL_STATE_DATABASE_NAME, + kind: 'file', + source: 'filtered_runtime_sqlite', + }, + ]; + const includedEntries = [OPERATIONAL_STATE_DATABASE_NAME]; + if (artifacts.length > 0) { + entries.push({ relativePath: 'artifacts', kind: 'directory', source: 'copy' }); + for (const artifact of artifacts) { + if (!isArtifactPathForSession(artifact.relativePath, input.sessionId)) { + throw new SessionBundleExportError( + 'path_escape', + `Artifact path does not belong to session ${input.sessionId}: ${artifact.relativePath}`, + ); } - continue; - } - if (isKnownProtectedEntry(entry.name)) { - excludedEntries.push(entry.name); - continue; + const relativePath = `artifacts/${artifact.relativePath}`; + await assertRegularFile(resolve(stateRoot, relativePath), relativePath); + entries.push({ relativePath, kind: 'file', source: 'copy' }); } - throw new SessionBundleExportError( - 'unknown_entry', - `Session bundle export encountered an unclassified top-level entry: ${entry.name}`, - ); - } - if (!sessionsClassified) { - throw new SessionBundleExportError( - 'invalid_root', - `Session bundle state root has no sessions tree: ${stateRoot}`, - ); + includedEntries.push('artifacts'); } - if (artifactsClassified !== artifactMetadata.artifactRootExists) { - throw artifactMetadataChanged(); - } - + const allowed = new Set([...SESSION_BUNDLE_STATE_ENTRIES]); + const excludedEntries = (await readdir(stateRoot)).filter((entry) => !allowed.has(entry)).sort(); return { - plan: { - stateRoot, - configRoot, - destinationRoot, - sessionId: input.sessionId, - includedEntries, - excludedEntries, - entries, - }, - artifactMetadata, - requestedDestinationRoot: input.destinationRoot, - stateRootIdentity, + stateRoot, + configRoot, + destinationRoot, + sessionId: input.sessionId, + includedEntries, + excludedEntries, + entries, }; } -async function resolveSessionBundleRoots( - input: SessionBundleExportInput, -): Promise { - assertSafeSessionId(input.sessionId); - const stateRoot = await canonicalizeExistingRoot(input.stateRoot, 'state'); - const stateRootIdentity = await readSessionBundleRootIdentity(stateRoot); - const configRoot = await canonicalizeExistingOrMissingRoot(input.configRoot, 'config'); - assertRootsDoNotOverlap(stateRoot, configRoot, input.allowShared === true); - - const destinationRoot = await canonicalizeExistingOrMissingRoot( - input.destinationRoot, - 'destination', - ); - assertRootsDoNotOverlap(stateRoot, destinationRoot, false); - assertRootsDoNotOverlap(configRoot, destinationRoot, false); - - return { stateRoot, stateRootIdentity, configRoot, destinationRoot }; -} - -/** - * Copy the planned state tree into an empty destination. This intentionally - * does not create an archive; callers can add compression around this stable, - * checked tree contract later without changing the policy. - */ export async function exportSessionBundleState( input: SessionBundleExportInput, ): Promise { - const preflight = await resolveSessionBundleRoots(input); - // A session_file reference is committed only after its Artifact create - // returns. Keep Artifact mutations fenced through session projection so a - // create cannot land between the Artifact snapshot and a copied reference. - return withArtifactWriterLock(preflight.stateRoot, async () => { - await assertSessionBundleRootIdentity(preflight.stateRoot, preflight.stateRootIdentity); - const current = await prepareSessionBundleExport(input); - if ( - current.plan.stateRoot !== preflight.stateRoot || - !sameSessionBundleRootIdentity(current.stateRootIdentity, preflight.stateRootIdentity) - ) { - throw new SessionBundleExportError( - 'invalid_root', - 'Session bundle state root changed before Artifact snapshot export', - ); - } - await ensureEmptyDestination(current.requestedDestinationRoot, current.plan.destinationRoot); - await assertSessionBundleRootIdentity(preflight.stateRoot, preflight.stateRootIdentity); - await exportPreparedArtifactState(current); - await assertSessionBundleRootIdentity(preflight.stateRoot, preflight.stateRootIdentity); - await exportPreparedNonArtifactState(current.plan, current.artifactMetadata); - await assertSessionBundleRootIdentity(preflight.stateRoot, preflight.stateRootIdentity); - return current.plan; - }); -} - -async function exportPreparedArtifactState(prepared: PreparedSessionBundleExport): Promise { - const { plan, artifactMetadata } = prepared; - const artifactEntries = plan.entries.filter((entry) => isArtifactEntry(entry)); - const directories = artifactEntries.filter((entry) => entry.kind === 'directory'); - const files = artifactEntries.filter((entry) => entry.kind === 'file' && entry.source === 'copy'); - for (const entry of directories) { - await mkdir(resolve(plan.destinationRoot, entry.relativePath), { recursive: true }); - } - for (const entry of files) { - const sourcePath = resolve(plan.stateRoot, entry.relativePath); - const destinationPath = resolve(plan.destinationRoot, entry.relativePath); - await mkdir(dirname(destinationPath), { recursive: true }); - await copyCheckedFile(sourcePath, destinationPath, plan.stateRoot, entry.relativePath); - } - await assertNoCanonicalArtifactTransactionResidue(plan.stateRoot); - await assertArtifactMetadataSnapshotUnchanged(plan.stateRoot, artifactMetadata); -} - -async function exportPreparedNonArtifactState( - plan: SessionBundleExportPlan, - artifactMetadata: ArtifactMetadataSnapshot, -): Promise { - const entries = plan.entries.filter((entry) => !isArtifactEntry(entry)); - const directories = entries.filter((entry) => entry.kind === 'directory'); - const files = entries.filter((entry) => entry.kind === 'file' && entry.source === 'copy'); - for (const entry of directories) { - await mkdir(resolve(plan.destinationRoot, entry.relativePath), { recursive: true }); - } - for (const entry of files) { - const sourcePath = resolve(plan.stateRoot, entry.relativePath); - const destinationPath = resolve(plan.destinationRoot, entry.relativePath); - await mkdir(dirname(destinationPath), { recursive: true }); - await copyCheckedFile(sourcePath, destinationPath, plan.stateRoot, entry.relativePath); - } - const authoritySnapshot = entries.some( - (entry) => - entry.source === 'selected_session_metadata' || entry.source === 'selected_session_boundary', - ) - ? await readSelectedSessionAuthoritySnapshot(plan) - : undefined; - for (const entry of entries) { - if (entry.source === 'selected_session_metadata') { - await exportSelectedSessionMetadata(plan, authoritySnapshot!); - } else if (entry.source === 'selected_session_boundary') { - await exportSelectedSessionBoundary(plan, authoritySnapshot!); - } else if (entry.source === 'filtered_runtime_sqlite') { - await exportFilteredRuntimeSqlite(plan, entry); - } - } - await exportSqliteArtifactMetadata(plan.destinationRoot, artifactMetadata.selectedRecords); -} - -function isArtifactEntry(entry: SessionBundleExportPlanEntry): boolean { - return entry.relativePath === 'artifacts' || entry.relativePath.startsWith('artifacts/'); -} - -async function planSelectedSessionTree( - sourcePath: string, - sessionId: string, - stateRoot: string, - excludedEntries: string[], - entries: SessionBundleExportPlanEntry[], -): Promise { - await assertDirectory(sourcePath, 'sessions root', stateRoot, 'sessions'); - entries.push({ relativePath: 'sessions', kind: 'directory', source: 'copy' }); - let selectedSessionFound = false; - for (const entry of (await readdir(sourcePath, { withFileTypes: true })).sort((a, b) => - a.name.localeCompare(b.name), - )) { - const childPath = resolve(sourcePath, entry.name); - assertNoSymlink(entry.isSymbolicLink(), `sessions/${entry.name}`); - if (entry.name === sessionId) { - selectedSessionFound = true; - await assertDirectory(childPath, `session ${sessionId}`, stateRoot, `sessions/${sessionId}`); - const transcriptPath = resolve(childPath, 'session.jsonl'); - await assertFile( - transcriptPath, - `session ${sessionId} transcript`, - stateRoot, - `sessions/${sessionId}/session.jsonl`, - ); - const children = await readdir(childPath, { withFileTypes: true }); - for (const child of children) { - const relativePath = `sessions/${sessionId}/${child.name}`; - const portablePath = resolve(childPath, child.name); - assertNoSymlink(child.isSymbolicLink(), relativePath); - if (child.name === 'session.jsonl') continue; - if (child.name === EXECUTION_BOUNDARY_TRANSFER_FILE) { - await assertFile( - portablePath, - 'portable execution boundary transfer', - stateRoot, - relativePath, - ); - continue; - } - if (portableSessionDirectories.has(child.name)) { - await assertDirectory( - portablePath, - `portable session entry ${child.name}`, - stateRoot, - relativePath, - ); - await inspectTree(portablePath, relativePath, stateRoot, entries, 'copy'); + return withArtifactWriterLock(input.stateRoot, async (stateRoot) => { + const plan = await planSessionBundleExport({ ...input, stateRoot }); + await assertDestinationMissing(plan.destinationRoot); + const stagingRoot = `${plan.destinationRoot}.${process.pid}.${randomUUID()}.tmp`; + try { + await mkdir(stagingRoot, { recursive: true }); + for (const entry of plan.entries) { + const destination = resolveInside(stagingRoot, entry.relativePath); + if (entry.kind === 'directory') { + await mkdir(destination, { recursive: true }); continue; } - if (portableSessionFiles.has(child.name)) { - await assertFile( - portablePath, - `portable session entry ${child.name}`, - stateRoot, - relativePath, - ); - entries.push({ relativePath, kind: 'file', source: 'copy' }); - continue; + await mkdir(dirname(destination), { recursive: true }); + if (entry.source === 'copy') { + await copyFile(resolveInside(plan.stateRoot, entry.relativePath), destination); + } else { + await exportFilteredDatabase(plan.stateRoot, destination, plan.sessionId); } - throw new SessionBundleExportError( - 'unknown_entry', - `Session bundle export encountered an unclassified selected-session entry: ${relativePath}`, - ); } - entries.push({ - relativePath: `sessions/${sessionId}`, - kind: 'directory', - source: 'copy', - }); - entries.push({ - relativePath: `sessions/${sessionId}/session.jsonl`, - kind: 'file', - source: 'selected_session_metadata', - }); - entries.push({ - relativePath: `sessions/${sessionId}/${EXECUTION_BOUNDARY_TRANSFER_FILE}`, - kind: 'file', - source: 'selected_session_boundary', - }); - continue; - } - if (entry.isDirectory() && isSafeSessionId(entry.name)) { - excludedEntries.push(`sessions/${entry.name}`); - continue; - } - if (isKnownProtectedEntry(entry.name)) { - excludedEntries.push(`sessions/${entry.name}`); - continue; + await mkdir(dirname(plan.destinationRoot), { recursive: true }); + await rename(stagingRoot, plan.destinationRoot); + return plan; + } catch (error) { + await rm(stagingRoot, { recursive: true, force: true }).catch(() => {}); + throw error; } - throw new SessionBundleExportError( - 'unknown_entry', - `Session bundle export encountered an unclassified sessions entry: ${entry.name}`, - ); - } - if (!selectedSessionFound) { - throw new SessionBundleExportError( - 'invalid_root', - `Session bundle session does not exist: ${sessionId}`, - ); - } + }); } -async function exportSelectedSessionMetadata( - plan: SessionBundleExportPlan, - snapshot: SessionAuthoritySnapshot, +async function exportFilteredDatabase( + stateRoot: string, + destinationPath: string, + sessionId: string, ): Promise { - const stagingRoot = `${plan.destinationRoot}.selected-session-${randomUUID()}`; + const lease = acquireOperationalStateDatabase(stateRoot); try { - await exportLegacySessionTreeSnapshot({ - workspaceRoot: plan.stateRoot, - destinationRoot: stagingRoot, - records: [snapshot.record], - boundaries: new Map([[plan.sessionId, snapshot.boundary]]), - sessionIds: [plan.sessionId], - }); - const relativePath = `sessions/${plan.sessionId}/session.jsonl`; - const sourcePath = resolve(stagingRoot, relativePath); - const destinationPath = resolve(plan.destinationRoot, relativePath); - await mkdir(dirname(destinationPath), { recursive: true }); - await copyCheckedFile(sourcePath, destinationPath, stagingRoot, relativePath); + await lease.backup(destinationPath); } finally { - await rm(stagingRoot, { recursive: true, force: true }).catch(() => {}); + lease.close(); } -} - -async function exportSelectedSessionBoundary( - plan: SessionBundleExportPlan, - snapshot: SessionAuthoritySnapshot, -): Promise { - const destinationPath = resolve( - plan.destinationRoot, - 'sessions', - plan.sessionId, - EXECUTION_BOUNDARY_TRANSFER_FILE, - ); - await mkdir(dirname(destinationPath), { recursive: true }); - await writeFile(destinationPath, encodeExecutionBoundaryTransfer(snapshot.boundary), 'utf8'); -} - -async function readSelectedSessionAuthoritySnapshot( - plan: SessionBundleExportPlan, -): Promise { - const metadata = createSqliteSessionMetadataStore( - resolve(plan.stateRoot, OPERATIONAL_STATE_DATABASE_NAME), - ); + const database = new DatabaseSync(destinationPath); try { - return await metadata.readSessionAuthoritySnapshot(plan.sessionId); - } finally { - metadata.close(); - } -} - -async function planSelectedArtifactTree( - sourcePath: string, - sessionId: string, - stateRoot: string, - excludedEntries: string[], - entries: SessionBundleExportPlanEntry[], - artifactMetadata: ArtifactMetadataSnapshot, -): Promise { - if (!artifactMetadata.artifactRootExists) throw artifactMetadataChanged(); - await assertDirectory(sourcePath, 'artifacts root', stateRoot, 'artifacts'); - entries.push({ relativePath: 'artifacts', kind: 'directory', source: 'copy' }); - let metadataClassified = false; - let selectedSessionDirectoryClassified = false; - for (const entry of (await readdir(sourcePath, { withFileTypes: true })).sort((a, b) => - a.name.localeCompare(b.name), - )) { - const childPath = resolve(sourcePath, entry.name); - assertNoSymlink(entry.isSymbolicLink(), `artifacts/${entry.name}`); - if (entry.name === sessionId) { - selectedSessionDirectoryClassified = true; - await assertDirectory( - childPath, - `artifacts for session ${sessionId}`, - stateRoot, - `artifacts/${sessionId}`, + database.exec('PRAGMA foreign_keys = OFF; BEGIN IMMEDIATE'); + const tables = database + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'") + .all() as Array<{ name?: unknown }>; + for (const row of tables) { + if (typeof row.name !== 'string' || PORTABLE_GLOBAL_TABLES.has(row.name)) continue; + const columns = database + .prepare(`PRAGMA table_info(${quoteIdentifier(row.name)})`) + .all() as Array<{ name?: unknown }>; + const names = new Set( + columns + .map((column) => column.name) + .filter((name): name is string => typeof name === 'string'), ); - await planSelectedArtifactSessionTree( - childPath, - sessionId, - stateRoot, - entries, - artifactMetadata.selectedRecords, - ); - continue; - } - if (entry.name === 'metadata.jsonl') { - metadataClassified = true; - if (!artifactMetadata.metadataExists) throw artifactMetadataChanged(); - excludedEntries.push('artifacts/metadata.jsonl'); - continue; - } - if ( - entry.name === ARTIFACT_PURGE_INTENT_FILE || - isCanonicalArtifactRecoveryTempName(entry.name) - ) { - throwArtifactRecoveryRequired(`artifacts/${entry.name}`); - } - if (entry.isDirectory() && isSafeSessionId(entry.name)) { - await assertNoArtifactPublicationResidue(childPath, entry.name); - excludedEntries.push(`artifacts/${entry.name}`); - continue; - } - if (isKnownProtectedEntry(entry.name)) { - excludedEntries.push(`artifacts/${entry.name}`); - continue; - } - throw new SessionBundleExportError( - 'unknown_entry', - `Session bundle export encountered an unclassified artifacts entry: ${entry.name}`, - ); - } - if (metadataClassified !== artifactMetadata.metadataExists) throw artifactMetadataChanged(); - if (!selectedSessionDirectoryClassified && artifactMetadata.selectedRecords.length > 0) { - throw new SessionBundleExportError( - 'invalid_root', - `Artifact metadata references a missing selected-session payload directory: artifacts/${sessionId}`, - ); - } -} - -async function planSelectedArtifactSessionTree( - sourcePath: string, - sessionId: string, - stateRoot: string, - entries: SessionBundleExportPlanEntry[], - selectedRecords: readonly ArtifactRecord[], -): Promise { - const sessionRelativePath = `artifacts/${sessionId}`; - const expectedByName = new Map( - selectedRecords.map((record) => [basename(record.relativePath), record] as const), - ); - const foundExpectedNames = new Set(); - for (const entry of (await readdir(sourcePath, { withFileTypes: true })).sort((a, b) => - a.name.localeCompare(b.name), - )) { - const relativePath = `${sessionRelativePath}/${entry.name}`; - assertNoSymlink(entry.isSymbolicLink(), relativePath); - assertArtifactPublicationEntryIsPortable(entry.name, relativePath); - const expected = expectedByName.get(entry.name); - if (expected) { - await assertFile( - resolve(sourcePath, entry.name), - `artifact payload ${expected.id}`, - stateRoot, - relativePath, - ); - foundExpectedNames.add(entry.name); - continue; - } - if (entry.isFile()) { - throwArtifactRecoveryRequired(relativePath); - } - if (entry.isDirectory()) { - throw new SessionBundleExportError( - 'unknown_entry', - `Session bundle export encountered an unclassified selected-session artifact entry: ${relativePath}`, + const sessionColumns = ['session_id', 'source_session_id', 'target_session_id'].filter( + (name) => names.has(name), ); + if (sessionColumns.length > 0) { + const predicate = sessionColumns + .map((name) => `${quoteIdentifier(name)} <> ?`) + .join(' OR '); + database + .prepare(`DELETE FROM ${quoteIdentifier(row.name)} WHERE ${predicate}`) + .run(...sessionColumns.map(() => sessionId)); + } else if (!PORTABLE_DERIVED_TABLES.has(row.name)) { + database.exec(`DELETE FROM ${quoteIdentifier(row.name)}`); + } } - throw new SessionBundleExportError( - 'unsupported_entry', - `Session bundle export cannot classify selected-session artifact entry: ${relativePath}`, - ); - } - for (const [name, record] of expectedByName) { - if (foundExpectedNames.has(name)) continue; - throw new SessionBundleExportError( - 'invalid_root', - `Artifact metadata references a missing payload: artifacts/${record.relativePath}`, - ); - } - if (selectedRecords.length === 0) return; - entries.push({ relativePath: sessionRelativePath, kind: 'directory', source: 'copy' }); - for (const record of selectedRecords) { - entries.push({ - relativePath: `artifacts/${record.relativePath}`, - kind: 'file', - source: 'copy', - }); - } -} - -async function assertNoArtifactPublicationResidue( - sessionArtifactRoot: string, - sessionId: string, -): Promise { - for (const entry of await readdir(sessionArtifactRoot, { withFileTypes: true })) { - if (!entry.name.startsWith('.artifact-publish.')) continue; - const relativePath = `artifacts/${sessionId}/${entry.name}`; - assertNoSymlink(entry.isSymbolicLink(), relativePath); - assertArtifactPublicationEntryIsPortable(entry.name, relativePath); - } -} - -async function assertNoCanonicalArtifactTransactionResidue(stateRoot: string): Promise { - const artifactRoot = resolve(stateRoot, 'artifacts'); - let rootEntries; - try { - rootEntries = await readdir(artifactRoot, { withFileTypes: true }); + database + .prepare(` + DELETE FROM tool_journal_events + WHERE NOT EXISTS ( + SELECT 1 FROM runtime_events + WHERE runtime_events.invocation_id = tool_journal_events.invocation_id + ) + `) + .run(); + database + .prepare(` + DELETE FROM tool_operations + WHERE NOT EXISTS ( + SELECT 1 FROM runtime_events + WHERE runtime_events.invocation_id = tool_operations.invocation_id + ) + `) + .run(); + database + .prepare(` + DELETE FROM core_interaction_outcomes + WHERE NOT EXISTS ( + SELECT 1 FROM core_interaction_requests + WHERE core_interaction_requests.request_id = core_interaction_outcomes.request_id + ) + `) + .run(); + database + .prepare(` + DELETE FROM core_message_host_epochs + WHERE NOT EXISTS ( + SELECT 1 FROM core_message_receipts + WHERE core_message_receipts.host_epoch = core_message_host_epochs.host_epoch + ) + `) + .run(); + database.exec('COMMIT'); + const foreignKeyViolation = database.prepare('PRAGMA foreign_key_check').get(); + if (foreignKeyViolation) throw new Error('Filtered session database has dangling references'); + const session = database + .prepare('SELECT 1 AS present FROM session_metadata WHERE session_id = ?') + .get(sessionId); + if (!session) throw new Error(`Filtered session is missing: ${sessionId}`); } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + try { + database.exec('ROLLBACK'); + } catch {} throw error; - } - for (const entry of rootEntries) { - const relativePath = `artifacts/${entry.name}`; - assertNoSymlink(entry.isSymbolicLink(), relativePath); - if ( - entry.name === ARTIFACT_PURGE_INTENT_FILE || - isCanonicalArtifactRecoveryTempName(entry.name) - ) { - throwArtifactRecoveryRequired(relativePath); - } - if (entry.isDirectory() && isSafeSessionId(entry.name)) { - await assertNoArtifactPublicationResidue(resolve(artifactRoot, entry.name), entry.name); - } + } finally { + database.close(); } } -function assertArtifactPublicationEntryIsPortable(entryName: string, relativePath: string): void { - if (ARTIFACT_PUBLICATION_STAGING_PATTERN.test(entryName)) { - throwArtifactRecoveryRequired(relativePath); - } - if (entryName.startsWith('.artifact-publish.')) { - throw new SessionBundleExportError( - 'unknown_entry', - `Session bundle export encountered an unclassified artifact entry: ${relativePath}`, - ); - } -} +const PORTABLE_GLOBAL_TABLES = new Set([ + 'operational_schema_migrations', + 'session_metadata_schema', + 'runtime_capabilities', + 'session_catalog_state', + 'automation_authority_state', +]); -function throwArtifactRecoveryRequired(relativePath: string): never { - throw new SessionBundleExportError( - 'unsupported_entry', - `Artifact write authority recovery is required before session bundle export: ${relativePath}`, - ); -} - -async function readArtifactMetadataSnapshot( - stateRoot: string, - sessionId: string, -): Promise { - const source = await readArtifactMetadataSource(stateRoot); - let records: ArtifactRecord[]; - try { - records = decodeArtifactMetadata(source.canonicalText); - } catch (error) { - throw new SessionBundleExportError( - 'unsupported_entry', - 'Artifact metadata cannot be reopened by the Artifact store', - { cause: error }, - ); - } - const selectedRecords = records.filter((record) => record.sessionId === sessionId); - return { - ...source, - selectedRecords, - }; -} +const PORTABLE_DERIVED_TABLES = new Set([ + 'tool_journal_events', + 'tool_operations', + 'core_interaction_outcomes', + 'core_message_host_epochs', +]); -async function readArtifactMetadataSource( - stateRoot: string, -): Promise< - Pick -> { - const legacy = await readLegacyArtifactMetadataSource(stateRoot); - const canonicalText = await readCanonicalSqliteArtifactMetadata( - stateRoot, - legacy.metadataExists ? legacy.canonicalText : undefined, +export function isArtifactPathForSession(relativePath: string, sessionId: string): boolean { + const parts = relativePath.split(/[\\/]+/); + return ( + parts.length >= 2 && + parts[0] === sessionId && + parts.every((part) => part.length > 0 && part !== '.' && part !== '..') ); - return canonicalText === undefined ? legacy : { ...legacy, canonicalText }; } -async function readLegacyArtifactMetadataSource( - stateRoot: string, -): Promise< - Pick -> { - const artifactRoot = resolve(stateRoot, 'artifacts'); - let artifactRootMetadata; +async function canonicalRoot(path: string, role: string, allowMissing = false): Promise { + const requested = resolve(path); try { - artifactRootMetadata = await lstat(artifactRoot); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return { artifactRootExists: false, metadataExists: false, canonicalText: '' }; + const metadata = await lstat(requested); + if (metadata.isSymbolicLink()) { + throw new SessionBundleExportError('symlink', `${role} root cannot be a symlink`); } - throw error; - } - assertNoSymlink(artifactRootMetadata.isSymbolicLink(), 'artifacts'); - await assertCanonicalPathInside(artifactRoot, stateRoot, 'artifacts'); - if (!artifactRootMetadata.isDirectory()) { - throw new SessionBundleExportError( - 'unsupported_entry', - `Session bundle artifacts root is not a directory: ${artifactRoot}`, - ); - } - - const metadataPath = resolve(artifactRoot, 'metadata.jsonl'); - let metadata; - try { - metadata = await lstat(metadataPath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') { - return { artifactRootExists: true, metadataExists: false, canonicalText: '' }; + if (!metadata.isDirectory()) { + throw new SessionBundleExportError('invalid_root', `${role} root is not a directory`); } - throw error; - } - assertNoSymlink(metadata.isSymbolicLink(), 'artifacts/metadata.jsonl'); - await assertCanonicalPathInside(metadataPath, stateRoot, 'artifacts/metadata.jsonl'); - if (!metadata.isFile()) { - throw new SessionBundleExportError( - 'unsupported_entry', - `Session bundle artifact metadata is not a regular file: ${metadataPath}`, - ); - } - return { - artifactRootExists: true, - metadataExists: true, - canonicalText: await readFile(metadataPath, 'utf8'), - }; -} - -async function readCanonicalSqliteArtifactMetadata( - stateRoot: string, - legacySource: string | undefined, -): Promise { - const databasePath = resolve(stateRoot, OPERATIONAL_STATE_DATABASE_NAME); - let metadata; - try { - metadata = await lstat(databasePath); + return realpath(requested); } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; - throw error; - } - assertNoSymlink(metadata.isSymbolicLink(), OPERATIONAL_STATE_DATABASE_NAME); - await assertCanonicalPathInside(databasePath, stateRoot, OPERATIONAL_STATE_DATABASE_NAME); - if (!metadata.isFile()) { - throw new SessionBundleExportError( - 'unsupported_entry', - `Session bundle runtime SQLite is not a regular file: ${databasePath}`, - ); - } - - const Database = loadDatabaseSync(); - const database = new Database(databasePath, { readOnly: true }); - let snapshotOpen = false; - try { - database.exec('PRAGMA busy_timeout = 5000; PRAGMA query_only = ON'); - database.exec('BEGIN'); - snapshotOpen = true; - if (!sqliteTableExists(database, 'cutover_journal')) return undefined; - const journal = database - .prepare(` - SELECT source_fingerprint, state - FROM cutover_journal - WHERE store_name = 'artifact_metadata' - `) - .get() as { source_fingerprint?: unknown; state?: unknown } | undefined; - if (!journal) { - if (sqliteTableExists(database, 'artifact_records')) { - const row = database.prepare('SELECT COUNT(*) AS count FROM artifact_records').get() as { - count?: unknown; - }; - if (row.count !== 0) { - throw new SessionBundleExportError( - 'unsupported_entry', - 'Artifact SQLite rows exist without a completed cutover journal', - ); - } - } - return undefined; - } - if (journal.state !== 'completed' || typeof journal.source_fingerprint !== 'string') { - throw new SessionBundleExportError( - 'unsupported_entry', - 'Artifact metadata cutover must complete before session bundle export', - ); - } - if (journal.source_fingerprint !== artifactMetadataSourceFingerprint(legacySource)) { - throw artifactMetadataChanged(); - } - if (!sqliteTableExists(database, 'artifact_records')) { - throw new SessionBundleExportError( - 'unsupported_entry', - 'Completed Artifact metadata cutover is missing artifact_records', - ); - } - const rows = database - .prepare(` - SELECT record_json - FROM artifact_records - ORDER BY created_at, storage_key - `) - .all() as Array<{ record_json?: unknown }>; - if (rows.some((row) => typeof row.record_json !== 'string')) { - throw new SessionBundleExportError( - 'unsupported_entry', - 'Artifact SQLite metadata contains an invalid record', - ); - } - return rows.length > 0 ? `${rows.map((row) => row.record_json as string).join('\n')}\n` : ''; - } finally { - try { - if (snapshotOpen) database.exec('ROLLBACK'); - } finally { - database.close(); - } - } -} - -async function assertArtifactMetadataSnapshotUnchanged( - stateRoot: string, - snapshot: ArtifactMetadataSnapshot, -): Promise { - const current = await readArtifactMetadataSource(stateRoot); - if ( - current.artifactRootExists !== snapshot.artifactRootExists || - current.metadataExists !== snapshot.metadataExists || - current.canonicalText !== snapshot.canonicalText - ) { - throw artifactMetadataChanged(); + if (allowMissing && (error as NodeJS.ErrnoException).code === 'ENOENT') return requested; + if (error instanceof SessionBundleExportError) throw error; + throw new SessionBundleExportError('invalid_root', `${role} root does not exist`, { + cause: error, + }); } } -function artifactMetadataChanged(): SessionBundleExportError { - return new SessionBundleExportError( - 'unsupported_entry', - 'Artifact metadata changed during session bundle export', - ); -} - -async function assertDirectory( - path: string, - role: string, - stateRoot: string, - relativePath: string, -): Promise { - let metadata; - try { - metadata = await lstat(path); - } catch (error) { - throw new SessionBundleExportError( - 'invalid_root', - `Session bundle ${role} does not exist: ${path}`, - { cause: error }, - ); +async function assertRegularFile(path: string, label: string): Promise { + const metadata = await lstat(path).catch((error) => { + throw new SessionBundleExportError('invalid_root', `Missing ${label}`, { cause: error }); + }); + if (metadata.isSymbolicLink()) { + throw new SessionBundleExportError('symlink', `${label} cannot be a symlink`); } - assertNoSymlink(metadata.isSymbolicLink(), relativePath); - await assertCanonicalPathInside(path, stateRoot, relativePath); - if (!metadata.isDirectory()) { - throw new SessionBundleExportError( - 'unsupported_entry', - `Session bundle ${role} is not a directory: ${path}`, - ); + if (!metadata.isFile()) { + throw new SessionBundleExportError('unsupported_entry', `${label} is not a regular file`); } } -async function assertFile( - path: string, - role: string, - stateRoot: string, - relativePath: string, -): Promise { - let metadata; +async function assertDestinationMissing(path: string): Promise { try { - metadata = await lstat(path); + await lstat(path); } catch (error) { - throw new SessionBundleExportError( - 'invalid_root', - `Session bundle ${role} does not exist: ${path}`, - { cause: error }, - ); - } - assertNoSymlink(metadata.isSymbolicLink(), relativePath); - await assertCanonicalPathInside(path, stateRoot, relativePath); - if (!metadata.isFile()) { - throw new SessionBundleExportError( - 'unsupported_entry', - `Session bundle ${role} is not a regular file: ${path}`, - ); - } -} - -async function inspectTree( - sourcePath: string, - relativePath: string, - stateRoot: string, - entries: SessionBundleExportPlanEntry[], - source: SessionBundleExportPlanEntry['source'], -): Promise { - const metadata = await lstat(sourcePath); - assertNoSymlink(metadata.isSymbolicLink(), relativePath); - await assertCanonicalPathInside(sourcePath, stateRoot, relativePath); - - if (metadata.isDirectory()) { - entries.push({ relativePath, kind: 'directory', source: 'copy' }); - for (const child of (await readdir(sourcePath, { withFileTypes: true })).sort((a, b) => - a.name.localeCompare(b.name), - )) { - await inspectTree( - resolve(sourcePath, child.name), - `${relativePath}/${child.name}`, - stateRoot, - entries, - 'copy', - ); - } - return; - } - if (metadata.isFile()) { - entries.push({ relativePath, kind: 'file', source }); - return; + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; + throw error; } throw new SessionBundleExportError( - 'unsupported_entry', - `Session bundle export cannot include special file: ${relativePath}`, + 'destination_not_empty', + `Session bundle destination already exists: ${path}`, ); } -async function copyCheckedFile( - sourcePath: string, - destinationPath: string, - stateRoot: string, - relativePath: string, -): Promise { - const metadata = await lstat(sourcePath); - assertNoSymlink(metadata.isSymbolicLink(), relativePath); - await assertCanonicalPathInside(sourcePath, stateRoot, relativePath); - if (!metadata.isFile()) { - throw new SessionBundleExportError( - 'unsupported_entry', - `Session bundle export source changed before copy: ${relativePath}`, - ); - } - await copyFile(sourcePath, destinationPath); -} - -async function exportFilteredRuntimeSqlite( - plan: SessionBundleExportPlan, - entry: SessionBundleExportPlanEntry, -): Promise { - const sourcePath = resolve(plan.stateRoot, entry.relativePath); - const destinationPath = resolve(plan.destinationRoot, entry.relativePath); - await assertFile(sourcePath, 'runtime SQLite', plan.stateRoot, entry.relativePath); - - const source = createSqliteRuntimeStore(sourcePath); - const filteredPath = `${destinationPath}.filtered`; - try { - const destination = createSqliteRuntimeStore(filteredPath); - try { - const eventsByRun = new Map< - string, - { sessionId: string; runId: string; events: RuntimeEvent[] } - >(); - for (const event of await source.readSessionRuntimeEvents(plan.sessionId)) { - const batch = eventsByRun.get(event.runId) ?? { - sessionId: plan.sessionId, - runId: event.runId, - events: [], - }; - batch.events.push(event); - eventsByRun.set(event.runId, batch); - } - for (const batch of eventsByRun.values()) await destination.importRuntimeEventsBatch(batch); - await destination.rebuildToolProjectionsFromRuntimeEvents(); - } finally { - destination.close(); - } - } finally { - source.close(); - } - await rename(filteredPath, destinationPath); -} - -async function exportSqliteArtifactMetadata( - destinationRoot: string, - records: readonly ArtifactRecord[], -): Promise { - const repository = createSqliteArtifactMetadataRepository(destinationRoot); - try { - await repository.ready(); - repository.replaceAll(records); - const reopened = repository.readAll(); - if (!sameArtifactRecordSet(reopened, records)) { - throw new SessionBundleExportError( - 'unsupported_entry', - 'Exported Artifact SQLite metadata failed validation', - ); - } - } finally { - repository.close(); - } -} - -function sameArtifactRecordSet( - left: readonly ArtifactRecord[], - right: readonly ArtifactRecord[], -): boolean { - if (left.length !== right.length) return false; - const canonical = (records: readonly ArtifactRecord[]) => - records.map((record) => JSON.stringify(record)).sort((a, b) => a.localeCompare(b)); - const leftRecords = canonical(left); - const rightRecords = canonical(right); - return leftRecords.every((record, index) => record === rightRecords[index]); -} - -async function ensureEmptyDestination(requestedPath: string, canonicalPath: string): Promise { - let metadata; - try { - metadata = await lstat(resolve(requestedPath)); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - await mkdir(resolve(requestedPath), { recursive: true }); - return; - } - assertNoSymlink(metadata.isSymbolicLink(), 'destination root'); - if (!metadata.isDirectory()) { - throw new SessionBundleExportError( - 'invalid_root', - `Session bundle destination is not a directory: ${requestedPath}`, - ); - } - const entries = await readdir(canonicalPath); - if (entries.length > 0) { - throw new SessionBundleExportError( - 'destination_not_empty', - `Session bundle export destination must be empty: ${requestedPath}`, - ); +function resolveInside(root: string, path: string): string { + const candidate = resolve(root, path); + const rel = relative(root, candidate); + if (rel === '' || rel === '..' || rel.startsWith(`..${sep}`) || isAbsolute(rel)) { + throw new SessionBundleExportError('path_escape', `Path escapes bundle root: ${path}`); } + return candidate; } -async function canonicalizeExistingRoot(path: string, role: string): Promise { - const requestedPath = resolve(path); - let metadata; - try { - metadata = await lstat(requestedPath); - } catch (error) { - throw new SessionBundleExportError( - 'invalid_root', - `Session bundle ${role} root does not exist: ${requestedPath}`, - { cause: error }, - ); - } - assertNoSymlink(metadata.isSymbolicLink(), `${role} root`); - if (!metadata.isDirectory()) { - throw new SessionBundleExportError( - 'invalid_root', - `Session bundle ${role} root is not a directory: ${requestedPath}`, - ); - } - return realpath(requestedPath); -} - -async function readSessionBundleRootIdentity( - stateRoot: string, -): Promise { - let metadata; - try { - metadata = await lstat(stateRoot, { bigint: true }); - } catch (error) { - throw new SessionBundleExportError( - 'invalid_root', - `Unable to inspect session bundle state root identity: ${stateRoot}`, - { cause: error }, - ); - } - if (!metadata.isDirectory() || metadata.isSymbolicLink()) { - throw new SessionBundleExportError( - 'invalid_root', - `Session bundle state root identity is not a directory: ${stateRoot}`, - ); - } - return { dev: metadata.dev, ino: metadata.ino }; -} - -async function assertSessionBundleRootIdentity( - stateRoot: string, - expected: SessionBundleRootIdentity, -): Promise { - const actual = await readSessionBundleRootIdentity(stateRoot); - if (!sameSessionBundleRootIdentity(actual, expected)) { - throw new SessionBundleExportError( - 'invalid_root', - `Session bundle state root changed during export: ${stateRoot}`, - ); - } -} - -function sameSessionBundleRootIdentity( - left: SessionBundleRootIdentity, - right: SessionBundleRootIdentity, -): boolean { - return left.dev === right.dev && left.ino === right.ino; -} - -async function canonicalizeExistingOrMissingRoot(path: string, role: string): Promise { - const requestedPath = resolve(path); - let metadata; - try { - metadata = await lstat(requestedPath); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { - throw new SessionBundleExportError( - 'invalid_root', - `Unable to inspect session bundle ${role} root: ${requestedPath}`, - { cause: error }, - ); - } - const { parent, suffix } = await nearestExistingDirectoryWithSuffix(requestedPath, role); - return resolve(parent, ...suffix); - } - assertNoSymlink(metadata.isSymbolicLink(), `${role} root`); - if (!metadata.isDirectory()) { - throw new SessionBundleExportError( - 'invalid_root', - `Session bundle ${role} root is not a directory: ${requestedPath}`, - ); - } - return realpath(requestedPath); -} - -async function nearestExistingDirectoryWithSuffix( - path: string, - role: string, -): Promise<{ parent: string; suffix: string[] }> { - let candidate = resolve(path); - const suffix: string[] = []; - while (true) { - try { - const metadata = await lstat(candidate); - assertNoSymlink(metadata.isSymbolicLink(), `${role} root parent`); - if (!metadata.isDirectory()) { - throw new SessionBundleExportError( - 'invalid_root', - `Session bundle ${role} root parent is not a directory: ${candidate}`, - ); - } - return { parent: await realpath(candidate), suffix: suffix.reverse() }; - } catch (error) { - if (error instanceof SessionBundleExportError) throw error; - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') { - throw new SessionBundleExportError( - 'invalid_root', - `Unable to resolve session bundle ${role} root parent: ${candidate}`, - { cause: error }, - ); - } - const parent = dirname(candidate); - if (parent === candidate) { - throw new SessionBundleExportError( - 'invalid_root', - `Unable to resolve session bundle ${role} root parent: ${candidate}`, - ); - } - suffix.push(basename(candidate)); - candidate = parent; - } - } -} - -function assertRootsDoNotOverlap(left: string, right: string, allowSame: boolean): void { +function assertRootsSeparate(left: string, right: string, allowSame: boolean): void { if (left === right) { if (allowSame) return; - throw new SessionBundleExportError( - 'overlapping_roots', - `Session bundle roots overlap at ${left}`, - ); + throw new SessionBundleExportError('overlapping_roots', 'Session bundle roots overlap'); } - if (isPathInside(left, right) || isPathInside(right, left)) { - throw new SessionBundleExportError( - 'overlapping_roots', - `Session bundle roots overlap unsafely: ${left} and ${right}`, - ); - } -} - -async function assertCanonicalPathInside( - path: string, - root: string, - relativePath: string, -): Promise { - let canonicalPath: string; - try { - canonicalPath = await realpath(path); - } catch (error) { - throw new SessionBundleExportError( - 'path_escape', - `Unable to resolve exported path: ${relativePath}`, - { cause: error }, - ); - } - if (!isPathInside(root, canonicalPath)) { - throw new SessionBundleExportError( - 'path_escape', - `Exported path escapes the state root: ${relativePath}`, - ); - } -} - -function assertNoSymlink(isSymlink: boolean, relativePath: string): void { - if (isSymlink) { - throw new SessionBundleExportError( - 'symlink', - `Session bundle export rejects symlinks: ${relativePath}`, - ); + const leftToRight = relative(left, right); + const rightToLeft = relative(right, left); + if ( + (!leftToRight.startsWith('..') && !isAbsolute(leftToRight)) || + (!rightToLeft.startsWith('..') && !isAbsolute(rightToLeft)) + ) { + throw new SessionBundleExportError('overlapping_roots', 'Session bundle roots overlap'); } } -function isKnownProtectedEntry(name: string): boolean { - return ( - protectedEntries.has(name) || - name === 'tmp' || - name === 'activation-input' || - name.endsWith('.log') - ); -} - function assertSafeSessionId(sessionId: string): void { - if (!isSafeSessionId(sessionId)) { + if (!/^[A-Za-z0-9_-]{1,128}$/.test(sessionId)) { throw new SessionBundleExportError('invalid_root', `Invalid session id: ${sessionId}`); } } -function isSafeSessionId(sessionId: string): boolean { - return /^[A-Za-z0-9_-]{1,128}$/.test(sessionId); -} - -export function isArtifactPathForSession(relativePath: string, sessionId: string): boolean { - const parts = relativePath.split(/[\\/]+/); - return ( - parts.length >= 2 && - parts[0] === sessionId && - parts.every((part) => part.length > 0 && part !== '.' && part !== '..') - ); -} - -function isPathInside(root: string, candidate: string): boolean { - const path = relative(root, candidate); - return path === '' || (!path.startsWith(`..${sep}`) && path !== '..' && !isAbsolute(path)); -} - -function sqliteTableExists(database: DatabaseSync, table: string): boolean { - return ( - database.prepare(`SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?`).get(table) !== - undefined - ); -} - -function loadDatabaseSync(): typeof import('node:sqlite').DatabaseSync { - return (require('node:sqlite') as typeof import('node:sqlite')).DatabaseSync; +function quoteIdentifier(value: string): string { + return `"${value.replaceAll('"', '""')}"`; } diff --git a/packages/storage/src/session-metadata-maintenance.ts b/packages/storage/src/session-metadata-maintenance.ts deleted file mode 100644 index 97b746cc09..0000000000 --- a/packages/storage/src/session-metadata-maintenance.ts +++ /dev/null @@ -1,223 +0,0 @@ -import { randomUUID } from 'node:crypto'; -import { mkdir, readFile, rename, rm, stat, writeFile } from 'node:fs/promises'; -import { dirname, join, relative, resolve } from 'node:path'; -import type { ExecutionBoundary, SessionHeader } from '@maka/core'; -import { - encodeExecutionBoundaryTransfer, - EXECUTION_BOUNDARY_TRANSFER_FILE, -} from './session-metadata-transfer.js'; -import { decodeSessionHeader } from './session-store.js'; -import { - acquireOperationalStateDatabase, - OPERATIONAL_STATE_DATABASE_NAME, -} from './operational-state-store.js'; -import { - createSqliteSessionMetadataStore, - type SessionMetadataRecord, -} from './sqlite-session-metadata-store.js'; -import { decodeSessionTranscriptMarker, isSessionTranscriptMarker } from './session-transcript.js'; - -export const SESSION_METADATA_EXPORT_FORMAT = 'maka-session-metadata-export'; -export const SESSION_METADATA_EXPORT_SCHEMA_VERSION = 1; -export const SESSION_METADATA_EXPORT_MANIFEST_NAME = 'session-metadata.json'; - -export interface SessionMetadataExportManifest { - format: typeof SESSION_METADATA_EXPORT_FORMAT; - schemaVersion: typeof SESSION_METADATA_EXPORT_SCHEMA_VERSION; - exportedAt: number; - sessions: SessionMetadataRecord[]; -} - -export interface LegacySessionTreeExportReport { - destinationRoot: string; - sessionsExported: number; - manifestPath: string; -} - -/** - * Export a self-contained legacy-compatible session tree without changing the - * canonical SQLite database or the live transcript files. - */ -export async function exportLegacySessionTree(input: { - workspaceRoot: string; - destinationRoot: string; - /** Optional selected-session export; omitted preserves the full backup behavior. */ - sessionIds?: readonly string[]; - now?: () => number; -}): Promise { - const workspaceRoot = resolve(input.workspaceRoot); - const databasePath = join(workspaceRoot, OPERATIONAL_STATE_DATABASE_NAME); - await assertFileExists(databasePath, 'SQLite session metadata database'); - const metadata = createSqliteSessionMetadataStore(databasePath); - try { - const snapshots = await Promise.all( - (await metadata.list()).map((record) => - metadata.readSessionAuthoritySnapshot(record.header.id), - ), - ); - return exportLegacySessionTreeSnapshot({ - ...input, - records: snapshots.map((snapshot) => snapshot.record), - boundaries: new Map( - snapshots.map((snapshot) => [snapshot.record.header.id, snapshot.boundary]), - ), - }); - } finally { - metadata.close(); - } -} - -export async function exportLegacySessionTreeSnapshot(input: { - workspaceRoot: string; - destinationRoot: string; - records: readonly SessionMetadataRecord[]; - boundaries: ReadonlyMap; - /** Optional selected-session export; omitted preserves the full backup behavior. */ - sessionIds?: readonly string[]; - now?: () => number; -}): Promise { - const workspaceRoot = resolve(input.workspaceRoot); - const destinationRoot = resolve(input.destinationRoot); - const sourceSessionsRoot = join(workspaceRoot, 'sessions'); - const destinationSessionsRoot = join(destinationRoot, 'sessions'); - if ( - destinationRoot === workspaceRoot || - isInsideOrSamePath(sourceSessionsRoot, destinationRoot) || - destinationSessionsRoot === sourceSessionsRoot - ) { - throw new Error('Session metadata export destination overlaps the live session tree'); - } - await assertPathMissing(destinationRoot, 'Session metadata export destination'); - - const stagingRoot = `${destinationRoot}.${process.pid}.${randomUUID()}.tmp`; - try { - const selectedIds = input.sessionIds === undefined ? undefined : new Set(input.sessionIds); - if (selectedIds?.size !== input.sessionIds?.length) { - throw new Error('Session metadata export contains duplicate session ids'); - } - const records = input.records - .filter((record) => selectedIds === undefined || selectedIds.has(record.header.id)) - .sort((a, b) => a.header.id.localeCompare(b.header.id)); - if (selectedIds !== undefined) { - const exportedIds = new Set(records.map((record) => record.header.id)); - const missing = [...selectedIds].filter((sessionId) => !exportedIds.has(sessionId)); - if (missing.length > 0) { - throw new Error(`Session metadata export session does not exist: ${missing.join(', ')}`); - } - if (records.length === 0) { - throw new Error('Session metadata export requires at least one selected session'); - } - } - const missingBoundaryIds = records - .map((record) => record.header.id) - .filter((sessionId) => !input.boundaries.has(sessionId)); - if (missingBoundaryIds.length > 0) { - throw new Error( - `Session metadata export execution boundary is missing: ${missingBoundaryIds.join(', ')}`, - ); - } - await mkdir(join(stagingRoot, 'sessions'), { recursive: true }); - for (const record of records) { - const sourcePath = join(sourceSessionsRoot, record.header.id, 'session.jsonl'); - const transcript = await readFile(sourcePath, 'utf8'); - const body = legacyCompatibleTranscriptBody(transcript, record.header.id, record.header); - const destinationPath = join(stagingRoot, 'sessions', record.header.id, 'session.jsonl'); - await mkdir(dirname(destinationPath), { recursive: true }); - await writeFile(destinationPath, body, 'utf8'); - const boundary = input.boundaries.get(record.header.id)!; - await writeFile( - join(dirname(destinationPath), EXECUTION_BOUNDARY_TRANSFER_FILE), - encodeExecutionBoundaryTransfer(boundary), - 'utf8', - ); - } - const manifest: SessionMetadataExportManifest = { - format: SESSION_METADATA_EXPORT_FORMAT, - schemaVersion: SESSION_METADATA_EXPORT_SCHEMA_VERSION, - exportedAt: (input.now ?? Date.now)(), - sessions: records, - }; - await writeFile( - join(stagingRoot, SESSION_METADATA_EXPORT_MANIFEST_NAME), - `${JSON.stringify(manifest, null, 2)}\n`, - 'utf8', - ); - await mkdir(dirname(destinationRoot), { recursive: true }); - await rename(stagingRoot, destinationRoot); - return { - destinationRoot, - sessionsExported: records.length, - manifestPath: join(destinationRoot, SESSION_METADATA_EXPORT_MANIFEST_NAME), - }; - } finally { - await rm(stagingRoot, { recursive: true, force: true }).catch(() => {}); - } -} - -export async function backupSessionMetadataDatabase(input: { - workspaceRoot: string; - destinationPath: string; -}): Promise<{ destinationPath: string; pagesCopied: number }> { - const destinationPath = resolve(input.destinationPath); - const workspaceRoot = resolve(input.workspaceRoot); - await assertFileExists( - join(workspaceRoot, OPERATIONAL_STATE_DATABASE_NAME), - 'SQLite session metadata database', - ); - const database = acquireOperationalStateDatabase(workspaceRoot); - try { - return { - destinationPath, - pagesCopied: await database.backup(destinationPath), - }; - } finally { - database.close(); - } -} - -function legacyCompatibleTranscriptBody( - transcript: string, - sessionId: string, - header: SessionHeader, -): string { - const firstNewline = transcript.indexOf('\n'); - if (firstNewline < 0) { - throw new Error(`Session ${sessionId}: cannot find first JSONL record`); - } - const firstLine = transcript.slice(0, firstNewline); - let firstRecord: unknown; - try { - firstRecord = JSON.parse(firstLine) as unknown; - if (isSessionTranscriptMarker(firstRecord)) { - decodeSessionTranscriptMarker(firstRecord, sessionId); - } else { - decodeSessionHeader(firstRecord, sessionId); - } - } catch (error) { - throw new Error(`Session ${sessionId}: invalid first JSONL record`, { cause: error }); - } - return `${JSON.stringify(header)}\n${transcript.slice(firstNewline + 1)}`; -} - -async function assertPathMissing(path: string, label: string): Promise { - try { - await stat(path); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; - throw error; - } - throw new Error(`${label} already exists: ${path}`); -} - -async function assertFileExists(path: string, label: string): Promise { - const info = await stat(path).catch((error: NodeJS.ErrnoException) => { - if (error.code === 'ENOENT') throw new Error(`${label} does not exist: ${path}`); - throw error; - }); - if (!info.isFile()) throw new Error(`${label} is not a file: ${path}`); -} - -function isInsideOrSamePath(root: string, candidate: string): boolean { - const path = relative(root, candidate); - return path === '' || (!path.startsWith('..') && !path.includes(':')); -} diff --git a/packages/storage/src/session-metadata-transfer.ts b/packages/storage/src/session-metadata-transfer.ts deleted file mode 100644 index 35c9724b13..0000000000 --- a/packages/storage/src/session-metadata-transfer.ts +++ /dev/null @@ -1,284 +0,0 @@ -import { createHash } from 'node:crypto'; -import { lstat, open, readFile, readdir, rm } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; -import { - assertExecutionBoundaryCapacity, - decodeExecutionBoundary, - MAX_EXECUTION_BOUNDARY_SERIALIZED_BYTES, - type ExecutionBoundary, -} from '@maka/core'; -import { decodeSessionHeader, isSafeSessionId } from './session-store.js'; -import { - createSessionTranscriptMarker, - decodeSessionTranscriptMarker, - isSessionTranscriptMarker, -} from './session-transcript.js'; -import type { - SessionMetadataImportEntry, - SqliteSessionMetadataStore, -} from './sqlite-session-metadata-store.js'; - -const LEGACY_SESSION_HEADER_MAX_BYTES = 1024 * 1024; -const LEGACY_SESSION_HEADER_READ_BYTES = 8192; -// A transfer contains the cumulative session boundary, not one 64 KiB expansion request. -export const EXECUTION_BOUNDARY_TRANSFER_MAX_BYTES = MAX_EXECUTION_BOUNDARY_SERIALIZED_BYTES + 64; -export const EXECUTION_BOUNDARY_TRANSFER_FILE = 'execution-boundary.json'; - -export function encodeExecutionBoundaryTransfer(boundary: ExecutionBoundary): string { - assertExecutionBoundaryCapacity(boundary); - const raw = `${JSON.stringify({ schemaVersion: 1, boundary })}\n`; - if (Buffer.byteLength(raw, 'utf8') > EXECUTION_BOUNDARY_TRANSFER_MAX_BYTES) { - throw new Error('Execution boundary transfer exceeds the serialized size limit'); - } - return raw; -} - -class MalformedLegacySessionHeaderError extends Error { - constructor(sourcePath: string, cause?: unknown) { - super(`Invalid legacy session header at ${sourcePath}`, { cause }); - this.name = 'MalformedLegacySessionHeaderError'; - } -} - -export interface LegacySessionMetadataImportReport { - filesScanned: number; - headersRead: number; - headersImported: number; - headersExisting: number; - sourcesAlreadyImported: number; - sourcesTombstoned: number; -} - -export interface LegacySessionMetadataEntry extends SessionMetadataImportEntry { - boundaryTransferPath?: string; -} - -/** - * Import every legacy line-1 SessionHeader in one SQLite transaction. - * - * The scan and decode phase completes before the transaction begins. - * Malformed headers are skipped (not tombstoned) so one corrupt session - * cannot block the rest of the catalog. Skipping without tombstoning means - * a repaired header will be picked up on the next launch. - */ -export async function importLegacySessionMetadataTree(input: { - workspaceRoot: string; - destination: SqliteSessionMetadataStore; -}): Promise { - const sessionsRoot = join(input.workspaceRoot, 'sessions'); - const entries: LegacySessionMetadataEntry[] = []; - const transcriptMarkerSessionIds: string[] = []; - const directories = await sessionDirectoryNames(sessionsRoot); - for (const directory of directories) { - const sourcePath = join(sessionsRoot, directory, 'session.jsonl'); - try { - const entry = await readLegacySessionMetadataEntry(sourcePath, directory); - if (entry) { - entries.push(entry); - } else { - transcriptMarkerSessionIds.push(directory); - } - } catch (error) { - if (isNotFound(error)) { - const canonicalStateExists = - (await input.destination.has(directory)) || - (await input.destination.isTombstoned(directory)); - if (canonicalStateExists) continue; - throw error; - } - // Corrupt or malformed legacy session headers should not crash the - // entire import. Skip the session without tombstoning it — the - // deletion tombstone is permanent and would suppress a repaired - // header on subsequent launches. Skipping means the malformed - // session is retried next launch, which is harmless (it will be - // skipped again until repaired). - if (error instanceof MalformedLegacySessionHeaderError) { - continue; - } - throw error; - } - } - for (const sessionId of transcriptMarkerSessionIds) { - if ( - !(await input.destination.has(sessionId)) && - !(await input.destination.isTombstoned(sessionId)) - ) { - if (await removeRecoverableOrphanTranscriptMarker(sessionsRoot, sessionId)) continue; - throw new Error(`Session transcript marker has no SQLite metadata: ${sessionId}`); - } - } - const result = await input.destination.importEntries(entries); - await Promise.all( - entries.map((entry) => - entry.boundaryTransferPath - ? rm(entry.boundaryTransferPath, { force: true }) - : Promise.resolve(), - ), - ); - const headersImported = result.created.filter(Boolean).length; - return { - filesScanned: directories.length, - headersRead: entries.length, - headersImported, - headersExisting: result.created.length - headersImported, - sourcesAlreadyImported: result.sourcesAlreadyImported, - sourcesTombstoned: result.sourcesTombstoned, - }; -} - -/** - * Recover only the exact filesystem state published before SQLite admission: - * one canonical marker record in an otherwise empty Session directory. - * - * Any extra transcript byte or directory entry may contain user/runtime state - * and therefore remains a fail-closed corruption boundary. - */ -async function removeRecoverableOrphanTranscriptMarker( - sessionsRoot: string, - sessionId: string, -): Promise { - const sessionDir = join(sessionsRoot, sessionId); - const entries = await readdir(sessionDir, { withFileTypes: true }); - if (entries.length !== 1 || entries[0]?.name !== 'session.jsonl' || !entries[0].isFile()) { - return false; - } - const path = join(sessionDir, 'session.jsonl'); - const actual = await readFile(path, 'utf8'); - const expected = `${JSON.stringify(createSessionTranscriptMarker(sessionId))}\n`; - if (actual !== expected) return false; - await rm(sessionDir, { recursive: true }); - return true; -} - -export async function readLegacySessionMetadataEntry( - sourcePath: string, - sessionId: string, -): Promise { - const headerLine = await readFirstJsonlRecord(sourcePath); - let value: unknown; - try { - value = JSON.parse(headerLine) as unknown; - } catch (error) { - throw new MalformedLegacySessionHeaderError(sourcePath, error); - } - if (isSessionTranscriptMarker(value)) { - decodeSessionTranscriptMarker(value, sessionId); - return null; - } - let header; - try { - header = decodeSessionHeader(value, sessionId); - } catch (error) { - throw new MalformedLegacySessionHeaderError(sourcePath, error); - } - const boundaryTransfer = await readExecutionBoundaryTransfer(sourcePath); - const fingerprintSource = boundaryTransfer - ? `${headerLine}\n${boundaryTransfer.raw}` - : headerLine; - return { - header, - ...(boundaryTransfer - ? { - initialBoundary: boundaryTransfer.boundary, - boundaryTransferPath: boundaryTransfer.path, - } - : {}), - source: { - path: sourcePath, - fingerprint: createHash('sha256').update(fingerprintSource).digest('hex'), - }, - }; -} - -async function readExecutionBoundaryTransfer(sourcePath: string): Promise< - | { - boundary: ExecutionBoundary; - path: string; - raw: string; - } - | undefined -> { - const path = join(dirname(sourcePath), EXECUTION_BOUNDARY_TRANSFER_FILE); - let info; - try { - info = await lstat(path); - } catch (error) { - if (isNotFound(error)) return undefined; - throw error; - } - if (!info.isFile() || info.size > EXECUTION_BOUNDARY_TRANSFER_MAX_BYTES) { - throw new MalformedLegacySessionHeaderError( - sourcePath, - new Error(`Invalid execution boundary transfer at ${path}`), - ); - } - const raw = await readFile(path, 'utf8'); - try { - const value = JSON.parse(raw) as unknown; - if ( - typeof value !== 'object' || - value === null || - !('schemaVersion' in value) || - value.schemaVersion !== 1 || - !('boundary' in value) - ) { - throw new Error('Invalid execution boundary transfer envelope'); - } - return { - boundary: decodeExecutionBoundary(value.boundary), - path, - raw, - }; - } catch (error) { - throw new MalformedLegacySessionHeaderError(sourcePath, error); - } -} - -function isNotFound(error: unknown): boolean { - return typeof error === 'object' && error !== null && 'code' in error && error.code === 'ENOENT'; -} - -async function sessionDirectoryNames(root: string): Promise { - let entries; - try { - entries = await readdir(root, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const names: string[] = []; - for (const entry of entries) { - if (!entry.isDirectory()) continue; - if (!isSafeSessionId(entry.name)) { - throw new Error(`Invalid Session entry: ${entry.name}`); - } - names.push(entry.name); - } - return names.sort(); -} - -async function readFirstJsonlRecord(path: string): Promise { - const handle = await open(path, 'r'); - try { - const chunks: Buffer[] = []; - let offset = 0; - while (offset < LEGACY_SESSION_HEADER_MAX_BYTES) { - const buffer = Buffer.alloc( - Math.min(LEGACY_SESSION_HEADER_READ_BYTES, LEGACY_SESSION_HEADER_MAX_BYTES - offset), - ); - const { bytesRead } = await handle.read(buffer, 0, buffer.length, offset); - if (bytesRead === 0) break; - chunks.push(buffer.subarray(0, bytesRead)); - const text = Buffer.concat(chunks).toString('utf8'); - const newline = text.indexOf('\n'); - if (newline >= 0) return text.slice(0, newline); - offset += bytesRead; - } - throw new MalformedLegacySessionHeaderError( - path, - new Error(`Cannot read legacy session header from ${path}`), - ); - } finally { - await handle.close(); - } -} diff --git a/packages/storage/src/session-store.ts b/packages/storage/src/session-store.ts index 65d633f8a8..342f2f7403 100644 --- a/packages/storage/src/session-store.ts +++ b/packages/storage/src/session-store.ts @@ -1,14 +1,5 @@ -import { mkdir, open, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; +import { join } from 'node:path'; import { createHash, randomUUID } from 'node:crypto'; -import { setTimeout as delay } from 'node:timers/promises'; -import { - decodeStoredMessageForRead, - decodeStoredMessageForRecovery, -} from './execution-record-codec.js'; -import { appendJsonl } from './jsonl-append.js'; -import { classifyJsonRecord } from './json-prefix.js'; -import { importLegacySessionMetadataTree } from './session-metadata-transfer.js'; import { createSqliteSessionMetadataStore, type SessionConfigurationMetadataUpdate, @@ -20,23 +11,13 @@ import { type StableSessionCreateProbe, type VersionedSessionIdentity, } from './sqlite-session-metadata-store.js'; -import { - isDiscardableConversationCopy, - isValidConversationCopyTransition, -} from './session-conversation-copy.js'; -import { - createSessionTranscriptMarker, - decodeSessionTranscriptMarker, - isSessionTranscriptMarker, -} from './session-transcript.js'; -import { chainWrite } from './write-queue.js'; +import { isDiscardableConversationCopy } from './session-conversation-copy.js'; import { acquireOperationalStateDatabase, OPERATIONAL_STATE_DATABASE_NAME, } from './operational-state-store.js'; import { DEFAULT_SESSION_NAME, - DurableStoreWriteError, deriveTurnRecords, isCollaborationMode, isOrchestrationMode, @@ -52,7 +33,6 @@ import { subagentSessionRuntimeSummary, WORKSPACE_AUTHORITY_SESSION_ID, } from '@maka/core'; -import { syncDirectoryChain, syncFile } from './stable-storage.js'; import type { AgentGraphOperatorProvisionRequest, AgentGraphOperatorProvisionResult, @@ -72,9 +52,14 @@ import type { } from '@maka/core'; const SESSION_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; -/** @deprecated Session metadata is canonical in the operational runtime.sqlite database. */ -export const SQLITE_SESSION_METADATA_DATABASE_NAME = OPERATIONAL_STATE_DATABASE_NAME; +export function isSafeSessionId(sessionId: string): boolean { + return SESSION_ID_PATTERN.test(sessionId); +} + +export function assertSafeSessionId(sessionId: string): void { + if (!isSafeSessionId(sessionId)) throw new Error(`Invalid Session id: ${sessionId}`); +} export class SessionNotFoundError extends Error { readonly name = 'SessionNotFoundError'; readonly code = 'session_not_found'; @@ -171,7 +156,7 @@ export interface SessionStore { readHeaderSnapshot(sessionId: string): Promise; /** Read durable messages without triggering connection-lock self-healing. */ readMessagesSnapshot(sessionId: string): Promise; - /** Read messages for startup recovery, rejecting durable JSONL corruption. */ + /** Read durable messages for startup recovery. */ readMessagesForRecovery(sessionId: string): Promise; /** Derive durable turns without triggering connection-lock self-healing. */ listTurnsSnapshot(sessionId: string): Promise; @@ -262,7 +247,6 @@ export interface SessionAuthorityStore extends SessionStore { removeSessionsVersioned(sessions: readonly VersionedSessionIdentity[]): Promise; reconcileOrphanedAgentGraphRetirements(): Promise; listPendingSessionRetirementCleanupIds(sessionId?: string): Promise; - purgeRemovedSessionTranscript(sessionId: string): Promise; completeSessionRetirementCleanup(sessionId: string): Promise; } @@ -282,44 +266,18 @@ export function createSessionStoreWithTestDependencies( return new SqliteSessionStore(workspaceRoot, dependencies); } -/** Legacy JSONL-header store retained only for migration and compatibility tests. */ -export function createLegacyFileSessionStore(workspaceRoot: string): SessionStore { - return new FileSessionStore(workspaceRoot); -} - class SqliteSessionStore implements SessionAuthorityStore { - private readonly files: FileSessionStore; private readonly metadata: SqliteSessionMetadataStore; - private readonly ready: Promise; + private readonly workspaceRoot: string; private closePromise: Promise | null = null; - private activeCatalogProjectionWrites = 0; - private catalogProjectionWritesIdle: Promise = Promise.resolve(); - private resolveCatalogProjectionWritesIdle: (() => void) | undefined; - private catalogProjectionRecovery: Promise | null = null; - private catalogProjectionFailure: unknown; - private readonly removeTranscript: (sessionId: string) => Promise; - - constructor(workspaceRoot: string, dependencies: SessionAuthorityStoreTestDependencies) { - this.files = new FileSessionStore(workspaceRoot, true); - this.removeTranscript = async (sessionId) => { - await dependencies.beforeTranscriptRemoval?.(sessionId); - await this.files.remove(sessionId); - }; + + constructor(workspaceRoot: string, _dependencies: SessionAuthorityStoreTestDependencies) { + this.workspaceRoot = workspaceRoot; const databaseLease = acquireOperationalStateDatabase(workspaceRoot); this.metadata = createSqliteSessionMetadataStore( join(workspaceRoot, OPERATIONAL_STATE_DATABASE_NAME), { databaseLease }, ); - this.ready = importLegacySessionMetadataTree({ - workspaceRoot, - destination: this.metadata, - }).then(async (report) => { - if (report.headersImported > 0) { - await this.metadata.requireCatalogProjectionRecovery(); - } - await this.recoverCatalogProjections(); - }); - void this.ready.catch(() => {}); } async create( @@ -331,13 +289,9 @@ class SqliteSessionStore implements SessionAuthorityStore { if (input.subagentSpawn) { throw new Error('Subagent spawn metadata requires createSubagent()'); } - const staged = await this.files.createTranscript(input); - try { - return (await this.metadata.create(staged, initialBoundary)).header; - } catch (error) { - await this.files.remove(staged.id).catch(() => {}); - throw error; - } + return ( + await this.metadata.create(buildSessionHeader(this.workspaceRoot, input), initialBoundary) + ).header; } async probeStableSessionCreate( @@ -373,9 +327,13 @@ class SqliteSessionStore implements SessionAuthorityStore { } if (probe.kind === 'conflict') return probe; - const staged = await this.files.ensureStableTranscript(request.input, request.sessionId); const result = await this.metadata.createStableSession( - staged, + buildSessionHeader( + this.workspaceRoot, + request.input, + request.sessionId, + request.input.conversationCopy, + ), request.requestFingerprint, initialBoundary, ); @@ -405,7 +363,6 @@ class SqliteSessionStore implements SessionAuthorityStore { throw new Error('Only a matching incomplete conversation copy can be discarded'); } } - await this.files.remove(sessionId); return this.metadata.discardStableSessionCreate(sessionId, requestFingerprint); } @@ -415,15 +372,11 @@ class SqliteSessionStore implements SessionAuthorityStore { ): Promise<{ header: SessionHeader; created: boolean }> { await this.ensureReady(); assertNoConversationCopyMetadata(input); - const staged = await this.files.createTranscript(input); - try { - const result = await this.metadata.createSubagent(staged, initialBoundary); - if (!result.created) await this.files.remove(staged.id); - return { header: result.record.header, created: result.created }; - } catch (error) { - await this.files.remove(staged.id).catch(() => {}); - throw error; - } + const result = await this.metadata.createSubagent( + buildSessionHeader(this.workspaceRoot, input), + initialBoundary, + ); + return { header: result.record.header, created: result.created }; } async createAgentGraphOperator( @@ -434,24 +387,17 @@ class SqliteSessionStore implements SessionAuthorityStore { ): Promise<{ header: SessionHeader } & AgentGraphOperatorProvisionResult> { await this.ensureReady(); assertNoConversationCopyMetadata(input); - const staged = await this.files.createTranscript(input); - try { - const result = await this.metadata.createAgentGraphOperator( - staged, - request, - expectedRevision, - initialBoundary, - ); - if (!result.created) await this.files.remove(staged.id); - return { - header: result.record.header, - provision: result.provision, - created: result.created, - }; - } catch (error) { - await this.files.remove(staged.id).catch(() => {}); - throw error; - } + const result = await this.metadata.createAgentGraphOperator( + buildSessionHeader(this.workspaceRoot, input), + request, + expectedRevision, + initialBoundary, + ); + return { + header: result.record.header, + provision: result.provision, + created: result.created, + }; } async readExecutionBoundary(sessionId: string): Promise { @@ -513,9 +459,7 @@ class SqliteSessionStore implements SessionAuthorityStore { previewMessages: StoredMessage[]; }> = []; for (const record of records) { - const previewMessages = await this.files - .readPreviewMessages(record.header.id) - .catch(() => []); + const previewMessages = await this.metadata.readPreviewMessages(record.header.id); withPreviews.push({ record, previewMessages }); } withPreviews.sort((a, b) => { @@ -537,9 +481,7 @@ class SqliteSessionStore implements SessionAuthorityStore { const { header } = record; let messages = previewMessages.slice(-10); if (index < 3) { - messages = ( - await this.files.readTranscriptMessagesSnapshot(header.id, header).catch(() => messages) - ).slice(-10); + messages = (await this.metadata.readMessages(header.id)).slice(-10); } summaries.push(toSummary(header, messages)); } @@ -577,7 +519,7 @@ class SqliteSessionStore implements SessionAuthorityStore { async listForRecovery(): Promise { const headers = await this.listHeaders(); for (const header of headers) { - await this.files.readTranscriptMessagesForRecovery(header.id, header); + await this.metadata.readMessagesForRecovery(header.id); } return headers; } @@ -609,14 +551,12 @@ class SqliteSessionStore implements SessionAuthorityStore { async readMessagesSnapshot(sessionId: string): Promise { await this.ensureReady(); - const header = (await this.metadata.read(sessionId)).header; - return this.files.readTranscriptMessagesSnapshot(sessionId, header); + return this.metadata.readMessages(sessionId); } async readMessagesForRecovery(sessionId: string): Promise { await this.ensureReady(); - const header = (await this.metadata.read(sessionId)).header; - return this.files.readTranscriptMessagesForRecovery(sessionId, header); + return this.metadata.readMessagesForRecovery(sessionId); } async listTurnsSnapshot(sessionId: string): Promise { @@ -645,26 +585,11 @@ class SqliteSessionStore implements SessionAuthorityStore { async appendMessages(sessionId: string, messages: StoredMessage[]): Promise { if (messages.length === 0) return; - const release = await this.acquireCatalogProjectionWrite(); - try { - await this.metadata.beginCatalogProjectionWrite(); - await this.files.appendMessages(sessionId, messages); - await this.metadata.commitCatalogProjectionWrite( - sessionId, - catalogMessageProjection(messages), - ); - } catch (error) { - const recovery = this.scheduleCatalogProjectionRecovery(); - release(); - try { - await recovery; - } catch { - throw error; - } - throw error; - } finally { - release(); - } + await this.metadata.appendMessages( + sessionId, + messages, + projectSessionCatalogMessages(messages), + ); } async updateHeader(sessionId: string, patch: Partial): Promise { @@ -750,25 +675,7 @@ class SqliteSessionStore implements SessionAuthorityStore { async listPendingSessionRetirementCleanupIds(sessionId?: string): Promise { await this.ensureReady(); - const [pending, existing] = await Promise.all([ - this.metadata.listPendingSessionRetirementCleanupIds(sessionId), - this.files.listSessionDirectoryIds(), - ]); - const resurrected = await this.metadata.listTombstonedSessionIdsAmong(existing); - const relevantResurrected = - sessionId === undefined - ? resurrected - : resurrected.filter((candidate) => candidate === sessionId); - return [...new Set([...pending, ...relevantResurrected])].sort(); - } - - async purgeRemovedSessionTranscript(sessionId: string): Promise { - await this.ensureReady(); - const probe = await this.metadata.probeRemoval(sessionId); - if (probe.kind !== 'removed') { - throw new Error(`Cannot purge transcript for a Session that is ${probe.kind}`); - } - await this.removeTranscript(sessionId); + return this.metadata.listPendingSessionRetirementCleanupIds(sessionId); } async completeSessionRetirementCleanup(sessionId: string): Promise { @@ -843,7 +750,6 @@ class SqliteSessionStore implements SessionAuthorityStore { async remove(sessionId: string): Promise { await this.ensureReady(); await this.metadata.remove(sessionId); - await this.removeTranscript(sessionId); } close(): Promise { @@ -852,8 +758,6 @@ class SqliteSessionStore implements SessionAuthorityStore { } private async closeAfterReady(): Promise { - await this.ready.catch(() => {}); - await this.catalogProjectionRecovery?.catch(() => {}); this.metadata.close(); } @@ -862,980 +766,83 @@ class SqliteSessionStore implements SessionAuthorityStore { knownMessages?: StoredMessage[], ): Promise { if (header.connectionLocked) return header; - const messages = - knownMessages ?? (await this.files.readTranscriptMessagesSnapshot(header.id, header)); + const messages = knownMessages ?? (await this.metadata.readMessages(header.id)); if (!messages.some((message) => message.type === 'user')) return header; return this.updateHeader(header.id, { connectionLocked: true }); } - private async recoverCatalogProjections(): Promise { - if (!(await this.metadata.hasPendingCatalogProjectionWrites())) return; - const projections = new Map>(); - for (const record of await this.metadata.list()) { - try { - const messages = await this.files.readTranscriptMessagesForRecovery( - record.header.id, - record.header, - ); - projections.set(record.header.id, catalogMessageProjection(messages)); - } catch (error) { - if (!isDiscardableConversationCopy(record.header)) throw error; - } - } - await this.metadata.recoverCatalogProjections(projections); - } - - private async ensureReady(): Promise { - await this.ready; - if (this.catalogProjectionRecovery) await this.catalogProjectionRecovery; - if (this.catalogProjectionFailure) throw this.catalogProjectionFailure; - } + private async ensureReady(): Promise {} private async ensureCatalogProjectionReadable(): Promise { await this.ensureReady(); - if ( - this.activeCatalogProjectionWrites === 0 && - (await this.metadata.hasPendingCatalogProjectionWrites()) - ) { - await this.scheduleCatalogProjectionRecovery(); - } - } - - private async acquireCatalogProjectionWrite(): Promise<() => void> { - while (true) { - await this.ensureReady(); - if (!this.catalogProjectionRecovery) break; - } - if (this.activeCatalogProjectionWrites === 0) { - this.catalogProjectionWritesIdle = new Promise((resolve) => { - this.resolveCatalogProjectionWritesIdle = resolve; - }); - } - this.activeCatalogProjectionWrites += 1; - let released = false; - return () => { - if (released) return; - released = true; - this.activeCatalogProjectionWrites -= 1; - if (this.activeCatalogProjectionWrites === 0) { - this.resolveCatalogProjectionWritesIdle?.(); - this.resolveCatalogProjectionWritesIdle = undefined; - } - }; - } - - private scheduleCatalogProjectionRecovery(): Promise { - if (this.catalogProjectionRecovery) return this.catalogProjectionRecovery; - const recovery = (async () => { - await this.catalogProjectionWritesIdle; - if (await this.metadata.hasPendingCatalogProjectionWrites()) { - await this.recoverCatalogProjections(); - } - })(); - this.catalogProjectionRecovery = recovery; - void recovery.then( - () => { - if (this.catalogProjectionRecovery === recovery) { - this.catalogProjectionRecovery = null; - } - }, - (error: unknown) => { - this.catalogProjectionFailure = error; - if (this.catalogProjectionRecovery === recovery) { - this.catalogProjectionRecovery = null; - } - }, - ); - return recovery; } } -class FileSessionStore implements SessionStore { - private static readonly HEADER_BUDGET = 8192; - private static readonly MAX_HEADER_BYTES = 1024 * 1024; - private static readonly TAIL_PREVIEW_BUDGET = 64 * 1024; - private readonly sessionsRoot: string; - private readonly writeQueues = new Map>(); - - constructor( - private readonly workspaceRoot: string, - private readonly durableTranscripts = false, +function buildSessionHeader( + workspaceRoot: string, + input: CreateSessionInput, + sessionId: string = randomUUID(), + conversationCopy?: SessionConversationCopy, +): SessionHeader { + if ( + input.projectId !== undefined && + input.projectId !== null && + (typeof input.projectId !== 'string' || input.projectId.length === 0) ) { - this.sessionsRoot = join(workspaceRoot, 'sessions'); - } - - async create(input: CreateSessionInput): Promise { - assertNoConversationCopyMetadata(input); - if (input.subagentSpawn) { - throw new Error('Child-session idempotency requires the SQLite metadata control plane'); - } - return this.createWithInitialRecord(input, 'legacy-header'); - } - - async createTranscript(input: CreateSessionInput, sessionId?: string): Promise { - assertNoConversationCopyMetadata(input); - return this.createWithInitialRecord(input, 'transcript-marker', sessionId); - } - - async ensureStableTranscript( - input: StableSessionCreateInput, - sessionId: string, - ): Promise { - return this.createWithInitialRecord( - input, - 'transcript-marker', - sessionId, - true, - input.conversationCopy, - ); - } - - private async createWithInitialRecord( - input: CreateSessionInput, - initialRecord: 'legacy-header' | 'transcript-marker', - sessionId?: string, - reuseStableTranscript = false, - conversationCopy?: SessionConversationCopy, - ): Promise { - if ( - input.projectId !== undefined && - input.projectId !== null && - (typeof input.projectId !== 'string' || input.projectId.length === 0) - ) { - throw new Error('Invalid project id'); - } - const now = Date.now(); - const id = sessionId ?? randomUUID(); - assertSafeSessionId(id); - // PR-UI-IPC-2 (@kenji msg 0474c3fe + @xuan msg 88d96a87): - // session name write contract. If caller passed undefined, - // use the canonical default; otherwise normalize the - // user-supplied name through the same `normalizeUserSessionName` - // gate that `rename` and `branchFromTurn` use. Empty-after- - // sanitize on an explicit input is a REJECT — we do NOT - // silently fall back to default, that would swallow the - // user's intent (per @xuan caller-semantics lock). - let resolvedName: string; - if (input.name === undefined) { - resolvedName = DEFAULT_SESSION_NAME; - } else { - const normalized = normalizeUserSessionName(input.name); - if (!normalized.ok) { - throw new Error(normalized.error); - } - resolvedName = normalized.value; - } - const header: SessionHeader = { - id, - workspaceRoot: this.workspaceRoot, - cwd: input.cwd, - ...(input.projectId !== undefined ? { projectId: input.projectId } : {}), - createdAt: now, - lastUsedAt: now, - name: resolvedName, - titleIsManual: false, - isFlagged: false, - labels: input.labels ?? [], - isArchived: false, - status: input.status ?? 'active', - ...(input.blockedReason ? { blockedReason: input.blockedReason } : {}), - statusUpdatedAt: now, - ...(input.parentSessionId ? { parentSessionId: input.parentSessionId } : {}), - ...(input.branchOfTurnId ? { branchOfTurnId: input.branchOfTurnId } : {}), - ...(input.subagentParent ? { subagentParent: input.subagentParent } : {}), - ...(input.subagentRuntime ? { subagentRuntime: input.subagentRuntime } : {}), - ...(input.subagentSpawn ? { subagentSpawn: input.subagentSpawn } : {}), - ...(input.subagentWorkspace ? { subagentWorkspace: input.subagentWorkspace } : {}), - ...(conversationCopy ? { conversationCopy } : {}), - ...(input.revisionRootSessionId - ? { revisionRootSessionId: input.revisionRootSessionId } - : {}), - ...(input.revisionParentSessionId - ? { revisionParentSessionId: input.revisionParentSessionId } - : {}), - ...(input.revisionOfTurnId ? { revisionOfTurnId: input.revisionOfTurnId } : {}), - ...(input.revisionIndex !== undefined ? { revisionIndex: input.revisionIndex } : {}), - ...(input.revisionState ? { revisionState: input.revisionState } : {}), - hasUnread: false, - backend: input.backend, - llmConnectionSlug: input.llmConnectionSlug, - connectionLocked: false, - model: input.model ?? 'default', - permissionMode: input.permissionMode, - collaborationMode: input.collaborationMode ?? 'agent', - orchestrationMode: input.orchestrationMode ?? 'default', - ...(input.thinkingLevel !== undefined ? { thinkingLevel: input.thinkingLevel } : {}), - schemaVersion: 1, - }; - - assertValidSessionLineage(header); - - await this.withQueue(id, async () => { - await mkdir(this.sessionsRoot, { recursive: true }); - if (reuseStableTranscript) { - try { - await mkdir(this.sessionDir(id)); - } catch (error) { - if (!hasErrorCode(error, 'EEXIST')) throw error; - } - await this.ensureMarkerOnlyTranscript(id); - return; - } - await mkdir(this.sessionDir(id)); - const firstRecord = - initialRecord === 'legacy-header' ? header : createSessionTranscriptMarker(header.id); - try { - await writeNewTranscript( - this.sessionPath(id), - JSON.stringify(firstRecord) + '\n', - this.durableTranscripts ? this.workspaceRoot : undefined, - ); - } catch (error) { - await rm(this.sessionDir(id), { recursive: true, force: true }).catch(() => {}); - throw error; - } - }); - - return header; - } - - private async ensureMarkerOnlyTranscript(sessionId: string): Promise { - const path = this.sessionPath(sessionId); - const marker = JSON.stringify(createSessionTranscriptMarker(sessionId)) + '\n'; - const entries = await readdir(this.sessionDir(sessionId)); - if (entries.length === 0) { - try { - await writeNewTranscript( - path, - marker, - this.durableTranscripts ? this.workspaceRoot : undefined, - ); - return; - } catch (error) { - if (!hasErrorCode(error, 'EEXIST')) throw error; - } - } else if (entries.length !== 1 || entries[0] !== 'session.jsonl') { - throw new Error(`Session ${sessionId}: stable transcript path is not recoverable`); - } - - const text = await readFile(path, 'utf8'); - if (text === marker) { - if (this.durableTranscripts) { - await stabilizeTranscript(path, this.workspaceRoot); - } - return; - } - if (marker.startsWith(text)) { - await this.writeAtomic(path, marker); - if (this.durableTranscripts) { - await stabilizeTranscript(path, this.workspaceRoot); - } - return; - } - const records = text.split('\n').filter((line) => line.trim().length > 0); - if (records.length !== 1 || !records[0]) { - throw new Error(`Session ${sessionId}: stable transcript is not marker-only`); - } - decodeSessionTranscriptMarker(JSON.parse(records[0]), sessionId); - await this.writeAtomic(path, marker); - if (this.durableTranscripts) { - await stabilizeTranscript(path, this.workspaceRoot); - } - } - - async list(filter?: SessionListFilter): Promise { - if (filter?.subagentParentSessionId !== undefined) { - throw new Error('Subagent session relation queries require SQLite session metadata'); - } - let entries; - try { - entries = await readdir(this.sessionsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - - // Phase 1: read each header plus a bounded tail preview. That keeps - // list() proportional to the number of sessions rather than full - // transcript size, while preserving sidebar previews and timestamp - // fallback for sessions outside the top few. - const withHeaders: Array<{ - id: string; - header: SessionHeader; - previewMessages: StoredMessage[]; - }> = []; - for (const entry of entries) { - if (!entry.isDirectory()) continue; - if (!isSafeSessionId(entry.name)) continue; - try { - const header = await this.readHeaderOnly(entry.name); - if (filter?.isArchived !== undefined && header.isArchived !== filter.isArchived) continue; - if (filter?.isFlagged !== undefined && header.isFlagged !== filter.isFlagged) continue; - if (filter?.labelSlug && !header.labels.includes(filter.labelSlug)) continue; - const previewMessages = await this.readTailPreviewMessages(entry.name).catch(() => []); - withHeaders.push({ id: entry.name, header, previewMessages }); - } catch { - // Ignore malformed session folders in the sidebar. - } - } - - // Secondary key on id (lexicographic) so sessions with identical - // lastMessageAt always sort in the same order - fixtures with - // multiple sessions seeded at the same frozen timestamp would - // otherwise drift across runs based on filesystem readdir order - // (PR108k-yj per @kenji e2e-fixture determinism). Negligible cost - // for real users; identical lastMessageAt is rare in production. - withHeaders.sort((a, b) => { - const aLastMessageAt = maxTimestamp( - a.header.lastMessageAt, - latestVisibleMessageAt(a.previewMessages), - ); - const bLastMessageAt = maxTimestamp( - b.header.lastMessageAt, - latestVisibleMessageAt(b.previewMessages), - ); - const tsDelta = (bLastMessageAt ?? 0) - (aLastMessageAt ?? 0); - if (tsDelta !== 0) return tsDelta; - return a.header.id.localeCompare(b.header.id); - }); - - // Phase 2: full detail read only for the most recent 3 sessions. - // For those, keep only the last 10 messages as preview. Remaining - // sessions use the bounded tail preview from phase 1. - const TOP_N = 3; - const summaries: SessionSummary[] = []; - for (let i = 0; i < withHeaders.length; i++) { - const { header, previewMessages } = withHeaders[i]; - let messages: StoredMessage[] = previewMessages.slice(-10); - if (i < TOP_N) { - try { - const result = await this.readFilePartsUnlocked(header.id); - messages = result.messages.slice(-10); - } catch { - // Fall through to the bounded tail preview from phase 1. - } - } - summaries.push(toSummary(header, messages)); - } - return summaries; - } - - async listSessionDirectoryIds(): Promise { - let entries; - try { - entries = await readdir(this.sessionsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - return entries - .filter((entry) => entry.isDirectory() && SESSION_ID_PATTERN.test(entry.name)) - .map((entry) => entry.name) - .sort(); - } - - async listForRecovery(): Promise { - return this.listHeaders(); - } - - async listHeaders(): Promise { - let entries; - try { - entries = await readdir(this.sessionsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const headers: SessionHeader[] = []; - for (const entry of entries) { - if (!entry.isDirectory() || !isSafeSessionId(entry.name)) { - throw new Error(`Invalid Session entry: ${entry.name}`); - } - headers.push(await this.readHeaderOnly(entry.name)); - } - return headers.sort((a, b) => a.id.localeCompare(b.id)); - } - - async readHeader(sessionId: string): Promise { - const { header, messages } = await this.readFileParts(sessionId); - if (!header.connectionLocked && messages.some((message) => message.type === 'user')) { - return this.updateHeader(sessionId, { connectionLocked: true }); - } - return header; - } - - async readHeaderSnapshot(sessionId: string): Promise { - try { - return await this.readHeaderOnly(sessionId); - } catch (error) { - if ((error as NodeJS.ErrnoException | undefined)?.code === 'ENOENT') { - throw new SessionNotFoundError(sessionId); - } - throw error; - } - } - - async readMessages(sessionId: string): Promise { - const { header, messages } = await this.readFileParts(sessionId); - if (!header.connectionLocked && messages.some((message) => message.type === 'user')) { - await this.updateHeader(sessionId, { connectionLocked: true }); - } - return messages; - } - - async readMessagesSnapshot(sessionId: string): Promise { - return (await this.readFileParts(sessionId)).messages; - } - - async readPreviewMessages(sessionId: string): Promise { - return this.readTailPreviewMessages(sessionId); - } - - async readTranscriptMessagesSnapshot( - sessionId: string, - header: SessionHeader, - ): Promise { - return this.readTranscriptMessagesUnlocked(sessionId, header); - } - - async readTranscriptMessagesForRecovery( - sessionId: string, - header: SessionHeader, - ): Promise { - return this.readTranscriptMessagesUnlocked(sessionId, header, true); - } - - async readMessagesForRecovery(sessionId: string): Promise { - return (await this.readFilePartsUnlocked(sessionId, true)).messages; - } - - async listTurnsSnapshot(sessionId: string): Promise { - return deriveTurnRecords(await this.readMessagesSnapshot(sessionId)); - } - - async listTurns(sessionId: string): Promise { - return deriveTurnRecords(await this.readMessages(sessionId)); - } - - async appendMessage(sessionId: string, message: StoredMessage): Promise { - await this.appendMessages(sessionId, [message]); - } - - async appendMessages(sessionId: string, messages: StoredMessage[]): Promise { - if (messages.length === 0) return; - await this.withQueue(sessionId, async () => { - const payload = messages.map((message) => JSON.stringify(message)).join('\n') + '\n'; - await appendJsonl(this.sessionPath(sessionId), payload, { - durable: this.durableTranscripts, - ...(this.durableTranscripts ? { durabilityRoot: this.workspaceRoot } : {}), - requireExistingRecord: true, - }); - }); - } - - async updateHeader(sessionId: string, patch: Partial): Promise { - if (Object.prototype.hasOwnProperty.call(patch, 'subagentParent')) { - throw new Error('Subagent session parent relation is immutable'); - } - if (Object.prototype.hasOwnProperty.call(patch, 'subagentRuntime')) { - throw new Error('Subagent session runtime snapshot is immutable'); - } - if (Object.prototype.hasOwnProperty.call(patch, 'subagentSpawn')) { - throw new Error('Subagent session spawn identity is immutable'); - } - if (Object.prototype.hasOwnProperty.call(patch, 'subagentWorkspace')) { - throw new Error('Subagent session workspace binding is immutable'); - } - let nextHeader: SessionHeader | undefined; - await this.withQueue(sessionId, async () => { - const { header, messages } = await this.readFilePartsUnlocked(sessionId); - assertConversationCopyTransition(header, patch); - nextHeader = { ...header, ...patch }; - assertValidSessionLineage(nextHeader); - const lines = [ - JSON.stringify(nextHeader), - ...messages.map((message) => JSON.stringify(message)), - ]; - await this.writeAtomic(this.sessionPath(sessionId), lines.join('\n') + '\n'); - }); - if (!nextHeader) throw new Error(`Failed to update session ${sessionId}`); - return nextHeader; - } - - async markSessionReadThrough(sessionId: string, readThroughTs: number): Promise { - let nextHeader: SessionHeader | undefined; - await this.withQueue(sessionId, async () => { - const { header, messages } = await this.readFilePartsUnlocked(sessionId); - const effectiveLastMessageAt = maxTimestamp( - header.lastMessageAt, - latestVisibleMessageAt(messages), - ); - if ( - !Number.isFinite(readThroughTs) || - !header.hasUnread || - (effectiveLastMessageAt !== undefined && effectiveLastMessageAt > readThroughTs) - ) { - nextHeader = header; - return; - } - nextHeader = { ...header, hasUnread: false }; - const lines = [ - JSON.stringify(nextHeader), - ...messages.map((message) => JSON.stringify(message)), - ]; - await this.writeAtomic(this.sessionPath(sessionId), lines.join('\n') + '\n'); - }); - if (!nextHeader) throw new Error(`Failed to update session ${sessionId}`); - return nextHeader; - } - - async archive(sessionId: string): Promise { - const now = Date.now(); - await this.updateHeader(sessionId, { - isArchived: true, - archivedAt: now, - status: 'archived', - statusUpdatedAt: now, - }); - } - - async unarchive(sessionId: string): Promise { - await this.updateHeader(sessionId, { - isArchived: false, - archivedAt: undefined, - status: 'active', - blockedReason: undefined, - statusUpdatedAt: Date.now(), - }); - } - - async setFlagged(sessionId: string, isFlagged: boolean): Promise { - await this.updateHeader(sessionId, { isFlagged }); - } - - async rename(sessionId: string, name: string): Promise { - // PR-UI-IPC-2: same `normalizeUserSessionName` chokepoint as - // create + branch. Replaces the older inline trim + length-80 - // cap with the shared helper so all three write paths go - // through a single contract (control char strip, bidi/zero- - // width defense, NFC, code-point cap, typed reject). - const normalized = normalizeUserSessionName(name); - if (!normalized.ok) { - throw new Error(normalized.error); - } - await this.updateHeader(sessionId, { name: normalized.value, titleIsManual: true }); - } - - async setGeneratedTitleIfAbsent(sessionId: string, title: string): Promise { - const normalized = normalizeUserSessionName(title); - if (!normalized.ok) return null; - let nextHeader: SessionHeader | null = null; - await this.withQueue(sessionId, async () => { - const { header, messages } = await this.readFilePartsUnlocked(sessionId); - if (header.titleIsManual || header.name !== DEFAULT_SESSION_NAME) return; - if (normalized.value === header.name) return; - nextHeader = { ...header, name: normalized.value }; - const lines = [ - JSON.stringify(nextHeader), - ...messages.map((message) => JSON.stringify(message)), - ]; - await this.writeAtomic(this.sessionPath(sessionId), lines.join('\n') + '\n'); - }); - return nextHeader; - } - - async remove(sessionId: string): Promise { - await this.withQueue(sessionId, async () => { - await rm(this.sessionDir(sessionId), { recursive: true, force: true }); - }); - } - - private sessionDir(sessionId: string): string { - assertSafeSessionId(sessionId); - return join(this.sessionsRoot, sessionId); - } - - private sessionPath(sessionId: string): string { - return join(this.sessionDir(sessionId), 'session.jsonl'); - } - - private async readHeaderOnly(sessionId: string): Promise { - // Fast path: read only the first JSON line (the header) without - // parsing any message payload. Used by list() to quickly scan - // all sessions before deciding which ones need detail reads. - const path = this.sessionPath(sessionId); - const handle = await open(path, 'r'); - try { - const chunks: Buffer[] = []; - let offset = 0; - while (offset < FileSessionStore.MAX_HEADER_BYTES) { - const buf = Buffer.alloc( - Math.min(FileSessionStore.HEADER_BUDGET, FileSessionStore.MAX_HEADER_BYTES - offset), - ); - const { bytesRead } = await handle.read(buf, 0, buf.length, offset); - if (bytesRead === 0) break; - chunks.push(buf.subarray(0, bytesRead)); - const region = Buffer.concat(chunks).toString('utf8'); - const firstNl = region.indexOf('\n'); - if (firstNl !== -1) { - return decodeSessionHeader(JSON.parse(region.slice(0, firstNl)), sessionId); - } - offset += bytesRead; - } - throw new Error(`Session ${sessionId}: cannot find header line`); - } finally { - await handle.close(); - } - } - - private async readTailPreviewMessages(sessionId: string): Promise { - const path = this.sessionPath(sessionId); - const handle = await open(path, 'r'); - try { - const { size } = await handle.stat(); - const start = Math.max(0, size - FileSessionStore.TAIL_PREVIEW_BUDGET); - const length = size - start; - if (length <= 0) return []; - const buf = Buffer.alloc(length); - const { bytesRead } = await handle.read(buf, 0, length, start); - const text = buf.toString('utf8', 0, bytesRead); - const rawLines = text.split('\n'); - // The first tail line is either the header (start === 0) or a partial JSONL line. - const lines = rawLines.slice(1); - const completeLines = text.endsWith('\n') ? lines : lines.slice(0, -1); - const messages: StoredMessage[] = []; - for (const line of completeLines) { - if (line.trim().length === 0) continue; - try { - messages.push(decodeStoredMessageForRead(JSON.parse(line))); - } catch { - // Tail previews are best-effort; full reads still surface durable corruption notes. - } - } - return messages; - } finally { - await handle.close(); - } - } - - private async readFileParts( - sessionId: string, - ): Promise<{ header: SessionHeader; messages: StoredMessage[] }> { - return this.readFilePartsUnlocked(sessionId); - } - - private async readFilePartsUnlocked( - sessionId: string, - strict = false, - ): Promise<{ header: SessionHeader; messages: StoredMessage[] }> { - const text = await readFile(this.sessionPath(sessionId), 'utf8'); - const rawLines = text.split('\n'); - const endsWithNewline = text.endsWith('\n'); - const lines = rawLines - .map((line, index) => ({ line, lineNumber: index + 1 })) - .filter((entry) => entry.line.trim().length > 0); - if (lines.length === 0 || !lines[0]) throw new Error(`Session ${sessionId} is empty`); - const header = decodeSessionHeader(JSON.parse(lines[0].line), sessionId); - const messages: StoredMessage[] = []; - const lastLineNumber = lines.at(-1)?.lineNumber; - for (const entry of lines.slice(1)) { - let parsed: unknown; - try { - parsed = JSON.parse(entry.line); - } catch (error) { - if ( - !endsWithNewline && - entry.lineNumber === lastLineNumber && - classifyJsonRecord(entry.line) === 'incomplete-prefix' - ) - continue; - if (strict) { - const detail = error instanceof Error ? error.message : String(error); - throw new Error( - `Session ${sessionId} has a corrupt JSONL record at line ${entry.lineNumber}: ${detail}`, - ); - } - messages.push(createJsonlCorruptionNote(header, entry.lineNumber, error)); - continue; - } - try { - messages.push( - strict ? decodeStoredMessageForRecovery(parsed) : decodeStoredMessageForRead(parsed), - ); - } catch (error) { - if (strict) { - const detail = error instanceof Error ? error.message : String(error); - throw new Error( - `Session ${sessionId} has a corrupt JSONL record at line ${entry.lineNumber}: ${detail}`, - ); - } - messages.push(createJsonlCorruptionNote(header, entry.lineNumber, error)); - } - } - return { header, messages }; - } - - private async readTranscriptMessagesUnlocked( - sessionId: string, - header: SessionHeader, - strict = false, - ): Promise { - const text = await readFile(this.sessionPath(sessionId), 'utf8'); - const rawLines = text.split('\n'); - const endsWithNewline = text.endsWith('\n'); - const lines = rawLines - .map((line, index) => ({ line, lineNumber: index + 1 })) - .filter((entry) => entry.line.trim().length > 0); - if (lines.length === 0 || !lines[0]) throw new Error(`Session ${sessionId} is empty`); - - let firstRecord: unknown; - try { - firstRecord = JSON.parse(lines[0].line) as unknown; - if (isSessionTranscriptMarker(firstRecord)) { - decodeSessionTranscriptMarker(firstRecord, sessionId); - } else { - decodeSessionHeader(firstRecord, sessionId); - } - } catch (error) { - const detail = error instanceof Error ? error.message : String(error); - throw new Error(`Session ${sessionId} has an invalid first JSONL record: ${detail}`); - } - - const messages: StoredMessage[] = []; - const lastLineNumber = lines.at(-1)?.lineNumber; - for (const entry of lines.slice(1)) { - let parsed: unknown; - try { - parsed = JSON.parse(entry.line); - } catch (error) { - if ( - !endsWithNewline && - entry.lineNumber === lastLineNumber && - classifyJsonRecord(entry.line) === 'incomplete-prefix' - ) { - continue; - } - if (strict) { - const detail = error instanceof Error ? error.message : String(error); - throw new Error( - `Session ${sessionId} has a corrupt JSONL record at line ${entry.lineNumber}: ${detail}`, - ); - } - messages.push(createJsonlCorruptionNote(header, entry.lineNumber, error)); - continue; - } - try { - messages.push( - strict ? decodeStoredMessageForRecovery(parsed) : decodeStoredMessageForRead(parsed), - ); - } catch (error) { - if (strict) { - const detail = error instanceof Error ? error.message : String(error); - throw new Error( - `Session ${sessionId} has a corrupt JSONL record at line ${entry.lineNumber}: ${detail}`, - ); - } - messages.push(createJsonlCorruptionNote(header, entry.lineNumber, error)); - } - } - return messages; - } - - private async writeAtomic(path: string, content: string): Promise { - await mkdir(dirname(path), { recursive: true }); - const tempPath = `${path}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`; - await writeFile(tempPath, content, { encoding: 'utf8', mode: 0o600 }); - try { - await replaceFileWithWindowsReaderRetry(tempPath, path); - } finally { - await rm(tempPath, { force: true }).catch(() => {}); - } - } - - private withQueue(sessionId: string, operation: () => Promise): Promise { - assertSafeSessionId(sessionId); - return chainWrite(this.writeQueues, sessionId, operation); - } -} - -async function replaceFileWithWindowsReaderRetry(tempPath: string, path: string): Promise { - const attempts = process.platform === 'win32' ? 6 : 1; - for (let attempt = 1; attempt <= attempts; attempt += 1) { - try { - await rename(tempPath, path); - return; - } catch (error) { - const code = (error as NodeJS.ErrnoException).code; - const retryable = process.platform === 'win32' && (code === 'EPERM' || code === 'EACCES'); - if (!retryable || attempt === attempts) throw error; - await delay(attempt * 10); - } - } -} - -/** Shared guard for stores that derive filesystem paths from a session id. */ -export function assertSafeSessionId(sessionId: string): void { - if (!isSafeSessionId(sessionId)) { - throw new Error('Invalid session id'); - } -} - -export function isSafeSessionId(sessionId: string): boolean { - return SESSION_ID_PATTERN.test(sessionId) && sessionId !== WORKSPACE_AUTHORITY_SESSION_ID; -} - -type StoredSessionHeader = Omit< - SessionHeader, - | 'backend' - | 'model' - | 'permissionMode' - | 'collaborationMode' - | 'orchestrationMode' - | 'status' - | 'blockedReason' - | 'titleIsManual' -> & { - backend: string; - model?: unknown; - permissionMode?: unknown; - collaborationMode?: unknown; - orchestrationMode?: unknown; - status?: unknown; - blockedReason?: unknown; - titleIsManual?: unknown; - /** Accepted only while decoding old session headers and dropped on normalization. */ - pendingCwdReminder?: unknown; -}; - -function createJsonlCorruptionNote( - header: SessionHeader, - lineNumber: number, - error: unknown, -): StoredMessage { - return { - type: 'system_note', - id: `jsonl-corrupt-${lineNumber}`, - ts: header.lastUsedAt ?? header.createdAt, - kind: 'error', - data: { - code: 'jsonl_parse_error', - lineNumber, - message: error instanceof Error ? error.message : 'Invalid JSONL message line', - }, - }; -} - -/** - * Decode the legacy line-1 JSONL header into the current canonical shape. - * - * Kept public for one-way importers so file and SQLite storage apply exactly - * the same compatibility defaults and validation rules. - */ -export function decodeSessionHeader(value: unknown, sessionId: string): SessionHeader { - if (!value || typeof value !== 'object' || Array.isArray(value)) { - throw new Error(`Invalid session header for session ${sessionId}: expected an object`); - } - const header = value as StoredSessionHeader; - const permissionMode = isPermissionMode(header.permissionMode) ? header.permissionMode : 'ask'; - const collaborationMode = isCollaborationMode(header.collaborationMode) - ? header.collaborationMode - : 'agent'; - const orchestrationMode = isOrchestrationMode(header.orchestrationMode) - ? header.orchestrationMode - : 'default'; - const model = - typeof header.model === 'string' && header.model.length > 0 ? header.model : 'default'; - const status = resolveMigratedStatus(header); - const blockedReason = - status === 'blocked' && isSessionBlockedReason(header.blockedReason) - ? header.blockedReason - : undefined; - const statusFields = { - status, - blockedReason, - statusUpdatedAt: - header.statusUpdatedAt ?? - header.archivedAt ?? - header.lastMessageAt ?? - header.lastUsedAt ?? - header.createdAt, + throw new Error('Invalid project id'); + } + const now = Date.now(); + assertSafeSessionId(sessionId); + const name = + input.name === undefined ? DEFAULT_SESSION_NAME : normalizeRequiredSessionName(input.name); + const header: SessionHeader = { + id: sessionId, + workspaceRoot, + cwd: input.cwd, + ...(input.projectId !== undefined ? { projectId: input.projectId } : {}), + createdAt: now, + lastUsedAt: now, + name, + titleIsManual: false, + isFlagged: false, + labels: input.labels ?? [], + isArchived: false, + status: input.status ?? 'active', + ...(input.blockedReason ? { blockedReason: input.blockedReason } : {}), + statusUpdatedAt: now, + ...(input.parentSessionId ? { parentSessionId: input.parentSessionId } : {}), + ...(input.branchOfTurnId ? { branchOfTurnId: input.branchOfTurnId } : {}), + ...(input.subagentParent ? { subagentParent: input.subagentParent } : {}), + ...(input.subagentRuntime ? { subagentRuntime: input.subagentRuntime } : {}), + ...(input.subagentSpawn ? { subagentSpawn: input.subagentSpawn } : {}), + ...(input.subagentWorkspace ? { subagentWorkspace: input.subagentWorkspace } : {}), + ...(conversationCopy ? { conversationCopy } : {}), + ...(input.revisionRootSessionId ? { revisionRootSessionId: input.revisionRootSessionId } : {}), + ...(input.revisionParentSessionId + ? { revisionParentSessionId: input.revisionParentSessionId } + : {}), + ...(input.revisionOfTurnId ? { revisionOfTurnId: input.revisionOfTurnId } : {}), + ...(input.revisionIndex !== undefined ? { revisionIndex: input.revisionIndex } : {}), + ...(input.revisionState ? { revisionState: input.revisionState } : {}), + hasUnread: false, + backend: input.backend, + llmConnectionSlug: input.llmConnectionSlug, + connectionLocked: false, + model: input.model ?? 'default', + permissionMode: input.permissionMode, + collaborationMode: input.collaborationMode ?? 'agent', + orchestrationMode: input.orchestrationMode ?? 'default', + ...(input.thinkingLevel !== undefined ? { thinkingLevel: input.thinkingLevel } : {}), + schemaVersion: 1, }; - const titleIsManual = - typeof header.titleIsManual === 'boolean' - ? header.titleIsManual - : normalizeSessionName(header.name) !== DEFAULT_SESSION_NAME; - if (header.backend === 'claude') { - return normalizeMigratedHeader( - { - ...header, - ...statusFields, - titleIsManual, - backend: 'ai-sdk', - model, - permissionMode, - collaborationMode, - orchestrationMode, - }, - sessionId, - ); - } - if (header.backend === 'pi-agent') { - return normalizeMigratedHeader( - { - ...header, - ...statusFields, - titleIsManual, - backend: 'pi-agent', - model, - permissionMode, - collaborationMode, - orchestrationMode, - }, - sessionId, - ); - } - if (header.backend === 'pi') { - return normalizeMigratedHeader( - { - ...header, - ...statusFields, - titleIsManual, - backend: 'pi-agent', - model, - permissionMode, - collaborationMode, - orchestrationMode, - }, - sessionId, - ); - } - return normalizeMigratedHeader( - { - ...header, - ...statusFields, - titleIsManual, - backend: header.backend === 'ai-sdk' ? 'ai-sdk' : 'fake', - model, - permissionMode, - collaborationMode, - orchestrationMode, - }, - sessionId, - ); + assertValidSessionLineage(header); + return header; } -function resolveMigratedStatus(header: StoredSessionHeader): SessionHeader['status'] { - if (header.isArchived) return 'archived'; - if (isSessionStatus(header.status) && header.status !== 'archived') return header.status; - return 'active'; -} - -function normalizeMigratedHeader( - header: SessionHeader & { pendingCwdReminder?: unknown }, - sessionId: string, -): SessionHeader { - const { pendingCwdReminder: _legacyPendingCwdReminder, ...normalizedHeader } = header; - return normalizeSessionHeader(normalizedHeader, sessionId); +function normalizeRequiredSessionName(name: string): string { + const normalized = normalizeUserSessionName(name); + if (!normalized.ok) throw new Error(normalized.error); + return normalized.value; } /** Validate and normalize a current SessionHeader before canonical persistence. */ @@ -1952,16 +959,6 @@ function isValidConversationCopyLineage(header: SessionHeader): boolean { ); } -function assertConversationCopyTransition( - current: SessionHeader, - patch: Partial, -): void { - if (!Object.prototype.hasOwnProperty.call(patch, 'conversationCopy')) return; - if (!isValidConversationCopyTransition(current, patch.conversationCopy)) { - throw new Error('Session conversation-copy identity is immutable'); - } -} - function isValidSubagentSessionLineage(header: SessionHeader): boolean { if (header.subagentParent === undefined) { return ( @@ -2002,10 +999,6 @@ function isFiniteNumber(value: unknown): value is number { return typeof value === 'number' && Number.isFinite(value); } -function hasErrorCode(error: unknown, code: string): boolean { - return (error as NodeJS.ErrnoException | undefined)?.code === code; -} - function assertNoConversationCopyMetadata(input: CreateSessionInput): void { if (Object.prototype.hasOwnProperty.call(input, 'conversationCopy')) { throw new Error('Conversation copy metadata requires createStableSession()'); @@ -2095,7 +1088,7 @@ function toCatalogSummary( }; } -function catalogMessageProjection(messages: StoredMessage[]): { +export function projectSessionCatalogMessages(messages: readonly StoredMessage[]): { readonly lastMessageAt?: number; readonly lastMessagePreview?: string; } { @@ -2107,7 +1100,7 @@ function catalogMessageProjection(messages: StoredMessage[]): { }; } -function latestVisibleMessageAt(messages: StoredMessage[]): number | undefined { +function latestVisibleMessageAt(messages: readonly StoredMessage[]): number | undefined { for (let index = messages.length - 1; index >= 0; index -= 1) { const message = messages[index]!; if (isVisibleSessionMessage(message)) return message.ts; @@ -2131,7 +1124,7 @@ function normalizeSessionName(name: string): string { return name === 'New Session' ? DEFAULT_SESSION_NAME : name; } -function lastMessagePreviewForMessages(messages: StoredMessage[]): string | undefined { +function lastMessagePreviewForMessages(messages: readonly StoredMessage[]): string | undefined { for (let index = messages.length - 1; index >= 0; index -= 1) { const message = messages[index]!; if (message.type === 'user') { @@ -2159,44 +1152,6 @@ function truncatePreview(text: string, maxLength = 96): string { return `${chars.slice(0, maxLength - 1).join('')}…`; } -async function writeNewTranscript( - path: string, - payload: string, - durabilityRoot?: string, -): Promise { - try { - const handle = await open(path, 'wx', 0o600); - try { - await handle.writeFile(payload, 'utf8'); - if (durabilityRoot) await handle.sync(); - } finally { - await handle.close(); - } - if (durabilityRoot) { - await syncDirectoryChain(dirname(path), durabilityRoot); - } - } catch (error) { - if (!durabilityRoot || error instanceof DurableStoreWriteError) throw error; - throw new DurableStoreWriteError( - `Durable Session transcript did not reach stable storage: ${path}`, - error, - ); - } -} - -async function stabilizeTranscript(path: string, durabilityRoot: string): Promise { - try { - await syncFile(path); - await syncDirectoryChain(dirname(path), durabilityRoot); - } catch (error) { - if (error instanceof DurableStoreWriteError) throw error; - throw new DurableStoreWriteError( - `Session transcript durability could not be re-established: ${path}`, - error, - ); - } -} - export function createUserMessage(input: { turnId: string; text: string; diff --git a/packages/storage/src/session-transcript.ts b/packages/storage/src/session-transcript.ts deleted file mode 100644 index 32b8141989..0000000000 --- a/packages/storage/src/session-transcript.ts +++ /dev/null @@ -1,38 +0,0 @@ -export const SESSION_TRANSCRIPT_RECORD_TYPE = 'session_transcript'; -export const SESSION_TRANSCRIPT_SCHEMA_VERSION = 1; - -export interface SessionTranscriptMarker { - type: typeof SESSION_TRANSCRIPT_RECORD_TYPE; - sessionId: string; - schemaVersion: typeof SESSION_TRANSCRIPT_SCHEMA_VERSION; -} - -export function createSessionTranscriptMarker(sessionId: string): SessionTranscriptMarker { - return { - type: SESSION_TRANSCRIPT_RECORD_TYPE, - sessionId, - schemaVersion: SESSION_TRANSCRIPT_SCHEMA_VERSION, - }; -} - -export function isSessionTranscriptMarker(value: unknown): value is SessionTranscriptMarker { - if (!value || typeof value !== 'object' || Array.isArray(value)) return false; - const record = value as Record; - return record.type === SESSION_TRANSCRIPT_RECORD_TYPE; -} - -export function decodeSessionTranscriptMarker( - value: unknown, - expectedSessionId: string, -): SessionTranscriptMarker { - if (!isSessionTranscriptMarker(value)) { - throw new Error(`Session ${expectedSessionId}: missing transcript marker`); - } - if ( - value.sessionId !== expectedSessionId || - value.schemaVersion !== SESSION_TRANSCRIPT_SCHEMA_VERSION - ) { - throw new Error(`Session ${expectedSessionId}: invalid transcript marker`); - } - return value; -} diff --git a/packages/storage/src/shell-run-store.ts b/packages/storage/src/shell-run-store.ts index 448f63a99c..0ade1c5fc1 100644 --- a/packages/storage/src/shell-run-store.ts +++ b/packages/storage/src/shell-run-store.ts @@ -1,9 +1,5 @@ -import { createHash, randomUUID } from 'node:crypto'; -import { access, mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; import { isDeepStrictEqual } from 'node:util'; import { - isValidLegacyShellRunState, isShellOutput, isShellRunId, isShellRunSourceToolCallId, @@ -15,13 +11,9 @@ import { type ShellRunPatch, type ShellRunStore, } from '@maka/core'; -import { syncDirectoryChain, syncFile } from './stable-storage.js'; -import { chainWrite } from './write-queue.js'; import { acquireOperationalStateDatabase, - completeOperationalStoreCutover, type OperationalStateDatabaseLease, - type OperationalStoreCutoverFailpoint, } from './operational-state-store.js'; const SESSION_ID_PATTERN = /^[A-Za-z0-9_-]{1,128}$/; @@ -55,71 +47,30 @@ const SHELL_RUN_RECORD_KEYS = new Set([ 'observedAt', 'output', ]); -const LEGACY_SHELL_RUN_RECORD_KEYS = new Set([ - 'shellRunId', - 'sessionId', - 'sourceRunId', - 'sourceTurnId', - 'sourceToolCallId', - 'cwd', - 'command', - 'status', - 'startedAt', - 'updatedAt', - 'completedAt', - 'timeoutMs', - 'exitCode', - 'failureMessage', - 'stdoutTail', - 'stderrTail', - 'latestOutputStream', - 'stdoutTruncated', - 'stderrTruncated', - 'observedAt', - 'orphanedReason', - 'pid', -]); - -export function createShellRunStore(workspaceRoot: string): ShellRunStore { - return new FileShellRunStore(workspaceRoot); -} - -export interface SqliteShellRunStoreOptions { - readonly failpoint?: (point: OperationalStoreCutoverFailpoint) => void; -} - export interface ClosableShellRunStore extends ShellRunStore { ready(): Promise; close(): void; } -export function createSqliteShellRunStore( - workspaceRoot: string, - options: SqliteShellRunStoreOptions = {}, -): ClosableShellRunStore { - return new SqliteShellRunStore(workspaceRoot, options); +export function createSqliteShellRunStore(workspaceRoot: string): ClosableShellRunStore { + return new SqliteShellRunStore(workspaceRoot); } class SqliteShellRunStore implements ClosableShellRunStore { - readonly #root: string; readonly #lease: OperationalStateDatabaseLease; - readonly #ready: Promise; - constructor(workspaceRoot: string, options: SqliteShellRunStoreOptions) { - this.#root = workspaceRoot; + constructor(workspaceRoot: string) { this.#lease = acquireOperationalStateDatabase(workspaceRoot); - this.#ready = importLegacyShellRuns(workspaceRoot, this.#lease, options); } ready(): Promise { - return this.#ready; + return Promise.resolve(); } async createShellRun(record: ShellRunRecord): Promise { assertSessionId(record.sessionId); assertShellRunId(record.shellRunId); const normalized = normalizeShellRunRecord(record, record.sessionId, record.shellRunId); - await this.#ready; this.#lease.transaction('write', () => { const result = this.#lease.database .prepare(` @@ -148,7 +99,6 @@ class SqliteShellRunStore implements ClosableShellRunStore { assertSessionId(sessionId); assertShellRunId(shellRunId); assertShellRunPatch(patch); - await this.#ready; return this.#lease.transaction('write', () => { const current = readSqliteShellRun(this.#lease.database, sessionId, shellRunId); if (patch.output && patch.output.mode !== current.output.mode) { @@ -185,13 +135,11 @@ class SqliteShellRunStore implements ClosableShellRunStore { async readShellRun(sessionId: string, shellRunId: string): Promise { assertSessionId(sessionId); assertShellRunId(shellRunId); - await this.#ready; return readSqliteShellRun(this.#lease.database, sessionId, shellRunId); } async listSessionShellRuns(sessionId: string): Promise { assertSessionId(sessionId); - await this.#ready; const rows = this.#lease.database .prepare(` SELECT shell_run_id, record_json @@ -213,218 +161,6 @@ class SqliteShellRunStore implements ClosableShellRunStore { } } -class FileShellRunStore implements ShellRunStore { - private readonly durabilityRoot: string; - private readonly sessionsRoot: string; - private readonly writeQueues = new Map>(); - - constructor(workspaceRoot: string) { - this.durabilityRoot = workspaceRoot; - this.sessionsRoot = join(workspaceRoot, 'sessions'); - } - - async createShellRun(record: ShellRunRecord): Promise { - assertSessionId(record.sessionId); - assertShellRunId(record.shellRunId); - const normalized = normalizeShellRunRecord(record, record.sessionId, record.shellRunId); - await this.withQueue(record.sessionId, record.shellRunId, async () => { - if (await pathExists(this.shellRunPath(record.sessionId, record.shellRunId))) { - throw new Error(`ShellRun already exists: ${record.shellRunId}`); - } - await mkdir(this.shellRunDir(record.sessionId, record.shellRunId), { recursive: true }); - await writeAtomic( - this.shellRunPath(record.sessionId, record.shellRunId), - JSON.stringify(normalized, sanitizeJson) + '\n', - this.durabilityRoot, - true, - ); - }); - return normalized; - } - - async updateShellRun( - sessionId: string, - shellRunId: string, - patch: ShellRunPatch, - ): Promise { - let next: ShellRunRecord | undefined; - await this.withQueue(sessionId, shellRunId, async () => { - assertShellRunPatch(patch); - const hasDurableIntent = Object.hasOwn(patch, 'status') || Object.hasOwn(patch, 'observedAt'); - const current = await this.readShellRunUnlocked(sessionId, shellRunId); - if (patch.output && patch.output.mode !== current.output.mode) { - throw new Error(`ShellRun output mode is immutable: ${current.output.mode}`); - } - const effectivePatch = - current.observedAt !== undefined && Object.hasOwn(patch, 'observedAt') - ? { ...patch, observedAt: current.observedAt } - : patch; - const candidate = normalizeShellRunRecord( - { ...current, ...effectivePatch, sessionId, shellRunId, revision: current.revision }, - sessionId, - shellRunId, - ); - assertShellRunTransition(current, candidate); - if (isDeepStrictEqual(candidate, current)) { - if (hasDurableIntent) { - const path = this.shellRunPath(sessionId, shellRunId); - await syncFile(path); - await syncDirectoryChain(dirname(path), this.durabilityRoot); - } - next = current; - return; - } - next = normalizeShellRunRecord( - { ...candidate, revision: current.revision + 1 }, - sessionId, - shellRunId, - ); - await writeAtomic( - this.shellRunPath(sessionId, shellRunId), - JSON.stringify(next, sanitizeJson) + '\n', - this.durabilityRoot, - hasDurableIntent, - ); - }); - if (!next) throw new Error(`Failed to update shell run ${shellRunId}`); - return next; - } - - async readShellRun(sessionId: string, shellRunId: string): Promise { - return this.readShellRunUnlocked(sessionId, shellRunId); - } - - async listSessionShellRuns(sessionId: string): Promise { - assertSessionId(sessionId); - let entries; - try { - entries = await readdir(this.shellRunsRoot(sessionId), { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const records: ShellRunRecord[] = []; - for (const entry of entries) { - if (!entry.isDirectory() || !isShellRunId(entry.name)) continue; - try { - records.push(await this.readShellRunUnlocked(sessionId, entry.name)); - } catch { - // Malformed shell run folders should not hide healthy runs. - } - } - return records.sort( - (a, b) => a.startedAt - b.startedAt || a.shellRunId.localeCompare(b.shellRunId), - ); - } - - private async readShellRunUnlocked( - sessionId: string, - shellRunId: string, - ): Promise { - assertSessionId(sessionId); - assertShellRunId(shellRunId); - return normalizeShellRunRecord( - JSON.parse(await readFile(this.shellRunPath(sessionId, shellRunId), 'utf8')), - sessionId, - shellRunId, - ); - } - - private shellRunsRoot(sessionId: string): string { - assertSessionId(sessionId); - return join(this.sessionsRoot, sessionId, 'shell-runs'); - } - - private shellRunDir(sessionId: string, shellRunId: string): string { - assertShellRunId(shellRunId); - return join(this.shellRunsRoot(sessionId), shellRunId); - } - - private shellRunPath(sessionId: string, shellRunId: string): string { - return join(this.shellRunDir(sessionId, shellRunId), 'shell-run.json'); - } - - private withQueue( - sessionId: string, - shellRunId: string, - operation: () => Promise, - ): Promise { - assertSessionId(sessionId); - assertShellRunId(shellRunId); - const key = `${sessionId}:${shellRunId}`; - return chainWrite(this.writeQueues, key, operation); - } -} - -async function importLegacyShellRuns( - root: string, - lease: OperationalStateDatabaseLease, - options: SqliteShellRunStoreOptions, -): Promise { - const records = await readLegacyShellRuns(root); - const fingerprint = createHash('sha256').update(JSON.stringify(records)).digest('hex'); - completeOperationalStoreCutover(lease, { - storeName: 'shell_runs', - sourcePath: join(root, 'sessions'), - sourceFingerprint: `sha256:${fingerprint}`, - failpoint: options.failpoint, - importAndValidate: (db) => { - for (const record of records) insertOrValidateShellRun(db, record); - return { shell_runs: records.length }; - }, - }); -} - -async function readLegacyShellRuns(root: string): Promise { - const sessionsRoot = join(root, 'sessions'); - let sessions; - try { - sessions = await readdir(sessionsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const legacy = new FileShellRunStore(root); - const records: ShellRunRecord[] = []; - for (const session of sessions.sort((a, b) => a.name.localeCompare(b.name))) { - if (!session.isDirectory() || !SESSION_ID_PATTERN.test(session.name)) continue; - const shellRunsRoot = join(sessionsRoot, session.name, 'shell-runs'); - let entries; - try { - entries = await readdir(shellRunsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue; - throw error; - } - for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { - if (!entry.isDirectory() || !isShellRunId(entry.name)) { - throw new Error(`Invalid ShellRun entry for session ${session.name}: ${entry.name}`); - } - records.push(await legacy.readShellRun(session.name, entry.name)); - } - } - return records; -} - -function insertOrValidateShellRun( - db: import('node:sqlite').DatabaseSync, - record: ShellRunRecord, -): void { - const encoded = JSON.stringify(record, sanitizeJson); - const result = db - .prepare(` - INSERT OR IGNORE INTO core_shell_runs( - session_id, shell_run_id, started_at, record_json - ) VALUES (?, ?, ?, ?) - `) - .run(record.sessionId, record.shellRunId, record.startedAt, encoded); - if (result.changes !== 0) return; - const existing = readSqliteShellRun(db, record.sessionId, record.shellRunId); - if (!isDeepStrictEqual(existing, record)) { - throw new Error(`ShellRun cutover conflict: ${record.shellRunId}`); - } -} - function readSqliteShellRun( db: import('node:sqlite').DatabaseSync, sessionId: string, @@ -442,35 +178,6 @@ function readSqliteShellRun( return normalizeShellRunRecord(JSON.parse(row.record_json), sessionId, shellRunId); } -async function writeAtomic( - path: string, - content: string, - durabilityRoot: string, - durable: boolean, -): Promise { - await mkdir(dirname(path), { recursive: true }); - const tempPath = `${path}.${process.pid}.${Date.now()}.${randomUUID()}.tmp`; - try { - await writeFile(tempPath, content, 'utf8'); - if (durable) await syncFile(tempPath); - await rename(tempPath, path); - if (durable) await syncDirectoryChain(dirname(path), durabilityRoot); - } catch (error) { - await rm(tempPath, { force: true }).catch(() => undefined); - throw error; - } -} - -async function pathExists(path: string): Promise { - try { - await access(path); - return true; - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return false; - throw error; - } -} - function normalizeShellRunRecord( value: unknown, sessionId: string, @@ -479,8 +186,7 @@ function normalizeShellRunRecord( if (!value || typeof value !== 'object' || Array.isArray(value)) { throw new Error(`Invalid ShellRun record for ${shellRunId}: expected an object`); } - const record = (normalizeLegacyShellRunRecord(value, sessionId, shellRunId) ?? - value) as Partial; + const record = value as Partial; const requiredStrings = [ record.shellRunId, record.sessionId, @@ -517,81 +223,6 @@ function normalizeShellRunRecord( return canonicalShellRunRecord(record as ShellRunRecord); } -function normalizeLegacyShellRunRecord( - value: object, - sessionId: string, - shellRunId: string, -): ShellRunRecord | undefined { - if (!hasOnlyKeys(value, LEGACY_SHELL_RUN_RECORD_KEYS)) return undefined; - const record = value as Record; - if ( - record.output !== undefined || - record.revision !== undefined || - record.shellRunId !== shellRunId || - record.sessionId !== sessionId || - typeof record.sourceTurnId !== 'string' || - !isShellRunSourceToolCallId(record.sourceToolCallId) || - typeof record.cwd !== 'string' || - typeof record.command !== 'string' || - !isShellRunStatus(record.status) || - !isFiniteNumber(record.startedAt) || - !isFiniteNumber(record.updatedAt) || - typeof record.stdoutTail !== 'string' || - typeof record.stderrTail !== 'string' || - typeof record.stdoutTruncated !== 'boolean' || - typeof record.stderrTruncated !== 'boolean' || - (record.sourceRunId !== undefined && typeof record.sourceRunId !== 'string') || - (record.completedAt !== undefined && !isFiniteNumber(record.completedAt)) || - (record.timeoutMs !== undefined && !isFiniteNumber(record.timeoutMs)) || - (record.exitCode !== undefined && !isFiniteNumber(record.exitCode)) || - (record.failureMessage !== undefined && typeof record.failureMessage !== 'string') || - (record.observedAt !== undefined && !isFiniteNumber(record.observedAt)) || - (record.orphanedReason !== undefined && typeof record.orphanedReason !== 'string') || - (record.pid !== undefined && !isFiniteNumber(record.pid)) || - (record.latestOutputStream !== undefined && - record.latestOutputStream !== 'stdout' && - record.latestOutputStream !== 'stderr') || - !isValidLegacyShellRunState(record) - ) - return undefined; - - const failureMessage = - typeof record.failureMessage === 'string' - ? record.failureMessage - : record.status === 'orphaned' - ? (record.orphanedReason as string) - : undefined; - return { - shellRunId, - sessionId, - ...(typeof record.sourceRunId === 'string' ? { sourceRunId: record.sourceRunId } : {}), - sourceTurnId: record.sourceTurnId, - sourceToolCallId: record.sourceToolCallId, - cwd: record.cwd, - command: record.command, - status: record.status, - startedAt: record.startedAt, - updatedAt: record.updatedAt, - ...(isFiniteNumber(record.completedAt) ? { completedAt: record.completedAt } : {}), - ...(isFiniteNumber(record.timeoutMs) ? { timeoutMs: record.timeoutMs } : {}), - ...(isFiniteNumber(record.exitCode) ? { exitCode: record.exitCode } : {}), - ...(failureMessage !== undefined ? { failureMessage } : {}), - revision: 1, - ...(isFiniteNumber(record.observedAt) ? { observedAt: record.observedAt } : {}), - output: { - mode: 'pipes', - stdout: record.stdoutTail, - stderr: record.stderrTail, - ...(record.latestOutputStream === 'stdout' || record.latestOutputStream === 'stderr' - ? { latestStream: record.latestOutputStream } - : {}), - stdoutTruncated: record.stdoutTruncated, - stderrTruncated: record.stderrTruncated, - redacted: false, - }, - }; -} - function assertSessionId(value: string): void { if (!SESSION_ID_PATTERN.test(value)) throw new Error('Invalid session id'); } diff --git a/packages/storage/src/sqlite-artifact-metadata.ts b/packages/storage/src/sqlite-artifact-metadata.ts index 432063bd25..b7a514e78a 100644 --- a/packages/storage/src/sqlite-artifact-metadata.ts +++ b/packages/storage/src/sqlite-artifact-metadata.ts @@ -1,53 +1,30 @@ import { createHash } from 'node:crypto'; -import { readFile } from 'node:fs/promises'; -import { join, resolve } from 'node:path'; +import { resolve } from 'node:path'; import type { ArtifactRecord } from '@maka/core/artifacts'; import type { ArtifactMetadataRepository } from './artifact-metadata-repository.js'; -import { decodeArtifactMetadata } from './artifact-metadata-codec.js'; +import { decodeArtifactRecordJsons } from './artifact-metadata-codec.js'; import { acquireOperationalStateDatabase, - completeOperationalStoreCutover, type OperationalStateDatabaseLease, - type OperationalStoreCutoverFailpoint, } from './operational-state-store.js'; -export interface CreateSqliteArtifactMetadataOptions { - readonly failpoint?: (point: OperationalStoreCutoverFailpoint) => void; -} - -export function artifactMetadataSourceFingerprint(source: string | undefined): string { - return `sha256:${createHash('sha256') - .update(source === undefined ? 'missing' : source) - .digest('hex')}`; -} - export function createSqliteArtifactMetadataRepository( workspaceRoot: string, - options: CreateSqliteArtifactMetadataOptions = {}, ): ArtifactMetadataRepository { - return new SqliteArtifactMetadataRepository(workspaceRoot, options.failpoint); + return new SqliteArtifactMetadataRepository(workspaceRoot); } class SqliteArtifactMetadataRepository implements ArtifactMetadataRepository { - readonly #root: string; readonly #lease: OperationalStateDatabaseLease; - readonly #failpoint?: (point: OperationalStoreCutoverFailpoint) => void; - #ready: Promise | undefined; #closed = false; - constructor( - workspaceRoot: string, - failpoint?: (point: OperationalStoreCutoverFailpoint) => void, - ) { - this.#root = resolve(workspaceRoot); - this.#lease = acquireOperationalStateDatabase(this.#root); - this.#failpoint = failpoint; + constructor(workspaceRoot: string) { + this.#lease = acquireOperationalStateDatabase(resolve(workspaceRoot)); } ready(): Promise { this.assertOpen(); - this.#ready ??= importLegacyArtifactMetadata(this.#root, this.#lease, this.#failpoint); - return this.#ready; + return Promise.resolve(); } readAll(): ArtifactRecord[] { @@ -102,84 +79,10 @@ class SqliteArtifactMetadataRepository implements ArtifactMetadataRepository { } } -async function importLegacyArtifactMetadata( - root: string, - lease: OperationalStateDatabaseLease, - failpoint?: (point: OperationalStoreCutoverFailpoint) => void, -): Promise { - const sourcePath = join(root, 'artifacts', 'metadata.jsonl'); - const source = await readOptionalText(sourcePath); - const records = source === undefined ? [] : decodeArtifactMetadata(source); - const fingerprint = artifactMetadataSourceFingerprint(source); - completeOperationalStoreCutover(lease, { - storeName: 'artifact_metadata', - sourcePath, - sourceFingerprint: fingerprint, - failpoint, - importAndValidate: (database) => { - const insert = database.prepare(` - INSERT INTO artifact_records( - storage_key, - artifact_id, - session_id, - created_at, - status, - relative_path, - record_json - ) VALUES (?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(storage_key) DO UPDATE SET - artifact_id = excluded.artifact_id, - session_id = excluded.session_id, - created_at = excluded.created_at, - status = excluded.status, - relative_path = excluded.relative_path, - record_json = excluded.record_json - `); - for (const record of records) { - insert.run( - artifactIdentityKey(record.id), - record.id, - record.sessionId, - record.createdAt, - record.status, - record.relativePath, - JSON.stringify(record), - ); - } - const persisted = database - .prepare('SELECT record_json FROM artifact_records ORDER BY created_at, storage_key') - .all() as Array<{ record_json: string }>; - const decoded = decodeRows(persisted); - if (decoded.length !== records.length || !sameRecordSet(decoded, records)) { - throw new Error('Artifact metadata cutover validation failed'); - } - return { records: records.length }; - }, - }); -} - function decodeRows(rows: readonly { record_json: string }[]): ArtifactRecord[] { - const text = rows.map((row) => row.record_json).join('\n'); - return decodeArtifactMetadata(text ? `${text}\n` : ''); -} - -function sameRecordSet(left: readonly ArtifactRecord[], right: readonly ArtifactRecord[]): boolean { - const canonical = (records: readonly ArtifactRecord[]) => - [...records].map((record) => JSON.stringify(record)).sort((a, b) => a.localeCompare(b)); - const leftRecords = canonical(left); - const rightRecords = canonical(right); - return leftRecords.every((record, index) => record === rightRecords[index]); + return decodeArtifactRecordJsons(rows.map((row) => row.record_json)); } function artifactIdentityKey(id: string): string { return createHash('sha256').update(JSON.stringify(id)).digest('hex'); } - -async function readOptionalText(path: string): Promise { - try { - return await readFile(path, 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; - throw error; - } -} diff --git a/packages/storage/src/sqlite-runtime-schema.ts b/packages/storage/src/sqlite-runtime-schema.ts index cc6ae4d37c..e936904c30 100644 --- a/packages/storage/src/sqlite-runtime-schema.ts +++ b/packages/storage/src/sqlite-runtime-schema.ts @@ -1,6 +1,6 @@ import type { DatabaseSync } from 'node:sqlite'; -export const SQLITE_RUNTIME_SCHEMA_VERSION = 7; +export const SQLITE_RUNTIME_SCHEMA_VERSION = 8; export const RUNTIME_RECOVERY_AUTHORITY_CAPABILITY = 'runtime_recovery_authority'; export const RUNTIME_RECOVERY_AUTHORITY_CAPABILITY_VERSION = 1; export const RUNTIME_CONTINUATION_AUTHORITY_CAPABILITY = 'runtime_continuation_authority'; @@ -92,11 +92,7 @@ const MIGRATIONS: ReadonlyMap = new Map([ [ 3, ` - CREATE TABLE runtime_import_sources ( - source_path TEXT PRIMARY KEY, - fingerprint TEXT NOT NULL, - imported_at INTEGER NOT NULL - ); + SELECT 1; `, ], [ @@ -241,6 +237,18 @@ const MIGRATIONS: ReadonlyMap = new Map([ VALUES ('runtime_workspace_version_authority', 1); `, ], + [ + 8, + ` + CREATE TABLE headless_task_run_events ( + task_run_id TEXT NOT NULL, + sequence INTEGER NOT NULL CHECK (sequence >= 0), + event_id TEXT NOT NULL, + record_json TEXT NOT NULL, + PRIMARY KEY (task_run_id, sequence) + ); + `, + ], ]); export function configureSqliteRuntimeDatabase(db: DatabaseSync): void { diff --git a/packages/storage/src/sqlite-runtime-store.ts b/packages/storage/src/sqlite-runtime-store.ts index c3c2e86458..702cbaff8a 100644 --- a/packages/storage/src/sqlite-runtime-store.ts +++ b/packages/storage/src/sqlite-runtime-store.ts @@ -147,7 +147,6 @@ export interface ToolCommitResult { export interface RuntimeEventBatchImportResult { created: boolean[]; - sourceAlreadyImported: boolean; } export interface ToolProjectionRebuildResult { @@ -346,7 +345,6 @@ export class SqliteRuntimeStore sessionId: string; runId: string; events: readonly RuntimeEvent[]; - source?: { path: string; fingerprint: string }; }): Promise { const events = input.events.map(canonicalizeRuntimeEventForStorage); for (const event of events) { @@ -356,32 +354,11 @@ export class SqliteRuntimeStore } } return this.transaction(() => { - if (input.source) { - const existing = this.db - .prepare(` - SELECT fingerprint FROM runtime_import_sources WHERE source_path = ? - `) - .get(input.source.path) as { fingerprint: string } | undefined; - if (existing?.fingerprint === input.source.fingerprint) { - return { created: [], sourceAlreadyImported: true }; - } - } if (events.some(isToolLedgerBearingEvent)) { this.assertToolLedgerTransition(events, 'generic_append'); } const created = events.map((event) => this.importRuntimeEventSync(event)); - if (input.source) { - this.db - .prepare(` - INSERT INTO runtime_import_sources (source_path, fingerprint, imported_at) - VALUES (?, ?, ?) - ON CONFLICT(source_path) DO UPDATE SET - fingerprint = excluded.fingerprint, - imported_at = excluded.imported_at - `) - .run(input.source.path, input.source.fingerprint, Date.now()); - } - return { created, sourceAlreadyImported: false }; + return { created }; }); } @@ -445,15 +422,6 @@ export class SqliteRuntimeStore }); } - async isRuntimeImportSourceCurrent(path: string, fingerprint: string): Promise { - const existing = this.db - .prepare(` - SELECT fingerprint FROM runtime_import_sources WHERE source_path = ? - `) - .get(path) as { fingerprint: string } | undefined; - return existing?.fingerprint === fingerprint; - } - async readRuntimeEvents(sessionId: string, runId: string): Promise { return this.readRuntimeEventsSync(sessionId, runId); } diff --git a/packages/storage/src/sqlite-session-metadata-schema.ts b/packages/storage/src/sqlite-session-metadata-schema.ts index 4c04ed7c04..3e3f119211 100644 --- a/packages/storage/src/sqlite-session-metadata-schema.ts +++ b/packages/storage/src/sqlite-session-metadata-schema.ts @@ -1,6 +1,6 @@ import type { DatabaseSync } from 'node:sqlite'; -export const SQLITE_SESSION_METADATA_SCHEMA_VERSION = 19; +export const SQLITE_SESSION_METADATA_SCHEMA_VERSION = 20; export const SQLITE_AGENT_GRAPH_CONTROL_TABLES = [ 'agent_graph_intent_claims', @@ -66,13 +66,6 @@ const MIGRATIONS: ReadonlyMap = new Map([ CREATE INDEX session_metadata_labels_by_label ON session_metadata_labels(label, session_id); - CREATE TABLE session_metadata_import_sources ( - source_path TEXT PRIMARY KEY, - fingerprint TEXT NOT NULL, - session_id TEXT NOT NULL, - imported_at INTEGER NOT NULL, - FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE - ); `, ], [ @@ -784,6 +777,27 @@ const MIGRATIONS: ReadonlyMap = new Map([ ON agent_graph_supervisor_wakes(status, updated_at, graph_id, wake_id); `, ], + [ + 20, + ` + CREATE TABLE session_messages ( + session_id TEXT NOT NULL, + sequence INTEGER NOT NULL CHECK (sequence >= 0), + message_id TEXT NOT NULL, + message_type TEXT NOT NULL, + message_ts INTEGER NOT NULL CHECK (message_ts >= 0), + record_json TEXT NOT NULL, + PRIMARY KEY(session_id, sequence), + FOREIGN KEY(session_id) REFERENCES session_metadata(session_id) ON DELETE CASCADE + ); + + CREATE INDEX session_messages_by_identity + ON session_messages(session_id, message_id); + + CREATE INDEX session_messages_by_time + ON session_messages(session_id, message_ts, sequence); + `, + ], ]); export function configureSqliteSessionMetadataDatabase(db: DatabaseSync): void { diff --git a/packages/storage/src/sqlite-session-metadata-store.ts b/packages/storage/src/sqlite-session-metadata-store.ts index a45bc9c3c3..999702e18f 100644 --- a/packages/storage/src/sqlite-session-metadata-store.ts +++ b/packages/storage/src/sqlite-session-metadata-store.ts @@ -58,8 +58,11 @@ import { type SettleSandboxBoundaryRequest, type SessionHeader, type SessionListFilter, + type StoredMessage, type SubagentSessionParent, type SupersedeAgentGraphSupervisorWakesRequest, + decodeStoredMessageForRead, + decodeStoredMessageForRecovery, } from '@maka/core'; import { assertSafeSessionId, @@ -109,7 +112,6 @@ function loadSqliteModule(): typeof import('node:sqlite') { export type SqliteSessionMetadataStoreFailpoint = | 'after_session_row_write' | 'after_session_labels_write' - | 'after_session_import_marker_write' | 'after_agent_graph_intent_claim_write' | 'after_agent_graph_schedule_update_write' | 'after_agent_graph_operator_provision_write' @@ -228,21 +230,6 @@ export interface IdempotentAgentGraphOperatorMetadataResult record: SessionMetadataRecord; } -export interface SessionMetadataImportEntry { - header: SessionHeader; - initialBoundary?: ExecutionBoundary; - source: { - path: string; - fingerprint: string; - }; -} - -export interface SessionMetadataImportResult { - created: boolean[]; - sourcesAlreadyImported: number; - sourcesTombstoned: number; -} - export class SessionMetadataConflictError extends Error { readonly name: string = 'SessionMetadataConflictError'; } @@ -1210,6 +1197,84 @@ export class SqliteSessionMetadataStore { return this.readCatalogRevisionSync(); } + async appendMessages( + sessionId: string, + messages: readonly StoredMessage[], + projection: SessionCatalogMessageProjection, + ): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + assertCatalogMessageProjection(projection); + if (messages.length === 0) return; + const encoded = messages.map((message) => { + const json = JSON.stringify(message); + const canonical = decodeStoredMessageForRecovery(JSON.parse(json) as unknown); + return { message: canonical, json }; + }); + this.transaction(() => { + if (!this.readRecordSync(sessionId)) throw new SessionNotFoundError(sessionId); + const row = this.db + .prepare( + 'SELECT COALESCE(MAX(sequence), -1) AS last_sequence FROM session_messages WHERE session_id = ?', + ) + .get(sessionId) as { last_sequence?: unknown }; + if ( + typeof row.last_sequence !== 'number' || + !Number.isSafeInteger(row.last_sequence) || + row.last_sequence < -1 + ) { + throw new Error(`Invalid Session message sequence for ${sessionId}`); + } + const insert = this.db.prepare(` + INSERT INTO session_messages( + session_id, sequence, message_id, message_type, message_ts, record_json + ) VALUES (?, ?, ?, ?, ?, ?) + `); + let sequence = row.last_sequence + 1; + for (const entry of encoded) { + insert.run( + sessionId, + sequence, + entry.message.id, + entry.message.type, + entry.message.ts, + entry.json, + ); + sequence += 1; + } + this.updateCatalogProjectionSync(sessionId, projection, false); + }); + } + + async readMessages(sessionId: string): Promise { + return this.readMessagesWith(sessionId, decodeStoredMessageForRead); + } + + async readMessagesForRecovery(sessionId: string): Promise { + return this.readMessagesWith(sessionId, decodeStoredMessageForRecovery); + } + + async readPreviewMessages(sessionId: string, limit = 10): Promise { + this.assertOpen(); + assertSafeSessionId(sessionId); + if (!Number.isSafeInteger(limit) || limit < 1 || limit > 128) { + throw new Error('Session message preview limit must be between 1 and 128'); + } + if (!this.readRecordSync(sessionId)) throw new SessionNotFoundError(sessionId); + const rows = this.db + .prepare(` + SELECT record_json + FROM session_messages + WHERE session_id = ? + ORDER BY sequence DESC + LIMIT ? + `) + .all(sessionId, limit) as Array<{ record_json?: unknown }>; + return rows + .reverse() + .map((row, index) => decodeStoredMessageRow(row.record_json, sessionId, index, false)); + } + async beginCatalogProjectionWrite(): Promise { this.assertOpen(); this.transaction(() => { @@ -2468,112 +2533,6 @@ export class SqliteSessionMetadataStore { }); } - async importEntries( - entries: readonly SessionMetadataImportEntry[], - ): Promise { - this.assertOpen(); - const sourcePaths = new Set(); - const normalized = entries.map((entry) => { - const header = normalizeSessionHeader(entry.header); - assertSafeSessionId(header.id); - if (!entry.source.path || !entry.source.fingerprint) { - throw new Error(`Invalid session metadata import source for ${header.id}`); - } - if (sourcePaths.has(entry.source.path)) { - throw new Error(`Duplicate session metadata import source: ${entry.source.path}`); - } - sourcePaths.add(entry.source.path); - return { - header, - ...(entry.initialBoundary - ? { - initialBoundary: { - ...decodeExecutionBoundary(entry.initialBoundary), - revision: 0, - }, - } - : {}), - source: entry.source, - }; - }); - return this.transaction(() => { - const created: boolean[] = []; - let sourcesAlreadyImported = 0; - let sourcesTombstoned = 0; - for (const entry of normalized) { - if (this.hasTombstone(entry.header.id)) { - sourcesTombstoned += 1; - continue; - } - const source = this.db - .prepare(` - SELECT fingerprint - FROM session_metadata_import_sources - WHERE source_path = ? - `) - .get(entry.source.path) as { fingerprint: string } | undefined; - if (source?.fingerprint === entry.source.fingerprint) { - const existing = this.readRecordSync(entry.header.id); - if (!existing) { - throw new SessionMetadataConflictError( - `Imported session metadata is missing: ${entry.header.id}`, - ); - } - sourcesAlreadyImported += 1; - continue; - } - const existing = this.readRecordSync(entry.header.id); - if (existing) { - if (!isDeepStrictEqual(existing.header, entry.header)) { - throw new SessionMetadataConflictError( - `Session metadata import conflict for ${entry.header.id}`, - ); - } - if ( - entry.initialBoundary && - !isDeepStrictEqual( - this.readCurrentExecutionBoundarySync(entry.header.id), - entry.initialBoundary, - ) - ) { - throw new SessionMetadataConflictError( - `Session execution boundary import conflict for ${entry.header.id}`, - ); - } - if (entry.header.subagentSpawn) { - this.assertMatchingSubagentSpawnClaim(entry.header); - } - created.push(false); - } else { - if (entry.header.subagentSpawn) { - const claim = this.tryClaimSubagentSpawn(entry.header, this.now()); - if (!claim.created && claim.childSessionId !== entry.header.id) { - throw new SessionMetadataConflictError( - `Child-session spawn identity already belongs to ${claim.childSessionId}`, - ); - } - this.assertMatchingSubagentSpawnClaim(entry.header); - } - this.insertHeader(entry.header, 1, this.now(), entry.initialBoundary); - created.push(true); - } - this.db - .prepare(` - INSERT INTO session_metadata_import_sources( - source_path, fingerprint, session_id, imported_at - ) VALUES (?, ?, ?, ?) - ON CONFLICT(source_path) DO UPDATE SET - fingerprint = excluded.fingerprint, - session_id = excluded.session_id, - imported_at = excluded.imported_at - `) - .run(entry.source.path, entry.source.fingerprint, entry.header.id, this.now()); - this.options.failpoint?.('after_session_import_marker_write'); - } - return { created, sourcesAlreadyImported, sourcesTombstoned }; - }); - } - private insertHeader( header: SessionHeader, metadataVersion: number, @@ -2961,7 +2920,7 @@ export class SqliteSessionMetadataStore { ? 'ask' : record.header.permissionMode); if ((projectedMode === 'bypass') !== (kind === 'bypass')) { - throw new Error('Execution boundary kind and legacy permission mode disagree'); + throw new Error('Execution boundary kind and projected permission mode disagree'); } let boundary: ExecutionBoundary = current; @@ -3050,6 +3009,33 @@ export class SqliteSessionMetadataStore { return row ? decodeRecord(row) : undefined; } + private readMessagesWith( + sessionId: string, + decode: (value: unknown) => StoredMessage, + ): StoredMessage[] { + this.assertOpen(); + assertSafeSessionId(sessionId); + if (!this.readRecordSync(sessionId)) throw new SessionNotFoundError(sessionId); + const rows = this.db + .prepare(` + SELECT record_json + FROM session_messages + WHERE session_id = ? + ORDER BY sequence + `) + .all(sessionId) as Array<{ record_json?: unknown }>; + return rows.map((row, index) => { + if (typeof row.record_json !== 'string') { + throw new Error(`Invalid Session message row ${index} for ${sessionId}`); + } + try { + return decode(JSON.parse(row.record_json) as unknown); + } catch (error) { + throw new Error(`Invalid Session message row ${index} for ${sessionId}`, { cause: error }); + } + }); + } + private readCatalogPreviewSync(sessionId: string): string | undefined { const row = this.db .prepare(` @@ -4308,3 +4294,20 @@ function assertGraphIntentId(value: string): void { throw new Error('Invalid agent graph intent id'); } } + +function decodeStoredMessageRow( + value: unknown, + sessionId: string, + index: number, + recovery: boolean, +): StoredMessage { + if (typeof value !== 'string') { + throw new Error(`Invalid Session message row ${index} for ${sessionId}`); + } + try { + const parsed = JSON.parse(value) as unknown; + return recovery ? decodeStoredMessageForRecovery(parsed) : decodeStoredMessageForRead(parsed); + } catch (error) { + throw new Error(`Invalid Session message row ${index} for ${sessionId}`, { cause: error }); + } +} diff --git a/packages/storage/src/sqlite-usage-store.ts b/packages/storage/src/sqlite-usage-store.ts index 3c1e0e3dab..ad80c3bb46 100644 --- a/packages/storage/src/sqlite-usage-store.ts +++ b/packages/storage/src/sqlite-usage-store.ts @@ -1,6 +1,5 @@ import { createHash } from 'node:crypto'; -import { readFile } from 'node:fs/promises'; -import { join, resolve } from 'node:path'; +import { resolve } from 'node:path'; import type { PricingConfig, UsageBucket, @@ -17,7 +16,6 @@ import { } from '@maka/core/usage-stats/pricing'; import { usageBucketKey } from '@maka/core/usage-stats/bucket-key'; import { - createPricingStore, PricingRevisionConflictError, PricingStoreClosedError, PricingStoreNotLoadedError, @@ -30,18 +28,13 @@ import { } from './pricing-store.js'; import { acquireOperationalStateDatabase, - completeOperationalStoreCutover, type OperationalStateDatabaseLease, - type OperationalStoreCutoverFailpoint, } from './operational-state-store.js'; import { decodePersistedLlmCallRecord, decodePersistedToolInvocationRecord, - decodeTelemetryFile, - emptyTelemetryFile, type PersistedLlmCallRecord, type PersistedToolInvocationRecord, - type TelemetryFile, } from './telemetry-file-schema.js'; import { resolveRange, @@ -54,9 +47,7 @@ import { type ToolUsageQuery, } from './telemetry-repo.js'; -export interface CreateSqliteUsageStoreOptions { - readonly failpoint?: (point: OperationalStoreCutoverFailpoint) => void; -} +export interface CreateSqliteUsageStoreOptions {} export function createSqliteTelemetryRepo( workspaceRoot: string, @@ -66,7 +57,6 @@ export function createSqliteTelemetryRepo( workspaceRoot, options.createIfMissing ?? true, options.managePricing ?? true, - options.failpoint, ); } @@ -74,36 +64,24 @@ export function createSqlitePricingStore( workspaceRoot: string, options: CreatePricingStoreOptions & CreateSqliteUsageStoreOptions = {}, ): PricingStore { - return new SqlitePricingStore( - workspaceRoot, - options.createIfMissing ?? true, - options.initialOverrides ?? [], - options.failpoint, - ); + return new SqlitePricingStore(workspaceRoot, options.createIfMissing ?? true); } class SqliteTelemetryRepo implements TelemetryRepo { readonly #root: string; readonly #lease: OperationalStateDatabaseLease; readonly #pricingStore: PricingStore | undefined; - readonly #failpoint: ((point: OperationalStoreCutoverFailpoint) => void) | undefined; #loaded = false; #state: 'open' | 'draining' | 'closed' = 'open'; #queue: Promise = Promise.resolve(); #loadPromise: Promise | undefined; #closePromise: Promise | undefined; - constructor( - workspaceRoot: string, - createIfMissing: boolean, - managePricing: boolean, - failpoint?: (point: OperationalStoreCutoverFailpoint) => void, - ) { + constructor(workspaceRoot: string, createIfMissing: boolean, managePricing: boolean) { this.#root = resolve(workspaceRoot); this.#lease = acquireOperationalStateDatabase(this.#root); - this.#failpoint = failpoint; this.#pricingStore = managePricing - ? createSqlitePricingStore(this.#root, { createIfMissing, failpoint }) + ? createSqlitePricingStore(this.#root, { createIfMissing }) : undefined; } @@ -111,12 +89,10 @@ class SqliteTelemetryRepo implements TelemetryRepo { if (this.#loaded) return Promise.resolve(); this.assertOpen(); if (this.#loadPromise) return this.#loadPromise; - const operation = importLegacyUsageState(this.#root, this.#lease, this.#failpoint).then( - async () => { - if (this.#pricingStore) await this.#pricingStore.load(); - this.#loaded = true; - }, - ); + const operation = (async () => { + if (this.#pricingStore) await this.#pricingStore.load(); + this.#loaded = true; + })(); this.#loadPromise = operation; void operation.catch(() => { if (this.#state === 'open' && this.#loadPromise === operation) { @@ -257,15 +233,6 @@ class SqliteTelemetryRepo implements TelemetryRepo { await store.delete(store.snapshot().revision, modelKey); } - legacyPricingOverrides(): readonly unknown[] { - this.assertReady(); - return []; - } - - publishCanonical(): Promise { - return this.flush(); - } - async flush(): Promise { this.assertLoaded(); await this.#queue; @@ -342,7 +309,7 @@ class SqliteTelemetryRepo implements TelemetryRepo { private requireManagedPricing(): PricingStore { this.assertReady(); if (!this.#pricingStore) { - throw new Error('Telemetry repository does not own the compatibility pricing facade'); + throw new Error('Telemetry repository does not own the managed pricing store'); } return this.#pricingStore; } @@ -364,36 +331,22 @@ class SqliteTelemetryRepo implements TelemetryRepo { class SqlitePricingStore implements PricingStore { readonly #root: string; readonly #lease: OperationalStateDatabaseLease; - readonly #initialOverrides: readonly unknown[]; - readonly #failpoint: ((point: OperationalStoreCutoverFailpoint) => void) | undefined; #loaded = false; #state: 'open' | 'draining' | 'closed' = 'open'; #queue: Promise = Promise.resolve(); #loadPromise: Promise | undefined; #closePromise: Promise | undefined; - constructor( - workspaceRoot: string, - createIfMissing: boolean, - initialOverrides: readonly unknown[], - failpoint?: (point: OperationalStoreCutoverFailpoint) => void, - ) { + constructor(workspaceRoot: string, createIfMissing: boolean) { this.#root = resolve(workspaceRoot); this.#lease = acquireOperationalStateDatabase(this.#root); - this.#initialOverrides = initialOverrides; - this.#failpoint = failpoint; } load(): Promise { if (this.#loaded) return Promise.resolve(); this.assertOpen(); if (this.#loadPromise) return this.#loadPromise; - const operation = importLegacyUsageState( - this.#root, - this.#lease, - this.#failpoint, - this.#initialOverrides, - ).then(() => { + const operation = Promise.resolve().then(() => { this.#loaded = true; }); this.#loadPromise = operation; @@ -530,115 +483,6 @@ class SqlitePricingStore implements PricingStore { } } -interface LegacyUsageState { - readonly telemetry: TelemetryFile; - readonly pricing: PricingSnapshot; - readonly fingerprint: string; -} - -async function importLegacyUsageState( - root: string, - lease: OperationalStateDatabaseLease, - failpoint?: (point: OperationalStoreCutoverFailpoint) => void, - fallbackPricing: readonly unknown[] = [], -): Promise { - const legacy = await readLegacyUsageState(root, fallbackPricing); - completeOperationalStoreCutover(lease, { - storeName: 'usage_pricing', - sourcePath: root, - sourceFingerprint: legacy.fingerprint, - failpoint, - importAndValidate: (database) => { - const insertLlm = database.prepare(` - INSERT INTO usage_llm_calls(storage_key, id, ts, record_json) - VALUES (?, ?, ?, ?) - ON CONFLICT(storage_key) DO UPDATE SET - id = excluded.id, - ts = excluded.ts, - record_json = excluded.record_json - `); - for (const record of legacy.telemetry.usageRecords) { - insertLlm.run(usageIdentityKey(record.id), record.id, record.ts, JSON.stringify(record)); - } - const insertTool = database.prepare(` - INSERT INTO usage_tool_invocations(storage_key, id, ts, record_json) - VALUES (?, ?, ?, ?) - ON CONFLICT(storage_key) DO UPDATE SET - id = excluded.id, - ts = excluded.ts, - record_json = excluded.record_json - `); - for (const record of legacy.telemetry.toolInvocations) { - insertTool.run(usageIdentityKey(record.id), record.id, record.ts, JSON.stringify(record)); - } - database - .prepare(` - INSERT INTO usage_pricing_authority(singleton, revision) - VALUES (1, ?) - ON CONFLICT(singleton) DO UPDATE SET revision = excluded.revision - `) - .run(legacy.pricing.revision); - const insertPricing = database.prepare(` - INSERT INTO usage_pricing_overrides(model_key, record_json) - VALUES (?, ?) - ON CONFLICT(model_key) DO UPDATE SET record_json = excluded.record_json - `); - for (const override of legacy.pricing.overrides) { - insertPricing.run(override.modelKey, JSON.stringify(override)); - } - const counts = { - llm: countRows(database, 'usage_llm_calls'), - tools: countRows(database, 'usage_tool_invocations'), - pricing: countRows(database, 'usage_pricing_overrides'), - }; - if ( - counts.llm !== legacy.telemetry.usageRecords.length || - counts.tools !== legacy.telemetry.toolInvocations.length || - counts.pricing !== legacy.pricing.overrides.length - ) { - throw new Error('Usage/pricing cutover row-count validation failed'); - } - return counts; - }, - }); -} - -async function readLegacyUsageState( - root: string, - fallbackPricing: readonly unknown[], -): Promise { - const telemetryPath = join(root, 'telemetry.json'); - const telemetryText = await readOptionalText(telemetryPath); - const decoded = - telemetryText === undefined - ? { file: emptyTelemetryFile(), legacyPricingOverrides: fallbackPricing } - : decodeTelemetryFile(JSON.parse(telemetryText)); - const pricingPath = join(root, 'pricing.json'); - const pricingText = await readOptionalText(pricingPath); - const pricingStore = createPricingStore(root, { - createIfMissing: false, - initialOverrides: decoded.legacyPricingOverrides, - }); - await pricingStore.load(); - const pricing = pricingStore.snapshot(); - await pricingStore.close(); - const fingerprint = `sha256:${createHash('sha256') - .update(telemetryText === undefined ? 'telemetry:missing' : `telemetry:${telemetryText}`) - .update('\0') - .update(pricingText === undefined ? 'pricing:missing' : `pricing:${pricingText}`) - .digest('hex')}`; - return { telemetry: decoded.file, pricing, fingerprint }; -} - -async function readOptionalText(path: string): Promise { - try { - return await readFile(path, 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return undefined; - throw error; - } -} - function readPricingSnapshot(lease: OperationalStateDatabaseLease): PricingSnapshot { const authority = lease.database .prepare('SELECT revision FROM usage_pricing_authority WHERE singleton = 1') diff --git a/packages/storage/src/sqlite-workflow-schema.ts b/packages/storage/src/sqlite-workflow-schema.ts index a77133c5e6..385128d33b 100644 --- a/packages/storage/src/sqlite-workflow-schema.ts +++ b/packages/storage/src/sqlite-workflow-schema.ts @@ -1,6 +1,6 @@ import type { DatabaseSync } from 'node:sqlite'; -export const SQLITE_WORKFLOW_SCHEMA_VERSION = 1; +export const SQLITE_WORKFLOW_SCHEMA_VERSION = 2; export function migrateSqliteWorkflowDatabase(db: DatabaseSync): void { db.exec(` @@ -53,5 +53,25 @@ export function migrateSqliteWorkflowDatabase(db: DatabaseSync): void { CREATE INDEX IF NOT EXISTS workflow_plan_reminders_order ON workflow_plan_reminders(created_at, reminder_id); + + CREATE TABLE IF NOT EXISTS workflow_quote_companion_cleanup ( + session_id TEXT PRIMARY KEY, + tracked_at INTEGER NOT NULL + ); + + CREATE TABLE IF NOT EXISTS workflow_daily_review_state ( + singleton INTEGER PRIMARY KEY CHECK (singleton = 1), + config_json TEXT NOT NULL + ); + + CREATE TABLE IF NOT EXISTS workflow_daily_review_archives ( + archive_id TEXT PRIMARY KEY, + generated_at INTEGER NOT NULL, + day_from_ms INTEGER NOT NULL, + record_json TEXT NOT NULL + ); + + CREATE INDEX IF NOT EXISTS workflow_daily_review_archives_order + ON workflow_daily_review_archives(generated_at DESC, day_from_ms DESC, archive_id); `); } diff --git a/packages/storage/src/task-ledger-store.ts b/packages/storage/src/task-ledger-store.ts index c02906f36c..421cd74e77 100644 --- a/packages/storage/src/task-ledger-store.ts +++ b/packages/storage/src/task-ledger-store.ts @@ -1,5 +1,4 @@ -import { appendFile, mkdir, readFile, readdir, rename, rm, writeFile } from 'node:fs/promises'; -import { dirname, join, resolve } from 'node:path'; +import { resolve } from 'node:path'; import { createHash, randomUUID } from 'node:crypto'; import type { DatabaseSync } from 'node:sqlite'; import { @@ -10,13 +9,9 @@ import { isTaskKey, isTaskOwner, isTerminalTaskStatus, - isTaskStatus, isTaskLedgerEvent, normalizeUpdateTaskInput, normalizeCreateTaskInput, - normalizeResumeTrust, - normalizeTaskEvidenceText, - normalizeTaskSubject, projectTaskLedgerEvents, taskLedgerEventTypeForCreate, taskLedgerEventTypeForUpdate, @@ -27,7 +22,6 @@ import { type TaskAvailableClaimScope, type TaskLedgerChangedEvent, type TaskLedgerEvent, - type TaskLedgerEventTaskSnapshot, type TaskLedgerListOptions, type TaskLedgerMutationContext, type TaskLedgerStore, @@ -35,13 +29,10 @@ import { } from '@maka/core/task-ledger'; import { chainWrite } from './write-queue.js'; import { assertSafeSessionId } from './session-store.js'; -import { syncDirectory } from './stable-storage.js'; import { registerTaskLedgerCanonicalReader } from './task-ledger-store-internal.js'; import { acquireOperationalStateDatabase, - completeOperationalStoreCutover, type OperationalStateDatabaseLease, - type OperationalStoreCutoverFailpoint, } from './operational-state-store.js'; export type { TaskLedgerStore } from '@maka/core/task-ledger'; @@ -62,39 +53,36 @@ export interface TaskLedgerAuthorityStore extends TaskLedgerStore { purgeConversationTaskLedger(sessionId: string): Promise; } -export function createTaskLedgerStore(workspaceRoot: string): TaskLedgerAuthorityStore { - return new FileTaskLedgerStore(workspaceRoot); -} - export interface SqliteTaskLedgerStore extends TaskLedgerAuthorityStore { ready(): Promise; close(): void; } -export interface CreateSqliteTaskLedgerStoreOptions { - failpoint?: (point: OperationalStoreCutoverFailpoint) => void; -} - -export function createSqliteTaskLedgerStore( - workspaceRoot: string, - options: CreateSqliteTaskLedgerStoreOptions = {}, -): SqliteTaskLedgerStore { - return new SqliteTaskLedgerStoreImpl(workspaceRoot, options); +export function createSqliteTaskLedgerStore(workspaceRoot: string): SqliteTaskLedgerStore { + return new SqliteTaskLedgerStoreImpl(workspaceRoot); } -class FileTaskLedgerStore implements TaskLedgerAuthorityStore { - private readonly sessionsRoot: string; +class SqliteTaskLedgerStoreImpl implements SqliteTaskLedgerStore { + readonly #lease: OperationalStateDatabaseLease; private readonly writeQueues = new Map>(); private readonly listeners = new Set<(event: TaskLedgerChangedEvent) => void>(); constructor(workspaceRoot: string) { - this.sessionsRoot = join(workspaceRoot, 'sessions'); + this.#lease = acquireOperationalStateDatabase(resolve(workspaceRoot)); registerTaskLedgerCanonicalReader(this, { list: (sessionId, options) => this.#listCanonical(sessionId, options), get: (sessionId, id, options) => this.#getCanonical(sessionId, id, options), }); } + ready(): Promise { + return Promise.resolve(); + } + + close(): void { + this.#lease.close(); + } + async list(sessionId: string, options: TaskLedgerListOptions = {}): Promise { assertSafeSessionId(sessionId); return this.applyListOptions(await this.readForRender(sessionId), options); @@ -179,42 +167,21 @@ class FileTaskLedgerStore implements TaskLedgerAuthorityStore { } await chainWrite(this.writeQueues, input.targetSessionId, async () => { - if ( - await this.copyCanonicalConversationTaskLedger( - input.targetSessionId, - selected, - projection.tasks, - ) - ) { - return; - } - const eventsPath = this.eventsPath(input.targetSessionId); - const tasksPath = this.filePath(input.targetSessionId); - await assertConversationCopyTargetAbsent(eventsPath); - await assertConversationCopyTargetAbsent(tasksPath); - await mkdir(dirname(eventsPath), { recursive: true }); - await writeFile( - eventsPath, - selected.map((event) => JSON.stringify(event)).join('\n') + '\n', - { encoding: 'utf8', flag: 'wx' }, - ); - await this.write(input.targetSessionId, projection.tasks); + this.copyConversationLedger(input.targetSessionId, selected, projection.tasks); }); } async purgeConversationTaskLedger(sessionId: string): Promise { assertSafeSessionId(sessionId); await chainWrite(this.writeQueues, sessionId, async () => { - if (await this.purgeCanonicalConversationTaskLedger(sessionId)) return; - const eventsPath = this.eventsPath(sessionId); - const tasksPath = this.filePath(sessionId); - await rm(eventsPath, { force: true }); - await rm(tasksPath, { force: true }); - try { - await syncDirectory(dirname(eventsPath)); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - } + this.#lease.transaction('write', () => { + this.#lease.database + .prepare('DELETE FROM workflow_task_ledger_events WHERE session_id = ?') + .run(sessionId); + this.#lease.database + .prepare('DELETE FROM workflow_task_ledger_projections WHERE session_id = ?') + .run(sessionId); + }); }); } @@ -559,108 +526,15 @@ class FileTaskLedgerStore implements TaskLedgerAuthorityStore { return { updated, total: all.length }; } - private filePath(sessionId: string): string { - return join(this.sessionsRoot, sessionId, 'tasks.json'); - } - - private eventsPath(sessionId: string): string { - return join(this.sessionsRoot, sessionId, 'task-events.jsonl'); - } - - /** - * Render-path read: a damaged event ledger falls back to the projection cache - * as untrusted when possible, so resume/debug surfaces retain conservative - * state without allowing writes to proceed from that cache. - */ private async readForRender(sessionId: string): Promise { - try { - return (await this.readProjected(sessionId)).tasks; - } catch (eventError) { - if (this.usesCanonicalEventStore()) throw eventError; - try { - await readFile(this.eventsPath(sessionId), 'utf8'); - return await this.readUntrustedCache(sessionId); - } catch (readEventError) { - if ((readEventError as NodeJS.ErrnoException).code !== 'ENOENT') { - return await this.readUntrustedCache(sessionId); - } - } - try { - return projectLegacySnapshots( - decodeTaskSnapshots(await readFile(this.filePath(sessionId), 'utf8')), - ).tasks; - } catch { - return []; - } - } - } - - private async readUntrustedCache(sessionId: string): Promise { - try { - const tasks = projectLegacySnapshots( - decodeTaskSnapshots(await readFile(this.filePath(sessionId), 'utf8')), - ).tasks; - return tasks.map((task) => ({ ...task, resumeTrust: 'untrusted' })); - } catch { - return []; - } + return (await this.readProjected(sessionId)).tasks; } - /** - * Mutate-path read: only ENOENT means a legitimately fresh ledger. Any - * other read error, undecodable JSON, or a non-array payload throws so the - * mutation fails closed instead of rebuilding the ledger from [] and - * silently overwriting whatever is on disk. - */ - private async readForMutateWithSource(sessionId: string): Promise<{ - tasks: Task[]; - source: 'events' | 'legacy'; - backfilledTaskIds: string[]; - }> { - try { - const projected = await this.readProjected(sessionId); - return { - tasks: projected.tasks, - source: 'events', - backfilledTaskIds: projected.backfilledTaskIds, - }; - } catch (eventError) { - if (this.usesCanonicalEventStore()) throw eventError; - try { - await readFile(this.eventsPath(sessionId), 'utf8'); - throw eventError; - } catch (readEventError) { - if ((readEventError as NodeJS.ErrnoException).code !== 'ENOENT') { - throw eventError; - } - } - } - let text: string; - try { - text = await readFile(this.filePath(sessionId), 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') - return { tasks: [], source: 'legacy', backfilledTaskIds: [] }; - throw error; - } - try { - const projected = projectLegacySnapshots(decodeTaskSnapshots(text)); - return { - tasks: projected.tasks, - source: 'legacy', - backfilledTaskIds: projected.backfilledTaskIds, - }; - } catch (error) { - throw new Error( - `Task ledger file for session ${sessionId} is corrupt; refusing to overwrite it: ` + - (error instanceof Error ? error.message : String(error)), - ); - } + private async readForMutateWithSource(sessionId: string): Promise<{ tasks: Task[] }> { + return this.readProjected(sessionId); } - private async readProjected( - sessionId: string, - ): Promise<{ tasks: Task[]; backfilledTaskIds: string[] }> { + private async readProjected(sessionId: string): Promise<{ tasks: Task[] }> { const events = await this.readTaskEvents(sessionId); const projection = projectTaskLedgerEvents(events); if (projection.diagnostics.length > 0) { @@ -673,47 +547,15 @@ class FileTaskLedgerStore implements TaskLedgerAuthorityStore { `task event ledger has ${projection.tasks.length} tasks, exceeding the ${TASK_LEDGER_MAX_TASKS}-task cap; refusing to load an unbounded ledger`, ); } - return { tasks: projection.tasks, backfilledTaskIds: projection.backfilledTaskIds }; + return { tasks: projection.tasks }; } private async readTaskEvents(sessionId: string): Promise { - const canonical = await this.readCanonicalTaskEvents(sessionId); - if (canonical) return canonical; - const text = await readFile(this.eventsPath(sessionId), 'utf8'); - const events: TaskLedgerEvent[] = []; - const lines = text.split(/\n/); - for (let index = 0; index < lines.length; index += 1) { - const line = lines[index]; - if (line.trim().length === 0) continue; - let parsed: unknown; - try { - parsed = JSON.parse(line); - } catch (error) { - throw new Error( - `Invalid task event JSONL line ${index + 1}: ${error instanceof Error ? error.message : String(error)}`, - ); - } - if (!isTaskLedgerEvent(parsed)) { - throw new Error(`Invalid task event JSONL line ${index + 1}: unexpected event shape`); - } - events.push(parsed); - } - return events; + return readSqliteTaskLedgerEvents(this.#lease.database, sessionId); } private async readConversationCopyEvents(sessionId: string): Promise { - try { - return await this.readTaskEvents(sessionId); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - const legacy = await this.readForMutateWithSource(sessionId); - if (legacy.tasks.length > 0) { - throw new Error( - 'Legacy Task Ledger snapshots cannot be placed at an exact conversation boundary', - ); - } - return []; - } + return this.readTaskEvents(sessionId); } private async mutate( @@ -727,39 +569,10 @@ class FileTaskLedgerStore implements TaskLedgerAuthorityStore { const current = currentRead.tasks; next = fn(current); const mutationEvents = eventsForMutation(next); - const compatibilityEvents = - currentRead.source === 'legacy' - ? current.map((task) => - buildTaskLedgerEvent({ - type: 'task_imported', - sessionId, - task, - context: { source: 'import', actor: 'system' }, - }), - ) - : currentRead.backfilledTaskIds.flatMap((taskId) => { - const task = current.find((candidate) => candidate.id === taskId); - return task - ? [ - buildTaskLedgerEvent({ - type: 'task_updated', - sessionId, - task, - previous: task, - context: { - source: 'recovery', - actor: 'system', - reason: 'backfilled task-ledger v2 fields', - }, - }), - ] - : []; - }); - const appended = [...compatibilityEvents, ...mutationEvents]; - await this.appendEvents(sessionId, appended); + await this.appendEvents(sessionId, mutationEvents); this.emitChanged({ sessionId, - taskIds: [...new Set(appended.map((event) => event.taskId))], + taskIds: [...new Set(mutationEvents.map((event) => event.taskId))], at: Date.now(), }); await this.write(sessionId, next); @@ -769,59 +582,36 @@ class FileTaskLedgerStore implements TaskLedgerAuthorityStore { private async appendEvents(sessionId: string, events: TaskLedgerEvent[]): Promise { if (events.length === 0) return; - if (await this.appendCanonicalTaskEvents(sessionId, events)) return; - const filePath = this.eventsPath(sessionId); - await mkdir(dirname(filePath), { recursive: true }); - await appendFile( - filePath, - events.map((event) => JSON.stringify(event)).join('\n') + '\n', - 'utf8', - ); + this.#lease.transaction('write', () => { + for (const event of events) insertTaskLedgerEvent(this.#lease.database, sessionId, event); + }); } private async write(sessionId: string, tasks: Task[]): Promise { - if (await this.writeCanonicalTaskProjection(sessionId, tasks)) return; - const filePath = this.filePath(sessionId); - await mkdir(dirname(filePath), { recursive: true }); - const tempPath = `${filePath}.${process.pid}.${Date.now()}.tmp`; - await writeFile(tempPath, JSON.stringify(tasks, null, 2) + '\n', 'utf8'); - await rename(tempPath, filePath); - } - - protected usesCanonicalEventStore(): boolean { - return false; - } - - protected async readCanonicalTaskEvents( - _sessionId: string, - ): Promise { - return undefined; - } - - protected async appendCanonicalTaskEvents( - _sessionId: string, - _events: TaskLedgerEvent[], - ): Promise { - return false; - } - - protected async writeCanonicalTaskProjection( - _sessionId: string, - _tasks: Task[], - ): Promise { - return false; - } - - protected async copyCanonicalConversationTaskLedger( - _sessionId: string, - _events: readonly TaskLedgerEvent[], - _tasks: readonly Task[], - ): Promise { - return false; + this.#lease.transaction('write', () => { + writeTaskLedgerProjection(this.#lease.database, sessionId, tasks); + }); } - protected async purgeCanonicalConversationTaskLedger(_sessionId: string): Promise { - return false; + private copyConversationLedger( + sessionId: string, + events: readonly TaskLedgerEvent[], + tasks: readonly Task[], + ): void { + this.#lease.transaction('write', () => { + const existing = this.#lease.database + .prepare(` + SELECT + (SELECT COUNT(*) FROM workflow_task_ledger_events WHERE session_id = ?) + + (SELECT COUNT(*) FROM workflow_task_ledger_projections WHERE session_id = ?) AS count + `) + .get(sessionId, sessionId) as { count?: unknown }; + if (existing.count !== 0) { + throw new Error('Task Ledger conversation-copy target already exists'); + } + for (const event of events) insertTaskLedgerEvent(this.#lease.database, sessionId, event); + writeTaskLedgerProjection(this.#lease.database, sessionId, [...tasks]); + }); } private applyListOptions(tasks: Task[], options: TaskLedgerListOptions): Task[] { @@ -856,214 +646,6 @@ class FileTaskLedgerStore implements TaskLedgerAuthorityStore { } } -class SqliteTaskLedgerStoreImpl extends FileTaskLedgerStore implements SqliteTaskLedgerStore { - readonly #lease: OperationalStateDatabaseLease; - readonly #ready: Promise; - - constructor(workspaceRoot: string, options: CreateSqliteTaskLedgerStoreOptions) { - super(workspaceRoot); - const root = resolve(workspaceRoot); - this.#lease = acquireOperationalStateDatabase(root); - this.#ready = importLegacyTaskLedgerState(root, this.#lease, options.failpoint); - } - - ready(): Promise { - return this.#ready; - } - - close(): void { - this.#lease.close(); - } - - protected override usesCanonicalEventStore(): boolean { - return true; - } - - protected override async readCanonicalTaskEvents(sessionId: string): Promise { - await this.#ready; - return readSqliteTaskLedgerEvents(this.#lease.database, sessionId); - } - - protected override async appendCanonicalTaskEvents( - sessionId: string, - events: TaskLedgerEvent[], - ): Promise { - await this.#ready; - this.#lease.transaction('write', () => { - for (const event of events) insertTaskLedgerEvent(this.#lease.database, sessionId, event); - }); - return true; - } - - protected override async writeCanonicalTaskProjection( - sessionId: string, - tasks: Task[], - ): Promise { - await this.#ready; - this.#lease.transaction('write', () => { - writeTaskLedgerProjection(this.#lease.database, sessionId, tasks); - }); - return true; - } - - protected override async copyCanonicalConversationTaskLedger( - sessionId: string, - events: readonly TaskLedgerEvent[], - tasks: readonly Task[], - ): Promise { - await this.#ready; - this.#lease.transaction('write', () => { - const existing = this.#lease.database - .prepare(` - SELECT - (SELECT COUNT(*) FROM workflow_task_ledger_events WHERE session_id = ?) + - (SELECT COUNT(*) FROM workflow_task_ledger_projections WHERE session_id = ?) AS count - `) - .get(sessionId, sessionId) as { count?: unknown }; - if (existing.count !== 0) { - throw new Error('Task Ledger conversation-copy target already exists'); - } - for (const event of events) { - insertTaskLedgerEvent(this.#lease.database, sessionId, event); - } - writeTaskLedgerProjection(this.#lease.database, sessionId, [...tasks]); - }); - return true; - } - - protected override async purgeCanonicalConversationTaskLedger( - sessionId: string, - ): Promise { - await this.#ready; - this.#lease.transaction('write', () => { - this.#lease.database - .prepare('DELETE FROM workflow_task_ledger_events WHERE session_id = ?') - .run(sessionId); - this.#lease.database - .prepare('DELETE FROM workflow_task_ledger_projections WHERE session_id = ?') - .run(sessionId); - }); - return true; - } -} - -interface LegacyTaskLedger { - sessionId: string; - events: TaskLedgerEvent[]; - tasks: Task[]; -} - -async function importLegacyTaskLedgerState( - root: string, - lease: OperationalStateDatabaseLease, - failpoint?: (point: OperationalStoreCutoverFailpoint) => void, -): Promise { - const ledgers = await readLegacyTaskLedgers(root); - const fingerprint = `sha256:${createHash('sha256') - .update(JSON.stringify(ledgers)) - .digest('hex')}`; - completeOperationalStoreCutover(lease, { - storeName: 'workflow_task_ledger', - sourcePath: join(root, 'sessions'), - sourceFingerprint: fingerprint, - failpoint, - importAndValidate: (database) => { - let eventCount = 0; - for (const ledger of ledgers) { - for (const event of ledger.events) { - insertOrValidateTaskLedgerEvent(database, ledger.sessionId, event); - eventCount += 1; - } - writeTaskLedgerProjection(database, ledger.sessionId, ledger.tasks); - } - const persisted = database - .prepare('SELECT COUNT(*) AS count FROM workflow_task_ledger_events') - .get() as { count?: unknown }; - if (persisted.count !== eventCount) { - throw new Error('Task ledger cutover row-count validation failed'); - } - return { sessions: ledgers.length, events: eventCount }; - }, - }); -} - -async function readLegacyTaskLedgers(root: string): Promise { - const sessionsRoot = join(root, 'sessions'); - let entries; - try { - entries = await readdir(sessionsRoot, { withFileTypes: true }); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; - throw error; - } - const result: LegacyTaskLedger[] = []; - for (const entry of entries.sort((left, right) => left.name.localeCompare(right.name))) { - if (!entry.isDirectory()) continue; - assertSafeSessionId(entry.name); - const eventsPath = join(sessionsRoot, entry.name, 'task-events.jsonl'); - let events: TaskLedgerEvent[] | undefined; - try { - events = decodeLegacyTaskLedgerEvents(await readFile(eventsPath, 'utf8'), entry.name); - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - } - if (!events) { - const tasksPath = join(sessionsRoot, entry.name, 'tasks.json'); - let snapshots: TaskLedgerEventTaskSnapshot[]; - try { - snapshots = decodeTaskSnapshots(await readFile(tasksPath, 'utf8')); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') continue; - throw error; - } - const projected = projectLegacySnapshots(snapshots); - events = projected.tasks.map( - (task, index): TaskLedgerEvent => ({ - eventId: `legacy-import-${index}`, - type: 'task_imported', - ts: task.createdAt, - sessionId: entry.name, - taskId: task.id, - nextStatus: task.status, - task, - source: 'import', - actor: 'system', - }), - ); - } - const projection = projectTaskLedgerEvents(events); - if (projection.diagnostics.length > 0) { - throw new Error( - `task event ledger has projection diagnostics: ${projection.diagnostics.join('; ')}`, - ); - } - result.push({ sessionId: entry.name, events, tasks: projection.tasks }); - } - return result; -} - -function decodeLegacyTaskLedgerEvents(text: string, sessionId: string): TaskLedgerEvent[] { - const events: TaskLedgerEvent[] = []; - for (const [index, line] of text.split(/\n/).entries()) { - if (!line.trim()) continue; - let parsed: unknown; - try { - parsed = JSON.parse(line); - } catch (error) { - throw new Error( - `Invalid task event JSONL line ${index + 1}: ${ - error instanceof Error ? error.message : String(error) - }`, - ); - } - if (!isTaskLedgerEvent(parsed) || parsed.sessionId !== sessionId) { - throw new Error(`Invalid task event JSONL line ${index + 1}: unexpected event shape`); - } - events.push(parsed); - } - return events; -} - function readSqliteTaskLedgerEvents(database: DatabaseSync, sessionId: string): TaskLedgerEvent[] { assertSafeSessionId(sessionId); const rows = database @@ -1110,27 +692,6 @@ function insertTaskLedgerEvent( .run(sessionId, row.sequence, event.eventId, JSON.stringify(event)); } -function insertOrValidateTaskLedgerEvent( - database: DatabaseSync, - sessionId: string, - event: TaskLedgerEvent, -): void { - const existing = database - .prepare(` - SELECT record_json - FROM workflow_task_ledger_events - WHERE session_id = ? AND event_id = ? - `) - .get(sessionId, event.eventId) as { record_json?: unknown } | undefined; - if (existing) { - if (existing.record_json !== JSON.stringify(event)) { - throw new Error(`Task ledger cutover conflict: ${event.eventId}`); - } - return; - } - insertTaskLedgerEvent(database, sessionId, event); -} - function writeTaskLedgerProjection(database: DatabaseSync, sessionId: string, tasks: Task[]): void { database .prepare(` @@ -1141,144 +702,6 @@ function writeTaskLedgerProjection(database: DatabaseSync, sessionId: string, ta .run(sessionId, JSON.stringify(tasks)); } -function decodeTaskSnapshots(text: string): TaskLedgerEventTaskSnapshot[] { - const parsed = JSON.parse(text) as unknown; - if (!Array.isArray(parsed)) { - throw new Error('expected a JSON array of tasks'); - } - const tasks: TaskLedgerEventTaskSnapshot[] = []; - const seenIds = new Set(); - for (const value of parsed) { - const task = normalizePersistedTask(value); - if (!task) continue; - // A tasks.json with two records sharing an id would render two - // indistinguishable tasks in the turn tail, and TaskUpdate's first-match - // lookup would only ever touch the first -- the second is unreachable and - // a mutate would silently keep both. Treat a duplicate id as corrupt so - // the render path degrades to empty and the mutate path stays fail-closed - // instead of rewriting a "half-correct" file. - if (seenIds.has(task.id)) { - throw new Error( - `task ledger has a duplicate id "${task.id}"; refusing to load an ambiguous ledger`, - ); - } - seenIds.add(task.id); - tasks.push(task); - } - // Enforce the same total-task cap as the write path on read. A hand-edited, - // legacy, or externally-written tasks.json could otherwise carry an - // unbounded number of valid records, which `list()` would inject into the - // turn tail every turn. Treat over-cap as corrupt so the render path - // degrades to empty (its caller already try/catches) and the mutate path - // stays fail-closed instead of silently truncating-and-overwriting. - if (tasks.length > TASK_LEDGER_MAX_TASKS) { - throw new Error( - `task ledger has ${tasks.length} tasks, exceeding the ${TASK_LEDGER_MAX_TASKS}-task cap; refusing to load an unbounded ledger`, - ); - } - return tasks; -} - -function normalizePersistedTask(value: unknown): TaskLedgerEventTaskSnapshot | undefined { - if (typeof value !== 'object' || value === null || Array.isArray(value)) return undefined; - const record = value as Partial; - // Timestamps must be finite: a hand-edited `1e999` parses to Infinity, and - // JSON.stringify(Infinity) writes null, so the record would silently vanish - // on the next write. Reject it up front (per-record drop) instead. - if ( - typeof record.id !== 'string' || - !isSafeTaskId(record.id) || - typeof record.createdAt !== 'number' || - !Number.isFinite(record.createdAt) || - typeof record.updatedAt !== 'number' || - !Number.isFinite(record.updatedAt) || - !isTaskStatus(record.status) - ) { - return undefined; - } - // Re-apply the same subject normalization as the write path (NFC, whitespace - // collapse, trim, length cap, non-empty) so a manually-edited or legacy - // tasks.json cannot inject an overlong/blank subject into the turn tail - // every turn. Invalid subjects drop the whole record, matching the existing - // "single malformed entry discarded" semantic. - const subject = normalizeTaskSubject(record.subject); - if (!subject.ok) return undefined; - return { - id: record.id, - ...(record.key && isTaskKey(record.key) ? { key: record.key } : {}), - subject: subject.value, - status: record.status, - createdAt: record.createdAt, - updatedAt: record.updatedAt, - ...(record.parentId && isSafeTaskId(record.parentId) ? { parentId: record.parentId } : {}), - ...normalizeOptionalOwner(record.owner), - ...(typeof record.endedAt === 'number' && Number.isFinite(record.endedAt) - ? { endedAt: record.endedAt } - : {}), - ...normalizeOptionalEvidence(record.blockedReason, 'blockedReason'), - ...normalizeOptionalEvidence(record.failureReason, 'failureReason'), - ...normalizeOptionalEvidence(record.completionEvidence, 'completionEvidence'), - ...normalizeOptionalResumeTrust(record.resumeTrust), - }; -} - -function normalizeOptionalEvidence( - value: unknown, - field: 'blockedReason' | 'failureReason' | 'completionEvidence', -): Partial { - if (value === undefined) return {}; - const normalized = normalizeTaskEvidenceText(value, field); - if (!normalized.ok) return {}; - return { [field]: normalized.value } as Partial; -} - -function normalizeOptionalResumeTrust( - value: unknown, -): Pick | Record { - if (value === undefined) return {}; - const normalized = normalizeResumeTrust(value); - if (!normalized.ok) return {}; - return { resumeTrust: normalized.value }; -} - -function normalizeOptionalOwner(value: unknown): Pick | Record { - if (!isTaskOwner(value)) return {}; - const owner = value; - return { - owner: { - actor: owner.actor, - ...(owner.sessionId ? { sessionId: owner.sessionId } : {}), - ...(owner.agentId ? { agentId: owner.agentId } : {}), - ...(owner.runId ? { runId: owner.runId } : {}), - ...(owner.turnId ? { turnId: owner.turnId } : {}), - }, - }; -} - -function projectLegacySnapshots(tasks: readonly TaskLedgerEventTaskSnapshot[]) { - const projection = projectTaskLedgerEvents( - tasks.map( - (task, index): TaskLedgerEvent => ({ - eventId: `legacy-import-${index}`, - type: 'task_imported', - ts: task.createdAt, - sessionId: 'legacy', - taskId: task.id, - nextStatus: task.status, - task, - source: 'import', - actor: 'system', - }), - ), - ); - if (projection.diagnostics.length > 0) { - throw new Error( - `legacy task ledger has projection diagnostics: ${projection.diagnostics.join('; ')}`, - ); - } - return projection; -} - function nextTaskKey(tasks: readonly Task[], parent: Task | undefined): string { const siblings = tasks.filter((task) => task.parentId === parent?.id); const prefix = parent ? `${parent.key}.` : 'T'; @@ -1393,16 +816,6 @@ function requiredConversationCopyRunId( return targetRunId; } -async function assertConversationCopyTargetAbsent(path: string): Promise { - try { - await readFile(path, 'utf8'); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return; - throw error; - } - throw new Error('Task Ledger conversation-copy target already exists'); -} - function buildTaskLedgerEvent(input: { type: TaskLedgerEvent['type']; sessionId: string; diff --git a/packages/storage/src/telemetry-file-schema.ts b/packages/storage/src/telemetry-file-schema.ts index fef1466642..e43862eca2 100644 --- a/packages/storage/src/telemetry-file-schema.ts +++ b/packages/storage/src/telemetry-file-schema.ts @@ -27,18 +27,6 @@ export type PersistedToolInvocationRecord = ToolInvocationRecord & { ts: number; }; -export interface TelemetryFile { - readonly version: 1; - readonly usageRecords: readonly PersistedLlmCallRecord[]; - readonly toolInvocations: readonly PersistedToolInvocationRecord[]; -} - -export interface DecodedTelemetryFile { - readonly file: TelemetryFile; - readonly legacyPricingOverrides: readonly unknown[]; - readonly requiresCanonicalPublication: boolean; -} - type ExactKeyShape = { readonly [Key in keyof Value]-?: true; }; @@ -47,7 +35,6 @@ function exactKeys(shape: ExactKeyShape): ReadonlyS return new Set(Object.keys(shape)); } -const FILE_KEYS = new Set(['version', 'usageRecords', 'toolInvocations']); const LLM_KEYS = exactKeys({ sessionId: true, turnId: true, @@ -120,108 +107,6 @@ const TOOL_SCHEMA_CHANGE_REASONS = new Set([ 'tool_source_state_changed', ]); -export function emptyTelemetryFile(): TelemetryFile { - return { version: 1, usageRecords: [], toolInvocations: [] }; -} - -export function decodeTelemetryFile(input: unknown): DecodedTelemetryFile { - if (!isRecord(input)) throw invalid('expected an object'); - if (input.version === undefined) return decodeLegacyTelemetryFile(input); - if (!hasOnlyKeys(input, FILE_KEYS)) { - throw invalid('expected exactly version, usageRecords, toolInvocations'); - } - if (input.version !== 1) throw invalid('expected version 1'); - assertArray(input, 'usageRecords'); - assertArray(input, 'toolInvocations'); - return { - file: { - version: 1, - usageRecords: input.usageRecords.map(decodePersistedLlmCallRecord), - toolInvocations: input.toolInvocations.map(decodePersistedToolInvocationRecord), - }, - legacyPricingOverrides: [], - requiresCanonicalPublication: false, - }; -} - -function decodeLegacyTelemetryFile(input: Record): DecodedTelemetryFile { - const hasKnownSection = - 'usageRecords' in input || 'toolInvocations' in input || 'pricingOverrides' in input; - if (!hasKnownSection) throw invalid('expected known telemetry sections'); - assertOptionalArray(input, 'usageRecords'); - assertOptionalArray(input, 'toolInvocations'); - assertOptionalArray(input, 'pricingOverrides'); - const usageRecords = (input.usageRecords ?? []) as unknown[]; - const toolInvocations = (input.toolInvocations ?? []) as unknown[]; - const pricingOverrides = (input.pricingOverrides ?? []) as unknown[]; - return { - file: { - version: 1, - usageRecords: usageRecords.map((row) => - decodePersistedLlmCallRecord(normalizeLlmCallRecord(row)), - ), - toolInvocations: toolInvocations.map((row) => - decodePersistedToolInvocationRecord(normalizeToolInvocationRecord(row)), - ), - }, - legacyPricingOverrides: pricingOverrides, - requiresCanonicalPublication: true, - }; -} - -export function normalizeLlmCallRecord(input: unknown): PersistedLlmCallRecord { - if (!isRecord(input)) throw invalid('LLM record must be an object'); - const row = input as Partial; - const inputTokens = finiteNumber(row.inputTokens) ?? 0; - const outputTokens = finiteNumber(row.outputTokens) ?? 0; - const cacheHitInputTokens = - finiteNumber(row.cacheHitInputTokens) ?? finiteNumber(row.cachedInputTokens) ?? 0; - const cacheWriteInputTokens = finiteNumber(row.cacheWriteInputTokens) ?? 0; - const cacheMissInputTokens = - finiteNumber(row.cacheMissInputTokens) ?? - Math.max(0, inputTokens - cacheHitInputTokens - cacheWriteInputTokens); - const reasoningTokens = finiteNumber(row.reasoningTokens) ?? 0; - const ts = finiteNumber(row.ts) ?? finiteNumber(row.startedAt) ?? 0; - return { - ...pickKnown(input, LLM_KEYS), - id: typeof row.id === 'string' ? row.id : `usage_${row.turnId ?? ts}`, - providerId: typeof row.providerId === 'string' ? row.providerId : 'unknown', - modelId: typeof row.modelId === 'string' ? row.modelId : 'unknown', - inputTokens, - outputTokens, - cacheHitInputTokens, - cacheMissInputTokens, - cachedInputTokens: cacheHitInputTokens, - cacheWriteInputTokens, - reasoningTokens, - totalTokens: finiteNumber(row.totalTokens) ?? inputTokens + outputTokens + reasoningTokens, - costUsd: finiteNumber(row.costUsd) ?? 0, - latencyMs: finiteNumber(row.latencyMs) ?? 0, - status: row.status === 'error' || row.status === 'aborted' ? row.status : 'success', - startedAt: finiteNumber(row.startedAt) ?? ts, - date: typeof row.date === 'string' ? row.date : new Date(ts).toISOString().slice(0, 10), - ts, - } as PersistedLlmCallRecord; -} - -export function normalizeToolInvocationRecord(input: unknown): PersistedToolInvocationRecord { - if (!isRecord(input)) throw invalid('tool invocation must be an object'); - const row = input as Partial; - const ts = finiteNumber(row.ts) ?? finiteNumber(row.startedAt) ?? 0; - return { - ...pickKnown(input, TOOL_KEYS), - id: typeof row.id === 'string' ? row.id : `tool_${row.toolCallId ?? row.turnId ?? ts}`, - toolName: typeof row.toolName === 'string' ? row.toolName : 'unknown', - durationMs: finiteNumber(row.durationMs) ?? 0, - status: row.status === 'error' || row.status === 'aborted' ? row.status : 'success', - bytesIn: finiteNumber(row.bytesIn) ?? 0, - bytesOut: finiteNumber(row.bytesOut) ?? 0, - startedAt: finiteNumber(row.startedAt) ?? ts, - date: typeof row.date === 'string' ? row.date : new Date(ts).toISOString().slice(0, 10), - ts, - } as PersistedToolInvocationRecord; -} - export function decodePersistedLlmCallRecord(input: unknown): PersistedLlmCallRecord { if (!isRecord(input) || !hasOnlyKeys(input, LLM_KEYS)) throw invalid('invalid LLM row keys'); if (!strings(input, ['id', 'providerId', 'modelId', 'date'])) { @@ -421,27 +306,6 @@ function isToolResultSummary(input: unknown): boolean { ); } -function assertArray( - value: Record, - key: 'usageRecords' | 'toolInvocations', -): asserts value is Record & Record { - if (!Array.isArray(value[key])) throw invalid(`${key} must be an array`); -} - -function assertOptionalArray( - value: Record, - key: 'usageRecords' | 'toolInvocations' | 'pricingOverrides', -): void { - if (key in value && !Array.isArray(value[key])) throw invalid(`${key} must be an array`); -} - -function pickKnown( - value: Record, - allowed: ReadonlySet, -): Record { - return Object.fromEntries(Object.entries(value).filter(([key]) => allowed.has(key))); -} - function hasOnlyKeys(value: Record, allowed: ReadonlySet): boolean { return Object.keys(value).every((key) => allowed.has(key)); } @@ -473,10 +337,6 @@ function isNonNegativeFinite(value: unknown): value is number { return typeof value === 'number' && Number.isFinite(value) && value >= 0; } -function finiteNumber(value: unknown): number | undefined { - return isNonNegativeFinite(value) ? value : undefined; -} - function optionalEnum(value: unknown, allowed: ReadonlySet): boolean { return value === undefined || (typeof value === 'string' && allowed.has(value)); } @@ -538,5 +398,5 @@ function isRecord(value: unknown): value is Record { } function invalid(message: string): Error { - return new Error(`Invalid telemetry file: ${message}`); + return new Error(`Invalid telemetry record: ${message}`); } diff --git a/packages/storage/src/telemetry-repo.ts b/packages/storage/src/telemetry-repo.ts index 23587832ad..a6fcf7c4bc 100644 --- a/packages/storage/src/telemetry-repo.ts +++ b/packages/storage/src/telemetry-repo.ts @@ -1,6 +1,3 @@ -import { randomUUID } from 'node:crypto'; -import { mkdir, open, readFile, rename, rm } from 'node:fs/promises'; -import { dirname, join } from 'node:path'; import type { PricingConfig, UsageBucket, @@ -9,18 +6,9 @@ import type { UsageQuery, UsageSummaryV2, } from '@maka/core/usage-stats/types'; -import { usageBucketKey } from '@maka/core/usage-stats/bucket-key'; -import { throwDeduplicatedFailures } from './failure-utils.js'; -import { createPricingStore, type PricingStore } from './pricing-store.js'; -import { syncDirectory } from './stable-storage.js'; import { - decodeTelemetryFile, - decodePersistedLlmCallRecord, - decodePersistedToolInvocationRecord, - emptyTelemetryFile, type PersistedLlmCallRecord, type PersistedToolInvocationRecord, - type TelemetryFile, } from './telemetry-file-schema.js'; export type { @@ -49,8 +37,6 @@ export interface TelemetryRepo { listPricingOverrides(): PricingConfig[]; upsertPricing(pricing: PricingConfig): Promise; deletePricing(modelKey: string): Promise; - legacyPricingOverrides(): readonly unknown[]; - publishCanonical(): Promise; load(): Promise; flush(): Promise; close(): Promise; @@ -99,382 +85,6 @@ export class TelemetryRepoPublicationError extends Error { } } -export function createTelemetryRepo( - workspaceRoot: string, - options: CreateTelemetryRepoOptions = {}, -): TelemetryRepo { - return new FileTelemetryRepo( - workspaceRoot, - options.createIfMissing ?? true, - options.managePricing ?? true, - ); -} - -class FileTelemetryRepo implements TelemetryRepo { - private readonly path: string; - private file: TelemetryFile = emptyTelemetryFile(); - private legacyPricing: readonly unknown[] = []; - private requiresCanonicalPublication = false; - private pricingStore: PricingStore | undefined; - private loaded = false; - private queue: Promise = Promise.resolve(); - private failure: TelemetryRepoPublicationError | undefined; - private state: 'open' | 'draining' | 'closed' = 'open'; - private loadPromise: Promise | undefined; - private closePromise: Promise | undefined; - - constructor( - workspaceRoot: string, - private readonly createIfMissing: boolean, - private readonly managePricing: boolean, - ) { - this.path = join(workspaceRoot, 'telemetry.json'); - } - - load(): Promise { - if (this.loaded) return Promise.resolve(); - this.assertOpen(); - if (this.loadPromise) return this.loadPromise; - const operation = this.loadFromDisk(); - this.loadPromise = operation; - void operation.catch(() => { - if (this.state === 'open' && this.loadPromise === operation) { - this.loadPromise = undefined; - } - }); - return operation; - } - - private async loadFromDisk(): Promise { - let missing = false; - let file: TelemetryFile; - let legacyPricing: readonly unknown[]; - let requiresCanonicalPublication: boolean; - let pricingStore: PricingStore | undefined; - try { - const decoded = decodeTelemetryFile(JSON.parse(await readFile(this.path, 'utf8'))); - file = decoded.file; - legacyPricing = decoded.legacyPricingOverrides; - requiresCanonicalPublication = decoded.requiresCanonicalPublication; - } catch (error) { - if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; - missing = true; - file = emptyTelemetryFile(); - legacyPricing = []; - requiresCanonicalPublication = false; - } - - if (this.managePricing) { - pricingStore = createPricingStore(dirname(this.path), { - createIfMissing: this.createIfMissing, - initialOverrides: legacyPricing, - }); - await pricingStore.load(); - } - if (this.createIfMissing && (missing || (this.managePricing && requiresCanonicalPublication))) { - await this.publish(file); - requiresCanonicalPublication = false; - } - - this.file = file; - this.legacyPricing = legacyPricing; - this.requiresCanonicalPublication = requiresCanonicalPublication; - this.pricingStore = pricingStore; - this.loaded = true; - } - - insertLlmCall(record: PersistedLlmCallRecord): Promise { - let admitted: PersistedLlmCallRecord; - try { - admitted = decodePersistedLlmCallRecord(record); - } catch (error) { - return Promise.reject(error); - } - return this.enqueueMutation((file) => ({ - ...file, - usageRecords: upsertById(file.usageRecords, admitted), - })); - } - - insertToolInvocation(record: PersistedToolInvocationRecord): Promise { - let admitted: PersistedToolInvocationRecord; - try { - admitted = decodePersistedToolInvocationRecord(record); - } catch (error) { - return Promise.reject(error); - } - return this.enqueueMutation((file) => ({ - ...file, - toolInvocations: upsertById(file.toolInvocations, admitted), - })); - } - - summary(query: UsageQuery): UsageSummaryV2 { - this.assertReady(); - const { from, to } = resolveRange(query.range); - const rows = this.filteredUsageRows(query, from, to); - return detached({ - range: { from, to }, - totalRequests: rows.length, - totalCostUsd: sum(rows.map((row) => row.costUsd)), - totalTokens: { - input: sum(rows.map((row) => row.inputTokens)), - output: sum(rows.map((row) => row.outputTokens)), - cacheMiss: sum(rows.map((row) => row.cacheMissInputTokens)), - cacheRead: sum(rows.map((row) => row.cacheHitInputTokens)), - cacheWrite: sum(rows.map((row) => row.cacheWriteInputTokens)), - reasoning: sum(rows.map((row) => row.reasoningTokens)), - total: sum(rows.map((row) => row.totalTokens)), - }, - cacheHitRequests: rows.filter((row) => row.cacheHitInputTokens > 0).length, - cacheCreateRequests: rows.filter((row) => row.cacheWriteInputTokens > 0).length, - errorRequests: rows.filter((row) => row.status === 'error').length, - }); - } - - buckets(query: UsageQuery, groupBy: UsageGroupBy): UsageBucket[] { - this.assertReady(); - const { from, to } = resolveRange(query.range); - if (groupBy === 'tool') { - return detached(toolBuckets(this.filteredToolRows(query, from, to))); - } - const groups = new Map(); - for (const row of this.filteredUsageRows(query, from, to)) { - const key = usageBucketKey(row, groupBy); - let group = groups.get(key); - if (!group) { - group = []; - groups.set(key, group); - } - group.push(row); - } - return detached( - [...groups.entries()] - .map(([key, rows]) => usageBucket(key, rows)) - .sort((left, right) => right.requests - left.requests), - ); - } - - logs(query: UsageQuery, offset = 0, limit = 100): { rows: UsageLogRow[]; total: number } { - this.assertReady(); - if (query.toolName !== undefined) { - throw new TelemetryQueryValidationError('toolName is not applicable to LLM logs'); - } - const { from, to } = resolveRange(query.range); - const rows = this.filteredUsageRows(query, from, to).sort((left, right) => right.ts - left.ts); - const total = rows.length; - return detached({ - rows: rows.slice(offset, offset + limit).map(toUsageLogRow), - total, - }); - } - - toolLogs( - query: ToolUsageQuery, - offset = 0, - limit = 100, - ): { rows: PersistedToolInvocationRecord[]; total: number } { - this.assertReady(); - assertToolUsageQuery(query); - const { from, to } = resolveRange(query.range); - const rows = this.filteredToolRows(query, from, to).sort((left, right) => right.ts - left.ts); - return detached({ rows: rows.slice(offset, offset + limit), total: rows.length }); - } - - latestLlmRuntimeProbe(connectionSlug: string, modelId?: string): UsageLogRow | undefined { - return this.logs({ range: 'all', connectionSlug, ...(modelId ? { modelId } : {}) }, 0, 1) - .rows[0]; - } - - listPricingOverrides(): PricingConfig[] { - this.assertReady(); - if (this.pricingStore) - return this.pricingStore.snapshot().overrides.map((item) => ({ ...item })); - return []; - } - - async upsertPricing(pricing: PricingConfig): Promise { - const store = this.requireManagedPricing(); - const snapshot = store.snapshot(); - await store.upsert(snapshot.revision, pricing); - } - - async deletePricing(modelKey: string): Promise { - const store = this.requireManagedPricing(); - const snapshot = store.snapshot(); - await store.delete(snapshot.revision, modelKey); - } - - legacyPricingOverrides(): readonly unknown[] { - this.assertReady(); - return structuredClone(this.legacyPricing); - } - - publishCanonical(): Promise { - this.assertReady(); - if (!this.requiresCanonicalPublication) return this.flush(); - return this.enqueueMutation((file) => file); - } - - async flush(): Promise { - this.assertLoaded(); - await this.queue; - if (this.failure) throw this.failure; - } - - close(): Promise { - if (this.closePromise) return this.closePromise; - this.state = 'draining'; - this.closePromise = this.closeResources().finally(() => { - this.state = 'closed'; - }); - return this.closePromise; - } - - private async closeResources(): Promise { - const settled = await Promise.allSettled([ - this.loadPromise ?? Promise.resolve(), - this.queue.then(() => { - if (this.failure) throw this.failure; - }), - ]); - const pricingResult = await Promise.allSettled([ - this.pricingStore?.close() ?? Promise.resolve(), - ]); - throwDeduplicatedFailures('Unable to close telemetry repository', [ - ...settled.flatMap((result) => (result.status === 'rejected' ? [result.reason] : [])), - ...pricingResult.flatMap((result) => (result.status === 'rejected' ? [result.reason] : [])), - ]); - } - - private enqueueMutation(mutate: (file: TelemetryFile) => TelemetryFile): Promise { - this.assertReady(); - const operation = this.queue.then(async () => { - if (this.failure) throw this.failure; - const candidate = mutate(this.file); - try { - await this.publish(candidate); - } catch (error) { - const failure = - error instanceof TelemetryRepoPublicationError - ? error - : new TelemetryRepoPublicationError(false, { cause: error }); - this.failure = failure; - throw failure; - } - this.file = candidate; - this.requiresCanonicalPublication = false; - }); - this.queue = operation.catch(() => undefined); - return operation; - } - - private async publish(file: TelemetryFile): Promise { - await mkdir(dirname(this.path), { recursive: true }); - const temporaryPath = `${this.path}.${randomUUID()}.tmp`; - let handle: Awaited> | undefined; - let published = false; - try { - handle = await open(temporaryPath, 'wx', 0o600); - await handle.writeFile(JSON.stringify(file, null, 2) + '\n', 'utf8'); - await handle.sync(); - await handle.close(); - handle = undefined; - await rename(temporaryPath, this.path); - published = true; - await syncDirectory(dirname(this.path)); - } catch (cause) { - throw new TelemetryRepoPublicationError(published, { cause }); - } finally { - await handle?.close().catch(() => undefined); - if (!published) await rm(temporaryPath, { force: true }).catch(() => undefined); - } - } - - private filteredUsageRows(query: UsageQuery, from: number, to: number) { - return this.file.usageRecords.filter((row) => { - if (row.ts < from || row.ts > to) return false; - if (query.connectionSlug && row.connectionSlug !== query.connectionSlug) return false; - if (query.providerId && row.providerId !== query.providerId) return false; - if (query.modelId && row.modelId !== query.modelId) return false; - if (query.status && query.status !== 'all' && row.status !== query.status) return false; - return true; - }); - } - - private filteredToolRows(query: UsageQuery, from: number, to: number) { - return this.file.toolInvocations.filter((row) => { - if (row.ts < from || row.ts > to) return false; - if (query.toolName && row.toolName !== query.toolName) return false; - if (query.status && query.status !== 'all' && row.status !== query.status) return false; - return true; - }); - } - - private requireManagedPricing(): PricingStore { - this.assertReady(); - if (!this.pricingStore) { - throw new Error('Telemetry repository does not own the compatibility pricing facade'); - } - return this.pricingStore; - } - - private assertLoaded(): void { - if (!this.loaded) throw new TelemetryRepoNotLoadedError(); - } - - private assertOpen(): void { - if (this.state !== 'open') throw new TelemetryRepoClosedError(); - } - - private assertReady(): void { - this.assertOpen(); - this.assertLoaded(); - if (this.failure?.commitUnknown) throw this.failure; - } -} - -function toUsageLogRow(row: PersistedLlmCallRecord): UsageLogRow { - return { - id: row.id, - ts: row.ts, - ...(row.callKind ? { callKind: row.callKind } : {}), - ...(row.callId ? { callId: row.callId } : {}), - ...(row.connectionSlug ? { connectionSlug: row.connectionSlug } : {}), - providerId: row.providerId, - modelId: row.modelId, - inputTokens: row.inputTokens, - outputTokens: row.outputTokens, - cacheMissTokens: row.cacheMissInputTokens, - cacheReadTokens: row.cacheHitInputTokens, - cacheWriteTokens: row.cacheWriteInputTokens, - ...(row.cacheMissInputSource ? { cacheMissInputSource: row.cacheMissInputSource } : {}), - reasoningTokens: row.reasoningTokens, - totalTokens: row.totalTokens, - costUsd: row.costUsd, - latencyMs: row.latencyMs, - status: row.status, - ...(row.errorClass ? { errorClass: row.errorClass } : {}), - ...(row.sessionId ? { sessionId: row.sessionId } : {}), - ...(row.turnId ? { turnId: row.turnId } : {}), - ...(row.systemPromptHash ? { systemPromptHash: row.systemPromptHash } : {}), - ...(row.prefixHash ? { prefixHash: row.prefixHash } : {}), - ...(row.prefixChangeReason ? { prefixChangeReason: row.prefixChangeReason } : {}), - ...(row.requestShapeHash ? { requestShapeHash: row.requestShapeHash } : {}), - ...(row.requestShapeChangeReason - ? { requestShapeChangeReason: row.requestShapeChangeReason } - : {}), - ...(row.toolSchemaChangeReason ? { toolSchemaChangeReason: row.toolSchemaChangeReason } : {}), - ...(row.toolAvailability ? { toolAvailability: row.toolAvailability } : {}), - ...(row.promptSegments ? { promptSegments: row.promptSegments } : {}), - ...(row.contextBudget ? { contextBudget: row.contextBudget } : {}), - }; -} - -function upsertById(rows: readonly T[], row: T): T[] { - return [...rows.filter((current) => current.id !== row.id), row]; -} - export function resolveRange(range: UsageQuery['range']): { from: number; to: number } { if (typeof range === 'object') return range; const now = Date.now(); @@ -489,79 +99,3 @@ export function resolveRange(range: UsageQuery['range']): { from: number; to: nu return { from: 0, to: now }; } } - -function usageBucket(key: string, rows: readonly PersistedLlmCallRecord[]): UsageBucket { - const errors = rows.filter((row) => row.status === 'error').length; - return { - key, - label: key, - requests: rows.length, - inputTokens: sum(rows.map((row) => row.inputTokens)), - outputTokens: sum(rows.map((row) => row.outputTokens)), - cacheMissTokens: sum(rows.map((row) => row.cacheMissInputTokens)), - cacheReadTokens: sum(rows.map((row) => row.cacheHitInputTokens)), - cacheWriteTokens: sum(rows.map((row) => row.cacheWriteInputTokens)), - reasoningTokens: sum(rows.map((row) => row.reasoningTokens)), - totalTokens: sum(rows.map((row) => row.totalTokens)), - costUsd: sum(rows.map((row) => row.costUsd)), - avgLatencyMs: rows.length ? Math.round(sum(rows.map((row) => row.latencyMs)) / rows.length) : 0, - errorRate: rows.length ? errors / rows.length : 0, - }; -} - -function toolBuckets(rows: readonly PersistedToolInvocationRecord[]): UsageBucket[] { - const groups = new Map(); - for (const row of rows) { - let group = groups.get(row.toolName); - if (!group) { - group = []; - groups.set(row.toolName, group); - } - group.push(row); - } - return [...groups.entries()] - .map(([key, group]) => { - const errors = group.filter((row) => row.status === 'error').length; - const bytesIn = sum(group.map((row) => row.bytesIn)); - const bytesOut = sum(group.map((row) => row.bytesOut)); - return { - key, - label: key, - requests: group.length, - inputTokens: bytesIn, - outputTokens: bytesOut, - cacheMissTokens: 0, - cacheReadTokens: 0, - cacheWriteTokens: 0, - reasoningTokens: 0, - totalTokens: bytesIn + bytesOut, - costUsd: 0, - avgLatencyMs: group.length - ? Math.round(sum(group.map((row) => row.durationMs)) / group.length) - : 0, - errorRate: group.length ? errors / group.length : 0, - }; - }) - .sort((left, right) => right.requests - left.requests); -} - -function sum(values: readonly number[]): number { - return values.reduce((total, value) => total + value, 0); -} - -function assertToolUsageQuery(query: ToolUsageQuery): void { - const keys = Object.keys(query); - if (keys.some((key) => !['range', 'toolName', 'status'].includes(key))) { - throw new TelemetryQueryValidationError('tool logs accept only range, toolName, and status'); - } -} - -function detached(value: T): T { - return deepFreeze(structuredClone(value)); -} - -function deepFreeze(value: T): T { - if (typeof value !== 'object' || value === null || Object.isFrozen(value)) return value; - for (const nested of Object.values(value)) deepFreeze(nested); - return Object.freeze(value); -} diff --git a/packages/storage/src/usage-stats-store.ts b/packages/storage/src/usage-stats-store.ts index d5b0da829f..e83e4d2093 100644 --- a/packages/storage/src/usage-stats-store.ts +++ b/packages/storage/src/usage-stats-store.ts @@ -1,4 +1,3 @@ -import { readFile, readdir, stat } from 'node:fs/promises'; import { join } from 'node:path'; import type { UsageRange, UsageStats } from '@maka/core'; import type { SessionHeader } from '@maka/core/session'; @@ -120,67 +119,28 @@ export async function readUsageStats( async function readStoredSessions( workspaceRoot: string, ): Promise> { - const sessionsRoot = join(workspaceRoot, 'sessions'); + const metadata = createSqliteSessionMetadataStore( + join(workspaceRoot, OPERATIONAL_STATE_DATABASE_NAME), + ); try { - const canonicalHeaders = await readCanonicalUsageHeaders(workspaceRoot); - const entries = await readdir(sessionsRoot, { withFileTypes: true }); const sessions: Array<{ header: UsageSessionHeader; messages: UsageMessage[] }> = []; - for (const entry of entries) { - if (!entry.isDirectory()) continue; - try { - const text = await readFile(join(sessionsRoot, entry.name, 'session.jsonl'), 'utf8'); - const lines = text.split('\n').filter((line) => line.trim()); - if (!lines[0]) continue; - const header = - canonicalHeaders === null - ? normalizeUsageSessionHeader(JSON.parse(lines[0]), entry.name) - : canonicalHeaders.get(entry.name); - if (!header) continue; - const messages: UsageMessage[] = []; - for (const line of lines.slice(1)) { - try { - const message = normalizeUsageMessage(JSON.parse(line)); - if (message) messages.push(message); - } catch { - // A partially-written/corrupt message line must not hide valid usage rows from the same session. - } - } - sessions.push({ - header, - messages, - }); - } catch { - // Ignore partially-written or legacy session folders. - } - } - return sessions; - } catch { - return []; - } -} - -async function readCanonicalUsageHeaders( - workspaceRoot: string, -): Promise | null> { - const path = join(workspaceRoot, OPERATIONAL_STATE_DATABASE_NAME); - try { - await stat(path); - } catch (error) { - if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null; - throw error; - } - const metadata = createSqliteSessionMetadataStore(path); - try { - return new Map( - (await metadata.list()).map(({ header }) => [ - header.id, - { + for (const { header } of await metadata.list()) { + const messages = (await metadata.readMessages(header.id)).flatMap((value) => { + const message = normalizeUsageMessage(value); + return message ? [message] : []; + }); + sessions.push({ + header: { id: header.id, llmConnectionSlug: header.llmConnectionSlug, model: header.model, }, - ]), - ); + messages, + }); + } + return sessions; + } catch { + return []; } finally { metadata.close(); } diff --git a/packages/storage/src/usage-stores.ts b/packages/storage/src/usage-stores.ts index 87493b99ef..a53620cb81 100644 --- a/packages/storage/src/usage-stores.ts +++ b/packages/storage/src/usage-stores.ts @@ -288,10 +288,7 @@ async function openRepos( const telemetry = createSqliteTelemetryRepo(root, { createIfMissing, managePricing: false }); await telemetry.load(); const modelCalls = createSqliteModelCallLedger(root); - const pricing = createSqlitePricingStore(root, { - createIfMissing, - initialOverrides: telemetry.legacyPricingOverrides(), - }); + const pricing = createSqlitePricingStore(root, { createIfMissing }); try { await pricing.load(); return { telemetry, modelCalls, pricing }; From de2a4b3458143872789561804adcdd3beb42e8c5 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 3 Aug 2026 20:30:44 +0800 Subject: [PATCH 02/10] fix(storage): seed usage pricing authority singleton on fresh workspaces The migration created usage_pricing_authority but never inserted the singleton row; the deleted importLegacyUsageState used to do it. Every fresh workspace then threw PricingValidationError from readPricingSnapshot, breaking createHostAiSdkBackend, ensureUsageReady, and the first send. Mirror the automation authority seeding (INSERT OR IGNORE revision 0). --- packages/storage/src/sqlite-usage-schema.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/packages/storage/src/sqlite-usage-schema.ts b/packages/storage/src/sqlite-usage-schema.ts index 1c5a31bc2e..8bebf2a256 100644 --- a/packages/storage/src/sqlite-usage-schema.ts +++ b/packages/storage/src/sqlite-usage-schema.ts @@ -52,6 +52,9 @@ export function migrateSqliteUsageDatabase(db: DatabaseSync): void { revision INTEGER NOT NULL CHECK (revision >= 0) ); + INSERT OR IGNORE INTO usage_pricing_authority(singleton, revision) + VALUES (1, 0); + CREATE TABLE IF NOT EXISTS usage_pricing_overrides ( model_key TEXT PRIMARY KEY, record_json TEXT NOT NULL From 195ffbd4a2bc9b6e5149db570700e2e554e8d5fd Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 3 Aug 2026 20:30:44 +0800 Subject: [PATCH 03/10] fix(storage): restore ENOENT code on missing ShellRun reads The SQLite not-found error was a plain Error, so callers gated on isNotFoundError (code ENOENT) surfaced the raw storage message and echoed the id back. Match agent-run-store's ENOENT-coded error. --- packages/storage/src/shell-run-store.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/storage/src/shell-run-store.ts b/packages/storage/src/shell-run-store.ts index 0ade1c5fc1..3af623ffcc 100644 --- a/packages/storage/src/shell-run-store.ts +++ b/packages/storage/src/shell-run-store.ts @@ -173,7 +173,11 @@ function readSqliteShellRun( WHERE session_id = ? AND shell_run_id = ? `) .get(sessionId, shellRunId) as { record_json?: unknown } | undefined; - if (!row) throw new Error(`ShellRun does not exist: ${shellRunId}`); + if (!row) { + const error = new Error(`ShellRun does not exist: ${shellRunId}`) as Error & { code?: string }; + error.code = 'ENOENT'; + throw error; + } if (typeof row.record_json !== 'string') throw new Error('Invalid SQLite ShellRun row'); return normalizeShellRunRecord(JSON.parse(row.record_json), sessionId, shellRunId); } From 9dcd5108c7e300cc09188fc7ffb6341c6115605e Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 3 Aug 2026 20:30:52 +0800 Subject: [PATCH 04/10] fix(storage): sync legacy-schema rewind fixtures with migrations 8 and 20 Migration 20 (session_messages) and migration 8 (headless_task_run_events) are plain CREATE TABLE, so rewind fixtures that rebuild an old version from a current database must drop them; the runtime schema mock also has to report version 8 as current. Fixes the 14 'table already exists' failures in the legacy-schema simulation suite. --- .../storage/src/__tests__/agent-graph-intent-claims.test.ts | 1 + .../src/__tests__/agent-graph-supervisor-wakes.test.ts | 1 + .../src/__tests__/recovery-persistence-authority.test.ts | 4 ++-- .../storage/src/__tests__/sqlite-recovery-concurrency.test.ts | 2 +- packages/storage/src/__tests__/sqlite-runtime-schema.test.ts | 2 +- packages/storage/src/__tests__/sqlite-runtime-store.test.ts | 1 + .../src/__tests__/sqlite-session-metadata-store.test.ts | 3 +++ 7 files changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/storage/src/__tests__/agent-graph-intent-claims.test.ts b/packages/storage/src/__tests__/agent-graph-intent-claims.test.ts index 1a9064ee62..724ff457e1 100644 --- a/packages/storage/src/__tests__/agent-graph-intent-claims.test.ts +++ b/packages/storage/src/__tests__/agent-graph-intent-claims.test.ts @@ -141,6 +141,7 @@ describe('SQLite agent graph intent claims', () => { ALTER TABLE session_metadata_tombstones DROP COLUMN retirement_unit_id; DROP TABLE session_create_claims; DROP TABLE sandbox_boundary_log; + DROP TABLE session_messages; DROP TABLE agent_graph_supervisor_wake_attempts; DROP TABLE agent_graph_supervisor_wakes; DROP TABLE agent_graph_client_applied_records; diff --git a/packages/storage/src/__tests__/agent-graph-supervisor-wakes.test.ts b/packages/storage/src/__tests__/agent-graph-supervisor-wakes.test.ts index ba4226ddeb..f09c6c7eb5 100644 --- a/packages/storage/src/__tests__/agent-graph-supervisor-wakes.test.ts +++ b/packages/storage/src/__tests__/agent-graph-supervisor-wakes.test.ts @@ -228,6 +228,7 @@ describe('SQLite Agent Graph supervisor wakes', () => { ALTER TABLE session_metadata_tombstones DROP COLUMN retirement_unit_id; DROP TABLE session_create_claims; DROP TABLE sandbox_boundary_log; + DROP TABLE session_messages; `); v11 .prepare(`UPDATE session_metadata_schema SET version = 11 WHERE scope = 'session_metadata'`) diff --git a/packages/storage/src/__tests__/recovery-persistence-authority.test.ts b/packages/storage/src/__tests__/recovery-persistence-authority.test.ts index a53d13e9f5..1f86dcf940 100644 --- a/packages/storage/src/__tests__/recovery-persistence-authority.test.ts +++ b/packages/storage/src/__tests__/recovery-persistence-authority.test.ts @@ -91,7 +91,7 @@ describe('SQLite recovery persistence authority', () => { dispatch.ts, ); db.exec( - 'DROP TABLE runtime_workspace_heads; DROP TABLE runtime_workspace_versions; DROP TABLE runtime_workspace_epochs; DROP TABLE runtime_continuation_claims; DROP TABLE runtime_capabilities; PRAGMA user_version = 4;', + 'DROP TABLE runtime_workspace_heads; DROP TABLE runtime_workspace_versions; DROP TABLE runtime_workspace_epochs; DROP TABLE headless_task_run_events; DROP TABLE runtime_continuation_claims; DROP TABLE runtime_capabilities; PRAGMA user_version = 4;', ); db.close(); @@ -187,7 +187,7 @@ describe('SQLite recovery persistence authority', () => { 2, ); db.exec( - 'DROP TABLE runtime_workspace_heads; DROP TABLE runtime_workspace_versions; DROP TABLE runtime_workspace_epochs; DROP TABLE runtime_continuation_claims; DROP TABLE runtime_capabilities; PRAGMA user_version = 4;', + 'DROP TABLE runtime_workspace_heads; DROP TABLE runtime_workspace_versions; DROP TABLE runtime_workspace_epochs; DROP TABLE headless_task_run_events; DROP TABLE runtime_continuation_claims; DROP TABLE runtime_capabilities; PRAGMA user_version = 4;', ); db.close(); diff --git a/packages/storage/src/__tests__/sqlite-recovery-concurrency.test.ts b/packages/storage/src/__tests__/sqlite-recovery-concurrency.test.ts index 38ff83f867..1e9ca3c600 100644 --- a/packages/storage/src/__tests__/sqlite-recovery-concurrency.test.ts +++ b/packages/storage/src/__tests__/sqlite-recovery-concurrency.test.ts @@ -220,7 +220,7 @@ describe('SQLite recovery authority multi-process races', () => { const db = new DatabaseSync(dbPath); try { db.exec( - "DROP TABLE runtime_workspace_heads; DROP TABLE runtime_workspace_versions; DROP TABLE runtime_workspace_epochs; DELETE FROM runtime_capabilities WHERE capability = 'runtime_workspace_version_authority'; PRAGMA user_version = 6;", + "DROP TABLE runtime_workspace_heads; DROP TABLE runtime_workspace_versions; DROP TABLE runtime_workspace_epochs; DROP TABLE headless_task_run_events; DELETE FROM runtime_capabilities WHERE capability = 'runtime_workspace_version_authority'; PRAGMA user_version = 6;", ); } finally { db.close(); diff --git a/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts b/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts index 75b19ed7ec..11079d2740 100644 --- a/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts +++ b/packages/storage/src/__tests__/sqlite-runtime-schema.test.ts @@ -13,7 +13,7 @@ describe('SQLite runtime schema migration', () => { return { get() { versionReads += 1; - return { user_version: versionReads === 1 ? 4 : 7 }; + return { user_version: versionReads === 1 ? 4 : 8 }; }, }; }, diff --git a/packages/storage/src/__tests__/sqlite-runtime-store.test.ts b/packages/storage/src/__tests__/sqlite-runtime-store.test.ts index 670bab904d..f4880a21ad 100644 --- a/packages/storage/src/__tests__/sqlite-runtime-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-runtime-store.test.ts @@ -51,6 +51,7 @@ describe('SqliteRuntimeStore', () => { DROP TABLE runtime_workspace_heads; DROP TABLE runtime_workspace_versions; DROP TABLE runtime_workspace_epochs; + DROP TABLE headless_task_run_events; DELETE FROM runtime_capabilities WHERE capability = 'runtime_workspace_version_authority'; PRAGMA user_version = 6; diff --git a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts index 4f04b89197..b2a90012a4 100644 --- a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts @@ -236,6 +236,7 @@ describe('SqliteSessionMetadataStore', () => { ALTER TABLE session_metadata_tombstones DROP COLUMN retirement_unit_id; DROP TABLE session_create_claims; DROP TABLE sandbox_boundary_log; + DROP TABLE session_messages; UPDATE session_metadata_schema SET version = 12 WHERE scope = 'session_metadata'; @@ -997,6 +998,7 @@ describe('SqliteSessionMetadataStore', () => { ALTER TABLE session_metadata_tombstones DROP COLUMN cleanup_pending; ALTER TABLE session_metadata_tombstones DROP COLUMN retirement_unit_id; DROP TABLE session_create_claims; + DROP TABLE session_messages; ALTER TABLE sandbox_boundary_log DROP COLUMN turn_id; ALTER TABLE sandbox_boundary_log DROP COLUMN run_id; DROP INDEX sandbox_boundary_log_settled_closures; @@ -1505,6 +1507,7 @@ describe('SqliteSessionMetadataStore', () => { ALTER TABLE session_metadata_tombstones DROP COLUMN retirement_unit_id; DROP TABLE session_create_claims; DROP TABLE sandbox_boundary_log; + DROP TABLE session_messages; DROP TABLE agent_graph_supervisor_wake_attempts; DROP TABLE agent_graph_supervisor_wakes; DROP TABLE agent_graph_client_applied_records; From 707d8c0358fd4ade515e36507c7ed820e97f283a Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 3 Aug 2026 20:30:52 +0800 Subject: [PATCH 05/10] test(storage): drop stale legacy-import and file-path assertions usage-stores and the desktop usage-ipc test still seeded telemetry.json and asserted the deleted legacy import; the claimed-graph admission test read a turn-admissions file the SQLite store never writes. Remove the assertions, keep the surviving IPC serialization coverage. --- .../src/main/__tests__/usage-ipc-main.test.ts | 26 ++--- ...claimed-agent-graph-root-admission.test.ts | 10 +- .../src/__tests__/usage-stores.test.ts | 110 ------------------ 3 files changed, 9 insertions(+), 137 deletions(-) diff --git a/apps/desktop/src/main/__tests__/usage-ipc-main.test.ts b/apps/desktop/src/main/__tests__/usage-ipc-main.test.ts index e0a7820f94..510c9eddf3 100644 --- a/apps/desktop/src/main/__tests__/usage-ipc-main.test.ts +++ b/apps/desktop/src/main/__tests__/usage-ipc-main.test.ts @@ -72,18 +72,8 @@ test('usage IPC leaves settings usage session-derived while detailed usage waits } }); -test('pricing IPC mutations serialize through the canonical SQLite repo after legacy import', async () => { +test('pricing IPC mutations serialize through the canonical SQLite repo', async () => { const root = await mkdtemp(join(tmpdir(), 'maka-usage-ipc-pricing-')); - const legacy = pricing('openai:legacy'); - await writeFile( - join(root, 'telemetry.json'), - JSON.stringify({ - usageRecords: [], - toolInvocations: [], - pricingOverrides: [legacy], - }), - 'utf8', - ); const telemetryRepo = createSqliteTelemetryRepo(root); const handlers = new Map(); const firstWrite = deferred(); @@ -133,11 +123,11 @@ test('pricing IPC mutations serialize through the canonical SQLite repo after le assert.ok(list); assert.ok(put); assert.ok(reset); - assert.deepEqual(await list({}), { ok: true, data: [legacy] }); + assert.deepEqual(await list({}), { ok: true, data: [] }); const first = put({}, pricing('openai:first')); const second = put({}, pricing('openai:second')); - const third = reset({}, 'openai:legacy'); + const third = reset({}, 'openai:first'); await firstWriteStarted.promise; assert.deepEqual(events, ['upsert:openai:first']); @@ -147,18 +137,18 @@ test('pricing IPC mutations serialize through the canonical SQLite repo after le assert.equal((await third).ok, true); assert.deepEqual( telemetryRepo.listPricingOverrides().map((item) => item.modelKey), - ['openai:first', 'openai:second'], + ['openai:second'], ); assert.deepEqual(events, [ 'upsert:openai:first', - 'refresh:2', + 'refresh:1', 'notify:usage:pricing:changed', 'upsert:openai:second', - 'refresh:3', - 'notify:usage:pricing:changed', - 'delete:openai:legacy', 'refresh:2', 'notify:usage:pricing:changed', + 'delete:openai:first', + 'refresh:1', + 'notify:usage:pricing:changed', ]); } finally { await telemetryRepo.close().catch(() => undefined); diff --git a/packages/storage/src/__tests__/claimed-agent-graph-root-admission.test.ts b/packages/storage/src/__tests__/claimed-agent-graph-root-admission.test.ts index 2a7a8fdaa1..715f2ac377 100644 --- a/packages/storage/src/__tests__/claimed-agent-graph-root-admission.test.ts +++ b/packages/storage/src/__tests__/claimed-agent-graph-root-admission.test.ts @@ -1,5 +1,5 @@ import assert from 'node:assert/strict'; -import { access, mkdtemp, readFile, rm } from 'node:fs/promises'; +import { access, mkdtemp, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { describe, test } from 'node:test'; @@ -28,14 +28,6 @@ describe('claimed agent graph root admission', () => { if (stored?.execution.kind === 'claimed_agent_graph_intent') { assert.equal(Object.isFrozen(stored.execution.claim), true); } - - const serialized = JSON.parse( - await readFile( - join(root, 'sessions', 'session-child', 'turn-admissions', 'turn-next.json'), - 'utf8', - ), - ) as { execution: unknown }; - assert.deepEqual(serialized.execution, admissionInput().execution); }); }); diff --git a/packages/storage/src/__tests__/usage-stores.test.ts b/packages/storage/src/__tests__/usage-stores.test.ts index d4e4efafb8..c1159230de 100644 --- a/packages/storage/src/__tests__/usage-stores.test.ts +++ b/packages/storage/src/__tests__/usage-stores.test.ts @@ -192,112 +192,6 @@ describe('InteractiveUsageStores', () => { } }); - test('migrates legacy usage, tools, and pricing idempotently', async () => { - await withInteractiveRoot(async ({ root, capability }) => { - const legacy = { - usageRecords: [llmRecord({ cachedInputTokens: 3, cacheHitInputTokens: undefined })], - toolInvocations: [toolRecord()], - pricingOverrides: [pricing('openai:gpt-5')], - }; - const legacyBytes = JSON.stringify(legacy, null, 2) + '\n'; - await writeFile(join(root, 'telemetry.json'), legacyBytes); - - const owner = await tryAcquireInteractiveRootOwner(capability); - assert(owner); - const stores = await openInteractiveUsageStoresForWrite(owner.lease); - assert.equal((await stores.telemetry.logs({ range: 'all' })).total, 1); - assert.deepEqual(await stores.pricing.snapshot(), { - revision: 0, - overrides: [pricing('openai:gpt-5')], - }); - await stores.close(); - await owner.close(); - - assert.equal(await readFile(join(root, 'telemetry.json'), 'utf8'), legacyBytes); - await assert.rejects( - () => readFile(join(root, 'pricing.json'), 'utf8'), - (error: NodeJS.ErrnoException) => error.code === 'ENOENT', - ); - - const successor = await tryAcquireInteractiveRootOwner(capability); - assert(successor); - const reopened = await openInteractiveUsageStoresForWrite(successor.lease); - assert.equal((await reopened.telemetry.buckets({ range: 'all' }, 'tool'))[0]?.requests, 1); - assert.deepEqual(await reopened.pricing.snapshot(), { - revision: 0, - overrides: [pricing('openai:gpt-5')], - }); - await reopened.close(); - await successor.close(); - assert.equal(await readFile(join(root, 'telemetry.json'), 'utf8'), legacyBytes); - await assert.rejects( - () => readFile(join(root, 'pricing.json'), 'utf8'), - (error: NodeJS.ErrnoException) => error.code === 'ENOENT', - ); - }); - }); - - test('an existing pricing authority wins over stale embedded overrides', async () => { - await withInteractiveRoot(async ({ root, capability }) => { - await writeFile( - join(root, 'telemetry.json'), - JSON.stringify({ - usageRecords: [], - toolInvocations: [], - pricingOverrides: [{ modelKey: '', inputUsdPer1M: -1 }], - }), - ); - await writeFile( - join(root, 'pricing.json'), - JSON.stringify({ - version: 1, - revision: 7, - overrides: [pricing('current:model')], - }), - ); - const owner = await tryAcquireInteractiveRootOwner(capability); - assert(owner); - const stores = await openInteractiveUsageStoresForWrite(owner.lease); - assert.deepEqual(await stores.pricing.snapshot(), { - revision: 7, - overrides: [pricing('current:model')], - }); - await stores.close(); - await owner.close(); - }); - }); - - test('migrates and reopens more than 128 legacy pricing overrides without loss', async () => { - await withInteractiveRoot(async ({ root, capability }) => { - const overrides = Array.from({ length: 140 }, (_, index) => - pricing(`provider:model-${String(139 - index).padStart(3, '0')}`), - ); - await writeFile( - join(root, 'telemetry.json'), - JSON.stringify({ usageRecords: [], toolInvocations: [], pricingOverrides: overrides }), - ); - - const owner = await tryAcquireInteractiveRootOwner(capability); - assert(owner); - const stores = await openInteractiveUsageStoresForWrite(owner.lease); - const migrated = await stores.pricing.snapshot(); - assert.equal(migrated.overrides.length, 140); - assert.deepEqual( - migrated.overrides.map((item) => item.modelKey), - overrides.map((item) => item.modelKey).sort(), - ); - await stores.close(); - await owner.close(); - - const successor = await tryAcquireInteractiveRootOwner(capability); - assert(successor); - const reopened = await openInteractiveUsageStoresForWrite(successor.lease); - assert.deepEqual(await reopened.pricing.snapshot(), migrated); - await reopened.close(); - await successor.close(); - }); - }); - test('drain waits accepted writes and rejects new admission', async () => { await withInteractiveRoot(async ({ root, capability }) => { const owner = await tryAcquireInteractiveRootOwner(capability); @@ -428,7 +322,3 @@ function toolRecord() { >['telemetry']['recordToolInvocation'] >[0]; } - -function pricing(modelKey: string) { - return { modelKey, inputUsdPer1M: 1.25, outputUsdPer1M: 10 }; -} From 3d1f9544a21464f5594016241ab6b28eaef02607 Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 3 Aug 2026 20:30:58 +0800 Subject: [PATCH 06/10] test(runtime): adapt shell-run startup-failure test to the SQLite store The old mechanism blocked the file store by writing a file at /sessions; the SQLite store ignores it. Fail the first durable ShellRun create through the wrapper store instead, keeping the slot-release assertions. --- .../src/__tests__/shell-run-manager.test.ts | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/packages/runtime/src/__tests__/shell-run-manager.test.ts b/packages/runtime/src/__tests__/shell-run-manager.test.ts index 5ac614a8fd..f73e9c6b2e 100644 --- a/packages/runtime/src/__tests__/shell-run-manager.test.ts +++ b/packages/runtime/src/__tests__/shell-run-manager.test.ts @@ -2094,10 +2094,23 @@ describe('ShellRunProcessManager', () => { test('releases failed startup slots and enforces total and PTY capacities independently', async () => { const cwd = await workspace(); - const storageRoot = await workspace(); - const sessionsPath = join(storageRoot, 'sessions'); - await writeFile(sessionsPath, 'blocks durable ShellRun creation', 'utf8'); - const store = createSqliteShellRunStore(storageRoot); + // Durable ShellRun creation fails once (the SQLite authority has no + // filesystem seam to block on), then succeeds; the manager must release + // the reserved slot on the failed startup. + const backingStore = createSqliteShellRunStore(cwd); + let durableFailureArmed = true; + const store: ShellRunStore = { + async createShellRun(record) { + if (durableFailureArmed) { + durableFailureArmed = false; + throw new Error('durable ShellRun creation failed'); + } + return backingStore.createShellRun(record); + }, + updateShellRun: (...args) => backingStore.updateShellRun(...args), + readShellRun: (...args) => backingStore.readShellRun(...args), + listSessionShellRuns: (...args) => backingStore.listSessionShellRuns(...args), + }; const manager = createManager(store, undefined, { maxLiveShellRuns: 2, maxLivePtyRuns: 1 }); try { await assert.rejects(() => @@ -2111,7 +2124,6 @@ describe('ShellRunProcessManager', () => { ); assert.equal(manager.liveCount(), 0); assert.equal(manager.livePtyCount(), 0); - await rm(sessionsPath, { force: true }); assert.deepEqual(await store.listSessionShellRuns('session-1'), []); const ptyRun = await manager.runBackgroundBash( From 61769bca78a38dcc0abc6773aaf2c8278aa3387c Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 3 Aug 2026 20:30:58 +0800 Subject: [PATCH 07/10] test(runtime-host): trigger startup failure via invalid host epoch The composition startup-failure test corrupted task-events.jsonl, which the legacy import read; no JSONL path remains. Reject an invalid host epoch in beginHostEpoch instead: it runs after the long-term memory store is opened, so the fail-closed cleanup assertion still holds. --- .../src/__tests__/execution-composition.test.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/packages/runtime-host/src/__tests__/execution-composition.test.ts b/packages/runtime-host/src/__tests__/execution-composition.test.ts index a2883db92c..de915ad93f 100644 --- a/packages/runtime-host/src/__tests__/execution-composition.test.ts +++ b/packages/runtime-host/src/__tests__/execution-composition.test.ts @@ -1,6 +1,6 @@ import assert from 'node:assert/strict'; import { createRequire } from 'node:module'; -import { mkdir, mkdtemp, rm, stat, writeFile } from 'node:fs/promises'; +import { mkdtemp, rm, stat } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; @@ -72,15 +72,20 @@ test('production composition closes long-term memory after a later startup failu model: 'fake-model', permissionMode: 'ask', }); - const sessionRoot = join(root, 'sessions', session.id); - await mkdir(sessionRoot, { recursive: true }); - await writeFile(join(sessionRoot, 'task-events.jsonl'), '{not-json}\n'); const memory = await openInteractiveLongTermMemoryStoreForWrite(owner.lease); - await assert.rejects(createExecutionRuntimeHostComposition(compositionContext(owner))); + // Fail the composition after the memory store is opened: beginHostEpoch + // runs later in the startup sequence and rejects an invalid host epoch, + // so the composition must close every resource it opened, including + // long-term memory. + await assert.rejects( + createExecutionRuntimeHostComposition({ + ...compositionContext(owner), + hostEpoch: 'invalid host epoch!', + }), + ); await assert.rejects(memory.readItem('after-failed-start'), /closed/); - await writeFile(join(sessionRoot, 'task-events.jsonl'), ''); await owner.close(); const recoveredCapability = await resolveStorageRoot({ path: root, kind: 'interactive' }); const recoveredOwner = await tryAcquireInteractiveRootOwner(recoveredCapability); From 47fc5c61f8a3e74580ab0fb694617dce545deb8f Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 3 Aug 2026 20:30:58 +0800 Subject: [PATCH 08/10] test(desktop): align session-health-notice E2E with the surviving fixture The stale-sessions fixture no longer seeds the legacy Claude session the spec clicked. Switch the switching assertion to the healthy session (notice must hide) and back to the stale fake session for the settings click-through; the deleted-connection notice variant stays covered by deriveSessionHealthNotice unit tests. --- apps/desktop/e2e/fixtures.ts | 3 +-- .../desktop/e2e/session-health-notice.spec.ts | 23 +++++++++++-------- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/apps/desktop/e2e/fixtures.ts b/apps/desktop/e2e/fixtures.ts index 7cb3894c55..97f25d2fec 100644 --- a/apps/desktop/e2e/fixtures.ts +++ b/apps/desktop/e2e/fixtures.ts @@ -357,8 +357,7 @@ export const test = base.extend<{ }, // Stale sessions: boots the e2e-fixture `stale-sessions` fixture — one // healthy session (zai-live, secret seeded), one unlocked fake session - // (opened active), and one locked legacy session whose connection is - // gone. Exercises the #1038 health-notice authority against real IPC + // (opened active). Exercises the #1038 health-notice authority against real IPC // (connection list, hasSecret probe, connectionLocked summaries). // Readiness = turns on screen: the fake session is open. staleSessionsWindow: async ({}, use) => { diff --git a/apps/desktop/e2e/session-health-notice.spec.ts b/apps/desktop/e2e/session-health-notice.spec.ts index 9c38bbb930..59b7aab2e9 100644 --- a/apps/desktop/e2e/session-health-notice.spec.ts +++ b/apps/desktop/e2e/session-health-notice.spec.ts @@ -3,14 +3,15 @@ // "will the next send fail?" from the same facts as the send gate // (connection list, hasSecret probe, connectionLocked on the summary). // -// The stale-sessions e2e-fixture seeds the exact on-disk states, and -// both stale sessions carry user messages, so storage self-heals them to -// `connectionLocked: true` on first read — the send can neither use -// their connections nor silently rebind, even though a healthy default +// The stale-sessions e2e-fixture seeds the exact on-disk states: one +// locked fake-backend session and one healthy ai-sdk session. The stale +// session carries user messages, so storage self-heals it to +// `connectionLocked: true` on first read — the send can neither use its +// connection nor silently rebind, even though a healthy default // connection exists. The old "default exists && enabled" proxy hid the // notice in exactly this state (#1038 case 1); it must now show. -// The silent-rebind counterpart (unlocked empty stale session) is -// covered by the projection and notice unit tests. +// The deleted-connection and legacy-backend notice variants are covered +// by deriveSessionHealthNotice unit tests. import { test, expect } from './fixtures'; @@ -48,12 +49,14 @@ test('locked stale sessions show the health notice even with a ready default', a expect(alignment.leftDelta).toBeLessThanOrEqual(1); expect(alignment.rightDelta).toBeLessThanOrEqual(1); - // Switch to the locked legacy session → its deleted-connection notice. + // A healthy session must not show the notice. await page.getByRole('button', { name: '展开侧边栏' }).click(); - await page.getByText('旧的 Claude 连接会话').first().click(); - await expect(page.getByText('连接已删除')).toBeVisible(); + await page.getByText('正常会话（Z.ai Live）').first().click(); + await expect(page.getByText('会话已过期 · 请先配置真实模型')).toBeHidden(); - // Click-through lands in Settings · 模型. + // Back on the stale session, click-through lands in Settings · 模型. + await page.getByText('旧的本地模拟会话').first().click(); + await expect(page.getByText('会话已过期 · 请先配置真实模型')).toBeVisible(); await page.getByRole('button', { name: '去模型' }).click(); await expect(page.getByLabel('设置内容')).toBeVisible(); // The connection list itself, not just the settings shell: `add-connection` From fa59d57dd2d2e0e09eb435047060c9ea3299598e Mon Sep 17 00:00:00 2001 From: AstroHan Date: Mon, 3 Aug 2026 20:30:58 +0800 Subject: [PATCH 09/10] docs: point runtime-resume architecture at runtime-event-persistence --- docs/architecture/runtime-resume-architecture.md | 2 +- docs/architecture/runtime-resume-architecture.zh-CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/architecture/runtime-resume-architecture.md b/docs/architecture/runtime-resume-architecture.md index 8e0272f097..a2dd29c420 100644 --- a/docs/architecture/runtime-resume-architecture.md +++ b/docs/architecture/runtime-resume-architecture.md @@ -882,7 +882,7 @@ The two most important follow-ups are: 1. `packages/storage/src/sqlite-runtime-schema.ts` 2. `packages/storage/src/sqlite-runtime-store.ts` -3. `packages/storage/src/runtime-event-transfer.ts` +3. `packages/storage/src/runtime-event-persistence.ts` 4. `packages/storage/src/agent-run-store.ts` 5. `packages/storage/src/workspace-identity.ts` diff --git a/docs/architecture/runtime-resume-architecture.zh-CN.md b/docs/architecture/runtime-resume-architecture.zh-CN.md index 900514d156..afa7fbf26b 100644 --- a/docs/architecture/runtime-resume-architecture.zh-CN.md +++ b/docs/architecture/runtime-resume-architecture.zh-CN.md @@ -920,7 +920,7 @@ Process crash、SQLite transaction atomicity 和应用级 `fsync` 不能自动 1. `packages/storage/src/sqlite-runtime-schema.ts`：schema、WAL、FULL synchronous 和 capability。 2. `packages/storage/src/sqlite-runtime-store.ts`：T1、T2、bundle transaction 与 projection rebuild。 -3. `packages/storage/src/runtime-event-transfer.ts`：sticky SQLite selection、legacy import 和 export。 +3. `packages/storage/src/runtime-event-persistence.ts`：runtime 事件的 SQLite 持久化。 4. `packages/storage/src/agent-run-store.ts`：AgentRun 与 legacy JSONL authority gate。 5. `packages/storage/src/workspace-identity.ts`：workspace UUID marker 和 canonical path。 From 713e2a05803650c5bbdc4706ac0aefa537b8c777 Mon Sep 17 00:00:00 2001 From: jackwener Date: Mon, 3 Aug 2026 20:33:25 +0800 Subject: [PATCH 10/10] fix(storage): address SQLite authority review --- README.md | 107 ++------ SECURITY.md | 11 +- apps/desktop/README.md | 2 +- apps/desktop/e2e/fixtures.ts | 2 +- .../desktop/e2e/session-health-notice.spec.ts | 2 +- .../agent-graph-intent-claims.test.ts | 1 + .../agent-graph-supervisor-wakes.test.ts | 1 + .../operational-state-backup.test.ts | 57 +++- .../sqlite-core-execution-store.test.ts | 11 + .../__tests__/sqlite-runtime-store.test.ts | 1 + .../sqlite-session-metadata-store.test.ts | 3 + .../src/__tests__/usage-stores.test.ts | 14 + .../storage/src/operational-state-backup.ts | 248 ++++++++++++++++-- 13 files changed, 351 insertions(+), 109 deletions(-) diff --git a/README.md b/README.md index 62ce79fe4a..38bc3ae59e 100644 --- a/README.md +++ b/README.md @@ -152,7 +152,7 @@ Start with [ARCHITECTURE.md](./ARCHITECTURE.md). It provides the system map, cod apps/desktop/ Electron main / preload / React renderer packages/core/ Pure contracts for Sessions, Events, Permissions, and Connections -packages/storage/ SQLite operational state, legacy importers, and payload stores +packages/storage/ SQLite operational state, configuration, and payload stores packages/runtime/ AgentRun, model adapters, tools, context, and recovery packages/headless/ TaskRun, Autonomous Loop, Self-check, eval, and AHE packages/cli/ TUI and non-interactive CLI @@ -168,17 +168,18 @@ Maka stores workspace data under Electron `userData` by default: ```text /workspaces/default/ + runtime.sqlite llm-connections.json credentials.json settings.json - sessions/ + artifacts/ ``` Current boundaries that matter: -- Sessions and connection metadata live in the local filesystem; +- Sessions, messages, execution ledgers, workflows, usage, Automations, Daily Review, and Headless TaskRuns live in `runtime.sqlite`; - Runtime credentials such as API keys, bot tokens, and proxy passwords currently live in local plaintext `credentials.json`, behind the OS account boundary, with POSIX directory mode `0700` and file mode `0600` enforced; -- Subscription OAuth tokens (Claude, Codex, GitHub Copilot, and the Cursor/Antigravity previews) live in the same `credentials.json` — the single authority for desktop, TUI, and headless; Electron `safeStorage` only decrypts pre-existing legacy token files once at desktop startup (#1125); +- Subscription OAuth tokens (Claude, Codex, GitHub Copilot, and the Cursor/Antigravity previews) live in the same `credentials.json` — the single authority for desktop, TUI, and headless. Pre-existing Electron `safeStorage` credential/token files are not imported; affected users must re-authenticate; - Renderer does not receive plaintext credentials. File writes, Shell, and dangerous tool calls pass through the permission engine; - Headless real-model evaluation fails closed by default and requires an explicit external isolation boundary. @@ -186,65 +187,30 @@ Read [SECURITY.md](./SECURITY.md) for security reporting and policy, and [docs/R ## Runtime storage and recovery -RuntimeEvent persistence is always canonical in `runtime.sqlite`. On the first -write, Maka batch-idempotently imports legacy RuntimeEvent JSONL without -rewriting it. Legacy-only workspaces remain available to read-only inspection -until that first write. - -Session metadata and Agent Graph control tables now use the same process-local -operational database owner and the same `runtime.sqlite` transaction authority. -The first operational open copies a WAL-consistent `sessions.sqlite` source -into `runtime.sqlite`, validates every source row, and records the source digest -and result in `cutover_journal`. An interrupted copy resumes without partial -rows; a legacy database changed after cutover fails closed. The old database is -retained as migration evidence but is no longer a production writer. Session -transcript bodies remain append-only JSONL. - -Core execution state now shares that authority too: AgentRun headers and event -ledgers, event projections, root-turn admissions and source proofs, -Interactions, Host Epoch message receipts, and ShellRun records are canonical -in `runtime.sqlite` across CLI, Desktop, Runtime Host, and Headless. Each legacy -file store is fingerprinted and imported through its own durable -`cutover_journal` entry before the corresponding repository opens. Copy and -validation are one SQLite transaction, retries are idempotent, and a changed -legacy source after cutover fails closed. The legacy files are retained only as -migration evidence; new execution writes do not modify them. - -Workflow state is migrating in reviewable slices. Task Ledger events and -projections, Plan events and projections, Deep Research events, and Plan -Reminder records are now canonical in `runtime.sqlite`. -Desktop and Runtime Host production wiring opens these SQLite repositories; -their JSON/JSONL predecessors are read only during a fingerprinted, crash-safe -cutover and are never updated by later mutations. - -Usage telemetry and pricing authority now use that same operational database. -Legacy `telemetry.json` and `pricing.json` sources are decoded together and -fingerprinted before their rows and pricing revision are committed atomically. -After cutover, Desktop and Runtime Host write only `runtime.sqlite`; the source -files remain unchanged as migration evidence. - -Artifact metadata and lifecycle state now follow the same rule. Payload bytes -remain files, but their records are canonical in `runtime.sqlite` after a -fingerprinted `metadata.jsonl` cutover. Payload publication keeps its durable -staging/link protocol: recovery removes bytes whose metadata transaction did not -commit and preserves committed bytes while cleaning staging residue. Purge -intent recovery likewise completes against the SQLite metadata authority. - -Selected-session bundle export now reads that SQLite authority through a -WAL-consistent snapshot, verifies that retained legacy evidence still matches -its completed cutover, and writes only the selected Artifact rows into the -bundle's `runtime.sqlite`. Payload bytes are copied under the Artifact writer -lock, cross-session rows and payloads are excluded, and bundles no longer emit -`artifacts/metadata.jsonl`. - -Full operational backup now uses the shared database owner's online SQLite -backup API rather than copying `runtime.sqlite` or its WAL sidecars. A strict -manifest binds the standalone database snapshot to every active session -transcript and canonical Artifact payload by size and SHA-256. Restore verifies -SQLite integrity, foreign keys, supported schema versions, relational identity -sets, transcript decodability, and the exact manifested file tree before -atomically publishing a new state root. Interrupted backup or restore staging -is removed and can be retried without changing either source. +`runtime.sqlite` is the sole operational authority. It owns RuntimeEvents, +session metadata and message history, Agent Graph control, core execution state, +workflow state, usage and pricing, Artifact metadata, Automations, Daily Review, +and Headless TaskRuns. Artifact payload bytes remain regular files under +`artifacts/`; connections, credentials, settings, MCP configuration, skills, +and device identity remain configuration files. + +This storage generation does not import earlier File/JSONL authorities. On +upgrade, legacy session titles may still be discoverable through current +metadata, but conversation history that exists only in legacy transcript files +is not copied into `session_messages` and opens as an empty thread. Likewise, +pre-version or `safeStorage`-encrypted credential/token files are not migrated; +users with only those copies must re-authenticate. This data-loss boundary is +intentional for this release and must be considered before upgrading an +existing workspace. + +Full operational backup uses the database owner's online SQLite backup API and +copies canonical Artifact payloads under the Artifact writer lock. Its manifest +binds every file by size and SHA-256. Validation checks the standalone SQLite +snapshot's integrity, foreign keys, schema registry and required tables, +decodes canonical session-message and Artifact records, and verifies Artifact +payload sizes against SQLite metadata before restore. Backup and restore use +owner-only file modes, file and directory synchronization, staging, and atomic +publication. Headless trajectory hydration now consumes a frozen selected-session export from that SQLite Artifact authority. The cell publishes `trajectory-state` @@ -254,21 +220,6 @@ validated snapshot. It does not copy a live WAL or fall back to `artifacts/metadata.jsonl`. Missing, corrupt, unsupported, or mismatched evidence fails closed to a summary trajectory instead of mixing authorities. -The remaining storage work is deliberately classified rather than implied -complete: - -- Artifact metadata no longer exposes a production JSONL writer; - `artifacts/metadata.jsonl` is accepted only as fingerprinted, read-only - cutover evidence, and can be removed after the migration/cutover matrix is - complete; -- StoredMessage transcript bodies remain append-only JSONL; -- automation, connections, credentials, settings, MCP configuration, skills, - and device identity are configuration state and stay outside this operational - migration; -- Headless TaskRun evaluation ledgers, imported foreign-session caches, Daily - Review archives, and quote-cleanup bookkeeping are separate product/evaluation - domains and are not part of issue #1649. - Runtime continuation remains opt-in: - `MAKA_RUNTIME_SAFE_BOUNDARY_RESUME=1` enables the Desktop interrupted-turn diff --git a/SECURITY.md b/SECURITY.md index fc501b0052..43a4ef34e1 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -97,9 +97,9 @@ supply their own boundary. See is the single authority every surface — Desktop, TUI, headless — reads and writes, under the same OS-account and 0o700/0o600 boundary as other runtime credentials. Electron safeStorage is not part of - this boundary anymore; Desktop startup imports pre-existing - safeStorage-encrypted token files into the store once and removes - them (#1125). + this boundary anymore. Pre-existing safeStorage-encrypted credential + or token files are not imported; users with only those copies must + re-authenticate. 3. **Renderer process sandbox + preload IPC bridge.** The renderer cannot reach files, network, or shell directly. Every IPC handler in `apps/desktop/src/main/main.ts` is the trust @@ -152,7 +152,7 @@ are welcome as ordinary issues, not security advisories. Beyond security, Maka treats the following as user-facing privacy commitments: -- **Workspace JSONL stays local.** Session messages, tool +- **Workspace state stays local.** Session messages, tool results, telemetry, settings are stored under `app.getPath('userData')`. Cloud sync is not shipped. - **Tool query strings are NEVER logged.** The WebSearch tool's @@ -179,8 +179,7 @@ boundary in §2.3 was crossed. Examples: looser than the 0o700/0o600 boundary. - A production OAuth path writes or requires a safeStorage-encrypted token copy again. `credentials.json` is the single documented token - authority; safeStorage exists only inside the one-shot legacy import - (#1125). + authority; there is no safeStorage legacy import fallback. - A cleartext secret crosses main→renderer IPC under any circumstance (including error paths, settings preview, IPC result envelopes, log lines). diff --git a/apps/desktop/README.md b/apps/desktop/README.md index a346aa2ece..515909334c 100644 --- a/apps/desktop/README.md +++ b/apps/desktop/README.md @@ -23,7 +23,7 @@ The Electron desktop app: `main` (Node/Electron main process) + `preload` (conte Sub-folders: `browser/` (embedded browser view), `oauth/`, `search/` (thread search), `web-search/`, `types/`. The browser IPC handler itself (`browser-ipc-main.ts`) is flat in `src/main/`, not under `browser/`. -`main.ts` startup order: stores and the runtime/controller are created synchronously at module load; `registerIpc()` runs at top level, **before** `app.whenReady()`; inside `whenReady`, the main window is created **hidden** early and background startup (credential migration, connection bootstrapping, telemetry, bots, schedulers) runs concurrently without blocking first paint. The window is created hidden and revealed after the renderer's first AppShell paint (the `window:notifyRendererReady` gate in `app.tsx`); a fallback timer reveals it if the renderer never signals, so a fail-soft loading state can show (e.g. if `main.tsx`'s onboarding prefetch times out). The real invariant for IPC: handlers must be registered before the renderer entry runs, because `main.tsx` prefetches the onboarding snapshot before mounting React. Background startup may mutate state after the renderer's first read, so don't assume it has already settled when wiring the UI. +`main.ts` startup order: stores and the runtime/controller are created synchronously at module load; `registerIpc()` runs at top level, **before** `app.whenReady()`; inside `whenReady`, the main window is created **hidden** early and background startup (connection bootstrapping, telemetry, bots, schedulers) runs concurrently without blocking first paint. The window is created hidden and revealed after the renderer's first AppShell paint (the `window:notifyRendererReady` gate in `app.tsx`); a fallback timer reveals it if the renderer never signals, so a fail-soft loading state can show (e.g. if `main.tsx`'s onboarding prefetch times out). The real invariant for IPC: handlers must be registered before the renderer entry runs, because `main.tsx` prefetches the onboarding snapshot before mounting React. Background startup may mutate state after the renderer's first read, so don't assume it has already settled when wiring the UI. ## IPC contract diff --git a/apps/desktop/e2e/fixtures.ts b/apps/desktop/e2e/fixtures.ts index 97f25d2fec..c37a7c353a 100644 --- a/apps/desktop/e2e/fixtures.ts +++ b/apps/desktop/e2e/fixtures.ts @@ -356,7 +356,7 @@ export const test = base.extend<{ ); }, // Stale sessions: boots the e2e-fixture `stale-sessions` fixture — one - // healthy session (zai-live, secret seeded), one unlocked fake session + // healthy session (zai-live, secret seeded) and one locked fake-backend session // (opened active). Exercises the #1038 health-notice authority against real IPC // (connection list, hasSecret probe, connectionLocked summaries). // Readiness = turns on screen: the fake session is open. diff --git a/apps/desktop/e2e/session-health-notice.spec.ts b/apps/desktop/e2e/session-health-notice.spec.ts index 59b7aab2e9..d9a168cf0b 100644 --- a/apps/desktop/e2e/session-health-notice.spec.ts +++ b/apps/desktop/e2e/session-health-notice.spec.ts @@ -40,7 +40,7 @@ async function probeNoticeAlignment(page: import('@playwright/test').Page) { }); } -test('locked stale sessions show the health notice even with a ready default', async ({ staleSessionsWindow: page }) => { +test('a locked stale session shows the health notice even with a ready default', async ({ staleSessionsWindow: page }) => { // Active = stale fake session (locked by its history): notice shows. await expect(page.getByText('会话已过期 · 请先配置真实模型')).toBeVisible(); diff --git a/packages/storage/src/__tests__/agent-graph-intent-claims.test.ts b/packages/storage/src/__tests__/agent-graph-intent-claims.test.ts index 724ff457e1..227360ed62 100644 --- a/packages/storage/src/__tests__/agent-graph-intent-claims.test.ts +++ b/packages/storage/src/__tests__/agent-graph-intent-claims.test.ts @@ -136,6 +136,7 @@ describe('SQLite agent graph intent claims', () => { const legacy = new DatabaseSync(path); legacy.exec(` + DROP TABLE session_messages; DROP INDEX session_metadata_tombstones_by_retirement_unit; ALTER TABLE session_metadata_tombstones DROP COLUMN cleanup_pending; ALTER TABLE session_metadata_tombstones DROP COLUMN retirement_unit_id; diff --git a/packages/storage/src/__tests__/agent-graph-supervisor-wakes.test.ts b/packages/storage/src/__tests__/agent-graph-supervisor-wakes.test.ts index f09c6c7eb5..0774a9ec4c 100644 --- a/packages/storage/src/__tests__/agent-graph-supervisor-wakes.test.ts +++ b/packages/storage/src/__tests__/agent-graph-supervisor-wakes.test.ts @@ -223,6 +223,7 @@ describe('SQLite Agent Graph supervisor wakes', () => { const v11 = new DatabaseSync(path); v11.exec(` + DROP TABLE session_messages; DROP INDEX session_metadata_tombstones_by_retirement_unit; ALTER TABLE session_metadata_tombstones DROP COLUMN cleanup_pending; ALTER TABLE session_metadata_tombstones DROP COLUMN retirement_unit_id; diff --git a/packages/storage/src/__tests__/operational-state-backup.test.ts b/packages/storage/src/__tests__/operational-state-backup.test.ts index 95e7ed6743..5734b5602e 100644 --- a/packages/storage/src/__tests__/operational-state-backup.test.ts +++ b/packages/storage/src/__tests__/operational-state-backup.test.ts @@ -1,11 +1,13 @@ import assert from 'node:assert/strict'; -import { mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { test } from 'node:test'; +import { createSqliteArtifactStore } from '../artifact-store.js'; import { createSessionStore } from '../session-store.js'; import { createOperationalStateBackup, + OperationalBackupError, restoreOperationalStateBackup, validateOperationalStateBackup, } from '../operational-state-backup.js'; @@ -34,8 +36,18 @@ test('backs up and restores runtime.sqlite plus artifact bytes', async () => { text: 'durable', }); await sessions.close?.(); - await mkdir(join(stateRoot, 'artifacts'), { recursive: true }); - await writeFile(join(stateRoot, 'artifacts', 'note.txt'), 'artifact'); + const artifacts = createSqliteArtifactStore(stateRoot); + const artifact = await artifacts.create({ + id: 'artifact-1', + sessionId: session.id, + turnId: 'turn-1', + name: 'note.txt', + kind: 'file', + content: 'artifact', + source: 'fixture', + now: 2, + }); + artifacts.close?.(); await createOperationalStateBackup({ stateRoot, destinationRoot: backupRoot, now: () => 10 }); assert.equal((await validateOperationalStateBackup(backupRoot)).createdAt, 10); @@ -44,7 +56,10 @@ test('backs up and restores runtime.sqlite plus artifact bytes', async () => { const restored = createSessionStore(restoreRoot); try { assert.equal((await restored.readMessages(session.id))[0]?.id, 'message-1'); - assert.equal(await readFile(join(restoreRoot, 'artifacts', 'note.txt'), 'utf8'), 'artifact'); + assert.equal( + await readFile(join(restoreRoot, 'artifacts', artifact.relativePath), 'utf8'), + 'artifact', + ); } finally { await restored.close?.(); } @@ -52,3 +67,37 @@ test('backs up and restores runtime.sqlite plus artifact bytes', async () => { await rm(base, { recursive: true, force: true }); } }); + +test('rejects a backup whose SQLite Artifact metadata has no matching payload', async () => { + const base = await mkdtemp(join(tmpdir(), 'maka-operational-backup-artifact-')); + const stateRoot = join(base, 'state'); + try { + const artifacts = createSqliteArtifactStore(stateRoot); + const artifact = await artifacts.create({ + id: 'artifact-1', + sessionId: 'session-1', + turnId: 'turn-1', + name: 'note.txt', + kind: 'file', + content: 'artifact', + source: 'fixture', + now: 2, + }); + artifacts.close?.(); + await rm(join(stateRoot, 'artifacts', artifact.relativePath)); + + await assert.rejects( + createOperationalStateBackup({ + stateRoot, + destinationRoot: join(base, 'backup'), + now: () => 10, + }), + (error: unknown) => + error instanceof OperationalBackupError && + error.code === 'corrupt_backup' && + /artifact payload/i.test(error.message), + ); + } finally { + await rm(base, { recursive: true, force: true }); + } +}); diff --git a/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts b/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts index 8b05298279..9772591ada 100644 --- a/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-core-execution-store.test.ts @@ -53,6 +53,17 @@ describe('SQLite core execution stores', () => { }); }); + test('reports a missing ShellRun with the ENOENT store contract', async () => { + await withRoot(async (root) => { + const store = createSqliteShellRunStore(root); + try { + await assert.rejects(store.readShellRun('session-1', 'missing-shell'), { code: 'ENOENT' }); + } finally { + store.close(); + } + }); + }); + test('persists message receipts', async () => { await withRoot(async (root) => { const store = createSqliteMessageReceiptStore(root); diff --git a/packages/storage/src/__tests__/sqlite-runtime-store.test.ts b/packages/storage/src/__tests__/sqlite-runtime-store.test.ts index f4880a21ad..ce5584dc5c 100644 --- a/packages/storage/src/__tests__/sqlite-runtime-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-runtime-store.test.ts @@ -48,6 +48,7 @@ describe('SqliteRuntimeStore', () => { const legacy = new DatabaseSync(dbPath); legacy.exec(` + DROP TABLE headless_task_run_events; DROP TABLE runtime_workspace_heads; DROP TABLE runtime_workspace_versions; DROP TABLE runtime_workspace_epochs; diff --git a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts index b2a90012a4..723ad99039 100644 --- a/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts +++ b/packages/storage/src/__tests__/sqlite-session-metadata-store.test.ts @@ -231,6 +231,7 @@ describe('SqliteSessionMetadataStore', () => { const v12 = new DatabaseSync(path); v12.exec(` + DROP TABLE session_messages; DROP INDEX session_metadata_tombstones_by_retirement_unit; ALTER TABLE session_metadata_tombstones DROP COLUMN cleanup_pending; ALTER TABLE session_metadata_tombstones DROP COLUMN retirement_unit_id; @@ -994,6 +995,7 @@ describe('SqliteSessionMetadataStore', () => { // Rewind to the pre-provenance shape a shipped database would have. const v13 = new DatabaseSync(path); v13.exec(` + DROP TABLE session_messages; DROP INDEX session_metadata_tombstones_by_retirement_unit; ALTER TABLE session_metadata_tombstones DROP COLUMN cleanup_pending; ALTER TABLE session_metadata_tombstones DROP COLUMN retirement_unit_id; @@ -1502,6 +1504,7 @@ describe('SqliteSessionMetadataStore', () => { const v4 = new DatabaseSync(path); v4.exec(` + DROP TABLE session_messages; DROP INDEX session_metadata_tombstones_by_retirement_unit; ALTER TABLE session_metadata_tombstones DROP COLUMN cleanup_pending; ALTER TABLE session_metadata_tombstones DROP COLUMN retirement_unit_id; diff --git a/packages/storage/src/__tests__/usage-stores.test.ts b/packages/storage/src/__tests__/usage-stores.test.ts index c1159230de..77893be6fb 100644 --- a/packages/storage/src/__tests__/usage-stores.test.ts +++ b/packages/storage/src/__tests__/usage-stores.test.ts @@ -115,6 +115,20 @@ describe('InteractiveUsageStores', () => { }); }); + test('seeds an empty pricing authority for a fresh workspace', async () => { + await withInteractiveRoot(async ({ capability }) => { + const owner = await tryAcquireInteractiveRootOwner(capability); + assert(owner); + const stores = await openInteractiveUsageStoresForWrite(owner.lease); + try { + assert.deepEqual(await stores.pricing.snapshot(), { revision: 0, overrides: [] }); + } finally { + await stores.close(); + await owner.close(); + } + }); + }); + test('classifies a renamed or replaced live root as a draining persistence failure', async () => { for (const replacement of [false, true]) { await withInteractiveRoot(async ({ root, capability }) => { diff --git a/packages/storage/src/operational-state-backup.ts b/packages/storage/src/operational-state-backup.ts index 52652be8ba..ce5c05fcf5 100644 --- a/packages/storage/src/operational-state-backup.ts +++ b/packages/storage/src/operational-state-backup.ts @@ -1,6 +1,7 @@ import { createHash, randomUUID } from 'node:crypto'; -import { createReadStream } from 'node:fs'; +import { createReadStream, lstatSync } from 'node:fs'; import { + chmod, copyFile, lstat, mkdir, @@ -13,7 +14,9 @@ import { } from 'node:fs/promises'; import { dirname, relative, resolve } from 'node:path'; import { DatabaseSync } from 'node:sqlite'; +import { decodeArtifactRecordJsons } from './artifact-metadata-codec.js'; import { withArtifactWriterLock } from './artifact-writer-lock.js'; +import { decodeStoredMessageForRecovery } from './execution-record-codec.js'; import { acquireOperationalStateDatabase, OPERATIONAL_STATE_DATABASE_NAME, @@ -26,6 +29,7 @@ import { SQLITE_RUNTIME_SCHEMA_VERSION } from './sqlite-runtime-schema.js'; import { SQLITE_SESSION_METADATA_SCHEMA_VERSION } from './sqlite-session-metadata-schema.js'; import { SQLITE_USAGE_SCHEMA_VERSION } from './sqlite-usage-schema.js'; import { SQLITE_WORKFLOW_SCHEMA_VERSION } from './sqlite-workflow-schema.js'; +import { syncDirectory, syncDirectoryChain, syncFile } from './stable-storage.js'; export const OPERATIONAL_BACKUP_FORMAT = 'maka-operational-backup'; export const OPERATIONAL_BACKUP_SCHEMA_VERSION = 3 as const; @@ -84,16 +88,25 @@ export async function createOperationalStateBackup( assertSeparateRoots(canonicalStateRoot, destinationRoot); const stagingRoot = `${destinationRoot}.${process.pid}.${randomUUID()}.tmp`; try { - await mkdir(stagingRoot, { recursive: true }); + await mkdir(stagingRoot, { recursive: true, mode: 0o700 }); const database = acquireOperationalStateDatabase(canonicalStateRoot); + const databasePath = resolve(stagingRoot, OPERATIONAL_STATE_DATABASE_NAME); try { - await database.backup(resolve(stagingRoot, OPERATIONAL_STATE_DATABASE_NAME)); + await database.backup(databasePath); } finally { database.close(); } + normalizeStandaloneSqliteSnapshot(databasePath); + await chmod(databasePath, 0o600); + await syncFile(databasePath); const artifactRoot = resolve(canonicalStateRoot, 'artifacts'); if (await pathExists(artifactRoot)) { - await copyRegularTree(artifactRoot, resolve(stagingRoot, 'artifacts'), artifactRoot); + await copyRegularTree( + artifactRoot, + resolve(stagingRoot, 'artifacts'), + artifactRoot, + stagingRoot, + ); } const createdAt = (input.now ?? Date.now)(); if (!Number.isSafeInteger(createdAt) || createdAt < 0) { @@ -105,13 +118,18 @@ export async function createOperationalStateBackup( createdAt, files: await inventory(stagingRoot), }; - await writeFile( - resolve(stagingRoot, OPERATIONAL_BACKUP_MANIFEST_FILE), - `${JSON.stringify(manifest, null, 2)}\n`, - 'utf8', - ); + const manifestPath = resolve(stagingRoot, OPERATIONAL_BACKUP_MANIFEST_FILE); + await writeFile(manifestPath, `${JSON.stringify(manifest, null, 2)}\n`, { + encoding: 'utf8', + flag: 'wx', + mode: 0o600, + }); + await syncFile(manifestPath); + await validateOperationalStateBackup(stagingRoot); + await syncDirectoryChain(stagingRoot, stagingRoot); await mkdir(dirname(destinationRoot), { recursive: true }); await rename(stagingRoot, destinationRoot); + await syncDirectory(dirname(destinationRoot)); return manifest; } catch (error) { await rm(stagingRoot, { recursive: true, force: true }).catch(() => {}); @@ -137,7 +155,7 @@ export async function validateOperationalStateBackup( if (JSON.stringify(actual) !== JSON.stringify(manifest.files)) { throw new OperationalBackupError('corrupt_backup', 'Backup file inventory does not match'); } - validateSqlite(resolve(root, OPERATIONAL_STATE_DATABASE_NAME)); + validateSqlite(resolve(root, OPERATIONAL_STATE_DATABASE_NAME), manifest.files); return manifest; } @@ -151,15 +169,25 @@ export async function restoreOperationalStateBackup( const manifest = await validateOperationalStateBackup(backupRoot); const stagingRoot = `${destinationRoot}.${process.pid}.${randomUUID()}.tmp`; try { - await mkdir(stagingRoot, { recursive: true }); + await mkdir(stagingRoot, { recursive: true, mode: 0o700 }); for (const file of manifest.files) { const source = resolveInside(backupRoot, file.path); const destination = resolveInside(stagingRoot, file.path); await mkdir(dirname(destination), { recursive: true }); await copyFile(source, destination); + await chmod(destination, 0o600); + await syncFile(destination); + await syncDirectoryChain(dirname(destination), stagingRoot); + } + const actual = await inventory(stagingRoot); + if (JSON.stringify(actual) !== JSON.stringify(manifest.files)) { + throw new OperationalBackupError('corrupt_backup', 'Restored file inventory does not match'); } + validateSqlite(resolve(stagingRoot, OPERATIONAL_STATE_DATABASE_NAME), manifest.files); + await syncDirectoryChain(stagingRoot, stagingRoot); await mkdir(dirname(destinationRoot), { recursive: true }); await rename(stagingRoot, destinationRoot); + await syncDirectory(dirname(destinationRoot)); return manifest; } catch (error) { await rm(stagingRoot, { recursive: true, force: true }).catch(() => {}); @@ -261,15 +289,25 @@ async function describeFile(root: string, path: string): Promise { +async function copyRegularTree( + source: string, + destination: string, + root: string, + destinationRoot: string, +): Promise { const metadata = await lstat(source); if (metadata.isSymbolicLink()) { throw new OperationalBackupError('invalid_root', 'Artifact tree cannot contain symlinks'); } if (metadata.isDirectory()) { - await mkdir(destination, { recursive: true }); + await mkdir(destination, { recursive: true, mode: 0o700 }); for (const entry of await readdir(source)) { - await copyRegularTree(resolve(source, entry), resolve(destination, entry), root); + await copyRegularTree( + resolve(source, entry), + resolve(destination, entry), + root, + destinationRoot, + ); } return; } @@ -277,16 +315,31 @@ async function copyRegularTree(source: string, destination: string, root: string throw new OperationalBackupError('invalid_root', 'Artifact tree contains a non-regular file'); } resolveInside(root, relative(root, source)); - await mkdir(dirname(destination), { recursive: true }); + await mkdir(dirname(destination), { recursive: true, mode: 0o700 }); await copyFile(source, destination); + await chmod(destination, 0o600); + await syncFile(destination); + await syncDirectoryChain(dirname(destination), destinationRoot); } -function validateSqlite(path: string): void { +function validateSqlite(path: string, files: readonly OperationalBackupFile[]): void { try { + const metadata = lstatSync(path); + if (!metadata.isFile() || metadata.isSymbolicLink()) { + throw new Error('runtime.sqlite is not a regular file'); + } const database = new DatabaseSync(path, { readOnly: true }); try { - const row = database.prepare('PRAGMA quick_check').get() as { quick_check?: unknown }; - if (row.quick_check !== 'ok') throw new Error('quick_check failed'); + database.exec('PRAGMA query_only = ON; PRAGMA foreign_keys = ON; BEGIN'); + const integrity = database.prepare('PRAGMA integrity_check').all() as Array<{ + integrity_check?: unknown; + }>; + if (integrity.length !== 1 || integrity[0]?.integrity_check !== 'ok') { + throw new Error('integrity_check failed'); + } + if (database.prepare('PRAGMA foreign_key_check').all().length > 0) { + throw new Error('foreign_key_check failed'); + } const expected = new Map([ ['runtime', SQLITE_RUNTIME_SCHEMA_VERSION], ['session_metadata', SQLITE_SESSION_METADATA_SCHEMA_VERSION], @@ -308,7 +361,149 @@ function validateSqlite(path: string): void { ) { throw new Error('operational schema versions do not match'); } + + const requiredTables = [ + 'operational_schema_migrations', + 'runtime_events', + 'tool_journal_events', + 'tool_operations', + 'runtime_partial_snapshots', + 'runtime_capabilities', + 'runtime_continuation_claims', + 'runtime_workspace_epochs', + 'runtime_workspace_versions', + 'runtime_workspace_heads', + 'headless_task_run_events', + 'session_metadata_schema', + 'session_metadata', + 'session_metadata_labels', + 'session_metadata_tombstones', + 'subagent_spawns', + 'agent_graph_intent_claims', + 'agent_graph_schedule_updates', + 'agent_graph_operator_provisions', + 'agent_graph_client_projections', + 'agent_graph_client_operator_projections', + 'agent_graph_client_terminal_activity', + 'agent_graph_client_applied_records', + 'agent_graph_supervisor_wakes', + 'agent_graph_supervisor_wake_attempts', + 'sandbox_boundary_log', + 'session_create_claims', + 'session_catalog_state', + 'session_catalog_projection', + 'session_catalog_label_projection', + 'session_messages', + 'core_agent_runs', + 'core_agent_run_events', + 'core_agent_run_projections', + 'core_root_turn_admissions', + 'core_root_source_message_proofs', + 'core_interaction_requests', + 'core_interaction_outcomes', + 'core_message_host_epochs', + 'core_message_receipts', + 'core_shell_runs', + 'workflow_task_ledger_events', + 'workflow_task_ledger_projections', + 'workflow_plan_events', + 'workflow_plan_projections', + 'workflow_deep_research_events', + 'workflow_plan_reminders', + 'workflow_quote_companion_cleanup', + 'workflow_daily_review_state', + 'workflow_daily_review_archives', + 'usage_llm_calls', + 'usage_tool_invocations', + 'usage_model_call_attempts', + 'usage_model_call_reprojection', + 'usage_pricing_authority', + 'usage_pricing_overrides', + 'artifact_records', + 'automation_authority_state', + 'automation_definitions', + 'automation_pending_fires', + ]; + const tableExists = database.prepare( + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", + ); + for (const table of requiredTables) { + if (tableExists.get(table) === undefined) { + throw new Error(`required table is missing: ${table}`); + } + } + + const artifactRows = database + .prepare(` + SELECT artifact_id, session_id, created_at, status, relative_path, record_json + FROM artifact_records + ORDER BY created_at, storage_key + `) + .all() as Array<{ + artifact_id?: unknown; + session_id?: unknown; + created_at?: unknown; + status?: unknown; + relative_path?: unknown; + record_json?: unknown; + }>; + const artifacts = decodeArtifactRecordJsons(artifactRows.map((row) => row.record_json)); + const filesByPath = new Map(files.map((file) => [file.path, file])); + for (const [index, record] of artifacts.entries()) { + const row = artifactRows[index]; + if ( + row?.artifact_id !== record.id || + row.session_id !== record.sessionId || + row.created_at !== record.createdAt || + row.status !== record.status || + row.relative_path !== record.relativePath + ) { + throw new Error(`artifact indexes do not match record: ${record.id}`); + } + const payload = filesByPath.get(`artifacts/${record.relativePath}`); + if (!payload || payload.size !== record.sizeBytes) { + throw new Error(`artifact payload does not match metadata: ${record.id}`); + } + } + + const messageRows = database + .prepare(` + SELECT session_id, sequence, message_id, message_type, message_ts, record_json + FROM session_messages + ORDER BY session_id, sequence + `) + .all() as Array<{ + session_id?: unknown; + sequence?: unknown; + message_id?: unknown; + message_type?: unknown; + message_ts?: unknown; + record_json?: unknown; + }>; + for (const row of messageRows) { + if ( + typeof row.session_id !== 'string' || + !Number.isSafeInteger(row.sequence) || + (row.sequence as number) < 0 || + typeof row.record_json !== 'string' + ) { + throw new Error('session message index is invalid'); + } + const message = decodeStoredMessageForRecovery(JSON.parse(row.record_json)); + if ( + message.id !== row.message_id || + message.type !== row.message_type || + message.ts !== row.message_ts + ) { + throw new Error(`session message indexes do not match record: ${message.id}`); + } + } } finally { + try { + database.exec('ROLLBACK'); + } catch { + // Preserve the validation error. + } database.close(); } } catch (error) { @@ -318,6 +513,23 @@ function validateSqlite(path: string): void { } } +function normalizeStandaloneSqliteSnapshot(path: string): void { + const database = new DatabaseSync(path); + try { + const row = database.prepare('PRAGMA journal_mode = DELETE').get() as + | { journal_mode?: unknown } + | undefined; + if (row?.journal_mode !== 'delete') { + throw new OperationalBackupError( + 'corrupt_backup', + 'Unable to make the SQLite backup self-contained', + ); + } + } finally { + database.close(); + } +} + function resolveInside(root: string, path: string): string { const candidate = resolve(root, path); const rel = relative(root, candidate);