diff --git a/CHANGELOG.md b/CHANGELOG.md index e48267c6e..34775a0a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased - Minor] + +### Added + +- `relay codex run` starts a Relay-managed local Codex app-server/thread. Live session teleport is dark-launched behind the default-off local `RELAY_LIVE_SESSION_TELEPORT_ENABLED=true` switch, captured strictly from the ambient process environment before dotenv loading and reported by `relay codex status`; when enabled, `relay codex teleport` seals its active Relayfile poll mount at the next turn boundary, binds exact Relayfile destination verification to the source checkpoint, requires a 40-minute Cloud turn lease, and requires confirmed fencing plus ownership handback and mount readiness before local rollback. After notification loss, Relay never replays model execution: it durably redelivers reconciled output at least once, so an already-rendered live prefix can repeat, and it transparently runs the same prompt locally only after a confirmed pre-submission cutover failure. + ## [Unreleased - Patch] ### Fixed diff --git a/packages/cli/README.md b/packages/cli/README.md index e69c1ec18..51d1494eb 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -20,6 +20,31 @@ agent-relay message post --channel general --text "hello" agent-relay workspace list ``` +## Relay-managed Codex teleport + +`relay codex run` always supports a local managed Codex app-server session. Live +session teleport is dark-launched and remains disabled unless the controller +process starts with the exact local opt-in below: + +```bash +RELAY_LIVE_SESSION_TELEPORT_ENABLED=true relay codex run +relay codex teleport +``` + +Unset, `false`, `TRUE`, whitespace-padded values, `1`, and all other values keep +new execution local: a fresh or already-local controller does not authenticate +to Cloud, prewarm, acquire, or route a turn remotely. The opt-in must come from +the ambient process environment; a cwd `.env` file cannot enable it. The +`teleport`, `rollback`, and `status` commands remain discoverable, but a disabled +controller rejects teleport requests. `relay codex status` reports the effective +startup switch and why it is enabled or disabled. + +The flag is read when `relay codex run` starts. To roll back the capability, +stop the controller and restart it with the variable unset or set to `false`. +If the prior process left a prewarm or remote environment behind, startup first +revokes that Cloud generation and restores the same thread and Relayfile mount +locally; it does not prewarm again while disabled. + ## This machine's node The `node` command group manages the broker on your machine and the agents it runs: diff --git a/packages/cli/src/cli/bootstrap.test.ts b/packages/cli/src/cli/bootstrap.test.ts index 067c42181..18b9b8091 100644 --- a/packages/cli/src/cli/bootstrap.test.ts +++ b/packages/cli/src/cli/bootstrap.test.ts @@ -5,7 +5,8 @@ import path from 'node:path'; import { Command } from 'commander'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { createProgram, propagateTelemetryContextToChildren } from './bootstrap.js'; +import { createProgram, loadCliEnvironment, propagateTelemetryContextToChildren } from './bootstrap.js'; +import { RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV } from './lib/codex-live-controller.js'; const expectedLeafCommands = [ // node broker + agent group (local is a hidden alias, filtered out below) @@ -40,6 +41,11 @@ const expectedLeafCommands = [ 'reflex off', 'reflex status', 'session replay', + // Relay-managed Codex execution teleport + 'codex run', + 'codex teleport', + 'codex rollback', + 'codex status', // fleet (serve is a hidden error stub, filtered out below) 'fleet agent list', 'fleet config', @@ -198,6 +204,27 @@ describe('createProgram output redaction', () => { }); describe('bootstrap CLI', () => { + it('does not let a cwd .env enable live teleport after the trusted startup capture', () => { + const originalCwd = process.cwd(); + const originalValue = process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV]; + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-codex-dotenv-')); + fs.writeFileSync(path.join(cwd, '.env'), `${RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV}=true\n`); + delete process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV]; + process.chdir(cwd); + + try { + const startupSwitch = loadCliEnvironment(); + + expect(startupSwitch).toEqual({ enabled: false, reason: 'ambient-unset' }); + expect(process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV]).toBe('true'); + } finally { + process.chdir(originalCwd); + if (originalValue === undefined) delete process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV]; + else process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV] = originalValue; + fs.rmSync(cwd, { recursive: true, force: true }); + } + }); + it('uses the expected program name', () => { const program = createProgram(); expect(program.name()).toBe('agent-relay'); @@ -221,6 +248,7 @@ describe('bootstrap CLI', () => { 'fleet', 'reflex', 'session', + 'codex', 'status', 'observer', 'version', diff --git a/packages/cli/src/cli/bootstrap.ts b/packages/cli/src/cli/bootstrap.ts index 7184ed7b7..de8554922 100644 --- a/packages/cli/src/cli/bootstrap.ts +++ b/packages/cli/src/cli/bootstrap.ts @@ -50,8 +50,25 @@ import { registerCapabilitiesCommands } from './commands/capabilities.js'; import { registerFleetCommands } from './commands/fleet.js'; import { registerSkillsCommands } from './commands/skills.js'; import { registerSessionCommands } from './commands/session.js'; +import { registerCodexCommands } from './commands/codex.js'; +import { + DEFAULT_CODEX_LIVE_TELEPORT_STARTUP_SWITCH, + RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV, + resolveCodexLiveTeleportStartupSwitch, + type CodexLiveTeleportStartupSwitch, +} from './lib/codex-live-controller.js'; -dotenvConfig({ quiet: true }); +/** + * Capture the trusted local teleport switch before cwd dotenv loading can + * mutate process.env. Production calls this once, at the start of runCli. + */ +export function loadCliEnvironment(): CodexLiveTeleportStartupSwitch { + const liveTeleportStartupSwitch = resolveCodexLiveTeleportStartupSwitch( + process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV] + ); + dotenvConfig({ quiet: true }); + return liveTeleportStartupSwitch; +} const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -373,7 +390,9 @@ function installExitHooks(): void { }); } -export function createProgram(options: { name?: string } = {}): Command { +export function createProgram( + options: { name?: string; liveTeleportStartupSwitch?: CodexLiveTeleportStartupSwitch } = {} +): Command { const program = new Command(); // Commander echoes offending tokens verbatim (`error: unknown option @@ -423,6 +442,10 @@ export function createProgram(options: { name?: string } = {}): Command { registerCapabilitiesCommands(program); registerSkillsCommands(program); registerSessionCommands(program); + registerCodexCommands(program, { + liveTeleportStartupSwitch: + options.liveTeleportStartupSwitch ?? DEFAULT_CODEX_LIVE_TELEPORT_STARTUP_SWITCH, + }); program .command('mcp') @@ -471,6 +494,7 @@ function collectTopLevelVerbs(program: Command): Set { } export async function runCli(argv: string[] = process.argv): Promise { + const liveTeleportStartupSwitch = loadCliEnvironment(); assertSupportedNodeVersion(); ensureWebSocketGlobal(); maybeRunUpdateCheck(VERSION, argv); @@ -487,7 +511,7 @@ export async function runCli(argv: string[] = process.argv): Promise { }); } - const program = createProgram({ name: resolveProgramName(argv) }); + const program = createProgram({ name: resolveProgramName(argv), liveTeleportStartupSwitch }); installTelemetryHooks(program); installExitHooks(); diff --git a/packages/cli/src/cli/commands/codex-defaults.test.ts b/packages/cli/src/cli/commands/codex-defaults.test.ts new file mode 100644 index 000000000..34460d8ed --- /dev/null +++ b/packages/cli/src/cli/commands/codex-defaults.test.ts @@ -0,0 +1,138 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { Readable } from 'node:stream'; + +import { Command } from 'commander'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => { + const cloudClient = { + prewarm: vi.fn(), + status: vi.fn(), + acquire: vi.fn(), + revoke: vi.fn(), + }; + const appServer = { + initialize: vi.fn(async () => undefined), + startThread: vi.fn(async () => 'thread-local-1'), + resumeThread: vi.fn(async () => undefined), + addEnvironment: vi.fn(async () => undefined), + environmentStatus: vi.fn(async () => ({ status: 'ready' })), + runTurn: vi.fn(async () => { + const turn = { + id: 'turn-local-1', + status: 'completed', + itemsView: 'full', + items: [{ id: 'answer-1', type: 'agentMessage', text: 'local answer' }], + }; + return { + turnId: turn.id, + response: { turn }, + completed: { method: 'turn/completed', params: { threadId: 'thread-local-1', turn } }, + }; + }), + turnOutcome: vi.fn(async () => ({ status: 'absent' as const })), + close: vi.fn(async () => undefined), + }; + const checkpointAndSeal = vi.fn(); + return { + appServer, + checkpointAndSeal, + cloudClient, + cloudConstructor: vi.fn(function () { + return cloudClient; + }), + ensureCloudSession: vi.fn(), + probeCapability: vi.fn(), + resumePersistedLocalMount: vi.fn(async () => undefined), + }; +}); + +vi.mock('@agent-relay/cloud', async (importOriginal) => ({ + ...(await importOriginal()), + CloudLiveTeleportClient: mocks.cloudConstructor, + ensureCloudSession: mocks.ensureCloudSession, +})); + +vi.mock('../lib/codex-app-server.js', async (importOriginal) => ({ + ...(await importOriginal()), + probeCodexEnvironmentCapability: mocks.probeCapability, + StdioCodexAppServerSession: { spawn: vi.fn(async () => mocks.appServer) }, +})); + +vi.mock('../lib/codex-relayfile-seal.js', async (importOriginal) => ({ + ...(await importOriginal()), + createRelayfileSealLifecycle: () => ({ + checkpointAndSeal: mocks.checkpointAndSeal, + resumePersistedLocalMount: mocks.resumePersistedLocalMount, + }), +})); + +import { registerCodexCommands } from './codex.js'; +import { RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV } from '../lib/codex-live-controller.js'; + +describe('default Relay-managed Codex command wiring', () => { + const temporaryRoots: string[] = []; + const originalFlag = process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV]; + const originalStateDir = process.env.AGENT_RELAY_STATE_DIR; + + afterEach(() => { + vi.clearAllMocks(); + if (originalFlag === undefined) delete process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV]; + else process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV] = originalFlag; + if (originalStateDir === undefined) delete process.env.AGENT_RELAY_STATE_DIR; + else process.env.AGENT_RELAY_STATE_DIR = originalStateDir; + for (const root of temporaryRoots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); + }); + + it('keeps a fresh durable run local when the captured startup switch is disabled', async () => { + // Unix-domain socket paths are short on macOS; /tmp keeps the production + // control socket below that limit while still exercising the real server. + const root = fs.mkdtempSync(path.join('/tmp', 'relay-codex-')); + temporaryRoots.push(root); + const workspace = path.join(root, 'workspace'); + const stateDir = path.join(root, 'state'); + fs.mkdirSync(workspace); + const workspaceRoot = fs.realpathSync(workspace); + process.env.AGENT_RELAY_STATE_DIR = stateDir; + + // Simulate dotenv or later bootstrap code mutating process.env after the + // trusted ambient value was captured. Production wiring must use only the + // immutable switch passed into registration. + process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV] = 'true'; + const program = new Command().exitOverride(); + registerCodexCommands(program, { + liveTeleportStartupSwitch: { enabled: false, reason: 'ambient-unset' }, + cwd: () => workspace, + input: Readable.from([]), + log: vi.fn(), + }); + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + + try { + await program.parseAsync(['node', 'relay', 'codex', 'run', 'stay local']); + } finally { + stderr.mockRestore(); + } + + expect(mocks.ensureCloudSession).not.toHaveBeenCalled(); + expect(mocks.cloudConstructor).not.toHaveBeenCalled(); + expect(mocks.probeCapability).not.toHaveBeenCalled(); + expect(mocks.cloudClient.prewarm).not.toHaveBeenCalled(); + expect(mocks.cloudClient.status).not.toHaveBeenCalled(); + expect(mocks.cloudClient.acquire).not.toHaveBeenCalled(); + expect(mocks.cloudClient.revoke).not.toHaveBeenCalled(); + expect(mocks.checkpointAndSeal).not.toHaveBeenCalled(); + expect(mocks.resumePersistedLocalMount).not.toHaveBeenCalled(); + expect(mocks.appServer.initialize).toHaveBeenCalledOnce(); + expect(mocks.appServer.startThread).toHaveBeenCalledWith({ cwd: workspaceRoot }); + expect(mocks.appServer.addEnvironment).not.toHaveBeenCalled(); + expect(mocks.appServer.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'stay local', + execution: { kind: 'local', workspaceRoot }, + }) + ); + expect(fs.existsSync(path.join(stateDir, 'codex-live', 'active.json'))).toBe(true); + }); +}); diff --git a/packages/cli/src/cli/commands/codex.test.ts b/packages/cli/src/cli/commands/codex.test.ts new file mode 100644 index 000000000..31710206e --- /dev/null +++ b/packages/cli/src/cli/commands/codex.test.ts @@ -0,0 +1,458 @@ +import { Command } from 'commander'; +import { describe, expect, it, vi } from 'vitest'; + +import { CodexTurnRecordedError } from '../lib/codex-live-controller.js'; +import { + nodeCallbackWriter, + registerCodexCommands, + runManagedCodexTurn, + surfaceRecoveredCodexTerminal, + withCodexControllerShutdown, +} from './codex.js'; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +function reconciledResult() { + const turn = { + id: 'turn-reconciled', + status: 'completed', + itemsView: 'full', + items: [{ id: 'answer-1', type: 'agentMessage', text: 'the recovered answer' }], + }; + return { + turnId: turn.id, + response: { turn }, + completed: { method: 'turn/completed', params: { threadId: 'thread-1', turn } }, + reconciled: true as const, + }; +} + +describe('nodeCallbackWriter', () => { + it('waits for the Node write callback even when write reports backpressure', async () => { + let callback!: (error?: Error | null) => void; + const stream = { + write: vi.fn((_message: string, done: (error?: Error | null) => void) => { + callback = done; + return false; + }), + }; + const completed = vi.fn(); + const pending = nodeCallbackWriter(stream)('durable output').then(completed); + + await Promise.resolve(); + expect(stream.write).toHaveBeenCalledWith('durable output', expect.any(Function)); + expect(completed).not.toHaveBeenCalled(); + + callback(); + await pending; + expect(completed).toHaveBeenCalledOnce(); + }); + + it('rejects callback errors and synchronous stream throws', async () => { + const callbackError = new Error('callback failed'); + const callbackStream = { + write: vi.fn((_message: string, done: (error?: Error | null) => void) => { + done(callbackError); + return true; + }), + }; + await expect(nodeCallbackWriter(callbackStream)('output')).rejects.toBe(callbackError); + + const thrown = new Error('write threw'); + const throwingStream = { + write: vi.fn(() => { + throw thrown; + }), + }; + await expect(nodeCallbackWriter(throwingStream as never)('output')).rejects.toBe(thrown); + }); +}); + +describe('withCodexControllerShutdown', () => { + it('marks only a normally completed command as an explicit pending-teleport cancellation', async () => { + const close = vi.fn(async () => undefined); + await expect(withCodexControllerShutdown({ close } as never, async () => 'complete')).resolves.toBe( + 'complete' + ); + expect(close).toHaveBeenLastCalledWith({ cancelPendingTeleport: true }); + + const terminal = new Error('recorded terminal'); + await expect( + withCodexControllerShutdown({ close } as never, async () => Promise.reject(terminal)) + ).rejects.toBe(terminal); + expect(close).toHaveBeenLastCalledWith({ cancelPendingTeleport: false }); + }); + + it('still closes the controller when control-server cleanup fails', async () => { + const close = vi.fn(async () => undefined); + const cleanupError = new Error('control server close failed'); + await expect( + withCodexControllerShutdown( + { close } as never, + async () => undefined, + async () => Promise.reject(cleanupError) + ) + ).rejects.toBe(cleanupError); + expect(close).toHaveBeenCalledWith({ cancelPendingTeleport: true }); + }); +}); + +describe('runManagedCodexTurn', () => { + it('keeps a completed recovery unacknowledged until the async output writer resolves', async () => { + const write = deferred(); + const controller = { + runTurn: vi.fn(async () => reconciledResult()), + acknowledgeRecoveredOutcome: vi.fn(), + }; + const running = runManagedCodexTurn(controller as never, 'recover me', { + json: false, + writeError: vi.fn(async () => undefined), + writeOutput: vi.fn(() => write.promise), + }); + + await vi.waitFor(() => expect(controller.runTurn).toHaveBeenCalledOnce()); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); + write.resolve(); + await running; + expect(controller.acknowledgeRecoveredOutcome).toHaveBeenCalledOnce(); + }); + + it.each(['failed', 'interrupted'] as const)( + 'keeps a %s recovery unacknowledged until the async error writer resolves', + async (status) => { + const write = deferred(); + const terminal = new CodexTurnRecordedError(status, `recorded ${status}`, true); + const controller = { + runTurn: vi.fn().mockRejectedValue(terminal), + acknowledgeRecoveredOutcome: vi.fn(), + }; + const running = runManagedCodexTurn(controller as never, 'recover me', { + json: false, + writeError: vi.fn(() => write.promise), + }); + + await vi.waitFor(() => expect(controller.runTurn).toHaveBeenCalledOnce()); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); + write.resolve(); + await expect(running).rejects.toBe(terminal); + expect(controller.acknowledgeRecoveredOutcome).toHaveBeenCalledOnce(); + } + ); + + it.each(['completed', 'failed', 'interrupted'] as const)( + 'keeps a %s recovery durable when the Node write callback fails', + async (status) => { + const writeError = new Error(`write callback failed for ${status}`); + const writer = nodeCallbackWriter({ + write: vi.fn((_message: string, done: (error?: Error | null) => void) => { + done(writeError); + return true; + }), + }); + const terminal = + status === 'completed' ? undefined : new CodexTurnRecordedError(status, `recorded ${status}`, true); + const controller = { + runTurn: terminal ? vi.fn().mockRejectedValue(terminal) : vi.fn(async () => reconciledResult()), + acknowledgeRecoveredOutcome: vi.fn(), + }; + + await expect( + runManagedCodexTurn(controller as never, 'recover me', { + json: false, + writeError: writer, + writeOutput: writer, + }) + ).rejects.toBe(writeError); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); + } + ); + + it('renders an exact assistant answer recovered after completion-notification loss', async () => { + const turn = { + id: 'turn-reconciled', + status: 'completed', + itemsView: 'full', + items: [{ id: 'answer-1', type: 'agentMessage', text: 'the recovered answer' }], + }; + const controller = { + runTurn: vi.fn(async () => ({ + turnId: turn.id, + response: { turn }, + completed: { method: 'turn/completed', params: { threadId: 'thread-1', turn } }, + reconciled: true as const, + })), + acknowledgeRecoveredOutcome: vi.fn(), + status: vi.fn(), + }; + const writeOutput = vi.fn(); + + await runManagedCodexTurn(controller as never, 'recover me', { + json: false, + writeError: vi.fn(), + writeOutput, + }); + + expect(writeOutput).toHaveBeenCalledWith('the recovered answer\n'); + expect(controller.acknowledgeRecoveredOutcome).toHaveBeenCalledOnce(); + }); + + it('does not acknowledge a live recovered completion until its exact answer is rendered', async () => { + const turn = { + id: 'turn-reconciled', + status: 'completed', + itemsView: 'full', + items: [{ id: 'answer-1', type: 'agentMessage', text: 'the recovered answer' }], + }; + const renderError = new Error('stdout unavailable'); + const controller = { + runTurn: vi.fn(async () => ({ + turnId: turn.id, + response: { turn }, + completed: { method: 'turn/completed', params: { threadId: 'thread-1', turn } }, + reconciled: true as const, + })), + acknowledgeRecoveredOutcome: vi.fn(), + }; + + await expect( + runManagedCodexTurn(controller as never, 'recover me', { + json: false, + writeError: vi.fn(), + writeOutput: () => { + throw renderError; + }, + }) + ).rejects.toBe(renderError); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); + }); + + it('does not report a successful command for a recorded failed turn', async () => { + const terminal = new CodexTurnRecordedError('failed', 'Codex turn ended failed.'); + const runTurn = vi.fn().mockRejectedValue(terminal); + const controller = { + runTurn, + acknowledgeRecoveredOutcome: vi.fn(), + status: vi.fn(() => ({ + phase: 'local' as const, + threadId: 'thread-1', + generation: 1, + })), + }; + const writeError = vi.fn(); + + await expect( + runManagedCodexTurn(controller as never, 'failed input', { json: false, writeError }) + ).rejects.toBe(terminal); + expect(writeError).toHaveBeenCalledWith(expect.stringContaining('recorded the turn as failed')); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); + }); + + it.each(['failed', 'interrupted'] as const)( + 'acknowledges a live reconciled %s terminal only after rendering it', + async (status) => { + const terminal = new CodexTurnRecordedError(status, `recorded ${status}`, true); + const controller = { + runTurn: vi.fn().mockRejectedValue(terminal), + acknowledgeRecoveredOutcome: vi.fn(), + }; + const writeError = vi.fn(); + + await expect( + runManagedCodexTurn(controller as never, 'do not replay', { json: false, writeError }) + ).rejects.toBe(terminal); + expect(writeError).toHaveBeenCalledWith(expect.stringContaining(`turn as ${status}`)); + expect(controller.acknowledgeRecoveredOutcome).toHaveBeenCalledOnce(); + } + ); + + it.each(['failed', 'interrupted'] as const)( + 'does not acknowledge a live reconciled %s terminal when rendering throws', + async (status) => { + const terminal = new CodexTurnRecordedError(status, `recorded ${status}`, true); + const renderError = new Error('stderr unavailable'); + const controller = { + runTurn: vi.fn().mockRejectedValue(terminal), + acknowledgeRecoveredOutcome: vi.fn(), + }; + + await expect( + runManagedCodexTurn(controller as never, 'do not replay', { + json: false, + writeError: () => { + throw renderError; + }, + }) + ).rejects.toBe(renderError); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); + } + ); + + it.each(['failed', 'interrupted'] as const)( + 'surfaces a crash-reconciled %s turn nonzero before any new prompt runs', + async (status) => { + const terminal = new CodexTurnRecordedError(status, `recorded ${status}`); + const controller = { + takeRecoveredTerminal: vi.fn().mockReturnValueOnce(terminal).mockReturnValue(undefined), + acknowledgeRecoveredOutcome: vi.fn(), + runTurn: vi.fn(), + }; + const writeError = vi.fn(); + + await expect( + surfaceRecoveredCodexTerminal(controller as never, { json: false, writeError }) + ).rejects.toBe(terminal); + + expect(writeError).toHaveBeenCalledWith(expect.stringContaining(`turn as ${status}`)); + expect(controller.acknowledgeRecoveredOutcome).toHaveBeenCalledOnce(); + await expect( + surfaceRecoveredCodexTerminal(controller as never, { json: false, writeError }) + ).resolves.toBeUndefined(); + expect(writeError).toHaveBeenCalledTimes(1); + expect(controller.runTurn).not.toHaveBeenCalled(); + } + ); + + it.each(['failed', 'interrupted'] as const)( + 'does not acknowledge durable %s terminal evidence when rendering throws', + async (status) => { + const terminal = new CodexTurnRecordedError(status, `recorded ${status}`); + const controller = { + takeRecoveredTerminal: vi.fn(() => terminal), + acknowledgeRecoveredOutcome: vi.fn(), + }; + const renderError = new Error('stderr unavailable'); + + await expect( + surfaceRecoveredCodexTerminal(controller as never, { + json: false, + writeError: () => { + throw renderError; + }, + }) + ).rejects.toBe(renderError); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); + } + ); + + it('fails closed instead of continuing when fencing is unconfirmed', async () => { + const controller = { + runTurn: vi.fn(async () => Promise.reject(new Error('revoke unconfirmed'))), + status: vi.fn(() => ({ phase: 'fenced', threadId: 'thread-1', generation: 1 })), + }; + + await expect( + runManagedCodexTurn(controller as never, 'must not continue', { + json: false, + writeError: vi.fn(), + }) + ).rejects.toThrow('cannot continue'); + }); +}); + +describe('registerCodexCommands', () => { + it('makes teleport one command against the active persisted generation', async () => { + const sendControl = vi.fn(async () => ({ + ok: true as const, + status: { + version: 1 as const, + sessionId: 'session-1', + threadId: 'thread-1', + workspaceRoot: '/repo', + generation: 4, + phase: 'teleport_pending' as const, + controllerPid: 10, + socketPath: '/state/controller.sock', + turnActive: false, + pending: { requestId: 'request-1', expectedGeneration: 4 }, + lastRequestId: 'request-1', + updatedAt: '2026-08-23T12:00:00.000Z', + controller: 'local' as const, + execution: 'local' as const, + liveTeleport: { enabled: true, reason: 'ambient-exact-true' as const }, + workspaceSource: 'relayfile-checkpoint-seal' as const, + }, + })); + const log = vi.fn(); + const program = new Command().exitOverride(); + registerCodexCommands(program, { + readState: () => ({ generation: 4 }) as never, + sendControl, + requestId: () => 'request-1', + log, + }); + + await program.parseAsync(['node', 'relay', 'codex', 'teleport']); + + expect(sendControl).toHaveBeenCalledWith({ + operation: 'teleport', + requestId: 'request-1', + expectedGeneration: 4, + }); + expect(log).toHaveBeenCalledWith(expect.stringContaining('confirmed pre-submission recovery')); + }); + + it('refuses teleport when the session was not started under Relay control', async () => { + const sendControl = vi.fn(); + const program = new Command().exitOverride(); + registerCodexCommands(program, { readState: () => null, sendControl }); + + await expect(program.parseAsync(['node', 'relay', 'codex', 'teleport'])).rejects.toThrow( + 'No active Relay-managed Codex session' + ); + expect(sendControl).not.toHaveBeenCalled(); + }); + + it('prints the immutable startup switch and reason in human and JSON status', async () => { + const status = { + version: 1 as const, + sessionId: 'session-1', + threadId: 'thread-1', + workspaceRoot: '/repo', + generation: 4, + phase: 'local' as const, + controllerPid: 10, + socketPath: '/state/controller.sock', + turnActive: false, + cloudLifecycle: 'none' as const, + updatedAt: '2026-08-23T12:00:00.000Z', + controller: 'local' as const, + execution: 'local' as const, + liveTeleport: { enabled: false, reason: 'ambient-value-not-exact-true' as const }, + workspaceSource: 'unavailable' as const, + }; + const sendControl = vi.fn(async () => ({ ok: true as const, status })); + const humanLog = vi.fn(); + const humanProgram = new Command().exitOverride(); + registerCodexCommands(humanProgram, { + readState: () => ({ generation: 4 }) as never, + sendControl, + log: humanLog, + }); + + await humanProgram.parseAsync(['node', 'relay', 'codex', 'status']); + + expect(humanLog).toHaveBeenCalledWith('Live teleport: disabled (ambient-value-not-exact-true)'); + + const jsonLog = vi.fn(); + const jsonProgram = new Command().exitOverride(); + registerCodexCommands(jsonProgram, { + readState: () => ({ generation: 4 }) as never, + sendControl, + log: jsonLog, + }); + + await jsonProgram.parseAsync(['node', 'relay', 'codex', 'status', '--json']); + + expect(JSON.parse(String(jsonLog.mock.calls[0]?.[0]))).toMatchObject({ + liveTeleport: { enabled: false, reason: 'ambient-value-not-exact-true' }, + }); + }); +}); diff --git a/packages/cli/src/cli/commands/codex.ts b/packages/cli/src/cli/commands/codex.ts new file mode 100644 index 000000000..3387ac022 --- /dev/null +++ b/packages/cli/src/cli/commands/codex.ts @@ -0,0 +1,569 @@ +import fs from 'node:fs'; +import net from 'node:net'; +import os from 'node:os'; +import path from 'node:path'; +import readline from 'node:readline'; +import { randomUUID } from 'node:crypto'; + +import { CloudLiveTeleportClient, ensureCloudSession } from '@agent-relay/cloud'; +import { Command } from 'commander'; + +import { + probeCodexEnvironmentCapability, + StdioCodexAppServerSession, + type CodexNotification, + type CodexTurnResult, +} from '../lib/codex-app-server.js'; +import { + CodexLiveController, + CodexTurnOutcomeUncertainError, + CodexTurnRecordedError, + FileCodexControllerStateStore, + codexControllerStateMayOwnCloudResources, + DEFAULT_CODEX_LIVE_TELEPORT_STARTUP_SWITCH, + type CodexControllerState, + type CodexLiveTeleportStartupSwitch, + type CodexPersistedMountResumeProvider, + type CodexWorkspaceSealProvider, + type PublicCodexControllerStatus, +} from '../lib/codex-live-controller.js'; +import { createRelayfileSealLifecycle } from '../lib/codex-relayfile-seal.js'; + +export type CodexControllerPaths = { + directory: string; + statePath: string; + socketPath: string; +}; + +type ControlRequest = + | { operation: 'status' } + | { operation: 'teleport'; requestId: string; expectedGeneration: number } + | { operation: 'rollback' }; + +type ControlResponse = { ok: true; status: PublicCodexControllerStatus } | { ok: false; error: string }; + +export type CodexAsyncWriter = (message: string) => Promise; +type CodexCommandInput = NodeJS.ReadableStream & { isTTY?: boolean }; + +type NodeCallbackWritable = { + write(message: string, callback: (error?: Error | null) => void): boolean; +}; + +/** + * Adapts Node's callback-based stream contract into a durable-output promise. + * A false return value is backpressure, not completion; only the callback ACKs + * the write. Callback errors and synchronous throws both reject the promise. + */ +export function nodeCallbackWriter(stream: NodeCallbackWritable): CodexAsyncWriter { + return (message) => + new Promise((resolve, reject) => { + try { + stream.write(message, (error) => { + if (error) reject(error); + else resolve(); + }); + } catch (error) { + reject(error); + } + }); +} + +/** + * Gives controller shutdown the same semantic outcome as the command body. + * A rejected command is an error unwind, never implicit cancellation of a + * teleport that the controller already accepted durably. + */ +export async function withCodexControllerShutdown( + controller: Pick, + run: () => Promise, + beforeClose: () => Promise = async () => undefined +): Promise { + let completedNormally = false; + try { + const result = await run(); + completedNormally = true; + return result; + } finally { + try { + await beforeClose(); + } finally { + await controller.close({ cancelPendingTeleport: completedNormally }); + } + } +} + +export interface CodexCommandDependencies { + runManaged(options: { cwd: string; model?: string; prompt?: string; json?: boolean }): Promise; + /** Immutable value captured before bootstrap loads dotenv. */ + liveTeleportStartupSwitch: CodexLiveTeleportStartupSwitch; + checkpointAndSeal: CodexWorkspaceSealProvider; + resumePersistedLocalMount: CodexPersistedMountResumeProvider; + readState(): CodexControllerState | null; + sendControl(request: ControlRequest): Promise; + requestId(): string; + cwd(): string; + input: CodexCommandInput; + log(message: string): void; +} + +export function codexControllerPaths(env: NodeJS.ProcessEnv = process.env): CodexControllerPaths { + const root = env.AGENT_RELAY_STATE_DIR?.trim() || path.join(os.homedir(), '.agent-relay'); + const directory = path.join(root, 'codex-live'); + return { + directory, + statePath: path.join(directory, 'active.json'), + socketPath: path.join(directory, 'controller.sock'), + }; +} + +function describeError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function agentMessageDelta(notification: CodexNotification): string | undefined { + if (!/agentMessage.*delta/i.test(notification.method)) return undefined; + const params = notification.params; + if (!params || typeof params !== 'object' || Array.isArray(params)) return undefined; + const delta = (params as Record).delta; + return typeof delta === 'string' ? delta : undefined; +} + +export async function runManagedCodexTurn( + controller: Pick, + text: string, + options: { + json: boolean; + writeError: CodexAsyncWriter; + writeOutput?: CodexAsyncWriter; + } +): Promise { + let result: CodexTurnResult; + try { + result = await controller.runTurn(text); + } catch (error) { + if (error instanceof CodexTurnRecordedError) { + if (error.requiresRecoveryAcknowledgment) { + await writeCodexRecordedTerminal(error, options); + controller.acknowledgeRecoveredOutcome(); + throw error; + } + await surfaceCodexRecordedTerminal(error, options); + } + if (error instanceof CodexTurnOutcomeUncertainError) { + throw new Error('Relay-managed Codex stopped because the last turn outcome is uncertain.', { + cause: error, + }); + } + throw new Error( + 'Relay-managed Codex cannot continue because execution fencing or local recovery is unconfirmed.', + { cause: error } + ); + } + if (result.reconciled) { + await writeReconciledTurn(result, options); + controller.acknowledgeRecoveredOutcome(); + } +} + +export async function surfaceCodexRecordedTerminal( + error: CodexTurnRecordedError, + options: { json: boolean; writeError: CodexAsyncWriter } +): Promise { + await writeCodexRecordedTerminal(error, options); + throw error; +} + +async function writeCodexRecordedTerminal( + error: CodexTurnRecordedError, + options: { json: boolean; writeError: CodexAsyncWriter } +): Promise { + await options.writeError( + options.json + ? `${JSON.stringify({ + method: 'relay/codexTurnRecordedTerminal', + params: { code: 'TURN_RECORDED_TERMINAL', status: error.status }, + })}\n` + : `Codex recorded the turn as ${error.status}; it was not replayed.\n` + ); +} + +export async function surfaceRecoveredCodexTerminal( + controller: Pick, + options: { json: boolean; writeError: CodexAsyncWriter } +): Promise { + const terminal = controller.takeRecoveredTerminal(); + if (!terminal) return; + await writeCodexRecordedTerminal(terminal, options); + controller.acknowledgeRecoveredOutcome(); + throw terminal; +} + +async function writeReconciledTurn( + result: CodexTurnResult, + options: { json: boolean; writeOutput?: CodexAsyncWriter } +): Promise { + if (!options.writeOutput) { + throw new Error('Reconciled Codex completion cannot be acknowledged without an output writer.'); + } + if (options.json) { + await options.writeOutput(`${JSON.stringify(result.completed)}\n`); + return; + } + const params = result.completed.params; + const turn = + params && typeof params === 'object' && !Array.isArray(params) + ? (params as Record).turn + : undefined; + const items = + turn && typeof turn === 'object' && !Array.isArray(turn) + ? (turn as Record).items + : undefined; + if (!Array.isArray(items)) { + throw new Error('Reconciled Codex completion did not contain an observable assistant answer.'); + } + const answers = items.flatMap((item) => + item && + typeof item === 'object' && + !Array.isArray(item) && + (item as Record).type === 'agentMessage' && + typeof (item as Record).text === 'string' + ? [(item as Record).text as string] + : [] + ); + if (answers.length === 0) { + throw new Error('Reconciled Codex completion did not contain an observable assistant answer.'); + } + await options.writeOutput(`${answers.join('\n')}\n`); +} + +async function listen(server: net.Server, socketPath: string): Promise { + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(socketPath, () => { + server.off('error', reject); + resolve(); + }); + }); + fs.chmodSync(socketPath, 0o600); +} + +async function closeServer(server: net.Server): Promise { + if (!server.listening) return; + await new Promise((resolve) => server.close(() => resolve())); +} + +function createControlServer(controller: CodexLiveController): net.Server { + let serialized = Promise.resolve(); + return net.createServer((socket) => { + socket.setEncoding('utf8'); + let buffer = ''; + socket.on('data', (chunk: string) => { + buffer += chunk; + const newline = buffer.indexOf('\n'); + if (newline < 0) return; + const line = buffer.slice(0, newline); + socket.pause(); + serialized = serialized.then(async () => { + let response: ControlResponse; + try { + const request = JSON.parse(line) as ControlRequest; + if (request.operation === 'status') { + response = { ok: true, status: controller.status() }; + } else if (request.operation === 'teleport') { + response = { + ok: true, + status: controller.requestTeleport({ + requestId: request.requestId, + expectedGeneration: request.expectedGeneration, + }), + }; + } else if (request.operation === 'rollback') { + response = { ok: true, status: await controller.rollback() }; + } else { + throw new Error('Unsupported Codex controller operation.'); + } + } catch (error) { + response = { ok: false, error: describeError(error) }; + } + socket.end(`${JSON.stringify(response)}\n`); + }); + }); + }); +} + +async function sendSocketControl(socketPath: string, request: ControlRequest): Promise { + return new Promise((resolve, reject) => { + const socket = net.createConnection(socketPath); + socket.setEncoding('utf8'); + socket.setTimeout(5_000); + let buffer = ''; + socket.once('connect', () => socket.write(`${JSON.stringify(request)}\n`)); + socket.on('data', (chunk: string) => { + buffer += chunk; + const newline = buffer.indexOf('\n'); + if (newline < 0) return; + socket.destroy(); + try { + resolve(JSON.parse(buffer.slice(0, newline)) as ControlResponse); + } catch (error) { + reject(new Error('Codex controller returned an invalid response.', { cause: error })); + } + }); + socket.once('timeout', () => { + socket.destroy(); + reject(new Error('Timed out contacting the active Codex controller.')); + }); + socket.once('error', (error) => + reject(new Error('No active Relay-managed Codex session.', { cause: error })) + ); + }); +} + +function withDefaults(overrides: Partial = {}): CodexCommandDependencies { + const paths = codexControllerPaths(); + const relayfileLifecycle = createRelayfileSealLifecycle(); + const liveTeleportStartupSwitch = Object.freeze({ + ...(overrides.liveTeleportStartupSwitch ?? DEFAULT_CODEX_LIVE_TELEPORT_STARTUP_SWITCH), + }); + const inputStream = overrides.input ?? process.stdin; + const checkpointAndSeal = overrides.checkpointAndSeal ?? relayfileLifecycle.checkpointAndSeal; + const resumePersistedLocalMount = + overrides.resumePersistedLocalMount ?? relayfileLifecycle.resumePersistedLocalMount; + return { + runManaged: + overrides.runManaged ?? + (async (options) => { + const workspaceRoot = fs.realpathSync(path.resolve(options.cwd)); + const liveTeleportEnabled = liveTeleportStartupSwitch.enabled; + if (liveTeleportEnabled) await probeCodexEnvironmentCapability('codex'); + fs.mkdirSync(paths.directory, { recursive: true, mode: 0o700 }); + const store = new FileCodexControllerStateStore(paths.statePath); + const prior = store.read(); + if (prior && fs.existsSync(paths.socketPath) && processIsAlive(prior.controllerPid)) { + throw new Error( + `Relay-managed Codex thread ${prior.threadId} is already controlled by process ${prior.controllerPid}.` + ); + } + const needsCloud = + liveTeleportEnabled || Boolean(prior && codexControllerStateMayOwnCloudResources(prior)); + const cloud = needsCloud + ? await (async () => { + const session = await ensureCloudSession({ interactive: true }); + return new CloudLiveTeleportClient( + (requestPath, init) => session.client.fetch(requestPath, init), + session.client.snapshot().apiUrl + ); + })() + : undefined; + + const controller = new CodexLiveController( + { + workspaceRoot, + socketPath: paths.socketPath, + liveTeleportStartupSwitch, + ...(options.model ? { model: options.model } : {}), + }, + { + ...(cloud ? { cloud } : {}), + store, + createAppServer: async () => + StdioCodexAppServerSession.spawn({ + cwd: workspaceRoot, + onNotification: (notification) => { + // Live notifications are best-effort display hints, not the + // durable delivery ACK. If a terminal notification is lost, + // restart recovery writes the full answer at least once; it + // may therefore duplicate a prefix already emitted here. + // The underlying model turn is never submitted again. + if (options.json) process.stdout.write(`${JSON.stringify(notification)}\n`); + else { + const delta = agentMessageDelta(notification); + if (delta) process.stdout.write(delta); + } + }, + }), + // Enabled startup probes before interactive Cloud login. Disabled + // recovery only fences prior ownership and does not need Codex's + // environment-add schema. Keeping this seam injectable lets + // restart/adversarial tests prove enabled startup fails closed. + probeCapability: async () => undefined, + checkpointAndSeal, + resumePersistedLocalMount, + sleep: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)), + now: () => new Date(), + sessionId: randomUUID, + operationId: randomUUID, + pid: process.pid, + } + ); + + try { + fs.unlinkSync(paths.socketPath); + } catch (error) { + if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error; + } + const server = createControlServer(controller); + try { + await withCodexControllerShutdown( + controller, + async () => { + const status = await controller.initialize(); + await surfaceRecoveredCodexTerminal(controller, { + json: Boolean(options.json), + writeError: nodeCallbackWriter(process.stderr), + }); + await listen(server, paths.socketPath); + process.stderr.write( + `Relay-managed Codex ${status.threadId} is ready locally (generation ${status.generation}).\n` + ); + const recoveredTurn = controller.takeRecoveredTurn(); + if (recoveredTurn) { + await writeReconciledTurn(recoveredTurn, { + json: Boolean(options.json), + writeOutput: nodeCallbackWriter(process.stdout), + }); + controller.acknowledgeRecoveredOutcome(); + } + + if (options.prompt) { + await runManagedCodexTurn(controller, options.prompt, { + json: Boolean(options.json), + writeError: nodeCallbackWriter(process.stderr), + writeOutput: nodeCallbackWriter(process.stdout), + }); + } + const input = readline.createInterface({ + input: inputStream, + terminal: Boolean(inputStream.isTTY), + }); + for await (const line of input) { + if (!line.trim()) continue; + await runManagedCodexTurn(controller, line, { + json: Boolean(options.json), + writeError: nodeCallbackWriter(process.stderr), + writeOutput: nodeCallbackWriter(process.stdout), + }); + if (!options.json) process.stdout.write('\n'); + } + }, + () => closeServer(server) + ); + } finally { + try { + fs.unlinkSync(paths.socketPath); + } catch { + // Already gone. + } + } + }), + liveTeleportStartupSwitch, + checkpointAndSeal, + resumePersistedLocalMount, + readState: + overrides.readState ?? + (() => { + try { + return new FileCodexControllerStateStore(paths.statePath).read(); + } catch { + return null; + } + }), + sendControl: overrides.sendControl ?? ((request) => sendSocketControl(paths.socketPath, request)), + requestId: overrides.requestId ?? randomUUID, + cwd: overrides.cwd ?? (() => process.cwd()), + input: inputStream, + log: overrides.log ?? ((message) => console.log(message)), + }; +} + +function processIsAlive(pid: number): boolean { + if (!Number.isSafeInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch (error) { + return Boolean(error instanceof Error && 'code' in error && error.code === 'EPERM'); + } +} + +function requireActiveState(deps: CodexCommandDependencies): CodexControllerState { + const state = deps.readState(); + if (!state) throw new Error('No active Relay-managed Codex session. Start one with `relay codex run`.'); + return state; +} + +function requireSuccess(response: ControlResponse): PublicCodexControllerStatus { + if (!response.ok) throw new Error(response.error); + return response.status; +} + +export function registerCodexCommands( + program: Command, + overrides: Partial = {} +): void { + const deps = withDefaults(overrides); + const group = program + .command('codex') + .description('Run and relocate a Relay-managed local Codex app-server session'); + + group + .command('run') + .description('Run a long-lived local Codex app-server and thread under Relay control') + .argument('[prompt...]', 'Optional first turn') + .option('--cwd ', 'Managed workspace root') + .option('--model ', 'Codex model') + .option('--json', 'Write raw app-server notifications as JSON lines') + .action(async (prompt: string[], options: { cwd?: string; model?: string; json?: boolean }) => { + await deps.runManaged({ + cwd: options.cwd ?? deps.cwd(), + ...(options.model ? { model: options.model } : {}), + ...(prompt.length ? { prompt: prompt.join(' ') } : {}), + ...(options.json ? { json: true } : {}), + }); + }); + + group + .command('teleport') + .description('Move execution for the active managed Codex session to Cloud at the next turn boundary') + .action(async () => { + const state = requireActiveState(deps); + const status = requireSuccess( + await deps.sendControl({ + operation: 'teleport', + requestId: deps.requestId(), + expectedGeneration: state.generation, + }) + ); + deps.log( + `Codex execution teleport queued for generation ${status.generation}; the next prompt runs in Cloud after cutover, or locally after confirmed pre-submission recovery.` + ); + }); + + group + .command('rollback') + .description('Revoke Cloud execution and resume the same thread through a fresh local controller') + .action(async () => { + requireActiveState(deps); + const status = requireSuccess(await deps.sendControl({ operation: 'rollback' })); + deps.log(`Codex thread ${status.threadId} resumed locally at generation ${status.generation}.`); + }); + + group + .command('status') + .description('Show where execution runs and where the conversation controller lives') + .option('--json', 'Write machine-readable JSON') + .action(async (options: { json?: boolean }) => { + requireActiveState(deps); + const status = requireSuccess(await deps.sendControl({ operation: 'status' })); + if (options.json) deps.log(JSON.stringify(status)); + else { + deps.log(`Execution: ${status.execution}`); + deps.log('Controller: local (keep this process and laptop running)'); + deps.log( + `Live teleport: ${status.liveTeleport.enabled ? 'enabled' : 'disabled'} (${status.liveTeleport.reason})` + ); + deps.log(`Thread: ${status.threadId}`); + deps.log(`Generation: ${status.generation}`); + } + }); +} diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 57f63189e..2ebfdeca5 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -670,6 +670,7 @@ describe('fleet command support', () => { relayWorkspaceId: 'rw_abc', relayfileMounted: true, relayfileMountPath: '/workspace', + providerId: 'agent37' as const, })); const deleteCloudFleetSandbox = vi.fn(async () => undefined); const logs: string[] = []; @@ -715,6 +716,7 @@ describe('fleet command support', () => { maxAgents: 1, mountRelayfile: true, forceProvision: true, + workloadProfile: 'long-running-agent', waitTimeoutMs: 90_000, name: 'daytona-codex', }); @@ -749,7 +751,11 @@ describe('fleet command support', () => { ); expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); expect(JSON.parse(logs[0]!)).toMatchObject({ - sandbox: { nodeName: 'daytona-codex', relayfileMountPath: '/workspace' }, + sandbox: { + nodeName: 'daytona-codex', + relayfileMountPath: '/workspace', + providerId: 'agent37', + }, invocation: { invocationId: 'inv_sandbox' }, attachCommand: "agent-relay node agent attach 'sandbox-worker' --node 'daytona-codex' --mode drive", }); @@ -783,6 +789,7 @@ describe('fleet command support', () => { relayWorkspaceId: 'rw_abc', relayfileMounted: true, relayfileMountPath: '/workspace', + providerId: 'agent37' as const, })), deleteCloudFleetSandbox, createFleetWorkspaceClient: vi.fn() as never, @@ -962,6 +969,7 @@ describe('fleet command support', () => { sandboxId: 'sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: false, + providerId: 'agent37' as const, })), deleteCloudFleetSandbox: vi.fn(async () => Promise.reject(new Error('delete failed'))), createFleetWorkspaceClient: vi.fn() as never, diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 9c20f19f0..05e2af731 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -236,6 +236,7 @@ export function registerFleetCommands( maxAgents: 1, mountRelayfile: mountSandboxRelayfile, forceProvision: true, + workloadProfile: 'long-running-agent', waitTimeoutMs: 90_000, name: requestedSandboxName, }); @@ -248,7 +249,7 @@ export function registerFleetCommands( }) .catch((cleanupError) => { deps.warn( - `Provisioning failed after Daytona created sandbox '${error.sandboxId}', and automatic cleanup failed: ${ + `Provisioning failed after Cloud created sandbox '${error.sandboxId}', and automatic cleanup failed: ${ cleanupError instanceof Error ? cleanupError.message : String(cleanupError) }` ); @@ -257,7 +258,7 @@ export function registerFleetCommands( deps.warn( `Cloud did not return a complete provisioning response. The outcome is unknown; check Cloud Fleet for node '${ error.nodeName ?? requestedSandboxName - }' before retrying so a Daytona sandbox is not left running.` + }' before retrying so a Cloud sandbox is not left running.` ); } throw error; @@ -276,7 +277,7 @@ export function registerFleetCommands( ); }); throw new Error( - `Daytona node '${sandbox.nodeName}' did not become ready within ${sandbox.waitedMs}ms.` + `${sandbox.providerId} node '${sandbox.nodeName}' did not become ready within ${sandbox.waitedMs}ms.` ); } if ( @@ -297,7 +298,7 @@ export function registerFleetCommands( ); }); } - throw new Error('Cloud returned a Daytona node without the required Relayfile mount.'); + throw new Error('Cloud returned a sandbox node without the required Relayfile mount.'); } targetNode = sandbox.nodeName; if (!workerCwd && sandbox.outcome === 'provisioned' && sandbox.relayfileMounted) { diff --git a/packages/cli/src/cli/lib/codex-app-server.test.ts b/packages/cli/src/cli/lib/codex-app-server.test.ts new file mode 100644 index 000000000..4ce744a2d --- /dev/null +++ b/packages/cli/src/cli/lib/codex-app-server.test.ts @@ -0,0 +1,556 @@ +import { EventEmitter } from 'node:events'; +import { PassThrough } from 'node:stream'; +import type { ChildProcessWithoutNullStreams } from 'node:child_process'; +import { describe, expect, it, vi } from 'vitest'; + +import { probeCodexEnvironmentCapability, StdioCodexAppServerSession } from './codex-app-server.js'; + +function fakeChild() { + const child = Object.assign(new EventEmitter(), { + stdin: new PassThrough(), + stdout: new PassThrough(), + stderr: new PassThrough(), + exitCode: null as number | null, + kill: vi.fn(() => { + queueMicrotask(() => { + child.exitCode = 0; + child.emit('exit', 0, null); + }); + return true; + }), + }); + return child as unknown as ChildProcessWithoutNullStreams; +} + +async function nextRequest(child: ChildProcessWithoutNullStreams): Promise> { + return new Promise((resolve) => { + child.stdin.once('data', (chunk: Buffer) => resolve(JSON.parse(chunk.toString('utf8').trim()))); + }); +} + +function turnPolicySchema(): string { + return JSON.stringify({ + properties: { + approvalPolicy: { $ref: '#/definitions/approval' }, + sandboxPolicy: { + oneOf: [{ $ref: '#/definitions/workspace' }, { $ref: '#/definitions/remote' }], + }, + clientUserMessageId: {}, + }, + definitions: { + approval: { enum: ['never'] }, + workspace: { + properties: { writableRoots: {}, type: { enum: ['workspaceWrite'] } }, + }, + remote: { properties: { type: { enum: ['dangerFullAccess'] } } }, + }, + }); +} + +function turnsListParamsSchema(): string { + return JSON.stringify({ + properties: { + itemsView: { $ref: '#/definitions/items' }, + sortDirection: { $ref: '#/definitions/sort' }, + }, + definitions: { items: { enum: ['full'] }, sort: { enum: ['desc'] } }, + }); +} + +function turnsListResponseSchema(): string { + return JSON.stringify({ + properties: { data: { $ref: '#/definitions/data' } }, + definitions: { + data: { + properties: { + clientId: {}, + text: {}, + type: { enum: ['userMessage', 'agentMessage'] }, + status: { enum: ['completed', 'interrupted', 'failed', 'inProgress'] }, + }, + }, + }, + }); +} + +function turnCompletedSchema(): string { + return JSON.stringify({ + required: ['threadId', 'turn'], + properties: { threadId: { type: 'string' }, turn: { $ref: '#/definitions/turn' } }, + definitions: { + turn: { + required: ['id', 'items', 'status'], + properties: { + id: { type: 'string' }, + items: { type: 'array' }, + status: { enum: ['completed', 'failed', 'interrupted', 'inProgress'] }, + }, + }, + }, + }); +} + +function turnPayload( + status: 'completed' | 'failed' | 'interrupted' | 'inProgress' = 'completed', + id = 'turn-1', + clientId = 'client-turn-1' +) { + return { + id, + status, + itemsView: 'full', + items: [ + { id: `user-${id}`, type: 'userMessage', clientId, content: [] }, + { id: `agent-${id}`, type: 'agentMessage', text: 'observable assistant answer' }, + ], + }; +} + +function successfulProbeFile(file: string): string { + if (file.endsWith('ClientRequest.json')) { + return '{"methods":["environment/add","environment/status","thread/turns/list"]}'; + } + if (file.endsWith('TurnStartParams.json')) return turnPolicySchema(); + if (file.endsWith('ThreadTurnsListParams.json')) return turnsListParamsSchema(); + if (file.endsWith('ThreadTurnsListResponse.json')) return turnsListResponseSchema(); + if (file.endsWith('TurnCompletedNotification.json')) return turnCompletedSchema(); + return '{"required":["environmentId","execServerUrl"],"properties":{"environmentId":{},"execServerUrl":{}}}'; +} + +describe('probeCodexEnvironmentCapability', () => { + it('requires add and status in the locally generated experimental schema', async () => { + const readFile = vi.fn(async (file: string) => successfulProbeFile(file)); + await expect( + probeCodexEnvironmentCapability('codex', { + makeTempDir: async () => '/schema', + execFile: async () => undefined, + readFile, + remove: async () => undefined, + }) + ).resolves.toMatchObject({ + environmentAdd: true, + environmentStatus: true, + explicitTurnPolicy: true, + }); + }); + + it('fails closed when environment/status is unsupported', async () => { + const remove = vi.fn(async () => undefined); + await expect( + probeCodexEnvironmentCapability('codex', { + makeTempDir: async () => '/schema', + execFile: async () => undefined, + readFile: async (file) => { + if (file.endsWith('ClientRequest.json')) return '{"methods":["environment/add"]}'; + return successfulProbeFile(file); + }, + remove, + }) + ).rejects.toThrow('does not expose both'); + expect(remove).toHaveBeenCalledWith('/schema'); + }); + + it('rejects schema drift that could move provider credentials into Relay', async () => { + await expect( + probeCodexEnvironmentCapability('codex', { + makeTempDir: async () => '/schema', + execFile: async () => undefined, + readFile: async (file) => { + if (!file.endsWith('EnvironmentAddParams.json')) return successfulProbeFile(file); + return '{"required":["environmentId","execServerUrl"],"properties":{"headers":{}}}'; + }, + remove: async () => undefined, + }) + ).rejects.toThrow('credential field'); + }); + + it('fails closed when turn/start cannot pin approval and sandbox policy', async () => { + await expect( + probeCodexEnvironmentCapability('codex', { + makeTempDir: async () => '/schema', + execFile: async () => undefined, + readFile: async (file) => { + if (file.endsWith('ClientRequest.json')) return successfulProbeFile(file); + if (file.endsWith('TurnStartParams.json')) return '{"properties":{}}'; + return successfulProbeFile(file); + }, + remove: async () => undefined, + }) + ).rejects.toThrow('execution-policy contract'); + }); + + it('fails closed when full turn-history reconciliation is unavailable', async () => { + await expect( + probeCodexEnvironmentCapability('codex', { + makeTempDir: async () => '/schema', + execFile: async () => undefined, + readFile: async (file) => { + if (file.endsWith('ThreadTurnsListResponse.json')) return '{"properties":{"data":{}}}'; + return successfulProbeFile(file); + }, + remove: async () => undefined, + }) + ).rejects.toThrow('turn-reconciliation contract'); + }); + + it('fails closed if turn/completed does not carry generated terminal status and items', async () => { + await expect( + probeCodexEnvironmentCapability('codex', { + makeTempDir: async () => '/schema', + execFile: async () => undefined, + readFile: async (file) => { + if (file.endsWith('TurnCompletedNotification.json')) { + return JSON.stringify({ + properties: { turn: { $ref: '#/definitions/turn' } }, + definitions: { + turn: { + required: ['id', 'status'], + properties: { + id: {}, + status: { enum: ['completed', 'failed', 'interrupted', 'inProgress'] }, + }, + }, + }, + }); + } + return successfulProbeFile(file); + }, + remove: async () => undefined, + }) + ).rejects.toThrow('terminal-turn contract'); + }); +}); + +describe('StdioCodexAppServerSession', () => { + it('uses Codex newline RPC without a jsonrpc field and opts into the experimental API', async () => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child); + const requestPromise = nextRequest(child); + const initializing = session.initialize(); + const request = await requestPromise; + + expect(request).toEqual({ + id: 1, + method: 'initialize', + params: { + clientInfo: { name: 'agent-relay', title: 'Agent Relay managed Codex', version: '1' }, + capabilities: { experimentalApi: true }, + }, + }); + expect(request).not.toHaveProperty('jsonrpc'); + child.stdout.write(`${JSON.stringify({ id: 1, result: {} })}\n`); + await expect(initializing).resolves.toBeUndefined(); + await session.close(); + }); + + it('uses never + provider-isolated full access for a selected Cloud environment', async () => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child); + const requestPromise = nextRequest(child); + const running = session.runTurn({ + threadId: 'thread-1', + text: 'continue', + clientUserMessageId: 'client-turn-1', + execution: { + kind: 'remote', + environment: { environmentId: 'environment-3', cwd: '/workspace' }, + }, + }); + const request = await requestPromise; + expect(request).toMatchObject({ + id: 1, + method: 'turn/start', + params: { + threadId: 'thread-1', + clientUserMessageId: 'client-turn-1', + approvalPolicy: 'never', + sandboxPolicy: { type: 'dangerFullAccess' }, + environments: [{ environmentId: 'environment-3', cwd: '/workspace' }], + }, + }); + child.stdout.write(`${JSON.stringify({ id: 1, result: { turn: { id: 'turn-1' } } })}\n`); + child.stdout.write( + `${JSON.stringify({ method: 'turn/completed', params: { threadId: 'thread-1', turn: turnPayload() } })}\n` + ); + await expect(running).resolves.toMatchObject({ turnId: 'turn-1' }); + await session.close(); + }); + + it('reconciles a persisted client user message through full turn history', async () => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child); + const requestPromise = nextRequest(child); + const reconciling = session.turnOutcome({ + threadId: 'thread-1', + clientUserMessageId: 'client-turn-1', + }); + const request = await requestPromise; + expect(request).toMatchObject({ + method: 'thread/turns/list', + params: { + threadId: 'thread-1', + itemsView: 'full', + sortDirection: 'desc', + limit: 100, + }, + }); + child.stdout.write( + `${JSON.stringify({ + id: request.id, + result: { + data: [ + { + ...turnPayload(), + }, + ], + nextCursor: null, + }, + })}\n` + ); + await expect(reconciling).resolves.toMatchObject({ + status: 'completed', + turnId: 'turn-1', + result: { + response: { + turn: { + items: [ + expect.objectContaining({ type: 'userMessage' }), + expect.objectContaining({ type: 'agentMessage', text: 'observable assistant answer' }), + ], + }, + }, + completed: { + method: 'turn/completed', + params: { + turn: expect.objectContaining({ status: 'completed' }), + }, + }, + }, + }); + await session.close(); + }); + + it('fails completed reconciliation when full history has no recoverable assistant answer', async () => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child); + const requestPromise = nextRequest(child); + const reconciling = session.turnOutcome({ + threadId: 'thread-1', + clientUserMessageId: 'client-turn-1', + }); + const request = await requestPromise; + child.stdout.write( + `${JSON.stringify({ + id: request.id, + result: { + data: [ + { + ...turnPayload(), + items: [{ id: 'user-1', type: 'userMessage', clientId: 'client-turn-1', content: [] }], + }, + ], + nextCursor: null, + }, + })}\n` + ); + + await expect(reconciling).rejects.toThrow('could not recover the completed assistant answer exactly'); + await session.close(); + }); + + it.each(['completed', 'failed', 'interrupted', 'inProgress'] as const)( + 'interprets generated turn/completed status %s without collapsing it to success', + async (status) => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child); + const requestPromise = nextRequest(child); + const running = session.runTurn({ + threadId: 'thread-1', + text: 'status test', + clientUserMessageId: 'client-turn-1', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }); + const request = await requestPromise; + child.stdout.write(`${JSON.stringify({ id: request.id, result: { turn: { id: 'turn-1' } } })}\n`); + child.stdout.write( + `${JSON.stringify({ + method: 'turn/completed', + params: { threadId: 'thread-1', turn: turnPayload(status) }, + })}\n` + ); + + if (status === 'completed') { + await expect(running).resolves.toMatchObject({ turnId: 'turn-1' }); + } else if (status === 'failed' || status === 'interrupted') { + await expect(running).rejects.toMatchObject({ + name: 'CodexAppServerTurnTerminalError', + status, + turnId: 'turn-1', + }); + } else { + await expect(running).rejects.toThrow('non-terminal status inProgress'); + } + await session.close(); + } + ); + + it('pins local turns to a non-networked workspace sandbox and rejects an unexpected approval request', async () => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child); + const requestPromise = nextRequest(child); + const running = session.runTurn({ + threadId: 'thread-1', + text: 'edit the workspace', + clientUserMessageId: 'client-turn-local', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }); + const request = await requestPromise; + expect(request).toMatchObject({ + method: 'turn/start', + params: { + approvalPolicy: 'never', + clientUserMessageId: 'client-turn-local', + sandboxPolicy: { + type: 'workspaceWrite', + writableRoots: ['/repo'], + networkAccess: false, + }, + environments: [], + }, + }); + + const approvalResponse = nextRequest(child); + child.stdout.write( + `${JSON.stringify({ id: 91, method: 'item/commandExecution/requestApproval', params: {} })}\n` + ); + await expect(approvalResponse).resolves.toEqual({ + id: 91, + error: { code: -32601, message: 'Client request not supported' }, + }); + + child.stdout.write(`${JSON.stringify({ id: request.id, result: { turnId: 'turn-local' } })}\n`); + child.stdout.write( + `${JSON.stringify({ + method: 'turn/completed', + params: { + threadId: 'thread-1', + turn: turnPayload('completed', 'turn-local', 'client-turn-local'), + }, + })}\n` + ); + await expect(running).resolves.toMatchObject({ turnId: 'turn-local' }); + await session.close(); + }); + + it('rejects app-server initiated numbered requests instead of leaving them hanging', async () => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child); + const responsePromise = nextRequest(child); + + child.stdout.write(`${JSON.stringify({ id: 91, method: 'item/tool/request', params: {} })}\n`); + + await expect(responsePromise).resolves.toEqual({ + id: 91, + error: { code: -32601, message: 'Client request not supported' }, + }); + await session.close(); + }); + + it.each([ + ['malformed JSON', '{definitely-not-json}\n'], + ['non-object JSON', '[]\n'], + ['oversized unterminated frame', 'x'.repeat(65)], + ])('fails closed on %s', async (_name, frame) => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child, 30_000, 30_000, undefined, { + maxFrameBytes: 64, + closeTimeoutMs: 10, + forceKillTimeoutMs: 10, + }); + + child.stdout.write(frame); + await vi.waitFor(() => expect(child.kill).toHaveBeenCalledWith('SIGTERM')); + await expect(session.initialize()).rejects.toThrow('not running'); + await session.close(); + }); + + it('fails closed when the notification queue reaches its bound', async () => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child, 30_000, 30_000, undefined, { + maxBufferedNotifications: 1, + closeTimeoutMs: 10, + forceKillTimeoutMs: 10, + }); + + child.stdout.write( + `${JSON.stringify({ method: 'turn/completed', params: { turn: turnPayload('completed', 'one') } })}\n` + ); + child.stdout.write( + `${JSON.stringify({ method: 'turn/completed', params: { turn: turnPayload('completed', 'two') } })}\n` + ); + + await vi.waitFor(() => expect(child.kill).toHaveBeenCalledWith('SIGTERM')); + await session.close(); + }); + + it('forwards and drops more than 1024 observed deltas without starving turn/completed', async () => { + const child = fakeChild(); + const onNotification = vi.fn(); + const session = new StdioCodexAppServerSession(child, 30_000, 30_000, onNotification, { + maxBufferedNotifications: 2, + closeTimeoutMs: 10, + forceKillTimeoutMs: 10, + }); + const requestPromise = nextRequest(child); + const running = session.runTurn({ + threadId: 'thread-1', + text: 'long turn', + clientUserMessageId: 'client-turn-long', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }); + const request = await requestPromise; + + for (let index = 0; index < 1_100; index += 1) { + child.stdout.write( + `${JSON.stringify({ method: 'item/agentMessage/delta', params: { delta: String(index) } })}\n` + ); + } + child.stdout.write(`${JSON.stringify({ id: request.id, result: { turnId: 'turn-long' } })}\n`); + child.stdout.write( + `${JSON.stringify({ + method: 'turn/completed', + params: { + threadId: 'thread-1', + turn: turnPayload('completed', 'turn-long', 'client-turn-long'), + }, + })}\n` + ); + + await expect(running).resolves.toMatchObject({ turnId: 'turn-long' }); + expect(onNotification).toHaveBeenCalledTimes(1_101); + expect(child.kill).not.toHaveBeenCalled(); + await session.close(); + }); + + it('waits for exit and escalates to SIGKILL before allowing replacement', async () => { + const child = fakeChild(); + vi.mocked(child.kill).mockImplementation((signal) => { + if (signal === 'SIGKILL') { + queueMicrotask(() => { + (child as unknown as { exitCode: number | null }).exitCode = 137; + child.emit('exit', null, 'SIGKILL'); + }); + } + return true; + }); + const session = new StdioCodexAppServerSession(child, 30_000, 30_000, undefined, { + closeTimeoutMs: 1, + forceKillTimeoutMs: 20, + }); + + await session.close(); + + expect(child.kill).toHaveBeenNthCalledWith(1, 'SIGTERM'); + expect(child.kill).toHaveBeenNthCalledWith(2, 'SIGKILL'); + }); +}); diff --git a/packages/cli/src/cli/lib/codex-app-server.ts b/packages/cli/src/cli/lib/codex-app-server.ts new file mode 100644 index 000000000..ac6e54f56 --- /dev/null +++ b/packages/cli/src/cli/lib/codex-app-server.ts @@ -0,0 +1,776 @@ +import fsp from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { execFile, spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +export type CodexNotification = { + method: string; + params?: unknown; +}; + +export type CodexTurnResult = { + turnId?: string; + response: unknown; + completed: CodexNotification; + /** Relay reconstructed this exact completion from full persisted turn history. */ + reconciled?: true; +}; + +export type CodexTurnOutcome = + | { status: 'completed'; turnId: string; result: CodexTurnResult } + | { status: 'failed' | 'interrupted' | 'inProgress'; turnId: string } + | { status: 'absent' }; + +export class CodexAppServerTurnTerminalError extends Error { + constructor( + readonly status: 'failed' | 'interrupted', + readonly turnId: string, + readonly completed: CodexNotification + ) { + super(`Codex turn ended ${status}.`); + this.name = 'CodexAppServerTurnTerminalError'; + } +} + +export type CodexTurnExecution = + | { kind: 'local'; workspaceRoot: string } + | { + kind: 'remote'; + environment?: { environmentId: string; cwd: string }; + }; + +export interface CodexAppServerSession { + initialize(): Promise; + startThread(input: { cwd: string; model?: string }): Promise; + resumeThread(input: { threadId: string; cwd: string }): Promise; + addEnvironment(input: { + environmentId: string; + execServerUrl: string; + connectTimeoutMs: number; + }): Promise; + environmentStatus(environmentId: string): Promise; + runTurn(input: { + threadId: string; + text: string; + clientUserMessageId: string; + execution: CodexTurnExecution; + }): Promise; + turnOutcome(input: { threadId: string; clientUserMessageId: string }): Promise; + close(): Promise; +} + +export type CodexEnvironmentCapability = { + environmentAdd: true; + environmentStatus: true; + explicitTurnPolicy: true; +}; + +export type CodexCapabilityProbeDependencies = { + makeTempDir: () => Promise; + execFile: (file: string, args: string[]) => Promise; + readFile: (file: string) => Promise; + remove: (directory: string) => Promise; +}; + +function defaultCapabilityProbeDependencies(): CodexCapabilityProbeDependencies { + return { + makeTempDir: () => fsp.mkdtemp(path.join(os.tmpdir(), 'relay-codex-schema-')), + execFile: async (file, args) => { + await execFileAsync(file, args, { maxBuffer: 4 * 1024 * 1024 }); + }, + readFile: (file) => fsp.readFile(file, 'utf8'), + remove: (directory) => fsp.rm(directory, { recursive: true, force: true }), + }; +} + +function assertEnvironmentAddSchema(addParams: string): void { + const schema = JSON.parse(addParams) as { + required?: unknown; + properties?: Record; + }; + const required = Array.isArray(schema.required) ? schema.required : []; + if (!required.includes('environmentId') || !required.includes('execServerUrl')) { + throw new Error('Codex EnvironmentAddParams does not match the required execution-teleport schema.'); + } + if (schema.properties && ('headers' in schema.properties || 'token' in schema.properties)) { + throw new Error( + 'Codex EnvironmentAddParams unexpectedly contains a credential field; upgrade Relay first.' + ); + } +} + +function assertTurnPolicySchema(turnParams: string): void { + const turnSchema = JSON.parse(turnParams) as { + properties?: Record; + }; + if ( + !turnSchema.properties?.approvalPolicy || + !turnSchema.properties?.sandboxPolicy || + !turnSchema.properties?.clientUserMessageId || + !schemaNodeContainsEnum(turnSchema, turnSchema.properties.approvalPolicy, 'never') || + !schemaNodeContainsEnum(turnSchema, turnSchema.properties.sandboxPolicy, 'workspaceWrite') || + !schemaNodeContainsProperty(turnSchema, turnSchema.properties.sandboxPolicy, 'writableRoots') || + !schemaNodeContainsEnum(turnSchema, turnSchema.properties.sandboxPolicy, 'dangerFullAccess') + ) { + throw new Error("Codex TurnStartParams does not support Relay's explicit execution-policy contract."); + } +} + +function assertTurnReconciliationSchema(requests: string, listParams: string, listResponse: string): void { + const params = JSON.parse(listParams) as { properties?: Record }; + const response = JSON.parse(listResponse) as { properties?: Record }; + if ( + !requests.includes('"thread/turns/list"') || + !params.properties?.itemsView || + !params.properties?.sortDirection || + !response.properties?.data || + !schemaNodeContainsEnum(params, params.properties.itemsView, 'full') || + !schemaNodeContainsEnum(params, params.properties.sortDirection, 'desc') || + !schemaNodeContainsProperty(response, response.properties.data, 'clientId') || + !schemaNodeContainsEnum(response, response.properties.data, 'userMessage') || + !schemaNodeContainsEnum(response, response.properties.data, 'agentMessage') || + !schemaNodeContainsProperty(response, response.properties.data, 'text') || + !schemaNodeContainsEnum(response, response.properties.data, 'completed') || + !schemaNodeContainsEnum(response, response.properties.data, 'interrupted') || + !schemaNodeContainsEnum(response, response.properties.data, 'failed') || + !schemaNodeContainsEnum(response, response.properties.data, 'inProgress') + ) { + throw new Error("Codex thread/turns/list does not support Relay's turn-reconciliation contract."); + } +} + +function assertTurnCompletedSchema(completedNotification: string): void { + const notification = JSON.parse(completedNotification) as { + required?: unknown; + properties?: Record; + }; + const required = Array.isArray(notification.required) ? notification.required : []; + const turn = notification.properties?.turn; + if ( + !required.includes('threadId') || + !required.includes('turn') || + !turn || + !schemaNodeRequiresProperty(notification, turn, 'id') || + !schemaNodeRequiresProperty(notification, turn, 'items') || + !schemaNodeRequiresProperty(notification, turn, 'status') || + !schemaNodeContainsEnum(notification, turn, 'completed') || + !schemaNodeContainsEnum(notification, turn, 'failed') || + !schemaNodeContainsEnum(notification, turn, 'interrupted') || + !schemaNodeContainsEnum(notification, turn, 'inProgress') + ) { + throw new Error("Codex turn/completed does not match Relay's terminal-turn contract."); + } +} + +function resolveSchemaRef(root: unknown, value: unknown): unknown { + if (!value || typeof value !== 'object' || Array.isArray(value)) return value; + const ref = (value as Record).$ref; + if (typeof ref !== 'string' || !ref.startsWith('#/')) return value; + let cursor: unknown = root; + for (const segment of ref.slice(2).split('/')) { + if (!cursor || typeof cursor !== 'object' || Array.isArray(cursor)) return value; + cursor = (cursor as Record)[segment.replaceAll('~1', '/').replaceAll('~0', '~')]; + } + return cursor; +} + +function schemaNodeContainsEnum(root: unknown, value: unknown, expected: string): boolean { + const resolved = resolveSchemaRef(root, value); + if (Array.isArray(resolved)) { + return resolved.some((entry) => schemaNodeContainsEnum(root, entry, expected)); + } + if (!resolved || typeof resolved !== 'object') return false; + const object = resolved as Record; + if (Array.isArray(object.enum) && object.enum.includes(expected)) return true; + return Object.entries(object) + .filter(([key]) => key !== 'definitions') + .some(([, entry]) => schemaNodeContainsEnum(root, entry, expected)); +} + +function schemaNodeContainsProperty(root: unknown, value: unknown, expected: string): boolean { + const resolved = resolveSchemaRef(root, value); + if (Array.isArray(resolved)) { + return resolved.some((entry) => schemaNodeContainsProperty(root, entry, expected)); + } + if (!resolved || typeof resolved !== 'object') return false; + const object = resolved as Record; + if (object.properties && typeof object.properties === 'object' && expected in object.properties) { + return true; + } + return Object.entries(object) + .filter(([key]) => key !== 'definitions') + .some(([, entry]) => schemaNodeContainsProperty(root, entry, expected)); +} + +function schemaNodeRequiresProperty(root: unknown, value: unknown, expected: string): boolean { + const resolved = resolveSchemaRef(root, value); + if (Array.isArray(resolved)) { + return resolved.some((entry) => schemaNodeRequiresProperty(root, entry, expected)); + } + if (!resolved || typeof resolved !== 'object') return false; + const object = resolved as Record; + if (Array.isArray(object.required) && object.required.includes(expected)) return true; + return Object.entries(object) + .filter(([key]) => key !== 'definitions') + .some(([, entry]) => schemaNodeRequiresProperty(root, entry, expected)); +} + +/** + * Probe the locally installed binary rather than assuming an experimental + * protocol from Relay's build-time Codex version. Both methods and the exact + * no-header EnvironmentAddParams seam are required before a managed session + * can become teleport-capable. + */ +export async function probeCodexEnvironmentCapability( + codexBinary = 'codex', + overrides: Partial = {} +): Promise { + const deps = { ...defaultCapabilityProbeDependencies(), ...overrides }; + const directory = await deps.makeTempDir(); + try { + await deps.execFile(codexBinary, [ + 'app-server', + 'generate-json-schema', + '--experimental', + '--out', + directory, + ]); + const [requests, addParams, turnParams, turnsListParams, turnsListResponse, turnCompletedNotification] = + await Promise.all([ + deps.readFile(path.join(directory, 'ClientRequest.json')), + deps.readFile(path.join(directory, 'v2', 'EnvironmentAddParams.json')), + deps.readFile(path.join(directory, 'v2', 'TurnStartParams.json')), + deps.readFile(path.join(directory, 'v2', 'ThreadTurnsListParams.json')), + deps.readFile(path.join(directory, 'v2', 'ThreadTurnsListResponse.json')), + deps.readFile(path.join(directory, 'v2', 'TurnCompletedNotification.json')), + ]); + if (!requests.includes('"environment/add"') || !requests.includes('"environment/status"')) { + throw new Error( + 'This Codex app-server does not expose both experimental environment/add and environment/status.' + ); + } + + assertEnvironmentAddSchema(addParams); + assertTurnPolicySchema(turnParams); + assertTurnReconciliationSchema(requests, turnsListParams, turnsListResponse); + assertTurnCompletedSchema(turnCompletedNotification); + + return { environmentAdd: true, environmentStatus: true, explicitTurnPolicy: true }; + } finally { + await deps.remove(directory); + } +} + +type PendingRequest = { + resolve: (value: unknown) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; +}; + +export type CodexAppServerTransportLimits = { + maxFrameBytes?: number; + maxBufferedNotifications?: number; + closeTimeoutMs?: number; + forceKillTimeoutMs?: number; +}; + +const DEFAULT_MAX_FRAME_BYTES = 1024 * 1024; +const DEFAULT_MAX_BUFFERED_NOTIFICATIONS = 1024; +const DEFAULT_CLOSE_TIMEOUT_MS = 5_000; +const DEFAULT_FORCE_KILL_TIMEOUT_MS = 1_000; + +function isObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function stringAt(value: unknown, ...keys: string[]): string | undefined { + let cursor: unknown = value; + for (const key of keys) { + if (!isObject(cursor)) return undefined; + cursor = cursor[key]; + } + return typeof cursor === 'string' && cursor.trim() ? cursor.trim() : undefined; +} + +function notificationThreadId(notification: CodexNotification): string | undefined { + return ( + stringAt(notification.params, 'threadId') ?? + stringAt(notification.params, 'thread', 'id') ?? + stringAt(notification.params, 'turn', 'threadId') + ); +} + +function notificationTurnId(notification: CodexNotification): string | undefined { + return stringAt(notification.params, 'turnId') ?? stringAt(notification.params, 'turn', 'id'); +} + +type CodexTurnPayload = Record & { + id: string; + items: unknown[]; + status: 'completed' | 'failed' | 'interrupted' | 'inProgress'; +}; + +function requiredTurnPayload(value: unknown, context: string): CodexTurnPayload { + if (!isObject(value) || typeof value.id !== 'string' || !value.id.trim() || !Array.isArray(value.items)) { + throw new Error(`Codex ${context} returned an invalid turn payload.`); + } + if ( + value.status !== 'completed' && + value.status !== 'failed' && + value.status !== 'interrupted' && + value.status !== 'inProgress' + ) { + throw new Error(`Codex ${context} returned an invalid turn status.`); + } + return value as CodexTurnPayload; +} + +function hasExactAssistantAnswer(turn: CodexTurnPayload): boolean { + if (turn.itemsView !== undefined && turn.itemsView !== 'full') return false; + return turn.items.some( + (item) => isObject(item) && item.type === 'agentMessage' && typeof item.text === 'string' + ); +} + +/** Newline-delimited Codex app-server transport. Codex omits the jsonrpc field. */ +export class StdioCodexAppServerSession implements CodexAppServerSession { + private readonly pending = new Map(); + private readonly notifications: CodexNotification[] = []; + private readonly notificationWaiters = new Set<{ + predicate: (notification: CodexNotification) => boolean; + resolve: (notification: CodexNotification) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; + }>(); + private nextId = 1; + private buffer = ''; + private closed = false; + private exited = false; + private shutdownStarted = false; + private writeChain = Promise.resolve(); + private readonly exitPromise: Promise; + private readonly maxFrameBytes: number; + private readonly maxBufferedNotifications: number; + private readonly closeTimeoutMs: number; + private readonly forceKillTimeoutMs: number; + + constructor( + private readonly child: ChildProcessWithoutNullStreams, + private readonly requestTimeoutMs = 30_000, + private readonly turnTimeoutMs = 30 * 60_000, + private readonly onNotification?: (notification: CodexNotification) => void, + limits: CodexAppServerTransportLimits = {} + ) { + this.maxFrameBytes = limits.maxFrameBytes ?? DEFAULT_MAX_FRAME_BYTES; + this.maxBufferedNotifications = limits.maxBufferedNotifications ?? DEFAULT_MAX_BUFFERED_NOTIFICATIONS; + this.closeTimeoutMs = limits.closeTimeoutMs ?? DEFAULT_CLOSE_TIMEOUT_MS; + this.forceKillTimeoutMs = limits.forceKillTimeoutMs ?? DEFAULT_FORCE_KILL_TIMEOUT_MS; + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk: string) => this.onData(chunk)); + child.stderr.setEncoding('utf8'); + // Drain stderr so the child cannot block, but never reflect provider or + // model diagnostics into persisted/public controller errors. + child.stderr.on('data', () => undefined); + this.exitPromise = new Promise((resolve) => { + child.once('exit', (code, signal) => { + this.exited = true; + this.closed = true; + this.rejectAll( + new Error(`Codex app-server exited before completing the request (${code ?? signal ?? 'unknown'}).`) + ); + resolve(); + }); + }); + child.stdin.once('error', (error) => this.failProtocol(`stdin write failed: ${error.message}`)); + } + + static spawn(options: { + codexBinary?: string; + cwd: string; + onNotification?: (notification: CodexNotification) => void; + }): StdioCodexAppServerSession { + const child = spawn(options.codexBinary ?? 'codex', ['app-server', '--stdio'], { + cwd: options.cwd, + stdio: ['pipe', 'pipe', 'pipe'], + env: process.env, + }); + return new StdioCodexAppServerSession(child, 30_000, 30 * 60_000, options.onNotification); + } + + async initialize(): Promise { + await this.request('initialize', { + clientInfo: { name: 'agent-relay', title: 'Agent Relay managed Codex', version: '1' }, + capabilities: { experimentalApi: true }, + }); + } + + async startThread(input: { cwd: string; model?: string }): Promise { + const result = await this.request('thread/start', { + cwd: input.cwd, + ...(input.model ? { model: input.model } : {}), + }); + const threadId = + stringAt(result, 'threadId') ?? stringAt(result, 'thread', 'id') ?? stringAt(result, 'id'); + if (!threadId) throw new Error('Codex thread/start returned no thread id.'); + return threadId; + } + + async resumeThread(input: { threadId: string; cwd: string }): Promise { + await this.request('thread/resume', { threadId: input.threadId, cwd: input.cwd }); + } + + async addEnvironment(input: { + environmentId: string; + execServerUrl: string; + connectTimeoutMs: number; + }): Promise { + await this.request('environment/add', input); + } + + environmentStatus(environmentId: string): Promise { + return this.request('environment/status', { environmentId }); + } + + async runTurn(input: { + threadId: string; + text: string; + clientUserMessageId: string; + execution: CodexTurnExecution; + }): Promise { + let executionParams: Record; + if (input.execution.kind === 'remote') { + executionParams = { + approvalPolicy: 'never', + sandboxPolicy: { type: 'dangerFullAccess' }, + ...(input.execution.environment ? { environments: [input.execution.environment] } : {}), + }; + } else { + const workspaceRoot = input.execution.workspaceRoot; + if (!path.isAbsolute(workspaceRoot) || path.normalize(workspaceRoot) !== workspaceRoot) { + throw new Error('Local Codex execution requires an absolute normalized workspace root.'); + } + executionParams = { + approvalPolicy: 'never', + sandboxPolicy: { + type: 'workspaceWrite', + writableRoots: [workspaceRoot], + networkAccess: false, + }, + // Clear any sticky remote environment before a recovered local turn. + environments: [], + }; + } + const response = await this.request( + 'turn/start', + { + threadId: input.threadId, + clientUserMessageId: input.clientUserMessageId, + input: [{ type: 'text', text: input.text }], + ...executionParams, + }, + this.turnTimeoutMs + ); + const turnId = + stringAt(response, 'turnId') ?? stringAt(response, 'turn', 'id') ?? stringAt(response, 'id'); + const completed = await this.waitForNotification( + (notification) => + notification.method === 'turn/completed' && + (!notificationThreadId(notification) || notificationThreadId(notification) === input.threadId) && + (!turnId || !notificationTurnId(notification) || notificationTurnId(notification) === turnId), + this.turnTimeoutMs + ); + const completedTurn = requiredTurnPayload( + isObject(completed.params) ? completed.params.turn : undefined, + 'turn/completed' + ); + if (turnId && completedTurn.id !== turnId) { + throw new Error('Codex turn/completed returned a mismatched turn id.'); + } + if (completedTurn.status === 'failed' || completedTurn.status === 'interrupted') { + throw new CodexAppServerTurnTerminalError(completedTurn.status, completedTurn.id, completed); + } + if (completedTurn.status !== 'completed') { + throw new Error(`Codex turn/completed returned non-terminal status ${completedTurn.status}.`); + } + return { turnId, response, completed }; + } + + async turnOutcome(input: { threadId: string; clientUserMessageId: string }): Promise { + let cursor: string | undefined; + for (let page = 0; page < 10; page += 1) { + const response = await this.request('thread/turns/list', { + threadId: input.threadId, + itemsView: 'full', + sortDirection: 'desc', + limit: 100, + ...(cursor ? { cursor } : {}), + }); + if (!isObject(response) || !Array.isArray(response.data)) { + throw new Error('Codex thread/turns/list returned an invalid response.'); + } + for (const candidate of response.data) { + if (!isObject(candidate) || typeof candidate.id !== 'string' || !Array.isArray(candidate.items)) { + continue; + } + const matches = candidate.items.some( + (item) => + isObject(item) && item.type === 'userMessage' && item.clientId === input.clientUserMessageId + ); + if (!matches) continue; + const turn = requiredTurnPayload(candidate, 'thread/turns/list'); + if (turn.status === 'completed') { + if (!hasExactAssistantAnswer(turn)) { + throw new Error( + 'Codex thread/turns/list could not recover the completed assistant answer exactly.' + ); + } + return { + status: 'completed', + turnId: turn.id, + result: { + turnId: turn.id, + response: { turn }, + completed: { + method: 'turn/completed', + params: { threadId: input.threadId, turn }, + }, + reconciled: true, + }, + }; + } + return { status: turn.status, turnId: turn.id }; + } + const nextCursor = typeof response.nextCursor === 'string' ? response.nextCursor : undefined; + if (!nextCursor) return { status: 'absent' }; + cursor = nextCursor; + } + throw new Error('Codex thread/turns/list exceeded the reconciliation page bound.'); + } + + async close(): Promise { + this.beginShutdown(new Error('Codex app-server closed.')); + if (this.exited || this.child.exitCode !== null) return; + if (await this.waitForExit(this.closeTimeoutMs)) return; + this.child.kill('SIGKILL'); + if (!(await this.waitForExit(this.forceKillTimeoutMs))) { + throw new Error('Codex app-server did not exit after SIGKILL.'); + } + } + + private request(method: string, params?: unknown, timeoutMs = this.requestTimeoutMs): Promise { + if (this.closed) return Promise.reject(new Error('Codex app-server is not running.')); + const id = this.nextId++; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(id); + reject(new Error(`Timed out waiting for Codex ${method}.`)); + }, timeoutMs); + this.pending.set(id, { resolve, reject, timer }); + void this.writeMessage({ id, method, ...(params === undefined ? {} : { params }) }).catch( + (error: unknown) => { + const pending = this.pending.get(id); + if (!pending) return; + this.pending.delete(id); + clearTimeout(pending.timer); + pending.reject(new Error(`Could not write Codex ${method}: ${errorMessage(error)}`)); + this.failProtocol(`failed to serialize a client request: ${errorMessage(error)}`); + } + ); + }); + } + + private waitForNotification( + predicate: (notification: CodexNotification) => boolean, + timeoutMs: number + ): Promise { + const queuedIndex = this.notifications.findIndex(predicate); + if (queuedIndex >= 0) return Promise.resolve(this.notifications.splice(queuedIndex, 1)[0]!); + + return new Promise((resolve, reject) => { + const waiter = { + predicate, + resolve: (notification: CodexNotification) => { + clearTimeout(waiter.timer); + this.notificationWaiters.delete(waiter); + resolve(notification); + }, + reject: (error: Error) => { + clearTimeout(waiter.timer); + this.notificationWaiters.delete(waiter); + reject(error); + }, + timer: setTimeout(() => { + this.notificationWaiters.delete(waiter); + reject(new Error('Timed out waiting for Codex turn/completed.')); + }, timeoutMs), + }; + this.notificationWaiters.add(waiter); + }); + } + + private onData(chunk: string): void { + if (this.closed) return; + this.buffer += chunk; + let newline: number; + while ((newline = this.buffer.indexOf('\n')) >= 0) { + const frame = this.buffer.slice(0, newline); + this.buffer = this.buffer.slice(newline + 1); + if (Buffer.byteLength(frame, 'utf8') > this.maxFrameBytes) { + this.failProtocol('received an oversized frame'); + return; + } + const line = frame.trim(); + if (!line) continue; + let message: unknown; + try { + message = JSON.parse(line) as unknown; + } catch { + this.failProtocol('received malformed JSON'); + return; + } + if (!isObject(message)) { + this.failProtocol('received a non-object frame'); + return; + } + this.handleMessage(message); + if (this.closed) return; + } + if (Buffer.byteLength(this.buffer, 'utf8') > this.maxFrameBytes) { + this.failProtocol('received an oversized unterminated frame'); + } + } + + private handleMessage(message: Record): void { + if (typeof message.id === 'number' && ('result' in message || 'error' in message)) { + this.handleResponse(message.id, message); + return; + } + if ( + (typeof message.id === 'number' || typeof message.id === 'string') && + typeof message.method === 'string' + ) { + void this.writeMessage({ + id: message.id, + error: { code: -32601, message: 'Client request not supported' }, + }).catch((error: unknown) => { + this.failProtocol(`could not reject an app-server request: ${errorMessage(error)}`); + }); + return; + } + if (typeof message.method !== 'string') { + this.failProtocol('received a frame without a response or method'); + return; + } + const notification = { + method: message.method, + ...('params' in message ? { params: message.params } : {}), + }; + try { + this.onNotification?.(notification); + } catch (error) { + this.failProtocol(`notification handler failed: ${errorMessage(error)}`); + return; + } + const waiter = [...this.notificationWaiters].find((candidate) => candidate.predicate(notification)); + if (waiter) waiter.resolve(notification); + else if (notification.method === 'turn/completed') { + if (this.notifications.length >= this.maxBufferedNotifications) { + this.failProtocol('notification queue exceeded its bound'); + return; + } + this.notifications.push(notification); + } + } + + private handleResponse(id: number, message: Record): void { + const pending = this.pending.get(id); + if (!pending) return; + this.pending.delete(id); + clearTimeout(pending.timer); + if ('error' in message) { + if (!isObject(message.error)) { + pending.reject(new Error('Codex app-server returned a malformed error response.')); + this.failProtocol('received a malformed error response'); + return; + } + pending.reject( + new Error( + `${typeof message.error.code === 'number' ? `${message.error.code}: ` : ''}${ + typeof message.error.message === 'string' ? message.error.message : 'Codex request failed' + }` + ) + ); + return; + } + pending.resolve(message.result); + } + + private rejectAll(error: Error): void { + for (const pending of this.pending.values()) { + clearTimeout(pending.timer); + pending.reject(error); + } + this.pending.clear(); + for (const waiter of this.notificationWaiters) waiter.reject(error); + this.notificationWaiters.clear(); + } + + private writeMessage(message: Record): Promise { + const frame = `${JSON.stringify(message)}\n`; + if (Buffer.byteLength(frame, 'utf8') > this.maxFrameBytes) { + return Promise.reject(new Error('outbound Codex frame exceeds the transport bound')); + } + const write = this.writeChain.then( + () => + new Promise((resolve, reject) => { + if (this.closed || this.child.stdin.destroyed) { + reject(new Error('Codex app-server is not writable.')); + return; + } + this.child.stdin.write(frame, (error?: Error | null) => { + if (error) reject(error); + else resolve(); + }); + }) + ); + this.writeChain = write.catch(() => undefined); + return write; + } + + private failProtocol(detail: string): void { + this.beginShutdown(new Error(`Codex app-server protocol violation: ${detail}.`)); + } + + private beginShutdown(error: Error): void { + if (!this.closed) { + this.closed = true; + this.rejectAll(error); + } + if (this.shutdownStarted || this.exited || this.child.exitCode !== null) return; + this.shutdownStarted = true; + this.child.stdin.end(); + this.child.kill('SIGTERM'); + } + + private async waitForExit(timeoutMs: number): Promise { + if (this.exited || this.child.exitCode !== null) return true; + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + this.exitPromise.then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), timeoutMs); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } + } +} diff --git a/packages/cli/src/cli/lib/codex-live-controller.test.ts b/packages/cli/src/cli/lib/codex-live-controller.test.ts new file mode 100644 index 000000000..724ad15b6 --- /dev/null +++ b/packages/cli/src/cli/lib/codex-live-controller.test.ts @@ -0,0 +1,2501 @@ +import { describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; + +import type { LiveTeleportCloudClient, LiveTeleportLifecycleStatus } from '@agent-relay/cloud'; + +import { + CodexLiveController, + FileCodexControllerStateStore, + resolveCodexLiveTeleportStartupSwitch, + type CodexControllerState, + type CodexControllerStateStore, + type CodexLiveTeleportStartupSwitch, + type CodexPersistedMountResumeProvider, + type CodexWorkspaceSealHandle, + type CodexWorkspaceSealProvider, +} from './codex-live-controller.js'; +import type { CodexAppServerSession, CodexTurnResult } from './codex-app-server.js'; +import { CodexAppServerTurnTerminalError } from './codex-app-server.js'; +import { surfaceRecoveredCodexTerminal, withCodexControllerShutdown } from '../commands/codex.js'; + +const source = { + kind: 'relayfile-checkpoint-seal' as const, + receipt: { + sealId: 'seal-1', + sealToken: 'opaque', + workspaceId: 'workspace-1', + root: '/', + sessionId: 'session-1', + generation: 1, + digest: `sha256:${'a'.repeat(64)}`, + workspaceRevision: 'rev_10', + eventCursor: 'evt_10', + }, +}; + +function sealHandle(overrides: Partial = {}): CodexWorkspaceSealHandle { + return { + source, + restore: { + lifecycleId: 'session-1:1:operation-1', + resumeId: 'resume-1', + workspaceId: 'workspace-1', + localRoot: '/repo', + }, + resumeLocal: vi.fn(async () => undefined), + close: vi.fn(async () => undefined), + ...overrides, + }; +} +const verification = { + version: 1 as const, + kind: 'relayfile-destination-verification' as const, + verificationId: 'verify-1', + workspaceId: 'workspace-1', + localRoot: '/workspace', + remoteRoot: '/' as const, + sessionId: 'session-1', + generation: 1, + status: 'converged' as const, + observed: { + digest: `sha256:${'a'.repeat(64)}`, + workspaceRevision: 'rev_10', + eventCursor: 'evt_10', + }, + health: { + pendingWriteback: 0 as const, + conflicts: 0 as const, + outboxPending: 0 as const, + outboxNeedsAttention: false as const, + }, + verifiedAt: '2026-08-23T11:59:00.000Z', +}; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +function turnResult(turnId = 'turn-1'): CodexTurnResult { + const turn = { + id: turnId, + status: 'completed', + itemsView: 'full', + items: [{ id: `answer-${turnId}`, type: 'agentMessage', text: `answer for ${turnId}` }], + }; + return { + turnId, + response: { turn }, + completed: { method: 'turn/completed', params: { threadId: 'thread-1', turn } }, + }; +} + +function completedOutcome( + turnId: string, + answer = `answer for ${turnId}`, + clientUserMessageId = 'client-persisted-1' +) { + const turn = { + id: turnId, + status: 'completed', + itemsView: 'full', + items: [ + { id: `user-${turnId}`, type: 'userMessage', clientId: clientUserMessageId, text: 'prompt' }, + { id: `answer-${turnId}`, type: 'agentMessage', text: answer }, + ], + }; + return { + status: 'completed' as const, + turnId, + result: { + turnId, + response: { turn }, + completed: { method: 'turn/completed', params: { threadId: 'thread-1', turn } }, + reconciled: true as const, + }, + }; +} + +function memoryStore(initial: CodexControllerState | null = null): CodexControllerStateStore & { + value: CodexControllerState | null; +} { + return { + value: initial, + read() { + return this.value ? structuredClone(this.value) : null; + }, + write(state) { + this.value = structuredClone(state); + }, + }; +} + +function fileStore(filePath: string, initial: CodexControllerState) { + const durable = new FileCodexControllerStateStore(filePath); + durable.write(initial); + return { + value: structuredClone(initial) as CodexControllerState | null, + read() { + this.value = durable.read(); + return this.value ? structuredClone(this.value) : null; + }, + write(state: CodexControllerState) { + durable.write(state); + this.value = structuredClone(state); + }, + }; +} + +function appServer(overrides: Partial = {}): CodexAppServerSession { + return { + initialize: vi.fn(async () => undefined), + startThread: vi.fn(async () => 'thread-1'), + resumeThread: vi.fn(async () => undefined), + addEnvironment: vi.fn(async () => undefined), + environmentStatus: vi.fn(async () => ({ status: 'ready' })), + runTurn: vi.fn(async () => turnResult()), + turnOutcome: vi.fn(async () => ({ status: 'absent' as const })), + close: vi.fn(async () => undefined), + ...overrides, + }; +} + +function lifecycle( + input: { sessionId: string; generation: number }, + status: LiveTeleportLifecycleStatus['status'] +): LiveTeleportLifecycleStatus { + return { sessionId: input.sessionId, generation: input.generation, status }; +} + +function cloud(overrides: Partial = {}): LiveTeleportCloudClient { + return { + prewarm: vi.fn(async (input) => ({ + sessionId: input.sessionId, + prewarmId: `prewarm-${input.generation}`, + generation: input.generation, + status: 'ready' as const, + expiresAt: '2026-08-23T12:30:00.000Z', + })), + status: vi.fn(async (input) => ({ + ...lifecycle(input, 'active'), + leaseExpiresAt: '2026-08-23T13:00:00.000Z', + })), + acquire: vi.fn(async (input) => ({ + sessionId: input.sessionId, + generation: input.generation, + environmentId: `environment-${input.generation}`, + connectPath: `/api/v1/live-teleports/connect/${input.sessionId}/${input.generation}`, + execServerUrl: `wss://cloud.agentrelay.test/api/v1/live-teleports/connect/${input.sessionId}/${input.generation}`, + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:05:00.000Z', + leaseExpiresAt: '2026-08-23T13:00:00.000Z', + verification: { ...verification, generation: input.generation, sessionId: input.sessionId }, + })), + revoke: vi.fn(async (input) => ({ ...lifecycle(input, 'revoked'), status: 'revoked' as const })), + ...overrides, + }; +} + +function createController( + options: { + store?: ReturnType; + cloud?: LiveTeleportCloudClient | null; + sessions?: CodexAppServerSession[]; + probe?: () => Promise; + checkpointAndSeal?: CodexWorkspaceSealProvider; + resumePersistedLocalMount?: CodexPersistedMountResumeProvider; + lifecycleDeadlineMs?: number; + lifecyclePollIntervalMs?: number; + liveTeleportEnabled?: boolean; + liveTeleportStartupSwitch?: CodexLiveTeleportStartupSwitch; + now?: () => Date; + } = {} +) { + const store = options.store ?? memoryStore(); + const cloudClient = options.cloud === null ? undefined : (options.cloud ?? cloud()); + const sessions = options.sessions ?? [appServer()]; + const createAppServer = vi.fn(async () => { + const session = sessions.shift(); + if (!session) throw new Error('no app server'); + return session; + }); + const checkpointAndSeal = vi.fn( + options.checkpointAndSeal ?? + (async (input) => + sealHandle({ + restore: { + lifecycleId: input.lifecycleId, + resumeId: 'resume-1', + workspaceId: 'workspace-1', + localRoot: '/repo', + }, + })) + ); + const resumePersistedLocalMount = vi.fn(options.resumePersistedLocalMount ?? (async () => undefined)); + let operation = 0; + const controller = new CodexLiveController( + { + workspaceRoot: '/repo', + socketPath: '/state/controller.sock', + liveTeleportStartupSwitch: + options.liveTeleportStartupSwitch ?? + (options.liveTeleportEnabled === false + ? { enabled: false, reason: 'ambient-unset' } + : { enabled: true, reason: 'ambient-exact-true' }), + ...(options.lifecycleDeadlineMs ? { lifecycleDeadlineMs: options.lifecycleDeadlineMs } : {}), + ...(options.lifecyclePollIntervalMs + ? { lifecyclePollIntervalMs: options.lifecyclePollIntervalMs } + : {}), + }, + { + ...(cloudClient ? { cloud: cloudClient } : {}), + store, + createAppServer, + probeCapability: options.probe ?? (async () => undefined), + checkpointAndSeal, + resumePersistedLocalMount, + sleep: async () => undefined, + now: options.now ?? (() => new Date('2026-08-23T12:00:00.000Z')), + sessionId: () => 'session-1', + operationId: () => `operation-${++operation}`, + pid: 123, + } + ); + return { + controller, + store, + cloud: cloudClient, + createAppServer, + checkpointAndSeal, + resumePersistedLocalMount, + }; +} + +function persistedRemote(overrides: Partial = {}): CodexControllerState { + return { + version: 1, + sessionId: 'session-1', + threadId: 'thread-1', + workspaceRoot: '/repo', + source: { kind: source.kind }, + mountRestore: { + lifecycleId: 'session-1:7:operation-7', + resumeId: 'resume-7', + workspaceId: 'workspace-1', + localRoot: '/repo', + }, + generation: 7, + phase: 'remote', + controllerPid: 99, + socketPath: '/old.sock', + turnActive: false, + remote: { + sessionId: 'session-1', + generation: 7, + environmentId: 'env-7', + connectPath: '/api/v1/live-teleports/connect/session-1/7', + execServerUrl: 'wss://cloud.agentrelay.test/api/v1/live-teleports/connect/session-1/7', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:05:00.000Z', + leaseExpiresAt: '2026-08-23T13:00:00.000Z', + attached: true, + verification: { ...verification, generation: 7 }, + }, + updatedAt: '2026-08-23T11:00:00.000Z', + ...overrides, + }; +} + +function persistedRemoteForFile(overrides: Partial = {}): CodexControllerState { + const persisted = persistedRemote(overrides); + if (persisted.remote) { + const remote = persisted.remote as typeof persisted.remote & Record; + delete remote.connectPath; + delete remote.execServerUrl; + } + return persisted; +} + +function persistedLocal(overrides: Partial = {}): CodexControllerState { + return { + version: 1, + sessionId: 'session-1', + threadId: 'thread-1', + workspaceRoot: '/repo', + generation: 3, + phase: 'local', + cloudLifecycle: 'none', + controllerPid: 99, + socketPath: '/old.sock', + turnActive: false, + updatedAt: '2026-08-23T11:00:00.000Z', + ...overrides, + }; +} + +describe('CodexLiveController', () => { + it.each([ + ['unset', undefined, false, 'ambient-unset'], + ['false', 'false', false, 'ambient-value-not-exact-true'], + ['exact true', 'true', true, 'ambient-exact-true'], + ['uppercase true', 'TRUE', false, 'ambient-value-not-exact-true'], + ['leading whitespace', ' true', false, 'ambient-value-not-exact-true'], + ['trailing whitespace', 'true ', false, 'ambient-value-not-exact-true'], + ['noncanonical truthy value', '1', false, 'ambient-value-not-exact-true'], + ['empty', '', false, 'ambient-value-not-exact-true'], + ] as const)('treats the ambient teleport flag as %s', (_name, raw, enabled, reason) => { + expect(resolveCodexLiveTeleportStartupSwitch(raw)).toEqual({ enabled, reason }); + }); + + it('keeps a fresh managed Codex session entirely local when teleport is disabled', async () => { + const local = appServer(); + const cloudClient = cloud(); + const probe = vi.fn(async () => { + throw new Error('teleport capability probe must remain dormant'); + }); + const { controller, checkpointAndSeal } = createController({ + liveTeleportEnabled: false, + cloud: cloudClient, + sessions: [local], + probe, + }); + + await expect(controller.initialize()).resolves.toMatchObject({ + phase: 'local', + execution: 'local', + cloudLifecycle: 'none', + liveTeleport: { enabled: false, reason: 'ambient-unset' }, + }); + await expect(controller.runTurn('local only')).resolves.toMatchObject({ turnId: 'turn-1' }); + + expect(probe).not.toHaveBeenCalled(); + expect(cloudClient.prewarm).not.toHaveBeenCalled(); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + expect(cloudClient.status).not.toHaveBeenCalled(); + expect(cloudClient.revoke).not.toHaveBeenCalled(); + expect(checkpointAndSeal).not.toHaveBeenCalled(); + expect(local.addEnvironment).not.toHaveBeenCalled(); + expect(local.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ execution: { kind: 'local', workspaceRoot: '/repo' } }) + ); + expect(() => + controller.requestTeleport({ requestId: 'disabled-request', expectedGeneration: 1 }) + ).toThrow('RELAY_LIVE_SESSION_TELEPORT_ENABLED=true'); + }); + + it('starts and runs locally while disabled without any Cloud client', async () => { + const local = appServer(); + const { controller } = createController({ + liveTeleportEnabled: false, + cloud: null, + sessions: [local], + }); + + await expect(controller.initialize()).resolves.toMatchObject({ execution: 'local' }); + await expect(controller.runTurn('no Cloud login')).resolves.toMatchObject({ turnId: 'turn-1' }); + expect(local.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ execution: { kind: 'local', workspaceRoot: '/repo' } }) + ); + }); + + it('fences earlier Cloud ownership on disabled startup, then stays local without rewarming', async () => { + const store = memoryStore(persistedRemote()); + const replacement = appServer(); + const cloudClient = cloud(); + const probe = vi.fn(async () => undefined); + const { controller, resumePersistedLocalMount } = createController({ + store, + cloud: cloudClient, + sessions: [replacement], + liveTeleportEnabled: false, + probe, + }); + + await expect(controller.initialize()).resolves.toMatchObject({ + generation: 8, + phase: 'local', + execution: 'local', + cloudLifecycle: 'none', + liveTeleport: { enabled: false, reason: 'ambient-unset' }, + }); + await controller.runTurn('recovered local turn'); + + expect(probe).not.toHaveBeenCalled(); + expect(cloudClient.revoke).toHaveBeenCalledWith( + expect.objectContaining({ generation: 7, idempotencyKey: 'session-1:7:revoke' }) + ); + expect(resumePersistedLocalMount).toHaveBeenCalledOnce(); + expect(replacement.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + expect(replacement.addEnvironment).not.toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ execution: { kind: 'local', workspaceRoot: '/repo' } }) + ); + expect(cloudClient.prewarm).not.toHaveBeenCalled(); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + expect(cloudClient.status).not.toHaveBeenCalled(); + expect(store.value?.pending).toBeUndefined(); + }); + + it('cancels a queued teleport on disabled startup without contacting Cloud', async () => { + const store = memoryStore( + persistedLocal({ + phase: 'teleport_pending', + pending: { requestId: 'queued-before-disable', expectedGeneration: 3 }, + }) + ); + const replacement = appServer(); + const cloudClient = cloud(); + const { controller } = createController({ + store, + cloud: cloudClient, + sessions: [replacement], + liveTeleportEnabled: false, + }); + + await expect(controller.initialize()).resolves.toMatchObject({ + generation: 3, + phase: 'local', + execution: 'local', + cloudLifecycle: 'none', + }); + await controller.runTurn('still local'); + + expect(store.value?.pending).toBeUndefined(); + expect(cloudClient.prewarm).not.toHaveBeenCalled(); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + expect(cloudClient.status).not.toHaveBeenCalled(); + expect(cloudClient.revoke).not.toHaveBeenCalled(); + expect(replacement.addEnvironment).not.toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ execution: { kind: 'local', workspaceRoot: '/repo' } }) + ); + }); + + it('fails before starting an app-server when the local experimental capability is unsupported', async () => { + const { controller, createAppServer } = createController({ + probe: async () => { + throw new Error('environment/status unsupported'); + }, + }); + + await expect(controller.initialize()).rejects.toThrow('environment/status unsupported'); + expect(createAppServer).not.toHaveBeenCalled(); + }); + + it('prewarms without stopping or sealing the active local mount', async () => { + const cloudClient = cloud(); + const { controller, checkpointAndSeal } = createController({ + cloud: cloudClient, + liveTeleportEnabled: true, + }); + + await controller.initialize(); + + expect(checkpointAndSeal).not.toHaveBeenCalled(); + expect(cloudClient.prewarm).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: 'session-1', + generation: 1, + workspaceRoot: '/', + idempotencyKey: 'session-1:1:prewarm', + }) + ); + }); + + it('does not block local initialization on a stalled background prewarm', async () => { + const pendingPrewarm = deferred<{ + sessionId: string; + prewarmId: string; + generation: number; + status: 'ready'; + expiresAt: string; + }>(); + const cloudClient = cloud({ prewarm: vi.fn(() => pendingPrewarm.promise) }); + const { controller } = createController({ cloud: cloudClient }); + + await expect(controller.initialize()).resolves.toMatchObject({ phase: 'local' }); + pendingPrewarm.resolve({ + sessionId: 'session-1', + prewarmId: 'prewarm-1', + generation: 1, + status: 'ready', + expiresAt: '2026-08-23T12:30:00.000Z', + }); + await vi.waitFor(() => expect(controller.status().prewarmStatus).toBe('ready')); + }); + + it('retains durable Cloud ownership intent when a prewarm response is lost', async () => { + const cloudClient = cloud({ + prewarm: vi.fn(async () => Promise.reject(new Error('response lost'))), + }); + const { controller, store } = createController({ cloud: cloudClient }); + + await controller.initialize(); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('failed')); + + expect(store.value).toMatchObject({ cloudLifecycle: 'prewarm_requested' }); + }); + + it('cleans up a failed no-row prewarm through Cloud idempotent terminal revoke', async () => { + const cloudClient = cloud({ + prewarm: vi.fn(async () => Promise.reject(new Error('feature disabled'))), + }); + const { controller, store } = createController({ cloud: cloudClient }); + await controller.initialize(); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('failed')); + + await controller.close(); + + expect(cloudClient.revoke).toHaveBeenCalledWith( + expect.objectContaining({ idempotencyKey: 'session-1:1:revoke' }) + ); + expect(cloudClient.status).not.toHaveBeenCalled(); + expect(store.value).toMatchObject({ cloudLifecycle: 'none', generation: 2 }); + }); + + it('advances prewarm identity after graceful fence before close → initialize → teleport', async () => { + const store = memoryStore(); + const cloudClient = cloud(); + const first = createController({ store, cloud: cloudClient, sessions: [appServer()] }); + await first.controller.initialize(); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + + await first.controller.close(); + + expect(store.value).toMatchObject({ generation: 2, cloudLifecycle: 'none', phase: 'local' }); + expect(cloudClient.revoke).toHaveBeenCalledWith( + expect.objectContaining({ generation: 1, idempotencyKey: 'session-1:1:revoke' }) + ); + + const resumed = appServer(); + const second = createController({ store, cloud: cloudClient, sessions: [resumed] }); + await second.controller.initialize(); + second.controller.requestTeleport({ requestId: 'request-generation-2', expectedGeneration: 2 }); + await second.controller.runTurn('fresh identity'); + + expect(cloudClient.prewarm).toHaveBeenCalledWith( + expect.objectContaining({ generation: 2, idempotencyKey: 'session-1:2:prewarm' }) + ); + expect(cloudClient.acquire).toHaveBeenCalledWith( + expect.objectContaining({ generation: 2, idempotencyKey: 'session-1:2:acquire' }) + ); + expect(resumed.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'fresh identity', + execution: expect.objectContaining({ kind: 'remote' }), + }) + ); + }); + + it('atomically finalizes a pending teleport when close consumes its prewarm generation', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-controller-close-pending-')); + const fileStore = new FileCodexControllerStateStore(path.join(directory, 'state.json')); + const store = { + value: null as CodexControllerState | null, + read() { + this.value = fileStore.read(); + return this.value; + }, + write(state: CodexControllerState) { + fileStore.write(state); + this.value = structuredClone(state); + }, + }; + try { + const cloudClient = cloud(); + const first = createController({ store, cloud: cloudClient, sessions: [appServer()] }); + await first.controller.initialize(); + await vi.waitFor(() => expect(fileStore.read()?.prewarmStatus).toBe('ready')); + first.controller.requestTeleport({ requestId: 'request-consumed-on-close', expectedGeneration: 1 }); + + await first.controller.close(); + + expect(fileStore.read()).toMatchObject({ + generation: 2, + phase: 'local', + cloudLifecycle: 'none', + lastRequestId: 'request-consumed-on-close', + }); + expect(fileStore.read()?.pending).toBeUndefined(); + + const second = createController({ store, cloud: cloudClient, sessions: [appServer()] }); + await expect(second.controller.initialize()).resolves.toMatchObject({ + generation: 2, + phase: 'local', + }); + await vi.waitFor(() => expect(fileStore.read()?.prewarmStatus).toBe('ready')); + await second.controller.close(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it('persists checkpoint lifecycle intent before Relayfile can stop the mount', async () => { + const checkpoint = deferred(); + let lifecycleId = ''; + const { controller, store } = createController({ + checkpointAndSeal: async (input) => { + lifecycleId = input.lifecycleId; + return checkpoint.promise; + }, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + const running = controller.runTurn('boundary turn'); + await vi.waitFor(() => expect(lifecycleId).not.toBe('')); + expect(store.value?.mountRestore).toEqual({ lifecycleId, localRoot: '/repo' }); + expect(store.value?.source).toBeUndefined(); + checkpoint.resolve( + sealHandle({ + restore: { + lifecycleId, + resumeId: 'resume-1', + workspaceId: 'workspace-1', + localRoot: '/repo', + }, + }) + ); + await running; + }); + + it('waits for aborted checkpoint cleanup to settle before starting local recovery', async () => { + const cleanup = deferred(); + const aborted = deferred(); + const original = appServer(); + const replacement = appServer(); + const { controller } = createController({ + sessions: [original, replacement], + lifecycleDeadlineMs: 5, + checkpointAndSeal: async (input) => { + input.signal?.addEventListener('abort', () => aborted.resolve(), { once: true }); + return cleanup.promise; + }, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + const running = controller.runTurn('deadline'); + await aborted.promise; + expect(original.close).not.toHaveBeenCalled(); + + cleanup.reject(Object.assign(new Error('checkpoint aborted and source ready'), { name: 'AbortError' })); + await expect(running).resolves.toMatchObject({ turnId: 'turn-1' }); + expect(original.close).toHaveBeenCalled(); + expect(replacement.resumeThread).toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ text: 'deadline', execution: { kind: 'local', workspaceRoot: '/repo' } }) + ); + }); + + it('fences without inverse effects when checkpoint ignores abort and never settles', async () => { + const original = appServer(); + const sealed = sealHandle(); + const { controller } = createController({ + sessions: [original], + lifecycleDeadlineMs: 5, + checkpointAndSeal: async () => new Promise(() => undefined), + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('unknown checkpoint')).rejects.toThrow('lifecycle outcome is unknown'); + + expect(original.close).not.toHaveBeenCalled(); + expect(sealed.resumeLocal).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ + phase: 'fenced', + lastError: 'LIFECYCLE_EFFECT_OUTCOME_UNKNOWN', + }); + await expect(controller.runTurn('must remain blocked')).rejects.toThrow('controller is fenced'); + }); + + it('fences without resuming the source when Cloud acquire never settles', async () => { + const original = appServer(); + const sealed = sealHandle(); + const cloudClient = cloud({ + acquire: vi.fn(async () => new Promise(() => undefined)), + }); + const { controller } = createController({ + sessions: [original], + cloud: cloudClient, + lifecycleDeadlineMs: 5, + checkpointAndSeal: async () => sealed, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('unknown acquire')).rejects.toThrow('lifecycle outcome is unknown'); + + expect(sealed.resumeLocal).not.toHaveBeenCalled(); + expect(original.close).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'fenced' }); + }); + + it('rejects stale generations and applies a mid-turn teleport only at the next boundary', async () => { + const firstTurn = deferred(); + const original = appServer({ + runTurn: vi + .fn() + .mockImplementationOnce(() => firstTurn.promise) + .mockResolvedValueOnce(turnResult('turn-2')), + }); + const cloudClient = cloud(); + const { controller } = createController({ sessions: [original], cloud: cloudClient }); + await controller.initialize(); + + expect(() => controller.requestTeleport({ requestId: 'stale', expectedGeneration: 0 })).toThrow( + 'Stale teleport generation' + ); + const running = controller.runTurn('local turn'); + await vi.waitFor(() => expect(original.runTurn).toHaveBeenCalledTimes(1)); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + + firstTurn.resolve(turnResult()); + await running; + await controller.runTurn('remote turn'); + + expect(original.runTurn).toHaveBeenNthCalledWith(1, { + threadId: 'thread-1', + text: 'local turn', + clientUserMessageId: 'operation-1', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }); + expect(original.runTurn).toHaveBeenNthCalledWith(2, { + threadId: 'thread-1', + text: 'remote turn', + clientUserMessageId: 'operation-3', + execution: { + kind: 'remote', + environment: { environmentId: 'environment-1', cwd: '/workspace' }, + }, + }); + + expect(cloudClient.acquire).toHaveBeenCalledWith( + expect.objectContaining({ + generation: 1, + workspaceRoot: '/', + source, + prewarmId: 'prewarm-1', + idempotencyKey: 'session-1:1:acquire', + }) + ); + }); + + it('reports verifying—not remote—until the first Cloud turn completes', async () => { + const firstRemoteTurn = deferred(); + const original = appServer({ runTurn: vi.fn(() => firstRemoteTurn.promise) }); + const { controller, store } = createController({ sessions: [original] }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + const running = controller.runTurn('first remote turn'); + await vi.waitFor(() => expect(controller.status().phase).toBe('verifying')); + expect(controller.status()).toMatchObject({ execution: 'verifying', phase: 'verifying' }); + expect(controller.status()).not.toHaveProperty('remote'); + + firstRemoteTurn.resolve(turnResult()); + await running; + expect(controller.status()).toMatchObject({ execution: 'cloud', phase: 'remote' }); + expect(controller.status().remote).toMatchObject({ attached: true, environmentId: 'environment-1' }); + expect(controller.status().remote).not.toHaveProperty('execServerUrl'); + expect(controller.status().remote).not.toHaveProperty('connectPath'); + expect(controller.status()).not.toHaveProperty('source'); + expect(controller.status()).not.toHaveProperty('mountRestore'); + expect(store.value?.source).toEqual({ kind: 'relayfile-checkpoint-seal' }); + expect(JSON.stringify(store.value)).not.toMatch(/opaque|ticket=|wss:\/\//); + + await controller.runTurn('subsequent remote turn'); + expect(original.runTurn).toHaveBeenNthCalledWith(2, { + threadId: 'thread-1', + text: 'subsequent remote turn', + clientUserMessageId: 'operation-3', + execution: { kind: 'remote' }, + }); + }); + + it.each(['failed', 'interrupted'] as const)( + 'surfaces a normal turn/completed %s terminal and keeps it recorded', + async (terminalStatus) => { + const completed = { + method: 'turn/completed', + params: { + threadId: 'thread-1', + turn: { id: 'turn-terminal', status: terminalStatus, items: [] }, + }, + }; + const original = appServer({ + runTurn: vi.fn(async () => { + throw new CodexAppServerTurnTerminalError(terminalStatus, 'turn-terminal', completed); + }), + }); + const { controller, store } = createController({ sessions: [original] }); + await controller.initialize(); + + await expect(controller.runTurn('terminal prompt')).rejects.toMatchObject({ + name: 'CodexTurnRecordedError', + status: terminalStatus, + }); + + expect(store.value?.inFlightTurn).toBeUndefined(); + expect(controller.status()).toMatchObject({ + phase: 'local', + lastError: `TURN_${terminalStatus.toUpperCase()}`, + }); + } + ); + + it('admits an acquired lease at the exact 40-minute turn horizon', async () => { + const cloudClient = cloud({ + acquire: vi.fn(async (input) => ({ + sessionId: input.sessionId, + generation: input.generation, + environmentId: 'environment-boundary', + connectPath: '/api/v1/live-teleports/connect/boundary', + execServerUrl: 'wss://cloud.agentrelay.test/api/v1/live-teleports/connect/boundary', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:05:00.000Z', + leaseExpiresAt: '2026-08-23T12:40:00.000Z', + verification: { ...verification, generation: input.generation }, + })), + }); + const original = appServer(); + const { controller } = createController({ cloud: cloudClient, sessions: [original] }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-boundary', expectedGeneration: 1 }); + + await expect(controller.runTurn('boundary lease')).resolves.toMatchObject({ turnId: 'turn-1' }); + expect(original.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ execution: expect.objectContaining({ kind: 'remote' }) }) + ); + }); + + it('fences a short acquired lease and transparently submits the same prompt locally', async () => { + const original = appServer(); + const replacement = appServer(); + const cloudClient = cloud({ + acquire: vi.fn(async (input) => ({ + sessionId: input.sessionId, + generation: input.generation, + environmentId: 'environment-short', + connectPath: '/api/v1/live-teleports/connect/short', + execServerUrl: 'wss://cloud.agentrelay.test/api/v1/live-teleports/connect/short', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:05:00.000Z', + leaseExpiresAt: '2026-08-23T12:39:59.999Z', + verification: { ...verification, generation: input.generation }, + })), + }); + const { controller } = createController({ + cloud: cloudClient, + sessions: [original, replacement], + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-short', expectedGeneration: 1 }); + + await expect(controller.runTurn('preserve this prompt')).resolves.toMatchObject({ turnId: 'turn-1' }); + expect(cloudClient.revoke).toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith({ + threadId: 'thread-1', + text: 'preserve this prompt', + clientUserMessageId: 'operation-2', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }); + }); + + it('persists a newer active lease only after same-generation 40-minute preflight renewal', async () => { + let now = new Date('2026-08-23T12:00:00.000Z'); + const original = appServer(); + const cloudClient = cloud({ + status: vi.fn(async (input) => ({ + ...lifecycle(input, 'active'), + leaseExpiresAt: '2026-08-23T13:10:00.000Z', + })), + }); + const { controller, store } = createController({ + cloud: cloudClient, + sessions: [original], + now: () => now, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-renew', expectedGeneration: 1 }); + await controller.runTurn('first remote'); + + now = new Date('2026-08-23T12:20:00.000Z'); + await controller.runTurn('after renewal'); + + expect(store.value?.remote?.leaseExpiresAt).toBe('2026-08-23T13:10:00.000Z'); + expect(original.runTurn).toHaveBeenCalledTimes(2); + }); + + it.each([ + ['missing', undefined], + ['short', '2026-08-23T12:59:59.999Z'], + ])('recovers locally when active status has a %s next-turn lease', async (_name, leaseExpiresAt) => { + let now = new Date('2026-08-23T12:00:00.000Z'); + const original = appServer(); + const replacement = appServer(); + const cloudClient = cloud({ + status: vi.fn(async (input) => ({ + ...lifecycle(input, 'active'), + ...(leaseExpiresAt ? { leaseExpiresAt } : {}), + })), + }); + const { controller } = createController({ + cloud: cloudClient, + sessions: [original, replacement], + now: () => now, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: `request-${_name}`, expectedGeneration: 1 }); + await controller.runTurn('first remote'); + + now = new Date('2026-08-23T12:20:00.000Z'); + await controller.runTurn('must recover locally'); + + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'must recover locally', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }) + ); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); + }); + + it('never revives a locally expired lease from a later active status', async () => { + let now = new Date('2026-08-23T12:00:00.000Z'); + const original = appServer(); + const replacement = appServer(); + const status = vi.fn(async (input) => ({ + ...lifecycle(input, 'active'), + leaseExpiresAt: '2026-08-23T14:00:00.000Z', + })); + const cloudClient = cloud({ status }); + const { controller } = createController({ + cloud: cloudClient, + sessions: [original, replacement], + now: () => now, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-expired', expectedGeneration: 1 }); + await controller.runTurn('first remote'); + + now = new Date('2026-08-23T13:00:00.001Z'); + await controller.runTurn('expired locally'); + + expect(status).not.toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'expired locally', + execution: expect.objectContaining({ kind: 'local' }), + }) + ); + }); + + it.each(['failed', 'interrupted'] as const)( + 'durably preserves a live remote %s reconciliation across two crashes without replay', + async (status) => { + const original = appServer({ runTurn: vi.fn(async () => Promise.reject(new Error('remote died'))) }); + const replacement = appServer({ + turnOutcome: vi.fn(async () => ({ status, turnId: `turn-remote-${status}` })), + }); + const cloudClient = cloud(); + const sealed = sealHandle(); + const { controller, store } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + checkpointAndSeal: async () => sealed, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('do not replay me')).rejects.toMatchObject({ + status, + requiresRecoveryAcknowledgment: true, + }); + + expect(cloudClient.revoke).toHaveBeenCalledWith( + expect.objectContaining({ idempotencyKey: 'session-1:1:revoke' }) + ); + expect(original.close).toHaveBeenCalled(); + expect(sealed.resumeLocal).toHaveBeenCalled(); + expect(replacement.initialize).toHaveBeenCalled(); + expect(replacement.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + expect(replacement.runTurn).not.toHaveBeenCalled(); + expect(store.value?.inFlightTurn).toBeUndefined(); + expect(store.value?.recoveredOutcome).toMatchObject({ + sessionId: 'session-1', + threadId: 'thread-1', + generation: 2, + clientUserMessageId: 'operation-2', + turnId: `turn-remote-${status}`, + status, + }); + expect(controller.status()).toMatchObject({ generation: 2, phase: 'local', execution: 'local' }); + + const prewarmCallsAfterRecovery = vi.mocked(cloudClient.prewarm).mock.calls.length; + const restartedOnce = appServer(); + const second = createController({ store, cloud: cloudClient, sessions: [restartedOnce] }); + await second.controller.initialize(); + expect(restartedOnce.turnOutcome).not.toHaveBeenCalled(); + expect(restartedOnce.runTurn).not.toHaveBeenCalled(); + expect(second.controller.takeRecoveredTerminal()).toMatchObject({ status }); + expect(vi.mocked(cloudClient.prewarm)).toHaveBeenCalledTimes(prewarmCallsAfterRecovery); + + const restartedTwice = appServer(); + const third = createController({ store, cloud: cloudClient, sessions: [restartedTwice] }); + await third.controller.initialize(); + expect(restartedTwice.turnOutcome).not.toHaveBeenCalled(); + expect(restartedTwice.runTurn).not.toHaveBeenCalled(); + expect(third.controller.takeRecoveredTerminal()).toMatchObject({ status }); + third.controller.acknowledgeRecoveredOutcome(); + expect(store.value?.recoveredOutcome).toBeUndefined(); + expect(third.controller.takeRecoveredTerminal()).toBeUndefined(); + } + ); + + it('reconciles a completed remote turn after notification loss without replaying it', async () => { + const original = appServer({ + runTurn: vi.fn(async () => Promise.reject(new Error('notification lost'))), + }); + const replacement = appServer({ + turnOutcome: vi.fn(async () => + completedOutcome('turn-remote-1', 'recovered exact answer', 'operation-2') + ), + }); + const cloudClient = cloud(); + const { controller, store } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('exactly once')).resolves.toMatchObject({ + turnId: 'turn-remote-1', + response: { + turn: { + items: expect.arrayContaining([ + expect.objectContaining({ type: 'agentMessage', text: 'recovered exact answer' }), + ]), + }, + }, + completed: { + params: { + turn: expect.objectContaining({ id: 'turn-remote-1', status: 'completed' }), + }, + }, + }); + expect(replacement.runTurn).not.toHaveBeenCalled(); + expect(store.value?.inFlightTurn).toBeUndefined(); + expect(store.value?.recoveredOutcome).toMatchObject({ + sessionId: 'session-1', + threadId: 'thread-1', + generation: 2, + clientUserMessageId: 'operation-2', + turnId: 'turn-remote-1', + status: 'completed', + }); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); + + const prewarmCallsAfterRecovery = vi.mocked(cloudClient.prewarm).mock.calls.length; + const restartedOnce = appServer(); + const second = createController({ store, cloud: cloudClient, sessions: [restartedOnce] }); + await second.controller.initialize(); + expect(restartedOnce.turnOutcome).not.toHaveBeenCalled(); + expect(restartedOnce.runTurn).not.toHaveBeenCalled(); + expect(second.controller.takeRecoveredTurn()).toMatchObject({ + turnId: 'turn-remote-1', + response: { + turn: { + items: expect.arrayContaining([ + expect.objectContaining({ type: 'agentMessage', text: 'recovered exact answer' }), + ]), + }, + }, + }); + expect(vi.mocked(cloudClient.prewarm)).toHaveBeenCalledTimes(prewarmCallsAfterRecovery); + + const restartedTwice = appServer(); + const third = createController({ store, cloud: cloudClient, sessions: [restartedTwice] }); + await third.controller.initialize(); + expect(restartedTwice.turnOutcome).not.toHaveBeenCalled(); + expect(restartedTwice.runTurn).not.toHaveBeenCalled(); + expect(third.controller.takeRecoveredTurn()).toMatchObject({ turnId: 'turn-remote-1' }); + third.controller.acknowledgeRecoveredOutcome(); + expect(store.value?.recoveredOutcome).toBeUndefined(); + expect(third.controller.takeRecoveredTurn()).toBeUndefined(); + }); + + it.each(['completed', 'failed', 'interrupted'] as const)( + 'keeps a file-backed %s delivery fenced across restart, resume failure, close failure, and restart', + async (status) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), `relay-controller-delivery-${status}-`)); + const statePath = path.join(directory, 'state.json'); + try { + const clientUserMessageId = `client-file-${status}`; + const store = fileStore( + statePath, + persistedRemoteForFile({ + inFlightTurn: { clientUserMessageId, execution: 'remote' }, + }) + ); + const outcome = + status === 'completed' + ? completedOutcome('turn-file-completed', 'file-backed answer', clientUserMessageId) + : { status, turnId: `turn-file-${status}` }; + const reconciler = appServer({ turnOutcome: vi.fn(async () => outcome) }); + const first = createController({ store, sessions: [reconciler] }); + + await expect(first.controller.initialize()).resolves.toMatchObject({ + phase: 'local', + generation: 8, + }); + await first.controller.close(); + + const durableAfterReconcile = new FileCodexControllerStateStore(statePath).read(); + expect(durableAfterReconcile).toMatchObject({ + phase: 'local', + generation: 8, + recoveredOutcome: { + generation: 8, + clientUserMessageId, + status, + }, + }); + expect(durableAfterReconcile?.pending).toBeUndefined(); + expect(durableAfterReconcile?.remote).toBeUndefined(); + + const resumeFailure = appServer({ + resumeThread: vi.fn(async () => Promise.reject(new Error('resume unavailable'))), + }); + const second = createController({ store, sessions: [resumeFailure] }); + await expect(second.controller.initialize()).rejects.toThrow('CODEX_THREAD_RESUME_FAILED_ON_RESTART'); + + const durableAfterResumeFailure = new FileCodexControllerStateStore(statePath).read(); + expect(durableAfterResumeFailure).toMatchObject({ + phase: 'recovery_failed', + generation: 8, + recoveredOutcome: { clientUserMessageId, status }, + }); + + const closeError = new Error('close unavailable'); + const closeFailure = appServer({ close: vi.fn(async () => Promise.reject(closeError)) }); + const third = createController({ store, sessions: [closeFailure] }); + await expect(third.controller.initialize()).resolves.toMatchObject({ + phase: 'local', + generation: 8, + }); + + expect(() => + third.controller.requestTeleport({ requestId: 'must-not-queue', expectedGeneration: 8 }) + ).toThrow('awaiting durable output acknowledgment'); + await expect(third.controller.runTurn('must not execute')).rejects.toThrow( + 'awaiting durable output acknowledgment' + ); + await expect(third.controller.rollback()).rejects.toThrow('awaiting durable output acknowledgment'); + expect(closeFailure.runTurn).not.toHaveBeenCalled(); + expect(closeFailure.addEnvironment).not.toHaveBeenCalled(); + expect(store.value?.pending).toBeUndefined(); + expect(store.value?.generation).toBe(8); + + await expect(third.controller.close()).rejects.toBe(closeError); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + phase: 'recovery_failed', + generation: 8, + recoveredOutcome: { clientUserMessageId, status }, + }); + + const finalSession = appServer(); + const fourth = createController({ store, sessions: [finalSession] }); + await expect(fourth.controller.initialize()).resolves.toMatchObject({ + phase: 'local', + generation: 8, + }); + if (status === 'completed') { + expect(fourth.controller.takeRecoveredTurn()).toMatchObject({ + turnId: 'turn-file-completed', + }); + } else { + expect(fourth.controller.takeRecoveredTerminal()).toMatchObject({ status }); + } + fourth.controller.acknowledgeRecoveredOutcome(); + expect(new FileCodexControllerStateStore(statePath).read()?.recoveredOutcome).toBeUndefined(); + await fourth.controller.close(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + } + ); + + it.each( + (['ready', 'failed', 'in-flight'] as const).flatMap((prewarmState) => + (['completed', 'failed', 'interrupted'] as const).map( + (outcomeStatus) => [prewarmState, outcomeStatus] as const + ) + ) + )( + 'keeps a file-backed local %s prewarm independent from an unacknowledged %s outcome', + async (prewarmState, outcomeStatus) => { + const directory = fs.mkdtempSync( + path.join(os.tmpdir(), `relay-controller-local-${prewarmState}-${outcomeStatus}-`) + ); + const statePath = path.join(directory, 'state.json'); + const pendingPrewarm = deferred<{ + sessionId: string; + prewarmId: string; + generation: number; + status: 'ready'; + expiresAt: string; + }>(); + let prewarmCalls = 0; + const prewarm = vi.fn(async (input: Parameters[0]) => { + prewarmCalls += 1; + if (prewarmCalls === 1) { + if (prewarmState === 'failed') throw new Error('first prewarm failed ambiguously'); + if (prewarmState === 'in-flight') return pendingPrewarm.promise; + } + return { + sessionId: input.sessionId, + prewarmId: `prewarm-${input.generation}`, + generation: input.generation, + status: 'ready' as const, + expiresAt: '2026-08-23T12:30:00.000Z', + }; + }); + const cloudClient = cloud({ prewarm }); + const store = fileStore(statePath, persistedLocal({ generation: 1 })); + const outcome = + outcomeStatus === 'completed' + ? completedOutcome('turn-local-recovered', 'file-backed local answer', 'operation-1') + : { status: outcomeStatus, turnId: `turn-local-${outcomeStatus}` }; + const local = appServer({ + runTurn: vi.fn(async () => Promise.reject(new Error('terminal notification lost'))), + turnOutcome: vi.fn(async () => outcome), + }); + + try { + const first = createController({ store, cloud: cloudClient, sessions: [local] }); + await first.controller.initialize(); + if (prewarmState === 'ready') { + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + } else if (prewarmState === 'failed') { + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('failed')); + } + + if (outcomeStatus === 'completed') { + await expect(first.controller.runTurn('do not replay local')).resolves.toMatchObject({ + turnId: 'turn-local-recovered', + }); + } else { + await expect(first.controller.runTurn('do not replay local')).rejects.toMatchObject({ + status: outcomeStatus, + requiresRecoveryAcknowledgment: true, + }); + } + expect(local.runTurn).toHaveBeenCalledTimes(1); + expect(local.turnOutcome).toHaveBeenCalledTimes(1); + + const durableBeforeRestart = new FileCodexControllerStateStore(statePath).read(); + expect(durableBeforeRestart).toMatchObject({ + generation: 1, + phase: 'local', + cloudLifecycle: prewarmState === 'ready' ? 'prewarmed' : 'prewarm_requested', + recoveredOutcome: { + generation: 1, + clientUserMessageId: 'operation-1', + status: outcomeStatus, + }, + }); + if (prewarmState === 'in-flight') { + expect(durableBeforeRestart?.prewarmStatus).toBeUndefined(); + } + + const closeError = new Error('close unavailable after durable recovery'); + const afterCrash = appServer({ close: vi.fn(async () => Promise.reject(closeError)) }); + const second = createController({ store, cloud: cloudClient, sessions: [afterCrash] }); + await expect(second.controller.initialize()).resolves.toMatchObject({ + generation: 2, + phase: 'local', + }); + expect(cloudClient.revoke).toHaveBeenCalledWith( + expect.objectContaining({ generation: 1, idempotencyKey: 'session-1:1:revoke' }) + ); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + generation: 2, + cloudLifecycle: 'none', + recoveredOutcome: { generation: 1, status: outcomeStatus }, + }); + expect(new FileCodexControllerStateStore(statePath).read()?.prewarmId).toBeUndefined(); + expect(new FileCodexControllerStateStore(statePath).read()?.prewarmStatus).toBeUndefined(); + + await expect(second.controller.close()).rejects.toBe(closeError); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + phase: 'recovery_failed', + generation: 2, + recoveredOutcome: { generation: 1, status: outcomeStatus }, + }); + + const resumeFailure = appServer({ + resumeThread: vi.fn(async () => Promise.reject(new Error('resume unavailable'))), + }); + const third = createController({ store, cloud: cloudClient, sessions: [resumeFailure] }); + await expect(third.controller.initialize()).rejects.toThrow('CODEX_THREAD_RESUME_FAILED_ON_RESTART'); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + phase: 'recovery_failed', + generation: 2, + recoveredOutcome: { generation: 1, status: outcomeStatus }, + }); + + const finalSession = appServer(); + const fourth = createController({ store, cloud: cloudClient, sessions: [finalSession] }); + await fourth.controller.initialize(); + expect(finalSession.runTurn).not.toHaveBeenCalled(); + if (outcomeStatus === 'completed') { + expect(fourth.controller.takeRecoveredTurn()).toMatchObject({ + turnId: 'turn-local-recovered', + }); + } else { + expect(fourth.controller.takeRecoveredTerminal()).toMatchObject({ status: outcomeStatus }); + } + fourth.controller.acknowledgeRecoveredOutcome(); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + expect(prewarm).toHaveBeenLastCalledWith( + expect.objectContaining({ generation: 2, idempotencyKey: 'session-1:2:prewarm' }) + ); + await fourth.controller.close(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + } + ); + + it('keeps late background prewarm completion from invalidating or erasing recovered output', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-controller-late-prewarm-')); + const statePath = path.join(directory, 'state.json'); + const pendingPrewarm = deferred<{ + sessionId: string; + prewarmId: string; + generation: number; + status: 'ready'; + expiresAt: string; + }>(); + const prewarm = vi.fn(() => pendingPrewarm.promise); + const cloudClient = cloud({ prewarm }); + const store = fileStore(statePath, persistedLocal({ generation: 1 })); + const local = appServer({ + runTurn: vi.fn(async () => Promise.reject(new Error('terminal notification lost'))), + turnOutcome: vi.fn(async () => + completedOutcome('turn-local-late-prewarm', 'exact recovered answer', 'operation-1') + ), + }); + + try { + const { controller } = createController({ store, cloud: cloudClient, sessions: [local] }); + await controller.initialize(); + await controller.runTurn('do not replay local'); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + cloudLifecycle: 'prewarm_requested', + recoveredOutcome: { turnId: 'turn-local-late-prewarm', status: 'completed' }, + }); + + pendingPrewarm.resolve({ + sessionId: 'session-1', + prewarmId: 'prewarm-1', + generation: 1, + status: 'ready', + expiresAt: '2026-08-23T12:30:00.000Z', + }); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + cloudLifecycle: 'prewarmed', + prewarmId: 'prewarm-1', + recoveredOutcome: { turnId: 'turn-local-late-prewarm', status: 'completed' }, + }); + + controller.acknowledgeRecoveredOutcome(); + expect(prewarm).toHaveBeenCalledTimes(1); + expect(new FileCodexControllerStateStore(statePath).read()?.recoveredOutcome).toBeUndefined(); + await controller.close(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it('retains a turn-boundary teleport queued while recovered local output awaits acknowledgment', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-controller-recovered-pending-')); + const statePath = path.join(directory, 'state.json'); + const lostTerminal = deferred(); + const local = appServer({ + runTurn: vi + .fn() + .mockImplementationOnce(() => lostTerminal.promise) + .mockResolvedValue(turnResult()), + turnOutcome: vi.fn(async () => + completedOutcome('turn-before-queued-teleport', 'durable exact answer', 'operation-1') + ), + }); + const store = fileStore(statePath, persistedLocal({ generation: 1 })); + const cloudClient = cloud(); + + try { + const { controller } = createController({ store, cloud: cloudClient, sessions: [local] }); + await controller.initialize(); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + + const running = controller.runTurn('turn before queued teleport'); + await vi.waitFor(() => expect(local.runTurn).toHaveBeenCalledTimes(1)); + controller.requestTeleport({ requestId: 'queued-during-turn', expectedGeneration: 1 }); + lostTerminal.reject(new Error('terminal notification lost')); + await expect(running).resolves.toMatchObject({ turnId: 'turn-before-queued-teleport' }); + + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + phase: 'teleport_pending', + pending: { requestId: 'queued-during-turn', expectedGeneration: 1 }, + recoveredOutcome: { turnId: 'turn-before-queued-teleport', status: 'completed' }, + }); + await expect(controller.runTurn('must wait for durable output')).rejects.toThrow( + 'awaiting durable output acknowledgment' + ); + + controller.acknowledgeRecoveredOutcome(); + await expect(controller.runTurn('first turn after acknowledgment')).resolves.toMatchObject({ + turnId: 'turn-1', + }); + expect(cloudClient.acquire).toHaveBeenCalledTimes(1); + expect(cloudClient.acquire).toHaveBeenCalledWith( + expect.objectContaining({ generation: 1, idempotencyKey: 'session-1:1:acquire' }) + ); + expect(local.runTurn).toHaveBeenCalledTimes(2); + await controller.close(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it.each( + (['ready', 'failed', 'in-flight'] as const).flatMap((prewarmState) => + (['completed', 'failed', 'interrupted'] as const).map( + (outcomeStatus) => [prewarmState, outcomeStatus] as const + ) + ) + )( + 'rebases a queued teleport after a crash with %s prewarm and %s recovery evidence', + async (prewarmState, outcomeStatus) => { + const directory = fs.mkdtempSync( + path.join(os.tmpdir(), `relay-controller-pending-restart-${prewarmState}-${outcomeStatus}-`) + ); + const statePath = path.join(directory, 'state.json'); + const neverSettledPrewarm = deferred<{ + sessionId: string; + prewarmId: string; + generation: number; + status: 'ready'; + expiresAt: string; + }>(); + let prewarmCalls = 0; + const prewarm = vi.fn(async (input: Parameters[0]) => { + prewarmCalls += 1; + if (prewarmCalls === 1) { + if (prewarmState === 'failed') throw new Error('first prewarm failed ambiguously'); + if (prewarmState === 'in-flight') return neverSettledPrewarm.promise; + } + return { + sessionId: input.sessionId, + prewarmId: `prewarm-${input.generation}`, + generation: input.generation, + status: 'ready' as const, + expiresAt: '2026-08-23T12:30:00.000Z', + }; + }); + const cloudClient = cloud({ prewarm }); + const store = fileStore(statePath, persistedLocal({ generation: 1 })); + const lostTerminal = deferred(); + const outcome = + outcomeStatus === 'completed' + ? completedOutcome('turn-before-crash', 'durable answer before crash', 'operation-1') + : { status: outcomeStatus, turnId: `turn-before-crash-${outcomeStatus}` }; + const firstSession = appServer({ + runTurn: vi.fn(() => lostTerminal.promise), + turnOutcome: vi.fn(async () => outcome), + }); + + try { + const first = createController({ store, cloud: cloudClient, sessions: [firstSession] }); + await first.controller.initialize(); + if (prewarmState === 'ready') { + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + } else if (prewarmState === 'failed') { + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('failed')); + } else { + expect(store.value).toMatchObject({ cloudLifecycle: 'prewarm_requested' }); + expect(store.value?.prewarmStatus).toBeUndefined(); + } + + const running = first.controller.runTurn('turn before process crash'); + await vi.waitFor(() => expect(firstSession.runTurn).toHaveBeenCalledTimes(1)); + first.controller.requestTeleport({ requestId: 'queued-before-crash', expectedGeneration: 1 }); + lostTerminal.reject(new Error('terminal notification lost before process crash')); + if (outcomeStatus === 'completed') { + await expect(running).resolves.toMatchObject({ turnId: 'turn-before-crash' }); + } else { + await expect(running).rejects.toMatchObject({ + status: outcomeStatus, + requiresRecoveryAcknowledgment: true, + }); + } + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + generation: 1, + phase: 'teleport_pending', + pending: { requestId: 'queued-before-crash', expectedGeneration: 1 }, + recoveredOutcome: { generation: 1, status: outcomeStatus }, + }); + + // Simulate a new process whose first restart cannot resume the Codex + // thread after the old Cloud identity has been fenced. + const resumeFailure = appServer({ + resumeThread: vi.fn(async () => Promise.reject(new Error('resume unavailable'))), + }); + const second = createController({ store, cloud: cloudClient, sessions: [resumeFailure] }); + await expect(second.controller.initialize()).rejects.toThrow('CODEX_THREAD_RESUME_FAILED_ON_RESTART'); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + generation: 1, + phase: 'recovery_failed', + pending: { requestId: 'queued-before-crash', expectedGeneration: 1 }, + recoveredOutcome: { generation: 1, status: outcomeStatus }, + }); + expect(prewarm).toHaveBeenCalledTimes(1); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + + // A later restart succeeds and atomically moves both the controller and + // accepted request onto generation 2. A close failure must not erase it. + const closeError = new Error('close unavailable after restart'); + const closeFailure = appServer({ close: vi.fn(async () => Promise.reject(closeError)) }); + const third = createController({ store, cloud: cloudClient, sessions: [closeFailure] }); + await expect(third.controller.initialize()).resolves.toMatchObject({ + generation: 2, + phase: 'teleport_pending', + pending: { requestId: 'queued-before-crash', expectedGeneration: 2 }, + }); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + generation: 2, + pending: { requestId: 'queued-before-crash', expectedGeneration: 2 }, + recoveredOutcome: { generation: 1, status: outcomeStatus }, + }); + expect(prewarm).toHaveBeenCalledTimes(1); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + await expect(third.controller.close()).rejects.toBe(closeError); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + phase: 'recovery_failed', + generation: 2, + pending: { requestId: 'queued-before-crash', expectedGeneration: 2 }, + recoveredOutcome: { generation: 1, status: outcomeStatus }, + }); + + const finalSession = appServer(); + const fourth = createController({ store, cloud: cloudClient, sessions: [finalSession] }); + await expect(fourth.controller.initialize()).resolves.toMatchObject({ + generation: 2, + phase: 'teleport_pending', + }); + expect(prewarm).toHaveBeenCalledTimes(1); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + + if (outcomeStatus === 'completed') { + expect(fourth.controller.takeRecoveredTurn()).toMatchObject({ turnId: 'turn-before-crash' }); + fourth.controller.acknowledgeRecoveredOutcome(); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + await fourth.controller.close({ cancelPendingTeleport: false }); + } else { + const writeError = vi.fn(async () => undefined); + await expect( + withCodexControllerShutdown(fourth.controller, () => + surfaceRecoveredCodexTerminal(fourth.controller, { json: false, writeError }) + ) + ).rejects.toMatchObject({ status: outcomeStatus }); + expect(writeError).toHaveBeenCalledWith( + `Codex recorded the turn as ${outcomeStatus}; it was not replayed.\n` + ); + } + expect(prewarm).toHaveBeenCalledTimes(2); + expect(prewarm).toHaveBeenLastCalledWith( + expect.objectContaining({ generation: 2, idempotencyKey: 'session-1:2:prewarm' }) + ); + + // This is the production terminal/error-unwind seam: stderr delivery + // was acknowledged, but throwing the recorded terminal must not turn + // a successful controller close into user cancellation. Because the + // generation-2 prewarm exists, close fences it and atomically rebases + // the accepted request onto generation 3. + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + generation: 3, + phase: 'teleport_pending', + pending: { requestId: 'queued-before-crash', expectedGeneration: 3 }, + cloudLifecycle: 'none', + }); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + + const fifthSession = appServer(); + const fifth = createController({ store, cloud: cloudClient, sessions: [fifthSession] }); + await expect(fifth.controller.initialize()).resolves.toMatchObject({ + generation: 3, + phase: 'teleport_pending', + pending: { requestId: 'queued-before-crash', expectedGeneration: 3 }, + }); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + expect(prewarm).toHaveBeenCalledTimes(3); + expect(prewarm).toHaveBeenLastCalledWith( + expect.objectContaining({ generation: 3, idempotencyKey: 'session-1:3:prewarm' }) + ); + await expect(fifth.controller.runTurn('first acknowledged Cloud turn')).resolves.toMatchObject({ + turnId: 'turn-1', + }); + expect(cloudClient.acquire).toHaveBeenCalledTimes(1); + expect(cloudClient.acquire).toHaveBeenCalledWith( + expect.objectContaining({ generation: 3, idempotencyKey: 'session-1:3:acquire' }) + ); + await fifth.controller.close(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + } + ); + + it('retains the consumed generation fence if restart crashes after terminal revoke', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-controller-terminal-fence-')); + const statePath = path.join(directory, 'state.json'); + const recovered = completedOutcome( + 'turn-before-terminal-fence', + 'durable answer before terminal fence', + 'client-before-terminal-fence' + ); + const store = fileStore( + statePath, + persistedLocal({ + generation: 3, + cloudLifecycle: 'prewarmed', + prewarmId: 'prewarm-3', + prewarmStatus: 'ready', + recoveredOutcome: { + sessionId: 'session-1', + threadId: 'thread-1', + generation: 3, + clientUserMessageId: 'client-before-terminal-fence', + turnId: recovered.turnId, + status: 'completed', + result: recovered.result, + }, + }) + ); + const cloudClient = cloud(); + const startFailure = appServer({ + initialize: vi.fn(async () => Promise.reject(new Error('process crashed after revoke'))), + }); + + try { + const first = createController({ store, cloud: cloudClient, sessions: [startFailure] }); + await expect(first.controller.initialize()).rejects.toThrow('process crashed after revoke'); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + generation: 3, + phase: 'rolling_back', + cloudLifecycle: 'cleanup_requested', + recoveredOutcome: { generation: 3, turnId: recovered.turnId }, + }); + + const resumed = appServer(); + const second = createController({ store, cloud: cloudClient, sessions: [resumed] }); + await expect(second.controller.initialize()).resolves.toMatchObject({ + generation: 4, + phase: 'local', + cloudLifecycle: 'none', + }); + expect(cloudClient.revoke).toHaveBeenCalledTimes(2); + expect(cloudClient.revoke).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ generation: 3, idempotencyKey: 'session-1:3:revoke' }) + ); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + generation: 4, + recoveredOutcome: { generation: 3, turnId: recovered.turnId }, + }); + expect(new FileCodexControllerStateStore(statePath).read()?.prewarmId).toBeUndefined(); + expect(new FileCodexControllerStateStore(statePath).read()?.prewarmStatus).toBeUndefined(); + + expect(second.controller.takeRecoveredTurn()).toMatchObject({ turnId: recovered.turnId }); + second.controller.acknowledgeRecoveredOutcome(); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + expect(cloudClient.prewarm).toHaveBeenCalledWith( + expect.objectContaining({ generation: 4, idempotencyKey: 'session-1:4:prewarm' }) + ); + await second.controller.close(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it('fails outcome-uncertain when an accepted remote turn cannot be found after fencing', async () => { + const original = appServer({ runTurn: vi.fn(async () => Promise.reject(new Error('transport lost'))) }); + const replacement = appServer({ turnOutcome: vi.fn(async () => ({ status: 'absent' as const })) }); + const { controller } = createController({ sessions: [original, replacement] }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('ambiguous')).rejects.toThrow('outcome is uncertain'); + expect(replacement.runTurn).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'outcome_uncertain', execution: 'fenced' }); + }); + + it('fails outcome-uncertain instead of claiming completion when history lost the exact answer', async () => { + const original = appServer({ + runTurn: vi.fn(async () => Promise.reject(new Error('notification lost'))), + }); + const replacement = appServer({ + turnOutcome: vi.fn(async () => { + throw new Error('completed history omitted assistant answer'); + }), + }); + const { controller } = createController({ sessions: [original, replacement] }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-missing-answer', expectedGeneration: 1 }); + + await expect(controller.runTurn('need exact answer')).rejects.toThrow('outcome is uncertain'); + expect(controller.status()).toMatchObject({ + phase: 'outcome_uncertain', + lastError: 'TURN_RECONCILIATION_UNAVAILABLE', + }); + }); + + it('recovers an expired later-turn lease before submitting the next prompt locally', async () => { + const original = appServer(); + const replacement = appServer(); + const cloudClient = cloud({ status: vi.fn(async (input) => lifecycle(input, 'expired')) }); + const { controller } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + await controller.runTurn('first remote'); + + await controller.runTurn('after lease expiry'); + + expect(replacement.runTurn).toHaveBeenCalledWith({ + threadId: 'thread-1', + text: 'after lease expiry', + clientUserMessageId: 'operation-3', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); + }); + + it('fails fenced and never resumes locally when first-turn revoke is unconfirmed', async () => { + const original = appServer({ runTurn: vi.fn(async () => Promise.reject(new Error('remote died'))) }); + const replacement = appServer(); + const sealed = sealHandle(); + const cloudClient = cloud({ + revoke: vi.fn(async () => Promise.reject(new Error('revoke timeout'))), + status: vi.fn(async (input) => lifecycle(input, 'ready')), + }); + const { controller, createAppServer } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + checkpointAndSeal: async () => sealed, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('must not run locally')).rejects.toThrow('CLOUD_FENCE_UNCONFIRMED'); + + expect(original.close).toHaveBeenCalled(); + expect(createAppServer).toHaveBeenCalledTimes(1); + expect(sealed.resumeLocal).not.toHaveBeenCalled(); + expect(replacement.resumeThread).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'fenced', execution: 'fenced' }); + }); + + it('does not trust a mount state or arbitrary receipt when no real seal provider is available', async () => { + const seal = vi + .fn() + .mockRejectedValue(new Error('checkpoint-and-seal API unavailable')); + const original = appServer(); + const replacement = appServer(); + const cloudClient = cloud(); + const { controller } = createController({ + checkpointAndSeal: seal, + sessions: [original, replacement], + cloud: cloudClient, + }); + await controller.initialize(); + expect(controller.status()).toMatchObject({ phase: 'local', workspaceSource: 'unavailable' }); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('local only')).resolves.toMatchObject({ turnId: 'turn-1' }); + + expect(cloudClient.acquire).not.toHaveBeenCalled(); + expect(cloudClient.revoke).toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ text: 'local only', execution: { kind: 'local', workspaceRoot: '/repo' } }) + ); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); + }); + + it('does not persist a seal whose restore identity targets another local root', async () => { + const sealed = sealHandle({ + restore: { + lifecycleId: 'session-1:1:operation-1', + resumeId: 'resume-1', + workspaceId: 'workspace-1', + localRoot: '/other', + }, + }); + const original = appServer(); + const replacement = appServer(); + const cloudClient = cloud(); + const { controller, store } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + checkpointAndSeal: async () => sealed, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('must stay on this mount')).resolves.toMatchObject({ turnId: 'turn-1' }); + + expect(cloudClient.acquire).not.toHaveBeenCalled(); + expect(sealed.resumeLocal).toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'must stay on this mount', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }) + ); + expect(store.value?.source).toBeUndefined(); + expect(store.value?.mountRestore).toBeUndefined(); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); + }); + + it('thaws the exact sealed mount after acquire failure before local resume', async () => { + const sealed = sealHandle(); + const original = appServer(); + const replacement = appServer(); + const cloudClient = cloud({ acquire: vi.fn(async () => Promise.reject(new Error('acquire failed'))) }); + const { controller } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + checkpointAndSeal: async () => sealed, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('local after abort')).resolves.toMatchObject({ turnId: 'turn-1' }); + + expect(sealed.resumeLocal).toHaveBeenCalled(); + expect(replacement.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); + expect(replacement.runTurn).toHaveBeenCalledWith({ + threadId: 'thread-1', + text: 'local after abort', + clientUserMessageId: 'operation-2', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }); + }); + + it('does not resume local execution when a fenced mount cannot be thawed', async () => { + const sealed = sealHandle({ resumeLocal: vi.fn(async () => Promise.reject(new Error('mount dead'))) }); + const original = appServer(); + const replacement = appServer(); + const cloudClient = cloud({ acquire: vi.fn(async () => Promise.reject(new Error('acquire failed'))) }); + const { controller, createAppServer } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + checkpointAndSeal: async () => sealed, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('must stay fenced')).rejects.toThrow( + 'RELAYFILE_RESUME_FAILED_AFTER_FENCE' + ); + + expect(createAppServer).toHaveBeenCalledTimes(1); + expect(replacement.resumeThread).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'recovery_failed', execution: 'fenced' }); + }); + + it('polls warming lifecycle status to ready before acquisition', async () => { + const cloudClient = cloud({ + prewarm: vi.fn(async (input) => ({ + sessionId: input.sessionId, + prewarmId: `prewarm-${input.generation}`, + generation: input.generation, + status: 'warming' as const, + expiresAt: '2026-08-23T12:30:00.000Z', + retryAfterMs: 1, + })), + status: vi + .fn() + .mockImplementationOnce(async (input) => ({ + ...lifecycle(input, 'warming'), + prewarmId: 'prewarm-1', + retryAfterMs: 1, + })) + .mockImplementationOnce(async (input) => ({ + ...lifecycle(input, 'ready'), + prewarmId: 'prewarm-1', + })), + }); + const { controller, checkpointAndSeal } = createController({ cloud: cloudClient }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await controller.runTurn('after convergence'); + + expect(cloudClient.status).toHaveBeenCalledTimes(2); + expect(vi.mocked(cloudClient.status).mock.invocationCallOrder.at(-1)).toBeLessThan( + checkpointAndSeal.mock.invocationCallOrder[0]! + ); + expect(cloudClient.acquire).toHaveBeenCalled(); + }); + + it('bounds non-convergence and recovers locally instead of blocking forever', async () => { + const cloudClient = cloud({ + prewarm: vi.fn(async (input) => ({ + sessionId: input.sessionId, + prewarmId: `prewarm-${input.generation}`, + generation: input.generation, + status: 'warming' as const, + expiresAt: '2026-08-23T12:30:00.000Z', + retryAfterMs: 1, + })), + status: vi.fn(async (input) => ({ ...lifecycle(input, 'warming'), retryAfterMs: 1 })), + }); + const original = appServer(); + const replacement = appServer(); + const { controller } = createController({ + cloud: cloudClient, + sessions: [original, replacement], + lifecycleDeadlineMs: 5, + lifecyclePollIntervalMs: 1, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('bounded')).resolves.toMatchObject({ turnId: 'turn-1' }); + + expect(cloudClient.acquire).not.toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ text: 'bounded', execution: { kind: 'local', workspaceRoot: '/repo' } }) + ); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); + }); + + it('polls Codex environment/status until ready', async () => { + const original = appServer({ + environmentStatus: vi + .fn() + .mockResolvedValueOnce({ status: 'connecting' }) + .mockResolvedValueOnce({ status: 'ready' }), + }); + const { controller } = createController({ sessions: [original] }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await controller.runTurn('ready after poll'); + + expect(original.environmentStatus).toHaveBeenCalledTimes(2); + }); + + it('rolls back only after a confirmed fence and resumes through a fresh app-server', async () => { + const original = appServer(); + const replacement = appServer(); + const sealed = sealHandle(); + const cloudClient = cloud(); + const { controller } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + checkpointAndSeal: async () => sealed, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + await controller.runTurn('remote'); + + const status = await controller.rollback(); + + expect(original.close).toHaveBeenCalled(); + expect(sealed.resumeLocal).toHaveBeenCalled(); + expect(vi.mocked(cloudClient.revoke).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(sealed.resumeLocal).mock.invocationCallOrder[0]! + ); + expect(replacement.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + expect(status).toMatchObject({ phase: 'local', generation: 2, execution: 'local' }); + }); + + it('does not resume a consumed checkpoint while Cloud handback is cleanup_pending', async () => { + const handback = deferred(); + const original = appServer(); + const replacement = appServer(); + const sealed = sealHandle(); + const cloudClient = cloud({ + revoke: vi.fn(async (input) => lifecycle(input, 'cleanup_pending')), + status: vi.fn(() => handback.promise), + }); + const { controller } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + checkpointAndSeal: async () => sealed, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + await controller.runTurn('remote'); + + const rollingBack = controller.rollback(); + await vi.waitFor(() => expect(cloudClient.status).toHaveBeenCalled()); + expect(sealed.resumeLocal).not.toHaveBeenCalled(); + + handback.resolve(lifecycle({ sessionId: 'session-1', generation: 1 }, 'revoked')); + await rollingBack; + expect(sealed.resumeLocal).toHaveBeenCalledOnce(); + }); + + it('keeps the source sealed when revoke ignores abort and never settles', async () => { + const original = appServer(); + const replacement = appServer(); + const sealed = sealHandle(); + const cloudClient = cloud({ revoke: vi.fn(async () => new Promise(() => undefined)) }); + const { controller } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + lifecycleDeadlineMs: 5, + checkpointAndSeal: async () => sealed, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + await controller.runTurn('remote'); + + await expect(controller.rollback()).rejects.toThrow('CLOUD_FENCE_UNCONFIRMED'); + + expect(sealed.resumeLocal).not.toHaveBeenCalled(); + expect(replacement.resumeThread).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'fenced' }); + }); + + it('restarts a clean local session without trying to revoke or poll Cloud', async () => { + const store = memoryStore(persistedLocal()); + const resumed = appServer(); + const cloudClient = cloud(); + const { controller } = createController({ store, sessions: [resumed], cloud: cloudClient }); + + await expect(controller.initialize()).resolves.toMatchObject({ phase: 'local', generation: 3 }); + + expect(cloudClient.revoke).not.toHaveBeenCalled(); + expect(cloudClient.status).not.toHaveBeenCalled(); + expect(resumed.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + }); + + it('fences a persisted prewarm request whose response may have been lost', async () => { + const store = memoryStore(persistedLocal({ cloudLifecycle: 'prewarm_requested' })); + const resumed = appServer(); + const cloudClient = cloud(); + const { controller } = createController({ store, sessions: [resumed], cloud: cloudClient }); + + await expect(controller.initialize()).resolves.toMatchObject({ phase: 'local', generation: 4 }); + + expect(cloudClient.revoke).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: 'session-1', generation: 3, idempotencyKey: 'session-1:3:revoke' }) + ); + expect(resumed.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + }); + + it('persists fenced on restart and never constructs a local app-server after unconfirmed revoke', async () => { + const persisted = persistedRemote(); + persisted.remote!.leaseExpiresAt = '2026-08-23T11:00:00.000Z'; + const store = memoryStore(persisted); + const cloudClient = cloud({ + revoke: vi.fn(async () => Promise.reject(new Error('timeout'))), + status: vi.fn(async (input) => lifecycle(input, 'ready')), + }); + const { controller, createAppServer, resumePersistedLocalMount } = createController({ + store, + cloud: cloudClient, + liveTeleportEnabled: false, + }); + + await expect(controller.initialize()).rejects.toThrow('CLOUD_FENCE_UNCONFIRMED_ON_RESTART'); + + expect(createAppServer).not.toHaveBeenCalled(); + expect(resumePersistedLocalMount).not.toHaveBeenCalled(); + expect(cloudClient.prewarm).not.toHaveBeenCalled(); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + expect(store.value).toMatchObject({ phase: 'fenced', generation: 7 }); + expect(controller.status()).toMatchObject({ + execution: 'fenced', + liveTeleport: { enabled: false, reason: 'ambient-unset' }, + }); + }); + + it('resumes locally only after Cloud authoritatively confirms the persisted lease expired', async () => { + const expired = persistedRemote({ + remote: { ...persistedRemote().remote!, leaseExpiresAt: '2026-08-23T11:59:59.000Z' }, + }); + const store = memoryStore(expired); + const resumed = appServer(); + const cloudClient = cloud({ + revoke: vi.fn(async (input) => ({ ...lifecycle(input, 'expired'), status: 'expired' as const })), + }); + const { controller, resumePersistedLocalMount } = createController({ + store, + sessions: [resumed], + cloud: cloudClient, + }); + + await expect(controller.initialize()).resolves.toMatchObject({ + phase: 'local', + generation: 8, + threadId: 'thread-1', + }); + + expect(cloudClient.revoke).toHaveBeenCalled(); + expect(resumePersistedLocalMount).toHaveBeenCalledWith( + expect.objectContaining({ source: { kind: 'relayfile-checkpoint-seal' } }) + ); + expect(resumed.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + expect(store.value?.source).toBeUndefined(); + expect(store.value?.mountRestore).toBeUndefined(); + }); + + it('reconciles a persisted completed turn before reopening local execution after restart', async () => { + const store = memoryStore( + persistedRemote({ + inFlightTurn: { clientUserMessageId: 'client-persisted-1', execution: 'remote' }, + }) + ); + const resumed = appServer({ + turnOutcome: vi.fn(async () => completedOutcome('turn-persisted-1')), + }); + const { controller } = createController({ store, sessions: [resumed] }); + + await expect(controller.initialize()).resolves.toMatchObject({ phase: 'local', generation: 8 }); + expect(resumed.turnOutcome).toHaveBeenCalledWith({ + threadId: 'thread-1', + clientUserMessageId: 'client-persisted-1', + }); + expect(store.value?.inFlightTurn).toBeUndefined(); + expect(controller.takeRecoveredTurn()).toMatchObject({ + reconciled: true, + response: { + turn: { + items: expect.arrayContaining([ + expect.objectContaining({ type: 'agentMessage', text: 'answer for turn-persisted-1' }), + ]), + }, + }, + }); + expect(controller.takeRecoveredTurn()).toBeUndefined(); + expect(store.value?.recoveredOutcome).toMatchObject({ + sessionId: 'session-1', + threadId: 'thread-1', + generation: 8, + clientUserMessageId: 'client-persisted-1', + turnId: 'turn-persisted-1', + status: 'completed', + }); + + const restartedOnce = appServer(); + const second = createController({ store, sessions: [restartedOnce] }); + await second.controller.initialize(); + expect(restartedOnce.turnOutcome).not.toHaveBeenCalled(); + expect(second.controller.takeRecoveredTurn()).toMatchObject({ + completed: { + params: { + turn: { + items: expect.arrayContaining([expect.objectContaining({ text: 'answer for turn-persisted-1' })]), + }, + }, + }, + }); + + const restartedTwice = appServer(); + const third = createController({ store, sessions: [restartedTwice] }); + await third.controller.initialize(); + expect(restartedTwice.turnOutcome).not.toHaveBeenCalled(); + expect(third.controller.takeRecoveredTurn()).toMatchObject({ turnId: 'turn-persisted-1' }); + third.controller.acknowledgeRecoveredOutcome(); + expect(store.value?.recoveredOutcome).toBeUndefined(); + }); + + it.each(['failed', 'interrupted'] as const)( + 'preserves a persisted crash-reconciled %s terminal for one-shot CLI failure', + async (status) => { + const store = memoryStore( + persistedRemote({ + inFlightTurn: { clientUserMessageId: `client-persisted-${status}`, execution: 'remote' }, + }) + ); + const resumed = appServer({ + turnOutcome: vi.fn(async () => ({ status, turnId: `turn-persisted-${status}` })), + }); + const cloudClient = cloud(); + const { controller } = createController({ store, sessions: [resumed], cloud: cloudClient }); + + await expect(controller.initialize()).resolves.toMatchObject({ phase: 'local', generation: 8 }); + expect(resumed.runTurn).not.toHaveBeenCalled(); + expect(cloudClient.prewarm).not.toHaveBeenCalled(); + expect(store.value?.inFlightTurn).toBeUndefined(); + expect(controller.takeRecoveredTerminal()).toMatchObject({ + name: 'CodexTurnRecordedError', + status, + recordedTerminal: true, + }); + expect(controller.takeRecoveredTerminal()).toBeUndefined(); + expect(controller.takeRecoveredTurn()).toBeUndefined(); + expect(store.value?.recoveredOutcome).toMatchObject({ + sessionId: 'session-1', + threadId: 'thread-1', + generation: 8, + clientUserMessageId: `client-persisted-${status}`, + turnId: `turn-persisted-${status}`, + status, + }); + + const restartedOnce = appServer(); + const second = createController({ store, cloud: cloudClient, sessions: [restartedOnce] }); + await second.controller.initialize(); + expect(restartedOnce.turnOutcome).not.toHaveBeenCalled(); + expect(second.controller.takeRecoveredTerminal()).toMatchObject({ status }); + + const restartedTwice = appServer(); + const third = createController({ store, cloud: cloudClient, sessions: [restartedTwice] }); + await third.controller.initialize(); + expect(restartedTwice.turnOutcome).not.toHaveBeenCalled(); + expect(third.controller.takeRecoveredTerminal()).toMatchObject({ status }); + third.controller.acknowledgeRecoveredOutcome(); + expect(store.value?.recoveredOutcome).toBeUndefined(); + } + ); + + it('keeps restart recovery outcome-uncertain when completed history cannot return the exact answer', async () => { + const store = memoryStore( + persistedRemote({ + inFlightTurn: { clientUserMessageId: 'client-persisted-missing', execution: 'remote' }, + }) + ); + const resumed = appServer({ + turnOutcome: vi.fn(async () => { + throw new Error('completed history omitted assistant answer'); + }), + }); + const { controller } = createController({ store, sessions: [resumed] }); + + await expect(controller.initialize()).rejects.toThrow('outcome is uncertain'); + expect(store.value).toMatchObject({ + phase: 'outcome_uncertain', + generation: 7, + lastError: 'TURN_RECONCILIATION_UNAVAILABLE', + inFlightTurn: { clientUserMessageId: 'client-persisted-missing' }, + }); + expect(controller.takeRecoveredTurn()).toBeUndefined(); + }); + + it('fails recovery if the same thread cannot be resumed after a confirmed fence', async () => { + const store = memoryStore(persistedRemote()); + const resumed = appServer({ resumeThread: vi.fn(async () => Promise.reject(new Error('gone'))) }); + const { controller } = createController({ store, sessions: [resumed] }); + + await expect(controller.initialize()).rejects.toThrow('CODEX_THREAD_RESUME_FAILED_ON_RESTART'); + + expect(store.value).toMatchObject({ phase: 'recovery_failed', generation: 7 }); + }); + + it('does not adopt a persisted thread from a different workspace', async () => { + const store = memoryStore(persistedRemote({ workspaceRoot: '/different-repo' })); + const { controller, createAppServer } = createController({ store }); + + await expect(controller.initialize()).rejects.toThrow('belongs to /different-repo'); + expect(createAppServer).not.toHaveBeenCalled(); + }); + + it('marks shutdown fenced when revoke cannot be confirmed', async () => { + const original = appServer(); + const cloudClient = cloud({ + revoke: vi.fn(async () => Promise.reject(new Error('timeout'))), + status: vi.fn(async (input) => lifecycle(input, 'ready')), + }); + const { controller } = createController({ sessions: [original], cloud: cloudClient }); + await controller.initialize(); + + await controller.close(); + + expect(original.close).toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'fenced', execution: 'fenced' }); + }); + + it('cancels and fences a racing prewarm without a late state mutation', async () => { + const prewarmStarted = deferred(); + const cloudClient = cloud({ + prewarm: vi.fn( + (input) => + new Promise((_, reject) => { + prewarmStarted.resolve(); + input.signal?.addEventListener( + 'abort', + () => reject(Object.assign(new Error('aborted'), { name: 'AbortError' })), + { once: true } + ); + }) + ), + }); + const { controller, store } = createController({ cloud: cloudClient }); + await controller.initialize(); + await prewarmStarted.promise; + + await controller.close(); + await Promise.resolve(); + + expect(cloudClient.revoke).toHaveBeenCalled(); + expect(store.value).toMatchObject({ cloudLifecycle: 'none' }); + expect(store.value?.prewarmStatus).toBeUndefined(); + }); + + it('bounds shutdown when prewarm ignores abort, then fences the ambiguous request', async () => { + const original = appServer(); + const cloudClient = cloud({ + prewarm: vi.fn(async () => new Promise(() => undefined)), + }); + const { controller } = createController({ + sessions: [original], + cloud: cloudClient, + lifecycleDeadlineMs: 5, + }); + await controller.initialize(); + + await controller.close(); + + expect(cloudClient.revoke).toHaveBeenCalled(); + expect(original.close).toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ cloudLifecycle: 'none' }); + }); + + it('resumes and verifies the sealed local mount on ordinary shutdown after remote execution', async () => { + const sealed = sealHandle(); + const original = appServer(); + const { controller } = createController({ + sessions: [original], + checkpointAndSeal: async () => sealed, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + await controller.runTurn('remote turn'); + + await controller.close(); + + expect(original.close).toHaveBeenCalled(); + expect(sealed.resumeLocal).toHaveBeenCalled(); + expect(sealed.close).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2, execution: 'local' }); + }); +}); + +describe('FileCodexControllerStateStore', () => { + it.each([ + ['foreign version', { ...persistedLocal(), version: 2 }], + ['negative generation', { ...persistedLocal(), generation: -1 }], + ['invalid pid', { ...persistedLocal(), controllerPid: 0 }], + ['malformed phase', { ...persistedLocal(), phase: 'teleported' }], + ['malformed pending', { ...persistedLocal(), pending: { requestId: '', expectedGeneration: 3 } }], + ['secret-bearing source', { ...persistedLocal(), source: { ...source } }], + [ + 'cross-root restore', + { + ...persistedLocal(), + mountRestore: { lifecycleId: 'lifecycle-3', resumeId: 'resume-3', localRoot: '/other' }, + }, + ], + ['partial remote', { ...persistedLocal(), remote: { sessionId: 'session-1', generation: 3 } }], + [ + 'cross-generation remote', + { + ...persistedRemote(), + remote: { ...persistedRemote().remote!, generation: 8 }, + }, + ], + [ + 'cross-root destination verification', + { + ...persistedRemote(), + remote: { + ...persistedRemote().remote!, + verification: { + ...persistedRemote().remote!.verification, + localRoot: '/different-workspace', + }, + }, + }, + ], + [ + 'malformed in-flight turn', + { ...persistedLocal(), inFlightTurn: { clientUserMessageId: '', execution: 'remote' } }, + ], + [ + 'cross-session recovered outcome', + { + ...persistedLocal(), + recoveredOutcome: { + sessionId: 'other-session', + threadId: 'thread-1', + generation: 3, + clientUserMessageId: 'client-recovered', + turnId: 'turn-recovered', + status: 'completed', + result: completedOutcome('turn-recovered').result, + }, + }, + ], + [ + 'future-generation recovered outcome', + { + ...persistedLocal(), + recoveredOutcome: { + sessionId: 'session-1', + threadId: 'thread-1', + generation: 4, + clientUserMessageId: 'client-future', + turnId: 'turn-future', + status: 'failed', + }, + }, + ], + [ + 'two-generations-stale recovered outcome', + { + ...persistedLocal(), + recoveredOutcome: { + sessionId: 'session-1', + threadId: 'thread-1', + generation: 1, + clientUserMessageId: 'client-stale', + turnId: 'turn-stale', + status: 'interrupted', + }, + }, + ], + ])('rejects %s instead of adopting malformed controller state', (_name, value) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-controller-state-')); + const file = path.join(directory, 'state.json'); + try { + fs.writeFileSync(file, JSON.stringify(value)); + expect(() => new FileCodexControllerStateStore(file).read()).toThrow( + 'invalid or from an unsupported version' + ); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/cli/src/cli/lib/codex-live-controller.ts b/packages/cli/src/cli/lib/codex-live-controller.ts new file mode 100644 index 000000000..7614e435d --- /dev/null +++ b/packages/cli/src/cli/lib/codex-live-controller.ts @@ -0,0 +1,1719 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; + +import type { + LiveTeleportCloudClient, + LiveTeleportEnvironment, + LiveTeleportLifecycleStatus, + LiveTeleportWorkspaceSource, +} from '@agent-relay/cloud'; +import { LIVE_TELEPORT_MIN_TURN_LEASE_MS } from '@agent-relay/cloud'; + +import { + CodexAppServerTurnTerminalError, + type CodexAppServerSession, + type CodexTurnOutcome, + type CodexTurnResult, +} from './codex-app-server.js'; + +export type CodexControllerPhase = + | 'local' + | 'teleport_pending' + | 'acquiring' + | 'verifying' + | 'remote' + | 'rolling_back' + | 'outcome_uncertain' + | 'fenced' + | 'recovery_failed'; + +export type CodexCloudLifecycleIntent = + | 'none' + | 'prewarm_requested' + | 'prewarmed' + | 'acquire_requested' + | 'acquired' + | 'cleanup_requested'; + +export type CodexTeleportRequest = { + requestId: string; + expectedGeneration: number; +}; + +export const RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV = 'RELAY_LIVE_SESSION_TELEPORT_ENABLED'; + +export type CodexLiveTeleportStartupReason = + | 'ambient-exact-true' + | 'ambient-unset' + | 'ambient-value-not-exact-true'; + +export type CodexLiveTeleportStartupSwitch = Readonly<{ + enabled: boolean; + reason: CodexLiveTeleportStartupReason; +}>; + +export const DEFAULT_CODEX_LIVE_TELEPORT_STARTUP_SWITCH: CodexLiveTeleportStartupSwitch = Object.freeze({ + enabled: false, + reason: 'ambient-unset', +}); + +/** + * Live execution teleport is a dark-launched, local-only capability. Only the + * exact, unmodified string `true` enables it. The caller must pass the value + * captured from the ambient process environment before dotenv is loaded; + * unset, whitespace/case variants, and every unknown value fail closed. + */ +export function resolveCodexLiveTeleportStartupSwitch( + ambientRawValue: string | undefined +): CodexLiveTeleportStartupSwitch { + if (ambientRawValue === 'true') { + return Object.freeze({ enabled: true, reason: 'ambient-exact-true' }); + } + if (ambientRawValue === undefined) return DEFAULT_CODEX_LIVE_TELEPORT_STARTUP_SWITCH; + return Object.freeze({ enabled: false, reason: 'ambient-value-not-exact-true' }); +} + +type CodexRecoveredOutcome = { + sessionId: string; + threadId: string; + generation: number; + clientUserMessageId: string; + turnId: string; +} & ({ status: 'completed'; result: CodexTurnResult } | { status: 'failed' | 'interrupted' }); + +export type CodexControllerState = { + version: 1; + sessionId: string; + threadId: string; + workspaceRoot: string; + source?: { kind: LiveTeleportWorkspaceSource['kind'] }; + mountRestore?: CodexMountRestoreIdentity; + generation: number; + phase: CodexControllerPhase; + controllerPid: number; + socketPath: string; + turnActive: boolean; + pending?: CodexTeleportRequest; + lastRequestId?: string; + prewarmId?: string; + prewarmStatus?: 'warming' | 'ready' | 'failed'; + /** Durable evidence that Cloud may own resources for this generation. */ + cloudLifecycle?: CodexCloudLifecycleIntent; + remote?: Omit & { + /** True only after the first remote turn/completed notification. */ + attached: boolean; + }; + inFlightTurn?: { + clientUserMessageId: string; + execution: 'local' | 'remote'; + }; + /** + * Durable, at-least-once output delivery retained until the CLI acknowledges + * that its asynchronous writer completed. Model execution is never replayed. + */ + recoveredOutcome?: CodexRecoveredOutcome; + lastError?: string; + updatedAt: string; +}; + +export type PublicCodexControllerStatus = Omit< + CodexControllerState, + 'source' | 'remote' | 'mountRestore' | 'recoveredOutcome' +> & { + execution: 'local' | 'verifying' | 'cloud' | 'fenced'; + controller: 'local'; + /** Immutable process-start gate; never inferred from dotenv-mutated state. */ + liveTeleport: CodexLiveTeleportStartupSwitch; + remote?: Pick< + LiveTeleportEnvironment, + 'environmentId' | 'generation' | 'workspaceCwd' | 'connectExpiresAt' | 'leaseExpiresAt' + > & { + attached: true; + }; + workspaceSource: LiveTeleportWorkspaceSource['kind'] | 'unavailable'; +}; + +export interface CodexControllerStateStore { + read(): CodexControllerState | null; + write(state: CodexControllerState): void; +} + +export class FileCodexControllerStateStore implements CodexControllerStateStore { + constructor(readonly filePath: string) {} + + read(): CodexControllerState | null { + try { + return validatePersistedState(JSON.parse(fs.readFileSync(this.filePath, 'utf8')) as unknown); + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return null; + throw error; + } + } + + write(state: CodexControllerState): void { + fs.mkdirSync(path.dirname(this.filePath), { recursive: true, mode: 0o700 }); + const temporary = `${this.filePath}.tmp-${process.pid}-${randomUUID()}`; + fs.writeFileSync(temporary, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 }); + const temporaryFd = fs.openSync(temporary, 'r'); + try { + fs.fsyncSync(temporaryFd); + } finally { + fs.closeSync(temporaryFd); + } + fs.renameSync(temporary, this.filePath); + fs.chmodSync(this.filePath, 0o600); + try { + const directoryFd = fs.openSync(path.dirname(this.filePath), 'r'); + try { + fs.fsyncSync(directoryFd); + } finally { + fs.closeSync(directoryFd); + } + } catch (error) { + if ( + !( + error instanceof Error && + 'code' in error && + ['EBADF', 'EISDIR', 'EINVAL', 'ENOTSUP', 'EPERM'].includes(String(error.code)) + ) + ) { + throw error; + } + } + } +} + +export type CodexWorkspaceSealInput = { + sessionId: string; + generation: number; + threadId: string; + workspaceRoot: string; + lifecycleId: string; + signal?: AbortSignal; +}; + +export type CodexWorkspaceSealHandle = { + source: LiveTeleportWorkspaceSource; + restore: CodexMountRestoreIdentity; + /** Restarts/thaws the exact sealed mount and resolves only after readiness. */ + resumeLocal(signal?: AbortSignal): Promise; + /** Releases resources when the controller is closing without local resume. */ + close(): Promise; +}; + +export type CodexControllerCloseOptions = { + /** + * Explicit operator/session completion is the only boundary allowed to + * cancel an accepted, not-yet-consumed teleport. Error unwinds retain it. + */ + cancelPendingTeleport?: boolean; +}; + +export type CodexMountRestoreIdentity = { + lifecycleId: string; + resumeId?: string; + workspaceId?: string; + localRoot: string; +}; + +export type CodexWorkspaceSealProvider = ( + input: CodexWorkspaceSealInput +) => Promise; +// Contract: a rejection (including AbortSignal) must leave the local mount +// ready, or restore it before rejecting. A fulfilled handle transfers mount +// lifecycle ownership to the controller until resumeLocal()/close(). + +export type CodexPersistedMountResumeProvider = ( + input: CodexWorkspaceSealInput & { + source?: { kind: LiveTeleportWorkspaceSource['kind'] }; + restore: CodexMountRestoreIdentity; + } +) => Promise; + +export type CodexLiveControllerDependencies = { + cloud?: LiveTeleportCloudClient; + store: CodexControllerStateStore; + createAppServer: () => Promise; + probeCapability: () => Promise; + checkpointAndSeal: CodexWorkspaceSealProvider; + resumePersistedLocalMount: CodexPersistedMountResumeProvider; + sleep: (milliseconds: number) => Promise; + now: () => Date; + sessionId: () => string; + operationId: () => string; + pid: number; +}; + +export type CodexLiveControllerOptions = { + workspaceRoot: string; + socketPath: string; + model?: string; + /** Defaults off. Production passes the immutable pre-dotenv startup gate. */ + liveTeleportStartupSwitch?: CodexLiveTeleportStartupSwitch; + lifecycleDeadlineMs?: number; + lifecyclePollIntervalMs?: number; +}; + +const DEFAULT_LIFECYCLE_DEADLINE_MS = 60_000; +const DEFAULT_LIFECYCLE_POLL_INTERVAL_MS = 500; +const TURN_BLOCKED_PHASES = new Set([ + 'recovery_failed', + 'outcome_uncertain', + 'fenced', + 'rolling_back', + 'acquiring', + 'verifying', +]); +const CONTROLLER_PHASES = new Set([ + 'local', + 'teleport_pending', + 'acquiring', + 'verifying', + 'remote', + 'rolling_back', + 'outcome_uncertain', + 'fenced', + 'recovery_failed', +]); +const CLOUD_LIFECYCLE_INTENTS = new Set([ + 'none', + 'prewarm_requested', + 'prewarmed', + 'acquire_requested', + 'acquired', + 'cleanup_requested', +]); + +function record(value: unknown): Record | undefined { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function nonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.trim().length > 0; +} + +function validOptionalString(value: unknown): boolean { + return value === undefined || nonEmptyString(value); +} + +function validTimestamp(value: unknown): value is string { + return nonEmptyString(value) && !Number.isNaN(Date.parse(value)); +} + +function hasOnlyKeys(value: Record, keys: readonly string[]): boolean { + return Object.keys(value).every((key) => keys.includes(key)); +} + +function invalidNestedState(state: Record): boolean { + const pending = state.pending === undefined ? undefined : record(state.pending); + const source = state.source === undefined ? undefined : record(state.source); + const restore = state.mountRestore === undefined ? undefined : record(state.mountRestore); + const remote = state.remote === undefined ? undefined : record(state.remote); + const inFlight = state.inFlightTurn === undefined ? undefined : record(state.inFlightTurn); + const recovered = state.recoveredOutcome === undefined ? undefined : record(state.recoveredOutcome); + const generation = Number(state.generation); + const workspaceRoot = String(state.workspaceRoot); + if ( + (pending !== undefined && + (!nonEmptyString(pending.requestId) || pending.expectedGeneration !== generation)) || + (state.pending !== undefined && !pending) || + (source !== undefined && + (!hasOnlyKeys(source, ['kind']) || source.kind !== 'relayfile-checkpoint-seal')) || + (state.source !== undefined && !source) || + (restore !== undefined && + (!nonEmptyString(restore.lifecycleId) || + !validOptionalString(restore.resumeId) || + !validOptionalString(restore.workspaceId) || + !nonEmptyString(restore.localRoot) || + !path.isAbsolute(restore.localRoot) || + path.resolve(restore.localRoot) !== path.resolve(workspaceRoot))) || + (state.mountRestore !== undefined && !restore) || + (source !== undefined && !restore) || + (inFlight !== undefined && + (!nonEmptyString(inFlight.clientUserMessageId) || + (inFlight.execution !== 'local' && inFlight.execution !== 'remote'))) || + (state.inFlightTurn !== undefined && !inFlight) || + (state.recoveredOutcome !== undefined && !recovered) || + (inFlight !== undefined && recovered !== undefined) || + (recovered !== undefined && !validRecoveredOutcome(state, recovered)) || + (recovered !== undefined && + state.phase !== 'local' && + state.phase !== 'teleport_pending' && + state.phase !== 'rolling_back' && + state.phase !== 'fenced' && + state.phase !== 'recovery_failed') || + (recovered !== undefined && (source !== undefined || restore !== undefined || remote !== undefined)) || + !validOptionalString(state.prewarmId) || + (state.prewarmStatus !== undefined && + state.prewarmStatus !== 'warming' && + state.prewarmStatus !== 'ready' && + state.prewarmStatus !== 'failed') + ) { + return true; + } + if (!remote) return state.remote !== undefined || state.phase === 'remote' || state.phase === 'verifying'; + const verification = record(remote.verification); + const observed = record(verification?.observed); + const health = record(verification?.health); + return ( + 'connectPath' in remote || + 'execServerUrl' in remote || + remote.sessionId !== state.sessionId || + remote.generation !== generation || + !nonEmptyString(remote.environmentId) || + !nonEmptyString(remote.workspaceCwd) || + !validTimestamp(remote.connectExpiresAt) || + !validTimestamp(remote.leaseExpiresAt) || + typeof remote.attached !== 'boolean' || + !verification || + verification.version !== 1 || + verification.kind !== 'relayfile-destination-verification' || + verification.status !== 'converged' || + verification.remoteRoot !== '/' || + verification.sessionId !== state.sessionId || + verification.generation !== generation || + !nonEmptyString(verification.verificationId) || + !nonEmptyString(verification.workspaceId) || + !nonEmptyString(verification.localRoot) || + verification.localRoot !== remote.workspaceCwd || + !validTimestamp(verification.verifiedAt) || + !observed || + typeof observed.digest !== 'string' || + !/^sha256:[a-f0-9]{64}$/.test(observed.digest) || + typeof observed.workspaceRevision !== 'string' || + !/^(?:0|rev_[0-9]+)$/.test(observed.workspaceRevision) || + typeof observed.eventCursor !== 'string' || + !/^(?:0|evt_[0-9]+)$/.test(observed.eventCursor) || + !health || + health.pendingWriteback !== 0 || + health.conflicts !== 0 || + health.outboxPending !== 0 || + health.outboxNeedsAttention !== false + ); +} + +function validRecoveredOutcome(state: Record, recovered: Record): boolean { + const commonKeys = [ + 'sessionId', + 'threadId', + 'generation', + 'clientUserMessageId', + 'turnId', + 'status', + ] as const; + const bindingMatches = [ + recovered.sessionId === state.sessionId, + recovered.threadId === state.threadId, + Number.isSafeInteger(recovered.generation) && + (recovered.generation === state.generation || recovered.generation === Number(state.generation) - 1), + nonEmptyString(recovered.clientUserMessageId), + nonEmptyString(recovered.turnId), + ].every(Boolean); + if (!bindingMatches) { + return false; + } + if (recovered.status === 'failed' || recovered.status === 'interrupted') { + return hasOnlyKeys(recovered, commonKeys); + } + if (recovered.status !== 'completed' || !hasOnlyKeys(recovered, [...commonKeys, 'result'])) { + return false; + } + return validRecoveredCompletion(state, recovered); +} + +function validRecoveredCompletion( + state: Record, + recovered: Record +): boolean { + const result = record(recovered.result) ?? {}; + const response = record(result.response) ?? {}; + const responseTurn = record(response.turn) ?? {}; + const completed = record(result.completed) ?? {}; + const params = record(completed.params) ?? {}; + const turn = record(params.turn) ?? {}; + const items = Array.isArray(turn.items) ? turn.items : []; + const hasBoundUserMessage = items.some((item) => { + const entry = record(item); + return entry?.type === 'userMessage' && entry.clientId === recovered.clientUserMessageId; + }); + const hasAssistantAnswer = items.some((item) => { + const entry = record(item); + return entry?.type === 'agentMessage' && typeof entry.text === 'string'; + }); + return [ + result.reconciled === true, + result.turnId === recovered.turnId, + responseTurn.id === recovered.turnId, + responseTurn.status === 'completed', + completed.method === 'turn/completed', + params.threadId === state.threadId, + turn.id === recovered.turnId, + turn.status === 'completed', + [undefined, 'full'].includes(turn.itemsView as string | undefined), + Array.isArray(turn.items), + hasBoundUserMessage, + hasAssistantAnswer, + ].every(Boolean); +} + +function inferredCloudLifecycle(value: Record): CodexCloudLifecycleIntent { + if (value.remote) return 'acquired'; + if (value.phase === 'acquiring' || value.phase === 'verifying') return 'acquire_requested'; + if (value.prewarmId || value.prewarmStatus === 'ready') return 'prewarmed'; + if (CLOUD_LIFECYCLE_INTENTS.has(value.cloudLifecycle as CodexCloudLifecycleIntent)) { + return value.cloudLifecycle as CodexCloudLifecycleIntent; + } + return 'none'; +} + +/** True when startup must contact Cloud to fence an earlier enabled process. */ +export function codexControllerStateMayOwnCloudResources(state: CodexControllerState): boolean { + return inferredCloudLifecycle(state as unknown as Record) !== 'none'; +} + +function validatePersistedState(value: unknown): CodexControllerState { + const state = record(value); + if ( + !state || + state.version !== 1 || + !nonEmptyString(state.sessionId) || + !nonEmptyString(state.threadId) || + !nonEmptyString(state.workspaceRoot) || + !nonEmptyString(state.socketPath) || + !Number.isSafeInteger(state.generation) || + Number(state.generation) < 1 || + !Number.isSafeInteger(state.controllerPid) || + Number(state.controllerPid) < 1 || + typeof state.turnActive !== 'boolean' || + !CONTROLLER_PHASES.has(state.phase as CodexControllerPhase) || + !nonEmptyString(state.updatedAt) || + Number.isNaN(Date.parse(state.updatedAt)) || + (state.cloudLifecycle !== undefined && + !CLOUD_LIFECYCLE_INTENTS.has(state.cloudLifecycle as CodexCloudLifecycleIntent)) || + invalidNestedState(state) + ) { + throw new Error('Persisted managed Codex controller state is invalid or from an unsupported version.'); + } + return { ...(state as CodexControllerState), cloudLifecycle: inferredCloudLifecycle(state) }; +} + +export class CodexTurnRecordedError extends Error { + readonly recordedTerminal = true; + + constructor( + readonly status: 'failed' | 'interrupted', + message: string, + readonly requiresRecoveryAcknowledgment = false + ) { + super(message); + this.name = 'CodexTurnRecordedError'; + } +} + +export class CodexTurnOutcomeUncertainError extends Error { + readonly outcomeUncertain = true; + + constructor( + message = 'Codex turn outcome is uncertain; execution is fenced pending reconciliation.', + options?: ErrorOptions + ) { + super(message, options); + this.name = 'CodexTurnOutcomeUncertainError'; + } +} + +class CodexLifecycleEffectUnsettledError extends Error { + constructor(operationName: string) { + super(`${operationName} did not settle after abort; lifecycle outcome is unknown.`); + this.name = 'CodexLifecycleEffectUnsettledError'; + } +} + +function isReadyStatus(value: unknown): boolean { + if (typeof value === 'string') return /^(?:ready|connected|active)$/i.test(value); + if (value && typeof value === 'object') { + for (const [key, nested] of Object.entries(value)) { + if ((key === 'status' || key === 'state') && isReadyStatus(nested)) return true; + } + } + return false; +} + +function sameRecoveredIdentity(left: CodexRecoveredOutcome, right: CodexRecoveredOutcome): boolean { + return ( + left.sessionId === right.sessionId && + left.threadId === right.threadId && + left.generation === right.generation && + left.clientUserMessageId === right.clientUserMessageId && + left.turnId === right.turnId && + left.status === right.status + ); +} + +function publicStatus( + state: CodexControllerState, + liveTeleport: CodexLiveTeleportStartupSwitch +): PublicCodexControllerStatus { + const { source, remote, mountRestore: _mountRestore, recoveredOutcome: _recoveredOutcome, ...rest } = state; + const execution = + state.phase === 'remote' + ? 'cloud' + : state.phase === 'verifying' + ? 'verifying' + : state.phase === 'fenced' || + state.phase === 'recovery_failed' || + state.phase === 'outcome_uncertain' || + state.phase === 'acquiring' || + state.phase === 'rolling_back' + ? 'fenced' + : 'local'; + return { + ...rest, + controller: 'local', + execution, + liveTeleport, + workspaceSource: source?.kind ?? 'unavailable', + ...(remote && state.phase === 'remote' && remote.attached + ? { + remote: { + environmentId: remote.environmentId, + generation: remote.generation, + workspaceCwd: remote.workspaceCwd, + connectExpiresAt: remote.connectExpiresAt, + leaseExpiresAt: remote.leaseExpiresAt, + attached: true, + }, + } + : {}), + }; +} + +/** + * Owns one local Codex app-server/thread. The queued teleport is consumed only + * immediately before turn/start. A remote lease is never followed by local + * execution until Cloud has positively confirmed revoke or expiry. + */ +export class CodexLiveController { + private appServer: CodexAppServerSession | null = null; + private state: CodexControllerState | null = null; + private sealedWorkspace: CodexWorkspaceSealHandle | null = null; + private prewarmPromise: Promise | null = null; + private prewarmAbort: AbortController | null = null; + private recoveredEvidenceTaken: CodexRecoveredOutcome | undefined; + private closing = false; + private readonly liveTeleportStartupSwitch: CodexLiveTeleportStartupSwitch; + + constructor( + private readonly options: CodexLiveControllerOptions, + private readonly deps: CodexLiveControllerDependencies + ) { + const startupSwitch = options.liveTeleportStartupSwitch ?? DEFAULT_CODEX_LIVE_TELEPORT_STARTUP_SWITCH; + this.liveTeleportStartupSwitch = Object.freeze({ ...startupSwitch }); + } + + async initialize(): Promise { + this.recoveredEvidenceTaken = undefined; + const persisted = this.deps.store.read(); + if (persisted && path.resolve(persisted.workspaceRoot) !== path.resolve(this.options.workspaceRoot)) { + throw new Error( + `The persisted managed Codex thread belongs to ${persisted.workspaceRoot}, not ${this.options.workspaceRoot}.` + ); + } + if (this.liveTeleportEnabled() || (persisted && this.cloudMayOwnResources(persisted))) { + this.requireCloud(); + } + if (this.liveTeleportEnabled()) await this.deps.probeCapability(); + + if (persisted) { + const mustFenceCloud = this.cloudMayOwnResources(persisted); + const mustRecoverMount = Boolean(persisted.source || persisted.mountRestore || persisted.remote); + this.state = { + ...persisted, + // Disabling the switch cancels dormant intent before the new process + // can observe a turn boundary. Existing Cloud ownership is still + // fenced below before local execution is allowed to resume. + pending: this.liveTeleportEnabled() ? persisted.pending : undefined, + controllerPid: this.deps.pid, + socketPath: this.options.socketPath, + workspaceRoot: this.options.workspaceRoot, + turnActive: false, + phase: mustFenceCloud || mustRecoverMount ? 'rolling_back' : 'local', + lastError: undefined, + updatedAt: this.timestamp(), + }; + this.persist(); + + if (mustFenceCloud) { + try { + await this.confirmFence('restart-revoke'); + } catch (error) { + this.markFenced('CLOUD_FENCE_UNCONFIRMED_ON_RESTART'); + throw new Error(this.requireState().lastError, { cause: error }); + } + } + + if (mustRecoverMount) { + try { + await this.resumeLocalMount(); + } catch (error) { + this.markRecoveryFailed('RELAYFILE_RESUME_FAILED_ON_RESTART'); + throw new Error(this.requireState().lastError, { cause: error }); + } + } + + const replacement = await this.createInitializedAppServer(); + this.appServer = replacement; + try { + await replacement.resumeThread({ + threadId: persisted.threadId, + cwd: this.options.workspaceRoot, + }); + } catch (error) { + this.markRecoveryFailed('CODEX_THREAD_RESUME_FAILED_ON_RESTART'); + await replacement.close().catch(() => undefined); + this.appServer = null; + throw new Error(this.requireState().lastError, { cause: error }); + } + + let recovered: + | Extract + | { status: 'failed' | 'interrupted'; turnId: string } + | undefined; + const inFlight = this.requireState().inFlightTurn; + if (inFlight) { + const outcome = await this.reconcileInFlightTurn().catch((error) => { + this.markOutcomeUncertain('TURN_RECONCILIATION_UNAVAILABLE'); + throw new CodexTurnOutcomeUncertainError(undefined, { cause: error }); + }); + if (outcome.status === 'absent' || outcome.status === 'inProgress') { + this.markOutcomeUncertain(`TURN_${outcome.status.toUpperCase()}`); + throw new CodexTurnOutcomeUncertainError(); + } + if (outcome.status === 'completed') { + recovered = outcome; + } else if (outcome.status === 'failed' || outcome.status === 'interrupted') { + recovered = { status: outcome.status, turnId: outcome.turnId }; + } + } + + const state = this.requireState(); + state.generation = persisted.generation + (mustFenceCloud || mustRecoverMount ? 1 : 0); + if (state.pending && this.liveTeleportEnabled()) { + // A queued teleport is an accepted durable intent, independent from + // recovered output delivery. Fencing consumes the old Cloud identity, + // so rebind the request in the same write that publishes the new + // generation; never silently discard it across a process restart. + state.pending = { ...state.pending, expectedGeneration: state.generation }; + } + state.phase = state.pending ? 'teleport_pending' : 'local'; + state.cloudLifecycle = 'none'; + state.remote = undefined; + state.source = undefined; + state.mountRestore = undefined; + state.prewarmId = undefined; + state.prewarmStatus = undefined; + if (recovered) this.stageRecoveredOutcome(recovered); + state.lastError = undefined; + state.updatedAt = this.timestamp(); + this.persist(); + } else { + const appServer = await this.createInitializedAppServer(); + this.appServer = appServer; + const threadId = await appServer.startThread({ + cwd: this.options.workspaceRoot, + ...(this.options.model ? { model: this.options.model } : {}), + }); + this.state = { + version: 1, + sessionId: this.deps.sessionId(), + threadId, + workspaceRoot: this.options.workspaceRoot, + generation: 1, + phase: 'local', + controllerPid: this.deps.pid, + socketPath: this.options.socketPath, + turnActive: false, + cloudLifecycle: 'none', + updatedAt: this.timestamp(), + }; + this.persist(); + } + + if (this.liveTeleportEnabled() && !this.requireState().recoveredOutcome) this.schedulePrewarm(); + return this.status(); + } + + status(): PublicCodexControllerStatus { + return publicStatus(this.requireState(), this.liveTeleportStartupSwitch); + } + + /** + * Claims crash-reconciled completion evidence without clearing its durable + * copy. Rendering the full recovered answer is intentionally at-least-once: + * a process can have emitted notification deltas before it lost the terminal + * notification, so a recovered answer may duplicate an already-observed + * prefix. The model turn itself is not replayed. + */ + takeRecoveredTurn(): CodexTurnResult | undefined { + const recovered = this.requireState().recoveredOutcome; + if (!recovered || recovered.status !== 'completed' || this.recoveredEvidenceTaken) return undefined; + this.recoveredEvidenceTaken = recovered; + return recovered.result; + } + + /** Claims terminal recovery evidence in memory without clearing its durable copy. */ + takeRecoveredTerminal(): CodexTurnRecordedError | undefined { + const recovered = this.requireState().recoveredOutcome; + if (!recovered || recovered.status === 'completed' || this.recoveredEvidenceTaken) return undefined; + this.recoveredEvidenceTaken = recovered; + return new CodexTurnRecordedError( + recovered.status, + `Codex recorded the crash-reconciled turn as ${recovered.status}; it was not replayed.`, + true + ); + } + + /** Durably acknowledges successful CLI rendering of the currently claimed recovery evidence. */ + acknowledgeRecoveredOutcome(): void { + const state = this.requireState(); + const durable = state.recoveredOutcome; + const claimed = this.recoveredEvidenceTaken; + if (!durable || !claimed || !sameRecoveredIdentity(durable, claimed)) { + throw new Error('Recovered Codex turn acknowledgment does not match durable recovery evidence.'); + } + state.recoveredOutcome = undefined; + state.updatedAt = this.timestamp(); + try { + this.persist(); + } catch (error) { + state.recoveredOutcome = durable; + throw error; + } + this.recoveredEvidenceTaken = undefined; + if (state.phase === 'local' || state.phase === 'teleport_pending') this.schedulePrewarm(); + } + + requestTeleport(request: CodexTeleportRequest): PublicCodexControllerStatus { + if (!this.liveTeleportEnabled()) { + throw new Error( + `Codex live session teleport is disabled. Set ${RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV}=true before starting \`relay codex run\`.` + ); + } + const state = this.requireState(); + this.assertNoUnacknowledgedOutcome('queue a teleport'); + if (request.expectedGeneration !== state.generation) { + throw new Error( + `Stale teleport generation ${request.expectedGeneration}; active generation is ${state.generation}.` + ); + } + if (state.lastRequestId === request.requestId || state.pending?.requestId === request.requestId) { + return this.status(); + } + if (state.phase === 'remote') throw new Error('This managed Codex session already executes in Cloud.'); + if (state.pending) throw new Error(`Teleport request ${state.pending.requestId} is already pending.`); + if (state.phase !== 'local') { + throw new Error(`Cannot queue a teleport while the controller is ${state.phase}.`); + } + + state.pending = request; + state.phase = 'teleport_pending'; + state.lastRequestId = request.requestId; + state.updatedAt = this.timestamp(); + this.persist(); + return this.status(); + } + + async runTurn(text: string): Promise { + const state = this.requireState(); + this.assertNoUnacknowledgedOutcome('start a turn'); + if (state.turnActive) throw new Error('A Codex turn is already active.'); + if (TURN_BLOCKED_PHASES.has(state.phase)) { + throw new Error(`Cannot start a turn while the controller is ${state.phase}.`); + } + + // Concrete turn boundary: snapshot before turnActive. A later request is + // persisted for the following invocation and cannot splice this turn. + const pendingAtBoundary = state.pending; + state.turnActive = true; + state.updatedAt = this.timestamp(); + this.persist(); + try { + if (pendingAtBoundary) await this.applyPendingTeleport(pendingAtBoundary); + if (this.liveTeleportEnabled() && this.requireState().phase === 'remote') { + await this.ensureRemoteActiveOrRecover(); + } + const execution = + this.liveTeleportEnabled() && + (this.requireState().phase === 'remote' || this.requireState().phase === 'verifying') + ? 'remote' + : 'local'; + state.inFlightTurn = { + clientUserMessageId: this.deps.operationId(), + execution, + }; + state.updatedAt = this.timestamp(); + this.persist(); + const result = await this.executeTurn(text, state.inFlightTurn.clientUserMessageId); + state.inFlightTurn = undefined; + state.updatedAt = this.timestamp(); + this.persist(); + return result; + } finally { + state.turnActive = false; + state.updatedAt = this.timestamp(); + this.persist(); + } + } + + private async executeTurn(text: string, clientUserMessageId: string): Promise { + const state = this.requireState(); + const phase = state.phase as CodexControllerPhase; + const remote = + this.liveTeleportEnabled() && (phase === 'verifying' || phase === 'remote') ? state.remote : undefined; + try { + const result = await this.requireAppServer().runTurn({ + threadId: state.threadId, + text, + clientUserMessageId, + execution: remote + ? { + kind: 'remote', + ...(!remote.attached + ? { environment: { environmentId: remote.environmentId, cwd: remote.workspaceCwd } } + : {}), + } + : { kind: 'local', workspaceRoot: state.workspaceRoot }, + }); + if (this.requireState().phase === 'verifying' && remote && !remote.attached) { + remote.attached = true; + state.phase = 'remote'; + state.lastError = undefined; + state.updatedAt = this.timestamp(); + this.persist(); + } + return result; + } catch (error) { + if (error instanceof CodexAppServerTurnTerminalError) { + if (remote && this.requireState().phase === 'verifying' && !remote.attached) { + remote.attached = true; + state.phase = 'remote'; + } + this.clearInFlightTurn(`TURN_${error.status.toUpperCase()}`); + throw new CodexTurnRecordedError(error.status, `Codex turn ended ${error.status}.`); + } + if (remote) { + const outcome = await this.recoverLocal('remote-turn-error-revoke', 'REMOTE_TURN_RECONCILED'); + if (outcome?.status === 'completed') return this.resultFromOutcome(outcome); + if (outcome?.status === 'failed' || outcome?.status === 'interrupted') { + throw new CodexTurnRecordedError( + outcome.status, + `Cloud turn ended ${outcome.status}; the recorded terminal turn was not replayed.`, + true + ); + } + throw new CodexTurnOutcomeUncertainError(undefined, { cause: error }); + } + let outcome: CodexTurnOutcome; + try { + outcome = await this.reconcileInFlightTurn(); + } catch (reconcileError) { + this.markOutcomeUncertain('TURN_RECONCILIATION_UNAVAILABLE'); + throw new CodexTurnOutcomeUncertainError(undefined, { cause: reconcileError }); + } + if (outcome.status === 'completed') { + this.persistRecoveredOutcome(outcome, 'LOCAL_TURN_RECONCILED'); + return this.resultFromOutcome(outcome); + } + if (outcome.status === 'failed' || outcome.status === 'interrupted') { + this.persistRecoveredOutcome( + { status: outcome.status, turnId: outcome.turnId }, + 'LOCAL_TURN_RECONCILED' + ); + throw new CodexTurnRecordedError( + outcome.status, + `Codex turn ended ${outcome.status}; the recorded terminal turn was not replayed.`, + true + ); + } + this.markOutcomeUncertain(`TURN_${outcome.status.toUpperCase()}`); + throw new CodexTurnOutcomeUncertainError(undefined, { cause: error }); + } + } + + async rollback(): Promise { + const state = this.requireState(); + this.assertNoUnacknowledgedOutcome('roll back'); + if (state.turnActive) throw new Error('Rollback is only allowed at a Codex turn boundary.'); + await this.recoverLocal('rollback-revoke', 'Operator requested local rollback.'); + return this.status(); + } + + async close(options: CodexControllerCloseOptions = {}): Promise { + const state = this.state; + const cancelPendingTeleport = options.cancelPendingTeleport ?? true; + const lifecycleIdentityConsumed = Boolean( + state && + (this.cloudMayOwnResources(state) || this.sealedWorkspace || state.source || state.mountRestore) + ); + this.closing = true; + this.prewarmAbort?.abort(); + await this.prewarmPromise?.catch(() => undefined); + let fenceConfirmed = false; + if (state && this.cloudMayOwnResources(state)) { + try { + await this.confirmFence('shutdown-revoke'); + fenceConfirmed = true; + } catch { + this.markFenced('CLOUD_FENCE_UNCONFIRMED_ON_SHUTDOWN'); + } + } + try { + await this.appServer?.close(); + } catch (error) { + if (state && (fenceConfirmed || !this.cloudMayOwnResources(state))) { + this.markRecoveryFailed('CONTROLLER_CLOSE_FAILED_ON_SHUTDOWN'); + } + throw error; + } + this.appServer = null; + if (state && (fenceConfirmed || !this.cloudMayOwnResources(state))) { + if (this.sealedWorkspace || state.source || state.mountRestore) { + try { + await this.resumeLocalMount(); + } catch (error) { + this.markRecoveryFailed('RELAYFILE_RESUME_FAILED_ON_SHUTDOWN'); + throw new Error(state.lastError, { cause: error }); + } + } + if (state.inFlightTurn) { + state.phase = 'outcome_uncertain'; + state.remote = undefined; + state.lastError = 'TURN_RECONCILIATION_REQUIRED_ON_RESTART'; + state.updatedAt = this.timestamp(); + this.persist(); + return; + } + if (!lifecycleIdentityConsumed) { + // Only explicit session completion is a cancellation boundary. A + // terminal/error unwind can successfully close the app-server too, + // but must leave an already accepted request durable for restart. + if (state.pending) { + if (cancelPendingTeleport) state.pending = undefined; + state.phase = cancelPendingTeleport ? 'local' : 'teleport_pending'; + state.updatedAt = this.timestamp(); + this.persist(); + } + return; + } + // A confirmed prewarm-only fence consumes this generation's idempotency + // identity just as surely as acquire/revoke. Never reinitialize and + // replay a revoked resource under the same generation. + state.generation += 1; + // Finalize or atomically rebase a queued request in the same durable + // write as the consumed identity. Error unwind is not cancellation. + if (state.pending) { + if (cancelPendingTeleport) state.pending = undefined; + else state.pending.expectedGeneration = state.generation; + } + state.phase = state.pending ? 'teleport_pending' : 'local'; + state.cloudLifecycle = 'none'; + state.remote = undefined; + state.source = undefined; + state.mountRestore = undefined; + state.prewarmId = undefined; + state.prewarmStatus = undefined; + state.lastError = undefined; + state.updatedAt = this.timestamp(); + this.persist(); + } + } + + private async applyPendingTeleport(pendingAtBoundary: CodexTeleportRequest): Promise { + const state = this.requireState(); + if (!this.liveTeleportEnabled()) { + state.pending = undefined; + state.phase = 'local'; + state.updatedAt = this.timestamp(); + this.persist(); + return; + } + const pending = state.pending; + if (!pending || pending.requestId !== pendingAtBoundary.requestId) return; + if (pending.expectedGeneration !== state.generation) { + throw new Error('Pending teleport generation became stale before the turn boundary.'); + } + + state.phase = 'acquiring'; + state.updatedAt = this.timestamp(); + this.persist(); + try { + // Keep the poll mount alive while Cloud warms. Admission is already + // stopped at this turn boundary, but the short-lived seal is minted only + // when Cloud is ready to consume it. + await this.prewarmPromise; + if (state.prewarmId && state.prewarmStatus !== 'ready') { + await this.waitForCloudReady(state.prewarmId); + } + + const lifecycleId = `${state.sessionId}:${state.generation}:${this.deps.operationId()}`; + state.mountRestore = { lifecycleId, localRoot: state.workspaceRoot }; + state.updatedAt = this.timestamp(); + this.persist(); + const sealedWorkspace = await this.withAbortableLifecycleDeadline( + (signal) => + this.deps.checkpointAndSeal({ + sessionId: state.sessionId, + generation: state.generation, + threadId: state.threadId, + workspaceRoot: state.workspaceRoot, + lifecycleId, + signal, + }), + 'Relayfile checkpoint-and-seal' + ); + const source = sealedWorkspace.source; + this.sealedWorkspace = sealedWorkspace; + this.assertSeal(source, sealedWorkspace.restore); + state.source = { kind: source.kind }; + state.mountRestore = sealedWorkspace.restore; + state.updatedAt = this.timestamp(); + this.persist(); + + state.cloudLifecycle = 'acquire_requested'; + state.updatedAt = this.timestamp(); + this.persist(); + const environment = await this.withAbortableLifecycleDeadline( + (signal) => + this.requireCloud().acquire({ + sessionId: state.sessionId, + threadId: state.threadId, + generation: state.generation, + workspaceRoot: '/', + source, + ...(state.prewarmId ? { prewarmId: state.prewarmId } : {}), + idempotencyKey: `${state.sessionId}:${state.generation}:acquire`, + signal, + }), + 'Cloud acquire/convergence' + ); + if (environment.sessionId !== state.sessionId || environment.generation !== state.generation) { + throw new Error('Cloud returned a stale or cross-session live-teleport generation.'); + } + this.requireTurnLeaseHorizon(environment.leaseExpiresAt); + + const { + connectPath: _connectPath, + execServerUrl: _execServerUrl, + ...persistedEnvironment + } = environment; + state.remote = { ...persistedEnvironment, attached: false }; + state.cloudLifecycle = 'acquired'; + state.pending = undefined; + state.phase = 'acquiring'; + state.updatedAt = this.timestamp(); + this.persist(); + + await this.requireAppServer().addEnvironment({ + environmentId: environment.environmentId, + execServerUrl: environment.execServerUrl, + connectTimeoutMs: Math.min(10_000, this.lifecycleDeadlineMs()), + }); + await this.waitForEnvironmentReady(environment.environmentId); + + state.phase = 'verifying'; + state.lastError = undefined; + state.updatedAt = this.timestamp(); + this.persist(); + } catch (error) { + if (error instanceof CodexLifecycleEffectUnsettledError) { + this.markFenced('LIFECYCLE_EFFECT_OUTCOME_UNKNOWN'); + throw error; + } + await this.recoverLocal('failed-acquire-revoke', 'TELEPORT_ACQUIRE_FAILED_RECOVERED'); + // No clientUserMessageId exists until applyPendingTeleport returns, so a + // settled failure here is proven pre-submission. Continue this same + // prompt locally on the recovered app-server with a fresh message id. + return; + } + } + + private async recoverLocal(reason: string, context: string): Promise { + const state = this.requireState(); + const old = this.requireAppServer(); + state.phase = 'rolling_back'; + state.pending = undefined; + state.updatedAt = this.timestamp(); + this.persist(); + + if (this.cloudMayOwnResources(state)) { + try { + await this.confirmFence(reason); + } catch (error) { + this.markFenced('CLOUD_FENCE_UNCONFIRMED'); + await old.close().catch(() => { + state.lastError = 'CLOUD_FENCE_UNCONFIRMED_AND_CONTROLLER_CLOSE_FAILED'; + state.updatedAt = this.timestamp(); + this.persist(); + }); + this.appServer = null; + throw new Error(state.lastError, { cause: error }); + } + } + + try { + await old.close(); + } catch (error) { + this.markRecoveryFailed('CONTROLLER_CLOSE_FAILED_AFTER_FENCE'); + throw new Error(state.lastError, { cause: error }); + } + this.appServer = null; + try { + await this.resumeLocalMount(); + } catch (error) { + this.markRecoveryFailed('RELAYFILE_RESUME_FAILED_AFTER_FENCE'); + throw new Error(state.lastError, { cause: error }); + } + let replacement: CodexAppServerSession; + try { + replacement = await this.createInitializedAppServer(); + } catch (error) { + this.markRecoveryFailed('LOCAL_APP_SERVER_START_FAILED_AFTER_FENCE'); + throw new Error(state.lastError, { cause: error }); + } + this.appServer = replacement; + try { + await replacement.resumeThread({ threadId: state.threadId, cwd: state.workspaceRoot }); + } catch (error) { + this.markRecoveryFailed('CODEX_THREAD_RESUME_FAILED_AFTER_FENCE'); + await replacement.close().catch(() => undefined); + this.appServer = null; + throw new Error(state.lastError, { cause: error }); + } + + let outcome: CodexTurnOutcome | undefined; + if (state.inFlightTurn) { + try { + outcome = await this.reconcileInFlightTurn(); + } catch (error) { + this.markOutcomeUncertain('TURN_RECONCILIATION_UNAVAILABLE'); + throw new CodexTurnOutcomeUncertainError(undefined, { cause: error }); + } + if (outcome.status === 'absent' || outcome.status === 'inProgress') { + this.markOutcomeUncertain(`TURN_${outcome.status.toUpperCase()}`); + throw new CodexTurnOutcomeUncertainError(); + } + } + + state.generation += 1; + state.phase = 'local'; + state.cloudLifecycle = 'none'; + state.remote = undefined; + state.source = undefined; + state.mountRestore = undefined; + state.prewarmId = undefined; + state.prewarmStatus = undefined; + if (outcome?.status === 'completed') { + this.stageRecoveredOutcome(outcome); + } else if (outcome?.status === 'failed' || outcome?.status === 'interrupted') { + this.stageRecoveredOutcome({ status: outcome.status, turnId: outcome.turnId }); + } + state.lastError = context; + state.updatedAt = this.timestamp(); + this.persist(); + if (outcome) { + this.recoveredEvidenceTaken = state.recoveredOutcome; + } else { + this.schedulePrewarm(); + } + return outcome; + } + + private async confirmFence(_reason: string): Promise { + const state = this.requireState(); + state.cloudLifecycle = 'cleanup_requested'; + // Once cleanup owns the generation, stale warm metadata must not cause + // legacy-state inference to downgrade the durable cleanup intent. + state.prewarmId = undefined; + state.prewarmStatus = undefined; + state.updatedAt = this.timestamp(); + this.persist(); + await this.withAbortableLifecycleDeadline(async (signal) => { + let lastError: unknown; + try { + const revoked = await this.requireCloud().revoke({ + sessionId: state.sessionId, + generation: state.generation, + idempotencyKey: `${state.sessionId}:${state.generation}:revoke`, + signal, + }); + this.assertLifecycleIdentity(revoked); + // The Cloud contract may publish revoked/expired only after the + // destination is stopped, flushed, and its consumer-bound Relayfile + // ownership has been handed back. cleanup_pending is not a fence and + // must never allow the source mount to resume. + if (revoked.status === 'revoked' || revoked.status === 'expired') { + return; + } + } catch (error) { + lastError = error; + } + for (let attempt = 0; attempt < this.lifecycleAttempts(); attempt += 1) { + try { + const status = await this.requireCloud().status({ + sessionId: state.sessionId, + generation: state.generation, + signal, + }); + this.assertLifecycleIdentity(status); + if (status.status === 'revoked' || status.status === 'expired') { + return; + } + lastError = new Error(`Cloud fence remains ${status.status}.`); + } catch (error) { + lastError = error; + } + await this.deps.sleep(this.lifecyclePollIntervalMs()); + } + throw new Error('Cloud fencing did not reach a terminal state.', { cause: lastError }); + }, 'Cloud revoke confirmation'); + } + + private async waitForCloudReady(prewarmId: string): Promise { + const state = this.requireState(); + const attempts = this.lifecycleAttempts(); + await this.withAbortableLifecycleDeadline(async (signal) => { + for (let attempt = 0; attempt < attempts; attempt += 1) { + const status = await this.requireCloud().status({ + sessionId: state.sessionId, + generation: state.generation, + prewarmId, + signal, + }); + this.assertLifecycleIdentity(status); + if (status.prewarmId && status.prewarmId !== prewarmId) { + throw new Error('Cloud returned a stale or cross-prewarm lifecycle status.'); + } + if (status.status === 'ready') { + state.prewarmStatus = 'ready'; + state.updatedAt = this.timestamp(); + this.persist(); + return; + } + if (status.status !== 'warming') { + throw new Error(`Cloud prewarm entered terminal state ${status.status}.`); + } + await this.deps.sleep( + Math.min(status.retryAfterMs ?? this.lifecyclePollIntervalMs(), this.lifecyclePollIntervalMs()) + ); + } + throw new Error('Cloud prewarm did not converge before the lifecycle deadline.'); + }, 'Cloud prewarm convergence'); + } + + private async waitForEnvironmentReady(environmentId: string): Promise { + const attempts = this.lifecycleAttempts(); + await this.withLifecycleDeadline( + (async () => { + for (let attempt = 0; attempt < attempts; attempt += 1) { + const environmentStatus = await this.requireAppServer().environmentStatus(environmentId); + if (isReadyStatus(environmentStatus)) return; + await this.deps.sleep(this.lifecyclePollIntervalMs()); + } + throw new Error('Codex did not report the Cloud execution environment ready before the deadline.'); + })(), + 'Codex environment readiness' + ); + } + + private schedulePrewarm(): void { + if (!this.liveTeleportEnabled() || this.prewarmPromise || this.closing) return; + const state = this.requireState(); + // Recovered output is an independent durable delivery queue. Do not start + // another lifecycle while it is unacknowledged, and do not duplicate an + // already-started lifecycle after its background promise has settled. + if (state.recoveredOutcome || this.cloudMayOwnResources(state)) return; + const generation = state.generation; + state.cloudLifecycle = 'prewarm_requested'; + state.updatedAt = this.timestamp(); + this.persist(); + const abort = new AbortController(); + this.prewarmAbort = abort; + const pending = this.startPrewarm(generation, abort.signal).finally(() => { + if (this.prewarmPromise === pending) this.prewarmPromise = null; + if (this.prewarmAbort === abort) this.prewarmAbort = null; + }); + this.prewarmPromise = pending; + } + + private async startPrewarm(generation: number, outerSignal: AbortSignal): Promise { + const state = this.requireState(); + try { + const prewarm = await this.withAbortableLifecycleDeadline( + (signal) => + this.requireCloud().prewarm({ + sessionId: state.sessionId, + generation, + workspaceRoot: '/', + idempotencyKey: `${state.sessionId}:${generation}:prewarm`, + signal, + }), + 'Cloud prewarm', + outerSignal + ); + if (prewarm.generation !== generation) { + throw new Error('Cloud returned a stale prewarm generation.'); + } + if (this.closing || this.requireState().generation !== generation) return; + state.source = undefined; + state.prewarmId = prewarm.prewarmId; + state.prewarmStatus = prewarm.status; + state.cloudLifecycle = 'prewarmed'; + } catch { + if (this.closing || this.requireState().generation !== generation) return; + state.source = undefined; + state.prewarmId = undefined; + state.prewarmStatus = 'failed'; + // Response loss is ambiguous. Retain prewarm_requested so restart or + // shutdown fences any Cloud resource that may have been created. + state.cloudLifecycle = 'prewarm_requested'; + state.lastError = 'CLOUD_PREWARM_UNAVAILABLE'; + } + state.updatedAt = this.timestamp(); + this.persist(); + } + + private assertSeal(source: LiveTeleportWorkspaceSource, restore: CodexMountRestoreIdentity): void { + if ( + source.kind !== 'relayfile-checkpoint-seal' || + !source.receipt || + typeof source.receipt !== 'object' || + Array.isArray(source.receipt) + ) { + throw new Error('Relayfile checkpoint-and-seal provider returned an invalid proof.'); + } + if ( + typeof restore.resumeId !== 'string' || + restore.resumeId.length === 0 || + typeof restore.workspaceId !== 'string' || + restore.workspaceId.length === 0 || + typeof restore.lifecycleId !== 'string' || + restore.lifecycleId.length === 0 || + restore.lifecycleId !== this.requireState().mountRestore?.lifecycleId || + path.resolve(restore.localRoot) !== path.resolve(this.options.workspaceRoot) + ) { + throw new Error('Relayfile checkpoint-and-seal provider returned an invalid restore identity.'); + } + } + + private async ensureRemoteActiveOrRecover(): Promise { + const state = this.requireState(); + const remote = state.remote; + let active: boolean; + if (!remote || Date.parse(remote.leaseExpiresAt) <= this.deps.now().getTime()) { + await this.recoverLocal('remote-preflight-revoke', 'REMOTE_LEASE_RECOVERED'); + return; + } + try { + const status = await this.withAbortableLifecycleDeadline( + (signal) => + this.requireCloud().status({ + sessionId: state.sessionId, + generation: state.generation, + signal, + }), + 'Cloud active lease preflight' + ); + this.assertLifecycleIdentity(status); + active = status.status === 'active' && status.leaseExpiresAt !== undefined; + if (active) { + this.requireTurnLeaseHorizon(status.leaseExpiresAt!); + if (Date.parse(status.leaseExpiresAt!) > Date.parse(remote.leaseExpiresAt)) { + remote.leaseExpiresAt = status.leaseExpiresAt!; + state.updatedAt = this.timestamp(); + this.persist(); + } + } + } catch { + active = false; + } + if (!active) await this.recoverLocal('remote-preflight-revoke', 'REMOTE_LEASE_RECOVERED'); + } + + private reconcileInFlightTurn(): Promise { + const state = this.requireState(); + const inFlight = state.inFlightTurn; + if (!inFlight) return Promise.resolve({ status: 'absent' }); + return this.requireAppServer().turnOutcome({ + threadId: state.threadId, + clientUserMessageId: inFlight.clientUserMessageId, + }); + } + + private resultFromOutcome(outcome: Extract): CodexTurnResult { + return outcome.result; + } + + private persistRecoveredOutcome( + outcome: + | Extract + | { status: 'failed' | 'interrupted'; turnId: string }, + context: string + ): void { + const state = this.requireState(); + this.stageRecoveredOutcome(outcome); + state.lastError = context; + state.updatedAt = this.timestamp(); + this.persist(); + this.recoveredEvidenceTaken = state.recoveredOutcome; + } + + /** Move one reconciled in-flight turn into durable evidence in the caller's atomic state write. */ + private stageRecoveredOutcome( + outcome: + | Extract + | { status: 'failed' | 'interrupted'; turnId: string } + ): void { + const state = this.requireState(); + const inFlight = state.inFlightTurn; + if (!inFlight || state.recoveredOutcome) { + throw new Error('Codex reconciliation evidence does not match one unique in-flight turn.'); + } + const common = { + sessionId: state.sessionId, + threadId: state.threadId, + generation: state.generation, + clientUserMessageId: inFlight.clientUserMessageId, + turnId: outcome.turnId, + }; + state.recoveredOutcome = + outcome.status === 'completed' + ? { ...common, status: 'completed', result: outcome.result } + : { ...common, status: outcome.status }; + state.inFlightTurn = undefined; + } + + private requireTurnLeaseHorizon(leaseExpiresAt: string): void { + const leaseDeadline = Date.parse(leaseExpiresAt); + const minimumDeadline = this.deps.now().getTime() + LIVE_TELEPORT_MIN_TURN_LEASE_MS; + if (Number.isNaN(leaseDeadline) || leaseDeadline < minimumDeadline) { + throw new Error( + `Cloud active lease does not cover the ${LIVE_TELEPORT_MIN_TURN_LEASE_MS / 60_000}-minute turn horizon.` + ); + } + } + + private clearInFlightTurn(code?: string): void { + const state = this.requireState(); + state.inFlightTurn = undefined; + state.lastError = code; + state.updatedAt = this.timestamp(); + this.persist(); + } + + private async resumeLocalMount(): Promise { + const state = this.requireState(); + if (this.sealedWorkspace) { + await this.withAbortableLifecycleDeadline( + (signal) => this.sealedWorkspace!.resumeLocal(signal), + 'Relayfile local mount resume/readiness' + ); + this.sealedWorkspace = null; + return; + } + if (!state.mountRestore) { + if (state.remote) { + throw new Error('Persisted remote execution has no Relayfile seal identity to restore.'); + } + return; + } + const restore = state.mountRestore; + await this.withAbortableLifecycleDeadline( + (signal) => + this.deps.resumePersistedLocalMount({ + sessionId: state.sessionId, + generation: state.generation, + threadId: state.threadId, + workspaceRoot: state.workspaceRoot, + lifecycleId: restore.lifecycleId, + ...(state.source ? { source: state.source } : {}), + restore, + signal, + }), + 'Persisted Relayfile local mount resume/readiness' + ); + } + + private assertLifecycleIdentity(status: LiveTeleportLifecycleStatus): void { + const state = this.requireState(); + if (status.sessionId !== state.sessionId || status.generation !== state.generation) { + throw new Error('Cloud returned a stale or cross-session lifecycle status.'); + } + } + + private cloudMayOwnResources(state = this.requireState()): boolean { + return codexControllerStateMayOwnCloudResources(state); + } + + private liveTeleportEnabled(): boolean { + return this.liveTeleportStartupSwitch.enabled; + } + + private requireCloud(): LiveTeleportCloudClient { + if (!this.deps.cloud) { + throw new Error('Codex live session teleport requires an authenticated Cloud client.'); + } + return this.deps.cloud; + } + + private assertNoUnacknowledgedOutcome(operation: string): void { + if (this.requireState().recoveredOutcome) { + throw new Error( + `Cannot ${operation} while a recovered Codex turn is awaiting durable output acknowledgment.` + ); + } + } + + private async createInitializedAppServer(): Promise { + const appServer = await this.deps.createAppServer(); + await appServer.initialize(); + return appServer; + } + + private markFenced(message: string): void { + const state = this.requireState(); + state.phase = 'fenced'; + state.lastError = message; + state.updatedAt = this.timestamp(); + this.persist(); + } + + private markRecoveryFailed(message: string): void { + const state = this.requireState(); + state.phase = 'recovery_failed'; + state.remote = undefined; + state.lastError = message; + state.updatedAt = this.timestamp(); + this.persist(); + } + + private markOutcomeUncertain(code: string): void { + const state = this.requireState(); + state.phase = 'outcome_uncertain'; + state.lastError = code; + state.updatedAt = this.timestamp(); + this.persist(); + } + + private withAbortableLifecycleDeadline( + operation: (signal: AbortSignal) => Promise, + operationName: string, + outerSignal?: AbortSignal + ): Promise { + const controller = new AbortController(); + const abortFromOuter = () => controller.abort(); + outerSignal?.addEventListener('abort', abortFromOuter, { once: true }); + if (outerSignal?.aborted) controller.abort(); + const timeoutMs = this.lifecycleDeadlineMs(); + return new Promise((resolve, reject) => { + let timedOut = false; + let settled = false; + let settlementTimer: NodeJS.Timeout | undefined; + const timer = setTimeout(() => { + timedOut = true; + controller.abort(); + settlementTimer = setTimeout( + () => { + if (!settled) { + outerSignal?.removeEventListener('abort', abortFromOuter); + reject(new CodexLifecycleEffectUnsettledError(operationName)); + } + }, + Math.min(5_000, timeoutMs) + ); + }, timeoutMs); + let promise: Promise; + try { + promise = operation(controller.signal); + } catch (error) { + clearTimeout(timer); + outerSignal?.removeEventListener('abort', abortFromOuter); + reject(error); + return; + } + promise.then( + (value) => { + settled = true; + clearTimeout(timer); + if (settlementTimer) clearTimeout(settlementTimer); + outerSignal?.removeEventListener('abort', abortFromOuter); + if (timedOut) reject(new Error(`${operationName} exceeded the ${timeoutMs}ms lifecycle deadline.`)); + else resolve(value); + }, + (error) => { + settled = true; + clearTimeout(timer); + if (settlementTimer) clearTimeout(settlementTimer); + outerSignal?.removeEventListener('abort', abortFromOuter); + if (timedOut) + reject( + new Error(`${operationName} exceeded the ${timeoutMs}ms lifecycle deadline.`, { cause: error }) + ); + else reject(error); + } + ); + }); + } + + private withLifecycleDeadline(promise: Promise, operation: string): Promise { + const timeoutMs = this.lifecycleDeadlineMs(); + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`${operation} exceeded the ${timeoutMs}ms lifecycle deadline.`)), + timeoutMs + ); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error) => { + clearTimeout(timer); + reject(error); + } + ); + }); + } + + private lifecycleDeadlineMs(): number { + return this.options.lifecycleDeadlineMs ?? DEFAULT_LIFECYCLE_DEADLINE_MS; + } + + private lifecyclePollIntervalMs(): number { + return this.options.lifecyclePollIntervalMs ?? DEFAULT_LIFECYCLE_POLL_INTERVAL_MS; + } + + private lifecycleAttempts(): number { + return Math.max(1, Math.ceil(this.lifecycleDeadlineMs() / this.lifecyclePollIntervalMs())); + } + + private persist(): void { + this.deps.store.write(this.requireState()); + } + + private timestamp(): string { + return this.deps.now().toISOString(); + } + + private requireState(): CodexControllerState { + if (!this.state) throw new Error('Codex controller is not initialized.'); + return this.state; + } + + private requireAppServer(): CodexAppServerSession { + if (!this.appServer) throw new Error('Codex app-server is not initialized.'); + return this.appServer; + } +} diff --git a/packages/cli/src/cli/lib/codex-relayfile-seal.test.ts b/packages/cli/src/cli/lib/codex-relayfile-seal.test.ts new file mode 100644 index 000000000..0bd44017e --- /dev/null +++ b/packages/cli/src/cli/lib/codex-relayfile-seal.test.ts @@ -0,0 +1,239 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { + createRelayfileSealLifecycle, + runRelayfileLifecycleJsonCommand, + type RelayfileLifecycleCommandRunner, +} from './codex-relayfile-seal.js'; + +function receipt(overrides: Record = {}) { + return { + sealId: 'cps_123', + sealToken: 'one-use-opaque-token', + workspaceId: 'ws_123', + root: '/', + sessionId: 'session-1', + generation: 4, + digest: `sha256:${'a'.repeat(64)}`, + workspaceRevision: 'rev_40', + eventCursor: 'evt_50', + issuedAt: '2026-08-23T12:00:00.000Z', + expiresAt: '2026-08-23T12:01:00.000Z', + ...overrides, + }; +} + +function checkpointOutput(overrides: Record = {}) { + return { + version: 1, + kind: 'relayfile-checkpoint-seal', + status: 'sealed', + workspaceId: 'ws_123', + localRoot: '/repo', + sessionId: 'session-1', + generation: 4, + receipt: receipt(), + health: { + pendingWriteback: 0, + conflicts: 0, + outboxPending: 0, + outboxNeedsAttention: false, + }, + resumeId: 'lifecycle-4', + sealedAt: '2026-08-23T12:00:00.000Z', + ...overrides, + }; +} + +function resumeOutput(overrides: Record = {}) { + return { + version: 1, + kind: 'relayfile-resume-seal', + workspaceId: 'ws_123', + localRoot: '/repo', + resumeId: 'lifecycle-4', + status: 'ready', + resumedAt: '2026-08-23T12:00:10.000Z', + ...overrides, + }; +} + +describe('createRelayfileSealLifecycle', () => { + it('binds the exact checkpoint command and keeps resumeId out of argv', async () => { + const runner = vi + .fn() + .mockResolvedValueOnce(checkpointOutput()) + .mockResolvedValueOnce(resumeOutput()); + const lifecycle = createRelayfileSealLifecycle({ binary: 'relayfile-test', runner }); + + const handle = await lifecycle.checkpointAndSeal({ + sessionId: 'session-1', + generation: 4, + threadId: 'thread-1', + workspaceRoot: '/repo', + lifecycleId: 'lifecycle-4', + }); + + expect(runner).toHaveBeenNthCalledWith(1, { + binary: 'relayfile-test', + args: [ + 'mount', + 'checkpoint-seal', + '--root', + '/repo', + '--session', + 'session-1', + '--generation', + '4', + '--lifecycle-id', + 'lifecycle-4', + '--timeout', + '30s', + '--ttl', + '60s', + '--json', + ], + signal: undefined, + }); + expect(handle.source).toEqual({ kind: 'relayfile-checkpoint-seal', receipt: receipt() }); + expect(handle.restore).toEqual({ + lifecycleId: 'lifecycle-4', + resumeId: 'lifecycle-4', + workspaceId: 'ws_123', + localRoot: '/repo', + }); + + await handle.resumeLocal(); + + const resumeCall = runner.mock.calls[1]![0]; + expect(resumeCall.args).toEqual(['mount', 'resume-seal', '--root', '/repo', '--json']); + expect(resumeCall.args.join(' ')).not.toContain('lifecycle-4'); + expect(resumeCall.stdin).toBe(`${JSON.stringify({ resumeId: 'lifecycle-4' })}\n`); + }); + + it('uses the same stdin-only resume contract after controller restart', async () => { + const runner = vi.fn(async () => + resumeOutput({ resumeId: 'resume_opaque_123' }) + ); + const lifecycle = createRelayfileSealLifecycle({ runner }); + + await lifecycle.resumePersistedLocalMount({ + sessionId: 'session-1', + generation: 4, + threadId: 'thread-1', + workspaceRoot: '/repo', + lifecycleId: 'lifecycle-4', + source: { kind: 'relayfile-checkpoint-seal' }, + restore: { + lifecycleId: 'lifecycle-4', + resumeId: 'resume_opaque_123', + workspaceId: 'ws_123', + localRoot: '/repo', + }, + }); + + expect(runner).toHaveBeenCalledWith( + expect.objectContaining({ + args: ['mount', 'resume-seal', '--root', '/repo', '--json'], + stdin: `${JSON.stringify({ resumeId: 'resume_opaque_123' })}\n`, + }) + ); + }); + + it('reconciles an in-flight checkpoint from persisted lifecycle intent alone', async () => { + const runner = vi.fn(async () => resumeOutput()); + const lifecycle = createRelayfileSealLifecycle({ runner }); + + await lifecycle.resumePersistedLocalMount({ + sessionId: 'session-1', + generation: 4, + threadId: 'thread-1', + workspaceRoot: '/repo', + lifecycleId: 'lifecycle-4', + restore: { lifecycleId: 'lifecycle-4', localRoot: '/repo' }, + }); + + expect(runner).toHaveBeenCalledWith( + expect.objectContaining({ + args: ['mount', 'resume-seal', '--root', '/repo', '--json'], + stdin: `${JSON.stringify({ resumeId: 'lifecycle-4' })}\n`, + }) + ); + }); + + it.each([ + ['wrong local root', checkpointOutput({ localRoot: '/other' })], + ['wrong session', checkpointOutput({ sessionId: 'session-other' })], + ['wrong generation', checkpointOutput({ generation: 5 })], + ['non-logical receipt root', checkpointOutput({ receipt: receipt({ root: '/repo' }) })], + ['caller-shaped digest', checkpointOutput({ receipt: receipt({ digest: 'caller-says-ok' }) })], + ['uppercase digest', checkpointOutput({ receipt: receipt({ digest: `sha256:${'A'.repeat(64)}` }) })], + ['bare revision', checkpointOutput({ receipt: receipt({ workspaceRevision: '40' }) })], + ['wrong revision namespace', checkpointOutput({ receipt: receipt({ workspaceRevision: 'evt_40' }) })], + ['bare cursor', checkpointOutput({ receipt: receipt({ eventCursor: '50' }) })], + ['wrong cursor namespace', checkpointOutput({ receipt: receipt({ eventCursor: 'rev_50' }) })], + [ + 'unsettled health', + checkpointOutput({ + health: { pendingWriteback: 1, conflicts: 0, outboxPending: 0, outboxNeedsAttention: false }, + }), + ], + ['missing one-use token', checkpointOutput({ receipt: receipt({ sealToken: '' }) })], + ])('fails closed on %s', async (_name, output) => { + const lifecycle = createRelayfileSealLifecycle({ runner: async () => output }); + await expect( + lifecycle.checkpointAndSeal({ + sessionId: 'session-1', + generation: 4, + threadId: 'thread-1', + workspaceRoot: '/repo', + lifecycleId: 'lifecycle-4', + }) + ).rejects.toThrow(/relayfile|checkpoint/); + }); + + it('rejects a resume response until readiness and identity are exact', async () => { + const lifecycle = createRelayfileSealLifecycle({ + runner: vi + .fn() + .mockResolvedValueOnce(checkpointOutput()) + .mockResolvedValueOnce(resumeOutput({ status: 'warming' })), + }); + const handle = await lifecycle.checkpointAndSeal({ + sessionId: 'session-1', + generation: 4, + threadId: 'thread-1', + workspaceRoot: '/repo', + lifecycleId: 'lifecycle-4', + }); + await expect(handle.resumeLocal()).rejects.toThrow('did not confirm mount readiness'); + }); +}); + +describe('runRelayfileLifecycleJsonCommand', () => { + it('kills an aborted child and rejects with AbortError', async () => { + const controller = new AbortController(); + const running = runRelayfileLifecycleJsonCommand({ + binary: process.execPath, + args: ['-e', 'setInterval(() => {}, 1000)'], + signal: controller.signal, + }); + setTimeout(() => controller.abort(), 10); + + await expect(running).rejects.toMatchObject({ name: 'AbortError' }); + }); + + it('reports only the stable error code and never echoes arbitrary stderr', async () => { + const error = await runRelayfileLifecycleJsonCommand({ + binary: process.execPath, + args: [ + '-e', + "process.stderr.write('secret-value\\nerror: checkpoint_fuse_unsupported: hidden'); process.exit(2)", + ], + }).catch((caught: unknown) => caught); + + expect(String(error)).toContain('checkpoint_fuse_unsupported'); + expect(String(error)).not.toContain('secret-value'); + expect(String(error)).not.toContain('hidden'); + }); +}); diff --git a/packages/cli/src/cli/lib/codex-relayfile-seal.ts b/packages/cli/src/cli/lib/codex-relayfile-seal.ts new file mode 100644 index 000000000..f522bbeac --- /dev/null +++ b/packages/cli/src/cli/lib/codex-relayfile-seal.ts @@ -0,0 +1,327 @@ +import path from 'node:path'; +import { spawn } from 'node:child_process'; + +import type { LiveTeleportWorkspaceSource } from '@agent-relay/cloud'; + +import type { + CodexMountRestoreIdentity, + CodexPersistedMountResumeProvider, + CodexWorkspaceSealHandle, + CodexWorkspaceSealProvider, +} from './codex-live-controller.js'; + +const MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024; +const FORCE_KILL_AFTER_MS = 1_000; + +export type RelayfileLifecycleCommandInput = { + binary: string; + args: string[]; + stdin?: string; + signal?: AbortSignal; +}; + +export type RelayfileLifecycleCommandRunner = (input: RelayfileLifecycleCommandInput) => Promise; + +export type RelayfileSealLifecycle = { + checkpointAndSeal: CodexWorkspaceSealProvider; + resumePersistedLocalMount: CodexPersistedMountResumeProvider; +}; + +type JsonObject = Record; + +function isObject(value: unknown): value is JsonObject { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function requiredString(value: unknown, field: string): string { + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`relayfile lifecycle response is missing ${field}.`); + } + return value.trim(); +} + +function requiredExactLocalRoot(value: unknown, workspaceRoot: string): string { + const localRoot = requiredString(value, 'localRoot'); + if ( + !path.isAbsolute(localRoot) || + localRoot !== workspaceRoot || + path.resolve(localRoot) !== workspaceRoot + ) { + throw new Error('relayfile lifecycle response returned a mismatched local root.'); + } + return localRoot; +} + +function requiredResumeId(value: unknown): string { + const resumeId = requiredString(value, 'resumeId'); + if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,511}$/.test(resumeId)) { + throw new Error('relayfile lifecycle response returned an invalid resumeId.'); + } + return resumeId; +} + +function requiredTimestamp(value: unknown, field: string): string { + const timestamp = requiredString(value, field); + if (Number.isNaN(Date.parse(timestamp))) { + throw new Error(`relayfile lifecycle response returned an invalid ${field}.`); + } + return timestamp; +} + +function requiredRelayfilePosition(value: unknown, field: string, prefix: 'rev' | 'evt'): string { + const position = requiredString(value, field); + if (!new RegExp(`^(?:0|${prefix}_[0-9]+)$`).test(position)) { + throw new Error(`relayfile lifecycle response returned an invalid ${field}.`); + } + return position; +} + +function requireConvergedHealth(value: unknown): void { + if ( + !isObject(value) || + value.pendingWriteback !== 0 || + value.conflicts !== 0 || + value.outboxPending !== 0 || + value.outboxNeedsAttention !== false + ) { + throw new Error('relayfile checkpoint response did not report converged health.'); + } +} + +function validateReceipt( + value: unknown, + input: { sessionId: string; generation: number; workspaceId: string } +): JsonObject { + if (!isObject(value)) throw new Error('relayfile checkpoint response is missing receipt.'); + const workspaceId = requiredString(value.workspaceId, 'receipt.workspaceId'); + const sessionId = requiredString(value.sessionId, 'receipt.sessionId'); + const generation = value.generation; + const digest = requiredString(value.digest, 'receipt.digest'); + requiredString(value.sealId, 'receipt.sealId'); + requiredString(value.sealToken, 'receipt.sealToken'); + if (requiredString(value.root, 'receipt.root') !== '/') { + throw new Error('relayfile checkpoint receipt root must be logical / for live teleport v1.'); + } + requiredRelayfilePosition(value.workspaceRevision, 'receipt.workspaceRevision', 'rev'); + requiredRelayfilePosition(value.eventCursor, 'receipt.eventCursor', 'evt'); + requiredTimestamp(value.issuedAt, 'receipt.issuedAt'); + requiredTimestamp(value.expiresAt, 'receipt.expiresAt'); + if ( + workspaceId !== input.workspaceId || + sessionId !== input.sessionId || + generation !== input.generation || + !/^sha256:[a-f0-9]{64}$/.test(digest) + ) { + throw new Error('relayfile checkpoint receipt is unbound or has an invalid digest.'); + } + return value; +} + +function validateCheckpointOutput( + value: unknown, + input: { workspaceRoot: string; sessionId: string; generation: number; lifecycleId: string } +): { source: LiveTeleportWorkspaceSource; restore: CodexMountRestoreIdentity } { + if ( + !isObject(value) || + value.version !== 1 || + value.kind !== 'relayfile-checkpoint-seal' || + value.status !== 'sealed' + ) { + throw new Error('relayfile checkpoint command returned an invalid response contract.'); + } + const workspaceId = requiredString(value.workspaceId, 'workspaceId'); + const localRoot = requiredExactLocalRoot(value.localRoot, input.workspaceRoot); + if ( + requiredString(value.sessionId, 'sessionId') !== input.sessionId || + value.generation !== input.generation + ) { + throw new Error('relayfile checkpoint command returned a stale or cross-session response.'); + } + requireConvergedHealth(value.health); + requiredTimestamp(value.sealedAt, 'sealedAt'); + const resumeId = requiredResumeId(value.resumeId); + if (resumeId !== input.lifecycleId) { + throw new Error('relayfile checkpoint command returned a mismatched lifecycle identity.'); + } + const receipt = validateReceipt(value.receipt, { + sessionId: input.sessionId, + generation: input.generation, + workspaceId, + }); + return { + source: { kind: 'relayfile-checkpoint-seal', receipt }, + restore: { lifecycleId: input.lifecycleId, resumeId, workspaceId, localRoot }, + }; +} + +function validateResumeOutput(value: unknown, restore: CodexMountRestoreIdentity): void { + if ( + !isObject(value) || + value.version !== 1 || + value.kind !== 'relayfile-resume-seal' || + value.status !== 'ready' + ) { + throw new Error('relayfile resume command did not confirm mount readiness.'); + } + if ( + requiredResumeId(value.resumeId) !== (restore.resumeId ?? restore.lifecycleId) || + (restore.workspaceId !== undefined && + requiredString(value.workspaceId, 'workspaceId') !== restore.workspaceId) || + requiredExactLocalRoot(value.localRoot, restore.localRoot) !== restore.localRoot + ) { + throw new Error('relayfile resume command returned a mismatched restore identity.'); + } + requiredTimestamp(value.resumedAt, 'resumedAt'); +} + +function abortError(): Error { + const error = new Error('relayfile lifecycle command was aborted.'); + error.name = 'AbortError'; + return error; +} + +export const runRelayfileLifecycleJsonCommand: RelayfileLifecycleCommandRunner = async (input) => { + if (input.signal?.aborted) throw abortError(); + const child = spawn(input.binary, input.args, { stdio: ['pipe', 'pipe', 'pipe'] }); + + return new Promise((resolve, reject) => { + let stdout = ''; + let stderr = ''; + let outputError: Error | undefined; + let aborted = false; + let forceKillTimer: NodeJS.Timeout | undefined; + + const append = (current: string, chunk: Buffer | string): string => { + const next = current + chunk.toString(); + if (Buffer.byteLength(next, 'utf8') > MAX_COMMAND_OUTPUT_BYTES && !outputError) { + outputError = new Error('relayfile lifecycle command output exceeded 1 MiB.'); + child.kill('SIGKILL'); + } + return next.slice(-MAX_COMMAND_OUTPUT_BYTES); + }; + const cleanup = () => { + input.signal?.removeEventListener('abort', onAbort); + if (forceKillTimer) clearTimeout(forceKillTimer); + }; + const onAbort = () => { + if (aborted) return; + aborted = true; + child.kill('SIGTERM'); + forceKillTimer = setTimeout(() => child.kill('SIGKILL'), FORCE_KILL_AFTER_MS); + forceKillTimer.unref?.(); + }; + + child.stdout.on('data', (chunk: Buffer | string) => { + stdout = append(stdout, chunk); + }); + child.stderr.on('data', (chunk: Buffer | string) => { + stderr = append(stderr, chunk); + }); + child.once('error', (error) => { + cleanup(); + reject(new Error(`Could not start relayfile lifecycle command: ${error.message}`)); + }); + child.once('exit', (code, signal) => { + cleanup(); + if (aborted) { + reject(abortError()); + return; + } + if (outputError) { + reject(outputError); + return; + } + if (code !== 0) { + const stableCode = /(?:^|\n)error:\s*([a-z0-9_]+):/i.exec(stderr)?.[1]; + reject( + new Error( + `relayfile lifecycle command failed (${signal ?? `exit ${code ?? 'unknown'}`}${ + stableCode ? `, ${stableCode}` : '' + }).` + ) + ); + return; + } + if (stderr.trim()) { + reject(new Error('relayfile lifecycle command produced unexpected stderr on success.')); + return; + } + try { + resolve(JSON.parse(stdout) as unknown); + } catch { + reject(new Error('relayfile lifecycle command returned malformed JSON.')); + } + }); + + input.signal?.addEventListener('abort', onAbort, { once: true }); + if (input.signal?.aborted) onAbort(); + if (input.stdin === undefined) child.stdin.end(); + else child.stdin.end(input.stdin); + }); +}; + +export function createRelayfileSealLifecycle( + options: { + binary?: string; + runner?: RelayfileLifecycleCommandRunner; + } = {} +): RelayfileSealLifecycle { + const binary = options.binary ?? 'relayfile'; + const runner = options.runner ?? runRelayfileLifecycleJsonCommand; + + const resume = async (restore: CodexMountRestoreIdentity, signal?: AbortSignal): Promise => { + const output = await runner({ + binary, + args: ['mount', 'resume-seal', '--root', restore.localRoot, '--json'], + stdin: `${JSON.stringify({ resumeId: restore.resumeId ?? restore.lifecycleId })}\n`, + signal, + }); + validateResumeOutput(output, restore); + }; + + return { + checkpointAndSeal: async (input): Promise => { + let sealed: ReturnType; + try { + const output = await runner({ + binary, + args: [ + 'mount', + 'checkpoint-seal', + '--root', + input.workspaceRoot, + '--session', + input.sessionId, + '--generation', + String(input.generation), + '--lifecycle-id', + input.lifecycleId, + '--timeout', + '30s', + '--ttl', + '60s', + '--json', + ], + signal: input.signal, + }); + sealed = validateCheckpointOutput(output, input); + } catch (error) { + await resume({ lifecycleId: input.lifecycleId, localRoot: input.workspaceRoot }).catch( + () => undefined + ); + throw error; + } + return { + ...sealed, + resumeLocal: (signal) => resume(sealed.restore, signal), + close: async () => undefined, + }; + }, + resumePersistedLocalMount: async (input): Promise => { + if (input.restore.localRoot !== input.workspaceRoot) { + throw new Error('Persisted relayfile restore identity does not match the managed workspace root.'); + } + await resume(input.restore, input.signal); + }, + }; +} diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index 6cae84254..55b086d69 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -51,6 +51,7 @@ describe('Cloud fleet sandbox client', () => { relayWorkspaceId: 'rw_abc', relayfileMounted: true, relayfileMountPath: '/workspace', + providerId: 'agent37', }, { status: 201 } ), @@ -64,6 +65,7 @@ describe('Cloud fleet sandbox client', () => { maxAgents: 1, mountRelayfile: true, forceProvision: true, + workloadProfile: 'long-running-agent', waitTimeoutMs: 90_000, }); @@ -84,6 +86,7 @@ describe('Cloud fleet sandbox client', () => { maxAgents: 1, mountRelayfile: true, forceProvision: true, + workloadProfile: 'long-running-agent', waitTimeoutMs: 90_000, }); expect(result).toEqual({ @@ -95,6 +98,7 @@ describe('Cloud fleet sandbox client', () => { relayWorkspaceId: 'rw_abc', relayfileMounted: true, relayfileMountPath: '/workspace', + providerId: 'agent37', }); }); @@ -112,6 +116,7 @@ describe('Cloud fleet sandbox client', () => { relayWorkspaceId: 'rw_abc', nodeName: 'daytona-codex', waitedMs: 90_000, + providerId: 'agent37', }, { status: 202 } ), @@ -154,6 +159,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: true, + providerId: 'daytona', }, { status: 201 } ), @@ -192,6 +198,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: true, + providerId: 'daytona', }, { status: 201 } ), diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 84b0e0edb..ad0fe42d8 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -57,9 +57,14 @@ export type EnsureCloudFleetSandboxInput = { maxAgents?: number; mountRelayfile?: boolean; forceProvision?: boolean; + /** Provider-neutral semantics; Cloud owns the provider decision. */ + workloadProfile?: CloudFleetSandboxWorkloadProfile; waitTimeoutMs?: number; }; +export type CloudFleetSandboxWorkloadProfile = 'standard' | 'long-running-agent'; +export type CloudFleetSandboxProviderId = 'daytona' | 'e2b' | 'vercel' | 'freestyle' | 'agent37'; + export type CloudFleetSandboxReady = { outcome: 'provisioned'; cloudWorkspaceId: string; @@ -69,6 +74,7 @@ export type CloudFleetSandboxReady = { relayWorkspaceId: string; relayfileMounted: boolean; relayfileMountPath?: string; + providerId: CloudFleetSandboxProviderId; }; export type CloudFleetSandboxReused = { @@ -88,6 +94,7 @@ export type CloudFleetSandboxProvisioningTimeout = { relayWorkspaceId: string; nodeName: string; waitedMs: number; + providerId: CloudFleetSandboxProviderId; }; export type EnsureCloudFleetSandboxResult = @@ -166,6 +173,14 @@ function requiredString(payload: JsonRecord, key: string, context: string): stri return value; } +function requiredProviderId(payload: JsonRecord): CloudFleetSandboxProviderId { + const value = requiredString(payload, 'providerId', 'Cloud fleet sandbox'); + if (!['daytona', 'e2b', 'vercel', 'freestyle', 'agent37'].includes(value)) { + throw new Error('Cloud fleet sandbox response has an unknown providerId.'); + } + return value as CloudFleetSandboxProviderId; +} + async function resolveCloudWorkspaceId( workspaceId: string, auth: Awaited>['auth'], @@ -210,6 +225,7 @@ function normalizeEnsureResult(payload: unknown, cloudWorkspaceId: string): Ensu sandboxId: requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'), relayWorkspaceId: requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'), relayfileMounted: payload.relayfileMounted, + providerId: requiredProviderId(payload), ...(readString(payload, 'relayfileMountPath') ? { relayfileMountPath: readString(payload, 'relayfileMountPath') } : {}), @@ -236,6 +252,7 @@ function normalizeEnsureResult(payload: unknown, cloudWorkspaceId: string): Ensu relayWorkspaceId: requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'), nodeName, waitedMs: requiredNumber(payload, 'waitedMs', 'Cloud fleet sandbox'), + providerId: requiredProviderId(payload), }; } @@ -274,6 +291,7 @@ export async function ensureCloudFleetSandbox( ...(input.maxAgents !== undefined ? { maxAgents: input.maxAgents } : {}), ...(input.mountRelayfile !== undefined ? { mountRelayfile: input.mountRelayfile } : {}), ...(input.forceProvision !== undefined ? { forceProvision: input.forceProvision } : {}), + ...(input.workloadProfile !== undefined ? { workloadProfile: input.workloadProfile } : {}), ...(input.waitTimeoutMs !== undefined ? { waitTimeoutMs: input.waitTimeoutMs } : {}), }), }, diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index 4f7f5f107..17451863b 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -53,6 +53,23 @@ export { type CloudApiClientSnapshot, } from './api-client.js'; +export { + CloudLiveTeleportClient, + LIVE_TELEPORT_MIN_TURN_LEASE_MS, + type LiveTeleportCloudClient, + type LiveTeleportWorkspaceSource, + type LiveTeleportPrewarmInput, + type LiveTeleportPrewarm, + type LiveTeleportAcquireInput, + type LiveTeleportEnvironment, + type LiveTeleportDestinationVerification, + type LiveTeleportRevokeInput, + type LiveTeleportStatusInput, + type LiveTeleportLifecycleStatus, + type LiveTeleportRollout, + type LiveTeleportRevocation, +} from './live-teleport.js'; + export { runWorkflow, scheduleWorkflow, @@ -86,6 +103,8 @@ export { type CloudFleetSandboxReady, type CloudFleetSandboxReused, type CloudFleetSandboxProvisioningTimeout, + type CloudFleetSandboxProviderId, + type CloudFleetSandboxWorkloadProfile, type DeleteCloudFleetSandboxInput, type CloudFleetSandboxRequestOptions, } from './fleet-sandbox.js'; diff --git a/packages/cloud/src/live-teleport.test.ts b/packages/cloud/src/live-teleport.test.ts new file mode 100644 index 000000000..d82d0f56b --- /dev/null +++ b/packages/cloud/src/live-teleport.test.ts @@ -0,0 +1,965 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { CloudLiveTeleportClient } from './live-teleport.js'; + +const receipt = { + sealId: 'seal-1', + sealToken: 'opaque-seal-token', + workspaceId: 'workspace-1', + root: '/', + sessionId: 'session-1', + generation: 2, + digest: `sha256:${'a'.repeat(64)}`, + workspaceRevision: 'rev_12', + eventCursor: 'evt_19313', +}; + +const input = { + sessionId: 'session-1', + threadId: 'thread-1', + generation: 2, + workspaceRoot: '/workspace', + source: { + kind: 'relayfile-checkpoint-seal' as const, + receipt, + }, + idempotencyKey: 'session-1:2:acquire', +}; + +function verification( + overrides: { + digest?: string; + workspaceRevision?: string; + eventCursor?: string; + workspaceId?: string; + localRoot?: string; + remoteRoot?: string; + sessionId?: string; + generation?: number; + pendingWriteback?: number; + } = {} +) { + return { + version: 1, + kind: 'relayfile-destination-verification', + verificationId: 'verify-2', + workspaceId: overrides.workspaceId ?? 'workspace-1', + localRoot: overrides.localRoot ?? '/workspace', + remoteRoot: overrides.remoteRoot ?? '/', + sessionId: overrides.sessionId ?? 'session-1', + generation: overrides.generation ?? 2, + status: 'converged', + observed: { + digest: overrides.digest ?? `sha256:${'a'.repeat(64)}`, + workspaceRevision: overrides.workspaceRevision ?? 'rev_12', + eventCursor: overrides.eventCursor ?? 'evt_19313', + }, + health: { + pendingWriteback: overrides.pendingWriteback ?? 0, + conflicts: 0, + outboxPending: 0, + outboxNeedsAttention: false, + }, + verifiedAt: '2026-08-23T11:59:00.000Z', + }; +} + +// Frozen constructor parity fixtures derived from Cloud e3f9e5ddc: +// prewarm/route.ts, acquire/route.ts, status/route.ts, and feature-flag.ts. +const cloudRollout = { + masterEnabled: true, + eligible: true, + reason: 'workspace-allowlist' as const, + percentage: 25, +}; + +function cloudPrewarmWarming() { + return { + sessionId: 'session-1', + generation: 2, + prewarmId: 'prewarm-2', + status: 'warming', + expiresAt: '2026-08-23T12:30:00.000Z', + retryAfterMs: 1_000, + }; +} + +function cloudAcquireVerifying() { + return { + sessionId: 'session-1', + generation: 2, + status: 'verifying', + retryAfterMs: 1_000, + }; +} + +function cloudAcquireActive() { + return { + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:05:00.000Z', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + verification: verification(), + }; +} + +function cloudActiveStatus(leaseExpiresAt = '2026-08-23T12:45:00.000Z') { + return { + sessionId: 'session-1', + generation: 2, + status: 'active', + prewarmId: 'prewarm-2', + expiresAt: leaseExpiresAt, + leaseExpiresAt, + rollout: cloudRollout, + }; +} + +describe('CloudLiveTeleportClient', () => { + it('polls an exact 202 acquire with the same idempotency body until active', async () => { + const fetcher = vi + .fn() + .mockResolvedValueOnce(Response.json(cloudAcquireVerifying(), { status: 202 })) + .mockResolvedValueOnce( + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: verification(), + }) + ); + const client = new CloudLiveTeleportClient(fetcher, 'https://cloud.agentrelay.test'); + + await expect(client.acquire(input)).resolves.toMatchObject({ environmentId: 'env-2' }); + expect(fetcher).toHaveBeenCalledTimes(2); + expect(vi.mocked(fetcher).mock.calls[0]?.[1]?.body).toBe(vi.mocked(fetcher).mock.calls[1]?.[1]?.body); + }); + + it('accepts active acquire only with HTTP 200 and rejects other successful codes', async () => { + const exact = new CloudLiveTeleportClient( + async () => Response.json(cloudAcquireActive(), { status: 200 }), + 'https://cloud.agentrelay.test' + ); + await expect(exact.acquire(input)).resolves.toMatchObject({ environmentId: 'env-2' }); + + for (const status of [201, 206]) { + const client = new CloudLiveTeleportClient( + async () => Response.json(cloudAcquireActive(), { status }), + 'https://cloud.agentrelay.test' + ); + await expect(client.acquire(input)).rejects.toThrow('invalid active lifecycle metadata'); + } + }); + + it('rejects acquire HTTP/body state mismatches', async () => { + const activePending = new CloudLiveTeleportClient( + async () => Response.json(cloudAcquireActive(), { status: 202 }), + 'https://cloud.agentrelay.test' + ); + await expect(activePending.acquire(input)).rejects.toThrow('invalid pending response'); + + const verifyingSuccess = new CloudLiveTeleportClient( + async () => Response.json(cloudAcquireVerifying(), { status: 200 }), + 'https://cloud.agentrelay.test' + ); + await expect(verifyingSuccess.acquire(input)).rejects.toThrow('invalid active lifecycle metadata'); + }); + + it('composes the frozen Cloud e3f9e5ddc warming → verifying → active and renewal constructors', async () => { + const renewedLease = '2026-08-23T13:15:00.000Z'; + const fetcher = vi + .fn() + .mockResolvedValueOnce(Response.json(cloudPrewarmWarming(), { status: 202 })) + .mockResolvedValueOnce(Response.json(cloudAcquireVerifying(), { status: 202 })) + .mockResolvedValueOnce(Response.json(cloudAcquireActive())) + .mockResolvedValueOnce(Response.json(cloudActiveStatus(renewedLease))); + const client = new CloudLiveTeleportClient(fetcher, 'https://cloud.agentrelay.test'); + + await expect( + client.prewarm({ + sessionId: 'session-1', + generation: 2, + workspaceRoot: '/', + idempotencyKey: 'session-1:2:prewarm', + }) + ).resolves.toEqual(cloudPrewarmWarming()); + await expect(client.acquire(input)).resolves.toMatchObject({ + sessionId: 'session-1', + generation: 2, + environmentId: 'env-2', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + }); + await expect(client.status({ sessionId: 'session-1', generation: 2 })).resolves.toEqual( + cloudActiveStatus(renewedLease) + ); + + expect(Object.keys(cloudPrewarmWarming()).sort()).toEqual( + ['expiresAt', 'generation', 'prewarmId', 'retryAfterMs', 'sessionId', 'status'].sort() + ); + expect(Object.keys(cloudAcquireVerifying()).sort()).toEqual( + ['generation', 'retryAfterMs', 'sessionId', 'status'].sort() + ); + expect(Object.keys(cloudActiveStatus()).sort()).toEqual( + ['expiresAt', 'generation', 'leaseExpiresAt', 'prewarmId', 'rollout', 'sessionId', 'status'].sort() + ); + expect(fetcher.mock.calls.map(([path]) => path)).toEqual([ + '/api/v1/live-teleports/prewarm', + '/api/v1/live-teleports/acquire', + '/api/v1/live-teleports/acquire', + '/api/v1/live-teleports/status', + ]); + }); + + it('rejects identity or required-field drift in frozen Cloud prewarm and pending acquire shapes', async () => { + const stalePrewarm = new CloudLiveTeleportClient( + async () => Response.json({ ...cloudPrewarmWarming(), sessionId: 'other-session' }, { status: 202 }), + 'https://cloud.agentrelay.test' + ); + await expect( + stalePrewarm.prewarm({ + sessionId: 'session-1', + generation: 2, + workspaceRoot: '/', + idempotencyKey: 'session-1:2:prewarm', + }) + ).rejects.toThrow('mismatched lifecycle metadata'); + + const missingExpiry = new CloudLiveTeleportClient(async () => { + const { expiresAt: _expiresAt, ...response } = cloudPrewarmWarming(); + return Response.json(response, { status: 202 }); + }, 'https://cloud.agentrelay.test'); + await expect( + missingExpiry.prewarm({ + sessionId: 'session-1', + generation: 2, + workspaceRoot: '/', + idempotencyKey: 'session-1:2:prewarm', + }) + ).rejects.toThrow('expiresAt'); + + const stalePending = new CloudLiveTeleportClient( + async () => Response.json({ ...cloudAcquireVerifying(), generation: 3 }, { status: 202 }), + 'https://cloud.agentrelay.test' + ); + await expect(stalePending.acquire(input)).rejects.toThrow('stale lifecycle identity'); + }); + + it('enforces Cloud prewarm HTTP, expiry, and retry semantics', async () => { + const readyResponse = { + ...cloudPrewarmWarming(), + status: 'ready', + }; + const { retryAfterMs: _retryAfterMs, ...ready } = readyResponse; + const readyClient = new CloudLiveTeleportClient( + async () => Response.json(ready), + 'https://cloud.agentrelay.test' + ); + await expect( + readyClient.prewarm({ + sessionId: 'session-1', + generation: 2, + workspaceRoot: '/', + idempotencyKey: 'session-1:2:prewarm', + }) + ).resolves.toEqual(ready); + + for (const response of [ + { body: cloudPrewarmWarming(), status: 200 }, + { + body: { ...cloudPrewarmWarming(), status: 'ready' }, + status: 200, + }, + { + body: (() => { + const { retryAfterMs: _retry, ...withoutRetry } = cloudPrewarmWarming(); + return withoutRetry; + })(), + status: 202, + }, + ]) { + const client = new CloudLiveTeleportClient( + async () => Response.json(response.body, { status: response.status }), + 'https://cloud.agentrelay.test' + ); + await expect( + client.prewarm({ + sessionId: 'session-1', + generation: 2, + workspaceRoot: '/', + idempotencyKey: 'session-1:2:prewarm', + }) + ).rejects.toThrow('mismatched lifecycle metadata'); + } + }); + + it('accepts only the provider-neutral Cloud WSS bridge contract', async () => { + const fetcher = vi.fn(async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/session-1/g/2?ticket=opaque', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: verification(), + }) + ); + + await expect( + new CloudLiveTeleportClient(fetcher, 'https://cloud.agentrelay.test').acquire(input) + ).resolves.toMatchObject({ + environmentId: 'env-2', + generation: 2, + execServerUrl: 'wss://cloud.agentrelay.test/api/v1/live-teleports/connect/session-1/g/2?ticket=opaque', + }); + expect(fetcher).toHaveBeenCalledWith( + '/api/v1/live-teleports/acquire', + expect.objectContaining({ method: 'POST' }) + ); + }); + + it('fails closed if Cloud exposes a provider credential or URL', async () => { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: verification(), + providerUrl: 'wss://provider.invalid/raw', + trafficAccessToken: 'secret', + }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.acquire(input)).rejects.toThrow('forbidden sensitive field'); + }); + + it('recursively rejects credential-shaped fields at every remote response boundary', async () => { + const active = { + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + verification: verification(), + }; + const cases = [ + { + response: { prewarmId: 'prewarm-2', generation: 2, status: 'ready', nested: { daytonaApiKey: 'x' } }, + invoke: (client: CloudLiveTeleportClient) => + client.prewarm({ + sessionId: 'session-1', + generation: 2, + workspaceRoot: '/', + idempotencyKey: 'session-1:2:prewarm', + }), + }, + { + response: { + sessionId: 'session-1', + generation: 2, + status: 'active', + metadata: { e2bApiKey: 'x' }, + }, + invoke: (client: CloudLiveTeleportClient) => client.status({ sessionId: 'session-1', generation: 2 }), + }, + { + response: { ...active, metadata: { child: { accessToken: 'x' } } }, + invoke: (client: CloudLiveTeleportClient) => client.acquire(input), + }, + { + response: { + sessionId: 'session-1', + generation: 2, + status: 'revoked', + metadata: { authorization: 'Bearer x' }, + }, + invoke: (client: CloudLiveTeleportClient) => + client.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'session-1:2:revoke' }), + }, + { + response: { + sessionId: 'session-1', + generation: 2, + status: 'revoked', + metadata: { nested: { secret: 'x' } }, + }, + invoke: (client: CloudLiveTeleportClient) => + client.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'session-1:2:revoke' }), + }, + ]; + + for (const scenario of cases) { + const client = new CloudLiveTeleportClient( + async () => Response.json(scenario.response), + 'https://cloud.agentrelay.test' + ); + await expect(scenario.invoke(client)).rejects.toThrow('forbidden sensitive field'); + } + }); + + it('rejects unknown non-sensitive fields instead of retaining unparsed remote response data', async () => { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + status: 'active', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + metadata: { region: 'unknown' }, + }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'unexpected response field metadata' + ); + }); + + it('rejects an arbitrary execution URL even when it points at Cloud', async () => { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + execServerUrl: 'ws://127.0.0.1:4500', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: verification(), + }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.acquire(input)).rejects.toThrow('must not return an arbitrary execServerUrl'); + }); + + it('rejects absolute, cross-origin, and traversal connect paths', async () => { + for (const connectPath of [ + 'wss://provider.invalid/raw', + '//provider.invalid/raw', + '/api/v1/live-teleports/connect/../../provider', + ]) { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath, + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: verification(), + }), + 'https://cloud.agentrelay.test' + ); + await expect(client.acquire(input)).rejects.toThrow(/connectPath/); + } + }); + + it('rejects a verification whose authoritative digest is malformed', async () => { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: verification({ digest: 'f'.repeat(64) }), + }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.acquire(input)).rejects.toThrow('invalid verification.observed.digest'); + }); + + it.each([ + ['bare revision', verification({ workspaceRevision: '12' })], + ['wrong revision namespace', verification({ workspaceRevision: 'evt_12' })], + ['bare cursor', verification({ eventCursor: '19313' })], + ['wrong cursor namespace', verification({ eventCursor: 'rev_19313' })], + ])('rejects %s in Relayfile destination verification', async (_name, proof) => { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: proof, + }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.acquire(input)).rejects.toThrow(/verification\.observed/); + }); + + it.each([ + ['workspace', verification({ workspaceId: 'workspace-other' })], + ['remote root', verification({ remoteRoot: '/other' })], + ['session', verification({ sessionId: 'session-other' })], + ['generation', verification({ generation: 3 })], + ['digest', verification({ digest: `sha256:${'b'.repeat(64)}` })], + ['workspace revision', verification({ workspaceRevision: 'rev_13' })], + ['event cursor', verification({ eventCursor: 'evt_19314' })], + ])('rejects destination verification that does not bind the receipt %s', async (_name, proof) => { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + verification: proof, + }), + 'https://cloud.agentrelay.test' + ); + + const error = await client.acquire(input).catch((caught: unknown) => caught); + expect(String(error)).toContain('mismatched Relayfile verification'); + expect(String(error)).not.toContain(receipt.sealToken); + }); + + it('rejects destination verification for a different acquired workspace cwd', async () => { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + verification: verification({ localRoot: '/different-workspace' }), + }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.acquire(input)).rejects.toThrow('mismatched Relayfile verification'); + }); + + it('rejects any response that echoes the one-use checkpoint capability', async () => { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + verification: { ...verification(), sealToken: receipt.sealToken }, + }), + 'https://cloud.agentrelay.test' + ); + + const error = await client.acquire(input).catch((caught: unknown) => caught); + expect(String(error)).toContain('forbidden sensitive field'); + expect(String(error)).not.toContain(receipt.sealToken); + }); + + it('bounds a permanently verifying acquire while replaying the same request', async () => { + const fetcher = vi.fn(async () => + Response.json({ ...cloudAcquireVerifying(), retryAfterMs: 1 }, { status: 202 }) + ); + const client = new CloudLiveTeleportClient(fetcher, 'https://cloud.agentrelay.test'); + + await expect(client.acquire(input)).rejects.toThrow('bounded verification poll limit'); + expect(fetcher).toHaveBeenCalledTimes(120); + expect(new Set(fetcher.mock.calls.map((call) => call[1]?.body))).toHaveLength(1); + }); + + it('rejects matching hashes when the destination outbox is not drained', async () => { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: verification({ pendingWriteback: 1 }), + }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.acquire(input)).rejects.toThrow('mismatched Relayfile verification'); + }); + + it('does not echo a Cloud error containing an opaque bridge or provider secret', async () => { + const client = new CloudLiveTeleportClient( + async () => + Response.json( + { error: 'provider rejected https://provider.invalid/?token=opaque-secret-value' }, + { status: 502, statusText: 'Bad Gateway' } + ), + 'https://cloud.agentrelay.test' + ); + + const error = await client.acquire(input).catch((caught: unknown) => caught); + expect(String(error)).toContain('Bad Gateway'); + expect(String(error)).not.toContain('opaque-secret-value'); + expect(String(error)).not.toContain('provider.invalid'); + }); + + it('does not leak the seal token through an acquire transport diagnostic', async () => { + const client = new CloudLiveTeleportClient(async (_path, init) => { + throw new Error(`transport rejected ${String(init?.body)}`); + }, 'https://cloud.agentrelay.test'); + + const error = await client.acquire(input).catch((caught: unknown) => caught); + expect(String(error)).toContain('acquire transport failed'); + expect(String(error)).not.toContain(receipt.sealToken); + }); + + it('parses bounded lifecycle polling and requires explicit revoke confirmation', async () => { + const fetcher = vi + .fn() + .mockResolvedValueOnce( + Response.json({ + sessionId: 'session-1', + generation: 2, + prewarmId: 'prewarm-2', + status: 'warming', + retryAfterMs: 250, + expiresAt: '2026-08-23T12:30:00.000Z', + rollout: cloudRollout, + }) + ) + .mockResolvedValueOnce(Response.json({ sessionId: 'session-1', generation: 2, status: 'revoked' })); + const client = new CloudLiveTeleportClient(fetcher, 'https://cloud.agentrelay.test'); + + await expect(client.status({ sessionId: 'session-1', generation: 2 })).resolves.toMatchObject({ + status: 'warming', + retryAfterMs: 250, + }); + expect(fetcher).toHaveBeenNthCalledWith( + 1, + '/api/v1/live-teleports/status', + expect.objectContaining({ method: 'POST' }) + ); + await expect( + client.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'revoke-2' }) + ).resolves.toMatchObject({ status: 'revoked' }); + }); + + it('requires exact status HTTP codes while accepting the compact 202 cleanup contract', async () => { + const activeWith202 = new CloudLiveTeleportClient( + async () => Response.json(cloudActiveStatus(), { status: 202 }), + 'https://cloud.agentrelay.test' + ); + await expect(activeWith202.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'invalid HTTP success code' + ); + + for (const status of [201, 206]) { + const client = new CloudLiveTeleportClient( + async () => Response.json(cloudActiveStatus(), { status }), + 'https://cloud.agentrelay.test' + ); + await expect(client.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'invalid HTTP success code' + ); + } + + const cleanup = new CloudLiveTeleportClient( + async () => + Response.json( + { + sessionId: 'session-1', + generation: 2, + prewarmId: 'prewarm-2', + status: 'cleanup_pending', + retryAfterMs: 250, + }, + { status: 202 } + ), + 'https://cloud.agentrelay.test' + ); + await expect(cleanup.status({ sessionId: 'session-1', generation: 2 })).resolves.toMatchObject({ + status: 'cleanup_pending', + retryAfterMs: 250, + }); + + const fullCleanup = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + prewarmId: 'prewarm-2', + status: 'cleanup_pending', + retryAfterMs: 250, + expiresAt: '2026-08-23T12:45:00.000Z', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + rollout: cloudRollout, + }), + 'https://cloud.agentrelay.test' + ); + await expect(fullCleanup.status({ sessionId: 'session-1', generation: 2 })).resolves.toMatchObject({ + status: 'cleanup_pending', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + }); + }); + + it.each(['warming', 'ready', 'verifying'] as const)( + 'rejects leaseExpiresAt while Cloud is only %s', + async (status) => { + const expiresAt = '2026-08-23T12:30:00.000Z'; + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + prewarmId: 'prewarm-2', + status, + expiresAt, + leaseExpiresAt: expiresAt, + rollout: cloudRollout, + ...(status === 'warming' || status === 'verifying' ? { retryAfterMs: 250 } : {}), + }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'lease before the environment became active' + ); + } + ); + + it('pins active, cleanup, and terminal lease metadata semantics', async () => { + const activeWithoutLease = new CloudLiveTeleportClient(async () => { + const { leaseExpiresAt: _leaseExpiresAt, ...response } = cloudActiveStatus(); + return Response.json(response); + }, 'https://cloud.agentrelay.test'); + await expect(activeWithoutLease.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'inconsistent active lease metadata' + ); + + const mismatchedActiveLease = new CloudLiveTeleportClient( + async () => + Response.json({ + ...cloudActiveStatus(), + leaseExpiresAt: '2026-08-23T12:44:59.000Z', + }), + 'https://cloud.agentrelay.test' + ); + await expect(mismatchedActiveLease.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'inconsistent active lease metadata' + ); + + for (const status of ['failed', 'revoked', 'expired'] as const) { + const terminal = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + prewarmId: 'prewarm-2', + status, + expiresAt: '2026-08-23T12:30:00.000Z', + // A terminal row may retain its historical active lease. It no + // longer grants execution and need not equal the terminal expiry. + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + rollout: cloudRollout, + }), + 'https://cloud.agentrelay.test' + ); + await expect(terminal.status({ sessionId: 'session-1', generation: 2 })).resolves.toMatchObject({ + status, + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + }); + } + }); + + it('parses the exact active lease deadline and rejects non-canonical timestamps', async () => { + const fetcher = vi + .fn() + .mockResolvedValueOnce(Response.json(cloudActiveStatus())) + .mockResolvedValueOnce( + Response.json({ + ...cloudActiveStatus(), + leaseExpiresAt: '2026-08-23T14:45:00+02:00', + }) + ); + const client = new CloudLiveTeleportClient(fetcher, 'https://cloud.agentrelay.test'); + + await expect(client.status({ sessionId: 'session-1', generation: 2 })).resolves.toMatchObject({ + status: 'active', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + }); + await expect(client.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'invalid leaseExpiresAt' + ); + }); + + it.each([ + ['unknown reason', { ...cloudRollout, reason: 'manual-override' }], + ['out-of-range percentage', { ...cloudRollout, percentage: 101 }], + ['inconsistent eligibility', { ...cloudRollout, reason: 'not-targeted', eligible: true }], + ['zero-percent admission', { ...cloudRollout, reason: 'percentage', percentage: 0 }], + ['100-percent exclusion', { ...cloudRollout, reason: 'not-targeted', percentage: 100, eligible: false }], + ['unexpected rollout field', { ...cloudRollout, cohort: 'canary' }], + ])('rejects %s in Cloud status rollout metadata', async (_name, rollout) => { + const client = new CloudLiveTeleportClient( + async () => Response.json({ ...cloudActiveStatus(), rollout }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow(/rollout/); + }); + + it.each(['warming', 'ready', 'verifying', 'active'] as const)( + 'rejects Cloud %s while the rollout is disabled', + async (status) => { + const expiresAt = '2026-08-23T12:45:00.000Z'; + const response = { + sessionId: 'session-1', + generation: 2, + prewarmId: 'prewarm-2', + status, + expiresAt, + rollout: { + masterEnabled: false, + eligible: false, + reason: 'disabled', + percentage: 25, + }, + ...(status === 'warming' || status === 'verifying' ? { retryAfterMs: 250 } : {}), + ...(status === 'active' ? { leaseExpiresAt: expiresAt } : {}), + }; + const client = new CloudLiveTeleportClient( + async () => Response.json(response), + 'https://cloud.agentrelay.test' + ); + + await expect(client.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'nonterminal state for an ineligible rollout' + ); + } + ); + + it('rejects a cross-session status response even when its rollout and lease are valid', async () => { + const client = new CloudLiveTeleportClient( + async () => Response.json({ ...cloudActiveStatus(), sessionId: 'other-session' }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'stale or cross-session' + ); + }); + + it('accepts cleanup_pending as a non-terminal revoke acknowledgement', async () => { + const client = new CloudLiveTeleportClient( + async () => + Response.json( + { sessionId: 'session-1', generation: 2, status: 'cleanup_pending', retryAfterMs: 10 }, + { status: 202 } + ), + 'https://cloud.agentrelay.test' + ); + await expect( + client.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'revoke-2' }) + ).resolves.toMatchObject({ status: 'cleanup_pending' }); + }); + + it('accepts only the exact identity-matching HTTP 200 terminal no-row revoke response', async () => { + const exact = new CloudLiveTeleportClient( + async () => + Response.json({ sessionId: 'session-1', generation: 2, status: 'revoked' }, { status: 200 }), + 'https://cloud.agentrelay.test' + ); + await expect( + exact.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'revoke-no-row-2' }) + ).resolves.toEqual({ sessionId: 'session-1', generation: 2, status: 'revoked' }); + + const mismatched = new CloudLiveTeleportClient( + async () => + Response.json({ sessionId: 'other-session', generation: 2, status: 'revoked' }, { status: 200 }), + 'https://cloud.agentrelay.test' + ); + await expect( + mismatched.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'revoke-no-row-2' }) + ).rejects.toThrow('stale or cross-session'); + + const notFound = new CloudLiveTeleportClient( + async () => + Response.json({ sessionId: 'session-1', generation: 2, status: 'revoked' }, { status: 404 }), + 'https://cloud.agentrelay.test' + ); + await expect( + notFound.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'revoke-no-row-2' }) + ).rejects.toThrow('request failed (404)'); + }); + + it('requires encrypted transport except for explicit loopback development', () => { + expect(() => new CloudLiveTeleportClient(vi.fn(), 'http://cloud.example.test')).toThrow('must use HTTPS'); + expect(() => new CloudLiveTeleportClient(vi.fn(), 'http://127.0.0.1:3000')).not.toThrow(); + }); + + it('rejects a successful revoke response that does not prove fencing', async () => { + const client = new CloudLiveTeleportClient( + async () => Response.json({ sessionId: 'session-1', generation: 2, status: 'ready' }), + 'https://cloud.agentrelay.test' + ); + await expect( + client.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'revoke-2' }) + ).rejects.toThrow('was not confirmed'); + }); +}); diff --git a/packages/cloud/src/live-teleport.ts b/packages/cloud/src/live-teleport.ts new file mode 100644 index 000000000..7f99d9496 --- /dev/null +++ b/packages/cloud/src/live-teleport.ts @@ -0,0 +1,793 @@ +export type LiveTeleportWorkspaceSource = { + kind: 'relayfile-checkpoint-seal'; + receipt: Record; +}; + +export type LiveTeleportPrewarmInput = { + sessionId: string; + generation: number; + workspaceRoot: string; + idempotencyKey: string; + signal?: AbortSignal; +}; + +export type LiveTeleportPrewarm = { + sessionId: string; + prewarmId: string; + generation: number; + status: 'warming' | 'ready'; + expiresAt: string; + retryAfterMs?: number; +}; + +export type LiveTeleportRollout = { + masterEnabled: boolean; + eligible: boolean; + reason: 'disabled' | 'workspace-allowlist' | 'user-allowlist' | 'percentage' | 'not-targeted'; + percentage: number; +}; + +export type LiveTeleportAcquireInput = LiveTeleportPrewarmInput & { + threadId: string; + source: LiveTeleportWorkspaceSource; + prewarmId?: string; +}; + +export type LiveTeleportDestinationVerification = { + version: 1; + kind: 'relayfile-destination-verification'; + verificationId: string; + workspaceId: string; + localRoot: string; + remoteRoot: '/'; + sessionId: string; + generation: number; + status: 'converged'; + observed: { + digest: string; + workspaceRevision: string; + eventCursor: string; + }; + health: { + pendingWriteback: 0; + conflicts: 0; + outboxPending: 0; + outboxNeedsAttention: false; + }; + verifiedAt: string; +}; + +export type LiveTeleportEnvironment = { + sessionId: string; + generation: number; + environmentId: string; + /** Relative, provider-neutral path returned by Cloud. */ + connectPath: string; + /** Derived locally from the constructor-pinned Cloud gateway origin. */ + execServerUrl: string; + workspaceCwd: string; + connectExpiresAt: string; + leaseExpiresAt: string; + verification: LiveTeleportDestinationVerification; +}; + +export type LiveTeleportRevokeInput = { + sessionId: string; + generation: number; + idempotencyKey: string; + signal?: AbortSignal; +}; + +export type LiveTeleportStatusInput = { + sessionId: string; + generation: number; + prewarmId?: string; + signal?: AbortSignal; +}; + +export type LiveTeleportLifecycleStatus = { + sessionId: string; + generation: number; + status: 'warming' | 'ready' | 'verifying' | 'active' | 'cleanup_pending' | 'failed' | 'revoked' | 'expired'; + prewarmId?: string; + retryAfterMs?: number; + expiresAt?: string; + /** Exact Cloud-authoritative lease deadline, present for active status. */ + leaseExpiresAt?: string; + /** Present on authoritative status responses; omitted by cleanup-pending recovery responses. */ + rollout?: LiveTeleportRollout; +}; + +export type LiveTeleportRevocation = LiveTeleportLifecycleStatus & { + status: 'revoked' | 'expired'; +}; + +export interface LiveTeleportCloudClient { + prewarm(input: LiveTeleportPrewarmInput): Promise; + status(input: LiveTeleportStatusInput): Promise; + acquire(input: LiveTeleportAcquireInput): Promise; + revoke(input: LiveTeleportRevokeInput): Promise; +} + +type Fetcher = (path: string, init?: RequestInit) => Promise; + +const SENSITIVE_RESPONSE_FIELD = + /(?:api[-_]?key|access[-_]?token|authorization|bearer|credential|password|private[-_]?key|secret|provider.*(?:url|token)|trafficAccessToken|signedPreviewUrl|sealToken|^ticket$)/i; +const MAX_ACQUIRE_ATTEMPTS = 120; +const MAX_ACQUIRE_RETRY_AFTER_MS = 1_000; +/** Remote turns can wait up to 30m for completion; Cloud reserves another 10m for fencing. */ +export const LIVE_TELEPORT_MIN_TURN_LEASE_MS = 40 * 60_000; + +function isObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function assertNoSensitiveResponseFields(value: unknown, path = 'response'): void { + if (Array.isArray(value)) { + value.forEach((entry, index) => assertNoSensitiveResponseFields(entry, `${path}[${index}]`)); + return; + } + if (!isObject(value)) return; + + for (const [key, entry] of Object.entries(value)) { + if (SENSITIVE_RESPONSE_FIELD.test(key)) { + throw new Error(`Cloud live-teleport response exposed forbidden sensitive field ${path}.${key}.`); + } + assertNoSensitiveResponseFields(entry, `${path}.${key}`); + } +} + +function assertAllowedResponseKeys( + value: Record, + allowed: readonly string[], + context: string +): void { + const unexpected = Object.keys(value).find((key) => !allowed.includes(key)); + if (unexpected) { + throw new Error(`Cloud live-teleport ${context} returned unexpected response field ${unexpected}.`); + } +} + +async function readPayload(response: Response): Promise { + const payload = (await response.json().catch(() => null)) as unknown; + assertNoSensitiveResponseFields(payload); + if (!response.ok) { + // Cloud error prose can accidentally interpolate an opaque ticket or + // provider URL. Only a short machine code is safe to relay to callers. + const detail = + isObject(payload) && typeof payload.code === 'string' && /^[A-Z0-9_]{1,64}$/.test(payload.code) + ? payload.code + : response.statusText; + throw new Error(`Cloud live-teleport request failed (${response.status}): ${detail || 'unknown error'}`); + } + return payload; +} + +async function fetchAcquireSafely(fetcher: Fetcher, path: string, init: RequestInit): Promise { + try { + return await fetcher(path, init); + } catch (error) { + if (error instanceof DOMException && error.name === 'AbortError') throw error; + // A transport adapter can include the serialized request body in its + // diagnostic. Acquire carries the one-use seal token, so never relay that + // arbitrary prose to callers. + throw new Error('Cloud live-teleport acquire transport failed.'); + } +} + +function requiredString(value: unknown, field: string): string { + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`Cloud live-teleport response is missing ${field}.`); + } + return value.trim(); +} + +function requiredGeneration(value: unknown): number { + if (!Number.isSafeInteger(value) || Number(value) < 1) { + throw new Error('Cloud live-teleport response has an invalid generation.'); + } + return Number(value); +} + +function requiredPositiveInteger(value: unknown, field: string): number { + if (!Number.isSafeInteger(value) || Number(value) < 1) { + throw new Error(`Cloud live-teleport response has an invalid ${field}.`); + } + return Number(value); +} + +function requiredDigest(value: unknown, field: string): string { + const digest = requiredString(value, field); + if (!/^sha256:[a-f0-9]{64}$/.test(digest)) { + throw new Error(`Cloud live-teleport verification has an invalid ${field}.`); + } + return digest; +} + +function requiredRelayfilePosition(value: unknown, field: string, prefix: 'rev' | 'evt'): string { + const position = requiredString(value, field); + if (!new RegExp(`^(?:0|${prefix}_[0-9]+)$`).test(position)) { + throw new Error(`Cloud live-teleport verification has an invalid ${field}.`); + } + return position; +} + +function requiredExactTimestamp(value: unknown, field: string): string { + const timestamp = requiredString(value, field); + const milliseconds = Date.parse(timestamp); + if (Number.isNaN(milliseconds) || new Date(milliseconds).toISOString() !== timestamp) { + throw new Error(`Cloud live-teleport response has an invalid ${field}.`); + } + return timestamp; +} + +function requiredRollout(value: unknown): LiveTeleportRollout { + if (!isObject(value)) { + throw new Error('Cloud live-teleport status is missing rollout metadata.'); + } + assertAllowedResponseKeys(value, ['masterEnabled', 'eligible', 'reason', 'percentage'], 'status rollout'); + const reason = value.reason; + const validReason = + reason === 'disabled' || + reason === 'workspace-allowlist' || + reason === 'user-allowlist' || + reason === 'percentage' || + reason === 'not-targeted'; + const percentage = value.percentage; + if ( + typeof value.masterEnabled !== 'boolean' || + typeof value.eligible !== 'boolean' || + !validReason || + !Number.isSafeInteger(percentage) || + Number(percentage) < 0 || + Number(percentage) > 100 + ) { + throw new Error('Cloud live-teleport status returned invalid rollout metadata.'); + } + const expected = + reason === 'disabled' + ? { masterEnabled: false, eligible: false } + : reason === 'not-targeted' + ? { masterEnabled: true, eligible: false } + : { masterEnabled: true, eligible: true }; + if (value.masterEnabled !== expected.masterEnabled || value.eligible !== expected.eligible) { + throw new Error('Cloud live-teleport status returned inconsistent rollout metadata.'); + } + if ((reason === 'percentage' && percentage === 0) || (reason === 'not-targeted' && percentage === 100)) { + throw new Error('Cloud live-teleport status returned an impossible rollout percentage.'); + } + return { + masterEnabled: value.masterEnabled, + eligible: value.eligible, + reason, + percentage: Number(percentage), + }; +} + +type LiveTeleportReceiptBinding = { + workspaceId: string; + remoteRoot: '/'; + sessionId: string; + generation: number; + digest: string; + workspaceRevision: string; + eventCursor: string; +}; + +function requiredReceiptBinding(source: LiveTeleportWorkspaceSource): LiveTeleportReceiptBinding { + if (source.kind !== 'relayfile-checkpoint-seal' || !isObject(source.receipt)) { + throw new Error('Cloud live-teleport acquire requires a Relayfile checkpoint receipt.'); + } + const receipt = source.receipt; + // Validate the capability without ever retaining it in verification output or + // interpolating it into an error. + requiredString(receipt.sealToken, 'source.receipt.sealToken'); + if (requiredString(receipt.root, 'source.receipt.root') !== '/') { + throw new Error('Cloud live-teleport acquire requires logical Relayfile root /.'); + } + return { + workspaceId: requiredString(receipt.workspaceId, 'source.receipt.workspaceId'), + remoteRoot: '/', + sessionId: requiredString(receipt.sessionId, 'source.receipt.sessionId'), + generation: requiredGeneration(receipt.generation), + digest: requiredDigest(receipt.digest, 'source.receipt.digest'), + workspaceRevision: requiredRelayfilePosition( + receipt.workspaceRevision, + 'source.receipt.workspaceRevision', + 'rev' + ), + eventCursor: requiredRelayfilePosition(receipt.eventCursor, 'source.receipt.eventCursor', 'evt'), + }; +} + +function requiredVerification( + value: unknown, + expected: LiveTeleportReceiptBinding & { localRoot: string } +): LiveTeleportDestinationVerification { + if (!isObject(value) || !isObject(value.observed) || !isObject(value.health)) { + throw new Error('Cloud live-teleport acquire did not return Relayfile destination verification.'); + } + assertAllowedResponseKeys( + value, + [ + 'version', + 'kind', + 'verificationId', + 'workspaceId', + 'localRoot', + 'remoteRoot', + 'sessionId', + 'generation', + 'status', + 'observed', + 'health', + 'verifiedAt', + ], + 'verification' + ); + assertAllowedResponseKeys( + value.observed, + ['digest', 'workspaceRevision', 'eventCursor'], + 'verification.observed' + ); + assertAllowedResponseKeys( + value.health, + ['pendingWriteback', 'conflicts', 'outboxPending', 'outboxNeedsAttention'], + 'verification.health' + ); + const verifiedAt = requiredString(value.verifiedAt, 'verification.verifiedAt'); + const workspaceId = requiredString(value.workspaceId, 'verification.workspaceId'); + const localRoot = requiredString(value.localRoot, 'verification.localRoot'); + const digest = requiredDigest(value.observed.digest, 'verification.observed.digest'); + const workspaceRevision = requiredRelayfilePosition( + value.observed.workspaceRevision, + 'verification.observed.workspaceRevision', + 'rev' + ); + const eventCursor = requiredRelayfilePosition( + value.observed.eventCursor, + 'verification.observed.eventCursor', + 'evt' + ); + if ( + value.version !== 1 || + value.kind !== 'relayfile-destination-verification' || + value.status !== 'converged' || + workspaceId !== expected.workspaceId || + localRoot !== expected.localRoot || + value.remoteRoot !== expected.remoteRoot || + value.sessionId !== expected.sessionId || + value.generation !== expected.generation || + digest !== expected.digest || + workspaceRevision !== expected.workspaceRevision || + eventCursor !== expected.eventCursor || + value.health.pendingWriteback !== 0 || + value.health.conflicts !== 0 || + value.health.outboxPending !== 0 || + value.health.outboxNeedsAttention !== false || + Number.isNaN(Date.parse(verifiedAt)) + ) { + throw new Error('Cloud live-teleport acquire returned a mismatched Relayfile verification.'); + } + return { + version: 1, + kind: 'relayfile-destination-verification', + verificationId: requiredString(value.verificationId, 'verification.verificationId'), + workspaceId, + localRoot, + remoteRoot: '/', + sessionId: expected.sessionId, + generation: expected.generation, + status: 'converged', + observed: { + digest, + workspaceRevision, + eventCursor, + }, + health: { + pendingWriteback: 0, + conflicts: 0, + outboxPending: 0, + outboxNeedsAttention: false, + }, + verifiedAt, + }; +} + +function wait(milliseconds: number, signal?: AbortSignal): Promise { + if (signal?.aborted) return Promise.reject(new DOMException('Aborted', 'AbortError')); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal?.removeEventListener('abort', abort); + resolve(); + }, milliseconds); + const abort = () => { + clearTimeout(timer); + reject(new DOMException('Aborted', 'AbortError')); + }; + signal?.addEventListener('abort', abort, { once: true }); + }); +} + +/** + * Provider-neutral Cloud control-plane client. The only execution address Relay + * accepts is Cloud's short-lived WSS bridge; raw provider URLs and credentials + * are rejected even if a buggy server includes them in an otherwise-valid body. + */ +export class CloudLiveTeleportClient implements LiveTeleportCloudClient { + private readonly gatewayOrigin: URL; + + constructor( + private readonly fetcher: Fetcher, + gatewayOrigin: string + ) { + let parsed: URL; + try { + parsed = new URL(gatewayOrigin); + } catch { + throw new Error('Cloud live-teleport requires a valid pinned gateway origin.'); + } + const loopback = + parsed.hostname === '127.0.0.1' || parsed.hostname === '::1' || parsed.hostname === 'localhost'; + if (parsed.protocol !== 'https:' && !(parsed.protocol === 'http:' && loopback)) { + throw new Error('Cloud live-teleport gateway origin must use HTTPS (HTTP is loopback-only).'); + } + if (parsed.username || parsed.password) { + throw new Error('Cloud live-teleport gateway origin must not contain credentials.'); + } + this.gatewayOrigin = new URL(parsed.origin); + } + + async prewarm(input: LiveTeleportPrewarmInput): Promise { + const { signal, ...request } = input; + const response = await this.fetcher('/api/v1/live-teleports/prewarm', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request), + signal, + }); + const payload = await readPayload(response); + if (!isObject(payload)) throw new Error('Cloud live-teleport prewarm returned an invalid response.'); + assertAllowedResponseKeys( + payload, + ['sessionId', 'generation', 'prewarmId', 'status', 'expiresAt', 'retryAfterMs'], + 'prewarm' + ); + + const status = payload.status; + if (status !== 'warming' && status !== 'ready') { + throw new Error('Cloud live-teleport prewarm returned an invalid status.'); + } + const sessionId = requiredString(payload.sessionId, 'sessionId'); + const generation = requiredGeneration(payload.generation); + const expiresAt = requiredExactTimestamp(payload.expiresAt, 'expiresAt'); + const retryAfterMs = + payload.retryAfterMs === undefined + ? undefined + : requiredPositiveInteger(payload.retryAfterMs, 'retryAfterMs'); + if ( + sessionId !== input.sessionId || + generation !== input.generation || + (status === 'warming' && (response.status !== 202 || retryAfterMs === undefined)) || + (status === 'ready' && (response.status !== 200 || retryAfterMs !== undefined)) + ) { + throw new Error('Cloud live-teleport prewarm returned mismatched lifecycle metadata.'); + } + return { + sessionId, + prewarmId: requiredString(payload.prewarmId, 'prewarmId'), + generation, + status, + expiresAt, + ...(retryAfterMs === undefined ? {} : { retryAfterMs }), + }; + } + + async status(input: LiveTeleportStatusInput): Promise { + const { signal, ...request } = input; + const response = await this.fetcher('/api/v1/live-teleports/status', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request), + signal, + }); + const payload = await readPayload(response); + if (!isObject(payload)) throw new Error('Cloud live-teleport status returned an invalid response.'); + if (response.status !== 200 && !(response.status === 202 && payload.status === 'cleanup_pending')) { + throw new Error('Cloud live-teleport status returned an invalid HTTP success code.'); + } + const status = this.parseLifecycleStatus(payload, 'status', response.status); + if (status.sessionId !== input.sessionId || status.generation !== input.generation) { + throw new Error('Cloud live-teleport status returned a stale or cross-session lifecycle identity.'); + } + return status; + } + + async acquire(input: LiveTeleportAcquireInput): Promise { + const { signal, ...request } = input; + const receiptBinding = requiredReceiptBinding(input.source); + if (receiptBinding.sessionId !== input.sessionId || receiptBinding.generation !== input.generation) { + throw new Error('Cloud live-teleport acquire received a stale or cross-session checkpoint receipt.'); + } + let payload: unknown; + let responseStatus: number | undefined; + let stillVerifying = false; + for (let attempt = 0; attempt < MAX_ACQUIRE_ATTEMPTS; attempt += 1) { + const response = await fetchAcquireSafely(this.fetcher, '/api/v1/live-teleports/acquire', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request), + signal, + }); + payload = await readPayload(response); + responseStatus = response.status; + if (response.status !== 202) { + stillVerifying = false; + break; + } + stillVerifying = true; + if (!isObject(payload) || payload.status !== 'verifying') { + throw new Error('Cloud live-teleport acquire returned an invalid pending response.'); + } + assertAllowedResponseKeys( + payload, + ['sessionId', 'generation', 'status', 'retryAfterMs'], + 'pending acquire' + ); + const pendingSessionId = requiredString(payload.sessionId, 'sessionId'); + const pendingGeneration = requiredGeneration(payload.generation); + if (pendingSessionId !== input.sessionId || pendingGeneration !== input.generation) { + throw new Error('Cloud live-teleport pending acquire returned a stale lifecycle identity.'); + } + await wait( + Math.min(requiredPositiveInteger(payload.retryAfterMs, 'retryAfterMs'), MAX_ACQUIRE_RETRY_AFTER_MS), + signal + ); + } + if (stillVerifying) { + throw new Error('Cloud live-teleport acquire exceeded the bounded verification poll limit.'); + } + if (!isObject(payload)) throw new Error('Cloud live-teleport acquire returned an invalid response.'); + if (responseStatus !== 200 || payload.status !== 'active') { + throw new Error('Cloud live-teleport acquire returned invalid active lifecycle metadata.'); + } + + if ('execServerUrl' in payload) { + throw new Error('Cloud live-teleport acquire must not return an arbitrary execServerUrl.'); + } + assertAllowedResponseKeys( + payload, + [ + 'sessionId', + 'generation', + 'threadId', + 'status', + 'environmentId', + 'connectPath', + 'workspaceCwd', + 'connectExpiresAt', + 'leaseExpiresAt', + 'verification', + ], + 'acquire' + ); + const connectPath = this.requiredConnectPath(payload.connectPath); + const execServerUrl = this.execServerUrl(connectPath); + + const sessionId = requiredString(payload.sessionId, 'sessionId'); + const generation = requiredGeneration(payload.generation); + const threadId = requiredString(payload.threadId, 'threadId'); + const workspaceCwd = requiredString(payload.workspaceCwd, 'workspaceCwd'); + const connectExpiresAt = requiredExactTimestamp(payload.connectExpiresAt, 'connectExpiresAt'); + const leaseExpiresAt = requiredExactTimestamp(payload.leaseExpiresAt, 'leaseExpiresAt'); + if ( + sessionId !== input.sessionId || + generation !== input.generation || + threadId !== input.threadId || + Date.parse(leaseExpiresAt) <= Date.parse(connectExpiresAt) + ) { + throw new Error('Cloud live-teleport acquire returned invalid active lifecycle metadata.'); + } + + return { + sessionId, + generation, + environmentId: requiredString(payload.environmentId, 'environmentId'), + connectPath, + execServerUrl, + workspaceCwd, + connectExpiresAt, + leaseExpiresAt, + verification: requiredVerification(payload.verification, { + ...receiptBinding, + localRoot: workspaceCwd, + }), + }; + } + + async revoke(input: LiveTeleportRevokeInput): Promise { + const { signal, ...request } = input; + const response = await this.fetcher('/api/v1/live-teleports/revoke', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request), + signal, + }); + const payload = await readPayload(response); + if (!isObject(payload)) throw new Error('Cloud live-teleport revoke returned an invalid response.'); + const status = this.parseLifecycleStatus(payload, 'revoke', response.status); + if (status.sessionId !== input.sessionId || status.generation !== input.generation) { + throw new Error('Cloud live-teleport revoke returned a stale or cross-session lifecycle identity.'); + } + if (status.status !== 'cleanup_pending' && status.status !== 'revoked') { + throw new Error('Cloud live-teleport revoke was not confirmed.'); + } + return status; + } + + private parseLifecycleStatus( + payload: Record, + boundary: 'status' | 'revoke', + httpStatus: number + ): LiveTeleportLifecycleStatus { + assertAllowedResponseKeys( + payload, + boundary === 'status' + ? [ + 'sessionId', + 'generation', + 'status', + 'prewarmId', + 'retryAfterMs', + 'expiresAt', + 'leaseExpiresAt', + 'rollout', + ] + : ['sessionId', 'generation', 'status', 'retryAfterMs'], + 'lifecycle status' + ); + const status = payload.status; + if ( + status !== 'warming' && + status !== 'ready' && + status !== 'verifying' && + status !== 'active' && + status !== 'cleanup_pending' && + status !== 'failed' && + status !== 'revoked' && + status !== 'expired' + ) { + throw new Error('Cloud live-teleport status returned an invalid lifecycle state.'); + } + const expiresAt = + payload.expiresAt === undefined ? undefined : requiredExactTimestamp(payload.expiresAt, 'expiresAt'); + const leaseExpiresAt = + payload.leaseExpiresAt === undefined + ? undefined + : requiredExactTimestamp(payload.leaseExpiresAt, 'leaseExpiresAt'); + const retryAfterMs = + payload.retryAfterMs === undefined + ? undefined + : requiredPositiveInteger(payload.retryAfterMs, 'retryAfterMs'); + const rollout = payload.rollout === undefined ? undefined : requiredRollout(payload.rollout); + this.assertLifecycleResponseSemantics({ + boundary, + httpStatus, + status, + prewarmId: payload.prewarmId, + retryAfterMs, + expiresAt, + leaseExpiresAt, + rollout, + }); + return { + sessionId: requiredString(payload.sessionId, 'sessionId'), + generation: requiredGeneration(payload.generation), + status, + ...(payload.prewarmId === undefined + ? {} + : { prewarmId: requiredString(payload.prewarmId, 'prewarmId') }), + ...(retryAfterMs === undefined ? {} : { retryAfterMs }), + ...(expiresAt ? { expiresAt } : {}), + ...(leaseExpiresAt ? { leaseExpiresAt } : {}), + ...(rollout ? { rollout } : {}), + }; + } + + private assertLifecycleResponseSemantics(input: { + boundary: 'status' | 'revoke'; + httpStatus: number; + status: LiveTeleportLifecycleStatus['status']; + prewarmId: unknown; + retryAfterMs?: number; + expiresAt?: string; + leaseExpiresAt?: string; + rollout?: LiveTeleportRollout; + }): void { + if (input.boundary === 'revoke') { + const valid = + (input.status === 'cleanup_pending' && + input.httpStatus === 202 && + input.retryAfterMs !== undefined) || + (input.status !== 'cleanup_pending' && input.httpStatus === 200 && input.retryAfterMs === undefined); + if (!valid) throw new Error('Cloud live-teleport revoke returned invalid lifecycle semantics.'); + return; + } + + if (input.prewarmId === undefined) { + throw new Error('Cloud live-teleport status is missing prewarmId.'); + } + if (input.status === 'cleanup_pending') { + const compactCleanup = + input.httpStatus === 202 && + input.retryAfterMs !== undefined && + input.expiresAt === undefined && + input.leaseExpiresAt === undefined && + input.rollout === undefined; + const fullCleanup = + input.httpStatus === 200 && + input.retryAfterMs !== undefined && + input.expiresAt !== undefined && + input.rollout !== undefined && + (input.leaseExpiresAt === undefined || input.leaseExpiresAt === input.expiresAt); + if (!compactCleanup && !fullCleanup) { + throw new Error('Cloud live-teleport status returned invalid cleanup lifecycle semantics.'); + } + return; + } + + if (!input.expiresAt || !input.rollout) { + throw new Error('Cloud live-teleport status is missing authoritative lifecycle metadata.'); + } + if (input.httpStatus !== 200) { + throw new Error('Cloud live-teleport status returned an invalid HTTP success code.'); + } + const polling = input.status === 'warming' || input.status === 'verifying'; + if (polling !== (input.retryAfterMs !== undefined)) { + throw new Error('Cloud live-teleport status returned invalid retryAfterMs semantics.'); + } + if (input.status === 'active' && (!input.leaseExpiresAt || input.leaseExpiresAt !== input.expiresAt)) { + throw new Error('Cloud live-teleport status returned inconsistent active lease metadata.'); + } + if ( + (input.status === 'warming' || input.status === 'ready' || input.status === 'verifying') && + input.leaseExpiresAt !== undefined + ) { + throw new Error('Cloud live-teleport status returned a lease before the environment became active.'); + } + const requiresEligibleRollout = + input.status === 'warming' || + input.status === 'ready' || + input.status === 'verifying' || + input.status === 'active'; + if (requiresEligibleRollout && (!input.rollout.masterEnabled || !input.rollout.eligible)) { + throw new Error('Cloud live-teleport status returned a nonterminal state for an ineligible rollout.'); + } + } + + private requiredConnectPath(value: unknown): string { + const connectPath = requiredString(value, 'connectPath'); + if ( + !connectPath.startsWith('/api/v1/live-teleports/connect/') || + connectPath.startsWith('//') || + connectPath.includes('\\') || + connectPath.includes('#') + ) { + throw new Error('Cloud live-teleport acquire returned an invalid connectPath.'); + } + const parsed = new URL(connectPath, this.gatewayOrigin); + if ( + parsed.origin !== this.gatewayOrigin.origin || + !parsed.pathname.startsWith('/api/v1/live-teleports/connect/') + ) { + throw new Error('Cloud live-teleport acquire returned a cross-origin connectPath.'); + } + return `${parsed.pathname}${parsed.search}`; + } + + private execServerUrl(connectPath: string): string { + const url = new URL(connectPath, this.gatewayOrigin); + url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:'; + return url.toString(); + } +} diff --git a/tests/relayflows/cases/1605-long-running-provider-attribution/case.json b/tests/relayflows/cases/1605-long-running-provider-attribution/case.json new file mode 100644 index 000000000..beaacde33 --- /dev/null +++ b/tests/relayflows/cases/1605-long-running-provider-attribution/case.json @@ -0,0 +1,20 @@ +{ + "version": 1, + "id": "1605-long-running-provider-attribution", + "kind": "feature", + "title": "Send semantic long-running workload intent and preserve Cloud provider attribution", + "runner": { + "command": ["node", "tests/relayflows/cases/1605-long-running-provider-attribution/run.mjs"] + }, + "timeoutSeconds": 900, + "expected": { + "base": { + "outcome": "absent", + "signature": "semantic_profile_and_attribution_absent" + }, + "head": { + "outcome": "fixed", + "signature": "long_running_profile_with_agent37_attribution" + } + } +} diff --git a/tests/relayflows/cases/1605-long-running-provider-attribution/run.mjs b/tests/relayflows/cases/1605-long-running-provider-attribution/run.mjs new file mode 100644 index 000000000..a69d82161 --- /dev/null +++ b/tests/relayflows/cases/1605-long-running-provider-attribution/run.mjs @@ -0,0 +1,185 @@ +#!/usr/bin/env node + +import { execFileSync, spawnSync } from 'node:child_process'; +import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const CASE_ID = '1605-long-running-provider-attribution'; +const arm = requiredValue('RELAY_PR_PROOF_ARM'); +const targetDir = path.resolve(requiredValue('RELAY_PR_PROOF_TARGET_DIR')); +const harnessDir = path.resolve(requiredValue('RELAY_PR_PROOF_HARNESS_DIR')); +const resultPath = requiredValue('RELAY_PR_PROOF_RESULT_PATH'); +if (arm !== 'base' && arm !== 'head') throw new Error('RelayFlow proof arm must be base or head'); + +const expectedSha = requiredValue(arm === 'base' ? 'RELAY_PR_PROOF_BASE_SHA' : 'RELAY_PR_PROOF_HEAD_SHA'); +const targetSha = execFileSync('git', ['-C', targetDir, 'rev-parse', 'HEAD'], { + encoding: 'utf8', +}).trim(); +if (targetSha !== expectedSha) { + throw new Error(`Target checkout ${targetSha} does not match exact ${arm} SHA ${expectedSha}`); +} +const runnerPath = fileURLToPath(import.meta.url); +if (!isWithin(harnessDir, runnerPath)) { + throw new Error('RelayFlow runner must execute from the exact-head harness checkout'); +} + +const probePath = path.join( + targetDir, + 'packages/cloud/src/.relayflow-1605-long-running-provider-attribution.test.ts' +); +const observationPath = path.join( + targetDir, + '.relayflow-1605-long-running-provider-attribution-observation.json' +); +const configPath = path.join( + targetDir, + '.relayflow-1605-long-running-provider-attribution.vitest.config.mjs' +); + +const probeSource = String.raw`import { expect, test, vi } from 'vitest'; +import { writeFile } from 'node:fs/promises'; + +const mocks = vi.hoisted(() => ({ + ensureCloudSession: vi.fn(), + authorizedApiFetch: vi.fn(), +})); + +vi.mock('./auth.js', () => ({ + ensureCloudSession: mocks.ensureCloudSession, + authorizedApiFetch: mocks.authorizedApiFetch, +})); + +import { ensureCloudFleetSandbox } from './fleet-sandbox.js'; + +const auth = { + accessToken: 'relayflow-access', + refreshToken: 'relayflow-refresh', + accessTokenExpiresAt: '2099-01-01T00:00:00Z', + apiUrl: 'https://relayflow.invalid', +}; + +test('observes semantic request routing and provider attribution', async () => { + const observationPath = process.env.RELAY_PR1605_OBSERVATION_PATH; + if (!observationPath) throw new Error('Missing RELAY_PR1605_OBSERVATION_PATH'); + mocks.ensureCloudSession.mockResolvedValue({ auth, client: {} }); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99' }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ + outcome: 'provisioned', + nodeId: 'node-agent37-proof', + nodeName: 'agent37-proof', + sandboxId: 'sandbox-agent37-proof', + relayWorkspaceId: 'rw_agent37_proof', + relayfileMounted: true, + providerId: 'agent37', + }, { status: 201 }), + auth, + }); + + const request = { + workspaceId: 'rw_agent37_proof', + requiredCapability: 'spawn:codex', + mountRelayfile: true, + forceProvision: true, + workloadProfile: 'long-running-agent', + } as Parameters[0] & Record; + const result = await ensureCloudFleetSandbox(request); + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + const ensureCall = mocks.authorizedApiFetch.mock.calls[1]; + const body = JSON.parse(String(ensureCall?.[2]?.body ?? '{}')); + const providerId = (result as unknown as { providerId?: string }).providerId; + await writeFile(observationPath, JSON.stringify({ body, providerId }), 'utf8'); +}); +`; +const configSource = `export default { test: { environment: 'node', include: ['packages/cloud/src/.relayflow-1605-long-running-provider-attribution.test.ts'], setupFiles: [] } };\n`; + +try { + run( + 'npm', + ['ci', '--ignore-scripts', '--workspace', 'packages/cloud', '--include-workspace-root=false'], + targetDir, + 'Cloud workspace dependency installation' + ); + run('npm', ['run', 'build:config'], targetDir, 'configuration package build'); + run('npm', ['run', 'build:cloud'], targetDir, 'Cloud package build'); + await writeFile(probePath, probeSource, { encoding: 'utf8', flag: 'wx' }); + await writeFile(configPath, configSource, { encoding: 'utf8', flag: 'wx' }); + run( + 'npm', + ['exec', '--', 'vitest', 'run', '--config', path.relative(targetDir, configPath)], + targetDir, + 'semantic fleet request probe', + { RELAY_PR1605_OBSERVATION_PATH: observationPath } + ); + + const observation = JSON.parse(await readFile(observationPath, 'utf8')); + const fixed = + observation?.body?.workloadProfile === 'long-running-agent' && observation?.providerId === 'agent37'; + const absent = observation?.body?.workloadProfile === undefined && observation?.providerId === undefined; + if (!fixed && !absent) { + throw new Error(`Unexpected semantic routing observation: ${JSON.stringify(observation)}`); + } + const outcome = fixed ? 'fixed' : 'absent'; + const signature = fixed + ? 'long_running_profile_with_agent37_attribution' + : 'semantic_profile_and_attribution_absent'; + await mkdir(path.dirname(resultPath), { recursive: true }); + await writeFile( + resultPath, + `${JSON.stringify( + { + version: 1, + caseId: CASE_ID, + arm, + outcome, + signature, + details: fixed + ? 'Cloud request carried long-running-agent and the normalized result retained Agent37 attribution.' + : 'Cloud request omitted semantic workload intent and discarded provider attribution.', + }, + null, + 2 + )}\n`, + 'utf8' + ); +} finally { + await rm(probePath, { force: true }); + await rm(configPath, { force: true }); + await rm(observationPath, { force: true }); +} + +function requiredValue(name) { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`Missing required environment variable ${name}`); + return value; +} + +function isWithin(directory, candidate) { + const relative = path.relative(directory, candidate); + return ( + relative === '' || + (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)) + ); +} + +function run(command, args, cwd, label, extraEnv = {}) { + const completed = spawnSync(command, args, { + cwd, + env: { ...process.env, ...extraEnv }, + stdio: ['ignore', 'inherit', 'inherit'], + }); + if (completed.error) throw new Error(`${label} could not start: ${completed.error.message}`); + if (completed.status !== 0) { + throw new Error( + `${label} failed with ${ + completed.signal ? `signal ${completed.signal}` : `exit code ${completed.status ?? 'unknown'}` + }` + ); + } +}