From 523dbb7cd11077332150827eea00deffe291738f Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Sun, 23 Aug 2026 15:19:29 +0200 Subject: [PATCH 01/16] feat(codex): add dormant live execution teleport controller --- CHANGELOG.md | 6 + packages/cli/src/cli/bootstrap.test.ts | 6 + packages/cli/src/cli/bootstrap.ts | 2 + packages/cli/src/cli/commands/codex.test.ts | 97 ++++ packages/cli/src/cli/commands/codex.ts | 374 +++++++++++++++ .../cli/src/cli/lib/codex-app-server.test.ts | 120 +++++ packages/cli/src/cli/lib/codex-app-server.ts | 375 +++++++++++++++ .../src/cli/lib/codex-live-controller.test.ts | 412 ++++++++++++++++ .../cli/src/cli/lib/codex-live-controller.ts | 451 ++++++++++++++++++ packages/cloud/src/index.ts | 13 + packages/cloud/src/live-teleport.test.ts | 141 ++++++ packages/cloud/src/live-teleport.ts | 291 +++++++++++ 12 files changed, 2288 insertions(+) create mode 100644 packages/cli/src/cli/commands/codex.test.ts create mode 100644 packages/cli/src/cli/commands/codex.ts create mode 100644 packages/cli/src/cli/lib/codex-app-server.test.ts create mode 100644 packages/cli/src/cli/lib/codex-app-server.ts create mode 100644 packages/cli/src/cli/lib/codex-live-controller.test.ts create mode 100644 packages/cli/src/cli/lib/codex-live-controller.ts create mode 100644 packages/cloud/src/live-teleport.test.ts create mode 100644 packages/cloud/src/live-teleport.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index e48267c6e..6aa5b3ce1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to Agent Relay will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased - Minor] + +### Added + +- `relay codex run` starts a Relay-managed local Codex app-server/thread, and `relay codex teleport` queues Cloud execution for its next turn boundary with persisted generations and local-resume rollback. + ## [Unreleased - Patch] ### Fixed diff --git a/packages/cli/src/cli/bootstrap.test.ts b/packages/cli/src/cli/bootstrap.test.ts index 067c42181..4a93f2670 100644 --- a/packages/cli/src/cli/bootstrap.test.ts +++ b/packages/cli/src/cli/bootstrap.test.ts @@ -40,6 +40,11 @@ const expectedLeafCommands = [ 'reflex off', 'reflex status', 'session replay', + // Relay-managed Codex execution teleport + 'codex run', + 'codex teleport', + 'codex rollback', + 'codex status', // fleet (serve is a hidden error stub, filtered out below) 'fleet agent list', 'fleet config', @@ -221,6 +226,7 @@ describe('bootstrap CLI', () => { 'fleet', 'reflex', 'session', + 'codex', 'status', 'observer', 'version', diff --git a/packages/cli/src/cli/bootstrap.ts b/packages/cli/src/cli/bootstrap.ts index 7184ed7b7..ebd4782ce 100644 --- a/packages/cli/src/cli/bootstrap.ts +++ b/packages/cli/src/cli/bootstrap.ts @@ -50,6 +50,7 @@ import { registerCapabilitiesCommands } from './commands/capabilities.js'; import { registerFleetCommands } from './commands/fleet.js'; import { registerSkillsCommands } from './commands/skills.js'; import { registerSessionCommands } from './commands/session.js'; +import { registerCodexCommands } from './commands/codex.js'; dotenvConfig({ quiet: true }); @@ -423,6 +424,7 @@ export function createProgram(options: { name?: string } = {}): Command { registerCapabilitiesCommands(program); registerSkillsCommands(program); registerSessionCommands(program); + registerCodexCommands(program); program .command('mcp') diff --git a/packages/cli/src/cli/commands/codex.test.ts b/packages/cli/src/cli/commands/codex.test.ts new file mode 100644 index 000000000..1b92d2e27 --- /dev/null +++ b/packages/cli/src/cli/commands/codex.test.ts @@ -0,0 +1,97 @@ +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { Command } from 'commander'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +import { registerCodexCommands, resolveLiveTeleportWorkspaceSource } from './codex.js'; + +const temporary: string[] = []; + +afterEach(() => { + temporary.splice(0).forEach((directory) => fs.rmSync(directory, { recursive: true, force: true })); +}); + +describe('resolveLiveTeleportWorkspaceSource', () => { + it('fails closed for a plain unmanaged or Git-only cwd', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-codex-unmanaged-')); + temporary.push(root); + fs.mkdirSync(path.join(root, '.git')); + + expect(() => resolveLiveTeleportWorkspaceSource(root)).toThrow('fails closed for an unmanaged'); + }); + + it('recognizes a Relayfile mount', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-codex-relayfile-')); + temporary.push(root); + const mountStatePath = path.join(root, '.relayfile-mount-state.json'); + fs.writeFileSync(mountStatePath, '{}'); + + expect(resolveLiveTeleportWorkspaceSource(root)).toEqual({ kind: 'relayfile-mount', mountStatePath }); + }); + + it('accepts an explicit opaque convergence receipt for Cloud verification', () => { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-codex-receipt-')); + temporary.push(root); + const receipt = path.join(root, 'receipt.jwt'); + fs.writeFileSync(receipt, 'signed.convergence.receipt\n'); + + expect(resolveLiveTeleportWorkspaceSource(root, receipt)).toEqual({ + kind: 'verified-convergence-receipt', + receipt: 'signed.convergence.receipt', + }); + }); +}); + +describe('registerCodexCommands', () => { + it('makes teleport one command against the active persisted generation', async () => { + const sendControl = vi.fn(async () => ({ + ok: true as const, + status: { + version: 1 as const, + sessionId: 'session-1', + threadId: 'thread-1', + workspaceRoot: '/repo', + generation: 4, + phase: 'teleport_pending' as const, + controllerPid: 10, + socketPath: '/state/controller.sock', + turnActive: false, + pending: { requestId: 'request-1', expectedGeneration: 4 }, + lastRequestId: 'request-1', + updatedAt: '2026-08-23T12:00:00.000Z', + controller: 'local' as const, + execution: 'local' as const, + workspaceSource: 'relayfile-mount' as const, + }, + })); + const log = vi.fn(); + const program = new Command().exitOverride(); + registerCodexCommands(program, { + readState: () => ({ generation: 4 }) as never, + sendControl, + requestId: () => 'request-1', + log, + }); + + await program.parseAsync(['node', 'relay', 'codex', 'teleport']); + + expect(sendControl).toHaveBeenCalledWith({ + operation: 'teleport', + requestId: 'request-1', + expectedGeneration: 4, + }); + expect(log).toHaveBeenCalledWith(expect.stringContaining('local controller remains authoritative')); + }); + + it('refuses teleport when the session was not started under Relay control', async () => { + const sendControl = vi.fn(); + const program = new Command().exitOverride(); + registerCodexCommands(program, { readState: () => null, sendControl }); + + await expect(program.parseAsync(['node', 'relay', 'codex', 'teleport'])).rejects.toThrow( + 'No active Relay-managed Codex session' + ); + expect(sendControl).not.toHaveBeenCalled(); + }); +}); diff --git a/packages/cli/src/cli/commands/codex.ts b/packages/cli/src/cli/commands/codex.ts new file mode 100644 index 000000000..1fac14f61 --- /dev/null +++ b/packages/cli/src/cli/commands/codex.ts @@ -0,0 +1,374 @@ +import fs from 'node:fs'; +import net from 'node:net'; +import os from 'node:os'; +import path from 'node:path'; +import readline from 'node:readline'; +import { randomUUID } from 'node:crypto'; + +import { + CloudLiveTeleportClient, + ensureCloudSession, + type LiveTeleportWorkspaceSource, +} from '@agent-relay/cloud'; +import { Command } from 'commander'; + +import { + isRelayfileMount, + probeCodexEnvironmentCapability, + StdioCodexAppServerSession, + type CodexNotification, +} from '../lib/codex-app-server.js'; +import { + CodexLiveController, + FileCodexControllerStateStore, + type CodexControllerState, + type PublicCodexControllerStatus, +} from '../lib/codex-live-controller.js'; + +export type CodexControllerPaths = { + directory: string; + statePath: string; + socketPath: string; +}; + +type ControlRequest = + | { operation: 'status' } + | { operation: 'teleport'; requestId: string; expectedGeneration: number } + | { operation: 'rollback' }; + +type ControlResponse = { ok: true; status: PublicCodexControllerStatus } | { ok: false; error: string }; + +export interface CodexCommandDependencies { + runManaged(options: { + cwd: string; + model?: string; + receiptFile?: string; + prompt?: string; + json?: boolean; + }): Promise; + readState(): CodexControllerState | null; + sendControl(request: ControlRequest): Promise; + requestId(): string; + cwd(): string; + log(message: string): void; +} + +export function codexControllerPaths(env: NodeJS.ProcessEnv = process.env): CodexControllerPaths { + const root = env.AGENT_RELAY_STATE_DIR?.trim() || path.join(os.homedir(), '.agent-relay'); + const directory = path.join(root, 'codex-live'); + return { + directory, + statePath: path.join(directory, 'active.json'), + socketPath: path.join(directory, 'controller.sock'), + }; +} + +export function resolveLiveTeleportWorkspaceSource( + workspaceRoot: string, + receiptFile?: string +): LiveTeleportWorkspaceSource { + if (receiptFile) { + const receipt = fs.readFileSync(path.resolve(receiptFile), 'utf8').trim(); + if (!receipt) throw new Error('The convergence receipt file is empty.'); + return { kind: 'verified-convergence-receipt', receipt }; + } + const relayfile = isRelayfileMount(workspaceRoot); + if (relayfile) return relayfile; + throw new Error( + 'Live Codex teleport fails closed for an unmanaged working directory. ' + + 'Use a Relayfile-mounted workspace or pass --convergence-receipt with a Cloud-verifiable receipt.' + ); +} + +function describeError(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function agentMessageDelta(notification: CodexNotification): string | undefined { + if (!/agentMessage.*delta/i.test(notification.method)) return undefined; + const params = notification.params; + if (!params || typeof params !== 'object' || Array.isArray(params)) return undefined; + const delta = (params as Record).delta; + return typeof delta === 'string' ? delta : undefined; +} + +async function listen(server: net.Server, socketPath: string): Promise { + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen(socketPath, () => { + server.off('error', reject); + resolve(); + }); + }); + fs.chmodSync(socketPath, 0o600); +} + +async function closeServer(server: net.Server): Promise { + if (!server.listening) return; + await new Promise((resolve) => server.close(() => resolve())); +} + +function createControlServer(controller: CodexLiveController): net.Server { + let serialized = Promise.resolve(); + return net.createServer((socket) => { + socket.setEncoding('utf8'); + let buffer = ''; + socket.on('data', (chunk: string) => { + buffer += chunk; + const newline = buffer.indexOf('\n'); + if (newline < 0) return; + const line = buffer.slice(0, newline); + socket.pause(); + serialized = serialized.then(async () => { + let response: ControlResponse; + try { + const request = JSON.parse(line) as ControlRequest; + if (request.operation === 'status') { + response = { ok: true, status: controller.status() }; + } else if (request.operation === 'teleport') { + response = { + ok: true, + status: controller.requestTeleport({ + requestId: request.requestId, + expectedGeneration: request.expectedGeneration, + }), + }; + } else if (request.operation === 'rollback') { + response = { ok: true, status: await controller.rollback() }; + } else { + throw new Error('Unsupported Codex controller operation.'); + } + } catch (error) { + response = { ok: false, error: describeError(error) }; + } + socket.end(`${JSON.stringify(response)}\n`); + }); + }); + }); +} + +async function sendSocketControl(socketPath: string, request: ControlRequest): Promise { + return new Promise((resolve, reject) => { + const socket = net.createConnection(socketPath); + socket.setEncoding('utf8'); + socket.setTimeout(5_000); + let buffer = ''; + socket.once('connect', () => socket.write(`${JSON.stringify(request)}\n`)); + socket.on('data', (chunk: string) => { + buffer += chunk; + const newline = buffer.indexOf('\n'); + if (newline < 0) return; + socket.destroy(); + try { + resolve(JSON.parse(buffer.slice(0, newline)) as ControlResponse); + } catch (error) { + reject(new Error('Codex controller returned an invalid response.', { cause: error })); + } + }); + socket.once('timeout', () => { + socket.destroy(); + reject(new Error('Timed out contacting the active Codex controller.')); + }); + socket.once('error', (error) => + reject(new Error('No active Relay-managed Codex session.', { cause: error })) + ); + }); +} + +function withDefaults(overrides: Partial = {}): CodexCommandDependencies { + const paths = codexControllerPaths(); + return { + runManaged: + overrides.runManaged ?? + (async (options) => { + const workspaceRoot = fs.realpathSync(path.resolve(options.cwd)); + const source = resolveLiveTeleportWorkspaceSource(workspaceRoot, options.receiptFile); + await probeCodexEnvironmentCapability('codex'); + fs.mkdirSync(paths.directory, { recursive: true, mode: 0o700 }); + const store = new FileCodexControllerStateStore(paths.statePath); + const prior = store.read(); + if (prior && fs.existsSync(paths.socketPath) && processIsAlive(prior.controllerPid)) { + throw new Error( + `Relay-managed Codex thread ${prior.threadId} is already controlled by process ${prior.controllerPid}.` + ); + } + const session = await ensureCloudSession({ interactive: true }); + const cloud = new CloudLiveTeleportClient((requestPath, init) => + session.client.fetch(requestPath, init) + ); + + const controller = new CodexLiveController( + { + workspaceRoot, + source, + socketPath: paths.socketPath, + ...(options.model ? { model: options.model } : {}), + }, + { + cloud, + store, + createAppServer: async () => + StdioCodexAppServerSession.spawn({ + cwd: workspaceRoot, + onNotification: (notification) => { + if (options.json) process.stdout.write(`${JSON.stringify(notification)}\n`); + else { + const delta = agentMessageDelta(notification); + if (delta) process.stdout.write(delta); + } + }, + }), + // Production already probed before interactive Cloud login. Keeping + // the controller seam injectable lets restart/adversarial tests + // prove an unsupported local binary still fails closed. + probeCapability: async () => undefined, + now: () => new Date(), + sessionId: randomUUID, + pid: process.pid, + } + ); + + try { + fs.unlinkSync(paths.socketPath); + } catch (error) { + if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error; + } + const server = createControlServer(controller); + try { + const status = await controller.initialize(); + await listen(server, paths.socketPath); + process.stderr.write( + `Relay-managed Codex ${status.threadId} is ready locally (generation ${status.generation}).\n` + ); + + if (options.prompt) await controller.runTurn(options.prompt); + const input = readline.createInterface({ + input: process.stdin, + terminal: Boolean(process.stdin.isTTY), + }); + for await (const line of input) { + if (!line.trim()) continue; + await controller.runTurn(line); + if (!options.json) process.stdout.write('\n'); + } + } finally { + await closeServer(server); + await controller.close(); + try { + fs.unlinkSync(paths.socketPath); + } catch { + // Already gone. + } + } + }), + readState: + overrides.readState ?? + (() => { + try { + return new FileCodexControllerStateStore(paths.statePath).read(); + } catch { + return null; + } + }), + sendControl: overrides.sendControl ?? ((request) => sendSocketControl(paths.socketPath, request)), + requestId: overrides.requestId ?? randomUUID, + cwd: overrides.cwd ?? (() => process.cwd()), + log: overrides.log ?? ((message) => console.log(message)), + }; +} + +function processIsAlive(pid: number): boolean { + if (!Number.isSafeInteger(pid) || pid <= 0) return false; + try { + process.kill(pid, 0); + return true; + } catch (error) { + return Boolean(error instanceof Error && 'code' in error && error.code === 'EPERM'); + } +} + +function requireActiveState(deps: CodexCommandDependencies): CodexControllerState { + const state = deps.readState(); + if (!state) throw new Error('No active Relay-managed Codex session. Start one with `relay codex run`.'); + return state; +} + +function requireSuccess(response: ControlResponse): PublicCodexControllerStatus { + if (!response.ok) throw new Error(response.error); + return response.status; +} + +export function registerCodexCommands( + program: Command, + overrides: Partial = {} +): void { + const deps = withDefaults(overrides); + const group = program + .command('codex') + .description('Run and relocate a Relay-managed local Codex app-server session'); + + group + .command('run') + .description('Run a long-lived local Codex app-server and thread under Relay control') + .argument('[prompt...]', 'Optional first turn') + .option('--cwd ', 'Managed workspace root') + .option('--model ', 'Codex model') + .option('--convergence-receipt ', 'Cloud-verifiable receipt for a non-Relayfile workspace') + .option('--json', 'Write raw app-server notifications as JSON lines') + .action( + async ( + prompt: string[], + options: { cwd?: string; model?: string; convergenceReceipt?: string; json?: boolean } + ) => { + await deps.runManaged({ + cwd: options.cwd ?? deps.cwd(), + ...(options.model ? { model: options.model } : {}), + ...(options.convergenceReceipt ? { receiptFile: options.convergenceReceipt } : {}), + ...(prompt.length ? { prompt: prompt.join(' ') } : {}), + ...(options.json ? { json: true } : {}), + }); + } + ); + + group + .command('teleport') + .description('Move execution for the active managed Codex session to Cloud at the next turn boundary') + .action(async () => { + const state = requireActiveState(deps); + const status = requireSuccess( + await deps.sendControl({ + operation: 'teleport', + requestId: deps.requestId(), + expectedGeneration: state.generation, + }) + ); + deps.log( + `Codex execution teleport queued for generation ${status.generation}; the local controller remains authoritative.` + ); + }); + + group + .command('rollback') + .description('Revoke Cloud execution and resume the same thread through a fresh local controller') + .action(async () => { + requireActiveState(deps); + const status = requireSuccess(await deps.sendControl({ operation: 'rollback' })); + deps.log(`Codex thread ${status.threadId} resumed locally at generation ${status.generation}.`); + }); + + group + .command('status') + .description('Show where execution runs and where the conversation controller lives') + .option('--json', 'Write machine-readable JSON') + .action(async (options: { json?: boolean }) => { + requireActiveState(deps); + const status = requireSuccess(await deps.sendControl({ operation: 'status' })); + if (options.json) deps.log(JSON.stringify(status)); + else { + deps.log(`Execution: ${status.execution}`); + deps.log('Controller: local (keep this process and laptop running)'); + deps.log(`Thread: ${status.threadId}`); + deps.log(`Generation: ${status.generation}`); + } + }); +} diff --git a/packages/cli/src/cli/lib/codex-app-server.test.ts b/packages/cli/src/cli/lib/codex-app-server.test.ts new file mode 100644 index 000000000..ea31d092a --- /dev/null +++ b/packages/cli/src/cli/lib/codex-app-server.test.ts @@ -0,0 +1,120 @@ +import { EventEmitter } from 'node:events'; +import { PassThrough } from 'node:stream'; +import type { ChildProcessWithoutNullStreams } from 'node:child_process'; +import { describe, expect, it, vi } from 'vitest'; + +import { probeCodexEnvironmentCapability, StdioCodexAppServerSession } from './codex-app-server.js'; + +function fakeChild() { + const child = Object.assign(new EventEmitter(), { + stdin: new PassThrough(), + stdout: new PassThrough(), + stderr: new PassThrough(), + exitCode: null as number | null, + kill: vi.fn(() => true), + }); + return child as unknown as ChildProcessWithoutNullStreams; +} + +async function nextRequest(child: ChildProcessWithoutNullStreams): Promise> { + return new Promise((resolve) => { + child.stdin.once('data', (chunk: Buffer) => resolve(JSON.parse(chunk.toString('utf8').trim()))); + }); +} + +describe('probeCodexEnvironmentCapability', () => { + it('requires add and status in the locally generated experimental schema', async () => { + const readFile = vi.fn(async (file: string) => + file.endsWith('ClientRequest.json') + ? '{"methods":["environment/add","environment/status"]}' + : '{"required":["environmentId","execServerUrl"],"properties":{"environmentId":{},"execServerUrl":{}}}' + ); + await expect( + probeCodexEnvironmentCapability('codex', { + makeTempDir: async () => '/schema', + execFile: async () => undefined, + readFile, + remove: async () => undefined, + }) + ).resolves.toMatchObject({ environmentAdd: true, environmentStatus: true }); + }); + + it('fails closed when environment/status is unsupported', async () => { + const remove = vi.fn(async () => undefined); + await expect( + probeCodexEnvironmentCapability('codex', { + makeTempDir: async () => '/schema', + execFile: async () => undefined, + readFile: async (file) => + file.endsWith('ClientRequest.json') + ? '{"methods":["environment/add"]}' + : '{"required":["environmentId","execServerUrl"]}', + remove, + }) + ).rejects.toThrow('does not expose both'); + expect(remove).toHaveBeenCalledWith('/schema'); + }); + + it('rejects schema drift that could move provider credentials into Relay', async () => { + await expect( + probeCodexEnvironmentCapability('codex', { + makeTempDir: async () => '/schema', + execFile: async () => undefined, + readFile: async (file) => + file.endsWith('ClientRequest.json') + ? '{"methods":["environment/add","environment/status"]}' + : '{"required":["environmentId","execServerUrl"],"properties":{"headers":{}}}', + remove: async () => undefined, + }) + ).rejects.toThrow('credential field'); + }); +}); + +describe('StdioCodexAppServerSession', () => { + it('uses Codex newline RPC without a jsonrpc field and opts into the experimental API', async () => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child); + const requestPromise = nextRequest(child); + const initializing = session.initialize(); + const request = await requestPromise; + + expect(request).toEqual({ + id: 1, + method: 'initialize', + params: { + clientInfo: { name: 'agent-relay', title: 'Agent Relay managed Codex', version: '1' }, + capabilities: { experimentalApi: true }, + }, + }); + expect(request).not.toHaveProperty('jsonrpc'); + child.stdout.write(`${JSON.stringify({ id: 1, result: {} })}\n`); + await expect(initializing).resolves.toBeUndefined(); + await session.close(); + }); + + it('waits for turn/completed and sends the Cloud environment only on the attaching turn', async () => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child); + const requestPromise = nextRequest(child); + const running = session.runTurn({ + threadId: 'thread-1', + text: 'continue', + environment: { environmentId: 'environment-3', cwd: '/workspace' }, + }); + const request = await requestPromise; + expect(request).toMatchObject({ + id: 1, + method: 'turn/start', + params: { + threadId: 'thread-1', + environments: [{ environmentId: 'environment-3', cwd: '/workspace' }], + }, + }); + child.stdout.write(`${JSON.stringify({ id: 1, result: { turn: { id: 'turn-1' } } })}\n`); + child.stdout.write( + `${JSON.stringify({ method: 'turn/completed', params: { threadId: 'thread-1', turn: { id: 'turn-1' } } })}\n` + ); + await expect(running).resolves.toMatchObject({ turnId: 'turn-1' }); + await session.close(); + }); +}); diff --git a/packages/cli/src/cli/lib/codex-app-server.ts b/packages/cli/src/cli/lib/codex-app-server.ts new file mode 100644 index 000000000..c741a139d --- /dev/null +++ b/packages/cli/src/cli/lib/codex-app-server.ts @@ -0,0 +1,375 @@ +import fs from 'node:fs'; +import fsp from 'node:fs/promises'; +import os from 'node:os'; +import path from 'node:path'; +import { execFile, spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +export type CodexNotification = { + method: string; + params?: unknown; +}; + +export type CodexTurnResult = { + turnId?: string; + response: unknown; + completed: CodexNotification; +}; + +export interface CodexAppServerSession { + initialize(): Promise; + startThread(input: { cwd: string; model?: string }): Promise; + resumeThread(input: { threadId: string; cwd: string }): Promise; + addEnvironment(input: { + environmentId: string; + execServerUrl: string; + connectTimeoutMs: number; + }): Promise; + environmentStatus(environmentId: string): Promise; + runTurn(input: { + threadId: string; + text: string; + environment?: { environmentId: string; cwd: string }; + }): Promise; + close(): Promise; +} + +export type CodexEnvironmentCapability = { + environmentAdd: true; + environmentStatus: true; +}; + +export type CodexCapabilityProbeDependencies = { + makeTempDir: () => Promise; + execFile: (file: string, args: string[]) => Promise; + readFile: (file: string) => Promise; + remove: (directory: string) => Promise; +}; + +function defaultCapabilityProbeDependencies(): CodexCapabilityProbeDependencies { + return { + makeTempDir: () => fsp.mkdtemp(path.join(os.tmpdir(), 'relay-codex-schema-')), + execFile: async (file, args) => { + await execFileAsync(file, args, { maxBuffer: 4 * 1024 * 1024 }); + }, + readFile: (file) => fsp.readFile(file, 'utf8'), + remove: (directory) => fsp.rm(directory, { recursive: true, force: true }), + }; +} + +/** + * Probe the locally installed binary rather than assuming an experimental + * protocol from Relay's build-time Codex version. Both methods and the exact + * no-header EnvironmentAddParams seam are required before a managed session + * can become teleport-capable. + */ +export async function probeCodexEnvironmentCapability( + codexBinary = 'codex', + overrides: Partial = {} +): Promise { + const deps = { ...defaultCapabilityProbeDependencies(), ...overrides }; + const directory = await deps.makeTempDir(); + try { + await deps.execFile(codexBinary, [ + 'app-server', + 'generate-json-schema', + '--experimental', + '--out', + directory, + ]); + const [requests, addParams] = await Promise.all([ + deps.readFile(path.join(directory, 'ClientRequest.json')), + deps.readFile(path.join(directory, 'v2', 'EnvironmentAddParams.json')), + ]); + if (!requests.includes('"environment/add"') || !requests.includes('"environment/status"')) { + throw new Error( + 'This Codex app-server does not expose both experimental environment/add and environment/status.' + ); + } + + const schema = JSON.parse(addParams) as { + required?: unknown; + properties?: Record; + }; + const required = Array.isArray(schema.required) ? schema.required : []; + if (!required.includes('environmentId') || !required.includes('execServerUrl')) { + throw new Error('Codex EnvironmentAddParams does not match the required execution-teleport schema.'); + } + if (schema.properties && ('headers' in schema.properties || 'token' in schema.properties)) { + throw new Error( + 'Codex EnvironmentAddParams unexpectedly contains a credential field; upgrade Relay first.' + ); + } + + return { environmentAdd: true, environmentStatus: true }; + } finally { + await deps.remove(directory); + } +} + +type PendingRequest = { + resolve: (value: unknown) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; +}; + +function isObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function stringAt(value: unknown, ...keys: string[]): string | undefined { + let cursor: unknown = value; + for (const key of keys) { + if (!isObject(cursor)) return undefined; + cursor = cursor[key]; + } + return typeof cursor === 'string' && cursor.trim() ? cursor.trim() : undefined; +} + +function notificationThreadId(notification: CodexNotification): string | undefined { + return ( + stringAt(notification.params, 'threadId') ?? + stringAt(notification.params, 'thread', 'id') ?? + stringAt(notification.params, 'turn', 'threadId') + ); +} + +function notificationTurnId(notification: CodexNotification): string | undefined { + return stringAt(notification.params, 'turnId') ?? stringAt(notification.params, 'turn', 'id'); +} + +/** Newline-delimited Codex app-server transport. Codex omits the jsonrpc field. */ +export class StdioCodexAppServerSession implements CodexAppServerSession { + private readonly pending = new Map(); + private readonly notifications: CodexNotification[] = []; + private readonly notificationWaiters = new Set<{ + predicate: (notification: CodexNotification) => boolean; + resolve: (notification: CodexNotification) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; + }>(); + private nextId = 1; + private buffer = ''; + private stderr = ''; + private closed = false; + + constructor( + private readonly child: ChildProcessWithoutNullStreams, + private readonly requestTimeoutMs = 30_000, + private readonly turnTimeoutMs = 30 * 60_000, + private readonly onNotification?: (notification: CodexNotification) => void + ) { + child.stdout.setEncoding('utf8'); + child.stdout.on('data', (chunk: string) => this.onData(chunk)); + child.stderr.setEncoding('utf8'); + child.stderr.on('data', (chunk: string) => { + this.stderr = `${this.stderr}${chunk}`.slice(-8_192); + }); + child.once('exit', (code, signal) => { + this.closed = true; + const detail = this.stderr.trim().split('\n').at(-1); + this.rejectAll( + new Error( + `Codex app-server exited before completing the request (${code ?? signal ?? 'unknown'}).${ + detail ? ` ${detail}` : '' + }` + ) + ); + }); + } + + static spawn(options: { + codexBinary?: string; + cwd: string; + onNotification?: (notification: CodexNotification) => void; + }): StdioCodexAppServerSession { + const child = spawn(options.codexBinary ?? 'codex', ['app-server', '--stdio'], { + cwd: options.cwd, + stdio: ['pipe', 'pipe', 'pipe'], + env: process.env, + }); + return new StdioCodexAppServerSession(child, 30_000, 30 * 60_000, options.onNotification); + } + + async initialize(): Promise { + await this.request('initialize', { + clientInfo: { name: 'agent-relay', title: 'Agent Relay managed Codex', version: '1' }, + capabilities: { experimentalApi: true }, + }); + } + + async startThread(input: { cwd: string; model?: string }): Promise { + const result = await this.request('thread/start', { + cwd: input.cwd, + ...(input.model ? { model: input.model } : {}), + }); + const threadId = + stringAt(result, 'threadId') ?? stringAt(result, 'thread', 'id') ?? stringAt(result, 'id'); + if (!threadId) throw new Error('Codex thread/start returned no thread id.'); + return threadId; + } + + async resumeThread(input: { threadId: string; cwd: string }): Promise { + await this.request('thread/resume', { threadId: input.threadId, cwd: input.cwd }); + } + + async addEnvironment(input: { + environmentId: string; + execServerUrl: string; + connectTimeoutMs: number; + }): Promise { + await this.request('environment/add', input); + } + + environmentStatus(environmentId: string): Promise { + return this.request('environment/status', { environmentId }); + } + + async runTurn(input: { + threadId: string; + text: string; + environment?: { environmentId: string; cwd: string }; + }): Promise { + const response = await this.request( + 'turn/start', + { + threadId: input.threadId, + input: [{ type: 'text', text: input.text }], + ...(input.environment ? { environments: [input.environment] } : {}), + }, + this.turnTimeoutMs + ); + const turnId = + stringAt(response, 'turnId') ?? stringAt(response, 'turn', 'id') ?? stringAt(response, 'id'); + const completed = await this.waitForNotification( + (notification) => + notification.method === 'turn/completed' && + (!notificationThreadId(notification) || notificationThreadId(notification) === input.threadId) && + (!turnId || !notificationTurnId(notification) || notificationTurnId(notification) === turnId), + this.turnTimeoutMs + ); + return { turnId, response, completed }; + } + + async close(): Promise { + if (this.closed) return; + this.closed = true; + this.rejectAll(new Error('Codex app-server closed.')); + this.child.stdin.end(); + if (this.child.exitCode === null) this.child.kill('SIGTERM'); + } + + private request(method: string, params?: unknown, timeoutMs = this.requestTimeoutMs): Promise { + if (this.closed) return Promise.reject(new Error('Codex app-server is not running.')); + const id = this.nextId++; + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + this.pending.delete(id); + reject(new Error(`Timed out waiting for Codex ${method}.`)); + }, timeoutMs); + this.pending.set(id, { resolve, reject, timer }); + this.child.stdin.write( + `${JSON.stringify({ id, method, ...(params === undefined ? {} : { params }) })}\n` + ); + }); + } + + private waitForNotification( + predicate: (notification: CodexNotification) => boolean, + timeoutMs: number + ): Promise { + const queuedIndex = this.notifications.findIndex(predicate); + if (queuedIndex >= 0) return Promise.resolve(this.notifications.splice(queuedIndex, 1)[0]!); + + return new Promise((resolve, reject) => { + const waiter = { + predicate, + resolve: (notification: CodexNotification) => { + clearTimeout(waiter.timer); + this.notificationWaiters.delete(waiter); + resolve(notification); + }, + reject: (error: Error) => { + clearTimeout(waiter.timer); + this.notificationWaiters.delete(waiter); + reject(error); + }, + timer: setTimeout(() => { + this.notificationWaiters.delete(waiter); + reject(new Error('Timed out waiting for Codex turn/completed.')); + }, timeoutMs), + }; + this.notificationWaiters.add(waiter); + }); + } + + private onData(chunk: string): void { + this.buffer += chunk; + let newline: number; + while ((newline = this.buffer.indexOf('\n')) >= 0) { + const line = this.buffer.slice(0, newline).trim(); + this.buffer = this.buffer.slice(newline + 1); + if (!line) continue; + let message: unknown; + try { + message = JSON.parse(line) as unknown; + } catch { + continue; + } + if (isObject(message)) this.handleMessage(message); + } + } + + private handleMessage(message: Record): void { + if (typeof message.id === 'number' && ('result' in message || 'error' in message)) { + this.handleResponse(message.id, message); + return; + } + if (typeof message.method !== 'string') return; + const notification = { + method: message.method, + ...('params' in message ? { params: message.params } : {}), + }; + this.onNotification?.(notification); + const waiter = [...this.notificationWaiters].find((candidate) => candidate.predicate(notification)); + if (waiter) waiter.resolve(notification); + else this.notifications.push(notification); + } + + private handleResponse(id: number, message: Record): void { + const pending = this.pending.get(id); + if (!pending) return; + this.pending.delete(id); + clearTimeout(pending.timer); + if ('error' in message && isObject(message.error)) { + pending.reject( + new Error( + `${typeof message.error.code === 'number' ? `${message.error.code}: ` : ''}${ + typeof message.error.message === 'string' ? message.error.message : 'Codex request failed' + }` + ) + ); + return; + } + pending.resolve(message.result); + } + + private rejectAll(error: Error): void { + for (const pending of this.pending.values()) { + clearTimeout(pending.timer); + pending.reject(error); + } + this.pending.clear(); + for (const waiter of this.notificationWaiters) waiter.reject(error); + this.notificationWaiters.clear(); + } +} + +export function isRelayfileMount( + workspaceRoot: string +): { kind: 'relayfile-mount'; mountStatePath: string } | null { + const mountStatePath = path.join(workspaceRoot, '.relayfile-mount-state.json'); + return fs.existsSync(mountStatePath) ? { kind: 'relayfile-mount', mountStatePath } : null; +} diff --git a/packages/cli/src/cli/lib/codex-live-controller.test.ts b/packages/cli/src/cli/lib/codex-live-controller.test.ts new file mode 100644 index 000000000..41594ab02 --- /dev/null +++ b/packages/cli/src/cli/lib/codex-live-controller.test.ts @@ -0,0 +1,412 @@ +import { describe, expect, it, vi } from 'vitest'; + +import type { LiveTeleportCloudClient } from '@agent-relay/cloud'; + +import { + CodexLiveController, + type CodexControllerState, + type CodexControllerStateStore, +} from './codex-live-controller.js'; +import type { CodexAppServerSession } from './codex-app-server.js'; + +const source = { kind: 'relayfile-mount' as const, mountStatePath: '/repo/.relayfile-mount-state.json' }; +const convergence = { + verdict: 'converged' as const, + source: { + cursor: 'evt_10', + manifestSha256: 'a'.repeat(64), + files: 2, + bytes: 20, + conflictArtifacts: [], + conflictDigest: 'b'.repeat(64), + sealedAt: '2026-08-23T11:59:00.000Z', + }, + destination: { + cursor: 'evt_10', + manifestSha256: 'a'.repeat(64), + files: 2, + bytes: 20, + conflictArtifacts: [], + conflictDigest: 'b'.repeat(64), + pendingWriteback: 0 as const, + hasPendingWriteback: false as const, + outboxNeedsAttention: false as const, + ephemeralPaths: [] as [], + }, +}; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +function memoryStore(initial: CodexControllerState | null = null): CodexControllerStateStore & { + value: CodexControllerState | null; +} { + return { + value: initial, + read() { + return this.value ? structuredClone(this.value) : null; + }, + write(state) { + this.value = structuredClone(state); + }, + }; +} + +function appServer(overrides: Partial = {}): CodexAppServerSession { + return { + initialize: vi.fn(async () => undefined), + startThread: vi.fn(async () => 'thread-1'), + resumeThread: vi.fn(async () => undefined), + addEnvironment: vi.fn(async () => undefined), + environmentStatus: vi.fn(async () => ({ status: 'ready' })), + runTurn: vi.fn(async () => ({ + turnId: 'turn-1', + response: {}, + completed: { method: 'turn/completed' }, + })), + close: vi.fn(async () => undefined), + ...overrides, + }; +} + +function cloud(overrides: Partial = {}): LiveTeleportCloudClient { + return { + prewarm: vi.fn(async (input) => ({ + prewarmId: `prewarm-${input.generation}`, + generation: input.generation, + status: 'ready' as const, + })), + acquire: vi.fn(async (input) => ({ + sessionId: input.sessionId, + generation: input.generation, + environmentId: `environment-${input.generation}`, + execServerUrl: 'wss://exec.agentrelay.test/ticket', + workspaceCwd: '/workspace', + expiresAt: '2026-08-23T13:00:00.000Z', + convergence, + })), + revoke: vi.fn(async () => undefined), + ...overrides, + }; +} + +function createController( + options: { + store?: ReturnType; + cloud?: LiveTeleportCloudClient; + sessions?: CodexAppServerSession[]; + probe?: () => Promise; + } = {} +) { + const store = options.store ?? memoryStore(); + const cloudClient = options.cloud ?? cloud(); + const sessions = options.sessions ?? [appServer()]; + const createAppServer = vi.fn(async () => { + const session = sessions.shift(); + if (!session) throw new Error('no app server'); + return session; + }); + const controller = new CodexLiveController( + { workspaceRoot: '/repo', source, socketPath: '/state/controller.sock' }, + { + cloud: cloudClient, + store, + createAppServer, + probeCapability: options.probe ?? (async () => undefined), + now: () => new Date('2026-08-23T12:00:00.000Z'), + sessionId: () => 'session-1', + pid: 123, + } + ); + return { controller, store, cloud: cloudClient, createAppServer }; +} + +describe('CodexLiveController', () => { + it('fails closed before starting an app-server when the local experimental capability is unsupported', async () => { + const { controller, createAppServer } = createController({ + probe: async () => { + throw new Error('environment/status unsupported'); + }, + }); + + await expect(controller.initialize()).rejects.toThrow('environment/status unsupported'); + expect(createAppServer).not.toHaveBeenCalled(); + }); + + it('rejects a stale generation and makes duplicate request ids idempotent', async () => { + const { controller } = createController(); + await controller.initialize(); + + expect(() => controller.requestTeleport({ requestId: 'request-stale', expectedGeneration: 0 })).toThrow( + 'Stale teleport generation' + ); + const first = controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + const duplicate = controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + expect(first.phase).toBe('teleport_pending'); + expect(duplicate.pending?.requestId).toBe('request-1'); + }); + + it('queues a mid-turn request and applies it only at the following turn boundary', async () => { + const firstTurn = deferred<{ + turnId: string; + response: object; + completed: { method: string }; + }>(); + const session = appServer({ + runTurn: vi + .fn() + .mockImplementationOnce(() => firstTurn.promise) + .mockResolvedValueOnce({ turnId: 'turn-2', response: {}, completed: { method: 'turn/completed' } }), + }); + const cloudClient = cloud(); + const { controller } = createController({ sessions: [session], cloud: cloudClient }); + await controller.initialize(); + + const running = controller.runTurn('local turn'); + await vi.waitFor(() => expect(session.runTurn).toHaveBeenCalledTimes(1)); + const queued = controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + expect(queued.turnActive).toBe(true); + expect(queued.phase).toBe('teleport_pending'); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + + firstTurn.resolve({ turnId: 'turn-1', response: {}, completed: { method: 'turn/completed' } }); + await running; + expect(cloudClient.acquire).not.toHaveBeenCalled(); + + await controller.runTurn('remote turn'); + expect(cloudClient.acquire).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: 'session-1', + threadId: 'thread-1', + generation: 1, + source, + prewarmId: 'prewarm-1', + idempotencyKey: 'session-1:1:acquire', + }) + ); + expect(vi.mocked(session.addEnvironment).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(session.runTurn).mock.invocationCallOrder[1]! + ); + expect(session.runTurn).toHaveBeenLastCalledWith({ + threadId: 'thread-1', + text: 'remote turn', + environment: { environmentId: 'environment-1', cwd: '/workspace' }, + }); + }); + + it('keeps execution local when Cloud acquisition fails before turn/start', async () => { + const session = appServer(); + const cloudClient = cloud({ acquire: vi.fn(async () => Promise.reject(new Error('Cloud unavailable'))) }); + const { controller } = createController({ sessions: [session], cloud: cloudClient }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('must not run remotely')).rejects.toThrow('execution remains local'); + expect(session.addEnvironment).not.toHaveBeenCalled(); + expect(session.runTurn).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ execution: 'local', phase: 'local', turnActive: false }); + expect(cloudClient.revoke).toHaveBeenCalled(); + }); + + it('rejects a stale generation returned by Cloud', async () => { + const session = appServer(); + const cloudClient = cloud({ + acquire: vi.fn(async (input) => ({ + sessionId: input.sessionId, + generation: input.generation + 1, + environmentId: 'stale', + execServerUrl: 'wss://exec.agentrelay.test/stale', + workspaceCwd: '/workspace', + expiresAt: '2026-08-23T13:00:00.000Z', + convergence, + })), + }); + const { controller } = createController({ sessions: [session], cloud: cloudClient }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('no cross-generation execution')).rejects.toThrow( + 'stale or cross-session' + ); + expect(session.addEnvironment).not.toHaveBeenCalled(); + }); + + it('revokes and stays local when Codex cannot verify the environment ready', async () => { + const session = appServer({ + environmentStatus: vi.fn(async () => ({ status: 'connecting' })), + }); + const cloudClient = cloud(); + const { controller } = createController({ sessions: [session], cloud: cloudClient }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('must stay local')).rejects.toThrow( + 'did not report the Cloud execution environment ready' + ); + expect(session.runTurn).not.toHaveBeenCalled(); + expect(cloudClient.revoke).toHaveBeenCalledWith( + expect.objectContaining({ idempotencyKey: 'session-1:1:failed-acquire-revoke' }) + ); + expect(controller.status()).toMatchObject({ phase: 'local', execution: 'local' }); + }); + + it('uses Codex stickiness after explicitly attaching the first remote turn', async () => { + const session = appServer(); + const { controller } = createController({ sessions: [session] }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await controller.runTurn('first remote turn'); + await controller.runTurn('sticky remote turn'); + + expect(session.runTurn).toHaveBeenNthCalledWith(1, { + threadId: 'thread-1', + text: 'first remote turn', + environment: { environmentId: 'environment-1', cwd: '/workspace' }, + }); + expect(session.runTurn).toHaveBeenNthCalledWith(2, { + threadId: 'thread-1', + text: 'sticky remote turn', + }); + expect(controller.status().remote).not.toHaveProperty('execServerUrl'); + }); + + it('rolls back by restarting, initializing, and resuming the same thread locally', async () => { + const original = appServer(); + const replacement = appServer(); + const cloudClient = cloud(); + const { controller } = createController({ sessions: [original, replacement], cloud: cloudClient }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + await controller.runTurn('remote turn'); + + const status = await controller.rollback(); + + expect(original.close).toHaveBeenCalled(); + expect(replacement.initialize).toHaveBeenCalled(); + expect(replacement.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + expect(status).toMatchObject({ generation: 2, phase: 'local', execution: 'local' }); + await controller.runTurn('local again'); + expect(replacement.runTurn).toHaveBeenCalledWith({ threadId: 'thread-1', text: 'local again' }); + }); + + it('fails closed if rollback cannot resume the same thread', async () => { + const original = appServer(); + const replacement = appServer({ + resumeThread: vi.fn(async () => Promise.reject(new Error('thread missing'))), + }); + const { controller } = createController({ sessions: [original, replacement] }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + await controller.runTurn('remote turn'); + + await expect(controller.rollback()).rejects.toThrow('Could not resume Codex thread thread-1'); + expect(controller.status()).toMatchObject({ phase: 'recovery_failed', execution: 'local' }); + await expect(controller.runTurn('must not continue')).rejects.toThrow('recovery_failed'); + }); + + it('recovers persisted remote state locally on controller restart', async () => { + const store = memoryStore({ + version: 1, + sessionId: 'session-1', + threadId: 'thread-1', + workspaceRoot: '/repo', + source, + generation: 7, + phase: 'remote', + controllerPid: 99, + socketPath: '/old.sock', + turnActive: false, + remote: { + sessionId: 'session-1', + generation: 7, + environmentId: 'env-7', + execServerUrl: 'wss://exec.agentrelay.test/old', + workspaceCwd: '/workspace', + expiresAt: '2026-08-23T13:00:00.000Z', + attached: true, + convergence, + }, + updatedAt: '2026-08-23T11:00:00.000Z', + }); + const resumed = appServer(); + const cloudClient = cloud(); + const { controller } = createController({ store, sessions: [resumed], cloud: cloudClient }); + + await expect(controller.initialize()).resolves.toMatchObject({ + generation: 8, + phase: 'local', + execution: 'local', + threadId: 'thread-1', + }); + expect(cloudClient.revoke).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: 'session-1', generation: 7 }) + ); + expect(resumed.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + }); + + it('persists recovery_failed when restart cannot resume the thread', async () => { + const store = memoryStore({ + version: 1, + sessionId: 'session-1', + threadId: 'thread-1', + workspaceRoot: '/repo', + source, + generation: 7, + phase: 'local', + controllerPid: 99, + socketPath: '/old.sock', + turnActive: false, + updatedAt: '2026-08-23T11:00:00.000Z', + }); + const resumed = appServer({ resumeThread: vi.fn(async () => Promise.reject(new Error('gone'))) }); + const { controller } = createController({ store, sessions: [resumed] }); + + await expect(controller.initialize()).rejects.toThrow('Could not resume Codex thread thread-1'); + expect(store.value).toMatchObject({ phase: 'recovery_failed', generation: 7 }); + }); + + it('does not adopt a persisted thread from a different workspace', async () => { + const store = memoryStore({ + version: 1, + sessionId: 'session-1', + threadId: 'thread-other', + workspaceRoot: '/different-repo', + source, + generation: 3, + phase: 'local', + controllerPid: 99, + socketPath: '/old.sock', + turnActive: false, + updatedAt: '2026-08-23T11:00:00.000Z', + }); + const session = appServer(); + const { controller, createAppServer } = createController({ store, sessions: [session] }); + + await expect(controller.initialize()).rejects.toThrow('belongs to /different-repo'); + expect(createAppServer).not.toHaveBeenCalled(); + }); + + it('revokes the exact generation on shutdown so prewarms and remote boxes cannot leak', async () => { + const session = appServer(); + const cloudClient = cloud(); + const { controller } = createController({ sessions: [session], cloud: cloudClient }); + await controller.initialize(); + + await controller.close(); + + expect(cloudClient.revoke).toHaveBeenCalledWith({ + sessionId: 'session-1', + generation: 1, + idempotencyKey: 'session-1:1:shutdown-revoke', + }); + expect(session.close).toHaveBeenCalled(); + }); +}); diff --git a/packages/cli/src/cli/lib/codex-live-controller.ts b/packages/cli/src/cli/lib/codex-live-controller.ts new file mode 100644 index 000000000..762a21bdf --- /dev/null +++ b/packages/cli/src/cli/lib/codex-live-controller.ts @@ -0,0 +1,451 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { randomUUID } from 'node:crypto'; + +import type { + LiveTeleportCloudClient, + LiveTeleportEnvironment, + LiveTeleportWorkspaceSource, +} from '@agent-relay/cloud'; + +import type { CodexAppServerSession, CodexTurnResult } from './codex-app-server.js'; + +export type CodexControllerPhase = + | 'local' + | 'teleport_pending' + | 'remote' + | 'rolling_back' + | 'failed' + | 'recovery_failed'; + +export type CodexTeleportRequest = { + requestId: string; + expectedGeneration: number; +}; + +export type CodexControllerState = { + version: 1; + sessionId: string; + threadId: string; + workspaceRoot: string; + source: LiveTeleportWorkspaceSource; + generation: number; + phase: CodexControllerPhase; + controllerPid: number; + socketPath: string; + turnActive: boolean; + pending?: CodexTeleportRequest; + lastRequestId?: string; + prewarmId?: string; + prewarmStatus?: 'warming' | 'ready' | 'failed'; + remote?: Omit & { + /** Stored privately for controller recovery; never returned from public status. */ + execServerUrl: string; + attached: boolean; + }; + lastError?: string; + updatedAt: string; +}; + +export type PublicCodexControllerStatus = Omit & { + execution: 'local' | 'cloud'; + controller: 'local'; + remote?: Pick & { + attached: boolean; + }; + workspaceSource: LiveTeleportWorkspaceSource['kind']; +}; + +export interface CodexControllerStateStore { + read(): CodexControllerState | null; + write(state: CodexControllerState): void; +} + +export class FileCodexControllerStateStore implements CodexControllerStateStore { + constructor(readonly filePath: string) {} + + read(): CodexControllerState | null { + try { + return JSON.parse(fs.readFileSync(this.filePath, 'utf8')) as CodexControllerState; + } catch (error) { + if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return null; + throw error; + } + } + + write(state: CodexControllerState): void { + fs.mkdirSync(path.dirname(this.filePath), { recursive: true, mode: 0o700 }); + const temporary = `${this.filePath}.tmp-${process.pid}-${randomUUID()}`; + fs.writeFileSync(temporary, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 }); + fs.renameSync(temporary, this.filePath); + fs.chmodSync(this.filePath, 0o600); + } +} + +export type CodexLiveControllerDependencies = { + cloud: LiveTeleportCloudClient; + store: CodexControllerStateStore; + createAppServer: () => Promise; + probeCapability: () => Promise; + now: () => Date; + sessionId: () => string; + pid: number; +}; + +export type CodexLiveControllerOptions = { + workspaceRoot: string; + source: LiveTeleportWorkspaceSource; + socketPath: string; + model?: string; +}; + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + +function isReadyStatus(value: unknown): boolean { + if (typeof value === 'string') return /^(?:ready|connected|active)$/i.test(value); + if (value && typeof value === 'object') { + for (const [key, nested] of Object.entries(value)) { + if ((key === 'status' || key === 'state') && isReadyStatus(nested)) return true; + } + } + return false; +} + +function publicStatus(state: CodexControllerState): PublicCodexControllerStatus { + const { source, remote, ...rest } = state; + return { + ...rest, + controller: 'local', + execution: remote && state.phase === 'remote' ? 'cloud' : 'local', + workspaceSource: source.kind, + ...(remote + ? { + remote: { + environmentId: remote.environmentId, + generation: remote.generation, + workspaceCwd: remote.workspaceCwd, + expiresAt: remote.expiresAt, + attached: remote.attached, + }, + } + : {}), + }; +} + +/** + * Owns one local Codex app-server/thread. There is exactly one mutation seam: + * a queued teleport is consumed immediately before a turn/start request. + */ +export class CodexLiveController { + private appServer: CodexAppServerSession | null = null; + private state: CodexControllerState | null = null; + + constructor( + private readonly options: CodexLiveControllerOptions, + private readonly deps: CodexLiveControllerDependencies + ) {} + + async initialize(): Promise { + await this.deps.probeCapability(); + const persisted = this.deps.store.read(); + if (persisted && path.resolve(persisted.workspaceRoot) !== path.resolve(this.options.workspaceRoot)) { + throw new Error( + `The persisted managed Codex thread belongs to ${persisted.workspaceRoot}, not ${this.options.workspaceRoot}.` + ); + } + this.appServer = await this.deps.createAppServer(); + await this.appServer.initialize(); + + if (persisted) { + this.state = { + ...persisted, + controllerPid: this.deps.pid, + socketPath: this.options.socketPath, + workspaceRoot: this.options.workspaceRoot, + source: this.options.source, + turnActive: false, + phase: 'rolling_back', + pending: undefined, + lastError: undefined, + updatedAt: this.timestamp(), + }; + this.persist(); + + if (persisted.remote) { + await this.deps.cloud + .revoke({ + sessionId: persisted.sessionId, + generation: persisted.generation, + idempotencyKey: `${persisted.sessionId}:${persisted.generation}:restart-revoke`, + }) + .catch(() => undefined); + } + + try { + await this.appServer.resumeThread({ + threadId: persisted.threadId, + cwd: this.options.workspaceRoot, + }); + } catch (error) { + this.state.phase = 'recovery_failed'; + this.state.lastError = `Could not resume Codex thread ${persisted.threadId}: ${errorMessage(error)}`; + this.state.remote = undefined; + this.state.updatedAt = this.timestamp(); + this.persist(); + throw new Error(this.state.lastError, { cause: error }); + } + + this.state.generation = persisted.generation + 1; + this.state.phase = 'local'; + this.state.remote = undefined; + this.state.updatedAt = this.timestamp(); + this.persist(); + } else { + const threadId = await this.appServer.startThread({ + cwd: this.options.workspaceRoot, + ...(this.options.model ? { model: this.options.model } : {}), + }); + this.state = { + version: 1, + sessionId: this.deps.sessionId(), + threadId, + workspaceRoot: this.options.workspaceRoot, + source: this.options.source, + generation: 1, + phase: 'local', + controllerPid: this.deps.pid, + socketPath: this.options.socketPath, + turnActive: false, + updatedAt: this.timestamp(), + }; + this.persist(); + } + + await this.startPrewarm(); + return this.status(); + } + + status(): PublicCodexControllerStatus { + return publicStatus(this.requireState()); + } + + requestTeleport(request: CodexTeleportRequest): PublicCodexControllerStatus { + const state = this.requireState(); + if (request.expectedGeneration !== state.generation) { + throw new Error( + `Stale teleport generation ${request.expectedGeneration}; active generation is ${state.generation}.` + ); + } + if (state.lastRequestId === request.requestId || state.pending?.requestId === request.requestId) { + return this.status(); + } + if (state.phase === 'remote') throw new Error('This managed Codex session already executes in Cloud.'); + if (state.pending) throw new Error(`Teleport request ${state.pending.requestId} is already pending.`); + if (state.phase !== 'local') + throw new Error(`Cannot queue a teleport while the controller is ${state.phase}.`); + + state.pending = request; + state.phase = 'teleport_pending'; + state.lastRequestId = request.requestId; + state.updatedAt = this.timestamp(); + this.persist(); + return this.status(); + } + + async runTurn(text: string): Promise { + const state = this.requireState(); + const appServer = this.requireAppServer(); + if (state.turnActive) throw new Error('A Codex turn is already active.'); + if (state.phase === 'recovery_failed' || state.phase === 'failed' || state.phase === 'rolling_back') { + throw new Error(`Cannot start a turn while the controller is ${state.phase}.`); + } + + // This is the concrete turn boundary: snapshot the pending request before + // marking the turn active. A request arriving after this snapshot is + // persisted for the following invocation, never spliced into this turn. + const pendingAtBoundary = state.pending; + state.turnActive = true; + state.updatedAt = this.timestamp(); + this.persist(); + try { + if (pendingAtBoundary) await this.applyPendingTeleport(pendingAtBoundary); + const remote = state.phase === 'remote' ? state.remote : undefined; + const result = await appServer.runTurn({ + threadId: state.threadId, + text, + ...(remote && !remote.attached + ? { environment: { environmentId: remote.environmentId, cwd: remote.workspaceCwd } } + : {}), + }); + if (remote && !remote.attached) remote.attached = true; + return result; + } finally { + state.turnActive = false; + state.updatedAt = this.timestamp(); + this.persist(); + } + } + + async rollback(): Promise { + const state = this.requireState(); + if (state.turnActive) throw new Error('Rollback is only allowed at a Codex turn boundary.'); + const old = this.requireAppServer(); + state.phase = 'rolling_back'; + state.pending = undefined; + state.updatedAt = this.timestamp(); + this.persist(); + + if (state.remote) { + await this.deps.cloud.revoke({ + sessionId: state.sessionId, + generation: state.generation, + idempotencyKey: `${state.sessionId}:${state.generation}:rollback-revoke`, + }); + } + + await old.close(); + const replacement = await this.deps.createAppServer(); + this.appServer = replacement; + await replacement.initialize(); + try { + await replacement.resumeThread({ threadId: state.threadId, cwd: state.workspaceRoot }); + } catch (error) { + state.phase = 'recovery_failed'; + state.remote = undefined; + state.lastError = `Could not resume Codex thread ${state.threadId}: ${errorMessage(error)}`; + state.updatedAt = this.timestamp(); + this.persist(); + throw new Error(state.lastError, { cause: error }); + } + + state.generation += 1; + state.phase = 'local'; + state.remote = undefined; + state.lastError = undefined; + state.updatedAt = this.timestamp(); + this.persist(); + await this.startPrewarm(); + return this.status(); + } + + async close(): Promise { + const state = this.state; + if (state) { + await this.deps.cloud + .revoke({ + sessionId: state.sessionId, + generation: state.generation, + idempotencyKey: `${state.sessionId}:${state.generation}:shutdown-revoke`, + }) + .catch((error) => { + state.lastError = `Cloud shutdown revoke failed; the generation must expire server-side: ${errorMessage( + error + )}`; + state.updatedAt = this.timestamp(); + this.persist(); + }); + } + await this.appServer?.close(); + this.appServer = null; + } + + private async applyPendingTeleport(pendingAtBoundary: CodexTeleportRequest): Promise { + const state = this.requireState(); + const pending = state.pending; + if (!pending || pending.requestId !== pendingAtBoundary.requestId) return; + if (pending.expectedGeneration !== state.generation) { + throw new Error('Pending teleport generation became stale before the turn boundary.'); + } + + try { + const environment = await this.deps.cloud.acquire({ + sessionId: state.sessionId, + threadId: state.threadId, + generation: state.generation, + workspaceRoot: state.workspaceRoot, + source: state.source, + ...(state.prewarmId ? { prewarmId: state.prewarmId } : {}), + idempotencyKey: `${state.sessionId}:${state.generation}:acquire`, + }); + if (environment.sessionId !== state.sessionId || environment.generation !== state.generation) { + throw new Error('Cloud returned a stale or cross-session live-teleport generation.'); + } + + await this.requireAppServer().addEnvironment({ + environmentId: environment.environmentId, + execServerUrl: environment.execServerUrl, + connectTimeoutMs: 10_000, + }); + const environmentStatus = await this.requireAppServer().environmentStatus(environment.environmentId); + if (!isReadyStatus(environmentStatus)) { + throw new Error('Codex did not report the Cloud execution environment ready.'); + } + + state.remote = { ...environment, attached: false }; + state.pending = undefined; + state.phase = 'remote'; + state.lastError = undefined; + state.updatedAt = this.timestamp(); + this.persist(); + } catch (error) { + await this.deps.cloud + .revoke({ + sessionId: state.sessionId, + generation: state.generation, + idempotencyKey: `${state.sessionId}:${state.generation}:failed-acquire-revoke`, + }) + .catch(() => undefined); + state.pending = undefined; + state.phase = 'local'; + state.remote = undefined; + state.lastError = `Teleport failed before turn/start; execution remains local: ${errorMessage(error)}`; + state.updatedAt = this.timestamp(); + this.persist(); + throw new Error(state.lastError, { cause: error }); + } + } + + private async startPrewarm(): Promise { + const state = this.requireState(); + try { + const prewarm = await this.deps.cloud.prewarm({ + sessionId: state.sessionId, + generation: state.generation, + workspaceRoot: state.workspaceRoot, + source: state.source, + idempotencyKey: `${state.sessionId}:${state.generation}:prewarm`, + }); + if (prewarm.generation !== state.generation) { + throw new Error('Cloud returned a stale prewarm generation.'); + } + state.prewarmId = prewarm.prewarmId; + state.prewarmStatus = prewarm.status; + } catch (error) { + state.prewarmId = undefined; + state.prewarmStatus = 'failed'; + state.lastError = `Cloud prewarm unavailable; local Codex remains usable: ${errorMessage(error)}`; + } + state.updatedAt = this.timestamp(); + this.persist(); + } + + private persist(): void { + this.deps.store.write(this.requireState()); + } + + private timestamp(): string { + return this.deps.now().toISOString(); + } + + private requireState(): CodexControllerState { + if (!this.state) throw new Error('Codex controller is not initialized.'); + return this.state; + } + + private requireAppServer(): CodexAppServerSession { + if (!this.appServer) throw new Error('Codex app-server is not initialized.'); + return this.appServer; + } +} diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index 4f7f5f107..774078b80 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -53,6 +53,19 @@ export { type CloudApiClientSnapshot, } from './api-client.js'; +export { + CloudLiveTeleportClient, + type LiveTeleportCloudClient, + type LiveTeleportWorkspaceSource, + type LiveTeleportPrewarmInput, + type LiveTeleportPrewarm, + type LiveTeleportAcquireInput, + type LiveTeleportEnvironment, + type LiveTeleportConvergenceWatermark, + type LiveTeleportConvergenceProof, + type LiveTeleportRevokeInput, +} from './live-teleport.js'; + export { runWorkflow, scheduleWorkflow, diff --git a/packages/cloud/src/live-teleport.test.ts b/packages/cloud/src/live-teleport.test.ts new file mode 100644 index 000000000..72ebfb013 --- /dev/null +++ b/packages/cloud/src/live-teleport.test.ts @@ -0,0 +1,141 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { CloudLiveTeleportClient } from './live-teleport.js'; + +const input = { + sessionId: 'session-1', + threadId: 'thread-1', + generation: 2, + workspaceRoot: '/workspace', + source: { kind: 'relayfile-mount' as const, mountStatePath: '/workspace/.relayfile-mount-state.json' }, + idempotencyKey: 'session-1:2:acquire', +}; + +function convergence(overrides: { destinationSha?: string; pendingWriteback?: number } = {}) { + const watermark = { + cursor: 'evt_19312', + manifestSha256: 'a'.repeat(64), + files: 12, + bytes: 2048, + conflictArtifacts: ['.relay/conflicts/shared.txt.writer-b'], + conflictDigest: 'b'.repeat(64), + }; + return { + verdict: 'converged', + source: { ...watermark, sealedAt: '2026-08-23T11:59:00.000Z' }, + destination: { + ...watermark, + cursor: 'evt_19313', + manifestSha256: overrides.destinationSha ?? watermark.manifestSha256, + pendingWriteback: overrides.pendingWriteback ?? 0, + hasPendingWriteback: false, + outboxNeedsAttention: false, + ephemeralPaths: [], + }, + }; +} + +describe('CloudLiveTeleportClient', () => { + it('accepts only the provider-neutral Cloud WSS bridge contract', async () => { + const fetcher = vi.fn(async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + environmentId: 'env-2', + execServerUrl: 'wss://exec.agentrelay.test/t/session-1/g/2', + workspaceCwd: '/workspace', + expiresAt: '2026-08-23T12:00:00.000Z', + convergence: convergence(), + }) + ); + + await expect(new CloudLiveTeleportClient(fetcher).acquire(input)).resolves.toMatchObject({ + environmentId: 'env-2', + generation: 2, + }); + expect(fetcher).toHaveBeenCalledWith( + '/api/v1/live-teleports/acquire', + expect.objectContaining({ method: 'POST' }) + ); + }); + + it('fails closed if Cloud exposes a provider credential or URL', async () => { + const client = new CloudLiveTeleportClient(async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + environmentId: 'env-2', + execServerUrl: 'wss://exec.agentrelay.test/ticket', + workspaceCwd: '/workspace', + expiresAt: '2026-08-23T12:00:00.000Z', + convergence: convergence(), + providerUrl: 'wss://provider.invalid/raw', + trafficAccessToken: 'secret', + }) + ); + + await expect(client.acquire(input)).rejects.toThrow('forbidden provider field'); + }); + + it('rejects a non-TLS execution address', async () => { + const client = new CloudLiveTeleportClient(async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + environmentId: 'env-2', + execServerUrl: 'ws://127.0.0.1:4500', + workspaceCwd: '/workspace', + expiresAt: '2026-08-23T12:00:00.000Z', + convergence: convergence(), + }) + ); + + await expect(client.acquire(input)).rejects.toThrow('Cloud WSS bridge'); + }); + + it('rejects a time-based convergence claim whose destination hash differs', async () => { + const client = new CloudLiveTeleportClient(async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + environmentId: 'env-2', + execServerUrl: 'wss://exec.agentrelay.test/ticket', + workspaceCwd: '/workspace', + expiresAt: '2026-08-23T12:00:00.000Z', + convergence: convergence({ destinationSha: 'f'.repeat(64) }), + }) + ); + + await expect(client.acquire(input)).rejects.toThrow('non-converged hash/cursor proof'); + }); + + it('rejects matching hashes when the destination outbox is not drained', async () => { + const client = new CloudLiveTeleportClient(async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + environmentId: 'env-2', + execServerUrl: 'wss://exec.agentrelay.test/ticket', + workspaceCwd: '/workspace', + expiresAt: '2026-08-23T12:00:00.000Z', + convergence: convergence({ pendingWriteback: 1 }), + }) + ); + + await expect(client.acquire(input)).rejects.toThrow('non-converged hash/cursor proof'); + }); + + it('does not echo a Cloud error containing an opaque bridge or provider secret', async () => { + const client = new CloudLiveTeleportClient(async () => + Response.json( + { error: 'provider rejected https://provider.invalid/?token=opaque-secret-value' }, + { status: 502, statusText: 'Bad Gateway' } + ) + ); + + const error = await client.acquire(input).catch((caught: unknown) => caught); + expect(String(error)).toContain('Bad Gateway'); + expect(String(error)).not.toContain('opaque-secret-value'); + expect(String(error)).not.toContain('provider.invalid'); + }); +}); diff --git a/packages/cloud/src/live-teleport.ts b/packages/cloud/src/live-teleport.ts new file mode 100644 index 000000000..1510e722b --- /dev/null +++ b/packages/cloud/src/live-teleport.ts @@ -0,0 +1,291 @@ +export type LiveTeleportWorkspaceSource = + | { + kind: 'relayfile-mount'; + mountStatePath: string; + } + | { + kind: 'verified-convergence-receipt'; + receipt: string; + }; + +export type LiveTeleportPrewarmInput = { + sessionId: string; + generation: number; + workspaceRoot: string; + source: LiveTeleportWorkspaceSource; + idempotencyKey: string; +}; + +export type LiveTeleportPrewarm = { + prewarmId: string; + generation: number; + status: 'warming' | 'ready'; +}; + +export type LiveTeleportAcquireInput = LiveTeleportPrewarmInput & { + threadId: string; + prewarmId?: string; +}; + +export type LiveTeleportConvergenceWatermark = { + cursor: string; + manifestSha256: string; + files: number; + bytes: number; + conflictArtifacts: string[]; + conflictDigest: string; +}; + +export type LiveTeleportConvergenceProof = { + verdict: 'converged'; + source: LiveTeleportConvergenceWatermark & { sealedAt: string }; + destination: LiveTeleportConvergenceWatermark & { + pendingWriteback: 0; + hasPendingWriteback: false; + outboxNeedsAttention: false; + ephemeralPaths: []; + }; +}; + +export type LiveTeleportEnvironment = { + sessionId: string; + generation: number; + environmentId: string; + execServerUrl: string; + workspaceCwd: string; + expiresAt: string; + convergence: LiveTeleportConvergenceProof; +}; + +export type LiveTeleportRevokeInput = { + sessionId: string; + generation: number; + idempotencyKey: string; +}; + +export interface LiveTeleportCloudClient { + prewarm(input: LiveTeleportPrewarmInput): Promise; + acquire(input: LiveTeleportAcquireInput): Promise; + revoke(input: LiveTeleportRevokeInput): Promise; +} + +type Fetcher = (path: string, init?: RequestInit) => Promise; + +const FORBIDDEN_PROVIDER_FIELD = + /(?:provider.*(?:url|token|credential)|trafficAccessToken|signedPreviewUrl)/i; + +function isObject(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function assertNoProviderSecrets(value: unknown, path = 'response'): void { + if (Array.isArray(value)) { + value.forEach((entry, index) => assertNoProviderSecrets(entry, `${path}[${index}]`)); + return; + } + if (!isObject(value)) return; + + for (const [key, entry] of Object.entries(value)) { + if (FORBIDDEN_PROVIDER_FIELD.test(key)) { + throw new Error(`Cloud live-teleport response exposed forbidden provider field ${path}.${key}.`); + } + assertNoProviderSecrets(entry, `${path}.${key}`); + } +} + +async function readPayload(response: Response): Promise { + const payload = (await response.json().catch(() => null)) as unknown; + assertNoProviderSecrets(payload); + if (!response.ok) { + // Cloud error prose can accidentally interpolate an opaque ticket or + // provider URL. Only a short machine code is safe to relay to callers. + const detail = + isObject(payload) && typeof payload.code === 'string' && /^[A-Z0-9_]{1,64}$/.test(payload.code) + ? payload.code + : response.statusText; + throw new Error(`Cloud live-teleport request failed (${response.status}): ${detail || 'unknown error'}`); + } + return payload; +} + +function requiredString(value: unknown, field: string): string { + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`Cloud live-teleport response is missing ${field}.`); + } + return value.trim(); +} + +function requiredGeneration(value: unknown): number { + if (!Number.isSafeInteger(value) || Number(value) < 1) { + throw new Error('Cloud live-teleport response has an invalid generation.'); + } + return Number(value); +} + +function requiredNonNegativeInteger(value: unknown, field: string): number { + if (!Number.isSafeInteger(value) || Number(value) < 0) { + throw new Error(`Cloud live-teleport convergence proof has an invalid ${field}.`); + } + return Number(value); +} + +function requiredStringArray(value: unknown, field: string): string[] { + if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) { + throw new Error(`Cloud live-teleport convergence proof has an invalid ${field}.`); + } + return value as string[]; +} + +function requiredDigest(value: unknown, field: string): string { + const digest = requiredString(value, field); + if (!/^[a-f0-9]{64}$/i.test(digest)) { + throw new Error(`Cloud live-teleport convergence proof has an invalid ${field}.`); + } + return digest.toLowerCase(); +} + +function requiredWatermark(value: unknown, field: string): LiveTeleportConvergenceWatermark { + if (!isObject(value)) throw new Error(`Cloud live-teleport convergence proof is missing ${field}.`); + return { + cursor: requiredString(value.cursor, `${field}.cursor`), + manifestSha256: requiredDigest(value.manifestSha256, `${field}.manifestSha256`), + files: requiredNonNegativeInteger(value.files, `${field}.files`), + bytes: requiredNonNegativeInteger(value.bytes, `${field}.bytes`), + conflictArtifacts: requiredStringArray(value.conflictArtifacts, `${field}.conflictArtifacts`), + conflictDigest: requiredDigest(value.conflictDigest, `${field}.conflictDigest`), + }; +} + +function parseCounter(value: string): { prefix: string; ordinal: number } | null { + const match = /^([A-Za-z][A-Za-z0-9]*_)?(\d+)$/.exec(value); + if (!match) return null; + const ordinal = Number.parseInt(match[2]!, 10); + return Number.isSafeInteger(ordinal) ? { prefix: match[1] ?? '', ordinal } : null; +} + +function sameStrings(left: readonly string[], right: readonly string[]): boolean { + return left.length === right.length && left.every((entry, index) => entry === right[index]); +} + +function requiredConvergenceProof(value: unknown): LiveTeleportConvergenceProof { + if (!isObject(value) || value.verdict !== 'converged') { + throw new Error('Cloud live-teleport acquire did not return a converged hash/cursor proof.'); + } + const source = requiredWatermark(value.source, 'source'); + const destination = requiredWatermark(value.destination, 'destination'); + const sourceObject = value.source as Record; + const destinationObject = value.destination as Record; + const sealedAt = requiredString(sourceObject.sealedAt, 'source.sealedAt'); + if (Number.isNaN(Date.parse(sealedAt))) { + throw new Error('Cloud live-teleport convergence proof has an invalid source.sealedAt.'); + } + const sourceCursor = parseCounter(source.cursor); + const destinationCursor = parseCounter(destination.cursor); + const sameCursorNamespace = + sourceCursor && destinationCursor && sourceCursor.prefix === destinationCursor.prefix; + const hashesMatch = + source.manifestSha256 === destination.manifestSha256 && + source.files === destination.files && + source.bytes === destination.bytes && + sameStrings([...source.conflictArtifacts].sort(), [...destination.conflictArtifacts].sort()) && + source.conflictDigest === destination.conflictDigest; + const outboxHealthy = + destinationObject.pendingWriteback === 0 && + destinationObject.hasPendingWriteback === false && + destinationObject.outboxNeedsAttention === false && + Array.isArray(destinationObject.ephemeralPaths) && + destinationObject.ephemeralPaths.length === 0; + if ( + !sameCursorNamespace || + destinationCursor.ordinal < sourceCursor.ordinal || + !hashesMatch || + !outboxHealthy + ) { + throw new Error('Cloud live-teleport acquire returned a non-converged hash/cursor proof.'); + } + return { + verdict: 'converged', + source: { ...source, sealedAt }, + destination: { + ...destination, + pendingWriteback: 0, + hasPendingWriteback: false, + outboxNeedsAttention: false, + ephemeralPaths: [], + }, + }; +} + +/** + * Provider-neutral Cloud control-plane client. The only execution address Relay + * accepts is Cloud's short-lived WSS bridge; raw provider URLs and credentials + * are rejected even if a buggy server includes them in an otherwise-valid body. + */ +export class CloudLiveTeleportClient implements LiveTeleportCloudClient { + constructor(private readonly fetcher: Fetcher) {} + + async prewarm(input: LiveTeleportPrewarmInput): Promise { + const response = await this.fetcher('/api/v1/live-teleports/prewarm', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(input), + }); + const payload = await readPayload(response); + if (!isObject(payload)) throw new Error('Cloud live-teleport prewarm returned an invalid response.'); + + const status = payload.status; + if (status !== 'warming' && status !== 'ready') { + throw new Error('Cloud live-teleport prewarm returned an invalid status.'); + } + return { + prewarmId: requiredString(payload.prewarmId, 'prewarmId'), + generation: requiredGeneration(payload.generation), + status, + }; + } + + async acquire(input: LiveTeleportAcquireInput): Promise { + const response = await this.fetcher('/api/v1/live-teleports/acquire', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(input), + }); + const payload = await readPayload(response); + if (!isObject(payload)) throw new Error('Cloud live-teleport acquire returned an invalid response.'); + + const execServerUrl = requiredString(payload.execServerUrl, 'execServerUrl'); + let parsed: URL; + try { + parsed = new URL(execServerUrl); + } catch { + throw new Error('Cloud live-teleport acquire returned an invalid execServerUrl.'); + } + if (parsed.protocol !== 'wss:') { + throw new Error('Cloud live-teleport acquire must return a Cloud WSS bridge URL.'); + } + + const expiresAt = requiredString(payload.expiresAt, 'expiresAt'); + if (Number.isNaN(Date.parse(expiresAt))) { + throw new Error('Cloud live-teleport acquire returned an invalid expiresAt.'); + } + + return { + sessionId: requiredString(payload.sessionId, 'sessionId'), + generation: requiredGeneration(payload.generation), + environmentId: requiredString(payload.environmentId, 'environmentId'), + execServerUrl, + workspaceCwd: requiredString(payload.workspaceCwd, 'workspaceCwd'), + expiresAt, + convergence: requiredConvergenceProof(payload.convergence), + }; + } + + async revoke(input: LiveTeleportRevokeInput): Promise { + const response = await this.fetcher('/api/v1/live-teleports/revoke', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(input), + }); + await readPayload(response); + } +} From 76426bae8992bb79830fbe121d5e982fe1d3d4f7 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Sun, 23 Aug 2026 17:53:16 +0200 Subject: [PATCH 02/16] fix(codex): fence live teleport recovery --- CHANGELOG.md | 2 +- packages/cli/src/cli/commands/codex.test.ts | 70 +- packages/cli/src/cli/commands/codex.ts | 120 +-- .../cli/src/cli/lib/codex-app-server.test.ts | 232 +++++- packages/cli/src/cli/lib/codex-app-server.ts | 310 ++++++-- .../src/cli/lib/codex-live-controller.test.ts | 595 ++++++++++----- .../cli/src/cli/lib/codex-live-controller.ts | 720 ++++++++++++++---- .../src/cli/lib/codex-relayfile-seal.test.ts | 186 +++++ .../cli/src/cli/lib/codex-relayfile-seal.ts | 287 +++++++ packages/cloud/src/index.ts | 3 + packages/cloud/src/live-teleport.test.ts | 178 +++-- packages/cloud/src/live-teleport.ts | 174 ++++- 12 files changed, 2314 insertions(+), 563 deletions(-) create mode 100644 packages/cli/src/cli/lib/codex-relayfile-seal.test.ts create mode 100644 packages/cli/src/cli/lib/codex-relayfile-seal.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 6aa5b3ce1..1dc5813a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `relay codex run` starts a Relay-managed local Codex app-server/thread, and `relay codex teleport` queues Cloud execution for its next turn boundary with persisted generations and local-resume rollback. +- `relay codex run` starts a Relay-managed local Codex app-server/thread, and `relay codex teleport` seals its active Relayfile poll mount at the next turn boundary, verifies the first Cloud turn through Relay's gateway, and requires confirmed fencing plus mount readiness before local rollback. ## [Unreleased - Patch] diff --git a/packages/cli/src/cli/commands/codex.test.ts b/packages/cli/src/cli/commands/codex.test.ts index 1b92d2e27..8b68f8f58 100644 --- a/packages/cli/src/cli/commands/codex.test.ts +++ b/packages/cli/src/cli/commands/codex.test.ts @@ -1,45 +1,45 @@ -import fs from 'node:fs'; -import os from 'node:os'; -import path from 'node:path'; import { Command } from 'commander'; -import { afterEach, describe, expect, it, vi } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; -import { registerCodexCommands, resolveLiveTeleportWorkspaceSource } from './codex.js'; +import { CodexTurnRecoveredError } from '../lib/codex-live-controller.js'; +import { registerCodexCommands, runManagedCodexTurn } from './codex.js'; -const temporary: string[] = []; - -afterEach(() => { - temporary.splice(0).forEach((directory) => fs.rmSync(directory, { recursive: true, force: true })); -}); - -describe('resolveLiveTeleportWorkspaceSource', () => { - it('fails closed for a plain unmanaged or Git-only cwd', () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-codex-unmanaged-')); - temporary.push(root); - fs.mkdirSync(path.join(root, '.git')); - - expect(() => resolveLiveTeleportWorkspaceSource(root)).toThrow('fails closed for an unmanaged'); - }); +describe('runManagedCodexTurn', () => { + it('reports a recovered acquire failure and accepts the next input on the same controller', async () => { + const runTurn = vi + .fn() + .mockRejectedValueOnce(new CodexTurnRecoveredError('acquire failed but recovered')) + .mockResolvedValueOnce({}); + const controller = { + runTurn, + status: vi.fn(() => ({ + phase: 'local' as const, + threadId: 'thread-1', + generation: 2, + })), + }; + const writeError = vi.fn(); - it('recognizes a Relayfile mount', () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-codex-relayfile-')); - temporary.push(root); - const mountStatePath = path.join(root, '.relayfile-mount-state.json'); - fs.writeFileSync(mountStatePath, '{}'); + await runManagedCodexTurn(controller as never, 'first input', { json: false, writeError }); + await runManagedCodexTurn(controller as never, 'second input', { json: false, writeError }); - expect(resolveLiveTeleportWorkspaceSource(root)).toEqual({ kind: 'relayfile-mount', mountStatePath }); + expect(runTurn).toHaveBeenNthCalledWith(1, 'first input'); + expect(runTurn).toHaveBeenNthCalledWith(2, 'second input'); + expect(writeError).toHaveBeenCalledWith(expect.stringContaining('recovered locally')); }); - it('accepts an explicit opaque convergence receipt for Cloud verification', () => { - const root = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-codex-receipt-')); - temporary.push(root); - const receipt = path.join(root, 'receipt.jwt'); - fs.writeFileSync(receipt, 'signed.convergence.receipt\n'); + it('fails closed instead of continuing when fencing is unconfirmed', async () => { + const controller = { + runTurn: vi.fn(async () => Promise.reject(new Error('revoke unconfirmed'))), + status: vi.fn(() => ({ phase: 'fenced', threadId: 'thread-1', generation: 1 })), + }; - expect(resolveLiveTeleportWorkspaceSource(root, receipt)).toEqual({ - kind: 'verified-convergence-receipt', - receipt: 'signed.convergence.receipt', - }); + await expect( + runManagedCodexTurn(controller as never, 'must not continue', { + json: false, + writeError: vi.fn(), + }) + ).rejects.toThrow('cannot continue'); }); }); @@ -62,7 +62,7 @@ describe('registerCodexCommands', () => { updatedAt: '2026-08-23T12:00:00.000Z', controller: 'local' as const, execution: 'local' as const, - workspaceSource: 'relayfile-mount' as const, + workspaceSource: 'relayfile-checkpoint-seal' as const, }, })); const log = vi.fn(); diff --git a/packages/cli/src/cli/commands/codex.ts b/packages/cli/src/cli/commands/codex.ts index 1fac14f61..37c23013b 100644 --- a/packages/cli/src/cli/commands/codex.ts +++ b/packages/cli/src/cli/commands/codex.ts @@ -5,25 +5,24 @@ import path from 'node:path'; import readline from 'node:readline'; import { randomUUID } from 'node:crypto'; -import { - CloudLiveTeleportClient, - ensureCloudSession, - type LiveTeleportWorkspaceSource, -} from '@agent-relay/cloud'; +import { CloudLiveTeleportClient, ensureCloudSession } from '@agent-relay/cloud'; import { Command } from 'commander'; import { - isRelayfileMount, probeCodexEnvironmentCapability, StdioCodexAppServerSession, type CodexNotification, } from '../lib/codex-app-server.js'; import { CodexLiveController, + CodexTurnRecoveredError, FileCodexControllerStateStore, type CodexControllerState, + type CodexPersistedMountResumeProvider, + type CodexWorkspaceSealProvider, type PublicCodexControllerStatus, } from '../lib/codex-live-controller.js'; +import { createRelayfileSealLifecycle } from '../lib/codex-relayfile-seal.js'; export type CodexControllerPaths = { directory: string; @@ -39,13 +38,9 @@ type ControlRequest = type ControlResponse = { ok: true; status: PublicCodexControllerStatus } | { ok: false; error: string }; export interface CodexCommandDependencies { - runManaged(options: { - cwd: string; - model?: string; - receiptFile?: string; - prompt?: string; - json?: boolean; - }): Promise; + runManaged(options: { cwd: string; model?: string; prompt?: string; json?: boolean }): Promise; + checkpointAndSeal: CodexWorkspaceSealProvider; + resumePersistedLocalMount: CodexPersistedMountResumeProvider; readState(): CodexControllerState | null; sendControl(request: ControlRequest): Promise; requestId(): string; @@ -63,23 +58,6 @@ export function codexControllerPaths(env: NodeJS.ProcessEnv = process.env): Code }; } -export function resolveLiveTeleportWorkspaceSource( - workspaceRoot: string, - receiptFile?: string -): LiveTeleportWorkspaceSource { - if (receiptFile) { - const receipt = fs.readFileSync(path.resolve(receiptFile), 'utf8').trim(); - if (!receipt) throw new Error('The convergence receipt file is empty.'); - return { kind: 'verified-convergence-receipt', receipt }; - } - const relayfile = isRelayfileMount(workspaceRoot); - if (relayfile) return relayfile; - throw new Error( - 'Live Codex teleport fails closed for an unmanaged working directory. ' + - 'Use a Relayfile-mounted workspace or pass --convergence-receipt with a Cloud-verifiable receipt.' - ); -} - function describeError(error: unknown): string { return error instanceof Error ? error.message : String(error); } @@ -92,6 +70,37 @@ function agentMessageDelta(notification: CodexNotification): string | undefined return typeof delta === 'string' ? delta : undefined; } +export async function runManagedCodexTurn( + controller: Pick, + text: string, + options: { json: boolean; writeError: (message: string) => void } +): Promise { + try { + await controller.runTurn(text); + } catch (error) { + const status = controller.status(); + if (error instanceof CodexTurnRecoveredError && status.phase === 'local') { + options.writeError( + options.json + ? `${JSON.stringify({ + method: 'relay/codexTurnRecovered', + params: { + code: 'TURN_FAILED_RECOVERED_LOCALLY', + threadId: status.threadId, + generation: status.generation, + }, + })}\n` + : 'Cloud turn failed; Cloud was fenced and the same Codex thread recovered locally. Retry the turn.\n' + ); + return; + } + throw new Error( + 'Relay-managed Codex cannot continue because execution fencing or local recovery is unconfirmed.', + { cause: error } + ); + } +} + async function listen(server: net.Server, socketPath: string): Promise { await new Promise((resolve, reject) => { server.once('error', reject); @@ -177,12 +186,15 @@ async function sendSocketControl(socketPath: string, request: ControlRequest): P function withDefaults(overrides: Partial = {}): CodexCommandDependencies { const paths = codexControllerPaths(); + const relayfileLifecycle = createRelayfileSealLifecycle(); + const checkpointAndSeal = overrides.checkpointAndSeal ?? relayfileLifecycle.checkpointAndSeal; + const resumePersistedLocalMount = + overrides.resumePersistedLocalMount ?? relayfileLifecycle.resumePersistedLocalMount; return { runManaged: overrides.runManaged ?? (async (options) => { const workspaceRoot = fs.realpathSync(path.resolve(options.cwd)); - const source = resolveLiveTeleportWorkspaceSource(workspaceRoot, options.receiptFile); await probeCodexEnvironmentCapability('codex'); fs.mkdirSync(paths.directory, { recursive: true, mode: 0o700 }); const store = new FileCodexControllerStateStore(paths.statePath); @@ -193,14 +205,14 @@ function withDefaults(overrides: Partial = {}): CodexC ); } const session = await ensureCloudSession({ interactive: true }); - const cloud = new CloudLiveTeleportClient((requestPath, init) => - session.client.fetch(requestPath, init) + const cloud = new CloudLiveTeleportClient( + (requestPath, init) => session.client.fetch(requestPath, init), + session.client.snapshot().apiUrl ); const controller = new CodexLiveController( { workspaceRoot, - source, socketPath: paths.socketPath, ...(options.model ? { model: options.model } : {}), }, @@ -222,6 +234,9 @@ function withDefaults(overrides: Partial = {}): CodexC // the controller seam injectable lets restart/adversarial tests // prove an unsupported local binary still fails closed. probeCapability: async () => undefined, + checkpointAndSeal, + resumePersistedLocalMount, + sleep: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)), now: () => new Date(), sessionId: randomUUID, pid: process.pid, @@ -241,14 +256,22 @@ function withDefaults(overrides: Partial = {}): CodexC `Relay-managed Codex ${status.threadId} is ready locally (generation ${status.generation}).\n` ); - if (options.prompt) await controller.runTurn(options.prompt); + if (options.prompt) { + await runManagedCodexTurn(controller, options.prompt, { + json: Boolean(options.json), + writeError: (message) => process.stderr.write(message), + }); + } const input = readline.createInterface({ input: process.stdin, terminal: Boolean(process.stdin.isTTY), }); for await (const line of input) { if (!line.trim()) continue; - await controller.runTurn(line); + await runManagedCodexTurn(controller, line, { + json: Boolean(options.json), + writeError: (message) => process.stderr.write(message), + }); if (!options.json) process.stdout.write('\n'); } } finally { @@ -261,6 +284,8 @@ function withDefaults(overrides: Partial = {}): CodexC } } }), + checkpointAndSeal, + resumePersistedLocalMount, readState: overrides.readState ?? (() => { @@ -313,22 +338,15 @@ export function registerCodexCommands( .argument('[prompt...]', 'Optional first turn') .option('--cwd ', 'Managed workspace root') .option('--model ', 'Codex model') - .option('--convergence-receipt ', 'Cloud-verifiable receipt for a non-Relayfile workspace') .option('--json', 'Write raw app-server notifications as JSON lines') - .action( - async ( - prompt: string[], - options: { cwd?: string; model?: string; convergenceReceipt?: string; json?: boolean } - ) => { - await deps.runManaged({ - cwd: options.cwd ?? deps.cwd(), - ...(options.model ? { model: options.model } : {}), - ...(options.convergenceReceipt ? { receiptFile: options.convergenceReceipt } : {}), - ...(prompt.length ? { prompt: prompt.join(' ') } : {}), - ...(options.json ? { json: true } : {}), - }); - } - ); + .action(async (prompt: string[], options: { cwd?: string; model?: string; json?: boolean }) => { + await deps.runManaged({ + cwd: options.cwd ?? deps.cwd(), + ...(options.model ? { model: options.model } : {}), + ...(prompt.length ? { prompt: prompt.join(' ') } : {}), + ...(options.json ? { json: true } : {}), + }); + }); group .command('teleport') diff --git a/packages/cli/src/cli/lib/codex-app-server.test.ts b/packages/cli/src/cli/lib/codex-app-server.test.ts index ea31d092a..d22e09596 100644 --- a/packages/cli/src/cli/lib/codex-app-server.test.ts +++ b/packages/cli/src/cli/lib/codex-app-server.test.ts @@ -11,7 +11,13 @@ function fakeChild() { stdout: new PassThrough(), stderr: new PassThrough(), exitCode: null as number | null, - kill: vi.fn(() => true), + kill: vi.fn(() => { + queueMicrotask(() => { + child.exitCode = 0; + child.emit('exit', 0, null); + }); + return true; + }), }); return child as unknown as ChildProcessWithoutNullStreams; } @@ -22,13 +28,30 @@ async function nextRequest(child: ChildProcessWithoutNullStreams): Promise { it('requires add and status in the locally generated experimental schema', async () => { - const readFile = vi.fn(async (file: string) => - file.endsWith('ClientRequest.json') - ? '{"methods":["environment/add","environment/status"]}' - : '{"required":["environmentId","execServerUrl"],"properties":{"environmentId":{},"execServerUrl":{}}}' - ); + const readFile = vi.fn(async (file: string) => { + if (file.endsWith('ClientRequest.json')) { + return '{"methods":["environment/add","environment/status"]}'; + } + if (file.endsWith('TurnStartParams.json')) { + return turnPolicySchema(); + } + return '{"required":["environmentId","execServerUrl"],"properties":{"environmentId":{},"execServerUrl":{}}}'; + }); await expect( probeCodexEnvironmentCapability('codex', { makeTempDir: async () => '/schema', @@ -36,7 +59,11 @@ describe('probeCodexEnvironmentCapability', () => { readFile, remove: async () => undefined, }) - ).resolves.toMatchObject({ environmentAdd: true, environmentStatus: true }); + ).resolves.toMatchObject({ + environmentAdd: true, + environmentStatus: true, + explicitTurnPolicy: true, + }); }); it('fails closed when environment/status is unsupported', async () => { @@ -45,10 +72,13 @@ describe('probeCodexEnvironmentCapability', () => { probeCodexEnvironmentCapability('codex', { makeTempDir: async () => '/schema', execFile: async () => undefined, - readFile: async (file) => - file.endsWith('ClientRequest.json') - ? '{"methods":["environment/add"]}' - : '{"required":["environmentId","execServerUrl"]}', + readFile: async (file) => { + if (file.endsWith('ClientRequest.json')) return '{"methods":["environment/add"]}'; + if (file.endsWith('TurnStartParams.json')) { + return turnPolicySchema(); + } + return '{"required":["environmentId","execServerUrl"]}'; + }, remove, }) ).rejects.toThrow('does not expose both'); @@ -60,14 +90,36 @@ describe('probeCodexEnvironmentCapability', () => { probeCodexEnvironmentCapability('codex', { makeTempDir: async () => '/schema', execFile: async () => undefined, - readFile: async (file) => - file.endsWith('ClientRequest.json') - ? '{"methods":["environment/add","environment/status"]}' - : '{"required":["environmentId","execServerUrl"],"properties":{"headers":{}}}', + readFile: async (file) => { + if (file.endsWith('ClientRequest.json')) { + return '{"methods":["environment/add","environment/status"]}'; + } + if (file.endsWith('TurnStartParams.json')) { + return turnPolicySchema(); + } + return '{"required":["environmentId","execServerUrl"],"properties":{"headers":{}}}'; + }, remove: async () => undefined, }) ).rejects.toThrow('credential field'); }); + + it('fails closed when turn/start cannot pin approval and sandbox policy', async () => { + await expect( + probeCodexEnvironmentCapability('codex', { + makeTempDir: async () => '/schema', + execFile: async () => undefined, + readFile: async (file) => { + if (file.endsWith('ClientRequest.json')) { + return '{"methods":["environment/add","environment/status"]}'; + } + if (file.endsWith('TurnStartParams.json')) return '{"properties":{}}'; + return '{"required":["environmentId","execServerUrl"]}'; + }, + remove: async () => undefined, + }) + ).rejects.toThrow('execution-policy contract'); + }); }); describe('StdioCodexAppServerSession', () => { @@ -92,14 +144,17 @@ describe('StdioCodexAppServerSession', () => { await session.close(); }); - it('waits for turn/completed and sends the Cloud environment only on the attaching turn', async () => { + it('uses never + provider-isolated full access for a selected Cloud environment', async () => { const child = fakeChild(); const session = new StdioCodexAppServerSession(child); const requestPromise = nextRequest(child); const running = session.runTurn({ threadId: 'thread-1', text: 'continue', - environment: { environmentId: 'environment-3', cwd: '/workspace' }, + execution: { + kind: 'remote', + environment: { environmentId: 'environment-3', cwd: '/workspace' }, + }, }); const request = await requestPromise; expect(request).toMatchObject({ @@ -107,6 +162,8 @@ describe('StdioCodexAppServerSession', () => { method: 'turn/start', params: { threadId: 'thread-1', + approvalPolicy: 'never', + sandboxPolicy: { type: 'dangerFullAccess' }, environments: [{ environmentId: 'environment-3', cwd: '/workspace' }], }, }); @@ -117,4 +174,145 @@ describe('StdioCodexAppServerSession', () => { await expect(running).resolves.toMatchObject({ turnId: 'turn-1' }); await session.close(); }); + + it('pins local turns to a non-networked workspace sandbox and rejects an unexpected approval request', async () => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child); + const requestPromise = nextRequest(child); + const running = session.runTurn({ + threadId: 'thread-1', + text: 'edit the workspace', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }); + const request = await requestPromise; + expect(request).toMatchObject({ + method: 'turn/start', + params: { + approvalPolicy: 'never', + sandboxPolicy: { + type: 'workspaceWrite', + writableRoots: ['/repo'], + networkAccess: false, + }, + environments: [], + }, + }); + + const approvalResponse = nextRequest(child); + child.stdout.write( + `${JSON.stringify({ id: 91, method: 'item/commandExecution/requestApproval', params: {} })}\n` + ); + await expect(approvalResponse).resolves.toEqual({ + id: 91, + error: { code: -32601, message: 'Client request not supported' }, + }); + + child.stdout.write(`${JSON.stringify({ id: request.id, result: { turnId: 'turn-local' } })}\n`); + child.stdout.write( + `${JSON.stringify({ method: 'turn/completed', params: { threadId: 'thread-1', turnId: 'turn-local' } })}\n` + ); + await expect(running).resolves.toMatchObject({ turnId: 'turn-local' }); + await session.close(); + }); + + it('rejects app-server initiated numbered requests instead of leaving them hanging', async () => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child); + const responsePromise = nextRequest(child); + + child.stdout.write(`${JSON.stringify({ id: 91, method: 'item/tool/request', params: {} })}\n`); + + await expect(responsePromise).resolves.toEqual({ + id: 91, + error: { code: -32601, message: 'Client request not supported' }, + }); + await session.close(); + }); + + it.each([ + ['malformed JSON', '{definitely-not-json}\n'], + ['non-object JSON', '[]\n'], + ['oversized unterminated frame', 'x'.repeat(65)], + ])('fails closed on %s', async (_name, frame) => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child, 30_000, 30_000, undefined, { + maxFrameBytes: 64, + closeTimeoutMs: 10, + forceKillTimeoutMs: 10, + }); + + child.stdout.write(frame); + await vi.waitFor(() => expect(child.kill).toHaveBeenCalledWith('SIGTERM')); + await expect(session.initialize()).rejects.toThrow('not running'); + await session.close(); + }); + + it('fails closed when the notification queue reaches its bound', async () => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child, 30_000, 30_000, undefined, { + maxBufferedNotifications: 1, + closeTimeoutMs: 10, + forceKillTimeoutMs: 10, + }); + + child.stdout.write(`${JSON.stringify({ method: 'turn/completed', params: { turnId: 'one' } })}\n`); + child.stdout.write(`${JSON.stringify({ method: 'turn/completed', params: { turnId: 'two' } })}\n`); + + await vi.waitFor(() => expect(child.kill).toHaveBeenCalledWith('SIGTERM')); + await session.close(); + }); + + it('forwards and drops more than 1024 observed deltas without starving turn/completed', async () => { + const child = fakeChild(); + const onNotification = vi.fn(); + const session = new StdioCodexAppServerSession(child, 30_000, 30_000, onNotification, { + maxBufferedNotifications: 2, + closeTimeoutMs: 10, + forceKillTimeoutMs: 10, + }); + const requestPromise = nextRequest(child); + const running = session.runTurn({ + threadId: 'thread-1', + text: 'long turn', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }); + const request = await requestPromise; + + for (let index = 0; index < 1_100; index += 1) { + child.stdout.write( + `${JSON.stringify({ method: 'item/agentMessage/delta', params: { delta: String(index) } })}\n` + ); + } + child.stdout.write(`${JSON.stringify({ id: request.id, result: { turnId: 'turn-long' } })}\n`); + child.stdout.write( + `${JSON.stringify({ method: 'turn/completed', params: { threadId: 'thread-1', turnId: 'turn-long' } })}\n` + ); + + await expect(running).resolves.toMatchObject({ turnId: 'turn-long' }); + expect(onNotification).toHaveBeenCalledTimes(1_101); + expect(child.kill).not.toHaveBeenCalled(); + await session.close(); + }); + + it('waits for exit and escalates to SIGKILL before allowing replacement', async () => { + const child = fakeChild(); + vi.mocked(child.kill).mockImplementation((signal) => { + if (signal === 'SIGKILL') { + queueMicrotask(() => { + (child as unknown as { exitCode: number | null }).exitCode = 137; + child.emit('exit', null, 'SIGKILL'); + }); + } + return true; + }); + const session = new StdioCodexAppServerSession(child, 30_000, 30_000, undefined, { + closeTimeoutMs: 1, + forceKillTimeoutMs: 20, + }); + + await session.close(); + + expect(child.kill).toHaveBeenNthCalledWith(1, 'SIGTERM'); + expect(child.kill).toHaveBeenNthCalledWith(2, 'SIGKILL'); + }); }); diff --git a/packages/cli/src/cli/lib/codex-app-server.ts b/packages/cli/src/cli/lib/codex-app-server.ts index c741a139d..40df2712a 100644 --- a/packages/cli/src/cli/lib/codex-app-server.ts +++ b/packages/cli/src/cli/lib/codex-app-server.ts @@ -1,4 +1,3 @@ -import fs from 'node:fs'; import fsp from 'node:fs/promises'; import os from 'node:os'; import path from 'node:path'; @@ -18,6 +17,13 @@ export type CodexTurnResult = { completed: CodexNotification; }; +export type CodexTurnExecution = + | { kind: 'local'; workspaceRoot: string } + | { + kind: 'remote'; + environment?: { environmentId: string; cwd: string }; + }; + export interface CodexAppServerSession { initialize(): Promise; startThread(input: { cwd: string; model?: string }): Promise; @@ -28,17 +34,14 @@ export interface CodexAppServerSession { connectTimeoutMs: number; }): Promise; environmentStatus(environmentId: string): Promise; - runTurn(input: { - threadId: string; - text: string; - environment?: { environmentId: string; cwd: string }; - }): Promise; + runTurn(input: { threadId: string; text: string; execution: CodexTurnExecution }): Promise; close(): Promise; } export type CodexEnvironmentCapability = { environmentAdd: true; environmentStatus: true; + explicitTurnPolicy: true; }; export type CodexCapabilityProbeDependencies = { @@ -59,6 +62,56 @@ function defaultCapabilityProbeDependencies(): CodexCapabilityProbeDependencies }; } +function assertEnvironmentAddSchema(addParams: string): void { + const schema = JSON.parse(addParams) as { + required?: unknown; + properties?: Record; + }; + const required = Array.isArray(schema.required) ? schema.required : []; + if (!required.includes('environmentId') || !required.includes('execServerUrl')) { + throw new Error('Codex EnvironmentAddParams does not match the required execution-teleport schema.'); + } + if (schema.properties && ('headers' in schema.properties || 'token' in schema.properties)) { + throw new Error( + 'Codex EnvironmentAddParams unexpectedly contains a credential field; upgrade Relay first.' + ); + } +} + +function assertTurnPolicySchema(turnParams: string): void { + const turnSchema = JSON.parse(turnParams) as { + properties?: Record; + }; + if ( + !turnSchema.properties?.approvalPolicy || + !turnSchema.properties?.sandboxPolicy || + !schemaContainsEnum(turnSchema, 'never') || + !schemaContainsEnum(turnSchema, 'workspaceWrite') || + !schemaContainsProperty(turnSchema, 'writableRoots') || + !schemaContainsEnum(turnSchema, 'dangerFullAccess') + ) { + throw new Error("Codex TurnStartParams does not support Relay's explicit execution-policy contract."); + } +} + +function schemaContainsEnum(value: unknown, expected: string): boolean { + if (Array.isArray(value)) return value.some((entry) => schemaContainsEnum(entry, expected)); + if (!value || typeof value !== 'object') return false; + const object = value as Record; + if (Array.isArray(object.enum) && object.enum.includes(expected)) return true; + return Object.values(object).some((entry) => schemaContainsEnum(entry, expected)); +} + +function schemaContainsProperty(value: unknown, expected: string): boolean { + if (Array.isArray(value)) return value.some((entry) => schemaContainsProperty(entry, expected)); + if (!value || typeof value !== 'object') return false; + const object = value as Record; + if (object.properties && typeof object.properties === 'object' && expected in object.properties) { + return true; + } + return Object.values(object).some((entry) => schemaContainsProperty(entry, expected)); +} + /** * Probe the locally installed binary rather than assuming an experimental * protocol from Relay's build-time Codex version. Both methods and the exact @@ -79,9 +132,10 @@ export async function probeCodexEnvironmentCapability( '--out', directory, ]); - const [requests, addParams] = await Promise.all([ + const [requests, addParams, turnParams] = await Promise.all([ deps.readFile(path.join(directory, 'ClientRequest.json')), deps.readFile(path.join(directory, 'v2', 'EnvironmentAddParams.json')), + deps.readFile(path.join(directory, 'v2', 'TurnStartParams.json')), ]); if (!requests.includes('"environment/add"') || !requests.includes('"environment/status"')) { throw new Error( @@ -89,21 +143,10 @@ export async function probeCodexEnvironmentCapability( ); } - const schema = JSON.parse(addParams) as { - required?: unknown; - properties?: Record; - }; - const required = Array.isArray(schema.required) ? schema.required : []; - if (!required.includes('environmentId') || !required.includes('execServerUrl')) { - throw new Error('Codex EnvironmentAddParams does not match the required execution-teleport schema.'); - } - if (schema.properties && ('headers' in schema.properties || 'token' in schema.properties)) { - throw new Error( - 'Codex EnvironmentAddParams unexpectedly contains a credential field; upgrade Relay first.' - ); - } + assertEnvironmentAddSchema(addParams); + assertTurnPolicySchema(turnParams); - return { environmentAdd: true, environmentStatus: true }; + return { environmentAdd: true, environmentStatus: true, explicitTurnPolicy: true }; } finally { await deps.remove(directory); } @@ -115,10 +158,26 @@ type PendingRequest = { timer: NodeJS.Timeout; }; +export type CodexAppServerTransportLimits = { + maxFrameBytes?: number; + maxBufferedNotifications?: number; + closeTimeoutMs?: number; + forceKillTimeoutMs?: number; +}; + +const DEFAULT_MAX_FRAME_BYTES = 1024 * 1024; +const DEFAULT_MAX_BUFFERED_NOTIFICATIONS = 1024; +const DEFAULT_CLOSE_TIMEOUT_MS = 5_000; +const DEFAULT_FORCE_KILL_TIMEOUT_MS = 1_000; + function isObject(value: unknown): value is Record { return value !== null && typeof value === 'object' && !Array.isArray(value); } +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); +} + function stringAt(value: unknown, ...keys: string[]): string | undefined { let cursor: unknown = value; for (const key of keys) { @@ -152,32 +211,44 @@ export class StdioCodexAppServerSession implements CodexAppServerSession { }>(); private nextId = 1; private buffer = ''; - private stderr = ''; private closed = false; + private exited = false; + private shutdownStarted = false; + private writeChain = Promise.resolve(); + private readonly exitPromise: Promise; + private readonly maxFrameBytes: number; + private readonly maxBufferedNotifications: number; + private readonly closeTimeoutMs: number; + private readonly forceKillTimeoutMs: number; constructor( private readonly child: ChildProcessWithoutNullStreams, private readonly requestTimeoutMs = 30_000, private readonly turnTimeoutMs = 30 * 60_000, - private readonly onNotification?: (notification: CodexNotification) => void + private readonly onNotification?: (notification: CodexNotification) => void, + limits: CodexAppServerTransportLimits = {} ) { + this.maxFrameBytes = limits.maxFrameBytes ?? DEFAULT_MAX_FRAME_BYTES; + this.maxBufferedNotifications = limits.maxBufferedNotifications ?? DEFAULT_MAX_BUFFERED_NOTIFICATIONS; + this.closeTimeoutMs = limits.closeTimeoutMs ?? DEFAULT_CLOSE_TIMEOUT_MS; + this.forceKillTimeoutMs = limits.forceKillTimeoutMs ?? DEFAULT_FORCE_KILL_TIMEOUT_MS; child.stdout.setEncoding('utf8'); child.stdout.on('data', (chunk: string) => this.onData(chunk)); child.stderr.setEncoding('utf8'); - child.stderr.on('data', (chunk: string) => { - this.stderr = `${this.stderr}${chunk}`.slice(-8_192); - }); - child.once('exit', (code, signal) => { - this.closed = true; - const detail = this.stderr.trim().split('\n').at(-1); - this.rejectAll( - new Error( - `Codex app-server exited before completing the request (${code ?? signal ?? 'unknown'}).${ - detail ? ` ${detail}` : '' - }` - ) - ); + // Drain stderr so the child cannot block, but never reflect provider or + // model diagnostics into persisted/public controller errors. + child.stderr.on('data', () => undefined); + this.exitPromise = new Promise((resolve) => { + child.once('exit', (code, signal) => { + this.exited = true; + this.closed = true; + this.rejectAll( + new Error(`Codex app-server exited before completing the request (${code ?? signal ?? 'unknown'}).`) + ); + resolve(); + }); }); + child.stdin.once('error', (error) => this.failProtocol(`stdin write failed: ${error.message}`)); } static spawn(options: { @@ -230,14 +301,37 @@ export class StdioCodexAppServerSession implements CodexAppServerSession { async runTurn(input: { threadId: string; text: string; - environment?: { environmentId: string; cwd: string }; + execution: CodexTurnExecution; }): Promise { + let executionParams: Record; + if (input.execution.kind === 'remote') { + executionParams = { + approvalPolicy: 'never', + sandboxPolicy: { type: 'dangerFullAccess' }, + ...(input.execution.environment ? { environments: [input.execution.environment] } : {}), + }; + } else { + const workspaceRoot = input.execution.workspaceRoot; + if (!path.isAbsolute(workspaceRoot) || path.normalize(workspaceRoot) !== workspaceRoot) { + throw new Error('Local Codex execution requires an absolute normalized workspace root.'); + } + executionParams = { + approvalPolicy: 'never', + sandboxPolicy: { + type: 'workspaceWrite', + writableRoots: [workspaceRoot], + networkAccess: false, + }, + // Clear any sticky remote environment before a recovered local turn. + environments: [], + }; + } const response = await this.request( 'turn/start', { threadId: input.threadId, input: [{ type: 'text', text: input.text }], - ...(input.environment ? { environments: [input.environment] } : {}), + ...executionParams, }, this.turnTimeoutMs ); @@ -254,11 +348,13 @@ export class StdioCodexAppServerSession implements CodexAppServerSession { } async close(): Promise { - if (this.closed) return; - this.closed = true; - this.rejectAll(new Error('Codex app-server closed.')); - this.child.stdin.end(); - if (this.child.exitCode === null) this.child.kill('SIGTERM'); + this.beginShutdown(new Error('Codex app-server closed.')); + if (this.exited || this.child.exitCode !== null) return; + if (await this.waitForExit(this.closeTimeoutMs)) return; + this.child.kill('SIGKILL'); + if (!(await this.waitForExit(this.forceKillTimeoutMs))) { + throw new Error('Codex app-server did not exit after SIGKILL.'); + } } private request(method: string, params?: unknown, timeoutMs = this.requestTimeoutMs): Promise { @@ -270,8 +366,15 @@ export class StdioCodexAppServerSession implements CodexAppServerSession { reject(new Error(`Timed out waiting for Codex ${method}.`)); }, timeoutMs); this.pending.set(id, { resolve, reject, timer }); - this.child.stdin.write( - `${JSON.stringify({ id, method, ...(params === undefined ? {} : { params }) })}\n` + void this.writeMessage({ id, method, ...(params === undefined ? {} : { params }) }).catch( + (error: unknown) => { + const pending = this.pending.get(id); + if (!pending) return; + this.pending.delete(id); + clearTimeout(pending.timer); + pending.reject(new Error(`Could not write Codex ${method}: ${errorMessage(error)}`)); + this.failProtocol(`failed to serialize a client request: ${errorMessage(error)}`); + } ); }); } @@ -306,19 +409,34 @@ export class StdioCodexAppServerSession implements CodexAppServerSession { } private onData(chunk: string): void { + if (this.closed) return; this.buffer += chunk; let newline: number; while ((newline = this.buffer.indexOf('\n')) >= 0) { - const line = this.buffer.slice(0, newline).trim(); + const frame = this.buffer.slice(0, newline); this.buffer = this.buffer.slice(newline + 1); + if (Buffer.byteLength(frame, 'utf8') > this.maxFrameBytes) { + this.failProtocol('received an oversized frame'); + return; + } + const line = frame.trim(); if (!line) continue; let message: unknown; try { message = JSON.parse(line) as unknown; } catch { - continue; + this.failProtocol('received malformed JSON'); + return; + } + if (!isObject(message)) { + this.failProtocol('received a non-object frame'); + return; } - if (isObject(message)) this.handleMessage(message); + this.handleMessage(message); + if (this.closed) return; + } + if (Buffer.byteLength(this.buffer, 'utf8') > this.maxFrameBytes) { + this.failProtocol('received an oversized unterminated frame'); } } @@ -327,15 +445,41 @@ export class StdioCodexAppServerSession implements CodexAppServerSession { this.handleResponse(message.id, message); return; } - if (typeof message.method !== 'string') return; + if ( + (typeof message.id === 'number' || typeof message.id === 'string') && + typeof message.method === 'string' + ) { + void this.writeMessage({ + id: message.id, + error: { code: -32601, message: 'Client request not supported' }, + }).catch((error: unknown) => { + this.failProtocol(`could not reject an app-server request: ${errorMessage(error)}`); + }); + return; + } + if (typeof message.method !== 'string') { + this.failProtocol('received a frame without a response or method'); + return; + } const notification = { method: message.method, ...('params' in message ? { params: message.params } : {}), }; - this.onNotification?.(notification); + try { + this.onNotification?.(notification); + } catch (error) { + this.failProtocol(`notification handler failed: ${errorMessage(error)}`); + return; + } const waiter = [...this.notificationWaiters].find((candidate) => candidate.predicate(notification)); if (waiter) waiter.resolve(notification); - else this.notifications.push(notification); + else if (notification.method === 'turn/completed') { + if (this.notifications.length >= this.maxBufferedNotifications) { + this.failProtocol('notification queue exceeded its bound'); + return; + } + this.notifications.push(notification); + } } private handleResponse(id: number, message: Record): void { @@ -343,7 +487,12 @@ export class StdioCodexAppServerSession implements CodexAppServerSession { if (!pending) return; this.pending.delete(id); clearTimeout(pending.timer); - if ('error' in message && isObject(message.error)) { + if ('error' in message) { + if (!isObject(message.error)) { + pending.reject(new Error('Codex app-server returned a malformed error response.')); + this.failProtocol('received a malformed error response'); + return; + } pending.reject( new Error( `${typeof message.error.code === 'number' ? `${message.error.code}: ` : ''}${ @@ -365,11 +514,56 @@ export class StdioCodexAppServerSession implements CodexAppServerSession { for (const waiter of this.notificationWaiters) waiter.reject(error); this.notificationWaiters.clear(); } -} -export function isRelayfileMount( - workspaceRoot: string -): { kind: 'relayfile-mount'; mountStatePath: string } | null { - const mountStatePath = path.join(workspaceRoot, '.relayfile-mount-state.json'); - return fs.existsSync(mountStatePath) ? { kind: 'relayfile-mount', mountStatePath } : null; + private writeMessage(message: Record): Promise { + const frame = `${JSON.stringify(message)}\n`; + if (Buffer.byteLength(frame, 'utf8') > this.maxFrameBytes) { + return Promise.reject(new Error('outbound Codex frame exceeds the transport bound')); + } + const write = this.writeChain.then( + () => + new Promise((resolve, reject) => { + if (this.closed || this.child.stdin.destroyed) { + reject(new Error('Codex app-server is not writable.')); + return; + } + this.child.stdin.write(frame, (error?: Error | null) => { + if (error) reject(error); + else resolve(); + }); + }) + ); + this.writeChain = write.catch(() => undefined); + return write; + } + + private failProtocol(detail: string): void { + this.beginShutdown(new Error(`Codex app-server protocol violation: ${detail}.`)); + } + + private beginShutdown(error: Error): void { + if (!this.closed) { + this.closed = true; + this.rejectAll(error); + } + if (this.shutdownStarted || this.exited || this.child.exitCode !== null) return; + this.shutdownStarted = true; + this.child.stdin.end(); + this.child.kill('SIGTERM'); + } + + private async waitForExit(timeoutMs: number): Promise { + if (this.exited || this.child.exitCode !== null) return true; + let timer: NodeJS.Timeout | undefined; + try { + return await Promise.race([ + this.exitPromise.then(() => true), + new Promise((resolve) => { + timer = setTimeout(() => resolve(false), timeoutMs); + }), + ]); + } finally { + if (timer) clearTimeout(timer); + } + } } diff --git a/packages/cli/src/cli/lib/codex-live-controller.test.ts b/packages/cli/src/cli/lib/codex-live-controller.test.ts index 41594ab02..ac11a9a3e 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.test.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.test.ts @@ -1,15 +1,31 @@ import { describe, expect, it, vi } from 'vitest'; -import type { LiveTeleportCloudClient } from '@agent-relay/cloud'; +import type { LiveTeleportCloudClient, LiveTeleportLifecycleStatus } from '@agent-relay/cloud'; import { CodexLiveController, type CodexControllerState, type CodexControllerStateStore, + type CodexPersistedMountResumeProvider, + type CodexWorkspaceSealHandle, + type CodexWorkspaceSealProvider, } from './codex-live-controller.js'; -import type { CodexAppServerSession } from './codex-app-server.js'; +import type { CodexAppServerSession, CodexTurnResult } from './codex-app-server.js'; -const source = { kind: 'relayfile-mount' as const, mountStatePath: '/repo/.relayfile-mount-state.json' }; +const source = { + kind: 'relayfile-checkpoint-seal' as const, + receipt: { sealId: 'seal-1', sealToken: 'opaque' }, +}; + +function sealHandle(overrides: Partial = {}): CodexWorkspaceSealHandle { + return { + source, + restore: { resumeId: 'resume-1', workspaceId: 'workspace-1', localRoot: '/repo' }, + resumeLocal: vi.fn(async () => undefined), + close: vi.fn(async () => undefined), + ...overrides, + }; +} const convergence = { verdict: 'converged' as const, source: { @@ -45,6 +61,10 @@ function deferred() { return { promise, resolve, reject }; } +function turnResult(turnId = 'turn-1'): CodexTurnResult { + return { turnId, response: {}, completed: { method: 'turn/completed' } }; +} + function memoryStore(initial: CodexControllerState | null = null): CodexControllerStateStore & { value: CodexControllerState | null; } { @@ -66,16 +86,19 @@ function appServer(overrides: Partial = {}): CodexAppServ resumeThread: vi.fn(async () => undefined), addEnvironment: vi.fn(async () => undefined), environmentStatus: vi.fn(async () => ({ status: 'ready' })), - runTurn: vi.fn(async () => ({ - turnId: 'turn-1', - response: {}, - completed: { method: 'turn/completed' }, - })), + runTurn: vi.fn(async () => turnResult()), close: vi.fn(async () => undefined), ...overrides, }; } +function lifecycle( + input: { sessionId: string; generation: number }, + status: LiveTeleportLifecycleStatus['status'] +): LiveTeleportLifecycleStatus { + return { sessionId: input.sessionId, generation: input.generation, status }; +} + function cloud(overrides: Partial = {}): LiveTeleportCloudClient { return { prewarm: vi.fn(async (input) => ({ @@ -83,16 +106,18 @@ function cloud(overrides: Partial = {}): LiveTeleportCl generation: input.generation, status: 'ready' as const, })), + status: vi.fn(async (input) => lifecycle(input, 'ready')), acquire: vi.fn(async (input) => ({ sessionId: input.sessionId, generation: input.generation, environmentId: `environment-${input.generation}`, - execServerUrl: 'wss://exec.agentrelay.test/ticket', + connectPath: `/api/v1/live-teleports/connect/${input.sessionId}/${input.generation}`, + execServerUrl: `wss://cloud.agentrelay.test/api/v1/live-teleports/connect/${input.sessionId}/${input.generation}`, workspaceCwd: '/workspace', expiresAt: '2026-08-23T13:00:00.000Z', convergence, })), - revoke: vi.fn(async () => undefined), + revoke: vi.fn(async (input) => ({ ...lifecycle(input, 'revoked'), status: 'revoked' as const })), ...overrides, }; } @@ -103,6 +128,10 @@ function createController( cloud?: LiveTeleportCloudClient; sessions?: CodexAppServerSession[]; probe?: () => Promise; + checkpointAndSeal?: CodexWorkspaceSealProvider; + resumePersistedLocalMount?: CodexPersistedMountResumeProvider; + lifecycleDeadlineMs?: number; + lifecyclePollIntervalMs?: number; } = {} ) { const store = options.store ?? memoryStore(); @@ -113,23 +142,71 @@ function createController( if (!session) throw new Error('no app server'); return session; }); + const checkpointAndSeal = vi.fn(options.checkpointAndSeal ?? (async () => sealHandle())); + const resumePersistedLocalMount = vi.fn(options.resumePersistedLocalMount ?? (async () => undefined)); const controller = new CodexLiveController( - { workspaceRoot: '/repo', source, socketPath: '/state/controller.sock' }, + { + workspaceRoot: '/repo', + socketPath: '/state/controller.sock', + ...(options.lifecycleDeadlineMs ? { lifecycleDeadlineMs: options.lifecycleDeadlineMs } : {}), + ...(options.lifecyclePollIntervalMs + ? { lifecyclePollIntervalMs: options.lifecyclePollIntervalMs } + : {}), + }, { cloud: cloudClient, store, createAppServer, probeCapability: options.probe ?? (async () => undefined), + checkpointAndSeal, + resumePersistedLocalMount, + sleep: async () => undefined, now: () => new Date('2026-08-23T12:00:00.000Z'), sessionId: () => 'session-1', pid: 123, } ); - return { controller, store, cloud: cloudClient, createAppServer }; + return { + controller, + store, + cloud: cloudClient, + createAppServer, + checkpointAndSeal, + resumePersistedLocalMount, + }; +} + +function persistedRemote(overrides: Partial = {}): CodexControllerState { + return { + version: 1, + sessionId: 'session-1', + threadId: 'thread-1', + workspaceRoot: '/repo', + source, + mountRestore: { resumeId: 'resume-7', workspaceId: 'workspace-1', localRoot: '/repo' }, + generation: 7, + phase: 'remote', + controllerPid: 99, + socketPath: '/old.sock', + turnActive: false, + remote: { + sessionId: 'session-1', + generation: 7, + environmentId: 'env-7', + connectPath: '/api/v1/live-teleports/connect/session-1/7', + execServerUrl: 'wss://cloud.agentrelay.test/api/v1/live-teleports/connect/session-1/7', + workspaceCwd: '/workspace', + expiresAt: '2026-08-23T13:00:00.000Z', + attached: true, + convergence, + }, + updatedAt: '2026-08-23T11:00:00.000Z', + ...overrides, + }; } describe('CodexLiveController', () => { - it('fails closed before starting an app-server when the local experimental capability is unsupported', async () => { + it('fails before starting an app-server when the local experimental capability is unsupported', async () => { const { controller, createAppServer } = createController({ probe: async () => { throw new Error('environment/status unsupported'); @@ -140,273 +217,429 @@ describe('CodexLiveController', () => { expect(createAppServer).not.toHaveBeenCalled(); }); - it('rejects a stale generation and makes duplicate request ids idempotent', async () => { - const { controller } = createController(); + it('prewarms without stopping or sealing the active local mount', async () => { + const cloudClient = cloud(); + const { controller, checkpointAndSeal } = createController({ cloud: cloudClient }); + await controller.initialize(); - expect(() => controller.requestTeleport({ requestId: 'request-stale', expectedGeneration: 0 })).toThrow( - 'Stale teleport generation' + expect(checkpointAndSeal).not.toHaveBeenCalled(); + expect(cloudClient.prewarm).toHaveBeenCalledWith( + expect.objectContaining({ + sessionId: 'session-1', + generation: 1, + workspaceRoot: '/', + idempotencyKey: 'session-1:1:prewarm', + }) ); - const first = controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - const duplicate = controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - expect(first.phase).toBe('teleport_pending'); - expect(duplicate.pending?.requestId).toBe('request-1'); }); - it('queues a mid-turn request and applies it only at the following turn boundary', async () => { - const firstTurn = deferred<{ - turnId: string; - response: object; - completed: { method: string }; - }>(); - const session = appServer({ + it('rejects stale generations and applies a mid-turn teleport only at the next boundary', async () => { + const firstTurn = deferred(); + const original = appServer({ runTurn: vi .fn() .mockImplementationOnce(() => firstTurn.promise) - .mockResolvedValueOnce({ turnId: 'turn-2', response: {}, completed: { method: 'turn/completed' } }), + .mockResolvedValueOnce(turnResult('turn-2')), }); const cloudClient = cloud(); - const { controller } = createController({ sessions: [session], cloud: cloudClient }); + const { controller } = createController({ sessions: [original], cloud: cloudClient }); await controller.initialize(); + expect(() => controller.requestTeleport({ requestId: 'stale', expectedGeneration: 0 })).toThrow( + 'Stale teleport generation' + ); const running = controller.runTurn('local turn'); - await vi.waitFor(() => expect(session.runTurn).toHaveBeenCalledTimes(1)); - const queued = controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - expect(queued.turnActive).toBe(true); - expect(queued.phase).toBe('teleport_pending'); + await vi.waitFor(() => expect(original.runTurn).toHaveBeenCalledTimes(1)); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); expect(cloudClient.acquire).not.toHaveBeenCalled(); - firstTurn.resolve({ turnId: 'turn-1', response: {}, completed: { method: 'turn/completed' } }); + firstTurn.resolve(turnResult()); await running; - expect(cloudClient.acquire).not.toHaveBeenCalled(); - await controller.runTurn('remote turn'); + + expect(original.runTurn).toHaveBeenNthCalledWith(1, { + threadId: 'thread-1', + text: 'local turn', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }); + expect(original.runTurn).toHaveBeenNthCalledWith(2, { + threadId: 'thread-1', + text: 'remote turn', + execution: { + kind: 'remote', + environment: { environmentId: 'environment-1', cwd: '/workspace' }, + }, + }); + expect(cloudClient.acquire).toHaveBeenCalledWith( expect.objectContaining({ - sessionId: 'session-1', - threadId: 'thread-1', generation: 1, + workspaceRoot: '/', source, prewarmId: 'prewarm-1', idempotencyKey: 'session-1:1:acquire', }) ); - expect(vi.mocked(session.addEnvironment).mock.invocationCallOrder[0]).toBeLessThan( - vi.mocked(session.runTurn).mock.invocationCallOrder[1]! - ); - expect(session.runTurn).toHaveBeenLastCalledWith({ + }); + + it('reports verifying—not remote—until the first Cloud turn completes', async () => { + const firstRemoteTurn = deferred(); + const original = appServer({ runTurn: vi.fn(() => firstRemoteTurn.promise) }); + const { controller } = createController({ sessions: [original] }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + const running = controller.runTurn('first remote turn'); + await vi.waitFor(() => expect(controller.status().phase).toBe('verifying')); + expect(controller.status()).toMatchObject({ execution: 'verifying', phase: 'verifying' }); + expect(controller.status()).not.toHaveProperty('remote'); + + firstRemoteTurn.resolve(turnResult()); + await running; + expect(controller.status()).toMatchObject({ execution: 'cloud', phase: 'remote' }); + expect(controller.status().remote).toMatchObject({ attached: true, environmentId: 'environment-1' }); + expect(controller.status().remote).not.toHaveProperty('execServerUrl'); + expect(controller.status().remote).not.toHaveProperty('connectPath'); + expect(controller.status()).not.toHaveProperty('source'); + expect(controller.status()).not.toHaveProperty('mountRestore'); + + await controller.runTurn('subsequent remote turn'); + expect(original.runTurn).toHaveBeenNthCalledWith(2, { threadId: 'thread-1', - text: 'remote turn', - environment: { environmentId: 'environment-1', cwd: '/workspace' }, + text: 'subsequent remote turn', + execution: { kind: 'remote' }, }); }); - it('keeps execution local when Cloud acquisition fails before turn/start', async () => { - const session = appServer(); - const cloudClient = cloud({ acquire: vi.fn(async () => Promise.reject(new Error('Cloud unavailable'))) }); - const { controller } = createController({ sessions: [session], cloud: cloudClient }); + it('confirms revoke, closes the old controller, and resumes the same thread after first-turn failure', async () => { + const original = appServer({ runTurn: vi.fn(async () => Promise.reject(new Error('remote died'))) }); + const replacement = appServer(); + const cloudClient = cloud(); + const sealed = sealHandle(); + const { controller } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + checkpointAndSeal: async () => sealed, + }); await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await expect(controller.runTurn('must not run remotely')).rejects.toThrow('execution remains local'); - expect(session.addEnvironment).not.toHaveBeenCalled(); - expect(session.runTurn).not.toHaveBeenCalled(); - expect(controller.status()).toMatchObject({ execution: 'local', phase: 'local', turnActive: false }); - expect(cloudClient.revoke).toHaveBeenCalled(); + await expect(controller.runTurn('do not replay me')).rejects.toThrow('resumed locally'); + + expect(cloudClient.revoke).toHaveBeenCalledWith( + expect.objectContaining({ idempotencyKey: 'session-1:1:first-turn-failed-revoke' }) + ); + expect(original.close).toHaveBeenCalled(); + expect(sealed.resumeLocal).toHaveBeenCalled(); + expect(replacement.initialize).toHaveBeenCalled(); + expect(replacement.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + expect(replacement.runTurn).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ generation: 2, phase: 'local', execution: 'local' }); }); - it('rejects a stale generation returned by Cloud', async () => { - const session = appServer(); + it('fails fenced and never resumes locally when first-turn revoke is unconfirmed', async () => { + const original = appServer({ runTurn: vi.fn(async () => Promise.reject(new Error('remote died'))) }); + const replacement = appServer(); const cloudClient = cloud({ - acquire: vi.fn(async (input) => ({ - sessionId: input.sessionId, - generation: input.generation + 1, - environmentId: 'stale', - execServerUrl: 'wss://exec.agentrelay.test/stale', - workspaceCwd: '/workspace', - expiresAt: '2026-08-23T13:00:00.000Z', - convergence, - })), + revoke: vi.fn(async () => Promise.reject(new Error('revoke timeout'))), + status: vi.fn(async (input) => lifecycle(input, 'ready')), + }); + const { controller, createAppServer } = createController({ + sessions: [original, replacement], + cloud: cloudClient, }); - const { controller } = createController({ sessions: [session], cloud: cloudClient }); await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await expect(controller.runTurn('no cross-generation execution')).rejects.toThrow( - 'stale or cross-session' - ); - expect(session.addEnvironment).not.toHaveBeenCalled(); + await expect(controller.runTurn('must not run locally')).rejects.toThrow('could not be confirmed'); + + expect(original.close).toHaveBeenCalled(); + expect(createAppServer).toHaveBeenCalledTimes(1); + expect(replacement.resumeThread).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'fenced', execution: 'fenced' }); }); - it('revokes and stays local when Codex cannot verify the environment ready', async () => { - const session = appServer({ - environmentStatus: vi.fn(async () => ({ status: 'connecting' })), + it('does not trust a mount state or arbitrary receipt when no real seal provider is available', async () => { + const seal = vi + .fn() + .mockRejectedValue(new Error('checkpoint-and-seal API unavailable')); + const original = appServer(); + const replacement = appServer(); + const cloudClient = cloud(); + const { controller } = createController({ + checkpointAndSeal: seal, + sessions: [original, replacement], + cloud: cloudClient, }); + await controller.initialize(); + expect(controller.status()).toMatchObject({ phase: 'local', workspaceSource: 'unavailable' }); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('local only')).rejects.toThrow('checkpoint-and-seal API unavailable'); + + expect(cloudClient.acquire).not.toHaveBeenCalled(); + expect(cloudClient.revoke).toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); + }); + + it('does not persist a seal whose restore identity targets another local root', async () => { + const sealed = sealHandle({ + restore: { resumeId: 'resume-1', workspaceId: 'workspace-1', localRoot: '/other' }, + }); + const original = appServer(); + const replacement = appServer(); const cloudClient = cloud(); - const { controller } = createController({ sessions: [session], cloud: cloudClient }); + const { controller, store } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + checkpointAndSeal: async () => sealed, + }); await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await expect(controller.runTurn('must stay local')).rejects.toThrow( - 'did not report the Cloud execution environment ready' - ); - expect(session.runTurn).not.toHaveBeenCalled(); - expect(cloudClient.revoke).toHaveBeenCalledWith( - expect.objectContaining({ idempotencyKey: 'session-1:1:failed-acquire-revoke' }) - ); - expect(controller.status()).toMatchObject({ phase: 'local', execution: 'local' }); + await expect(controller.runTurn('must stay on this mount')).rejects.toThrow('invalid restore identity'); + + expect(cloudClient.acquire).not.toHaveBeenCalled(); + expect(sealed.resumeLocal).toHaveBeenCalled(); + expect(store.value?.source).toBeUndefined(); + expect(store.value?.mountRestore).toBeUndefined(); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); }); - it('uses Codex stickiness after explicitly attaching the first remote turn', async () => { - const session = appServer(); - const { controller } = createController({ sessions: [session] }); + it('thaws the exact sealed mount after acquire failure before local resume', async () => { + const sealed = sealHandle(); + const original = appServer(); + const replacement = appServer(); + const cloudClient = cloud({ acquire: vi.fn(async () => Promise.reject(new Error('acquire failed'))) }); + const { controller } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + checkpointAndSeal: async () => sealed, + }); await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await controller.runTurn('first remote turn'); - await controller.runTurn('sticky remote turn'); + await expect(controller.runTurn('local after abort')).rejects.toThrow('acquire failed'); - expect(session.runTurn).toHaveBeenNthCalledWith(1, { + expect(sealed.resumeLocal).toHaveBeenCalled(); + expect(replacement.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); + await controller.runTurn('second input'); + expect(replacement.runTurn).toHaveBeenCalledWith({ threadId: 'thread-1', - text: 'first remote turn', - environment: { environmentId: 'environment-1', cwd: '/workspace' }, + text: 'second input', + execution: { kind: 'local', workspaceRoot: '/repo' }, }); - expect(session.runTurn).toHaveBeenNthCalledWith(2, { - threadId: 'thread-1', - text: 'sticky remote turn', + }); + + it('does not resume local execution when a fenced mount cannot be thawed', async () => { + const sealed = sealHandle({ resumeLocal: vi.fn(async () => Promise.reject(new Error('mount dead'))) }); + const original = appServer(); + const replacement = appServer(); + const cloudClient = cloud({ acquire: vi.fn(async () => Promise.reject(new Error('acquire failed'))) }); + const { controller, createAppServer } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + checkpointAndSeal: async () => sealed, }); - expect(controller.status().remote).not.toHaveProperty('execServerUrl'); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('must stay fenced')).rejects.toThrow('could not resume'); + + expect(createAppServer).toHaveBeenCalledTimes(1); + expect(replacement.resumeThread).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'recovery_failed', execution: 'fenced' }); + }); + + it('polls warming lifecycle status to ready before acquisition', async () => { + const cloudClient = cloud({ + prewarm: vi.fn(async (input) => ({ + prewarmId: `prewarm-${input.generation}`, + generation: input.generation, + status: 'warming' as const, + })), + status: vi + .fn() + .mockImplementationOnce(async (input) => ({ + ...lifecycle(input, 'warming'), + prewarmId: 'prewarm-1', + retryAfterMs: 1, + })) + .mockImplementationOnce(async (input) => ({ + ...lifecycle(input, 'ready'), + prewarmId: 'prewarm-1', + })), + }); + const { controller, checkpointAndSeal } = createController({ cloud: cloudClient }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await controller.runTurn('after convergence'); + + expect(cloudClient.status).toHaveBeenCalledTimes(2); + expect(vi.mocked(cloudClient.status).mock.invocationCallOrder.at(-1)).toBeLessThan( + checkpointAndSeal.mock.invocationCallOrder[0]! + ); + expect(cloudClient.acquire).toHaveBeenCalled(); }); - it('rolls back by restarting, initializing, and resuming the same thread locally', async () => { + it('bounds non-convergence and recovers locally instead of blocking forever', async () => { + const cloudClient = cloud({ + prewarm: vi.fn(async (input) => ({ + prewarmId: `prewarm-${input.generation}`, + generation: input.generation, + status: 'warming' as const, + })), + status: vi.fn(async (input) => ({ ...lifecycle(input, 'warming'), retryAfterMs: 1 })), + }); const original = appServer(); const replacement = appServer(); - const cloudClient = cloud(); - const { controller } = createController({ sessions: [original, replacement], cloud: cloudClient }); + const { controller } = createController({ + cloud: cloudClient, + sessions: [original, replacement], + lifecycleDeadlineMs: 5, + lifecyclePollIntervalMs: 1, + }); await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await controller.runTurn('remote turn'); - const status = await controller.rollback(); + await expect(controller.runTurn('bounded')).rejects.toThrow('did not converge'); - expect(original.close).toHaveBeenCalled(); - expect(replacement.initialize).toHaveBeenCalled(); - expect(replacement.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); - expect(status).toMatchObject({ generation: 2, phase: 'local', execution: 'local' }); - await controller.runTurn('local again'); - expect(replacement.runTurn).toHaveBeenCalledWith({ threadId: 'thread-1', text: 'local again' }); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); + }); + + it('polls Codex environment/status until ready', async () => { + const original = appServer({ + environmentStatus: vi + .fn() + .mockResolvedValueOnce({ status: 'connecting' }) + .mockResolvedValueOnce({ status: 'ready' }), + }); + const { controller } = createController({ sessions: [original] }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await controller.runTurn('ready after poll'); + + expect(original.environmentStatus).toHaveBeenCalledTimes(2); }); - it('fails closed if rollback cannot resume the same thread', async () => { + it('rolls back only after a confirmed fence and resumes through a fresh app-server', async () => { const original = appServer(); - const replacement = appServer({ - resumeThread: vi.fn(async () => Promise.reject(new Error('thread missing'))), + const replacement = appServer(); + const sealed = sealHandle(); + const { controller } = createController({ + sessions: [original, replacement], + checkpointAndSeal: async () => sealed, }); - const { controller } = createController({ sessions: [original, replacement] }); await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await controller.runTurn('remote turn'); + await controller.runTurn('remote'); + + const status = await controller.rollback(); - await expect(controller.rollback()).rejects.toThrow('Could not resume Codex thread thread-1'); - expect(controller.status()).toMatchObject({ phase: 'recovery_failed', execution: 'local' }); - await expect(controller.runTurn('must not continue')).rejects.toThrow('recovery_failed'); + expect(original.close).toHaveBeenCalled(); + expect(sealed.resumeLocal).toHaveBeenCalled(); + expect(replacement.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + expect(status).toMatchObject({ phase: 'local', generation: 2, execution: 'local' }); }); - it('recovers persisted remote state locally on controller restart', async () => { - const store = memoryStore({ - version: 1, - sessionId: 'session-1', - threadId: 'thread-1', - workspaceRoot: '/repo', - source, - generation: 7, - phase: 'remote', - controllerPid: 99, - socketPath: '/old.sock', - turnActive: false, - remote: { - sessionId: 'session-1', - generation: 7, - environmentId: 'env-7', - execServerUrl: 'wss://exec.agentrelay.test/old', - workspaceCwd: '/workspace', - expiresAt: '2026-08-23T13:00:00.000Z', - attached: true, - convergence, - }, - updatedAt: '2026-08-23T11:00:00.000Z', + it('persists fenced on restart and never constructs a local app-server after unconfirmed revoke', async () => { + const persisted = persistedRemote(); + persisted.remote!.expiresAt = '2026-08-23T11:00:00.000Z'; + const store = memoryStore(persisted); + const cloudClient = cloud({ + revoke: vi.fn(async () => Promise.reject(new Error('timeout'))), + status: vi.fn(async (input) => lifecycle(input, 'ready')), + }); + const { controller, createAppServer } = createController({ store, cloud: cloudClient }); + + await expect(controller.initialize()).rejects.toThrow('could not confirm Cloud fencing'); + + expect(createAppServer).not.toHaveBeenCalled(); + expect(store.value).toMatchObject({ phase: 'fenced', generation: 7 }); + }); + + it('resumes locally only after Cloud authoritatively confirms the persisted lease expired', async () => { + const expired = persistedRemote({ + remote: { ...persistedRemote().remote!, expiresAt: '2026-08-23T11:59:59.000Z' }, }); + const store = memoryStore(expired); const resumed = appServer(); - const cloudClient = cloud(); - const { controller } = createController({ store, sessions: [resumed], cloud: cloudClient }); + const cloudClient = cloud({ + revoke: vi.fn(async (input) => ({ ...lifecycle(input, 'expired'), status: 'expired' as const })), + }); + const { controller, resumePersistedLocalMount } = createController({ + store, + sessions: [resumed], + cloud: cloudClient, + }); await expect(controller.initialize()).resolves.toMatchObject({ - generation: 8, phase: 'local', - execution: 'local', + generation: 8, threadId: 'thread-1', }); - expect(cloudClient.revoke).toHaveBeenCalledWith( - expect.objectContaining({ sessionId: 'session-1', generation: 7 }) - ); + + expect(cloudClient.revoke).toHaveBeenCalled(); + expect(resumePersistedLocalMount).toHaveBeenCalledWith(expect.objectContaining({ source })); expect(resumed.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + expect(store.value?.source).toBeUndefined(); + expect(store.value?.mountRestore).toBeUndefined(); }); - it('persists recovery_failed when restart cannot resume the thread', async () => { - const store = memoryStore({ - version: 1, - sessionId: 'session-1', - threadId: 'thread-1', - workspaceRoot: '/repo', - source, - generation: 7, - phase: 'local', - controllerPid: 99, - socketPath: '/old.sock', - turnActive: false, - updatedAt: '2026-08-23T11:00:00.000Z', - }); + it('fails recovery if the same thread cannot be resumed after a confirmed fence', async () => { + const store = memoryStore(persistedRemote()); const resumed = appServer({ resumeThread: vi.fn(async () => Promise.reject(new Error('gone'))) }); const { controller } = createController({ store, sessions: [resumed] }); await expect(controller.initialize()).rejects.toThrow('Could not resume Codex thread thread-1'); + expect(store.value).toMatchObject({ phase: 'recovery_failed', generation: 7 }); }); it('does not adopt a persisted thread from a different workspace', async () => { - const store = memoryStore({ - version: 1, - sessionId: 'session-1', - threadId: 'thread-other', - workspaceRoot: '/different-repo', - source, - generation: 3, - phase: 'local', - controllerPid: 99, - socketPath: '/old.sock', - turnActive: false, - updatedAt: '2026-08-23T11:00:00.000Z', - }); - const session = appServer(); - const { controller, createAppServer } = createController({ store, sessions: [session] }); + const store = memoryStore(persistedRemote({ workspaceRoot: '/different-repo' })); + const { controller, createAppServer } = createController({ store }); await expect(controller.initialize()).rejects.toThrow('belongs to /different-repo'); expect(createAppServer).not.toHaveBeenCalled(); }); - it('revokes the exact generation on shutdown so prewarms and remote boxes cannot leak', async () => { - const session = appServer(); - const cloudClient = cloud(); - const { controller } = createController({ sessions: [session], cloud: cloudClient }); + it('marks shutdown fenced when revoke cannot be confirmed', async () => { + const original = appServer(); + const cloudClient = cloud({ + revoke: vi.fn(async () => Promise.reject(new Error('timeout'))), + status: vi.fn(async (input) => lifecycle(input, 'ready')), + }); + const { controller } = createController({ sessions: [original], cloud: cloudClient }); await controller.initialize(); await controller.close(); - expect(cloudClient.revoke).toHaveBeenCalledWith({ - sessionId: 'session-1', - generation: 1, - idempotencyKey: 'session-1:1:shutdown-revoke', + expect(original.close).toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'fenced', execution: 'fenced' }); + }); + + it('resumes and verifies the sealed local mount on ordinary shutdown after remote execution', async () => { + const sealed = sealHandle(); + const original = appServer(); + const { controller } = createController({ + sessions: [original], + checkpointAndSeal: async () => sealed, }); - expect(session.close).toHaveBeenCalled(); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + await controller.runTurn('remote turn'); + + await controller.close(); + + expect(original.close).toHaveBeenCalled(); + expect(sealed.resumeLocal).toHaveBeenCalled(); + expect(sealed.close).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2, execution: 'local' }); }); }); diff --git a/packages/cli/src/cli/lib/codex-live-controller.ts b/packages/cli/src/cli/lib/codex-live-controller.ts index 762a21bdf..c2138061f 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.ts @@ -5,6 +5,7 @@ import { randomUUID } from 'node:crypto'; import type { LiveTeleportCloudClient, LiveTeleportEnvironment, + LiveTeleportLifecycleStatus, LiveTeleportWorkspaceSource, } from '@agent-relay/cloud'; @@ -13,9 +14,11 @@ import type { CodexAppServerSession, CodexTurnResult } from './codex-app-server. export type CodexControllerPhase = | 'local' | 'teleport_pending' + | 'acquiring' + | 'verifying' | 'remote' | 'rolling_back' - | 'failed' + | 'fenced' | 'recovery_failed'; export type CodexTeleportRequest = { @@ -28,7 +31,8 @@ export type CodexControllerState = { sessionId: string; threadId: string; workspaceRoot: string; - source: LiveTeleportWorkspaceSource; + source?: LiveTeleportWorkspaceSource; + mountRestore?: CodexMountRestoreIdentity; generation: number; phase: CodexControllerPhase; controllerPid: number; @@ -38,22 +42,21 @@ export type CodexControllerState = { lastRequestId?: string; prewarmId?: string; prewarmStatus?: 'warming' | 'ready' | 'failed'; - remote?: Omit & { - /** Stored privately for controller recovery; never returned from public status. */ - execServerUrl: string; + remote?: LiveTeleportEnvironment & { + /** True only after the first remote turn/completed notification. */ attached: boolean; }; lastError?: string; updatedAt: string; }; -export type PublicCodexControllerStatus = Omit & { - execution: 'local' | 'cloud'; +export type PublicCodexControllerStatus = Omit & { + execution: 'local' | 'verifying' | 'cloud' | 'fenced'; controller: 'local'; remote?: Pick & { - attached: boolean; + attached: true; }; - workspaceSource: LiveTeleportWorkspaceSource['kind']; + workspaceSource: LiveTeleportWorkspaceSource['kind'] | 'unavailable'; }; export interface CodexControllerStateStore { @@ -82,11 +85,51 @@ export class FileCodexControllerStateStore implements CodexControllerStateStore } } +export type CodexWorkspaceSealInput = { + sessionId: string; + generation: number; + threadId: string; + workspaceRoot: string; + signal?: AbortSignal; +}; + +export type CodexWorkspaceSealHandle = { + source: LiveTeleportWorkspaceSource; + restore: CodexMountRestoreIdentity; + /** Restarts/thaws the exact sealed mount and resolves only after readiness. */ + resumeLocal(signal?: AbortSignal): Promise; + /** Releases resources when the controller is closing without local resume. */ + close(): Promise; +}; + +export type CodexMountRestoreIdentity = { + resumeId: string; + workspaceId: string; + localRoot: string; +}; + +export type CodexWorkspaceSealProvider = ( + input: CodexWorkspaceSealInput +) => Promise; +// Contract: a rejection (including AbortSignal) must leave the local mount +// ready, or restore it before rejecting. A fulfilled handle transfers mount +// lifecycle ownership to the controller until resumeLocal()/close(). + +export type CodexPersistedMountResumeProvider = ( + input: CodexWorkspaceSealInput & { + source: LiveTeleportWorkspaceSource; + restore: CodexMountRestoreIdentity; + } +) => Promise; + export type CodexLiveControllerDependencies = { cloud: LiveTeleportCloudClient; store: CodexControllerStateStore; createAppServer: () => Promise; probeCapability: () => Promise; + checkpointAndSeal: CodexWorkspaceSealProvider; + resumePersistedLocalMount: CodexPersistedMountResumeProvider; + sleep: (milliseconds: number) => Promise; now: () => Date; sessionId: () => string; pid: number; @@ -94,11 +137,31 @@ export type CodexLiveControllerDependencies = { export type CodexLiveControllerOptions = { workspaceRoot: string; - source: LiveTeleportWorkspaceSource; socketPath: string; model?: string; + lifecycleDeadlineMs?: number; + lifecyclePollIntervalMs?: number; }; +const DEFAULT_LIFECYCLE_DEADLINE_MS = 60_000; +const DEFAULT_LIFECYCLE_POLL_INTERVAL_MS = 500; +const TURN_BLOCKED_PHASES = new Set([ + 'recovery_failed', + 'fenced', + 'rolling_back', + 'acquiring', + 'verifying', +]); + +export class CodexTurnRecoveredError extends Error { + readonly recoveredLocally = true; + + constructor(message: string, options?: ErrorOptions) { + super(message, options); + this.name = 'CodexTurnRecoveredError'; + } +} + function errorMessage(error: unknown): string { return error instanceof Error ? error.message : String(error); } @@ -114,20 +177,28 @@ function isReadyStatus(value: unknown): boolean { } function publicStatus(state: CodexControllerState): PublicCodexControllerStatus { - const { source, remote, ...rest } = state; + const { source, remote, mountRestore: _mountRestore, ...rest } = state; + const execution = + state.phase === 'remote' + ? 'cloud' + : state.phase === 'verifying' + ? 'verifying' + : state.phase === 'fenced' || state.phase === 'recovery_failed' + ? 'fenced' + : 'local'; return { ...rest, controller: 'local', - execution: remote && state.phase === 'remote' ? 'cloud' : 'local', - workspaceSource: source.kind, - ...(remote + execution, + workspaceSource: source?.kind ?? 'unavailable', + ...(remote && state.phase === 'remote' && remote.attached ? { remote: { environmentId: remote.environmentId, generation: remote.generation, workspaceCwd: remote.workspaceCwd, expiresAt: remote.expiresAt, - attached: remote.attached, + attached: true, }, } : {}), @@ -135,12 +206,14 @@ function publicStatus(state: CodexControllerState): PublicCodexControllerStatus } /** - * Owns one local Codex app-server/thread. There is exactly one mutation seam: - * a queued teleport is consumed immediately before a turn/start request. + * Owns one local Codex app-server/thread. The queued teleport is consumed only + * immediately before turn/start. A remote lease is never followed by local + * execution until Cloud has positively confirmed revoke or expiry. */ export class CodexLiveController { private appServer: CodexAppServerSession | null = null; private state: CodexControllerState | null = null; + private sealedWorkspace: CodexWorkspaceSealHandle | null = null; constructor( private readonly options: CodexLiveControllerOptions, @@ -155,8 +228,6 @@ export class CodexLiveController { `The persisted managed Codex thread belongs to ${persisted.workspaceRoot}, not ${this.options.workspaceRoot}.` ); } - this.appServer = await this.deps.createAppServer(); - await this.appServer.initialize(); if (persisted) { this.state = { @@ -164,7 +235,6 @@ export class CodexLiveController { controllerPid: this.deps.pid, socketPath: this.options.socketPath, workspaceRoot: this.options.workspaceRoot, - source: this.options.source, turnActive: false, phase: 'rolling_back', pending: undefined, @@ -173,37 +243,51 @@ export class CodexLiveController { }; this.persist(); - if (persisted.remote) { - await this.deps.cloud - .revoke({ - sessionId: persisted.sessionId, - generation: persisted.generation, - idempotencyKey: `${persisted.sessionId}:${persisted.generation}:restart-revoke`, - }) - .catch(() => undefined); + try { + await this.confirmFence('restart-revoke'); + } catch (error) { + this.markFenced(`Controller restart could not confirm Cloud fencing: ${errorMessage(error)}`); + throw new Error(this.requireState().lastError, { cause: error }); } try { - await this.appServer.resumeThread({ + await this.resumeLocalMount(); + } catch (error) { + this.markRecoveryFailed( + `Cloud was fenced but the persisted Relayfile mount could not resume: ${errorMessage(error)}` + ); + throw new Error(this.requireState().lastError, { cause: error }); + } + + const replacement = await this.createInitializedAppServer(); + this.appServer = replacement; + try { + await replacement.resumeThread({ threadId: persisted.threadId, cwd: this.options.workspaceRoot, }); } catch (error) { - this.state.phase = 'recovery_failed'; - this.state.lastError = `Could not resume Codex thread ${persisted.threadId}: ${errorMessage(error)}`; - this.state.remote = undefined; - this.state.updatedAt = this.timestamp(); - this.persist(); - throw new Error(this.state.lastError, { cause: error }); + this.markRecoveryFailed( + `Could not resume Codex thread ${persisted.threadId}: ${errorMessage(error)}` + ); + await replacement.close().catch(() => undefined); + this.appServer = null; + throw new Error(this.requireState().lastError, { cause: error }); } - this.state.generation = persisted.generation + 1; - this.state.phase = 'local'; - this.state.remote = undefined; - this.state.updatedAt = this.timestamp(); + const state = this.requireState(); + state.generation = persisted.generation + 1; + state.phase = 'local'; + state.remote = undefined; + state.source = undefined; + state.mountRestore = undefined; + state.lastError = undefined; + state.updatedAt = this.timestamp(); this.persist(); } else { - const threadId = await this.appServer.startThread({ + const appServer = await this.createInitializedAppServer(); + this.appServer = appServer; + const threadId = await appServer.startThread({ cwd: this.options.workspaceRoot, ...(this.options.model ? { model: this.options.model } : {}), }); @@ -212,7 +296,6 @@ export class CodexLiveController { sessionId: this.deps.sessionId(), threadId, workspaceRoot: this.options.workspaceRoot, - source: this.options.source, generation: 1, phase: 'local', controllerPid: this.deps.pid, @@ -243,8 +326,9 @@ export class CodexLiveController { } if (state.phase === 'remote') throw new Error('This managed Codex session already executes in Cloud.'); if (state.pending) throw new Error(`Teleport request ${state.pending.requestId} is already pending.`); - if (state.phase !== 'local') + if (state.phase !== 'local') { throw new Error(`Cannot queue a teleport while the controller is ${state.phase}.`); + } state.pending = request; state.phase = 'teleport_pending'; @@ -256,31 +340,20 @@ export class CodexLiveController { async runTurn(text: string): Promise { const state = this.requireState(); - const appServer = this.requireAppServer(); if (state.turnActive) throw new Error('A Codex turn is already active.'); - if (state.phase === 'recovery_failed' || state.phase === 'failed' || state.phase === 'rolling_back') { + if (TURN_BLOCKED_PHASES.has(state.phase)) { throw new Error(`Cannot start a turn while the controller is ${state.phase}.`); } - // This is the concrete turn boundary: snapshot the pending request before - // marking the turn active. A request arriving after this snapshot is - // persisted for the following invocation, never spliced into this turn. + // Concrete turn boundary: snapshot before turnActive. A later request is + // persisted for the following invocation and cannot splice this turn. const pendingAtBoundary = state.pending; state.turnActive = true; state.updatedAt = this.timestamp(); this.persist(); try { if (pendingAtBoundary) await this.applyPendingTeleport(pendingAtBoundary); - const remote = state.phase === 'remote' ? state.remote : undefined; - const result = await appServer.runTurn({ - threadId: state.threadId, - text, - ...(remote && !remote.attached - ? { environment: { environmentId: remote.environmentId, cwd: remote.workspaceCwd } } - : {}), - }); - if (remote && !remote.attached) remote.attached = true; - return result; + return await this.executeTurn(text); } finally { state.turnActive = false; state.updatedAt = this.timestamp(); @@ -288,67 +361,90 @@ export class CodexLiveController { } } - async rollback(): Promise { + private async executeTurn(text: string): Promise { const state = this.requireState(); - if (state.turnActive) throw new Error('Rollback is only allowed at a Codex turn boundary.'); - const old = this.requireAppServer(); - state.phase = 'rolling_back'; - state.pending = undefined; - state.updatedAt = this.timestamp(); - this.persist(); - - if (state.remote) { - await this.deps.cloud.revoke({ - sessionId: state.sessionId, - generation: state.generation, - idempotencyKey: `${state.sessionId}:${state.generation}:rollback-revoke`, - }); - } - - await old.close(); - const replacement = await this.deps.createAppServer(); - this.appServer = replacement; - await replacement.initialize(); + const phase = state.phase as CodexControllerPhase; + const remote = phase === 'verifying' || phase === 'remote' ? state.remote : undefined; try { - await replacement.resumeThread({ threadId: state.threadId, cwd: state.workspaceRoot }); + const result = await this.requireAppServer().runTurn({ + threadId: state.threadId, + text, + execution: remote + ? { + kind: 'remote', + ...(!remote.attached + ? { environment: { environmentId: remote.environmentId, cwd: remote.workspaceCwd } } + : {}), + } + : { kind: 'local', workspaceRoot: state.workspaceRoot }, + }); + if (this.requireState().phase === 'verifying' && remote && !remote.attached) { + remote.attached = true; + state.phase = 'remote'; + state.lastError = undefined; + state.updatedAt = this.timestamp(); + this.persist(); + } + return result; } catch (error) { - state.phase = 'recovery_failed'; - state.remote = undefined; - state.lastError = `Could not resume Codex thread ${state.threadId}: ${errorMessage(error)}`; - state.updatedAt = this.timestamp(); - this.persist(); - throw new Error(state.lastError, { cause: error }); + if (this.requireState().phase !== 'verifying') throw error; + await this.recoverLocal('first-turn-failed-revoke', `First Cloud turn failed: ${errorMessage(error)}`); + throw new CodexTurnRecoveredError( + `First Cloud turn failed; Cloud fencing was confirmed and thread ${state.threadId} resumed locally: ${errorMessage(error)}`, + { cause: error } + ); } + } - state.generation += 1; - state.phase = 'local'; - state.remote = undefined; - state.lastError = undefined; - state.updatedAt = this.timestamp(); - this.persist(); - await this.startPrewarm(); + async rollback(): Promise { + const state = this.requireState(); + if (state.turnActive) throw new Error('Rollback is only allowed at a Codex turn boundary.'); + await this.recoverLocal('rollback-revoke', 'Operator requested local rollback.'); return this.status(); } async close(): Promise { const state = this.state; + let fenceConfirmed = false; if (state) { - await this.deps.cloud - .revoke({ - sessionId: state.sessionId, - generation: state.generation, - idempotencyKey: `${state.sessionId}:${state.generation}:shutdown-revoke`, - }) - .catch((error) => { - state.lastError = `Cloud shutdown revoke failed; the generation must expire server-side: ${errorMessage( - error - )}`; - state.updatedAt = this.timestamp(); - this.persist(); - }); + try { + await this.confirmFence('shutdown-revoke'); + fenceConfirmed = true; + } catch (error) { + this.markFenced(`Cloud shutdown revoke was not confirmed: ${errorMessage(error)}`); + } + } + try { + await this.appServer?.close(); + } catch (error) { + if (state && fenceConfirmed) { + this.markRecoveryFailed( + `Cloud was fenced during shutdown but the Codex app-server did not exit: ${errorMessage(error)}` + ); + } + throw error; } - await this.appServer?.close(); this.appServer = null; + if (state && fenceConfirmed && (this.sealedWorkspace || state.source || state.mountRestore)) { + try { + await this.resumeLocalMount(); + } catch (error) { + this.markRecoveryFailed( + `Cloud was fenced during shutdown but the Relayfile mount did not resume: ${errorMessage(error)}` + ); + throw new Error(state.lastError, { cause: error }); + } + state.generation += 1; + state.phase = 'local'; + state.remote = undefined; + state.source = undefined; + state.mountRestore = undefined; + state.prewarmId = undefined; + state.prewarmStatus = undefined; + state.lastError = undefined; + state.updatedAt = this.timestamp(); + this.persist(); + } } private async applyPendingTeleport(pendingAtBoundary: CodexTeleportRequest): Promise { @@ -359,70 +455,262 @@ export class CodexLiveController { throw new Error('Pending teleport generation became stale before the turn boundary.'); } + state.phase = 'acquiring'; + state.updatedAt = this.timestamp(); + this.persist(); try { - const environment = await this.deps.cloud.acquire({ - sessionId: state.sessionId, - threadId: state.threadId, - generation: state.generation, - workspaceRoot: state.workspaceRoot, - source: state.source, - ...(state.prewarmId ? { prewarmId: state.prewarmId } : {}), - idempotencyKey: `${state.sessionId}:${state.generation}:acquire`, - }); + // Keep the poll mount alive while Cloud warms. Admission is already + // stopped at this turn boundary, but the short-lived seal is minted only + // when Cloud is ready to consume it. + if (state.prewarmId && state.prewarmStatus !== 'ready') { + await this.waitForCloudReady(state.prewarmId); + } + + const sealedWorkspace = await this.withAbortableLifecycleDeadline( + (signal) => + this.deps.checkpointAndSeal({ + sessionId: state.sessionId, + generation: state.generation, + threadId: state.threadId, + workspaceRoot: state.workspaceRoot, + signal, + }), + 'Relayfile checkpoint-and-seal' + ); + const source = sealedWorkspace.source; + this.sealedWorkspace = sealedWorkspace; + this.assertSeal(source, sealedWorkspace.restore); + state.source = source; + state.mountRestore = sealedWorkspace.restore; + state.updatedAt = this.timestamp(); + this.persist(); + + const environment = await this.withAbortableLifecycleDeadline( + (signal) => + this.deps.cloud.acquire({ + sessionId: state.sessionId, + threadId: state.threadId, + generation: state.generation, + workspaceRoot: '/', + source, + ...(state.prewarmId ? { prewarmId: state.prewarmId } : {}), + idempotencyKey: `${state.sessionId}:${state.generation}:acquire`, + signal, + }), + 'Cloud acquire/convergence' + ); if (environment.sessionId !== state.sessionId || environment.generation !== state.generation) { throw new Error('Cloud returned a stale or cross-session live-teleport generation.'); } + state.remote = { ...environment, attached: false }; + state.pending = undefined; + state.phase = 'acquiring'; + state.updatedAt = this.timestamp(); + this.persist(); + await this.requireAppServer().addEnvironment({ environmentId: environment.environmentId, execServerUrl: environment.execServerUrl, - connectTimeoutMs: 10_000, + connectTimeoutMs: Math.min(10_000, this.lifecycleDeadlineMs()), }); - const environmentStatus = await this.requireAppServer().environmentStatus(environment.environmentId); - if (!isReadyStatus(environmentStatus)) { - throw new Error('Codex did not report the Cloud execution environment ready.'); - } + await this.waitForEnvironmentReady(environment.environmentId); - state.remote = { ...environment, attached: false }; - state.pending = undefined; - state.phase = 'remote'; + state.phase = 'verifying'; state.lastError = undefined; state.updatedAt = this.timestamp(); this.persist(); } catch (error) { - await this.deps.cloud - .revoke({ - sessionId: state.sessionId, - generation: state.generation, - idempotencyKey: `${state.sessionId}:${state.generation}:failed-acquire-revoke`, - }) - .catch(() => undefined); - state.pending = undefined; - state.phase = 'local'; - state.remote = undefined; - state.lastError = `Teleport failed before turn/start; execution remains local: ${errorMessage(error)}`; - state.updatedAt = this.timestamp(); - this.persist(); + await this.recoverLocal( + 'failed-acquire-revoke', + `Teleport failed before the first Cloud turn completed: ${errorMessage(error)}` + ); + throw new CodexTurnRecoveredError( + `Teleport failed; Cloud fencing was confirmed and execution resumed locally: ${errorMessage(error)}`, + { cause: error } + ); + } + } + + private async recoverLocal(reason: string, context: string): Promise { + const state = this.requireState(); + const old = this.requireAppServer(); + state.phase = 'rolling_back'; + state.pending = undefined; + state.updatedAt = this.timestamp(); + this.persist(); + + try { + await this.confirmFence(reason); + } catch (error) { + this.markFenced(`${context} Cloud fencing could not be confirmed: ${errorMessage(error)}`); + await old.close().catch((closeError) => { + state.lastError = `${state.lastError} Old controller shutdown also failed: ${errorMessage(closeError)}`; + state.updatedAt = this.timestamp(); + this.persist(); + }); + this.appServer = null; + throw new Error(state.lastError, { cause: error }); + } + + try { + await old.close(); + } catch (error) { + this.markRecoveryFailed( + `Cloud was fenced but the previous Codex app-server did not exit: ${errorMessage(error)}` + ); + throw new Error(state.lastError, { cause: error }); + } + this.appServer = null; + try { + await this.resumeLocalMount(); + } catch (error) { + this.markRecoveryFailed( + `Cloud was fenced but the sealed Relayfile mount could not resume: ${errorMessage(error)}` + ); + throw new Error(state.lastError, { cause: error }); + } + let replacement: CodexAppServerSession; + try { + replacement = await this.createInitializedAppServer(); + } catch (error) { + this.markRecoveryFailed( + `Cloud was fenced and the mount resumed, but a local Codex app-server could not start: ${errorMessage(error)}` + ); + throw new Error(state.lastError, { cause: error }); + } + this.appServer = replacement; + try { + await replacement.resumeThread({ threadId: state.threadId, cwd: state.workspaceRoot }); + } catch (error) { + this.markRecoveryFailed(`Could not resume Codex thread ${state.threadId}: ${errorMessage(error)}`); + await replacement.close().catch(() => undefined); + this.appServer = null; throw new Error(state.lastError, { cause: error }); } + + state.generation += 1; + state.phase = 'local'; + state.remote = undefined; + state.source = undefined; + state.mountRestore = undefined; + state.prewarmId = undefined; + state.prewarmStatus = undefined; + state.lastError = context; + state.updatedAt = this.timestamp(); + this.persist(); + await this.startPrewarm(); + } + + private async confirmFence(reason: string): Promise { + const state = this.requireState(); + + try { + const revoked = await this.withAbortableLifecycleDeadline( + (signal) => + this.deps.cloud.revoke({ + sessionId: state.sessionId, + generation: state.generation, + idempotencyKey: `${state.sessionId}:${state.generation}:${reason}`, + signal, + }), + 'Cloud revoke confirmation' + ); + this.assertLifecycleIdentity(revoked); + if (revoked.status === 'revoked' || revoked.status === 'expired') return; + } catch (revokeError) { + try { + const status = await this.withAbortableLifecycleDeadline( + (signal) => + this.deps.cloud.status({ + sessionId: state.sessionId, + generation: state.generation, + signal, + }), + 'Cloud fence status confirmation' + ); + this.assertLifecycleIdentity(status); + if (status.status === 'revoked' || status.status === 'expired') return; + } catch (statusError) { + throw new Error( + `revoke failed (${errorMessage(revokeError)}); status was unconfirmed (${errorMessage(statusError)})`, + { cause: statusError } + ); + } + throw revokeError; + } + throw new Error('Cloud revoke returned without a terminal fence state.'); + } + + private async waitForCloudReady(prewarmId: string): Promise { + const state = this.requireState(); + const attempts = this.lifecycleAttempts(); + await this.withAbortableLifecycleDeadline(async (signal) => { + for (let attempt = 0; attempt < attempts; attempt += 1) { + const status = await this.deps.cloud.status({ + sessionId: state.sessionId, + generation: state.generation, + prewarmId, + signal, + }); + this.assertLifecycleIdentity(status); + if (status.prewarmId && status.prewarmId !== prewarmId) { + throw new Error('Cloud returned a stale or cross-prewarm lifecycle status.'); + } + if (status.status === 'ready') { + state.prewarmStatus = 'ready'; + state.updatedAt = this.timestamp(); + this.persist(); + return; + } + if (status.status !== 'warming') { + throw new Error(`Cloud prewarm entered terminal state ${status.status}.`); + } + await this.deps.sleep( + Math.min(status.retryAfterMs ?? this.lifecyclePollIntervalMs(), this.lifecyclePollIntervalMs()) + ); + } + throw new Error('Cloud prewarm did not converge before the lifecycle deadline.'); + }, 'Cloud prewarm convergence'); + } + + private async waitForEnvironmentReady(environmentId: string): Promise { + const attempts = this.lifecycleAttempts(); + await this.withLifecycleDeadline( + (async () => { + for (let attempt = 0; attempt < attempts; attempt += 1) { + const environmentStatus = await this.requireAppServer().environmentStatus(environmentId); + if (isReadyStatus(environmentStatus)) return; + await this.deps.sleep(this.lifecyclePollIntervalMs()); + } + throw new Error('Codex did not report the Cloud execution environment ready before the deadline.'); + })(), + 'Codex environment readiness' + ); } private async startPrewarm(): Promise { const state = this.requireState(); try { - const prewarm = await this.deps.cloud.prewarm({ - sessionId: state.sessionId, - generation: state.generation, - workspaceRoot: state.workspaceRoot, - source: state.source, - idempotencyKey: `${state.sessionId}:${state.generation}:prewarm`, - }); + const prewarm = await this.withAbortableLifecycleDeadline( + (signal) => + this.deps.cloud.prewarm({ + sessionId: state.sessionId, + generation: state.generation, + workspaceRoot: '/', + idempotencyKey: `${state.sessionId}:${state.generation}:prewarm`, + signal, + }), + 'Cloud prewarm' + ); if (prewarm.generation !== state.generation) { throw new Error('Cloud returned a stale prewarm generation.'); } + state.source = undefined; state.prewarmId = prewarm.prewarmId; state.prewarmStatus = prewarm.status; } catch (error) { + state.source = undefined; state.prewarmId = undefined; state.prewarmStatus = 'failed'; state.lastError = `Cloud prewarm unavailable; local Codex remains usable: ${errorMessage(error)}`; @@ -431,6 +719,154 @@ export class CodexLiveController { this.persist(); } + private assertSeal(source: LiveTeleportWorkspaceSource, restore: CodexMountRestoreIdentity): void { + if ( + source.kind !== 'relayfile-checkpoint-seal' || + !source.receipt || + typeof source.receipt !== 'object' || + Array.isArray(source.receipt) + ) { + throw new Error('Relayfile checkpoint-and-seal provider returned an invalid proof.'); + } + if ( + typeof restore.resumeId !== 'string' || + restore.resumeId.length === 0 || + typeof restore.workspaceId !== 'string' || + restore.workspaceId.length === 0 || + path.resolve(restore.localRoot) !== path.resolve(this.options.workspaceRoot) + ) { + throw new Error('Relayfile checkpoint-and-seal provider returned an invalid restore identity.'); + } + } + + private async resumeLocalMount(): Promise { + const state = this.requireState(); + if (this.sealedWorkspace) { + await this.withAbortableLifecycleDeadline( + (signal) => this.sealedWorkspace!.resumeLocal(signal), + 'Relayfile local mount resume/readiness' + ); + this.sealedWorkspace = null; + return; + } + if (!state.source || !state.mountRestore) { + if (state.remote) { + throw new Error('Persisted remote execution has no Relayfile seal identity to restore.'); + } + return; + } + const source = state.source; + const restore = state.mountRestore; + await this.withAbortableLifecycleDeadline( + (signal) => + this.deps.resumePersistedLocalMount({ + sessionId: state.sessionId, + generation: state.generation, + threadId: state.threadId, + workspaceRoot: state.workspaceRoot, + source, + restore, + signal, + }), + 'Persisted Relayfile local mount resume/readiness' + ); + } + + private assertLifecycleIdentity(status: LiveTeleportLifecycleStatus): void { + const state = this.requireState(); + if (status.sessionId !== state.sessionId || status.generation !== state.generation) { + throw new Error('Cloud returned a stale or cross-session lifecycle status.'); + } + } + + private async createInitializedAppServer(): Promise { + const appServer = await this.deps.createAppServer(); + await appServer.initialize(); + return appServer; + } + + private markFenced(message: string): void { + const state = this.requireState(); + state.phase = 'fenced'; + state.pending = undefined; + state.lastError = message; + state.updatedAt = this.timestamp(); + this.persist(); + } + + private markRecoveryFailed(message: string): void { + const state = this.requireState(); + state.phase = 'recovery_failed'; + state.remote = undefined; + state.lastError = message; + state.updatedAt = this.timestamp(); + this.persist(); + } + + private withAbortableLifecycleDeadline( + operation: (signal: AbortSignal) => Promise, + operationName: string + ): Promise { + const controller = new AbortController(); + const timeoutMs = this.lifecycleDeadlineMs(); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + controller.abort(); + reject(new Error(`${operationName} exceeded the ${timeoutMs}ms lifecycle deadline.`)); + }, timeoutMs); + let promise: Promise; + try { + promise = operation(controller.signal); + } catch (error) { + clearTimeout(timer); + reject(error); + return; + } + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error) => { + clearTimeout(timer); + reject(error); + } + ); + }); + } + + private withLifecycleDeadline(promise: Promise, operation: string): Promise { + const timeoutMs = this.lifecycleDeadlineMs(); + return new Promise((resolve, reject) => { + const timer = setTimeout( + () => reject(new Error(`${operation} exceeded the ${timeoutMs}ms lifecycle deadline.`)), + timeoutMs + ); + promise.then( + (value) => { + clearTimeout(timer); + resolve(value); + }, + (error) => { + clearTimeout(timer); + reject(error); + } + ); + }); + } + + private lifecycleDeadlineMs(): number { + return this.options.lifecycleDeadlineMs ?? DEFAULT_LIFECYCLE_DEADLINE_MS; + } + + private lifecyclePollIntervalMs(): number { + return this.options.lifecyclePollIntervalMs ?? DEFAULT_LIFECYCLE_POLL_INTERVAL_MS; + } + + private lifecycleAttempts(): number { + return Math.max(1, Math.ceil(this.lifecycleDeadlineMs() / this.lifecyclePollIntervalMs())); + } + private persist(): void { this.deps.store.write(this.requireState()); } diff --git a/packages/cli/src/cli/lib/codex-relayfile-seal.test.ts b/packages/cli/src/cli/lib/codex-relayfile-seal.test.ts new file mode 100644 index 000000000..baf8d4df7 --- /dev/null +++ b/packages/cli/src/cli/lib/codex-relayfile-seal.test.ts @@ -0,0 +1,186 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { + createRelayfileSealLifecycle, + runRelayfileLifecycleJsonCommand, + type RelayfileLifecycleCommandRunner, +} from './codex-relayfile-seal.js'; + +function receipt(overrides: Record = {}) { + return { + sealId: 'cps_123', + sealToken: 'one-use-opaque-token', + workspaceId: 'ws_123', + root: '/', + sessionId: 'session-1', + generation: 4, + digest: `sha256:${'a'.repeat(64)}`, + workspaceRevision: 'rev-40', + eventCursor: 'evt-50', + issuedAt: '2026-08-23T12:00:00.000Z', + expiresAt: '2026-08-23T12:01:00.000Z', + ...overrides, + }; +} + +function checkpointOutput(overrides: Record = {}) { + return { + version: 1, + kind: 'relayfile-checkpoint-seal', + workspaceId: 'ws_123', + localRoot: '/repo', + sessionId: 'session-1', + generation: 4, + receipt: receipt(), + resumeId: 'resume_opaque_123', + sealedAt: '2026-08-23T12:00:00.000Z', + ...overrides, + }; +} + +function resumeOutput(overrides: Record = {}) { + return { + version: 1, + kind: 'relayfile-resume-seal', + workspaceId: 'ws_123', + localRoot: '/repo', + resumeId: 'resume_opaque_123', + status: 'ready', + resumedAt: '2026-08-23T12:00:10.000Z', + ...overrides, + }; +} + +describe('createRelayfileSealLifecycle', () => { + it('binds the exact checkpoint command and keeps resumeId out of argv', async () => { + const runner = vi + .fn() + .mockResolvedValueOnce(checkpointOutput()) + .mockResolvedValueOnce(resumeOutput()); + const lifecycle = createRelayfileSealLifecycle({ binary: 'relayfile-test', runner }); + + const handle = await lifecycle.checkpointAndSeal({ + sessionId: 'session-1', + generation: 4, + threadId: 'thread-1', + workspaceRoot: '/repo', + }); + + expect(runner).toHaveBeenNthCalledWith(1, { + binary: 'relayfile-test', + args: [ + 'mount', + 'checkpoint-seal', + '--root', + '/repo', + '--session', + 'session-1', + '--generation', + '4', + '--timeout', + '30s', + '--ttl', + '60s', + '--json', + ], + signal: undefined, + }); + expect(handle.source).toEqual({ kind: 'relayfile-checkpoint-seal', receipt: receipt() }); + expect(handle.restore).toEqual({ + resumeId: 'resume_opaque_123', + workspaceId: 'ws_123', + localRoot: '/repo', + }); + + await handle.resumeLocal(); + + const resumeCall = runner.mock.calls[1]![0]; + expect(resumeCall.args).toEqual(['mount', 'resume-seal', '--root', '/repo', '--json']); + expect(resumeCall.args.join(' ')).not.toContain('resume_opaque_123'); + expect(resumeCall.stdin).toBe(`${JSON.stringify({ resumeId: 'resume_opaque_123' })}\n`); + }); + + it('uses the same stdin-only resume contract after controller restart', async () => { + const runner = vi.fn(async () => resumeOutput()); + const lifecycle = createRelayfileSealLifecycle({ runner }); + + await lifecycle.resumePersistedLocalMount({ + sessionId: 'session-1', + generation: 4, + threadId: 'thread-1', + workspaceRoot: '/repo', + source: { kind: 'relayfile-checkpoint-seal', receipt: receipt() }, + restore: { resumeId: 'resume_opaque_123', workspaceId: 'ws_123', localRoot: '/repo' }, + }); + + expect(runner).toHaveBeenCalledWith( + expect.objectContaining({ + args: ['mount', 'resume-seal', '--root', '/repo', '--json'], + stdin: `${JSON.stringify({ resumeId: 'resume_opaque_123' })}\n`, + }) + ); + }); + + it.each([ + ['wrong local root', checkpointOutput({ localRoot: '/other' })], + ['wrong session', checkpointOutput({ sessionId: 'session-other' })], + ['wrong generation', checkpointOutput({ generation: 5 })], + ['non-logical receipt root', checkpointOutput({ receipt: receipt({ root: '/repo' }) })], + ['caller-shaped digest', checkpointOutput({ receipt: receipt({ digest: 'caller-says-ok' }) })], + ['missing one-use token', checkpointOutput({ receipt: receipt({ sealToken: '' }) })], + ])('fails closed on %s', async (_name, output) => { + const lifecycle = createRelayfileSealLifecycle({ runner: async () => output }); + await expect( + lifecycle.checkpointAndSeal({ + sessionId: 'session-1', + generation: 4, + threadId: 'thread-1', + workspaceRoot: '/repo', + }) + ).rejects.toThrow(/relayfile|checkpoint/); + }); + + it('rejects a resume response until readiness and identity are exact', async () => { + const lifecycle = createRelayfileSealLifecycle({ + runner: vi + .fn() + .mockResolvedValueOnce(checkpointOutput()) + .mockResolvedValueOnce(resumeOutput({ status: 'warming' })), + }); + const handle = await lifecycle.checkpointAndSeal({ + sessionId: 'session-1', + generation: 4, + threadId: 'thread-1', + workspaceRoot: '/repo', + }); + await expect(handle.resumeLocal()).rejects.toThrow('did not confirm mount readiness'); + }); +}); + +describe('runRelayfileLifecycleJsonCommand', () => { + it('kills an aborted child and rejects with AbortError', async () => { + const controller = new AbortController(); + const running = runRelayfileLifecycleJsonCommand({ + binary: process.execPath, + args: ['-e', 'setInterval(() => {}, 1000)'], + signal: controller.signal, + }); + setTimeout(() => controller.abort(), 10); + + await expect(running).rejects.toMatchObject({ name: 'AbortError' }); + }); + + it('reports only the stable error code and never echoes arbitrary stderr', async () => { + const error = await runRelayfileLifecycleJsonCommand({ + binary: process.execPath, + args: [ + '-e', + "process.stderr.write('secret-value\\nerror: checkpoint_fuse_unsupported: hidden'); process.exit(2)", + ], + }).catch((caught: unknown) => caught); + + expect(String(error)).toContain('checkpoint_fuse_unsupported'); + expect(String(error)).not.toContain('secret-value'); + expect(String(error)).not.toContain('hidden'); + }); +}); diff --git a/packages/cli/src/cli/lib/codex-relayfile-seal.ts b/packages/cli/src/cli/lib/codex-relayfile-seal.ts new file mode 100644 index 000000000..1ddd81c7b --- /dev/null +++ b/packages/cli/src/cli/lib/codex-relayfile-seal.ts @@ -0,0 +1,287 @@ +import path from 'node:path'; +import { spawn } from 'node:child_process'; + +import type { LiveTeleportWorkspaceSource } from '@agent-relay/cloud'; + +import type { + CodexMountRestoreIdentity, + CodexPersistedMountResumeProvider, + CodexWorkspaceSealHandle, + CodexWorkspaceSealProvider, +} from './codex-live-controller.js'; + +const MAX_COMMAND_OUTPUT_BYTES = 1024 * 1024; +const FORCE_KILL_AFTER_MS = 1_000; + +export type RelayfileLifecycleCommandInput = { + binary: string; + args: string[]; + stdin?: string; + signal?: AbortSignal; +}; + +export type RelayfileLifecycleCommandRunner = (input: RelayfileLifecycleCommandInput) => Promise; + +export type RelayfileSealLifecycle = { + checkpointAndSeal: CodexWorkspaceSealProvider; + resumePersistedLocalMount: CodexPersistedMountResumeProvider; +}; + +type JsonObject = Record; + +function isObject(value: unknown): value is JsonObject { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function requiredString(value: unknown, field: string): string { + if (typeof value !== 'string' || !value.trim()) { + throw new Error(`relayfile lifecycle response is missing ${field}.`); + } + return value.trim(); +} + +function requiredExactLocalRoot(value: unknown, workspaceRoot: string): string { + const localRoot = requiredString(value, 'localRoot'); + if ( + !path.isAbsolute(localRoot) || + localRoot !== workspaceRoot || + path.resolve(localRoot) !== workspaceRoot + ) { + throw new Error('relayfile lifecycle response returned a mismatched local root.'); + } + return localRoot; +} + +function requiredResumeId(value: unknown): string { + const resumeId = requiredString(value, 'resumeId'); + if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,511}$/.test(resumeId)) { + throw new Error('relayfile lifecycle response returned an invalid resumeId.'); + } + return resumeId; +} + +function requiredTimestamp(value: unknown, field: string): string { + const timestamp = requiredString(value, field); + if (Number.isNaN(Date.parse(timestamp))) { + throw new Error(`relayfile lifecycle response returned an invalid ${field}.`); + } + return timestamp; +} + +function validateReceipt( + value: unknown, + input: { sessionId: string; generation: number; workspaceId: string } +): JsonObject { + if (!isObject(value)) throw new Error('relayfile checkpoint response is missing receipt.'); + const workspaceId = requiredString(value.workspaceId, 'receipt.workspaceId'); + const sessionId = requiredString(value.sessionId, 'receipt.sessionId'); + const generation = value.generation; + const digest = requiredString(value.digest, 'receipt.digest'); + requiredString(value.sealId, 'receipt.sealId'); + requiredString(value.sealToken, 'receipt.sealToken'); + if (requiredString(value.root, 'receipt.root') !== '/') { + throw new Error('relayfile checkpoint receipt root must be logical / for live teleport v1.'); + } + requiredString(value.workspaceRevision, 'receipt.workspaceRevision'); + requiredString(value.eventCursor, 'receipt.eventCursor'); + requiredTimestamp(value.issuedAt, 'receipt.issuedAt'); + requiredTimestamp(value.expiresAt, 'receipt.expiresAt'); + if ( + workspaceId !== input.workspaceId || + sessionId !== input.sessionId || + generation !== input.generation || + !/^sha256:[a-f0-9]{64}$/i.test(digest) + ) { + throw new Error('relayfile checkpoint receipt is unbound or has an invalid digest.'); + } + return value; +} + +function validateCheckpointOutput( + value: unknown, + input: { workspaceRoot: string; sessionId: string; generation: number } +): { source: LiveTeleportWorkspaceSource; restore: CodexMountRestoreIdentity } { + if (!isObject(value) || value.version !== 1 || value.kind !== 'relayfile-checkpoint-seal') { + throw new Error('relayfile checkpoint command returned an invalid response contract.'); + } + const workspaceId = requiredString(value.workspaceId, 'workspaceId'); + const localRoot = requiredExactLocalRoot(value.localRoot, input.workspaceRoot); + if ( + requiredString(value.sessionId, 'sessionId') !== input.sessionId || + value.generation !== input.generation + ) { + throw new Error('relayfile checkpoint command returned a stale or cross-session response.'); + } + requiredTimestamp(value.sealedAt, 'sealedAt'); + const resumeId = requiredResumeId(value.resumeId); + const receipt = validateReceipt(value.receipt, { + sessionId: input.sessionId, + generation: input.generation, + workspaceId, + }); + return { + source: { kind: 'relayfile-checkpoint-seal', receipt }, + restore: { resumeId, workspaceId, localRoot }, + }; +} + +function validateResumeOutput(value: unknown, restore: CodexMountRestoreIdentity): void { + if ( + !isObject(value) || + value.version !== 1 || + value.kind !== 'relayfile-resume-seal' || + value.status !== 'ready' + ) { + throw new Error('relayfile resume command did not confirm mount readiness.'); + } + if ( + requiredString(value.workspaceId, 'workspaceId') !== restore.workspaceId || + requiredExactLocalRoot(value.localRoot, restore.localRoot) !== restore.localRoot || + requiredResumeId(value.resumeId) !== restore.resumeId + ) { + throw new Error('relayfile resume command returned a mismatched restore identity.'); + } + requiredTimestamp(value.resumedAt, 'resumedAt'); +} + +function abortError(): Error { + const error = new Error('relayfile lifecycle command was aborted.'); + error.name = 'AbortError'; + return error; +} + +export const runRelayfileLifecycleJsonCommand: RelayfileLifecycleCommandRunner = async (input) => { + if (input.signal?.aborted) throw abortError(); + const child = spawn(input.binary, input.args, { stdio: ['pipe', 'pipe', 'pipe'] }); + + return new Promise((resolve, reject) => { + let stdout = ''; + let stderr = ''; + let outputError: Error | undefined; + let aborted = false; + let forceKillTimer: NodeJS.Timeout | undefined; + + const append = (current: string, chunk: Buffer | string): string => { + const next = current + chunk.toString(); + if (Buffer.byteLength(next, 'utf8') > MAX_COMMAND_OUTPUT_BYTES && !outputError) { + outputError = new Error('relayfile lifecycle command output exceeded 1 MiB.'); + child.kill('SIGKILL'); + } + return next.slice(-MAX_COMMAND_OUTPUT_BYTES); + }; + const cleanup = () => { + input.signal?.removeEventListener('abort', onAbort); + if (forceKillTimer) clearTimeout(forceKillTimer); + }; + const onAbort = () => { + if (aborted) return; + aborted = true; + child.kill('SIGTERM'); + forceKillTimer = setTimeout(() => child.kill('SIGKILL'), FORCE_KILL_AFTER_MS); + forceKillTimer.unref?.(); + }; + + child.stdout.on('data', (chunk: Buffer | string) => { + stdout = append(stdout, chunk); + }); + child.stderr.on('data', (chunk: Buffer | string) => { + stderr = append(stderr, chunk); + }); + child.once('error', (error) => { + cleanup(); + reject(new Error(`Could not start relayfile lifecycle command: ${error.message}`)); + }); + child.once('exit', (code, signal) => { + cleanup(); + if (aborted) { + reject(abortError()); + return; + } + if (outputError) { + reject(outputError); + return; + } + if (code !== 0) { + const stableCode = /(?:^|\n)error:\s*([a-z0-9_]+):/i.exec(stderr)?.[1]; + reject( + new Error( + `relayfile lifecycle command failed (${signal ?? `exit ${code ?? 'unknown'}`}${ + stableCode ? `, ${stableCode}` : '' + }).` + ) + ); + return; + } + if (stderr.trim()) { + reject(new Error('relayfile lifecycle command produced unexpected stderr on success.')); + return; + } + try { + resolve(JSON.parse(stdout) as unknown); + } catch { + reject(new Error('relayfile lifecycle command returned malformed JSON.')); + } + }); + + input.signal?.addEventListener('abort', onAbort, { once: true }); + if (input.signal?.aborted) onAbort(); + if (input.stdin === undefined) child.stdin.end(); + else child.stdin.end(input.stdin); + }); +}; + +export function createRelayfileSealLifecycle( + options: { + binary?: string; + runner?: RelayfileLifecycleCommandRunner; + } = {} +): RelayfileSealLifecycle { + const binary = options.binary ?? 'relayfile'; + const runner = options.runner ?? runRelayfileLifecycleJsonCommand; + + const resume = async (restore: CodexMountRestoreIdentity, signal?: AbortSignal): Promise => { + const output = await runner({ + binary, + args: ['mount', 'resume-seal', '--root', restore.localRoot, '--json'], + stdin: `${JSON.stringify({ resumeId: restore.resumeId })}\n`, + signal, + }); + validateResumeOutput(output, restore); + }; + + return { + checkpointAndSeal: async (input): Promise => { + const output = await runner({ + binary, + args: [ + 'mount', + 'checkpoint-seal', + '--root', + input.workspaceRoot, + '--session', + input.sessionId, + '--generation', + String(input.generation), + '--timeout', + '30s', + '--ttl', + '60s', + '--json', + ], + signal: input.signal, + }); + const sealed = validateCheckpointOutput(output, input); + return { + ...sealed, + resumeLocal: (signal) => resume(sealed.restore, signal), + close: async () => undefined, + }; + }, + resumePersistedLocalMount: async (input): Promise => { + if (input.restore.localRoot !== input.workspaceRoot) { + throw new Error('Persisted relayfile restore identity does not match the managed workspace root.'); + } + await resume(input.restore, input.signal); + }, + }; +} diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index 774078b80..ee2daed1c 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -64,6 +64,9 @@ export { type LiveTeleportConvergenceWatermark, type LiveTeleportConvergenceProof, type LiveTeleportRevokeInput, + type LiveTeleportStatusInput, + type LiveTeleportLifecycleStatus, + type LiveTeleportRevocation, } from './live-teleport.js'; export { diff --git a/packages/cloud/src/live-teleport.test.ts b/packages/cloud/src/live-teleport.test.ts index 72ebfb013..d2cd911fa 100644 --- a/packages/cloud/src/live-teleport.test.ts +++ b/packages/cloud/src/live-teleport.test.ts @@ -7,7 +7,10 @@ const input = { threadId: 'thread-1', generation: 2, workspaceRoot: '/workspace', - source: { kind: 'relayfile-mount' as const, mountStatePath: '/workspace/.relayfile-mount-state.json' }, + source: { + kind: 'relayfile-checkpoint-seal' as const, + receipt: { sealId: 'seal-1', sealToken: 'opaque' }, + }, idempotencyKey: 'session-1:2:acquire', }; @@ -42,16 +45,19 @@ describe('CloudLiveTeleportClient', () => { sessionId: 'session-1', generation: 2, environmentId: 'env-2', - execServerUrl: 'wss://exec.agentrelay.test/t/session-1/g/2', + connectPath: '/api/v1/live-teleports/connect/session-1/g/2?ticket=opaque', workspaceCwd: '/workspace', expiresAt: '2026-08-23T12:00:00.000Z', convergence: convergence(), }) ); - await expect(new CloudLiveTeleportClient(fetcher).acquire(input)).resolves.toMatchObject({ + await expect( + new CloudLiveTeleportClient(fetcher, 'https://cloud.agentrelay.test').acquire(input) + ).resolves.toMatchObject({ environmentId: 'env-2', generation: 2, + execServerUrl: 'wss://cloud.agentrelay.test/api/v1/live-teleports/connect/session-1/g/2?ticket=opaque', }); expect(fetcher).toHaveBeenCalledWith( '/api/v1/live-teleports/acquire', @@ -60,77 +66,111 @@ describe('CloudLiveTeleportClient', () => { }); it('fails closed if Cloud exposes a provider credential or URL', async () => { - const client = new CloudLiveTeleportClient(async () => - Response.json({ - sessionId: 'session-1', - generation: 2, - environmentId: 'env-2', - execServerUrl: 'wss://exec.agentrelay.test/ticket', - workspaceCwd: '/workspace', - expiresAt: '2026-08-23T12:00:00.000Z', - convergence: convergence(), - providerUrl: 'wss://provider.invalid/raw', - trafficAccessToken: 'secret', - }) + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + expiresAt: '2026-08-23T12:00:00.000Z', + convergence: convergence(), + providerUrl: 'wss://provider.invalid/raw', + trafficAccessToken: 'secret', + }), + 'https://cloud.agentrelay.test' ); await expect(client.acquire(input)).rejects.toThrow('forbidden provider field'); }); - it('rejects a non-TLS execution address', async () => { - const client = new CloudLiveTeleportClient(async () => - Response.json({ - sessionId: 'session-1', - generation: 2, - environmentId: 'env-2', - execServerUrl: 'ws://127.0.0.1:4500', - workspaceCwd: '/workspace', - expiresAt: '2026-08-23T12:00:00.000Z', - convergence: convergence(), - }) + it('rejects an arbitrary execution URL even when it points at Cloud', async () => { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + environmentId: 'env-2', + execServerUrl: 'ws://127.0.0.1:4500', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + expiresAt: '2026-08-23T12:00:00.000Z', + convergence: convergence(), + }), + 'https://cloud.agentrelay.test' ); - await expect(client.acquire(input)).rejects.toThrow('Cloud WSS bridge'); + await expect(client.acquire(input)).rejects.toThrow('must not return an arbitrary execServerUrl'); + }); + + it('rejects absolute, cross-origin, and traversal connect paths', async () => { + for (const connectPath of [ + 'wss://provider.invalid/raw', + '//provider.invalid/raw', + '/api/v1/live-teleports/connect/../../provider', + ]) { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + environmentId: 'env-2', + connectPath, + workspaceCwd: '/workspace', + expiresAt: '2026-08-23T12:00:00.000Z', + convergence: convergence(), + }), + 'https://cloud.agentrelay.test' + ); + await expect(client.acquire(input)).rejects.toThrow(/connectPath/); + } }); it('rejects a time-based convergence claim whose destination hash differs', async () => { - const client = new CloudLiveTeleportClient(async () => - Response.json({ - sessionId: 'session-1', - generation: 2, - environmentId: 'env-2', - execServerUrl: 'wss://exec.agentrelay.test/ticket', - workspaceCwd: '/workspace', - expiresAt: '2026-08-23T12:00:00.000Z', - convergence: convergence({ destinationSha: 'f'.repeat(64) }), - }) + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + expiresAt: '2026-08-23T12:00:00.000Z', + convergence: convergence({ destinationSha: 'f'.repeat(64) }), + }), + 'https://cloud.agentrelay.test' ); await expect(client.acquire(input)).rejects.toThrow('non-converged hash/cursor proof'); }); it('rejects matching hashes when the destination outbox is not drained', async () => { - const client = new CloudLiveTeleportClient(async () => - Response.json({ - sessionId: 'session-1', - generation: 2, - environmentId: 'env-2', - execServerUrl: 'wss://exec.agentrelay.test/ticket', - workspaceCwd: '/workspace', - expiresAt: '2026-08-23T12:00:00.000Z', - convergence: convergence({ pendingWriteback: 1 }), - }) + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + expiresAt: '2026-08-23T12:00:00.000Z', + convergence: convergence({ pendingWriteback: 1 }), + }), + 'https://cloud.agentrelay.test' ); await expect(client.acquire(input)).rejects.toThrow('non-converged hash/cursor proof'); }); it('does not echo a Cloud error containing an opaque bridge or provider secret', async () => { - const client = new CloudLiveTeleportClient(async () => - Response.json( - { error: 'provider rejected https://provider.invalid/?token=opaque-secret-value' }, - { status: 502, statusText: 'Bad Gateway' } - ) + const client = new CloudLiveTeleportClient( + async () => + Response.json( + { error: 'provider rejected https://provider.invalid/?token=opaque-secret-value' }, + { status: 502, statusText: 'Bad Gateway' } + ), + 'https://cloud.agentrelay.test' ); const error = await client.acquire(input).catch((caught: unknown) => caught); @@ -138,4 +178,38 @@ describe('CloudLiveTeleportClient', () => { expect(String(error)).not.toContain('opaque-secret-value'); expect(String(error)).not.toContain('provider.invalid'); }); + + it('parses bounded lifecycle polling and requires explicit revoke confirmation', async () => { + const fetcher = vi + .fn() + .mockResolvedValueOnce( + Response.json({ + sessionId: 'session-1', + generation: 2, + prewarmId: 'prewarm-2', + status: 'warming', + retryAfterMs: 250, + }) + ) + .mockResolvedValueOnce(Response.json({ sessionId: 'session-1', generation: 2, status: 'revoked' })); + const client = new CloudLiveTeleportClient(fetcher, 'https://cloud.agentrelay.test'); + + await expect(client.status({ sessionId: 'session-1', generation: 2 })).resolves.toMatchObject({ + status: 'warming', + retryAfterMs: 250, + }); + await expect( + client.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'revoke-2' }) + ).resolves.toMatchObject({ status: 'revoked' }); + }); + + it('rejects a successful revoke response that does not prove fencing', async () => { + const client = new CloudLiveTeleportClient( + async () => Response.json({ sessionId: 'session-1', generation: 2, status: 'ready' }), + 'https://cloud.agentrelay.test' + ); + await expect( + client.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'revoke-2' }) + ).rejects.toThrow('was not confirmed'); + }); }); diff --git a/packages/cloud/src/live-teleport.ts b/packages/cloud/src/live-teleport.ts index 1510e722b..b9f631790 100644 --- a/packages/cloud/src/live-teleport.ts +++ b/packages/cloud/src/live-teleport.ts @@ -1,19 +1,14 @@ -export type LiveTeleportWorkspaceSource = - | { - kind: 'relayfile-mount'; - mountStatePath: string; - } - | { - kind: 'verified-convergence-receipt'; - receipt: string; - }; +export type LiveTeleportWorkspaceSource = { + kind: 'relayfile-checkpoint-seal'; + receipt: Record; +}; export type LiveTeleportPrewarmInput = { sessionId: string; generation: number; workspaceRoot: string; - source: LiveTeleportWorkspaceSource; idempotencyKey: string; + signal?: AbortSignal; }; export type LiveTeleportPrewarm = { @@ -24,6 +19,7 @@ export type LiveTeleportPrewarm = { export type LiveTeleportAcquireInput = LiveTeleportPrewarmInput & { threadId: string; + source: LiveTeleportWorkspaceSource; prewarmId?: string; }; @@ -51,6 +47,9 @@ export type LiveTeleportEnvironment = { sessionId: string; generation: number; environmentId: string; + /** Relative, provider-neutral path returned by Cloud. */ + connectPath: string; + /** Derived locally from the constructor-pinned Cloud gateway origin. */ execServerUrl: string; workspaceCwd: string; expiresAt: string; @@ -61,12 +60,34 @@ export type LiveTeleportRevokeInput = { sessionId: string; generation: number; idempotencyKey: string; + signal?: AbortSignal; +}; + +export type LiveTeleportStatusInput = { + sessionId: string; + generation: number; + prewarmId?: string; + signal?: AbortSignal; +}; + +export type LiveTeleportLifecycleStatus = { + sessionId: string; + generation: number; + status: 'warming' | 'ready' | 'failed' | 'revoked' | 'expired'; + prewarmId?: string; + retryAfterMs?: number; + expiresAt?: string; +}; + +export type LiveTeleportRevocation = LiveTeleportLifecycleStatus & { + status: 'revoked' | 'expired'; }; export interface LiveTeleportCloudClient { prewarm(input: LiveTeleportPrewarmInput): Promise; + status(input: LiveTeleportStatusInput): Promise; acquire(input: LiveTeleportAcquireInput): Promise; - revoke(input: LiveTeleportRevokeInput): Promise; + revoke(input: LiveTeleportRevokeInput): Promise; } type Fetcher = (path: string, init?: RequestInit) => Promise; @@ -122,6 +143,11 @@ function requiredGeneration(value: unknown): number { return Number(value); } +function optionalNonNegativeInteger(value: unknown, field: string): number | undefined { + if (value === undefined) return undefined; + return requiredNonNegativeInteger(value, field); +} + function requiredNonNegativeInteger(value: unknown, field: string): number { if (!Number.isSafeInteger(value) || Number(value) < 0) { throw new Error(`Cloud live-teleport convergence proof has an invalid ${field}.`); @@ -222,13 +248,34 @@ function requiredConvergenceProof(value: unknown): LiveTeleportConvergenceProof * are rejected even if a buggy server includes them in an otherwise-valid body. */ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { - constructor(private readonly fetcher: Fetcher) {} + private readonly gatewayOrigin: URL; + + constructor( + private readonly fetcher: Fetcher, + gatewayOrigin: string + ) { + let parsed: URL; + try { + parsed = new URL(gatewayOrigin); + } catch { + throw new Error('Cloud live-teleport requires a valid pinned gateway origin.'); + } + if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') { + throw new Error('Cloud live-teleport gateway origin must use HTTP or HTTPS.'); + } + if (parsed.username || parsed.password) { + throw new Error('Cloud live-teleport gateway origin must not contain credentials.'); + } + this.gatewayOrigin = new URL(parsed.origin); + } async prewarm(input: LiveTeleportPrewarmInput): Promise { + const { signal, ...request } = input; const response = await this.fetcher('/api/v1/live-teleports/prewarm', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(input), + body: JSON.stringify(request), + signal, }); const payload = await readPayload(response); if (!isObject(payload)) throw new Error('Cloud live-teleport prewarm returned an invalid response.'); @@ -244,25 +291,35 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { }; } + async status(input: LiveTeleportStatusInput): Promise { + const { signal, ...request } = input; + const response = await this.fetcher('/api/v1/live-teleports/status', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request), + signal, + }); + const payload = await readPayload(response); + if (!isObject(payload)) throw new Error('Cloud live-teleport status returned an invalid response.'); + return this.parseLifecycleStatus(payload); + } + async acquire(input: LiveTeleportAcquireInput): Promise { + const { signal, ...request } = input; const response = await this.fetcher('/api/v1/live-teleports/acquire', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(input), + body: JSON.stringify(request), + signal, }); const payload = await readPayload(response); if (!isObject(payload)) throw new Error('Cloud live-teleport acquire returned an invalid response.'); - const execServerUrl = requiredString(payload.execServerUrl, 'execServerUrl'); - let parsed: URL; - try { - parsed = new URL(execServerUrl); - } catch { - throw new Error('Cloud live-teleport acquire returned an invalid execServerUrl.'); - } - if (parsed.protocol !== 'wss:') { - throw new Error('Cloud live-teleport acquire must return a Cloud WSS bridge URL.'); + if ('execServerUrl' in payload) { + throw new Error('Cloud live-teleport acquire must not return an arbitrary execServerUrl.'); } + const connectPath = this.requiredConnectPath(payload.connectPath); + const execServerUrl = this.execServerUrl(connectPath); const expiresAt = requiredString(payload.expiresAt, 'expiresAt'); if (Number.isNaN(Date.parse(expiresAt))) { @@ -273,6 +330,7 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { sessionId: requiredString(payload.sessionId, 'sessionId'), generation: requiredGeneration(payload.generation), environmentId: requiredString(payload.environmentId, 'environmentId'), + connectPath, execServerUrl, workspaceCwd: requiredString(payload.workspaceCwd, 'workspaceCwd'), expiresAt, @@ -280,12 +338,76 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { }; } - async revoke(input: LiveTeleportRevokeInput): Promise { + async revoke(input: LiveTeleportRevokeInput): Promise { + const { signal, ...request } = input; const response = await this.fetcher('/api/v1/live-teleports/revoke', { method: 'POST', headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(input), + body: JSON.stringify(request), + signal, }); - await readPayload(response); + const payload = await readPayload(response); + if (!isObject(payload)) throw new Error('Cloud live-teleport revoke returned an invalid response.'); + const status = this.parseLifecycleStatus(payload); + if (status.status !== 'revoked' && status.status !== 'expired') { + throw new Error('Cloud live-teleport revoke was not confirmed.'); + } + return { ...status, status: status.status }; + } + + private parseLifecycleStatus(payload: Record): LiveTeleportLifecycleStatus { + const status = payload.status; + if ( + status !== 'warming' && + status !== 'ready' && + status !== 'failed' && + status !== 'revoked' && + status !== 'expired' + ) { + throw new Error('Cloud live-teleport status returned an invalid lifecycle state.'); + } + const expiresAt = + payload.expiresAt === undefined ? undefined : requiredString(payload.expiresAt, 'expiresAt'); + if (expiresAt && Number.isNaN(Date.parse(expiresAt))) { + throw new Error('Cloud live-teleport status returned an invalid expiresAt.'); + } + return { + sessionId: requiredString(payload.sessionId, 'sessionId'), + generation: requiredGeneration(payload.generation), + status, + ...(payload.prewarmId === undefined + ? {} + : { prewarmId: requiredString(payload.prewarmId, 'prewarmId') }), + ...(payload.retryAfterMs === undefined + ? {} + : { retryAfterMs: optionalNonNegativeInteger(payload.retryAfterMs, 'retryAfterMs')! }), + ...(expiresAt ? { expiresAt } : {}), + }; + } + + private requiredConnectPath(value: unknown): string { + const connectPath = requiredString(value, 'connectPath'); + if ( + !connectPath.startsWith('/api/v1/live-teleports/connect/') || + connectPath.startsWith('//') || + connectPath.includes('\\') || + connectPath.includes('#') + ) { + throw new Error('Cloud live-teleport acquire returned an invalid connectPath.'); + } + const parsed = new URL(connectPath, this.gatewayOrigin); + if ( + parsed.origin !== this.gatewayOrigin.origin || + !parsed.pathname.startsWith('/api/v1/live-teleports/connect/') + ) { + throw new Error('Cloud live-teleport acquire returned a cross-origin connectPath.'); + } + return `${parsed.pathname}${parsed.search}`; + } + + private execServerUrl(connectPath: string): string { + const url = new URL(connectPath, this.gatewayOrigin); + url.protocol = url.protocol === 'https:' ? 'wss:' : 'ws:'; + return url.toString(); } } From 2644d0c6cf5adad574725231eca98f338b7c1d49 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Sun, 23 Aug 2026 18:43:24 +0200 Subject: [PATCH 03/16] fix(codex): harden live teleport cutover seams --- CHANGELOG.md | 2 +- packages/cli/src/cli/commands/codex.ts | 21 +- .../cli/src/cli/lib/codex-app-server.test.ts | 127 +++- packages/cli/src/cli/lib/codex-app-server.ts | 131 +++- .../src/cli/lib/codex-live-controller.test.ts | 543 +++++++++++-- .../cli/src/cli/lib/codex-live-controller.ts | 716 +++++++++++++++--- .../src/cli/lib/codex-relayfile-seal.test.ts | 73 +- .../cli/src/cli/lib/codex-relayfile-seal.ts | 100 ++- packages/cloud/src/index.ts | 3 +- packages/cloud/src/live-teleport.test.ts | 177 ++++- packages/cloud/src/live-teleport.ts | 263 ++++--- 11 files changed, 1761 insertions(+), 395 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1dc5813a6..3f5fafc7d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `relay codex run` starts a Relay-managed local Codex app-server/thread, and `relay codex teleport` seals its active Relayfile poll mount at the next turn boundary, verifies the first Cloud turn through Relay's gateway, and requires confirmed fencing plus mount readiness before local rollback. +- `relay codex run` starts a Relay-managed local Codex app-server/thread, and `relay codex teleport` seals its active Relayfile poll mount at the next turn boundary, requires exact Relayfile destination verification before Cloud execution, reconciles lost turn responses by stable client message ID, and requires confirmed fencing plus ownership handback and mount readiness before local rollback. ## [Unreleased - Patch] diff --git a/packages/cli/src/cli/commands/codex.ts b/packages/cli/src/cli/commands/codex.ts index 37c23013b..d23ba68f2 100644 --- a/packages/cli/src/cli/commands/codex.ts +++ b/packages/cli/src/cli/commands/codex.ts @@ -15,6 +15,8 @@ import { } from '../lib/codex-app-server.js'; import { CodexLiveController, + CodexTurnOutcomeUncertainError, + CodexTurnRecordedError, CodexTurnRecoveredError, FileCodexControllerStateStore, type CodexControllerState, @@ -90,10 +92,26 @@ export async function runManagedCodexTurn( generation: status.generation, }, })}\n` - : 'Cloud turn failed; Cloud was fenced and the same Codex thread recovered locally. Retry the turn.\n' + : 'Cloud execution was fenced and the same Codex thread recovered locally; no turn was replayed.\n' ); return; } + if (error instanceof CodexTurnRecordedError) { + options.writeError( + options.json + ? `${JSON.stringify({ + method: 'relay/codexTurnRecordedTerminal', + params: { code: 'TURN_RECORDED_TERMINAL', status: error.status }, + })}\n` + : `Codex recorded the turn as ${error.status}; it was not replayed.\n` + ); + return; + } + if (error instanceof CodexTurnOutcomeUncertainError) { + throw new Error('Relay-managed Codex stopped because the last turn outcome is uncertain.', { + cause: error, + }); + } throw new Error( 'Relay-managed Codex cannot continue because execution fencing or local recovery is unconfirmed.', { cause: error } @@ -239,6 +257,7 @@ function withDefaults(overrides: Partial = {}): CodexC sleep: (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds)), now: () => new Date(), sessionId: randomUUID, + operationId: randomUUID, pid: process.pid, } ); diff --git a/packages/cli/src/cli/lib/codex-app-server.test.ts b/packages/cli/src/cli/lib/codex-app-server.test.ts index d22e09596..882e6c77a 100644 --- a/packages/cli/src/cli/lib/codex-app-server.test.ts +++ b/packages/cli/src/cli/lib/codex-app-server.test.ts @@ -30,7 +30,13 @@ async function nextRequest(child: ChildProcessWithoutNullStreams): Promise { it('requires add and status in the locally generated experimental schema', async () => { - const readFile = vi.fn(async (file: string) => { - if (file.endsWith('ClientRequest.json')) { - return '{"methods":["environment/add","environment/status"]}'; - } - if (file.endsWith('TurnStartParams.json')) { - return turnPolicySchema(); - } - return '{"required":["environmentId","execServerUrl"],"properties":{"environmentId":{},"execServerUrl":{}}}'; - }); + const readFile = vi.fn(async (file: string) => successfulProbeFile(file)); await expect( probeCodexEnvironmentCapability('codex', { makeTempDir: async () => '/schema', @@ -74,10 +107,7 @@ describe('probeCodexEnvironmentCapability', () => { execFile: async () => undefined, readFile: async (file) => { if (file.endsWith('ClientRequest.json')) return '{"methods":["environment/add"]}'; - if (file.endsWith('TurnStartParams.json')) { - return turnPolicySchema(); - } - return '{"required":["environmentId","execServerUrl"]}'; + return successfulProbeFile(file); }, remove, }) @@ -91,12 +121,7 @@ describe('probeCodexEnvironmentCapability', () => { makeTempDir: async () => '/schema', execFile: async () => undefined, readFile: async (file) => { - if (file.endsWith('ClientRequest.json')) { - return '{"methods":["environment/add","environment/status"]}'; - } - if (file.endsWith('TurnStartParams.json')) { - return turnPolicySchema(); - } + if (!file.endsWith('EnvironmentAddParams.json')) return successfulProbeFile(file); return '{"required":["environmentId","execServerUrl"],"properties":{"headers":{}}}'; }, remove: async () => undefined, @@ -110,16 +135,28 @@ describe('probeCodexEnvironmentCapability', () => { makeTempDir: async () => '/schema', execFile: async () => undefined, readFile: async (file) => { - if (file.endsWith('ClientRequest.json')) { - return '{"methods":["environment/add","environment/status"]}'; - } + if (file.endsWith('ClientRequest.json')) return successfulProbeFile(file); if (file.endsWith('TurnStartParams.json')) return '{"properties":{}}'; - return '{"required":["environmentId","execServerUrl"]}'; + return successfulProbeFile(file); }, remove: async () => undefined, }) ).rejects.toThrow('execution-policy contract'); }); + + it('fails closed when full turn-history reconciliation is unavailable', async () => { + await expect( + probeCodexEnvironmentCapability('codex', { + makeTempDir: async () => '/schema', + execFile: async () => undefined, + readFile: async (file) => { + if (file.endsWith('ThreadTurnsListResponse.json')) return '{"properties":{"data":{}}}'; + return successfulProbeFile(file); + }, + remove: async () => undefined, + }) + ).rejects.toThrow('turn-reconciliation contract'); + }); }); describe('StdioCodexAppServerSession', () => { @@ -151,6 +188,7 @@ describe('StdioCodexAppServerSession', () => { const running = session.runTurn({ threadId: 'thread-1', text: 'continue', + clientUserMessageId: 'client-turn-1', execution: { kind: 'remote', environment: { environmentId: 'environment-3', cwd: '/workspace' }, @@ -162,6 +200,7 @@ describe('StdioCodexAppServerSession', () => { method: 'turn/start', params: { threadId: 'thread-1', + clientUserMessageId: 'client-turn-1', approvalPolicy: 'never', sandboxPolicy: { type: 'dangerFullAccess' }, environments: [{ environmentId: 'environment-3', cwd: '/workspace' }], @@ -175,6 +214,43 @@ describe('StdioCodexAppServerSession', () => { await session.close(); }); + it('reconciles a persisted client user message through full turn history', async () => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child); + const requestPromise = nextRequest(child); + const reconciling = session.turnOutcome({ + threadId: 'thread-1', + clientUserMessageId: 'client-turn-1', + }); + const request = await requestPromise; + expect(request).toMatchObject({ + method: 'thread/turns/list', + params: { + threadId: 'thread-1', + itemsView: 'full', + sortDirection: 'desc', + limit: 100, + }, + }); + child.stdout.write( + `${JSON.stringify({ + id: request.id, + result: { + data: [ + { + id: 'turn-1', + status: 'completed', + items: [{ type: 'userMessage', clientId: 'client-turn-1' }], + }, + ], + nextCursor: null, + }, + })}\n` + ); + await expect(reconciling).resolves.toEqual({ status: 'completed', turnId: 'turn-1' }); + await session.close(); + }); + it('pins local turns to a non-networked workspace sandbox and rejects an unexpected approval request', async () => { const child = fakeChild(); const session = new StdioCodexAppServerSession(child); @@ -182,6 +258,7 @@ describe('StdioCodexAppServerSession', () => { const running = session.runTurn({ threadId: 'thread-1', text: 'edit the workspace', + clientUserMessageId: 'client-turn-local', execution: { kind: 'local', workspaceRoot: '/repo' }, }); const request = await requestPromise; @@ -189,6 +266,7 @@ describe('StdioCodexAppServerSession', () => { method: 'turn/start', params: { approvalPolicy: 'never', + clientUserMessageId: 'client-turn-local', sandboxPolicy: { type: 'workspaceWrite', writableRoots: ['/repo'], @@ -274,6 +352,7 @@ describe('StdioCodexAppServerSession', () => { const running = session.runTurn({ threadId: 'thread-1', text: 'long turn', + clientUserMessageId: 'client-turn-long', execution: { kind: 'local', workspaceRoot: '/repo' }, }); const request = await requestPromise; diff --git a/packages/cli/src/cli/lib/codex-app-server.ts b/packages/cli/src/cli/lib/codex-app-server.ts index 40df2712a..773b670f6 100644 --- a/packages/cli/src/cli/lib/codex-app-server.ts +++ b/packages/cli/src/cli/lib/codex-app-server.ts @@ -17,6 +17,11 @@ export type CodexTurnResult = { completed: CodexNotification; }; +export type CodexTurnOutcome = + | { status: 'completed'; turnId: string } + | { status: 'failed' | 'interrupted' | 'inProgress'; turnId: string } + | { status: 'absent' }; + export type CodexTurnExecution = | { kind: 'local'; workspaceRoot: string } | { @@ -34,7 +39,13 @@ export interface CodexAppServerSession { connectTimeoutMs: number; }): Promise; environmentStatus(environmentId: string): Promise; - runTurn(input: { threadId: string; text: string; execution: CodexTurnExecution }): Promise; + runTurn(input: { + threadId: string; + text: string; + clientUserMessageId: string; + execution: CodexTurnExecution; + }): Promise; + turnOutcome(input: { threadId: string; clientUserMessageId: string }): Promise; close(): Promise; } @@ -85,31 +96,75 @@ function assertTurnPolicySchema(turnParams: string): void { if ( !turnSchema.properties?.approvalPolicy || !turnSchema.properties?.sandboxPolicy || - !schemaContainsEnum(turnSchema, 'never') || - !schemaContainsEnum(turnSchema, 'workspaceWrite') || - !schemaContainsProperty(turnSchema, 'writableRoots') || - !schemaContainsEnum(turnSchema, 'dangerFullAccess') + !turnSchema.properties?.clientUserMessageId || + !schemaNodeContainsEnum(turnSchema, turnSchema.properties.approvalPolicy, 'never') || + !schemaNodeContainsEnum(turnSchema, turnSchema.properties.sandboxPolicy, 'workspaceWrite') || + !schemaNodeContainsProperty(turnSchema, turnSchema.properties.sandboxPolicy, 'writableRoots') || + !schemaNodeContainsEnum(turnSchema, turnSchema.properties.sandboxPolicy, 'dangerFullAccess') ) { throw new Error("Codex TurnStartParams does not support Relay's explicit execution-policy contract."); } } -function schemaContainsEnum(value: unknown, expected: string): boolean { - if (Array.isArray(value)) return value.some((entry) => schemaContainsEnum(entry, expected)); - if (!value || typeof value !== 'object') return false; - const object = value as Record; +function assertTurnReconciliationSchema(requests: string, listParams: string, listResponse: string): void { + const params = JSON.parse(listParams) as { properties?: Record }; + const response = JSON.parse(listResponse) as { properties?: Record }; + if ( + !requests.includes('"thread/turns/list"') || + !params.properties?.itemsView || + !params.properties?.sortDirection || + !response.properties?.data || + !schemaNodeContainsEnum(params, params.properties.itemsView, 'full') || + !schemaNodeContainsEnum(params, params.properties.sortDirection, 'desc') || + !schemaNodeContainsProperty(response, response.properties.data, 'clientId') || + !schemaNodeContainsEnum(response, response.properties.data, 'userMessage') || + !schemaNodeContainsEnum(response, response.properties.data, 'completed') || + !schemaNodeContainsEnum(response, response.properties.data, 'interrupted') || + !schemaNodeContainsEnum(response, response.properties.data, 'failed') || + !schemaNodeContainsEnum(response, response.properties.data, 'inProgress') + ) { + throw new Error("Codex thread/turns/list does not support Relay's turn-reconciliation contract."); + } +} + +function resolveSchemaRef(root: unknown, value: unknown): unknown { + if (!value || typeof value !== 'object' || Array.isArray(value)) return value; + const ref = (value as Record).$ref; + if (typeof ref !== 'string' || !ref.startsWith('#/')) return value; + let cursor: unknown = root; + for (const segment of ref.slice(2).split('/')) { + if (!cursor || typeof cursor !== 'object' || Array.isArray(cursor)) return value; + cursor = (cursor as Record)[segment.replaceAll('~1', '/').replaceAll('~0', '~')]; + } + return cursor; +} + +function schemaNodeContainsEnum(root: unknown, value: unknown, expected: string): boolean { + const resolved = resolveSchemaRef(root, value); + if (Array.isArray(resolved)) { + return resolved.some((entry) => schemaNodeContainsEnum(root, entry, expected)); + } + if (!resolved || typeof resolved !== 'object') return false; + const object = resolved as Record; if (Array.isArray(object.enum) && object.enum.includes(expected)) return true; - return Object.values(object).some((entry) => schemaContainsEnum(entry, expected)); + return Object.entries(object) + .filter(([key]) => key !== 'definitions') + .some(([, entry]) => schemaNodeContainsEnum(root, entry, expected)); } -function schemaContainsProperty(value: unknown, expected: string): boolean { - if (Array.isArray(value)) return value.some((entry) => schemaContainsProperty(entry, expected)); - if (!value || typeof value !== 'object') return false; - const object = value as Record; +function schemaNodeContainsProperty(root: unknown, value: unknown, expected: string): boolean { + const resolved = resolveSchemaRef(root, value); + if (Array.isArray(resolved)) { + return resolved.some((entry) => schemaNodeContainsProperty(root, entry, expected)); + } + if (!resolved || typeof resolved !== 'object') return false; + const object = resolved as Record; if (object.properties && typeof object.properties === 'object' && expected in object.properties) { return true; } - return Object.values(object).some((entry) => schemaContainsProperty(entry, expected)); + return Object.entries(object) + .filter(([key]) => key !== 'definitions') + .some(([, entry]) => schemaNodeContainsProperty(root, entry, expected)); } /** @@ -132,10 +187,12 @@ export async function probeCodexEnvironmentCapability( '--out', directory, ]); - const [requests, addParams, turnParams] = await Promise.all([ + const [requests, addParams, turnParams, turnsListParams, turnsListResponse] = await Promise.all([ deps.readFile(path.join(directory, 'ClientRequest.json')), deps.readFile(path.join(directory, 'v2', 'EnvironmentAddParams.json')), deps.readFile(path.join(directory, 'v2', 'TurnStartParams.json')), + deps.readFile(path.join(directory, 'v2', 'ThreadTurnsListParams.json')), + deps.readFile(path.join(directory, 'v2', 'ThreadTurnsListResponse.json')), ]); if (!requests.includes('"environment/add"') || !requests.includes('"environment/status"')) { throw new Error( @@ -145,6 +202,7 @@ export async function probeCodexEnvironmentCapability( assertEnvironmentAddSchema(addParams); assertTurnPolicySchema(turnParams); + assertTurnReconciliationSchema(requests, turnsListParams, turnsListResponse); return { environmentAdd: true, environmentStatus: true, explicitTurnPolicy: true }; } finally { @@ -301,6 +359,7 @@ export class StdioCodexAppServerSession implements CodexAppServerSession { async runTurn(input: { threadId: string; text: string; + clientUserMessageId: string; execution: CodexTurnExecution; }): Promise { let executionParams: Record; @@ -330,6 +389,7 @@ export class StdioCodexAppServerSession implements CodexAppServerSession { 'turn/start', { threadId: input.threadId, + clientUserMessageId: input.clientUserMessageId, input: [{ type: 'text', text: input.text }], ...executionParams, }, @@ -347,6 +407,45 @@ export class StdioCodexAppServerSession implements CodexAppServerSession { return { turnId, response, completed }; } + async turnOutcome(input: { threadId: string; clientUserMessageId: string }): Promise { + let cursor: string | undefined; + for (let page = 0; page < 10; page += 1) { + const response = await this.request('thread/turns/list', { + threadId: input.threadId, + itemsView: 'full', + sortDirection: 'desc', + limit: 100, + ...(cursor ? { cursor } : {}), + }); + if (!isObject(response) || !Array.isArray(response.data)) { + throw new Error('Codex thread/turns/list returned an invalid response.'); + } + for (const candidate of response.data) { + if (!isObject(candidate) || typeof candidate.id !== 'string' || !Array.isArray(candidate.items)) { + continue; + } + const matches = candidate.items.some( + (item) => + isObject(item) && item.type === 'userMessage' && item.clientId === input.clientUserMessageId + ); + if (!matches) continue; + if ( + candidate.status !== 'completed' && + candidate.status !== 'failed' && + candidate.status !== 'interrupted' && + candidate.status !== 'inProgress' + ) { + throw new Error('Codex thread/turns/list returned an invalid turn status.'); + } + return { status: candidate.status, turnId: candidate.id }; + } + const nextCursor = typeof response.nextCursor === 'string' ? response.nextCursor : undefined; + if (!nextCursor) return { status: 'absent' }; + cursor = nextCursor; + } + throw new Error('Codex thread/turns/list exceeded the reconciliation page bound.'); + } + async close(): Promise { this.beginShutdown(new Error('Codex app-server closed.')); if (this.exited || this.child.exitCode !== null) return; diff --git a/packages/cli/src/cli/lib/codex-live-controller.test.ts b/packages/cli/src/cli/lib/codex-live-controller.test.ts index ac11a9a3e..cefe43e8f 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.test.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.test.ts @@ -1,9 +1,13 @@ import { describe, expect, it, vi } from 'vitest'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; import type { LiveTeleportCloudClient, LiveTeleportLifecycleStatus } from '@agent-relay/cloud'; import { CodexLiveController, + FileCodexControllerStateStore, type CodexControllerState, type CodexControllerStateStore, type CodexPersistedMountResumeProvider, @@ -20,35 +24,39 @@ const source = { function sealHandle(overrides: Partial = {}): CodexWorkspaceSealHandle { return { source, - restore: { resumeId: 'resume-1', workspaceId: 'workspace-1', localRoot: '/repo' }, + restore: { + lifecycleId: 'session-1:1:operation-1', + resumeId: 'resume-1', + workspaceId: 'workspace-1', + localRoot: '/repo', + }, resumeLocal: vi.fn(async () => undefined), close: vi.fn(async () => undefined), ...overrides, }; } -const convergence = { - verdict: 'converged' as const, - source: { - cursor: 'evt_10', - manifestSha256: 'a'.repeat(64), - files: 2, - bytes: 20, - conflictArtifacts: [], - conflictDigest: 'b'.repeat(64), - sealedAt: '2026-08-23T11:59:00.000Z', +const verification = { + version: 1 as const, + kind: 'relayfile-destination-verification' as const, + verificationId: 'verify-1', + workspaceId: 'workspace-1', + localRoot: '/workspace', + remoteRoot: '/' as const, + sessionId: 'session-1', + generation: 1, + status: 'converged' as const, + observed: { + digest: `sha256:${'a'.repeat(64)}`, + workspaceRevision: 'rev_10', + eventCursor: 'evt_10', }, - destination: { - cursor: 'evt_10', - manifestSha256: 'a'.repeat(64), - files: 2, - bytes: 20, - conflictArtifacts: [], - conflictDigest: 'b'.repeat(64), + health: { pendingWriteback: 0 as const, - hasPendingWriteback: false as const, + conflicts: 0 as const, + outboxPending: 0 as const, outboxNeedsAttention: false as const, - ephemeralPaths: [] as [], }, + verifiedAt: '2026-08-23T11:59:00.000Z', }; function deferred() { @@ -87,6 +95,7 @@ function appServer(overrides: Partial = {}): CodexAppServ addEnvironment: vi.fn(async () => undefined), environmentStatus: vi.fn(async () => ({ status: 'ready' })), runTurn: vi.fn(async () => turnResult()), + turnOutcome: vi.fn(async () => ({ status: 'absent' as const })), close: vi.fn(async () => undefined), ...overrides, }; @@ -106,7 +115,7 @@ function cloud(overrides: Partial = {}): LiveTeleportCl generation: input.generation, status: 'ready' as const, })), - status: vi.fn(async (input) => lifecycle(input, 'ready')), + status: vi.fn(async (input) => lifecycle(input, 'active')), acquire: vi.fn(async (input) => ({ sessionId: input.sessionId, generation: input.generation, @@ -114,8 +123,9 @@ function cloud(overrides: Partial = {}): LiveTeleportCl connectPath: `/api/v1/live-teleports/connect/${input.sessionId}/${input.generation}`, execServerUrl: `wss://cloud.agentrelay.test/api/v1/live-teleports/connect/${input.sessionId}/${input.generation}`, workspaceCwd: '/workspace', - expiresAt: '2026-08-23T13:00:00.000Z', - convergence, + connectExpiresAt: '2026-08-23T12:05:00.000Z', + leaseExpiresAt: '2026-08-23T13:00:00.000Z', + verification: { ...verification, generation: input.generation, sessionId: input.sessionId }, })), revoke: vi.fn(async (input) => ({ ...lifecycle(input, 'revoked'), status: 'revoked' as const })), ...overrides, @@ -142,8 +152,20 @@ function createController( if (!session) throw new Error('no app server'); return session; }); - const checkpointAndSeal = vi.fn(options.checkpointAndSeal ?? (async () => sealHandle())); + const checkpointAndSeal = vi.fn( + options.checkpointAndSeal ?? + (async (input) => + sealHandle({ + restore: { + lifecycleId: input.lifecycleId, + resumeId: 'resume-1', + workspaceId: 'workspace-1', + localRoot: '/repo', + }, + })) + ); const resumePersistedLocalMount = vi.fn(options.resumePersistedLocalMount ?? (async () => undefined)); + let operation = 0; const controller = new CodexLiveController( { workspaceRoot: '/repo', @@ -163,6 +185,7 @@ function createController( sleep: async () => undefined, now: () => new Date('2026-08-23T12:00:00.000Z'), sessionId: () => 'session-1', + operationId: () => `operation-${++operation}`, pid: 123, } ); @@ -182,8 +205,13 @@ function persistedRemote(overrides: Partial = {}): CodexCo sessionId: 'session-1', threadId: 'thread-1', workspaceRoot: '/repo', - source, - mountRestore: { resumeId: 'resume-7', workspaceId: 'workspace-1', localRoot: '/repo' }, + source: { kind: source.kind }, + mountRestore: { + lifecycleId: 'session-1:7:operation-7', + resumeId: 'resume-7', + workspaceId: 'workspace-1', + localRoot: '/repo', + }, generation: 7, phase: 'remote', controllerPid: 99, @@ -196,15 +224,33 @@ function persistedRemote(overrides: Partial = {}): CodexCo connectPath: '/api/v1/live-teleports/connect/session-1/7', execServerUrl: 'wss://cloud.agentrelay.test/api/v1/live-teleports/connect/session-1/7', workspaceCwd: '/workspace', - expiresAt: '2026-08-23T13:00:00.000Z', + connectExpiresAt: '2026-08-23T12:05:00.000Z', + leaseExpiresAt: '2026-08-23T13:00:00.000Z', attached: true, - convergence, + verification: { ...verification, generation: 7 }, }, updatedAt: '2026-08-23T11:00:00.000Z', ...overrides, }; } +function persistedLocal(overrides: Partial = {}): CodexControllerState { + return { + version: 1, + sessionId: 'session-1', + threadId: 'thread-1', + workspaceRoot: '/repo', + generation: 3, + phase: 'local', + cloudLifecycle: 'none', + controllerPid: 99, + socketPath: '/old.sock', + turnActive: false, + updatedAt: '2026-08-23T11:00:00.000Z', + ...overrides, + }; +} + describe('CodexLiveController', () => { it('fails before starting an app-server when the local experimental capability is unsupported', async () => { const { controller, createAppServer } = createController({ @@ -234,6 +280,148 @@ describe('CodexLiveController', () => { ); }); + it('does not block local initialization on a stalled background prewarm', async () => { + const pendingPrewarm = deferred<{ + prewarmId: string; + generation: number; + status: 'ready'; + }>(); + const cloudClient = cloud({ prewarm: vi.fn(() => pendingPrewarm.promise) }); + const { controller } = createController({ cloud: cloudClient }); + + await expect(controller.initialize()).resolves.toMatchObject({ phase: 'local' }); + pendingPrewarm.resolve({ prewarmId: 'prewarm-1', generation: 1, status: 'ready' }); + await vi.waitFor(() => expect(controller.status().prewarmStatus).toBe('ready')); + }); + + it('retains durable Cloud ownership intent when a prewarm response is lost', async () => { + const cloudClient = cloud({ + prewarm: vi.fn(async () => Promise.reject(new Error('response lost'))), + }); + const { controller, store } = createController({ cloud: cloudClient }); + + await controller.initialize(); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('failed')); + + expect(store.value).toMatchObject({ cloudLifecycle: 'prewarm_requested' }); + }); + + it('cleans up a failed no-row prewarm through Cloud idempotent terminal revoke', async () => { + const cloudClient = cloud({ + prewarm: vi.fn(async () => Promise.reject(new Error('feature disabled'))), + }); + const { controller, store } = createController({ cloud: cloudClient }); + await controller.initialize(); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('failed')); + + await controller.close(); + + expect(cloudClient.revoke).toHaveBeenCalledWith( + expect.objectContaining({ idempotencyKey: 'session-1:1:revoke' }) + ); + expect(cloudClient.status).not.toHaveBeenCalled(); + expect(store.value).toMatchObject({ cloudLifecycle: 'none' }); + }); + + it('persists checkpoint lifecycle intent before Relayfile can stop the mount', async () => { + const checkpoint = deferred(); + let lifecycleId = ''; + const { controller, store } = createController({ + checkpointAndSeal: async (input) => { + lifecycleId = input.lifecycleId; + return checkpoint.promise; + }, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + const running = controller.runTurn('boundary turn'); + await vi.waitFor(() => expect(lifecycleId).not.toBe('')); + expect(store.value?.mountRestore).toEqual({ lifecycleId, localRoot: '/repo' }); + expect(store.value?.source).toBeUndefined(); + checkpoint.resolve( + sealHandle({ + restore: { + lifecycleId, + resumeId: 'resume-1', + workspaceId: 'workspace-1', + localRoot: '/repo', + }, + }) + ); + await running; + }); + + it('waits for aborted checkpoint cleanup to settle before starting local recovery', async () => { + const cleanup = deferred(); + const aborted = deferred(); + const original = appServer(); + const replacement = appServer(); + const { controller } = createController({ + sessions: [original, replacement], + lifecycleDeadlineMs: 5, + checkpointAndSeal: async (input) => { + input.signal?.addEventListener('abort', () => aborted.resolve(), { once: true }); + return cleanup.promise; + }, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + const running = controller.runTurn('deadline'); + await aborted.promise; + expect(original.close).not.toHaveBeenCalled(); + + cleanup.reject(Object.assign(new Error('checkpoint aborted and source ready'), { name: 'AbortError' })); + await expect(running).rejects.toThrow('Teleport failed before turn submission'); + expect(original.close).toHaveBeenCalled(); + expect(replacement.resumeThread).toHaveBeenCalled(); + }); + + it('fences without inverse effects when checkpoint ignores abort and never settles', async () => { + const original = appServer(); + const sealed = sealHandle(); + const { controller } = createController({ + sessions: [original], + lifecycleDeadlineMs: 5, + checkpointAndSeal: async () => new Promise(() => undefined), + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('unknown checkpoint')).rejects.toThrow('lifecycle outcome is unknown'); + + expect(original.close).not.toHaveBeenCalled(); + expect(sealed.resumeLocal).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ + phase: 'fenced', + lastError: 'LIFECYCLE_EFFECT_OUTCOME_UNKNOWN', + }); + await expect(controller.runTurn('must remain blocked')).rejects.toThrow('controller is fenced'); + }); + + it('fences without resuming the source when Cloud acquire never settles', async () => { + const original = appServer(); + const sealed = sealHandle(); + const cloudClient = cloud({ + acquire: vi.fn(async () => new Promise(() => undefined)), + }); + const { controller } = createController({ + sessions: [original], + cloud: cloudClient, + lifecycleDeadlineMs: 5, + checkpointAndSeal: async () => sealed, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('unknown acquire')).rejects.toThrow('lifecycle outcome is unknown'); + + expect(sealed.resumeLocal).not.toHaveBeenCalled(); + expect(original.close).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'fenced' }); + }); + it('rejects stale generations and applies a mid-turn teleport only at the next boundary', async () => { const firstTurn = deferred(); const original = appServer({ @@ -261,11 +449,13 @@ describe('CodexLiveController', () => { expect(original.runTurn).toHaveBeenNthCalledWith(1, { threadId: 'thread-1', text: 'local turn', + clientUserMessageId: 'operation-1', execution: { kind: 'local', workspaceRoot: '/repo' }, }); expect(original.runTurn).toHaveBeenNthCalledWith(2, { threadId: 'thread-1', text: 'remote turn', + clientUserMessageId: 'operation-3', execution: { kind: 'remote', environment: { environmentId: 'environment-1', cwd: '/workspace' }, @@ -286,7 +476,7 @@ describe('CodexLiveController', () => { it('reports verifying—not remote—until the first Cloud turn completes', async () => { const firstRemoteTurn = deferred(); const original = appServer({ runTurn: vi.fn(() => firstRemoteTurn.promise) }); - const { controller } = createController({ sessions: [original] }); + const { controller, store } = createController({ sessions: [original] }); await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); @@ -303,18 +493,23 @@ describe('CodexLiveController', () => { expect(controller.status().remote).not.toHaveProperty('connectPath'); expect(controller.status()).not.toHaveProperty('source'); expect(controller.status()).not.toHaveProperty('mountRestore'); + expect(store.value?.source).toEqual({ kind: 'relayfile-checkpoint-seal' }); + expect(JSON.stringify(store.value)).not.toMatch(/opaque|ticket=|wss:\/\//); await controller.runTurn('subsequent remote turn'); expect(original.runTurn).toHaveBeenNthCalledWith(2, { threadId: 'thread-1', text: 'subsequent remote turn', + clientUserMessageId: 'operation-3', execution: { kind: 'remote' }, }); }); - it('confirms revoke, closes the old controller, and resumes the same thread after first-turn failure', async () => { + it('confirms revoke, reconciles a recorded failed turn, and resumes the same thread without replay', async () => { const original = appServer({ runTurn: vi.fn(async () => Promise.reject(new Error('remote died'))) }); - const replacement = appServer(); + const replacement = appServer({ + turnOutcome: vi.fn(async () => ({ status: 'failed' as const, turnId: 'turn-remote-1' })), + }); const cloudClient = cloud(); const sealed = sealHandle(); const { controller } = createController({ @@ -325,10 +520,10 @@ describe('CodexLiveController', () => { await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await expect(controller.runTurn('do not replay me')).rejects.toThrow('resumed locally'); + await expect(controller.runTurn('do not replay me')).rejects.toThrow('recorded terminal turn'); expect(cloudClient.revoke).toHaveBeenCalledWith( - expect.objectContaining({ idempotencyKey: 'session-1:1:first-turn-failed-revoke' }) + expect.objectContaining({ idempotencyKey: 'session-1:1:revoke' }) ); expect(original.close).toHaveBeenCalled(); expect(sealed.resumeLocal).toHaveBeenCalled(); @@ -338,9 +533,64 @@ describe('CodexLiveController', () => { expect(controller.status()).toMatchObject({ generation: 2, phase: 'local', execution: 'local' }); }); + it('reconciles a completed remote turn after notification loss without replaying it', async () => { + const original = appServer({ + runTurn: vi.fn(async () => Promise.reject(new Error('notification lost'))), + }); + const replacement = appServer({ + turnOutcome: vi.fn(async () => ({ status: 'completed' as const, turnId: 'turn-remote-1' })), + }); + const { controller } = createController({ sessions: [original, replacement] }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('exactly once')).resolves.toMatchObject({ + turnId: 'turn-remote-1', + response: { reconciled: true }, + }); + expect(replacement.runTurn).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); + }); + + it('fails outcome-uncertain when an accepted remote turn cannot be found after fencing', async () => { + const original = appServer({ runTurn: vi.fn(async () => Promise.reject(new Error('transport lost'))) }); + const replacement = appServer({ turnOutcome: vi.fn(async () => ({ status: 'absent' as const })) }); + const { controller } = createController({ sessions: [original, replacement] }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + + await expect(controller.runTurn('ambiguous')).rejects.toThrow('outcome is uncertain'); + expect(replacement.runTurn).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'outcome_uncertain', execution: 'fenced' }); + }); + + it('recovers an expired later-turn lease before submitting the next prompt locally', async () => { + const original = appServer(); + const replacement = appServer(); + const cloudClient = cloud({ status: vi.fn(async (input) => lifecycle(input, 'expired')) }); + const { controller } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + await controller.runTurn('first remote'); + + await controller.runTurn('after lease expiry'); + + expect(replacement.runTurn).toHaveBeenCalledWith({ + threadId: 'thread-1', + text: 'after lease expiry', + clientUserMessageId: 'operation-3', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); + }); + it('fails fenced and never resumes locally when first-turn revoke is unconfirmed', async () => { const original = appServer({ runTurn: vi.fn(async () => Promise.reject(new Error('remote died'))) }); const replacement = appServer(); + const sealed = sealHandle(); const cloudClient = cloud({ revoke: vi.fn(async () => Promise.reject(new Error('revoke timeout'))), status: vi.fn(async (input) => lifecycle(input, 'ready')), @@ -348,14 +598,16 @@ describe('CodexLiveController', () => { const { controller, createAppServer } = createController({ sessions: [original, replacement], cloud: cloudClient, + checkpointAndSeal: async () => sealed, }); await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await expect(controller.runTurn('must not run locally')).rejects.toThrow('could not be confirmed'); + await expect(controller.runTurn('must not run locally')).rejects.toThrow('CLOUD_FENCE_UNCONFIRMED'); expect(original.close).toHaveBeenCalled(); expect(createAppServer).toHaveBeenCalledTimes(1); + expect(sealed.resumeLocal).not.toHaveBeenCalled(); expect(replacement.resumeThread).not.toHaveBeenCalled(); expect(controller.status()).toMatchObject({ phase: 'fenced', execution: 'fenced' }); }); @@ -376,7 +628,7 @@ describe('CodexLiveController', () => { expect(controller.status()).toMatchObject({ phase: 'local', workspaceSource: 'unavailable' }); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await expect(controller.runTurn('local only')).rejects.toThrow('checkpoint-and-seal API unavailable'); + await expect(controller.runTurn('local only')).rejects.toThrow('Teleport failed before turn submission'); expect(cloudClient.acquire).not.toHaveBeenCalled(); expect(cloudClient.revoke).toHaveBeenCalled(); @@ -385,7 +637,12 @@ describe('CodexLiveController', () => { it('does not persist a seal whose restore identity targets another local root', async () => { const sealed = sealHandle({ - restore: { resumeId: 'resume-1', workspaceId: 'workspace-1', localRoot: '/other' }, + restore: { + lifecycleId: 'session-1:1:operation-1', + resumeId: 'resume-1', + workspaceId: 'workspace-1', + localRoot: '/other', + }, }); const original = appServer(); const replacement = appServer(); @@ -398,7 +655,9 @@ describe('CodexLiveController', () => { await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await expect(controller.runTurn('must stay on this mount')).rejects.toThrow('invalid restore identity'); + await expect(controller.runTurn('must stay on this mount')).rejects.toThrow( + 'Teleport failed before turn submission' + ); expect(cloudClient.acquire).not.toHaveBeenCalled(); expect(sealed.resumeLocal).toHaveBeenCalled(); @@ -420,7 +679,9 @@ describe('CodexLiveController', () => { await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await expect(controller.runTurn('local after abort')).rejects.toThrow('acquire failed'); + await expect(controller.runTurn('local after abort')).rejects.toThrow( + 'Teleport failed before turn submission' + ); expect(sealed.resumeLocal).toHaveBeenCalled(); expect(replacement.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); @@ -429,6 +690,7 @@ describe('CodexLiveController', () => { expect(replacement.runTurn).toHaveBeenCalledWith({ threadId: 'thread-1', text: 'second input', + clientUserMessageId: 'operation-2', execution: { kind: 'local', workspaceRoot: '/repo' }, }); }); @@ -446,7 +708,9 @@ describe('CodexLiveController', () => { await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await expect(controller.runTurn('must stay fenced')).rejects.toThrow('could not resume'); + await expect(controller.runTurn('must stay fenced')).rejects.toThrow( + 'RELAYFILE_RESUME_FAILED_AFTER_FENCE' + ); expect(createAppServer).toHaveBeenCalledTimes(1); expect(replacement.resumeThread).not.toHaveBeenCalled(); @@ -505,7 +769,7 @@ describe('CodexLiveController', () => { await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await expect(controller.runTurn('bounded')).rejects.toThrow('did not converge'); + await expect(controller.runTurn('bounded')).rejects.toThrow('Teleport failed before turn submission'); expect(cloudClient.acquire).not.toHaveBeenCalled(); expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); @@ -531,8 +795,10 @@ describe('CodexLiveController', () => { const original = appServer(); const replacement = appServer(); const sealed = sealHandle(); + const cloudClient = cloud(); const { controller } = createController({ sessions: [original, replacement], + cloud: cloudClient, checkpointAndSeal: async () => sealed, }); await controller.initialize(); @@ -543,13 +809,92 @@ describe('CodexLiveController', () => { expect(original.close).toHaveBeenCalled(); expect(sealed.resumeLocal).toHaveBeenCalled(); + expect(vi.mocked(cloudClient.revoke).mock.invocationCallOrder[0]).toBeLessThan( + vi.mocked(sealed.resumeLocal).mock.invocationCallOrder[0]! + ); expect(replacement.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); expect(status).toMatchObject({ phase: 'local', generation: 2, execution: 'local' }); }); + it('does not resume a consumed checkpoint while Cloud handback is cleanup_pending', async () => { + const handback = deferred(); + const original = appServer(); + const replacement = appServer(); + const sealed = sealHandle(); + const cloudClient = cloud({ + revoke: vi.fn(async (input) => lifecycle(input, 'cleanup_pending')), + status: vi.fn(() => handback.promise), + }); + const { controller } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + checkpointAndSeal: async () => sealed, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + await controller.runTurn('remote'); + + const rollingBack = controller.rollback(); + await vi.waitFor(() => expect(cloudClient.status).toHaveBeenCalled()); + expect(sealed.resumeLocal).not.toHaveBeenCalled(); + + handback.resolve(lifecycle({ sessionId: 'session-1', generation: 1 }, 'revoked')); + await rollingBack; + expect(sealed.resumeLocal).toHaveBeenCalledOnce(); + }); + + it('keeps the source sealed when revoke ignores abort and never settles', async () => { + const original = appServer(); + const replacement = appServer(); + const sealed = sealHandle(); + const cloudClient = cloud({ revoke: vi.fn(async () => new Promise(() => undefined)) }); + const { controller } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + lifecycleDeadlineMs: 5, + checkpointAndSeal: async () => sealed, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + await controller.runTurn('remote'); + + await expect(controller.rollback()).rejects.toThrow('CLOUD_FENCE_UNCONFIRMED'); + + expect(sealed.resumeLocal).not.toHaveBeenCalled(); + expect(replacement.resumeThread).not.toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ phase: 'fenced' }); + }); + + it('restarts a clean local session without trying to revoke or poll Cloud', async () => { + const store = memoryStore(persistedLocal()); + const resumed = appServer(); + const cloudClient = cloud(); + const { controller } = createController({ store, sessions: [resumed], cloud: cloudClient }); + + await expect(controller.initialize()).resolves.toMatchObject({ phase: 'local', generation: 3 }); + + expect(cloudClient.revoke).not.toHaveBeenCalled(); + expect(cloudClient.status).not.toHaveBeenCalled(); + expect(resumed.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + }); + + it('fences a persisted prewarm request whose response may have been lost', async () => { + const store = memoryStore(persistedLocal({ cloudLifecycle: 'prewarm_requested' })); + const resumed = appServer(); + const cloudClient = cloud(); + const { controller } = createController({ store, sessions: [resumed], cloud: cloudClient }); + + await expect(controller.initialize()).resolves.toMatchObject({ phase: 'local', generation: 4 }); + + expect(cloudClient.revoke).toHaveBeenCalledWith( + expect.objectContaining({ sessionId: 'session-1', generation: 3, idempotencyKey: 'session-1:3:revoke' }) + ); + expect(resumed.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + }); + it('persists fenced on restart and never constructs a local app-server after unconfirmed revoke', async () => { const persisted = persistedRemote(); - persisted.remote!.expiresAt = '2026-08-23T11:00:00.000Z'; + persisted.remote!.leaseExpiresAt = '2026-08-23T11:00:00.000Z'; const store = memoryStore(persisted); const cloudClient = cloud({ revoke: vi.fn(async () => Promise.reject(new Error('timeout'))), @@ -557,7 +902,7 @@ describe('CodexLiveController', () => { }); const { controller, createAppServer } = createController({ store, cloud: cloudClient }); - await expect(controller.initialize()).rejects.toThrow('could not confirm Cloud fencing'); + await expect(controller.initialize()).rejects.toThrow('CLOUD_FENCE_UNCONFIRMED_ON_RESTART'); expect(createAppServer).not.toHaveBeenCalled(); expect(store.value).toMatchObject({ phase: 'fenced', generation: 7 }); @@ -565,7 +910,7 @@ describe('CodexLiveController', () => { it('resumes locally only after Cloud authoritatively confirms the persisted lease expired', async () => { const expired = persistedRemote({ - remote: { ...persistedRemote().remote!, expiresAt: '2026-08-23T11:59:59.000Z' }, + remote: { ...persistedRemote().remote!, leaseExpiresAt: '2026-08-23T11:59:59.000Z' }, }); const store = memoryStore(expired); const resumed = appServer(); @@ -585,18 +930,39 @@ describe('CodexLiveController', () => { }); expect(cloudClient.revoke).toHaveBeenCalled(); - expect(resumePersistedLocalMount).toHaveBeenCalledWith(expect.objectContaining({ source })); + expect(resumePersistedLocalMount).toHaveBeenCalledWith( + expect.objectContaining({ source: { kind: 'relayfile-checkpoint-seal' } }) + ); expect(resumed.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); expect(store.value?.source).toBeUndefined(); expect(store.value?.mountRestore).toBeUndefined(); }); + it('reconciles a persisted completed turn before reopening local execution after restart', async () => { + const store = memoryStore( + persistedRemote({ + inFlightTurn: { clientUserMessageId: 'client-persisted-1', execution: 'remote' }, + }) + ); + const resumed = appServer({ + turnOutcome: vi.fn(async () => ({ status: 'completed' as const, turnId: 'turn-persisted-1' })), + }); + const { controller } = createController({ store, sessions: [resumed] }); + + await expect(controller.initialize()).resolves.toMatchObject({ phase: 'local', generation: 8 }); + expect(resumed.turnOutcome).toHaveBeenCalledWith({ + threadId: 'thread-1', + clientUserMessageId: 'client-persisted-1', + }); + expect(store.value?.inFlightTurn).toBeUndefined(); + }); + it('fails recovery if the same thread cannot be resumed after a confirmed fence', async () => { const store = memoryStore(persistedRemote()); const resumed = appServer({ resumeThread: vi.fn(async () => Promise.reject(new Error('gone'))) }); const { controller } = createController({ store, sessions: [resumed] }); - await expect(controller.initialize()).rejects.toThrow('Could not resume Codex thread thread-1'); + await expect(controller.initialize()).rejects.toThrow('CODEX_THREAD_RESUME_FAILED_ON_RESTART'); expect(store.value).toMatchObject({ phase: 'recovery_failed', generation: 7 }); }); @@ -624,6 +990,52 @@ describe('CodexLiveController', () => { expect(controller.status()).toMatchObject({ phase: 'fenced', execution: 'fenced' }); }); + it('cancels and fences a racing prewarm without a late state mutation', async () => { + const prewarmStarted = deferred(); + const cloudClient = cloud({ + prewarm: vi.fn( + (input) => + new Promise((_, reject) => { + prewarmStarted.resolve(); + input.signal?.addEventListener( + 'abort', + () => reject(Object.assign(new Error('aborted'), { name: 'AbortError' })), + { once: true } + ); + }) + ), + }); + const { controller, store } = createController({ cloud: cloudClient }); + await controller.initialize(); + await prewarmStarted.promise; + + await controller.close(); + await Promise.resolve(); + + expect(cloudClient.revoke).toHaveBeenCalled(); + expect(store.value).toMatchObject({ cloudLifecycle: 'none' }); + expect(store.value?.prewarmStatus).toBeUndefined(); + }); + + it('bounds shutdown when prewarm ignores abort, then fences the ambiguous request', async () => { + const original = appServer(); + const cloudClient = cloud({ + prewarm: vi.fn(async () => new Promise(() => undefined)), + }); + const { controller } = createController({ + sessions: [original], + cloud: cloudClient, + lifecycleDeadlineMs: 5, + }); + await controller.initialize(); + + await controller.close(); + + expect(cloudClient.revoke).toHaveBeenCalled(); + expect(original.close).toHaveBeenCalled(); + expect(controller.status()).toMatchObject({ cloudLifecycle: 'none' }); + }); + it('resumes and verifies the sealed local mount on ordinary shutdown after remote execution', async () => { const sealed = sealHandle(); const original = appServer(); @@ -643,3 +1055,44 @@ describe('CodexLiveController', () => { expect(controller.status()).toMatchObject({ phase: 'local', generation: 2, execution: 'local' }); }); }); + +describe('FileCodexControllerStateStore', () => { + it.each([ + ['foreign version', { ...persistedLocal(), version: 2 }], + ['negative generation', { ...persistedLocal(), generation: -1 }], + ['invalid pid', { ...persistedLocal(), controllerPid: 0 }], + ['malformed phase', { ...persistedLocal(), phase: 'teleported' }], + ['malformed pending', { ...persistedLocal(), pending: { requestId: '', expectedGeneration: 3 } }], + ['secret-bearing source', { ...persistedLocal(), source: { ...source } }], + [ + 'cross-root restore', + { + ...persistedLocal(), + mountRestore: { lifecycleId: 'lifecycle-3', resumeId: 'resume-3', localRoot: '/other' }, + }, + ], + ['partial remote', { ...persistedLocal(), remote: { sessionId: 'session-1', generation: 3 } }], + [ + 'cross-generation remote', + { + ...persistedRemote(), + remote: { ...persistedRemote().remote!, generation: 8 }, + }, + ], + [ + 'malformed in-flight turn', + { ...persistedLocal(), inFlightTurn: { clientUserMessageId: '', execution: 'remote' } }, + ], + ])('rejects %s instead of adopting malformed controller state', (_name, value) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-controller-state-')); + const file = path.join(directory, 'state.json'); + try { + fs.writeFileSync(file, JSON.stringify(value)); + expect(() => new FileCodexControllerStateStore(file).read()).toThrow( + 'invalid or from an unsupported version' + ); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/cli/src/cli/lib/codex-live-controller.ts b/packages/cli/src/cli/lib/codex-live-controller.ts index c2138061f..8bdc66f0e 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.ts @@ -9,7 +9,7 @@ import type { LiveTeleportWorkspaceSource, } from '@agent-relay/cloud'; -import type { CodexAppServerSession, CodexTurnResult } from './codex-app-server.js'; +import type { CodexAppServerSession, CodexTurnOutcome, CodexTurnResult } from './codex-app-server.js'; export type CodexControllerPhase = | 'local' @@ -18,9 +18,18 @@ export type CodexControllerPhase = | 'verifying' | 'remote' | 'rolling_back' + | 'outcome_uncertain' | 'fenced' | 'recovery_failed'; +export type CodexCloudLifecycleIntent = + | 'none' + | 'prewarm_requested' + | 'prewarmed' + | 'acquire_requested' + | 'acquired' + | 'cleanup_requested'; + export type CodexTeleportRequest = { requestId: string; expectedGeneration: number; @@ -31,7 +40,7 @@ export type CodexControllerState = { sessionId: string; threadId: string; workspaceRoot: string; - source?: LiveTeleportWorkspaceSource; + source?: { kind: LiveTeleportWorkspaceSource['kind'] }; mountRestore?: CodexMountRestoreIdentity; generation: number; phase: CodexControllerPhase; @@ -42,10 +51,16 @@ export type CodexControllerState = { lastRequestId?: string; prewarmId?: string; prewarmStatus?: 'warming' | 'ready' | 'failed'; - remote?: LiveTeleportEnvironment & { + /** Durable evidence that Cloud may own resources for this generation. */ + cloudLifecycle?: CodexCloudLifecycleIntent; + remote?: Omit & { /** True only after the first remote turn/completed notification. */ attached: boolean; }; + inFlightTurn?: { + clientUserMessageId: string; + execution: 'local' | 'remote'; + }; lastError?: string; updatedAt: string; }; @@ -53,7 +68,10 @@ export type CodexControllerState = { export type PublicCodexControllerStatus = Omit & { execution: 'local' | 'verifying' | 'cloud' | 'fenced'; controller: 'local'; - remote?: Pick & { + remote?: Pick< + LiveTeleportEnvironment, + 'environmentId' | 'generation' | 'workspaceCwd' | 'connectExpiresAt' | 'leaseExpiresAt' + > & { attached: true; }; workspaceSource: LiveTeleportWorkspaceSource['kind'] | 'unavailable'; @@ -69,7 +87,7 @@ export class FileCodexControllerStateStore implements CodexControllerStateStore read(): CodexControllerState | null { try { - return JSON.parse(fs.readFileSync(this.filePath, 'utf8')) as CodexControllerState; + return validatePersistedState(JSON.parse(fs.readFileSync(this.filePath, 'utf8')) as unknown); } catch (error) { if (error instanceof Error && 'code' in error && error.code === 'ENOENT') return null; throw error; @@ -80,8 +98,32 @@ export class FileCodexControllerStateStore implements CodexControllerStateStore fs.mkdirSync(path.dirname(this.filePath), { recursive: true, mode: 0o700 }); const temporary = `${this.filePath}.tmp-${process.pid}-${randomUUID()}`; fs.writeFileSync(temporary, `${JSON.stringify(state, null, 2)}\n`, { mode: 0o600 }); + const temporaryFd = fs.openSync(temporary, 'r'); + try { + fs.fsyncSync(temporaryFd); + } finally { + fs.closeSync(temporaryFd); + } fs.renameSync(temporary, this.filePath); fs.chmodSync(this.filePath, 0o600); + try { + const directoryFd = fs.openSync(path.dirname(this.filePath), 'r'); + try { + fs.fsyncSync(directoryFd); + } finally { + fs.closeSync(directoryFd); + } + } catch (error) { + if ( + !( + error instanceof Error && + 'code' in error && + ['EBADF', 'EISDIR', 'EINVAL', 'ENOTSUP', 'EPERM'].includes(String(error.code)) + ) + ) { + throw error; + } + } } } @@ -90,6 +132,7 @@ export type CodexWorkspaceSealInput = { generation: number; threadId: string; workspaceRoot: string; + lifecycleId: string; signal?: AbortSignal; }; @@ -103,8 +146,9 @@ export type CodexWorkspaceSealHandle = { }; export type CodexMountRestoreIdentity = { - resumeId: string; - workspaceId: string; + lifecycleId: string; + resumeId?: string; + workspaceId?: string; localRoot: string; }; @@ -117,7 +161,7 @@ export type CodexWorkspaceSealProvider = ( export type CodexPersistedMountResumeProvider = ( input: CodexWorkspaceSealInput & { - source: LiveTeleportWorkspaceSource; + source?: { kind: LiveTeleportWorkspaceSource['kind'] }; restore: CodexMountRestoreIdentity; } ) => Promise; @@ -132,6 +176,7 @@ export type CodexLiveControllerDependencies = { sleep: (milliseconds: number) => Promise; now: () => Date; sessionId: () => string; + operationId: () => string; pid: number; }; @@ -147,11 +192,165 @@ const DEFAULT_LIFECYCLE_DEADLINE_MS = 60_000; const DEFAULT_LIFECYCLE_POLL_INTERVAL_MS = 500; const TURN_BLOCKED_PHASES = new Set([ 'recovery_failed', + 'outcome_uncertain', 'fenced', 'rolling_back', 'acquiring', 'verifying', ]); +const CONTROLLER_PHASES = new Set([ + 'local', + 'teleport_pending', + 'acquiring', + 'verifying', + 'remote', + 'rolling_back', + 'outcome_uncertain', + 'fenced', + 'recovery_failed', +]); +const CLOUD_LIFECYCLE_INTENTS = new Set([ + 'none', + 'prewarm_requested', + 'prewarmed', + 'acquire_requested', + 'acquired', + 'cleanup_requested', +]); + +function record(value: unknown): Record | undefined { + return value !== null && typeof value === 'object' && !Array.isArray(value) + ? (value as Record) + : undefined; +} + +function nonEmptyString(value: unknown): value is string { + return typeof value === 'string' && value.trim().length > 0; +} + +function validOptionalString(value: unknown): boolean { + return value === undefined || nonEmptyString(value); +} + +function validTimestamp(value: unknown): value is string { + return nonEmptyString(value) && !Number.isNaN(Date.parse(value)); +} + +function hasOnlyKeys(value: Record, keys: readonly string[]): boolean { + return Object.keys(value).every((key) => keys.includes(key)); +} + +function invalidNestedState(state: Record): boolean { + const pending = state.pending === undefined ? undefined : record(state.pending); + const source = state.source === undefined ? undefined : record(state.source); + const restore = state.mountRestore === undefined ? undefined : record(state.mountRestore); + const remote = state.remote === undefined ? undefined : record(state.remote); + const inFlight = state.inFlightTurn === undefined ? undefined : record(state.inFlightTurn); + const generation = Number(state.generation); + const workspaceRoot = String(state.workspaceRoot); + if ( + (pending !== undefined && + (!nonEmptyString(pending.requestId) || pending.expectedGeneration !== generation)) || + (state.pending !== undefined && !pending) || + (source !== undefined && + (!hasOnlyKeys(source, ['kind']) || source.kind !== 'relayfile-checkpoint-seal')) || + (state.source !== undefined && !source) || + (restore !== undefined && + (!nonEmptyString(restore.lifecycleId) || + !validOptionalString(restore.resumeId) || + !validOptionalString(restore.workspaceId) || + !nonEmptyString(restore.localRoot) || + !path.isAbsolute(restore.localRoot) || + path.resolve(restore.localRoot) !== path.resolve(workspaceRoot))) || + (state.mountRestore !== undefined && !restore) || + (source !== undefined && !restore) || + (inFlight !== undefined && + (!nonEmptyString(inFlight.clientUserMessageId) || + (inFlight.execution !== 'local' && inFlight.execution !== 'remote'))) || + (state.inFlightTurn !== undefined && !inFlight) || + !validOptionalString(state.prewarmId) || + (state.prewarmStatus !== undefined && + state.prewarmStatus !== 'warming' && + state.prewarmStatus !== 'ready' && + state.prewarmStatus !== 'failed') + ) { + return true; + } + if (!remote) return state.remote !== undefined || state.phase === 'remote' || state.phase === 'verifying'; + const verification = record(remote.verification); + const observed = record(verification?.observed); + const health = record(verification?.health); + return ( + 'connectPath' in remote || + 'execServerUrl' in remote || + remote.sessionId !== state.sessionId || + remote.generation !== generation || + !nonEmptyString(remote.environmentId) || + !nonEmptyString(remote.workspaceCwd) || + !validTimestamp(remote.connectExpiresAt) || + !validTimestamp(remote.leaseExpiresAt) || + typeof remote.attached !== 'boolean' || + !verification || + verification.version !== 1 || + verification.kind !== 'relayfile-destination-verification' || + verification.status !== 'converged' || + verification.remoteRoot !== '/' || + verification.sessionId !== state.sessionId || + verification.generation !== generation || + !nonEmptyString(verification.verificationId) || + !nonEmptyString(verification.workspaceId) || + !nonEmptyString(verification.localRoot) || + !validTimestamp(verification.verifiedAt) || + !observed || + typeof observed.digest !== 'string' || + !/^sha256:[a-f0-9]{64}$/.test(observed.digest) || + typeof observed.workspaceRevision !== 'string' || + !/^(?:0|rev_[0-9]+)$/.test(observed.workspaceRevision) || + typeof observed.eventCursor !== 'string' || + !/^(?:0|evt_[0-9]+)$/.test(observed.eventCursor) || + !health || + health.pendingWriteback !== 0 || + health.conflicts !== 0 || + health.outboxPending !== 0 || + health.outboxNeedsAttention !== false + ); +} + +function inferredCloudLifecycle(value: Record): CodexCloudLifecycleIntent { + if (value.remote) return 'acquired'; + if (value.phase === 'acquiring' || value.phase === 'verifying') return 'acquire_requested'; + if (value.prewarmId || value.prewarmStatus === 'ready') return 'prewarmed'; + if (CLOUD_LIFECYCLE_INTENTS.has(value.cloudLifecycle as CodexCloudLifecycleIntent)) { + return value.cloudLifecycle as CodexCloudLifecycleIntent; + } + return 'none'; +} + +function validatePersistedState(value: unknown): CodexControllerState { + const state = record(value); + if ( + !state || + state.version !== 1 || + !nonEmptyString(state.sessionId) || + !nonEmptyString(state.threadId) || + !nonEmptyString(state.workspaceRoot) || + !nonEmptyString(state.socketPath) || + !Number.isSafeInteger(state.generation) || + Number(state.generation) < 1 || + !Number.isSafeInteger(state.controllerPid) || + Number(state.controllerPid) < 1 || + typeof state.turnActive !== 'boolean' || + !CONTROLLER_PHASES.has(state.phase as CodexControllerPhase) || + !nonEmptyString(state.updatedAt) || + Number.isNaN(Date.parse(state.updatedAt)) || + (state.cloudLifecycle !== undefined && + !CLOUD_LIFECYCLE_INTENTS.has(state.cloudLifecycle as CodexCloudLifecycleIntent)) || + invalidNestedState(state) + ) { + throw new Error('Persisted managed Codex controller state is invalid or from an unsupported version.'); + } + return { ...(state as CodexControllerState), cloudLifecycle: inferredCloudLifecycle(state) }; +} export class CodexTurnRecoveredError extends Error { readonly recoveredLocally = true; @@ -162,8 +361,35 @@ export class CodexTurnRecoveredError extends Error { } } -function errorMessage(error: unknown): string { - return error instanceof Error ? error.message : String(error); +export class CodexTurnRecordedError extends Error { + readonly recordedTerminal = true; + + constructor( + readonly status: 'failed' | 'interrupted', + message: string + ) { + super(message); + this.name = 'CodexTurnRecordedError'; + } +} + +export class CodexTurnOutcomeUncertainError extends Error { + readonly outcomeUncertain = true; + + constructor( + message = 'Codex turn outcome is uncertain; execution is fenced pending reconciliation.', + options?: ErrorOptions + ) { + super(message, options); + this.name = 'CodexTurnOutcomeUncertainError'; + } +} + +class CodexLifecycleEffectUnsettledError extends Error { + constructor(operationName: string) { + super(`${operationName} did not settle after abort; lifecycle outcome is unknown.`); + this.name = 'CodexLifecycleEffectUnsettledError'; + } } function isReadyStatus(value: unknown): boolean { @@ -183,7 +409,11 @@ function publicStatus(state: CodexControllerState): PublicCodexControllerStatus ? 'cloud' : state.phase === 'verifying' ? 'verifying' - : state.phase === 'fenced' || state.phase === 'recovery_failed' + : state.phase === 'fenced' || + state.phase === 'recovery_failed' || + state.phase === 'outcome_uncertain' || + state.phase === 'acquiring' || + state.phase === 'rolling_back' ? 'fenced' : 'local'; return { @@ -197,7 +427,8 @@ function publicStatus(state: CodexControllerState): PublicCodexControllerStatus environmentId: remote.environmentId, generation: remote.generation, workspaceCwd: remote.workspaceCwd, - expiresAt: remote.expiresAt, + connectExpiresAt: remote.connectExpiresAt, + leaseExpiresAt: remote.leaseExpiresAt, attached: true, }, } @@ -214,6 +445,9 @@ export class CodexLiveController { private appServer: CodexAppServerSession | null = null; private state: CodexControllerState | null = null; private sealedWorkspace: CodexWorkspaceSealHandle | null = null; + private prewarmPromise: Promise | null = null; + private prewarmAbort: AbortController | null = null; + private closing = false; constructor( private readonly options: CodexLiveControllerOptions, @@ -230,33 +464,37 @@ export class CodexLiveController { } if (persisted) { + const mustFenceCloud = this.cloudMayOwnResources(persisted); + const mustRecoverMount = Boolean(persisted.source || persisted.mountRestore || persisted.remote); this.state = { ...persisted, controllerPid: this.deps.pid, socketPath: this.options.socketPath, workspaceRoot: this.options.workspaceRoot, turnActive: false, - phase: 'rolling_back', + phase: mustFenceCloud || mustRecoverMount ? 'rolling_back' : 'local', pending: undefined, lastError: undefined, updatedAt: this.timestamp(), }; this.persist(); - try { - await this.confirmFence('restart-revoke'); - } catch (error) { - this.markFenced(`Controller restart could not confirm Cloud fencing: ${errorMessage(error)}`); - throw new Error(this.requireState().lastError, { cause: error }); + if (mustFenceCloud) { + try { + await this.confirmFence('restart-revoke'); + } catch (error) { + this.markFenced('CLOUD_FENCE_UNCONFIRMED_ON_RESTART'); + throw new Error(this.requireState().lastError, { cause: error }); + } } - try { - await this.resumeLocalMount(); - } catch (error) { - this.markRecoveryFailed( - `Cloud was fenced but the persisted Relayfile mount could not resume: ${errorMessage(error)}` - ); - throw new Error(this.requireState().lastError, { cause: error }); + if (mustRecoverMount) { + try { + await this.resumeLocalMount(); + } catch (error) { + this.markRecoveryFailed('RELAYFILE_RESUME_FAILED_ON_RESTART'); + throw new Error(this.requireState().lastError, { cause: error }); + } } const replacement = await this.createInitializedAppServer(); @@ -267,17 +505,32 @@ export class CodexLiveController { cwd: this.options.workspaceRoot, }); } catch (error) { - this.markRecoveryFailed( - `Could not resume Codex thread ${persisted.threadId}: ${errorMessage(error)}` - ); + this.markRecoveryFailed('CODEX_THREAD_RESUME_FAILED_ON_RESTART'); await replacement.close().catch(() => undefined); this.appServer = null; throw new Error(this.requireState().lastError, { cause: error }); } + if (this.requireState().inFlightTurn) { + const outcome = await this.reconcileInFlightTurn().catch((error) => { + this.markOutcomeUncertain('TURN_RECONCILIATION_UNAVAILABLE'); + throw new CodexTurnOutcomeUncertainError(undefined, { cause: error }); + }); + if (outcome.status === 'absent' || outcome.status === 'inProgress') { + this.markOutcomeUncertain(`TURN_${outcome.status.toUpperCase()}`); + throw new CodexTurnOutcomeUncertainError(); + } + this.clearInFlightTurn( + outcome.status === 'completed' + ? 'TURN_COMPLETED_DURING_RECOVERY' + : `TURN_${outcome.status.toUpperCase()}` + ); + } + const state = this.requireState(); - state.generation = persisted.generation + 1; + state.generation = persisted.generation + (mustFenceCloud || mustRecoverMount ? 1 : 0); state.phase = 'local'; + state.cloudLifecycle = 'none'; state.remote = undefined; state.source = undefined; state.mountRestore = undefined; @@ -301,12 +554,13 @@ export class CodexLiveController { controllerPid: this.deps.pid, socketPath: this.options.socketPath, turnActive: false, + cloudLifecycle: 'none', updatedAt: this.timestamp(), }; this.persist(); } - await this.startPrewarm(); + this.schedulePrewarm(); return this.status(); } @@ -353,7 +607,22 @@ export class CodexLiveController { this.persist(); try { if (pendingAtBoundary) await this.applyPendingTeleport(pendingAtBoundary); - return await this.executeTurn(text); + if (this.requireState().phase === 'remote') await this.ensureRemoteActiveOrRecover(); + const execution = + this.requireState().phase === 'remote' || this.requireState().phase === 'verifying' + ? 'remote' + : 'local'; + state.inFlightTurn = { + clientUserMessageId: this.deps.operationId(), + execution, + }; + state.updatedAt = this.timestamp(); + this.persist(); + const result = await this.executeTurn(text, state.inFlightTurn.clientUserMessageId); + state.inFlightTurn = undefined; + state.updatedAt = this.timestamp(); + this.persist(); + return result; } finally { state.turnActive = false; state.updatedAt = this.timestamp(); @@ -361,7 +630,7 @@ export class CodexLiveController { } } - private async executeTurn(text: string): Promise { + private async executeTurn(text: string, clientUserMessageId: string): Promise { const state = this.requireState(); const phase = state.phase as CodexControllerPhase; const remote = phase === 'verifying' || phase === 'remote' ? state.remote : undefined; @@ -369,6 +638,7 @@ export class CodexLiveController { const result = await this.requireAppServer().runTurn({ threadId: state.threadId, text, + clientUserMessageId, execution: remote ? { kind: 'remote', @@ -387,12 +657,31 @@ export class CodexLiveController { } return result; } catch (error) { - if (this.requireState().phase !== 'verifying') throw error; - await this.recoverLocal('first-turn-failed-revoke', `First Cloud turn failed: ${errorMessage(error)}`); - throw new CodexTurnRecoveredError( - `First Cloud turn failed; Cloud fencing was confirmed and thread ${state.threadId} resumed locally: ${errorMessage(error)}`, - { cause: error } - ); + if (remote) { + const outcome = await this.recoverLocal('remote-turn-error-revoke', 'REMOTE_TURN_RECONCILED'); + if (outcome?.status === 'completed') return this.resultFromOutcome(outcome); + if (outcome?.status === 'failed' || outcome?.status === 'interrupted') { + throw new CodexTurnRecordedError( + outcome.status, + `Cloud turn ended ${outcome.status}; the recorded terminal turn was not replayed.` + ); + } + throw new CodexTurnOutcomeUncertainError(undefined, { cause: error }); + } + let outcome: CodexTurnOutcome; + try { + outcome = await this.reconcileInFlightTurn(); + } catch (reconcileError) { + this.markOutcomeUncertain('TURN_RECONCILIATION_UNAVAILABLE'); + throw new CodexTurnOutcomeUncertainError(undefined, { cause: reconcileError }); + } + if (outcome.status === 'completed') return this.resultFromOutcome(outcome); + if (outcome.status === 'failed' || outcome.status === 'interrupted') { + this.clearInFlightTurn(`TURN_${outcome.status.toUpperCase()}`); + throw new CodexTurnRecordedError(outcome.status, `Codex turn ended ${outcome.status}.`); + } + this.markOutcomeUncertain(`TURN_${outcome.status.toUpperCase()}`); + throw new CodexTurnOutcomeUncertainError(undefined, { cause: error }); } } @@ -405,37 +694,49 @@ export class CodexLiveController { async close(): Promise { const state = this.state; + this.closing = true; + this.prewarmAbort?.abort(); + await this.prewarmPromise?.catch(() => undefined); let fenceConfirmed = false; - if (state) { + if (state && this.cloudMayOwnResources(state)) { try { await this.confirmFence('shutdown-revoke'); fenceConfirmed = true; - } catch (error) { - this.markFenced(`Cloud shutdown revoke was not confirmed: ${errorMessage(error)}`); + } catch { + this.markFenced('CLOUD_FENCE_UNCONFIRMED_ON_SHUTDOWN'); } } try { await this.appServer?.close(); } catch (error) { - if (state && fenceConfirmed) { - this.markRecoveryFailed( - `Cloud was fenced during shutdown but the Codex app-server did not exit: ${errorMessage(error)}` - ); + if (state && (fenceConfirmed || !this.cloudMayOwnResources(state))) { + this.markRecoveryFailed('CONTROLLER_CLOSE_FAILED_ON_SHUTDOWN'); } throw error; } this.appServer = null; - if (state && fenceConfirmed && (this.sealedWorkspace || state.source || state.mountRestore)) { + if ( + state && + (fenceConfirmed || !this.cloudMayOwnResources(state)) && + (this.sealedWorkspace || state.source || state.mountRestore) + ) { try { await this.resumeLocalMount(); } catch (error) { - this.markRecoveryFailed( - `Cloud was fenced during shutdown but the Relayfile mount did not resume: ${errorMessage(error)}` - ); + this.markRecoveryFailed('RELAYFILE_RESUME_FAILED_ON_SHUTDOWN'); throw new Error(state.lastError, { cause: error }); } + if (state.inFlightTurn) { + state.phase = 'outcome_uncertain'; + state.remote = undefined; + state.lastError = 'TURN_RECONCILIATION_REQUIRED_ON_RESTART'; + state.updatedAt = this.timestamp(); + this.persist(); + return; + } state.generation += 1; state.phase = 'local'; + state.cloudLifecycle = 'none'; state.remote = undefined; state.source = undefined; state.mountRestore = undefined; @@ -462,10 +763,15 @@ export class CodexLiveController { // Keep the poll mount alive while Cloud warms. Admission is already // stopped at this turn boundary, but the short-lived seal is minted only // when Cloud is ready to consume it. + await this.prewarmPromise; if (state.prewarmId && state.prewarmStatus !== 'ready') { await this.waitForCloudReady(state.prewarmId); } + const lifecycleId = `${state.sessionId}:${state.generation}:${this.deps.operationId()}`; + state.mountRestore = { lifecycleId, localRoot: state.workspaceRoot }; + state.updatedAt = this.timestamp(); + this.persist(); const sealedWorkspace = await this.withAbortableLifecycleDeadline( (signal) => this.deps.checkpointAndSeal({ @@ -473,6 +779,7 @@ export class CodexLiveController { generation: state.generation, threadId: state.threadId, workspaceRoot: state.workspaceRoot, + lifecycleId, signal, }), 'Relayfile checkpoint-and-seal' @@ -480,11 +787,14 @@ export class CodexLiveController { const source = sealedWorkspace.source; this.sealedWorkspace = sealedWorkspace; this.assertSeal(source, sealedWorkspace.restore); - state.source = source; + state.source = { kind: source.kind }; state.mountRestore = sealedWorkspace.restore; state.updatedAt = this.timestamp(); this.persist(); + state.cloudLifecycle = 'acquire_requested'; + state.updatedAt = this.timestamp(); + this.persist(); const environment = await this.withAbortableLifecycleDeadline( (signal) => this.deps.cloud.acquire({ @@ -503,7 +813,13 @@ export class CodexLiveController { throw new Error('Cloud returned a stale or cross-session live-teleport generation.'); } - state.remote = { ...environment, attached: false }; + const { + connectPath: _connectPath, + execServerUrl: _execServerUrl, + ...persistedEnvironment + } = environment; + state.remote = { ...persistedEnvironment, attached: false }; + state.cloudLifecycle = 'acquired'; state.pending = undefined; state.phase = 'acquiring'; state.updatedAt = this.timestamp(); @@ -521,18 +837,19 @@ export class CodexLiveController { state.updatedAt = this.timestamp(); this.persist(); } catch (error) { - await this.recoverLocal( - 'failed-acquire-revoke', - `Teleport failed before the first Cloud turn completed: ${errorMessage(error)}` - ); + if (error instanceof CodexLifecycleEffectUnsettledError) { + this.markFenced('LIFECYCLE_EFFECT_OUTCOME_UNKNOWN'); + throw error; + } + await this.recoverLocal('failed-acquire-revoke', 'TELEPORT_ACQUIRE_FAILED_RECOVERED'); throw new CodexTurnRecoveredError( - `Teleport failed; Cloud fencing was confirmed and execution resumed locally: ${errorMessage(error)}`, + 'Teleport failed before turn submission; Cloud fencing and local recovery were confirmed.', { cause: error } ); } } - private async recoverLocal(reason: string, context: string): Promise { + private async recoverLocal(reason: string, context: string): Promise { const state = this.requireState(); const old = this.requireAppServer(); state.phase = 'rolling_back'; @@ -540,57 +857,69 @@ export class CodexLiveController { state.updatedAt = this.timestamp(); this.persist(); - try { - await this.confirmFence(reason); - } catch (error) { - this.markFenced(`${context} Cloud fencing could not be confirmed: ${errorMessage(error)}`); - await old.close().catch((closeError) => { - state.lastError = `${state.lastError} Old controller shutdown also failed: ${errorMessage(closeError)}`; - state.updatedAt = this.timestamp(); - this.persist(); - }); - this.appServer = null; - throw new Error(state.lastError, { cause: error }); + if (this.cloudMayOwnResources(state)) { + try { + await this.confirmFence(reason); + } catch (error) { + this.markFenced('CLOUD_FENCE_UNCONFIRMED'); + await old.close().catch(() => { + state.lastError = 'CLOUD_FENCE_UNCONFIRMED_AND_CONTROLLER_CLOSE_FAILED'; + state.updatedAt = this.timestamp(); + this.persist(); + }); + this.appServer = null; + throw new Error(state.lastError, { cause: error }); + } } try { await old.close(); } catch (error) { - this.markRecoveryFailed( - `Cloud was fenced but the previous Codex app-server did not exit: ${errorMessage(error)}` - ); + this.markRecoveryFailed('CONTROLLER_CLOSE_FAILED_AFTER_FENCE'); throw new Error(state.lastError, { cause: error }); } this.appServer = null; try { await this.resumeLocalMount(); } catch (error) { - this.markRecoveryFailed( - `Cloud was fenced but the sealed Relayfile mount could not resume: ${errorMessage(error)}` - ); + this.markRecoveryFailed('RELAYFILE_RESUME_FAILED_AFTER_FENCE'); throw new Error(state.lastError, { cause: error }); } let replacement: CodexAppServerSession; try { replacement = await this.createInitializedAppServer(); } catch (error) { - this.markRecoveryFailed( - `Cloud was fenced and the mount resumed, but a local Codex app-server could not start: ${errorMessage(error)}` - ); + this.markRecoveryFailed('LOCAL_APP_SERVER_START_FAILED_AFTER_FENCE'); throw new Error(state.lastError, { cause: error }); } this.appServer = replacement; try { await replacement.resumeThread({ threadId: state.threadId, cwd: state.workspaceRoot }); } catch (error) { - this.markRecoveryFailed(`Could not resume Codex thread ${state.threadId}: ${errorMessage(error)}`); + this.markRecoveryFailed('CODEX_THREAD_RESUME_FAILED_AFTER_FENCE'); await replacement.close().catch(() => undefined); this.appServer = null; throw new Error(state.lastError, { cause: error }); } + let outcome: CodexTurnOutcome | undefined; + if (state.inFlightTurn) { + try { + outcome = await this.reconcileInFlightTurn(); + } catch (error) { + this.markOutcomeUncertain('TURN_RECONCILIATION_UNAVAILABLE'); + throw new CodexTurnOutcomeUncertainError(undefined, { cause: error }); + } + if (outcome.status === 'absent' || outcome.status === 'inProgress') { + this.markOutcomeUncertain(`TURN_${outcome.status.toUpperCase()}`); + throw new CodexTurnOutcomeUncertainError(); + } + state.inFlightTurn = undefined; + } + state.generation += 1; state.phase = 'local'; + state.cloudLifecycle = 'none'; state.remote = undefined; state.source = undefined; state.mountRestore = undefined; @@ -599,47 +928,56 @@ export class CodexLiveController { state.lastError = context; state.updatedAt = this.timestamp(); this.persist(); - await this.startPrewarm(); + this.schedulePrewarm(); + return outcome; } - private async confirmFence(reason: string): Promise { + private async confirmFence(_reason: string): Promise { const state = this.requireState(); - - try { - const revoked = await this.withAbortableLifecycleDeadline( - (signal) => - this.deps.cloud.revoke({ + state.cloudLifecycle = 'cleanup_requested'; + state.updatedAt = this.timestamp(); + this.persist(); + await this.withAbortableLifecycleDeadline(async (signal) => { + let lastError: unknown; + try { + const revoked = await this.deps.cloud.revoke({ + sessionId: state.sessionId, + generation: state.generation, + idempotencyKey: `${state.sessionId}:${state.generation}:revoke`, + signal, + }); + this.assertLifecycleIdentity(revoked); + // The Cloud contract may publish revoked/expired only after the + // destination is stopped, flushed, and its consumer-bound Relayfile + // ownership has been handed back. cleanup_pending is not a fence and + // must never allow the source mount to resume. + if (revoked.status === 'revoked' || revoked.status === 'expired') { + this.markCloudFenced(); + return; + } + } catch (error) { + lastError = error; + } + for (let attempt = 0; attempt < this.lifecycleAttempts(); attempt += 1) { + try { + const status = await this.deps.cloud.status({ sessionId: state.sessionId, generation: state.generation, - idempotencyKey: `${state.sessionId}:${state.generation}:${reason}`, signal, - }), - 'Cloud revoke confirmation' - ); - this.assertLifecycleIdentity(revoked); - if (revoked.status === 'revoked' || revoked.status === 'expired') return; - } catch (revokeError) { - try { - const status = await this.withAbortableLifecycleDeadline( - (signal) => - this.deps.cloud.status({ - sessionId: state.sessionId, - generation: state.generation, - signal, - }), - 'Cloud fence status confirmation' - ); - this.assertLifecycleIdentity(status); - if (status.status === 'revoked' || status.status === 'expired') return; - } catch (statusError) { - throw new Error( - `revoke failed (${errorMessage(revokeError)}); status was unconfirmed (${errorMessage(statusError)})`, - { cause: statusError } - ); + }); + this.assertLifecycleIdentity(status); + if (status.status === 'revoked' || status.status === 'expired') { + this.markCloudFenced(); + return; + } + lastError = new Error(`Cloud fence remains ${status.status}.`); + } catch (error) { + lastError = error; + } + await this.deps.sleep(this.lifecyclePollIntervalMs()); } - throw revokeError; - } - throw new Error('Cloud revoke returned without a terminal fence state.'); + throw new Error('Cloud fencing did not reach a terminal state.', { cause: lastError }); + }, 'Cloud revoke confirmation'); } private async waitForCloudReady(prewarmId: string): Promise { @@ -689,31 +1027,54 @@ export class CodexLiveController { ); } - private async startPrewarm(): Promise { + private schedulePrewarm(): void { + if (this.prewarmPromise || this.closing) return; + const state = this.requireState(); + const generation = state.generation; + state.cloudLifecycle = 'prewarm_requested'; + state.updatedAt = this.timestamp(); + this.persist(); + const abort = new AbortController(); + this.prewarmAbort = abort; + const pending = this.startPrewarm(generation, abort.signal).finally(() => { + if (this.prewarmPromise === pending) this.prewarmPromise = null; + if (this.prewarmAbort === abort) this.prewarmAbort = null; + }); + this.prewarmPromise = pending; + } + + private async startPrewarm(generation: number, outerSignal: AbortSignal): Promise { const state = this.requireState(); try { const prewarm = await this.withAbortableLifecycleDeadline( (signal) => this.deps.cloud.prewarm({ sessionId: state.sessionId, - generation: state.generation, + generation, workspaceRoot: '/', - idempotencyKey: `${state.sessionId}:${state.generation}:prewarm`, + idempotencyKey: `${state.sessionId}:${generation}:prewarm`, signal, }), - 'Cloud prewarm' + 'Cloud prewarm', + outerSignal ); - if (prewarm.generation !== state.generation) { + if (prewarm.generation !== generation) { throw new Error('Cloud returned a stale prewarm generation.'); } + if (this.closing || this.requireState().generation !== generation) return; state.source = undefined; state.prewarmId = prewarm.prewarmId; state.prewarmStatus = prewarm.status; - } catch (error) { + state.cloudLifecycle = 'prewarmed'; + } catch { + if (this.closing || this.requireState().generation !== generation) return; state.source = undefined; state.prewarmId = undefined; state.prewarmStatus = 'failed'; - state.lastError = `Cloud prewarm unavailable; local Codex remains usable: ${errorMessage(error)}`; + // Response loss is ambiguous. Retain prewarm_requested so restart or + // shutdown fences any Cloud resource that may have been created. + state.cloudLifecycle = 'prewarm_requested'; + state.lastError = 'CLOUD_PREWARM_UNAVAILABLE'; } state.updatedAt = this.timestamp(); this.persist(); @@ -733,12 +1094,66 @@ export class CodexLiveController { restore.resumeId.length === 0 || typeof restore.workspaceId !== 'string' || restore.workspaceId.length === 0 || + typeof restore.lifecycleId !== 'string' || + restore.lifecycleId.length === 0 || + restore.lifecycleId !== this.requireState().mountRestore?.lifecycleId || path.resolve(restore.localRoot) !== path.resolve(this.options.workspaceRoot) ) { throw new Error('Relayfile checkpoint-and-seal provider returned an invalid restore identity.'); } } + private async ensureRemoteActiveOrRecover(): Promise { + const state = this.requireState(); + let active: boolean; + try { + const status = await this.withAbortableLifecycleDeadline( + (signal) => + this.deps.cloud.status({ + sessionId: state.sessionId, + generation: state.generation, + signal, + }), + 'Cloud active lease preflight' + ); + this.assertLifecycleIdentity(status); + active = status.status === 'active'; + } catch { + active = false; + } + if (!active) await this.recoverLocal('remote-preflight-revoke', 'REMOTE_LEASE_RECOVERED'); + } + + private reconcileInFlightTurn(): Promise { + const state = this.requireState(); + const inFlight = state.inFlightTurn; + if (!inFlight) return Promise.resolve({ status: 'absent' }); + return this.requireAppServer().turnOutcome({ + threadId: state.threadId, + clientUserMessageId: inFlight.clientUserMessageId, + }); + } + + private resultFromOutcome(outcome: Extract): CodexTurnResult { + const state = this.requireState(); + return { + turnId: outcome.turnId, + response: { reconciled: true }, + completed: { + method: 'turn/completed', + params: { threadId: state.threadId, turn: { id: outcome.turnId, status: 'completed' } }, + }, + }; + } + + private clearInFlightTurn(code?: string): void { + const state = this.requireState(); + state.inFlightTurn = undefined; + state.lastError = code; + state.updatedAt = this.timestamp(); + this.persist(); + } + private async resumeLocalMount(): Promise { const state = this.requireState(); if (this.sealedWorkspace) { @@ -749,13 +1164,12 @@ export class CodexLiveController { this.sealedWorkspace = null; return; } - if (!state.source || !state.mountRestore) { + if (!state.mountRestore) { if (state.remote) { throw new Error('Persisted remote execution has no Relayfile seal identity to restore.'); } return; } - const source = state.source; const restore = state.mountRestore; await this.withAbortableLifecycleDeadline( (signal) => @@ -764,7 +1178,8 @@ export class CodexLiveController { generation: state.generation, threadId: state.threadId, workspaceRoot: state.workspaceRoot, - source, + lifecycleId: restore.lifecycleId, + ...(state.source ? { source: state.source } : {}), restore, signal, }), @@ -779,6 +1194,19 @@ export class CodexLiveController { } } + private cloudMayOwnResources(state = this.requireState()): boolean { + return inferredCloudLifecycle(state as unknown as Record) !== 'none'; + } + + private markCloudFenced(): void { + const state = this.requireState(); + state.cloudLifecycle = 'none'; + state.prewarmId = undefined; + state.prewarmStatus = undefined; + state.updatedAt = this.timestamp(); + this.persist(); + } + private async createInitializedAppServer(): Promise { const appServer = await this.deps.createAppServer(); await appServer.initialize(); @@ -803,33 +1231,69 @@ export class CodexLiveController { this.persist(); } + private markOutcomeUncertain(code: string): void { + const state = this.requireState(); + state.phase = 'outcome_uncertain'; + state.lastError = code; + state.updatedAt = this.timestamp(); + this.persist(); + } + private withAbortableLifecycleDeadline( operation: (signal: AbortSignal) => Promise, - operationName: string + operationName: string, + outerSignal?: AbortSignal ): Promise { const controller = new AbortController(); + const abortFromOuter = () => controller.abort(); + outerSignal?.addEventListener('abort', abortFromOuter, { once: true }); + if (outerSignal?.aborted) controller.abort(); const timeoutMs = this.lifecycleDeadlineMs(); return new Promise((resolve, reject) => { + let timedOut = false; + let settled = false; + let settlementTimer: NodeJS.Timeout | undefined; const timer = setTimeout(() => { + timedOut = true; controller.abort(); - reject(new Error(`${operationName} exceeded the ${timeoutMs}ms lifecycle deadline.`)); + settlementTimer = setTimeout( + () => { + if (!settled) { + outerSignal?.removeEventListener('abort', abortFromOuter); + reject(new CodexLifecycleEffectUnsettledError(operationName)); + } + }, + Math.min(5_000, timeoutMs) + ); }, timeoutMs); let promise: Promise; try { promise = operation(controller.signal); } catch (error) { clearTimeout(timer); + outerSignal?.removeEventListener('abort', abortFromOuter); reject(error); return; } promise.then( (value) => { + settled = true; clearTimeout(timer); - resolve(value); + if (settlementTimer) clearTimeout(settlementTimer); + outerSignal?.removeEventListener('abort', abortFromOuter); + if (timedOut) reject(new Error(`${operationName} exceeded the ${timeoutMs}ms lifecycle deadline.`)); + else resolve(value); }, (error) => { + settled = true; clearTimeout(timer); - reject(error); + if (settlementTimer) clearTimeout(settlementTimer); + outerSignal?.removeEventListener('abort', abortFromOuter); + if (timedOut) + reject( + new Error(`${operationName} exceeded the ${timeoutMs}ms lifecycle deadline.`, { cause: error }) + ); + else reject(error); } ); }); diff --git a/packages/cli/src/cli/lib/codex-relayfile-seal.test.ts b/packages/cli/src/cli/lib/codex-relayfile-seal.test.ts index baf8d4df7..0bd44017e 100644 --- a/packages/cli/src/cli/lib/codex-relayfile-seal.test.ts +++ b/packages/cli/src/cli/lib/codex-relayfile-seal.test.ts @@ -15,8 +15,8 @@ function receipt(overrides: Record = {}) { sessionId: 'session-1', generation: 4, digest: `sha256:${'a'.repeat(64)}`, - workspaceRevision: 'rev-40', - eventCursor: 'evt-50', + workspaceRevision: 'rev_40', + eventCursor: 'evt_50', issuedAt: '2026-08-23T12:00:00.000Z', expiresAt: '2026-08-23T12:01:00.000Z', ...overrides, @@ -27,12 +27,19 @@ function checkpointOutput(overrides: Record = {}) { return { version: 1, kind: 'relayfile-checkpoint-seal', + status: 'sealed', workspaceId: 'ws_123', localRoot: '/repo', sessionId: 'session-1', generation: 4, receipt: receipt(), - resumeId: 'resume_opaque_123', + health: { + pendingWriteback: 0, + conflicts: 0, + outboxPending: 0, + outboxNeedsAttention: false, + }, + resumeId: 'lifecycle-4', sealedAt: '2026-08-23T12:00:00.000Z', ...overrides, }; @@ -44,7 +51,7 @@ function resumeOutput(overrides: Record = {}) { kind: 'relayfile-resume-seal', workspaceId: 'ws_123', localRoot: '/repo', - resumeId: 'resume_opaque_123', + resumeId: 'lifecycle-4', status: 'ready', resumedAt: '2026-08-23T12:00:10.000Z', ...overrides, @@ -64,6 +71,7 @@ describe('createRelayfileSealLifecycle', () => { generation: 4, threadId: 'thread-1', workspaceRoot: '/repo', + lifecycleId: 'lifecycle-4', }); expect(runner).toHaveBeenNthCalledWith(1, { @@ -77,6 +85,8 @@ describe('createRelayfileSealLifecycle', () => { 'session-1', '--generation', '4', + '--lifecycle-id', + 'lifecycle-4', '--timeout', '30s', '--ttl', @@ -87,7 +97,8 @@ describe('createRelayfileSealLifecycle', () => { }); expect(handle.source).toEqual({ kind: 'relayfile-checkpoint-seal', receipt: receipt() }); expect(handle.restore).toEqual({ - resumeId: 'resume_opaque_123', + lifecycleId: 'lifecycle-4', + resumeId: 'lifecycle-4', workspaceId: 'ws_123', localRoot: '/repo', }); @@ -96,12 +107,14 @@ describe('createRelayfileSealLifecycle', () => { const resumeCall = runner.mock.calls[1]![0]; expect(resumeCall.args).toEqual(['mount', 'resume-seal', '--root', '/repo', '--json']); - expect(resumeCall.args.join(' ')).not.toContain('resume_opaque_123'); - expect(resumeCall.stdin).toBe(`${JSON.stringify({ resumeId: 'resume_opaque_123' })}\n`); + expect(resumeCall.args.join(' ')).not.toContain('lifecycle-4'); + expect(resumeCall.stdin).toBe(`${JSON.stringify({ resumeId: 'lifecycle-4' })}\n`); }); it('uses the same stdin-only resume contract after controller restart', async () => { - const runner = vi.fn(async () => resumeOutput()); + const runner = vi.fn(async () => + resumeOutput({ resumeId: 'resume_opaque_123' }) + ); const lifecycle = createRelayfileSealLifecycle({ runner }); await lifecycle.resumePersistedLocalMount({ @@ -109,8 +122,14 @@ describe('createRelayfileSealLifecycle', () => { generation: 4, threadId: 'thread-1', workspaceRoot: '/repo', - source: { kind: 'relayfile-checkpoint-seal', receipt: receipt() }, - restore: { resumeId: 'resume_opaque_123', workspaceId: 'ws_123', localRoot: '/repo' }, + lifecycleId: 'lifecycle-4', + source: { kind: 'relayfile-checkpoint-seal' }, + restore: { + lifecycleId: 'lifecycle-4', + resumeId: 'resume_opaque_123', + workspaceId: 'ws_123', + localRoot: '/repo', + }, }); expect(runner).toHaveBeenCalledWith( @@ -121,12 +140,44 @@ describe('createRelayfileSealLifecycle', () => { ); }); + it('reconciles an in-flight checkpoint from persisted lifecycle intent alone', async () => { + const runner = vi.fn(async () => resumeOutput()); + const lifecycle = createRelayfileSealLifecycle({ runner }); + + await lifecycle.resumePersistedLocalMount({ + sessionId: 'session-1', + generation: 4, + threadId: 'thread-1', + workspaceRoot: '/repo', + lifecycleId: 'lifecycle-4', + restore: { lifecycleId: 'lifecycle-4', localRoot: '/repo' }, + }); + + expect(runner).toHaveBeenCalledWith( + expect.objectContaining({ + args: ['mount', 'resume-seal', '--root', '/repo', '--json'], + stdin: `${JSON.stringify({ resumeId: 'lifecycle-4' })}\n`, + }) + ); + }); + it.each([ ['wrong local root', checkpointOutput({ localRoot: '/other' })], ['wrong session', checkpointOutput({ sessionId: 'session-other' })], ['wrong generation', checkpointOutput({ generation: 5 })], ['non-logical receipt root', checkpointOutput({ receipt: receipt({ root: '/repo' }) })], ['caller-shaped digest', checkpointOutput({ receipt: receipt({ digest: 'caller-says-ok' }) })], + ['uppercase digest', checkpointOutput({ receipt: receipt({ digest: `sha256:${'A'.repeat(64)}` }) })], + ['bare revision', checkpointOutput({ receipt: receipt({ workspaceRevision: '40' }) })], + ['wrong revision namespace', checkpointOutput({ receipt: receipt({ workspaceRevision: 'evt_40' }) })], + ['bare cursor', checkpointOutput({ receipt: receipt({ eventCursor: '50' }) })], + ['wrong cursor namespace', checkpointOutput({ receipt: receipt({ eventCursor: 'rev_50' }) })], + [ + 'unsettled health', + checkpointOutput({ + health: { pendingWriteback: 1, conflicts: 0, outboxPending: 0, outboxNeedsAttention: false }, + }), + ], ['missing one-use token', checkpointOutput({ receipt: receipt({ sealToken: '' }) })], ])('fails closed on %s', async (_name, output) => { const lifecycle = createRelayfileSealLifecycle({ runner: async () => output }); @@ -136,6 +187,7 @@ describe('createRelayfileSealLifecycle', () => { generation: 4, threadId: 'thread-1', workspaceRoot: '/repo', + lifecycleId: 'lifecycle-4', }) ).rejects.toThrow(/relayfile|checkpoint/); }); @@ -152,6 +204,7 @@ describe('createRelayfileSealLifecycle', () => { generation: 4, threadId: 'thread-1', workspaceRoot: '/repo', + lifecycleId: 'lifecycle-4', }); await expect(handle.resumeLocal()).rejects.toThrow('did not confirm mount readiness'); }); diff --git a/packages/cli/src/cli/lib/codex-relayfile-seal.ts b/packages/cli/src/cli/lib/codex-relayfile-seal.ts index 1ddd81c7b..f522bbeac 100644 --- a/packages/cli/src/cli/lib/codex-relayfile-seal.ts +++ b/packages/cli/src/cli/lib/codex-relayfile-seal.ts @@ -68,6 +68,26 @@ function requiredTimestamp(value: unknown, field: string): string { return timestamp; } +function requiredRelayfilePosition(value: unknown, field: string, prefix: 'rev' | 'evt'): string { + const position = requiredString(value, field); + if (!new RegExp(`^(?:0|${prefix}_[0-9]+)$`).test(position)) { + throw new Error(`relayfile lifecycle response returned an invalid ${field}.`); + } + return position; +} + +function requireConvergedHealth(value: unknown): void { + if ( + !isObject(value) || + value.pendingWriteback !== 0 || + value.conflicts !== 0 || + value.outboxPending !== 0 || + value.outboxNeedsAttention !== false + ) { + throw new Error('relayfile checkpoint response did not report converged health.'); + } +} + function validateReceipt( value: unknown, input: { sessionId: string; generation: number; workspaceId: string } @@ -82,15 +102,15 @@ function validateReceipt( if (requiredString(value.root, 'receipt.root') !== '/') { throw new Error('relayfile checkpoint receipt root must be logical / for live teleport v1.'); } - requiredString(value.workspaceRevision, 'receipt.workspaceRevision'); - requiredString(value.eventCursor, 'receipt.eventCursor'); + requiredRelayfilePosition(value.workspaceRevision, 'receipt.workspaceRevision', 'rev'); + requiredRelayfilePosition(value.eventCursor, 'receipt.eventCursor', 'evt'); requiredTimestamp(value.issuedAt, 'receipt.issuedAt'); requiredTimestamp(value.expiresAt, 'receipt.expiresAt'); if ( workspaceId !== input.workspaceId || sessionId !== input.sessionId || generation !== input.generation || - !/^sha256:[a-f0-9]{64}$/i.test(digest) + !/^sha256:[a-f0-9]{64}$/.test(digest) ) { throw new Error('relayfile checkpoint receipt is unbound or has an invalid digest.'); } @@ -99,9 +119,14 @@ function validateReceipt( function validateCheckpointOutput( value: unknown, - input: { workspaceRoot: string; sessionId: string; generation: number } + input: { workspaceRoot: string; sessionId: string; generation: number; lifecycleId: string } ): { source: LiveTeleportWorkspaceSource; restore: CodexMountRestoreIdentity } { - if (!isObject(value) || value.version !== 1 || value.kind !== 'relayfile-checkpoint-seal') { + if ( + !isObject(value) || + value.version !== 1 || + value.kind !== 'relayfile-checkpoint-seal' || + value.status !== 'sealed' + ) { throw new Error('relayfile checkpoint command returned an invalid response contract.'); } const workspaceId = requiredString(value.workspaceId, 'workspaceId'); @@ -112,8 +137,12 @@ function validateCheckpointOutput( ) { throw new Error('relayfile checkpoint command returned a stale or cross-session response.'); } + requireConvergedHealth(value.health); requiredTimestamp(value.sealedAt, 'sealedAt'); const resumeId = requiredResumeId(value.resumeId); + if (resumeId !== input.lifecycleId) { + throw new Error('relayfile checkpoint command returned a mismatched lifecycle identity.'); + } const receipt = validateReceipt(value.receipt, { sessionId: input.sessionId, generation: input.generation, @@ -121,7 +150,7 @@ function validateCheckpointOutput( }); return { source: { kind: 'relayfile-checkpoint-seal', receipt }, - restore: { resumeId, workspaceId, localRoot }, + restore: { lifecycleId: input.lifecycleId, resumeId, workspaceId, localRoot }, }; } @@ -135,9 +164,10 @@ function validateResumeOutput(value: unknown, restore: CodexMountRestoreIdentity throw new Error('relayfile resume command did not confirm mount readiness.'); } if ( - requiredString(value.workspaceId, 'workspaceId') !== restore.workspaceId || - requiredExactLocalRoot(value.localRoot, restore.localRoot) !== restore.localRoot || - requiredResumeId(value.resumeId) !== restore.resumeId + requiredResumeId(value.resumeId) !== (restore.resumeId ?? restore.lifecycleId) || + (restore.workspaceId !== undefined && + requiredString(value.workspaceId, 'workspaceId') !== restore.workspaceId) || + requiredExactLocalRoot(value.localRoot, restore.localRoot) !== restore.localRoot ) { throw new Error('relayfile resume command returned a mismatched restore identity.'); } @@ -243,7 +273,7 @@ export function createRelayfileSealLifecycle( const output = await runner({ binary, args: ['mount', 'resume-seal', '--root', restore.localRoot, '--json'], - stdin: `${JSON.stringify({ resumeId: restore.resumeId })}\n`, + stdin: `${JSON.stringify({ resumeId: restore.resumeId ?? restore.lifecycleId })}\n`, signal, }); validateResumeOutput(output, restore); @@ -251,26 +281,36 @@ export function createRelayfileSealLifecycle( return { checkpointAndSeal: async (input): Promise => { - const output = await runner({ - binary, - args: [ - 'mount', - 'checkpoint-seal', - '--root', - input.workspaceRoot, - '--session', - input.sessionId, - '--generation', - String(input.generation), - '--timeout', - '30s', - '--ttl', - '60s', - '--json', - ], - signal: input.signal, - }); - const sealed = validateCheckpointOutput(output, input); + let sealed: ReturnType; + try { + const output = await runner({ + binary, + args: [ + 'mount', + 'checkpoint-seal', + '--root', + input.workspaceRoot, + '--session', + input.sessionId, + '--generation', + String(input.generation), + '--lifecycle-id', + input.lifecycleId, + '--timeout', + '30s', + '--ttl', + '60s', + '--json', + ], + signal: input.signal, + }); + sealed = validateCheckpointOutput(output, input); + } catch (error) { + await resume({ lifecycleId: input.lifecycleId, localRoot: input.workspaceRoot }).catch( + () => undefined + ); + throw error; + } return { ...sealed, resumeLocal: (signal) => resume(sealed.restore, signal), diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index ee2daed1c..04ed4cf9b 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -61,8 +61,7 @@ export { type LiveTeleportPrewarm, type LiveTeleportAcquireInput, type LiveTeleportEnvironment, - type LiveTeleportConvergenceWatermark, - type LiveTeleportConvergenceProof, + type LiveTeleportDestinationVerification, type LiveTeleportRevokeInput, type LiveTeleportStatusInput, type LiveTeleportLifecycleStatus, diff --git a/packages/cloud/src/live-teleport.test.ts b/packages/cloud/src/live-teleport.test.ts index d2cd911fa..0827452ae 100644 --- a/packages/cloud/src/live-teleport.test.ts +++ b/packages/cloud/src/live-teleport.test.ts @@ -14,41 +14,78 @@ const input = { idempotencyKey: 'session-1:2:acquire', }; -function convergence(overrides: { destinationSha?: string; pendingWriteback?: number } = {}) { - const watermark = { - cursor: 'evt_19312', - manifestSha256: 'a'.repeat(64), - files: 12, - bytes: 2048, - conflictArtifacts: ['.relay/conflicts/shared.txt.writer-b'], - conflictDigest: 'b'.repeat(64), - }; +function verification( + overrides: { + digest?: string; + workspaceRevision?: string; + eventCursor?: string; + pendingWriteback?: number; + } = {} +) { return { - verdict: 'converged', - source: { ...watermark, sealedAt: '2026-08-23T11:59:00.000Z' }, - destination: { - ...watermark, - cursor: 'evt_19313', - manifestSha256: overrides.destinationSha ?? watermark.manifestSha256, + version: 1, + kind: 'relayfile-destination-verification', + verificationId: 'verify-2', + workspaceId: 'workspace-1', + localRoot: '/workspace', + remoteRoot: '/', + sessionId: 'session-1', + generation: 2, + status: 'converged', + observed: { + digest: overrides.digest ?? `sha256:${'a'.repeat(64)}`, + workspaceRevision: overrides.workspaceRevision ?? 'rev_12', + eventCursor: overrides.eventCursor ?? 'evt_19313', + }, + health: { pendingWriteback: overrides.pendingWriteback ?? 0, - hasPendingWriteback: false, + conflicts: 0, + outboxPending: 0, outboxNeedsAttention: false, - ephemeralPaths: [], }, + verifiedAt: '2026-08-23T11:59:00.000Z', }; } describe('CloudLiveTeleportClient', () => { + it('polls an exact 202 acquire with the same idempotency body until active', async () => { + const fetcher = vi + .fn() + .mockResolvedValueOnce(Response.json({ status: 'verifying', retryAfterMs: 0 }, { status: 202 })) + .mockResolvedValueOnce( + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: verification(), + }) + ); + const client = new CloudLiveTeleportClient(fetcher, 'https://cloud.agentrelay.test'); + + await expect(client.acquire(input)).resolves.toMatchObject({ environmentId: 'env-2' }); + expect(fetcher).toHaveBeenCalledTimes(2); + expect(vi.mocked(fetcher).mock.calls[0]?.[1]?.body).toBe(vi.mocked(fetcher).mock.calls[1]?.[1]?.body); + }); + it('accepts only the provider-neutral Cloud WSS bridge contract', async () => { const fetcher = vi.fn(async () => Response.json({ sessionId: 'session-1', generation: 2, + threadId: 'thread-1', + status: 'active', environmentId: 'env-2', connectPath: '/api/v1/live-teleports/connect/session-1/g/2?ticket=opaque', workspaceCwd: '/workspace', - expiresAt: '2026-08-23T12:00:00.000Z', - convergence: convergence(), + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: verification(), }) ); @@ -71,11 +108,14 @@ describe('CloudLiveTeleportClient', () => { Response.json({ sessionId: 'session-1', generation: 2, + threadId: 'thread-1', + status: 'active', environmentId: 'env-2', connectPath: '/api/v1/live-teleports/connect/ticket', workspaceCwd: '/workspace', - expiresAt: '2026-08-23T12:00:00.000Z', - convergence: convergence(), + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: verification(), providerUrl: 'wss://provider.invalid/raw', trafficAccessToken: 'secret', }), @@ -91,12 +131,15 @@ describe('CloudLiveTeleportClient', () => { Response.json({ sessionId: 'session-1', generation: 2, + threadId: 'thread-1', + status: 'active', environmentId: 'env-2', execServerUrl: 'ws://127.0.0.1:4500', connectPath: '/api/v1/live-teleports/connect/ticket', workspaceCwd: '/workspace', - expiresAt: '2026-08-23T12:00:00.000Z', - convergence: convergence(), + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: verification(), }), 'https://cloud.agentrelay.test' ); @@ -115,11 +158,14 @@ describe('CloudLiveTeleportClient', () => { Response.json({ sessionId: 'session-1', generation: 2, + threadId: 'thread-1', + status: 'active', environmentId: 'env-2', connectPath, workspaceCwd: '/workspace', - expiresAt: '2026-08-23T12:00:00.000Z', - convergence: convergence(), + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: verification(), }), 'https://cloud.agentrelay.test' ); @@ -127,22 +173,62 @@ describe('CloudLiveTeleportClient', () => { } }); - it('rejects a time-based convergence claim whose destination hash differs', async () => { + it('rejects a verification whose authoritative digest is malformed', async () => { const client = new CloudLiveTeleportClient( async () => Response.json({ sessionId: 'session-1', generation: 2, + threadId: 'thread-1', + status: 'active', environmentId: 'env-2', connectPath: '/api/v1/live-teleports/connect/ticket', workspaceCwd: '/workspace', - expiresAt: '2026-08-23T12:00:00.000Z', - convergence: convergence({ destinationSha: 'f'.repeat(64) }), + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: verification({ digest: 'f'.repeat(64) }), }), 'https://cloud.agentrelay.test' ); - await expect(client.acquire(input)).rejects.toThrow('non-converged hash/cursor proof'); + await expect(client.acquire(input)).rejects.toThrow('invalid verification.observed.digest'); + }); + + it.each([ + ['bare revision', verification({ workspaceRevision: '12' })], + ['wrong revision namespace', verification({ workspaceRevision: 'evt_12' })], + ['bare cursor', verification({ eventCursor: '19313' })], + ['wrong cursor namespace', verification({ eventCursor: 'rev_19313' })], + ])('rejects %s in Relayfile destination verification', async (_name, proof) => { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: proof, + }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.acquire(input)).rejects.toThrow(/verification\.observed/); + }); + + it('bounds a permanently verifying acquire while replaying the same request', async () => { + const fetcher = vi.fn(async () => + Response.json({ status: 'verifying', retryAfterMs: 0 }, { status: 202 }) + ); + const client = new CloudLiveTeleportClient(fetcher, 'https://cloud.agentrelay.test'); + + await expect(client.acquire(input)).rejects.toThrow('bounded verification poll limit'); + expect(fetcher).toHaveBeenCalledTimes(120); + expect(new Set(fetcher.mock.calls.map((call) => call[1]?.body))).toHaveLength(1); }); it('rejects matching hashes when the destination outbox is not drained', async () => { @@ -151,16 +237,19 @@ describe('CloudLiveTeleportClient', () => { Response.json({ sessionId: 'session-1', generation: 2, + threadId: 'thread-1', + status: 'active', environmentId: 'env-2', connectPath: '/api/v1/live-teleports/connect/ticket', workspaceCwd: '/workspace', - expiresAt: '2026-08-23T12:00:00.000Z', - convergence: convergence({ pendingWriteback: 1 }), + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:30:00.000Z', + verification: verification({ pendingWriteback: 1 }), }), 'https://cloud.agentrelay.test' ); - await expect(client.acquire(input)).rejects.toThrow('non-converged hash/cursor proof'); + await expect(client.acquire(input)).rejects.toThrow('mismatched Relayfile verification'); }); it('does not echo a Cloud error containing an opaque bridge or provider secret', async () => { @@ -198,11 +287,35 @@ describe('CloudLiveTeleportClient', () => { status: 'warming', retryAfterMs: 250, }); + expect(fetcher).toHaveBeenNthCalledWith( + 1, + '/api/v1/live-teleports/status', + expect.objectContaining({ method: 'POST' }) + ); await expect( client.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'revoke-2' }) ).resolves.toMatchObject({ status: 'revoked' }); }); + it('accepts cleanup_pending as a non-terminal revoke acknowledgement', async () => { + const client = new CloudLiveTeleportClient( + async () => + Response.json( + { sessionId: 'session-1', generation: 2, status: 'cleanup_pending', retryAfterMs: 10 }, + { status: 202 } + ), + 'https://cloud.agentrelay.test' + ); + await expect( + client.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'revoke-2' }) + ).resolves.toMatchObject({ status: 'cleanup_pending' }); + }); + + it('requires encrypted transport except for explicit loopback development', () => { + expect(() => new CloudLiveTeleportClient(vi.fn(), 'http://cloud.example.test')).toThrow('must use HTTPS'); + expect(() => new CloudLiveTeleportClient(vi.fn(), 'http://127.0.0.1:3000')).not.toThrow(); + }); + it('rejects a successful revoke response that does not prove fencing', async () => { const client = new CloudLiveTeleportClient( async () => Response.json({ sessionId: 'session-1', generation: 2, status: 'ready' }), diff --git a/packages/cloud/src/live-teleport.ts b/packages/cloud/src/live-teleport.ts index b9f631790..813b78f6b 100644 --- a/packages/cloud/src/live-teleport.ts +++ b/packages/cloud/src/live-teleport.ts @@ -23,24 +23,28 @@ export type LiveTeleportAcquireInput = LiveTeleportPrewarmInput & { prewarmId?: string; }; -export type LiveTeleportConvergenceWatermark = { - cursor: string; - manifestSha256: string; - files: number; - bytes: number; - conflictArtifacts: string[]; - conflictDigest: string; -}; - -export type LiveTeleportConvergenceProof = { - verdict: 'converged'; - source: LiveTeleportConvergenceWatermark & { sealedAt: string }; - destination: LiveTeleportConvergenceWatermark & { +export type LiveTeleportDestinationVerification = { + version: 1; + kind: 'relayfile-destination-verification'; + verificationId: string; + workspaceId: string; + localRoot: string; + remoteRoot: '/'; + sessionId: string; + generation: number; + status: 'converged'; + observed: { + digest: string; + workspaceRevision: string; + eventCursor: string; + }; + health: { pendingWriteback: 0; - hasPendingWriteback: false; + conflicts: 0; + outboxPending: 0; outboxNeedsAttention: false; - ephemeralPaths: []; }; + verifiedAt: string; }; export type LiveTeleportEnvironment = { @@ -52,8 +56,9 @@ export type LiveTeleportEnvironment = { /** Derived locally from the constructor-pinned Cloud gateway origin. */ execServerUrl: string; workspaceCwd: string; - expiresAt: string; - convergence: LiveTeleportConvergenceProof; + connectExpiresAt: string; + leaseExpiresAt: string; + verification: LiveTeleportDestinationVerification; }; export type LiveTeleportRevokeInput = { @@ -73,7 +78,7 @@ export type LiveTeleportStatusInput = { export type LiveTeleportLifecycleStatus = { sessionId: string; generation: number; - status: 'warming' | 'ready' | 'failed' | 'revoked' | 'expired'; + status: 'warming' | 'ready' | 'verifying' | 'active' | 'cleanup_pending' | 'failed' | 'revoked' | 'expired'; prewarmId?: string; retryAfterMs?: number; expiresAt?: string; @@ -87,13 +92,15 @@ export interface LiveTeleportCloudClient { prewarm(input: LiveTeleportPrewarmInput): Promise; status(input: LiveTeleportStatusInput): Promise; acquire(input: LiveTeleportAcquireInput): Promise; - revoke(input: LiveTeleportRevokeInput): Promise; + revoke(input: LiveTeleportRevokeInput): Promise; } type Fetcher = (path: string, init?: RequestInit) => Promise; const FORBIDDEN_PROVIDER_FIELD = - /(?:provider.*(?:url|token|credential)|trafficAccessToken|signedPreviewUrl)/i; + /(?:provider.*(?:url|token|credential)|trafficAccessToken|signedPreviewUrl|sealToken|^ticket$)/i; +const MAX_ACQUIRE_ATTEMPTS = 120; +const MAX_ACQUIRE_RETRY_AFTER_MS = 1_000; function isObject(value: unknown): value is Record { return value !== null && typeof value === 'object' && !Array.isArray(value); @@ -155,93 +162,93 @@ function requiredNonNegativeInteger(value: unknown, field: string): number { return Number(value); } -function requiredStringArray(value: unknown, field: string): string[] { - if (!Array.isArray(value) || value.some((entry) => typeof entry !== 'string')) { - throw new Error(`Cloud live-teleport convergence proof has an invalid ${field}.`); - } - return value as string[]; -} - function requiredDigest(value: unknown, field: string): string { const digest = requiredString(value, field); - if (!/^[a-f0-9]{64}$/i.test(digest)) { - throw new Error(`Cloud live-teleport convergence proof has an invalid ${field}.`); + if (!/^sha256:[a-f0-9]{64}$/.test(digest)) { + throw new Error(`Cloud live-teleport verification has an invalid ${field}.`); } - return digest.toLowerCase(); -} - -function requiredWatermark(value: unknown, field: string): LiveTeleportConvergenceWatermark { - if (!isObject(value)) throw new Error(`Cloud live-teleport convergence proof is missing ${field}.`); - return { - cursor: requiredString(value.cursor, `${field}.cursor`), - manifestSha256: requiredDigest(value.manifestSha256, `${field}.manifestSha256`), - files: requiredNonNegativeInteger(value.files, `${field}.files`), - bytes: requiredNonNegativeInteger(value.bytes, `${field}.bytes`), - conflictArtifacts: requiredStringArray(value.conflictArtifacts, `${field}.conflictArtifacts`), - conflictDigest: requiredDigest(value.conflictDigest, `${field}.conflictDigest`), - }; -} - -function parseCounter(value: string): { prefix: string; ordinal: number } | null { - const match = /^([A-Za-z][A-Za-z0-9]*_)?(\d+)$/.exec(value); - if (!match) return null; - const ordinal = Number.parseInt(match[2]!, 10); - return Number.isSafeInteger(ordinal) ? { prefix: match[1] ?? '', ordinal } : null; + return digest; } -function sameStrings(left: readonly string[], right: readonly string[]): boolean { - return left.length === right.length && left.every((entry, index) => entry === right[index]); +function requiredRelayfilePosition(value: unknown, field: string, prefix: 'rev' | 'evt'): string { + const position = requiredString(value, field); + if (!new RegExp(`^(?:0|${prefix}_[0-9]+)$`).test(position)) { + throw new Error(`Cloud live-teleport verification has an invalid ${field}.`); + } + return position; } -function requiredConvergenceProof(value: unknown): LiveTeleportConvergenceProof { - if (!isObject(value) || value.verdict !== 'converged') { - throw new Error('Cloud live-teleport acquire did not return a converged hash/cursor proof.'); +function requiredVerification( + value: unknown, + identity: { sessionId: string; generation: number } +): LiveTeleportDestinationVerification { + if (!isObject(value) || !isObject(value.observed) || !isObject(value.health)) { + throw new Error('Cloud live-teleport acquire did not return Relayfile destination verification.'); } - const source = requiredWatermark(value.source, 'source'); - const destination = requiredWatermark(value.destination, 'destination'); - const sourceObject = value.source as Record; - const destinationObject = value.destination as Record; - const sealedAt = requiredString(sourceObject.sealedAt, 'source.sealedAt'); - if (Number.isNaN(Date.parse(sealedAt))) { - throw new Error('Cloud live-teleport convergence proof has an invalid source.sealedAt.'); - } - const sourceCursor = parseCounter(source.cursor); - const destinationCursor = parseCounter(destination.cursor); - const sameCursorNamespace = - sourceCursor && destinationCursor && sourceCursor.prefix === destinationCursor.prefix; - const hashesMatch = - source.manifestSha256 === destination.manifestSha256 && - source.files === destination.files && - source.bytes === destination.bytes && - sameStrings([...source.conflictArtifacts].sort(), [...destination.conflictArtifacts].sort()) && - source.conflictDigest === destination.conflictDigest; - const outboxHealthy = - destinationObject.pendingWriteback === 0 && - destinationObject.hasPendingWriteback === false && - destinationObject.outboxNeedsAttention === false && - Array.isArray(destinationObject.ephemeralPaths) && - destinationObject.ephemeralPaths.length === 0; + const verifiedAt = requiredString(value.verifiedAt, 'verification.verifiedAt'); if ( - !sameCursorNamespace || - destinationCursor.ordinal < sourceCursor.ordinal || - !hashesMatch || - !outboxHealthy + value.version !== 1 || + value.kind !== 'relayfile-destination-verification' || + value.status !== 'converged' || + value.remoteRoot !== '/' || + value.sessionId !== identity.sessionId || + value.generation !== identity.generation || + value.health.pendingWriteback !== 0 || + value.health.conflicts !== 0 || + value.health.outboxPending !== 0 || + value.health.outboxNeedsAttention !== false || + Number.isNaN(Date.parse(verifiedAt)) ) { - throw new Error('Cloud live-teleport acquire returned a non-converged hash/cursor proof.'); + throw new Error('Cloud live-teleport acquire returned a mismatched Relayfile verification.'); } return { - verdict: 'converged', - source: { ...source, sealedAt }, - destination: { - ...destination, + version: 1, + kind: 'relayfile-destination-verification', + verificationId: requiredString(value.verificationId, 'verification.verificationId'), + workspaceId: requiredString(value.workspaceId, 'verification.workspaceId'), + localRoot: requiredString(value.localRoot, 'verification.localRoot'), + remoteRoot: '/', + sessionId: identity.sessionId, + generation: identity.generation, + status: 'converged', + observed: { + digest: requiredDigest(value.observed.digest, 'verification.observed.digest'), + workspaceRevision: requiredRelayfilePosition( + value.observed.workspaceRevision, + 'verification.observed.workspaceRevision', + 'rev' + ), + eventCursor: requiredRelayfilePosition( + value.observed.eventCursor, + 'verification.observed.eventCursor', + 'evt' + ), + }, + health: { pendingWriteback: 0, - hasPendingWriteback: false, + conflicts: 0, + outboxPending: 0, outboxNeedsAttention: false, - ephemeralPaths: [], }, + verifiedAt, }; } +function wait(milliseconds: number, signal?: AbortSignal): Promise { + if (signal?.aborted) return Promise.reject(new DOMException('Aborted', 'AbortError')); + return new Promise((resolve, reject) => { + const timer = setTimeout(() => { + signal?.removeEventListener('abort', abort); + resolve(); + }, milliseconds); + const abort = () => { + clearTimeout(timer); + reject(new DOMException('Aborted', 'AbortError')); + }; + signal?.addEventListener('abort', abort, { once: true }); + }); +} + /** * Provider-neutral Cloud control-plane client. The only execution address Relay * accepts is Cloud's short-lived WSS bridge; raw provider URLs and credentials @@ -260,8 +267,10 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { } catch { throw new Error('Cloud live-teleport requires a valid pinned gateway origin.'); } - if (parsed.protocol !== 'https:' && parsed.protocol !== 'http:') { - throw new Error('Cloud live-teleport gateway origin must use HTTP or HTTPS.'); + const loopback = + parsed.hostname === '127.0.0.1' || parsed.hostname === '::1' || parsed.hostname === 'localhost'; + if (parsed.protocol !== 'https:' && !(parsed.protocol === 'http:' && loopback)) { + throw new Error('Cloud live-teleport gateway origin must use HTTPS (HTTP is loopback-only).'); } if (parsed.username || parsed.password) { throw new Error('Cloud live-teleport gateway origin must not contain credentials.'); @@ -306,13 +315,35 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { async acquire(input: LiveTeleportAcquireInput): Promise { const { signal, ...request } = input; - const response = await this.fetcher('/api/v1/live-teleports/acquire', { - method: 'POST', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(request), - signal, - }); - const payload = await readPayload(response); + let payload: unknown; + let stillVerifying = false; + for (let attempt = 0; attempt < MAX_ACQUIRE_ATTEMPTS; attempt += 1) { + const response = await this.fetcher('/api/v1/live-teleports/acquire', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(request), + signal, + }); + payload = await readPayload(response); + if (response.status !== 202) { + stillVerifying = false; + break; + } + stillVerifying = true; + if (!isObject(payload) || payload.status !== 'verifying') { + throw new Error('Cloud live-teleport acquire returned an invalid pending response.'); + } + await wait( + Math.min( + optionalNonNegativeInteger(payload.retryAfterMs, 'retryAfterMs') ?? 500, + MAX_ACQUIRE_RETRY_AFTER_MS + ), + signal + ); + } + if (stillVerifying) { + throw new Error('Cloud live-teleport acquire exceeded the bounded verification poll limit.'); + } if (!isObject(payload)) throw new Error('Cloud live-teleport acquire returned an invalid response.'); if ('execServerUrl' in payload) { @@ -321,24 +352,37 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { const connectPath = this.requiredConnectPath(payload.connectPath); const execServerUrl = this.execServerUrl(connectPath); - const expiresAt = requiredString(payload.expiresAt, 'expiresAt'); - if (Number.isNaN(Date.parse(expiresAt))) { - throw new Error('Cloud live-teleport acquire returned an invalid expiresAt.'); + const sessionId = requiredString(payload.sessionId, 'sessionId'); + const generation = requiredGeneration(payload.generation); + const threadId = requiredString(payload.threadId, 'threadId'); + const connectExpiresAt = requiredString(payload.connectExpiresAt, 'connectExpiresAt'); + const leaseExpiresAt = requiredString(payload.leaseExpiresAt, 'leaseExpiresAt'); + if ( + payload.status !== 'active' || + sessionId !== input.sessionId || + generation !== input.generation || + threadId !== input.threadId || + Number.isNaN(Date.parse(connectExpiresAt)) || + Number.isNaN(Date.parse(leaseExpiresAt)) || + Date.parse(leaseExpiresAt) <= Date.parse(connectExpiresAt) + ) { + throw new Error('Cloud live-teleport acquire returned invalid active lifecycle metadata.'); } return { - sessionId: requiredString(payload.sessionId, 'sessionId'), - generation: requiredGeneration(payload.generation), + sessionId, + generation, environmentId: requiredString(payload.environmentId, 'environmentId'), connectPath, execServerUrl, workspaceCwd: requiredString(payload.workspaceCwd, 'workspaceCwd'), - expiresAt, - convergence: requiredConvergenceProof(payload.convergence), + connectExpiresAt, + leaseExpiresAt, + verification: requiredVerification(payload.verification, { sessionId, generation }), }; } - async revoke(input: LiveTeleportRevokeInput): Promise { + async revoke(input: LiveTeleportRevokeInput): Promise { const { signal, ...request } = input; const response = await this.fetcher('/api/v1/live-teleports/revoke', { method: 'POST', @@ -349,10 +393,10 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { const payload = await readPayload(response); if (!isObject(payload)) throw new Error('Cloud live-teleport revoke returned an invalid response.'); const status = this.parseLifecycleStatus(payload); - if (status.status !== 'revoked' && status.status !== 'expired') { + if (status.status !== 'cleanup_pending' && status.status !== 'revoked' && status.status !== 'expired') { throw new Error('Cloud live-teleport revoke was not confirmed.'); } - return { ...status, status: status.status }; + return status; } private parseLifecycleStatus(payload: Record): LiveTeleportLifecycleStatus { @@ -360,6 +404,9 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { if ( status !== 'warming' && status !== 'ready' && + status !== 'verifying' && + status !== 'active' && + status !== 'cleanup_pending' && status !== 'failed' && status !== 'revoked' && status !== 'expired' From 46ca14e79fc43d134cde66c0848d62d43f32a029 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Sun, 23 Aug 2026 19:22:27 +0200 Subject: [PATCH 04/16] fix(codex): close teleport recovery review gaps --- CHANGELOG.md | 2 +- packages/cli/src/cli/commands/codex.test.ts | 76 +++- packages/cli/src/cli/commands/codex.ts | 114 +++-- .../cli/src/cli/lib/codex-app-server.test.ts | 179 +++++++- packages/cli/src/cli/lib/codex-app-server.ts | 140 ++++++- .../src/cli/lib/codex-live-controller.test.ts | 393 +++++++++++++++++- .../cli/src/cli/lib/codex-live-controller.ts | 121 ++++-- packages/cloud/src/index.ts | 1 + packages/cloud/src/live-teleport.test.ts | 149 ++++++- packages/cloud/src/live-teleport.ts | 121 +++++- 10 files changed, 1145 insertions(+), 151 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f5fafc7d..fdfb3ee4b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `relay codex run` starts a Relay-managed local Codex app-server/thread, and `relay codex teleport` seals its active Relayfile poll mount at the next turn boundary, requires exact Relayfile destination verification before Cloud execution, reconciles lost turn responses by stable client message ID, and requires confirmed fencing plus ownership handback and mount readiness before local rollback. +- `relay codex run` starts a Relay-managed local Codex app-server/thread, and `relay codex teleport` seals its active Relayfile poll mount at the next turn boundary, binds exact Relayfile destination verification to the source checkpoint, requires a 40-minute Cloud turn lease, preserves reconciled assistant answers, transparently runs the same prompt locally after a confirmed pre-submission cutover failure, and requires confirmed fencing plus ownership handback and mount readiness before local rollback. ## [Unreleased - Patch] diff --git a/packages/cli/src/cli/commands/codex.test.ts b/packages/cli/src/cli/commands/codex.test.ts index 8b68f8f58..4fb046472 100644 --- a/packages/cli/src/cli/commands/codex.test.ts +++ b/packages/cli/src/cli/commands/codex.test.ts @@ -1,33 +1,79 @@ import { Command } from 'commander'; import { describe, expect, it, vi } from 'vitest'; -import { CodexTurnRecoveredError } from '../lib/codex-live-controller.js'; -import { registerCodexCommands, runManagedCodexTurn } from './codex.js'; +import { CodexTurnRecordedError } from '../lib/codex-live-controller.js'; +import { registerCodexCommands, runManagedCodexTurn, surfaceRecoveredCodexTerminal } from './codex.js'; describe('runManagedCodexTurn', () => { - it('reports a recovered acquire failure and accepts the next input on the same controller', async () => { - const runTurn = vi - .fn() - .mockRejectedValueOnce(new CodexTurnRecoveredError('acquire failed but recovered')) - .mockResolvedValueOnce({}); + it('renders an exact assistant answer recovered after completion-notification loss', async () => { + const turn = { + id: 'turn-reconciled', + status: 'completed', + itemsView: 'full', + items: [{ id: 'answer-1', type: 'agentMessage', text: 'the recovered answer' }], + }; + const controller = { + runTurn: vi.fn(async () => ({ + turnId: turn.id, + response: { turn }, + completed: { method: 'turn/completed', params: { threadId: 'thread-1', turn } }, + reconciled: true as const, + })), + status: vi.fn(), + }; + const writeOutput = vi.fn(); + + await runManagedCodexTurn(controller as never, 'recover me', { + json: false, + writeError: vi.fn(), + writeOutput, + }); + + expect(writeOutput).toHaveBeenCalledWith('the recovered answer\n'); + }); + + it('does not report a successful command for a recorded failed turn', async () => { + const terminal = new CodexTurnRecordedError('failed', 'Codex turn ended failed.'); + const runTurn = vi.fn().mockRejectedValue(terminal); const controller = { runTurn, status: vi.fn(() => ({ phase: 'local' as const, threadId: 'thread-1', - generation: 2, + generation: 1, })), }; const writeError = vi.fn(); - await runManagedCodexTurn(controller as never, 'first input', { json: false, writeError }); - await runManagedCodexTurn(controller as never, 'second input', { json: false, writeError }); - - expect(runTurn).toHaveBeenNthCalledWith(1, 'first input'); - expect(runTurn).toHaveBeenNthCalledWith(2, 'second input'); - expect(writeError).toHaveBeenCalledWith(expect.stringContaining('recovered locally')); + await expect( + runManagedCodexTurn(controller as never, 'failed input', { json: false, writeError }) + ).rejects.toBe(terminal); + expect(writeError).toHaveBeenCalledWith(expect.stringContaining('recorded the turn as failed')); }); + it.each(['failed', 'interrupted'] as const)( + 'surfaces a crash-reconciled %s turn nonzero before any new prompt runs', + (status) => { + const terminal = new CodexTurnRecordedError(status, `recorded ${status}`); + const controller = { + takeRecoveredTerminal: vi.fn().mockReturnValueOnce(terminal).mockReturnValue(undefined), + runTurn: vi.fn(), + }; + const writeError = vi.fn(); + + expect(() => surfaceRecoveredCodexTerminal(controller as never, { json: false, writeError })).toThrow( + terminal + ); + + expect(writeError).toHaveBeenCalledWith(expect.stringContaining(`turn as ${status}`)); + expect(() => + surfaceRecoveredCodexTerminal(controller as never, { json: false, writeError }) + ).not.toThrow(); + expect(writeError).toHaveBeenCalledTimes(1); + expect(controller.runTurn).not.toHaveBeenCalled(); + } + ); + it('fails closed instead of continuing when fencing is unconfirmed', async () => { const controller = { runTurn: vi.fn(async () => Promise.reject(new Error('revoke unconfirmed'))), @@ -81,7 +127,7 @@ describe('registerCodexCommands', () => { requestId: 'request-1', expectedGeneration: 4, }); - expect(log).toHaveBeenCalledWith(expect.stringContaining('local controller remains authoritative')); + expect(log).toHaveBeenCalledWith(expect.stringContaining('confirmed pre-submission recovery')); }); it('refuses teleport when the session was not started under Relay control', async () => { diff --git a/packages/cli/src/cli/commands/codex.ts b/packages/cli/src/cli/commands/codex.ts index d23ba68f2..6f19449db 100644 --- a/packages/cli/src/cli/commands/codex.ts +++ b/packages/cli/src/cli/commands/codex.ts @@ -12,12 +12,12 @@ import { probeCodexEnvironmentCapability, StdioCodexAppServerSession, type CodexNotification, + type CodexTurnResult, } from '../lib/codex-app-server.js'; import { CodexLiveController, CodexTurnOutcomeUncertainError, CodexTurnRecordedError, - CodexTurnRecoveredError, FileCodexControllerStateStore, type CodexControllerState, type CodexPersistedMountResumeProvider, @@ -73,39 +73,20 @@ function agentMessageDelta(notification: CodexNotification): string | undefined } export async function runManagedCodexTurn( - controller: Pick, + controller: Pick, text: string, - options: { json: boolean; writeError: (message: string) => void } + options: { + json: boolean; + writeError: (message: string) => void; + writeOutput?: (message: string) => void; + } ): Promise { + let result: CodexTurnResult; try { - await controller.runTurn(text); + result = await controller.runTurn(text); } catch (error) { - const status = controller.status(); - if (error instanceof CodexTurnRecoveredError && status.phase === 'local') { - options.writeError( - options.json - ? `${JSON.stringify({ - method: 'relay/codexTurnRecovered', - params: { - code: 'TURN_FAILED_RECOVERED_LOCALLY', - threadId: status.threadId, - generation: status.generation, - }, - })}\n` - : 'Cloud execution was fenced and the same Codex thread recovered locally; no turn was replayed.\n' - ); - return; - } if (error instanceof CodexTurnRecordedError) { - options.writeError( - options.json - ? `${JSON.stringify({ - method: 'relay/codexTurnRecordedTerminal', - params: { code: 'TURN_RECORDED_TERMINAL', status: error.status }, - })}\n` - : `Codex recorded the turn as ${error.status}; it was not replayed.\n` - ); - return; + surfaceCodexRecordedTerminal(error, options); } if (error instanceof CodexTurnOutcomeUncertainError) { throw new Error('Relay-managed Codex stopped because the last turn outcome is uncertain.', { @@ -117,6 +98,66 @@ export async function runManagedCodexTurn( { cause: error } ); } + if (result.reconciled) writeReconciledTurn(result, options); +} + +export function surfaceCodexRecordedTerminal( + error: CodexTurnRecordedError, + options: { json: boolean; writeError: (message: string) => void } +): never { + options.writeError( + options.json + ? `${JSON.stringify({ + method: 'relay/codexTurnRecordedTerminal', + params: { code: 'TURN_RECORDED_TERMINAL', status: error.status }, + })}\n` + : `Codex recorded the turn as ${error.status}; it was not replayed.\n` + ); + throw error; +} + +export function surfaceRecoveredCodexTerminal( + controller: Pick, + options: { json: boolean; writeError: (message: string) => void } +): void { + const terminal = controller.takeRecoveredTerminal(); + if (terminal) surfaceCodexRecordedTerminal(terminal, options); +} + +function writeReconciledTurn( + result: CodexTurnResult, + options: { json: boolean; writeOutput?: (message: string) => void } +): void { + if (!options.writeOutput) return; + if (options.json) { + options.writeOutput(`${JSON.stringify(result.completed)}\n`); + return; + } + const params = result.completed.params; + const turn = + params && typeof params === 'object' && !Array.isArray(params) + ? (params as Record).turn + : undefined; + const items = + turn && typeof turn === 'object' && !Array.isArray(turn) + ? (turn as Record).items + : undefined; + if (!Array.isArray(items)) { + throw new Error('Reconciled Codex completion did not contain an observable assistant answer.'); + } + const answers = items.flatMap((item) => + item && + typeof item === 'object' && + !Array.isArray(item) && + (item as Record).type === 'agentMessage' && + typeof (item as Record).text === 'string' + ? [(item as Record).text as string] + : [] + ); + if (answers.length === 0) { + throw new Error('Reconciled Codex completion did not contain an observable assistant answer.'); + } + options.writeOutput(`${answers.join('\n')}\n`); } async function listen(server: net.Server, socketPath: string): Promise { @@ -270,15 +311,27 @@ function withDefaults(overrides: Partial = {}): CodexC const server = createControlServer(controller); try { const status = await controller.initialize(); + surfaceRecoveredCodexTerminal(controller, { + json: Boolean(options.json), + writeError: (message) => process.stderr.write(message), + }); await listen(server, paths.socketPath); process.stderr.write( `Relay-managed Codex ${status.threadId} is ready locally (generation ${status.generation}).\n` ); + const recoveredTurn = controller.takeRecoveredTurn(); + if (recoveredTurn) { + writeReconciledTurn(recoveredTurn, { + json: Boolean(options.json), + writeOutput: (message) => process.stdout.write(message), + }); + } if (options.prompt) { await runManagedCodexTurn(controller, options.prompt, { json: Boolean(options.json), writeError: (message) => process.stderr.write(message), + writeOutput: (message) => process.stdout.write(message), }); } const input = readline.createInterface({ @@ -290,6 +343,7 @@ function withDefaults(overrides: Partial = {}): CodexC await runManagedCodexTurn(controller, line, { json: Boolean(options.json), writeError: (message) => process.stderr.write(message), + writeOutput: (message) => process.stdout.write(message), }); if (!options.json) process.stdout.write('\n'); } @@ -380,7 +434,7 @@ export function registerCodexCommands( }) ); deps.log( - `Codex execution teleport queued for generation ${status.generation}; the local controller remains authoritative.` + `Codex execution teleport queued for generation ${status.generation}; the next prompt runs in Cloud after cutover, or locally after confirmed pre-submission recovery.` ); }); diff --git a/packages/cli/src/cli/lib/codex-app-server.test.ts b/packages/cli/src/cli/lib/codex-app-server.test.ts index 882e6c77a..4ce744a2d 100644 --- a/packages/cli/src/cli/lib/codex-app-server.test.ts +++ b/packages/cli/src/cli/lib/codex-app-server.test.ts @@ -64,7 +64,8 @@ function turnsListResponseSchema(): string { data: { properties: { clientId: {}, - type: { enum: ['userMessage'] }, + text: {}, + type: { enum: ['userMessage', 'agentMessage'] }, status: { enum: ['completed', 'interrupted', 'failed', 'inProgress'] }, }, }, @@ -72,6 +73,39 @@ function turnsListResponseSchema(): string { }); } +function turnCompletedSchema(): string { + return JSON.stringify({ + required: ['threadId', 'turn'], + properties: { threadId: { type: 'string' }, turn: { $ref: '#/definitions/turn' } }, + definitions: { + turn: { + required: ['id', 'items', 'status'], + properties: { + id: { type: 'string' }, + items: { type: 'array' }, + status: { enum: ['completed', 'failed', 'interrupted', 'inProgress'] }, + }, + }, + }, + }); +} + +function turnPayload( + status: 'completed' | 'failed' | 'interrupted' | 'inProgress' = 'completed', + id = 'turn-1', + clientId = 'client-turn-1' +) { + return { + id, + status, + itemsView: 'full', + items: [ + { id: `user-${id}`, type: 'userMessage', clientId, content: [] }, + { id: `agent-${id}`, type: 'agentMessage', text: 'observable assistant answer' }, + ], + }; +} + function successfulProbeFile(file: string): string { if (file.endsWith('ClientRequest.json')) { return '{"methods":["environment/add","environment/status","thread/turns/list"]}'; @@ -79,6 +113,7 @@ function successfulProbeFile(file: string): string { if (file.endsWith('TurnStartParams.json')) return turnPolicySchema(); if (file.endsWith('ThreadTurnsListParams.json')) return turnsListParamsSchema(); if (file.endsWith('ThreadTurnsListResponse.json')) return turnsListResponseSchema(); + if (file.endsWith('TurnCompletedNotification.json')) return turnCompletedSchema(); return '{"required":["environmentId","execServerUrl"],"properties":{"environmentId":{},"execServerUrl":{}}}'; } @@ -157,6 +192,33 @@ describe('probeCodexEnvironmentCapability', () => { }) ).rejects.toThrow('turn-reconciliation contract'); }); + + it('fails closed if turn/completed does not carry generated terminal status and items', async () => { + await expect( + probeCodexEnvironmentCapability('codex', { + makeTempDir: async () => '/schema', + execFile: async () => undefined, + readFile: async (file) => { + if (file.endsWith('TurnCompletedNotification.json')) { + return JSON.stringify({ + properties: { turn: { $ref: '#/definitions/turn' } }, + definitions: { + turn: { + required: ['id', 'status'], + properties: { + id: {}, + status: { enum: ['completed', 'failed', 'interrupted', 'inProgress'] }, + }, + }, + }, + }); + } + return successfulProbeFile(file); + }, + remove: async () => undefined, + }) + ).rejects.toThrow('terminal-turn contract'); + }); }); describe('StdioCodexAppServerSession', () => { @@ -208,7 +270,7 @@ describe('StdioCodexAppServerSession', () => { }); child.stdout.write(`${JSON.stringify({ id: 1, result: { turn: { id: 'turn-1' } } })}\n`); child.stdout.write( - `${JSON.stringify({ method: 'turn/completed', params: { threadId: 'thread-1', turn: { id: 'turn-1' } } })}\n` + `${JSON.stringify({ method: 'turn/completed', params: { threadId: 'thread-1', turn: turnPayload() } })}\n` ); await expect(running).resolves.toMatchObject({ turnId: 'turn-1' }); await session.close(); @@ -238,19 +300,100 @@ describe('StdioCodexAppServerSession', () => { result: { data: [ { - id: 'turn-1', - status: 'completed', - items: [{ type: 'userMessage', clientId: 'client-turn-1' }], + ...turnPayload(), + }, + ], + nextCursor: null, + }, + })}\n` + ); + await expect(reconciling).resolves.toMatchObject({ + status: 'completed', + turnId: 'turn-1', + result: { + response: { + turn: { + items: [ + expect.objectContaining({ type: 'userMessage' }), + expect.objectContaining({ type: 'agentMessage', text: 'observable assistant answer' }), + ], + }, + }, + completed: { + method: 'turn/completed', + params: { + turn: expect.objectContaining({ status: 'completed' }), + }, + }, + }, + }); + await session.close(); + }); + + it('fails completed reconciliation when full history has no recoverable assistant answer', async () => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child); + const requestPromise = nextRequest(child); + const reconciling = session.turnOutcome({ + threadId: 'thread-1', + clientUserMessageId: 'client-turn-1', + }); + const request = await requestPromise; + child.stdout.write( + `${JSON.stringify({ + id: request.id, + result: { + data: [ + { + ...turnPayload(), + items: [{ id: 'user-1', type: 'userMessage', clientId: 'client-turn-1', content: [] }], }, ], nextCursor: null, }, })}\n` ); - await expect(reconciling).resolves.toEqual({ status: 'completed', turnId: 'turn-1' }); + + await expect(reconciling).rejects.toThrow('could not recover the completed assistant answer exactly'); await session.close(); }); + it.each(['completed', 'failed', 'interrupted', 'inProgress'] as const)( + 'interprets generated turn/completed status %s without collapsing it to success', + async (status) => { + const child = fakeChild(); + const session = new StdioCodexAppServerSession(child); + const requestPromise = nextRequest(child); + const running = session.runTurn({ + threadId: 'thread-1', + text: 'status test', + clientUserMessageId: 'client-turn-1', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }); + const request = await requestPromise; + child.stdout.write(`${JSON.stringify({ id: request.id, result: { turn: { id: 'turn-1' } } })}\n`); + child.stdout.write( + `${JSON.stringify({ + method: 'turn/completed', + params: { threadId: 'thread-1', turn: turnPayload(status) }, + })}\n` + ); + + if (status === 'completed') { + await expect(running).resolves.toMatchObject({ turnId: 'turn-1' }); + } else if (status === 'failed' || status === 'interrupted') { + await expect(running).rejects.toMatchObject({ + name: 'CodexAppServerTurnTerminalError', + status, + turnId: 'turn-1', + }); + } else { + await expect(running).rejects.toThrow('non-terminal status inProgress'); + } + await session.close(); + } + ); + it('pins local turns to a non-networked workspace sandbox and rejects an unexpected approval request', async () => { const child = fakeChild(); const session = new StdioCodexAppServerSession(child); @@ -287,7 +430,13 @@ describe('StdioCodexAppServerSession', () => { child.stdout.write(`${JSON.stringify({ id: request.id, result: { turnId: 'turn-local' } })}\n`); child.stdout.write( - `${JSON.stringify({ method: 'turn/completed', params: { threadId: 'thread-1', turnId: 'turn-local' } })}\n` + `${JSON.stringify({ + method: 'turn/completed', + params: { + threadId: 'thread-1', + turn: turnPayload('completed', 'turn-local', 'client-turn-local'), + }, + })}\n` ); await expect(running).resolves.toMatchObject({ turnId: 'turn-local' }); await session.close(); @@ -333,8 +482,12 @@ describe('StdioCodexAppServerSession', () => { forceKillTimeoutMs: 10, }); - child.stdout.write(`${JSON.stringify({ method: 'turn/completed', params: { turnId: 'one' } })}\n`); - child.stdout.write(`${JSON.stringify({ method: 'turn/completed', params: { turnId: 'two' } })}\n`); + child.stdout.write( + `${JSON.stringify({ method: 'turn/completed', params: { turn: turnPayload('completed', 'one') } })}\n` + ); + child.stdout.write( + `${JSON.stringify({ method: 'turn/completed', params: { turn: turnPayload('completed', 'two') } })}\n` + ); await vi.waitFor(() => expect(child.kill).toHaveBeenCalledWith('SIGTERM')); await session.close(); @@ -364,7 +517,13 @@ describe('StdioCodexAppServerSession', () => { } child.stdout.write(`${JSON.stringify({ id: request.id, result: { turnId: 'turn-long' } })}\n`); child.stdout.write( - `${JSON.stringify({ method: 'turn/completed', params: { threadId: 'thread-1', turnId: 'turn-long' } })}\n` + `${JSON.stringify({ + method: 'turn/completed', + params: { + threadId: 'thread-1', + turn: turnPayload('completed', 'turn-long', 'client-turn-long'), + }, + })}\n` ); await expect(running).resolves.toMatchObject({ turnId: 'turn-long' }); diff --git a/packages/cli/src/cli/lib/codex-app-server.ts b/packages/cli/src/cli/lib/codex-app-server.ts index 773b670f6..ac6e54f56 100644 --- a/packages/cli/src/cli/lib/codex-app-server.ts +++ b/packages/cli/src/cli/lib/codex-app-server.ts @@ -15,13 +15,26 @@ export type CodexTurnResult = { turnId?: string; response: unknown; completed: CodexNotification; + /** Relay reconstructed this exact completion from full persisted turn history. */ + reconciled?: true; }; export type CodexTurnOutcome = - | { status: 'completed'; turnId: string } + | { status: 'completed'; turnId: string; result: CodexTurnResult } | { status: 'failed' | 'interrupted' | 'inProgress'; turnId: string } | { status: 'absent' }; +export class CodexAppServerTurnTerminalError extends Error { + constructor( + readonly status: 'failed' | 'interrupted', + readonly turnId: string, + readonly completed: CodexNotification + ) { + super(`Codex turn ended ${status}.`); + this.name = 'CodexAppServerTurnTerminalError'; + } +} + export type CodexTurnExecution = | { kind: 'local'; workspaceRoot: string } | { @@ -118,6 +131,8 @@ function assertTurnReconciliationSchema(requests: string, listParams: string, li !schemaNodeContainsEnum(params, params.properties.sortDirection, 'desc') || !schemaNodeContainsProperty(response, response.properties.data, 'clientId') || !schemaNodeContainsEnum(response, response.properties.data, 'userMessage') || + !schemaNodeContainsEnum(response, response.properties.data, 'agentMessage') || + !schemaNodeContainsProperty(response, response.properties.data, 'text') || !schemaNodeContainsEnum(response, response.properties.data, 'completed') || !schemaNodeContainsEnum(response, response.properties.data, 'interrupted') || !schemaNodeContainsEnum(response, response.properties.data, 'failed') || @@ -127,6 +142,29 @@ function assertTurnReconciliationSchema(requests: string, listParams: string, li } } +function assertTurnCompletedSchema(completedNotification: string): void { + const notification = JSON.parse(completedNotification) as { + required?: unknown; + properties?: Record; + }; + const required = Array.isArray(notification.required) ? notification.required : []; + const turn = notification.properties?.turn; + if ( + !required.includes('threadId') || + !required.includes('turn') || + !turn || + !schemaNodeRequiresProperty(notification, turn, 'id') || + !schemaNodeRequiresProperty(notification, turn, 'items') || + !schemaNodeRequiresProperty(notification, turn, 'status') || + !schemaNodeContainsEnum(notification, turn, 'completed') || + !schemaNodeContainsEnum(notification, turn, 'failed') || + !schemaNodeContainsEnum(notification, turn, 'interrupted') || + !schemaNodeContainsEnum(notification, turn, 'inProgress') + ) { + throw new Error("Codex turn/completed does not match Relay's terminal-turn contract."); + } +} + function resolveSchemaRef(root: unknown, value: unknown): unknown { if (!value || typeof value !== 'object' || Array.isArray(value)) return value; const ref = (value as Record).$ref; @@ -167,6 +205,19 @@ function schemaNodeContainsProperty(root: unknown, value: unknown, expected: str .some(([, entry]) => schemaNodeContainsProperty(root, entry, expected)); } +function schemaNodeRequiresProperty(root: unknown, value: unknown, expected: string): boolean { + const resolved = resolveSchemaRef(root, value); + if (Array.isArray(resolved)) { + return resolved.some((entry) => schemaNodeRequiresProperty(root, entry, expected)); + } + if (!resolved || typeof resolved !== 'object') return false; + const object = resolved as Record; + if (Array.isArray(object.required) && object.required.includes(expected)) return true; + return Object.entries(object) + .filter(([key]) => key !== 'definitions') + .some(([, entry]) => schemaNodeRequiresProperty(root, entry, expected)); +} + /** * Probe the locally installed binary rather than assuming an experimental * protocol from Relay's build-time Codex version. Both methods and the exact @@ -187,13 +238,15 @@ export async function probeCodexEnvironmentCapability( '--out', directory, ]); - const [requests, addParams, turnParams, turnsListParams, turnsListResponse] = await Promise.all([ - deps.readFile(path.join(directory, 'ClientRequest.json')), - deps.readFile(path.join(directory, 'v2', 'EnvironmentAddParams.json')), - deps.readFile(path.join(directory, 'v2', 'TurnStartParams.json')), - deps.readFile(path.join(directory, 'v2', 'ThreadTurnsListParams.json')), - deps.readFile(path.join(directory, 'v2', 'ThreadTurnsListResponse.json')), - ]); + const [requests, addParams, turnParams, turnsListParams, turnsListResponse, turnCompletedNotification] = + await Promise.all([ + deps.readFile(path.join(directory, 'ClientRequest.json')), + deps.readFile(path.join(directory, 'v2', 'EnvironmentAddParams.json')), + deps.readFile(path.join(directory, 'v2', 'TurnStartParams.json')), + deps.readFile(path.join(directory, 'v2', 'ThreadTurnsListParams.json')), + deps.readFile(path.join(directory, 'v2', 'ThreadTurnsListResponse.json')), + deps.readFile(path.join(directory, 'v2', 'TurnCompletedNotification.json')), + ]); if (!requests.includes('"environment/add"') || !requests.includes('"environment/status"')) { throw new Error( 'This Codex app-server does not expose both experimental environment/add and environment/status.' @@ -203,6 +256,7 @@ export async function probeCodexEnvironmentCapability( assertEnvironmentAddSchema(addParams); assertTurnPolicySchema(turnParams); assertTurnReconciliationSchema(requests, turnsListParams, turnsListResponse); + assertTurnCompletedSchema(turnCompletedNotification); return { environmentAdd: true, environmentStatus: true, explicitTurnPolicy: true }; } finally { @@ -257,6 +311,34 @@ function notificationTurnId(notification: CodexNotification): string | undefined return stringAt(notification.params, 'turnId') ?? stringAt(notification.params, 'turn', 'id'); } +type CodexTurnPayload = Record & { + id: string; + items: unknown[]; + status: 'completed' | 'failed' | 'interrupted' | 'inProgress'; +}; + +function requiredTurnPayload(value: unknown, context: string): CodexTurnPayload { + if (!isObject(value) || typeof value.id !== 'string' || !value.id.trim() || !Array.isArray(value.items)) { + throw new Error(`Codex ${context} returned an invalid turn payload.`); + } + if ( + value.status !== 'completed' && + value.status !== 'failed' && + value.status !== 'interrupted' && + value.status !== 'inProgress' + ) { + throw new Error(`Codex ${context} returned an invalid turn status.`); + } + return value as CodexTurnPayload; +} + +function hasExactAssistantAnswer(turn: CodexTurnPayload): boolean { + if (turn.itemsView !== undefined && turn.itemsView !== 'full') return false; + return turn.items.some( + (item) => isObject(item) && item.type === 'agentMessage' && typeof item.text === 'string' + ); +} + /** Newline-delimited Codex app-server transport. Codex omits the jsonrpc field. */ export class StdioCodexAppServerSession implements CodexAppServerSession { private readonly pending = new Map(); @@ -404,6 +486,19 @@ export class StdioCodexAppServerSession implements CodexAppServerSession { (!turnId || !notificationTurnId(notification) || notificationTurnId(notification) === turnId), this.turnTimeoutMs ); + const completedTurn = requiredTurnPayload( + isObject(completed.params) ? completed.params.turn : undefined, + 'turn/completed' + ); + if (turnId && completedTurn.id !== turnId) { + throw new Error('Codex turn/completed returned a mismatched turn id.'); + } + if (completedTurn.status === 'failed' || completedTurn.status === 'interrupted') { + throw new CodexAppServerTurnTerminalError(completedTurn.status, completedTurn.id, completed); + } + if (completedTurn.status !== 'completed') { + throw new Error(`Codex turn/completed returned non-terminal status ${completedTurn.status}.`); + } return { turnId, response, completed }; } @@ -429,15 +524,28 @@ export class StdioCodexAppServerSession implements CodexAppServerSession { isObject(item) && item.type === 'userMessage' && item.clientId === input.clientUserMessageId ); if (!matches) continue; - if ( - candidate.status !== 'completed' && - candidate.status !== 'failed' && - candidate.status !== 'interrupted' && - candidate.status !== 'inProgress' - ) { - throw new Error('Codex thread/turns/list returned an invalid turn status.'); + const turn = requiredTurnPayload(candidate, 'thread/turns/list'); + if (turn.status === 'completed') { + if (!hasExactAssistantAnswer(turn)) { + throw new Error( + 'Codex thread/turns/list could not recover the completed assistant answer exactly.' + ); + } + return { + status: 'completed', + turnId: turn.id, + result: { + turnId: turn.id, + response: { turn }, + completed: { + method: 'turn/completed', + params: { threadId: input.threadId, turn }, + }, + reconciled: true, + }, + }; } - return { status: candidate.status, turnId: candidate.id }; + return { status: turn.status, turnId: turn.id }; } const nextCursor = typeof response.nextCursor === 'string' ? response.nextCursor : undefined; if (!nextCursor) return { status: 'absent' }; diff --git a/packages/cli/src/cli/lib/codex-live-controller.test.ts b/packages/cli/src/cli/lib/codex-live-controller.test.ts index cefe43e8f..9daa28a06 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.test.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.test.ts @@ -15,10 +15,21 @@ import { type CodexWorkspaceSealProvider, } from './codex-live-controller.js'; import type { CodexAppServerSession, CodexTurnResult } from './codex-app-server.js'; +import { CodexAppServerTurnTerminalError } from './codex-app-server.js'; const source = { kind: 'relayfile-checkpoint-seal' as const, - receipt: { sealId: 'seal-1', sealToken: 'opaque' }, + receipt: { + sealId: 'seal-1', + sealToken: 'opaque', + workspaceId: 'workspace-1', + root: '/', + sessionId: 'session-1', + generation: 1, + digest: `sha256:${'a'.repeat(64)}`, + workspaceRevision: 'rev_10', + eventCursor: 'evt_10', + }, }; function sealHandle(overrides: Partial = {}): CodexWorkspaceSealHandle { @@ -70,7 +81,36 @@ function deferred() { } function turnResult(turnId = 'turn-1'): CodexTurnResult { - return { turnId, response: {}, completed: { method: 'turn/completed' } }; + const turn = { + id: turnId, + status: 'completed', + itemsView: 'full', + items: [{ id: `answer-${turnId}`, type: 'agentMessage', text: `answer for ${turnId}` }], + }; + return { + turnId, + response: { turn }, + completed: { method: 'turn/completed', params: { threadId: 'thread-1', turn } }, + }; +} + +function completedOutcome(turnId: string, answer = `answer for ${turnId}`) { + const turn = { + id: turnId, + status: 'completed', + itemsView: 'full', + items: [{ id: `answer-${turnId}`, type: 'agentMessage', text: answer }], + }; + return { + status: 'completed' as const, + turnId, + result: { + turnId, + response: { turn }, + completed: { method: 'turn/completed', params: { threadId: 'thread-1', turn } }, + reconciled: true as const, + }, + }; } function memoryStore(initial: CodexControllerState | null = null): CodexControllerStateStore & { @@ -115,7 +155,10 @@ function cloud(overrides: Partial = {}): LiveTeleportCl generation: input.generation, status: 'ready' as const, })), - status: vi.fn(async (input) => lifecycle(input, 'active')), + status: vi.fn(async (input) => ({ + ...lifecycle(input, 'active'), + leaseExpiresAt: '2026-08-23T13:00:00.000Z', + })), acquire: vi.fn(async (input) => ({ sessionId: input.sessionId, generation: input.generation, @@ -142,6 +185,7 @@ function createController( resumePersistedLocalMount?: CodexPersistedMountResumeProvider; lifecycleDeadlineMs?: number; lifecyclePollIntervalMs?: number; + now?: () => Date; } = {} ) { const store = options.store ?? memoryStore(); @@ -183,7 +227,7 @@ function createController( checkpointAndSeal, resumePersistedLocalMount, sleep: async () => undefined, - now: () => new Date('2026-08-23T12:00:00.000Z'), + now: options.now ?? (() => new Date('2026-08-23T12:00:00.000Z')), sessionId: () => 'session-1', operationId: () => `operation-${++operation}`, pid: 123, @@ -320,7 +364,41 @@ describe('CodexLiveController', () => { expect.objectContaining({ idempotencyKey: 'session-1:1:revoke' }) ); expect(cloudClient.status).not.toHaveBeenCalled(); - expect(store.value).toMatchObject({ cloudLifecycle: 'none' }); + expect(store.value).toMatchObject({ cloudLifecycle: 'none', generation: 2 }); + }); + + it('advances prewarm identity after graceful fence before close → initialize → teleport', async () => { + const store = memoryStore(); + const cloudClient = cloud(); + const first = createController({ store, cloud: cloudClient, sessions: [appServer()] }); + await first.controller.initialize(); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + + await first.controller.close(); + + expect(store.value).toMatchObject({ generation: 2, cloudLifecycle: 'none', phase: 'local' }); + expect(cloudClient.revoke).toHaveBeenCalledWith( + expect.objectContaining({ generation: 1, idempotencyKey: 'session-1:1:revoke' }) + ); + + const resumed = appServer(); + const second = createController({ store, cloud: cloudClient, sessions: [resumed] }); + await second.controller.initialize(); + second.controller.requestTeleport({ requestId: 'request-generation-2', expectedGeneration: 2 }); + await second.controller.runTurn('fresh identity'); + + expect(cloudClient.prewarm).toHaveBeenCalledWith( + expect.objectContaining({ generation: 2, idempotencyKey: 'session-1:2:prewarm' }) + ); + expect(cloudClient.acquire).toHaveBeenCalledWith( + expect.objectContaining({ generation: 2, idempotencyKey: 'session-1:2:acquire' }) + ); + expect(resumed.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'fresh identity', + execution: expect.objectContaining({ kind: 'remote' }), + }) + ); }); it('persists checkpoint lifecycle intent before Relayfile can stop the mount', async () => { @@ -373,9 +451,12 @@ describe('CodexLiveController', () => { expect(original.close).not.toHaveBeenCalled(); cleanup.reject(Object.assign(new Error('checkpoint aborted and source ready'), { name: 'AbortError' })); - await expect(running).rejects.toThrow('Teleport failed before turn submission'); + await expect(running).resolves.toMatchObject({ turnId: 'turn-1' }); expect(original.close).toHaveBeenCalled(); expect(replacement.resumeThread).toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ text: 'deadline', execution: { kind: 'local', workspaceRoot: '/repo' } }) + ); }); it('fences without inverse effects when checkpoint ignores abort and never settles', async () => { @@ -505,6 +586,184 @@ describe('CodexLiveController', () => { }); }); + it.each(['failed', 'interrupted'] as const)( + 'surfaces a normal turn/completed %s terminal and keeps it recorded', + async (terminalStatus) => { + const completed = { + method: 'turn/completed', + params: { + threadId: 'thread-1', + turn: { id: 'turn-terminal', status: terminalStatus, items: [] }, + }, + }; + const original = appServer({ + runTurn: vi.fn(async () => { + throw new CodexAppServerTurnTerminalError(terminalStatus, 'turn-terminal', completed); + }), + }); + const { controller, store } = createController({ sessions: [original] }); + await controller.initialize(); + + await expect(controller.runTurn('terminal prompt')).rejects.toMatchObject({ + name: 'CodexTurnRecordedError', + status: terminalStatus, + }); + + expect(store.value?.inFlightTurn).toBeUndefined(); + expect(controller.status()).toMatchObject({ + phase: 'local', + lastError: `TURN_${terminalStatus.toUpperCase()}`, + }); + } + ); + + it('admits an acquired lease at the exact 40-minute turn horizon', async () => { + const cloudClient = cloud({ + acquire: vi.fn(async (input) => ({ + sessionId: input.sessionId, + generation: input.generation, + environmentId: 'environment-boundary', + connectPath: '/api/v1/live-teleports/connect/boundary', + execServerUrl: 'wss://cloud.agentrelay.test/api/v1/live-teleports/connect/boundary', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:05:00.000Z', + leaseExpiresAt: '2026-08-23T12:40:00.000Z', + verification: { ...verification, generation: input.generation }, + })), + }); + const original = appServer(); + const { controller } = createController({ cloud: cloudClient, sessions: [original] }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-boundary', expectedGeneration: 1 }); + + await expect(controller.runTurn('boundary lease')).resolves.toMatchObject({ turnId: 'turn-1' }); + expect(original.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ execution: expect.objectContaining({ kind: 'remote' }) }) + ); + }); + + it('fences a short acquired lease and transparently submits the same prompt locally', async () => { + const original = appServer(); + const replacement = appServer(); + const cloudClient = cloud({ + acquire: vi.fn(async (input) => ({ + sessionId: input.sessionId, + generation: input.generation, + environmentId: 'environment-short', + connectPath: '/api/v1/live-teleports/connect/short', + execServerUrl: 'wss://cloud.agentrelay.test/api/v1/live-teleports/connect/short', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:05:00.000Z', + leaseExpiresAt: '2026-08-23T12:39:59.999Z', + verification: { ...verification, generation: input.generation }, + })), + }); + const { controller } = createController({ + cloud: cloudClient, + sessions: [original, replacement], + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-short', expectedGeneration: 1 }); + + await expect(controller.runTurn('preserve this prompt')).resolves.toMatchObject({ turnId: 'turn-1' }); + expect(cloudClient.revoke).toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith({ + threadId: 'thread-1', + text: 'preserve this prompt', + clientUserMessageId: 'operation-2', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }); + }); + + it('persists a newer active lease only after same-generation 40-minute preflight renewal', async () => { + let now = new Date('2026-08-23T12:00:00.000Z'); + const original = appServer(); + const cloudClient = cloud({ + status: vi.fn(async (input) => ({ + ...lifecycle(input, 'active'), + leaseExpiresAt: '2026-08-23T13:10:00.000Z', + })), + }); + const { controller, store } = createController({ + cloud: cloudClient, + sessions: [original], + now: () => now, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-renew', expectedGeneration: 1 }); + await controller.runTurn('first remote'); + + now = new Date('2026-08-23T12:20:00.000Z'); + await controller.runTurn('after renewal'); + + expect(store.value?.remote?.leaseExpiresAt).toBe('2026-08-23T13:10:00.000Z'); + expect(original.runTurn).toHaveBeenCalledTimes(2); + }); + + it.each([ + ['missing', undefined], + ['short', '2026-08-23T12:59:59.999Z'], + ])('recovers locally when active status has a %s next-turn lease', async (_name, leaseExpiresAt) => { + let now = new Date('2026-08-23T12:00:00.000Z'); + const original = appServer(); + const replacement = appServer(); + const cloudClient = cloud({ + status: vi.fn(async (input) => ({ + ...lifecycle(input, 'active'), + ...(leaseExpiresAt ? { leaseExpiresAt } : {}), + })), + }); + const { controller } = createController({ + cloud: cloudClient, + sessions: [original, replacement], + now: () => now, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: `request-${_name}`, expectedGeneration: 1 }); + await controller.runTurn('first remote'); + + now = new Date('2026-08-23T12:20:00.000Z'); + await controller.runTurn('must recover locally'); + + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'must recover locally', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }) + ); + expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); + }); + + it('never revives a locally expired lease from a later active status', async () => { + let now = new Date('2026-08-23T12:00:00.000Z'); + const original = appServer(); + const replacement = appServer(); + const status = vi.fn(async (input) => ({ + ...lifecycle(input, 'active'), + leaseExpiresAt: '2026-08-23T14:00:00.000Z', + })); + const cloudClient = cloud({ status }); + const { controller } = createController({ + cloud: cloudClient, + sessions: [original, replacement], + now: () => now, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-expired', expectedGeneration: 1 }); + await controller.runTurn('first remote'); + + now = new Date('2026-08-23T13:00:00.001Z'); + await controller.runTurn('expired locally'); + + expect(status).not.toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'expired locally', + execution: expect.objectContaining({ kind: 'local' }), + }) + ); + }); + it('confirms revoke, reconciles a recorded failed turn, and resumes the same thread without replay', async () => { const original = appServer({ runTurn: vi.fn(async () => Promise.reject(new Error('remote died'))) }); const replacement = appServer({ @@ -538,7 +797,7 @@ describe('CodexLiveController', () => { runTurn: vi.fn(async () => Promise.reject(new Error('notification lost'))), }); const replacement = appServer({ - turnOutcome: vi.fn(async () => ({ status: 'completed' as const, turnId: 'turn-remote-1' })), + turnOutcome: vi.fn(async () => completedOutcome('turn-remote-1', 'recovered exact answer')), }); const { controller } = createController({ sessions: [original, replacement] }); await controller.initialize(); @@ -546,7 +805,16 @@ describe('CodexLiveController', () => { await expect(controller.runTurn('exactly once')).resolves.toMatchObject({ turnId: 'turn-remote-1', - response: { reconciled: true }, + response: { + turn: { + items: [expect.objectContaining({ type: 'agentMessage', text: 'recovered exact answer' })], + }, + }, + completed: { + params: { + turn: expect.objectContaining({ id: 'turn-remote-1', status: 'completed' }), + }, + }, }); expect(replacement.runTurn).not.toHaveBeenCalled(); expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); @@ -564,6 +832,26 @@ describe('CodexLiveController', () => { expect(controller.status()).toMatchObject({ phase: 'outcome_uncertain', execution: 'fenced' }); }); + it('fails outcome-uncertain instead of claiming completion when history lost the exact answer', async () => { + const original = appServer({ + runTurn: vi.fn(async () => Promise.reject(new Error('notification lost'))), + }); + const replacement = appServer({ + turnOutcome: vi.fn(async () => { + throw new Error('completed history omitted assistant answer'); + }), + }); + const { controller } = createController({ sessions: [original, replacement] }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-missing-answer', expectedGeneration: 1 }); + + await expect(controller.runTurn('need exact answer')).rejects.toThrow('outcome is uncertain'); + expect(controller.status()).toMatchObject({ + phase: 'outcome_uncertain', + lastError: 'TURN_RECONCILIATION_UNAVAILABLE', + }); + }); + it('recovers an expired later-turn lease before submitting the next prompt locally', async () => { const original = appServer(); const replacement = appServer(); @@ -628,10 +916,13 @@ describe('CodexLiveController', () => { expect(controller.status()).toMatchObject({ phase: 'local', workspaceSource: 'unavailable' }); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await expect(controller.runTurn('local only')).rejects.toThrow('Teleport failed before turn submission'); + await expect(controller.runTurn('local only')).resolves.toMatchObject({ turnId: 'turn-1' }); expect(cloudClient.acquire).not.toHaveBeenCalled(); expect(cloudClient.revoke).toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ text: 'local only', execution: { kind: 'local', workspaceRoot: '/repo' } }) + ); expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); }); @@ -655,12 +946,16 @@ describe('CodexLiveController', () => { await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await expect(controller.runTurn('must stay on this mount')).rejects.toThrow( - 'Teleport failed before turn submission' - ); + await expect(controller.runTurn('must stay on this mount')).resolves.toMatchObject({ turnId: 'turn-1' }); expect(cloudClient.acquire).not.toHaveBeenCalled(); expect(sealed.resumeLocal).toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'must stay on this mount', + execution: { kind: 'local', workspaceRoot: '/repo' }, + }) + ); expect(store.value?.source).toBeUndefined(); expect(store.value?.mountRestore).toBeUndefined(); expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); @@ -679,17 +974,14 @@ describe('CodexLiveController', () => { await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await expect(controller.runTurn('local after abort')).rejects.toThrow( - 'Teleport failed before turn submission' - ); + await expect(controller.runTurn('local after abort')).resolves.toMatchObject({ turnId: 'turn-1' }); expect(sealed.resumeLocal).toHaveBeenCalled(); expect(replacement.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); - await controller.runTurn('second input'); expect(replacement.runTurn).toHaveBeenCalledWith({ threadId: 'thread-1', - text: 'second input', + text: 'local after abort', clientUserMessageId: 'operation-2', execution: { kind: 'local', workspaceRoot: '/repo' }, }); @@ -769,9 +1061,12 @@ describe('CodexLiveController', () => { await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await expect(controller.runTurn('bounded')).rejects.toThrow('Teleport failed before turn submission'); + await expect(controller.runTurn('bounded')).resolves.toMatchObject({ turnId: 'turn-1' }); expect(cloudClient.acquire).not.toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ text: 'bounded', execution: { kind: 'local', workspaceRoot: '/repo' } }) + ); expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); }); @@ -945,7 +1240,7 @@ describe('CodexLiveController', () => { }) ); const resumed = appServer({ - turnOutcome: vi.fn(async () => ({ status: 'completed' as const, turnId: 'turn-persisted-1' })), + turnOutcome: vi.fn(async () => completedOutcome('turn-persisted-1')), }); const { controller } = createController({ store, sessions: [resumed] }); @@ -955,6 +1250,66 @@ describe('CodexLiveController', () => { clientUserMessageId: 'client-persisted-1', }); expect(store.value?.inFlightTurn).toBeUndefined(); + expect(controller.takeRecoveredTurn()).toMatchObject({ + reconciled: true, + response: { + turn: { + items: [expect.objectContaining({ type: 'agentMessage', text: 'answer for turn-persisted-1' })], + }, + }, + }); + expect(controller.takeRecoveredTurn()).toBeUndefined(); + }); + + it.each(['failed', 'interrupted'] as const)( + 'preserves a persisted crash-reconciled %s terminal for one-shot CLI failure', + async (status) => { + const store = memoryStore( + persistedRemote({ + inFlightTurn: { clientUserMessageId: `client-persisted-${status}`, execution: 'remote' }, + }) + ); + const resumed = appServer({ + turnOutcome: vi.fn(async () => ({ status, turnId: `turn-persisted-${status}` })), + }); + const cloudClient = cloud(); + const { controller } = createController({ store, sessions: [resumed], cloud: cloudClient }); + + await expect(controller.initialize()).resolves.toMatchObject({ phase: 'local', generation: 8 }); + expect(resumed.runTurn).not.toHaveBeenCalled(); + expect(cloudClient.prewarm).not.toHaveBeenCalled(); + expect(store.value?.inFlightTurn).toBeUndefined(); + expect(controller.takeRecoveredTerminal()).toMatchObject({ + name: 'CodexTurnRecordedError', + status, + recordedTerminal: true, + }); + expect(controller.takeRecoveredTerminal()).toBeUndefined(); + expect(controller.takeRecoveredTurn()).toBeUndefined(); + } + ); + + it('keeps restart recovery outcome-uncertain when completed history cannot return the exact answer', async () => { + const store = memoryStore( + persistedRemote({ + inFlightTurn: { clientUserMessageId: 'client-persisted-missing', execution: 'remote' }, + }) + ); + const resumed = appServer({ + turnOutcome: vi.fn(async () => { + throw new Error('completed history omitted assistant answer'); + }), + }); + const { controller } = createController({ store, sessions: [resumed] }); + + await expect(controller.initialize()).rejects.toThrow('outcome is uncertain'); + expect(store.value).toMatchObject({ + phase: 'outcome_uncertain', + generation: 7, + lastError: 'TURN_RECONCILIATION_UNAVAILABLE', + inFlightTurn: { clientUserMessageId: 'client-persisted-missing' }, + }); + expect(controller.takeRecoveredTurn()).toBeUndefined(); }); it('fails recovery if the same thread cannot be resumed after a confirmed fence', async () => { diff --git a/packages/cli/src/cli/lib/codex-live-controller.ts b/packages/cli/src/cli/lib/codex-live-controller.ts index 8bdc66f0e..6d3e30a1c 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.ts @@ -8,8 +8,14 @@ import type { LiveTeleportLifecycleStatus, LiveTeleportWorkspaceSource, } from '@agent-relay/cloud'; +import { LIVE_TELEPORT_MIN_TURN_LEASE_MS } from '@agent-relay/cloud'; -import type { CodexAppServerSession, CodexTurnOutcome, CodexTurnResult } from './codex-app-server.js'; +import { + CodexAppServerTurnTerminalError, + type CodexAppServerSession, + type CodexTurnOutcome, + type CodexTurnResult, +} from './codex-app-server.js'; export type CodexControllerPhase = | 'local' @@ -352,15 +358,6 @@ function validatePersistedState(value: unknown): CodexControllerState { return { ...(state as CodexControllerState), cloudLifecycle: inferredCloudLifecycle(state) }; } -export class CodexTurnRecoveredError extends Error { - readonly recoveredLocally = true; - - constructor(message: string, options?: ErrorOptions) { - super(message, options); - this.name = 'CodexTurnRecoveredError'; - } -} - export class CodexTurnRecordedError extends Error { readonly recordedTerminal = true; @@ -447,6 +444,8 @@ export class CodexLiveController { private sealedWorkspace: CodexWorkspaceSealHandle | null = null; private prewarmPromise: Promise | null = null; private prewarmAbort: AbortController | null = null; + private recoveredTurn: CodexTurnResult | undefined; + private recoveredTerminal: CodexTurnRecordedError | undefined; private closing = false; constructor( @@ -520,6 +519,14 @@ export class CodexLiveController { this.markOutcomeUncertain(`TURN_${outcome.status.toUpperCase()}`); throw new CodexTurnOutcomeUncertainError(); } + if (outcome.status === 'completed') { + this.recoveredTurn = outcome.result; + } else if (outcome.status === 'failed' || outcome.status === 'interrupted') { + this.recoveredTerminal = new CodexTurnRecordedError( + outcome.status, + `Codex recorded the crash-reconciled turn as ${outcome.status}; it was not replayed.` + ); + } this.clearInFlightTurn( outcome.status === 'completed' ? 'TURN_COMPLETED_DURING_RECOVERY' @@ -560,7 +567,7 @@ export class CodexLiveController { this.persist(); } - this.schedulePrewarm(); + if (!this.recoveredTerminal) this.schedulePrewarm(); return this.status(); } @@ -568,6 +575,20 @@ export class CodexLiveController { return publicStatus(this.requireState()); } + /** Returns a crash-reconciled completion once so the command can render it. */ + takeRecoveredTurn(): CodexTurnResult | undefined { + const recovered = this.recoveredTurn; + this.recoveredTurn = undefined; + return recovered; + } + + /** Returns a crash-reconciled failed/interrupted terminal once for CLI failure surfacing. */ + takeRecoveredTerminal(): CodexTurnRecordedError | undefined { + const terminal = this.recoveredTerminal; + this.recoveredTerminal = undefined; + return terminal; + } + requestTeleport(request: CodexTeleportRequest): PublicCodexControllerStatus { const state = this.requireState(); if (request.expectedGeneration !== state.generation) { @@ -657,6 +678,14 @@ export class CodexLiveController { } return result; } catch (error) { + if (error instanceof CodexAppServerTurnTerminalError) { + if (remote && this.requireState().phase === 'verifying' && !remote.attached) { + remote.attached = true; + state.phase = 'remote'; + } + this.clearInFlightTurn(`TURN_${error.status.toUpperCase()}`); + throw new CodexTurnRecordedError(error.status, `Codex turn ended ${error.status}.`); + } if (remote) { const outcome = await this.recoverLocal('remote-turn-error-revoke', 'REMOTE_TURN_RECONCILED'); if (outcome?.status === 'completed') return this.resultFromOutcome(outcome); @@ -694,6 +723,10 @@ export class CodexLiveController { async close(): Promise { const state = this.state; + const lifecycleIdentityConsumed = Boolean( + state && + (this.cloudMayOwnResources(state) || this.sealedWorkspace || state.source || state.mountRestore) + ); this.closing = true; this.prewarmAbort?.abort(); await this.prewarmPromise?.catch(() => undefined); @@ -715,16 +748,14 @@ export class CodexLiveController { throw error; } this.appServer = null; - if ( - state && - (fenceConfirmed || !this.cloudMayOwnResources(state)) && - (this.sealedWorkspace || state.source || state.mountRestore) - ) { - try { - await this.resumeLocalMount(); - } catch (error) { - this.markRecoveryFailed('RELAYFILE_RESUME_FAILED_ON_SHUTDOWN'); - throw new Error(state.lastError, { cause: error }); + if (state && (fenceConfirmed || !this.cloudMayOwnResources(state))) { + if (this.sealedWorkspace || state.source || state.mountRestore) { + try { + await this.resumeLocalMount(); + } catch (error) { + this.markRecoveryFailed('RELAYFILE_RESUME_FAILED_ON_SHUTDOWN'); + throw new Error(state.lastError, { cause: error }); + } } if (state.inFlightTurn) { state.phase = 'outcome_uncertain'; @@ -734,6 +765,10 @@ export class CodexLiveController { this.persist(); return; } + if (!lifecycleIdentityConsumed) return; + // A confirmed prewarm-only fence consumes this generation's idempotency + // identity just as surely as acquire/revoke. Never reinitialize and + // replay a revoked resource under the same generation. state.generation += 1; state.phase = 'local'; state.cloudLifecycle = 'none'; @@ -812,6 +847,7 @@ export class CodexLiveController { if (environment.sessionId !== state.sessionId || environment.generation !== state.generation) { throw new Error('Cloud returned a stale or cross-session live-teleport generation.'); } + this.requireTurnLeaseHorizon(environment.leaseExpiresAt); const { connectPath: _connectPath, @@ -842,10 +878,10 @@ export class CodexLiveController { throw error; } await this.recoverLocal('failed-acquire-revoke', 'TELEPORT_ACQUIRE_FAILED_RECOVERED'); - throw new CodexTurnRecoveredError( - 'Teleport failed before turn submission; Cloud fencing and local recovery were confirmed.', - { cause: error } - ); + // No clientUserMessageId exists until applyPendingTeleport returns, so a + // settled failure here is proven pre-submission. Continue this same + // prompt locally on the recovered app-server with a fresh message id. + return; } } @@ -1105,7 +1141,12 @@ export class CodexLiveController { private async ensureRemoteActiveOrRecover(): Promise { const state = this.requireState(); + const remote = state.remote; let active: boolean; + if (!remote || Date.parse(remote.leaseExpiresAt) <= this.deps.now().getTime()) { + await this.recoverLocal('remote-preflight-revoke', 'REMOTE_LEASE_RECOVERED'); + return; + } try { const status = await this.withAbortableLifecycleDeadline( (signal) => @@ -1117,7 +1158,15 @@ export class CodexLiveController { 'Cloud active lease preflight' ); this.assertLifecycleIdentity(status); - active = status.status === 'active'; + active = status.status === 'active' && status.leaseExpiresAt !== undefined; + if (active) { + this.requireTurnLeaseHorizon(status.leaseExpiresAt!); + if (Date.parse(status.leaseExpiresAt!) > Date.parse(remote.leaseExpiresAt)) { + remote.leaseExpiresAt = status.leaseExpiresAt!; + state.updatedAt = this.timestamp(); + this.persist(); + } + } } catch { active = false; } @@ -1135,15 +1184,17 @@ export class CodexLiveController { } private resultFromOutcome(outcome: Extract): CodexTurnResult { - const state = this.requireState(); - return { - turnId: outcome.turnId, - response: { reconciled: true }, - completed: { - method: 'turn/completed', - params: { threadId: state.threadId, turn: { id: outcome.turnId, status: 'completed' } }, - }, - }; + return outcome.result; + } + + private requireTurnLeaseHorizon(leaseExpiresAt: string): void { + const leaseDeadline = Date.parse(leaseExpiresAt); + const minimumDeadline = this.deps.now().getTime() + LIVE_TELEPORT_MIN_TURN_LEASE_MS; + if (Number.isNaN(leaseDeadline) || leaseDeadline < minimumDeadline) { + throw new Error( + `Cloud active lease does not cover the ${LIVE_TELEPORT_MIN_TURN_LEASE_MS / 60_000}-minute turn horizon.` + ); + } } private clearInFlightTurn(code?: string): void { diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index 04ed4cf9b..10b8bc21b 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -55,6 +55,7 @@ export { export { CloudLiveTeleportClient, + LIVE_TELEPORT_MIN_TURN_LEASE_MS, type LiveTeleportCloudClient, type LiveTeleportWorkspaceSource, type LiveTeleportPrewarmInput, diff --git a/packages/cloud/src/live-teleport.test.ts b/packages/cloud/src/live-teleport.test.ts index 0827452ae..eecc29a9f 100644 --- a/packages/cloud/src/live-teleport.test.ts +++ b/packages/cloud/src/live-teleport.test.ts @@ -2,6 +2,18 @@ import { describe, expect, it, vi } from 'vitest'; import { CloudLiveTeleportClient } from './live-teleport.js'; +const receipt = { + sealId: 'seal-1', + sealToken: 'opaque-seal-token', + workspaceId: 'workspace-1', + root: '/', + sessionId: 'session-1', + generation: 2, + digest: `sha256:${'a'.repeat(64)}`, + workspaceRevision: 'rev_12', + eventCursor: 'evt_19313', +}; + const input = { sessionId: 'session-1', threadId: 'thread-1', @@ -9,7 +21,7 @@ const input = { workspaceRoot: '/workspace', source: { kind: 'relayfile-checkpoint-seal' as const, - receipt: { sealId: 'seal-1', sealToken: 'opaque' }, + receipt, }, idempotencyKey: 'session-1:2:acquire', }; @@ -19,6 +31,10 @@ function verification( digest?: string; workspaceRevision?: string; eventCursor?: string; + workspaceId?: string; + remoteRoot?: string; + sessionId?: string; + generation?: number; pendingWriteback?: number; } = {} ) { @@ -26,11 +42,11 @@ function verification( version: 1, kind: 'relayfile-destination-verification', verificationId: 'verify-2', - workspaceId: 'workspace-1', + workspaceId: overrides.workspaceId ?? 'workspace-1', localRoot: '/workspace', - remoteRoot: '/', - sessionId: 'session-1', - generation: 2, + remoteRoot: overrides.remoteRoot ?? '/', + sessionId: overrides.sessionId ?? 'session-1', + generation: overrides.generation ?? 2, status: 'converged', observed: { digest: overrides.digest ?? `sha256:${'a'.repeat(64)}`, @@ -220,6 +236,60 @@ describe('CloudLiveTeleportClient', () => { await expect(client.acquire(input)).rejects.toThrow(/verification\.observed/); }); + it.each([ + ['workspace', verification({ workspaceId: 'workspace-other' })], + ['remote root', verification({ remoteRoot: '/other' })], + ['session', verification({ sessionId: 'session-other' })], + ['generation', verification({ generation: 3 })], + ['digest', verification({ digest: `sha256:${'b'.repeat(64)}` })], + ['workspace revision', verification({ workspaceRevision: 'rev_13' })], + ['event cursor', verification({ eventCursor: 'evt_19314' })], + ])('rejects destination verification that does not bind the receipt %s', async (_name, proof) => { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + verification: proof, + }), + 'https://cloud.agentrelay.test' + ); + + const error = await client.acquire(input).catch((caught: unknown) => caught); + expect(String(error)).toContain('mismatched Relayfile verification'); + expect(String(error)).not.toContain(receipt.sealToken); + }); + + it('rejects any response that echoes the one-use checkpoint capability', async () => { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + verification: { ...verification(), sealToken: receipt.sealToken }, + }), + 'https://cloud.agentrelay.test' + ); + + const error = await client.acquire(input).catch((caught: unknown) => caught); + expect(String(error)).toContain('forbidden provider field'); + expect(String(error)).not.toContain(receipt.sealToken); + }); + it('bounds a permanently verifying acquire while replaying the same request', async () => { const fetcher = vi.fn(async () => Response.json({ status: 'verifying', retryAfterMs: 0 }, { status: 202 }) @@ -268,6 +338,16 @@ describe('CloudLiveTeleportClient', () => { expect(String(error)).not.toContain('provider.invalid'); }); + it('does not leak the seal token through an acquire transport diagnostic', async () => { + const client = new CloudLiveTeleportClient(async (_path, init) => { + throw new Error(`transport rejected ${String(init?.body)}`); + }, 'https://cloud.agentrelay.test'); + + const error = await client.acquire(input).catch((caught: unknown) => caught); + expect(String(error)).toContain('acquire transport failed'); + expect(String(error)).not.toContain(receipt.sealToken); + }); + it('parses bounded lifecycle polling and requires explicit revoke confirmation', async () => { const fetcher = vi .fn() @@ -297,6 +377,36 @@ describe('CloudLiveTeleportClient', () => { ).resolves.toMatchObject({ status: 'revoked' }); }); + it('parses the exact active lease deadline and rejects non-canonical timestamps', async () => { + const fetcher = vi + .fn() + .mockResolvedValueOnce( + Response.json({ + sessionId: 'session-1', + generation: 2, + status: 'active', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + }) + ) + .mockResolvedValueOnce( + Response.json({ + sessionId: 'session-1', + generation: 2, + status: 'active', + leaseExpiresAt: '2026-08-23T14:45:00+02:00', + }) + ); + const client = new CloudLiveTeleportClient(fetcher, 'https://cloud.agentrelay.test'); + + await expect(client.status({ sessionId: 'session-1', generation: 2 })).resolves.toMatchObject({ + status: 'active', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + }); + await expect(client.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'invalid leaseExpiresAt' + ); + }); + it('accepts cleanup_pending as a non-terminal revoke acknowledgement', async () => { const client = new CloudLiveTeleportClient( async () => @@ -311,6 +421,35 @@ describe('CloudLiveTeleportClient', () => { ).resolves.toMatchObject({ status: 'cleanup_pending' }); }); + it('accepts only the exact identity-matching HTTP 200 terminal no-row revoke response', async () => { + const exact = new CloudLiveTeleportClient( + async () => + Response.json({ sessionId: 'session-1', generation: 2, status: 'revoked' }, { status: 200 }), + 'https://cloud.agentrelay.test' + ); + await expect( + exact.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'revoke-no-row-2' }) + ).resolves.toEqual({ sessionId: 'session-1', generation: 2, status: 'revoked' }); + + const mismatched = new CloudLiveTeleportClient( + async () => + Response.json({ sessionId: 'other-session', generation: 2, status: 'revoked' }, { status: 200 }), + 'https://cloud.agentrelay.test' + ); + await expect( + mismatched.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'revoke-no-row-2' }) + ).rejects.toThrow('stale or cross-session'); + + const notFound = new CloudLiveTeleportClient( + async () => + Response.json({ sessionId: 'session-1', generation: 2, status: 'revoked' }, { status: 404 }), + 'https://cloud.agentrelay.test' + ); + await expect( + notFound.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'revoke-no-row-2' }) + ).rejects.toThrow('request failed (404)'); + }); + it('requires encrypted transport except for explicit loopback development', () => { expect(() => new CloudLiveTeleportClient(vi.fn(), 'http://cloud.example.test')).toThrow('must use HTTPS'); expect(() => new CloudLiveTeleportClient(vi.fn(), 'http://127.0.0.1:3000')).not.toThrow(); diff --git a/packages/cloud/src/live-teleport.ts b/packages/cloud/src/live-teleport.ts index 813b78f6b..0de3f76a3 100644 --- a/packages/cloud/src/live-teleport.ts +++ b/packages/cloud/src/live-teleport.ts @@ -82,6 +82,8 @@ export type LiveTeleportLifecycleStatus = { prewarmId?: string; retryAfterMs?: number; expiresAt?: string; + /** Exact Cloud-authoritative lease deadline, present for active status. */ + leaseExpiresAt?: string; }; export type LiveTeleportRevocation = LiveTeleportLifecycleStatus & { @@ -101,6 +103,8 @@ const FORBIDDEN_PROVIDER_FIELD = /(?:provider.*(?:url|token|credential)|trafficAccessToken|signedPreviewUrl|sealToken|^ticket$)/i; const MAX_ACQUIRE_ATTEMPTS = 120; const MAX_ACQUIRE_RETRY_AFTER_MS = 1_000; +/** Remote turns can wait up to 30m for completion; Cloud reserves another 10m for fencing. */ +export const LIVE_TELEPORT_MIN_TURN_LEASE_MS = 40 * 60_000; function isObject(value: unknown): value is Record { return value !== null && typeof value === 'object' && !Array.isArray(value); @@ -136,6 +140,18 @@ async function readPayload(response: Response): Promise { return payload; } +async function fetchAcquireSafely(fetcher: Fetcher, path: string, init: RequestInit): Promise { + try { + return await fetcher(path, init); + } catch (error) { + if (error instanceof DOMException && error.name === 'AbortError') throw error; + // A transport adapter can include the serialized request body in its + // diagnostic. Acquire carries the one-use seal token, so never relay that + // arbitrary prose to callers. + throw new Error('Cloud live-teleport acquire transport failed.'); + } +} + function requiredString(value: unknown, field: string): string { if (typeof value !== 'string' || !value.trim()) { throw new Error(`Cloud live-teleport response is missing ${field}.`); @@ -178,21 +194,82 @@ function requiredRelayfilePosition(value: unknown, field: string, prefix: 'rev' return position; } +function requiredExactTimestamp(value: unknown, field: string): string { + const timestamp = requiredString(value, field); + const milliseconds = Date.parse(timestamp); + if (Number.isNaN(milliseconds) || new Date(milliseconds).toISOString() !== timestamp) { + throw new Error(`Cloud live-teleport response has an invalid ${field}.`); + } + return timestamp; +} + +type LiveTeleportReceiptBinding = { + workspaceId: string; + remoteRoot: '/'; + sessionId: string; + generation: number; + digest: string; + workspaceRevision: string; + eventCursor: string; +}; + +function requiredReceiptBinding(source: LiveTeleportWorkspaceSource): LiveTeleportReceiptBinding { + if (source.kind !== 'relayfile-checkpoint-seal' || !isObject(source.receipt)) { + throw new Error('Cloud live-teleport acquire requires a Relayfile checkpoint receipt.'); + } + const receipt = source.receipt; + // Validate the capability without ever retaining it in verification output or + // interpolating it into an error. + requiredString(receipt.sealToken, 'source.receipt.sealToken'); + if (requiredString(receipt.root, 'source.receipt.root') !== '/') { + throw new Error('Cloud live-teleport acquire requires logical Relayfile root /.'); + } + return { + workspaceId: requiredString(receipt.workspaceId, 'source.receipt.workspaceId'), + remoteRoot: '/', + sessionId: requiredString(receipt.sessionId, 'source.receipt.sessionId'), + generation: requiredGeneration(receipt.generation), + digest: requiredDigest(receipt.digest, 'source.receipt.digest'), + workspaceRevision: requiredRelayfilePosition( + receipt.workspaceRevision, + 'source.receipt.workspaceRevision', + 'rev' + ), + eventCursor: requiredRelayfilePosition(receipt.eventCursor, 'source.receipt.eventCursor', 'evt'), + }; +} + function requiredVerification( value: unknown, - identity: { sessionId: string; generation: number } + expected: LiveTeleportReceiptBinding ): LiveTeleportDestinationVerification { if (!isObject(value) || !isObject(value.observed) || !isObject(value.health)) { throw new Error('Cloud live-teleport acquire did not return Relayfile destination verification.'); } const verifiedAt = requiredString(value.verifiedAt, 'verification.verifiedAt'); + const workspaceId = requiredString(value.workspaceId, 'verification.workspaceId'); + const digest = requiredDigest(value.observed.digest, 'verification.observed.digest'); + const workspaceRevision = requiredRelayfilePosition( + value.observed.workspaceRevision, + 'verification.observed.workspaceRevision', + 'rev' + ); + const eventCursor = requiredRelayfilePosition( + value.observed.eventCursor, + 'verification.observed.eventCursor', + 'evt' + ); if ( value.version !== 1 || value.kind !== 'relayfile-destination-verification' || value.status !== 'converged' || - value.remoteRoot !== '/' || - value.sessionId !== identity.sessionId || - value.generation !== identity.generation || + workspaceId !== expected.workspaceId || + value.remoteRoot !== expected.remoteRoot || + value.sessionId !== expected.sessionId || + value.generation !== expected.generation || + digest !== expected.digest || + workspaceRevision !== expected.workspaceRevision || + eventCursor !== expected.eventCursor || value.health.pendingWriteback !== 0 || value.health.conflicts !== 0 || value.health.outboxPending !== 0 || @@ -205,24 +282,16 @@ function requiredVerification( version: 1, kind: 'relayfile-destination-verification', verificationId: requiredString(value.verificationId, 'verification.verificationId'), - workspaceId: requiredString(value.workspaceId, 'verification.workspaceId'), + workspaceId, localRoot: requiredString(value.localRoot, 'verification.localRoot'), remoteRoot: '/', - sessionId: identity.sessionId, - generation: identity.generation, + sessionId: expected.sessionId, + generation: expected.generation, status: 'converged', observed: { - digest: requiredDigest(value.observed.digest, 'verification.observed.digest'), - workspaceRevision: requiredRelayfilePosition( - value.observed.workspaceRevision, - 'verification.observed.workspaceRevision', - 'rev' - ), - eventCursor: requiredRelayfilePosition( - value.observed.eventCursor, - 'verification.observed.eventCursor', - 'evt' - ), + digest, + workspaceRevision, + eventCursor, }, health: { pendingWriteback: 0, @@ -315,10 +384,14 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { async acquire(input: LiveTeleportAcquireInput): Promise { const { signal, ...request } = input; + const receiptBinding = requiredReceiptBinding(input.source); + if (receiptBinding.sessionId !== input.sessionId || receiptBinding.generation !== input.generation) { + throw new Error('Cloud live-teleport acquire received a stale or cross-session checkpoint receipt.'); + } let payload: unknown; let stillVerifying = false; for (let attempt = 0; attempt < MAX_ACQUIRE_ATTEMPTS; attempt += 1) { - const response = await this.fetcher('/api/v1/live-teleports/acquire', { + const response = await fetchAcquireSafely(this.fetcher, '/api/v1/live-teleports/acquire', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(request), @@ -378,7 +451,7 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { workspaceCwd: requiredString(payload.workspaceCwd, 'workspaceCwd'), connectExpiresAt, leaseExpiresAt, - verification: requiredVerification(payload.verification, { sessionId, generation }), + verification: requiredVerification(payload.verification, receiptBinding), }; } @@ -393,6 +466,9 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { const payload = await readPayload(response); if (!isObject(payload)) throw new Error('Cloud live-teleport revoke returned an invalid response.'); const status = this.parseLifecycleStatus(payload); + if (status.sessionId !== input.sessionId || status.generation !== input.generation) { + throw new Error('Cloud live-teleport revoke returned a stale or cross-session lifecycle identity.'); + } if (status.status !== 'cleanup_pending' && status.status !== 'revoked' && status.status !== 'expired') { throw new Error('Cloud live-teleport revoke was not confirmed.'); } @@ -418,6 +494,10 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { if (expiresAt && Number.isNaN(Date.parse(expiresAt))) { throw new Error('Cloud live-teleport status returned an invalid expiresAt.'); } + const leaseExpiresAt = + payload.leaseExpiresAt === undefined + ? undefined + : requiredExactTimestamp(payload.leaseExpiresAt, 'leaseExpiresAt'); return { sessionId: requiredString(payload.sessionId, 'sessionId'), generation: requiredGeneration(payload.generation), @@ -429,6 +509,7 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { ? {} : { retryAfterMs: optionalNonNegativeInteger(payload.retryAfterMs, 'retryAfterMs')! }), ...(expiresAt ? { expiresAt } : {}), + ...(leaseExpiresAt ? { leaseExpiresAt } : {}), }; } From 891281e352ab7ab326bce0903a1051fccb049546 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Sun, 23 Aug 2026 19:47:02 +0200 Subject: [PATCH 05/16] fix(codex): preserve teleport recovery evidence --- packages/cli/src/cli/commands/codex.test.ts | 21 ++ packages/cli/src/cli/commands/codex.ts | 17 +- .../src/cli/lib/codex-live-controller.test.ts | 123 ++++++++++- .../cli/src/cli/lib/codex-live-controller.ts | 194 +++++++++++++++--- packages/cloud/src/live-teleport.test.ts | 114 +++++++++- packages/cloud/src/live-teleport.ts | 92 +++++++-- 6 files changed, 516 insertions(+), 45 deletions(-) diff --git a/packages/cli/src/cli/commands/codex.test.ts b/packages/cli/src/cli/commands/codex.test.ts index 4fb046472..aac1d77c2 100644 --- a/packages/cli/src/cli/commands/codex.test.ts +++ b/packages/cli/src/cli/commands/codex.test.ts @@ -57,6 +57,7 @@ describe('runManagedCodexTurn', () => { const terminal = new CodexTurnRecordedError(status, `recorded ${status}`); const controller = { takeRecoveredTerminal: vi.fn().mockReturnValueOnce(terminal).mockReturnValue(undefined), + acknowledgeRecoveredOutcome: vi.fn(), runTurn: vi.fn(), }; const writeError = vi.fn(); @@ -66,6 +67,7 @@ describe('runManagedCodexTurn', () => { ); expect(writeError).toHaveBeenCalledWith(expect.stringContaining(`turn as ${status}`)); + expect(controller.acknowledgeRecoveredOutcome).toHaveBeenCalledOnce(); expect(() => surfaceRecoveredCodexTerminal(controller as never, { json: false, writeError }) ).not.toThrow(); @@ -74,6 +76,25 @@ describe('runManagedCodexTurn', () => { } ); + it('does not acknowledge durable terminal evidence when rendering fails', () => { + const terminal = new CodexTurnRecordedError('failed', 'recorded failed'); + const controller = { + takeRecoveredTerminal: vi.fn(() => terminal), + acknowledgeRecoveredOutcome: vi.fn(), + }; + const renderError = new Error('stderr unavailable'); + + expect(() => + surfaceRecoveredCodexTerminal(controller as never, { + json: false, + writeError: () => { + throw renderError; + }, + }) + ).toThrow(renderError); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); + }); + it('fails closed instead of continuing when fencing is unconfirmed', async () => { const controller = { runTurn: vi.fn(async () => Promise.reject(new Error('revoke unconfirmed'))), diff --git a/packages/cli/src/cli/commands/codex.ts b/packages/cli/src/cli/commands/codex.ts index 6f19449db..efc90029a 100644 --- a/packages/cli/src/cli/commands/codex.ts +++ b/packages/cli/src/cli/commands/codex.ts @@ -105,6 +105,14 @@ export function surfaceCodexRecordedTerminal( error: CodexTurnRecordedError, options: { json: boolean; writeError: (message: string) => void } ): never { + writeCodexRecordedTerminal(error, options); + throw error; +} + +function writeCodexRecordedTerminal( + error: CodexTurnRecordedError, + options: { json: boolean; writeError: (message: string) => void } +): void { options.writeError( options.json ? `${JSON.stringify({ @@ -113,15 +121,17 @@ export function surfaceCodexRecordedTerminal( })}\n` : `Codex recorded the turn as ${error.status}; it was not replayed.\n` ); - throw error; } export function surfaceRecoveredCodexTerminal( - controller: Pick, + controller: Pick, options: { json: boolean; writeError: (message: string) => void } ): void { const terminal = controller.takeRecoveredTerminal(); - if (terminal) surfaceCodexRecordedTerminal(terminal, options); + if (!terminal) return; + writeCodexRecordedTerminal(terminal, options); + controller.acknowledgeRecoveredOutcome(); + throw terminal; } function writeReconciledTurn( @@ -325,6 +335,7 @@ function withDefaults(overrides: Partial = {}): CodexC json: Boolean(options.json), writeOutput: (message) => process.stdout.write(message), }); + controller.acknowledgeRecoveredOutcome(); } if (options.prompt) { diff --git a/packages/cli/src/cli/lib/codex-live-controller.test.ts b/packages/cli/src/cli/lib/codex-live-controller.test.ts index 9daa28a06..72518410c 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.test.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.test.ts @@ -99,7 +99,10 @@ function completedOutcome(turnId: string, answer = `answer for ${turnId}`) { id: turnId, status: 'completed', itemsView: 'full', - items: [{ id: `answer-${turnId}`, type: 'agentMessage', text: answer }], + items: [ + { id: `user-${turnId}`, type: 'userMessage', clientId: 'client-persisted-1', text: 'prompt' }, + { id: `answer-${turnId}`, type: 'agentMessage', text: answer }, + ], }; return { status: 'completed' as const, @@ -401,6 +404,49 @@ describe('CodexLiveController', () => { ); }); + it('atomically finalizes a pending teleport when close consumes its prewarm generation', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-controller-close-pending-')); + const fileStore = new FileCodexControllerStateStore(path.join(directory, 'state.json')); + const store = { + value: null as CodexControllerState | null, + read() { + this.value = fileStore.read(); + return this.value; + }, + write(state: CodexControllerState) { + fileStore.write(state); + this.value = structuredClone(state); + }, + }; + try { + const cloudClient = cloud(); + const first = createController({ store, cloud: cloudClient, sessions: [appServer()] }); + await first.controller.initialize(); + await vi.waitFor(() => expect(fileStore.read()?.prewarmStatus).toBe('ready')); + first.controller.requestTeleport({ requestId: 'request-consumed-on-close', expectedGeneration: 1 }); + + await first.controller.close(); + + expect(fileStore.read()).toMatchObject({ + generation: 2, + phase: 'local', + cloudLifecycle: 'none', + lastRequestId: 'request-consumed-on-close', + }); + expect(fileStore.read()?.pending).toBeUndefined(); + + const second = createController({ store, cloud: cloudClient, sessions: [appServer()] }); + await expect(second.controller.initialize()).resolves.toMatchObject({ + generation: 2, + phase: 'local', + }); + await vi.waitFor(() => expect(fileStore.read()?.prewarmStatus).toBe('ready')); + await second.controller.close(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + it('persists checkpoint lifecycle intent before Relayfile can stop the mount', async () => { const checkpoint = deferred(); let lifecycleId = ''; @@ -807,7 +853,9 @@ describe('CodexLiveController', () => { turnId: 'turn-remote-1', response: { turn: { - items: [expect.objectContaining({ type: 'agentMessage', text: 'recovered exact answer' })], + items: expect.arrayContaining([ + expect.objectContaining({ type: 'agentMessage', text: 'recovered exact answer' }), + ]), }, }, completed: { @@ -1254,11 +1302,43 @@ describe('CodexLiveController', () => { reconciled: true, response: { turn: { - items: [expect.objectContaining({ type: 'agentMessage', text: 'answer for turn-persisted-1' })], + items: expect.arrayContaining([ + expect.objectContaining({ type: 'agentMessage', text: 'answer for turn-persisted-1' }), + ]), }, }, }); expect(controller.takeRecoveredTurn()).toBeUndefined(); + expect(store.value?.recoveredOutcome).toMatchObject({ + sessionId: 'session-1', + threadId: 'thread-1', + generation: 8, + clientUserMessageId: 'client-persisted-1', + turnId: 'turn-persisted-1', + status: 'completed', + }); + + const restartedOnce = appServer(); + const second = createController({ store, sessions: [restartedOnce] }); + await second.controller.initialize(); + expect(restartedOnce.turnOutcome).not.toHaveBeenCalled(); + expect(second.controller.takeRecoveredTurn()).toMatchObject({ + completed: { + params: { + turn: { + items: expect.arrayContaining([expect.objectContaining({ text: 'answer for turn-persisted-1' })]), + }, + }, + }, + }); + + const restartedTwice = appServer(); + const third = createController({ store, sessions: [restartedTwice] }); + await third.controller.initialize(); + expect(restartedTwice.turnOutcome).not.toHaveBeenCalled(); + expect(third.controller.takeRecoveredTurn()).toMatchObject({ turnId: 'turn-persisted-1' }); + third.controller.acknowledgeRecoveredOutcome(); + expect(store.value?.recoveredOutcome).toBeUndefined(); }); it.each(['failed', 'interrupted'] as const)( @@ -1286,6 +1366,28 @@ describe('CodexLiveController', () => { }); expect(controller.takeRecoveredTerminal()).toBeUndefined(); expect(controller.takeRecoveredTurn()).toBeUndefined(); + expect(store.value?.recoveredOutcome).toMatchObject({ + sessionId: 'session-1', + threadId: 'thread-1', + generation: 8, + clientUserMessageId: `client-persisted-${status}`, + turnId: `turn-persisted-${status}`, + status, + }); + + const restartedOnce = appServer(); + const second = createController({ store, cloud: cloudClient, sessions: [restartedOnce] }); + await second.controller.initialize(); + expect(restartedOnce.turnOutcome).not.toHaveBeenCalled(); + expect(second.controller.takeRecoveredTerminal()).toMatchObject({ status }); + + const restartedTwice = appServer(); + const third = createController({ store, cloud: cloudClient, sessions: [restartedTwice] }); + await third.controller.initialize(); + expect(restartedTwice.turnOutcome).not.toHaveBeenCalled(); + expect(third.controller.takeRecoveredTerminal()).toMatchObject({ status }); + third.controller.acknowledgeRecoveredOutcome(); + expect(store.value?.recoveredOutcome).toBeUndefined(); } ); @@ -1438,6 +1540,21 @@ describe('FileCodexControllerStateStore', () => { 'malformed in-flight turn', { ...persistedLocal(), inFlightTurn: { clientUserMessageId: '', execution: 'remote' } }, ], + [ + 'cross-session recovered outcome', + { + ...persistedLocal(), + recoveredOutcome: { + sessionId: 'other-session', + threadId: 'thread-1', + generation: 3, + clientUserMessageId: 'client-recovered', + turnId: 'turn-recovered', + status: 'completed', + result: completedOutcome('turn-recovered').result, + }, + }, + ], ])('rejects %s instead of adopting malformed controller state', (_name, value) => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-controller-state-')); const file = path.join(directory, 'state.json'); diff --git a/packages/cli/src/cli/lib/codex-live-controller.ts b/packages/cli/src/cli/lib/codex-live-controller.ts index 6d3e30a1c..9b7ed5fb3 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.ts @@ -41,6 +41,14 @@ export type CodexTeleportRequest = { expectedGeneration: number; }; +type CodexRecoveredOutcome = { + sessionId: string; + threadId: string; + generation: number; + clientUserMessageId: string; + turnId: string; +} & ({ status: 'completed'; result: CodexTurnResult } | { status: 'failed' | 'interrupted' }); + export type CodexControllerState = { version: 1; sessionId: string; @@ -67,11 +75,16 @@ export type CodexControllerState = { clientUserMessageId: string; execution: 'local' | 'remote'; }; + /** Durable, at-least-once recovery evidence retained until the CLI acknowledges rendering it. */ + recoveredOutcome?: CodexRecoveredOutcome; lastError?: string; updatedAt: string; }; -export type PublicCodexControllerStatus = Omit & { +export type PublicCodexControllerStatus = Omit< + CodexControllerState, + 'source' | 'remote' | 'mountRestore' | 'recoveredOutcome' +> & { execution: 'local' | 'verifying' | 'cloud' | 'fenced'; controller: 'local'; remote?: Pick< @@ -252,6 +265,7 @@ function invalidNestedState(state: Record): boolean { const restore = state.mountRestore === undefined ? undefined : record(state.mountRestore); const remote = state.remote === undefined ? undefined : record(state.remote); const inFlight = state.inFlightTurn === undefined ? undefined : record(state.inFlightTurn); + const recovered = state.recoveredOutcome === undefined ? undefined : record(state.recoveredOutcome); const generation = Number(state.generation); const workspaceRoot = String(state.workspaceRoot); if ( @@ -274,6 +288,18 @@ function invalidNestedState(state: Record): boolean { (!nonEmptyString(inFlight.clientUserMessageId) || (inFlight.execution !== 'local' && inFlight.execution !== 'remote'))) || (state.inFlightTurn !== undefined && !inFlight) || + (state.recoveredOutcome !== undefined && !recovered) || + (inFlight !== undefined && recovered !== undefined) || + (recovered !== undefined && !validRecoveredOutcome(state, recovered)) || + (recovered !== undefined && + (state.phase !== 'local' || + pending !== undefined || + source !== undefined || + restore !== undefined || + remote !== undefined || + state.cloudLifecycle !== 'none' || + state.prewarmId !== undefined || + state.prewarmStatus !== undefined)) || !validOptionalString(state.prewarmId) || (state.prewarmStatus !== undefined && state.prewarmStatus !== 'warming' && @@ -322,6 +348,69 @@ function invalidNestedState(state: Record): boolean { ); } +function validRecoveredOutcome(state: Record, recovered: Record): boolean { + const commonKeys = [ + 'sessionId', + 'threadId', + 'generation', + 'clientUserMessageId', + 'turnId', + 'status', + ] as const; + const bindingMatches = [ + recovered.sessionId === state.sessionId, + recovered.threadId === state.threadId, + recovered.generation === state.generation, + nonEmptyString(recovered.clientUserMessageId), + nonEmptyString(recovered.turnId), + ].every(Boolean); + if (!bindingMatches) { + return false; + } + if (recovered.status === 'failed' || recovered.status === 'interrupted') { + return hasOnlyKeys(recovered, commonKeys); + } + if (recovered.status !== 'completed' || !hasOnlyKeys(recovered, [...commonKeys, 'result'])) { + return false; + } + return validRecoveredCompletion(state, recovered); +} + +function validRecoveredCompletion( + state: Record, + recovered: Record +): boolean { + const result = record(recovered.result) ?? {}; + const response = record(result.response) ?? {}; + const responseTurn = record(response.turn) ?? {}; + const completed = record(result.completed) ?? {}; + const params = record(completed.params) ?? {}; + const turn = record(params.turn) ?? {}; + const items = Array.isArray(turn.items) ? turn.items : []; + const hasBoundUserMessage = items.some((item) => { + const entry = record(item); + return entry?.type === 'userMessage' && entry.clientId === recovered.clientUserMessageId; + }); + const hasAssistantAnswer = items.some((item) => { + const entry = record(item); + return entry?.type === 'agentMessage' && typeof entry.text === 'string'; + }); + return [ + result.reconciled === true, + result.turnId === recovered.turnId, + responseTurn.id === recovered.turnId, + responseTurn.status === 'completed', + completed.method === 'turn/completed', + params.threadId === state.threadId, + turn.id === recovered.turnId, + turn.status === 'completed', + [undefined, 'full'].includes(turn.itemsView as string | undefined), + Array.isArray(turn.items), + hasBoundUserMessage, + hasAssistantAnswer, + ].every(Boolean); +} + function inferredCloudLifecycle(value: Record): CodexCloudLifecycleIntent { if (value.remote) return 'acquired'; if (value.phase === 'acquiring' || value.phase === 'verifying') return 'acquire_requested'; @@ -399,8 +488,19 @@ function isReadyStatus(value: unknown): boolean { return false; } +function sameRecoveredIdentity(left: CodexRecoveredOutcome, right: CodexRecoveredOutcome): boolean { + return ( + left.sessionId === right.sessionId && + left.threadId === right.threadId && + left.generation === right.generation && + left.clientUserMessageId === right.clientUserMessageId && + left.turnId === right.turnId && + left.status === right.status + ); +} + function publicStatus(state: CodexControllerState): PublicCodexControllerStatus { - const { source, remote, mountRestore: _mountRestore, ...rest } = state; + const { source, remote, mountRestore: _mountRestore, recoveredOutcome: _recoveredOutcome, ...rest } = state; const execution = state.phase === 'remote' ? 'cloud' @@ -444,8 +544,7 @@ export class CodexLiveController { private sealedWorkspace: CodexWorkspaceSealHandle | null = null; private prewarmPromise: Promise | null = null; private prewarmAbort: AbortController | null = null; - private recoveredTurn: CodexTurnResult | undefined; - private recoveredTerminal: CodexTurnRecordedError | undefined; + private recoveredEvidenceTaken: CodexRecoveredOutcome | undefined; private closing = false; constructor( @@ -454,6 +553,7 @@ export class CodexLiveController { ) {} async initialize(): Promise { + this.recoveredEvidenceTaken = undefined; await this.deps.probeCapability(); const persisted = this.deps.store.read(); if (persisted && path.resolve(persisted.workspaceRoot) !== path.resolve(this.options.workspaceRoot)) { @@ -510,7 +610,16 @@ export class CodexLiveController { throw new Error(this.requireState().lastError, { cause: error }); } - if (this.requireState().inFlightTurn) { + let recovered: + | { + clientUserMessageId: string; + outcome: + | Extract + | { status: 'failed' | 'interrupted'; turnId: string }; + } + | undefined; + const inFlight = this.requireState().inFlightTurn; + if (inFlight) { const outcome = await this.reconcileInFlightTurn().catch((error) => { this.markOutcomeUncertain('TURN_RECONCILIATION_UNAVAILABLE'); throw new CodexTurnOutcomeUncertainError(undefined, { cause: error }); @@ -520,18 +629,13 @@ export class CodexLiveController { throw new CodexTurnOutcomeUncertainError(); } if (outcome.status === 'completed') { - this.recoveredTurn = outcome.result; + recovered = { clientUserMessageId: inFlight.clientUserMessageId, outcome }; } else if (outcome.status === 'failed' || outcome.status === 'interrupted') { - this.recoveredTerminal = new CodexTurnRecordedError( - outcome.status, - `Codex recorded the crash-reconciled turn as ${outcome.status}; it was not replayed.` - ); + recovered = { + clientUserMessageId: inFlight.clientUserMessageId, + outcome: { status: outcome.status, turnId: outcome.turnId }, + }; } - this.clearInFlightTurn( - outcome.status === 'completed' - ? 'TURN_COMPLETED_DURING_RECOVERY' - : `TURN_${outcome.status.toUpperCase()}` - ); } const state = this.requireState(); @@ -541,6 +645,20 @@ export class CodexLiveController { state.remote = undefined; state.source = undefined; state.mountRestore = undefined; + if (recovered) { + const common = { + sessionId: state.sessionId, + threadId: state.threadId, + generation: state.generation, + clientUserMessageId: recovered.clientUserMessageId, + turnId: recovered.outcome.turnId, + }; + state.recoveredOutcome = + recovered.outcome.status === 'completed' + ? { ...common, status: 'completed', result: recovered.outcome.result } + : { ...common, status: recovered.outcome.status }; + state.inFlightTurn = undefined; + } state.lastError = undefined; state.updatedAt = this.timestamp(); this.persist(); @@ -567,7 +685,7 @@ export class CodexLiveController { this.persist(); } - if (!this.recoveredTerminal) this.schedulePrewarm(); + if (!this.requireState().recoveredOutcome) this.schedulePrewarm(); return this.status(); } @@ -575,18 +693,43 @@ export class CodexLiveController { return publicStatus(this.requireState()); } - /** Returns a crash-reconciled completion once so the command can render it. */ + /** Claims crash-reconciled completion evidence in memory without clearing its durable copy. */ takeRecoveredTurn(): CodexTurnResult | undefined { - const recovered = this.recoveredTurn; - this.recoveredTurn = undefined; - return recovered; + const recovered = this.requireState().recoveredOutcome; + if (!recovered || recovered.status !== 'completed' || this.recoveredEvidenceTaken) return undefined; + this.recoveredEvidenceTaken = recovered; + return recovered.result; } - /** Returns a crash-reconciled failed/interrupted terminal once for CLI failure surfacing. */ + /** Claims terminal recovery evidence in memory without clearing its durable copy. */ takeRecoveredTerminal(): CodexTurnRecordedError | undefined { - const terminal = this.recoveredTerminal; - this.recoveredTerminal = undefined; - return terminal; + const recovered = this.requireState().recoveredOutcome; + if (!recovered || recovered.status === 'completed' || this.recoveredEvidenceTaken) return undefined; + this.recoveredEvidenceTaken = recovered; + return new CodexTurnRecordedError( + recovered.status, + `Codex recorded the crash-reconciled turn as ${recovered.status}; it was not replayed.` + ); + } + + /** Durably acknowledges successful CLI rendering of the currently claimed recovery evidence. */ + acknowledgeRecoveredOutcome(): void { + const state = this.requireState(); + const durable = state.recoveredOutcome; + const claimed = this.recoveredEvidenceTaken; + if (!durable || !claimed || !sameRecoveredIdentity(durable, claimed)) { + throw new Error('Recovered Codex turn acknowledgment does not match durable recovery evidence.'); + } + state.recoveredOutcome = undefined; + state.updatedAt = this.timestamp(); + try { + this.persist(); + this.recoveredEvidenceTaken = undefined; + if (durable.status === 'completed') this.schedulePrewarm(); + } catch (error) { + state.recoveredOutcome = durable; + throw error; + } } requestTeleport(request: CodexTeleportRequest): PublicCodexControllerStatus { @@ -769,6 +912,9 @@ export class CodexLiveController { // A confirmed prewarm-only fence consumes this generation's idempotency // identity just as surely as acquire/revoke. Never reinitialize and // replay a revoked resource under the same generation. + // Finalize any queued request in the same durable write so it cannot + // retain an expectedGeneration from the consumed identity. + state.pending = undefined; state.generation += 1; state.phase = 'local'; state.cloudLifecycle = 'none'; diff --git a/packages/cloud/src/live-teleport.test.ts b/packages/cloud/src/live-teleport.test.ts index eecc29a9f..84e3f7a46 100644 --- a/packages/cloud/src/live-teleport.test.ts +++ b/packages/cloud/src/live-teleport.test.ts @@ -32,6 +32,7 @@ function verification( workspaceRevision?: string; eventCursor?: string; workspaceId?: string; + localRoot?: string; remoteRoot?: string; sessionId?: string; generation?: number; @@ -43,7 +44,7 @@ function verification( kind: 'relayfile-destination-verification', verificationId: 'verify-2', workspaceId: overrides.workspaceId ?? 'workspace-1', - localRoot: '/workspace', + localRoot: overrides.localRoot ?? '/workspace', remoteRoot: overrides.remoteRoot ?? '/', sessionId: overrides.sessionId ?? 'session-1', generation: overrides.generation ?? 2, @@ -138,7 +139,93 @@ describe('CloudLiveTeleportClient', () => { 'https://cloud.agentrelay.test' ); - await expect(client.acquire(input)).rejects.toThrow('forbidden provider field'); + await expect(client.acquire(input)).rejects.toThrow('forbidden sensitive field'); + }); + + it('recursively rejects credential-shaped fields at every remote response boundary', async () => { + const active = { + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + verification: verification(), + }; + const cases = [ + { + response: { prewarmId: 'prewarm-2', generation: 2, status: 'ready', nested: { daytonaApiKey: 'x' } }, + invoke: (client: CloudLiveTeleportClient) => + client.prewarm({ + sessionId: 'session-1', + generation: 2, + workspaceRoot: '/', + idempotencyKey: 'session-1:2:prewarm', + }), + }, + { + response: { + sessionId: 'session-1', + generation: 2, + status: 'active', + metadata: { e2bApiKey: 'x' }, + }, + invoke: (client: CloudLiveTeleportClient) => client.status({ sessionId: 'session-1', generation: 2 }), + }, + { + response: { ...active, metadata: { child: { accessToken: 'x' } } }, + invoke: (client: CloudLiveTeleportClient) => client.acquire(input), + }, + { + response: { + sessionId: 'session-1', + generation: 2, + status: 'revoked', + metadata: { authorization: 'Bearer x' }, + }, + invoke: (client: CloudLiveTeleportClient) => + client.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'session-1:2:revoke' }), + }, + { + response: { + sessionId: 'session-1', + generation: 2, + status: 'revoked', + metadata: { nested: { secret: 'x' } }, + }, + invoke: (client: CloudLiveTeleportClient) => + client.revoke({ sessionId: 'session-1', generation: 2, idempotencyKey: 'session-1:2:revoke' }), + }, + ]; + + for (const scenario of cases) { + const client = new CloudLiveTeleportClient( + async () => Response.json(scenario.response), + 'https://cloud.agentrelay.test' + ); + await expect(scenario.invoke(client)).rejects.toThrow('forbidden sensitive field'); + } + }); + + it('rejects unknown non-sensitive fields instead of retaining unparsed remote response data', async () => { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + status: 'active', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + metadata: { region: 'unknown' }, + }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'unexpected response field metadata' + ); }); it('rejects an arbitrary execution URL even when it points at Cloud', async () => { @@ -267,6 +354,27 @@ describe('CloudLiveTeleportClient', () => { expect(String(error)).not.toContain(receipt.sealToken); }); + it('rejects destination verification for a different acquired workspace cwd', async () => { + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:00:00.000Z', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + verification: verification({ localRoot: '/different-workspace' }), + }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.acquire(input)).rejects.toThrow('mismatched Relayfile verification'); + }); + it('rejects any response that echoes the one-use checkpoint capability', async () => { const client = new CloudLiveTeleportClient( async () => @@ -286,7 +394,7 @@ describe('CloudLiveTeleportClient', () => { ); const error = await client.acquire(input).catch((caught: unknown) => caught); - expect(String(error)).toContain('forbidden provider field'); + expect(String(error)).toContain('forbidden sensitive field'); expect(String(error)).not.toContain(receipt.sealToken); }); diff --git a/packages/cloud/src/live-teleport.ts b/packages/cloud/src/live-teleport.ts index 0de3f76a3..5ead7fb60 100644 --- a/packages/cloud/src/live-teleport.ts +++ b/packages/cloud/src/live-teleport.ts @@ -99,8 +99,8 @@ export interface LiveTeleportCloudClient { type Fetcher = (path: string, init?: RequestInit) => Promise; -const FORBIDDEN_PROVIDER_FIELD = - /(?:provider.*(?:url|token|credential)|trafficAccessToken|signedPreviewUrl|sealToken|^ticket$)/i; +const SENSITIVE_RESPONSE_FIELD = + /(?:api[-_]?key|access[-_]?token|authorization|bearer|credential|password|private[-_]?key|secret|provider.*(?:url|token)|trafficAccessToken|signedPreviewUrl|sealToken|^ticket$)/i; const MAX_ACQUIRE_ATTEMPTS = 120; const MAX_ACQUIRE_RETRY_AFTER_MS = 1_000; /** Remote turns can wait up to 30m for completion; Cloud reserves another 10m for fencing. */ @@ -110,24 +110,35 @@ function isObject(value: unknown): value is Record { return value !== null && typeof value === 'object' && !Array.isArray(value); } -function assertNoProviderSecrets(value: unknown, path = 'response'): void { +function assertNoSensitiveResponseFields(value: unknown, path = 'response'): void { if (Array.isArray(value)) { - value.forEach((entry, index) => assertNoProviderSecrets(entry, `${path}[${index}]`)); + value.forEach((entry, index) => assertNoSensitiveResponseFields(entry, `${path}[${index}]`)); return; } if (!isObject(value)) return; for (const [key, entry] of Object.entries(value)) { - if (FORBIDDEN_PROVIDER_FIELD.test(key)) { - throw new Error(`Cloud live-teleport response exposed forbidden provider field ${path}.${key}.`); + if (SENSITIVE_RESPONSE_FIELD.test(key)) { + throw new Error(`Cloud live-teleport response exposed forbidden sensitive field ${path}.${key}.`); } - assertNoProviderSecrets(entry, `${path}.${key}`); + assertNoSensitiveResponseFields(entry, `${path}.${key}`); + } +} + +function assertAllowedResponseKeys( + value: Record, + allowed: readonly string[], + context: string +): void { + const unexpected = Object.keys(value).find((key) => !allowed.includes(key)); + if (unexpected) { + throw new Error(`Cloud live-teleport ${context} returned unexpected response field ${unexpected}.`); } } async function readPayload(response: Response): Promise { const payload = (await response.json().catch(() => null)) as unknown; - assertNoProviderSecrets(payload); + assertNoSensitiveResponseFields(payload); if (!response.ok) { // Cloud error prose can accidentally interpolate an opaque ticket or // provider URL. Only a short machine code is safe to relay to callers. @@ -241,13 +252,42 @@ function requiredReceiptBinding(source: LiveTeleportWorkspaceSource): LiveTelepo function requiredVerification( value: unknown, - expected: LiveTeleportReceiptBinding + expected: LiveTeleportReceiptBinding & { localRoot: string } ): LiveTeleportDestinationVerification { if (!isObject(value) || !isObject(value.observed) || !isObject(value.health)) { throw new Error('Cloud live-teleport acquire did not return Relayfile destination verification.'); } + assertAllowedResponseKeys( + value, + [ + 'version', + 'kind', + 'verificationId', + 'workspaceId', + 'localRoot', + 'remoteRoot', + 'sessionId', + 'generation', + 'status', + 'observed', + 'health', + 'verifiedAt', + ], + 'verification' + ); + assertAllowedResponseKeys( + value.observed, + ['digest', 'workspaceRevision', 'eventCursor'], + 'verification.observed' + ); + assertAllowedResponseKeys( + value.health, + ['pendingWriteback', 'conflicts', 'outboxPending', 'outboxNeedsAttention'], + 'verification.health' + ); const verifiedAt = requiredString(value.verifiedAt, 'verification.verifiedAt'); const workspaceId = requiredString(value.workspaceId, 'verification.workspaceId'); + const localRoot = requiredString(value.localRoot, 'verification.localRoot'); const digest = requiredDigest(value.observed.digest, 'verification.observed.digest'); const workspaceRevision = requiredRelayfilePosition( value.observed.workspaceRevision, @@ -264,6 +304,7 @@ function requiredVerification( value.kind !== 'relayfile-destination-verification' || value.status !== 'converged' || workspaceId !== expected.workspaceId || + localRoot !== expected.localRoot || value.remoteRoot !== expected.remoteRoot || value.sessionId !== expected.sessionId || value.generation !== expected.generation || @@ -283,7 +324,7 @@ function requiredVerification( kind: 'relayfile-destination-verification', verificationId: requiredString(value.verificationId, 'verification.verificationId'), workspaceId, - localRoot: requiredString(value.localRoot, 'verification.localRoot'), + localRoot, remoteRoot: '/', sessionId: expected.sessionId, generation: expected.generation, @@ -357,6 +398,7 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { }); const payload = await readPayload(response); if (!isObject(payload)) throw new Error('Cloud live-teleport prewarm returned an invalid response.'); + assertAllowedResponseKeys(payload, ['prewarmId', 'generation', 'status'], 'prewarm'); const status = payload.status; if (status !== 'warming' && status !== 'ready') { @@ -406,6 +448,7 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { if (!isObject(payload) || payload.status !== 'verifying') { throw new Error('Cloud live-teleport acquire returned an invalid pending response.'); } + assertAllowedResponseKeys(payload, ['status', 'retryAfterMs'], 'pending acquire'); await wait( Math.min( optionalNonNegativeInteger(payload.retryAfterMs, 'retryAfterMs') ?? 500, @@ -422,12 +465,29 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { if ('execServerUrl' in payload) { throw new Error('Cloud live-teleport acquire must not return an arbitrary execServerUrl.'); } + assertAllowedResponseKeys( + payload, + [ + 'sessionId', + 'generation', + 'threadId', + 'status', + 'environmentId', + 'connectPath', + 'workspaceCwd', + 'connectExpiresAt', + 'leaseExpiresAt', + 'verification', + ], + 'acquire' + ); const connectPath = this.requiredConnectPath(payload.connectPath); const execServerUrl = this.execServerUrl(connectPath); const sessionId = requiredString(payload.sessionId, 'sessionId'); const generation = requiredGeneration(payload.generation); const threadId = requiredString(payload.threadId, 'threadId'); + const workspaceCwd = requiredString(payload.workspaceCwd, 'workspaceCwd'); const connectExpiresAt = requiredString(payload.connectExpiresAt, 'connectExpiresAt'); const leaseExpiresAt = requiredString(payload.leaseExpiresAt, 'leaseExpiresAt'); if ( @@ -448,10 +508,13 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { environmentId: requiredString(payload.environmentId, 'environmentId'), connectPath, execServerUrl, - workspaceCwd: requiredString(payload.workspaceCwd, 'workspaceCwd'), + workspaceCwd, connectExpiresAt, leaseExpiresAt, - verification: requiredVerification(payload.verification, receiptBinding), + verification: requiredVerification(payload.verification, { + ...receiptBinding, + localRoot: workspaceCwd, + }), }; } @@ -476,6 +539,11 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { } private parseLifecycleStatus(payload: Record): LiveTeleportLifecycleStatus { + assertAllowedResponseKeys( + payload, + ['sessionId', 'generation', 'status', 'prewarmId', 'retryAfterMs', 'expiresAt', 'leaseExpiresAt'], + 'lifecycle status' + ); const status = payload.status; if ( status !== 'warming' && From cf416bbd22e0d1972f89b5892caf3860efee1fbf Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Sun, 23 Aug 2026 20:12:56 +0200 Subject: [PATCH 06/16] fix(codex): align teleport cloud response schemas --- .../src/cli/lib/codex-live-controller.test.ts | 31 ++- .../cli/src/cli/lib/codex-live-controller.ts | 1 + packages/cloud/src/index.ts | 1 + packages/cloud/src/live-teleport.test.ts | 227 +++++++++++++++++- packages/cloud/src/live-teleport.ts | 212 +++++++++++++--- 5 files changed, 427 insertions(+), 45 deletions(-) diff --git a/packages/cli/src/cli/lib/codex-live-controller.test.ts b/packages/cli/src/cli/lib/codex-live-controller.test.ts index 72518410c..65830147f 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.test.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.test.ts @@ -154,9 +154,11 @@ function lifecycle( function cloud(overrides: Partial = {}): LiveTeleportCloudClient { return { prewarm: vi.fn(async (input) => ({ + sessionId: input.sessionId, prewarmId: `prewarm-${input.generation}`, generation: input.generation, status: 'ready' as const, + expiresAt: '2026-08-23T12:30:00.000Z', })), status: vi.fn(async (input) => ({ ...lifecycle(input, 'active'), @@ -329,15 +331,23 @@ describe('CodexLiveController', () => { it('does not block local initialization on a stalled background prewarm', async () => { const pendingPrewarm = deferred<{ + sessionId: string; prewarmId: string; generation: number; status: 'ready'; + expiresAt: string; }>(); const cloudClient = cloud({ prewarm: vi.fn(() => pendingPrewarm.promise) }); const { controller } = createController({ cloud: cloudClient }); await expect(controller.initialize()).resolves.toMatchObject({ phase: 'local' }); - pendingPrewarm.resolve({ prewarmId: 'prewarm-1', generation: 1, status: 'ready' }); + pendingPrewarm.resolve({ + sessionId: 'session-1', + prewarmId: 'prewarm-1', + generation: 1, + status: 'ready', + expiresAt: '2026-08-23T12:30:00.000Z', + }); await vi.waitFor(() => expect(controller.status().prewarmStatus).toBe('ready')); }); @@ -1060,9 +1070,12 @@ describe('CodexLiveController', () => { it('polls warming lifecycle status to ready before acquisition', async () => { const cloudClient = cloud({ prewarm: vi.fn(async (input) => ({ + sessionId: input.sessionId, prewarmId: `prewarm-${input.generation}`, generation: input.generation, status: 'warming' as const, + expiresAt: '2026-08-23T12:30:00.000Z', + retryAfterMs: 1, })), status: vi .fn() @@ -1092,9 +1105,12 @@ describe('CodexLiveController', () => { it('bounds non-convergence and recovers locally instead of blocking forever', async () => { const cloudClient = cloud({ prewarm: vi.fn(async (input) => ({ + sessionId: input.sessionId, prewarmId: `prewarm-${input.generation}`, generation: input.generation, status: 'warming' as const, + expiresAt: '2026-08-23T12:30:00.000Z', + retryAfterMs: 1, })), status: vi.fn(async (input) => ({ ...lifecycle(input, 'warming'), retryAfterMs: 1 })), }); @@ -1536,6 +1552,19 @@ describe('FileCodexControllerStateStore', () => { remote: { ...persistedRemote().remote!, generation: 8 }, }, ], + [ + 'cross-root destination verification', + { + ...persistedRemote(), + remote: { + ...persistedRemote().remote!, + verification: { + ...persistedRemote().remote!.verification, + localRoot: '/different-workspace', + }, + }, + }, + ], [ 'malformed in-flight turn', { ...persistedLocal(), inFlightTurn: { clientUserMessageId: '', execution: 'remote' } }, diff --git a/packages/cli/src/cli/lib/codex-live-controller.ts b/packages/cli/src/cli/lib/codex-live-controller.ts index 9b7ed5fb3..85c56dc52 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.ts @@ -332,6 +332,7 @@ function invalidNestedState(state: Record): boolean { !nonEmptyString(verification.verificationId) || !nonEmptyString(verification.workspaceId) || !nonEmptyString(verification.localRoot) || + verification.localRoot !== remote.workspaceCwd || !validTimestamp(verification.verifiedAt) || !observed || typeof observed.digest !== 'string' || diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index 10b8bc21b..5d966399a 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -66,6 +66,7 @@ export { type LiveTeleportRevokeInput, type LiveTeleportStatusInput, type LiveTeleportLifecycleStatus, + type LiveTeleportRollout, type LiveTeleportRevocation, } from './live-teleport.js'; diff --git a/packages/cloud/src/live-teleport.test.ts b/packages/cloud/src/live-teleport.test.ts index 84e3f7a46..0c55abc18 100644 --- a/packages/cloud/src/live-teleport.test.ts +++ b/packages/cloud/src/live-teleport.test.ts @@ -64,11 +64,67 @@ function verification( }; } +// Frozen constructor parity fixtures derived from Cloud e3f9e5ddc: +// prewarm/route.ts, acquire/route.ts, status/route.ts, and feature-flag.ts. +const cloudRollout = { + masterEnabled: true, + eligible: true, + reason: 'workspace-allowlist' as const, + percentage: 25, +}; + +function cloudPrewarmWarming() { + return { + sessionId: 'session-1', + generation: 2, + prewarmId: 'prewarm-2', + status: 'warming', + expiresAt: '2026-08-23T12:30:00.000Z', + retryAfterMs: 1_000, + }; +} + +function cloudAcquireVerifying() { + return { + sessionId: 'session-1', + generation: 2, + status: 'verifying', + retryAfterMs: 1_000, + }; +} + +function cloudAcquireActive() { + return { + sessionId: 'session-1', + generation: 2, + threadId: 'thread-1', + status: 'active', + environmentId: 'env-2', + connectPath: '/api/v1/live-teleports/connect/ticket', + workspaceCwd: '/workspace', + connectExpiresAt: '2026-08-23T12:05:00.000Z', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + verification: verification(), + }; +} + +function cloudActiveStatus(leaseExpiresAt = '2026-08-23T12:45:00.000Z') { + return { + sessionId: 'session-1', + generation: 2, + status: 'active', + prewarmId: 'prewarm-2', + expiresAt: leaseExpiresAt, + leaseExpiresAt, + rollout: cloudRollout, + }; +} + describe('CloudLiveTeleportClient', () => { it('polls an exact 202 acquire with the same idempotency body until active', async () => { const fetcher = vi .fn() - .mockResolvedValueOnce(Response.json({ status: 'verifying', retryAfterMs: 0 }, { status: 202 })) + .mockResolvedValueOnce(Response.json(cloudAcquireVerifying(), { status: 202 })) .mockResolvedValueOnce( Response.json({ sessionId: 'session-1', @@ -90,6 +146,133 @@ describe('CloudLiveTeleportClient', () => { expect(vi.mocked(fetcher).mock.calls[0]?.[1]?.body).toBe(vi.mocked(fetcher).mock.calls[1]?.[1]?.body); }); + it('composes the frozen Cloud e3f9e5ddc warming → verifying → active and renewal constructors', async () => { + const renewedLease = '2026-08-23T13:15:00.000Z'; + const fetcher = vi + .fn() + .mockResolvedValueOnce(Response.json(cloudPrewarmWarming(), { status: 202 })) + .mockResolvedValueOnce(Response.json(cloudAcquireVerifying(), { status: 202 })) + .mockResolvedValueOnce(Response.json(cloudAcquireActive())) + .mockResolvedValueOnce(Response.json(cloudActiveStatus(renewedLease))); + const client = new CloudLiveTeleportClient(fetcher, 'https://cloud.agentrelay.test'); + + await expect( + client.prewarm({ + sessionId: 'session-1', + generation: 2, + workspaceRoot: '/', + idempotencyKey: 'session-1:2:prewarm', + }) + ).resolves.toEqual(cloudPrewarmWarming()); + await expect(client.acquire(input)).resolves.toMatchObject({ + sessionId: 'session-1', + generation: 2, + environmentId: 'env-2', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + }); + await expect(client.status({ sessionId: 'session-1', generation: 2 })).resolves.toEqual( + cloudActiveStatus(renewedLease) + ); + + expect(Object.keys(cloudPrewarmWarming()).sort()).toEqual( + ['expiresAt', 'generation', 'prewarmId', 'retryAfterMs', 'sessionId', 'status'].sort() + ); + expect(Object.keys(cloudAcquireVerifying()).sort()).toEqual( + ['generation', 'retryAfterMs', 'sessionId', 'status'].sort() + ); + expect(Object.keys(cloudActiveStatus()).sort()).toEqual( + ['expiresAt', 'generation', 'leaseExpiresAt', 'prewarmId', 'rollout', 'sessionId', 'status'].sort() + ); + expect(fetcher.mock.calls.map(([path]) => path)).toEqual([ + '/api/v1/live-teleports/prewarm', + '/api/v1/live-teleports/acquire', + '/api/v1/live-teleports/acquire', + '/api/v1/live-teleports/status', + ]); + }); + + it('rejects identity or required-field drift in frozen Cloud prewarm and pending acquire shapes', async () => { + const stalePrewarm = new CloudLiveTeleportClient( + async () => Response.json({ ...cloudPrewarmWarming(), sessionId: 'other-session' }, { status: 202 }), + 'https://cloud.agentrelay.test' + ); + await expect( + stalePrewarm.prewarm({ + sessionId: 'session-1', + generation: 2, + workspaceRoot: '/', + idempotencyKey: 'session-1:2:prewarm', + }) + ).rejects.toThrow('mismatched lifecycle metadata'); + + const missingExpiry = new CloudLiveTeleportClient(async () => { + const { expiresAt: _expiresAt, ...response } = cloudPrewarmWarming(); + return Response.json(response, { status: 202 }); + }, 'https://cloud.agentrelay.test'); + await expect( + missingExpiry.prewarm({ + sessionId: 'session-1', + generation: 2, + workspaceRoot: '/', + idempotencyKey: 'session-1:2:prewarm', + }) + ).rejects.toThrow('expiresAt'); + + const stalePending = new CloudLiveTeleportClient( + async () => Response.json({ ...cloudAcquireVerifying(), generation: 3 }, { status: 202 }), + 'https://cloud.agentrelay.test' + ); + await expect(stalePending.acquire(input)).rejects.toThrow('stale lifecycle identity'); + }); + + it('enforces Cloud prewarm HTTP, expiry, and retry semantics', async () => { + const readyResponse = { + ...cloudPrewarmWarming(), + status: 'ready', + }; + const { retryAfterMs: _retryAfterMs, ...ready } = readyResponse; + const readyClient = new CloudLiveTeleportClient( + async () => Response.json(ready), + 'https://cloud.agentrelay.test' + ); + await expect( + readyClient.prewarm({ + sessionId: 'session-1', + generation: 2, + workspaceRoot: '/', + idempotencyKey: 'session-1:2:prewarm', + }) + ).resolves.toEqual(ready); + + for (const response of [ + { body: cloudPrewarmWarming(), status: 200 }, + { + body: { ...cloudPrewarmWarming(), status: 'ready' }, + status: 200, + }, + { + body: (() => { + const { retryAfterMs: _retry, ...withoutRetry } = cloudPrewarmWarming(); + return withoutRetry; + })(), + status: 202, + }, + ]) { + const client = new CloudLiveTeleportClient( + async () => Response.json(response.body, { status: response.status }), + 'https://cloud.agentrelay.test' + ); + await expect( + client.prewarm({ + sessionId: 'session-1', + generation: 2, + workspaceRoot: '/', + idempotencyKey: 'session-1:2:prewarm', + }) + ).rejects.toThrow('mismatched lifecycle metadata'); + } + }); + it('accepts only the provider-neutral Cloud WSS bridge contract', async () => { const fetcher = vi.fn(async () => Response.json({ @@ -400,7 +583,7 @@ describe('CloudLiveTeleportClient', () => { it('bounds a permanently verifying acquire while replaying the same request', async () => { const fetcher = vi.fn(async () => - Response.json({ status: 'verifying', retryAfterMs: 0 }, { status: 202 }) + Response.json({ ...cloudAcquireVerifying(), retryAfterMs: 1 }, { status: 202 }) ); const client = new CloudLiveTeleportClient(fetcher, 'https://cloud.agentrelay.test'); @@ -466,6 +649,8 @@ describe('CloudLiveTeleportClient', () => { prewarmId: 'prewarm-2', status: 'warming', retryAfterMs: 250, + expiresAt: '2026-08-23T12:30:00.000Z', + rollout: cloudRollout, }) ) .mockResolvedValueOnce(Response.json({ sessionId: 'session-1', generation: 2, status: 'revoked' })); @@ -488,19 +673,10 @@ describe('CloudLiveTeleportClient', () => { it('parses the exact active lease deadline and rejects non-canonical timestamps', async () => { const fetcher = vi .fn() + .mockResolvedValueOnce(Response.json(cloudActiveStatus())) .mockResolvedValueOnce( Response.json({ - sessionId: 'session-1', - generation: 2, - status: 'active', - leaseExpiresAt: '2026-08-23T12:45:00.000Z', - }) - ) - .mockResolvedValueOnce( - Response.json({ - sessionId: 'session-1', - generation: 2, - status: 'active', + ...cloudActiveStatus(), leaseExpiresAt: '2026-08-23T14:45:00+02:00', }) ); @@ -515,6 +691,31 @@ describe('CloudLiveTeleportClient', () => { ); }); + it.each([ + ['unknown reason', { ...cloudRollout, reason: 'manual-override' }], + ['out-of-range percentage', { ...cloudRollout, percentage: 101 }], + ['inconsistent eligibility', { ...cloudRollout, reason: 'not-targeted', eligible: true }], + ['unexpected rollout field', { ...cloudRollout, cohort: 'canary' }], + ])('rejects %s in Cloud status rollout metadata', async (_name, rollout) => { + const client = new CloudLiveTeleportClient( + async () => Response.json({ ...cloudActiveStatus(), rollout }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow(/rollout/); + }); + + it('rejects a cross-session status response even when its rollout and lease are valid', async () => { + const client = new CloudLiveTeleportClient( + async () => Response.json({ ...cloudActiveStatus(), sessionId: 'other-session' }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'stale or cross-session' + ); + }); + it('accepts cleanup_pending as a non-terminal revoke acknowledgement', async () => { const client = new CloudLiveTeleportClient( async () => diff --git a/packages/cloud/src/live-teleport.ts b/packages/cloud/src/live-teleport.ts index 5ead7fb60..2e4680eac 100644 --- a/packages/cloud/src/live-teleport.ts +++ b/packages/cloud/src/live-teleport.ts @@ -12,9 +12,19 @@ export type LiveTeleportPrewarmInput = { }; export type LiveTeleportPrewarm = { + sessionId: string; prewarmId: string; generation: number; status: 'warming' | 'ready'; + expiresAt: string; + retryAfterMs?: number; +}; + +export type LiveTeleportRollout = { + masterEnabled: boolean; + eligible: boolean; + reason: 'disabled' | 'workspace-allowlist' | 'user-allowlist' | 'percentage' | 'not-targeted'; + percentage: number; }; export type LiveTeleportAcquireInput = LiveTeleportPrewarmInput & { @@ -84,6 +94,8 @@ export type LiveTeleportLifecycleStatus = { expiresAt?: string; /** Exact Cloud-authoritative lease deadline, present for active status. */ leaseExpiresAt?: string; + /** Present on authoritative status responses; omitted by cleanup-pending recovery responses. */ + rollout?: LiveTeleportRollout; }; export type LiveTeleportRevocation = LiveTeleportLifecycleStatus & { @@ -177,14 +189,9 @@ function requiredGeneration(value: unknown): number { return Number(value); } -function optionalNonNegativeInteger(value: unknown, field: string): number | undefined { - if (value === undefined) return undefined; - return requiredNonNegativeInteger(value, field); -} - -function requiredNonNegativeInteger(value: unknown, field: string): number { - if (!Number.isSafeInteger(value) || Number(value) < 0) { - throw new Error(`Cloud live-teleport convergence proof has an invalid ${field}.`); +function requiredPositiveInteger(value: unknown, field: string): number { + if (!Number.isSafeInteger(value) || Number(value) < 1) { + throw new Error(`Cloud live-teleport response has an invalid ${field}.`); } return Number(value); } @@ -214,6 +221,46 @@ function requiredExactTimestamp(value: unknown, field: string): string { return timestamp; } +function requiredRollout(value: unknown): LiveTeleportRollout { + if (!isObject(value)) { + throw new Error('Cloud live-teleport status is missing rollout metadata.'); + } + assertAllowedResponseKeys(value, ['masterEnabled', 'eligible', 'reason', 'percentage'], 'status rollout'); + const reason = value.reason; + const validReason = + reason === 'disabled' || + reason === 'workspace-allowlist' || + reason === 'user-allowlist' || + reason === 'percentage' || + reason === 'not-targeted'; + const percentage = value.percentage; + if ( + typeof value.masterEnabled !== 'boolean' || + typeof value.eligible !== 'boolean' || + !validReason || + !Number.isSafeInteger(percentage) || + Number(percentage) < 0 || + Number(percentage) > 100 + ) { + throw new Error('Cloud live-teleport status returned invalid rollout metadata.'); + } + const expected = + reason === 'disabled' + ? { masterEnabled: false, eligible: false } + : reason === 'not-targeted' + ? { masterEnabled: true, eligible: false } + : { masterEnabled: true, eligible: true }; + if (value.masterEnabled !== expected.masterEnabled || value.eligible !== expected.eligible) { + throw new Error('Cloud live-teleport status returned inconsistent rollout metadata.'); + } + return { + masterEnabled: value.masterEnabled, + eligible: value.eligible, + reason, + percentage: Number(percentage), + }; +} + type LiveTeleportReceiptBinding = { workspaceId: string; remoteRoot: '/'; @@ -398,16 +445,38 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { }); const payload = await readPayload(response); if (!isObject(payload)) throw new Error('Cloud live-teleport prewarm returned an invalid response.'); - assertAllowedResponseKeys(payload, ['prewarmId', 'generation', 'status'], 'prewarm'); + assertAllowedResponseKeys( + payload, + ['sessionId', 'generation', 'prewarmId', 'status', 'expiresAt', 'retryAfterMs'], + 'prewarm' + ); const status = payload.status; if (status !== 'warming' && status !== 'ready') { throw new Error('Cloud live-teleport prewarm returned an invalid status.'); } + const sessionId = requiredString(payload.sessionId, 'sessionId'); + const generation = requiredGeneration(payload.generation); + const expiresAt = requiredExactTimestamp(payload.expiresAt, 'expiresAt'); + const retryAfterMs = + payload.retryAfterMs === undefined + ? undefined + : requiredPositiveInteger(payload.retryAfterMs, 'retryAfterMs'); + if ( + sessionId !== input.sessionId || + generation !== input.generation || + (status === 'warming' && (response.status !== 202 || retryAfterMs === undefined)) || + (status === 'ready' && (response.status !== 200 || retryAfterMs !== undefined)) + ) { + throw new Error('Cloud live-teleport prewarm returned mismatched lifecycle metadata.'); + } return { + sessionId, prewarmId: requiredString(payload.prewarmId, 'prewarmId'), - generation: requiredGeneration(payload.generation), + generation, status, + expiresAt, + ...(retryAfterMs === undefined ? {} : { retryAfterMs }), }; } @@ -421,7 +490,11 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { }); const payload = await readPayload(response); if (!isObject(payload)) throw new Error('Cloud live-teleport status returned an invalid response.'); - return this.parseLifecycleStatus(payload); + const status = this.parseLifecycleStatus(payload, 'status'); + if (status.sessionId !== input.sessionId || status.generation !== input.generation) { + throw new Error('Cloud live-teleport status returned a stale or cross-session lifecycle identity.'); + } + return status; } async acquire(input: LiveTeleportAcquireInput): Promise { @@ -448,12 +521,18 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { if (!isObject(payload) || payload.status !== 'verifying') { throw new Error('Cloud live-teleport acquire returned an invalid pending response.'); } - assertAllowedResponseKeys(payload, ['status', 'retryAfterMs'], 'pending acquire'); + assertAllowedResponseKeys( + payload, + ['sessionId', 'generation', 'status', 'retryAfterMs'], + 'pending acquire' + ); + const pendingSessionId = requiredString(payload.sessionId, 'sessionId'); + const pendingGeneration = requiredGeneration(payload.generation); + if (pendingSessionId !== input.sessionId || pendingGeneration !== input.generation) { + throw new Error('Cloud live-teleport pending acquire returned a stale lifecycle identity.'); + } await wait( - Math.min( - optionalNonNegativeInteger(payload.retryAfterMs, 'retryAfterMs') ?? 500, - MAX_ACQUIRE_RETRY_AFTER_MS - ), + Math.min(requiredPositiveInteger(payload.retryAfterMs, 'retryAfterMs'), MAX_ACQUIRE_RETRY_AFTER_MS), signal ); } @@ -488,15 +567,13 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { const generation = requiredGeneration(payload.generation); const threadId = requiredString(payload.threadId, 'threadId'); const workspaceCwd = requiredString(payload.workspaceCwd, 'workspaceCwd'); - const connectExpiresAt = requiredString(payload.connectExpiresAt, 'connectExpiresAt'); - const leaseExpiresAt = requiredString(payload.leaseExpiresAt, 'leaseExpiresAt'); + const connectExpiresAt = requiredExactTimestamp(payload.connectExpiresAt, 'connectExpiresAt'); + const leaseExpiresAt = requiredExactTimestamp(payload.leaseExpiresAt, 'leaseExpiresAt'); if ( payload.status !== 'active' || sessionId !== input.sessionId || generation !== input.generation || threadId !== input.threadId || - Number.isNaN(Date.parse(connectExpiresAt)) || - Number.isNaN(Date.parse(leaseExpiresAt)) || Date.parse(leaseExpiresAt) <= Date.parse(connectExpiresAt) ) { throw new Error('Cloud live-teleport acquire returned invalid active lifecycle metadata.'); @@ -528,20 +605,34 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { }); const payload = await readPayload(response); if (!isObject(payload)) throw new Error('Cloud live-teleport revoke returned an invalid response.'); - const status = this.parseLifecycleStatus(payload); + const status = this.parseLifecycleStatus(payload, 'revoke'); if (status.sessionId !== input.sessionId || status.generation !== input.generation) { throw new Error('Cloud live-teleport revoke returned a stale or cross-session lifecycle identity.'); } - if (status.status !== 'cleanup_pending' && status.status !== 'revoked' && status.status !== 'expired') { + if (status.status !== 'cleanup_pending' && status.status !== 'revoked') { throw new Error('Cloud live-teleport revoke was not confirmed.'); } return status; } - private parseLifecycleStatus(payload: Record): LiveTeleportLifecycleStatus { + private parseLifecycleStatus( + payload: Record, + boundary: 'status' | 'revoke' + ): LiveTeleportLifecycleStatus { assertAllowedResponseKeys( payload, - ['sessionId', 'generation', 'status', 'prewarmId', 'retryAfterMs', 'expiresAt', 'leaseExpiresAt'], + boundary === 'status' + ? [ + 'sessionId', + 'generation', + 'status', + 'prewarmId', + 'retryAfterMs', + 'expiresAt', + 'leaseExpiresAt', + 'rollout', + ] + : ['sessionId', 'generation', 'status', 'retryAfterMs'], 'lifecycle status' ); const status = payload.status; @@ -558,14 +649,25 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { throw new Error('Cloud live-teleport status returned an invalid lifecycle state.'); } const expiresAt = - payload.expiresAt === undefined ? undefined : requiredString(payload.expiresAt, 'expiresAt'); - if (expiresAt && Number.isNaN(Date.parse(expiresAt))) { - throw new Error('Cloud live-teleport status returned an invalid expiresAt.'); - } + payload.expiresAt === undefined ? undefined : requiredExactTimestamp(payload.expiresAt, 'expiresAt'); const leaseExpiresAt = payload.leaseExpiresAt === undefined ? undefined : requiredExactTimestamp(payload.leaseExpiresAt, 'leaseExpiresAt'); + const retryAfterMs = + payload.retryAfterMs === undefined + ? undefined + : requiredPositiveInteger(payload.retryAfterMs, 'retryAfterMs'); + const rollout = payload.rollout === undefined ? undefined : requiredRollout(payload.rollout); + this.assertLifecycleResponseSemantics({ + boundary, + status, + prewarmId: payload.prewarmId, + retryAfterMs, + expiresAt, + leaseExpiresAt, + rollout, + }); return { sessionId: requiredString(payload.sessionId, 'sessionId'), generation: requiredGeneration(payload.generation), @@ -573,14 +675,62 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { ...(payload.prewarmId === undefined ? {} : { prewarmId: requiredString(payload.prewarmId, 'prewarmId') }), - ...(payload.retryAfterMs === undefined - ? {} - : { retryAfterMs: optionalNonNegativeInteger(payload.retryAfterMs, 'retryAfterMs')! }), + ...(retryAfterMs === undefined ? {} : { retryAfterMs }), ...(expiresAt ? { expiresAt } : {}), ...(leaseExpiresAt ? { leaseExpiresAt } : {}), + ...(rollout ? { rollout } : {}), }; } + private assertLifecycleResponseSemantics(input: { + boundary: 'status' | 'revoke'; + status: LiveTeleportLifecycleStatus['status']; + prewarmId: unknown; + retryAfterMs?: number; + expiresAt?: string; + leaseExpiresAt?: string; + rollout?: LiveTeleportRollout; + }): void { + if (input.boundary === 'revoke') { + const valid = + (input.status === 'cleanup_pending' && input.retryAfterMs !== undefined) || + (input.status !== 'cleanup_pending' && input.retryAfterMs === undefined); + if (!valid) throw new Error('Cloud live-teleport revoke returned invalid lifecycle semantics.'); + return; + } + + if (input.prewarmId === undefined) { + throw new Error('Cloud live-teleport status is missing prewarmId.'); + } + if (input.status === 'cleanup_pending') { + const compactCleanup = + input.retryAfterMs !== undefined && + input.expiresAt === undefined && + input.leaseExpiresAt === undefined && + input.rollout === undefined; + const fullCleanup = + input.retryAfterMs !== undefined && + input.expiresAt !== undefined && + input.rollout !== undefined && + (input.leaseExpiresAt === undefined || input.leaseExpiresAt === input.expiresAt); + if (!compactCleanup && !fullCleanup) { + throw new Error('Cloud live-teleport status returned invalid cleanup lifecycle semantics.'); + } + return; + } + + if (!input.expiresAt || !input.rollout) { + throw new Error('Cloud live-teleport status is missing authoritative lifecycle metadata.'); + } + const polling = input.status === 'warming' || input.status === 'verifying'; + if (polling !== (input.retryAfterMs !== undefined)) { + throw new Error('Cloud live-teleport status returned invalid retryAfterMs semantics.'); + } + if (input.status === 'active' && (!input.leaseExpiresAt || input.leaseExpiresAt !== input.expiresAt)) { + throw new Error('Cloud live-teleport status returned inconsistent active lease metadata.'); + } + } + private requiredConnectPath(value: unknown): string { const connectPath = requiredString(value, 'connectPath'); if ( From 3ab0835607d84c09d012425b8b3f7a6d36f29066 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Sun, 23 Aug 2026 20:35:06 +0200 Subject: [PATCH 07/16] fix(codex): persist live reconciliation evidence --- packages/cli/src/cli/commands/codex.test.ts | 71 +++++++++ packages/cli/src/cli/commands/codex.ts | 16 +- .../src/cli/lib/codex-live-controller.test.ts | 139 ++++++++++++++---- .../cli/src/cli/lib/codex-live-controller.ts | 106 ++++++++----- packages/cloud/src/live-teleport.test.ts | 120 +++++++++++++++ packages/cloud/src/live-teleport.ts | 40 ++++- 6 files changed, 421 insertions(+), 71 deletions(-) diff --git a/packages/cli/src/cli/commands/codex.test.ts b/packages/cli/src/cli/commands/codex.test.ts index aac1d77c2..13f93644d 100644 --- a/packages/cli/src/cli/commands/codex.test.ts +++ b/packages/cli/src/cli/commands/codex.test.ts @@ -19,6 +19,7 @@ describe('runManagedCodexTurn', () => { completed: { method: 'turn/completed', params: { threadId: 'thread-1', turn } }, reconciled: true as const, })), + acknowledgeRecoveredOutcome: vi.fn(), status: vi.fn(), }; const writeOutput = vi.fn(); @@ -30,6 +31,37 @@ describe('runManagedCodexTurn', () => { }); expect(writeOutput).toHaveBeenCalledWith('the recovered answer\n'); + expect(controller.acknowledgeRecoveredOutcome).toHaveBeenCalledOnce(); + }); + + it('does not acknowledge a live recovered completion until its exact answer is rendered', async () => { + const turn = { + id: 'turn-reconciled', + status: 'completed', + itemsView: 'full', + items: [{ id: 'answer-1', type: 'agentMessage', text: 'the recovered answer' }], + }; + const renderError = new Error('stdout unavailable'); + const controller = { + runTurn: vi.fn(async () => ({ + turnId: turn.id, + response: { turn }, + completed: { method: 'turn/completed', params: { threadId: 'thread-1', turn } }, + reconciled: true as const, + })), + acknowledgeRecoveredOutcome: vi.fn(), + }; + + await expect( + runManagedCodexTurn(controller as never, 'recover me', { + json: false, + writeError: vi.fn(), + writeOutput: () => { + throw renderError; + }, + }) + ).rejects.toBe(renderError); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); }); it('does not report a successful command for a recorded failed turn', async () => { @@ -37,6 +69,7 @@ describe('runManagedCodexTurn', () => { const runTurn = vi.fn().mockRejectedValue(terminal); const controller = { runTurn, + acknowledgeRecoveredOutcome: vi.fn(), status: vi.fn(() => ({ phase: 'local' as const, threadId: 'thread-1', @@ -49,6 +82,44 @@ describe('runManagedCodexTurn', () => { runManagedCodexTurn(controller as never, 'failed input', { json: false, writeError }) ).rejects.toBe(terminal); expect(writeError).toHaveBeenCalledWith(expect.stringContaining('recorded the turn as failed')); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); + }); + + it.each(['failed', 'interrupted'] as const)( + 'acknowledges a live reconciled %s terminal only after rendering it', + async (status) => { + const terminal = new CodexTurnRecordedError(status, `recorded ${status}`, true); + const controller = { + runTurn: vi.fn().mockRejectedValue(terminal), + acknowledgeRecoveredOutcome: vi.fn(), + }; + const writeError = vi.fn(); + + await expect( + runManagedCodexTurn(controller as never, 'do not replay', { json: false, writeError }) + ).rejects.toBe(terminal); + expect(writeError).toHaveBeenCalledWith(expect.stringContaining(`turn as ${status}`)); + expect(controller.acknowledgeRecoveredOutcome).toHaveBeenCalledOnce(); + } + ); + + it('does not acknowledge a live reconciled terminal when rendering fails', async () => { + const terminal = new CodexTurnRecordedError('failed', 'recorded failed', true); + const renderError = new Error('stderr unavailable'); + const controller = { + runTurn: vi.fn().mockRejectedValue(terminal), + acknowledgeRecoveredOutcome: vi.fn(), + }; + + await expect( + runManagedCodexTurn(controller as never, 'do not replay', { + json: false, + writeError: () => { + throw renderError; + }, + }) + ).rejects.toBe(renderError); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); }); it.each(['failed', 'interrupted'] as const)( diff --git a/packages/cli/src/cli/commands/codex.ts b/packages/cli/src/cli/commands/codex.ts index efc90029a..3e8d2ab58 100644 --- a/packages/cli/src/cli/commands/codex.ts +++ b/packages/cli/src/cli/commands/codex.ts @@ -73,7 +73,7 @@ function agentMessageDelta(notification: CodexNotification): string | undefined } export async function runManagedCodexTurn( - controller: Pick, + controller: Pick, text: string, options: { json: boolean; @@ -86,6 +86,11 @@ export async function runManagedCodexTurn( result = await controller.runTurn(text); } catch (error) { if (error instanceof CodexTurnRecordedError) { + if (error.requiresRecoveryAcknowledgment) { + writeCodexRecordedTerminal(error, options); + controller.acknowledgeRecoveredOutcome(); + throw error; + } surfaceCodexRecordedTerminal(error, options); } if (error instanceof CodexTurnOutcomeUncertainError) { @@ -98,7 +103,10 @@ export async function runManagedCodexTurn( { cause: error } ); } - if (result.reconciled) writeReconciledTurn(result, options); + if (result.reconciled) { + writeReconciledTurn(result, options); + controller.acknowledgeRecoveredOutcome(); + } } export function surfaceCodexRecordedTerminal( @@ -138,7 +146,9 @@ function writeReconciledTurn( result: CodexTurnResult, options: { json: boolean; writeOutput?: (message: string) => void } ): void { - if (!options.writeOutput) return; + if (!options.writeOutput) { + throw new Error('Reconciled Codex completion cannot be acknowledged without an output writer.'); + } if (options.json) { options.writeOutput(`${JSON.stringify(result.completed)}\n`); return; diff --git a/packages/cli/src/cli/lib/codex-live-controller.test.ts b/packages/cli/src/cli/lib/codex-live-controller.test.ts index 65830147f..bdb69868f 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.test.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.test.ts @@ -94,13 +94,17 @@ function turnResult(turnId = 'turn-1'): CodexTurnResult { }; } -function completedOutcome(turnId: string, answer = `answer for ${turnId}`) { +function completedOutcome( + turnId: string, + answer = `answer for ${turnId}`, + clientUserMessageId = 'client-persisted-1' +) { const turn = { id: turnId, status: 'completed', itemsView: 'full', items: [ - { id: `user-${turnId}`, type: 'userMessage', clientId: 'client-persisted-1', text: 'prompt' }, + { id: `user-${turnId}`, type: 'userMessage', clientId: clientUserMessageId, text: 'prompt' }, { id: `answer-${turnId}`, type: 'agentMessage', text: answer }, ], }; @@ -820,42 +824,82 @@ describe('CodexLiveController', () => { ); }); - it('confirms revoke, reconciles a recorded failed turn, and resumes the same thread without replay', async () => { - const original = appServer({ runTurn: vi.fn(async () => Promise.reject(new Error('remote died'))) }); - const replacement = appServer({ - turnOutcome: vi.fn(async () => ({ status: 'failed' as const, turnId: 'turn-remote-1' })), - }); - const cloudClient = cloud(); - const sealed = sealHandle(); - const { controller } = createController({ - sessions: [original, replacement], - cloud: cloudClient, - checkpointAndSeal: async () => sealed, - }); - await controller.initialize(); - controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); + it.each(['failed', 'interrupted'] as const)( + 'durably preserves a live remote %s reconciliation across two crashes without replay', + async (status) => { + const original = appServer({ runTurn: vi.fn(async () => Promise.reject(new Error('remote died'))) }); + const replacement = appServer({ + turnOutcome: vi.fn(async () => ({ status, turnId: `turn-remote-${status}` })), + }); + const cloudClient = cloud(); + const sealed = sealHandle(); + const { controller, store } = createController({ + sessions: [original, replacement], + cloud: cloudClient, + checkpointAndSeal: async () => sealed, + }); + await controller.initialize(); + controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); - await expect(controller.runTurn('do not replay me')).rejects.toThrow('recorded terminal turn'); + await expect(controller.runTurn('do not replay me')).rejects.toMatchObject({ + status, + requiresRecoveryAcknowledgment: true, + }); - expect(cloudClient.revoke).toHaveBeenCalledWith( - expect.objectContaining({ idempotencyKey: 'session-1:1:revoke' }) - ); - expect(original.close).toHaveBeenCalled(); - expect(sealed.resumeLocal).toHaveBeenCalled(); - expect(replacement.initialize).toHaveBeenCalled(); - expect(replacement.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); - expect(replacement.runTurn).not.toHaveBeenCalled(); - expect(controller.status()).toMatchObject({ generation: 2, phase: 'local', execution: 'local' }); - }); + expect(cloudClient.revoke).toHaveBeenCalledWith( + expect.objectContaining({ idempotencyKey: 'session-1:1:revoke' }) + ); + expect(original.close).toHaveBeenCalled(); + expect(sealed.resumeLocal).toHaveBeenCalled(); + expect(replacement.initialize).toHaveBeenCalled(); + expect(replacement.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + expect(replacement.runTurn).not.toHaveBeenCalled(); + expect(store.value?.inFlightTurn).toBeUndefined(); + expect(store.value?.recoveredOutcome).toMatchObject({ + sessionId: 'session-1', + threadId: 'thread-1', + generation: 2, + clientUserMessageId: 'operation-2', + turnId: `turn-remote-${status}`, + status, + }); + expect(controller.status()).toMatchObject({ generation: 2, phase: 'local', execution: 'local' }); + + const prewarmCallsAfterRecovery = vi.mocked(cloudClient.prewarm).mock.calls.length; + const restartedOnce = appServer(); + const second = createController({ store, cloud: cloudClient, sessions: [restartedOnce] }); + await second.controller.initialize(); + expect(restartedOnce.turnOutcome).not.toHaveBeenCalled(); + expect(restartedOnce.runTurn).not.toHaveBeenCalled(); + expect(second.controller.takeRecoveredTerminal()).toMatchObject({ status }); + expect(vi.mocked(cloudClient.prewarm)).toHaveBeenCalledTimes(prewarmCallsAfterRecovery); + + const restartedTwice = appServer(); + const third = createController({ store, cloud: cloudClient, sessions: [restartedTwice] }); + await third.controller.initialize(); + expect(restartedTwice.turnOutcome).not.toHaveBeenCalled(); + expect(restartedTwice.runTurn).not.toHaveBeenCalled(); + expect(third.controller.takeRecoveredTerminal()).toMatchObject({ status }); + third.controller.acknowledgeRecoveredOutcome(); + expect(store.value?.recoveredOutcome).toBeUndefined(); + expect(third.controller.takeRecoveredTerminal()).toBeUndefined(); + } + ); it('reconciles a completed remote turn after notification loss without replaying it', async () => { const original = appServer({ runTurn: vi.fn(async () => Promise.reject(new Error('notification lost'))), }); const replacement = appServer({ - turnOutcome: vi.fn(async () => completedOutcome('turn-remote-1', 'recovered exact answer')), + turnOutcome: vi.fn(async () => + completedOutcome('turn-remote-1', 'recovered exact answer', 'operation-2') + ), + }); + const cloudClient = cloud(); + const { controller, store } = createController({ + sessions: [original, replacement], + cloud: cloudClient, }); - const { controller } = createController({ sessions: [original, replacement] }); await controller.initialize(); controller.requestTeleport({ requestId: 'request-1', expectedGeneration: 1 }); @@ -875,7 +919,44 @@ describe('CodexLiveController', () => { }, }); expect(replacement.runTurn).not.toHaveBeenCalled(); + expect(store.value?.inFlightTurn).toBeUndefined(); + expect(store.value?.recoveredOutcome).toMatchObject({ + sessionId: 'session-1', + threadId: 'thread-1', + generation: 2, + clientUserMessageId: 'operation-2', + turnId: 'turn-remote-1', + status: 'completed', + }); expect(controller.status()).toMatchObject({ phase: 'local', generation: 2 }); + + const prewarmCallsAfterRecovery = vi.mocked(cloudClient.prewarm).mock.calls.length; + const restartedOnce = appServer(); + const second = createController({ store, cloud: cloudClient, sessions: [restartedOnce] }); + await second.controller.initialize(); + expect(restartedOnce.turnOutcome).not.toHaveBeenCalled(); + expect(restartedOnce.runTurn).not.toHaveBeenCalled(); + expect(second.controller.takeRecoveredTurn()).toMatchObject({ + turnId: 'turn-remote-1', + response: { + turn: { + items: expect.arrayContaining([ + expect.objectContaining({ type: 'agentMessage', text: 'recovered exact answer' }), + ]), + }, + }, + }); + expect(vi.mocked(cloudClient.prewarm)).toHaveBeenCalledTimes(prewarmCallsAfterRecovery); + + const restartedTwice = appServer(); + const third = createController({ store, cloud: cloudClient, sessions: [restartedTwice] }); + await third.controller.initialize(); + expect(restartedTwice.turnOutcome).not.toHaveBeenCalled(); + expect(restartedTwice.runTurn).not.toHaveBeenCalled(); + expect(third.controller.takeRecoveredTurn()).toMatchObject({ turnId: 'turn-remote-1' }); + third.controller.acknowledgeRecoveredOutcome(); + expect(store.value?.recoveredOutcome).toBeUndefined(); + expect(third.controller.takeRecoveredTurn()).toBeUndefined(); }); it('fails outcome-uncertain when an accepted remote turn cannot be found after fencing', async () => { diff --git a/packages/cli/src/cli/lib/codex-live-controller.ts b/packages/cli/src/cli/lib/codex-live-controller.ts index 85c56dc52..61f9af49f 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.ts @@ -453,7 +453,8 @@ export class CodexTurnRecordedError extends Error { constructor( readonly status: 'failed' | 'interrupted', - message: string + message: string, + readonly requiresRecoveryAcknowledgment = false ) { super(message); this.name = 'CodexTurnRecordedError'; @@ -612,12 +613,8 @@ export class CodexLiveController { } let recovered: - | { - clientUserMessageId: string; - outcome: - | Extract - | { status: 'failed' | 'interrupted'; turnId: string }; - } + | Extract + | { status: 'failed' | 'interrupted'; turnId: string } | undefined; const inFlight = this.requireState().inFlightTurn; if (inFlight) { @@ -630,12 +627,9 @@ export class CodexLiveController { throw new CodexTurnOutcomeUncertainError(); } if (outcome.status === 'completed') { - recovered = { clientUserMessageId: inFlight.clientUserMessageId, outcome }; + recovered = outcome; } else if (outcome.status === 'failed' || outcome.status === 'interrupted') { - recovered = { - clientUserMessageId: inFlight.clientUserMessageId, - outcome: { status: outcome.status, turnId: outcome.turnId }, - }; + recovered = { status: outcome.status, turnId: outcome.turnId }; } } @@ -646,20 +640,7 @@ export class CodexLiveController { state.remote = undefined; state.source = undefined; state.mountRestore = undefined; - if (recovered) { - const common = { - sessionId: state.sessionId, - threadId: state.threadId, - generation: state.generation, - clientUserMessageId: recovered.clientUserMessageId, - turnId: recovered.outcome.turnId, - }; - state.recoveredOutcome = - recovered.outcome.status === 'completed' - ? { ...common, status: 'completed', result: recovered.outcome.result } - : { ...common, status: recovered.outcome.status }; - state.inFlightTurn = undefined; - } + if (recovered) this.stageRecoveredOutcome(recovered); state.lastError = undefined; state.updatedAt = this.timestamp(); this.persist(); @@ -709,7 +690,8 @@ export class CodexLiveController { this.recoveredEvidenceTaken = recovered; return new CodexTurnRecordedError( recovered.status, - `Codex recorded the crash-reconciled turn as ${recovered.status}; it was not replayed.` + `Codex recorded the crash-reconciled turn as ${recovered.status}; it was not replayed.`, + true ); } @@ -836,7 +818,8 @@ export class CodexLiveController { if (outcome?.status === 'failed' || outcome?.status === 'interrupted') { throw new CodexTurnRecordedError( outcome.status, - `Cloud turn ended ${outcome.status}; the recorded terminal turn was not replayed.` + `Cloud turn ended ${outcome.status}; the recorded terminal turn was not replayed.`, + true ); } throw new CodexTurnOutcomeUncertainError(undefined, { cause: error }); @@ -848,10 +831,20 @@ export class CodexLiveController { this.markOutcomeUncertain('TURN_RECONCILIATION_UNAVAILABLE'); throw new CodexTurnOutcomeUncertainError(undefined, { cause: reconcileError }); } - if (outcome.status === 'completed') return this.resultFromOutcome(outcome); + if (outcome.status === 'completed') { + this.persistRecoveredOutcome(outcome, 'LOCAL_TURN_RECONCILED'); + return this.resultFromOutcome(outcome); + } if (outcome.status === 'failed' || outcome.status === 'interrupted') { - this.clearInFlightTurn(`TURN_${outcome.status.toUpperCase()}`); - throw new CodexTurnRecordedError(outcome.status, `Codex turn ended ${outcome.status}.`); + this.persistRecoveredOutcome( + { status: outcome.status, turnId: outcome.turnId }, + 'LOCAL_TURN_RECONCILED' + ); + throw new CodexTurnRecordedError( + outcome.status, + `Codex turn ended ${outcome.status}; the recorded terminal turn was not replayed.`, + true + ); } this.markOutcomeUncertain(`TURN_${outcome.status.toUpperCase()}`); throw new CodexTurnOutcomeUncertainError(undefined, { cause: error }); @@ -1097,7 +1090,6 @@ export class CodexLiveController { this.markOutcomeUncertain(`TURN_${outcome.status.toUpperCase()}`); throw new CodexTurnOutcomeUncertainError(); } - state.inFlightTurn = undefined; } state.generation += 1; @@ -1108,10 +1100,19 @@ export class CodexLiveController { state.mountRestore = undefined; state.prewarmId = undefined; state.prewarmStatus = undefined; + if (outcome?.status === 'completed') { + this.stageRecoveredOutcome(outcome); + } else if (outcome?.status === 'failed' || outcome?.status === 'interrupted') { + this.stageRecoveredOutcome({ status: outcome.status, turnId: outcome.turnId }); + } state.lastError = context; state.updatedAt = this.timestamp(); this.persist(); - this.schedulePrewarm(); + if (outcome) { + this.recoveredEvidenceTaken = state.recoveredOutcome; + } else { + this.schedulePrewarm(); + } return outcome; } @@ -1334,6 +1335,45 @@ export class CodexLiveController { return outcome.result; } + private persistRecoveredOutcome( + outcome: + | Extract + | { status: 'failed' | 'interrupted'; turnId: string }, + context: string + ): void { + const state = this.requireState(); + this.stageRecoveredOutcome(outcome); + state.lastError = context; + state.updatedAt = this.timestamp(); + this.persist(); + this.recoveredEvidenceTaken = state.recoveredOutcome; + } + + /** Move one reconciled in-flight turn into durable evidence in the caller's atomic state write. */ + private stageRecoveredOutcome( + outcome: + | Extract + | { status: 'failed' | 'interrupted'; turnId: string } + ): void { + const state = this.requireState(); + const inFlight = state.inFlightTurn; + if (!inFlight || state.recoveredOutcome) { + throw new Error('Codex reconciliation evidence does not match one unique in-flight turn.'); + } + const common = { + sessionId: state.sessionId, + threadId: state.threadId, + generation: state.generation, + clientUserMessageId: inFlight.clientUserMessageId, + turnId: outcome.turnId, + }; + state.recoveredOutcome = + outcome.status === 'completed' + ? { ...common, status: 'completed', result: outcome.result } + : { ...common, status: outcome.status }; + state.inFlightTurn = undefined; + } + private requireTurnLeaseHorizon(leaseExpiresAt: string): void { const leaseDeadline = Date.parse(leaseExpiresAt); const minimumDeadline = this.deps.now().getTime() + LIVE_TELEPORT_MIN_TURN_LEASE_MS; diff --git a/packages/cloud/src/live-teleport.test.ts b/packages/cloud/src/live-teleport.test.ts index 0c55abc18..9b755416b 100644 --- a/packages/cloud/src/live-teleport.test.ts +++ b/packages/cloud/src/live-teleport.test.ts @@ -146,6 +146,36 @@ describe('CloudLiveTeleportClient', () => { expect(vi.mocked(fetcher).mock.calls[0]?.[1]?.body).toBe(vi.mocked(fetcher).mock.calls[1]?.[1]?.body); }); + it('accepts active acquire only with HTTP 200 and rejects other successful codes', async () => { + const exact = new CloudLiveTeleportClient( + async () => Response.json(cloudAcquireActive(), { status: 200 }), + 'https://cloud.agentrelay.test' + ); + await expect(exact.acquire(input)).resolves.toMatchObject({ environmentId: 'env-2' }); + + for (const status of [201, 206]) { + const client = new CloudLiveTeleportClient( + async () => Response.json(cloudAcquireActive(), { status }), + 'https://cloud.agentrelay.test' + ); + await expect(client.acquire(input)).rejects.toThrow('invalid active lifecycle metadata'); + } + }); + + it('rejects acquire HTTP/body state mismatches', async () => { + const activePending = new CloudLiveTeleportClient( + async () => Response.json(cloudAcquireActive(), { status: 202 }), + 'https://cloud.agentrelay.test' + ); + await expect(activePending.acquire(input)).rejects.toThrow('invalid pending response'); + + const verifyingSuccess = new CloudLiveTeleportClient( + async () => Response.json(cloudAcquireVerifying(), { status: 200 }), + 'https://cloud.agentrelay.test' + ); + await expect(verifyingSuccess.acquire(input)).rejects.toThrow('invalid active lifecycle metadata'); + }); + it('composes the frozen Cloud e3f9e5ddc warming → verifying → active and renewal constructors', async () => { const renewedLease = '2026-08-23T13:15:00.000Z'; const fetcher = vi @@ -670,6 +700,64 @@ describe('CloudLiveTeleportClient', () => { ).resolves.toMatchObject({ status: 'revoked' }); }); + it('requires exact status HTTP codes while accepting the compact 202 cleanup contract', async () => { + const activeWith202 = new CloudLiveTeleportClient( + async () => Response.json(cloudActiveStatus(), { status: 202 }), + 'https://cloud.agentrelay.test' + ); + await expect(activeWith202.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'invalid HTTP success code' + ); + + for (const status of [201, 206]) { + const client = new CloudLiveTeleportClient( + async () => Response.json(cloudActiveStatus(), { status }), + 'https://cloud.agentrelay.test' + ); + await expect(client.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'invalid HTTP success code' + ); + } + + const cleanup = new CloudLiveTeleportClient( + async () => + Response.json( + { + sessionId: 'session-1', + generation: 2, + prewarmId: 'prewarm-2', + status: 'cleanup_pending', + retryAfterMs: 250, + }, + { status: 202 } + ), + 'https://cloud.agentrelay.test' + ); + await expect(cleanup.status({ sessionId: 'session-1', generation: 2 })).resolves.toMatchObject({ + status: 'cleanup_pending', + retryAfterMs: 250, + }); + + const fullCleanup = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + prewarmId: 'prewarm-2', + status: 'cleanup_pending', + retryAfterMs: 250, + expiresAt: '2026-08-23T12:45:00.000Z', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + rollout: cloudRollout, + }), + 'https://cloud.agentrelay.test' + ); + await expect(fullCleanup.status({ sessionId: 'session-1', generation: 2 })).resolves.toMatchObject({ + status: 'cleanup_pending', + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + }); + }); + it('parses the exact active lease deadline and rejects non-canonical timestamps', async () => { const fetcher = vi .fn() @@ -695,6 +783,8 @@ describe('CloudLiveTeleportClient', () => { ['unknown reason', { ...cloudRollout, reason: 'manual-override' }], ['out-of-range percentage', { ...cloudRollout, percentage: 101 }], ['inconsistent eligibility', { ...cloudRollout, reason: 'not-targeted', eligible: true }], + ['zero-percent admission', { ...cloudRollout, reason: 'percentage', percentage: 0 }], + ['100-percent exclusion', { ...cloudRollout, reason: 'not-targeted', percentage: 100, eligible: false }], ['unexpected rollout field', { ...cloudRollout, cohort: 'canary' }], ])('rejects %s in Cloud status rollout metadata', async (_name, rollout) => { const client = new CloudLiveTeleportClient( @@ -705,6 +795,36 @@ describe('CloudLiveTeleportClient', () => { await expect(client.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow(/rollout/); }); + it.each(['warming', 'ready', 'verifying', 'active'] as const)( + 'rejects Cloud %s while the rollout is disabled', + async (status) => { + const expiresAt = '2026-08-23T12:45:00.000Z'; + const response = { + sessionId: 'session-1', + generation: 2, + prewarmId: 'prewarm-2', + status, + expiresAt, + rollout: { + masterEnabled: false, + eligible: false, + reason: 'disabled', + percentage: 25, + }, + ...(status === 'warming' || status === 'verifying' ? { retryAfterMs: 250 } : {}), + ...(status === 'active' ? { leaseExpiresAt: expiresAt } : {}), + }; + const client = new CloudLiveTeleportClient( + async () => Response.json(response), + 'https://cloud.agentrelay.test' + ); + + await expect(client.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'nonterminal state for an ineligible rollout' + ); + } + ); + it('rejects a cross-session status response even when its rollout and lease are valid', async () => { const client = new CloudLiveTeleportClient( async () => Response.json({ ...cloudActiveStatus(), sessionId: 'other-session' }), diff --git a/packages/cloud/src/live-teleport.ts b/packages/cloud/src/live-teleport.ts index 2e4680eac..cd22d8e8c 100644 --- a/packages/cloud/src/live-teleport.ts +++ b/packages/cloud/src/live-teleport.ts @@ -253,6 +253,9 @@ function requiredRollout(value: unknown): LiveTeleportRollout { if (value.masterEnabled !== expected.masterEnabled || value.eligible !== expected.eligible) { throw new Error('Cloud live-teleport status returned inconsistent rollout metadata.'); } + if ((reason === 'percentage' && percentage === 0) || (reason === 'not-targeted' && percentage === 100)) { + throw new Error('Cloud live-teleport status returned an impossible rollout percentage.'); + } return { masterEnabled: value.masterEnabled, eligible: value.eligible, @@ -490,7 +493,10 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { }); const payload = await readPayload(response); if (!isObject(payload)) throw new Error('Cloud live-teleport status returned an invalid response.'); - const status = this.parseLifecycleStatus(payload, 'status'); + if (response.status !== 200 && !(response.status === 202 && payload.status === 'cleanup_pending')) { + throw new Error('Cloud live-teleport status returned an invalid HTTP success code.'); + } + const status = this.parseLifecycleStatus(payload, 'status', response.status); if (status.sessionId !== input.sessionId || status.generation !== input.generation) { throw new Error('Cloud live-teleport status returned a stale or cross-session lifecycle identity.'); } @@ -504,6 +510,7 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { throw new Error('Cloud live-teleport acquire received a stale or cross-session checkpoint receipt.'); } let payload: unknown; + let responseStatus: number | undefined; let stillVerifying = false; for (let attempt = 0; attempt < MAX_ACQUIRE_ATTEMPTS; attempt += 1) { const response = await fetchAcquireSafely(this.fetcher, '/api/v1/live-teleports/acquire', { @@ -513,6 +520,7 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { signal, }); payload = await readPayload(response); + responseStatus = response.status; if (response.status !== 202) { stillVerifying = false; break; @@ -540,6 +548,9 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { throw new Error('Cloud live-teleport acquire exceeded the bounded verification poll limit.'); } if (!isObject(payload)) throw new Error('Cloud live-teleport acquire returned an invalid response.'); + if (responseStatus !== 200 || payload.status !== 'active') { + throw new Error('Cloud live-teleport acquire returned invalid active lifecycle metadata.'); + } if ('execServerUrl' in payload) { throw new Error('Cloud live-teleport acquire must not return an arbitrary execServerUrl.'); @@ -570,7 +581,6 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { const connectExpiresAt = requiredExactTimestamp(payload.connectExpiresAt, 'connectExpiresAt'); const leaseExpiresAt = requiredExactTimestamp(payload.leaseExpiresAt, 'leaseExpiresAt'); if ( - payload.status !== 'active' || sessionId !== input.sessionId || generation !== input.generation || threadId !== input.threadId || @@ -605,7 +615,7 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { }); const payload = await readPayload(response); if (!isObject(payload)) throw new Error('Cloud live-teleport revoke returned an invalid response.'); - const status = this.parseLifecycleStatus(payload, 'revoke'); + const status = this.parseLifecycleStatus(payload, 'revoke', response.status); if (status.sessionId !== input.sessionId || status.generation !== input.generation) { throw new Error('Cloud live-teleport revoke returned a stale or cross-session lifecycle identity.'); } @@ -617,7 +627,8 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { private parseLifecycleStatus( payload: Record, - boundary: 'status' | 'revoke' + boundary: 'status' | 'revoke', + httpStatus: number ): LiveTeleportLifecycleStatus { assertAllowedResponseKeys( payload, @@ -661,6 +672,7 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { const rollout = payload.rollout === undefined ? undefined : requiredRollout(payload.rollout); this.assertLifecycleResponseSemantics({ boundary, + httpStatus, status, prewarmId: payload.prewarmId, retryAfterMs, @@ -684,6 +696,7 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { private assertLifecycleResponseSemantics(input: { boundary: 'status' | 'revoke'; + httpStatus: number; status: LiveTeleportLifecycleStatus['status']; prewarmId: unknown; retryAfterMs?: number; @@ -693,8 +706,10 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { }): void { if (input.boundary === 'revoke') { const valid = - (input.status === 'cleanup_pending' && input.retryAfterMs !== undefined) || - (input.status !== 'cleanup_pending' && input.retryAfterMs === undefined); + (input.status === 'cleanup_pending' && + input.httpStatus === 202 && + input.retryAfterMs !== undefined) || + (input.status !== 'cleanup_pending' && input.httpStatus === 200 && input.retryAfterMs === undefined); if (!valid) throw new Error('Cloud live-teleport revoke returned invalid lifecycle semantics.'); return; } @@ -704,11 +719,13 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { } if (input.status === 'cleanup_pending') { const compactCleanup = + input.httpStatus === 202 && input.retryAfterMs !== undefined && input.expiresAt === undefined && input.leaseExpiresAt === undefined && input.rollout === undefined; const fullCleanup = + input.httpStatus === 200 && input.retryAfterMs !== undefined && input.expiresAt !== undefined && input.rollout !== undefined && @@ -722,6 +739,9 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { if (!input.expiresAt || !input.rollout) { throw new Error('Cloud live-teleport status is missing authoritative lifecycle metadata.'); } + if (input.httpStatus !== 200) { + throw new Error('Cloud live-teleport status returned an invalid HTTP success code.'); + } const polling = input.status === 'warming' || input.status === 'verifying'; if (polling !== (input.retryAfterMs !== undefined)) { throw new Error('Cloud live-teleport status returned invalid retryAfterMs semantics.'); @@ -729,6 +749,14 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { if (input.status === 'active' && (!input.leaseExpiresAt || input.leaseExpiresAt !== input.expiresAt)) { throw new Error('Cloud live-teleport status returned inconsistent active lease metadata.'); } + const requiresEligibleRollout = + input.status === 'warming' || + input.status === 'ready' || + input.status === 'verifying' || + input.status === 'active'; + if (requiresEligibleRollout && (!input.rollout.masterEnabled || !input.rollout.eligible)) { + throw new Error('Cloud live-teleport status returned a nonterminal state for an ineligible rollout.'); + } } private requiredConnectPath(value: unknown): string { From 78d58bf5a8b33811a06a400450a2aa64d81a0682 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Sun, 23 Aug 2026 20:55:10 +0200 Subject: [PATCH 08/16] fix(codex): fence recovered output delivery --- CHANGELOG.md | 2 +- packages/cli/src/cli/commands/codex.test.ts | 228 ++++++++++++++---- packages/cli/src/cli/commands/codex.ts | 90 ++++--- .../src/cli/lib/codex-live-controller.test.ts | 127 ++++++++++ .../cli/src/cli/lib/codex-live-controller.ts | 32 ++- packages/cloud/src/live-teleport.test.ts | 69 ++++++ packages/cloud/src/live-teleport.ts | 6 + 7 files changed, 476 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fdfb3ee4b..04ce28822 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `relay codex run` starts a Relay-managed local Codex app-server/thread, and `relay codex teleport` seals its active Relayfile poll mount at the next turn boundary, binds exact Relayfile destination verification to the source checkpoint, requires a 40-minute Cloud turn lease, preserves reconciled assistant answers, transparently runs the same prompt locally after a confirmed pre-submission cutover failure, and requires confirmed fencing plus ownership handback and mount readiness before local rollback. +- `relay codex run` starts a Relay-managed local Codex app-server/thread, and `relay codex teleport` seals its active Relayfile poll mount at the next turn boundary, binds exact Relayfile destination verification to the source checkpoint, requires a 40-minute Cloud turn lease, and requires confirmed fencing plus ownership handback and mount readiness before local rollback. After notification loss, Relay never replays model execution: it durably redelivers reconciled output at least once, so an already-rendered live prefix can repeat, and it transparently runs the same prompt locally only after a confirmed pre-submission cutover failure. ## [Unreleased - Patch] diff --git a/packages/cli/src/cli/commands/codex.test.ts b/packages/cli/src/cli/commands/codex.test.ts index 13f93644d..6d5978105 100644 --- a/packages/cli/src/cli/commands/codex.test.ts +++ b/packages/cli/src/cli/commands/codex.test.ts @@ -2,9 +2,149 @@ import { Command } from 'commander'; import { describe, expect, it, vi } from 'vitest'; import { CodexTurnRecordedError } from '../lib/codex-live-controller.js'; -import { registerCodexCommands, runManagedCodexTurn, surfaceRecoveredCodexTerminal } from './codex.js'; +import { + nodeCallbackWriter, + registerCodexCommands, + runManagedCodexTurn, + surfaceRecoveredCodexTerminal, +} from './codex.js'; + +function deferred() { + let resolve!: (value: T) => void; + let reject!: (error: unknown) => void; + const promise = new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + return { promise, resolve, reject }; +} + +function reconciledResult() { + const turn = { + id: 'turn-reconciled', + status: 'completed', + itemsView: 'full', + items: [{ id: 'answer-1', type: 'agentMessage', text: 'the recovered answer' }], + }; + return { + turnId: turn.id, + response: { turn }, + completed: { method: 'turn/completed', params: { threadId: 'thread-1', turn } }, + reconciled: true as const, + }; +} + +describe('nodeCallbackWriter', () => { + it('waits for the Node write callback even when write reports backpressure', async () => { + let callback!: (error?: Error | null) => void; + const stream = { + write: vi.fn((_message: string, done: (error?: Error | null) => void) => { + callback = done; + return false; + }), + }; + const completed = vi.fn(); + const pending = nodeCallbackWriter(stream)('durable output').then(completed); + + await Promise.resolve(); + expect(stream.write).toHaveBeenCalledWith('durable output', expect.any(Function)); + expect(completed).not.toHaveBeenCalled(); + + callback(); + await pending; + expect(completed).toHaveBeenCalledOnce(); + }); + + it('rejects callback errors and synchronous stream throws', async () => { + const callbackError = new Error('callback failed'); + const callbackStream = { + write: vi.fn((_message: string, done: (error?: Error | null) => void) => { + done(callbackError); + return true; + }), + }; + await expect(nodeCallbackWriter(callbackStream)('output')).rejects.toBe(callbackError); + + const thrown = new Error('write threw'); + const throwingStream = { + write: vi.fn(() => { + throw thrown; + }), + }; + await expect(nodeCallbackWriter(throwingStream as never)('output')).rejects.toBe(thrown); + }); +}); describe('runManagedCodexTurn', () => { + it('keeps a completed recovery unacknowledged until the async output writer resolves', async () => { + const write = deferred(); + const controller = { + runTurn: vi.fn(async () => reconciledResult()), + acknowledgeRecoveredOutcome: vi.fn(), + }; + const running = runManagedCodexTurn(controller as never, 'recover me', { + json: false, + writeError: vi.fn(async () => undefined), + writeOutput: vi.fn(() => write.promise), + }); + + await vi.waitFor(() => expect(controller.runTurn).toHaveBeenCalledOnce()); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); + write.resolve(); + await running; + expect(controller.acknowledgeRecoveredOutcome).toHaveBeenCalledOnce(); + }); + + it.each(['failed', 'interrupted'] as const)( + 'keeps a %s recovery unacknowledged until the async error writer resolves', + async (status) => { + const write = deferred(); + const terminal = new CodexTurnRecordedError(status, `recorded ${status}`, true); + const controller = { + runTurn: vi.fn().mockRejectedValue(terminal), + acknowledgeRecoveredOutcome: vi.fn(), + }; + const running = runManagedCodexTurn(controller as never, 'recover me', { + json: false, + writeError: vi.fn(() => write.promise), + }); + + await vi.waitFor(() => expect(controller.runTurn).toHaveBeenCalledOnce()); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); + write.resolve(); + await expect(running).rejects.toBe(terminal); + expect(controller.acknowledgeRecoveredOutcome).toHaveBeenCalledOnce(); + } + ); + + it.each(['completed', 'failed', 'interrupted'] as const)( + 'keeps a %s recovery durable when the Node write callback fails', + async (status) => { + const writeError = new Error(`write callback failed for ${status}`); + const writer = nodeCallbackWriter({ + write: vi.fn((_message: string, done: (error?: Error | null) => void) => { + done(writeError); + return true; + }), + }); + const terminal = + status === 'completed' ? undefined : new CodexTurnRecordedError(status, `recorded ${status}`, true); + const controller = { + runTurn: terminal ? vi.fn().mockRejectedValue(terminal) : vi.fn(async () => reconciledResult()), + acknowledgeRecoveredOutcome: vi.fn(), + }; + + await expect( + runManagedCodexTurn(controller as never, 'recover me', { + json: false, + writeError: writer, + writeOutput: writer, + }) + ).rejects.toBe(writeError); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); + } + ); + it('renders an exact assistant answer recovered after completion-notification loss', async () => { const turn = { id: 'turn-reconciled', @@ -103,28 +243,31 @@ describe('runManagedCodexTurn', () => { } ); - it('does not acknowledge a live reconciled terminal when rendering fails', async () => { - const terminal = new CodexTurnRecordedError('failed', 'recorded failed', true); - const renderError = new Error('stderr unavailable'); - const controller = { - runTurn: vi.fn().mockRejectedValue(terminal), - acknowledgeRecoveredOutcome: vi.fn(), - }; + it.each(['failed', 'interrupted'] as const)( + 'does not acknowledge a live reconciled %s terminal when rendering throws', + async (status) => { + const terminal = new CodexTurnRecordedError(status, `recorded ${status}`, true); + const renderError = new Error('stderr unavailable'); + const controller = { + runTurn: vi.fn().mockRejectedValue(terminal), + acknowledgeRecoveredOutcome: vi.fn(), + }; - await expect( - runManagedCodexTurn(controller as never, 'do not replay', { - json: false, - writeError: () => { - throw renderError; - }, - }) - ).rejects.toBe(renderError); - expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); - }); + await expect( + runManagedCodexTurn(controller as never, 'do not replay', { + json: false, + writeError: () => { + throw renderError; + }, + }) + ).rejects.toBe(renderError); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); + } + ); it.each(['failed', 'interrupted'] as const)( 'surfaces a crash-reconciled %s turn nonzero before any new prompt runs', - (status) => { + async (status) => { const terminal = new CodexTurnRecordedError(status, `recorded ${status}`); const controller = { takeRecoveredTerminal: vi.fn().mockReturnValueOnce(terminal).mockReturnValue(undefined), @@ -133,38 +276,41 @@ describe('runManagedCodexTurn', () => { }; const writeError = vi.fn(); - expect(() => surfaceRecoveredCodexTerminal(controller as never, { json: false, writeError })).toThrow( - terminal - ); + await expect( + surfaceRecoveredCodexTerminal(controller as never, { json: false, writeError }) + ).rejects.toBe(terminal); expect(writeError).toHaveBeenCalledWith(expect.stringContaining(`turn as ${status}`)); expect(controller.acknowledgeRecoveredOutcome).toHaveBeenCalledOnce(); - expect(() => + await expect( surfaceRecoveredCodexTerminal(controller as never, { json: false, writeError }) - ).not.toThrow(); + ).resolves.toBeUndefined(); expect(writeError).toHaveBeenCalledTimes(1); expect(controller.runTurn).not.toHaveBeenCalled(); } ); - it('does not acknowledge durable terminal evidence when rendering fails', () => { - const terminal = new CodexTurnRecordedError('failed', 'recorded failed'); - const controller = { - takeRecoveredTerminal: vi.fn(() => terminal), - acknowledgeRecoveredOutcome: vi.fn(), - }; - const renderError = new Error('stderr unavailable'); + it.each(['failed', 'interrupted'] as const)( + 'does not acknowledge durable %s terminal evidence when rendering throws', + async (status) => { + const terminal = new CodexTurnRecordedError(status, `recorded ${status}`); + const controller = { + takeRecoveredTerminal: vi.fn(() => terminal), + acknowledgeRecoveredOutcome: vi.fn(), + }; + const renderError = new Error('stderr unavailable'); - expect(() => - surfaceRecoveredCodexTerminal(controller as never, { - json: false, - writeError: () => { - throw renderError; - }, - }) - ).toThrow(renderError); - expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); - }); + await expect( + surfaceRecoveredCodexTerminal(controller as never, { + json: false, + writeError: () => { + throw renderError; + }, + }) + ).rejects.toBe(renderError); + expect(controller.acknowledgeRecoveredOutcome).not.toHaveBeenCalled(); + } + ); it('fails closed instead of continuing when fencing is unconfirmed', async () => { const controller = { diff --git a/packages/cli/src/cli/commands/codex.ts b/packages/cli/src/cli/commands/codex.ts index 3e8d2ab58..3f5143f5e 100644 --- a/packages/cli/src/cli/commands/codex.ts +++ b/packages/cli/src/cli/commands/codex.ts @@ -39,6 +39,31 @@ type ControlRequest = type ControlResponse = { ok: true; status: PublicCodexControllerStatus } | { ok: false; error: string }; +export type CodexAsyncWriter = (message: string) => Promise; + +type NodeCallbackWritable = { + write(message: string, callback: (error?: Error | null) => void): boolean; +}; + +/** + * Adapts Node's callback-based stream contract into a durable-output promise. + * A false return value is backpressure, not completion; only the callback ACKs + * the write. Callback errors and synchronous throws both reject the promise. + */ +export function nodeCallbackWriter(stream: NodeCallbackWritable): CodexAsyncWriter { + return (message) => + new Promise((resolve, reject) => { + try { + stream.write(message, (error) => { + if (error) reject(error); + else resolve(); + }); + } catch (error) { + reject(error); + } + }); +} + export interface CodexCommandDependencies { runManaged(options: { cwd: string; model?: string; prompt?: string; json?: boolean }): Promise; checkpointAndSeal: CodexWorkspaceSealProvider; @@ -77,8 +102,8 @@ export async function runManagedCodexTurn( text: string, options: { json: boolean; - writeError: (message: string) => void; - writeOutput?: (message: string) => void; + writeError: CodexAsyncWriter; + writeOutput?: CodexAsyncWriter; } ): Promise { let result: CodexTurnResult; @@ -87,11 +112,11 @@ export async function runManagedCodexTurn( } catch (error) { if (error instanceof CodexTurnRecordedError) { if (error.requiresRecoveryAcknowledgment) { - writeCodexRecordedTerminal(error, options); + await writeCodexRecordedTerminal(error, options); controller.acknowledgeRecoveredOutcome(); throw error; } - surfaceCodexRecordedTerminal(error, options); + await surfaceCodexRecordedTerminal(error, options); } if (error instanceof CodexTurnOutcomeUncertainError) { throw new Error('Relay-managed Codex stopped because the last turn outcome is uncertain.', { @@ -104,24 +129,24 @@ export async function runManagedCodexTurn( ); } if (result.reconciled) { - writeReconciledTurn(result, options); + await writeReconciledTurn(result, options); controller.acknowledgeRecoveredOutcome(); } } -export function surfaceCodexRecordedTerminal( +export async function surfaceCodexRecordedTerminal( error: CodexTurnRecordedError, - options: { json: boolean; writeError: (message: string) => void } -): never { - writeCodexRecordedTerminal(error, options); + options: { json: boolean; writeError: CodexAsyncWriter } +): Promise { + await writeCodexRecordedTerminal(error, options); throw error; } -function writeCodexRecordedTerminal( +async function writeCodexRecordedTerminal( error: CodexTurnRecordedError, - options: { json: boolean; writeError: (message: string) => void } -): void { - options.writeError( + options: { json: boolean; writeError: CodexAsyncWriter } +): Promise { + await options.writeError( options.json ? `${JSON.stringify({ method: 'relay/codexTurnRecordedTerminal', @@ -131,26 +156,26 @@ function writeCodexRecordedTerminal( ); } -export function surfaceRecoveredCodexTerminal( +export async function surfaceRecoveredCodexTerminal( controller: Pick, - options: { json: boolean; writeError: (message: string) => void } -): void { + options: { json: boolean; writeError: CodexAsyncWriter } +): Promise { const terminal = controller.takeRecoveredTerminal(); if (!terminal) return; - writeCodexRecordedTerminal(terminal, options); + await writeCodexRecordedTerminal(terminal, options); controller.acknowledgeRecoveredOutcome(); throw terminal; } -function writeReconciledTurn( +async function writeReconciledTurn( result: CodexTurnResult, - options: { json: boolean; writeOutput?: (message: string) => void } -): void { + options: { json: boolean; writeOutput?: CodexAsyncWriter } +): Promise { if (!options.writeOutput) { throw new Error('Reconciled Codex completion cannot be acknowledged without an output writer.'); } if (options.json) { - options.writeOutput(`${JSON.stringify(result.completed)}\n`); + await options.writeOutput(`${JSON.stringify(result.completed)}\n`); return; } const params = result.completed.params; @@ -177,7 +202,7 @@ function writeReconciledTurn( if (answers.length === 0) { throw new Error('Reconciled Codex completion did not contain an observable assistant answer.'); } - options.writeOutput(`${answers.join('\n')}\n`); + await options.writeOutput(`${answers.join('\n')}\n`); } async function listen(server: net.Server, socketPath: string): Promise { @@ -302,6 +327,11 @@ function withDefaults(overrides: Partial = {}): CodexC StdioCodexAppServerSession.spawn({ cwd: workspaceRoot, onNotification: (notification) => { + // Live notifications are best-effort display hints, not the + // durable delivery ACK. If a terminal notification is lost, + // restart recovery writes the full answer at least once; it + // may therefore duplicate a prefix already emitted here. + // The underlying model turn is never submitted again. if (options.json) process.stdout.write(`${JSON.stringify(notification)}\n`); else { const delta = agentMessageDelta(notification); @@ -331,9 +361,9 @@ function withDefaults(overrides: Partial = {}): CodexC const server = createControlServer(controller); try { const status = await controller.initialize(); - surfaceRecoveredCodexTerminal(controller, { + await surfaceRecoveredCodexTerminal(controller, { json: Boolean(options.json), - writeError: (message) => process.stderr.write(message), + writeError: nodeCallbackWriter(process.stderr), }); await listen(server, paths.socketPath); process.stderr.write( @@ -341,9 +371,9 @@ function withDefaults(overrides: Partial = {}): CodexC ); const recoveredTurn = controller.takeRecoveredTurn(); if (recoveredTurn) { - writeReconciledTurn(recoveredTurn, { + await writeReconciledTurn(recoveredTurn, { json: Boolean(options.json), - writeOutput: (message) => process.stdout.write(message), + writeOutput: nodeCallbackWriter(process.stdout), }); controller.acknowledgeRecoveredOutcome(); } @@ -351,8 +381,8 @@ function withDefaults(overrides: Partial = {}): CodexC if (options.prompt) { await runManagedCodexTurn(controller, options.prompt, { json: Boolean(options.json), - writeError: (message) => process.stderr.write(message), - writeOutput: (message) => process.stdout.write(message), + writeError: nodeCallbackWriter(process.stderr), + writeOutput: nodeCallbackWriter(process.stdout), }); } const input = readline.createInterface({ @@ -363,8 +393,8 @@ function withDefaults(overrides: Partial = {}): CodexC if (!line.trim()) continue; await runManagedCodexTurn(controller, line, { json: Boolean(options.json), - writeError: (message) => process.stderr.write(message), - writeOutput: (message) => process.stdout.write(message), + writeError: nodeCallbackWriter(process.stderr), + writeOutput: nodeCallbackWriter(process.stdout), }); if (!options.json) process.stdout.write('\n'); } diff --git a/packages/cli/src/cli/lib/codex-live-controller.test.ts b/packages/cli/src/cli/lib/codex-live-controller.test.ts index bdb69868f..32de2bc89 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.test.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.test.ts @@ -134,6 +134,22 @@ function memoryStore(initial: CodexControllerState | null = null): CodexControll }; } +function fileStore(filePath: string, initial: CodexControllerState) { + const durable = new FileCodexControllerStateStore(filePath); + durable.write(initial); + return { + value: structuredClone(initial) as CodexControllerState | null, + read() { + this.value = durable.read(); + return this.value ? structuredClone(this.value) : null; + }, + write(state: CodexControllerState) { + durable.write(state); + this.value = structuredClone(state); + }, + }; +} + function appServer(overrides: Partial = {}): CodexAppServerSession { return { initialize: vi.fn(async () => undefined), @@ -287,6 +303,16 @@ function persistedRemote(overrides: Partial = {}): CodexCo }; } +function persistedRemoteForFile(overrides: Partial = {}): CodexControllerState { + const persisted = persistedRemote(overrides); + if (persisted.remote) { + const remote = persisted.remote as typeof persisted.remote & Record; + delete remote.connectPath; + delete remote.execServerUrl; + } + return persisted; +} + function persistedLocal(overrides: Partial = {}): CodexControllerState { return { version: 1, @@ -959,6 +985,107 @@ describe('CodexLiveController', () => { expect(third.controller.takeRecoveredTurn()).toBeUndefined(); }); + it.each(['completed', 'failed', 'interrupted'] as const)( + 'keeps a file-backed %s delivery fenced across restart, resume failure, close failure, and restart', + async (status) => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), `relay-controller-delivery-${status}-`)); + const statePath = path.join(directory, 'state.json'); + try { + const clientUserMessageId = `client-file-${status}`; + const store = fileStore( + statePath, + persistedRemoteForFile({ + inFlightTurn: { clientUserMessageId, execution: 'remote' }, + }) + ); + const outcome = + status === 'completed' + ? completedOutcome('turn-file-completed', 'file-backed answer', clientUserMessageId) + : { status, turnId: `turn-file-${status}` }; + const reconciler = appServer({ turnOutcome: vi.fn(async () => outcome) }); + const first = createController({ store, sessions: [reconciler] }); + + await expect(first.controller.initialize()).resolves.toMatchObject({ + phase: 'local', + generation: 8, + }); + await first.controller.close(); + + const durableAfterReconcile = new FileCodexControllerStateStore(statePath).read(); + expect(durableAfterReconcile).toMatchObject({ + phase: 'local', + generation: 8, + recoveredOutcome: { + generation: 8, + clientUserMessageId, + status, + }, + }); + expect(durableAfterReconcile?.pending).toBeUndefined(); + expect(durableAfterReconcile?.remote).toBeUndefined(); + + const resumeFailure = appServer({ + resumeThread: vi.fn(async () => Promise.reject(new Error('resume unavailable'))), + }); + const second = createController({ store, sessions: [resumeFailure] }); + await expect(second.controller.initialize()).rejects.toThrow('CODEX_THREAD_RESUME_FAILED_ON_RESTART'); + + const durableAfterResumeFailure = new FileCodexControllerStateStore(statePath).read(); + expect(durableAfterResumeFailure).toMatchObject({ + phase: 'recovery_failed', + generation: 8, + recoveredOutcome: { clientUserMessageId, status }, + }); + + const closeError = new Error('close unavailable'); + const closeFailure = appServer({ close: vi.fn(async () => Promise.reject(closeError)) }); + const third = createController({ store, sessions: [closeFailure] }); + await expect(third.controller.initialize()).resolves.toMatchObject({ + phase: 'local', + generation: 8, + }); + + expect(() => + third.controller.requestTeleport({ requestId: 'must-not-queue', expectedGeneration: 8 }) + ).toThrow('awaiting durable output acknowledgment'); + await expect(third.controller.runTurn('must not execute')).rejects.toThrow( + 'awaiting durable output acknowledgment' + ); + await expect(third.controller.rollback()).rejects.toThrow('awaiting durable output acknowledgment'); + expect(closeFailure.runTurn).not.toHaveBeenCalled(); + expect(closeFailure.addEnvironment).not.toHaveBeenCalled(); + expect(store.value?.pending).toBeUndefined(); + expect(store.value?.generation).toBe(8); + + await expect(third.controller.close()).rejects.toBe(closeError); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + phase: 'recovery_failed', + generation: 8, + recoveredOutcome: { clientUserMessageId, status }, + }); + + const finalSession = appServer(); + const fourth = createController({ store, sessions: [finalSession] }); + await expect(fourth.controller.initialize()).resolves.toMatchObject({ + phase: 'local', + generation: 8, + }); + if (status === 'completed') { + expect(fourth.controller.takeRecoveredTurn()).toMatchObject({ + turnId: 'turn-file-completed', + }); + } else { + expect(fourth.controller.takeRecoveredTerminal()).toMatchObject({ status }); + } + fourth.controller.acknowledgeRecoveredOutcome(); + expect(new FileCodexControllerStateStore(statePath).read()?.recoveredOutcome).toBeUndefined(); + await fourth.controller.close(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + } + ); + it('fails outcome-uncertain when an accepted remote turn cannot be found after fencing', async () => { const original = appServer({ runTurn: vi.fn(async () => Promise.reject(new Error('transport lost'))) }); const replacement = appServer({ turnOutcome: vi.fn(async () => ({ status: 'absent' as const })) }); diff --git a/packages/cli/src/cli/lib/codex-live-controller.ts b/packages/cli/src/cli/lib/codex-live-controller.ts index 61f9af49f..b7df73a47 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.ts @@ -75,7 +75,10 @@ export type CodexControllerState = { clientUserMessageId: string; execution: 'local' | 'remote'; }; - /** Durable, at-least-once recovery evidence retained until the CLI acknowledges rendering it. */ + /** + * Durable, at-least-once output delivery retained until the CLI acknowledges + * that its asynchronous writer completed. Model execution is never replayed. + */ recoveredOutcome?: CodexRecoveredOutcome; lastError?: string; updatedAt: string; @@ -291,9 +294,9 @@ function invalidNestedState(state: Record): boolean { (state.recoveredOutcome !== undefined && !recovered) || (inFlight !== undefined && recovered !== undefined) || (recovered !== undefined && !validRecoveredOutcome(state, recovered)) || + (recovered !== undefined && state.phase !== 'local' && state.phase !== 'recovery_failed') || (recovered !== undefined && - (state.phase !== 'local' || - pending !== undefined || + (pending !== undefined || source !== undefined || restore !== undefined || remote !== undefined || @@ -675,7 +678,13 @@ export class CodexLiveController { return publicStatus(this.requireState()); } - /** Claims crash-reconciled completion evidence in memory without clearing its durable copy. */ + /** + * Claims crash-reconciled completion evidence without clearing its durable + * copy. Rendering the full recovered answer is intentionally at-least-once: + * a process can have emitted notification deltas before it lost the terminal + * notification, so a recovered answer may duplicate an already-observed + * prefix. The model turn itself is not replayed. + */ takeRecoveredTurn(): CodexTurnResult | undefined { const recovered = this.requireState().recoveredOutcome; if (!recovered || recovered.status !== 'completed' || this.recoveredEvidenceTaken) return undefined; @@ -707,16 +716,17 @@ export class CodexLiveController { state.updatedAt = this.timestamp(); try { this.persist(); - this.recoveredEvidenceTaken = undefined; - if (durable.status === 'completed') this.schedulePrewarm(); } catch (error) { state.recoveredOutcome = durable; throw error; } + this.recoveredEvidenceTaken = undefined; + if (state.phase === 'local') this.schedulePrewarm(); } requestTeleport(request: CodexTeleportRequest): PublicCodexControllerStatus { const state = this.requireState(); + this.assertNoUnacknowledgedOutcome('queue a teleport'); if (request.expectedGeneration !== state.generation) { throw new Error( `Stale teleport generation ${request.expectedGeneration}; active generation is ${state.generation}.` @@ -741,6 +751,7 @@ export class CodexLiveController { async runTurn(text: string): Promise { const state = this.requireState(); + this.assertNoUnacknowledgedOutcome('start a turn'); if (state.turnActive) throw new Error('A Codex turn is already active.'); if (TURN_BLOCKED_PHASES.has(state.phase)) { throw new Error(`Cannot start a turn while the controller is ${state.phase}.`); @@ -853,6 +864,7 @@ export class CodexLiveController { async rollback(): Promise { const state = this.requireState(); + this.assertNoUnacknowledgedOutcome('roll back'); if (state.turnActive) throw new Error('Rollback is only allowed at a Codex turn boundary.'); await this.recoverLocal('rollback-revoke', 'Operator requested local rollback.'); return this.status(); @@ -1436,6 +1448,14 @@ export class CodexLiveController { return inferredCloudLifecycle(state as unknown as Record) !== 'none'; } + private assertNoUnacknowledgedOutcome(operation: string): void { + if (this.requireState().recoveredOutcome) { + throw new Error( + `Cannot ${operation} while a recovered Codex turn is awaiting durable output acknowledgment.` + ); + } + } + private markCloudFenced(): void { const state = this.requireState(); state.cloudLifecycle = 'none'; diff --git a/packages/cloud/src/live-teleport.test.ts b/packages/cloud/src/live-teleport.test.ts index 9b755416b..d82d0f56b 100644 --- a/packages/cloud/src/live-teleport.test.ts +++ b/packages/cloud/src/live-teleport.test.ts @@ -758,6 +758,75 @@ describe('CloudLiveTeleportClient', () => { }); }); + it.each(['warming', 'ready', 'verifying'] as const)( + 'rejects leaseExpiresAt while Cloud is only %s', + async (status) => { + const expiresAt = '2026-08-23T12:30:00.000Z'; + const client = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + prewarmId: 'prewarm-2', + status, + expiresAt, + leaseExpiresAt: expiresAt, + rollout: cloudRollout, + ...(status === 'warming' || status === 'verifying' ? { retryAfterMs: 250 } : {}), + }), + 'https://cloud.agentrelay.test' + ); + + await expect(client.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'lease before the environment became active' + ); + } + ); + + it('pins active, cleanup, and terminal lease metadata semantics', async () => { + const activeWithoutLease = new CloudLiveTeleportClient(async () => { + const { leaseExpiresAt: _leaseExpiresAt, ...response } = cloudActiveStatus(); + return Response.json(response); + }, 'https://cloud.agentrelay.test'); + await expect(activeWithoutLease.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'inconsistent active lease metadata' + ); + + const mismatchedActiveLease = new CloudLiveTeleportClient( + async () => + Response.json({ + ...cloudActiveStatus(), + leaseExpiresAt: '2026-08-23T12:44:59.000Z', + }), + 'https://cloud.agentrelay.test' + ); + await expect(mismatchedActiveLease.status({ sessionId: 'session-1', generation: 2 })).rejects.toThrow( + 'inconsistent active lease metadata' + ); + + for (const status of ['failed', 'revoked', 'expired'] as const) { + const terminal = new CloudLiveTeleportClient( + async () => + Response.json({ + sessionId: 'session-1', + generation: 2, + prewarmId: 'prewarm-2', + status, + expiresAt: '2026-08-23T12:30:00.000Z', + // A terminal row may retain its historical active lease. It no + // longer grants execution and need not equal the terminal expiry. + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + rollout: cloudRollout, + }), + 'https://cloud.agentrelay.test' + ); + await expect(terminal.status({ sessionId: 'session-1', generation: 2 })).resolves.toMatchObject({ + status, + leaseExpiresAt: '2026-08-23T12:45:00.000Z', + }); + } + }); + it('parses the exact active lease deadline and rejects non-canonical timestamps', async () => { const fetcher = vi .fn() diff --git a/packages/cloud/src/live-teleport.ts b/packages/cloud/src/live-teleport.ts index cd22d8e8c..7f99d9496 100644 --- a/packages/cloud/src/live-teleport.ts +++ b/packages/cloud/src/live-teleport.ts @@ -749,6 +749,12 @@ export class CloudLiveTeleportClient implements LiveTeleportCloudClient { if (input.status === 'active' && (!input.leaseExpiresAt || input.leaseExpiresAt !== input.expiresAt)) { throw new Error('Cloud live-teleport status returned inconsistent active lease metadata.'); } + if ( + (input.status === 'warming' || input.status === 'ready' || input.status === 'verifying') && + input.leaseExpiresAt !== undefined + ) { + throw new Error('Cloud live-teleport status returned a lease before the environment became active.'); + } const requiresEligibleRollout = input.status === 'warming' || input.status === 'ready' || From 1688a0abf26b1d3f480a0ac6aeece139b2af585d Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Sun, 23 Aug 2026 21:28:10 +0200 Subject: [PATCH 09/16] fix(codex): preserve recovered output across prewarm --- .../src/cli/lib/codex-live-controller.test.ts | 345 ++++++++++++++++++ .../cli/src/cli/lib/codex-live-controller.ts | 38 +- 2 files changed, 363 insertions(+), 20 deletions(-) diff --git a/packages/cli/src/cli/lib/codex-live-controller.test.ts b/packages/cli/src/cli/lib/codex-live-controller.test.ts index 32de2bc89..be6cab57b 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.test.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.test.ts @@ -1086,6 +1086,323 @@ describe('CodexLiveController', () => { } ); + it.each( + (['ready', 'failed', 'in-flight'] as const).flatMap((prewarmState) => + (['completed', 'failed', 'interrupted'] as const).map( + (outcomeStatus) => [prewarmState, outcomeStatus] as const + ) + ) + )( + 'keeps a file-backed local %s prewarm independent from an unacknowledged %s outcome', + async (prewarmState, outcomeStatus) => { + const directory = fs.mkdtempSync( + path.join(os.tmpdir(), `relay-controller-local-${prewarmState}-${outcomeStatus}-`) + ); + const statePath = path.join(directory, 'state.json'); + const pendingPrewarm = deferred<{ + sessionId: string; + prewarmId: string; + generation: number; + status: 'ready'; + expiresAt: string; + }>(); + let prewarmCalls = 0; + const prewarm = vi.fn(async (input: Parameters[0]) => { + prewarmCalls += 1; + if (prewarmCalls === 1) { + if (prewarmState === 'failed') throw new Error('first prewarm failed ambiguously'); + if (prewarmState === 'in-flight') return pendingPrewarm.promise; + } + return { + sessionId: input.sessionId, + prewarmId: `prewarm-${input.generation}`, + generation: input.generation, + status: 'ready' as const, + expiresAt: '2026-08-23T12:30:00.000Z', + }; + }); + const cloudClient = cloud({ prewarm }); + const store = fileStore(statePath, persistedLocal({ generation: 1 })); + const outcome = + outcomeStatus === 'completed' + ? completedOutcome('turn-local-recovered', 'file-backed local answer', 'operation-1') + : { status: outcomeStatus, turnId: `turn-local-${outcomeStatus}` }; + const local = appServer({ + runTurn: vi.fn(async () => Promise.reject(new Error('terminal notification lost'))), + turnOutcome: vi.fn(async () => outcome), + }); + + try { + const first = createController({ store, cloud: cloudClient, sessions: [local] }); + await first.controller.initialize(); + if (prewarmState === 'ready') { + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + } else if (prewarmState === 'failed') { + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('failed')); + } + + if (outcomeStatus === 'completed') { + await expect(first.controller.runTurn('do not replay local')).resolves.toMatchObject({ + turnId: 'turn-local-recovered', + }); + } else { + await expect(first.controller.runTurn('do not replay local')).rejects.toMatchObject({ + status: outcomeStatus, + requiresRecoveryAcknowledgment: true, + }); + } + expect(local.runTurn).toHaveBeenCalledTimes(1); + expect(local.turnOutcome).toHaveBeenCalledTimes(1); + + const durableBeforeRestart = new FileCodexControllerStateStore(statePath).read(); + expect(durableBeforeRestart).toMatchObject({ + generation: 1, + phase: 'local', + cloudLifecycle: prewarmState === 'ready' ? 'prewarmed' : 'prewarm_requested', + recoveredOutcome: { + generation: 1, + clientUserMessageId: 'operation-1', + status: outcomeStatus, + }, + }); + if (prewarmState === 'in-flight') { + expect(durableBeforeRestart?.prewarmStatus).toBeUndefined(); + } + + const closeError = new Error('close unavailable after durable recovery'); + const afterCrash = appServer({ close: vi.fn(async () => Promise.reject(closeError)) }); + const second = createController({ store, cloud: cloudClient, sessions: [afterCrash] }); + await expect(second.controller.initialize()).resolves.toMatchObject({ + generation: 2, + phase: 'local', + }); + expect(cloudClient.revoke).toHaveBeenCalledWith( + expect.objectContaining({ generation: 1, idempotencyKey: 'session-1:1:revoke' }) + ); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + generation: 2, + cloudLifecycle: 'none', + recoveredOutcome: { generation: 1, status: outcomeStatus }, + }); + expect(new FileCodexControllerStateStore(statePath).read()?.prewarmId).toBeUndefined(); + expect(new FileCodexControllerStateStore(statePath).read()?.prewarmStatus).toBeUndefined(); + + await expect(second.controller.close()).rejects.toBe(closeError); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + phase: 'recovery_failed', + generation: 2, + recoveredOutcome: { generation: 1, status: outcomeStatus }, + }); + + const resumeFailure = appServer({ + resumeThread: vi.fn(async () => Promise.reject(new Error('resume unavailable'))), + }); + const third = createController({ store, cloud: cloudClient, sessions: [resumeFailure] }); + await expect(third.controller.initialize()).rejects.toThrow('CODEX_THREAD_RESUME_FAILED_ON_RESTART'); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + phase: 'recovery_failed', + generation: 2, + recoveredOutcome: { generation: 1, status: outcomeStatus }, + }); + + const finalSession = appServer(); + const fourth = createController({ store, cloud: cloudClient, sessions: [finalSession] }); + await fourth.controller.initialize(); + expect(finalSession.runTurn).not.toHaveBeenCalled(); + if (outcomeStatus === 'completed') { + expect(fourth.controller.takeRecoveredTurn()).toMatchObject({ + turnId: 'turn-local-recovered', + }); + } else { + expect(fourth.controller.takeRecoveredTerminal()).toMatchObject({ status: outcomeStatus }); + } + fourth.controller.acknowledgeRecoveredOutcome(); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + expect(prewarm).toHaveBeenLastCalledWith( + expect.objectContaining({ generation: 2, idempotencyKey: 'session-1:2:prewarm' }) + ); + await fourth.controller.close(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + } + ); + + it('keeps late background prewarm completion from invalidating or erasing recovered output', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-controller-late-prewarm-')); + const statePath = path.join(directory, 'state.json'); + const pendingPrewarm = deferred<{ + sessionId: string; + prewarmId: string; + generation: number; + status: 'ready'; + expiresAt: string; + }>(); + const prewarm = vi.fn(() => pendingPrewarm.promise); + const cloudClient = cloud({ prewarm }); + const store = fileStore(statePath, persistedLocal({ generation: 1 })); + const local = appServer({ + runTurn: vi.fn(async () => Promise.reject(new Error('terminal notification lost'))), + turnOutcome: vi.fn(async () => + completedOutcome('turn-local-late-prewarm', 'exact recovered answer', 'operation-1') + ), + }); + + try { + const { controller } = createController({ store, cloud: cloudClient, sessions: [local] }); + await controller.initialize(); + await controller.runTurn('do not replay local'); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + cloudLifecycle: 'prewarm_requested', + recoveredOutcome: { turnId: 'turn-local-late-prewarm', status: 'completed' }, + }); + + pendingPrewarm.resolve({ + sessionId: 'session-1', + prewarmId: 'prewarm-1', + generation: 1, + status: 'ready', + expiresAt: '2026-08-23T12:30:00.000Z', + }); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + cloudLifecycle: 'prewarmed', + prewarmId: 'prewarm-1', + recoveredOutcome: { turnId: 'turn-local-late-prewarm', status: 'completed' }, + }); + + controller.acknowledgeRecoveredOutcome(); + expect(prewarm).toHaveBeenCalledTimes(1); + expect(new FileCodexControllerStateStore(statePath).read()?.recoveredOutcome).toBeUndefined(); + await controller.close(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it('retains a turn-boundary teleport queued while recovered local output awaits acknowledgment', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-controller-recovered-pending-')); + const statePath = path.join(directory, 'state.json'); + const lostTerminal = deferred(); + const local = appServer({ + runTurn: vi + .fn() + .mockImplementationOnce(() => lostTerminal.promise) + .mockResolvedValue(turnResult()), + turnOutcome: vi.fn(async () => + completedOutcome('turn-before-queued-teleport', 'durable exact answer', 'operation-1') + ), + }); + const store = fileStore(statePath, persistedLocal({ generation: 1 })); + const cloudClient = cloud(); + + try { + const { controller } = createController({ store, cloud: cloudClient, sessions: [local] }); + await controller.initialize(); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + + const running = controller.runTurn('turn before queued teleport'); + await vi.waitFor(() => expect(local.runTurn).toHaveBeenCalledTimes(1)); + controller.requestTeleport({ requestId: 'queued-during-turn', expectedGeneration: 1 }); + lostTerminal.reject(new Error('terminal notification lost')); + await expect(running).resolves.toMatchObject({ turnId: 'turn-before-queued-teleport' }); + + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + phase: 'teleport_pending', + pending: { requestId: 'queued-during-turn', expectedGeneration: 1 }, + recoveredOutcome: { turnId: 'turn-before-queued-teleport', status: 'completed' }, + }); + await expect(controller.runTurn('must wait for durable output')).rejects.toThrow( + 'awaiting durable output acknowledgment' + ); + + controller.acknowledgeRecoveredOutcome(); + await expect(controller.runTurn('first turn after acknowledgment')).resolves.toMatchObject({ + turnId: 'turn-1', + }); + expect(cloudClient.acquire).toHaveBeenCalledTimes(1); + expect(cloudClient.acquire).toHaveBeenCalledWith( + expect.objectContaining({ generation: 1, idempotencyKey: 'session-1:1:acquire' }) + ); + expect(local.runTurn).toHaveBeenCalledTimes(2); + await controller.close(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + + it('retains the consumed generation fence if restart crashes after terminal revoke', async () => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-controller-terminal-fence-')); + const statePath = path.join(directory, 'state.json'); + const recovered = completedOutcome( + 'turn-before-terminal-fence', + 'durable answer before terminal fence', + 'client-before-terminal-fence' + ); + const store = fileStore( + statePath, + persistedLocal({ + generation: 3, + cloudLifecycle: 'prewarmed', + prewarmId: 'prewarm-3', + prewarmStatus: 'ready', + recoveredOutcome: { + sessionId: 'session-1', + threadId: 'thread-1', + generation: 3, + clientUserMessageId: 'client-before-terminal-fence', + turnId: recovered.turnId, + status: 'completed', + result: recovered.result, + }, + }) + ); + const cloudClient = cloud(); + const startFailure = appServer({ + initialize: vi.fn(async () => Promise.reject(new Error('process crashed after revoke'))), + }); + + try { + const first = createController({ store, cloud: cloudClient, sessions: [startFailure] }); + await expect(first.controller.initialize()).rejects.toThrow('process crashed after revoke'); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + generation: 3, + phase: 'rolling_back', + cloudLifecycle: 'cleanup_requested', + recoveredOutcome: { generation: 3, turnId: recovered.turnId }, + }); + + const resumed = appServer(); + const second = createController({ store, cloud: cloudClient, sessions: [resumed] }); + await expect(second.controller.initialize()).resolves.toMatchObject({ + generation: 4, + phase: 'local', + cloudLifecycle: 'none', + }); + expect(cloudClient.revoke).toHaveBeenCalledTimes(2); + expect(cloudClient.revoke).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ generation: 3, idempotencyKey: 'session-1:3:revoke' }) + ); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + generation: 4, + recoveredOutcome: { generation: 3, turnId: recovered.turnId }, + }); + expect(new FileCodexControllerStateStore(statePath).read()?.prewarmId).toBeUndefined(); + expect(new FileCodexControllerStateStore(statePath).read()?.prewarmStatus).toBeUndefined(); + + expect(second.controller.takeRecoveredTurn()).toMatchObject({ turnId: recovered.turnId }); + second.controller.acknowledgeRecoveredOutcome(); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + expect(cloudClient.prewarm).toHaveBeenCalledWith( + expect.objectContaining({ generation: 4, idempotencyKey: 'session-1:4:prewarm' }) + ); + await second.controller.close(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + }); + it('fails outcome-uncertain when an accepted remote turn cannot be found after fencing', async () => { const original = appServer({ runTurn: vi.fn(async () => Promise.reject(new Error('transport lost'))) }); const replacement = appServer({ turnOutcome: vi.fn(async () => ({ status: 'absent' as const })) }); @@ -1792,6 +2109,34 @@ describe('FileCodexControllerStateStore', () => { }, }, ], + [ + 'future-generation recovered outcome', + { + ...persistedLocal(), + recoveredOutcome: { + sessionId: 'session-1', + threadId: 'thread-1', + generation: 4, + clientUserMessageId: 'client-future', + turnId: 'turn-future', + status: 'failed', + }, + }, + ], + [ + 'two-generations-stale recovered outcome', + { + ...persistedLocal(), + recoveredOutcome: { + sessionId: 'session-1', + threadId: 'thread-1', + generation: 1, + clientUserMessageId: 'client-stale', + turnId: 'turn-stale', + status: 'interrupted', + }, + }, + ], ])('rejects %s instead of adopting malformed controller state', (_name, value) => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-controller-state-')); const file = path.join(directory, 'state.json'); diff --git a/packages/cli/src/cli/lib/codex-live-controller.ts b/packages/cli/src/cli/lib/codex-live-controller.ts index b7df73a47..5ac86351b 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.ts @@ -294,15 +294,13 @@ function invalidNestedState(state: Record): boolean { (state.recoveredOutcome !== undefined && !recovered) || (inFlight !== undefined && recovered !== undefined) || (recovered !== undefined && !validRecoveredOutcome(state, recovered)) || - (recovered !== undefined && state.phase !== 'local' && state.phase !== 'recovery_failed') || (recovered !== undefined && - (pending !== undefined || - source !== undefined || - restore !== undefined || - remote !== undefined || - state.cloudLifecycle !== 'none' || - state.prewarmId !== undefined || - state.prewarmStatus !== undefined)) || + state.phase !== 'local' && + state.phase !== 'teleport_pending' && + state.phase !== 'rolling_back' && + state.phase !== 'fenced' && + state.phase !== 'recovery_failed') || + (recovered !== undefined && (source !== undefined || restore !== undefined || remote !== undefined)) || !validOptionalString(state.prewarmId) || (state.prewarmStatus !== undefined && state.prewarmStatus !== 'warming' && @@ -364,7 +362,8 @@ function validRecoveredOutcome(state: Record, recovered: Record const bindingMatches = [ recovered.sessionId === state.sessionId, recovered.threadId === state.threadId, - recovered.generation === state.generation, + Number.isSafeInteger(recovered.generation) && + (recovered.generation === state.generation || recovered.generation === Number(state.generation) - 1), nonEmptyString(recovered.clientUserMessageId), nonEmptyString(recovered.turnId), ].every(Boolean); @@ -643,6 +642,8 @@ export class CodexLiveController { state.remote = undefined; state.source = undefined; state.mountRestore = undefined; + state.prewarmId = undefined; + state.prewarmStatus = undefined; if (recovered) this.stageRecoveredOutcome(recovered); state.lastError = undefined; state.updatedAt = this.timestamp(); @@ -1131,6 +1132,10 @@ export class CodexLiveController { private async confirmFence(_reason: string): Promise { const state = this.requireState(); state.cloudLifecycle = 'cleanup_requested'; + // Once cleanup owns the generation, stale warm metadata must not cause + // legacy-state inference to downgrade the durable cleanup intent. + state.prewarmId = undefined; + state.prewarmStatus = undefined; state.updatedAt = this.timestamp(); this.persist(); await this.withAbortableLifecycleDeadline(async (signal) => { @@ -1148,7 +1153,6 @@ export class CodexLiveController { // ownership has been handed back. cleanup_pending is not a fence and // must never allow the source mount to resume. if (revoked.status === 'revoked' || revoked.status === 'expired') { - this.markCloudFenced(); return; } } catch (error) { @@ -1163,7 +1167,6 @@ export class CodexLiveController { }); this.assertLifecycleIdentity(status); if (status.status === 'revoked' || status.status === 'expired') { - this.markCloudFenced(); return; } lastError = new Error(`Cloud fence remains ${status.status}.`); @@ -1226,6 +1229,10 @@ export class CodexLiveController { private schedulePrewarm(): void { if (this.prewarmPromise || this.closing) return; const state = this.requireState(); + // Recovered output is an independent durable delivery queue. Do not start + // another lifecycle while it is unacknowledged, and do not duplicate an + // already-started lifecycle after its background promise has settled. + if (state.recoveredOutcome || this.cloudMayOwnResources(state)) return; const generation = state.generation; state.cloudLifecycle = 'prewarm_requested'; state.updatedAt = this.timestamp(); @@ -1456,15 +1463,6 @@ export class CodexLiveController { } } - private markCloudFenced(): void { - const state = this.requireState(); - state.cloudLifecycle = 'none'; - state.prewarmId = undefined; - state.prewarmStatus = undefined; - state.updatedAt = this.timestamp(); - this.persist(); - } - private async createInitializedAppServer(): Promise { const appServer = await this.deps.createAppServer(); await appServer.initialize(); From 5c58a78f3ca5a0e7d67f62e074a80aebe60206d9 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Sun, 23 Aug 2026 21:46:42 +0200 Subject: [PATCH 10/16] fix(codex): retain queued teleport across restart --- .../src/cli/lib/codex-live-controller.test.ts | 154 ++++++++++++++++++ .../cli/src/cli/lib/codex-live-controller.ts | 26 ++- 2 files changed, 175 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/cli/lib/codex-live-controller.test.ts b/packages/cli/src/cli/lib/codex-live-controller.test.ts index be6cab57b..35d4b9c2f 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.test.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.test.ts @@ -1331,6 +1331,160 @@ describe('CodexLiveController', () => { } }); + it.each( + (['ready', 'failed', 'in-flight'] as const).flatMap((prewarmState) => + (['completed', 'failed', 'interrupted'] as const).map( + (outcomeStatus) => [prewarmState, outcomeStatus] as const + ) + ) + )( + 'rebases a queued teleport after a crash with %s prewarm and %s recovery evidence', + async (prewarmState, outcomeStatus) => { + const directory = fs.mkdtempSync( + path.join(os.tmpdir(), `relay-controller-pending-restart-${prewarmState}-${outcomeStatus}-`) + ); + const statePath = path.join(directory, 'state.json'); + const neverSettledPrewarm = deferred<{ + sessionId: string; + prewarmId: string; + generation: number; + status: 'ready'; + expiresAt: string; + }>(); + let prewarmCalls = 0; + const prewarm = vi.fn(async (input: Parameters[0]) => { + prewarmCalls += 1; + if (prewarmCalls === 1) { + if (prewarmState === 'failed') throw new Error('first prewarm failed ambiguously'); + if (prewarmState === 'in-flight') return neverSettledPrewarm.promise; + } + return { + sessionId: input.sessionId, + prewarmId: `prewarm-${input.generation}`, + generation: input.generation, + status: 'ready' as const, + expiresAt: '2026-08-23T12:30:00.000Z', + }; + }); + const cloudClient = cloud({ prewarm }); + const store = fileStore(statePath, persistedLocal({ generation: 1 })); + const lostTerminal = deferred(); + const outcome = + outcomeStatus === 'completed' + ? completedOutcome('turn-before-crash', 'durable answer before crash', 'operation-1') + : { status: outcomeStatus, turnId: `turn-before-crash-${outcomeStatus}` }; + const firstSession = appServer({ + runTurn: vi.fn(() => lostTerminal.promise), + turnOutcome: vi.fn(async () => outcome), + }); + + try { + const first = createController({ store, cloud: cloudClient, sessions: [firstSession] }); + await first.controller.initialize(); + if (prewarmState === 'ready') { + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + } else if (prewarmState === 'failed') { + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('failed')); + } else { + expect(store.value).toMatchObject({ cloudLifecycle: 'prewarm_requested' }); + expect(store.value?.prewarmStatus).toBeUndefined(); + } + + const running = first.controller.runTurn('turn before process crash'); + await vi.waitFor(() => expect(firstSession.runTurn).toHaveBeenCalledTimes(1)); + first.controller.requestTeleport({ requestId: 'queued-before-crash', expectedGeneration: 1 }); + lostTerminal.reject(new Error('terminal notification lost before process crash')); + if (outcomeStatus === 'completed') { + await expect(running).resolves.toMatchObject({ turnId: 'turn-before-crash' }); + } else { + await expect(running).rejects.toMatchObject({ + status: outcomeStatus, + requiresRecoveryAcknowledgment: true, + }); + } + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + generation: 1, + phase: 'teleport_pending', + pending: { requestId: 'queued-before-crash', expectedGeneration: 1 }, + recoveredOutcome: { generation: 1, status: outcomeStatus }, + }); + + // Simulate a new process whose first restart cannot resume the Codex + // thread after the old Cloud identity has been fenced. + const resumeFailure = appServer({ + resumeThread: vi.fn(async () => Promise.reject(new Error('resume unavailable'))), + }); + const second = createController({ store, cloud: cloudClient, sessions: [resumeFailure] }); + await expect(second.controller.initialize()).rejects.toThrow('CODEX_THREAD_RESUME_FAILED_ON_RESTART'); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + generation: 1, + phase: 'recovery_failed', + pending: { requestId: 'queued-before-crash', expectedGeneration: 1 }, + recoveredOutcome: { generation: 1, status: outcomeStatus }, + }); + expect(prewarm).toHaveBeenCalledTimes(1); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + + // A later restart succeeds and atomically moves both the controller and + // accepted request onto generation 2. A close failure must not erase it. + const closeError = new Error('close unavailable after restart'); + const closeFailure = appServer({ close: vi.fn(async () => Promise.reject(closeError)) }); + const third = createController({ store, cloud: cloudClient, sessions: [closeFailure] }); + await expect(third.controller.initialize()).resolves.toMatchObject({ + generation: 2, + phase: 'teleport_pending', + pending: { requestId: 'queued-before-crash', expectedGeneration: 2 }, + }); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + generation: 2, + pending: { requestId: 'queued-before-crash', expectedGeneration: 2 }, + recoveredOutcome: { generation: 1, status: outcomeStatus }, + }); + expect(prewarm).toHaveBeenCalledTimes(1); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + await expect(third.controller.close()).rejects.toBe(closeError); + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + phase: 'recovery_failed', + generation: 2, + pending: { requestId: 'queued-before-crash', expectedGeneration: 2 }, + recoveredOutcome: { generation: 1, status: outcomeStatus }, + }); + + const finalSession = appServer(); + const fourth = createController({ store, cloud: cloudClient, sessions: [finalSession] }); + await expect(fourth.controller.initialize()).resolves.toMatchObject({ + generation: 2, + phase: 'teleport_pending', + }); + expect(prewarm).toHaveBeenCalledTimes(1); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + + if (outcomeStatus === 'completed') { + expect(fourth.controller.takeRecoveredTurn()).toMatchObject({ turnId: 'turn-before-crash' }); + } else { + expect(fourth.controller.takeRecoveredTerminal()).toMatchObject({ status: outcomeStatus }); + } + fourth.controller.acknowledgeRecoveredOutcome(); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + expect(prewarm).toHaveBeenCalledTimes(2); + expect(prewarm).toHaveBeenLastCalledWith( + expect.objectContaining({ generation: 2, idempotencyKey: 'session-1:2:prewarm' }) + ); + + await expect(fourth.controller.runTurn('first acknowledged Cloud turn')).resolves.toMatchObject({ + turnId: 'turn-1', + }); + expect(cloudClient.acquire).toHaveBeenCalledTimes(1); + expect(cloudClient.acquire).toHaveBeenCalledWith( + expect.objectContaining({ generation: 2, idempotencyKey: 'session-1:2:acquire' }) + ); + await fourth.controller.close(); + } finally { + fs.rmSync(directory, { recursive: true, force: true }); + } + } + ); + it('retains the consumed generation fence if restart crashes after terminal revoke', async () => { const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-controller-terminal-fence-')); const statePath = path.join(directory, 'state.json'); diff --git a/packages/cli/src/cli/lib/codex-live-controller.ts b/packages/cli/src/cli/lib/codex-live-controller.ts index 5ac86351b..664404344 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.ts @@ -576,7 +576,6 @@ export class CodexLiveController { workspaceRoot: this.options.workspaceRoot, turnActive: false, phase: mustFenceCloud || mustRecoverMount ? 'rolling_back' : 'local', - pending: undefined, lastError: undefined, updatedAt: this.timestamp(), }; @@ -637,7 +636,14 @@ export class CodexLiveController { const state = this.requireState(); state.generation = persisted.generation + (mustFenceCloud || mustRecoverMount ? 1 : 0); - state.phase = 'local'; + if (state.pending) { + // A queued teleport is an accepted durable intent, independent from + // recovered output delivery. Fencing consumes the old Cloud identity, + // so rebind the request in the same write that publishes the new + // generation; never silently discard it across a process restart. + state.pending = { ...state.pending, expectedGeneration: state.generation }; + } + state.phase = state.pending ? 'teleport_pending' : 'local'; state.cloudLifecycle = 'none'; state.remote = undefined; state.source = undefined; @@ -722,7 +728,7 @@ export class CodexLiveController { throw error; } this.recoveredEvidenceTaken = undefined; - if (state.phase === 'local') this.schedulePrewarm(); + if (state.phase === 'local' || state.phase === 'teleport_pending') this.schedulePrewarm(); } requestTeleport(request: CodexTeleportRequest): PublicCodexControllerStatus { @@ -915,7 +921,18 @@ export class CodexLiveController { this.persist(); return; } - if (!lifecycleIdentityConsumed) return; + if (!lifecycleIdentityConsumed) { + // A clean operator shutdown is the explicit cancellation boundary for + // an otherwise durable queued request. Crash/recovery failures never + // pass through this successful close path and therefore retain it. + if (state.pending) { + state.pending = undefined; + state.phase = 'local'; + state.updatedAt = this.timestamp(); + this.persist(); + } + return; + } // A confirmed prewarm-only fence consumes this generation's idempotency // identity just as surely as acquire/revoke. Never reinitialize and // replay a revoked resource under the same generation. @@ -1472,7 +1489,6 @@ export class CodexLiveController { private markFenced(message: string): void { const state = this.requireState(); state.phase = 'fenced'; - state.pending = undefined; state.lastError = message; state.updatedAt = this.timestamp(); this.persist(); From dadc2e199392de11a2ce4db8b7c9c11a69f20743 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Sun, 23 Aug 2026 22:18:05 +0200 Subject: [PATCH 11/16] fix(codex): retain teleport on terminal unwind --- packages/cli/src/cli/commands/codex.test.ts | 30 +++++ packages/cli/src/cli/commands/codex.ts | 106 +++++++++++------- .../src/cli/lib/codex-live-controller.test.ts | 47 +++++++- .../cli/src/cli/lib/codex-live-controller.ts | 32 ++++-- 4 files changed, 160 insertions(+), 55 deletions(-) diff --git a/packages/cli/src/cli/commands/codex.test.ts b/packages/cli/src/cli/commands/codex.test.ts index 6d5978105..2b0259fe8 100644 --- a/packages/cli/src/cli/commands/codex.test.ts +++ b/packages/cli/src/cli/commands/codex.test.ts @@ -7,6 +7,7 @@ import { registerCodexCommands, runManagedCodexTurn, surfaceRecoveredCodexTerminal, + withCodexControllerShutdown, } from './codex.js'; function deferred() { @@ -75,6 +76,35 @@ describe('nodeCallbackWriter', () => { }); }); +describe('withCodexControllerShutdown', () => { + it('marks only a normally completed command as an explicit pending-teleport cancellation', async () => { + const close = vi.fn(async () => undefined); + await expect(withCodexControllerShutdown({ close } as never, async () => 'complete')).resolves.toBe( + 'complete' + ); + expect(close).toHaveBeenLastCalledWith({ cancelPendingTeleport: true }); + + const terminal = new Error('recorded terminal'); + await expect( + withCodexControllerShutdown({ close } as never, async () => Promise.reject(terminal)) + ).rejects.toBe(terminal); + expect(close).toHaveBeenLastCalledWith({ cancelPendingTeleport: false }); + }); + + it('still closes the controller when control-server cleanup fails', async () => { + const close = vi.fn(async () => undefined); + const cleanupError = new Error('control server close failed'); + await expect( + withCodexControllerShutdown( + { close } as never, + async () => undefined, + async () => Promise.reject(cleanupError) + ) + ).rejects.toBe(cleanupError); + expect(close).toHaveBeenCalledWith({ cancelPendingTeleport: true }); + }); +}); + describe('runManagedCodexTurn', () => { it('keeps a completed recovery unacknowledged until the async output writer resolves', async () => { const write = deferred(); diff --git a/packages/cli/src/cli/commands/codex.ts b/packages/cli/src/cli/commands/codex.ts index 3f5143f5e..840cdaf58 100644 --- a/packages/cli/src/cli/commands/codex.ts +++ b/packages/cli/src/cli/commands/codex.ts @@ -64,6 +64,30 @@ export function nodeCallbackWriter(stream: NodeCallbackWritable): CodexAsyncWrit }); } +/** + * Gives controller shutdown the same semantic outcome as the command body. + * A rejected command is an error unwind, never implicit cancellation of a + * teleport that the controller already accepted durably. + */ +export async function withCodexControllerShutdown( + controller: Pick, + run: () => Promise, + beforeClose: () => Promise = async () => undefined +): Promise { + let completedNormally = false; + try { + const result = await run(); + completedNormally = true; + return result; + } finally { + try { + await beforeClose(); + } finally { + await controller.close({ cancelPendingTeleport: completedNormally }); + } + } +} + export interface CodexCommandDependencies { runManaged(options: { cwd: string; model?: string; prompt?: string; json?: boolean }): Promise; checkpointAndSeal: CodexWorkspaceSealProvider; @@ -360,47 +384,51 @@ function withDefaults(overrides: Partial = {}): CodexC } const server = createControlServer(controller); try { - const status = await controller.initialize(); - await surfaceRecoveredCodexTerminal(controller, { - json: Boolean(options.json), - writeError: nodeCallbackWriter(process.stderr), - }); - await listen(server, paths.socketPath); - process.stderr.write( - `Relay-managed Codex ${status.threadId} is ready locally (generation ${status.generation}).\n` + await withCodexControllerShutdown( + controller, + async () => { + const status = await controller.initialize(); + await surfaceRecoveredCodexTerminal(controller, { + json: Boolean(options.json), + writeError: nodeCallbackWriter(process.stderr), + }); + await listen(server, paths.socketPath); + process.stderr.write( + `Relay-managed Codex ${status.threadId} is ready locally (generation ${status.generation}).\n` + ); + const recoveredTurn = controller.takeRecoveredTurn(); + if (recoveredTurn) { + await writeReconciledTurn(recoveredTurn, { + json: Boolean(options.json), + writeOutput: nodeCallbackWriter(process.stdout), + }); + controller.acknowledgeRecoveredOutcome(); + } + + if (options.prompt) { + await runManagedCodexTurn(controller, options.prompt, { + json: Boolean(options.json), + writeError: nodeCallbackWriter(process.stderr), + writeOutput: nodeCallbackWriter(process.stdout), + }); + } + const input = readline.createInterface({ + input: process.stdin, + terminal: Boolean(process.stdin.isTTY), + }); + for await (const line of input) { + if (!line.trim()) continue; + await runManagedCodexTurn(controller, line, { + json: Boolean(options.json), + writeError: nodeCallbackWriter(process.stderr), + writeOutput: nodeCallbackWriter(process.stdout), + }); + if (!options.json) process.stdout.write('\n'); + } + }, + () => closeServer(server) ); - const recoveredTurn = controller.takeRecoveredTurn(); - if (recoveredTurn) { - await writeReconciledTurn(recoveredTurn, { - json: Boolean(options.json), - writeOutput: nodeCallbackWriter(process.stdout), - }); - controller.acknowledgeRecoveredOutcome(); - } - - if (options.prompt) { - await runManagedCodexTurn(controller, options.prompt, { - json: Boolean(options.json), - writeError: nodeCallbackWriter(process.stderr), - writeOutput: nodeCallbackWriter(process.stdout), - }); - } - const input = readline.createInterface({ - input: process.stdin, - terminal: Boolean(process.stdin.isTTY), - }); - for await (const line of input) { - if (!line.trim()) continue; - await runManagedCodexTurn(controller, line, { - json: Boolean(options.json), - writeError: nodeCallbackWriter(process.stderr), - writeOutput: nodeCallbackWriter(process.stdout), - }); - if (!options.json) process.stdout.write('\n'); - } } finally { - await closeServer(server); - await controller.close(); try { fs.unlinkSync(paths.socketPath); } catch { diff --git a/packages/cli/src/cli/lib/codex-live-controller.test.ts b/packages/cli/src/cli/lib/codex-live-controller.test.ts index 35d4b9c2f..1b7d4b436 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.test.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.test.ts @@ -16,6 +16,7 @@ import { } from './codex-live-controller.js'; import type { CodexAppServerSession, CodexTurnResult } from './codex-app-server.js'; import { CodexAppServerTurnTerminalError } from './codex-app-server.js'; +import { surfaceRecoveredCodexTerminal, withCodexControllerShutdown } from '../commands/codex.js'; const source = { kind: 'relayfile-checkpoint-seal' as const, @@ -1461,24 +1462,58 @@ describe('CodexLiveController', () => { if (outcomeStatus === 'completed') { expect(fourth.controller.takeRecoveredTurn()).toMatchObject({ turnId: 'turn-before-crash' }); + fourth.controller.acknowledgeRecoveredOutcome(); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + await fourth.controller.close({ cancelPendingTeleport: false }); } else { - expect(fourth.controller.takeRecoveredTerminal()).toMatchObject({ status: outcomeStatus }); + const writeError = vi.fn(async () => undefined); + await expect( + withCodexControllerShutdown(fourth.controller, () => + surfaceRecoveredCodexTerminal(fourth.controller, { json: false, writeError }) + ) + ).rejects.toMatchObject({ status: outcomeStatus }); + expect(writeError).toHaveBeenCalledWith( + `Codex recorded the turn as ${outcomeStatus}; it was not replayed.\n` + ); } - fourth.controller.acknowledgeRecoveredOutcome(); - await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); expect(prewarm).toHaveBeenCalledTimes(2); expect(prewarm).toHaveBeenLastCalledWith( expect.objectContaining({ generation: 2, idempotencyKey: 'session-1:2:prewarm' }) ); - await expect(fourth.controller.runTurn('first acknowledged Cloud turn')).resolves.toMatchObject({ + // This is the production terminal/error-unwind seam: stderr delivery + // was acknowledged, but throwing the recorded terminal must not turn + // a successful controller close into user cancellation. Because the + // generation-2 prewarm exists, close fences it and atomically rebases + // the accepted request onto generation 3. + expect(new FileCodexControllerStateStore(statePath).read()).toMatchObject({ + generation: 3, + phase: 'teleport_pending', + pending: { requestId: 'queued-before-crash', expectedGeneration: 3 }, + cloudLifecycle: 'none', + }); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + + const fifthSession = appServer(); + const fifth = createController({ store, cloud: cloudClient, sessions: [fifthSession] }); + await expect(fifth.controller.initialize()).resolves.toMatchObject({ + generation: 3, + phase: 'teleport_pending', + pending: { requestId: 'queued-before-crash', expectedGeneration: 3 }, + }); + await vi.waitFor(() => expect(store.value?.prewarmStatus).toBe('ready')); + expect(prewarm).toHaveBeenCalledTimes(3); + expect(prewarm).toHaveBeenLastCalledWith( + expect.objectContaining({ generation: 3, idempotencyKey: 'session-1:3:prewarm' }) + ); + await expect(fifth.controller.runTurn('first acknowledged Cloud turn')).resolves.toMatchObject({ turnId: 'turn-1', }); expect(cloudClient.acquire).toHaveBeenCalledTimes(1); expect(cloudClient.acquire).toHaveBeenCalledWith( - expect.objectContaining({ generation: 2, idempotencyKey: 'session-1:2:acquire' }) + expect.objectContaining({ generation: 3, idempotencyKey: 'session-1:3:acquire' }) ); - await fourth.controller.close(); + await fifth.controller.close(); } finally { fs.rmSync(directory, { recursive: true, force: true }); } diff --git a/packages/cli/src/cli/lib/codex-live-controller.ts b/packages/cli/src/cli/lib/codex-live-controller.ts index 664404344..e85a69e59 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.ts @@ -167,6 +167,14 @@ export type CodexWorkspaceSealHandle = { close(): Promise; }; +export type CodexControllerCloseOptions = { + /** + * Explicit operator/session completion is the only boundary allowed to + * cancel an accepted, not-yet-consumed teleport. Error unwinds retain it. + */ + cancelPendingTeleport?: boolean; +}; + export type CodexMountRestoreIdentity = { lifecycleId: string; resumeId?: string; @@ -877,8 +885,9 @@ export class CodexLiveController { return this.status(); } - async close(): Promise { + async close(options: CodexControllerCloseOptions = {}): Promise { const state = this.state; + const cancelPendingTeleport = options.cancelPendingTeleport ?? true; const lifecycleIdentityConsumed = Boolean( state && (this.cloudMayOwnResources(state) || this.sealedWorkspace || state.source || state.mountRestore) @@ -922,12 +931,12 @@ export class CodexLiveController { return; } if (!lifecycleIdentityConsumed) { - // A clean operator shutdown is the explicit cancellation boundary for - // an otherwise durable queued request. Crash/recovery failures never - // pass through this successful close path and therefore retain it. + // Only explicit session completion is a cancellation boundary. A + // terminal/error unwind can successfully close the app-server too, + // but must leave an already accepted request durable for restart. if (state.pending) { - state.pending = undefined; - state.phase = 'local'; + if (cancelPendingTeleport) state.pending = undefined; + state.phase = cancelPendingTeleport ? 'local' : 'teleport_pending'; state.updatedAt = this.timestamp(); this.persist(); } @@ -936,11 +945,14 @@ export class CodexLiveController { // A confirmed prewarm-only fence consumes this generation's idempotency // identity just as surely as acquire/revoke. Never reinitialize and // replay a revoked resource under the same generation. - // Finalize any queued request in the same durable write so it cannot - // retain an expectedGeneration from the consumed identity. - state.pending = undefined; state.generation += 1; - state.phase = 'local'; + // Finalize or atomically rebase a queued request in the same durable + // write as the consumed identity. Error unwind is not cancellation. + if (state.pending) { + if (cancelPendingTeleport) state.pending = undefined; + else state.pending.expectedGeneration = state.generation; + } + state.phase = state.pending ? 'teleport_pending' : 'local'; state.cloudLifecycle = 'none'; state.remote = undefined; state.source = undefined; From ff0845264150dab6a32ceab317343e7b3abf510f Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Sun, 23 Aug 2026 23:35:24 +0200 Subject: [PATCH 12/16] fix(codex): gate live teleport behind local flag --- CHANGELOG.md | 2 +- packages/cli/README.md | 22 +++ packages/cli/src/cli/commands/codex.ts | 31 ++-- .../src/cli/lib/codex-live-controller.test.ts | 146 +++++++++++++++++- .../cli/src/cli/lib/codex-live-controller.ts | 82 ++++++++-- 5 files changed, 253 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 04ce28822..b5e409de7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `relay codex run` starts a Relay-managed local Codex app-server/thread, and `relay codex teleport` seals its active Relayfile poll mount at the next turn boundary, binds exact Relayfile destination verification to the source checkpoint, requires a 40-minute Cloud turn lease, and requires confirmed fencing plus ownership handback and mount readiness before local rollback. After notification loss, Relay never replays model execution: it durably redelivers reconciled output at least once, so an already-rendered live prefix can repeat, and it transparently runs the same prompt locally only after a confirmed pre-submission cutover failure. +- `relay codex run` starts a Relay-managed local Codex app-server/thread. Live session teleport is dark-launched behind the default-off local `RELAY_LIVE_SESSION_TELEPORT_ENABLED=true` switch; when enabled, `relay codex teleport` seals its active Relayfile poll mount at the next turn boundary, binds exact Relayfile destination verification to the source checkpoint, requires a 40-minute Cloud turn lease, and requires confirmed fencing plus ownership handback and mount readiness before local rollback. After notification loss, Relay never replays model execution: it durably redelivers reconciled output at least once, so an already-rendered live prefix can repeat, and it transparently runs the same prompt locally only after a confirmed pre-submission cutover failure. ## [Unreleased - Patch] diff --git a/packages/cli/README.md b/packages/cli/README.md index e69c1ec18..a40cc76dc 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -20,6 +20,28 @@ agent-relay message post --channel general --text "hello" agent-relay workspace list ``` +## Relay-managed Codex teleport + +`relay codex run` always supports a local managed Codex app-server session. Live +session teleport is dark-launched and remains disabled unless the controller +process starts with the exact local opt-in below: + +```bash +RELAY_LIVE_SESSION_TELEPORT_ENABLED=true relay codex run +relay codex teleport +``` + +Unset, `false`, `1`, and all other values keep new execution local: a fresh or +already-local controller does not authenticate to Cloud, prewarm, acquire, or +route a turn remotely. The `teleport`, `rollback`, and `status` commands remain +discoverable, but a disabled controller rejects teleport requests. + +The flag is read when `relay codex run` starts. To roll back the capability, +stop the controller and restart it with the variable unset or set to `false`. +If the prior process left a prewarm or remote environment behind, startup first +revokes that Cloud generation and restores the same thread and Relayfile mount +locally; it does not prewarm again while disabled. + ## This machine's node The `node` command group manages the broker on your machine and the agents it runs: diff --git a/packages/cli/src/cli/commands/codex.ts b/packages/cli/src/cli/commands/codex.ts index 840cdaf58..a9a27d6f7 100644 --- a/packages/cli/src/cli/commands/codex.ts +++ b/packages/cli/src/cli/commands/codex.ts @@ -19,6 +19,8 @@ import { CodexTurnOutcomeUncertainError, CodexTurnRecordedError, FileCodexControllerStateStore, + codexControllerStateMayOwnCloudResources, + isCodexLiveTeleportEnabled, type CodexControllerState, type CodexPersistedMountResumeProvider, type CodexWorkspaceSealProvider, @@ -323,7 +325,8 @@ function withDefaults(overrides: Partial = {}): CodexC overrides.runManaged ?? (async (options) => { const workspaceRoot = fs.realpathSync(path.resolve(options.cwd)); - await probeCodexEnvironmentCapability('codex'); + const liveTeleportEnabled = isCodexLiveTeleportEnabled(); + if (liveTeleportEnabled) await probeCodexEnvironmentCapability('codex'); fs.mkdirSync(paths.directory, { recursive: true, mode: 0o700 }); const store = new FileCodexControllerStateStore(paths.statePath); const prior = store.read(); @@ -332,20 +335,27 @@ function withDefaults(overrides: Partial = {}): CodexC `Relay-managed Codex thread ${prior.threadId} is already controlled by process ${prior.controllerPid}.` ); } - const session = await ensureCloudSession({ interactive: true }); - const cloud = new CloudLiveTeleportClient( - (requestPath, init) => session.client.fetch(requestPath, init), - session.client.snapshot().apiUrl - ); + const needsCloud = + liveTeleportEnabled || Boolean(prior && codexControllerStateMayOwnCloudResources(prior)); + const cloud = needsCloud + ? await (async () => { + const session = await ensureCloudSession({ interactive: true }); + return new CloudLiveTeleportClient( + (requestPath, init) => session.client.fetch(requestPath, init), + session.client.snapshot().apiUrl + ); + })() + : undefined; const controller = new CodexLiveController( { workspaceRoot, socketPath: paths.socketPath, + liveTeleportEnabled, ...(options.model ? { model: options.model } : {}), }, { - cloud, + ...(cloud ? { cloud } : {}), store, createAppServer: async () => StdioCodexAppServerSession.spawn({ @@ -363,9 +373,10 @@ function withDefaults(overrides: Partial = {}): CodexC } }, }), - // Production already probed before interactive Cloud login. Keeping - // the controller seam injectable lets restart/adversarial tests - // prove an unsupported local binary still fails closed. + // Enabled startup probes before interactive Cloud login. Disabled + // recovery only fences prior ownership and does not need Codex's + // environment-add schema. Keeping this seam injectable lets + // restart/adversarial tests prove enabled startup fails closed. probeCapability: async () => undefined, checkpointAndSeal, resumePersistedLocalMount, diff --git a/packages/cli/src/cli/lib/codex-live-controller.test.ts b/packages/cli/src/cli/lib/codex-live-controller.test.ts index 1b7d4b436..36da1162f 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.test.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.test.ts @@ -8,6 +8,7 @@ import type { LiveTeleportCloudClient, LiveTeleportLifecycleStatus } from '@agen import { CodexLiveController, FileCodexControllerStateStore, + isCodexLiveTeleportEnabled, type CodexControllerState, type CodexControllerStateStore, type CodexPersistedMountResumeProvider, @@ -204,18 +205,19 @@ function cloud(overrides: Partial = {}): LiveTeleportCl function createController( options: { store?: ReturnType; - cloud?: LiveTeleportCloudClient; + cloud?: LiveTeleportCloudClient | null; sessions?: CodexAppServerSession[]; probe?: () => Promise; checkpointAndSeal?: CodexWorkspaceSealProvider; resumePersistedLocalMount?: CodexPersistedMountResumeProvider; lifecycleDeadlineMs?: number; lifecyclePollIntervalMs?: number; + liveTeleportEnabled?: boolean; now?: () => Date; } = {} ) { const store = options.store ?? memoryStore(); - const cloudClient = options.cloud ?? cloud(); + const cloudClient = options.cloud === null ? undefined : (options.cloud ?? cloud()); const sessions = options.sessions ?? [appServer()]; const createAppServer = vi.fn(async () => { const session = sessions.shift(); @@ -240,13 +242,14 @@ function createController( { workspaceRoot: '/repo', socketPath: '/state/controller.sock', + liveTeleportEnabled: options.liveTeleportEnabled ?? true, ...(options.lifecycleDeadlineMs ? { lifecycleDeadlineMs: options.lifecycleDeadlineMs } : {}), ...(options.lifecyclePollIntervalMs ? { lifecyclePollIntervalMs: options.lifecyclePollIntervalMs } : {}), }, { - cloud: cloudClient, + ...(cloudClient ? { cloud: cloudClient } : {}), store, createAppServer, probeCapability: options.probe ?? (async () => undefined), @@ -332,6 +335,138 @@ function persistedLocal(overrides: Partial = {}): CodexCon } describe('CodexLiveController', () => { + it.each([ + ['unset', {}, false], + ['false', { RELAY_LIVE_SESSION_TELEPORT_ENABLED: 'false' }, false], + ['true', { RELAY_LIVE_SESSION_TELEPORT_ENABLED: 'true' }, true], + ['trimmed uppercase true', { RELAY_LIVE_SESSION_TELEPORT_ENABLED: ' TRUE ' }, true], + ['noncanonical truthy value', { RELAY_LIVE_SESSION_TELEPORT_ENABLED: '1' }, false], + ] as const)('treats the local teleport flag as %s', (_name, env, enabled) => { + expect(isCodexLiveTeleportEnabled(env)).toBe(enabled); + }); + + it('keeps a fresh managed Codex session entirely local when teleport is disabled', async () => { + const local = appServer(); + const cloudClient = cloud(); + const probe = vi.fn(async () => { + throw new Error('teleport capability probe must remain dormant'); + }); + const { controller, checkpointAndSeal } = createController({ + liveTeleportEnabled: false, + cloud: cloudClient, + sessions: [local], + probe, + }); + + await expect(controller.initialize()).resolves.toMatchObject({ + phase: 'local', + execution: 'local', + cloudLifecycle: 'none', + }); + await expect(controller.runTurn('local only')).resolves.toMatchObject({ turnId: 'turn-1' }); + + expect(probe).not.toHaveBeenCalled(); + expect(cloudClient.prewarm).not.toHaveBeenCalled(); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + expect(cloudClient.status).not.toHaveBeenCalled(); + expect(cloudClient.revoke).not.toHaveBeenCalled(); + expect(checkpointAndSeal).not.toHaveBeenCalled(); + expect(local.addEnvironment).not.toHaveBeenCalled(); + expect(local.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ execution: { kind: 'local', workspaceRoot: '/repo' } }) + ); + expect(() => + controller.requestTeleport({ requestId: 'disabled-request', expectedGeneration: 1 }) + ).toThrow('RELAY_LIVE_SESSION_TELEPORT_ENABLED=true'); + }); + + it('starts and runs locally while disabled without any Cloud client', async () => { + const local = appServer(); + const { controller } = createController({ + liveTeleportEnabled: false, + cloud: null, + sessions: [local], + }); + + await expect(controller.initialize()).resolves.toMatchObject({ execution: 'local' }); + await expect(controller.runTurn('no Cloud login')).resolves.toMatchObject({ turnId: 'turn-1' }); + expect(local.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ execution: { kind: 'local', workspaceRoot: '/repo' } }) + ); + }); + + it('fences earlier Cloud ownership on disabled startup, then stays local without rewarming', async () => { + const store = memoryStore(persistedRemote()); + const replacement = appServer(); + const cloudClient = cloud(); + const probe = vi.fn(async () => undefined); + const { controller, resumePersistedLocalMount } = createController({ + store, + cloud: cloudClient, + sessions: [replacement], + liveTeleportEnabled: false, + probe, + }); + + await expect(controller.initialize()).resolves.toMatchObject({ + generation: 8, + phase: 'local', + execution: 'local', + cloudLifecycle: 'none', + }); + await controller.runTurn('recovered local turn'); + + expect(probe).not.toHaveBeenCalled(); + expect(cloudClient.revoke).toHaveBeenCalledWith( + expect.objectContaining({ generation: 7, idempotencyKey: 'session-1:7:revoke' }) + ); + expect(resumePersistedLocalMount).toHaveBeenCalledOnce(); + expect(replacement.resumeThread).toHaveBeenCalledWith({ threadId: 'thread-1', cwd: '/repo' }); + expect(replacement.addEnvironment).not.toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ execution: { kind: 'local', workspaceRoot: '/repo' } }) + ); + expect(cloudClient.prewarm).not.toHaveBeenCalled(); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + expect(cloudClient.status).not.toHaveBeenCalled(); + expect(store.value?.pending).toBeUndefined(); + }); + + it('cancels a queued teleport on disabled startup without contacting Cloud', async () => { + const store = memoryStore( + persistedLocal({ + phase: 'teleport_pending', + pending: { requestId: 'queued-before-disable', expectedGeneration: 3 }, + }) + ); + const replacement = appServer(); + const cloudClient = cloud(); + const { controller } = createController({ + store, + cloud: cloudClient, + sessions: [replacement], + liveTeleportEnabled: false, + }); + + await expect(controller.initialize()).resolves.toMatchObject({ + generation: 3, + phase: 'local', + execution: 'local', + cloudLifecycle: 'none', + }); + await controller.runTurn('still local'); + + expect(store.value?.pending).toBeUndefined(); + expect(cloudClient.prewarm).not.toHaveBeenCalled(); + expect(cloudClient.acquire).not.toHaveBeenCalled(); + expect(cloudClient.status).not.toHaveBeenCalled(); + expect(cloudClient.revoke).not.toHaveBeenCalled(); + expect(replacement.addEnvironment).not.toHaveBeenCalled(); + expect(replacement.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ execution: { kind: 'local', workspaceRoot: '/repo' } }) + ); + }); + it('fails before starting an app-server when the local experimental capability is unsupported', async () => { const { controller, createAppServer } = createController({ probe: async () => { @@ -345,7 +480,10 @@ describe('CodexLiveController', () => { it('prewarms without stopping or sealing the active local mount', async () => { const cloudClient = cloud(); - const { controller, checkpointAndSeal } = createController({ cloud: cloudClient }); + const { controller, checkpointAndSeal } = createController({ + cloud: cloudClient, + liveTeleportEnabled: true, + }); await controller.initialize(); diff --git a/packages/cli/src/cli/lib/codex-live-controller.ts b/packages/cli/src/cli/lib/codex-live-controller.ts index e85a69e59..0af38b438 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.ts @@ -41,6 +41,17 @@ export type CodexTeleportRequest = { expectedGeneration: number; }; +export const RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV = 'RELAY_LIVE_SESSION_TELEPORT_ENABLED'; + +/** + * Live execution teleport is a dark-launched, local-only capability. Only the + * explicit string `true` enables it; unset, false, and every unknown value + * fail closed so merging support code cannot activate remote execution. + */ +export function isCodexLiveTeleportEnabled(env: NodeJS.ProcessEnv = process.env): boolean { + return env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV]?.trim().toLowerCase() === 'true'; +} + type CodexRecoveredOutcome = { sessionId: string; threadId: string; @@ -197,7 +208,7 @@ export type CodexPersistedMountResumeProvider = ( ) => Promise; export type CodexLiveControllerDependencies = { - cloud: LiveTeleportCloudClient; + cloud?: LiveTeleportCloudClient; store: CodexControllerStateStore; createAppServer: () => Promise; probeCapability: () => Promise; @@ -214,6 +225,8 @@ export type CodexLiveControllerOptions = { workspaceRoot: string; socketPath: string; model?: string; + /** Defaults off. Enabling must always be an explicit local operator action. */ + liveTeleportEnabled?: boolean; lifecycleDeadlineMs?: number; lifecyclePollIntervalMs?: number; }; @@ -432,6 +445,11 @@ function inferredCloudLifecycle(value: Record): CodexCloudLifec return 'none'; } +/** True when startup must contact Cloud to fence an earlier enabled process. */ +export function codexControllerStateMayOwnCloudResources(state: CodexControllerState): boolean { + return inferredCloudLifecycle(state as unknown as Record) !== 'none'; +} + function validatePersistedState(value: unknown): CodexControllerState { const state = record(value); if ( @@ -566,19 +584,26 @@ export class CodexLiveController { async initialize(): Promise { this.recoveredEvidenceTaken = undefined; - await this.deps.probeCapability(); const persisted = this.deps.store.read(); if (persisted && path.resolve(persisted.workspaceRoot) !== path.resolve(this.options.workspaceRoot)) { throw new Error( `The persisted managed Codex thread belongs to ${persisted.workspaceRoot}, not ${this.options.workspaceRoot}.` ); } + if (this.liveTeleportEnabled() || (persisted && this.cloudMayOwnResources(persisted))) { + this.requireCloud(); + } + if (this.liveTeleportEnabled()) await this.deps.probeCapability(); if (persisted) { const mustFenceCloud = this.cloudMayOwnResources(persisted); const mustRecoverMount = Boolean(persisted.source || persisted.mountRestore || persisted.remote); this.state = { ...persisted, + // Disabling the switch cancels dormant intent before the new process + // can observe a turn boundary. Existing Cloud ownership is still + // fenced below before local execution is allowed to resume. + pending: this.liveTeleportEnabled() ? persisted.pending : undefined, controllerPid: this.deps.pid, socketPath: this.options.socketPath, workspaceRoot: this.options.workspaceRoot, @@ -644,7 +669,7 @@ export class CodexLiveController { const state = this.requireState(); state.generation = persisted.generation + (mustFenceCloud || mustRecoverMount ? 1 : 0); - if (state.pending) { + if (state.pending && this.liveTeleportEnabled()) { // A queued teleport is an accepted durable intent, independent from // recovered output delivery. Fencing consumes the old Cloud identity, // so rebind the request in the same write that publishes the new @@ -685,7 +710,7 @@ export class CodexLiveController { this.persist(); } - if (!this.requireState().recoveredOutcome) this.schedulePrewarm(); + if (this.liveTeleportEnabled() && !this.requireState().recoveredOutcome) this.schedulePrewarm(); return this.status(); } @@ -740,6 +765,11 @@ export class CodexLiveController { } requestTeleport(request: CodexTeleportRequest): PublicCodexControllerStatus { + if (!this.liveTeleportEnabled()) { + throw new Error( + `Codex live session teleport is disabled. Set ${RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV}=true before starting \`relay codex run\`.` + ); + } const state = this.requireState(); this.assertNoUnacknowledgedOutcome('queue a teleport'); if (request.expectedGeneration !== state.generation) { @@ -780,9 +810,12 @@ export class CodexLiveController { this.persist(); try { if (pendingAtBoundary) await this.applyPendingTeleport(pendingAtBoundary); - if (this.requireState().phase === 'remote') await this.ensureRemoteActiveOrRecover(); + if (this.liveTeleportEnabled() && this.requireState().phase === 'remote') { + await this.ensureRemoteActiveOrRecover(); + } const execution = - this.requireState().phase === 'remote' || this.requireState().phase === 'verifying' + this.liveTeleportEnabled() && + (this.requireState().phase === 'remote' || this.requireState().phase === 'verifying') ? 'remote' : 'local'; state.inFlightTurn = { @@ -806,7 +839,8 @@ export class CodexLiveController { private async executeTurn(text: string, clientUserMessageId: string): Promise { const state = this.requireState(); const phase = state.phase as CodexControllerPhase; - const remote = phase === 'verifying' || phase === 'remote' ? state.remote : undefined; + const remote = + this.liveTeleportEnabled() && (phase === 'verifying' || phase === 'remote') ? state.remote : undefined; try { const result = await this.requireAppServer().runTurn({ threadId: state.threadId, @@ -967,6 +1001,13 @@ export class CodexLiveController { private async applyPendingTeleport(pendingAtBoundary: CodexTeleportRequest): Promise { const state = this.requireState(); + if (!this.liveTeleportEnabled()) { + state.pending = undefined; + state.phase = 'local'; + state.updatedAt = this.timestamp(); + this.persist(); + return; + } const pending = state.pending; if (!pending || pending.requestId !== pendingAtBoundary.requestId) return; if (pending.expectedGeneration !== state.generation) { @@ -1014,7 +1055,7 @@ export class CodexLiveController { this.persist(); const environment = await this.withAbortableLifecycleDeadline( (signal) => - this.deps.cloud.acquire({ + this.requireCloud().acquire({ sessionId: state.sessionId, threadId: state.threadId, generation: state.generation, @@ -1170,7 +1211,7 @@ export class CodexLiveController { await this.withAbortableLifecycleDeadline(async (signal) => { let lastError: unknown; try { - const revoked = await this.deps.cloud.revoke({ + const revoked = await this.requireCloud().revoke({ sessionId: state.sessionId, generation: state.generation, idempotencyKey: `${state.sessionId}:${state.generation}:revoke`, @@ -1189,7 +1230,7 @@ export class CodexLiveController { } for (let attempt = 0; attempt < this.lifecycleAttempts(); attempt += 1) { try { - const status = await this.deps.cloud.status({ + const status = await this.requireCloud().status({ sessionId: state.sessionId, generation: state.generation, signal, @@ -1213,7 +1254,7 @@ export class CodexLiveController { const attempts = this.lifecycleAttempts(); await this.withAbortableLifecycleDeadline(async (signal) => { for (let attempt = 0; attempt < attempts; attempt += 1) { - const status = await this.deps.cloud.status({ + const status = await this.requireCloud().status({ sessionId: state.sessionId, generation: state.generation, prewarmId, @@ -1256,7 +1297,7 @@ export class CodexLiveController { } private schedulePrewarm(): void { - if (this.prewarmPromise || this.closing) return; + if (!this.liveTeleportEnabled() || this.prewarmPromise || this.closing) return; const state = this.requireState(); // Recovered output is an independent durable delivery queue. Do not start // another lifecycle while it is unacknowledged, and do not duplicate an @@ -1280,7 +1321,7 @@ export class CodexLiveController { try { const prewarm = await this.withAbortableLifecycleDeadline( (signal) => - this.deps.cloud.prewarm({ + this.requireCloud().prewarm({ sessionId: state.sessionId, generation, workspaceRoot: '/', @@ -1346,7 +1387,7 @@ export class CodexLiveController { try { const status = await this.withAbortableLifecycleDeadline( (signal) => - this.deps.cloud.status({ + this.requireCloud().status({ sessionId: state.sessionId, generation: state.generation, signal, @@ -1481,7 +1522,18 @@ export class CodexLiveController { } private cloudMayOwnResources(state = this.requireState()): boolean { - return inferredCloudLifecycle(state as unknown as Record) !== 'none'; + return codexControllerStateMayOwnCloudResources(state); + } + + private liveTeleportEnabled(): boolean { + return this.options.liveTeleportEnabled === true; + } + + private requireCloud(): LiveTeleportCloudClient { + if (!this.deps.cloud) { + throw new Error('Codex live session teleport requires an authenticated Cloud client.'); + } + return this.deps.cloud; } private assertNoUnacknowledgedOutcome(operation: string): void { From f86ea22874427095de7941df0567e04e6c4e7049 Mon Sep 17 00:00:00 2001 From: Proactive Runtime Bot Date: Mon, 24 Aug 2026 00:05:31 +0200 Subject: [PATCH 13/16] fix(codex): trust only ambient teleport opt-in --- CHANGELOG.md | 2 +- packages/cli/README.md | 11 +- packages/cli/src/cli/bootstrap.test.ts | 24 ++- packages/cli/src/cli/bootstrap.ts | 30 +++- .../src/cli/commands/codex-defaults.test.ts | 138 ++++++++++++++++++ packages/cli/src/cli/commands/codex.test.ts | 47 ++++++ packages/cli/src/cli/commands/codex.ts | 24 ++- .../src/cli/lib/codex-live-controller.test.ts | 42 ++++-- .../cli/src/cli/lib/codex-live-controller.ts | 52 +++++-- 9 files changed, 335 insertions(+), 35 deletions(-) create mode 100644 packages/cli/src/cli/commands/codex-defaults.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index b5e409de7..34775a0a2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- `relay codex run` starts a Relay-managed local Codex app-server/thread. Live session teleport is dark-launched behind the default-off local `RELAY_LIVE_SESSION_TELEPORT_ENABLED=true` switch; when enabled, `relay codex teleport` seals its active Relayfile poll mount at the next turn boundary, binds exact Relayfile destination verification to the source checkpoint, requires a 40-minute Cloud turn lease, and requires confirmed fencing plus ownership handback and mount readiness before local rollback. After notification loss, Relay never replays model execution: it durably redelivers reconciled output at least once, so an already-rendered live prefix can repeat, and it transparently runs the same prompt locally only after a confirmed pre-submission cutover failure. +- `relay codex run` starts a Relay-managed local Codex app-server/thread. Live session teleport is dark-launched behind the default-off local `RELAY_LIVE_SESSION_TELEPORT_ENABLED=true` switch, captured strictly from the ambient process environment before dotenv loading and reported by `relay codex status`; when enabled, `relay codex teleport` seals its active Relayfile poll mount at the next turn boundary, binds exact Relayfile destination verification to the source checkpoint, requires a 40-minute Cloud turn lease, and requires confirmed fencing plus ownership handback and mount readiness before local rollback. After notification loss, Relay never replays model execution: it durably redelivers reconciled output at least once, so an already-rendered live prefix can repeat, and it transparently runs the same prompt locally only after a confirmed pre-submission cutover failure. ## [Unreleased - Patch] diff --git a/packages/cli/README.md b/packages/cli/README.md index a40cc76dc..51d1494eb 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -31,10 +31,13 @@ RELAY_LIVE_SESSION_TELEPORT_ENABLED=true relay codex run relay codex teleport ``` -Unset, `false`, `1`, and all other values keep new execution local: a fresh or -already-local controller does not authenticate to Cloud, prewarm, acquire, or -route a turn remotely. The `teleport`, `rollback`, and `status` commands remain -discoverable, but a disabled controller rejects teleport requests. +Unset, `false`, `TRUE`, whitespace-padded values, `1`, and all other values keep +new execution local: a fresh or already-local controller does not authenticate +to Cloud, prewarm, acquire, or route a turn remotely. The opt-in must come from +the ambient process environment; a cwd `.env` file cannot enable it. The +`teleport`, `rollback`, and `status` commands remain discoverable, but a disabled +controller rejects teleport requests. `relay codex status` reports the effective +startup switch and why it is enabled or disabled. The flag is read when `relay codex run` starts. To roll back the capability, stop the controller and restart it with the variable unset or set to `false`. diff --git a/packages/cli/src/cli/bootstrap.test.ts b/packages/cli/src/cli/bootstrap.test.ts index 4a93f2670..18b9b8091 100644 --- a/packages/cli/src/cli/bootstrap.test.ts +++ b/packages/cli/src/cli/bootstrap.test.ts @@ -5,7 +5,8 @@ import path from 'node:path'; import { Command } from 'commander'; import { afterEach, describe, expect, it, vi } from 'vitest'; -import { createProgram, propagateTelemetryContextToChildren } from './bootstrap.js'; +import { createProgram, loadCliEnvironment, propagateTelemetryContextToChildren } from './bootstrap.js'; +import { RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV } from './lib/codex-live-controller.js'; const expectedLeafCommands = [ // node broker + agent group (local is a hidden alias, filtered out below) @@ -203,6 +204,27 @@ describe('createProgram output redaction', () => { }); describe('bootstrap CLI', () => { + it('does not let a cwd .env enable live teleport after the trusted startup capture', () => { + const originalCwd = process.cwd(); + const originalValue = process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV]; + const cwd = fs.mkdtempSync(path.join(os.tmpdir(), 'relay-codex-dotenv-')); + fs.writeFileSync(path.join(cwd, '.env'), `${RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV}=true\n`); + delete process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV]; + process.chdir(cwd); + + try { + const startupSwitch = loadCliEnvironment(); + + expect(startupSwitch).toEqual({ enabled: false, reason: 'ambient-unset' }); + expect(process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV]).toBe('true'); + } finally { + process.chdir(originalCwd); + if (originalValue === undefined) delete process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV]; + else process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV] = originalValue; + fs.rmSync(cwd, { recursive: true, force: true }); + } + }); + it('uses the expected program name', () => { const program = createProgram(); expect(program.name()).toBe('agent-relay'); diff --git a/packages/cli/src/cli/bootstrap.ts b/packages/cli/src/cli/bootstrap.ts index ebd4782ce..de8554922 100644 --- a/packages/cli/src/cli/bootstrap.ts +++ b/packages/cli/src/cli/bootstrap.ts @@ -51,8 +51,24 @@ import { registerFleetCommands } from './commands/fleet.js'; import { registerSkillsCommands } from './commands/skills.js'; import { registerSessionCommands } from './commands/session.js'; import { registerCodexCommands } from './commands/codex.js'; +import { + DEFAULT_CODEX_LIVE_TELEPORT_STARTUP_SWITCH, + RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV, + resolveCodexLiveTeleportStartupSwitch, + type CodexLiveTeleportStartupSwitch, +} from './lib/codex-live-controller.js'; -dotenvConfig({ quiet: true }); +/** + * Capture the trusted local teleport switch before cwd dotenv loading can + * mutate process.env. Production calls this once, at the start of runCli. + */ +export function loadCliEnvironment(): CodexLiveTeleportStartupSwitch { + const liveTeleportStartupSwitch = resolveCodexLiveTeleportStartupSwitch( + process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV] + ); + dotenvConfig({ quiet: true }); + return liveTeleportStartupSwitch; +} const __filename = fileURLToPath(import.meta.url); const __dirname = path.dirname(__filename); @@ -374,7 +390,9 @@ function installExitHooks(): void { }); } -export function createProgram(options: { name?: string } = {}): Command { +export function createProgram( + options: { name?: string; liveTeleportStartupSwitch?: CodexLiveTeleportStartupSwitch } = {} +): Command { const program = new Command(); // Commander echoes offending tokens verbatim (`error: unknown option @@ -424,7 +442,10 @@ export function createProgram(options: { name?: string } = {}): Command { registerCapabilitiesCommands(program); registerSkillsCommands(program); registerSessionCommands(program); - registerCodexCommands(program); + registerCodexCommands(program, { + liveTeleportStartupSwitch: + options.liveTeleportStartupSwitch ?? DEFAULT_CODEX_LIVE_TELEPORT_STARTUP_SWITCH, + }); program .command('mcp') @@ -473,6 +494,7 @@ function collectTopLevelVerbs(program: Command): Set { } export async function runCli(argv: string[] = process.argv): Promise { + const liveTeleportStartupSwitch = loadCliEnvironment(); assertSupportedNodeVersion(); ensureWebSocketGlobal(); maybeRunUpdateCheck(VERSION, argv); @@ -489,7 +511,7 @@ export async function runCli(argv: string[] = process.argv): Promise { }); } - const program = createProgram({ name: resolveProgramName(argv) }); + const program = createProgram({ name: resolveProgramName(argv), liveTeleportStartupSwitch }); installTelemetryHooks(program); installExitHooks(); diff --git a/packages/cli/src/cli/commands/codex-defaults.test.ts b/packages/cli/src/cli/commands/codex-defaults.test.ts new file mode 100644 index 000000000..34460d8ed --- /dev/null +++ b/packages/cli/src/cli/commands/codex-defaults.test.ts @@ -0,0 +1,138 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import { Readable } from 'node:stream'; + +import { Command } from 'commander'; +import { afterEach, describe, expect, it, vi } from 'vitest'; + +const mocks = vi.hoisted(() => { + const cloudClient = { + prewarm: vi.fn(), + status: vi.fn(), + acquire: vi.fn(), + revoke: vi.fn(), + }; + const appServer = { + initialize: vi.fn(async () => undefined), + startThread: vi.fn(async () => 'thread-local-1'), + resumeThread: vi.fn(async () => undefined), + addEnvironment: vi.fn(async () => undefined), + environmentStatus: vi.fn(async () => ({ status: 'ready' })), + runTurn: vi.fn(async () => { + const turn = { + id: 'turn-local-1', + status: 'completed', + itemsView: 'full', + items: [{ id: 'answer-1', type: 'agentMessage', text: 'local answer' }], + }; + return { + turnId: turn.id, + response: { turn }, + completed: { method: 'turn/completed', params: { threadId: 'thread-local-1', turn } }, + }; + }), + turnOutcome: vi.fn(async () => ({ status: 'absent' as const })), + close: vi.fn(async () => undefined), + }; + const checkpointAndSeal = vi.fn(); + return { + appServer, + checkpointAndSeal, + cloudClient, + cloudConstructor: vi.fn(function () { + return cloudClient; + }), + ensureCloudSession: vi.fn(), + probeCapability: vi.fn(), + resumePersistedLocalMount: vi.fn(async () => undefined), + }; +}); + +vi.mock('@agent-relay/cloud', async (importOriginal) => ({ + ...(await importOriginal()), + CloudLiveTeleportClient: mocks.cloudConstructor, + ensureCloudSession: mocks.ensureCloudSession, +})); + +vi.mock('../lib/codex-app-server.js', async (importOriginal) => ({ + ...(await importOriginal()), + probeCodexEnvironmentCapability: mocks.probeCapability, + StdioCodexAppServerSession: { spawn: vi.fn(async () => mocks.appServer) }, +})); + +vi.mock('../lib/codex-relayfile-seal.js', async (importOriginal) => ({ + ...(await importOriginal()), + createRelayfileSealLifecycle: () => ({ + checkpointAndSeal: mocks.checkpointAndSeal, + resumePersistedLocalMount: mocks.resumePersistedLocalMount, + }), +})); + +import { registerCodexCommands } from './codex.js'; +import { RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV } from '../lib/codex-live-controller.js'; + +describe('default Relay-managed Codex command wiring', () => { + const temporaryRoots: string[] = []; + const originalFlag = process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV]; + const originalStateDir = process.env.AGENT_RELAY_STATE_DIR; + + afterEach(() => { + vi.clearAllMocks(); + if (originalFlag === undefined) delete process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV]; + else process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV] = originalFlag; + if (originalStateDir === undefined) delete process.env.AGENT_RELAY_STATE_DIR; + else process.env.AGENT_RELAY_STATE_DIR = originalStateDir; + for (const root of temporaryRoots.splice(0)) fs.rmSync(root, { recursive: true, force: true }); + }); + + it('keeps a fresh durable run local when the captured startup switch is disabled', async () => { + // Unix-domain socket paths are short on macOS; /tmp keeps the production + // control socket below that limit while still exercising the real server. + const root = fs.mkdtempSync(path.join('/tmp', 'relay-codex-')); + temporaryRoots.push(root); + const workspace = path.join(root, 'workspace'); + const stateDir = path.join(root, 'state'); + fs.mkdirSync(workspace); + const workspaceRoot = fs.realpathSync(workspace); + process.env.AGENT_RELAY_STATE_DIR = stateDir; + + // Simulate dotenv or later bootstrap code mutating process.env after the + // trusted ambient value was captured. Production wiring must use only the + // immutable switch passed into registration. + process.env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV] = 'true'; + const program = new Command().exitOverride(); + registerCodexCommands(program, { + liveTeleportStartupSwitch: { enabled: false, reason: 'ambient-unset' }, + cwd: () => workspace, + input: Readable.from([]), + log: vi.fn(), + }); + const stderr = vi.spyOn(process.stderr, 'write').mockImplementation(() => true); + + try { + await program.parseAsync(['node', 'relay', 'codex', 'run', 'stay local']); + } finally { + stderr.mockRestore(); + } + + expect(mocks.ensureCloudSession).not.toHaveBeenCalled(); + expect(mocks.cloudConstructor).not.toHaveBeenCalled(); + expect(mocks.probeCapability).not.toHaveBeenCalled(); + expect(mocks.cloudClient.prewarm).not.toHaveBeenCalled(); + expect(mocks.cloudClient.status).not.toHaveBeenCalled(); + expect(mocks.cloudClient.acquire).not.toHaveBeenCalled(); + expect(mocks.cloudClient.revoke).not.toHaveBeenCalled(); + expect(mocks.checkpointAndSeal).not.toHaveBeenCalled(); + expect(mocks.resumePersistedLocalMount).not.toHaveBeenCalled(); + expect(mocks.appServer.initialize).toHaveBeenCalledOnce(); + expect(mocks.appServer.startThread).toHaveBeenCalledWith({ cwd: workspaceRoot }); + expect(mocks.appServer.addEnvironment).not.toHaveBeenCalled(); + expect(mocks.appServer.runTurn).toHaveBeenCalledWith( + expect.objectContaining({ + text: 'stay local', + execution: { kind: 'local', workspaceRoot }, + }) + ); + expect(fs.existsSync(path.join(stateDir, 'codex-live', 'active.json'))).toBe(true); + }); +}); diff --git a/packages/cli/src/cli/commands/codex.test.ts b/packages/cli/src/cli/commands/codex.test.ts index 2b0259fe8..31710206e 100644 --- a/packages/cli/src/cli/commands/codex.test.ts +++ b/packages/cli/src/cli/commands/codex.test.ts @@ -376,6 +376,7 @@ describe('registerCodexCommands', () => { updatedAt: '2026-08-23T12:00:00.000Z', controller: 'local' as const, execution: 'local' as const, + liveTeleport: { enabled: true, reason: 'ambient-exact-true' as const }, workspaceSource: 'relayfile-checkpoint-seal' as const, }, })); @@ -408,4 +409,50 @@ describe('registerCodexCommands', () => { ); expect(sendControl).not.toHaveBeenCalled(); }); + + it('prints the immutable startup switch and reason in human and JSON status', async () => { + const status = { + version: 1 as const, + sessionId: 'session-1', + threadId: 'thread-1', + workspaceRoot: '/repo', + generation: 4, + phase: 'local' as const, + controllerPid: 10, + socketPath: '/state/controller.sock', + turnActive: false, + cloudLifecycle: 'none' as const, + updatedAt: '2026-08-23T12:00:00.000Z', + controller: 'local' as const, + execution: 'local' as const, + liveTeleport: { enabled: false, reason: 'ambient-value-not-exact-true' as const }, + workspaceSource: 'unavailable' as const, + }; + const sendControl = vi.fn(async () => ({ ok: true as const, status })); + const humanLog = vi.fn(); + const humanProgram = new Command().exitOverride(); + registerCodexCommands(humanProgram, { + readState: () => ({ generation: 4 }) as never, + sendControl, + log: humanLog, + }); + + await humanProgram.parseAsync(['node', 'relay', 'codex', 'status']); + + expect(humanLog).toHaveBeenCalledWith('Live teleport: disabled (ambient-value-not-exact-true)'); + + const jsonLog = vi.fn(); + const jsonProgram = new Command().exitOverride(); + registerCodexCommands(jsonProgram, { + readState: () => ({ generation: 4 }) as never, + sendControl, + log: jsonLog, + }); + + await jsonProgram.parseAsync(['node', 'relay', 'codex', 'status', '--json']); + + expect(JSON.parse(String(jsonLog.mock.calls[0]?.[0]))).toMatchObject({ + liveTeleport: { enabled: false, reason: 'ambient-value-not-exact-true' }, + }); + }); }); diff --git a/packages/cli/src/cli/commands/codex.ts b/packages/cli/src/cli/commands/codex.ts index a9a27d6f7..3387ac022 100644 --- a/packages/cli/src/cli/commands/codex.ts +++ b/packages/cli/src/cli/commands/codex.ts @@ -20,8 +20,9 @@ import { CodexTurnRecordedError, FileCodexControllerStateStore, codexControllerStateMayOwnCloudResources, - isCodexLiveTeleportEnabled, + DEFAULT_CODEX_LIVE_TELEPORT_STARTUP_SWITCH, type CodexControllerState, + type CodexLiveTeleportStartupSwitch, type CodexPersistedMountResumeProvider, type CodexWorkspaceSealProvider, type PublicCodexControllerStatus, @@ -42,6 +43,7 @@ type ControlRequest = type ControlResponse = { ok: true; status: PublicCodexControllerStatus } | { ok: false; error: string }; export type CodexAsyncWriter = (message: string) => Promise; +type CodexCommandInput = NodeJS.ReadableStream & { isTTY?: boolean }; type NodeCallbackWritable = { write(message: string, callback: (error?: Error | null) => void): boolean; @@ -92,12 +94,15 @@ export async function withCodexControllerShutdown( export interface CodexCommandDependencies { runManaged(options: { cwd: string; model?: string; prompt?: string; json?: boolean }): Promise; + /** Immutable value captured before bootstrap loads dotenv. */ + liveTeleportStartupSwitch: CodexLiveTeleportStartupSwitch; checkpointAndSeal: CodexWorkspaceSealProvider; resumePersistedLocalMount: CodexPersistedMountResumeProvider; readState(): CodexControllerState | null; sendControl(request: ControlRequest): Promise; requestId(): string; cwd(): string; + input: CodexCommandInput; log(message: string): void; } @@ -317,6 +322,10 @@ async function sendSocketControl(socketPath: string, request: ControlRequest): P function withDefaults(overrides: Partial = {}): CodexCommandDependencies { const paths = codexControllerPaths(); const relayfileLifecycle = createRelayfileSealLifecycle(); + const liveTeleportStartupSwitch = Object.freeze({ + ...(overrides.liveTeleportStartupSwitch ?? DEFAULT_CODEX_LIVE_TELEPORT_STARTUP_SWITCH), + }); + const inputStream = overrides.input ?? process.stdin; const checkpointAndSeal = overrides.checkpointAndSeal ?? relayfileLifecycle.checkpointAndSeal; const resumePersistedLocalMount = overrides.resumePersistedLocalMount ?? relayfileLifecycle.resumePersistedLocalMount; @@ -325,7 +334,7 @@ function withDefaults(overrides: Partial = {}): CodexC overrides.runManaged ?? (async (options) => { const workspaceRoot = fs.realpathSync(path.resolve(options.cwd)); - const liveTeleportEnabled = isCodexLiveTeleportEnabled(); + const liveTeleportEnabled = liveTeleportStartupSwitch.enabled; if (liveTeleportEnabled) await probeCodexEnvironmentCapability('codex'); fs.mkdirSync(paths.directory, { recursive: true, mode: 0o700 }); const store = new FileCodexControllerStateStore(paths.statePath); @@ -351,7 +360,7 @@ function withDefaults(overrides: Partial = {}): CodexC { workspaceRoot, socketPath: paths.socketPath, - liveTeleportEnabled, + liveTeleportStartupSwitch, ...(options.model ? { model: options.model } : {}), }, { @@ -424,8 +433,8 @@ function withDefaults(overrides: Partial = {}): CodexC }); } const input = readline.createInterface({ - input: process.stdin, - terminal: Boolean(process.stdin.isTTY), + input: inputStream, + terminal: Boolean(inputStream.isTTY), }); for await (const line of input) { if (!line.trim()) continue; @@ -447,6 +456,7 @@ function withDefaults(overrides: Partial = {}): CodexC } } }), + liveTeleportStartupSwitch, checkpointAndSeal, resumePersistedLocalMount, readState: @@ -461,6 +471,7 @@ function withDefaults(overrides: Partial = {}): CodexC sendControl: overrides.sendControl ?? ((request) => sendSocketControl(paths.socketPath, request)), requestId: overrides.requestId ?? randomUUID, cwd: overrides.cwd ?? (() => process.cwd()), + input: inputStream, log: overrides.log ?? ((message) => console.log(message)), }; } @@ -548,6 +559,9 @@ export function registerCodexCommands( else { deps.log(`Execution: ${status.execution}`); deps.log('Controller: local (keep this process and laptop running)'); + deps.log( + `Live teleport: ${status.liveTeleport.enabled ? 'enabled' : 'disabled'} (${status.liveTeleport.reason})` + ); deps.log(`Thread: ${status.threadId}`); deps.log(`Generation: ${status.generation}`); } diff --git a/packages/cli/src/cli/lib/codex-live-controller.test.ts b/packages/cli/src/cli/lib/codex-live-controller.test.ts index 36da1162f..724ad15b6 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.test.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.test.ts @@ -8,9 +8,10 @@ import type { LiveTeleportCloudClient, LiveTeleportLifecycleStatus } from '@agen import { CodexLiveController, FileCodexControllerStateStore, - isCodexLiveTeleportEnabled, + resolveCodexLiveTeleportStartupSwitch, type CodexControllerState, type CodexControllerStateStore, + type CodexLiveTeleportStartupSwitch, type CodexPersistedMountResumeProvider, type CodexWorkspaceSealHandle, type CodexWorkspaceSealProvider, @@ -213,6 +214,7 @@ function createController( lifecycleDeadlineMs?: number; lifecyclePollIntervalMs?: number; liveTeleportEnabled?: boolean; + liveTeleportStartupSwitch?: CodexLiveTeleportStartupSwitch; now?: () => Date; } = {} ) { @@ -242,7 +244,11 @@ function createController( { workspaceRoot: '/repo', socketPath: '/state/controller.sock', - liveTeleportEnabled: options.liveTeleportEnabled ?? true, + liveTeleportStartupSwitch: + options.liveTeleportStartupSwitch ?? + (options.liveTeleportEnabled === false + ? { enabled: false, reason: 'ambient-unset' } + : { enabled: true, reason: 'ambient-exact-true' }), ...(options.lifecycleDeadlineMs ? { lifecycleDeadlineMs: options.lifecycleDeadlineMs } : {}), ...(options.lifecyclePollIntervalMs ? { lifecyclePollIntervalMs: options.lifecyclePollIntervalMs } @@ -336,13 +342,16 @@ function persistedLocal(overrides: Partial = {}): CodexCon describe('CodexLiveController', () => { it.each([ - ['unset', {}, false], - ['false', { RELAY_LIVE_SESSION_TELEPORT_ENABLED: 'false' }, false], - ['true', { RELAY_LIVE_SESSION_TELEPORT_ENABLED: 'true' }, true], - ['trimmed uppercase true', { RELAY_LIVE_SESSION_TELEPORT_ENABLED: ' TRUE ' }, true], - ['noncanonical truthy value', { RELAY_LIVE_SESSION_TELEPORT_ENABLED: '1' }, false], - ] as const)('treats the local teleport flag as %s', (_name, env, enabled) => { - expect(isCodexLiveTeleportEnabled(env)).toBe(enabled); + ['unset', undefined, false, 'ambient-unset'], + ['false', 'false', false, 'ambient-value-not-exact-true'], + ['exact true', 'true', true, 'ambient-exact-true'], + ['uppercase true', 'TRUE', false, 'ambient-value-not-exact-true'], + ['leading whitespace', ' true', false, 'ambient-value-not-exact-true'], + ['trailing whitespace', 'true ', false, 'ambient-value-not-exact-true'], + ['noncanonical truthy value', '1', false, 'ambient-value-not-exact-true'], + ['empty', '', false, 'ambient-value-not-exact-true'], + ] as const)('treats the ambient teleport flag as %s', (_name, raw, enabled, reason) => { + expect(resolveCodexLiveTeleportStartupSwitch(raw)).toEqual({ enabled, reason }); }); it('keeps a fresh managed Codex session entirely local when teleport is disabled', async () => { @@ -362,6 +371,7 @@ describe('CodexLiveController', () => { phase: 'local', execution: 'local', cloudLifecycle: 'none', + liveTeleport: { enabled: false, reason: 'ambient-unset' }, }); await expect(controller.runTurn('local only')).resolves.toMatchObject({ turnId: 'turn-1' }); @@ -413,6 +423,7 @@ describe('CodexLiveController', () => { phase: 'local', execution: 'local', cloudLifecycle: 'none', + liveTeleport: { enabled: false, reason: 'ambient-unset' }, }); await controller.runTurn('recovered local turn'); @@ -2111,12 +2122,23 @@ describe('CodexLiveController', () => { revoke: vi.fn(async () => Promise.reject(new Error('timeout'))), status: vi.fn(async (input) => lifecycle(input, 'ready')), }); - const { controller, createAppServer } = createController({ store, cloud: cloudClient }); + const { controller, createAppServer, resumePersistedLocalMount } = createController({ + store, + cloud: cloudClient, + liveTeleportEnabled: false, + }); await expect(controller.initialize()).rejects.toThrow('CLOUD_FENCE_UNCONFIRMED_ON_RESTART'); expect(createAppServer).not.toHaveBeenCalled(); + expect(resumePersistedLocalMount).not.toHaveBeenCalled(); + expect(cloudClient.prewarm).not.toHaveBeenCalled(); + expect(cloudClient.acquire).not.toHaveBeenCalled(); expect(store.value).toMatchObject({ phase: 'fenced', generation: 7 }); + expect(controller.status()).toMatchObject({ + execution: 'fenced', + liveTeleport: { enabled: false, reason: 'ambient-unset' }, + }); }); it('resumes locally only after Cloud authoritatively confirms the persisted lease expired', async () => { diff --git a/packages/cli/src/cli/lib/codex-live-controller.ts b/packages/cli/src/cli/lib/codex-live-controller.ts index 0af38b438..7614e435d 100644 --- a/packages/cli/src/cli/lib/codex-live-controller.ts +++ b/packages/cli/src/cli/lib/codex-live-controller.ts @@ -43,13 +43,35 @@ export type CodexTeleportRequest = { export const RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV = 'RELAY_LIVE_SESSION_TELEPORT_ENABLED'; +export type CodexLiveTeleportStartupReason = + | 'ambient-exact-true' + | 'ambient-unset' + | 'ambient-value-not-exact-true'; + +export type CodexLiveTeleportStartupSwitch = Readonly<{ + enabled: boolean; + reason: CodexLiveTeleportStartupReason; +}>; + +export const DEFAULT_CODEX_LIVE_TELEPORT_STARTUP_SWITCH: CodexLiveTeleportStartupSwitch = Object.freeze({ + enabled: false, + reason: 'ambient-unset', +}); + /** * Live execution teleport is a dark-launched, local-only capability. Only the - * explicit string `true` enables it; unset, false, and every unknown value - * fail closed so merging support code cannot activate remote execution. + * exact, unmodified string `true` enables it. The caller must pass the value + * captured from the ambient process environment before dotenv is loaded; + * unset, whitespace/case variants, and every unknown value fail closed. */ -export function isCodexLiveTeleportEnabled(env: NodeJS.ProcessEnv = process.env): boolean { - return env[RELAY_LIVE_SESSION_TELEPORT_ENABLED_ENV]?.trim().toLowerCase() === 'true'; +export function resolveCodexLiveTeleportStartupSwitch( + ambientRawValue: string | undefined +): CodexLiveTeleportStartupSwitch { + if (ambientRawValue === 'true') { + return Object.freeze({ enabled: true, reason: 'ambient-exact-true' }); + } + if (ambientRawValue === undefined) return DEFAULT_CODEX_LIVE_TELEPORT_STARTUP_SWITCH; + return Object.freeze({ enabled: false, reason: 'ambient-value-not-exact-true' }); } type CodexRecoveredOutcome = { @@ -101,6 +123,8 @@ export type PublicCodexControllerStatus = Omit< > & { execution: 'local' | 'verifying' | 'cloud' | 'fenced'; controller: 'local'; + /** Immutable process-start gate; never inferred from dotenv-mutated state. */ + liveTeleport: CodexLiveTeleportStartupSwitch; remote?: Pick< LiveTeleportEnvironment, 'environmentId' | 'generation' | 'workspaceCwd' | 'connectExpiresAt' | 'leaseExpiresAt' @@ -225,8 +249,8 @@ export type CodexLiveControllerOptions = { workspaceRoot: string; socketPath: string; model?: string; - /** Defaults off. Enabling must always be an explicit local operator action. */ - liveTeleportEnabled?: boolean; + /** Defaults off. Production passes the immutable pre-dotenv startup gate. */ + liveTeleportStartupSwitch?: CodexLiveTeleportStartupSwitch; lifecycleDeadlineMs?: number; lifecyclePollIntervalMs?: number; }; @@ -529,7 +553,10 @@ function sameRecoveredIdentity(left: CodexRecoveredOutcome, right: CodexRecovere ); } -function publicStatus(state: CodexControllerState): PublicCodexControllerStatus { +function publicStatus( + state: CodexControllerState, + liveTeleport: CodexLiveTeleportStartupSwitch +): PublicCodexControllerStatus { const { source, remote, mountRestore: _mountRestore, recoveredOutcome: _recoveredOutcome, ...rest } = state; const execution = state.phase === 'remote' @@ -547,6 +574,7 @@ function publicStatus(state: CodexControllerState): PublicCodexControllerStatus ...rest, controller: 'local', execution, + liveTeleport, workspaceSource: source?.kind ?? 'unavailable', ...(remote && state.phase === 'remote' && remote.attached ? { @@ -576,11 +604,15 @@ export class CodexLiveController { private prewarmAbort: AbortController | null = null; private recoveredEvidenceTaken: CodexRecoveredOutcome | undefined; private closing = false; + private readonly liveTeleportStartupSwitch: CodexLiveTeleportStartupSwitch; constructor( private readonly options: CodexLiveControllerOptions, private readonly deps: CodexLiveControllerDependencies - ) {} + ) { + const startupSwitch = options.liveTeleportStartupSwitch ?? DEFAULT_CODEX_LIVE_TELEPORT_STARTUP_SWITCH; + this.liveTeleportStartupSwitch = Object.freeze({ ...startupSwitch }); + } async initialize(): Promise { this.recoveredEvidenceTaken = undefined; @@ -715,7 +747,7 @@ export class CodexLiveController { } status(): PublicCodexControllerStatus { - return publicStatus(this.requireState()); + return publicStatus(this.requireState(), this.liveTeleportStartupSwitch); } /** @@ -1526,7 +1558,7 @@ export class CodexLiveController { } private liveTeleportEnabled(): boolean { - return this.options.liveTeleportEnabled === true; + return this.liveTeleportStartupSwitch.enabled; } private requireCloud(): LiveTeleportCloudClient { From 001c90d41f78b2995d6e289dac800c36b4338809 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Fri, 28 Aug 2026 16:43:07 +0200 Subject: [PATCH 14/16] feat(fleet): request long-running sandbox routing --- packages/cli/src/cli/commands/fleet.test.ts | 10 ++++++++- packages/cli/src/cli/commands/fleet.ts | 9 ++++---- packages/cloud/src/fleet-sandbox.test.ts | 7 ++++++ packages/cloud/src/fleet-sandbox.ts | 25 +++++++++++++++++++++ packages/cloud/src/index.ts | 2 ++ 5 files changed, 48 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/cli/commands/fleet.test.ts b/packages/cli/src/cli/commands/fleet.test.ts index 57f63189e..2ebfdeca5 100644 --- a/packages/cli/src/cli/commands/fleet.test.ts +++ b/packages/cli/src/cli/commands/fleet.test.ts @@ -670,6 +670,7 @@ describe('fleet command support', () => { relayWorkspaceId: 'rw_abc', relayfileMounted: true, relayfileMountPath: '/workspace', + providerId: 'agent37' as const, })); const deleteCloudFleetSandbox = vi.fn(async () => undefined); const logs: string[] = []; @@ -715,6 +716,7 @@ describe('fleet command support', () => { maxAgents: 1, mountRelayfile: true, forceProvision: true, + workloadProfile: 'long-running-agent', waitTimeoutMs: 90_000, name: 'daytona-codex', }); @@ -749,7 +751,11 @@ describe('fleet command support', () => { ); expect(deleteCloudFleetSandbox).not.toHaveBeenCalled(); expect(JSON.parse(logs[0]!)).toMatchObject({ - sandbox: { nodeName: 'daytona-codex', relayfileMountPath: '/workspace' }, + sandbox: { + nodeName: 'daytona-codex', + relayfileMountPath: '/workspace', + providerId: 'agent37', + }, invocation: { invocationId: 'inv_sandbox' }, attachCommand: "agent-relay node agent attach 'sandbox-worker' --node 'daytona-codex' --mode drive", }); @@ -783,6 +789,7 @@ describe('fleet command support', () => { relayWorkspaceId: 'rw_abc', relayfileMounted: true, relayfileMountPath: '/workspace', + providerId: 'agent37' as const, })), deleteCloudFleetSandbox, createFleetWorkspaceClient: vi.fn() as never, @@ -962,6 +969,7 @@ describe('fleet command support', () => { sandboxId: 'sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: false, + providerId: 'agent37' as const, })), deleteCloudFleetSandbox: vi.fn(async () => Promise.reject(new Error('delete failed'))), createFleetWorkspaceClient: vi.fn() as never, diff --git a/packages/cli/src/cli/commands/fleet.ts b/packages/cli/src/cli/commands/fleet.ts index 9c20f19f0..05e2af731 100644 --- a/packages/cli/src/cli/commands/fleet.ts +++ b/packages/cli/src/cli/commands/fleet.ts @@ -236,6 +236,7 @@ export function registerFleetCommands( maxAgents: 1, mountRelayfile: mountSandboxRelayfile, forceProvision: true, + workloadProfile: 'long-running-agent', waitTimeoutMs: 90_000, name: requestedSandboxName, }); @@ -248,7 +249,7 @@ export function registerFleetCommands( }) .catch((cleanupError) => { deps.warn( - `Provisioning failed after Daytona created sandbox '${error.sandboxId}', and automatic cleanup failed: ${ + `Provisioning failed after Cloud created sandbox '${error.sandboxId}', and automatic cleanup failed: ${ cleanupError instanceof Error ? cleanupError.message : String(cleanupError) }` ); @@ -257,7 +258,7 @@ export function registerFleetCommands( deps.warn( `Cloud did not return a complete provisioning response. The outcome is unknown; check Cloud Fleet for node '${ error.nodeName ?? requestedSandboxName - }' before retrying so a Daytona sandbox is not left running.` + }' before retrying so a Cloud sandbox is not left running.` ); } throw error; @@ -276,7 +277,7 @@ export function registerFleetCommands( ); }); throw new Error( - `Daytona node '${sandbox.nodeName}' did not become ready within ${sandbox.waitedMs}ms.` + `${sandbox.providerId} node '${sandbox.nodeName}' did not become ready within ${sandbox.waitedMs}ms.` ); } if ( @@ -297,7 +298,7 @@ export function registerFleetCommands( ); }); } - throw new Error('Cloud returned a Daytona node without the required Relayfile mount.'); + throw new Error('Cloud returned a sandbox node without the required Relayfile mount.'); } targetNode = sandbox.nodeName; if (!workerCwd && sandbox.outcome === 'provisioned' && sandbox.relayfileMounted) { diff --git a/packages/cloud/src/fleet-sandbox.test.ts b/packages/cloud/src/fleet-sandbox.test.ts index 6cae84254..55b086d69 100644 --- a/packages/cloud/src/fleet-sandbox.test.ts +++ b/packages/cloud/src/fleet-sandbox.test.ts @@ -51,6 +51,7 @@ describe('Cloud fleet sandbox client', () => { relayWorkspaceId: 'rw_abc', relayfileMounted: true, relayfileMountPath: '/workspace', + providerId: 'agent37', }, { status: 201 } ), @@ -64,6 +65,7 @@ describe('Cloud fleet sandbox client', () => { maxAgents: 1, mountRelayfile: true, forceProvision: true, + workloadProfile: 'long-running-agent', waitTimeoutMs: 90_000, }); @@ -84,6 +86,7 @@ describe('Cloud fleet sandbox client', () => { maxAgents: 1, mountRelayfile: true, forceProvision: true, + workloadProfile: 'long-running-agent', waitTimeoutMs: 90_000, }); expect(result).toEqual({ @@ -95,6 +98,7 @@ describe('Cloud fleet sandbox client', () => { relayWorkspaceId: 'rw_abc', relayfileMounted: true, relayfileMountPath: '/workspace', + providerId: 'agent37', }); }); @@ -112,6 +116,7 @@ describe('Cloud fleet sandbox client', () => { relayWorkspaceId: 'rw_abc', nodeName: 'daytona-codex', waitedMs: 90_000, + providerId: 'agent37', }, { status: 202 } ), @@ -154,6 +159,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: true, + providerId: 'daytona', }, { status: 201 } ), @@ -192,6 +198,7 @@ describe('Cloud fleet sandbox client', () => { sandboxId: 'sandbox-1', relayWorkspaceId: 'rw_abc', relayfileMounted: true, + providerId: 'daytona', }, { status: 201 } ), diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 84b0e0edb..44a8e7734 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -57,9 +57,19 @@ export type EnsureCloudFleetSandboxInput = { maxAgents?: number; mountRelayfile?: boolean; forceProvision?: boolean; + /** Provider-neutral semantics; Cloud owns the provider decision. */ + workloadProfile?: CloudFleetSandboxWorkloadProfile; waitTimeoutMs?: number; }; +export type CloudFleetSandboxWorkloadProfile = 'standard' | 'long-running-agent'; +export type CloudFleetSandboxProviderId = + | 'daytona' + | 'e2b' + | 'vercel' + | 'freestyle' + | 'agent37'; + export type CloudFleetSandboxReady = { outcome: 'provisioned'; cloudWorkspaceId: string; @@ -69,6 +79,7 @@ export type CloudFleetSandboxReady = { relayWorkspaceId: string; relayfileMounted: boolean; relayfileMountPath?: string; + providerId: CloudFleetSandboxProviderId; }; export type CloudFleetSandboxReused = { @@ -88,6 +99,7 @@ export type CloudFleetSandboxProvisioningTimeout = { relayWorkspaceId: string; nodeName: string; waitedMs: number; + providerId: CloudFleetSandboxProviderId; }; export type EnsureCloudFleetSandboxResult = @@ -166,6 +178,14 @@ function requiredString(payload: JsonRecord, key: string, context: string): stri return value; } +function requiredProviderId(payload: JsonRecord): CloudFleetSandboxProviderId { + const value = requiredString(payload, 'providerId', 'Cloud fleet sandbox'); + if (!['daytona', 'e2b', 'vercel', 'freestyle', 'agent37'].includes(value)) { + throw new Error('Cloud fleet sandbox response has an unknown providerId.'); + } + return value as CloudFleetSandboxProviderId; +} + async function resolveCloudWorkspaceId( workspaceId: string, auth: Awaited>['auth'], @@ -210,6 +230,7 @@ function normalizeEnsureResult(payload: unknown, cloudWorkspaceId: string): Ensu sandboxId: requiredString(payload, 'sandboxId', 'Cloud fleet sandbox'), relayWorkspaceId: requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'), relayfileMounted: payload.relayfileMounted, + providerId: requiredProviderId(payload), ...(readString(payload, 'relayfileMountPath') ? { relayfileMountPath: readString(payload, 'relayfileMountPath') } : {}), @@ -236,6 +257,7 @@ function normalizeEnsureResult(payload: unknown, cloudWorkspaceId: string): Ensu relayWorkspaceId: requiredString(payload, 'relayWorkspaceId', 'Cloud fleet sandbox'), nodeName, waitedMs: requiredNumber(payload, 'waitedMs', 'Cloud fleet sandbox'), + providerId: requiredProviderId(payload), }; } @@ -274,6 +296,9 @@ export async function ensureCloudFleetSandbox( ...(input.maxAgents !== undefined ? { maxAgents: input.maxAgents } : {}), ...(input.mountRelayfile !== undefined ? { mountRelayfile: input.mountRelayfile } : {}), ...(input.forceProvision !== undefined ? { forceProvision: input.forceProvision } : {}), + ...(input.workloadProfile !== undefined + ? { workloadProfile: input.workloadProfile } + : {}), ...(input.waitTimeoutMs !== undefined ? { waitTimeoutMs: input.waitTimeoutMs } : {}), }), }, diff --git a/packages/cloud/src/index.ts b/packages/cloud/src/index.ts index 5d966399a..17451863b 100644 --- a/packages/cloud/src/index.ts +++ b/packages/cloud/src/index.ts @@ -103,6 +103,8 @@ export { type CloudFleetSandboxReady, type CloudFleetSandboxReused, type CloudFleetSandboxProvisioningTimeout, + type CloudFleetSandboxProviderId, + type CloudFleetSandboxWorkloadProfile, type DeleteCloudFleetSandboxInput, type CloudFleetSandboxRequestOptions, } from './fleet-sandbox.js'; From d34b9f708c8429ce203aea7aa6e216e2539500f4 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 28 Aug 2026 16:04:17 +0000 Subject: [PATCH 15/16] style: auto-format with Prettier --- packages/cloud/src/fleet-sandbox.ts | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/packages/cloud/src/fleet-sandbox.ts b/packages/cloud/src/fleet-sandbox.ts index 44a8e7734..ad0fe42d8 100644 --- a/packages/cloud/src/fleet-sandbox.ts +++ b/packages/cloud/src/fleet-sandbox.ts @@ -63,12 +63,7 @@ export type EnsureCloudFleetSandboxInput = { }; export type CloudFleetSandboxWorkloadProfile = 'standard' | 'long-running-agent'; -export type CloudFleetSandboxProviderId = - | 'daytona' - | 'e2b' - | 'vercel' - | 'freestyle' - | 'agent37'; +export type CloudFleetSandboxProviderId = 'daytona' | 'e2b' | 'vercel' | 'freestyle' | 'agent37'; export type CloudFleetSandboxReady = { outcome: 'provisioned'; @@ -296,9 +291,7 @@ export async function ensureCloudFleetSandbox( ...(input.maxAgents !== undefined ? { maxAgents: input.maxAgents } : {}), ...(input.mountRelayfile !== undefined ? { mountRelayfile: input.mountRelayfile } : {}), ...(input.forceProvision !== undefined ? { forceProvision: input.forceProvision } : {}), - ...(input.workloadProfile !== undefined - ? { workloadProfile: input.workloadProfile } - : {}), + ...(input.workloadProfile !== undefined ? { workloadProfile: input.workloadProfile } : {}), ...(input.waitTimeoutMs !== undefined ? { waitTimeoutMs: input.waitTimeoutMs } : {}), }), }, From b2d6b00312530b77c4ba81b3709cf840cbf80158 Mon Sep 17 00:00:00 2001 From: kjgbot Date: Fri, 28 Aug 2026 18:08:34 +0200 Subject: [PATCH 16/16] test(relayflow): prove long-running provider attribution --- .../case.json | 20 ++ .../run.mjs | 185 ++++++++++++++++++ 2 files changed, 205 insertions(+) create mode 100644 tests/relayflows/cases/1605-long-running-provider-attribution/case.json create mode 100644 tests/relayflows/cases/1605-long-running-provider-attribution/run.mjs diff --git a/tests/relayflows/cases/1605-long-running-provider-attribution/case.json b/tests/relayflows/cases/1605-long-running-provider-attribution/case.json new file mode 100644 index 000000000..beaacde33 --- /dev/null +++ b/tests/relayflows/cases/1605-long-running-provider-attribution/case.json @@ -0,0 +1,20 @@ +{ + "version": 1, + "id": "1605-long-running-provider-attribution", + "kind": "feature", + "title": "Send semantic long-running workload intent and preserve Cloud provider attribution", + "runner": { + "command": ["node", "tests/relayflows/cases/1605-long-running-provider-attribution/run.mjs"] + }, + "timeoutSeconds": 900, + "expected": { + "base": { + "outcome": "absent", + "signature": "semantic_profile_and_attribution_absent" + }, + "head": { + "outcome": "fixed", + "signature": "long_running_profile_with_agent37_attribution" + } + } +} diff --git a/tests/relayflows/cases/1605-long-running-provider-attribution/run.mjs b/tests/relayflows/cases/1605-long-running-provider-attribution/run.mjs new file mode 100644 index 000000000..a69d82161 --- /dev/null +++ b/tests/relayflows/cases/1605-long-running-provider-attribution/run.mjs @@ -0,0 +1,185 @@ +#!/usr/bin/env node + +import { execFileSync, spawnSync } from 'node:child_process'; +import { mkdir, readFile, rm, writeFile } from 'node:fs/promises'; +import path from 'node:path'; +import process from 'node:process'; +import { fileURLToPath } from 'node:url'; + +const CASE_ID = '1605-long-running-provider-attribution'; +const arm = requiredValue('RELAY_PR_PROOF_ARM'); +const targetDir = path.resolve(requiredValue('RELAY_PR_PROOF_TARGET_DIR')); +const harnessDir = path.resolve(requiredValue('RELAY_PR_PROOF_HARNESS_DIR')); +const resultPath = requiredValue('RELAY_PR_PROOF_RESULT_PATH'); +if (arm !== 'base' && arm !== 'head') throw new Error('RelayFlow proof arm must be base or head'); + +const expectedSha = requiredValue(arm === 'base' ? 'RELAY_PR_PROOF_BASE_SHA' : 'RELAY_PR_PROOF_HEAD_SHA'); +const targetSha = execFileSync('git', ['-C', targetDir, 'rev-parse', 'HEAD'], { + encoding: 'utf8', +}).trim(); +if (targetSha !== expectedSha) { + throw new Error(`Target checkout ${targetSha} does not match exact ${arm} SHA ${expectedSha}`); +} +const runnerPath = fileURLToPath(import.meta.url); +if (!isWithin(harnessDir, runnerPath)) { + throw new Error('RelayFlow runner must execute from the exact-head harness checkout'); +} + +const probePath = path.join( + targetDir, + 'packages/cloud/src/.relayflow-1605-long-running-provider-attribution.test.ts' +); +const observationPath = path.join( + targetDir, + '.relayflow-1605-long-running-provider-attribution-observation.json' +); +const configPath = path.join( + targetDir, + '.relayflow-1605-long-running-provider-attribution.vitest.config.mjs' +); + +const probeSource = String.raw`import { expect, test, vi } from 'vitest'; +import { writeFile } from 'node:fs/promises'; + +const mocks = vi.hoisted(() => ({ + ensureCloudSession: vi.fn(), + authorizedApiFetch: vi.fn(), +})); + +vi.mock('./auth.js', () => ({ + ensureCloudSession: mocks.ensureCloudSession, + authorizedApiFetch: mocks.authorizedApiFetch, +})); + +import { ensureCloudFleetSandbox } from './fleet-sandbox.js'; + +const auth = { + accessToken: 'relayflow-access', + refreshToken: 'relayflow-refresh', + accessTokenExpiresAt: '2099-01-01T00:00:00Z', + apiUrl: 'https://relayflow.invalid', +}; + +test('observes semantic request routing and provider attribution', async () => { + const observationPath = process.env.RELAY_PR1605_OBSERVATION_PATH; + if (!observationPath) throw new Error('Missing RELAY_PR1605_OBSERVATION_PATH'); + mocks.ensureCloudSession.mockResolvedValue({ auth, client: {} }); + mocks.authorizedApiFetch + .mockResolvedValueOnce({ + response: Response.json({ cloudWorkspaceId: '50587328-441d-4acb-b8f3-dbe1b3c5de99' }), + auth, + }) + .mockResolvedValueOnce({ + response: Response.json({ + outcome: 'provisioned', + nodeId: 'node-agent37-proof', + nodeName: 'agent37-proof', + sandboxId: 'sandbox-agent37-proof', + relayWorkspaceId: 'rw_agent37_proof', + relayfileMounted: true, + providerId: 'agent37', + }, { status: 201 }), + auth, + }); + + const request = { + workspaceId: 'rw_agent37_proof', + requiredCapability: 'spawn:codex', + mountRelayfile: true, + forceProvision: true, + workloadProfile: 'long-running-agent', + } as Parameters[0] & Record; + const result = await ensureCloudFleetSandbox(request); + expect(mocks.authorizedApiFetch).toHaveBeenCalledTimes(2); + const ensureCall = mocks.authorizedApiFetch.mock.calls[1]; + const body = JSON.parse(String(ensureCall?.[2]?.body ?? '{}')); + const providerId = (result as unknown as { providerId?: string }).providerId; + await writeFile(observationPath, JSON.stringify({ body, providerId }), 'utf8'); +}); +`; +const configSource = `export default { test: { environment: 'node', include: ['packages/cloud/src/.relayflow-1605-long-running-provider-attribution.test.ts'], setupFiles: [] } };\n`; + +try { + run( + 'npm', + ['ci', '--ignore-scripts', '--workspace', 'packages/cloud', '--include-workspace-root=false'], + targetDir, + 'Cloud workspace dependency installation' + ); + run('npm', ['run', 'build:config'], targetDir, 'configuration package build'); + run('npm', ['run', 'build:cloud'], targetDir, 'Cloud package build'); + await writeFile(probePath, probeSource, { encoding: 'utf8', flag: 'wx' }); + await writeFile(configPath, configSource, { encoding: 'utf8', flag: 'wx' }); + run( + 'npm', + ['exec', '--', 'vitest', 'run', '--config', path.relative(targetDir, configPath)], + targetDir, + 'semantic fleet request probe', + { RELAY_PR1605_OBSERVATION_PATH: observationPath } + ); + + const observation = JSON.parse(await readFile(observationPath, 'utf8')); + const fixed = + observation?.body?.workloadProfile === 'long-running-agent' && observation?.providerId === 'agent37'; + const absent = observation?.body?.workloadProfile === undefined && observation?.providerId === undefined; + if (!fixed && !absent) { + throw new Error(`Unexpected semantic routing observation: ${JSON.stringify(observation)}`); + } + const outcome = fixed ? 'fixed' : 'absent'; + const signature = fixed + ? 'long_running_profile_with_agent37_attribution' + : 'semantic_profile_and_attribution_absent'; + await mkdir(path.dirname(resultPath), { recursive: true }); + await writeFile( + resultPath, + `${JSON.stringify( + { + version: 1, + caseId: CASE_ID, + arm, + outcome, + signature, + details: fixed + ? 'Cloud request carried long-running-agent and the normalized result retained Agent37 attribution.' + : 'Cloud request omitted semantic workload intent and discarded provider attribution.', + }, + null, + 2 + )}\n`, + 'utf8' + ); +} finally { + await rm(probePath, { force: true }); + await rm(configPath, { force: true }); + await rm(observationPath, { force: true }); +} + +function requiredValue(name) { + const value = process.env[name]?.trim(); + if (!value) throw new Error(`Missing required environment variable ${name}`); + return value; +} + +function isWithin(directory, candidate) { + const relative = path.relative(directory, candidate); + return ( + relative === '' || + (!relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)) + ); +} + +function run(command, args, cwd, label, extraEnv = {}) { + const completed = spawnSync(command, args, { + cwd, + env: { ...process.env, ...extraEnv }, + stdio: ['ignore', 'inherit', 'inherit'], + }); + if (completed.error) throw new Error(`${label} could not start: ${completed.error.message}`); + if (completed.status !== 0) { + throw new Error( + `${label} failed with ${ + completed.signal ? `signal ${completed.signal}` : `exit code ${completed.status ?? 'unknown'}` + }` + ); + } +}