diff --git a/src/remote-hosts.ts b/src/remote-hosts.ts index a82bdb37..2b9106f3 100644 --- a/src/remote-hosts.ts +++ b/src/remote-hosts.ts @@ -326,6 +326,59 @@ export async function probeRemoteCliVersion( } } +/** + * COD-108 — build the SSH command that asks whether THIS Codeman's durable + * remote tmux session (`-L codeman-remote -s codeman-ssh-`) is still alive + * on the remote host. + * + * `has-session` exits 0 when the session exists, non-zero otherwise (and + * stderr is swallowed). Connection options come from the shared + * `buildSshConnectionArgs` so this probe reaches exactly the hosts the launch + * can reach — same port/identity/proxy/jump-host as `buildRemoteLaunchCommand`. + */ +export function buildRemoteSessionAliveCommand( + host: Pick & RemoteSshOptions, + remoteSessionName: string +): string { + const [ssh, ...connectionArgs] = buildSshConnectionArgs(host); + const remoteCmd = `tmux -L codeman-remote has-session -t ${shellescape(remoteSessionName)} 2>/dev/null`; + return [ssh, ...connectionArgs, remoteSshTarget(host), shellescape(remoteCmd)].join(' '); +} + +/** + * COD-108 — resolve whether THIS Codeman's durable remote tmux session is still + * alive on the remote host, for the auto-reconnect watcher. + * + * Returns: + * - `true` → the remote tmux session exists (the agent is still running + * on the remote; the LOCAL pane died from a transport drop → + * safe to auto-reconnect). + * - `false` → the remote session is gone (the agent exited cleanly and + * the remote tmux tore down; reviving would relaunch a fresh + * agent — must NOT auto-reconnect). + * - `undefined` → probe failed (host unreachable, ssh error, tmux missing). + * Callers MUST treat this as "do not reconnect": an + * unreachable host is not a reason to relaunch the agent. + * + * VITEST guard — returns `true` under test so a real ssh never runs; the + * command construction is covered by `buildRemoteSessionAliveCommand`. + */ +export async function remoteTmuxSessionAlive( + remote: Pick & RemoteSshOptions, + remoteSessionName: string +): Promise { + if (process.env.VITEST) return true; + const command = buildRemoteSessionAliveCommand(remote, remoteSessionName); + try { + const { stdout } = await execAsync(command, { timeout: 15_000 }); + // has-session prints the session name on success (exit 0). Anything else is + // a non-zero exit → the session is gone. + return stdout.trim().length > 0; + } catch { + return undefined; + } +} + /** * COD-105 — build the SSH command that lists `codeman-*` tmux sessions on a * remote host's canonical `-L codeman` socket. diff --git a/src/remote-reconnect.ts b/src/remote-reconnect.ts index 6e5a709a..bbcfcd86 100644 --- a/src/remote-reconnect.ts +++ b/src/remote-reconnect.ts @@ -108,6 +108,15 @@ export interface ReconnectSessionView { isRemote: boolean; /** Result of `isPaneDead(muxName)` for this session. */ paneDead: boolean; + /** + * Whether the DURABLE remote tmux session is still alive on the remote host. + * Tri-state: `true` = transport drop with the agent still running (safe to + * reattach); `false` = the remote session is gone (the agent exited cleanly + * via ctrl-c/ctrl-d/exit and the remote tmux tore down); `undefined` = + * unknown/unresolvable. The watcher must NOT revive when the remote session + * is gone or unknown — a clean exit must never auto-relaunch the agent. + */ + remoteAlive: boolean | undefined; } /** @@ -130,7 +139,8 @@ export type ReconnectSkipReason = | 'in-flight' | 'not-due' | 'exhausted' - | 'disabled'; + | 'disabled' + | 'remote-gone'; export interface DecideReconnectInput { session: ReconnectSessionView; @@ -166,6 +176,14 @@ export function decideReconnect(input: DecideReconnectInput): ReconnectAction { if (!session.paneDead) return { kind: 'skip', reason: 'pane-alive' }; // Intentional kill / detach must NEVER be auto-revived. if (guarded) return { kind: 'skip', reason: 'guarded' }; + // A clean exit tears down the durable remote tmux (the session's only pane + // exiting destroys it). Reviving is ONLY correct for a transport drop: the + // agent is still running on the remote, so the durable session must still + // exist. When it is gone (or status is unknown — probe failed/unreachable), + // the agent exited intentionally and must not be auto-relaunched (found + // live 2026-08-29: remote omp/opencode ctrl-c/ctrl-d auto-respawned fresh + // sessions; only claude's `|| --resume` accidentally masked it). + if (session.remoteAlive !== true) return { kind: 'skip', reason: 'remote-gone' }; const s = state ?? freshReconnectState(); diff --git a/src/tmux-manager.ts b/src/tmux-manager.ts index 5088793d..cae32c19 100644 --- a/src/tmux-manager.ts +++ b/src/tmux-manager.ts @@ -64,6 +64,7 @@ import { defaultRemoteCommandForMode, remoteLoginShellCommand, remoteSshTarget, + remoteTmuxSessionAlive, } from './remote-hosts.js'; import { buildDockerBaseArgs, @@ -1674,6 +1675,18 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { * torn down (killed/detached/stopping). A guarded session is NEVER revived. */ private reconnectGuard: Set = new Set(); + /** + * Cached result of the remote tmux `has-session` probe (sessionId → alive). + * `true` = the durable remote tmux session exists (transport drop → reconnect + * is safe); `false` = remote session gone (agent exited cleanly → do NOT + * reconnect); `undefined` = not yet probed / probe failed. Only sessions + * whose pane is otherwise dead+eligible get probed, so a clean exit tears + * down the remote tmux and the probe reports false — killing the auto-revive + * (found live 2026-08-29: remote omp/opencode ctrl-c/ctrl-d auto-respawned + * fresh agents because the watcher couldn't tell a clean exit from a + * transport drop). + */ + private remoteAliveCache: Map = new Map(); private trueColorConfigured = false; /** tmux 3.7+ can resize pane history after creation; older releases cannot. */ @@ -2134,7 +2147,6 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { // Create tmux session in three steps to handle cold-start (no server running) // and avoid the race where the command exits before remain-on-exit is set: - // 1. Create session with default shell (starts tmux server, stays alive) // 2. Set remain-on-exit (server now exists, session won't vanish on exit) // 3. Replace shell with actual command via respawn-pane (no terminal echo) // Unset $TMUX so nested sessions work when the dev server itself runs inside tmux. @@ -3111,16 +3123,44 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { * applies the pure {@link decideReconnect} decision and translates the result * into events + backoff/state transitions. Public for tests + the watcher. */ + /** + * Refresh the cached remote-tmux liveness for a session whose pane is dead. + * Fire-and-forget (async, not awaited by the sync tick): the probe is a slow + * ssh round-trip, so it must not block the 5s watcher interval. On success it + * writes the cached result; the NEXT tick then makes the revive decision with + * fresh data. A clean exit makes the remote tmux session vanish, so the probe + * resolves false and the watcher stops reviving it (2026-08-29). + */ + private async refreshRemoteAlive(session: MuxSession): Promise { + if (!session.remote) return; + const remoteName = session.remote.remoteSessionName || remoteTmuxSessionName(session.sessionId); + try { + const alive = await remoteTmuxSessionAlive(session.remote, remoteName); + this.remoteAliveCache.set(session.sessionId, alive); + } catch { + this.remoteAliveCache.set(session.sessionId, undefined); + } + } + runRemoteReconnectTick(now: number, enabled: boolean): void { for (const session of this.sessions.values()) { if (!session.remote) continue; const sessionId = session.sessionId; const state = this.reconnectState.get(sessionId); + // Only probe when the pane is actually dead — otherwise the ssh round-trip + // would run every 5s for every healthy remote session. The cache is + // refreshed lazily so a clean exit (remote tmux gone) flips it to false + // on the next tick and stops the auto-revive. + const paneDead = this.isPaneDead(session.muxName); + if (paneDead && this.remoteAliveCache.get(sessionId) === undefined) { + void this.refreshRemoteAlive(session); + } const action = decideReconnect({ session: { sessionId, isRemote: true, - paneDead: this.isPaneDead(session.muxName), + paneDead, + remoteAlive: this.remoteAliveCache.get(sessionId), }, state, guarded: this.reconnectGuard.has(sessionId), @@ -3168,12 +3208,14 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { guardRemoteReconnect(sessionId: string): void { this.reconnectGuard.add(sessionId); this.reconnectState.delete(sessionId); + this.remoteAliveCache.delete(sessionId); } /** Clear all per-session reconnect + guard state (e.g. when a session is removed). */ clearRemoteReconnectState(sessionId: string): void { this.reconnectState.delete(sessionId); this.reconnectGuard.delete(sessionId); + this.remoteAliveCache.delete(sessionId); } destroy(): void { @@ -3182,6 +3224,7 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer { this.stopRemoteReconnectWatcher(); this.reconnectState.clear(); this.reconnectGuard.clear(); + this.remoteAliveCache.clear(); } registerSession(session: MuxSession): void { diff --git a/test/remote-auto-reconnect.test.ts b/test/remote-auto-reconnect.test.ts index 6be1dc0b..b09d2d61 100644 --- a/test/remote-auto-reconnect.test.ts +++ b/test/remote-auto-reconnect.test.ts @@ -106,7 +106,7 @@ describe('reconnect backoff schedule (pure)', () => { // ──────────────────────────────────────────────────────────────────────────── describe('decideReconnect (pure eligibility)', () => { - const deadRemote: ReconnectSessionView = { sessionId: 's1', isRemote: true, paneDead: true }; + const deadRemote: ReconnectSessionView = { sessionId: 's1', isRemote: true, paneDead: true, remoteAlive: true }; it('emits for a dead remote pane that is not guarded and is due', () => { const action = decideReconnect({ @@ -132,7 +132,7 @@ describe('decideReconnect (pure eligibility)', () => { it('skips non-remote sessions', () => { const action = decideReconnect({ - session: { sessionId: 's1', isRemote: false, paneDead: true }, + session: { sessionId: 's1', isRemote: false, paneDead: true, remoteAlive: true }, state: freshReconnectState(), guarded: false, enabled: true, @@ -143,7 +143,7 @@ describe('decideReconnect (pure eligibility)', () => { it('skips when the pane is alive', () => { const action = decideReconnect({ - session: { sessionId: 's1', isRemote: true, paneDead: false }, + session: { sessionId: 's1', isRemote: true, paneDead: false, remoteAlive: true }, state: freshReconnectState(), guarded: false, enabled: true, @@ -152,6 +152,28 @@ describe('decideReconnect (pure eligibility)', () => { expect(action).toEqual({ kind: 'skip', reason: 'pane-alive' }); }); + it('NEVER revives when the durable remote tmux is GONE (clean exit — the 2026-08-29 fix)', () => { + const action = decideReconnect({ + session: { sessionId: 's1', isRemote: true, paneDead: true, remoteAlive: false }, + state: freshReconnectState(), + guarded: false, + enabled: true, + now: 0, + }); + expect(action).toEqual({ kind: 'skip', reason: 'remote-gone' }); + }); + + it('NEVER revives when remote liveness is unknown (probe failed — fail closed)', () => { + const action = decideReconnect({ + session: { sessionId: 's1', isRemote: true, paneDead: true, remoteAlive: undefined }, + state: freshReconnectState(), + guarded: false, + enabled: true, + now: 0, + }); + expect(action).toEqual({ kind: 'skip', reason: 'remote-gone' }); + }); + it('skips when the kill-switch is off', () => { const action = decideReconnect({ session: deadRemote, @@ -222,6 +244,11 @@ describe('TmuxManager remote reconnect watcher (integration)', () => { registerRemote('aaaa1111'); // Force the watcher to see a dead pane regardless of test-mode isPaneDead. vi.spyOn(manager, 'isPaneDead').mockReturnValue(true); + // The durable remote tmux is still alive (transport drop) → reconnect allowed. + (manager as unknown as { remoteAliveCache: Map }).remoteAliveCache.set( + 'aaaa1111', + true + ); const dropped: Array<{ sessionId: string; attempt: number }> = []; const exhausted: Array<{ sessionId: string }> = []; @@ -263,6 +290,10 @@ describe('TmuxManager remote reconnect watcher (integration)', () => { it('resets backoff on a successful reattach (noteRemoteReconnect)', () => { registerRemote('cccc3333'); vi.spyOn(manager, 'isPaneDead').mockReturnValue(true); + (manager as unknown as { remoteAliveCache: Map }).remoteAliveCache.set( + 'cccc3333', + true + ); const dropped: Array<{ attempt: number }> = []; manager.on('remoteSessionDropped', (d) => dropped.push(d)); @@ -290,6 +321,10 @@ describe('TmuxManager remote reconnect watcher (integration)', () => { manager.clearRemoteReconnectState('eeee5555'); // After clearing the guard, a fresh dead-pane observation should emit again. vi.spyOn(manager, 'isPaneDead').mockReturnValue(true); + (manager as unknown as { remoteAliveCache: Map }).remoteAliveCache.set( + 'eeee5555', + true + ); const dropped: unknown[] = []; manager.on('remoteSessionDropped', (d) => dropped.push(d)); manager.runRemoteReconnectTick(0, true);