Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 53 additions & 0 deletions src/remote-hosts.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -326,6 +326,59 @@ export async function probeRemoteCliVersion(
}
}

/**
* COD-108 — build the SSH command that asks whether THIS Codeman's durable
* remote tmux session (`-L codeman-remote -s codeman-ssh-<id>`) is still alive
* on the remote host.
*
* `has-session` exits 0 when the session exists, non-zero otherwise (and
* stderr is swallowed). Connection options come from the shared
* `buildSshConnectionArgs` so this probe reaches exactly the hosts the launch
* can reach — same port/identity/proxy/jump-host as `buildRemoteLaunchCommand`.
*/
export function buildRemoteSessionAliveCommand(
host: Pick<RemoteHost, 'username' | 'host' | 'port'> & RemoteSshOptions,
remoteSessionName: string
): string {
const [ssh, ...connectionArgs] = buildSshConnectionArgs(host);
const remoteCmd = `tmux -L codeman-remote has-session -t ${shellescape(remoteSessionName)} 2>/dev/null`;
return [ssh, ...connectionArgs, remoteSshTarget(host), shellescape(remoteCmd)].join(' ');
}

/**
* COD-108 — resolve whether THIS Codeman's durable remote tmux session is still
* alive on the remote host, for the auto-reconnect watcher.
*
* Returns:
* - `true` → the remote tmux session exists (the agent is still running
* on the remote; the LOCAL pane died from a transport drop →
* safe to auto-reconnect).
* - `false` → the remote session is gone (the agent exited cleanly and
* the remote tmux tore down; reviving would relaunch a fresh
* agent — must NOT auto-reconnect).
* - `undefined` → probe failed (host unreachable, ssh error, tmux missing).
* Callers MUST treat this as "do not reconnect": an
* unreachable host is not a reason to relaunch the agent.
*
* VITEST guard — returns `true` under test so a real ssh never runs; the
* command construction is covered by `buildRemoteSessionAliveCommand`.
*/
export async function remoteTmuxSessionAlive(
remote: Pick<RemoteHost, 'username' | 'host' | 'port'> & RemoteSshOptions,
remoteSessionName: string
): Promise<boolean | undefined> {
if (process.env.VITEST) return true;
const command = buildRemoteSessionAliveCommand(remote, remoteSessionName);
try {
const { stdout } = await execAsync(command, { timeout: 15_000 });
// has-session prints the session name on success (exit 0). Anything else is
// a non-zero exit → the session is gone.
return stdout.trim().length > 0;
} catch {
return undefined;
}
}

/**
* COD-105 — build the SSH command that lists `codeman-*` tmux sessions on a
* remote host's canonical `-L codeman` socket.
Expand Down
20 changes: 19 additions & 1 deletion src/remote-reconnect.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -108,6 +108,15 @@ export interface ReconnectSessionView {
isRemote: boolean;
/** Result of `isPaneDead(muxName)` for this session. */
paneDead: boolean;
/**
* Whether the DURABLE remote tmux session is still alive on the remote host.
* Tri-state: `true` = transport drop with the agent still running (safe to
* reattach); `false` = the remote session is gone (the agent exited cleanly
* via ctrl-c/ctrl-d/exit and the remote tmux tore down); `undefined` =
* unknown/unresolvable. The watcher must NOT revive when the remote session
* is gone or unknown — a clean exit must never auto-relaunch the agent.
*/
remoteAlive: boolean | undefined;
}

/**
Expand All@@ -130,7 +139,8 @@ export type ReconnectSkipReason =
| 'in-flight'
| 'not-due'
| 'exhausted'
| 'disabled';
| 'disabled'
| 'remote-gone';

export interface DecideReconnectInput {
session: ReconnectSessionView;
Expand DownExpand Up@@ -166,6 +176,14 @@ export function decideReconnect(input: DecideReconnectInput): ReconnectAction {
if (!session.paneDead) return { kind: 'skip', reason: 'pane-alive' };
// Intentional kill / detach must NEVER be auto-revived.
if (guarded) return { kind: 'skip', reason: 'guarded' };
// A clean exit tears down the durable remote tmux (the session's only pane
// exiting destroys it). Reviving is ONLY correct for a transport drop: the
// agent is still running on the remote, so the durable session must still
// exist. When it is gone (or status is unknown — probe failed/unreachable),
// the agent exited intentionally and must not be auto-relaunched (found
// live 2026-08-29: remote omp/opencode ctrl-c/ctrl-d auto-respawned fresh
// sessions; only claude's `|| --resume` accidentally masked it).
if (session.remoteAlive !== true) return { kind: 'skip', reason: 'remote-gone' };

const s = state ?? freshReconnectState();

Expand Down
47 changes: 45 additions & 2 deletions src/tmux-manager.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -64,6 +64,7 @@ import {
defaultRemoteCommandForMode,
remoteLoginShellCommand,
remoteSshTarget,
remoteTmuxSessionAlive,
} from './remote-hosts.js';
import {
buildDockerBaseArgs,
Expand DownExpand Up@@ -1674,6 +1675,18 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer {
* torn down (killed/detached/stopping). A guarded session is NEVER revived.
*/
private reconnectGuard: Set<string> = new Set();
/**
* Cached result of the remote tmux `has-session` probe (sessionId → alive).
* `true` = the durable remote tmux session exists (transport drop → reconnect
* is safe); `false` = remote session gone (agent exited cleanly → do NOT
* reconnect); `undefined` = not yet probed / probe failed. Only sessions
* whose pane is otherwise dead+eligible get probed, so a clean exit tears
* down the remote tmux and the probe reports false — killing the auto-revive
* (found live 2026-08-29: remote omp/opencode ctrl-c/ctrl-d auto-respawned
* fresh agents because the watcher couldn't tell a clean exit from a
* transport drop).
*/
private remoteAliveCache: Map<string, boolean | undefined> = new Map();

private trueColorConfigured = false;
/** tmux 3.7+ can resize pane history after creation; older releases cannot. */
Expand DownExpand Up@@ -2134,7 +2147,6 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer {

// Create tmux session in three steps to handle cold-start (no server running)
// and avoid the race where the command exits before remain-on-exit is set:
// 1. Create session with default shell (starts tmux server, stays alive)
// 2. Set remain-on-exit (server now exists, session won't vanish on exit)
// 3. Replace shell with actual command via respawn-pane (no terminal echo)
// Unset $TMUX so nested sessions work when the dev server itself runs inside tmux.
Expand DownExpand Up@@ -3111,16 +3123,44 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer {
* applies the pure {@link decideReconnect} decision and translates the result
* into events + backoff/state transitions. Public for tests + the watcher.
*/
/**
* Refresh the cached remote-tmux liveness for a session whose pane is dead.
* Fire-and-forget (async, not awaited by the sync tick): the probe is a slow
* ssh round-trip, so it must not block the 5s watcher interval. On success it
* writes the cached result; the NEXT tick then makes the revive decision with
* fresh data. A clean exit makes the remote tmux session vanish, so the probe
* resolves false and the watcher stops reviving it (2026-08-29).
*/
private async refreshRemoteAlive(session: MuxSession): Promise<void> {
if (!session.remote) return;
const remoteName = session.remote.remoteSessionName || remoteTmuxSessionName(session.sessionId);
try {
const alive = await remoteTmuxSessionAlive(session.remote, remoteName);
this.remoteAliveCache.set(session.sessionId, alive);
} catch {
this.remoteAliveCache.set(session.sessionId, undefined);
}
}

runRemoteReconnectTick(now: number, enabled: boolean): void {
for (const session of this.sessions.values()) {
if (!session.remote) continue;
const sessionId = session.sessionId;
const state = this.reconnectState.get(sessionId);
// Only probe when the pane is actually dead — otherwise the ssh round-trip
// would run every 5s for every healthy remote session. The cache is
// refreshed lazily so a clean exit (remote tmux gone) flips it to false
// on the next tick and stops the auto-revive.
const paneDead = this.isPaneDead(session.muxName);
if (paneDead && this.remoteAliveCache.get(sessionId) === undefined) {
void this.refreshRemoteAlive(session);
}
const action = decideReconnect({
session: {
sessionId,
isRemote: true,
paneDead: this.isPaneDead(session.muxName),
paneDead,
remoteAlive: this.remoteAliveCache.get(sessionId),
},
state,
guarded: this.reconnectGuard.has(sessionId),
Expand DownExpand Up@@ -3168,12 +3208,14 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer {
guardRemoteReconnect(sessionId: string): void {
this.reconnectGuard.add(sessionId);
this.reconnectState.delete(sessionId);
this.remoteAliveCache.delete(sessionId);
}

/** Clear all per-session reconnect + guard state (e.g. when a session is removed). */
clearRemoteReconnectState(sessionId: string): void {
this.reconnectState.delete(sessionId);
this.reconnectGuard.delete(sessionId);
this.remoteAliveCache.delete(sessionId);
}

destroy(): void {
Expand All@@ -3182,6 +3224,7 @@ export class TmuxManager extends EventEmitter implements TerminalMultiplexer {
this.stopRemoteReconnectWatcher();
this.reconnectState.clear();
this.reconnectGuard.clear();
this.remoteAliveCache.clear();
}

registerSession(session: MuxSession): void {
Expand Down
41 changes: 38 additions & 3 deletions test/remote-auto-reconnect.test.ts
Original file line numberDiff line numberDiff line change
Expand Up@@ -106,7 +106,7 @@ describe('reconnect backoff schedule (pure)', () => {
// ────────────────────────────────────────────────────────────────────────────

describe('decideReconnect (pure eligibility)', () => {
const deadRemote: ReconnectSessionView = { sessionId: 's1', isRemote: true, paneDead: true };
const deadRemote: ReconnectSessionView = { sessionId: 's1', isRemote: true, paneDead: true, remoteAlive: true };

it('emits for a dead remote pane that is not guarded and is due', () => {
const action = decideReconnect({
Expand All@@ -132,7 +132,7 @@ describe('decideReconnect (pure eligibility)', () => {

it('skips non-remote sessions', () => {
const action = decideReconnect({
session: { sessionId: 's1', isRemote: false, paneDead: true },
session: { sessionId: 's1', isRemote: false, paneDead: true, remoteAlive: true },
state: freshReconnectState(),
guarded: false,
enabled: true,
Expand All@@ -143,7 +143,7 @@ describe('decideReconnect (pure eligibility)', () => {

it('skips when the pane is alive', () => {
const action = decideReconnect({
session: { sessionId: 's1', isRemote: true, paneDead: false },
session: { sessionId: 's1', isRemote: true, paneDead: false, remoteAlive: true },
state: freshReconnectState(),
guarded: false,
enabled: true,
Expand All@@ -152,6 +152,28 @@ describe('decideReconnect (pure eligibility)', () => {
expect(action).toEqual({ kind: 'skip', reason: 'pane-alive' });
});

it('NEVER revives when the durable remote tmux is GONE (clean exit — the 2026-08-29 fix)', () => {
const action = decideReconnect({
session: { sessionId: 's1', isRemote: true, paneDead: true, remoteAlive: false },
state: freshReconnectState(),
guarded: false,
enabled: true,
now: 0,
});
expect(action).toEqual({ kind: 'skip', reason: 'remote-gone' });
});

it('NEVER revives when remote liveness is unknown (probe failed — fail closed)', () => {
const action = decideReconnect({
session: { sessionId: 's1', isRemote: true, paneDead: true, remoteAlive: undefined },
state: freshReconnectState(),
guarded: false,
enabled: true,
now: 0,
});
expect(action).toEqual({ kind: 'skip', reason: 'remote-gone' });
});

it('skips when the kill-switch is off', () => {
const action = decideReconnect({
session: deadRemote,
Expand DownExpand Up@@ -222,6 +244,11 @@ describe('TmuxManager remote reconnect watcher (integration)', () => {
registerRemote('aaaa1111');
// Force the watcher to see a dead pane regardless of test-mode isPaneDead.
vi.spyOn(manager, 'isPaneDead').mockReturnValue(true);
// The durable remote tmux is still alive (transport drop) → reconnect allowed.
(manager as unknown as { remoteAliveCache: Map<string, boolean | undefined> }).remoteAliveCache.set(
'aaaa1111',
true
);

const dropped: Array<{ sessionId: string; attempt: number }> = [];
const exhausted: Array<{ sessionId: string }> = [];
Expand DownExpand Up@@ -263,6 +290,10 @@ describe('TmuxManager remote reconnect watcher (integration)', () => {
it('resets backoff on a successful reattach (noteRemoteReconnect)', () => {
registerRemote('cccc3333');
vi.spyOn(manager, 'isPaneDead').mockReturnValue(true);
(manager as unknown as { remoteAliveCache: Map<string, boolean | undefined> }).remoteAliveCache.set(
'cccc3333',
true
);

const dropped: Array<{ attempt: number }> = [];
manager.on('remoteSessionDropped', (d) => dropped.push(d));
Expand DownExpand Up@@ -290,6 +321,10 @@ describe('TmuxManager remote reconnect watcher (integration)', () => {
manager.clearRemoteReconnectState('eeee5555');
// After clearing the guard, a fresh dead-pane observation should emit again.
vi.spyOn(manager, 'isPaneDead').mockReturnValue(true);
(manager as unknown as { remoteAliveCache: Map<string, boolean | undefined> }).remoteAliveCache.set(
'eeee5555',
true
);
const dropped: unknown[] = [];
manager.on('remoteSessionDropped', (d) => dropped.push(d));
manager.runRemoteReconnectTick(0, true);
Expand Down