From bafe1b3aecc17d454e271061c4c403a2f093ba4a Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:29:30 -0500 Subject: [PATCH 001/106] agent-herdr: resolve herdr via PATH probe instead of hardcoded ~/.local/bin (S002) --- lib/__tests__/agent-herdr.test.ts | 22 +++++++++++++++++++++- lib/agent-herdr.ts | 30 ++++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/lib/__tests__/agent-herdr.test.ts b/lib/__tests__/agent-herdr.test.ts index 568fdc19..a8bd035c 100644 --- a/lib/__tests__/agent-herdr.test.ts +++ b/lib/__tests__/agent-herdr.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test"; -import { herdrAgentWait, launchInWorkspace, type HerdrRunner } from "../agent-herdr.ts"; +import { defaultHerdrRunner, herdrAgentWait, launchInWorkspace, resolveHerdrBin, type HerdrRunner } from "../agent-herdr.ts"; function scripted(responses: Record) { const calls: string[][] = []; @@ -90,3 +90,23 @@ test("herdrAgentWait builds the current verb (agent wait --until)", async () => await herdrAgentWait("wA:p1", ["idle", "done"], 45000, runner); expect(calls[0]).toEqual(["agent", "wait", "wA:p1", "--until", "idle", "--until", "done", "--timeout", "45000"]); }); + +test("resolveHerdrBin prefers HERDR_BIN over everything else", () => { + const bin = resolveHerdrBin({ HERDR_BIN: "/custom/herdr", HOME: "/home/x" }, () => "/opt/homebrew/bin/herdr"); + expect(bin).toBe("/custom/herdr"); +}); + +test("resolveHerdrBin prefers a PATH-resolved herdr over the ~/.local/bin fallback", () => { + const bin = resolveHerdrBin({ HOME: "/home/x" }, () => "/opt/homebrew/bin/herdr"); + expect(bin).toBe("/opt/homebrew/bin/herdr"); +}); + +test("resolveHerdrBin falls back to ~/.local/bin/herdr when PATH resolution fails", () => { + const bin = resolveHerdrBin({ HOME: "/home/x" }, () => null); + expect(bin).toBe("/home/x/.local/bin/herdr"); +}); + +test("defaultHerdrRunner throws a clear error when the resolved bin does not exist", async () => { + const runner = defaultHerdrRunner({ HERDR_BIN: "/nonexistent/herdr", HOME: "/home/x" }); + await expect(runner(["workspace", "list"])).rejects.toThrow(/herdr not found/); +}); diff --git a/lib/agent-herdr.ts b/lib/agent-herdr.ts index ec3dccee..34761dec 100644 --- a/lib/agent-herdr.ts +++ b/lib/agent-herdr.ts @@ -16,6 +16,7 @@ * visible pane, rt verifies the result from real state afterward. */ +import { existsSync } from "fs"; import { homedir } from "os"; import { join } from "path"; import { runCapture } from "./subprocess.ts"; @@ -23,11 +24,32 @@ import { runCapture } from "./subprocess.ts"; export interface HerdrResult { stdout: string; exitCode: number } export type HerdrRunner = (args: string[]) => Promise; -export function defaultHerdrRunner(): HerdrRunner { - const home = process.env.HOME ?? homedir(); - const bin = process.env.HERDR_BIN ?? join(home, ".local", "bin", "herdr"); - const socket = process.env.HERDR_SOCKET_PATH ?? join(home, ".config", "herdr", "herdr.sock"); +/** + * Mirrors lib/cswap.ts's cswapBin(): Bun.which reads process.env.PATH at + * call time, and the daemon overlays the user's full login PATH onto + * process.env.PATH at boot (lib/daemon.ts resolveUserPath), so this resolves + * a brew-installed herdr even though the daemon's start-env PATH does not + * carry it. The ~/.local/bin fallback preserves the vendor-script install. + */ +export function resolveHerdrBin( + env: NodeJS.ProcessEnv = process.env, + which: (cmd: string) => string | null = (cmd) => Bun.which(cmd), +): string { + if (env.HERDR_BIN) return env.HERDR_BIN; + const onPath = which("herdr"); + if (onPath) return onPath; + const home = env.HOME ?? homedir(); + return join(home, ".local", "bin", "herdr"); +} + +export function defaultHerdrRunner(env: NodeJS.ProcessEnv = process.env): HerdrRunner { + const home = env.HOME ?? homedir(); + const bin = resolveHerdrBin(env); + const socket = env.HERDR_SOCKET_PATH ?? join(home, ".config", "herdr", "herdr.sock"); return async (args) => { + if (!existsSync(bin)) { + throw new Error(`herdr not found at ${bin} (install via \`rt setup\` / brew)`); + } const r = await runCapture([bin, ...args], { timeoutMs: 15_000, stderr: "pipe", From fa5f64b2c565626fc11e262fb7b7cee975b04b31 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:31:47 -0500 Subject: [PATCH 002/106] notifier: retry the post-push queue removal so a busy-swallowed delete can't cause a duplicate delivery (S096) --- lib/__tests__/notifier.test.ts | 30 ++++++++++++++++++++++++++++++ lib/notifier.ts | 27 +++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/lib/__tests__/notifier.test.ts b/lib/__tests__/notifier.test.ts index ed044d38..f9e5e103 100644 --- a/lib/__tests__/notifier.test.ts +++ b/lib/__tests__/notifier.test.ts @@ -280,3 +280,33 @@ describe("conflict-free streak", () => { expect(__test__.shouldRearmConflicts(conflicted, snap)).toBe(false); }); }); + +describe("removeFromQueueWithRetry (busy-swallowed post-push removal)", () => { + test("succeeds on the first attempt when the removal actually took", async () => { + let queued = true; + const removeFn = () => { queued = false; }; + const isQueuedFn = () => queued; + const ok = await __test__.removeFromQueueWithRetry("evt-1", 3, 1, removeFn, isQueuedFn); + expect(ok).toBe(true); + }); + + test("retries when the removal is busy-swallowed, then succeeds", async () => { + let queued = true; + let calls = 0; + const removeFn = () => { + calls++; + if (calls >= 3) queued = false; // the first two deletes are swallowed by SQLITE_BUSY + }; + const isQueuedFn = () => queued; + const ok = await __test__.removeFromQueueWithRetry("evt-2", 5, 1, removeFn, isQueuedFn); + expect(ok).toBe(true); + expect(calls).toBe(3); + }); + + test("gives up after the attempt budget and reports failure instead of hanging", async () => { + const removeFn = () => {}; // never actually removes it + const isQueuedFn = () => true; + const ok = await __test__.removeFromQueueWithRetry("evt-3", 3, 1, removeFn, isQueuedFn); + expect(ok).toBe(false); + }); +}); diff --git a/lib/notifier.ts b/lib/notifier.ts index 55db3c32..0c235c87 100644 --- a/lib/notifier.ts +++ b/lib/notifier.ts @@ -214,6 +214,29 @@ export function peekNotifications(): NotificationEvent[] { * Attempt to push a notification event to the tray app via its Unix socket. * Returns true if the push succeeded, false if tray is unavailable. */ +/** + * removeQueuedNotification's own write already retries on SQLITE_BUSY (3 x + * 20ms, lib/state/notifier-store.ts) and gives up silently. When that + * happens after a successful tray push, the row stays queued and the next + * drainNotifications()/peekNotifications() redelivers it as a duplicate. + * Retry the removal at this layer, bounded, before reporting success. + */ +async function removeFromQueueWithRetry( + eventId: string, + attempts = 3, + delayMs = 50, + removeFn: (id: string) => void = removeQueuedNotification, + isQueuedFn: (id: string) => boolean = isNotificationQueued, +): Promise { + for (let i = 0; i < attempts; i++) { + removeFn(eventId); + if (!isQueuedFn(eventId)) return true; + if (i < attempts - 1) await new Promise((r) => setTimeout(r, delayMs)); + } + log.warn({ eventId }, "notification queue removal kept failing after a successful push; it may redeliver on the next drain"); + return false; +} + async function pushToTray(event: NotificationEvent): Promise { if (!existsSync(TRAY_SOCK_PATH)) return false; @@ -227,8 +250,7 @@ async function pushToTray(event: NotificationEvent): Promise { } as any); if (response.ok) { - // Push succeeded — remove from queue - removeQueuedNotification(event.id); + await removeFromQueueWithRetry(event.id); return true; } return false; @@ -878,6 +900,7 @@ export const __test__ = { notifyFallback, firedKey, pruneFiredForEvictedBranches, + removeFromQueueWithRetry, setFallbackNotifier(path: string | null): void { fallbackNotifier = path ?? "osascript"; }, From ce9c8d6ad3c6923e0c5769260be4f76f4bd36f34 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:35:36 -0500 Subject: [PATCH 003/106] cron: pass the daemon's resolved process.env to spawn instead of the launchd-frozen snapshot (S046) --- lib/daemon/__tests__/cron.test.ts | 34 ++++++++++++++++++++++++++++++- lib/daemon/cron.ts | 8 +++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/lib/daemon/__tests__/cron.test.ts b/lib/daemon/__tests__/cron.test.ts index 1c535b19..b09ab97b 100644 --- a/lib/daemon/__tests__/cron.test.ts +++ b/lib/daemon/__tests__/cron.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { mkdtempSync, realpathSync, rmSync } from "fs"; +import { mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; import { setSetting } from "../../settings/write.ts"; @@ -121,4 +121,36 @@ describe("startCron", () => { await sleep(40); expect(runs).toHaveLength(0); }); + + // The real (non-overridden) spawn path: Bun.spawn without an explicit + // `env` key gives the child the env snapshot from when THIS bun process + // started, not process.env as mutated at runtime (verified directly: + // omitting `env` leaves a child reading a var reassigned post-startup at + // its ORIGINAL value; `env: { ...process.env }` gives it the live one). + // That is exactly what the daemon's boot-time PATH overlay does to + // process.env.PATH, so this exercises the same shape with a plain var — + // an absolute argv[0] sidesteps unrelated PATH-search mechanics. + test("cron spawns with the live process.env, not the frozen start snapshot", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-cron-env-"))); + const marker = join(dir, "ran.txt"); + const script = join(dir, "marker.sh"); + writeFileSync(script, `#!/bin/sh\necho "$CRON_TEST_VALUE" > "${marker}"\n`, { mode: 0o755 }); + + const varName = "CRON_TEST_VALUE"; + delete process.env[varName]; // absent at this bun process's own startup + process.env[varName] = "live-value"; // set only after startup, like resolveUserPath does to PATH + try { + const cron = startCron( + { triggers: [{ name: "t", event: "tick", run: [script], debounceMs: 5 }] }, + { log }, + ); + cron.onBroadcast("tick", null); + await sleep(500); // real process spawn + exit, not a mocked runCommand + cron.dispose(); + expect(readFileSync(marker, "utf8").trim()).toBe("live-value"); + } finally { + delete process.env[varName]; + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/lib/daemon/cron.ts b/lib/daemon/cron.ts index 3720afc9..ef9cc4ee 100644 --- a/lib/daemon/cron.ts +++ b/lib/daemon/cron.ts @@ -75,7 +75,13 @@ function defaultRunCommand(argv: string[], trigger: CronTrigger, log: CronLog): // so a daemon restart kills an in-flight command. Accepted (spec section // 5): invoked programs must be idempotent one-shot passes, and the next // matching event simply re-runs them. - const proc = Bun.spawn(argv, { stdin: "ignore", stdout: "ignore", stderr: "ignore" }); + // Bun.spawn ignores assignments made to process.env after startup unless + // `env` is passed explicitly (mirrors lib/subprocess.ts's runCapture and + // lib/daemon/handlers/agent.ts's defaultSpawnHeadless) — omitting it here + // strands the PATH the daemon overlays onto process.env at boot + // (lib/daemon.ts resolveUserPath) and leaves any non-absolute argv[0] + // (a "#!/usr/bin/env node" shebang, "pnpm", ...) unresolvable. + const proc = Bun.spawn(argv, { stdin: "ignore", stdout: "ignore", stderr: "ignore", env: { ...process.env } }); log.info(`cron ${trigger.name}: spawned "${argv.join(" ")}" (pid ${proc.pid})`); void proc.exited.then((code) => log.info(`cron ${trigger.name}: exited ${code}`)); } catch (err) { From 52be3d9ea06affb8d134a1c8be843338895af15e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:37:43 -0500 Subject: [PATCH 004/106] agent handler: return ok:false when herdr dedups the tab label instead of a phantom record (S051) --- lib/daemon/__tests__/agent-handlers.test.ts | 29 +++++++++++++++++++++ lib/daemon/handlers/agent.ts | 14 +++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/lib/daemon/__tests__/agent-handlers.test.ts b/lib/daemon/__tests__/agent-handlers.test.ts index 80643fff..46f70fa6 100644 --- a/lib/daemon/__tests__/agent-handlers.test.ts +++ b/lib/daemon/__tests__/agent-handlers.test.ts @@ -4,6 +4,7 @@ import { join } from "path"; import { openStateDb } from "../../state/index.ts"; import { createAgentHandlers, type HeadlessChild } from "../handlers/agent.ts"; import type { HerdrRunner } from "../../agent-herdr.ts"; +import { repoLabel } from "../../repo-arg.ts"; let n = 0; const REPO = "remote:example.com%2Fa%2Fb"; @@ -61,6 +62,34 @@ test("agent:start herdr rolls back the inserted record when launch fails", async expect(list.data.agents).toHaveLength(0); }); +// Pins S051: a tab-label dedup must never report success with a phantom +// record nothing is listening on (rt agent resume on it would run +// `claude --resume` for a session that never started). +test("agent:start herdr returns ok:false and rolls back when herdr dedups the tab label", async () => { + const label = "!7"; + const focusCalls: string[][] = []; + const runner: HerdrRunner = async (args) => { + focusCalls.push(args); + if (args[0] === "workspace" && args[1] === "list") { + return { stdout: JSON.stringify({ result: { workspaces: [{ workspace_id: "w1", label: repoLabel(REPO) }] } }), exitCode: 0 }; + } + if (args[0] === "tab" && args[1] === "list") { + return { stdout: JSON.stringify({ result: { tabs: [{ tab_id: "w1:t9", label } ] } }), exitCode: 0 }; + } + return { stdout: "{}", exitCode: 0 }; + }; + const h = fresh({ runner }); + const res = await h["agent:start"]({ repo: REPO, cwd: "/tmp/x", prompt: "hi", surface: "herdr", tab: label }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toMatch(/already open/); + expect(focusCalls.some((c) => c[0] === "tab" && c[1] === "focus")).toBe(true); + expect(focusCalls.some((c) => c[0] === "pane" && c[1] === "run")).toBe(false); + const list = await h["agent:list"]({}); + if (!list.ok) throw new Error("unreachable"); + expect(list.data.agents).toHaveLength(0); +}); + // Pins the guard: a no-op insert (standing in for runCriticalWrite giving up // after sustained SQLITE_BUSY) must block the launch, not just the record. test("agent:start refuses to launch when the insert did not persist", async () => { diff --git a/lib/daemon/handlers/agent.ts b/lib/daemon/handlers/agent.ts index 82fd61ff..9c548642 100644 --- a/lib/daemon/handlers/agent.ts +++ b/lib/daemon/handlers/agent.ts @@ -95,11 +95,17 @@ export function createAgentHandlers(opts: { { workspaceLabel, tabLabel, paneCommand: buildPaneCommand(rec.cwd, inv) }, runner, ); - if (!out.focusedExisting) { - rec.paneId = out.paneId; - rec.tabId = out.tabId; - rec.workspaceId = out.workspaceId; + if (out.focusedExisting) { + // A live tab already answers to this label: launchInWorkspace + // focused it and ran nothing. Reporting ok:true here would insert a + // record with a freshly minted sessionId nothing is listening on — + // rt agent resume against it would run `claude --resume` for a + // session that was never started. + return { ok: false, error: `tab "${tabLabel}" already open; focused it` }; } + rec.paneId = out.paneId; + rec.tabId = out.tabId; + rec.workspaceId = out.workspaceId; return { ok: true, data: rec }; } From 94d6218581b36963490711902290dad08d8cd11f Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:39:06 -0500 Subject: [PATCH 005/106] chat handlers: guard emit/notify after the message commit so a throw can't surface as a failed post (S052) --- lib/daemon/__tests__/chat-handlers.test.ts | 34 +++++++++++++++ lib/daemon/handlers/chat.ts | 48 +++++++++++++++------- 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/lib/daemon/__tests__/chat-handlers.test.ts b/lib/daemon/__tests__/chat-handlers.test.ts index cfa55e66..b54a0774 100644 --- a/lib/daemon/__tests__/chat-handlers.test.ts +++ b/lib/daemon/__tests__/chat-handlers.test.ts @@ -56,6 +56,40 @@ test("chat:post returns the recipients and emits one wake event per recipient", expect(emitted).toEqual(["chat/r/msg", "chat/wake/b"]); }); +test("chat:post still reports success when emitEvent throws after the message is durable", async () => { + const h = freshHandlers(() => { throw new Error("events.db locked"); }); + await h["chat:join"]({ room: "r", handle: "a" }); + const res = await h["chat:post"]({ room: "r", handle: "a", body: "hi" }); + expect(res.ok).toBe(true); + if (!res.ok) throw new Error("unreachable"); + expect(res.data.id).toBeGreaterThan(0); +}); + +test("chat:post continues waking the rest of the recipients when one recipient's emit throws", async () => { + const emitted: string[] = []; + const h = freshHandlers((topic) => { + emitted.push(topic); + if (topic === "chat/wake/a") throw new Error("boom for a"); + return 0; + }); + await h["chat:join"]({ room: "r", handle: "a" }); + await h["chat:join"]({ room: "r", handle: "b" }); + await h["chat:join"]({ room: "r", handle: "c" }); + const res = await h["chat:post"]({ room: "r", handle: "poster", body: "@a @b @c hi" }); + expect(res.ok).toBe(true); + expect(emitted).toContain("chat/wake/a"); + expect(emitted).toContain("chat/wake/b"); + expect(emitted).toContain("chat/wake/c"); +}); + +test("chat:dm still reports success when emitEvent throws after the message is durable", async () => { + const h = freshHandlers(() => { throw new Error("events.db locked"); }); + const res = await h["chat:dm"]({ from: "agent", to: "matt", body: "ping" }); + expect(res.ok).toBe(true); + if (!res.ok) throw new Error("unreachable"); + expect(res.data.id).toBeGreaterThan(0); +}); + test("chat:post rejects an invalid mentions element with a reason rather than storing it", async () => { const h = freshHandlers(); await h["chat:join"]({ room: "r", handle: "a" }); diff --git a/lib/daemon/handlers/chat.ts b/lib/daemon/handlers/chat.ts index f67d14ee..98101e8a 100644 --- a/lib/daemon/handlers/chat.ts +++ b/lib/daemon/handlers/chat.ts @@ -42,9 +42,12 @@ import { chatViewerUrl, readChatViewerUrlSetting } from "../../chat-viewer-url.t import { getSetting } from "../../settings/resolve.ts"; import { herdrRequest, waitTimeout } from "../../herdr/client.ts"; import { herdrError } from "./pane.ts"; +import { lazyChildLogger } from "../../daemon-logger.ts"; import type { Commands } from "../../../packages/rt-client/src/commands.ts"; import type { CommandResult, TypedHandlers } from "./types.ts"; +const log = lazyChildLogger("chat"); + const CHAT_COMMANDS = [ "chat:join", "chat:leave", @@ -94,9 +97,22 @@ function postAndNotify( const { room, handle, body, mentions } = args; const posted = postMessage({ room, handle, body, mentions }, db); if (!posted) return undefined; - emitEvent(`chat/${room}/msg`, { id: posted.id }); + // The row is durable at this point. Every step below is best-effort: a + // throw here (a full disk, an orphan daemon holding an events.db lock) + // must never surface as a failed post — the caller would retry and post + // the message twice — and one recipient's failure must not skip the wake + // for the rest. + try { + emitEvent(`chat/${room}/msg`, { id: posted.id }); + } catch (err) { + log.warn({ err, id: posted.id, room }, "chat: emit for the posted message threw; message is durable, this emit was not"); + } for (const recipient of posted.recipients) { - emitEvent(`chat/wake/${recipient}`, { id: posted.id, room }); + try { + emitEvent(`chat/wake/${recipient}`, { id: posted.id, room }); + } catch (err) { + log.warn({ err, id: posted.id, room, recipient }, "chat: wake emit threw for one recipient; continuing to the rest"); + } } // Independent of chat_members / wake_on: agents create rooms via // join-creates, so the human is typically not a member yet, and a @@ -104,18 +120,22 @@ function postAndNotify( const humanHandle = getSetting("chat.humanHandle").value; const allMentions = mergeMentions(body, mentions); if (humanHandle && allMentions.includes(humanHandle)) { - const dm = dmParticipants(room, db); - const title = dm ? `DM from ${handle}` : `#${room}`; - // The click target: the viewer at this exact message, when the viewer is - // configured. The tray opens `url` on a default click for any category. - notifyEnabled( - CHAT_NOTIFICATION_CATEGORY, - title, - `${handle}: ${body}`, - chatViewerUrl(readChatViewerUrlSetting(), room, posted.id), - undefined, - `chat:${posted.id}`, - ); + try { + const dm = dmParticipants(room, db); + const title = dm ? `DM from ${handle}` : `#${room}`; + // The click target: the viewer at this exact message, when the viewer is + // configured. The tray opens `url` on a default click for any category. + notifyEnabled( + CHAT_NOTIFICATION_CATEGORY, + title, + `${handle}: ${body}`, + chatViewerUrl(readChatViewerUrlSetting(), room, posted.id), + undefined, + `chat:${posted.id}`, + ); + } catch (err) { + log.warn({ err, id: posted.id, room }, "chat: desk notify threw after a successful post"); + } } return posted; } From 7b061da35ddc55805cf6d1235f6f01aa89726a59 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:40:39 -0500 Subject: [PATCH 006/106] discussions:diffs: bound the GitLab fetch with a timeout+abort signal and report truncation at 100 files (S053, S086) --- .../__tests__/discussions-diffs.test.ts | 62 +++++++++++++++++++ lib/daemon/handlers/discussions.ts | 52 +++++++++++++--- 2 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 lib/daemon/__tests__/discussions-diffs.test.ts diff --git a/lib/daemon/__tests__/discussions-diffs.test.ts b/lib/daemon/__tests__/discussions-diffs.test.ts new file mode 100644 index 00000000..7ea75ddf --- /dev/null +++ b/lib/daemon/__tests__/discussions-diffs.test.ts @@ -0,0 +1,62 @@ +/** + * discussions:diffs (S053 + S086): the outbound GitLab fetch must carry a + * bound signal (so a stalled connection doesn't orphan the promise and the + * sops-decrypted token in its closure forever), and a full page (100 rows) + * must be reported as `truncated` rather than silently dropped. + */ +import { expect, test } from "bun:test"; +import { fetchMrDiffs } from "../handlers/discussions.ts"; + +function fakeDiffs(n: number) { + return Array.from({ length: n }, (_, i) => ({ new_path: `file${i}.ts`, diff: `@@ diff ${i}` })); +} + +test("truncated is false when fewer than a full page comes back", async () => { + const fetchFn = (async () => new Response(JSON.stringify(fakeDiffs(3)), { status: 200 })) as typeof fetch; + const out = await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); + expect(out.diffs).toHaveLength(3); + expect(out.truncated).toBe(false); +}); + +test("truncated is true when exactly a full page (100) comes back", async () => { + const fetchFn = (async () => new Response(JSON.stringify(fakeDiffs(100)), { status: 200 })) as typeof fetch; + const out = await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); + expect(out.diffs).toHaveLength(100); + expect(out.truncated).toBe(true); +}); + +test("maps new_path/diff to newPath/diff", async () => { + const fetchFn = (async () => new Response(JSON.stringify([{ new_path: "a.ts", diff: "@@" }]), { status: 200 })) as typeof fetch; + const out = await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); + expect(out.diffs).toEqual([{ newPath: "a.ts", diff: "@@" }]); +}); + +test("a non-ok response throws with the status", async () => { + const fetchFn = (async () => new Response("", { status: 502 })) as typeof fetch; + await expect(fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn })).rejects.toThrow(/502/); +}); + +test("the fetch call carries an AbortSignal", async () => { + let sawSignal: AbortSignal | undefined; + const fetchFn = (async (_url: any, init: any) => { + sawSignal = init?.signal; + return new Response(JSON.stringify([]), { status: 200 }); + }) as typeof fetch; + await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); + expect(sawSignal).toBeInstanceOf(AbortSignal); +}); + +test("aborting the caller's own request signal cancels the in-flight fetch", async () => { + const controller = new AbortController(); + let sawSignal: AbortSignal | undefined; + const fetchFn = (async (_url: any, init: any) => { + sawSignal = init?.signal; + return new Promise((_resolve, reject) => { + init.signal.addEventListener("abort", () => reject(new Error("aborted"))); + }); + }) as typeof fetch; + const promise = fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn, reqSignal: controller.signal }); + controller.abort(); + await expect(promise).rejects.toThrow(); + expect(sawSignal?.aborted).toBe(true); +}); diff --git a/lib/daemon/handlers/discussions.ts b/lib/daemon/handlers/discussions.ts index 86f5adab..5cbc3b3f 100644 --- a/lib/daemon/handlers/discussions.ts +++ b/lib/daemon/handlers/discussions.ts @@ -31,6 +31,40 @@ import type { Commands } from "../../../packages/rt-client/src/commands.ts"; /** Discussions are stable per push; 2min TTL keeps reads fast without going stale. */ const DISCUSSIONS_TTL_MS = 2 * 60 * 1000; +/** GitLab's page size for this endpoint; a full page means there may be more. */ +const DIFFS_PAGE_SIZE = 100; +const DIFFS_FETCH_TIMEOUT_MS = 30_000; + +/** + * Every other outbound fetch in the daemon carries a bound (linear.ts, + * notifier.ts, park.ts); this one previously had none, so a stalled GitLab + * connection left the promise (and the sops-decrypted token in its closure) + * pending indefinitely. `reqSignal` is the client's own request signal + * (handlers/types.ts's Handler(payload, signal?)) so a client giving up + * actually cancels the in-flight GitLab request instead of orphaning it. + * `truncated` reports a full page rather than silently dropping files past it. + */ +export async function fetchMrDiffs( + baseURL: string, + projectPath: string, + iid: number, + token: string, + opts: { reqSignal?: AbortSignal; fetchFn?: typeof fetch } = {}, +): Promise<{ diffs: Array<{ newPath: string; diff: string }>; truncated: boolean }> { + const fetchFn = opts.fetchFn ?? fetch; + const encoded = encodeURIComponent(projectPath); + const url = `${baseURL}/api/v4/projects/${encoded}/merge_requests/${iid}/diffs?per_page=${DIFFS_PAGE_SIZE}`; + const timeout = AbortSignal.timeout(DIFFS_FETCH_TIMEOUT_MS); + const signal = opts.reqSignal ? AbortSignal.any([timeout, opts.reqSignal]) : timeout; + const res = await fetchFn(url, { headers: { "PRIVATE-TOKEN": token }, signal }); + if (!res.ok) throw new Error(`GitLab diffs API: ${res.status}`); + const raw = (await res.json()) as Array<{ new_path: string; diff: string }>; + return { + diffs: raw.map((d) => ({ newPath: d.new_path, diff: d.diff })), + truncated: raw.length >= DIFFS_PAGE_SIZE, + }; +} + export function createDiscussionHandlers( ctx: HandlerContext, broadcast: BroadcastFn, @@ -122,7 +156,7 @@ export function createDiscussionHandlers( } }, - "discussions:diffs": async (payload) => { + "discussions:diffs": async (payload, signal) => { const repoName = payload?.repoName as string | undefined; const iid = payload?.iid as number | undefined; if (!repoName || typeof iid !== "number") { @@ -138,14 +172,14 @@ export function createDiscussionHandlers( const secrets = await loadSecrets(); if (!secrets.gitlabToken) return { ok: false, error: "no gitlabToken in secrets" }; - const encoded = encodeURIComponent(repoCtx.projectPath); - const url = `${repoCtx.provider.baseURL}/api/v4/projects/${encoded}/merge_requests/${iid}/diffs?per_page=100`; - const res = await fetch(url, { headers: { "PRIVATE-TOKEN": secrets.gitlabToken } }); - if (!res.ok) return { ok: false, error: `GitLab diffs API: ${res.status}` }; - - const raw = await res.json() as Array<{ new_path: string; diff: string }>; - const diffs = raw.map((d) => ({ newPath: d.new_path, diff: d.diff })); - return { ok: true, data: { diffs } }; + const { diffs, truncated } = await fetchMrDiffs( + repoCtx.provider.baseURL, + repoCtx.projectPath, + iid, + secrets.gitlabToken, + { reqSignal: signal }, + ); + return { ok: true, data: { diffs, truncated } }; } catch (err) { return { ok: false, error: String(err) }; } From cc924b5c84a13a1dc7abaa40144da2ad81cfe5bc Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:41:56 -0500 Subject: [PATCH 007/106] plan: Phase 0 honest-supervision implementation plan Co-Authored-By: Claude Opus 4.8 (1M context) --- .../plans/2026-08-28-p0-supervision.md | 1088 +++++++++++++++++ 1 file changed, 1088 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-28-p0-supervision.md diff --git a/docs/superpowers/plans/2026-08-28-p0-supervision.md b/docs/superpowers/plans/2026-08-28-p0-supervision.md new file mode 100644 index 00000000..bbbd14d5 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-p0-supervision.md @@ -0,0 +1,1088 @@ +# Phase 0 · Honest Supervision Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make rt daemon crashes visible and recoverable: every boot failure exits non-zero (or parks visibly) instead of becoming a silent zombie, crash/restart history is persisted and surfaced by `rt daemon status`, and start/stop/eviction stop deleting a live daemon's runtime files. + +**Architecture:** Retire the P0/P1 crash-and-hide class in `lib/daemon.ts` and its immediate collaborators. Boot failures become fatal (log + `process.exit(1)`) on the prod path, gated by a boot-phase flag so steady-state stray rejections still recover; crash handlers and stderr redirection move above every module-scope side effect; restart counters and last-exit reasons live in the existing `kv` table (no schema change); `rt daemon status` and `/api/status` gain `alive-not-serving` / `parked` / `boot-failed` / `crash-looping` verdicts; start/stop/eviction become ownership-aware. Two one-line correctness fixes (state.db flavor on the snapshot path, extended busy-code matching + `BEGIN IMMEDIATE`) make the contention policy real. + +**Tech Stack:** Bun, `bun:sqlite`, pino, TypeScript. Tests are `bun test` (unit) and `bun test --preload ./e2e/setup.ts` (e2e, isolated-HOME daemon spawns). + +**Spec:** `/Users/matt/Documents/GitHub/repo-tools/.claude/worktrees/daemon-stability-audit/docs/daemon-stability-audit-2026-08.md` — "Roadmap › Phase 0" plus Appendix A/B entries S001, S003, S004, S009, S011, S012, S026, S027, S028, S029, S030, S035, S036, S037, S043, S044, S060, S072, S073, S074, R001, R002, R007, R017. Each carries a failure scenario, prescribed fix, and fixer notes; read the relevant entry before implementing its task. + +## Global Constraints + +- **No `SCHEMA_VERSION` bump and no new/edited `V*_SCHEMA` block.** Persist all supervision state (restart counters, last-exit reason, boot-failed markers) in the existing `kv` table under namespace `daemon-supervision`, via `setKvValue`/`getKvValue` from `lib/state/kv-blob.ts`. If any task appears to need a schema change, STOP and ask the user (per the job brief's question format) — do not proceed. +- **Never start a daemon or run `rt` against the real machine.** Every daemon or `dist/rt` invocation in a test or check runs under `env -i HOME=` only (repo CLAUDE.md, "Operating on this machine"). e2e daemon spawns already do this via `e2e/setup.ts`; new e2e tests must follow the same isolation. +- **Write fence — do NOT modify these sibling-owned files** (ask the user if a task seems to need one): `lib/daemon/api-server.ts`, `lib/daemon/api-auth.ts`, `lib/daemon/socket-server.ts`, `lib/daemon/handlers/secrets.ts`, `lib/subprocess.ts`, `lib/daemon/cache-refresh.ts`, `lib/git-worktrees.ts`, `lib/daemon/freshness.ts`, `lib/daemon/pollers.ts`, `lib/daemon/worktree-process-kill.ts`, `lib/daemon/system-process-scanner.ts`, `lib/runs/store.ts`, `lib/notifier.ts`, `lib/daemon/handlers/discussions.ts`, `lib/daemon/handlers/chat.ts`, `lib/daemon/handlers/agent.ts`, `lib/daemon/handlers/pane.ts`, `lib/daemon/handlers/project-mrs.ts`, `lib/daemon/handlers/worktree.ts`, `lib/herdr/client.ts`, `lib/port-scanner.ts`, `lib/deps/links.ts`, `lib/worktree/trash.ts`, `lib/agent-herdr.ts`, `lib/daemon/cron.ts`, `lib/daemon/hooks-guard.ts`, `lib/home/age-key.ts`, `lib/daemon/discussions-store.ts`, `lib/state/presence-store.ts`. +- **Every subagent dispatched during execution carries an explicit `model`** (`sonnet` for mechanical tasks, `haiku` for lookups). +- **`packages/rt-client` is touched** (Task 5 edits `registry-defs.ts`). After that task and before the final whole-branch review, run `bun run build` inside `packages/rt-client` (keeps `dist/` and `dist-freshness.test.ts` green). +- **Verification (must pass before the work is done):** + - `bun test lib commands packages scripts` green (from the worktree root) + - `bunx tsc --noEmit` reports zero errors + - `bun run test:e2e` green, or at minimum `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts` (say which was run) + +## Deferred / out-of-fence items (documented, not implemented here) + +- **S073 · `presence-store.ts:signIn` → `.immediate()`** — `lib/state/presence-store.ts` is fenced. Task 7 converts the chat-store and notifier-store read-then-write transactions; the `signIn` caller is a follow-up for the presence-store owner. Not required for verification. +- **0.3 · `rt.apiPort` bind-time consumption** — `lib/daemon/api-server.ts` (the binder) is fenced. Task 5 registers the `rt.apiPort` setting and exposes `resolveApiPort()`; the bind-time read is the api-server sibling's hop. The escape hatch is therefore wired daemon-side but consumed sibling-side; do not claim it functions end-to-end until the sibling reads it. +- **Swift tray edits (S026 dot mapping, S028 `DaemonLifecycle` kickstart fallback, S029 `tray-crash.log` rotation, S060 `AppDelegate` comment)** — grouped as optional Task 16. They cannot be verified by the bun/tsc/e2e gate and the operating rules forbid rebuilding the blessed bundle. Scope confirmation is raised at the plan-review checkpoint. The verifiable CLI-side halves (S028 start→kickstart fallback, S060 exit-code policy) live in Tasks 14 and 12 and are done regardless. + +--- + +## Task 1: Status-verdict + exit-code design sketch + +The half-page design that Tasks 9–14 depend on. No production code; the deliverable is a committed design doc. (Retires nothing directly; anchors R001, R002, S036, S060, S026, S028.) + +**Files:** +- Create: `docs/daemon-supervision-design.md` + +**Interfaces:** +- Produces: the verdict names (`serving`, `alive-not-serving`, `parked`, `boot-failed`, `crash-looping`, `installed-not-running`, `not-installed`), the boot-phase names (`booting` → `ready`), the kv keys under ns `daemon-supervision`, the boot-breadcrumb shape, and the exit-code policy table. Tasks 9/10/12 consume these names verbatim. + +- [ ] **Step 1: Write the design doc** with exactly this content: + +```markdown +# Daemon supervision: status verdicts and exit-code semantics + +Phase 0 design anchor for the rt daemon stability roadmap (audit +2026-08). Tasks 9–14 of the Phase 0 plan implement this. + +## launchd contract + +The prod plist sets `KeepAlive = { SuccessfulExit: false }`: launchd +respawns the daemon ONLY on a non-zero exit. A zero exit means "stay +down". Every exit-code decision below follows from that single fact. + +## Exit-code policy + +| Path | Exit | Why | +|-----------------------------------------|------|-----| +| `startDaemon()` boot throw (prod path) | 1 | Visible + launchd relaunches. Paired with crash-loop detection so it cannot loop silently forever. | +| `shutdown` IPC/REST verb | 0 | Intentional stop; launchd must not respawn. Records `last-exit.kind = "shutdown"`. | +| Bare OS signal SIGTERM/SIGINT/SIGHUP | 1 | External kill (pkill, script, memory pressure); launchd SHOULD respawn. The sanctioned stop path goes through SMAppService.unregister, where the exit code is irrelevant, so exiting non-zero here does not break intended stops. | +| Crash-loop guard trips (N in M minutes) | 0 (park) | Stop the flapping; surface `crash-looping` so a human intervenes instead of launchd hammering every ~10s. | + +Mechanism: a module-scope `shuttingDownViaVerb` flag is set true by the +`shutdown` verb before it calls cleanup; `gracefulExit(signal)` reads it +— set → exit(0), unset (bare signal) → exit(1). + +Boot-phase gate: a module-scope `bootPhase: "booting" | "ready"` flips +to `"ready"` immediately before the `daemon ready` log. The +`unhandledRejection` handler exits(1) while `bootPhase === "booting"` +and only logs (recovers) once ready — so a boot-time stray rejection is +fatal but a steady-state one is not. + +## Status verdicts + +`rt daemon status` and `/api/status` classify by first match: + +1. `not-installed` — SMAppService not registered. +2. `serving` — ping on rt.sock succeeds. +3. `parked` — ping fails, a live rt pid exists, and the boot breadcrumb + phase is a flavor standoff (park). Named distinctly so the user is + told "another flavor owns the socket", not "wedged". +4. `alive-not-serving` — ping fails but a live rt pid exists + (`process.kill(pid,0)` on rt.pid, or `pgrep -f 'rt --daemon|lib/daemon.ts'`). + Sub-detail from the breadcrumb phase: `booting` / `wedged`, or + `quarantined` when a state.db/events.db boot-failed marker is present. + Prints "process is running but not answering rt.sock — rt daemon logs -t". +5. `crash-looping` — no live pid AND the kv failure record shows ≥ N + failures within the last M minutes (N=3, M=5). Prints the last reason. +6. `boot-failed` — no live pid AND the most recent kv exit record is a + boot throw (fewer than N failures). Prints the last reason + phase. +7. `installed-not-running` — registered, no live pid, clean/again-absent + exit record. + +## Persisted state (kv, ns `daemon-supervision`, no schema change) + +- `boot-attempts` (number) — incremented at the top of `runDaemon()`. +- `last-ready-at` (number, epoch ms) — stamped just before `daemon ready`. +- `recent-failures` (array of `{ at, phase, reason }`, capped to 10) — + appended by the boot fatal path and by state.db/events.db boot-failed + markers. Crash-loop = ≥ N entries newer than now − M minutes. +- `last-exit` (`{ at, kind: "shutdown" | "signal" | "boot-failed", code, reason? }`) + — written by the shutdown verb, the signal handlers, and the boot + fatal path. Lets status distinguish "cleanly stopped" from "died". + +## Boot breadcrumb + +`~/.mattstack/rt/daemon-boot.json` = `{ at, pid, flavor, phase }`, +rewritten at each boot phase: `start` → `crash-handlers` → `events-db` +→ `state-db` → `socket` → `api` → `ready`. Lets `alive-not-serving` +name where a live-but-silent daemon is stuck even when the logs are +unreadable. Removed (or stamped `ready`) on successful boot. +``` + +- [ ] **Step 2: Commit** + +```bash +git add docs/daemon-supervision-design.md +git commit -m "docs: sketch daemon supervision verdicts + exit-code semantics" +``` + +--- + +## Task 2: Fatal boot means exit (0.1 — S001, S037) + +Boot failures on the prod path currently become an `unhandledRejection` that only logs, leaving a live-pid zombie with no socket/API. Make a `runDaemon()` throw fatal, gate the rejection handler on a boot-phase flag, and move the rt.pid write to after both binds. + +**Files:** +- Modify: `lib/daemon-logger.ts:232-264` (`installCrashHandlers` gains a `booting` predicate) +- Modify: `lib/daemon.ts:385-514` (wrap `runDaemon()` body; boot-phase flag; move rt.pid write), `lib/daemon.ts:392` (pass predicate), `lib/daemon.ts:513` (flip flag) +- Test: `lib/__tests__/daemon-logger.test.ts` (rejection handler), `e2e/tests/daemon.test.ts` (fatal boot) + +**Interfaces:** +- Produces: `installCrashHandlers(logger, opts?: { booting?: () => boolean })` — when `booting()` is true, `unhandledRejection` logs `fatal` and `process.exit(1)`; otherwise it logs `error` only (today's behavior). Default (no `booting`) preserves today's error-only behavior. +- Produces: module-scope `let bootPhase: "booting" | "ready" = "booting"` in `lib/daemon.ts`, flipped to `"ready"` at line 513. + +- [ ] **Step 1: Write the failing unit test** in `lib/__tests__/daemon-logger.test.ts`: + +```ts +test("unhandledRejection exits(1) while booting, only logs once ready", () => { + const exits: number[] = []; + const origExit = process.exit; + // @ts-expect-error test stub + process.exit = (code?: number) => { exits.push(code ?? 0); }; + const fatal = mock(() => {}); + const error = mock(() => {}); + const logger = makeFakeLogger({ fatal, error }); // existing test helper + let booting = true; + installCrashHandlers(logger, { booting: () => booting }); + process.emit("unhandledRejection", new Error("boot boom"), Promise.resolve()); + expect(fatal).toHaveBeenCalledTimes(1); + expect(exits).toEqual([1]); + booting = false; + process.emit("unhandledRejection", new Error("steady boom"), Promise.resolve()); + expect(error).toHaveBeenCalledTimes(1); + expect(exits).toEqual([1]); // no second exit + process.exit = origExit; +}); +``` + +(If `makeFakeLogger` does not exist, build a minimal `{ info, warn, error, fatal }` of `mock(() => {})`. Remove the listeners this test adds in `afterEach` via `process.removeAllListeners("unhandledRejection")` scoped to the test, matching the file's existing cleanup convention.) + +- [ ] **Step 2: Run it — expect FAIL** (`booting` option not supported): + +Run: `bun test lib/__tests__/daemon-logger.test.ts -t "unhandledRejection exits"` +Expected: FAIL. + +- [ ] **Step 3: Implement in `lib/daemon-logger.ts`.** Change the signature and the `unhandledRejection` branch (currently lines 242-244): + +```ts +export function installCrashHandlers( + logger: Logger, + opts: { booting?: () => boolean } = {}, +): void { + // ... uncaughtException handler unchanged (still fatal + exit 1) ... + process.on("unhandledRejection", (reason) => { + if (opts.booting?.()) { + logger.fatal({ err: reason }, "unhandledRejection during boot"); + process.exit(1); + return; + } + logger.error({ err: reason }, "unhandledRejection"); + }); + // ... stderr interception unchanged ... +} +``` + +- [ ] **Step 4: Run the unit test — expect PASS.** + +- [ ] **Step 5: Wire the boot-phase flag + fatal wrap + rt.pid move in `lib/daemon.ts`.** + - Add near the top of module scope (after imports, before line 78): `let bootPhase: "booting" | "ready" = "booting";` + - At the `installCrashHandlers(loggerHandle)` call (line 392): `installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" });` + - Wrap the body of `runDaemon()` (385-514) in try/catch: + +```ts +async function runDaemon() { + try { + // ... existing body 386-513 ... + } catch (err) { + loggerHandle.fatal?.({ err }, "daemon boot failed"); + // Task 9 adds recordBootFailure(currentPhase, err) here. + try { loggerHandle.flush?.(); } catch {} + process.exit(1); + } +} +``` + + - **Move the rt.pid write** (currently `writeFileSync(DAEMON_PID_PATH, String(process.pid))` at line 416) to AFTER both binds — i.e. after the API bind at line 469 and the socket bind at line 468 (Task 5 will make API bind first; either way, rt.pid is written only once both `servers.socket` and `servers.api` are assigned). A failed boot then never leaves a live-pid file. + - Set `bootPhase = "ready";` immediately before `log.info({ pid }, "daemon ready")` at line 513. + +- [ ] **Step 6: Write the failing e2e test** in `e2e/tests/daemon.test.ts` (uses the isolated-HOME harness already in `e2e/setup.ts`): + +```ts +test("daemon boot with API port already bound exits non-zero and leaves no stale rt.pid", async () => { + // Bind the API port inside the isolated HOME so the daemon cannot. + const port = 9411; + const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + try { + const proc = Bun.spawn({ + cmd: [rtBinary, "--daemon"], + env: { ...isolatedEnv, RT_API_PORT: String(port) }, + stdout: "pipe", stderr: "pipe", + }); + const code = await proc.exited; + expect(code).not.toBe(0); + expect(existsSync(join(isolatedHome, ".mattstack/rt/rt.pid"))).toBe(false); + } finally { + squatter.stop(true); + } +}); +``` + +(Reuse the harness's existing `rtBinary`, `isolatedEnv`, `isolatedHome` fixtures — mirror `e2e/tests/daemon.test.ts`'s existing setup. The daemon's own `startApiServer` retries the bind 6× before throwing, so allow up to the 60s timeout.) + +- [ ] **Step 7: Run the e2e test — expect PASS.** Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "API port already bound"` + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon-logger.ts lib/daemon.ts lib/__tests__/daemon-logger.test.ts e2e/tests/daemon.test.ts +git commit -m "daemon: boot failure is fatal (exit 1), gated by boot-phase flag; rt.pid after binds" +``` + +--- + +## Task 3: Crash handlers first (0.2 — S003, S004, R007) + +Move `redirectNativeStderr()` and `installCrashHandlers()` above every module-scope side effect so a pre-`startDaemon` failure lands in the crash log instead of a discarded stderr. + +**Files:** +- Modify: `lib/daemon.ts` (hoist two calls to module scope; remove the duplicates inside `runDaemon()` at 391-392) +- Test: `e2e/tests/daemon.test.ts` + +**Interfaces:** +- Consumes: `installCrashHandlers(logger, { booting })` from Task 2. + +- [ ] **Step 1: Hoist `redirectNativeStderr()`** to the very first executable statement of `lib/daemon.ts` module scope — before `migrateLegacyRtDir()` at line 78. It depends only on `logsDir()` and has its own internal try/catch, so a subsequent module-scope throw's fd-2 output lands in `daemon-stderr.log`. + +- [ ] **Step 2: Hoist `installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" })`** to immediately after `getDaemonLogger()` resolves (right after line ~95, before `parkUntilIntended` at 103 and before `createEventsBus` at 191). The logger must exist first (line 85), so this is the earliest correct point. + +- [ ] **Step 3: Remove the now-duplicate `redirectNativeStderr()` and `installCrashHandlers()` calls** inside `runDaemon()` (lines 391-392). Keep `mkdirSync(RT_DIR, …)` at 386 (redirect needs the logs dir; `redirectNativeStderr` already mkdirs its own dir, and RT_DIR creation is idempotent — verify the hoisted `redirectNativeStderr` still finds/creates `logsDir()`). + +- [ ] **Step 4: Write the failing e2e test** in `e2e/tests/daemon.test.ts`: + +```ts +test("a corrupt events.db does not crash the daemon silently — error is captured", async () => { + // Pre-create a corrupt events.db in the isolated HOME. + const rtDir = join(isolatedHome, ".mattstack/rt"); + mkdirSync(rtDir, { recursive: true }); + writeFileSync(join(rtDir, "events.db"), "not a sqlite file at all"); + const proc = Bun.spawn({ cmd: [rtBinary, "--daemon"], env: isolatedEnv, stdout: "pipe", stderr: "pipe" }); + // Task 4 makes this self-heal; for Task 3 we only require the failure is captured, not /dev/null. + await Bun.sleep(3000); proc.kill(); + const stderrLog = join(rtDir, "logs", "daemon-stderr.log"); + const quarantined = readdirSync(rtDir).some((f) => f.startsWith("events.db.corrupt-")); + const captured = existsSync(stderrLog) || quarantined; + expect(captured).toBe(true); +}); +``` + +- [ ] **Step 5: Run — expect PASS** (the redirect now runs before events.db construction, so a corruption throw is captured in `daemon-stderr.log`; after Task 4 it is quarantined instead). Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "corrupt events.db"` + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon.ts e2e/tests/daemon.test.ts +git commit -m "daemon: install stderr redirect + crash handlers before every module-scope side effect" +``` + +--- + +## Task 4: events.db joins the discipline (0.4 — S009, S035) + +Give `events.db` the corruption quarantine + busy_timeout/synchronous pragmas state.db has, and guard the two sweep timers so a sync sqlite throw cannot exit the daemon. + +**Files:** +- Modify: `lib/daemon/events-bus.ts:64-224` (quarantine + pragmas in `createEventsBus`) +- Modify: `lib/daemon.ts:194,196` (wrap sweep timers; add a `safeInterval`/`safeTimeout` helper) +- Test: `lib/daemon/__tests__/events-bus.test.ts` + +**Interfaces:** +- Consumes: `isCorruptionError` and `quarantine` shape from `lib/state/db.ts` (reuse the `SQLITE_CORRUPT`/`SQLITE_NOTADB` detection; `events.db` is a bounded-retention journal, so total loss on quarantine is harmless — no migration concern). + +- [ ] **Step 1: Write the failing test** in `lib/daemon/__tests__/events-bus.test.ts`: + +```ts +test("createEventsBus quarantines and recreates a corrupt events.db instead of throwing", () => { + const dir = mkdtempSync(join(tmpdir(), "events-corrupt-")); + const dbPath = join(dir, "events.db"); + writeFileSync(dbPath, "garbage not sqlite"); + const bus = createEventsBus({ dbPath, log: silentLog }); + expect(readdirSync(dir).some((f) => f.startsWith("events.db.corrupt-"))).toBe(true); + // fresh db works: + bus.emit("test", { hi: 1 }); + expect(bus.list({ limit: 1 }).length).toBe(1); + bus.close(); +}); + +test("createEventsBus sets busy_timeout and synchronous=NORMAL", () => { + const dir = mkdtempSync(join(tmpdir(), "events-pragma-")); + const bus = createEventsBus({ dbPath: join(dir, "events.db"), log: silentLog }); + // @ts-expect-error internal handle access for the test, or expose a debug getter + const handle = bus.__db ?? getBusDb(bus); + expect(handle.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); + expect(handle.query("PRAGMA synchronous").get()).toEqual({ synchronous: 1 }); // NORMAL + bus.close(); +}); +``` + +(If the bus does not expose its handle, add a minimal `__db` back-reference or a `busyTimeout()` debug accessor in `events-bus.ts` for the test; do not expose it beyond the module's test needs.) + +- [ ] **Step 2: Run — expect FAIL.** Run: `bun test lib/daemon/__tests__/events-bus.test.ts -t "quarantine"` + +- [ ] **Step 3: Implement in `lib/daemon/events-bus.ts`** — wrap the open (lines 73-86): + +```ts +import { isCorruptionError } from "../state/db"; // export it if not already exported +// ... +mkdirSync(dirname(opts.dbPath), { recursive: true }); +let db: Database; +try { + db = new Database(opts.dbPath, { create: true }); + db.exec("PRAGMA busy_timeout = 250;"); // before journal_mode, matching state/db.ts ordering + db.exec("PRAGMA journal_mode = WAL;"); + db.exec("PRAGMA synchronous = NORMAL;"); + db.query("PRAGMA user_version").get(); // touch to force a real read that trips corruption +} catch (err) { + if (!isCorruptionError(err)) throw err; + quarantineEventsDb(opts.dbPath, opts.log); // rename to events.db.corrupt-, warn + db = new Database(opts.dbPath, { create: true }); + db.exec("PRAGMA busy_timeout = 250;"); + db.exec("PRAGMA journal_mode = WAL;"); + db.exec("PRAGMA synchronous = NORMAL;"); +} +// then the existing CREATE TABLE / CREATE INDEX (78-86) +``` + +Write a local `quarantineEventsDb(path, log)` mirroring `state/db.ts`'s `quarantine` (rename the db + `-wal`/`-shm` sidecars to `path.corrupt-`, `log.warn`). If `isCorruptionError` is not exported from `state/db.ts`, add the export (it is a pure predicate, safe to export). + +- [ ] **Step 4: Run the events-bus tests — expect PASS.** + +- [ ] **Step 5: Write the failing sweep-guard test** in `lib/daemon/__tests__/events-bus.test.ts` OR a small `lib/__tests__/daemon-sweep-guard.test.ts` for the helper: + +```ts +test("safeInterval swallows a throwing tick and logs warn", () => { + const warn = mock(() => {}); + const log = { ...silentLog, warn }; + let ticks = 0; + const handle = safeInterval(() => { ticks++; throw new Error("SQLITE_FULL"); }, 10, "test-sweep", log); + return new Promise((r) => setTimeout(() => { + clearInterval(handle); + expect(ticks).toBeGreaterThan(0); + expect(warn).toHaveBeenCalled(); + r(); + }, 45)); +}); +``` + +- [ ] **Step 6: Implement `safeInterval`/`safeTimeout`** in `lib/daemon.ts` (or a tiny `lib/daemon/safe-timers.ts` exporting both): + +```ts +export function safeInterval(fn: () => void, ms: number, label: string, log: Logger) { + return setInterval(() => { try { fn(); } catch (err) { log.warn({ err, label }, "timer tick failed"); } }, ms); +} +export function safeTimeout(fn: () => void, ms: number, label: string, log: Logger) { + return setTimeout(() => { try { fn(); } catch (err) { log.warn({ err, label }, "timer tick failed"); } }, ms); +} +``` + +Replace the two bare sweep timers at `lib/daemon.ts:194` and `:196` with `safeInterval(() => eventsBus.sweep(), 60*60*1000, "events-sweep", log)` and `safeTimeout(() => eventsBus.sweep(), 30_000, "events-sweep-boot", log)`. (The `pruneRuns`/`pruneLogs` timers at 200-248 already wrap their bodies; leaving them is fine, but converting them to `safeInterval` is a welcome DRY cleanup if trivial.) + +- [ ] **Step 7: Run — expect PASS.** Then `bun test lib/daemon/__tests__/events-bus.test.ts`. + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon/events-bus.ts lib/daemon.ts lib/state/db.ts lib/daemon/__tests__/events-bus.test.ts +git commit -m "events.db: corruption quarantine + busy_timeout/synchronous pragmas; guard sweep timers" +``` + +--- + +## Task 5: Bind order + rt.apiPort setting (0.3, my side — S043, S030 seam) + +Bind the API server before the unix socket so a failed API bind never strands a socket-bound zombie, and register the `rt.apiPort` escape-hatch setting (the api-server sibling consumes it at bind time). + +**Files:** +- Modify: `lib/daemon.ts:468-469` (swap: API bind first, then socket bind; rt.pid still written after both, per Task 2) +- Modify: `lib/daemon-config.ts:72` (add `resolveApiPort()`; keep `API_PORT` const non-breaking) +- Modify: `packages/rt-client/src/settings/registry-defs.ts` (register `rt.apiPort`) +- Test: `lib/__tests__/daemon-config.test.ts`, `packages/rt-client` settings test, `e2e/tests/daemon.test.ts` + +**Interfaces:** +- Produces: `resolveApiPort(): number` in `lib/daemon-config.ts` = `Number(process.env.RT_API_PORT) || getSetting("rt.apiPort").value || 9401`, resolved lazily (never at module load). `API_PORT` const stays as-is for `api-server.ts`'s existing import. + +- [ ] **Step 1: Register the setting.** Append to `packages/rt-client/src/settings/registry-defs.ts` (shaped like `rt.logRetentionDays` at 190-198): + +```ts +{ + key: "rt.apiPort", + type: "number", + scopes: ["machine", "user"], + default: 9401, + merge: "replace", + migrated: true, + description: "TCP port the daemon's local HTTP/WS API binds (escape hatch when 9401 is held).", +}, +``` + +- [ ] **Step 2: Rebuild rt-client** so the daemon and `dist-freshness` see the new key: + +Run: `cd packages/rt-client && bun run build && cd -` + +- [ ] **Step 3: Write the failing test** in `lib/__tests__/daemon-config.test.ts`: + +```ts +test("resolveApiPort: env wins, then setting, then 9401", () => { + const prev = process.env.RT_API_PORT; + process.env.RT_API_PORT = "12345"; + expect(resolveApiPort()).toBe(12345); + delete process.env.RT_API_PORT; + expect(resolveApiPort()).toBe(9401); // default setting value + if (prev !== undefined) process.env.RT_API_PORT = prev; +}); +``` + +- [ ] **Step 4: Run — expect FAIL** (`resolveApiPort` undefined). + +- [ ] **Step 5: Implement in `lib/daemon-config.ts`** (leave `API_PORT` at line 72 untouched): + +```ts +import { getSetting } from "./settings/resolve"; +export function resolveApiPort(): number { + const env = Number(process.env.RT_API_PORT); + if (env) return env; + try { return getSetting("rt.apiPort").value || 9401; } catch { return 9401; } +} +``` + +- [ ] **Step 6: Run — expect PASS.** + +- [ ] **Step 7: Swap the bind order in `lib/daemon.ts`.** Reorder so the API binds before the socket: + +```ts +servers.api = await startApiServer({ handleCommand, log }); // was line 469 — now first +servers.socket = startSocketServer({ handleCommand, log }); // was line 468 — now second +// rt.pid write (moved by Task 2) stays after BOTH assignments +``` + +- [ ] **Step 8: Write the failing e2e test** in `e2e/tests/daemon.test.ts` (extends the Task 2 squatter test): + +```ts +test("API-bind failure leaves neither rt.sock nor rt.pid", async () => { + const port = 9412; + const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + try { + const proc = Bun.spawn({ cmd: [rtBinary, "--daemon"], env: { ...isolatedEnv, RT_API_PORT: String(port) }, stdout: "pipe", stderr: "pipe" }); + await proc.exited; + expect(existsSync(join(isolatedHome, ".mattstack/rt/rt.sock"))).toBe(false); + expect(existsSync(join(isolatedHome, ".mattstack/rt/rt.pid"))).toBe(false); + } finally { squatter.stop(true); } +}); +``` + +- [ ] **Step 9: Run — expect PASS.** Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "API-bind failure"` + +- [ ] **Step 10: Commit** + +```bash +git add lib/daemon.ts lib/daemon-config.ts packages/rt-client/src/settings/registry-defs.ts packages/rt-client/dist lib/__tests__/daemon-config.test.ts e2e/tests/daemon.test.ts +git commit -m "daemon: bind API before socket; register rt.apiPort setting + resolveApiPort()" +``` + +--- + +## Task 6: home-snapshot opens state.db daemon-flavored (0.7 — S011) + +`startHomeSnapshot` opens the state.db singleton at module scope with the default `cli` flavor (5000ms busy_timeout), so the daemon runs with the wrong contention policy forever. Make its db lazy and daemon-flavored, and harden `getStateDb` against a silent flavor mismatch. + +**Files:** +- Modify: `lib/daemon/home-snapshot.ts:272,299` (lazy daemon-flavored db resolver) +- Modify: `lib/state/db.ts:522-530` (`getStateDb` mismatch hardening) +- Test: `lib/state/__tests__/db.test.ts` + +**Interfaces:** +- Consumes: `getStateDb("daemon")` — 250ms busy_timeout. + +- [ ] **Step 1: Write the failing test** in `lib/state/__tests__/db.test.ts` (`describe("pragma values per flavor")`): + +```ts +test("getStateDb('daemon') reports busy_timeout 250 even after a default open", () => { + const cli = getStateDb(); // opens singleton, cli flavor + expect(cli.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 5000 }); + const daemon = getStateDb("daemon"); // same singleton — must not stay at 5000 + expect(daemon.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); +}); +``` + +(Use the file's existing isolated-HOME / `closeStateDb` setup so this does not leak into other tests.) + +- [ ] **Step 2: Run — expect FAIL** (singleton keeps the cli 5000 timeout). + +- [ ] **Step 3: Harden `getStateDb` in `lib/state/db.ts:522-530`** — when the singleton is already open and a caller requests a stronger (shorter) flavor timeout, re-apply the pragma: + +```ts +export function getStateDb(flavor: DbFlavor = "cli"): Database { + const path = stateDbPath(); + if (singleton && singletonPath === path) { + const want = BUSY_TIMEOUT_MS[flavor]; + const have = Number((singleton.query("PRAGMA busy_timeout").get() as any)?.timeout ?? 0); + if (want < have) singleton.exec(`PRAGMA busy_timeout = ${want};`); + return singleton; + } + // ... existing (re)open path ... +} +``` + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Make home-snapshot's db lazy + daemon-flavored** in `lib/daemon/home-snapshot.ts`. Replace the eager `db: rawDeps.db ?? getStateDb()` (line 272) with a thunk defaulting to `() => getStateDb("daemon")`, and resolve it on first use inside `loadState`/`runNow`/`status` rather than at construction (line 299). Concretely: store `const resolveDb = rawDeps.db ? () => rawDeps.db! : () => getStateDb("daemon");` and call `resolveDb()` where `deps.db` was read, so no db opens until `startDaemon` has already opened it daemon-flavored via `openBranchCacheStore`. + +- [ ] **Step 6: Add a boot-order regression test** in `lib/state/__tests__/db.test.ts` (or `home-snapshot.test.ts`) asserting that constructing `startHomeSnapshot` does NOT open the state.db singleton (call it, then assert `getStateDb` was not yet invoked — spy on the module or assert no `state.db` file exists until first use in an isolated HOME). + +- [ ] **Step 7: Run the db tests — expect PASS.** Run: `bun test lib/state/__tests__/db.test.ts` + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon/home-snapshot.ts lib/state/db.ts lib/state/__tests__/db.test.ts +git commit -m "home-snapshot: lazy daemon-flavored state.db; getStateDb re-applies a stronger flavor timeout" +``` + +--- + +## Task 7: Extended busy codes + IMMEDIATE transactions (0.7 — S072, S073) + +`isBusyError` misses `SQLITE_BUSY_SNAPSHOT`/`_RECOVERY`, and read-then-write daemon transactions use a deferred `BEGIN` that produces snapshot conflicts busy_timeout cannot absorb. Widen the match and take the write lock up front. + +**Files:** +- Modify: `lib/state/busy.ts:39-41` (`isBusyError`) +- Modify: `lib/state/chat-store.ts` (`readUnread`, `joinRoom`, `archiveRoom`, `dmRoomFor` → `.immediate()`) +- Modify: `lib/state/notifier-store.ts` (`drainNotificationQueue` → `.immediate()`) +- Test: `lib/state/__tests__/busy.test.ts` +- **Deferred (fenced):** `lib/state/presence-store.ts:signIn` — documented follow-up, not done here. + +**Interfaces:** +- Produces: `isBusyError` returns true for any `code` starting `SQLITE_BUSY`. + +- [ ] **Step 1: Write the failing test** in `lib/state/__tests__/busy.test.ts` — a real two-connection snapshot conflict: + +```ts +test("isBusyError matches SQLITE_BUSY_SNAPSHOT from a real conflict", () => { + const dir = mkdtempSync(join(tmpdir(), "busy-snap-")); + const path = join(dir, "t.db"); + const a = new Database(path); a.exec("PRAGMA journal_mode=WAL; CREATE TABLE t(id INTEGER PRIMARY KEY, v INTEGER);"); + a.exec("INSERT INTO t(id,v) VALUES(1,0);"); + const b = new Database(path); + a.exec("BEGIN;"); a.query("SELECT v FROM t WHERE id=1").get(); // pin snapshot on A + b.exec("UPDATE t SET v=1 WHERE id=1;"); // B commits (autocommit) + let caught: unknown; + try { a.exec("UPDATE t SET v=2 WHERE id=1;"); } catch (e) { caught = e; } + expect(caught).toBeDefined(); + expect((caught as any).code?.startsWith("SQLITE_BUSY")).toBe(true); + expect(isBusyError(caught)).toBe(true); + try { a.exec("ROLLBACK;"); } catch {} + a.close(); b.close(); +}); +``` + +- [ ] **Step 2: Run — expect FAIL** (`isBusyError` returns false for `SQLITE_BUSY_SNAPSHOT`). + +- [ ] **Step 3: Widen `isBusyError`** in `lib/state/busy.ts:39-41`: + +```ts +export function isBusyError(err: unknown): boolean { + const code = (err as { code?: string })?.code; + return code === "SQLITE_BUSY" || (typeof code === "string" && code.startsWith("SQLITE_BUSY_")); +} +``` + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Convert read-then-write transactions to `.immediate()`.** In `lib/state/chat-store.ts`, change the four cited transaction builders (`readUnread` ~483, `joinRoom`, `archiveRoom`, `dmRoomFor`) from `db.transaction(fn)(...)` to `db.transaction(fn).immediate(...)` so the write lock is taken at `BEGIN IMMEDIATE`. Do the same for `drainNotificationQueue` in `lib/state/notifier-store.ts:111`. Leave a one-line comment at the first site: `// BEGIN IMMEDIATE: read-then-write must lock up front or SQLITE_BUSY_SNAPSHOT bypasses busy_timeout.` + +- [ ] **Step 6: Verify no regression** — run the chat/notifier/state suites: `bun test lib/state` and any `chat` command tests. Expect PASS (behavior identical under no contention; the change only affects lock acquisition timing). + +- [ ] **Step 7: Commit** + +```bash +git add lib/state/busy.ts lib/state/chat-store.ts lib/state/notifier-store.ts lib/state/__tests__/busy.test.ts +git commit -m "state: isBusyError matches SQLITE_BUSY_*; read-then-write daemon txns use BEGIN IMMEDIATE" +``` + +--- + +## Task 8: state.db importer isolation (0.3 — S074) + +A throwing legacy importer inside the v0 migration rolls back the whole migration, so `user_version` stays 0 and every subsequent open repeats the failure. Wrap each importer in a SAVEPOINT; on throw, roll back that one importer, warn, and still rename the file. + +**Files:** +- Modify: `lib/state/db.ts:400-417` (`importLegacyStores` — SAVEPOINT per importer) +- Test: `lib/state/__tests__/db.test.ts` + +**Interfaces:** +- Consumes: `LEGACY_IMPORTS` array (line 71), `runMigrations`'s `consumed` list (renames to `.migrated`). + +- [ ] **Step 1: Write the failing test** in `lib/state/__tests__/db.test.ts`: + +```ts +test("a throwing legacy importer is isolated: db reaches SCHEMA_VERSION, other stores import, file renamed", () => { + // Arrange an isolated HOME with a project-mrs.json whose keys "5" and "05" normalize to the same iid, + // plus one benign legacy file another importer consumes. + writeFileSync(join(rtDir, "project-mrs.json"), JSON.stringify({ "host/repo": { mrs: { "5": {...}, "05": {...} } } })); + writeFileSync(join(rtDir, ".json"), JSON.stringify(benignFixture)); + const db = openStateDb(join(rtDir, "state.db"), "daemon"); + expect(db.query("PRAGMA user_version").get()).toEqual({ user_version: SCHEMA_VERSION }); + // benign store's rows landed: + expect(db.query("SELECT COUNT(*) c FROM ").get()).toMatchObject({ c: benignCount }); + // offending file still renamed: + expect(existsSync(join(rtDir, "project-mrs.json"))).toBe(false); + expect(existsSync(join(rtDir, "project-mrs.json.migrated"))).toBe(true); +}); +``` + +(Fill ``/``/fixtures from an existing importer in `LEGACY_IMPORTS`; the db.test.ts legacy-import cases already build such fixtures — reuse one.) + +- [ ] **Step 2: Run — expect FAIL** (the throw rolls back the whole migration; `user_version` stays 0 / benign rows absent). + +- [ ] **Step 3: Implement SAVEPOINT-per-importer** in `importLegacyStores` (`lib/state/db.ts:400-417`). For each `LEGACY_IMPORTS` entry, wrap its `run(db)` in a savepoint scoped to that importer only (do NOT loosen the surrounding schema-DDL migration, which must stay loud): + +```ts +for (const imp of LEGACY_IMPORTS) { + if (!existsSync(imp.path())) continue; + db.exec("SAVEPOINT legacy_import;"); + try { + imp.run(db); + db.exec("RELEASE legacy_import;"); + consumed.push(imp.path()); + } catch (err) { + db.exec("ROLLBACK TO legacy_import; RELEASE legacy_import;"); + log?.warn?.({ err, file: imp.path() }, "legacy import failed; skipping (file will still be renamed)"); + consumed.push(imp.path()); // spec: corrupt = warn + skip, but still rename + } +} +``` + +(Match the exact `LegacyImport` shape at db.ts:64-69 — `path()`/`run(db)` names may differ; adapt to the real fields.) + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Commit** + +```bash +git add lib/state/db.ts lib/state/__tests__/db.test.ts +git commit -m "state.db: isolate each legacy importer in a SAVEPOINT so one bad file cannot wedge migration" +``` + +--- + +## Task 9: Restart counter + last-exit reason in kv (0.5 — R002, S037) + +Persist boot attempts, last-ready stamp, recent failures, and last-exit reason in the `kv` table (ns `daemon-supervision`), plus a boot breadcrumb file. Wire the record calls into the boot path, the shutdown verb, the signal handlers, and the boot fatal path. No schema change. + +**Files:** +- Create: `lib/daemon/supervision-state.ts` +- Create: `lib/daemon/__tests__/supervision-state.test.ts` +- Modify: `lib/daemon.ts` (record boot-attempt at `runDaemon` top; breadcrumb per phase; ready stamp; `recordBootFailure` in Task 2's catch; `recordCleanExit("shutdown")` in the shutdown verb), `lib/daemon/shutdown.ts` (record exit in `gracefulExit`) + +**Interfaces:** +- Produces: + - `recordBootAttempt(): void` — increments `boot-attempts`, appends nothing. + - `recordDaemonReady(): void` — sets `last-ready-at = Date.now()`. + - `recordBootFailure(phase: BootPhase, reason: string): void` — appends `{ at, phase, reason }` to `recent-failures` (cap 10), sets `last-exit = { at, kind: "boot-failed", code: 1, reason }`. + - `recordCleanExit(kind: "shutdown" | "signal", code: number): void` — sets `last-exit = { at, kind, code }`. + - `readSupervisionState(): { bootAttempts, lastReadyAt, recentFailures, lastExit }`. + - `isCrashLooping(state, now, n = 3, windowMs = 5*60_000): boolean` — ≥ n failures newer than `now - windowMs`. + - `writeBreadcrumb(phase: BootPhase): void` / `clearBreadcrumb(): void` — `~/.mattstack/rt/daemon-boot.json`. + - `type BootPhase = "start" | "crash-handlers" | "events-db" | "state-db" | "socket" | "api" | "ready"`. +- Consumes: `getKvValue`/`setKvValue` from `lib/state/kv-blob.ts`; `getStateDb("daemon")`. + +- [ ] **Step 1: Write the failing test** in `lib/daemon/__tests__/supervision-state.test.ts`: + +```ts +test("boot attempts, ready stamp, failures and last-exit round-trip through kv", () => { + recordBootAttempt(); recordBootAttempt(); + recordDaemonReady(); + recordBootFailure("api", "EADDRINUSE"); + const s = readSupervisionState(); + expect(s.bootAttempts).toBe(2); + expect(s.lastReadyAt).toBeGreaterThan(0); + expect(s.recentFailures.at(-1)).toMatchObject({ phase: "api", reason: "EADDRINUSE" }); + expect(s.lastExit).toMatchObject({ kind: "boot-failed", code: 1 }); +}); + +test("isCrashLooping true at >=3 failures within the window", () => { + const now = 1_000_000; + const fails = [now-10, now-20, now-30].map((at) => ({ at, phase: "api" as const, reason: "x" })); + expect(isCrashLooping({ bootAttempts: 3, lastReadyAt: 0, recentFailures: fails, lastExit: null }, now)).toBe(true); + const old = [{ at: now-10*60_000, phase: "api" as const, reason: "x" }]; + expect(isCrashLooping({ bootAttempts: 1, lastReadyAt: 0, recentFailures: old, lastExit: null }, now)).toBe(false); +}); +``` + +(Use the file's isolated-HOME convention — `bunfig` preload already repoints HOME for `bun test`; `recordBootAttempt` writes to the test state.db.) + +- [ ] **Step 2: Run — expect FAIL** (module does not exist). + +- [ ] **Step 3: Implement `lib/daemon/supervision-state.ts`** with the interfaces above. All reads/writes go through `getKvValue("daemon-supervision", key, fallback, getStateDb("daemon"))` / `setKvValue("daemon-supervision", key, value, getStateDb("daemon"))`. Cap `recent-failures` at 10 on append. Breadcrumb via `writeFileSync(join(RT_DIR, "daemon-boot.json"), JSON.stringify({ at, pid: process.pid, flavor: currentMode(), phase }))` inside a try/catch (never fatal). + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Wire into the boot path** (`lib/daemon.ts`): + - `recordBootAttempt(); writeBreadcrumb("start");` at the top of `runDaemon()` (after Task 3's crash handlers are already at module scope; call these first thing inside `runDaemon`). + - `writeBreadcrumb("events-db" | "state-db" | "socket" | "api")` at each corresponding phase (events.db is module-scope — write that breadcrumb right after `createEventsBus`; state-db right after `openBranchCacheStore`; socket/api at the binds). + - In Task 2's catch: `recordBootFailure(currentPhase, String(err));` (track `currentPhase` in a module var updated alongside each `writeBreadcrumb`). + - `recordDaemonReady(); writeBreadcrumb("ready");` right where `bootPhase = "ready"` is set (line 513). + - Shutdown verb (349-364): before `process.exit(0)`, `recordCleanExit("shutdown", 0);` and set the Task 12 `shuttingDownViaVerb = true` flag (Task 12 adds the flag; here just add the record call). + +- [ ] **Step 6: Wire into signal exit** (`lib/daemon/shutdown.ts` `gracefulExit`): before exit, `recordCleanExit("signal", code)` (Task 12 sets `code` to 1 for bare signals; for now record with the code it exits with). + +- [ ] **Step 7: Add an e2e assertion** in `e2e/tests/daemon.test.ts`: after the Task 5 API-bind-failure spawn, assert `daemon-boot.json` exists with `phase: "api"` and the kv `recent-failures` has an entry (read the state.db in the isolated HOME, or assert via `rt daemon status --json` once Task 10 lands — for Task 9, assert the breadcrumb file only). + +- [ ] **Step 8: Run the unit + e2e tests — expect PASS.** + +- [ ] **Step 9: Commit** + +```bash +git add lib/daemon/supervision-state.ts lib/daemon/__tests__/supervision-state.test.ts lib/daemon.ts lib/daemon/shutdown.ts e2e/tests/daemon.test.ts +git commit -m "daemon: persist boot attempts, failures, last-exit in kv + boot breadcrumb" +``` + +--- + +## Task 10: Status verdicts (0.5 — R001, S026 daemon-side) + +Extend `DaemonStatusVerdict` and `classifyDaemonStatus` with `alive-not-serving`, `parked`, `boot-failed`, `crash-looping`, read from the liveness probe + supervision state + breadcrumb. Expose the fields in `ping`/`/api/status`, and print them in `rt daemon status`. + +**Files:** +- Modify: `lib/daemon-status.ts:14-19` (verdict type), `31-…` (`classifyDaemonStatus`) +- Modify: `lib/daemon/handlers/status.ts:23` (`ping` includes supervision summary; NOT fenced) +- Modify: `commands/daemon.ts:379-…` (`statusLines` prints new verdicts) +- Test: `lib/__tests__/daemon-status.test.ts` + +**Interfaces:** +- Consumes: `readSupervisionState`, `isCrashLooping`, breadcrumb reader from Task 9. +- Produces: `DaemonStatusVerdict` extended with `{ state: "alive-not-serving"; pid: number; detail: "booting" | "wedged" | "quarantined" }`, `{ state: "parked"; pid: number; holderFlavor?: string }`, `{ state: "boot-failed"; reason: string; phase: string }`, `{ state: "crash-looping"; failures: number; reason: string }`. + +- [ ] **Step 1: Write the failing tests** in `lib/__tests__/daemon-status.test.ts`: + +```ts +test("alive pid + failed ping -> alive-not-serving with breadcrumb detail", () => { + const v = classifyDaemonStatus({ installed: true, pingOk: false, pidAlive: true, pid: 42, + breadcrumb: { phase: "socket" }, supervision: emptySupervision() }); + expect(v).toMatchObject({ state: "alive-not-serving", pid: 42, detail: "booting" }); +}); +test("no pid + >=3 recent failures -> crash-looping", () => { + const v = classifyDaemonStatus({ installed: true, pingOk: false, pidAlive: false, pid: null, + supervision: { recentFailures: threeRecentFailures(), lastExit: { kind: "boot-failed", reason: "EADDRINUSE" } } }); + expect(v).toMatchObject({ state: "crash-looping" }); +}); +test("no pid + single boot-failed -> boot-failed with reason", () => { + const v = classifyDaemonStatus({ installed: true, pingOk: false, pidAlive: false, pid: null, + supervision: { recentFailures: [oneFailure("api","EADDRINUSE")], lastExit: { kind: "boot-failed", reason: "EADDRINUSE", phase: "api" } } }); + expect(v).toMatchObject({ state: "boot-failed", reason: "EADDRINUSE", phase: "api" }); +}); +``` + +- [ ] **Step 2: Run — expect FAIL.** + +- [ ] **Step 3: Extend the verdict type and `classifyDaemonStatus`** in `lib/daemon-status.ts` following the resolution order from the design doc (Task 1): not-installed → serving → parked → alive-not-serving → crash-looping → boot-failed → installed-not-running. `classifyDaemonStatus` takes the already-gathered inputs (`pingOk`, `pidAlive`, `pid`, `breadcrumb`, `supervision`); keep it a pure function (the liveness probe and kv/breadcrumb reads happen in `commands/daemon.ts`/the caller, matching the existing `needsLivenessProbe` split). `parked` when breadcrumb phase indicates a flavor standoff; `alive-not-serving` detail = `booting` (phase < ready), `wedged` (phase == ready), `quarantined` (a `*.boot-failed` marker present). + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Surface the data.** In `lib/daemon/handlers/status.ts` `ping`, add a `supervision: { bootAttempts, lastReadyAt, recentFailures: recentFailures.slice(-3), lastExit }` field (read via Task 9). In `commands/daemon.ts` `showStatus`/liveness probe, gather `pidAlive` (`process.kill(pid,0)` on rt.pid, fallback `pgrep -f 'rt --daemon|lib/daemon.ts'` via `runCapture`) and read the breadcrumb + supervision state (when ping fails), then pass to `classifyDaemonStatus`. In `statusLines`, print a line per new verdict (see the design doc's status strings). + +- [ ] **Step 6: Add a `--json` assertion e2e** in `e2e/tests/daemon.test.ts`: after an API-bind-failure spawn, `rt daemon status --json` under the same isolated HOME reports `boot-failed` or `crash-looping` (whichever the failure count yields). Run under `env -i HOME=`. + +- [ ] **Step 7: Run the unit + e2e tests — expect PASS.** `bun test lib/__tests__/daemon-status.test.ts` + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon-status.ts lib/daemon/handlers/status.ts commands/daemon.ts lib/__tests__/daemon-status.test.ts e2e/tests/daemon.test.ts +git commit -m "daemon status: alive-not-serving / parked / boot-failed / crash-looping verdicts" +``` + +--- + +## Task 11: stderr log rotation + stale-crash stamp (0.5 — S029) + +`daemon-stderr.log` is never rotated and its stale contents are shown as "most recent crash". Rotate on open and gate the crash block on mtime. + +**Files:** +- Modify: `lib/daemon-logger.ts:204-219` (`redirectNativeStderr` rotate-on-open) +- Modify: `commands/daemon.ts:716-765` (`showLogs` mtime gate + label) +- Test: `lib/__tests__/daemon-logger.test.ts`, `commands/__tests__/daemon.test.ts` (or wherever `showLogs` is tested) + +**Interfaces:** +- Produces: rotated filename `daemon-stderr..log` (matches the janitor's `LOG_FILE_PATTERN`, so it prunes for free). + +- [ ] **Step 1: Write the failing test** in `lib/__tests__/daemon-logger.test.ts`: + +```ts +test("redirectNativeStderr rotates a non-empty daemon-stderr.log before reopening", () => { + // isolated logs dir with a pre-existing non-empty daemon-stderr.log + writeFileSync(join(logsDir(), "daemon-stderr.log"), "old panic\n"); + redirectNativeStderr(); + const rotated = readdirSync(logsDir()).filter((f) => /^daemon-stderr\.\d{4}-\d{2}-\d{2}\.log$/.test(f)); + expect(rotated.length).toBe(1); + expect(statSync(join(logsDir(), "daemon-stderr.log")).size).toBe(0); +}); +``` + +- [ ] **Step 2: Run — expect FAIL.** + +- [ ] **Step 3: Implement rotate-on-open** in `redirectNativeStderr` (`lib/daemon-logger.ts:204-219`): before `openSync(path, "a")`, if the file exists and is non-empty, `renameSync` it to `daemon-stderr..log` (dedupe with a `.N` suffix if that name already exists, matching the janitor's dated-file convention). Keep the existing swallow-on-failure behavior. + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Write the failing `showLogs` test** — the native-stderr block is hidden when the file mtime predates the current daemon's start: + +```ts +test("showLogs hides the native-stderr block when the file is older than the daemon start", () => { + // stub daemon startedAt newer than the file mtime; assert the red block is not printed +}); +``` + +- [ ] **Step 6: Implement in `showLogs`** (`commands/daemon.ts:724-737`): only print the native-stderr block when the file's `mtime` is newer than the current daemon's `startedAt` (from `ping`); include the mtime in the header (`native stderr (captured )`). When older, skip it silently (or print a one-line "no crash since this daemon started"). + +- [ ] **Step 7: Run — expect PASS.** + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon-logger.ts commands/daemon.ts lib/__tests__/daemon-logger.test.ts +git commit -m "logs: rotate daemon-stderr.log on open; hide stale crash block by mtime" +``` + +--- + +## Task 12: Exit-code semantics (0.6 — S036, S060) + +The `shutdown` verb correctly exits 0, but bare OS signals also exit 0, so an externally-killed daemon stays down. Reserve exit 0 for the verb; exit non-zero on bare signals. + +**Files:** +- Modify: `lib/daemon.ts:349-364` (shutdown verb sets `shuttingDownViaVerb = true`) +- Modify: `lib/daemon/shutdown.ts:57-71` (`gracefulExit` reads the flag) +- Test: `lib/daemon/__tests__/shutdown.test.ts` (new) + +**Interfaces:** +- Produces: a shared boolean the shutdown verb sets before cleanup; `installSignalHandlers` gains a `wasVerbShutdown: () => boolean` option (avoids a cross-module global). `gracefulExit` exits 0 when `wasVerbShutdown()` is true, else `process.exit(1)`. +- Consumes: `recordCleanExit` (Task 9). + +- [ ] **Step 1: Write the failing test** in `lib/daemon/__tests__/shutdown.test.ts`: + +```ts +test("gracefulExit exits 0 after the shutdown verb, 1 on a bare signal", () => { + const exits: number[] = []; + const exit = (c?: number) => { exits.push(c ?? 0); }; + let viaVerb = false; + const handlers = makeGracefulExit({ cleanup: () => {}, flushLogs: () => {}, log: silentLog, + wasVerbShutdown: () => viaVerb, exit, recordCleanExit: () => {} }); + handlers("SIGTERM"); + expect(exits).toEqual([1]); + viaVerb = true; + handlers("SIGTERM"); + expect(exits).toEqual([1, 0]); +}); +``` + +(Refactor `gracefulExit` into a testable `makeGracefulExit(deps)` that returns the handler, injecting `exit`/`recordCleanExit` so no real `process.exit` fires in the test.) + +- [ ] **Step 2: Run — expect FAIL.** + +- [ ] **Step 3: Implement.** In `lib/daemon/shutdown.ts`, refactor `installSignalHandlers`/`gracefulExit` to `makeGracefulExit(deps)` reading `deps.wasVerbShutdown()`: true → `recordCleanExit("shutdown", 0)` + `exit(0)`; false → `recordCleanExit("signal", 1)` + `exit(1)`. In `lib/daemon.ts`, add module-scope `let shuttingDownViaVerb = false;`, set it `true` in the shutdown verb before `cleanup()`, and pass `wasVerbShutdown: () => shuttingDownViaVerb` into `installSignalHandlers` at line 511. The shutdown verb keeps `process.exit(0)`. + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon.ts lib/daemon/shutdown.ts lib/daemon/__tests__/shutdown.test.ts +git commit -m "daemon: bare-signal exit is non-zero (launchd respawns); shutdown verb stays exit 0" +``` + +--- + +## Task 13: Ownership-aware cleanup + eviction death-confirmation (0.6 — S012, S044) + +`cleanup()` unlinks rt.sock/rt.pid unconditionally, and eviction sleeps a blind 300ms. Make cleanup compare-and-delete, and make eviction wait for the old pid to actually die. + +**Files:** +- Modify: `lib/daemon/shutdown.ts:44-46` (ownership-aware unlink) +- Modify: `lib/daemon/boot-reconcile.ts:16-26` (poll-to-death, escalate to SIGKILL) +- Test: `lib/daemon/__tests__/shutdown.test.ts`, `lib/daemon/__tests__/boot-reconcile.test.ts` (new) + +**Interfaces:** +- Produces: `cleanup()` unlinks rt.pid only when its content `=== String(process.pid)`, and gates the rt.sock unlink on that same check. `evictStaleDaemon` polls `process.kill(pid, 0)` up to a bound (e.g. 3s / 30×100ms), escalates to SIGKILL, and only returns once the pid is gone. + +- [ ] **Step 1: Write the failing cleanup test** in `lib/daemon/__tests__/shutdown.test.ts`: + +```ts +test("cleanup does not unlink rt.pid/rt.sock when the pid file belongs to another process", () => { + writeFileSync(DAEMON_PID_PATH, "999999"); // not our pid + writeFileSync(DAEMON_SOCK_PATH, ""); + createCleanup({ ...deps, pid: process.pid })(); + expect(existsSync(DAEMON_PID_PATH)).toBe(true); + expect(existsSync(DAEMON_SOCK_PATH)).toBe(true); +}); +test("cleanup unlinks when the pid file is ours", () => { + writeFileSync(DAEMON_PID_PATH, String(process.pid)); + writeFileSync(DAEMON_SOCK_PATH, ""); + createCleanup({ ...deps, pid: process.pid })(); + expect(existsSync(DAEMON_PID_PATH)).toBe(false); + expect(existsSync(DAEMON_SOCK_PATH)).toBe(false); +}); +``` + +(Inject `pid` into `createCleanup` deps for testability; default to `process.pid`.) + +- [ ] **Step 2: Run — expect FAIL.** + +- [ ] **Step 3: Implement ownership-aware unlink** in `createCleanup` (`lib/daemon/shutdown.ts:44-46`): + +```ts +try { + if (existsSync(DAEMON_PID_PATH) && readFileSync(DAEMON_PID_PATH, "utf8").trim() === String(deps.pid)) { + unlinkSync(DAEMON_PID_PATH); + if (existsSync(DAEMON_SOCK_PATH)) unlinkSync(DAEMON_SOCK_PATH); + } +} catch (err) { deps.log.warn({ err }, "cleanup unlink skipped"); } +``` + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Write the failing eviction test** in `lib/daemon/__tests__/boot-reconcile.test.ts`: + +```ts +test("evictStaleDaemon waits for the old pid to die, escalating to SIGKILL", async () => { + // Spawn a child that ignores SIGTERM; write its pid to rt.pid; assert evict SIGKILLs it and returns only once gone. + const child = Bun.spawn({ cmd: ["bash", "-c", "trap '' TERM; sleep 30"] }); + writeFileSync(DAEMON_PID_PATH, String(child.pid)); + const start = Date.now(); + await evictStaleDaemon(silentLog); // now async + expect(isAlive(child.pid)).toBe(false); + expect(Date.now() - start).toBeLessThan(5000); +}); +``` + +- [ ] **Step 6: Implement in `evictStaleDaemon`** (`lib/daemon/boot-reconcile.ts`): replace `Bun.sleepSync(300)` with an async poll — after SIGTERM, loop `process.kill(pid, 0)` every 100ms up to ~2.5s; if still alive, `process.kill(pid, "SIGKILL")` and poll another ~0.5s; return once `process.kill(pid,0)` throws (pid gone). Make the function `async` and `await` it at the call site (`lib/daemon.ts:396`). Keep the `previousPid === process.pid` self-guard. + +- [ ] **Step 7: Run — expect PASS.** Then `bun test lib/daemon/__tests__/boot-reconcile.test.ts lib/daemon/__tests__/shutdown.test.ts` + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon/shutdown.ts lib/daemon/boot-reconcile.ts lib/daemon.ts lib/daemon/__tests__/shutdown.test.ts lib/daemon/__tests__/boot-reconcile.test.ts +git commit -m "daemon: ownership-aware socket/pid unlink; eviction waits for pid death then SIGKILL" +``` + +--- + +## Task 14: uninstall + start guards (0.6 — S027, S030, S028 CLI-side) + +`rt daemon uninstall` deletes rt.sock/rt.pid from under a live daemon, and `rt daemon start` cannot revive a registered-but-exited-0 daemon. Guard uninstall on liveness; make start escalate to a kickstart route. + +**Files:** +- Modify: `commands/daemon.ts:210-231` (`uninstall` guard), `commands/daemon.ts:235-269` (`start` escalation) +- Modify: `lib/daemon-client.ts:171-185` (`attemptRestart` re-probes liveness) +- Test: `commands/__tests__/daemon.test.ts` (or the existing commands/daemon test location) + +**Interfaces:** +- Consumes: `isDaemonProcessRunning()` (daemon-config.ts:152), `probeSocketHolder()` (park.ts), `activeLaunchdLabel()` (daemon-config.ts:61), `isDaemonRunning()`. + +- [ ] **Step 1: Write the failing uninstall test**: + +```ts +test("uninstall leaves rt.sock/rt.pid and daemon.json when the daemon is still running", async () => { + // stub trayQuery('/daemon/stop') to fail, isDaemonProcessRunning -> true + await uninstall(); + expect(cleanupDaemonFilesSpy).not.toHaveBeenCalled(); + expect(printedRemedy).toContain("launchctl bootout"); +}); +``` + +- [ ] **Step 2: Run — expect FAIL.** + +- [ ] **Step 3: Implement the uninstall guard** (`commands/daemon.ts:210-231`): after a failed/absent `trayQuery("/daemon/stop")`, call `isDaemonProcessRunning()` (and `probeSocketHolder()` as a second signal). Only run `markDaemonUninstalled()` + `cleanupDaemonFiles()` when no live holder; otherwise print the remedy `launchctl bootout gui/$UID/${activeLaunchdLabel()}` and leave the files. Audit other callers of `cleanupDaemonFiles`/`markDaemonUninstalled` (e.g. the dev-mode toggle in `commands/settings.ts`) for the same missing guard and note any in the commit body. + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Write the failing start-escalation test**: + +```ts +test("start escalates to the restart/kickstart route when the tray acks but the socket stays absent", async () => { + // trayQuery('/daemon/start') returns ok, isDaemonRunning stays false through the poll + await start(); + expect(restartRouteSpy).toHaveBeenCalled(); +}); +``` + +- [ ] **Step 6: Implement start escalation** (`commands/daemon.ts:235-269`): when the tray acked `/daemon/start` but `isDaemonRunning()` stays false through the 12×250ms poll, fall back to the `/daemon/restart` route (kickstart). In `lib/daemon-client.ts:171-185`, make `attemptRestart` re-probe `isDaemonRunning()` after `trayQuery("/daemon/start")` and return `true` only when the daemon actually answers — so the `daemonQuery` nag stops misdirecting. Do NOT change signal-handler exit codes here (that is Task 12). + +- [ ] **Step 7: Run — expect PASS.** + +- [ ] **Step 8: Commit** + +```bash +git add commands/daemon.ts lib/daemon-client.ts commands/__tests__/daemon.test.ts +git commit -m "daemon CLI: uninstall guards on liveness; start escalates to kickstart; attemptRestart re-probes" +``` + +--- + +## Task 15: Retire the stale audit doc (0.8 — R017) + +Replace `docs/daemon-runner-health.md` (which audits deleted subsystems) with a pointer to the current audit and the new supervision design doc. + +**Files:** +- Modify: `docs/daemon-runner-health.md` + +- [ ] **Step 1: Replace the file's contents** with a short pointer: + +```markdown +# Daemon runner health — superseded + +This document audited subsystems (process-manager, remedy-engine, +runner.tsx, workspace-sync) that no longer exist. It is retained only as +a redirect. + +- Current stability audit + roadmap: `docs/daemon-stability-audit-2026-08.md` + (in the daemon-stability-audit worktree). +- Supervision verdicts + exit-code semantics: `docs/daemon-supervision-design.md`. +``` + +(If R058's comment sweep is trivially co-located, remove the one stale comment it names; otherwise leave it.) + +- [ ] **Step 2: Commit** + +```bash +git add docs/daemon-runner-health.md +git commit -m "docs: retire stale daemon-runner-health.md, point at the current audit + supervision design" +``` + +--- + +## Task 16 (OPTIONAL — pending plan-review scope confirmation): Swift tray consumption + +**Do not start without the reviewer's go-ahead** (raised in the plan-milestone report). These edits cannot be verified by the bun/tsc/e2e gate, and the operating rules forbid rebuilding the blessed bundle, so they ship as source-only, unverified changes following the fixer notes: + +- **S026** — `rt-tray/Sources/AppDelegate.swift`: give `.starting` an expiry (record `startingSince`; in `refreshStatus` treat `.starting` as expired after ~30s / 3 failed polls and fall through to `setHealth(.down)`), so the health dot stops sticking yellow; map the new daemon verdicts (Task 10) to dot colors. +- **S028** — `rt-tray/Sources/DaemonLifecycle.swift`: when `register()` returns already-registered but the socket stays unreachable, fall back to `launchctl kickstart` (Kickstart.arguments already exists). +- **S029** — `rt-tray/Sources/TrayLog.swift`: rotate `tray-crash.log` on open (rename-if-nonempty), matching Task 11's `daemon-stderr.log` treatment. +- **S060** — `rt-tray/Sources/AppDelegate.swift:625-627`: fix the stale comment to say `KeepAlive: SuccessfulExit=false` (not `KeepAlive=true`). + +If confirmed, do the comment fix (S060) first (trivial), then S026/S028/S029, one commit each, each commit body noting "source-only, unverified: blessed bundle not rebuilt". + +--- + +## Final verification (before the whole-branch review) + +- [ ] `cd packages/rt-client && bun run build && cd -` (rt-client was touched in Task 5) +- [ ] `bunx tsc --noEmit` — zero errors +- [ ] `bun test lib commands packages scripts` — green +- [ ] `bun run test:e2e` (or `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts` — record which) +- [ ] Request the whole-branch code review (superpowers:requesting-code-review); address findings; re-run the gate. From 71ca8a945c9cc5a201a2412abc95451203bd18fa Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:42:09 -0500 Subject: [PATCH 008/106] hooks-guard: handle fs.watch error events and reconcile stale watchers on refresh (S057) --- lib/daemon/__tests__/hooks-guard.test.ts | 57 ++++++++++++++++++++++++ lib/daemon/hooks-guard.ts | 30 ++++++++++++- 2 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 lib/daemon/__tests__/hooks-guard.test.ts diff --git a/lib/daemon/__tests__/hooks-guard.test.ts b/lib/daemon/__tests__/hooks-guard.test.ts new file mode 100644 index 00000000..ee72af08 --- /dev/null +++ b/lib/daemon/__tests__/hooks-guard.test.ts @@ -0,0 +1,57 @@ +/** + * S057: fs.watch handles in hooks-guard have no 'error' listener (an + * emitted error with no listener is an uncaught exception, which + * installCrashHandlers turns into a daemon exit(1) and a launchd relaunch + * that re-arms the same watchers and hits the same limit); and + * refreshWatchedRepos is add-only, so a relocated or removed repo's stale + * watcher on a dead .git dir is kept forever. + */ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import pino from "pino"; +import { createHooksGuard } from "../hooks-guard.ts"; + +const log = pino({ level: "silent" }); + +let dir: string; +beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "rt-hooks-guard-")); }); +afterEach(() => { rmSync(dir, { recursive: true, force: true }); }); + +function makeRepo(name: string): string { + const repoPath = join(dir, name); + mkdirSync(join(repoPath, ".git"), { recursive: true }); + writeFileSync(join(repoPath, ".git", "config"), "[core]\n"); + return repoPath; +} + +test("an emitted 'error' event on the watcher does not throw, and self-heals the bookkeeping", async () => { + const repoPath = makeRepo("a"); + const guard = createHooksGuard(log); + guard.startWatchingRepo("a", repoPath); + expect(guard.watchedConfigs.size).toBe(1); + + const [configPath, watcher] = [...guard.watchedConfigs.entries()][0]!; + expect(() => watcher.emit("error", new Error("EMFILE"))).not.toThrow(); + // give the close/delete a tick if it's deferred + await Bun.sleep(0); + expect(guard.watchedConfigs.has(configPath)).toBe(false); + guard.closeAll(); +}); + +test("refreshWatchedRepos closes and drops a watcher whose repo left the index (relocated or removed)", () => { + const repoPathA = makeRepo("a"); + const repoPathB = makeRepo("b"); + let index: Record = { a: repoPathA, b: repoPathB }; + const guard = createHooksGuard(log, { loadRepoIndexFn: () => index }); + + guard.refreshWatchedRepos(); + expect(guard.watchedConfigs.size).toBe(2); + + // "b" is relocated/removed: the index no longer carries it. + index = { a: repoPathA }; + guard.refreshWatchedRepos(); + expect(guard.watchedConfigs.size).toBe(1); + guard.closeAll(); +}); diff --git a/lib/daemon/hooks-guard.ts b/lib/daemon/hooks-guard.ts index f0b30d53..758b1569 100644 --- a/lib/daemon/hooks-guard.ts +++ b/lib/daemon/hooks-guard.ts @@ -12,6 +12,7 @@ import type { Logger } from "pino"; import { repoDataDir } from "../rt-paths.ts"; import { runCapture } from "../subprocess.ts"; import { loadRepoIndex, resolveGitConfigPath } from "./repo-index.ts"; +import type { RepoIndex } from "./handlers/types.ts"; export interface HooksGuard { /** Live map of repo git-config watchers (configPath → FSWatcher). */ @@ -26,7 +27,11 @@ export interface HooksGuard { closeAll(): void; } -export function createHooksGuard(log: Logger): HooksGuard { +export function createHooksGuard( + log: Logger, + deps: { loadRepoIndexFn?: () => RepoIndex } = {}, +): HooksGuard { + const loadRepoIndexFn = deps.loadRepoIndexFn ?? loadRepoIndex; const watchedConfigs = new Map(); async function checkAndRepairHooksPath(repoName: string, repoPath: string): Promise { @@ -90,6 +95,16 @@ export function createHooksGuard(log: Logger): HooksGuard { }, 100); // slightly longer debounce: rename events can cluster }); + // FSWatcher is an EventEmitter: an 'error' with no listener is an + // uncaught exception, which installCrashHandlers turns into a daemon + // exit(1) and a launchd relaunch that re-arms the same watchers and + // hits the same limit (EMFILE, a watched dir unlinked, ...). + watcher.on("error", (err) => { + log.warn({ err, repo: repoName, configPath }, "hooks-guard watcher error; dropping this watch"); + watchedConfigs.delete(configPath); + try { watcher.close(); } catch { /* already gone */ } + }); + watchedConfigs.set(configPath, watcher); log.debug({ repo: repoName, file: `${gitDir}/${configFile}` }, "watching repo"); @@ -98,11 +113,22 @@ export function createHooksGuard(log: Logger): HooksGuard { } function refreshWatchedRepos(): void { - const repos = loadRepoIndex(); + const repos = loadRepoIndexFn(); + const liveConfigPaths = new Set(); for (const [repoName, repoPath] of Object.entries(repos)) { if (!existsSync(repoPath)) continue; + const configPath = resolveGitConfigPath(repoPath); + if (configPath) liveConfigPaths.add(configPath); startWatchingRepo(repoName, repoPath); } + // Reconcile, not just add: a repo relocated (rt repos locate) or removed + // from the index leaves its old watcher pointed at a dead .git dir, and + // status().watchedRepos over-reports forever without this. + for (const [configPath, watcher] of watchedConfigs) { + if (liveConfigPaths.has(configPath)) continue; + try { watcher.close(); } catch { /* already gone */ } + watchedConfigs.delete(configPath); + } } function closeAll(): void { From b944a2d3ed0860890a423da3208b85efce3dcdc1 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:42:18 -0500 Subject: [PATCH 009/106] plan: Phase 3 trust-boundary implementation plan (S005/S006/S010/S040-043/S050/S054/S083-085/S092) --- .../plans/2026-08-28-p3-trust-boundary.md | 1745 +++++++++++++++++ 1 file changed, 1745 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-28-p3-trust-boundary.md diff --git a/docs/superpowers/plans/2026-08-28-p3-trust-boundary.md b/docs/superpowers/plans/2026-08-28-p3-trust-boundary.md new file mode 100644 index 00000000..5c8ce0d9 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-p3-trust-boundary.md @@ -0,0 +1,1745 @@ +# Phase 3 — The 127.0.0.1 Trust Boundary — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the daemon's `:9401` trust boundary — WS/CORS origin auth, token-gate the ungated mutating/destructive routes, one shared api-token loader, input hygiene (body-size cap, path-param decode safety, query coercion) — plus the two standalone items S043 (EADDRINUSE diagnostics + typed failure) and S083 (`pathParam()` 400s). + +**Architecture:** All logic lands as small, pure, dependency-injected functions in `lib/daemon/api-auth.ts` and `lib/daemon/api-server.ts` (mirroring the existing `bindApiServerWithRetry`/`needsToken`/`tokenOk` style already in this codebase) so every gate is unit-testable without spinning a real socket. The one exception is the request-body-size cap, which is a Bun.serve runtime option — that gets one small live `Bun.serve({port:0})` integration test. Two findings (S010, S050) have their fix location in sibling-owned files (`lib/daemon/handlers/worktree.ts`, `lib/daemon/freshness.ts`); this plan creates standalone, tested validator/utility modules for them and documents the one-line call-site wiring for the owning job, per the job brief's explicit instruction for S010 (mirrored here for S050 since its fix location is equally out of this job's write fence). + +**Tech Stack:** Bun, TypeScript, `bun:test`, existing `@mattstack/rt-client` settings resolver. + +**Spec:** `/Users/matt/Documents/GitHub/repo-tools/.claude/worktrees/daemon-stability-audit/docs/daemon-stability-audit-2026-08.md` — "Roadmap > Phase 3" (lines 72-79) plus Appendix A findings S005, S006, S010, S040, S041, S042, S043, S050, S054, S083, S084, S085, S092 (read-only input; never modify). + +## Global Constraints + +- Write fence: only `lib/daemon/api-server.ts`, `lib/daemon/api-auth.ts`, `lib/daemon/socket-server.ts`, `lib/daemon/handlers/secrets.ts`, new files under `lib/daemon/` (+ their tests under `lib/daemon/__tests__/`), `e2e/tests/`, `packages/rt-client/src/` (only if the client must send a token), the settings registry file, and `docs/`. Never touch `lib/daemon/handlers/worktree.ts`, `lib/daemon/freshness.ts`, or `lib/daemon.ts` — those are sibling-owned. +- Never start a daemon or run `rt`/`dist/rt` against the real machine; any such invocation in a test must run under `env -i HOME=`. +- `bun test lib commands packages scripts` must stay green; `bunx tsc --noEmit` must report zero errors. +- If `packages/rt-client` is touched, run `bun run build` inside it before the final review (dist-freshness test enforces this). +- Non-browser clients (no `Origin` header: the Swift tray, rt-client from Bun processes, mr-board, gitq, the VS Code extension) must keep working completely unchanged. A browser `Origin` must present the `X-RT-Token` (or, for `/ws`, a `?token=` query param) OR match the new `rt.trustedBrowserOrigins` settings allowlist. +- Never use em dashes or en dashes in code comments, commit messages, or docs (project convention — use parens or "..."). +- Follow TDD: write the failing test first, watch it fail, then implement. + +--- + +## File Map + +| File | Change | +|---|---| +| `lib/daemon/request-limits.ts` | **new** — shared `MAX_REQUEST_BODY_SIZE` constant (S092) | +| `lib/daemon/api-auth.ts` | token singleton + warn (S054); settings-backed origin allowlist + `isBrowserRequestTrusted` (S005/S006); `needsToken` invert-default (S040/S041/S084) | +| `lib/daemon/api-server.ts` | CORS default-deny + `/ws` gate (S005/S006); `broadcastToClients` backpressure/dead-client handling (S042); `pathParam()` + 400 (S083); `coerceQueryParams` (S085); `ApiPortInUseError` + lsof probe (S043) | +| `lib/daemon/socket-server.ts` | wire `MAX_REQUEST_BODY_SIZE` (S092) | +| `lib/daemon/handlers/secrets.ts` | use the shared token singleton (S054) | +| `packages/rt-client/src/settings/registry-defs.ts` | new `rt.trustedBrowserOrigins` key | +| `lib/daemon/git-ref-validation.ts` | **new** — S010 validator (standalone; sibling wires into `worktree.ts`) | +| `lib/daemon/redact-credentials.ts` | **new** — S050 utility (standalone; sibling wires into `freshness.ts`) | +| `docs/daemon-api-auth.md` | **new** — short note on the auth model + the two wiring pointers | + +All new tests live under `lib/daemon/__tests__/`. + +--- + +## Task 1: Shared request body-size cap (S092) + +**Files:** +- Create: `lib/daemon/request-limits.ts` +- Modify: `lib/daemon/api-server.ts` (add `maxRequestBodySize` to the `Bun.serve` options) +- Modify: `lib/daemon/socket-server.ts` (add `maxRequestBodySize` to the `Bun.serve` options) +- Test: `lib/daemon/__tests__/request-body-size.test.ts` + +**Interfaces:** +- Produces: `export const MAX_REQUEST_BODY_SIZE: number` (1 MiB) from `lib/daemon/request-limits.ts`, imported by both servers. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/request-body-size.test.ts +/** + * Bun enforces `maxRequestBodySize` itself (413 before the handler runs) -- + * this is a live-server test, not a pure-function one, because there is no + * pure function to unit test: the cap is a Bun.serve runtime option. + */ +import { describe, test, expect, afterEach } from "bun:test"; +import type { Server } from "bun"; +import { MAX_REQUEST_BODY_SIZE } from "../request-limits.ts"; + +let server: Server | undefined; + +afterEach(() => { + server?.stop(true); + server = undefined; +}); + +describe("MAX_REQUEST_BODY_SIZE", () => { + test("is set to 1 MiB", () => { + expect(MAX_REQUEST_BODY_SIZE).toBe(1024 * 1024); + }); + + test("Bun rejects a body over the cap with a 4xx before the handler runs", async () => { + let handlerRan = false; + server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + maxRequestBodySize: 10, // tiny cap for a fast, deterministic test + async fetch(req) { + handlerRan = true; + await req.text(); + return new Response("ok"); + }, + }); + const res = await fetch(`http://127.0.0.1:${server.port}/`, { + method: "POST", + body: "x".repeat(1000), + }); + expect(res.status).toBeGreaterThanOrEqual(400); + expect(res.status).toBeLessThan(500); + expect(handlerRan).toBe(false); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/request-body-size.test.ts` +Expected: FAIL — `request-limits.ts` does not exist yet (module not found). + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/request-limits.ts +/** + * Shared cap on request body size for both daemon servers (api-server.ts's + * :9401 HTTP/WS surface and socket-server.ts's unix-socket IPC channel). + * Neither transport authenticates reads, so an unbounded body (Bun's + * default is 128 MB) lets any same-user process or a cross-origin browser + * request stall the daemon's single event loop parsing a giant payload. + * Real payloads on both transports are kilobytes; 1 MiB costs nothing and + * turns an oversized request into an immediate 413 instead. + */ +export const MAX_REQUEST_BODY_SIZE = 1024 * 1024; +``` + +Then in `lib/daemon/api-server.ts`, add the import and option: + +```ts +import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; +``` + +...and inside the `Bun.serve({ ... })` options object passed to `bindApiServerWithRetry`, add: + +```ts + maxRequestBodySize: MAX_REQUEST_BODY_SIZE, +``` + +(alongside the existing `port`, `hostname`, `idleTimeout` keys). + +In `lib/daemon/socket-server.ts`, add the same import and, inside the `Bun.serve({ ... })` call, add: + +```ts + maxRequestBodySize: MAX_REQUEST_BODY_SIZE, +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/request-body-size.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice for a quick regression check** + +Run: `bun test lib/daemon` +Expected: PASS (no existing test asserts a specific absence of `maxRequestBodySize`) + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/request-limits.ts lib/daemon/api-server.ts lib/daemon/socket-server.ts lib/daemon/__tests__/request-body-size.test.ts +git commit -m "daemon: cap request body size at 1 MiB on both servers (S092)" +``` + +--- + +## Task 2: One shared api-token loader, with a warn on persist failure (S054) + +**Files:** +- Modify: `lib/daemon/api-auth.ts` +- Modify: `lib/daemon/api-server.ts` (use the new getter instead of the raw loader) +- Modify: `lib/daemon/handlers/secrets.ts` (use the new getter as the default override) +- Test: `lib/daemon/__tests__/api-auth.test.ts` (extend) + +**Interfaces:** +- Produces: `export function getApiToken(tokenPath?: string): string` and `export function reloadApiToken(tokenPath?: string): string` from `lib/daemon/api-auth.ts`. `loadOrCreateApiToken` keeps its existing signature and export (still the underlying file I/O primitive; `getApiToken`/`reloadApiToken` wrap it with an in-memory cache). +- Consumes: `lazyChildLogger` from `../daemon-logger.ts` (already exported; see `getDaemonLogger`/`lazyChildLogger` in `lib/daemon-logger.ts`). + +- [ ] **Step 1: Write the failing test** + +Append to `lib/daemon/__tests__/api-auth.test.ts`: + +```ts +import { getApiToken, reloadApiToken, loadOrCreateApiToken } from "../api-auth.ts"; +import { mkdtempSync, rmSync, writeFileSync, chmodSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +describe("getApiToken / reloadApiToken singleton", () => { + test("getApiToken caches: a second call does not re-read the file even if it changes underneath", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + const first = reloadApiToken(tokenPath); // seed the cache with a known path + writeFileSync(tokenPath, "a-different-token", { mode: 0o600 }); + const second = getApiToken(tokenPath); // ignores the new file content -- cached + expect(second).toBe(first); + expect(second).not.toBe("a-different-token"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("reloadApiToken re-reads and updates the cache", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + reloadApiToken(tokenPath); + writeFileSync(tokenPath, "rotated-token", { mode: 0o600 }); + const reloaded = reloadApiToken(tokenPath); + expect(reloaded).toBe("rotated-token"); + expect(getApiToken(tokenPath)).toBe("rotated-token"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("loadOrCreateApiToken still works standalone (unchanged primitive)", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + const a = loadOrCreateApiToken(tokenPath); + const b = loadOrCreateApiToken(tokenPath); + expect(a).toBe(b); + expect(a.length).toBeGreaterThan(0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-auth.test.ts` +Expected: FAIL — `getApiToken`/`reloadApiToken` are not exported yet. + +- [ ] **Step 3: Write minimal implementation** + +In `lib/daemon/api-auth.ts`, add near the top (after the existing imports) and after `loadOrCreateApiToken`: + +```ts +import { lazyChildLogger } from "../daemon-logger.ts"; + +const log = lazyChildLogger("api-auth"); +``` + +Change the existing `loadOrCreateApiToken`'s silent write-failure catch to log a warning (this is the only edit to that function's body): + +```ts +export function loadOrCreateApiToken(tokenPath: string = API_TOKEN_PATH): string { + try { + if (existsSync(tokenPath)) { + const existing = readFileSync(tokenPath, "utf8").trim(); + if (existing) return existing; + } + } catch { /* fall through to regenerate */ } + const token = randomUUID(); + try { + mkdirSync(RT_DIR, { recursive: true }); + writeFileSync(tokenPath, token, { mode: 0o600 }); + } catch (err) { + log.warn({ err, tokenPath }, "could not persist api-token; enforced in-memory only this run, so a client reading the file will disagree until the daemon restarts"); + } + return token; +} + +/** + * `getApiToken`/`reloadApiToken` share ONE in-memory value between + * api-server.ts and the secrets handler (S054): before this, api-server + * captured a token once at boot while the secrets handler called + * `loadOrCreateApiToken()` fresh on every request, so an external rotation + * (deleting api-token to force a new one) left the two permanently + * disagreeing about which token is current -- and if the token dir was + * unwritable, the secrets handler regenerated a brand new random token on + * every single call, never matching anything a client could read from disk. + * Both consumers now read the SAME cached value; a rotation only takes + * effect for both after `reloadApiToken()` runs or the daemon restarts, + * either of which was already the closest thing to a happy path before. + */ +let cachedApiToken: string | null = null; + +export function getApiToken(tokenPath: string = API_TOKEN_PATH): string { + if (cachedApiToken === null) cachedApiToken = loadOrCreateApiToken(tokenPath); + return cachedApiToken; +} + +export function reloadApiToken(tokenPath: string = API_TOKEN_PATH): string { + cachedApiToken = loadOrCreateApiToken(tokenPath); + return cachedApiToken; +} +``` + +In `lib/daemon/api-server.ts`, change the import and the one call site: + +```ts +import { needsToken, tokenOk, getApiToken } from "./api-auth.ts"; +``` + +```ts + const apiToken = getApiToken(); +``` + +(replaces `const apiToken = loadOrCreateApiToken();`) + +In `lib/daemon/handlers/secrets.ts`, change the import and the one default-override: + +```ts +import { getApiToken, tokenOk } from "../api-auth.ts"; +``` + +```ts + /** Defaults to `getApiToken` (the real ~/.mattstack/rt/api-token, shared with api-auth.ts and api-server.ts). */ + apiToken?: () => string; +``` + +(only the doc comment line changes; the field name/type is unchanged) + +```ts + const apiToken = overrides.apiToken ?? (() => getApiToken()); +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-auth.test.ts lib/daemon/__tests__/secrets-handler.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice** + +Run: `bun test lib/daemon` +Expected: PASS + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/api-auth.ts lib/daemon/api-server.ts lib/daemon/handlers/secrets.ts lib/daemon/__tests__/api-auth.test.ts +git commit -m "daemon: one shared api-token cache for api-server and secrets handler (S054)" +``` + +--- + +## Task 3: Origin allowlist settings key + trust helper + needsToken invert-default (S005, S006, S040, S041, S084) + +**Files:** +- Modify: `packages/rt-client/src/settings/registry-defs.ts` +- Modify: `lib/daemon/api-auth.ts` +- Test: `lib/daemon/__tests__/api-auth.test.ts` (extend) + +**Interfaces:** +- Produces: + - `export function getTrustedBrowserOrigins(): readonly string[]` (reads the `rt.trustedBrowserOrigins` setting, `[]` on any error or when unset) + - `export function isOriginAllowed(origin: string, allowedOrigins: readonly string[]): boolean` + - `export function isBrowserRequestTrusted(origin: string | null, token: string | null, apiToken: string, allowedOrigins: readonly string[]): boolean` (the shared decision Task 4 wires into both the CORS header and the `/ws` gate: no `Origin` header at all -> trusted (non-browser); otherwise a valid token OR an allowlisted origin) + - `needsToken(method, pathname)` changes shape: default-gated for every method except `GET`/`HEAD`/`OPTIONS`, plus two explicit GET exceptions (`/api/secrets`, `/api/notifications`) that are gated despite being reads. +- Consumes: `getSetting` from `../settings/resolve.ts` (already re-exported from `@mattstack/rt-client`; see `lib/chat-viewer-url.ts` for the exact import/usage pattern), `tokenOk` (already in this file). + +- [ ] **Step 1: Add the registry key** + +In `packages/rt-client/src/settings/registry-defs.ts`, add a new entry after the `rt.hooks` block (around line 207, right before the `// --- mattstack (installer-lane) ---` comment): + +```ts + { + key: "rt.trustedBrowserOrigins", + type: "array", + scopes: ["user", "machine"], + default: [], + merge: "replace", + description: "Browser Origins (scheme://host:port, exact string match) trusted to read the :9401 daemon API and subscribe to /ws without presenting the local api-token -- e.g. a locally-hosted console or chat-viewer dev server. Empty by default: every current mattstack consumer (the CLI, the Swift tray, rt-client from Bun/Node processes, the VS Code extension) is a non-browser client (sends no Origin header at all) and is unaffected either way.", + }, +``` + +- [ ] **Step 2: Build rt-client so the new key is live for tests that resolve it** + +Run: `cd packages/rt-client && bun run build && cd -` +Expected: build succeeds; `dist/` picks up the new registry row. + +- [ ] **Step 3: Write the failing tests** + +Append to `lib/daemon/__tests__/api-auth.test.ts`: + +```ts +import { isOriginAllowed, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "../api-auth.ts"; + +describe("isOriginAllowed", () => { + test("exact match", () => { + expect(isOriginAllowed("http://localhost:5544", ["http://localhost:5544"])).toBe(true); + }); + test("no match", () => { + expect(isOriginAllowed("http://evil.example", ["http://localhost:5544"])).toBe(false); + }); + test("empty allowlist matches nothing", () => { + expect(isOriginAllowed("http://localhost:5544", [])).toBe(false); + }); +}); + +describe("isBrowserRequestTrusted", () => { + const apiToken = "the-real-token"; + + test("no Origin header at all -- a non-browser client -- is trusted regardless of token or allowlist", () => { + expect(isBrowserRequestTrusted(null, null, apiToken, [])).toBe(true); + }); + + test("a browser Origin with the correct token is trusted even off the allowlist", () => { + expect(isBrowserRequestTrusted("http://evil.example", apiToken, apiToken, [])).toBe(true); + }); + + test("a browser Origin with a wrong token and not on the allowlist is rejected", () => { + expect(isBrowserRequestTrusted("http://evil.example", "wrong", apiToken, [])).toBe(false); + }); + + test("a browser Origin with no token but on the allowlist is trusted", () => { + expect(isBrowserRequestTrusted("http://localhost:5544", null, apiToken, ["http://localhost:5544"])).toBe(true); + }); + + test("a browser Origin with no token and not on the allowlist is rejected", () => { + expect(isBrowserRequestTrusted("http://localhost:5544", null, apiToken, [])).toBe(false); + }); +}); + +describe("getTrustedBrowserOrigins", () => { + test("returns an array (empty by default in an isolated test HOME)", () => { + const origins = getTrustedBrowserOrigins(); + expect(Array.isArray(origins)).toBe(true); + }); +}); +``` + +Also extend the existing `needsToken` describe block in the same file with the new cases (add these `test`s inside the existing `describe("needsToken", ...)`): + +```ts + test("refresh requires a token now (S040)", () => { + expect(needsToken("POST", "/api/refresh")).toBe(true); + }); + + test("hooks repair requires a token now (S040/S084)", () => { + expect(needsToken("POST", "/api/hooks/my-repo/repair")).toBe(true); + }); + + test("notifications GET (destructive drain) requires a token now (S041)", () => { + expect(needsToken("GET", "/api/notifications")).toBe(true); + }); + + test("every non-GET/HEAD/OPTIONS method defaults to requiring a token", () => { + expect(needsToken("POST", "/api/some-future-mutating-route")).toBe(true); + expect(needsToken("PUT", "/api/anything")).toBe(true); + expect(needsToken("DELETE", "/api/anything")).toBe(true); + }); + + test("plain reads still do not require a token", () => { + expect(needsToken("GET", "/api/repos")).toBe(false); + expect(needsToken("GET", "/api/cache")).toBe(false); + expect(needsToken("HEAD", "/api/repos")).toBe(false); + }); +``` + +- [ ] **Step 4: Run tests to verify they fail** + +Run: `bun test lib/daemon/__tests__/api-auth.test.ts` +Expected: FAIL — `isOriginAllowed`/`isBrowserRequestTrusted`/`getTrustedBrowserOrigins` not exported; the two new `needsToken` cases for `/api/refresh` and `/api/hooks/.../repair` and `/api/notifications` fail against the current allowlist-based implementation. + +- [ ] **Step 5: Write minimal implementation** + +In `lib/daemon/api-auth.ts`, add the import (alongside the existing ones) and the new functions: + +```ts +import { getSetting } from "../settings/resolve.ts"; +``` + +```ts +/** + * `rt.trustedBrowserOrigins` -- see registry-defs.ts. Read fresh on every + * call (the settings resolver is deliberately unmemoized), wrapped in a + * try/catch since a request-path settings read must never 500 the daemon + * over a malformed store file. + */ +export function getTrustedBrowserOrigins(): readonly string[] { + try { + const resolved = getSetting("rt.trustedBrowserOrigins"); + return Array.isArray(resolved.value) ? resolved.value : []; + } catch { + return []; + } +} + +export function isOriginAllowed(origin: string, allowedOrigins: readonly string[]): boolean { + return allowedOrigins.includes(origin); +} + +/** + * The 127.0.0.1 trust boundary (S005/S006): the daemon binds loopback-only, + * but any web page the user visits also runs on 127.0.0.1 and can send a + * request. A request with NO Origin header at all is not a browser fetch -- + * it is the CLI, the Swift tray, rt-client from a Bun/Node process, or the + * VS Code extension, none of which send one -- so it is trusted unchanged. + * A request that DOES carry an Origin header is trusted only if it presents + * the local api-token or its Origin is on the explicit allowlist. + */ +export function isBrowserRequestTrusted( + origin: string | null, + token: string | null, + apiToken: string, + allowedOrigins: readonly string[], +): boolean { + if (!origin) return true; + if (tokenOk(token, apiToken)) return true; + return isOriginAllowed(origin, allowedOrigins); +} +``` + +Replace the whole `needsToken` function with: + +```ts +/** + * True when a request mutates state, or (secrets/notifications) returns or + * drains something a GET should not silently consume, and must present the + * local token. Default-gated for every method except GET/HEAD/OPTIONS (S040: + * an allowlist-by-path guaranteed the next mutating route would ship + * unguarded) plus two explicit GET exceptions whose verb lies about being a + * read. + */ +export function needsToken(method: string, pathname: string): boolean { + if (method === "GET" || method === "HEAD" || method === "OPTIONS") { + // Gated despite being a GET: /api/secrets's response body IS a + // credential (S054); /api/notifications DRAINS the queue (S041), so its + // verb lies about being a read the way every other GET here is not. + if (pathname === "/api/secrets") return true; + if (pathname === "/api/notifications") return true; + return false; + } + return true; +} +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `bun test lib/daemon/__tests__/api-auth.test.ts` +Expected: PASS + +- [ ] **Step 7: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. + +- [ ] **Step 8: Commit** + +```bash +git add packages/rt-client/src/settings/registry-defs.ts packages/rt-client/dist lib/daemon/api-auth.ts lib/daemon/__tests__/api-auth.test.ts +git commit -m "daemon: rt.trustedBrowserOrigins allowlist + needsToken invert-default (S005/S006/S040/S041/S084)" +``` + +--- + +## Task 4: Wire CORS default-deny and the `/ws` origin/token gate into api-server.ts (S005, S006) + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Test: `lib/daemon/__tests__/api-server-cors-ws.test.ts` (new) + +**Interfaces:** +- Consumes: `isBrowserRequestTrusted`, `getTrustedBrowserOrigins` from `./api-auth.ts` (Task 3). +- Produces: `export function buildCorsHeaders(origin: string | null, trusted: boolean): Record` — a pure function so the header-shape logic is unit-testable without a real server. The live `/ws` gate and the live CORS-header wiring inside `fetch()` are exercised indirectly through this same function plus the `isBrowserRequestTrusted` tests from Task 3 (this codebase's existing convention -- see `bindApiServerWithRetry` -- is to keep the decision logic in pure, tested functions and keep the `Bun.serve` wiring itself thin). + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/api-server-cors-ws.test.ts +import { describe, test, expect } from "bun:test"; +import { buildCorsHeaders } from "../api-server.ts"; + +describe("buildCorsHeaders", () => { + test("no Origin header: no Access-Control-Allow-Origin is set (non-browser request, CORS is irrelevant)", () => { + const headers = buildCorsHeaders(null, true); + expect(headers["Access-Control-Allow-Origin"]).toBeUndefined(); + }); + + test("an untrusted Origin gets no Access-Control-Allow-Origin (default-deny, S006)", () => { + const headers = buildCorsHeaders("http://evil.example", false); + expect(headers["Access-Control-Allow-Origin"]).toBeUndefined(); + }); + + test("a trusted Origin is echoed back with Vary: Origin", () => { + const headers = buildCorsHeaders("http://localhost:5544", true); + expect(headers["Access-Control-Allow-Origin"]).toBe("http://localhost:5544"); + expect(headers["Vary"]).toBe("Origin"); + }); + + test("always advertises the methods/headers a preflight needs, trusted or not", () => { + const headers = buildCorsHeaders("http://evil.example", false); + expect(headers["Access-Control-Allow-Methods"]).toContain("POST"); + expect(headers["Access-Control-Allow-Headers"]).toContain("X-RT-Token"); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-server-cors-ws.test.ts` +Expected: FAIL — `buildCorsHeaders` is not exported yet. + +- [ ] **Step 3: Write minimal implementation** + +In `lib/daemon/api-server.ts`, change the import line to add the two new helpers: + +```ts +import { needsToken, tokenOk, getApiToken, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "./api-auth.ts"; +``` + +Add this exported pure function near the top of the file (after the `wsClients`/`broadcast` block, before `startApiServer`): + +```ts +/** + * CORS default-deny (S006): a browser page on an untrusted Origin still gets + * its request served (127.0.0.1 loopback + the per-route token gate are the + * real defenses), but the response carries no Access-Control-Allow-Origin, + * so the page's own JavaScript cannot read the body. A request with no + * Origin at all (every non-browser consumer today) needs no CORS headers. + */ +export function buildCorsHeaders(origin: string | null, trusted: boolean): Record { + const headers: Record = { + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type, X-RT-Token", + }; + if (origin && trusted) { + headers["Access-Control-Allow-Origin"] = origin; + headers["Vary"] = "Origin"; + } + return headers; +} +``` + +Now replace the body of `fetch(req, server)` from the top through the old `corsHeaders` declaration. The current code (for reference) is: + +```ts + async fetch(req, server) { + const url = new URL(req.url); + + // WebSocket upgrade — broadcast channel + if (url.pathname === "/ws") { + if (server.upgrade(req, { data: { kind: "broadcast" } })) return undefined as any; + return new Response("WebSocket upgrade failed", { status: 400 }); + } + + // CORS headers for local dev + const corsHeaders = { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type", + }; +``` + +Replace it with: + +```ts + async fetch(req, server) { + const url = new URL(req.url); + const origin = req.headers.get("origin"); + const allowedOrigins = getTrustedBrowserOrigins(); + + // WebSocket upgrade — broadcast channel. Browsers cannot set custom + // headers on a WS handshake, so the token (when a browser page wants + // to identify itself) travels as a ?token= query param instead of + // X-RT-Token (S005). + if (url.pathname === "/ws") { + const wsToken = url.searchParams.get("token"); + if (!isBrowserRequestTrusted(origin, wsToken, apiToken, allowedOrigins)) { + return new Response("origin not allowed", { status: 403 }); + } + if (server.upgrade(req, { data: { kind: "broadcast" } })) return undefined as any; + return new Response("WebSocket upgrade failed", { status: 400 }); + } + + // CORS: default-deny. A trusted Origin (token or allowlist) gets its + // Origin echoed back; anything else gets no Access-Control-Allow-Origin + // at all, so a malicious page's own JS cannot read the response (S006). + const trusted = isBrowserRequestTrusted(origin, req.headers.get("x-rt-token"), apiToken, allowedOrigins); + const corsHeaders = buildCorsHeaders(origin, trusted); +``` + +No other lines in `fetch()` need to change — every later reference to `corsHeaders` in the function already reads from this same local binding. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-server-cors-ws.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/__tests__/api-server-cors-ws.test.ts +git commit -m "daemon: default-deny CORS and gate /ws on origin/token (S005/S006)" +``` + +--- + +## Task 5: broadcast() drops dead/backpressured WS clients (S042) + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Test: `lib/daemon/__tests__/api-server-broadcast.test.ts` (new) + +**Interfaces:** +- Produces: `export function broadcastToClients(clients: Iterable, type: string, data: any, log: { warn: (o: unknown, m: string) => void }): void` where `BroadcastTarget = { send(data: string): number; close(): void }` (exported type). `broadcast()` becomes a thin wrapper calling `broadcastToClients(wsClients, type, data, log)`. +- Consumes: nothing new. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/api-server-broadcast.test.ts +import { describe, test, expect } from "bun:test"; +import { broadcastToClients, type BroadcastTarget } from "../api-server.ts"; + +function fakeClient(sendReturns: number[]): BroadcastTarget & { closed: boolean; sent: string[] } { + const sent: string[] = []; + let i = 0; + const client = { + closed: false, + sent, + send(data: string) { + sent.push(data); + const ret = sendReturns[Math.min(i, sendReturns.length - 1)]; + i++; + return ret; + }, + close() { client.closed = true; }, + }; + return client; +} + +function fakeLog() { + const warns: unknown[] = []; + return { warn: (o: unknown, _m: string) => { warns.push(o); }, warns }; +} + +describe("broadcastToClients", () => { + test("a healthy client (positive send return) is never closed", () => { + const client = fakeClient([42]); + broadcastToClients([client], "status", { ok: true }, fakeLog()); + expect(client.closed).toBe(false); + expect(client.sent.length).toBe(1); + }); + + test("a send() returning 0 (dropped frame) closes the client immediately and logs a warning", () => { + const client = fakeClient([0]); + const log = fakeLog(); + broadcastToClients([client], "status", { ok: true }, log); + expect(client.closed).toBe(true); + expect(log.warns.length).toBe(1); + }); + + test("a send() returning -1 (backpressure) is tolerated for a few sends before closing", () => { + const client = fakeClient([-1, -1, -1, -1]); + const log = fakeLog(); + broadcastToClients([client], "a", {}, log); + expect(client.closed).toBe(false); + broadcastToClients([client], "b", {}, log); + expect(client.closed).toBe(false); + broadcastToClients([client], "c", {}, log); + // third consecutive backpressure event closes the client + expect(client.closed).toBe(true); + }); + + test("a successful send resets the backpressure counter", () => { + const client = fakeClient([-1, -1, 99, -1, -1, -1]); + const log = fakeLog(); + broadcastToClients([client], "a", {}, log); // -1 (count=1) + broadcastToClients([client], "b", {}, log); // -1 (count=2) + broadcastToClients([client], "c", {}, log); // 99 -- resets to 0 + expect(client.closed).toBe(false); + broadcastToClients([client], "d", {}, log); // -1 (count=1) + broadcastToClients([client], "e", {}, log); // -1 (count=2) + expect(client.closed).toBe(false); + broadcastToClients([client], "f", {}, log); // -1 (count=3) -- closes + expect(client.closed).toBe(true); + }); + + test("a client whose send() throws is treated as gone: caught, not propagated", () => { + const client: BroadcastTarget = { + send() { throw new Error("ECONNRESET"); }, + close() { /* no-op */ }, + }; + expect(() => broadcastToClients([client], "a", {}, fakeLog())).not.toThrow(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-server-broadcast.test.ts` +Expected: FAIL — `broadcastToClients`/`BroadcastTarget` not exported yet. + +- [ ] **Step 3: Write minimal implementation** + +Replace the existing `broadcast()` function and the `wsClients` declaration block in `lib/daemon/api-server.ts`: + +Current code (for reference): + +```ts +const wsClients = new Set>(); + +/** Broadcast an event to all connected WebSocket clients. */ +export function broadcast(type: string, data: any): void { + if (wsClients.size === 0) return; + const msg = JSON.stringify({ type, data, timestamp: Date.now() }); + for (const ws of wsClients) { + try { ws.send(msg); } catch { /* client disconnected */ } + } +} +``` + +Replace with: + +```ts +const wsClients = new Set>(); + +/** Consecutive Bun `ws.send()` backpressure (-1) returns tolerated before a + client is dropped as chronically stalled. */ +const BACKPRESSURE_CLOSE_THRESHOLD = 3; +const backpressureCounts = new WeakMap(); + +export interface BroadcastTarget { + send(data: string): number; + close(): void; +} + +/** + * Sends one frame to every client, dropping any that Bun's own send() return + * value marks as gone (S042). `ws.send()` never throws on a dead socket -- + * it returns 0 (this send silently failed) or -1 (backpressure) -- so a + * disconnected or stalled console/chat-viewer tab used to keep receiving a + * SUBSET of frames forever with nothing logged. 0 means Bun already dropped + * this exact frame for this client: closing immediately (rather than + * counting) is correct because the client's own reconnect logic is the only + * way it recovers a consistent stream. -1 means backpressure, which can be + * transient, so a few in a row are tolerated before giving up on the client. + */ +export function broadcastToClients( + clients: Iterable, + type: string, + data: any, + log: { warn: (o: unknown, m: string) => void }, +): void { + const msg = JSON.stringify({ type, data, timestamp: Date.now() }); + for (const client of clients) { + let result: number; + try { + result = client.send(msg); + } catch (err) { + log.warn({ err }, "ws client send threw; dropping"); + try { client.close(); } catch { /* already gone */ } + continue; + } + if (result === 0) { + log.warn({ type }, "ws client dropped a frame (send()=0); closing so its reconnect resyncs"); + backpressureCounts.delete(client); + try { client.close(); } catch { /* already gone */ } + } else if (result === -1) { + const count = (backpressureCounts.get(client) ?? 0) + 1; + if (count >= BACKPRESSURE_CLOSE_THRESHOLD) { + log.warn({ type, count }, "ws client chronically backpressured; closing"); + backpressureCounts.delete(client); + try { client.close(); } catch { /* already gone */ } + } else { + backpressureCounts.set(client, count); + } + } else { + backpressureCounts.delete(client); + } + } +} + +/** Broadcast an event to all connected WebSocket clients. */ +export function broadcast(type: string, data: any): void { + if (wsClients.size === 0) return; + broadcastToClients(wsClients, type, data, apiServerLog); +} +``` + +This introduces one new module-level binding, `apiServerLog`, since `broadcast()` previously had no logger in scope at all (it is called from many places across the daemon, not just from inside `startApiServer`). Add it right after the existing `wsClients`-adjacent declarations, and set it from `startApiServer`: + +```ts +let apiServerLog: { warn: (o: unknown, m: string) => void } = { warn: () => {} }; +``` + +...and inside `startApiServer`, right after `const { handleCommand, log } = deps;`, add: + +```ts + apiServerLog = log; +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-server-broadcast.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. (`ServerWebSocket` structurally satisfies `BroadcastTarget` since it has both `send(string): number` and `close(): void`, so `broadcastToClients(wsClients, ...)` type-checks with no cast.) + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/__tests__/api-server-broadcast.test.ts +git commit -m "daemon: broadcast() drops dead/backpressured ws clients instead of silently dropping frames (S042)" +``` + +--- + +## Task 6: pathParam() helper — malformed %-encoding returns 400, not a logged 500 (S083) + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Test: `lib/daemon/__tests__/api-server-path-param.test.ts` (new) + +**Interfaces:** +- Produces: `export function pathParam(pathname: string, prefix: string, suffix?: string): string | undefined` (returns `undefined` when the pathname doesn't match the prefix/suffix shape, the captured segment is empty, or `decodeURIComponent` throws). + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/api-server-path-param.test.ts +import { describe, test, expect } from "bun:test"; +import { pathParam } from "../api-server.ts"; + +describe("pathParam", () => { + test("decodes a clean prefix-only param", () => { + expect(pathParam("/api/cache/main", "/api/cache/")).toBe("main"); + }); + + test("decodes a URL-encoded segment", () => { + expect(pathParam("/api/cache/feature%2Ffoo", "/api/cache/")).toBe("feature/foo"); + }); + + test("returns undefined for malformed %-encoding instead of throwing", () => { + expect(pathParam("/api/cache/%E0%A4%A", "/api/cache/")).toBeUndefined(); + }); + + test("returns undefined when the pathname doesn't start with the prefix", () => { + expect(pathParam("/api/other/main", "/api/cache/")).toBeUndefined(); + }); + + test("handles a prefix+suffix pair (hooks repair shape)", () => { + expect(pathParam("/api/hooks/my-repo/repair", "/api/hooks/", "/repair")).toBe("my-repo"); + }); + + test("prefix+suffix: malformed encoding still returns undefined", () => { + expect(pathParam("/api/hooks/%E0%A4%A/repair", "/api/hooks/", "/repair")).toBeUndefined(); + }); + + test("prefix+suffix: wrong suffix returns undefined", () => { + expect(pathParam("/api/hooks/my-repo/other", "/api/hooks/", "/repair")).toBeUndefined(); + }); + + test("an empty captured segment returns undefined", () => { + expect(pathParam("/api/cache/", "/api/cache/")).toBeUndefined(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-server-path-param.test.ts` +Expected: FAIL — `pathParam` not exported yet. + +- [ ] **Step 3: Write minimal implementation** + +Add this exported function to `lib/daemon/api-server.ts` (near `buildCorsHeaders`, before `startApiServer`): + +```ts +/** + * Decodes one path segment between a fixed prefix (and optional suffix), + * returning `undefined` (never throwing) on any shape mismatch or malformed + * %-encoding (S083). Before this, each parameterized route hand-rolled its + * own decodeURIComponent inside the route's try block, so a malformed + * segment fell through to the OUTER catch and came back as a logged 500; + * every route using this helper instead gets a clean 400. + */ +export function pathParam(pathname: string, prefix: string, suffix = ""): string | undefined { + if (!pathname.startsWith(prefix)) return undefined; + if (suffix && !pathname.endsWith(suffix)) return undefined; + const end = suffix ? pathname.length - suffix.length : pathname.length; + if (end <= prefix.length) return undefined; + const raw = pathname.slice(prefix.length, end); + try { + return decodeURIComponent(raw); + } catch { + return undefined; + } +} +``` + +Now wire it into the three routes inside `fetch()`. Replace the `/api/cache/:branch` block: + +```ts + // Single branch lookup: /api/cache/:branch + if (url.pathname.startsWith("/api/cache/") && req.method === "GET") { + const branch = pathParam(url.pathname, "/api/cache/"); + if (branch === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); + } + const result = await handleCommand("cache:read", { branches: [branch] }, req.signal); + return Response.json(result, { headers: corsHeaders }); + } +``` + +Replace the `/api/hooks/:repo/repair` block: + +```ts + // Hooks repair: /api/hooks/:repo/repair + if (url.pathname.startsWith("/api/hooks/") && url.pathname.endsWith("/repair") && req.method === "POST") { + const repo = pathParam(url.pathname, "/api/hooks/", "/repair"); + if (repo === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); + } + const result = await handleCommand("hooks:repair", { repo }, req.signal); + return Response.json(result, { headers: corsHeaders }); + } +``` + +Replace the `/api/runs/:repo/:runId` block (this one already guarded against the URIError; simplify it onto the shared helper so there is one decode path in the file, not two): + +```ts + // Runs detail: /api/runs/:repo/:runId + if (url.pathname.startsWith("/api/runs/") && req.method === "GET") { + const rest = pathParam(url.pathname, "/api/runs/"); + if (rest === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); + } + const slash = rest.indexOf("/"); + if (slash > 0 && slash < rest.length - 1) { + const result = await handleCommand("runs:get", { repo: rest.slice(0, slash), runId: rest.slice(slash + 1) }, req.signal); + return Response.json(result, { headers: corsHeaders }); + } + // falls through to the 404 path below for a shape mismatch, e.g. "/api/runs/onlyonesegment" + } +``` + +Note the behavior change from before: a malformed `/api/runs/...` now returns 400 instead of falling through to the generic 404. This is intentional and matches S083's ask across all three routes uniformly (a malformed path parameter is a 400, a genuinely unknown route is a 404); update no other test, since no existing test in this repo asserts the old runs-route 500-vs-404 distinction (grep `lib/daemon/__tests__/` for `runs:get` before assuming otherwise, and if one exists, adjust it to expect 400 for the malformed case instead of removing coverage). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-server-path-param.test.ts` +Expected: PASS + +- [ ] **Step 5: Check for an existing runs-route test that might need updating** + +Run: `grep -rn "runs:get\|/api/runs/" lib/daemon/__tests__/` +If a test asserts the old 500-on-malformed or 404-on-malformed behavior for `/api/runs/`, update its expected status to 400 to match the new shared helper. + +- [ ] **Step 6: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. + +- [ ] **Step 7: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/__tests__/api-server-path-param.test.ts +git commit -m "daemon: pathParam() helper -- malformed %-encoding is a 400, not a logged 500 (S083)" +``` + +--- + +## Task 7: GET query param coercion (S085) + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Test: `lib/daemon/__tests__/api-server-query-coerce.test.ts` (new) + +**Interfaces:** +- Produces: `export function coerceQueryParams(params: URLSearchParams): Record` — converts `"true"`/`"false"` to booleans and plain-integer/decimal strings to numbers; everything else stays a string. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/api-server-query-coerce.test.ts +import { describe, test, expect } from "bun:test"; +import { coerceQueryParams } from "../api-server.ts"; + +describe("coerceQueryParams", () => { + test("coerces maxAgeMs to a number (the documented cache:read flag)", () => { + const out = coerceQueryParams(new URLSearchParams("maxAgeMs=60000")); + expect(out.maxAgeMs).toBe(60000); + expect(typeof out.maxAgeMs).toBe("number"); + }); + + test("coerces refresh=true to a boolean (the documented ports flag)", () => { + const out = coerceQueryParams(new URLSearchParams("refresh=true")); + expect(out.refresh).toBe(true); + }); + + test("coerces refresh=false to a boolean false, not a truthy string", () => { + const out = coerceQueryParams(new URLSearchParams("refresh=false")); + expect(out.refresh).toBe(false); + }); + + test("leaves a non-numeric, non-boolean string alone", () => { + const out = coerceQueryParams(new URLSearchParams("repo=my-repo-name")); + expect(out.repo).toBe("my-repo-name"); + }); + + test("leaves an empty string alone rather than coercing to 0", () => { + const out = coerceQueryParams(new URLSearchParams("q=")); + expect(out.q).toBe(""); + }); + + test("coerces a decimal number too", () => { + const out = coerceQueryParams(new URLSearchParams("ratio=1.5")); + expect(out.ratio).toBe(1.5); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-server-query-coerce.test.ts` +Expected: FAIL — `coerceQueryParams` not exported yet. + +- [ ] **Step 3: Write minimal implementation** + +Add to `lib/daemon/api-server.ts` (near `pathParam`): + +```ts +const PLAIN_NUMBER_RE = /^-?\d+(\.\d+)?$/; + +/** + * REST query strings arrive as strings no matter what the client meant + * (S085): `?maxAgeMs=60000` and `?refresh=true` reached handlers that do a + * strict `typeof x === "number"` or `x === true` check, so the documented + * flag silently no-op'd over HTTP while working over the socket (where + * payloads are real JSON). One coercion at the REST seam fixes every such + * flag at once instead of a per-handler private parser. + */ +export function coerceQueryParams(params: URLSearchParams): Record { + const out: Record = {}; + for (const [key, value] of params) { + if (value === "true") out[key] = true; + else if (value === "false") out[key] = false; + else if (value !== "" && PLAIN_NUMBER_RE.test(value)) out[key] = Number(value); + else out[key] = value; + } + return out; +} +``` + +Change the payload-building line inside `fetch()` from: + +```ts + } else { + payload = Object.fromEntries(url.searchParams); + } +``` + +to: + +```ts + } else { + payload = coerceQueryParams(url.searchParams); + } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-server-query-coerce.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/__tests__/api-server-query-coerce.test.ts +git commit -m "daemon: coerce REST GET query params to number/boolean at the seam (S085)" +``` + +--- + +## Task 8: ApiPortInUseError + lsof diagnostics when bind retries are exhausted (S043 addendum) + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Test: `lib/daemon/__tests__/api-server-bind.test.ts` (extend) + +**Interfaces:** +- Produces: `export class ApiPortInUseError extends Error { readonly code: "EADDRINUSE"; readonly port: number }`. `BindRetryDeps` grows an optional `probePortHolder?: (port: number) => Promise` field (defaults to a real `lsof -i :` via `runCapture`). `bindApiServerWithRetry` throws `ApiPortInUseError` (instead of the raw EADDRINUSE `Error`) once `BIND_RETRY_ATTEMPTS` is exhausted, after logging the probe result at `warn`. +- Consumes: `runCapture` from `../subprocess.ts`. +- **Contract for the sibling job owning `lib/daemon.ts`'s caller side:** catch `ApiPortInUseError` (check `err instanceof ApiPortInUseError`, or `err.name === "ApiPortInUseError"`) around the `startApiServer()` call and route to a park-and-retry-with-backoff loop instead of letting it propagate to the top-level crash/unhandledRejection path. Every other error out of `startApiServer()` is a real misconfiguration and should keep crashing as it does today. + +- [ ] **Step 1: Write the failing test** + +Append to `lib/daemon/__tests__/api-server-bind.test.ts`: + +```ts +import { ApiPortInUseError, BIND_RETRY_ATTEMPTS } from "../api-server.ts"; + +function depsWithProbe(overrides: Partial = {}) { + const logs: Array<{ o: unknown; m: string }> = []; + const sleeps: number[] = []; + const probeCalls: number[] = []; + return { + sleep: async (ms: number) => { sleeps.push(ms); }, + log: { warn: (o: unknown, m: string) => logs.push({ o, m }) }, + probePortHolder: async (port: number) => { probeCalls.push(port); return "COMMAND PID USER\nnode 123 matt"; }, + logs, + sleeps, + probeCalls, + ...overrides, + }; +} + +describe("bindApiServerWithRetry — exhausted retries (S043)", () => { + test("throws ApiPortInUseError (not the raw EADDRINUSE Error) once attempts are exhausted", async () => { + const d = depsWithProbe(); + let error: unknown; + try { + await bindApiServerWithRetry(() => { throw eaddrinuse(); }, d); + } catch (err) { + error = err; + } + expect(error).toBeInstanceOf(ApiPortInUseError); + expect((error as ApiPortInUseError).code).toBe("EADDRINUSE"); + expect((error as Error).message).toContain("EADDRINUSE"); + }); + + test("probes the port holder exactly once, only after the final attempt", async () => { + const d = depsWithProbe(); + let calls = 0; + await expect( + bindApiServerWithRetry(() => { calls++; throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + expect(calls).toBe(BIND_RETRY_ATTEMPTS); + expect(d.probeCalls.length).toBe(1); + }); + + test("logs the probe result at warn before throwing", async () => { + const d = depsWithProbe(); + await expect( + bindApiServerWithRetry(() => { throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + const finalWarn = d.logs.at(-1)!; + expect(finalWarn.o).toMatchObject({ holder: expect.stringContaining("node") }); + }); + + test("a probe failure does not prevent the ApiPortInUseError from being thrown", async () => { + const d = depsWithProbe({ probePortHolder: async () => { throw new Error("lsof: command not found"); } }); + await expect( + bindApiServerWithRetry(() => { throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + }); + + test("a successful bind never probes", async () => { + const d = depsWithProbe(); + await bindApiServerWithRetry(() => "server" as any, d); + expect(d.probeCalls.length).toBe(0); + }); +}); +``` + +Also update the OLD test that currently asserts the raw-error message (it now gets a differently-shaped, but still EADDRINUSE-mentioning, error): + +Find this existing test: + +```ts + test("exhausting retries rethrows the original error after exactly BIND_RETRY_ATTEMPTS calls", async () => { + const d = deps(); + let calls = 0; + await expect( + bindApiServerWithRetry(() => { calls++; throw eaddrinuse(); }, d), + ).rejects.toThrow("EADDRINUSE"); + expect(calls).toBe(BIND_RETRY_ATTEMPTS); + expect(d.sleeps.length).toBe(BIND_RETRY_ATTEMPTS - 1); + }); +``` + +Change its assertion to also cover the new type, since `deps()` (the original helper, with no `probePortHolder` override) must still work by falling back to a default real-`lsof` probe — which would actually shell out in a test. To keep this test hermetic, add `probePortHolder: async () => "n/a"` to the base `deps()` helper's return object (it is an optional field on `BindRetryDeps`, so this is a backward-compatible addition, not a signature break): + +```ts +function deps(overrides: Partial = {}): BindRetryDeps & { logs: string[]; sleeps: number[] } { + const logs: string[] = []; + const sleeps: number[] = []; + return { + sleep: async (ms: number) => { sleeps.push(ms); }, + log: { warn: (_o: unknown, m: string) => logs.push(`warn:${m}`) }, + probePortHolder: async () => "n/a", + logs, + sleeps, + ...overrides, + }; +} +``` + +Then the pre-existing test's assertion becomes: + +```ts + test("exhausting retries rethrows as ApiPortInUseError after exactly BIND_RETRY_ATTEMPTS calls", async () => { + const d = deps(); + let calls = 0; + await expect( + bindApiServerWithRetry(() => { calls++; throw eaddrinuse(); }, d), + ).rejects.toThrow("EADDRINUSE"); + expect(calls).toBe(BIND_RETRY_ATTEMPTS); + expect(d.sleeps.length).toBe(BIND_RETRY_ATTEMPTS - 1); + }); +``` + +(only the test name and this doc comment change; `.rejects.toThrow("EADDRINUSE")` still matches since `ApiPortInUseError`'s message contains that substring.) + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-server-bind.test.ts` +Expected: FAIL — `ApiPortInUseError` not exported; `probePortHolder` not a recognized field yet. + +- [ ] **Step 3: Write minimal implementation** + +In `lib/daemon/api-server.ts`, add the import: + +```ts +import { runCapture } from "../subprocess.ts"; +``` + +Add the error class and update `BindRetryDeps`/`bindApiServerWithRetry`. Replace the current block: + +```ts +export interface BindRetryDeps { + sleep: (ms: number) => Promise; + log: { warn: (o: unknown, m: string) => void }; +} + +// evictStaleDaemon (lib/daemon/boot-reconcile.ts) already assumes a prior +// holder is gone after a 300ms Bun.sleepSync — 6 attempts at 500ms (~3s +// worst case, exported so tests assert against these, not hardcoded copies) +// gives that same assumption room to be wrong once before this gives up too. +export const BIND_RETRY_ATTEMPTS = 6; +export const BIND_RETRY_DELAY_MS = 500; + +/** + * evictStaleDaemon() SIGTERMs the previous holder of this port before a new + * daemon binds, but the kill isn't synchronous with the exit — a fresh + * daemon can reach this bind before the old one has actually released + * 9401. Only EADDRINUSE is retried (bounded, ~3s total); anything else is a + * real misconfiguration and fails on the first attempt, same as before. + */ +export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDeps): Promise { + for (let attempt = 1; ; attempt++) { + try { + return bind(); + } catch (err) { + const isAddrInUse = err instanceof Error && (err as NodeJS.ErrnoException).code === "EADDRINUSE"; + if (!isAddrInUse || attempt >= BIND_RETRY_ATTEMPTS) throw err; + deps.log.warn({ attempt, port: API_PORT }, "api port in use, retrying — another daemon is likely still shutting down"); + await deps.sleep(BIND_RETRY_DELAY_MS); + } + } +} +``` + +with: + +```ts +/** + * Thrown when every bind retry is exhausted with EADDRINUSE still held. A + * NAMED error type (S043) rather than the raw EADDRINUSE Error, so + * lib/daemon.ts's caller can distinguish "the port is genuinely squatted" + * from any other startup failure and park-and-retry with backoff instead of + * crash-looping (that caller-side change belongs to a sibling job; this + * class is the contract it wires into). + */ +export class ApiPortInUseError extends Error { + readonly code = "EADDRINUSE" as const; + readonly port: number; + constructor(port: number) { + super(`EADDRINUSE: api server port ${port} is still in use after ${BIND_RETRY_ATTEMPTS} bind attempts`); + this.name = "ApiPortInUseError"; + this.port = port; + } +} + +export interface BindRetryDeps { + sleep: (ms: number) => Promise; + log: { warn: (o: unknown, m: string) => void }; + /** Defaults to a real `lsof -i :` via runCapture; overridable so tests never shell out. */ + probePortHolder?: (port: number) => Promise; +} + +async function defaultProbePortHolder(port: number): Promise { + const result = await runCapture(["lsof", "-i", `:${port}`], { timeoutMs: 5_000, stderr: "pipe" }); + return result.stdout.trim(); +} + +// evictStaleDaemon (lib/daemon/boot-reconcile.ts) already assumes a prior +// holder is gone after a 300ms Bun.sleepSync — 6 attempts at 500ms (~3s +// worst case, exported so tests assert against these, not hardcoded copies) +// gives that same assumption room to be wrong once before this gives up too. +export const BIND_RETRY_ATTEMPTS = 6; +export const BIND_RETRY_DELAY_MS = 500; + +/** + * evictStaleDaemon() SIGTERMs the previous holder of this port before a new + * daemon binds, but the kill isn't synchronous with the exit — a fresh + * daemon can reach this bind before the old one has actually released + * 9401. Only EADDRINUSE is retried (bounded, ~3s total); anything else is a + * real misconfiguration and fails on the first attempt, same as before. + * + * Once retries are exhausted, this logs WHO holds the port (S043's + * diagnostic ask) and throws ApiPortInUseError instead of the bare + * EADDRINUSE Error, so a caller can tell "give up cleanly" apart from "the + * bind function itself is broken". + */ +export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDeps): Promise { + const probe = deps.probePortHolder ?? defaultProbePortHolder; + for (let attempt = 1; ; attempt++) { + try { + return bind(); + } catch (err) { + const isAddrInUse = err instanceof Error && (err as NodeJS.ErrnoException).code === "EADDRINUSE"; + if (!isAddrInUse) throw err; + if (attempt >= BIND_RETRY_ATTEMPTS) { + const holder = await probe(API_PORT).catch((probeErr) => `lsof failed: ${String(probeErr)}`); + deps.log.warn({ port: API_PORT, holder }, "api port still in use after retries; giving up bind — the daemon should park and retry with backoff rather than crash-loop"); + throw new ApiPortInUseError(API_PORT); + } + deps.log.warn({ attempt, port: API_PORT }, "api port in use, retrying — another daemon is likely still shutting down"); + await deps.sleep(BIND_RETRY_DELAY_MS); + } + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-server-bind.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/__tests__/api-server-bind.test.ts +git commit -m "daemon: log the EADDRINUSE port holder and throw a typed ApiPortInUseError (S043)" +``` + +--- + +## Task 9: git-ref validation utility (S010 — standalone, sibling wires it into worktree.ts) + +**Files:** +- Create: `lib/daemon/git-ref-validation.ts` +- Test: `lib/daemon/__tests__/git-ref-validation.test.ts` + +**Interfaces:** +- Produces: `export function isSafeGitRef(ref: string): boolean` and `export function validateGitRef(ref: string): { ok: true } | { ok: false; error: string }`. + +This module is NOT wired into `lib/daemon/handlers/worktree.ts` by this job — that file is sibling-owned per the write fence. The wiring is documented in `docs/daemon-api-auth.md` (Task 11) and the job report's Notes. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/git-ref-validation.test.ts +import { describe, test, expect } from "bun:test"; +import { isSafeGitRef, validateGitRef } from "../git-ref-validation.ts"; + +describe("isSafeGitRef", () => { + test("a normal branch name is safe", () => { + expect(isSafeGitRef("feature/my-branch")).toBe(true); + }); + + test("a leading dash is unsafe (option injection, S010)", () => { + expect(isSafeGitRef("--upload-pack=touch /tmp/x")).toBe(false); + }); + + test("a bare dash is unsafe", () => { + expect(isSafeGitRef("-")).toBe(false); + }); + + test("an empty string is unsafe", () => { + expect(isSafeGitRef("")).toBe(false); + }); + + test("a branch containing a dash mid-name is safe", () => { + expect(isSafeGitRef("job/p3-trust-boundary")).toBe(true); + }); +}); + +describe("validateGitRef", () => { + test("returns ok:true for a safe ref", () => { + expect(validateGitRef("main")).toEqual({ ok: true }); + }); + + test("returns ok:false with the offending ref named in the error for an unsafe one", () => { + const result = validateGitRef("--upload-pack=x"); + expect(result.ok).toBe(false); + expect((result as { ok: false; error: string }).error).toContain("--upload-pack=x"); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/git-ref-validation.test.ts` +Expected: FAIL — module does not exist. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/git-ref-validation.ts +/** + * Rejects a branch/ref that could be parsed by git as an OPTION rather than + * a ref (S010): a caller-supplied `branch` like "--upload-pack=touch /tmp/x" + * reaches `git fetch origin ` and `git rev-list ......` + * unescaped in lib/daemon/handlers/worktree.ts, and git happily runs it as + * an option since nothing on that path validates the string first. This is + * the ONE guard both call sites need; a future caller (or a consumer app + * like mr-board/console/the chat viewer forwarding an untrusted string as + * `branch`) inherits the same hole without it. + * + * Deliberately narrow: reject a leading '-' rather than allowlisting a + * character set, since `git check-ref-format --branch` accepts far more + * punctuation than is worth re-deriving here, and the vulnerable shape is + * specifically "parses as an option", not "contains an unusual character". + */ +export function isSafeGitRef(ref: string): boolean { + return ref.length > 0 && !ref.startsWith("-"); +} + +export function validateGitRef(ref: string): { ok: true } | { ok: false; error: string } { + if (!isSafeGitRef(ref)) { + return { ok: false, error: `unsafe git ref (starts with '-' or empty): ${ref}` }; + } + return { ok: true }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/git-ref-validation.test.ts` +Expected: PASS + +- [ ] **Step 5: Run tsc** + +Run: `bunx tsc --noEmit` +Expected: clean. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/git-ref-validation.ts lib/daemon/__tests__/git-ref-validation.test.ts +git commit -m "daemon: standalone git-ref validator for S010 (worktree.ts wiring documented, not wired here)" +``` + +--- + +## Task 10: credential redaction utility (S050 — standalone, sibling wires it into freshness.ts) + +**Files:** +- Create: `lib/daemon/redact-credentials.ts` +- Test: `lib/daemon/__tests__/redact-credentials.test.ts` + +**Interfaces:** +- Produces: `export function redactCredentials(text: string): string`. + +Same treatment as Task 9: `lib/daemon/freshness.ts` (where the audit's fixer notes say this belongs, at lines ~142/148/275/279) is not in this job's write fence. This module is standalone and tested; the wiring is documented in `docs/daemon-api-auth.md` (Task 11) and the report's Notes. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/redact-credentials.test.ts +import { describe, test, expect } from "bun:test"; +import { redactCredentials } from "../redact-credentials.ts"; + +describe("redactCredentials", () => { + test("redacts userinfo (user:token@) out of an https remote URL", () => { + const input = "https://oauth2:glpat-XXXXXXXXXXXXXXXXXXXX@gitlab.example.com/g/p.git"; + const out = redactCredentials(input); + expect(out).not.toContain("glpat-XXXXXXXXXXXXXXXXXXXX"); + expect(out).toContain("gitlab.example.com/g/p.git"); + }); + + test("redacts a GitHub PAT embedded in the URL", () => { + const input = "https://ghp_abcdefghijklmnopqrstuvwxyz012345@github.com/o/r.git"; + const out = redactCredentials(input); + expect(out).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz012345"); + expect(out).toContain("github.com/o/r.git"); + }); + + test("leaves a URL with no embedded credentials unchanged", () => { + const input = "https://gitlab.example.com/g/p.git"; + expect(redactCredentials(input)).toBe(input); + }); + + test("leaves plain text with no URL unchanged", () => { + const input = "local branch listing failed"; + expect(redactCredentials(input)).toBe(input); + }); + + test("redacts every match when more than one credential-bearing URL appears in the same string", () => { + const input = "tried https://oauth2:tok1@a.example/x then https://oauth2:tok2@b.example/y"; + const out = redactCredentials(input); + expect(out).not.toContain("tok1"); + expect(out).not.toContain("tok2"); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/redact-credentials.test.ts` +Expected: FAIL — module does not exist. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/redact-credentials.ts +/** + * Strips userinfo (user:token@ or user@) out of any http(s) URL embedded in + * a string (S050): freshness.ts logs `remote.origin.url` verbatim at info + * on every reconcile, and echoes it into thrown errors every mr/discussions + * handler returns to callers. A repo cloned as + * `https://oauth2:glpat-XXXX@gitlab.example.com/...` (routine for + * dotfiles/CI-derived clones) puts that token into ~/.rt/logs/daemon.*.log + * and into any client-facing error message -- logs are the first thing a + * user pastes into a bug report. + */ +const CREDENTIAL_URL_RE = /(https?:\/\/)[^/@\s]+@/gi; + +export function redactCredentials(text: string): string { + return text.replace(CREDENTIAL_URL_RE, "$1[redacted]@"); +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/redact-credentials.test.ts` +Expected: PASS + +- [ ] **Step 5: Run tsc** + +Run: `bunx tsc --noEmit` +Expected: clean. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/redact-credentials.ts lib/daemon/__tests__/redact-credentials.test.ts +git commit -m "daemon: standalone credential-redaction utility for S050 (freshness.ts wiring documented, not wired here)" +``` + +--- + +## Task 11: docs note + final whole-branch verification + +**Files:** +- Create: `docs/daemon-api-auth.md` + +- [ ] **Step 1: Write the doc** + +```markdown +# The :9401 trust boundary + +How the daemon's REST/WS surface decides who to trust, and the follow-up +wiring two standalone modules from this phase still need in sibling-owned +files. + +## The model + +- **No `Origin` header at all** (the CLI, the Swift tray, rt-client from a + Bun/Node process, mr-board, gitq, the VS Code extension): unaffected, no + gate applies beyond what already existed. None of today's consumers send + an `Origin` header to `:9401`. +- **A browser `Origin` header is present**: trusted only if the request + presents the local `X-RT-Token` (`?token=` query param for `/ws`, since + browsers cannot set custom headers on a WS handshake) OR the Origin is on + the `rt.trustedBrowserOrigins` settings allowlist (see + `packages/rt-client/src/settings/registry-defs.ts`; `docs/settings-architecture.md` + is the settings-system contract). Otherwise: no `Access-Control-Allow-Origin` + on REST reads (default-deny CORS), and a 403 on `/ws`. +- **Mutating routes** (every method except GET/HEAD/OPTIONS, plus + `/api/secrets` and `/api/notifications` despite being GETs) require the + local `X-RT-Token` regardless of Origin — this is the CSRF defense against + a browser form/simple-request bypassing CORS preflight entirely, and it is + orthogonal to the Origin check above. + +See `lib/daemon/api-auth.ts` (`isBrowserRequestTrusted`, `needsToken`, +`getTrustedBrowserOrigins`) and `lib/daemon/api-server.ts` +(`buildCorsHeaders`, the `/ws` gate in `fetch()`) for the implementation. + +## Follow-up wiring for sibling-owned files (not done in this job) + +**S010** (`lib/daemon/handlers/worktree.ts`): `lib/daemon/git-ref-validation.ts` +exports `validateGitRef(ref)`. Call it right after `payload.branch` is read +(around `worktree.ts:282`) and return `{ ok: false, error }` on a rejection +BEFORE any `runGit` call reaches it — that single call site also covers the +weaker secondary instance in `divergence()` (`worktree.ts:211-213`), since +both read the same `branch` value. + +**S050** (`lib/daemon/freshness.ts`): `lib/daemon/redact-credentials.ts` +exports `redactCredentials(text)`. Wrap every log/error interpolation of a +remote URL with it — the audit names `freshness.ts:142, 148, 275, 279` as the +current call sites. + +**S043 caller side** (`lib/daemon.ts`): `lib/daemon/api-server.ts` exports +`ApiPortInUseError` (a named `Error` subclass with `.name === "ApiPortInUseError"` +and `.port`). Catch it around the `startApiServer()` call and park-and-retry +with backoff instead of letting it reach the top-level crash path; any other +error out of `startApiServer()` is a genuine misconfiguration and should keep +crashing as it does today. +``` + +- [ ] **Step 2: Run the full verification suite** + +Run: `bun test lib commands packages scripts` +Expected: all green. + +Run: `bunx tsc --noEmit` +Expected: zero errors. + +Run: `cd packages/rt-client && bun run build && cd -` +Expected: clean build (already run in Task 3, but re-run here as the final gate since later tasks may have touched files rt-client's dist-freshness test watches). + +- [ ] **Step 3: Commit** + +```bash +git add docs/daemon-api-auth.md +git commit -m "docs: the :9401 trust boundary model and the S010/S050/S043 sibling wiring notes" +``` + +--- + +## Self-Review Notes (already applied above) + +- **Spec coverage:** S092 (Task 1), S054 (Task 2), S005/S006/S040/S041/S084 (Tasks 3-4), S042 (Task 5), S083 (Task 6), S085 (Task 7), S043 (Task 8), S010 (Task 9), S050 (Task 10). All 13 cited findings have a task. +- **Write fence:** every modified/created file is inside the job's write fence; Tasks 9-10 deliberately stop short of editing `worktree.ts`/`freshness.ts` and document the wiring instead, mirroring the job brief's explicit instruction for S010 and extending the same treatment to S050 since its fix location is equally out of fence. +- **Type consistency:** `isBrowserRequestTrusted(origin, token, apiToken, allowedOrigins)` has the same parameter order and names everywhere it's used (Task 3 definition, Task 4 call sites). `BroadcastTarget`/`broadcastToClients` names match between Task 5's definition and its test. `pathParam`/`coerceQueryParams` signatures match between definition and test across Tasks 6-7. From 2246ec948f8367efa51df9e580114ce0550e9287 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:45:30 -0500 Subject: [PATCH 010/106] deps links: never auto-unlink a DEFAULT_EXPOSED tool (rt/fast-browser/gitq/deck) on a same-named PATH collision (S066) --- lib/deps/__tests__/links.test.ts | 20 ++++++++++++++++++++ lib/deps/links.ts | 11 ++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/deps/__tests__/links.test.ts b/lib/deps/__tests__/links.test.ts index 5b9fef2e..63ea5048 100644 --- a/lib/deps/__tests__/links.test.ts +++ b/lib/deps/__tests__/links.test.ts @@ -289,6 +289,26 @@ describe("tagged PATH links", () => { expect(result).toEqual({ removed: ["gh"], kept: [] }); }); + // S066: a DEFAULT_EXPOSED tool (rt's own product surface, not a vendored + // third-party tool like "gh") is never auto-unlinked by a same-named PATH + // collision — an unrelated foreign tool of the same name (e.g. Kong's own + // "deck") must never shadow-remove mattstack's. + test("reconcile never auto-unlinks deck/gitq/rt even when a same-named binary appears elsewhere on PATH (S066)", () => { + const p = bundleProbe({ env: { PATH: "/opt/homebrew/bin" } }); + p.symlink(join(appRoot, HELPERS_DIR, "deck"), linkPath(home, "deck")); + p.symlink(join(appRoot, HELPERS_DIR, "gitq"), linkPath(home, "gitq")); + link(p, "rt", {}, { installRtBinary: (src, dest) => { p.symlink(src, dest); return dest; } }); + + // An unrelated foreign binary happens to share each name. + p.writeFile("/opt/homebrew/bin/deck", "kongs-deck-binary"); + p.writeFile("/opt/homebrew/bin/gitq", "some-other-gitq-binary"); + p.writeFile("/opt/homebrew/bin/rt", "some-other-rt-binary"); + + const result = reconcile(p); + expect(result.removed).toEqual([]); + expect(result.kept.sort()).toEqual(["deck", "gitq", "rt"]); + }); + test("real-fs repro (F1): after the daemon's boot-time PATH prepend, reconcile removes nothing", () => { // Real fs (existsSync/statSync/symlinkSync/readdirSync via createRealProbes) // is what actually matters for this proof — the file/directory distinction diff --git a/lib/deps/links.ts b/lib/deps/links.ts index e70d56a9..df570d16 100644 --- a/lib/deps/links.ts +++ b/lib/deps/links.ts @@ -147,11 +147,20 @@ export function unlink(p: Probes, tool: string): { removed: boolean } { export function reconcile(p: Probes): { removed: string[]; kept: string[] } { const dir = join(p.home, ".local", "bin"); const forced = new Set(readSetupState(p).forcedLinks); + const defaultExposed = new Set(DEFAULT_EXPOSED); const removed: string[] = []; const kept: string[] = []; for (const tool of p.readDir(dir)) { if (!isOurLink(p, tool)) continue; // not one of ours — nothing to reconcile - if (forced.has(tool) || !userCopyOnPath(p, tool)) { + // DEFAULT_EXPOSED (rt, fast-browser, gitq, deck) is mattstack's own + // product surface, not a vendored third-party tool like the bundled + // "gh" — userCopyOnPath matches by name only, so a same-named PATH + // entry is at least as likely to be an unrelated namesake (Kong's own + // "deck") as a genuine second copy of ours, and shadow-removing our own + // "rt" this way breaks every command. Only a vendored tool's step-aside + // (deferring to a real system copy the user installed) is safe to + // decide by name alone. + if (defaultExposed.has(tool) || forced.has(tool) || !userCopyOnPath(p, tool)) { kept.push(tool); } else { p.removeFile(linkPath(p.home, tool)); From def840fc8ed89d01c7af5a96047a92df0692ef9c Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:47:36 -0500 Subject: [PATCH 011/106] age-key: give the keychain spawn a bounded, distinguishable timeout (S070) --- lib/home/__tests__/age-key.test.ts | 35 ++++++++++++++++++++ lib/home/age-key.ts | 52 ++++++++++++++++++++++++++---- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/lib/home/__tests__/age-key.test.ts b/lib/home/__tests__/age-key.test.ts index 4aad547d..bc6ee48e 100644 --- a/lib/home/__tests__/age-key.test.ts +++ b/lib/home/__tests__/age-key.test.ts @@ -9,6 +9,8 @@ import { keyExport, withArgvRedaction, AgeKeyAbsentError, + AgeKeyTimeoutError, + createRealAgeKeySeam, type AgeExecResult, type AgeKeySeam, } from "../age-key.ts"; @@ -378,3 +380,36 @@ describe("keyExport", () => { expect('import { statfs } from "./statfs-helpers.ts";').not.toMatch(FS_IMPORT_RE); }); }); + +// S070: a locked keychain (screen-lock, or an ACL mismatch between the dev +// shim and mattstack.app) pops a GUI dialog that blocks the real spawn +// forever; every seam.run must have a bounded deadline instead. +describe("createRealAgeKeySeam timeout (S070)", () => { + test("a spawn that outlives its timeout rejects with a distinguished AgeKeyTimeoutError instead of hanging", async () => { + const seam = createRealAgeKeySeam(); + await expect(seam.run(["sleep", "5"], { timeoutMs: 50 })).rejects.toThrow(AgeKeyTimeoutError); + }); + + test("the timeout error names the command without leaking a sensitive -w value", async () => { + const seam = createRealAgeKeySeam(); + // sh -c's trailing args become unused positional params ($0, $1, ...) — + // this still sleeps for the script text alone, while still carrying a + // "-w " pair later in argv for redactArgv to find. + const cmd = ["sh", "-c", "sleep 5", "argv0", "-w", "AGE-SECRET-KEY-should-not-appear"]; + try { + await seam.run(cmd, { timeoutMs: 50, sensitive: true }); + throw new Error("expected a timeout"); + } catch (err) { + expect(err).toBeInstanceOf(AgeKeyTimeoutError); + expect((err as Error).message).not.toContain("AGE-SECRET-KEY-should-not-appear"); + expect((err as Error).message).toContain(""); + } + }); + + test("a spawn that finishes well within its timeout resolves normally", async () => { + const seam = createRealAgeKeySeam(); + const res = await seam.run(["echo", "hi"], { timeoutMs: 5000 }); + expect(res.code).toBe(0); + expect(res.stdout.trim()).toBe("hi"); + }); +}); diff --git a/lib/home/age-key.ts b/lib/home/age-key.ts index 1df556ab..5629ce6d 100644 --- a/lib/home/age-key.ts +++ b/lib/home/age-key.ts @@ -29,8 +29,26 @@ export interface AgeKeySeam { * the private key this way, keeping it out of argv). * `sensitive` marks a call whose argv or stdin carries key material, so a * logging wrapper (withArgvRedaction) knows to redact it. + * `timeoutMs` overrides the default kill-and-reject deadline (see + * AgeKeyTimeoutError) — a locked keychain or an unexpected access-control + * dialog otherwise blocks this call, and every caller behind it, forever. */ - run(cmd: string[], opts?: { input?: string; sensitive?: boolean }): Promise; + run(cmd: string[], opts?: { input?: string; sensitive?: boolean; timeoutMs?: number }): Promise; +} + +/** + * Thrown instead of resolving when the spawn outlives its timeout — a + * distinct class (not a generic Error, and never a resolved AgeExecResult) + * so a caller like loadSecrets's domainMemo can recognize "the keychain is + * probably showing a dialog right now" and avoid caching it as a permanent + * decrypt failure that would poison every later retry, including the one + * after the user dismisses the dialog. + */ +export class AgeKeyTimeoutError extends Error { + constructor(message: string) { + super(message); + this.name = "AgeKeyTimeoutError"; + } } const KEYCHAIN_ACCOUNT = "mattstack"; @@ -277,6 +295,9 @@ function debugLog(cmd: string[]): void { if (CLI_DEBUG) console.error(`[age-key] ${cmd.join(" ")}`); } +/** A locked keychain (screen-lock, or a keychain item whose ACL belongs to a different signed binary) pops a GUI dialog and blocks until clicked; this bounds every spawn against that. */ +const DEFAULT_AGE_KEY_TIMEOUT_MS = 30_000; + /** Bun.spawn-based capture, env passed live (PATH-snapshot gotcha). Unexported: only reachable wrapped, via createRealAgeKeySeam. */ function createRawAgeKeySeam(): AgeKeySeam { return { @@ -298,12 +319,29 @@ function createRawAgeKeySeam(): AgeKeySeam { // read it ignore the close, but age-keygen -y blocks on EOF otherwise. if (opts?.input !== undefined) proc.stdin.write(opts.input); proc.stdin.end(); - const [stdout, stderr, code] = await Promise.all([ - new Response(proc.stdout).text(), - new Response(proc.stderr).text(), - proc.exited, - ]); - return { code, stdout, stderr }; + + const timeoutMs = opts?.timeoutMs ?? DEFAULT_AGE_KEY_TIMEOUT_MS; + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + try { proc.kill(); } catch { /* already exited */ } + }, timeoutMs); + + try { + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + if (timedOut) { + throw new AgeKeyTimeoutError( + `${redactArgv(cmd).join(" ")}: did not exit within ${timeoutMs}ms (keychain prompt pending?)`, + ); + } + return { code, stdout, stderr }; + } finally { + clearTimeout(timer); + } }, }; } From c0c1ff00b0bf9c9de7012c53e54c1283cd5765ab Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:49:06 -0500 Subject: [PATCH 012/106] presence-store: honor the tail heartbeat in the offline rule and prune predicate (S075) --- lib/state/__tests__/presence-store.test.ts | 22 +++++++++++++++++++++ lib/state/presence-store.ts | 23 ++++++++++++++++------ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/lib/state/__tests__/presence-store.test.ts b/lib/state/__tests__/presence-store.test.ts index f41f6cd2..c83e9c60 100644 --- a/lib/state/__tests__/presence-store.test.ts +++ b/lib/state/__tests__/presence-store.test.ts @@ -59,6 +59,28 @@ test("buddyStatus: table order, first match wins, tail heartbeat is COALESCE(tai expect(buddyStatus({ lastSeenAt: now }, now)).toBe("idle"); }); +// S075: a >24h autonomous agent (Monitor-driven, no user prompt) keeps its +// tail heartbeat fresh but its session heartbeat (last_seen_at, which only +// advances on a user prompt) goes stale past pruneMs. Both the offline rule +// and the prune predicate must honor the tail heartbeat, or the handle gets +// reclaimed out from under a session that never left. +test("buddyStatus: a fresh tail heartbeat keeps an armed row live past the 24h pruneMs boundary", () => { + expect(buddyStatus({ lastSeenAt: now - 25 * HOUR, armedAt: now, tailSeenAt: now }, now)).toBe("live"); +}); + +test("buddyStatus: an armed row is still offline when BOTH heartbeats are stale past pruneMs", () => { + expect(buddyStatus({ lastSeenAt: now - 25 * HOUR, armedAt: now - 25 * HOUR, tailSeenAt: now - 25 * HOUR }, now)).toBe("offline"); +}); + +test("prune: an armed row with a stale session heartbeat but a fresh tail heartbeat survives past 24h", () => { + const db = fresh(); + signIn({ sessionId: "s1", baseHandle: "x", now }, db); // last_seen_at stays at `now`, 25h stale by the prune call below + const pruneNow = now + 25 * HOUR; + db.run("UPDATE chat_presence SET armed_at = ?, tail_seen_at = ? WHERE session_id = 's1'", [pruneNow, pruneNow]); // still touching + expect(prunePresence(pruneNow, db)).toBe(0); + expect(db.query("SELECT COUNT(*) c FROM chat_presence").get()).toMatchObject({ c: 1 }); +}); + test("pulse writes last_seen_at and deets only", () => { const db = fresh(); signIn({ sessionId: "s1", baseHandle: "x", now }, db); diff --git a/lib/state/presence-store.ts b/lib/state/presence-store.ts index 3b4401a6..0d9c235a 100644 --- a/lib/state/presence-store.ts +++ b/lib/state/presence-store.ts @@ -83,10 +83,13 @@ const RECLAIMABLE_SQL = `signed_out_at IS NOT NULL OR (last_seen_at < ? AND COAL /** * Prune's own predicate, deliberately never RECLAIMABLE_SQL: that fragment's * bare `signed_out_at IS NOT NULL` leg would delete every signed-out row at - * daemon startup and empty the offline window. Bind params in order: - * dayAgo, dayAgo (same cutoff, both legs). + * daemon startup and empty the offline window. The second leg also honors + * the tail heartbeat (COALESCE(tail_seen_at, armed_at, 0)), the same fold + * buddyStatus's offline check uses — an armed row a long autonomous turn is + * still touching must survive even once last_seen_at alone looks stale. + * Bind params in order: dayAgo, dayAgo, dayAgo (same cutoff, all three legs). */ -const PRUNABLE_SQL = `(signed_out_at IS NOT NULL AND signed_out_at < ?) OR last_seen_at < ?`; +const PRUNABLE_SQL = `(signed_out_at IS NOT NULL AND signed_out_at < ?) OR (last_seen_at < ? AND COALESCE(tail_seen_at, armed_at, 0) < ?)`; const SELECT_PRESENCE_BY_HANDLE_SQL = `SELECT ${PRESENCE_COLUMNS} FROM chat_presence WHERE handle = ?;`; const SELECT_PRESENCE_BY_SESSION_SQL = `SELECT ${PRESENCE_COLUMNS} FROM chat_presence WHERE session_id = ?;`; @@ -147,7 +150,15 @@ export function buddyStatus( ): BuddyStatus { if (row.signedOutAt !== undefined) return "offline"; const lastSeenAt = row.lastSeenAt ?? 0; - if (now - lastSeenAt > th.pruneMs) return "offline"; + // A >24h autonomous agent (Monitor-driven, no user prompt) keeps only its + // tail heartbeat fresh — last_seen_at advances on a user prompt alone, so + // it starves for hours while the tail keeps touching. The offline check + // must honor whichever heartbeat is newer, or the handle reads offline + // (and the next chat:sign-in by any session prunes and reclaims it) out + // from under a session that never left. + const tailLiveness = row.armedAt !== undefined ? (row.tailSeenAt ?? row.armedAt) : 0; + const liveness = Math.max(lastSeenAt, tailLiveness); + if (now - liveness > th.pruneMs) return "offline"; if (row.armedAt !== undefined) { const tailHeartbeat = row.tailSeenAt ?? row.armedAt; return now - tailHeartbeat <= th.tailStaleMs ? "live" : "deaf"; @@ -309,7 +320,7 @@ export function pulseSession( export function listBuddies(now: number, db: Database = getStateDb()): Array { const th = presenceThresholds(); const dayAgo = now - th.pruneMs; - const rows = db.query(SELECT_NON_PRUNABLE_PRESENCE_SQL).all(dayAgo, dayAgo) as PresenceRawRow[]; + const rows = db.query(SELECT_NON_PRUNABLE_PRESENCE_SQL).all(dayAgo, dayAgo, dayAgo) as PresenceRawRow[]; return rows.map((raw) => { const presence = rowToPresence(raw); return { ...presence, status: buddyStatus(presence, now, th) }; @@ -358,7 +369,7 @@ export function assertSessionSignedIn(sessionId: string, db: Database = getState export function prunePresence(now: number, db: Database = getStateDb()): number { const th = presenceThresholds(); const dayAgo = now - th.pruneMs; - return db.query(DELETE_PRUNABLE_PRESENCE_SQL).run(dayAgo, dayAgo).changes; + return db.query(DELETE_PRUNABLE_PRESENCE_SQL).run(dayAgo, dayAgo, dayAgo).changes; } // --- Internal wiring for chat-store.ts's dual-write (arm/touch/disarm). --- From fd626d8b4f4290c65311c0979371c7656c155fd7 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:49:52 -0500 Subject: [PATCH 013/106] chat:sign-in: reject a missing/empty sessionId instead of storing a NULL-keyed presence row (S076) --- lib/daemon/__tests__/chat-handlers.test.ts | 29 ++++++++++++++++++++++ lib/daemon/handlers/chat.ts | 8 ++++++ lib/state/presence-store.ts | 6 +++++ 3 files changed, 43 insertions(+) diff --git a/lib/daemon/__tests__/chat-handlers.test.ts b/lib/daemon/__tests__/chat-handlers.test.ts index b54a0774..fb75f284 100644 --- a/lib/daemon/__tests__/chat-handlers.test.ts +++ b/lib/daemon/__tests__/chat-handlers.test.ts @@ -221,6 +221,35 @@ test("sign-in rejects an invalid baseHandle with a reason rather than normalizin expect(res.error).toContain("handle"); }); +// S076: a missing/empty sessionId binds as NULL against session_id TEXT +// PRIMARY KEY, which SQLite accepts — the row then holds the UNIQUE handle +// but the reclaim-by-session_id path can never match it, wedging every +// later sign-in under that base handle with a UNIQUE constraint failure. +test("sign-in rejects a missing sessionId rather than storing a NULL-keyed row", async () => { + const h = freshHandlers(); + const res = await h["chat:sign-in"]({ baseHandle: "x" } as any); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("sessionId"); +}); + +test("sign-in rejects an empty-string sessionId the same way", async () => { + const h = freshHandlers(); + const res = await h["chat:sign-in"]({ sessionId: "", baseHandle: "x" }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("sessionId"); +}); + +test("a rejected sign-in never wedges the next sign-in under the same base handle", async () => { + const h = freshHandlers(); + await h["chat:sign-in"]({ baseHandle: "x" } as any); // rejected, must not persist a row + const ok = await h["chat:sign-in"]({ sessionId: "s1", baseHandle: "x" }); + expect(ok.ok).toBe(true); + if (!ok.ok) throw new Error("unreachable"); + expect(ok.data).toMatchObject({ handle: "x" }); +}); + test("a reclaimed handle refuses the old session's pulse with the reason", async () => { const h = freshHandlers(); await h["chat:sign-in"]({ sessionId: "s1", baseHandle: "x" }); diff --git a/lib/daemon/handlers/chat.ts b/lib/daemon/handlers/chat.ts index 98101e8a..470e846d 100644 --- a/lib/daemon/handlers/chat.ts +++ b/lib/daemon/handlers/chat.ts @@ -297,6 +297,14 @@ export function createChatHandlers(opts: { "chat:sign-in": async (payload: Commands["chat:sign-in"]["payload"]): Promise> => { const { sessionId, baseHandle, cwd, repo, branch, pane, statusText } = payload; + // A missing/empty sessionId binds as NULL against session_id TEXT + // PRIMARY KEY, which SQLite accepts silently: the row then holds the + // UNIQUE handle, but the reclaim-by-session_id path can never match + // it, so every later sign-in under this base handle 500s with a + // UNIQUE constraint failure until prunePresence eventually removes it. + if (typeof sessionId !== "string" || sessionId.length === 0) { + return { ok: false, error: "chat:sign-in requires a non-empty sessionId" }; + } if (!isValidChatName(baseHandle)) return { ok: false, error: `invalid handle "${baseHandle}"` }; const data = signIn({ sessionId, baseHandle, cwd, repo, branch, pane, statusText }, db); return { ok: true, data }; diff --git a/lib/state/presence-store.ts b/lib/state/presence-store.ts index 0d9c235a..0d052aac 100644 --- a/lib/state/presence-store.ts +++ b/lib/state/presence-store.ts @@ -224,6 +224,12 @@ export function signIn( db: Database = getStateDb(), ): { handle: string; reclaimed: boolean } { const { sessionId, baseHandle, statusText } = args; + // Defense in depth: the handler (lib/daemon/handlers/chat.ts) is the + // root-cause guard, but session_id is a bare TEXT PRIMARY KEY with no + // NOT NULL/CHECK constraint (bun:sqlite binds undefined as NULL, which + // SQLite accepts), so any future caller of this store function directly + // must not be able to wedge the same NULL-keyed-row failure mode. + if (!sessionId) throw new Error("signIn: sessionId is required"); const cwd = args.cwd ?? null; const repo = args.repo ?? null; const branch = args.branch ?? null; From 5651d9a16ed53300c86934d2fca9256d028a54c9 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:50:45 -0500 Subject: [PATCH 014/106] worktree trash: exclude .worktrees/ from the repo's git status on every retire, not just createTree's path (S078) --- lib/worktree/__tests__/trash.test.ts | 25 +++++++++++++++++++++++++ lib/worktree/trash.ts | 8 ++++++++ 2 files changed, 33 insertions(+) diff --git a/lib/worktree/__tests__/trash.test.ts b/lib/worktree/__tests__/trash.test.ts index c4ed2348..73c77411 100644 --- a/lib/worktree/__tests__/trash.test.ts +++ b/lib/worktree/__tests__/trash.test.ts @@ -164,6 +164,31 @@ describe("worktree trash", () => { expect(result.ok).toBe(false); expect(existsSync(tree)).toBe(true); }); + + // S078: `.worktrees/` was never added to info/exclude unless the tree + // went through createTree first (e.g. `rt worktree adopt`'s disposals, + // or any repo whose worktrees root differs from the default, skip that + // call entirely) — the retention store then shows up as `?? .worktrees/` + // in the user's own `git status`, and `git add -A` stages a whole second + // copy of the source tree into it. + test("retireTree excludes .worktrees/ from the repo's own git status, even without going through createTree first", async () => { + Bun.spawnSync(["git", "init", "-q", repo]); + Bun.spawnSync(["git", "-C", repo, "config", "user.email", "test@example.com"]); + Bun.spawnSync(["git", "-C", repo, "config", "user.name", "Test"]); + writeFileSync(join(repo, "README.md"), "hi\n"); + Bun.spawnSync(["git", "-C", repo, "add", "README.md"]); + Bun.spawnSync(["git", "-C", repo, "commit", "-q", "-m", "init"]); + + const tree = makeTree(root, "hotel"); + const result = await retireTree(tree, "hotel", repo); + expect(result.ok).toBe(true); + + const exclude = readFileSync(join(repo, ".git", "info", "exclude"), "utf8"); + expect(exclude).toContain(".worktrees/"); + + const status = Bun.spawnSync(["git", "-C", repo, "status", "--porcelain"]).stdout.toString(); + expect(status).not.toContain(".worktrees"); + }); }); describe("stripTrashDir", () => { diff --git a/lib/worktree/trash.ts b/lib/worktree/trash.ts index 9b39f72a..80292078 100644 --- a/lib/worktree/trash.ts +++ b/lib/worktree/trash.ts @@ -22,6 +22,7 @@ import { mkdir, readdir, rename } from "fs/promises"; import { basename, dirname, join } from "path"; +import { ensureInfoExclude } from "./git-async.ts"; /** Marks a directory as rt's to delete. Nothing without this prefix is ever reaped. */ export const TRASH_PREFIX = ".trash-"; @@ -110,6 +111,13 @@ export async function retireTree( if (!name || name.includes("/") || name.includes("\\")) { throw new Error(`worktree trash name must be a single path component: ${JSON.stringify(name)}`); } + // retainedTrashRoot is always /.worktrees/.trash regardless of + // cfg.root, so this exclude pattern is correct in every case — including + // a tree disposed without ever going through createTree first (e.g. `rt + // worktree adopt`'s disposals), whose repo may never have had + // ".worktrees/" excluded at all. Without it the retention store shows up + // as `?? .worktrees/` in the user's own `git status`. + await ensureInfoExclude(repoPath, ".worktrees/"); const root = retainedTrashRoot(repoPath); await mkdir(root, { recursive: true }); trashPath = join(root, `${name}-${Date.now()}`); From b64dac756f378e6f0e880ef113c9ee393b4eff0e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:50:52 -0500 Subject: [PATCH 015/106] docs: sketch daemon supervision verdicts + exit-code semantics --- docs/daemon-supervision-design.md | 69 +++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 docs/daemon-supervision-design.md diff --git a/docs/daemon-supervision-design.md b/docs/daemon-supervision-design.md new file mode 100644 index 00000000..f672cafb --- /dev/null +++ b/docs/daemon-supervision-design.md @@ -0,0 +1,69 @@ +# Daemon supervision: status verdicts and exit-code semantics + +Phase 0 design anchor for the rt daemon stability roadmap (audit +2026-08). Tasks 9–14 of the Phase 0 plan implement this. + +## launchd contract + +The prod plist sets `KeepAlive = { SuccessfulExit: false }`: launchd +respawns the daemon ONLY on a non-zero exit. A zero exit means "stay +down". Every exit-code decision below follows from that single fact. + +## Exit-code policy + +| Path | Exit | Why | +|-----------------------------------------|------|-----| +| `startDaemon()` boot throw (prod path) | 1 | Visible + launchd relaunches. Paired with crash-loop detection so it cannot loop silently forever. | +| `shutdown` IPC/REST verb | 0 | Intentional stop; launchd must not respawn. Records `last-exit.kind = "shutdown"`. | +| Bare OS signal SIGTERM/SIGINT/SIGHUP | 1 | External kill (pkill, script, memory pressure); launchd SHOULD respawn. The sanctioned stop path goes through SMAppService.unregister, where the exit code is irrelevant, so exiting non-zero here does not break intended stops. | +| Crash-loop guard trips (N in M minutes) | 0 (park) | Stop the flapping; surface `crash-looping` so a human intervenes instead of launchd hammering every ~10s. | + +Mechanism: a module-scope `shuttingDownViaVerb` flag is set true by the +`shutdown` verb before it calls cleanup; `gracefulExit(signal)` reads it +— set → exit(0), unset (bare signal) → exit(1). + +Boot-phase gate: a module-scope `bootPhase: "booting" | "ready"` flips +to `"ready"` immediately before the `daemon ready` log. The +`unhandledRejection` handler exits(1) while `bootPhase === "booting"` +and only logs (recovers) once ready — so a boot-time stray rejection is +fatal but a steady-state one is not. + +## Status verdicts + +`rt daemon status` and `/api/status` classify by first match: + +1. `not-installed` — SMAppService not registered. +2. `serving` — ping on rt.sock succeeds. +3. `parked` — ping fails, a live rt pid exists, and the boot breadcrumb + phase is a flavor standoff (park). Named distinctly so the user is + told "another flavor owns the socket", not "wedged". +4. `alive-not-serving` — ping fails but a live rt pid exists + (`process.kill(pid,0)` on rt.pid, or `pgrep -f 'rt --daemon|lib/daemon.ts'`). + Sub-detail from the breadcrumb phase: `booting` / `wedged`, or + `quarantined` when a state.db/events.db boot-failed marker is present. + Prints "process is running but not answering rt.sock — rt daemon logs -t". +5. `crash-looping` — no live pid AND the kv failure record shows ≥ N + failures within the last M minutes (N=3, M=5). Prints the last reason. +6. `boot-failed` — no live pid AND the most recent kv exit record is a + boot throw (fewer than N failures). Prints the last reason + phase. +7. `installed-not-running` — registered, no live pid, clean/again-absent + exit record. + +## Persisted state (kv, ns `daemon-supervision`, no schema change) + +- `boot-attempts` (number) — incremented at the top of `runDaemon()`. +- `last-ready-at` (number, epoch ms) — stamped just before `daemon ready`. +- `recent-failures` (array of `{ at, phase, reason }`, capped to 10) — + appended by the boot fatal path and by state.db/events.db boot-failed + markers. Crash-loop = ≥ N entries newer than now − M minutes. +- `last-exit` (`{ at, kind: "shutdown" | "signal" | "boot-failed", code, reason? }`) + — written by the shutdown verb, the signal handlers, and the boot + fatal path. Lets status distinguish "cleanly stopped" from "died". + +## Boot breadcrumb + +`~/.mattstack/rt/daemon-boot.json` = `{ at, pid, flavor, phase }`, +rewritten at each boot phase: `start` → `crash-handlers` → `events-db` +→ `state-db` → `socket` → `api` → `ready`. Lets `alive-not-serving` +name where a live-but-silent daemon is stuck even when the logs are +unreadable. Removed (or stamped `ready`) on successful boot. From f0ffdbbc384df978c1df0c500c37a47a80078b92 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:50:53 -0500 Subject: [PATCH 016/106] docs: retire stale daemon-runner-health.md, point at the current audit + supervision design --- docs/daemon-runner-health.md | 189 ++--------------------------------- 1 file changed, 7 insertions(+), 182 deletions(-) diff --git a/docs/daemon-runner-health.md b/docs/daemon-runner-health.md index 95b5ae62..9d6db779 100644 --- a/docs/daemon-runner-health.md +++ b/docs/daemon-runner-health.md @@ -1,184 +1,9 @@ -# Daemon & Runner Code-Health Audit +# Daemon runner health — superseded -Reference analysis for the `rt` daemon ([lib/daemon.ts](../lib/daemon.ts) and [lib/daemon/](../lib/daemon/)) and the runner TUI ([commands/runner.tsx](../commands/runner.tsx)). +This document audited subsystems (process-manager, remedy-engine, +runner.tsx, workspace-sync) that no longer exist. It is retained only as +a redirect. -Both subsystems are central — the daemon owns all process lifecycle and long-lived state; the runner is our flagship feature and the primary surface that touches that state. Defects here leak processes, corrupt UI state, or silently drop features without any obvious error. This document exists to catalog known issues so they can be fixed deliberately rather than re-discovered. - -## Legend - -- **P0** — Active correctness bug or resource leak with user-visible impact. -- **P1** — Reliability issue, silent feature drop, or unbounded growth under realistic use. -- **P2** — Edge case or latent bug that will bite under specific conditions. -- **P3** — Code quality / testability — not a bug, but the surrounding code will rot without it. - -Each item is scoped by file:line-link so a fix PR can jump straight to the site. - ---- - -## 1. Orphaned processes - -### ~~P0 — Child process groups are not reaped on kill~~ ✅ FIXED -[lib/daemon/process-manager.ts:34-54](../lib/daemon/process-manager.ts#L34-L54), [lib/daemon/process-manager.ts:140-147](../lib/daemon/process-manager.ts#L140-L147), [lib/daemon/process-manager.ts:182-190](../lib/daemon/process-manager.ts#L182-L190) - -`Bun.spawn` now passes `detached: true` so the child is a session/pgroup leader (pgid == pid). A new `killGroup(pid, signal)` helper sends signals to `-pid` with guards against pid ≤ 1 and ESRCH. Both the kill() path and the existing-process eviction path now use it. New regression test [`process-manager.test.ts`](../lib/daemon/__tests__/process-manager.test.ts) `"kill reaps grandchildren (detached pgroup)"` verifies the fix by spawning backgrounded sleepers, killing the parent, and asserting every grandchild pid is gone. - -### ~~P0 — `process:respawn` drops the remedy subscription~~ ✅ FIXED -[lib/daemon.ts:720-726](../lib/daemon.ts#L720-L726) - -Added `remedyEngine.onSpawn(id)` after `processManager.respawn(id)`. Re-uses stored `processMeta` (cwd/cmd unchanged across respawn). - -### ~~P1 — Warm (SIGSTOP) processes survive daemon death as stopped zombies~~ ✅ FIXED -[lib/daemon/state-store.ts:28-104](../lib/daemon/state-store.ts#L28-L104), [lib/daemon.ts:1398-1411](../lib/daemon.ts#L1398-L1411), [lib/daemon/process-manager.ts:168-174](../lib/daemon/process-manager.ts#L168-L174) - -`StateStore` now persists `{state, pid}` per id. `ProcessManager.spawn` records pid on spawn; terminal transitions (`stopped`/`crashed`) clear it. `reconcileAfterRestart()` returns orphans (non-stopped with a pid), which the daemon reaps via `killGroup(pid, SIGCONT)` followed by `killGroup(pid, SIGKILL)`. Legacy `{id: state}` persistence format is still accepted on load. Regression coverage in [state-store.test.ts](../lib/daemon/__tests__/state-store.test.ts#L200-L230). - -### ~~P1 — `subscribeToOutput` unsubscribers leave empty Sets behind~~ ✅ FIXED -[lib/daemon/process-manager.ts:100-109](../lib/daemon/process-manager.ts#L100-L109) - -Unsubscribe now deletes the Set from `outputHooks` once its size reaches zero. - ---- - -## 2. Memory leaks - -### ~~P0 — `:wt` git watcher is torn down every reconcile tick~~ ✅ FIXED -[commands/runner.tsx:1580-1584](../commands/runner.tsx#L1580-L1584) - -Reconcile loop now strips a trailing `:wt` from the key before checking `activeLaneIds`, so the companion watcher lives and dies with its parent lane instead of being torn down every tick. - -### ~~P1 — Global-remedy `fs.watch` has no handle and no cleanup~~ ✅ FIXED -[lib/daemon.ts:132-163](../lib/daemon.ts#L132-L163), [lib/daemon.ts:1320](../lib/daemon.ts#L1320) - -Handle retained in `globalRemedyWatcher` and closed in `cleanup()`. - -### ~~P1 — `ProcessManager.spawnConfigs` and `outputHooks` grow unbounded~~ ✅ FIXED -[lib/daemon.ts:770-779](../lib/daemon.ts#L770-L779), [commands/runner.tsx](../commands/runner.tsx) - -Added `process:remove` IPC handler that tears down all five daemon-side maps (remedy state, attach socket, log buffer, process config, state-store entry). The runner now calls it from every entry-deletion path (remove-entry, remove-lane, reset). Entries with fresh ids no longer leak daemon-side state. - -### ~~P2 — `RemedyEngine` can create orphan state for globals-only matches~~ ✅ FIXED -[lib/daemon.ts:773](../lib/daemon.ts#L773) - -`process:remove` handler (added in Session B) calls `remedyEngine.unregister(id)` before tearing down the other maps, so globals-only state is cleaned up when the runner deletes an entry. - -### ~~P2 — `refreshCache` has no in-flight guard~~ ✅ FIXED -[lib/daemon.ts:390-407](../lib/daemon.ts#L390-L407) - -Split into a coalescing `refreshCache()` wrapper and the original logic as `refreshCacheImpl()`. Concurrent callers await the same promise. - -### ~~P2 — outputHooks Map retains empty Sets~~ ✅ FIXED -Covered by the subscribeToOutput unsubscribe cleanup above. - ---- - -## 3. Bugs - -### ~~P1 — Invalid JSON during save wipes all globals~~ ✅ FIXED -[lib/runner-store.ts:154-168](../lib/runner-store.ts#L154-L168), [lib/daemon.ts:137-158](../lib/daemon.ts#L137-L158) - -`loadGlobalRemedies` now throws on parse failure / non-array shape; missing file still returns `[]`. Daemon watcher callback catches and logs without calling `reloadGlobals`, so live rules persist through transient invalid states. - -### ~~P1 — Global-remedy watcher is not debounced~~ ✅ FIXED -[lib/daemon.ts:147-161](../lib/daemon.ts#L147-L161) - -100ms settle timer collapses the rename+change burst into a single reload. - -### ~~P1 — Concurrent `cache:refresh` via REST API~~ ✅ FIXED -Resolved by the `refreshCache` in-flight guard above. - -### ~~P2 — `entry.id` basename collision~~ ✅ FIXED -[lib/runner-store.ts:410-420](../lib/runner-store.ts#L410-L420) - -`normalizeLane` now detects within-lane entry id duplicates and appends a 6-char sha1 of the worktree path to the loser (`~a1b2c3`). Collisions no longer silently alias two processes' PTY/state. - -### ~~P2 — `onSpawn` re-merge uses stale meta for callers that pass neither cwd nor cmd~~ ✅ FIXED -[lib/daemon.ts:753-761](../lib/daemon.ts#L753-L761), [lib/daemon/process-manager.ts:245-247](../lib/daemon/process-manager.ts#L245-L247) - -`process:respawn` — the only caller that used to pass no cwd/cmd — now pulls them from `processManager.getSpawnConfig(id)` so globals-only matches survive respawn. Other `onSpawn` callers already pass cwd/cmd from their handler payload. - -### ~~P2 — `reloadGlobals` didn't subscribe for processes registered with empty remedies~~ ✅ FIXED -[lib/daemon/remedy-engine.ts:108-127](../lib/daemon/remedy-engine.ts#L108-L127) - -If `register()` was called with no per-entry remedies and `onSpawn` early-returned (no subscription because merged was empty), a later `reloadGlobals` that added matching rules updated `s.remedies` but never subscribed to output — so nothing fired. Found while writing the RemedyEngine test suite. Fix: `reloadGlobals` now subscribes on empty→non-empty and unsubscribes on non-empty→empty. Regression test in [remedy-engine.test.ts "reloadGlobals mid-flight re-merges active states"](../lib/daemon/__tests__/remedy-engine.test.ts#L300-L319). - -### ~~P3 — Dead constant~~ ✅ FIXED -[lib/daemon.ts:48](../lib/daemon.ts#L48) - -`LINEAR_REFRESH_INTERVAL_MS` removed. - -### ~~P3 — Compaction ordering is interleaving-sensitive~~ ✅ FIXED -[lib/runner-store.ts:338-348](../lib/runner-store.ts#L338-L348) - -Each entry now gets its absolute input index as its position. Groups inherit the position of their first member. Output order follows input order deterministically. - -### ~~P3 — State-store permits invalid transitions silently~~ ✅ FIXED -[lib/daemon/state-store.ts:111-119](../lib/daemon/state-store.ts#L111-L119), [lib/daemon.ts:1403-1407](../lib/daemon.ts#L1403-L1407) - -Forced transitions remain permitted (needed for kill-of-warm, reconcile, etc.) but `StateStore` now exposes `onInvalidTransition(cb)` and fires it when the move isn't in `VALID_TRANSITIONS`. The daemon wires this to its log so drift surfaces in the daemon log instead of being silently swallowed. - ---- - -## 4. Code organization / testability - -### ~~P1 — `lib/daemon.ts` is 1400+ lines with no seams~~ ✅ FIXED -[lib/daemon.ts](../lib/daemon.ts), [lib/daemon/handlers/](../lib/daemon/handlers/) - -`handleCommand` split into four domain modules behind a routed-lookup-first-then-switch dispatch: - -- [lib/daemon/handlers/process.ts](../lib/daemon/handlers/process.ts) — `process:spawn|kill|respawn|remove|start|stop|restart|list|state|states|logs|attach-info|suspend|resume` -- [lib/daemon/handlers/cache.ts](../lib/daemon/handlers/cache.ts) — `cache:read|refresh` + `branch:enrich` -- [lib/daemon/handlers/remedy.ts](../lib/daemon/handlers/remedy.ts) — `remedy:set|clear|drain` -- [lib/daemon/handlers/proxy.ts](../lib/daemon/handlers/proxy.ts) — `proxy:start|stop|set-upstream|status|list` - -Each module is a factory that takes a `HandlerContext` and returns a `HandlerMap`. Daemon.ts constructs the ctx once and merges the maps into `routedHandlers`; `handleCommand` does `routedHandlers[cmd] ?? switch` so non-extracted commands (ping, hooks:*, repos, ports, status, tcc:check, notifications*, tray:status, group:*, port:*, shutdown) remain inline because they read daemon-local state (watchers, repos index, notifications, port allocator, groups) that wouldn't benefit from being pushed out. - -Live cache access goes through `ctx.cache.entries` — `loadCache()` now mutates `cache.entries` in place instead of reassigning, so handlers see disk reloads without plumbing getters. - -### ~~P1 — `lib/runner-store.ts` doubled in size with no tests~~ ✅ FIXED -[lib/runner-store/compact.ts](../lib/runner-store/compact.ts), [lib/__tests__/runner-store-compact.test.ts](../lib/__tests__/runner-store-compact.test.ts) - -Added 7 round-trip tests and physically extracted `normalizeRemedy`, `normalizeEntry`, `compactEntries` (and their helpers) to `lib/runner-store/compact.ts`. runner-store.ts now re-exports `compactEntries`/`normalizeEntry` and delegates all compact↔expand logic. The extracted module imports only types from `runner-store.ts`, so there's no runtime cycle. - -### ~~P1 — `commands/runner.tsx` is 2500+ lines~~ ✅ PARTIALLY FIXED -[lib/runner/git-watchers.ts](../lib/runner/git-watchers.ts) - -Git watcher setup, per-lane debounce, and linked-worktree `.git/worktrees//HEAD` handling extracted into `createGitWatcherPool(onChange) → { sync, dispose }`. `repoGitDir`, `readCurrentBranch`, `readCurrentBranchAsync` now live there too. runner.tsx drops from 2500 → ~2430 lines and no longer imports `fs.watch` or `FSWatcher` at all. Remaining deferrals (keymap-handler split, `reconcileWatchers` module) are cosmetic and not bugs. - -### ~~P2 — `ProcessManager.spawn` does too many things~~ ✅ PARTIALLY FIXED -[lib/daemon/process-manager.ts:31-44](../lib/daemon/process-manager.ts#L31-L44) - -Port eviction is now an async `evictPort(port)` helper — no more synchronous `Bun.spawnSync(["sh","-c","lsof..."])` stalling the event loop. The broader decomposition of `spawn()` is deferred with the other structural refactors. - -### ~~P2 — No unit tests for `RemedyEngine` lifecycle invariants~~ ✅ FIXED -[lib/daemon/__tests__/remedy-engine.test.ts](../lib/daemon/__tests__/remedy-engine.test.ts) - -Added 13 tests driving `register`/`onSpawn`/`unregister` against a `FakeProcessManager` stub. Covers all five invariants (hook accumulation, orphan on removal, concurrent triggers, cooldown gating, double-respawn guard) plus pattern handling (array OR, invalid regex tolerance, ANSI stripping) and global remedies (cwdContains match/non-match, reload mid-flight re-merge). Writing the tests uncovered the latent `reloadGlobals`-doesn't-subscribe bug fixed above. - -### ~~P3 — Repeated sync git reads on runner startup~~ ✅ FIXED -[commands/runner.tsx:483-498](../commands/runner.tsx#L483-L498), [commands/runner.tsx:2527-2545](../commands/runner.tsx#L2527-L2545) - -Added async `readCurrentBranchAsync` (uses `Bun.spawn` + `proc.exited`) and parallelized the startup fan-out with `Promise.all`. Cost is now ~max-spawn-time, not N×spawn-time. - ---- - -## Status summary (audit sweep) - -All correctness, leak, and resource-safety items from the audit have been addressed: - -- **P0 / P1 bugs & leaks:** all fixed — pgroup reaping, remedy respawn subscription, `:wt` watcher, global-remedy hardening, refreshCache coalescing, warm-process crash recovery, empty-Set cleanup, map growth. -- **P2 correctness:** all fixed — entry.id collision salting, onSpawn meta passthrough, reloadGlobals missed-subscribe, RemedyEngine lifecycle tests, state-store invalid-transition observability. -- **P3:** all fixed — dead constant removed, compaction ordering deterministic, async parallel git reads on startup. - -**Structural refactors completed:** -- `handleCommand` split into `lib/daemon/handlers/{process,cache,remedy,proxy}.ts` — routed-lookup dispatch in daemon.ts, 14 `process:*` + 3 `cache:*`/`branch:*` + 3 `remedy:*` + 5 `proxy:*` commands all flow through factory-injected `HandlerContext`. -- compact/expand physically extracted to `lib/runner-store/compact.ts`; runner-store.ts re-exports for back-compat. -- Git watcher pool extracted to `lib/runner/git-watchers.ts` — `createGitWatcherPool(onChange) → { sync, dispose }` with built-in 150ms per-lane debounce. - -**Remaining cosmetic deferrals:** -- `commands/runner.tsx` keymap handlers could be split into sibling files for testability, but that's pure reorg — no correctness signal behind it. - -## Not covered here - -- `SuspendManager`, `ExclusiveGroup`, `AttachServer`, `WorkspaceSync`, `PortAllocator` — not re-read this pass. Worth a follow-up audit on the same four axes. -- Test coverage itself — there is very little for the daemon. Any of the extraction work above should ship with the matching test file. -- Observability — `diag()` is sprinkled throughout but there's no sampling, no correlation id, no structured reader. A shared `makeChildLogger(component, id)` helper would clean up call sites and produce greppable logs. +- Current stability audit + roadmap: `docs/daemon-stability-audit-2026-08.md` + (in the daemon-stability-audit worktree). +- Supervision verdicts + exit-code semantics: `docs/daemon-supervision-design.md`. From a15ed3e00905543bb0218c5cf252015ee0db6d0e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:51:22 -0500 Subject: [PATCH 017/106] daemon: cap request body size at 1 MiB on both servers (S092) --- .../__tests__/request-body-size.test.ts | 42 +++++++++++++++++++ lib/daemon/api-server.ts | 2 + lib/daemon/request-limits.ts | 10 +++++ lib/daemon/socket-server.ts | 2 + 4 files changed, 56 insertions(+) create mode 100644 lib/daemon/__tests__/request-body-size.test.ts create mode 100644 lib/daemon/request-limits.ts diff --git a/lib/daemon/__tests__/request-body-size.test.ts b/lib/daemon/__tests__/request-body-size.test.ts new file mode 100644 index 00000000..b6ee04a5 --- /dev/null +++ b/lib/daemon/__tests__/request-body-size.test.ts @@ -0,0 +1,42 @@ +/** + * Bun enforces `maxRequestBodySize` itself (413 before the handler runs) -- + * this is a live-server test, not a pure-function one, because there is no + * pure function to unit test: the cap is a Bun.serve runtime option. + */ +import { describe, test, expect, afterEach } from "bun:test"; +import type { Server } from "bun"; +import { MAX_REQUEST_BODY_SIZE } from "../request-limits.ts"; + +let server: Server | undefined; + +afterEach(() => { + server?.stop(true); + server = undefined; +}); + +describe("MAX_REQUEST_BODY_SIZE", () => { + test("is set to 1 MiB", () => { + expect(MAX_REQUEST_BODY_SIZE).toBe(1024 * 1024); + }); + + test("Bun rejects a body over the cap with a 4xx before the handler runs", async () => { + let handlerRan = false; + server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + maxRequestBodySize: 10, // tiny cap for a fast, deterministic test + async fetch(req) { + handlerRan = true; + await req.text(); + return new Response("ok"); + }, + }); + const res = await fetch(`http://127.0.0.1:${server.port}/`, { + method: "POST", + body: "x".repeat(1000), + }); + expect(res.status).toBeGreaterThanOrEqual(400); + expect(res.status).toBeLessThan(500); + expect(handlerRan).toBe(false); + }); +}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index 7dab9b6c..39558743 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -12,6 +12,7 @@ import type { Logger } from "pino"; import { API_PORT } from "../daemon-config.ts"; import { needsToken, tokenOk, loadOrCreateApiToken } from "./api-auth.ts"; import { getAggregatedConnection } from "./freshness.ts"; +import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; const API_INDEX = { name: "rt daemon", @@ -140,6 +141,7 @@ export async function startApiServer(deps: ApiServerDeps): Promise> // clients aren't reaped every 10s ("[Bun.serve]: request timed out"). // 255 is the server max; the websocket block below sets its own larger one. idleTimeout: 255, + maxRequestBodySize: MAX_REQUEST_BODY_SIZE, async fetch(req, server) { const url = new URL(req.url); diff --git a/lib/daemon/request-limits.ts b/lib/daemon/request-limits.ts new file mode 100644 index 00000000..d8448818 --- /dev/null +++ b/lib/daemon/request-limits.ts @@ -0,0 +1,10 @@ +/** + * Shared cap on request body size for both daemon servers (api-server.ts's + * :9401 HTTP/WS surface and socket-server.ts's unix-socket IPC channel). + * Neither transport authenticates reads, so an unbounded body (Bun's + * default is 128 MB) lets any same-user process or a cross-origin browser + * request stall the daemon's single event loop parsing a giant payload. + * Real payloads on both transports are kilobytes; 1 MiB costs nothing and + * turns an oversized request into an immediate 413 instead. + */ +export const MAX_REQUEST_BODY_SIZE = 1024 * 1024; diff --git a/lib/daemon/socket-server.ts b/lib/daemon/socket-server.ts index 7ab37fb8..fee4c27e 100644 --- a/lib/daemon/socket-server.ts +++ b/lib/daemon/socket-server.ts @@ -9,6 +9,7 @@ import { existsSync, unlinkSync } from "fs"; import type { Server } from "bun"; import type { Logger } from "pino"; import { DAEMON_SOCK_PATH } from "../daemon-config.ts"; +import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; export function startSocketServer(opts: { handleCommand: (cmd: string, payload: any, signal?: AbortSignal) => Promise; @@ -33,6 +34,7 @@ export function startSocketServer(opts: { // on how long an idle connection may sit before Bun kills it — it never // holds connections open on its own. idleTimeout: 255, + maxRequestBodySize: MAX_REQUEST_BODY_SIZE, async fetch(req) { try { const url = new URL(req.url); From cd20722ce829db604e3b852e300d5a489bbf8eca Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:55:23 -0500 Subject: [PATCH 018/106] worktree trash reap: require a plausible rt-written epoch, and refuse a configured root that is an ancestor of the repo (S079) --- .../__tests__/worktree-reconciler.test.ts | 21 +++++++++++++++- lib/daemon/worktree-reconciler.ts | 24 +++++++++++++++++-- lib/worktree/__tests__/trash.test.ts | 13 ++++++++++ lib/worktree/trash.ts | 17 ++++++++++++- 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/lib/daemon/__tests__/worktree-reconciler.test.ts b/lib/daemon/__tests__/worktree-reconciler.test.ts index 8f30830a..8d2bca47 100644 --- a/lib/daemon/__tests__/worktree-reconciler.test.ts +++ b/lib/daemon/__tests__/worktree-reconciler.test.ts @@ -2,7 +2,7 @@ import { describe, test, expect, beforeEach, spyOn } from "bun:test"; import { execSync } from "child_process"; import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; -import { basename, join } from "path"; +import { basename, dirname, join } from "path"; import type { Logger } from "pino"; import { readJson, writeJson } from "../../json-store.ts"; import { closeStateDb, listKvValues, setKvValue } from "../../state/index.ts"; @@ -1380,4 +1380,23 @@ describe("reapRepoTrash", () => { await waitFor(() => !existsSync(leftover) && !existsSync(expired)); expect(existsSync(fresh)).toBe(true); }); + + // S079: sanitizeRoot (lib/worktree/config.ts) has no ancestor check, so a + // repo configured with e.g. `root: "${repoRoot}/.."` makes the crash sweep + // walk the parent directory of every sibling repo for `.trash-*` names. + test("refuses a configured root outside the repo and warns instead of sweeping it", async () => { + const parent = dirname(repo); + const siblingLeftover = join(parent, ".trash-should-survive-123"); + mkdirSync(siblingLeftover, { recursive: true }); + await declareWorktrees(repo, "acme", { root: parent }); + + const warns: unknown[][] = []; + const log = { info: () => {}, warn: (...a: unknown[]) => warns.push(a), error: () => {}, debug: () => {} } as unknown as Logger; + + await __test__.reapRepoTrash({ repoName: "acme", repoPath: repo, log }); + await new Promise((r) => setTimeout(r, 300)); // give a wrongly-spawned detached rm time to run + + expect(existsSync(siblingLeftover)).toBe(true); + expect(warns.some((w) => JSON.stringify(w).includes(parent))).toBe(true); + }); }); diff --git a/lib/daemon/worktree-reconciler.ts b/lib/daemon/worktree-reconciler.ts index d427c942..f2e20065 100644 --- a/lib/daemon/worktree-reconciler.ts +++ b/lib/daemon/worktree-reconciler.ts @@ -8,7 +8,7 @@ * `creationInFlight`). */ -import { basename, join } from "path"; +import { basename, isAbsolute, join, relative, resolve } from "path"; import { realpathSync } from "fs"; import type { Logger } from "pino"; import { rtDir } from "../rt-paths.ts"; @@ -1002,10 +1002,30 @@ async function replenishAndShrink( * parks trees stripped-but-recoverable (RT-51) — are reaped only past the * retention window. */ +/** + * Whether `root` is repoPath itself or a strict ancestor of it — + * sanitizeRoot (lib/worktree/config.ts) has no such check, so a value like + * `${repoRoot}/..` sweeps the parent directory shared by every sibling repo + * for `.trash-*` names. An unrelated, dedicated external root (the + * documented `root: "~/wt"` case) is fine to sweep — it's a repo-specific + * destination nothing else shares — so this only refuses the ancestor + * shape, not "root lies outside repoPath" in general. + */ +function isRootAnAncestorOfRepo(repoPath: string, root: string): boolean { + const rel = relative(resolve(root), resolve(repoPath)); + return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel)); +} + async function reapRepoTrash(deps: { repoName: string; repoPath: string; log: Logger }): Promise { const { repoName, repoPath, log } = deps; const cfg = await loadWorktreeRepoConfig(repoName, repoPath); - const reaped = await reapTrashInRoots([join(repoPath, ".worktrees"), cfg.root], log); + const roots = [join(repoPath, ".worktrees")]; + if (isRootAnAncestorOfRepo(repoPath, cfg.root)) { + log.warn({ repo: repoName, root: cfg.root, repoPath }, "worktree trash sweep refused a configured root that is an ancestor of the repo"); + } else { + roots.push(cfg.root); + } + const reaped = await reapTrashInRoots(roots, log); if (reaped > 0) log.info({ repo: repoName, count: reaped }, "worktree trash reaped"); const expired = await reapExpiredTrash(repoPath, log); if (expired > 0) log.info({ repo: repoName, count: expired }, "worktree retention trash reaped"); diff --git a/lib/worktree/__tests__/trash.test.ts b/lib/worktree/__tests__/trash.test.ts index 73c77411..f732e550 100644 --- a/lib/worktree/__tests__/trash.test.ts +++ b/lib/worktree/__tests__/trash.test.ts @@ -267,6 +267,19 @@ describe("worktree trash", () => { expect(existsSync(stray)).toBe(true); expect(warns.length).toBe(1); }); + + // S079: a trailing small integer (a manual "backup-3", "notes-42" dropped + // "with the other trash") parses as a number just fine and, taken at face + // value as a ms-epoch, is always ancient — rm -rf'd on the very next pass + // despite the doc comment promising non-rt entries are kept. This is a + // distinct failure shape from "not-rt-made" above (no digits at all). + test("an entry whose trailing digits are not a plausible rt epoch is kept and warned about", async () => { + const stray = makeTree(retainedTrashRoot(repo), "backup-3"); + const { log, warns } = capturingLog(); + expect(await reapExpiredTrash(repo, log, Date.now())).toBe(0); + expect(existsSync(stray)).toBe(true); + expect(warns.length).toBe(1); + }); }); describe("reapTrashDir on retained entries", () => { diff --git a/lib/worktree/trash.ts b/lib/worktree/trash.ts index 80292078..70921734 100644 --- a/lib/worktree/trash.ts +++ b/lib/worktree/trash.ts @@ -39,6 +39,21 @@ const RETAIN_DIR = ".trash"; /** How long a retained tree survives before the reconciler reaps it. */ export const RETENTION_MS = 14 * 24 * 60 * 60 * 1000; +/** + * A ms-epoch rt actually wrote (`${name}-${Date.now()}`) is always after + * this. A trailing small integer — a manual "backup-3" or "notes-42" + * dropped "with the other trash" — parses as a number just fine and, taken + * at face value as an epoch, is always ancient; without this floor it gets + * reaped on the very next pass despite not being rt's to delete. + */ +const EPOCH_FLOOR_MS = Date.UTC(2020, 0, 1); + +/** Whether `raw` looks like an epoch rt itself would have written, not merely any integer. */ +function looksLikeRtEpoch(raw: string): boolean { + const n = Number(raw); + return Number.isInteger(n) && n >= EPOCH_FLOOR_MS; +} + /** * Top-level dirs inside a retained tree that are reinstallable and deleted at * dispose time (exact names, plus the `dist-*` family). Mispredicting here @@ -202,7 +217,7 @@ export async function reapExpiredTrash( let reaped = 0; for (const entry of entries) { const epoch = /-(\d+)$/.exec(entry)?.[1]; - if (!epoch) { + if (!epoch || !looksLikeRtEpoch(epoch)) { log.warn({ root, entry }, "worktree retention sweep skipped an entry it did not write"); continue; } From 32fd1973f9d3f49e6c8594a5d407987ac6d14983 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:56:47 -0500 Subject: [PATCH 019/106] daemon: one shared api-token cache for api-server and secrets handler (S054) api-server captured the token once at boot while the secrets handler called loadOrCreateApiToken() fresh on every request; an external rotation or an unwritable token dir left the two permanently disagreeing about the current token. getApiToken/reloadApiToken share one in-memory cache between both consumers, and a persist failure now logs a warning instead of failing silently. --- lib/daemon/__tests__/api-auth.test.ts | 48 ++++++++++++++++++++++++++- lib/daemon/api-auth.ts | 32 +++++++++++++++++- lib/daemon/api-server.ts | 4 +-- lib/daemon/handlers/secrets.ts | 6 ++-- 4 files changed, 83 insertions(+), 7 deletions(-) diff --git a/lib/daemon/__tests__/api-auth.test.ts b/lib/daemon/__tests__/api-auth.test.ts index d9cda0b1..ad97eef9 100644 --- a/lib/daemon/__tests__/api-auth.test.ts +++ b/lib/daemon/__tests__/api-auth.test.ts @@ -4,7 +4,10 @@ */ import { describe, test, expect } from "bun:test"; -import { needsToken, tokenOk } from "../api-auth.ts"; +import { needsToken, tokenOk, getApiToken, reloadApiToken, loadOrCreateApiToken } from "../api-auth.ts"; +import { mkdtempSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; describe("needsToken", () => { test("shutdown requires a token", () => { @@ -58,3 +61,46 @@ describe("tokenOk", () => { expect(tokenOk("anything", "")).toBe(false); }); }); + +describe("getApiToken / reloadApiToken singleton", () => { + test("getApiToken caches: a second call does not re-read the file even if it changes underneath", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + const first = reloadApiToken(tokenPath); // seed the cache with a known path + writeFileSync(tokenPath, "a-different-token", { mode: 0o600 }); + const second = getApiToken(tokenPath); // ignores the new file content -- cached + expect(second).toBe(first); + expect(second).not.toBe("a-different-token"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("reloadApiToken re-reads and updates the cache", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + reloadApiToken(tokenPath); + writeFileSync(tokenPath, "rotated-token", { mode: 0o600 }); + const reloaded = reloadApiToken(tokenPath); + expect(reloaded).toBe("rotated-token"); + expect(getApiToken(tokenPath)).toBe("rotated-token"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("loadOrCreateApiToken still works standalone (unchanged primitive)", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + const a = loadOrCreateApiToken(tokenPath); + const b = loadOrCreateApiToken(tokenPath); + expect(a).toBe(b); + expect(a.length).toBeGreaterThan(0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/lib/daemon/api-auth.ts b/lib/daemon/api-auth.ts index e41b923b..89d07980 100644 --- a/lib/daemon/api-auth.ts +++ b/lib/daemon/api-auth.ts @@ -12,6 +12,9 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; import { join } from "path"; import { randomUUID } from "crypto"; import { RT_DIR } from "../daemon-config.ts"; +import { lazyChildLogger } from "../daemon-logger.ts"; + +const log = lazyChildLogger("api-auth"); /** Where the local API token is persisted (0600) for trusted local clients. */ export const API_TOKEN_PATH = join(RT_DIR, "api-token"); @@ -32,10 +35,37 @@ export function loadOrCreateApiToken(tokenPath: string = API_TOKEN_PATH): string try { mkdirSync(RT_DIR, { recursive: true }); writeFileSync(tokenPath, token, { mode: 0o600 }); - } catch { /* best-effort; token still enforced in-memory this run */ } + } catch (err) { + log.warn({ err, tokenPath }, "could not persist api-token; enforced in-memory only this run, so a client reading the file will disagree until the daemon restarts"); + } return token; } +/** + * `getApiToken`/`reloadApiToken` share ONE in-memory value between + * api-server.ts and the secrets handler (S054): before this, api-server + * captured a token once at boot while the secrets handler called + * `loadOrCreateApiToken()` fresh on every request, so an external rotation + * (deleting api-token to force a new one) left the two permanently + * disagreeing about which token is current -- and if the token dir was + * unwritable, the secrets handler regenerated a brand new random token on + * every single call, never matching anything a client could read from disk. + * Both consumers now read the SAME cached value; a rotation only takes + * effect for both after `reloadApiToken()` runs or the daemon restarts, + * either of which was already the closest thing to a happy path before. + */ +let cachedApiToken: string | null = null; + +export function getApiToken(tokenPath: string = API_TOKEN_PATH): string { + if (cachedApiToken === null) cachedApiToken = loadOrCreateApiToken(tokenPath); + return cachedApiToken; +} + +export function reloadApiToken(tokenPath: string = API_TOKEN_PATH): string { + cachedApiToken = loadOrCreateApiToken(tokenPath); + return cachedApiToken; +} + /** True when a request mutates state, or (secrets) returns raw credential values, and must present the local token. */ export function needsToken(method: string, pathname: string): boolean { if (method === "OPTIONS") return false; diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index 39558743..d61909fd 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -10,7 +10,7 @@ import type { Server, ServerWebSocket } from "bun"; import type { Logger } from "pino"; import { API_PORT } from "../daemon-config.ts"; -import { needsToken, tokenOk, loadOrCreateApiToken } from "./api-auth.ts"; +import { needsToken, tokenOk, getApiToken } from "./api-auth.ts"; import { getAggregatedConnection } from "./freshness.ts"; import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; @@ -131,7 +131,7 @@ export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDe export async function startApiServer(deps: ApiServerDeps): Promise> { const { handleCommand, log } = deps; - const apiToken = loadOrCreateApiToken(); + const apiToken = getApiToken(); const server = await bindApiServerWithRetry(() => Bun.serve({ port: API_PORT, diff --git a/lib/daemon/handlers/secrets.ts b/lib/daemon/handlers/secrets.ts index a0e33021..2608e603 100644 --- a/lib/daemon/handlers/secrets.ts +++ b/lib/daemon/handlers/secrets.ts @@ -46,7 +46,7 @@ import { loadSecrets } from "../../linear.ts"; import { parseIdentity } from "../../settings/identity.ts"; import { loadMachineRepoTracking, grants, type RepoTracking } from "../../repo-tracking.ts"; -import { loadOrCreateApiToken, tokenOk } from "../api-auth.ts"; +import { getApiToken, tokenOk } from "../api-auth.ts"; import { readSecret, createRealSecretsExecSeam, type SecretsSeams } from "../../secrets/store.ts"; import { createRealAgeKeySeam } from "../../home/age-key.ts"; import type { Commands, ForgeSlug } from "../../../packages/rt-client/src/commands.ts"; @@ -125,7 +125,7 @@ export interface SecretsHandlerOverrides { deckSecrets?: () => Promise<{ cfApiToken?: string; cfZoneId?: string }>; /** Defaults to `loadBoardSecrets` (cross-domain: `board` + `rt`) for secrets:read's "board" scope. */ boardSecrets?: () => Promise; - /** Defaults to `loadOrCreateApiToken` (the real ~/.mattstack/rt/api-token, shared with api-auth.ts). */ + /** Defaults to `getApiToken` (the real ~/.mattstack/rt/api-token, shared with api-auth.ts and api-server.ts). */ apiToken?: () => string; } @@ -142,7 +142,7 @@ export function createSecretsHandlers( const extensionSecrets = overrides.extensionSecrets ?? loadSecrets; const deckSecrets = overrides.deckSecrets ?? loadDeckSecrets; const boardSecrets = overrides.boardSecrets ?? loadBoardSecrets; - const apiToken = overrides.apiToken ?? (() => loadOrCreateApiToken()); + const apiToken = overrides.apiToken ?? (() => getApiToken()); return { "secrets:forge-token": async (payload: Commands["secrets:forge-token"]["payload"]) => { From 421d67b6ec4ae1338fa4bf2a62a039db6a637315 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:57:46 -0500 Subject: [PATCH 020/106] daemon: boot failure is fatal (exit 1), gated by boot-phase flag; rt.pid after binds Boot failures on the prod path used to leave a live-pid zombie: a stale rt.pid could get written before the socket/API binds even attempted, so a failed boot exited without ever removing it. runDaemon() now wraps its body in try/catch (log.fatal + flush + exit 1 on any failure), and rt.pid is only written once both servers.socket and servers.api are assigned. installCrashHandlers gains an opts.booting predicate: unhandledRejection is fatal + exit 1 while true (boot phase, nothing worth staying up for), and logs only (today's behavior) once bootPhase flips to "ready" right before "daemon ready". --- e2e/tests/daemon.test.ts | 18 ++ lib/__tests__/daemon-logger.test.ts | 42 ++++- lib/daemon-logger.ts | 16 +- lib/daemon.ts | 263 +++++++++++++++------------- 4 files changed, 212 insertions(+), 127 deletions(-) diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index 48345d0f..37982958 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -3,6 +3,24 @@ import { existsSync } from "fs"; import { join } from "path"; import { createTestHome, rt } from "../harness.ts"; +describe("fatal boot", () => { + test("daemon boot with API port already bound exits non-zero and leaves no stale rt.pid", async () => { + const { path: home, cleanup } = createTestHome(); + // Bind the API port inside the isolated HOME so the daemon cannot. + const port = 9411; + const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + try { + const result = await rt(["--daemon"], { home, env: { RT_API_PORT: String(port) } }); + + expect(result.exitCode).not.toBe(0); + expect(existsSync(join(home, ".mattstack", "rt", "rt.pid"))).toBe(false); + } finally { + squatter.stop(true); + cleanup(); + } + }, 60_000); +}); + describe("daemon", () => { describe("install creates config", () => { let home: string; diff --git a/lib/__tests__/daemon-logger.test.ts b/lib/__tests__/daemon-logger.test.ts index 1370be57..69bd3a9e 100644 --- a/lib/__tests__/daemon-logger.test.ts +++ b/lib/__tests__/daemon-logger.test.ts @@ -1,10 +1,18 @@ -import { describe, it, expect, beforeEach, afterEach } from "bun:test"; +import { describe, it, expect, beforeEach, afterEach, mock } from "bun:test"; import { mkdtempSync, rmSync, readFileSync, readdirSync, existsSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import type { Logger } from "pino"; // We import the factory (not the singleton) so each test gets isolation. -import { createDaemonLogger, lazyChildLogger, getDaemonLogger, __test__ } from "../daemon-logger.ts"; +import { + createDaemonLogger, + lazyChildLogger, + getDaemonLogger, + installCrashHandlers, + __test__, + type DaemonLoggerHandle, +} from "../daemon-logger.ts"; import { logsDir } from "../rt-paths.ts"; let logDir: string; @@ -191,3 +199,33 @@ describe("lazyChildLogger — Proxy guard", () => { expect(() => (log as any).level).toThrow(); }); }); + +describe("installCrashHandlers — boot-phase gating", () => { + it("unhandledRejection exits(1) while booting, only logs once ready", () => { + const exits: number[] = []; + const origExit = process.exit; + const origStderrWrite = process.stderr.write.bind(process.stderr); + // @ts-expect-error test stub — captures the exit code instead of terminating + process.exit = (code?: number) => { exits.push(code ?? 0); }; + const fatal = mock(() => {}); + const error = mock(() => {}); + const logger = { fatal, error } as unknown as Logger; + let booting = true; + try { + installCrashHandlers({ logger } as DaemonLoggerHandle, { booting: () => booting }); + process.emit("unhandledRejection", new Error("boot boom"), Promise.resolve()); + expect(fatal).toHaveBeenCalledTimes(1); + expect(exits).toEqual([1]); + + booting = false; + process.emit("unhandledRejection", new Error("steady boom"), Promise.resolve()); + expect(error).toHaveBeenCalledTimes(1); + expect(exits).toEqual([1]); // no second exit + } finally { + process.exit = origExit; + process.stderr.write = origStderrWrite; + process.removeAllListeners("unhandledRejection"); + process.removeAllListeners("uncaughtException"); + } + }); +}); diff --git a/lib/daemon-logger.ts b/lib/daemon-logger.ts index ecca0df9..bd4826e1 100644 --- a/lib/daemon-logger.ts +++ b/lib/daemon-logger.ts @@ -225,11 +225,18 @@ export function redirectNativeStderr(): void { * logger before exit. Call ONCE during daemon startup, AFTER logger init. * * - uncaughtException: log as fatal (sync), exit 1 - * - unhandledRejection: log as error, do NOT exit (let the daemon recover) + * - unhandledRejection: fatal + exit 1 while `opts.booting()` is true (no + * socket/API bound yet, nothing worth staying up for); log as error and + * stay alive once booted, so a stray steady-state rejection doesn't kill a + * daemon that's already serving. No `booting` given preserves the old + * always-log, never-exit behavior. * - process.stderr.write: intercept so console.error / anything writing to * stderr lands in the JSON log instead of vanishing. */ -export function installCrashHandlers(handle: DaemonLoggerHandle): void { +export function installCrashHandlers( + handle: DaemonLoggerHandle, + opts: { booting?: () => boolean } = {}, +): void { const { logger } = handle; // Because the pino-roll stream is opened with sync:true, logger.fatal() @@ -240,6 +247,11 @@ export function installCrashHandlers(handle: DaemonLoggerHandle): void { }); process.on("unhandledRejection", (reason) => { + if (opts.booting?.()) { + logger.fatal({ err: reason }, "unhandledRejection during boot"); + process.exit(1); + return; + } logger.error({ err: reason }, "unhandledRejection"); }); diff --git a/lib/daemon.ts b/lib/daemon.ts index fc2ba5ed..fcfe6ca5 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -75,6 +75,11 @@ import type { PortEntry } from "./port-scanner.ts"; // a clean rename of a real legacy tree into a conflict. Idempotent — the CLI // entry (cli.ts) also runs it, but `bun run lib/daemon.ts` skips cli.ts. import { migrateLegacyRtDir, LEGACY_RT_LABEL, RT_DIR_LABEL, logsDir } from "./rt-paths.ts"; + +// Gates installCrashHandlers' unhandledRejection handler: fatal during boot +// (no socket/API bound yet, nothing to recover), advisory-only once ready. +let bootPhase: "booting" | "ready" = "booting"; + const rtMigration = migrateLegacyRtDir(); // ─── Logging ───────────────────────────────────────────────────────────────── @@ -383,134 +388,146 @@ const cleanup = (): void => { // ─── Entry ─────────────────────────────────────────────────────────────────── async function runDaemon(): Promise { - mkdirSync(RT_DIR, { recursive: true }); - - // Capture native panics (bypass JS entirely) at the fd level, then wire - // uncaughtException + unhandledRejection through pino. Must run BEFORE - // any async work that could throw uncaught. - redirectNativeStderr(); - installCrashHandlers(loggerHandle); - - // If a previous daemon process is still alive (orphan from a failed - // restart), evict it before we bind the socket. - evictStaleDaemon(log); - - // Auto-unlink any tagged tool link whose tool now has a genuine user copy - // elsewhere on PATH (e.g. the user ran `brew install gh` after rt linked - // the bundled one). reconcile() itself is synchronous (a ~/.local/bin - // readDir plus a handful of stats) — wrapping the call in `async` alone - // would NOT defer it, since nothing inside actually awaits. setTimeout(0) - // is what actually pushes it past the rest of this function: the PID - // write, openBranchCacheStore, and both server binds below all run first, - // on this same synchronous pass, before the timer callback ever fires. - setTimeout(() => { - try { - const { removed } = reconcileLinks(createRealProbes()); - if (removed.length > 0) log.info({ removed }, "deps: auto-unlinked tools now shadowed by a user copy"); - } catch (err) { - log.warn({ err }, "deps: link reconcile failed"); - } - }, 0); - - log.info("daemon starting"); - writeFileSync(DAEMON_PID_PATH, String(process.pid)); - - // Open state.db and build the in-memory branch-cache map BEFORE serving - // (spec "Migration & contention"): the one long transaction is the - // legacy-JSON import, and it must never land inside the event loop. If a - // CLI process is mid-import right now, we block here, in startup. - openBranchCacheStore(); - log.info({ count: Object.keys(cache.entries).length }, "branch cache loaded from state.db"); - - // one-shot re-key of every legacy NAME-keyed store row onto its - // serialized repo identity. Fire-and-forget (not awaited) like the PATH - // reconcile above — the ordering guarantee this depends on (running before - // anything prunes the repo index) only needs this to be on the boot path, - // not blocking the socket bind; a prune only ever arrives as a command sent - // to an already-running daemon. - runBootIdentityMigration(log).catch((err) => { - log.warn({ err }, "boot identity migration failed"); - }); - - routedHandlers = buildRoutedHandlers({ - ctx: handlerCtx, - broadcast: emit, - systemProcessScanner, - worktree: { - emit, - kick: worktreeReconciler.kick, - creationInFlight: worktreeReconciler.creationInFlight, - }, - eventsBus, - homeSnapshot, - repos: { - withReconcilerHeld: worktreeReconciler.withReconcilerHeld, - refreshWatchedRepos: hooksGuard.refreshWatchedRepos, - }, - stateDb: getStateDb("daemon"), - }); - - // No waiter outlives the daemon, so every armed_at set at boot is stale; - // clearing must finish before the socket listens, or an agent that arms - // in the gap has its fresh armed_at wiped. - const clearedArmed = clearAllArmed(); - if (clearedArmed > 0) log.info({ clearedArmed }, "chat: cleared stale armed_at from previous daemon run"); - - // Daemon startup is one of the two moments a handle is about to be - // needed (spec "Pruning"); sign-in is the other, inside signIn itself. - // Pruning is best-effort cleanup, so a concurrent CLI writer's - // SQLITE_BUSY must not abort startup before the socket binds. - let prunedPresence = 0; - persistOrWarn("daemon", () => { prunedPresence = prunePresence(Date.now()); }, { op: "prunePresence" }); - if (prunedPresence > 0) log.info({ prunedPresence }, "chat: pruned stale presence rows at daemon startup"); - - // Socket server (Unix socket for CLI/tray) + REST/WS server (external clients) - servers.socket = startSocketServer({ handleCommand, log }); - servers.api = await startApiServer({ handleCommand, log }); - - // Wire notification broadcasts to WebSocket clients - onNotification(emit); - - // Discover and watch repos - hooksGuard.refreshWatchedRepos(); - - // Team tracking intent (mattstack.tracking) resolves through a primed - // identity→name map, not live derivation — loadRepoTracking is sync and - // runs on every freshness tick. Team intent is inert until this completes. - // The repo index moved into state.db (RT-50): there is no file to fs.watch - // for new-repo changes any more, so the 60s hooks-scan poller (pollers.ts) - // is the only re-prime mechanism now, not just the reliable one. - primeTeamTrackingIdentityMap(loadRepoIndex()).catch((err) => { - log.warn({ err }, "repo-tracking: failed to prime team-intent identity map"); - }); - - // Periodic background work: cache refresh, port scan, system-process scan, - // hooks-guard fallback rescan. - startPollers({ - log, refreshCache, portCacheRef, broadcast: emit, systemProcessScanner, - repoIndex: loadRepoIndex, - checkAndRepairHooksPath: hooksGuard.checkAndRepairHooksPath, - }); - - // Kick off the events watchers once the first refresh has populated the - // cache with repoName stamps. reconcileFreshness inside the cache refresher - // follows repo-index changes from there. - setTimeout(() => { - initFreshness(freshnessEnv).catch((err) => { - log.error({ err }, "freshness: init failed"); + try { + mkdirSync(RT_DIR, { recursive: true }); + + // Capture native panics (bypass JS entirely) at the fd level, then wire + // uncaughtException + unhandledRejection through pino. Must run BEFORE + // any async work that could throw uncaught. + redirectNativeStderr(); + installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" }); + + // If a previous daemon process is still alive (orphan from a failed + // restart), evict it before we bind the socket. + evictStaleDaemon(log); + + // Auto-unlink any tagged tool link whose tool now has a genuine user copy + // elsewhere on PATH (e.g. the user ran `brew install gh` after rt linked + // the bundled one). reconcile() itself is synchronous (a ~/.local/bin + // readDir plus a handful of stats) — wrapping the call in `async` alone + // would NOT defer it, since nothing inside actually awaits. setTimeout(0) + // is what actually pushes it past the rest of this function: the PID + // write, openBranchCacheStore, and both server binds below all run first, + // on this same synchronous pass, before the timer callback ever fires. + setTimeout(() => { + try { + const { removed } = reconcileLinks(createRealProbes()); + if (removed.length > 0) log.info({ removed }, "deps: auto-unlinked tools now shadowed by a user copy"); + } catch (err) { + log.warn({ err }, "deps: link reconcile failed"); + } + }, 0); + + log.info("daemon starting"); + + // Open state.db and build the in-memory branch-cache map BEFORE serving + // (spec "Migration & contention"): the one long transaction is the + // legacy-JSON import, and it must never land inside the event loop. If a + // CLI process is mid-import right now, we block here, in startup. + openBranchCacheStore(); + log.info({ count: Object.keys(cache.entries).length }, "branch cache loaded from state.db"); + + // one-shot re-key of every legacy NAME-keyed store row onto its + // serialized repo identity. Fire-and-forget (not awaited) like the PATH + // reconcile above — the ordering guarantee this depends on (running before + // anything prunes the repo index) only needs this to be on the boot path, + // not blocking the socket bind; a prune only ever arrives as a command sent + // to an already-running daemon. + runBootIdentityMigration(log).catch((err) => { + log.warn({ err }, "boot identity migration failed"); + }); + + routedHandlers = buildRoutedHandlers({ + ctx: handlerCtx, + broadcast: emit, + systemProcessScanner, + worktree: { + emit, + kick: worktreeReconciler.kick, + creationInFlight: worktreeReconciler.creationInFlight, + }, + eventsBus, + homeSnapshot, + repos: { + withReconcilerHeld: worktreeReconciler.withReconcilerHeld, + refreshWatchedRepos: hooksGuard.refreshWatchedRepos, + }, + stateDb: getStateDb("daemon"), + }); + + // No waiter outlives the daemon, so every armed_at set at boot is stale; + // clearing must finish before the socket listens, or an agent that arms + // in the gap has its fresh armed_at wiped. + const clearedArmed = clearAllArmed(); + if (clearedArmed > 0) log.info({ clearedArmed }, "chat: cleared stale armed_at from previous daemon run"); + + // Daemon startup is one of the two moments a handle is about to be + // needed (spec "Pruning"); sign-in is the other, inside signIn itself. + // Pruning is best-effort cleanup, so a concurrent CLI writer's + // SQLITE_BUSY must not abort startup before the socket binds. + let prunedPresence = 0; + persistOrWarn("daemon", () => { prunedPresence = prunePresence(Date.now()); }, { op: "prunePresence" }); + if (prunedPresence > 0) log.info({ prunedPresence }, "chat: pruned stale presence rows at daemon startup"); + + // Socket server (Unix socket for CLI/tray) + REST/WS server (external clients) + servers.socket = startSocketServer({ handleCommand, log }); + servers.api = await startApiServer({ handleCommand, log }); + + // Only write rt.pid once both servers are actually bound — a boot that + // fails before this point must never leave a live-pid file with no + // socket/API behind it. + writeFileSync(DAEMON_PID_PATH, String(process.pid)); + + // Wire notification broadcasts to WebSocket clients + onNotification(emit); + + // Discover and watch repos + hooksGuard.refreshWatchedRepos(); + + // Team tracking intent (mattstack.tracking) resolves through a primed + // identity→name map, not live derivation — loadRepoTracking is sync and + // runs on every freshness tick. Team intent is inert until this completes. + // The repo index moved into state.db (RT-50): there is no file to fs.watch + // for new-repo changes any more, so the 60s hooks-scan poller (pollers.ts) + // is the only re-prime mechanism now, not just the reliable one. + primeTeamTrackingIdentityMap(loadRepoIndex()).catch((err) => { + log.warn({ err }, "repo-tracking: failed to prime team-intent identity map"); + }); + + // Periodic background work: cache refresh, port scan, system-process scan, + // hooks-guard fallback rescan. + startPollers({ + log, refreshCache, portCacheRef, broadcast: emit, systemProcessScanner, + repoIndex: loadRepoIndex, + checkAndRepairHooksPath: hooksGuard.checkAndRepairHooksPath, }); - }, 7000); - // Background sweep for new MR comments → `discussions:new-comments` events. - startDiscussionsPoller({ ctx: handlerCtx, broadcast: emit }); + // Kick off the events watchers once the first refresh has populated the + // cache with repoName stamps. reconcileFreshness inside the cache refresher + // follows repo-index changes from there. + setTimeout(() => { + initFreshness(freshnessEnv).catch((err) => { + log.error({ err }, "freshness: init failed"); + }); + }, 7000); + + // Background sweep for new MR comments → `discussions:new-comments` events. + startDiscussionsPoller({ ctx: handlerCtx, broadcast: emit }); - // Sandbox ground-truth reconcile: port-forwards, dev-ports mirroring, and - // typed-event → notification fan-out (no-op while no controller answers). + // Sandbox ground-truth reconcile: port-forwards, dev-ports mirroring, and + // typed-event → notification fan-out (no-op while no controller answers). - // Graceful shutdown on all termination signals - installSignalHandlers({ cleanup, flushLogs: () => loggerHandle.flush?.(), log }); + // Graceful shutdown on all termination signals + installSignalHandlers({ cleanup, flushLogs: () => loggerHandle.flush?.(), log }); - log.info({ pid: process.pid }, "daemon ready"); + bootPhase = "ready"; + log.info({ pid: process.pid }, "daemon ready"); + } catch (err) { + log.fatal({ err }, "daemon boot failed"); + // Task 9 adds recordBootFailure(currentPhase, err) here. + try { loggerHandle.flush?.(); } catch { /* */ } + process.exit(1); + } } /** From 8be75f0d093d019105ac649384f1a3c3967cfb5e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:58:45 -0500 Subject: [PATCH 021/106] pane:spawn: check the caller's abort signal between steps and stop early instead of racing a retry into a second pane (S087) --- lib/daemon/__tests__/pane-handlers.test.ts | 22 ++++++++++++++++++ lib/daemon/handlers/pane.ts | 27 +++++++++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/lib/daemon/__tests__/pane-handlers.test.ts b/lib/daemon/__tests__/pane-handlers.test.ts index 6f09a1ee..2da4bbbf 100644 --- a/lib/daemon/__tests__/pane-handlers.test.ts +++ b/lib/daemon/__tests__/pane-handlers.test.ts @@ -279,6 +279,28 @@ test("pane:spawn stops polling for registration at the wall-clock budget, not a expect(calls.filter((c) => c === "agent.get").length).toBeLessThanOrEqual(2); }); +// S087: pane:spawn's worst-case daemon budget (register + idle wait + trust +// retry + prompt) exceeds rt-client's client-side timeout — the client +// gives up and reports failure while the daemon keeps working, so a retry +// spawns a second claude pane in the same cwd. Once the caller's own +// AbortSignal fires, the handler must stop spending further budget and +// return the pane it already created instead of continuing the full flow. +test("pane:spawn stops after tab creation once the caller's AbortSignal fires, returning the pane already created", async () => { + const controller = new AbortController(); + const { handler, calls } = spawnFake({ statuses: ["idle"] }); + const abortingHandler: typeof handler = (method, params) => { + if (method === "pane.send_input") controller.abort(); + return handler(method, params); + }; + const { pane } = harness(abortingHandler); + const res = await pane["pane:spawn"]({ cwd: "/repos/chat" }, controller.signal); + if (!res.ok) throw new Error(res.error); + expect(res.data.pane.paneId).toBe("w2:p7"); + expect(res.data.ready).toBe(false); + expect(calls).not.toContain("agent.get"); + expect(calls).not.toContain("agent.wait"); +}); + test("pane:spawn refuses an unknown cswap account before touching herdr", async () => { const { handler, calls } = spawnFake({ statuses: ["idle"] }); const { pane } = harness(handler); diff --git a/lib/daemon/handlers/pane.ts b/lib/daemon/handlers/pane.ts index 3eb2db2f..317ac067 100644 --- a/lib/daemon/handlers/pane.ts +++ b/lib/daemon/handlers/pane.ts @@ -124,7 +124,9 @@ export function createPaneHandlers(opts: { exec?: typeof runCapture; now?: () => number; registry?: (repoName: string) => Array<{ path: string; branch: string | null | undefined }>; -}): Pick & { db: Database } { +}): Pick + & { "pane:spawn": (payload: Commands["pane:spawn"]["payload"], signal?: AbortSignal) => Promise> } + & { db: Database } { const { db, repoIndex } = opts; const herdr = opts.herdr ?? herdrRequest; const exec = opts.exec ?? runCapture; @@ -190,7 +192,7 @@ export function createPaneHandlers(opts: { return { ok: true, data: { directories: out } }; }, - "pane:spawn": async (payload: Commands["pane:spawn"]["payload"]): Promise> => { + "pane:spawn": async (payload: Commands["pane:spawn"]["payload"], signal?: AbortSignal): Promise> => { const { cwd, account, model, effort, prompt } = payload; if (!cwd || !cwd.startsWith("/")) return { ok: false, error: "cwd must be an absolute path" }; if (account) { @@ -213,8 +215,23 @@ export function createPaneHandlers(opts: { if (!tab.ok) return herdrError(tab); const paneId = tab.result.root_pane.pane_id; + // pane:spawn's summed worst-case budget (register + idle wait + a + // blocked-trust retry + the opening prompt) can run longer than + // rt-client's own client-side timeout for this call. Once the caller + // has given up, continuing to spend the daemon's budget only risks a + // retry racing a second claude pane into the same cwd — so every step + // past tab creation checks the signal first and returns the pane + // already created (not-ready) rather than pressing on for a client + // that is no longer listening. + const earlyReturn = async (status: AgentStatus): Promise> => { + const ctx: PaneRowContext = { db, repoIndex, exec, now, workspaces: new Map([[workspaceId!, label]]), ...presenceMaps(db, now()) }; + const pane = await paneRow({ ...tab.result.root_pane, agent: "claude", agent_status: status }, ctx); + return { ok: true, data: { pane, ready: false } }; + }; + const sent = await herdr("pane.send_input", { pane_id: paneId, text: launchCommand({ cwd, account, model, effort }), keys: ["enter"] }); if (!sent.ok) return herdrError(sent); + if (signal?.aborted) return earlyReturn("unknown"); // herdr registers the agent a few hundred ms after the shell starts claude. // Bound the wait by wall-clock, not a fixed attempt count: a slow-but-alive @@ -223,7 +240,7 @@ export function createPaneHandlers(opts: { // holds its caller. let registered = false; const registerDeadline = now() + REGISTER_BUDGET_MS; - while (now() < registerDeadline) { + while (now() < registerDeadline && !signal?.aborted) { const got = await herdr("agent.get", { target: paneId }); if (got.ok) { registered = true; @@ -231,22 +248,26 @@ export function createPaneHandlers(opts: { } await Bun.sleep(REGISTER_POLL_MS); } + if (signal?.aborted) return earlyReturn("unknown"); let status: AgentStatus = "unknown"; let ready = false; if (registered) { const settled = await herdr<{ agent: HerdrAgent }>("agent.wait", { target: paneId, until: SETTLED, timeout_ms: IDLE_BUDGET_MS }, { timeoutMs: waitTimeout(IDLE_BUDGET_MS) }); if (settled.ok) status = settled.result.agent.agent_status; + if (signal?.aborted) return earlyReturn(status); if (status === "blocked") { const screen = await herdr<{ read: { text: string } }>("pane.read", { pane_id: paneId, source: "visible" }); if (screen.ok && /trust/i.test(screen.result.read.text)) { await herdr("pane.send_keys", { pane_id: paneId, keys: ["enter"] }); + if (signal?.aborted) return earlyReturn(status); const again = await herdr<{ agent: HerdrAgent }>("agent.wait", { target: paneId, until: SETTLED, timeout_ms: TRUST_BUDGET_MS }, { timeoutMs: waitTimeout(TRUST_BUDGET_MS) }); if (again.ok) status = again.result.agent.agent_status; } } ready = status === "idle" || status === "done"; } + if (signal?.aborted) return earlyReturn(status); if (ready && prompt) { await herdr("agent.prompt", { target: paneId, text: prompt, wait: { until: ["working"], timeout_ms: PROMPT_BUDGET_MS } }, { timeoutMs: waitTimeout(PROMPT_BUDGET_MS) }); From eea6451095898c8b359c1741344ba18b5dd80074 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:00:07 -0500 Subject: [PATCH 022/106] mr:by-branch: apply the same demand-scope gate to the forge write-back that the sync path enforces (S088) --- lib/daemon/__tests__/project-sync.test.ts | 29 +++++++++++++++++++++++ lib/daemon/handlers/project-mrs.ts | 13 +++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/lib/daemon/__tests__/project-sync.test.ts b/lib/daemon/__tests__/project-sync.test.ts index 2437da6f..57612384 100644 --- a/lib/daemon/__tests__/project-sync.test.ts +++ b/lib/daemon/__tests__/project-sync.test.ts @@ -494,6 +494,35 @@ describe("project-mrs:read handler", async () => { expect(forgeCalls).toBe(1); }); + // S088: mr:by-branch's forge write-back upserted unconditionally, bypassing + // the same scope/tagged gate upsertProject (lib/daemon/freshness.ts) + // enforces for every other write path — a demand-scoped repo could pick up + // a stranger's MR that the next delta sync filters right back out, then + // reappears on the next by-branch call. The caller still gets the PR (it + // asked for this exact branch); it just must not land in the store. + test("by-branch: forge write-back respects the same demand scope the sync path enforces", async () => { + const store = tmpStore(); + store.fullSync("remote:repo", "g/p", [], Date.now()); + store.setScope("remote:repo", { authors: ["alice"], windowDays: 30 }); + const h = createProjectMRsHandlers(fakeCtx, () => {}, { store, sync: async () => {}, tracking: grantedTracking, + fetchByBranch: async (_r, branch) => ({ pr: pr(9, { sourceBranch: branch, author: { id: 2, username: "stranger" } as any }), projectPath: "g/p" }) }); + const res = await h["mr:by-branch"]!({ repoName: "remote:repo", branches: ["feat-x"] }); + expect(res.ok).toBe(true); + expect((dataOf(res) as any).byBranch["feat-x"]).toMatchObject({ source: "forge", pr: { iid: 9 } }); + expect(store.read("remote:repo")!.mrs[9]).toBeUndefined(); + }); + + test("by-branch: forge write-back still stores an in-scope author's MR", async () => { + const store = tmpStore(); + store.fullSync("remote:repo", "g/p", [], Date.now()); + store.setScope("remote:repo", { authors: ["alice"], windowDays: 30 }); + const h = createProjectMRsHandlers(fakeCtx, () => {}, { store, sync: async () => {}, tracking: grantedTracking, + fetchByBranch: async (_r, branch) => ({ pr: pr(10, { sourceBranch: branch, author: { id: 1, username: "alice" } as any }), projectPath: "g/p" }) }); + const res = await h["mr:by-branch"]!({ repoName: "remote:repo", branches: ["feat-y"] }); + expect(res.ok).toBe(true); + expect(store.read("remote:repo")!.mrs[10]).toBeDefined(); + }); + test("by-branch: no MR anywhere is null; a per-branch forge failure is null with a warn, not a batch failure", async () => { const store = tmpStore(); store.fullSync("remote:repo", "g/p", [pr(1, { sourceBranch: "ok" })], Date.now()); diff --git a/lib/daemon/handlers/project-mrs.ts b/lib/daemon/handlers/project-mrs.ts index c9441a78..0395ecdc 100644 --- a/lib/daemon/handlers/project-mrs.ts +++ b/lib/daemon/handlers/project-mrs.ts @@ -245,7 +245,18 @@ export function createProjectMRsHandlers( try { const { pr, projectPath } = await fetchByBranch(repoName, branch); if (pr) { - store().upsert(repoName, projectPath, pr, "events"); + // Same scope/tagged gate upsertProject (lib/daemon/freshness.ts) + // enforces for every other write path: a demand-scoped repo + // must not pick up a stranger's MR here just because a client + // happened to ask for their branch by name — the next delta + // sync would filter it right back out, and it would reappear + // on the next by-branch call. The caller still gets the PR + // (it asked for this exact branch); it just isn't stored. + const rec = store().read(repoName); + const scope = rec?.scope; + const tagged = (rec?.mrs[pr.iid]?.codeownerSections?.length ?? 0) > 0; + const inScope = !scope || !pr.author?.username || scope.authors.includes(pr.author.username) || tagged; + if (inScope) store().upsert(repoName, projectPath, pr, "events"); byBranch[branch] = { pr, source: "forge" }; } else { byBranch[branch] = null; From 3bd511ba8d42c3a8af7636b6a193a6a76c1b1149 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:01:50 -0500 Subject: [PATCH 023/106] plan: Phase 1 event-loop implementation plan (RT-78, items 1.1-1.5) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../plans/2026-08-28-p1-event-loop.md | 1490 +++++++++++++++++ 1 file changed, 1490 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-28-p1-event-loop.md diff --git a/docs/superpowers/plans/2026-08-28-p1-event-loop.md b/docs/superpowers/plans/2026-08-28-p1-event-loop.md new file mode 100644 index 00000000..c8d35527 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-p1-event-loop.md @@ -0,0 +1,1490 @@ +# Phase 1 · Event-Loop Sacred Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Excise every synchronous subprocess call from daemon-reachable code, make `runCapture`'s timeout actually enforceable, add an import-graph gate that keeps sync-exec dead, and stop the coalesced refresh and background scans from wedging or taxing the event loop. + +**Architecture:** Route all daemon-thread subprocess work through the existing async helpers (`runCapture` in `lib/subprocess.ts`, `runGit`/`listWorktreesAsync` in `lib/worktree/git-async.ts`). Harden `runCapture` so a pipe-holding grandchild can no longer hold its promise open. Add a source-graph test that walks the daemon's import tree and fails on any sync-exec import. Bound the refresh cycle and the provider cache, and gate/cheapen the periodic scans. + +**Tech Stack:** Bun 1.3.13, TypeScript (strict), `bun:test`, `bun:sqlite`, pino. `@mattstack/glance` provides `GitLabProvider`. + +**Spec:** `/Users/matt/Documents/GitHub/repo-tools/.claude/worktrees/daemon-stability-audit/docs/daemon-stability-audit-2026-08.md` ... "Roadmap > Phase 1" (items 1.1-1.5) plus Appendix A/B for each finding (S007, S008, S015, S016, S021, S023, S024, S038, S039, S045, S048, S049, S055, S058, S061, S093, S098, S101, S104, R032). + +## Global Constraints + +- **Isolated HOME for any binary run.** Never start a daemon or run `dist/rt` against the real machine. Any daemon or `dist/rt` invocation runs under `env -i HOME=` only. Tests use `bun:test` with tmp HOME/`RT_RUNS_ROOT` fixtures ... never touch the developer's `~/.mattstack` or `~/.rt`. +- **Write fence.** These sibling-owned files must NOT be modified this run: `cli.ts`, `lib/daemon.ts`, `lib/daemon/park.ts`, `lib/daemon/boot-reconcile.ts`, `lib/daemon/boot-migrate.ts`, `lib/daemon/shutdown.ts`, `lib/daemon-logger.ts`, `lib/daemon-config.ts`, `lib/daemon-status.ts`, `commands/daemon.ts`, `lib/daemon/events-bus.ts`, `lib/daemon/home-snapshot.ts`, `lib/daemon/handlers/status.ts`, `lib/state/*`, `rt-tray/**`, `lib/daemon/api-server.ts`, `lib/daemon/api-auth.ts`, `lib/daemon/socket-server.ts`, `lib/daemon/handlers/secrets.ts`, `lib/notifier.ts`, `lib/daemon/handlers/discussions.ts`, `lib/daemon/handlers/chat.ts`, `lib/daemon/handlers/agent.ts`, `lib/daemon/handlers/pane.ts`, `lib/daemon/handlers/project-mrs.ts`, `lib/daemon/handlers/worktree.ts`, `lib/herdr/client.ts`, `lib/port-scanner.ts`, `lib/deps/links.ts`, `lib/worktree/trash.ts`, `lib/agent-herdr.ts`, `lib/daemon/cron.ts`, `lib/daemon/hooks-guard.ts`, `lib/home/age-key.ts`, `lib/daemon/discussions-store.ts`. +- **S055 carve-out.** `lib/daemon/handlers/status.ts` is sibling-owned. Do NOT edit it. The async replacement it needs (`listWorktreesAsync`) already lives in a file we own (`lib/worktree/git-async.ts`). The one-line call-site swap is documented in the report Notes, not applied here. The import-graph gate (Task 7) allowlists `status.ts` with a comment naming S055. +- **rt-client mirror.** `packages/rt-client/src/settings/exec.ts` is a byte-for-byte mirror of `lib/subprocess.ts`'s `runCapture`. Change `lib/subprocess.ts` first, mirror there, then run `bun run build` inside `packages/rt-client` (its `dist/` is gitignored and copied verbatim by `file:` consumers; `packages/rt-client/test/dist-freshness.test.ts` fails otherwise). +- **Canonical fixtures.** `listWorktreesAsync` returns git's canonicalized paths (`/private/var/...` on macOS tmpdirs). Tests that build temp repos must compare against `realpathSync`'d paths. +- **Subagent models.** Every subagent dispatched during execution carries an explicit `model` (`sonnet` for mechanical tasks, `haiku` for lookups). +- **Verification (all must pass before done):** `bun test lib commands packages scripts` green; `bunx tsc --noEmit` zero errors; the Task 7 gate passes AND is proven to fail when a forbidden import is reintroduced (show the RED run in the report). +- **Comments:** clean-code rules. A comment states a constraint the code cannot show (a parity anchor, an ordering trap, a non-obvious invariant). No narration, no finding IDs in source. No em dashes. + +--- + +### Task 1: Make `runCapture`'s timeout enforceable (S023, S024) + +`runCapture` awaits `new Response(proc.stdout).text()` unconditionally after the kill timer fires; a grandchild that inherited the pipe keeps that read pending forever, so the promise never settles and every in-flight guard latches. Fix: race the reads against the deadline so `runCapture` always settles within `timeoutMs`, and escalate SIGTERM→SIGKILL on the child. (Group-kill via `process.kill(-pid)` is NOT reliable on Bun 1.3.13 ... verified ESRCH once the direct child exits ... so this uses the race as the load-critical fix, per the S023 fixer note that racing the reads is mandatory and kill escalation is belt-and-suspenders. A surviving grandchild leaks its fd until the OS reaps it, which the audit accepts.) + +**Files:** +- Modify: `lib/subprocess.ts` (the `runCapture` body and `RunResult`) +- Mirror: `packages/rt-client/src/settings/exec.ts` (byte-for-byte) +- Test: `lib/__tests__/subprocess.test.ts` (create if absent) + +**Interfaces:** +- Produces: `runCapture(argv, opts) => Promise` where `RunResult` gains `timedOut?: boolean`. Contract unchanged otherwise: never throws; on spawn failure/timeout/read error returns `exitCode: -1` (callers already branch on `!== 0`). `timedOut: true` is additive and set only on the deadline path. + +- [ ] **Step 1: Write the failing test** + +Create `lib/__tests__/subprocess.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { runCapture } from "../subprocess.ts"; + +test("resolves within the deadline even when a grandchild holds the pipe", async () => { + // zsh exits after ~0.2s, but backgrounds `sleep 20` which inherits stdout. + const t0 = Date.now(); + const r = await runCapture( + ["/bin/zsh", "-c", "sleep 20 & echo started; sleep 0.2"], + { timeoutMs: 1000 }, + ); + const elapsed = Date.now() - t0; + expect(elapsed).toBeLessThan(4000); // must NOT wait for the 20s grandchild + expect(r.timedOut).toBe(true); + expect(r.exitCode).toBe(-1); +}); + +test("a SIGTERM-ignoring child is bounded by SIGKILL escalation", async () => { + const t0 = Date.now(); + const r = await runCapture( + ["/bin/zsh", "-c", "trap '' TERM; sleep 20"], + { timeoutMs: 800 }, + ); + expect(Date.now() - t0).toBeLessThan(4000); + expect(r.timedOut).toBe(true); +}); + +test("normal fast command still returns real stdout and exitCode 0", async () => { + const r = await runCapture(["/bin/echo", "hello"], { timeoutMs: 5000 }); + expect(r.stdout.trim()).toBe("hello"); + expect(r.exitCode).toBe(0); + expect(r.timedOut).toBeUndefined(); +}); + +test("timed-out call reports exitCode -1 so callers treat it as failure", async () => { + const r = await runCapture(["/bin/sleep", "20"], { timeoutMs: 500 }); + expect(r.exitCode).toBe(-1); + expect(r.timedOut).toBe(true); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/__tests__/subprocess.test.ts` +Expected: the grandchild test FAILS (elapsed ~20s, exceeds 4000ms) or times out ... proving the current unconditional-await bug. + +- [ ] **Step 3: Rewrite `runCapture` to race the reads against the deadline** + +In `lib/subprocess.ts`, add `timedOut` to `RunResult` and replace the timer + read block: + +```ts +export interface RunResult { + stdout: string; + stderr: string; + exitCode: number; + /** Set true only when the deadline fired before the child settled. */ + timedOut?: boolean; +} +``` + +Replace the body from the `const timer = setTimeout(...)` line through the closing of the `try/catch/finally` (lines 60-76) with: + +```ts + const timeoutMs = opts.timeoutMs ?? 10_000; + // SIGTERM at the deadline, SIGKILL a short grace later. A child that ignores + // SIGTERM (or a D-state descendant) cannot be reaped in-band, so the read is + // raced against the deadline below rather than awaited unconditionally: that + // is what lets runCapture settle while a grandchild still holds the pipe. + let killTimer: ReturnType | undefined; + const term = setTimeout(() => { + try { proc.kill("SIGTERM"); } catch { /* already exited */ } + killTimer = setTimeout(() => { + try { proc.kill("SIGKILL"); } catch { /* already exited */ } + }, 2000); + }, timeoutMs); + + const captured: Promise = (async () => { + try { + const stdoutPromise = new Response(proc.stdout as ReadableStream).text(); + const stderrPromise = captureStderr + ? new Response(proc.stderr as ReadableStream).text() + : Promise.resolve(""); + const [stdout, stderr, exitCode] = await Promise.all([ + stdoutPromise, + stderrPromise, + proc.exited, + ]); + return { stdout, stderr, exitCode }; + } catch { + return { stdout: "", stderr: "", exitCode: -1 }; + } + })(); + + const deadline: Promise = new Promise((resolve) => { + setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); + }); + + try { + return await Promise.race([captured, deadline]); + } finally { + clearTimeout(term); + if (killTimer) clearTimeout(killTimer); + } +``` + +(Keep the `Bun.spawn` block above it unchanged, including the env comment.) + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/__tests__/subprocess.test.ts` +Expected: PASS (all four). + +- [ ] **Step 5: Mirror into rt-client and rebuild** + +Apply the identical `RunResult`/body change to `packages/rt-client/src/settings/exec.ts` (its `runCapture` is the same shape; keep its own env comment). Then: + +Run: `bun run build` (from `packages/rt-client/`) +Then: `bun test packages/rt-client` +Expected: `dist-freshness.test.ts` PASS. + +- [ ] **Step 6: Commit** + +```bash +git add lib/subprocess.ts packages/rt-client/src/settings/exec.ts packages/rt-client/dist lib/__tests__/subprocess.test.ts +git commit -m "runCapture: race reads against the deadline so a pipe-holding grandchild can't wedge it (S023, S024)" +``` + +--- + +### Task 2: Long timeouts for mutating git verbs (S104) + +Mutating git verbs (checkout, merge, stash push/pop) and `status` share the 60s `DEFAULT_TIMEOUT_MS`; a large-repo checkout gets SIGKILLed half-applied at 60s. Give them the same 5-minute budget that fetch/`worktree add` already have. + +**Files:** +- Modify: `lib/worktree/git-async.ts` (add constant; thread `timeoutMs` into stash helpers and `statusPorcelainAsync`) +- Modify: `lib/daemon/worktree-reconciler.ts` (checkout/merge call sites in `freshenOne` and `autoReturnMain`) +- Test: `lib/__tests__/git-async-timeouts.test.ts` (create) + +**Interfaces:** +- Produces: `export const MUTATING_TIMEOUT_MS = 5 * 60_000;` in `git-async.ts`. `stashChangesAsync(cwd, label, opts?: { timeoutMs?: number })` and `popStashAsync(cwd, stashName, opts?: { timeoutMs?: number })` now accept an optional timeout, defaulting to `MUTATING_TIMEOUT_MS`. + +- [ ] **Step 1: Write the failing test** + +Create `lib/__tests__/git-async-timeouts.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { MUTATING_TIMEOUT_MS } from "../worktree/git-async.ts"; + +test("mutating timeout is 5 minutes", () => { + expect(MUTATING_TIMEOUT_MS).toBe(5 * 60_000); +}); +``` + +Also assert the stash helper signature accepts an override (compile-time guard): + +```ts +import { stashChangesAsync, popStashAsync } from "../worktree/git-async.ts"; +test("stash helpers accept a timeout override", () => { + // Type-level: these must type-check with an opts arg. + const a: typeof stashChangesAsync = stashChangesAsync; + const b: typeof popStashAsync = popStashAsync; + expect(typeof a).toBe("function"); + expect(typeof b).toBe("function"); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/__tests__/git-async-timeouts.test.ts` +Expected: FAIL ... `MUTATING_TIMEOUT_MS` not exported. + +- [ ] **Step 3: Add the constant and thread it** + +In `lib/worktree/git-async.ts`, below `const DEFAULT_TIMEOUT_MS = 60_000;`: + +```ts +/** Checkout/merge/stash on a large tree can legitimately exceed a minute; a + * 60s SIGKILL leaves the working tree half-switched. Match fetch/worktree-add. */ +export const MUTATING_TIMEOUT_MS = 5 * 60_000; +``` + +Update the helpers: + +```ts +export async function statusPorcelainAsync(cwd: string): Promise { + const r = await runGit(cwd, ["status", "--porcelain"], { timeoutMs: MUTATING_TIMEOUT_MS }); + return r.stdout; +} + +export async function stashChangesAsync( + cwd: string, + label: string, + opts: { timeoutMs?: number } = {}, +): Promise { + const message = `!!GitHub_Desktop<${label}>`; + await runGit(cwd, ["stash", "push", "-u", "-m", message], { + timeoutMs: opts.timeoutMs ?? MUTATING_TIMEOUT_MS, + }); +} + +export async function popStashAsync( + cwd: string, + stashName: string, + opts: { timeoutMs?: number } = {}, +): Promise { + await runGit(cwd, ["stash", "pop", stashName], { + timeoutMs: opts.timeoutMs ?? MUTATING_TIMEOUT_MS, + }); +} +``` + +- [ ] **Step 4: Bump the reconciler checkout/merge call sites** + +In `lib/daemon/worktree-reconciler.ts`, add `MUTATING_TIMEOUT_MS` to the existing `../worktree/git-async.ts` import (the file already imports `runGit` from there). Add `{ timeoutMs: MUTATING_TIMEOUT_MS }` to these `runGit` calls: +- `freshenOne`: `["checkout", "--", ...classify.discard]` (~line 727); `["merge", "--ff-only", defaultRef]` (~line 756). +- `autoReturnMain`: `["checkout", defaultBranch]` (~line 469); `["merge", "--ff-only", defaultRef]` (~line 475). + +Example: + +```ts +const merge = await runGit(rec.path, ["merge", "--ff-only", defaultRef], { timeoutMs: MUTATING_TIMEOUT_MS }); +``` + +The stash push/pop calls in these functions inherit the new default automatically. (Leave `worktree prune` and `branch -D` at the 60s default; they are not in S104's named set and are fast.) + +- [ ] **Step 5: Run tests** + +Run: `bun test lib/__tests__/git-async-timeouts.test.ts && bun test lib/daemon/__tests__/worktree-reconciler` +Expected: PASS. If a reconciler test asserts an exact `runGit` argv without opts, update it to allow the opts arg. + +- [ ] **Step 6: Commit** + +```bash +git add lib/worktree/git-async.ts lib/daemon/worktree-reconciler.ts lib/__tests__/git-async-timeouts.test.ts +git commit -m "git-async: 5-min timeout for checkout/merge/stash/status so a large-repo checkout isn't killed half-applied (S104)" +``` + +--- + +### Task 3: Excise sync exec from cache-refresh and git-worktrees (S008, S045, S021) + +The 5-minute refresh runs `execSync` (`for-each-ref`, `git config`) and the sync `listWorktrees`/`listWorktreeRoots` per repo on the daemon thread. Swap to the async helpers. Add the missing `listWorktreeRootsAsync` twin. Gate the doppler loop on grants (honors the "off = zero background work" contract). + +**Files:** +- Modify: `lib/worktree/git-async.ts` (add `listWorktreeRootsAsync`) +- Modify: `lib/daemon/cache-refresh.ts` (three swaps + doppler grant gate) +- Modify: `lib/daemon/__tests__/cache-refresh-gc.test.ts` (retarget the `listWorktrees`/`listWorktreeRoots` spies) +- Test: `lib/__tests__/git-async.test.ts` or add to existing git-async coverage for `listWorktreeRootsAsync` + +**Interfaces:** +- Consumes: `listWorktreesAsync(path) => Promise`, `runGit(cwd, args) => Promise` (Task 1/2 hardened). +- Produces: `listWorktreeRootsAsync(repoPath) => Promise` (returns `[]` on git failure, matching the sync contract). + +- [ ] **Step 1: Write the failing test for `listWorktreeRootsAsync`** + +Add to a git-async test file (create `lib/__tests__/git-worktree-roots-async.test.ts`): + +```ts +import { test, expect } from "bun:test"; +import { mkdtempSync } from "fs"; +import { realpathSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { listWorktreeRootsAsync } from "../worktree/git-async.ts"; +import { runGit } from "../worktree/git-async.ts"; + +test("listWorktreeRootsAsync returns the main worktree path", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-wt-"))); + await runGit(dir, ["init", "-q"]); + await runGit(dir, ["commit", "--allow-empty", "-m", "init", "-c", "user.email=a@b.c", "-c", "user.name=t"]); + const roots = await listWorktreeRootsAsync(dir); + expect(roots).toContain(dir); +}); + +test("listWorktreeRootsAsync returns [] on a non-repo", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-nonrepo-"))); + expect(await listWorktreeRootsAsync(dir)).toEqual([]); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/__tests__/git-worktree-roots-async.test.ts` +Expected: FAIL ... `listWorktreeRootsAsync` not exported. + +- [ ] **Step 3: Add `listWorktreeRootsAsync`** + +In `lib/worktree/git-async.ts`, after `listWorktreesAsync`: + +```ts +/** Worktree root paths (main + linked), existing-on-disk only. `[]` on git + * failure ... the async twin of git-worktrees.ts listWorktreeRoots. */ +export async function listWorktreeRootsAsync(repoPath: string): Promise { + return (await listWorktreesAsync(repoPath) ?? []).map((w) => w.path); +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/__tests__/git-worktree-roots-async.test.ts` +Expected: PASS. + +- [ ] **Step 5: Swap the cache-refresh call sites** + +In `lib/daemon/cache-refresh.ts`: + +Imports ... remove `import { execSync } from "child_process";` and `import { listWorktreeRoots, listWorktrees } from "../git-worktrees.ts";`; add `import { listWorktreesAsync, listWorktreeRootsAsync, runGit } from "../worktree/git-async.ts";`. + +Branch listing (line ~112): + +```ts +const branches: Array<{ path: string; branch: string }> = ((await listWorktreesAsync(repoPath)) ?? []) + .filter((w): w is { path: string; branch: string } => !!w.branch && !w.branch.startsWith("on-deck/")); +``` + +`for-each-ref` (lines ~118-133): + +```ts +const worktreeBranchSet = new Set(branches.map((b) => b.branch)); +const localBranches = await runGit(repoPath, ["for-each-ref", "--format=%(refname:short)", "refs/heads/"]); +if (localBranches.exitCode === 0) { + for (const name of localBranches.stdout.split("\n")) { + const trimmed = name.trim(); + if (!trimmed || worktreeBranchSet.has(trimmed) || trimmed.startsWith("on-deck/")) continue; + if (extractLinearId(trimmed)) branches.push({ path: repoPath, branch: trimmed }); + } +} else { + log.warn({ repo: repoPath }, "local branch listing failed"); +} +``` + +(argv form removes the shell, so the old `replace(/^'|'$/g, "")` quote-stripping is no longer needed ... the `--format=%(refname:short)` has no surrounding quotes without a shell.) + +Remote URL (lines ~138-142): + +```ts +let remoteUrl: string | undefined; +const remote = await runGit(repoPath, ["config", "--get", "remote.origin.url"]); +if (remote.exitCode === 0) remoteUrl = remote.stdout.trim() || undefined; +``` + +Doppler loop (line ~209): replace `listWorktreeRoots(repoPath)` with `await listWorktreeRootsAsync(repoPath)`, and gate the loop body on grants. Load tracking once above the loop and skip repos with no cache grant: + +```ts +const tracking = loadRepoTracking(); +for (const [repoName, repoPath] of Object.entries(repoIndex())) { + if (!existsSync(repoPath)) continue; + if (grants(tracking, repoName).caches.size === 0) continue; // off = zero background work + try { + const worktreeRoots = await listWorktreeRootsAsync(repoPath); + // ...unchanged body... +``` + +(`loadRepoTracking` and `grants` are already imported in this file.) + +- [ ] **Step 6: Retarget the GC test spies** + +In `lib/daemon/__tests__/cache-refresh-gc.test.ts` (~lines 105, 108), the test spies on `gitWorktreesModule.listWorktrees`/`listWorktreeRoots`. Retarget to the async twins: + +```ts +import * as gitAsync from "../../worktree/git-async.ts"; +// ... +spyOn(gitAsync, "listWorktreesAsync").mockResolvedValue([]); +spyOn(gitAsync, "listWorktreeRootsAsync").mockResolvedValue([]); +``` + +(Remove the now-unused `gitWorktreesModule` import if nothing else uses it.) + +- [ ] **Step 7: Run the cache-refresh tests** + +Run: `bun test lib/daemon/__tests__/cache-refresh-gc.test.ts` +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add lib/worktree/git-async.ts lib/daemon/cache-refresh.ts lib/daemon/__tests__/cache-refresh-gc.test.ts lib/__tests__/git-worktree-roots-async.test.ts +git commit -m "cache-refresh: async git + grant-gated doppler loop; add listWorktreeRootsAsync (S008, S045, S021)" +``` + +--- + +### Task 4: Async, cached `getRemoteUrl` in freshness (R032) + +`getRemoteUrl` uses `execSync` on the daemon thread inside every forge handler (mr:action, discussions:*, project sync) and the freshness reconcile loop. Make it async via `runCapture` with a 5s timeout, cached per repoPath for the process lifetime. + +**Files:** +- Modify: `lib/daemon/freshness.ts` +- Test: `lib/daemon/__tests__/freshness-remote-url.test.ts` (create) + +**Interfaces:** +- Produces: `getRemoteUrl(repoPath) => Promise` (was sync). All four call sites become `await getRemoteUrl(...)`. + +- [ ] **Step 1: Write the failing test** + +Create `lib/daemon/__tests__/freshness-remote-url.test.ts` ... since `getRemoteUrl` is module-private, test through a real repo via a tiny exported probe is not available; instead assert no `execSync`/`child_process` import remains in freshness.ts (the observable contract for R032) plus a behavioral cache test using a real temp repo through the public `getRepoContext` is heavy. Keep it to the source guard, which the Task 7 gate will also enforce, plus a targeted unit if a seam exists: + +```ts +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("freshness.ts no longer imports execSync/child_process", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + expect(src).not.toMatch(/from\s+["']child_process["']/); + expect(src).not.toMatch(/\bexecSync\b/); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/freshness-remote-url.test.ts` +Expected: FAIL ... `child_process`/`execSync` still present. + +- [ ] **Step 3: Convert `getRemoteUrl` and add the cache** + +In `lib/daemon/freshness.ts`: remove `import { execSync } from "child_process";`; add `import { runCapture } from "../subprocess.ts";`. Add a module-level cache near the other module state (below the `providers`/`userId` block ~line 91): + +```ts +const remoteUrlCache = new Map(); +``` + +Replace `getRemoteUrl` (lines 95-103): + +```ts +/** remote.origin.url, cached per repoPath for the process lifetime (remotes + * rarely change). Async so it never blocks the event loop. */ +async function getRemoteUrl(repoPath: string): Promise { + const cached = remoteUrlCache.get(repoPath); + if (cached !== undefined) return cached; + const r = await runCapture(["git", "config", "--get", "remote.origin.url"], { + cwd: repoPath, + timeoutMs: 5000, + stderr: "ignore", + }); + const url = r.exitCode === 0 ? (r.stdout.trim() || null) : null; + remoteUrlCache.set(repoPath, url); + return url; +} +``` + +Add `await` at the four call sites: `ensureProvider` (~135), `getRepoContext` (~273 and ~292), `reconcileFreshnessImpl` (~692). All three enclosing functions are already `async`. + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/freshness-remote-url.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc zero errors (the `await` additions type-check because callers are async). + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/freshness.ts lib/daemon/__tests__/freshness-remote-url.test.ts +git commit -m "freshness: async, cached getRemoteUrl via runCapture (R032)" +``` + +--- + +### Task 5: Async `killWorktreeProcesses` (S015, S016) + +`killWorktreeProcesses` runs three `execSync` calls (system-wide `lsof`, two `ps`) on the daemon thread per disposal and on the unattended reactor path; a stuck `lsof` wedges the loop. Port to `runCapture` (the identical async `lsof` already exists in `system-process-scanner.ts`). + +**Files:** +- Modify: `lib/daemon/worktree-process-kill.ts` (make `killWorktreeProcesses` async; three `runCapture` swaps) +- Modify: `lib/worktree/dispose.ts:238` (add `await`) +- Modify: `lib/daemon/worktree-reconciler.ts:447` (add `await`; update the "sync by design" comment) +- Test: `lib/daemon/__tests__/worktree-process-kill.test.ts` (add a source guard + keep `selectKillTargets` coverage) + +**Interfaces:** +- Produces: `killWorktreeProcesses(worktreePath) => Promise` (was sync). `WorktreeKillResult` unchanged. + +- [ ] **Step 1: Write the failing test** + +Add to `lib/daemon/__tests__/worktree-process-kill.test.ts`: + +```ts +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("worktree-process-kill.ts imports no sync exec", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "worktree-process-kill.ts"), "utf8"); + expect(src).not.toMatch(/from\s+["']child_process["']/); + expect(src).not.toMatch(/\bexecSync\b/); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/worktree-process-kill.test.ts` +Expected: FAIL ... `execSync` still imported. + +- [ ] **Step 3: Port the three calls to `runCapture`** + +In `lib/daemon/worktree-process-kill.ts`: remove `import { execSync } from "child_process";`; add `import { runCapture } from "../subprocess.ts";`. Make the function async and swap each call (argv form, no shell, no `2>/dev/null`): + +```ts +export async function killWorktreeProcesses(worktreePath: string): Promise { + const lsof = await runCapture(["lsof", "-d", "cwd", "-Fpn"], { timeoutMs: 10_000 }); + if (lsof.exitCode !== 0 && !lsof.stdout) { + log.warn({ exitCode: lsof.exitCode, worktreePath }, "lsof failed; skipping worktree process kill"); + return { terminated: [] }; + } + const lsofOut = lsof.stdout; + // ...existing parse of lsofOut into candidate pids (unchanged)... +``` + +Then the `ps -p -o pid=,ppid=,comm=,args=` call: + +```ts + const ps = await runCapture(["ps", "-p", pidList, "-o", "pid=,ppid=,comm=,args="], { timeoutMs: 5000 }); + if (ps.exitCode !== 0 && !ps.stdout) { + log.warn({ exitCode: ps.exitCode, worktreePath }, "ps failed; skipping worktree process kill"); + return { terminated: [] }; + } + for (const line of ps.stdout.split("\n")) { /* unchanged parse */ } +``` + +Then the `ps eww` label call: + +```ts + const eww = await runCapture(["ps", "eww", "-o", "pid=,command=", "-p", targets.map((t) => t.pid).join(",")], { timeoutMs: 5000 }); + if (eww.exitCode === 0 || eww.stdout) scripts = parsePackageScripts(eww.stdout); +``` + +(`pidList` is the same comma/space-joined pid string the old `ps` used; pass it as one argv element ... `ps -p` accepts a comma-separated list.) + +- [ ] **Step 4: Await at the two call sites** + +`lib/worktree/dispose.ts:238`: `const { terminated } = await killWorktreeProcesses(rec.path);` +`lib/daemon/worktree-reconciler.ts:447`: `const { terminated } = await killWorktreeProcesses(rec.path);` ... and replace the "ruled execSync exception (the process killer is sync by design)" comment with one that no longer claims sync (e.g. drop it; the try/catch keeps its "a failure here never blocks the return" rationale). + +- [ ] **Step 5: Run tests** + +Run: `bun test lib/daemon/__tests__/worktree-process-kill.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean (both call sites already sit in `async` functions). + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/worktree-process-kill.ts lib/worktree/dispose.ts lib/daemon/worktree-reconciler.ts lib/daemon/__tests__/worktree-process-kill.test.ts +git commit -m "worktree-process-kill: async lsof/ps via runCapture (S015, S016)" +``` + +--- + +### Task 6: Async index write in `resolveIndexPathForIdentity` (S098) + +`endpoint:claim`/`endpoint:lookup` can reach `resolveIndexPathForIdentity`, which on a legacy-key match calls `updateRepoIndex` → `observedMainPath` → a synchronous `git worktree list` on the daemon thread. Resolve the observed main path asynchronously. + +**Files:** +- Modify: `lib/repo-index.ts` (add `observedMainPathAsync`; use it on the async path) +- Test: `lib/__tests__/repo-index-async.test.ts` (create) + +**Interfaces:** +- Consumes: `listWorktreesAsync(path) => Promise`, `setIndexPath(key, mainPath)` (existing sync KV write, no subprocess). +- Produces: `observedMainPathAsync(repoRoot) => Promise` (degrades to `repoRoot`). + +- [ ] **Step 1: Write the failing test** + +Create `lib/__tests__/repo-index-async.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("resolveIndexPathForIdentity no longer reaches a sync git via observedMainPath", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "repo-index.ts"), "utf8"); + // observedMainPath (sync execSync) must not be called from the async resolver path. + expect(src).toMatch(/observedMainPathAsync/); +}); +``` + +(The Task 7 import-graph gate is the real enforcement that no sync git reaches the daemon graph; this test just anchors the async twin's existence.) + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/__tests__/repo-index-async.test.ts` +Expected: FAIL ... `observedMainPathAsync` not present. + +- [ ] **Step 3: Add `observedMainPathAsync` and use it** + +In `lib/repo-index.ts`, add `import { listWorktreesAsync } from "./worktree/git-async.ts";` (top imports). Add next to `observedMainPath`: + +```ts +/** Async twin of observedMainPath: the repo's MAIN worktree path as git + * reports it, degrading to repoRoot. Safe on the daemon thread. */ +async function observedMainPathAsync(repoRoot: string): Promise { + const wts = await listWorktreesAsync(repoRoot); + return wts?.[0]?.path ?? repoRoot; +} +``` + +In `resolveIndexPathForIdentity` (line ~277), replace `updateRepoIndex(serialized, path);` with: + +```ts +setIndexPath(serialized, await observedMainPathAsync(path)); +``` + +(`setIndexPath` is the raw KV write already exported in this file; it does the same persistence `updateRepoIndex` does, minus the sync git probe, which `observedMainPathAsync` now supplies.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/__tests__/repo-index-async.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean (`resolveIndexPathForIdentity` is already async). + +- [ ] **Step 5: Commit** + +```bash +git add lib/repo-index.ts lib/__tests__/repo-index-async.test.ts +git commit -m "repo-index: async observed-main-path on the endpoint:claim resolve path (S098)" +``` + +--- + +### Task 7: The import-graph gate (1.3) + +A test that walks the daemon's import graph from `lib/daemon.ts` and fails if any daemon-reachable module has a sync-exec call site (`execSync(`, `spawnSync(`, `Bun.spawnSync(`, `Bun.sleepSync(`). The rule has been re-broken twice; only a gate keeps it dead. Ships GREEN with an honest allowlist of every current out-of-Phase-1 offender, each commented with the finding/phase that removes it (decision: full-closure walk with honest allowlist). + +**Verified facts (from a throwaway walker run against the live tree):** +- Closure from `lib/daemon.ts` = 151 files. `lib/daemon.ts` and other entry files carry a `#!/usr/bin/env bun` shebang that makes `Bun.Transpiler.scanImports` throw ... strip a leading shebang before scanning. +- `.tsx` files (e.g. `lib/rt-render.tsx`, in the closure) need the `tsx` loader, not `ts`. +- Relative imports are spelled with explicit `.ts`/`.tsx` extensions throughout, so `resolve(dirname(file), importPath)` needs no extension guessing. +- Matching call sites (`\bexecSync\s*\(`) rather than bare `execSync` avoids false positives: `api-server.ts:105` mentions `Bun.sleepSync` in a comment with no paren, correctly NOT flagged. Raw-source matching yields the same 13 offenders as comment-stripped matching, so the test uses raw matching. +- After Tasks 3/4/5 remove cache-refresh.ts, freshness.ts, and worktree-process-kill.ts from the offender set, exactly these 10 remain (the allowlist): + +| Allowlist entry | Removing finding / phase | +|---|---| +| `lib/daemon/user-path.ts` | Phase 6 PATH rebuild (S013/S014/S062) | +| `lib/daemon/boot-reconcile.ts` | Phase 0.6 / S044 (`Bun.sleepSync`) | +| `lib/state/db.ts` | Phase 0.7 / S072-S073 busy-retry (`Bun.sleepSync`) | +| `lib/state/busy.ts` | Phase 0.7 / S072-S073 busy-retry (`Bun.sleepSync`) | +| `lib/git-worktrees.ts` | S055 (reached only via `handlers/status.ts`; that swap to `listWorktreesAsync` removes the edge) | +| `lib/daemon/handlers/status.ts` | S055 (the edge that pulls in git-worktrees.ts; own source has no sync-exec, listed per the brief) | +| `lib/repo-index.ts` | Phase 5.3 dedup (retains `execSync` at heal/derive paths; Task 6's endpoint path is covered by `repo-index-async.test.ts`) | +| `lib/repo.ts` | R050 / Phase 5.4 (reached via `handlers/system-processes.ts` → `repo-arg.ts`) | +| `lib/git.ts` | R050 / Phase 5.4 (via `repo.ts`) | +| `lib/herdr-launch.ts` | Phase 5 herdr (reached via `handlers/pane.ts`) | +| `lib/rt-render.tsx` | R050 / Phase 5.4 (daemon carries the TUI; `no-eager-tui` extension breaks the chain) | + +**Files:** +- Create: `lib/__tests__/no-daemon-sync-exec.test.ts` + +**Interfaces:** +- Self-contained test; consumes nothing from other tasks. Must run AFTER Tasks 3/4/5 land (else the three fixed files fail it). + +- [ ] **Step 1: Write the gate test (RED against current tree, before Tasks 3/4/5, or GREEN after)** + +Create `lib/__tests__/no-daemon-sync-exec.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { dirname, resolve } from "path"; + +// Files with sync-exec that Phase 1 does NOT remove. Each entry names the +// finding/phase that will delete it, so this list shrinks as later phases land. +// A regression that reintroduces sync-exec into any OTHER daemon-reachable +// module fails this gate (the rule has been re-broken twice). +const ALLOWLIST = new Set([ + "lib/daemon/user-path.ts", // Phase 6 PATH rebuild (S013/S014/S062) + "lib/daemon/boot-reconcile.ts", // Phase 0.6 / S044 (Bun.sleepSync) + "lib/state/db.ts", // Phase 0.7 / S072-S073 busy-retry + "lib/state/busy.ts", // Phase 0.7 / S072-S073 busy-retry + "lib/git-worktrees.ts", // S055: reached only via handlers/status.ts + "lib/daemon/handlers/status.ts", // S055: the edge into git-worktrees.ts + "lib/repo-index.ts", // Phase 5.3 dedup (heal/derive execSync) + "lib/repo.ts", // R050 / Phase 5.4 (via handlers/system-processes.ts) + "lib/git.ts", // R050 / Phase 5.4 (via repo.ts) + "lib/herdr-launch.ts", // Phase 5 herdr (via handlers/pane.ts) + "lib/rt-render.tsx", // R050 / Phase 5.4 (daemon carries the TUI) +]); + +const SYNC_EXEC = [ + /\bexecSync\s*\(/, + /\bspawnSync\s*\(/, + /\bBun\.spawnSync\s*\(/, + /\bBun\.sleepSync\s*\(/, +]; + +const REPO_ROOT = resolve(import.meta.dir, "..", ".."); +const stripShebang = (s: string) => s.replace(/^#!.*\n/, ""); +const tsT = new Bun.Transpiler({ loader: "ts" }); +const tsxT = new Bun.Transpiler({ loader: "tsx" }); +const loaderFor = (f: string) => (f.endsWith(".tsx") || f.endsWith(".jsx") ? tsxT : tsT); + +/** Files reachable from lib/daemon.ts via relative imports (the daemon graph). */ +function daemonClosure(): string[] { + const entry = resolve(REPO_ROOT, "lib/daemon.ts"); + const visited = new Set(); + const stack = [entry]; + while (stack.length) { + const file = stack.pop()!; + if (visited.has(file)) continue; + visited.add(file); + let src: string; + try { src = stripShebang(readFileSync(file, "utf8")); } catch { continue; } + let imports: { path: string }[]; + try { imports = loaderFor(file).scanImports(src); } catch { continue; } + for (const imp of imports) { + if (!imp.path.startsWith(".")) continue; // external package + stack.push(resolve(dirname(file), imp.path)); + } + } + return [...visited]; +} + +function hasSyncExec(source: string): boolean { + return SYNC_EXEC.some((re) => re.test(source)); +} + +test("no daemon-reachable module calls sync exec (outside the allowlist)", () => { + const offenders: string[] = []; + for (const file of daemonClosure()) { + const rel = file.replace(REPO_ROOT + "/", ""); + if (ALLOWLIST.has(rel)) continue; + let src: string; + try { src = readFileSync(file, "utf8"); } catch { continue; } + if (hasSyncExec(src)) offenders.push(rel); + } + expect(offenders).toEqual([]); +}); + +test("the checker flags a reintroduced sync-exec call (proves the gate bites)", () => { + // Permanent RED proof: the matcher must catch a fresh offense. + expect(hasSyncExec(`import { execSync } from "child_process";\nexecSync("true");`)).toBe(true); + expect(hasSyncExec(`await Bun.sleepSync(10);`)).toBe(true); + expect(hasSyncExec(`// a comment mentioning execSync without a call`)).toBe(false); +}); + +test("the daemon closure actually resolves (guards against a walker that finds nothing)", () => { + const closure = daemonClosure(); + expect(closure.length).toBeGreaterThan(50); // ~151 today; a collapse means the walk broke +}); +``` + +- [ ] **Step 2: Run the gate** + +Run: `bun test lib/__tests__/no-daemon-sync-exec.test.ts` +Expected: GREEN (Tasks 3/4/5 already removed cache-refresh, freshness, worktree-process-kill). If it lists an offender not in the allowlist, that file is either a Phase-1 target that regressed (fix it) or a daemon-reachable sync-exec outside Phase 1 (add it to the allowlist with a finding/phase comment). The closure-size guard and the checker RED-proof tests must also pass. + +- [ ] **Step 3: Demonstrate the gate fails on reintroduction (report evidence)** + +Temporarily add to `lib/daemon/pollers.ts` (a Phase-1-clean, non-allowlisted daemon file): `import { execSync } from "child_process";` and a call `execSync("true");`. Run: + +Run: `bun test lib/__tests__/no-daemon-sync-exec.test.ts` +Expected: FAIL ... offenders `["lib/daemon/pollers.ts"]`. Capture this RED output for the report, then revert both lines. + +- [ ] **Step 4: Commit** + +```bash +git add lib/__tests__/no-daemon-sync-exec.test.ts +git commit -m "gate: fail on sync-exec anywhere in the daemon import graph (1.3)" +``` + +--- + +### Task 8: Refresh cannot wedge (S007) + +One hung GitLab call latches `refreshInFlight` forever, freezing the coalesced refresh, and merged `runner.pending` grows unbounded. Add a whole-cycle deadline that clears the latch so the next tick can start, and cap the pending queue. + +**Files:** +- Modify: `lib/daemon/cache-refresh.ts` (extract a coalescer with a deadline) +- Modify: `lib/daemon/freshness.ts` (`applyInvalidationBatch` pending cap) +- Test: `lib/daemon/__tests__/cache-refresh-coalesce.test.ts` (create), `lib/daemon/__tests__/freshness-pending-cap.test.ts` (create) + +**Interfaces:** +- Produces: `makeCoalescer(run, deadlineMs, onTimeout) => () => Promise` exported from `cache-refresh.ts`. +- Produces: `PENDING_CAP` const in `freshness.ts`; `applyInvalidationBatch` dedupes-and-caps pending. + +- [ ] **Step 1: Write the failing coalescer test** + +Create `lib/daemon/__tests__/cache-refresh-coalesce.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { makeCoalescer } from "../cache-refresh.ts"; + +test("clears the in-flight latch after the deadline even if run never settles", async () => { + let starts = 0; + let timedOut = 0; + const coalesce = makeCoalescer( + () => { starts++; return new Promise(() => {}); }, // never resolves + 50, + () => { timedOut++; }, + ); + const t0 = Date.now(); + await coalesce(); // resolves at the deadline, not never + expect(Date.now() - t0).toBeLessThan(500); + expect(timedOut).toBe(1); + await coalesce(); // latch cleared, a new run can start + expect(starts).toBe(2); +}); + +test("coalesces concurrent callers onto one run", async () => { + let starts = 0; + let resolveRun!: () => void; + const coalesce = makeCoalescer( + () => { starts++; return new Promise((r) => { resolveRun = r; }); }, + 10_000, + () => {}, + ); + const a = coalesce(); + const b = coalesce(); + expect(starts).toBe(1); + resolveRun(); + await Promise.all([a, b]); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/cache-refresh-coalesce.test.ts` +Expected: FAIL ... `makeCoalescer` not exported. + +- [ ] **Step 3: Add `makeCoalescer` and use it** + +In `lib/daemon/cache-refresh.ts`, add above `createCacheRefresher`: + +```ts +/** Below the 5-min tick, above the slowest legitimate deep sync. */ +const REFRESH_CYCLE_DEADLINE_MS = 4 * 60 * 1000; + +/** + * Coalesce concurrent callers onto one in-flight run, but clear the latch after + * `deadlineMs` even if the run never settles, so a wedged cycle (a half-open + * GitLab socket that never rejects) cannot pin the latch forever. The wedged + * run's frame still leaks until the OS reaps the socket; this only frees the + * next tick. + */ +export function makeCoalescer( + run: () => Promise, + deadlineMs: number, + onTimeout: () => void, +): () => Promise { + let inFlight: Promise | null = null; + return () => { + if (inFlight) return inFlight; + const impl = run().catch(() => {}); // a rejected cycle still clears the latch + const guarded = Promise.race([ + impl, + new Promise((resolve) => setTimeout(() => { onTimeout(); resolve(); }, deadlineMs)), + ]).finally(() => { inFlight = null; }); + inFlight = guarded; + return guarded; + }; +} +``` + +Replace the `refreshInFlight` coalescer (lines 54-60) inside `createCacheRefresher`: + +```ts + const refreshCache = makeCoalescer( + refreshCacheImpl, + REFRESH_CYCLE_DEADLINE_MS, + () => log.warn("cache refresh timed out; cleared in-flight latch for next tick"), + ); +``` + +(Delete `let refreshInFlight` and the old `function refreshCache`. `refreshCacheImpl` stays as the async body; `return refreshCache;` at the end is unchanged.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/cache-refresh-coalesce.test.ts` +Expected: PASS. + +- [ ] **Step 5: Add the pending-cap test** + +Create `lib/daemon/__tests__/freshness-pending-cap.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { applyInvalidationBatch, PENDING_CAP } from "../freshness.ts"; + +test("merged pending is deduped by kind:ref and capped", async () => { + const runner: any = { processing: true, pending: [] }; + // Push more distinct keys than the cap; plus duplicates. + const keys = Array.from({ length: PENDING_CAP + 500 }, (_, i) => ({ kind: "mr", ref: String(i) })); + const dupes = [{ kind: "mr", ref: "0" }, { kind: "mr", ref: "0" }]; + await applyInvalidationBatch({} as any, {} as any, runner, [...keys, ...dupes], {}); + expect(runner.pending.length).toBeLessThanOrEqual(PENDING_CAP); + const ids = runner.pending.map((k: any) => `${k.kind}:${k.ref}`); + expect(new Set(ids).size).toBe(ids.length); // no duplicates +}); +``` + +- [ ] **Step 6: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/freshness-pending-cap.test.ts` +Expected: FAIL ... `PENDING_CAP` not exported (and current code pushes unbounded). + +- [ ] **Step 7: Cap the pending queue** + +In `lib/daemon/freshness.ts`, add near the module state (below `const watches`): + +```ts +/** Bound merged pending so a wedged processKeys cannot grow memory unbounded. */ +export const PENDING_CAP = 1000; +``` + +Replace the `if (runner.processing)` block in `applyInvalidationBatch` (lines 392-395): + +```ts + if (runner.processing) { + const seen = new Set(runner.pending.map((k) => `${k.kind}:${k.ref}`)); + for (const k of keys) { + if (runner.pending.length >= PENDING_CAP) break; + const id = `${k.kind}:${k.ref}`; + if (seen.has(id)) continue; + seen.add(id); + runner.pending.push(k); + } + return; + } +``` + +- [ ] **Step 8: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/freshness-pending-cap.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean. + +- [ ] **Step 9: Commit** + +```bash +git add lib/daemon/cache-refresh.ts lib/daemon/freshness.ts lib/daemon/__tests__/cache-refresh-coalesce.test.ts lib/daemon/__tests__/freshness-pending-cap.test.ts +git commit -m "refresh: whole-cycle deadline clears the coalesce latch; cap RepoWatch.pending (S007)" +``` + +--- + +### Task 9: Provider cache invalidation on token rotation (S048, S049) + +`providers` caches `GitLabProvider` instances by repoName with no token check, so a rotated `gitlabToken` never reaches watchers or forge handlers until a daemon restart. Key the cache on a token fingerprint and rebuild on mismatch; reset the `userIdResolved` latch. + +**Files:** +- Modify: `lib/daemon/freshness.ts` +- Test: `lib/daemon/__tests__/freshness-provider-rotation.test.ts` (create) + +**Interfaces:** +- Consumes: `loadSecrets()` (from `../linear.ts`, returns `{ gitlabToken }`), `makeProvider(host, token)`. +- Internal: `providers` map value becomes `{ provider: GitLabProvider; token: string }`. + +- [ ] **Step 1: Write the failing test** + +Create `lib/daemon/__tests__/freshness-provider-rotation.test.ts`. Since `ensureProvider` is module-private, test the observable contract: the source no longer returns a cached provider without comparing the current token. Assert the fingerprint mechanism exists and the `if (cached) return cached;` short-circuit is gone: + +```ts +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("ensureProvider compares the current token before reusing a cached provider", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + // The unconditional cache-hit return is the S049 bug; it must be gone. + expect(src).not.toMatch(/const cached = providers\.get\(repoName\);\s*\n\s*if \(cached\) return cached;/); + // A token fingerprint must be stored alongside the provider. + expect(src).toMatch(/providers\.set\(repoName,\s*\{\s*provider/); +}); +``` + +(A behavioral test would need a real GitLabProvider + secrets seam; the freshness module has no injection point for `loadSecrets`. This source-contract test plus `bunx tsc` is the achievable guard; note the limitation in the report.) + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/freshness-provider-rotation.test.ts` +Expected: FAIL ... the unconditional cache-hit return still present. + +- [ ] **Step 3: Key the provider cache on a token fingerprint** + +In `lib/daemon/freshness.ts`, change the `providers` map type (line 88): + +```ts +const providers = new Map(); +``` + +Rewrite `ensureProvider` so `loadSecrets()` runs before the cache decision and a token mismatch rebuilds (lines 125-155): + +```ts +async function ensureProvider(repoName: string, repoPath: string): Promise { + const secrets = await loadSecrets(); + if (!secrets.gitlabToken) { + log.info(`no gitlabToken; skipping ${repoName}`); + return null; + } + const cached = providers.get(repoName); + if (cached && cached.token === secrets.gitlabToken) return cached.provider; + if (cached) { + // Token rotated: drop the stale provider and any live watch built on it so + // the next reconcile rebuilds with the new token, and re-resolve userId. + stopWatch(repoName); + userIdResolved = false; + } + + const remoteUrl = await getRemoteUrl(repoPath); + if (!remoteUrl) { log.info(`no origin remote for ${repoName}; skipping`); return null; } + if (!isGitLabRemote(remoteUrl)) { log.info(`remote "${remoteUrl}" for ${repoName} is not GitLab; skipping events watch`); return null; } + const remote = parseRemoteUrl(remoteUrl); + if (!remote) { log.info(`could not parse remote "${remoteUrl}" for ${repoName}; skipping`); return null; } + + const provider = makeProvider(remote.host, secrets.gitlabToken); + providers.set(repoName, { provider, token: secrets.gitlabToken }); + return provider; +} +``` + +Update the other cache reads to the new value shape: +- `getRepoContext` line 251: `let provider = providers.get(repoName)?.provider ?? null;` +- `getRepoContext` line 282 (after building): `providers.set(repoName, { provider, token: secrets.gitlabToken });` +- `ensureUserId` line 160: `const anyProvider = providers.values().next().value?.provider as GitLabProvider | undefined;` +- `disposeFreshness` `providers.clear()` unchanged. + +(`stopWatch` is already defined in this module; calling it for a repo with no watch is a no-op ... verify its guard, add an early return if absent.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/freshness-provider-rotation.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean (all `providers.get/set/values` sites updated to the new shape). + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/freshness.ts lib/daemon/__tests__/freshness-provider-rotation.test.ts +git commit -m "freshness: rebuild the provider cache when gitlabToken rotates (S048, S049)" +``` + +--- + +### Task 10: A single failed lsof no longer resets runaway detection (S061) + +`gather()` returns `[]` on both a real empty scan and an `lsof`/`ps` failure; `scan()` then prunes every tracked pid, resetting `firstSeen` and the runaway sample window so a machine where `lsof` fails intermittently can never fire a runaway notification. Distinguish failure (null) from empty. + +**Files:** +- Modify: `lib/daemon/system-process-scanner.ts` (`getAllRepoPids`, `gather`, `scan`, `refresh`) +- Test: `lib/daemon/__tests__/system-process-scanner-resilience.test.ts` (create) + +**Interfaces:** +- Internal: `getAllRepoPids` and `gather` return `... | null` (null = the underlying `lsof`/`ps` scan failed). `scan`/`refresh` preserve `tracked`/`lastResult`/`lastScanAt` when `gather` returns null. +- Change `gather` from `private` to `protected` so a test subclass can drive its return value. + +- [ ] **Step 1: Write the failing test** + +Create `lib/daemon/__tests__/system-process-scanner-resilience.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { SystemProcessScanner } from "../system-process-scanner.ts"; + +class FakeScanner extends SystemProcessScanner { + next: any[] | null = []; + protected async gather(): Promise { return this.next; } +} + +test("a failed gather (null) keeps tracked windows and lastResult intact", async () => { + const s = new FakeScanner(); + // Seed a tracked process across enough scans that firstSeen is established. + s.next = [{ pid: 4242, cpuPercent: 95, command: "node", args: "x", ppid: 1, port: null, memoryMB: 10, etime: "01:00", repo: "r", linearTicket: null, packageScript: null }]; + const first = await s.scan(); + expect(first.find((p) => p.pid === 4242)).toBeTruthy(); + const firstSeen = s.getTracked().get(4242)?.firstSeen; + + // Now lsof fails: gather returns null. tracked and lastResult must survive. + s.next = null; + const during = await s.scan(); + expect(s.getTracked().get(4242)?.firstSeen).toBe(firstSeen); + expect(during.find((p) => p.pid === 4242)).toBeTruthy(); // lastResult carried forward +}); +``` + +(Field names on the fake process object follow `GatheredProcess`; adjust to the exact shape when implementing. `getTracked()` is an existing accessor.) + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/system-process-scanner-resilience.test.ts` +Expected: FAIL ... `gather` is `private` (can't override) and/or `scan` wipes `tracked` on the null tick. + +- [ ] **Step 3: Thread the failure signal** + +In `lib/daemon/system-process-scanner.ts`: + +`getAllRepoPids` ... return null on lsof failure instead of an empty Map: + +```ts +async function getAllRepoPids(trackedPaths: string[]): Promise | null> { + if (trackedPaths.length === 0) return new Map(); + const { stdout, exitCode } = await runCapture(["lsof", "-d", "cwd", "-Fpn"], { timeoutMs: 10_000 }); + if (exitCode !== 0 && !stdout) { + log.warn({ exitCode }, "lsof scan failed; preserving prior process state"); + return null; + } + return parseLsofCwdMap(stdout, trackedPaths); +} +``` + +`gather` ... change signature to `protected async gather(...): Promise` and propagate null (lines 380, 389): + +```ts + const cwdMap = await getAllRepoPids(trackedPaths); + if (cwdMap === null) return null; // lsof failed + if (cwdMap.size === 0) return []; // genuinely no tracked-cwd processes + // ... + const psRes = await runCapture([...], { timeoutMs: 5000 }); + if (psRes.exitCode !== 0 && !psRes.stdout) return null; // ps failed + if (!psRes.stdout) return []; +``` + +`scan` ... early-return on null before the try/finally so `tracked`/`lastResult`/`lastScanAt` are untouched: + +```ts + async scan(portEntries: PortEntry[] = []): Promise { + const gathered = await this.gather(portEntries); + if (gathered === null) return this.lastResult; // failed tick: preserve everything + try { + // ...existing loop over `gathered`, prune, this.lastResult = results... + return results; + } finally { + this.lastScanAt = Date.now(); + } + } +``` + +`refresh` ... same null guard: + +```ts + async refresh(portEntries: PortEntry[] = []): Promise { + const gathered = await this.gather(portEntries); + if (gathered === null) return this.lastResult; + try { + // ...existing map... + return results; + } finally { + this.lastScanAt = Date.now(); + } + } +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/system-process-scanner-resilience.test.ts && bun test lib/daemon/__tests__/system-process-scanner.test.ts && bunx tsc --noEmit` +Expected: PASS (new + existing scanner tests); tsc clean. + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/system-process-scanner.ts lib/daemon/__tests__/system-process-scanner-resilience.test.ts +git commit -m "system-process-scanner: a failed lsof preserves runaway windows (S061)" +``` + +--- + +### Task 11: Cheapen run-DB opens: mtime memoization + herdr backoff (S101, S038, S039) + +The agent-status poller and `runs:list` open every retained run's SQLite db every 10s (cost scales with retained runs, not live runs), and the poller spawns herdr every 10s forever on a herdr-less machine. Memoize finished-run summaries by state.db mtime, and back off the herdr probe after repeated failures. + +**Files:** +- Modify: `lib/runs/store.ts` (mtime cache in `listRuns`) +- Modify: `lib/daemon/agent-status-poller.ts` (consecutive-failure backoff) +- Test: `lib/runs/__tests__/store-memo.test.ts` (create), add to `lib/daemon/__tests__/agent-status-poller.test.ts` + +**Interfaces:** +- Internal: `listRuns` caches finished-run `RunSummary` keyed by `${repo}/${id}` → `{ mtimeMs, summary }`; running runs are never cached (their db still mutates and their liveness overlay changes). +- Internal: poller gains `FAILURE_THRESHOLD` / `BACKOFF_TICKS` backoff. + +- [ ] **Step 1: Write the failing memoization test** + +Create `lib/runs/__tests__/store-memo.test.ts`. Use the existing `seedRun`/`root` fixtures and a spy to prove a finished run's db is not reopened when its mtime is unchanged: + +```ts +import { test, expect, afterEach, spyOn } from "bun:test"; +import { Database } from "bun:sqlite"; +import { listRuns } from "../store.ts"; +import { root, seedRun } from "./fixtures.ts"; + +afterEach(() => { delete process.env.RT_RUNS_ROOT; }); + +test("a finished run's db is opened once, then served from the mtime cache", () => { + root(); // sets RT_RUNS_ROOT to a temp dir + seedRun("repoA", "run1", { status: "done" }); + const openSpy = spyOn(Database.prototype, "query"); + listRuns(); // first call opens + reads + const afterFirst = openSpy.mock.calls.length; + listRuns(); // second call: mtime unchanged -> no reopen + expect(openSpy.mock.calls.length).toBe(afterFirst); // no additional queries for run1 + openSpy.mockRestore(); +}); +``` + +(If `seedRun`'s signature differs, match it; the point is a finished run plus a proof the second `listRuns` does not re-query it.) + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/runs/__tests__/store-memo.test.ts` +Expected: FAIL ... every `listRuns` reopens and re-queries every run. + +- [ ] **Step 3: Add the mtime cache** + +In `lib/runs/store.ts`, add `import { statSync } from "fs";` (if absent) and a module cache: + +```ts +// Finished runs never change; skip the open+PRAGMA+4-reads when the db mtime +// is unchanged. Running runs are never cached: their db still mutates and their +// liveness overlay is recomputed per call. +const summaryCache = new Map(); +``` + +Rewrite the per-run body of `listRuns` (lines 122-130): + +```ts + for (const id of dirs(join(runsRoot(), r))) { + const dbPath = join(runsRoot(), r, id, "state.db"); + let mtimeMs: number; + try { mtimeMs = statSync(dbPath).mtimeMs; } catch { continue; } + const key = `${r}/${id}`; + const hit = summaryCache.get(key); + if (hit && hit.mtimeMs === mtimeMs) { out.push(hit.summary); continue; } + + const opened = openRun(r, id); + if (!opened) continue; + try { + const row = runRow(opened.db); + if (row) { + const summary = withAttention(opened.db, row, liveness); + out.push(summary); + if (summary.status !== "running") summaryCache.set(key, { mtimeMs, summary }); + } + } finally { + opened.db.close(); + } + } +``` + +(Confirm the field is `summary.status`; the poller reads `run.status`. If `RunSummary` names it `state`, use that.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/runs/__tests__/store-memo.test.ts && bun test lib/runs/__tests__/store.test.ts` +Expected: PASS (new + existing store tests, including corrupt/missing-db cases). + +- [ ] **Step 5: Write the failing herdr-backoff test** + +Add to `lib/daemon/__tests__/agent-status-poller.test.ts` (using the existing `probe`/`list` DI seams and manual `tick()`): + +```ts +test("backs off the herdr probe after repeated failures", async () => { + let probeCalls = 0; + const handle = startAgentStatusPoller({ + intervalMs: 3_600_000, // real timer never fires + probe: async () => { probeCalls++; return null; }, // herdr absent + list: () => [], + }); + for (let i = 0; i < 20; i++) await handle.tick(); + handle.stop(); + // Without backoff this would be 20; with backoff (threshold 3, 1-in-6) far fewer. + expect(probeCalls).toBeLessThan(10); +}); +``` + +- [ ] **Step 6: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/agent-status-poller.test.ts` +Expected: FAIL ... `probeCalls` is 20 (probes every tick). + +- [ ] **Step 7: Add the backoff** + +In `lib/daemon/agent-status-poller.ts`, add constants and counter state in the closure around `tick`: + +```ts +const FAILURE_THRESHOLD = 3; // consecutive null probes before backing off +const BACKOFF_TICKS = 6; // then probe once every 6 ticks (~60s at 10s cadence) +``` + +```ts + let consecutiveFailures = 0; + let ticksSkipped = 0; + async function tick() { + if (consecutiveFailures >= FAILURE_THRESHOLD) { + if (++ticksSkipped < BACKOFF_TICKS) return; + ticksSkipped = 0; + } + const entries = await probe(); + if (entries === null) { consecutiveFailures++; return; } + consecutiveFailures = 0; + // ...existing tick body... + } +``` + +- [ ] **Step 8: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/agent-status-poller.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean. + +- [ ] **Step 9: Commit** + +```bash +git add lib/runs/store.ts lib/daemon/agent-status-poller.ts lib/runs/__tests__/store-memo.test.ts lib/daemon/__tests__/agent-status-poller.test.ts +git commit -m "runs: mtime-memoize finished-run summaries; back off herdr probe (S101, S038, S039)" +``` + +--- + +### Task 12: Gate background scans on demand (S058, S093) + +The 10s system-process scan and 30s port scan run at full cadence with zero consumers (idle tray, battery). Gate them on a recent consumer read (tray/CLI/REST hit of `ports`/`system-processes`/`tray:status`). Wired via a new tracker read from `pollers.ts` and set by wrapping the demand commands in `command-router.ts` (no forbidden-file edit; verified reachable). + +**Files:** +- Create: `lib/daemon/demand-tracker.ts` +- Modify: `lib/daemon/command-router.ts` (wrap the demand-command entries) +- Modify: `lib/daemon/pollers.ts` (skip scans when no recent demand) +- Test: `lib/daemon/__tests__/demand-tracker.test.ts` (create) + +**Interfaces:** +- Produces: `recordDemand()`, `demandedWithin(ms) => boolean`, `wrapWithDemand(handlers, cmds) => handlers` in `demand-tracker.ts`. + +- [ ] **Step 1: Write the failing test** + +Create `lib/daemon/__tests__/demand-tracker.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { recordDemand, demandedWithin, wrapWithDemand } from "../demand-tracker.ts"; + +test("demandedWithin reflects a recent recordDemand", () => { + recordDemand(); + expect(demandedWithin(60_000)).toBe(true); + expect(demandedWithin(0)).toBe(false); // window of 0ms is never "recent" +}); + +test("wrapWithDemand records demand and delegates to the inner handler", async () => { + let called = false; + const handlers = { "system-processes": async () => { called = true; return { ok: true }; }, other: async () => ({ ok: true }) }; + wrapWithDemand(handlers, ["system-processes"]); + const before = demandedWithin(50); + await handlers["system-processes"](undefined as any); + expect(called).toBe(true); + expect(demandedWithin(1000)).toBe(true); + void before; +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/demand-tracker.test.ts` +Expected: FAIL ... module does not exist. + +- [ ] **Step 3: Create the tracker** + +Create `lib/daemon/demand-tracker.ts`: + +```ts +/** + * "A consumer is watching" signal for the background scans. The tray/CLI/console + * calling ports/system-processes/tray:status stamps demand here (via the + * command-router wrapper); pollers skip the 10s/30s scans when nothing has asked + * recently, so an idle machine stops paying the lsof/git tax (S058, S093). + */ +let lastDemandAt = 0; + +export function recordDemand(): void { + lastDemandAt = Date.now(); +} + +/** True when a consumer read a scan-backed command within `ms`. */ +export function demandedWithin(ms: number): boolean { + return lastDemandAt !== 0 && Date.now() - lastDemandAt < ms; +} + +/** Wrap the named handler entries so each call stamps demand, then delegates. */ +export function wrapWithDemand>(handlers: T, cmds: string[]): T { + for (const cmd of cmds) { + const inner = handlers[cmd]; + if (typeof inner !== "function") continue; + (handlers as any)[cmd] = (...args: any[]) => { recordDemand(); return inner(...args); }; + } + return handlers; +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/demand-tracker.test.ts` +Expected: PASS. + +- [ ] **Step 5: Wrap the demand commands in the router** + +In `lib/daemon/command-router.ts`: `import { wrapWithDemand } from "./demand-tracker.ts";`. Change the trailing `return { ...handlers... }` of `buildRoutedHandlers` to assign then wrap: + +```ts + const handlers: TypedHandlers & HandlerMap = { + ...createCacheHandlers(ctx), + // ...unchanged spreads... + "freshness:reconcile": async () => { + await reconcileFreshness({ ctx, broadcast }); + return { ok: true, data: getFreshnessSnapshot() }; + }, + }; + // A tray/CLI/console read of any scan-backed command means "someone is + // watching", which un-gates the background scans (see pollers.ts, S058/S093). + return wrapWithDemand(handlers, ["ports", "system-processes", "tray:status"]); +``` + +- [ ] **Step 6: Gate the scans in pollers** + +In `lib/daemon/pollers.ts`: `import { demandedWithin } from "./demand-tracker.ts";` and add: + +```ts +/** Consider a consumer "present" for 5 min after its last scan-backed read. */ +const DEMAND_WINDOW_MS = 5 * 60 * 1000; +``` + +Add the gate as the first line inside both scan bodies (after the in-flight guard): + +```ts + async function refreshPortCache(): Promise { + if (portScanInFlight) return; + if (!demandedWithin(DEMAND_WINDOW_MS)) return; // no consumer asked recently + portScanInFlight = true; + // ...unchanged... + } + + async function refreshSystemProcesses(): Promise { + if (processScanInFlight) return; + if (!demandedWithin(DEMAND_WINDOW_MS)) return; + processScanInFlight = true; + // ...unchanged... + } +``` + +(The tray's on-demand `system-processes` handler already calls `scanner.refresh()` when its cache is stale, so a freshly-connecting consumer gets immediate data and its read stamps demand, resuming the poller on the next tick. The hooks-scan and cache-refresh intervals are unaffected.) + +- [ ] **Step 7: Run tests + typecheck** + +Run: `bun test lib/daemon/__tests__/demand-tracker.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean. + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon/demand-tracker.ts lib/daemon/command-router.ts lib/daemon/pollers.ts lib/daemon/__tests__/demand-tracker.test.ts +git commit -m "pollers: gate the 10s/30s scans on recent consumer demand (S058, S093)" +``` + +--- + +### Final verification (run before the whole-branch review) + +- [ ] `bun run build` inside `packages/rt-client` (rt-client's `runCapture` mirror changed in Task 1). +- [ ] `bun test lib commands packages scripts` ... all green. +- [ ] `bunx tsc --noEmit` ... zero errors. +- [ ] `bun test lib/__tests__/no-daemon-sync-exec.test.ts` ... green; capture the Task 7 Step 3 RED demonstration for the report. +- [ ] Confirm no sibling-owned file in the write fence was modified (`git diff --name-only main...` reviewed against the fence). From b7694dd0b70d5bde7530307dd7f60173701a3752 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:02:31 -0500 Subject: [PATCH 024/106] daemon: drop startDaemon's dead-code catch and its now-stale JSDoc Review fix (round 1): runDaemon() now catches and exit(1)s internally on boot failure, so startDaemon()'s outer try/catch never runs, and its JSDoc claiming otherwise was misleading. Replaced with a one-line comment stating the real constraint. --- lib/daemon.ts | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/lib/daemon.ts b/lib/daemon.ts index fcfe6ca5..d06c8a51 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -530,25 +530,10 @@ async function runDaemon(): Promise { } } -/** - * Both real callers (cli.ts's `--daemon` entry, and this file's own - * import.meta.main guard below) invoke this fire-and-forget — neither awaits - * or catches. Left as a bare async function, ANY failure inside runDaemon() - * (this bind, openBranchCacheStore, anything else awaited) becomes an - * unhandledRejection, whose handler (installCrashHandlers, above) logs and - * deliberately does NOT exit — the daemon would stay alive with rt.sock - * possibly bound but startup never having reached signal handlers, pollers, - * or "daemon ready". Catching here and exiting explicitly gives every - * startup failure the same outcome a synchronous one already had: the - * process dies and whatever restarts a crashed daemon restarts this one too. - */ +// runDaemon() never rejects — it logs fatal and exit(1)s internally on any +// boot failure, so this wrapper needs no catch of its own. export async function startDaemon(): Promise { - try { - await runDaemon(); - } catch (err) { - log.fatal({ err }, "daemon startup failed — exiting"); - process.exit(1); - } + await runDaemon(); } // Auto-run when executed directly (source mode: bun run lib/daemon.ts) From b24bb2b1d057cc4713c32ce7404a26ced6edb59d Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:04:05 -0500 Subject: [PATCH 025/106] worktree create: serialize createTree per repoPath so provision and replenish don't race git ref locks (S089) --- .../__tests__/worktree-reconciler.test.ts | 54 ++++++++++++++++++- lib/daemon/handlers/worktree.ts | 12 +++-- lib/daemon/worktree-reconciler.ts | 26 ++++++++- 3 files changed, 86 insertions(+), 6 deletions(-) diff --git a/lib/daemon/__tests__/worktree-reconciler.test.ts b/lib/daemon/__tests__/worktree-reconciler.test.ts index 8d2bca47..30133a8c 100644 --- a/lib/daemon/__tests__/worktree-reconciler.test.ts +++ b/lib/daemon/__tests__/worktree-reconciler.test.ts @@ -19,7 +19,7 @@ import { import { createTree } from "../../worktree/create.ts"; import type { WorktreeAppConfig } from "../../worktree/config.ts"; import { RETENTION_MS } from "../../worktree/trash.ts"; -import { reconcileRepoRegistry, createWorktreeReconciler, __test__ } from "../worktree-reconciler.ts"; +import { reconcileRepoRegistry, createWorktreeReconciler, withCreateLock, __test__ } from "../worktree-reconciler.ts"; function makeRepo(): string { // realpathSync: git canonicalizes /var -> /private/var on macOS (Global Constraints) @@ -1400,3 +1400,55 @@ describe("reapRepoTrash", () => { expect(warns.some((w) => JSON.stringify(w).includes(parent))).toBe(true); }); }); + +// S089: a provision's cold createTree and the reconciler's own replenish +// createTree can run concurrently for the same repo, both `git fetch origin +// ` against the same repoPath — the loser fails to lock +// refs/remotes/origin/, and that failure gets charged to +// createBackoff (a 5-to-30-minute replenish hold) for what was really just +// contention, not a genuine failure. Serializing createTree per repoPath +// closes the race at its root. +describe("withCreateLock", () => { + test("serializes concurrent calls for the same repoPath — never two holders at once", async () => { + const order: string[] = []; + let active = 0; + let maxActive = 0; + const run = (id: string) => withCreateLock("/repo/a", async () => { + active++; + maxActive = Math.max(maxActive, active); + order.push(`start-${id}`); + await new Promise((r) => setTimeout(r, 10)); + order.push(`end-${id}`); + active--; + }); + await Promise.all([run("1"), run("2"), run("3")]); + expect(maxActive).toBe(1); + expect(order).toEqual(["start-1", "end-1", "start-2", "end-2", "start-3", "end-3"]); + }); + + test("different repoPaths are not serialized against each other", async () => { + let active = 0; + let maxActive = 0; + const run = (path: string) => withCreateLock(path, async () => { + active++; + maxActive = Math.max(maxActive, active); + await new Promise((r) => setTimeout(r, 10)); + active--; + }); + await Promise.all([run("/repo/b"), run("/repo/c")]); + expect(maxActive).toBe(2); + }); + + test("a holder that throws still releases the lock for the next caller", async () => { + await expect(withCreateLock("/repo/d", async () => { throw new Error("boom"); })).rejects.toThrow("boom"); + let ran = false; + await withCreateLock("/repo/d", async () => { ran = true; }); + expect(ran).toBe(true); + }); +}); + +test("both cold-create call sites in handlers/worktree.ts serialize createTree through the shared per-repo lock (S089)", () => { + const source = readFileSync(new URL("../handlers/worktree.ts", import.meta.url), "utf8"); + const matches = source.match(/withCreateLock\(/g) ?? []; + expect(matches.length).toBeGreaterThanOrEqual(2); +}); diff --git a/lib/daemon/handlers/worktree.ts b/lib/daemon/handlers/worktree.ts index b65f672a..bb98e53e 100644 --- a/lib/daemon/handlers/worktree.ts +++ b/lib/daemon/handlers/worktree.ts @@ -57,7 +57,7 @@ import { resolveReadySteps, } from "../../worktree/config.ts"; import { changedSince, runReadySteps, stepsToRun } from "../../worktree/ready.ts"; -import { freshenRepo, reconcileRepoRegistry } from "../worktree-reconciler.ts"; +import { freshenRepo, reconcileRepoRegistry, withCreateLock } from "../worktree-reconciler.ts"; import { repoDataDir, rtDir } from "../../rt-paths.ts"; const PROVISION_FETCH_TIMEOUT_MS = 5 * 60_000; @@ -313,9 +313,13 @@ export function createWorktreeHandlers( } } if (!rec) { - const created = await createTree({ + // Serialized against the reconciler's own replenish createTree for + // this repo (S089): both `git fetch origin ` against the + // same repoPath, and an unserialized race charges the loser's + // ref-lock failure to createBackoff for what was just contention. + const created = await withCreateLock(repoPath, () => createTree({ repoName, repoPath, emit: opts.emit, log: ctx.log, - }); + })); if (!created.ok) { if (created.error === "busy") return { ok: false, error: "busy" }; return { @@ -467,7 +471,7 @@ export function createWorktreeHandlers( const repoPath = repoName ? ctx.repoIndex()[repoName] : undefined; if (!repoName || !repoPath || parseIdentity(repoName) === null) return { ok: false, error: "repo-unknown" }; - const created = await createTree({ repoName, repoPath, emit: opts.emit, log: ctx.log }); + const created = await withCreateLock(repoPath, () => createTree({ repoName, repoPath, emit: opts.emit, log: ctx.log })); if (!created.ok) { if (created.error === "busy") return { ok: false, error: "busy" }; return { ok: false, error: createFailedError(created) }; diff --git a/lib/daemon/worktree-reconciler.ts b/lib/daemon/worktree-reconciler.ts index f2e20065..d7af150d 100644 --- a/lib/daemon/worktree-reconciler.ts +++ b/lib/daemon/worktree-reconciler.ts @@ -851,6 +851,30 @@ export async function freshenRepo( */ const createBackoff = new Map(); +/** + * S089: a provision's cold `createTree` (handlers/worktree.ts) and this + * reconciler's own replenish `createTree` can run concurrently for the same + * repo, both `git fetch origin ` against the same repoPath — the + * loser fails to lock refs/remotes/origin/, and that failure gets + * charged to createBackoff (a 5-to-30-minute replenish hold) for what was + * really just contention, not a genuine failure. Chained per repoPath so + * concurrent callers queue instead of racing; a rejected holder still + * releases the lock for the next one. + */ +const createLocks = new Map>(); + +export function withCreateLock(repoPath: string, fn: () => Promise): Promise { + const prior = createLocks.get(repoPath) ?? Promise.resolve(); + const ready = prior.catch(() => {}); // a previous holder's rejection must not block the next one + const result = ready.then(fn); + const tracked: Promise = result.then(() => undefined, () => undefined); + createLocks.set(repoPath, tracked); + void tracked.finally(() => { + if (createLocks.get(repoPath) === tracked) createLocks.delete(repoPath); + }); + return result; +} + /** The active backoff deadline for a repo, or null when creates may run now. */ function createBlockedUntil(repoName: string): string | null { const entry = createBackoff.get(repoName); @@ -920,7 +944,7 @@ async function replenishAndShrink( break; } budget--; - const p: Promise = createTree({ repoName, repoPath, emit, log }) + const p: Promise = withCreateLock(repoPath, () => createTree({ repoName, repoPath, emit, log })) .then((result) => { if (result.ok) { createBackoff.delete(repoName); From 8da0e048f4cf8e89cdfd800d39cee19492ee88cf Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:04:45 -0500 Subject: [PATCH 026/106] daemon: rt.trustedBrowserOrigins allowlist + needsToken invert-default (S005/S006/S040/S041/S084) --- lib/daemon/__tests__/api-auth.test.ts | 68 +++++++++++++++++++ lib/daemon/api-auth.ts | 67 +++++++++++++++--- .../rt-client/src/settings/registry-defs.ts | 8 +++ 3 files changed, 133 insertions(+), 10 deletions(-) diff --git a/lib/daemon/__tests__/api-auth.test.ts b/lib/daemon/__tests__/api-auth.test.ts index ad97eef9..8f8e102e 100644 --- a/lib/daemon/__tests__/api-auth.test.ts +++ b/lib/daemon/__tests__/api-auth.test.ts @@ -5,6 +5,7 @@ import { describe, test, expect } from "bun:test"; import { needsToken, tokenOk, getApiToken, reloadApiToken, loadOrCreateApiToken } from "../api-auth.ts"; +import { isOriginAllowed, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "../api-auth.ts"; import { mkdtempSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; @@ -42,6 +43,30 @@ describe("needsToken", () => { test("secrets requires a token even though it's a GET — the response body is a credential, not metadata", () => { expect(needsToken("GET", "/api/secrets")).toBe(true); }); + + test("refresh requires a token now (S040)", () => { + expect(needsToken("POST", "/api/refresh")).toBe(true); + }); + + test("hooks repair requires a token now (S040/S084)", () => { + expect(needsToken("POST", "/api/hooks/my-repo/repair")).toBe(true); + }); + + test("notifications GET (destructive drain) requires a token now (S041)", () => { + expect(needsToken("GET", "/api/notifications")).toBe(true); + }); + + test("every non-GET/HEAD/OPTIONS method defaults to requiring a token", () => { + expect(needsToken("POST", "/api/some-future-mutating-route")).toBe(true); + expect(needsToken("PUT", "/api/anything")).toBe(true); + expect(needsToken("DELETE", "/api/anything")).toBe(true); + }); + + test("plain reads still do not require a token", () => { + expect(needsToken("GET", "/api/repos")).toBe(false); + expect(needsToken("GET", "/api/cache")).toBe(false); + expect(needsToken("HEAD", "/api/repos")).toBe(false); + }); }); describe("tokenOk", () => { @@ -104,3 +129,46 @@ describe("getApiToken / reloadApiToken singleton", () => { } }); }); + +describe("isOriginAllowed", () => { + test("exact match", () => { + expect(isOriginAllowed("http://localhost:5544", ["http://localhost:5544"])).toBe(true); + }); + test("no match", () => { + expect(isOriginAllowed("http://evil.example", ["http://localhost:5544"])).toBe(false); + }); + test("empty allowlist matches nothing", () => { + expect(isOriginAllowed("http://localhost:5544", [])).toBe(false); + }); +}); + +describe("isBrowserRequestTrusted", () => { + const apiToken = "the-real-token"; + + test("no Origin header at all -- a non-browser client -- is trusted regardless of token or allowlist", () => { + expect(isBrowserRequestTrusted(null, null, apiToken, [])).toBe(true); + }); + + test("a browser Origin with the correct token is trusted even off the allowlist", () => { + expect(isBrowserRequestTrusted("http://evil.example", apiToken, apiToken, [])).toBe(true); + }); + + test("a browser Origin with a wrong token and not on the allowlist is rejected", () => { + expect(isBrowserRequestTrusted("http://evil.example", "wrong", apiToken, [])).toBe(false); + }); + + test("a browser Origin with no token but on the allowlist is trusted", () => { + expect(isBrowserRequestTrusted("http://localhost:5544", null, apiToken, ["http://localhost:5544"])).toBe(true); + }); + + test("a browser Origin with no token and not on the allowlist is rejected", () => { + expect(isBrowserRequestTrusted("http://localhost:5544", null, apiToken, [])).toBe(false); + }); +}); + +describe("getTrustedBrowserOrigins", () => { + test("returns an array (empty by default in an isolated test HOME)", () => { + const origins = getTrustedBrowserOrigins(); + expect(Array.isArray(origins)).toBe(true); + }); +}); diff --git a/lib/daemon/api-auth.ts b/lib/daemon/api-auth.ts index 89d07980..acc67076 100644 --- a/lib/daemon/api-auth.ts +++ b/lib/daemon/api-auth.ts @@ -13,6 +13,7 @@ import { join } from "path"; import { randomUUID } from "crypto"; import { RT_DIR } from "../daemon-config.ts"; import { lazyChildLogger } from "../daemon-logger.ts"; +import { getSetting } from "../settings/resolve.ts"; const log = lazyChildLogger("api-auth"); @@ -66,20 +67,66 @@ export function reloadApiToken(tokenPath: string = API_TOKEN_PATH): string { return cachedApiToken; } -/** True when a request mutates state, or (secrets) returns raw credential values, and must present the local token. */ +/** + * True when a request mutates state, or (secrets/notifications) returns or + * drains something a GET should not silently consume, and must present the + * local token. Default-gated for every method except GET/HEAD/OPTIONS (S040: + * an allowlist-by-path guaranteed the next mutating route would ship + * unguarded) plus two explicit GET exceptions whose verb lies about being a + * read. + */ export function needsToken(method: string, pathname: string): boolean { - if (method === "OPTIONS") return false; - if (pathname === "/api/shutdown") return true; - if (pathname === "/api/sdm/reconnect") return true; - if (pathname === "/api/events/emit") return true; - // Gated despite being a GET: every other read-only route returns metadata - // (branch names, MR titles, ports) safe under the open-CORS "reads are - // free" policy above; this one's response body IS the credential. - if (pathname === "/api/secrets") return true; - return false; + if (method === "GET" || method === "HEAD" || method === "OPTIONS") { + // Gated despite being a GET: /api/secrets's response body IS a + // credential (S054); /api/notifications DRAINS the queue (S041), so its + // verb lies about being a read the way every other GET here is not. + if (pathname === "/api/secrets") return true; + if (pathname === "/api/notifications") return true; + return false; + } + return true; } /** True when the presented token matches the configured one (and one exists). */ export function tokenOk(provided: string | null, expected: string): boolean { return expected.length > 0 && provided === expected; } + +/** + * `rt.trustedBrowserOrigins` -- see registry-defs.ts. Read fresh on every + * call (the settings resolver is deliberately unmemoized), wrapped in a + * try/catch since a request-path settings read must never 500 the daemon + * over a malformed store file. + */ +export function getTrustedBrowserOrigins(): readonly string[] { + try { + const resolved = getSetting("rt.trustedBrowserOrigins"); + return Array.isArray(resolved.value) ? resolved.value : []; + } catch { + return []; + } +} + +export function isOriginAllowed(origin: string, allowedOrigins: readonly string[]): boolean { + return allowedOrigins.includes(origin); +} + +/** + * The 127.0.0.1 trust boundary (S005/S006): the daemon binds loopback-only, + * but any web page the user visits also runs on 127.0.0.1 and can send a + * request. A request with NO Origin header at all is not a browser fetch -- + * it is the CLI, the Swift tray, rt-client from a Bun/Node process, or the + * VS Code extension, none of which send one -- so it is trusted unchanged. + * A request that DOES carry an Origin header is trusted only if it presents + * the local api-token or its Origin is on the explicit allowlist. + */ +export function isBrowserRequestTrusted( + origin: string | null, + token: string | null, + apiToken: string, + allowedOrigins: readonly string[], +): boolean { + if (!origin) return true; + if (tokenOk(token, apiToken)) return true; + return isOriginAllowed(origin, allowedOrigins); +} diff --git a/packages/rt-client/src/settings/registry-defs.ts b/packages/rt-client/src/settings/registry-defs.ts index f0109ddd..198a3d84 100644 --- a/packages/rt-client/src/settings/registry-defs.ts +++ b/packages/rt-client/src/settings/registry-defs.ts @@ -205,6 +205,14 @@ export const REGISTRY: readonly SettingDef[] = [ migrated: true, description: "Per-repo git hook enable/disable state ({enabled, hooks: {: boolean}}); ownership-latch port of repos//hooks.json, store wins per field once it owns the key — including per-hook-name entries inside the nested hooks map, each defaulting to enabled when absent. The installed git-hook shim still greps repos//hooks.json with zero process spawns (a hook fires on every git operation); that file is now a DERIVED CACHE this key writes through, kept current by commands/hooks.ts's regenerateHooksCache at every write seam.", }, + { + key: "rt.trustedBrowserOrigins", + type: "array", + scopes: ["user", "machine"], + default: [], + merge: "replace", + description: "Browser Origins (scheme://host:port, exact string match) trusted to read the :9401 daemon API and subscribe to /ws without presenting the local api-token -- e.g. a locally-hosted console or chat-viewer dev server. Empty by default: every current mattstack consumer (the CLI, the Swift tray, rt-client from Bun/Node processes, the VS Code extension) is a non-browser client (sends no Origin header at all) and is unaffected either way.", + }, // --- mattstack (installer-lane) ----------------------------------------- { From 40e1e35bf074b4b6f5726f1e625465315afe9237 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:05:30 -0500 Subject: [PATCH 027/106] herdr client: accumulate raw socket bytes and decode once instead of per-chunk, so a split multibyte char survives (S095) --- lib/herdr/__tests__/client.test.ts | 30 ++++++++++++++++++++++++++++++ lib/herdr/client.ts | 10 ++++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/lib/herdr/__tests__/client.test.ts b/lib/herdr/__tests__/client.test.ts index 431b7132..952601ab 100644 --- a/lib/herdr/__tests__/client.test.ts +++ b/lib/herdr/__tests__/client.test.ts @@ -66,3 +66,33 @@ test("herdrAvailable probes session.snapshot", async () => { test("waitTimeout adds the 5s margin herdr needs to answer at its own budget", () => { expect(waitTimeout(60_000)).toBe(65_000); }); + +// S095: herdrRequest decoded each socket chunk independently — a multibyte +// character (box-drawing, emoji, ...) split across a chunk boundary became +// two partial byte sequences, each decoding to U+FFFD, corrupting the reply +// even though JSON.parse still succeeded on the well-formed-but-wrong text. +test("a multibyte character split across two socket chunks decodes correctly", async () => { + const sock = join(tmpdir(), `herdr-split-${process.pid}.sock`); + const text = "a─b"; // U+2500 BOX DRAWINGS LIGHT HORIZONTAL: E2 94 80, 3 bytes + const payload = JSON.stringify({ id: "x", result: { text } }) + "\n"; + const bytes = Buffer.from(payload, "utf8"); + const charStart = bytes.indexOf(Buffer.from("─", "utf8")); + const splitAt = charStart + 1; // split inside the multibyte sequence, after its first byte + + const server = Bun.listen({ + unix: sock, + socket: { + open(socket) { + socket.write(bytes.subarray(0, splitAt)); + setTimeout(() => socket.write(bytes.subarray(splitAt)), 20); + }, + data() {}, + close() {}, + error() {}, + }, + }); + stops.push(() => server.stop(true)); + + const res = await herdrRequest<{ text: string }>("whatever", {}, { sockPath: sock }); + expect(res).toEqual({ ok: true, result: { text: "a─b" } }); +}); diff --git a/lib/herdr/client.ts b/lib/herdr/client.ts index eb1dc373..c252b406 100644 --- a/lib/herdr/client.ts +++ b/lib/herdr/client.ts @@ -37,7 +37,12 @@ export function herdrRequest( return new Promise((resolve) => { let settled = false; - let buf = ""; + // A multibyte UTF-8 character (box-drawing, emoji, ...) can straddle a + // socket chunk boundary; decoding each chunk independently turns each + // half into its own U+FFFD replacement character, corrupting the reply + // (JSON.parse still succeeds — the text is just wrong). Accumulating raw + // bytes and decoding the whole buffer together avoids that entirely. + const chunks: Buffer[] = []; let conn: { end(): void } | undefined; const finish = (r: HerdrResult) => { if (settled) return; @@ -55,6 +60,7 @@ export function herdrRequest( } const tryParseBuffered = (): HerdrResult | undefined => { + const buf = Buffer.concat(chunks).toString("utf8"); const nl = buf.indexOf("\n"); if (nl < 0) return undefined; const text = buf.slice(0, nl); @@ -81,7 +87,7 @@ export function herdrRequest( socket.write(line); }, data(_socket, chunk) { - buf += chunk.toString(); + chunks.push(chunk); const parsed = tryParseBuffered(); if (parsed) finish(parsed); }, From d34dfbf4cf275097938fe7774707b4b74fe3a246 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:07:30 -0500 Subject: [PATCH 028/106] daemon: install stderr redirect + crash handlers before every module-scope side effect Hoists redirectNativeStderr() to the first executable statement and installCrashHandlers() to right after the logger resolves, both before createEventsBus, cron, sweep timers, and home-snapshot construction. A pre-startDaemon throw (e.g. a corrupt events.db) now lands in daemon-stderr.log instead of vanishing down a discarded fd 2. Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/tests/daemon.test.ts | 25 ++++++++++++++++++++++++- lib/daemon.ts | 19 +++++++++++++------ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index 37982958..5bccbdfe 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from "bun:test"; -import { existsSync } from "fs"; +import { existsSync, mkdirSync, writeFileSync, readdirSync } from "fs"; import { join } from "path"; import { createTestHome, rt } from "../harness.ts"; @@ -19,6 +19,29 @@ describe("fatal boot", () => { cleanup(); } }, 60_000); + + test("a corrupt events.db does not crash the daemon silently — error is captured", async () => { + const { path: home, cleanup } = createTestHome(); + try { + // Pre-create a corrupt events.db in the isolated HOME, before the + // daemon ever runs — createEventsBus (module scope) throws opening it. + const rtDir = join(home, ".mattstack", "rt"); + mkdirSync(rtDir, { recursive: true }); + writeFileSync(join(rtDir, "events.db"), "not a sqlite file at all"); + + const result = await rt(["--daemon"], { home }); + + expect(result.exitCode).not.toBe(0); + // Task 4 makes this self-heal (quarantine + retry); for Task 3 we only + // require the failure is captured, not that boot recovers. + const stderrLog = join(rtDir, "logs", "daemon-stderr.log"); + const quarantined = existsSync(rtDir) && readdirSync(rtDir).some((f) => f.startsWith("events.db.corrupt-")); + const captured = existsSync(stderrLog) || quarantined; + expect(captured).toBe(true); + } finally { + cleanup(); + } + }, 60_000); }); describe("daemon", () => { diff --git a/lib/daemon.ts b/lib/daemon.ts index d06c8a51..2dbd549c 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -76,6 +76,13 @@ import type { PortEntry } from "./port-scanner.ts"; // entry (cli.ts) also runs it, but `bun run lib/daemon.ts` skips cli.ts. import { migrateLegacyRtDir, LEGACY_RT_LABEL, RT_DIR_LABEL, logsDir } from "./rt-paths.ts"; +// Capture native panics (bypass JS entirely) at the fd level before anything +// else in this module runs, so a throw during any later module-scope +// construction (createEventsBus, cron, home-snapshot, …) lands in +// daemon-stderr.log instead of vanishing down whatever fd 2 the launcher gave +// us. Depends only on logsDir() and mkdirs its own dir; no logger needed yet. +redirectNativeStderr(); + // Gates installCrashHandlers' unhandledRejection handler: fatal during boot // (no socket/API bound yet, nothing to recover), advisory-only once ready. let bootPhase: "booting" | "ready" = "booting"; @@ -90,6 +97,12 @@ const rtMigration = migrateLegacyRtDir(); const loggerHandle = await getDaemonLogger(); const log = loggerHandle.logger; +// Wire uncaughtException + unhandledRejection through pino as early as the +// logger allows: every module-scope side effect below this point +// (createEventsBus, cron, home-snapshot, sweep timers) can throw, and this +// must run BEFORE any of it does. +installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" }); + if (rtMigration === "migrated") { log.info(`migrated legacy ${LEGACY_RT_LABEL} state to ${RT_DIR_LABEL}`); } else if (rtMigration === "conflict") { @@ -391,12 +404,6 @@ async function runDaemon(): Promise { try { mkdirSync(RT_DIR, { recursive: true }); - // Capture native panics (bypass JS entirely) at the fd level, then wire - // uncaughtException + unhandledRejection through pino. Must run BEFORE - // any async work that could throw uncaught. - redirectNativeStderr(); - installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" }); - // If a previous daemon process is still alive (orphan from a failed // restart), evict it before we bind the socket. evictStaleDaemon(log); From 43c5486c8bfa248a00b26ab8679de771bd89ea07 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:07:37 -0500 Subject: [PATCH 029/106] port-scanner: canonicalize repo/worktree paths once per scan so a symlinked or case-variant root matches lsof (S097) --- lib/__tests__/port-scanner.test.ts | 64 ++++++++++++++++++++++++++++++ lib/port-scanner.ts | 47 +++++++++++++++++++--- 2 files changed, 106 insertions(+), 5 deletions(-) diff --git a/lib/__tests__/port-scanner.test.ts b/lib/__tests__/port-scanner.test.ts index 750c574b..033c0ada 100644 --- a/lib/__tests__/port-scanner.test.ts +++ b/lib/__tests__/port-scanner.test.ts @@ -1,10 +1,16 @@ import { describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; import { parseListeningLsof, parsePidValueMap, parseCwdMap, parseWorktreePorcelain, parseEtimeMs, + matchCwdToRepo, + canonicalizeRepoIndex, + canonicalizeWorktreeMap, } from "../port-scanner.ts"; describe("parseListeningLsof", () => { @@ -120,3 +126,61 @@ describe("parseEtimeMs", () => { expect(parseEtimeMs("")).toBeNull(); }); }); + +// S097: lsof reports the kernel's resolved (real) cwd; the repo index and +// worktree map carry whatever path the user cd'd through when registering +// a repo. A symlinked repo root (~/code -> /Volumes/Dev/code) previously +// matched nothing at all — no ports, no runaway detection, and stale dev +// servers kept running past dispose because killWorktreeProcesses found +// nothing either. +describe("canonicalizeRepoIndex / canonicalizeWorktreeMap (S097)", () => { + let dir: string; + let real: string; + let link: string; + + const setup = () => { + dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-port-scanner-"))); + real = join(dir, "real-repo"); + link = join(dir, "linked-repo"); + mkdirSync(real, { recursive: true }); + symlinkSync(real, link); + }; + const teardown = () => { rmSync(dir, { recursive: true, force: true }); }; + + test("a symlinked repo root resolves to the real path lsof would report", () => { + setup(); + try { + const repos = canonicalizeRepoIndex({ acme: link }); + expect(repos.acme).toBe(realpathSync(real)); + } finally { + teardown(); + } + }); + + test("a nonexistent path (deleted mid-scan) falls back to the literal rather than throwing", () => { + const repos = canonicalizeRepoIndex({ acme: "/does/not/exist/anywhere" }); + expect(repos.acme).toBe("/does/not/exist/anywhere"); + }); + + test("worktree map keys are canonicalized the same way", () => { + setup(); + try { + const wt = canonicalizeWorktreeMap(new Map([[link, { repo: "acme", branch: "main" }]])); + expect([...wt.keys()]).toEqual([realpathSync(real)]); + } finally { + teardown(); + } + }); + + test("end-to-end: matchCwdToRepo finds a repo registered under a symlinked root once canonicalized", () => { + setup(); + try { + const cwd = realpathSync(real); // what lsof would report + const repos = canonicalizeRepoIndex({ acme: link }); // what the index has + const match = matchCwdToRepo(cwd, repos, new Map()); + expect(match.repo).toBe("acme"); + } finally { + teardown(); + } + }); +}); diff --git a/lib/port-scanner.ts b/lib/port-scanner.ts index c437b620..8ff5b4d9 100644 --- a/lib/port-scanner.ts +++ b/lib/port-scanner.ts @@ -10,7 +10,7 @@ * would freeze the event loop long enough to time out status polls. */ -import { existsSync } from "fs"; +import { existsSync, realpathSync } from "fs"; import { homedir } from "os"; import { loadRepoIndex as loadRepoIndexFromStore } from "./repo-index.ts"; import { runCapture } from "./subprocess.ts"; @@ -167,6 +167,39 @@ export async function buildWorktreeMap( return new Map(perRepo.flat()); } +/** + * lsof reports the kernel's resolved (real, canonically-cased) path for a + * process's cwd. The repo index and worktree map instead carry whatever + * path the user cd'd through when the repo was registered (behind a + * symlink, under a case-insensitive APFS volume's non-canonical spelling, + * ...), so a raw string comparison against lsof's cwd can miss even a repo + * that is genuinely running. Falls back to the literal path when it does + * not (yet) exist on disk — mirrors lib/runs/prune.ts's canon(). + */ +function canon(path: string): string { + try { + return realpathSync(path); + } catch { + return path; + } +} + +/** Canonicalizes every path in the repo index once per scan (not once per matched port). */ +export function canonicalizeRepoIndex(repos: Record): Record { + const out: Record = {}; + for (const [name, path] of Object.entries(repos)) out[name] = canon(path); + return out; +} + +/** Canonicalizes every worktree path's key once per scan (not once per matched port). */ +export function canonicalizeWorktreeMap( + worktreeMap: Map, +): Map { + const out = new Map(); + for (const [path, info] of worktreeMap) out.set(canon(path), info); + return out; +} + export function matchCwdToRepo( cwd: string, repos: Record, @@ -215,13 +248,17 @@ export function matchCwdToRepo( * ports whose process CWD matches a known repo. */ export async function scanListeningPorts(): Promise { - const repos = loadRepoIndex(); - if (Object.keys(repos).length === 0) return []; + const rawRepos = loadRepoIndex(); + if (Object.keys(rawRepos).length === 0) return []; - const [worktreeMap, listenersRes] = await Promise.all([ - buildWorktreeMap(repos), + const [rawWorktreeMap, listenersRes] = await Promise.all([ + buildWorktreeMap(rawRepos), runCapture(["lsof", "-iTCP", "-sTCP:LISTEN", "-P", "-n"], { timeoutMs: 10_000 }), ]); + // Once per scan, not once per matched port: lsof's cwd is already real, + // so the index/worktree side is what needs canonicalizing to match it. + const repos = canonicalizeRepoIndex(rawRepos); + const worktreeMap = canonicalizeWorktreeMap(rawWorktreeMap); const listeners = parseListeningLsof(listenersRes.stdout); if (listeners.length === 0) return []; From ff751ee5ac657fecf8a4b54d42cda9543951504e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:09:38 -0500 Subject: [PATCH 030/106] rt-paths: only migrate a legacy ~/.rt that carries an actual rt signature (S099) --- lib/__tests__/rt-paths.test.ts | 48 ++++++++++++++++++++++++++++++++++ lib/rt-paths.ts | 24 ++++++++++++++--- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/lib/__tests__/rt-paths.test.ts b/lib/__tests__/rt-paths.test.ts index 50cf38a6..665eafdb 100644 --- a/lib/__tests__/rt-paths.test.ts +++ b/lib/__tests__/rt-paths.test.ts @@ -228,6 +228,7 @@ describe("rt-paths", () => { const home = makeHome(); process.env.HOME = home; mkdirSync(join(home, ".rt"), { recursive: true }); + writeFileSync(join(home, ".rt", "repos.json"), "{}"); // rt signature marker (S099) writeFileSync(join(home, ".rt", "old.json"), "old"); mkdirSync(join(home, ".mattstack", "rt"), { recursive: true }); writeFileSync(join(home, ".mattstack", "rt", "new.json"), "new"); @@ -241,11 +242,58 @@ describe("rt-paths", () => { const home = makeHome(); process.env.HOME = home; mkdirSync(join(home, ".rt"), { recursive: true }); + writeFileSync(join(home, ".rt", "repos.json"), "{}"); // rt signature marker (S099) expect(migrateLegacyRtDir()).toBe("migrated"); expect(migrateLegacyRtDir()).toBe("none"); rmSync(home, { recursive: true, force: true }); }); + // S099: a new user who has never run rt, but has another tool that also + // uses ~/.rt as its config dir, must not have that directory silently + // annexed as rt state (parsed, quarantined, or renamed) — only a + // directory carrying an actual rt signature is ever touched. + test("migrate: a foreign ~/.rt with no rt signature is left alone, no ~/.mattstack/rt materializes (S099)", () => { + const home = makeHome(); + process.env.HOME = home; + mkdirSync(join(home, ".rt"), { recursive: true }); + writeFileSync(join(home, ".rt", "config.toml"), "some-other-tools-config"); + const result = migrateLegacyRtDir(); + expect(result).not.toBe("migrated"); + expect(result).not.toBe("conflict"); + expect(readFileSync(join(home, ".rt", "config.toml"), "utf8")).toBe("some-other-tools-config"); + expect(existsSync(join(home, ".mattstack", "rt"))).toBe(false); + rmSync(home, { recursive: true, force: true }); + }); + + test("migrate: a foreign ~/.rt is left alone even when ~/.mattstack/rt already exists — not reported as a conflict (S099)", () => { + const home = makeHome(); + process.env.HOME = home; + mkdirSync(join(home, ".rt"), { recursive: true }); + writeFileSync(join(home, ".rt", "config.toml"), "some-other-tools-config"); + mkdirSync(join(home, ".mattstack", "rt"), { recursive: true }); + const result = migrateLegacyRtDir(); + expect(result).not.toBe("conflict"); + expect(readFileSync(join(home, ".rt", "config.toml"), "utf8")).toBe("some-other-tools-config"); + rmSync(home, { recursive: true, force: true }); + }); + + test("migrate: a bare 'logs' dir alone is a sufficient rt signature", () => { + const home = makeHome(); + process.env.HOME = home; + mkdirSync(join(home, ".rt", "logs"), { recursive: true }); + expect(migrateLegacyRtDir()).toBe("migrated"); + rmSync(home, { recursive: true, force: true }); + }); + + test("migrate: a bare 'state.db' file alone is a sufficient rt signature", () => { + const home = makeHome(); + process.env.HOME = home; + mkdirSync(join(home, ".rt"), { recursive: true }); + writeFileSync(join(home, ".rt", "state.db"), ""); + expect(migrateLegacyRtDir()).toBe("migrated"); + rmSync(home, { recursive: true, force: true }); + }); + // ── legacyDirsPresent (the canary probe) ──────────────────────────────────── test("canary: reports real legacy dirs, ignores symlinks, skips absent", () => { diff --git a/lib/rt-paths.ts b/lib/rt-paths.ts index 3fc07fa5..ce92b121 100644 --- a/lib/rt-paths.ts +++ b/lib/rt-paths.ts @@ -280,6 +280,20 @@ export type LegacyMigrationResult = "none" | "migrated" | "conflict"; export const LEGACY_RT_LABEL = "~/.rt"; export const RT_DIR_LABEL = "~/.mattstack/rt"; +/** + * Entries whose presence at the top of ~/.rt proves rt itself wrote it — + * not merely that something happens to live at that path. A new user who + * has never run an old rt but has an unrelated tool using ~/.rt as ITS + * config dir must never have that directory silently annexed as rt state + * (renamed, then parsed/quarantined as daemon.json/repos.json/state.db). + */ +const RT_SIGNATURE_ENTRIES = ["state.db", "logs", "repos.json"]; + +/** Whether `dir` carries an actual rt signature, not merely a directory of the same name. */ +function hasRtSignature(dir: string): boolean { + return RT_SIGNATURE_ENTRIES.some((name) => existsSync(join(dir, name))); +} + /** * One-shot migration of a real legacy ~/.rt directory to ~/.mattstack/rt. * Called early from the CLI entry and daemon boot — BEFORE anything (loggers @@ -287,9 +301,12 @@ export const RT_DIR_LABEL = "~/.mattstack/rt"; * ~/.rt would land in "conflict" instead of migrating. * * - ~/.rt absent, or a symlink (the RT-33 compat shim): "none", untouched. - * - real ~/.rt, no ~/.mattstack/rt: rename it into place → "migrated". - * - real ~/.rt AND ~/.mattstack/rt both exist: "conflict" — state is split - * and a human must merge; nothing is touched. + * - ~/.rt real but carries no rt signature (another tool's config dir of + * the same name): "none", untouched — never renamed, parsed, or reported + * as a conflict against ~/.mattstack/rt. + * - real ~/.rt (rt's), no ~/.mattstack/rt: rename it into place → "migrated". + * - real ~/.rt (rt's) AND ~/.mattstack/rt both exist: "conflict" — state is + * split and a human must merge; nothing is touched. */ export function migrateLegacyRtDir(): LegacyMigrationResult { const legacy = legacyRtDir(); @@ -300,6 +317,7 @@ export function migrateLegacyRtDir(): LegacyMigrationResult { return "none"; // no ~/.rt at all } if (legacyStat.isSymbolicLink() || !legacyStat.isDirectory()) return "none"; + if (!hasRtSignature(legacy)) return "none"; // real directory, but not rt's — never touch it const target = rtDir(); try { From 8e689b06f0c52b9f45f04720af75e6ec393e5fac Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:11:25 -0500 Subject: [PATCH 031/106] daemon: fix needsToken to never gate OPTIONS preflight (review fix) Splitting OPTIONS out of the GET/HEAD bucket restores the pre-existing guarantee that a CORS preflight is never gated, regardless of path, matching this file's own docblock and existing test coverage. --- lib/daemon/__tests__/api-auth.test.ts | 5 +++++ lib/daemon/api-auth.ts | 12 +++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/daemon/__tests__/api-auth.test.ts b/lib/daemon/__tests__/api-auth.test.ts index 8f8e102e..6abf9eab 100644 --- a/lib/daemon/__tests__/api-auth.test.ts +++ b/lib/daemon/__tests__/api-auth.test.ts @@ -67,6 +67,11 @@ describe("needsToken", () => { expect(needsToken("GET", "/api/cache")).toBe(false); expect(needsToken("HEAD", "/api/repos")).toBe(false); }); + + test("OPTIONS never requires a token even for secrets/notifications (preflight must never be gated)", () => { + expect(needsToken("OPTIONS", "/api/secrets")).toBe(false); + expect(needsToken("OPTIONS", "/api/notifications")).toBe(false); + }); }); describe("tokenOk", () => { diff --git a/lib/daemon/api-auth.ts b/lib/daemon/api-auth.ts index acc67076..dae00efa 100644 --- a/lib/daemon/api-auth.ts +++ b/lib/daemon/api-auth.ts @@ -70,13 +70,15 @@ export function reloadApiToken(tokenPath: string = API_TOKEN_PATH): string { /** * True when a request mutates state, or (secrets/notifications) returns or * drains something a GET should not silently consume, and must present the - * local token. Default-gated for every method except GET/HEAD/OPTIONS (S040: - * an allowlist-by-path guaranteed the next mutating route would ship - * unguarded) plus two explicit GET exceptions whose verb lies about being a - * read. + * local token. A CORS preflight (OPTIONS) can never present the custom + * X-RT-Token header, so it is never gated, on any path. Otherwise + * default-gated for every method except GET/HEAD (S040: an allowlist-by-path + * guaranteed the next mutating route would ship unguarded), plus two + * explicit GET exceptions whose verb lies about being a read. */ export function needsToken(method: string, pathname: string): boolean { - if (method === "GET" || method === "HEAD" || method === "OPTIONS") { + if (method === "OPTIONS") return false; + if (method === "GET" || method === "HEAD") { // Gated despite being a GET: /api/secrets's response body IS a // credential (S054); /api/notifications DRAINS the queue (S041), so its // verb lies about being a read the way every other GET here is not. From e8d0a7c87c93b7f16ec00fd86f72b2e7ef1469bd Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:15:15 -0500 Subject: [PATCH 032/106] daemon-client: return timed-out/refused attribution per query instead of shared module flags (S081) --- .../daemon-client-attribution.test.ts | 57 +++++++++ lib/daemon-client.ts | 119 +++++++++++------- 2 files changed, 134 insertions(+), 42 deletions(-) create mode 100644 lib/__tests__/daemon-client-attribution.test.ts diff --git a/lib/__tests__/daemon-client-attribution.test.ts b/lib/__tests__/daemon-client-attribution.test.ts new file mode 100644 index 00000000..91fc5558 --- /dev/null +++ b/lib/__tests__/daemon-client-attribution.test.ts @@ -0,0 +1,57 @@ +/** + * S081: daemon-client's timed-out/refused attribution used to live in + * module-level flags (`_lastQueryTimedOut`/`_lastQueryWasRefused`) shared + * across every concurrent query. A fast query resolving in between a slow + * query's own trySocketQuery call and the moment its caller reads + * lastQueryTimedOut() could reset those flags out from under it — the slow + * query's caller would then see "daemon unavailable" for a call that + * actually just exceeded its own window. daemonQueryAttributed returns the + * failure kind alongside the response instead, so it can never be + * clobbered by an unrelated concurrent call. + */ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { DAEMON_SOCK_PATH, markDaemonInstalled, markDaemonUninstalled } from "../daemon-config.ts"; +import { daemonQueryAttributed } from "../daemon-client.ts"; + +describe("daemonQueryAttributed", () => { + let server: ReturnType; + + beforeEach(() => { + markDaemonInstalled(); + server = Bun.serve({ + unix: DAEMON_SOCK_PATH, + async fetch(req) { + const url = new URL(req.url); + if (url.pathname === "/slow") { + await new Promise(() => {}); // never resolves — the client's own AbortSignal.timeout fires + } + return new Response(JSON.stringify({ ok: true, data: {} }), { headers: { "Content-Type": "application/json" } }); + }, + }); + }); + + afterEach(() => { + server.stop(true); + markDaemonUninstalled(); + }); + + test("a slow query that times out reports timedOut:true even resolved after a concurrent fast success", async () => { + const slow = daemonQueryAttributed("slow", {}, 50); + const fast = await daemonQueryAttributed("fast", {}); + expect(fast.response?.ok).toBe(true); + expect(fast.timedOut).toBe(false); + + const slowResult = await slow; + expect(slowResult.response).toBeNull(); + expect(slowResult.timedOut).toBe(true); + expect(slowResult.refused).toBe(false); + }); + + test("a fast query's own success is reported correctly even started after a slow one is already in flight", async () => { + const slow = daemonQueryAttributed("slow", {}, 200); + const fast = await daemonQueryAttributed("fast", {}); + expect(fast.response?.ok).toBe(true); + expect(fast.timedOut).toBe(false); + await slow; // drain — this one times out at 200ms, don't leave it dangling + }); +}); diff --git a/lib/daemon-client.ts b/lib/daemon-client.ts index 2a03a629..0fca23a8 100644 --- a/lib/daemon-client.ts +++ b/lib/daemon-client.ts @@ -30,19 +30,32 @@ export interface DaemonResponse { const REQUEST_TIMEOUT_MS = 2000; +// Deprecated shim state (see lastQueryTimedOut's doc): kept in sync so an +// external caller still reading it via the module-level accessor keeps +// working, but nothing in this file reads these two vars anymore — every +// internal caller carries its own attribution on its own return value +// instead, which a concurrent query can never clobber. let _lastQueryWasRefused = false; let _lastQueryTimedOut = false; +interface SocketQueryAttempt { + response: DaemonResponse | null; + /** True on ECONNREFUSED — the socket is stale, not merely slow. */ + refused: boolean; + /** True when the request itself exceeded timeoutMs. */ + timedOut: boolean; +} + async function trySocketQuery( cmd: string, payload?: Record, timeoutMs: number = REQUEST_TIMEOUT_MS, -): Promise { - // Reset per-query flags up front so a missing-socket early return doesn't - // leave a previous query's timeout/refused state visible to callers. - _lastQueryWasRefused = false; - _lastQueryTimedOut = false; - if (!existsSync(DAEMON_SOCK_PATH)) return null; +): Promise { + if (!existsSync(DAEMON_SOCK_PATH)) { + _lastQueryWasRefused = false; + _lastQueryTimedOut = false; + return { response: null, refused: false, timedOut: false }; + } try { const hasBody = payload && Object.keys(payload).length > 0; @@ -57,14 +70,16 @@ async function trySocketQuery( _lastQueryWasRefused = false; _lastQueryTimedOut = false; - return (await response.json()) as DaemonResponse; + return { response: (await response.json()) as DaemonResponse, refused: false, timedOut: false }; } catch (err) { const code = (err as any)?.code ?? ""; const name = (err as any)?.name ?? ""; const msg = err instanceof Error ? err.message : ""; - _lastQueryWasRefused = code === "ECONNREFUSED" || msg.includes("ECONNREFUSED") || msg.includes("Connection refused"); - _lastQueryTimedOut = name === "TimeoutError" || name === "AbortError" || msg.includes("timed out"); - return null; + const refused = code === "ECONNREFUSED" || msg.includes("ECONNREFUSED") || msg.includes("Connection refused"); + const timedOut = name === "TimeoutError" || name === "AbortError" || msg.includes("timed out"); + _lastQueryWasRefused = refused; + _lastQueryTimedOut = timedOut; + return { response: null, refused, timedOut }; } } @@ -102,7 +117,7 @@ export async function daemonSocketQuery( payload?: Record, timeoutMs?: number, ): Promise { - return trySocketQuery(cmd, payload, timeoutMs); + return (await trySocketQuery(cmd, payload, timeoutMs)).response; } // ─── Tray request client (MAT-383 setup verbs) ─────────────────────────────── @@ -205,48 +220,68 @@ export function suppressDaemonDownWarning(): void { // ─── Public API ────────────────────────────────────────────────────────────── /** - * Send a command to the daemon and return the response. - * - * Returns null if daemon is not available (either not installed or not running - * and can't be auto-restarted). Callers should fall back to direct execution. + * Same contract as `daemonQuery`, but the failure kind travels on the + * return value instead of the module-level `_lastQuery*` flags — so a + * concurrent query (a 30s mr:action merge racing a 2s status poll, say) + * can never clobber this call's own attribution between its trySocketQuery + * resolving and its caller reading it. Prefer this over + * `daemonQuery` + `lastQueryTimedOut()` for any new caller. */ -export async function daemonQuery( +export async function daemonQueryAttributed( cmd: string, payload?: Record, timeoutMs?: number, -): Promise { +): Promise { // 1. Try HTTP request over Unix socket - const result = await trySocketQuery(cmd, payload, timeoutMs); - if (result !== null) return result; + const first = await trySocketQuery(cmd, payload, timeoutMs); + if (first.response !== null) return first; // 2. Check if user opted in - if (!isDaemonInstalled()) return null; // not installed → silent fallback + if (!isDaemonInstalled()) return { response: null, refused: false, timedOut: false }; // not installed → silent fallback // 3. If the socket file still exists AND the connection wasn't refused, // the daemon IS running — this query timed out or hit a transient error. // Return null silently. But if the connection was refused, the socket is // stale (daemon died without cleaning up) — fall through to attempt restart. - if (existsSync(DAEMON_SOCK_PATH) && !_lastQueryWasRefused) return null; + if (existsSync(DAEMON_SOCK_PATH) && !first.refused) return first; // 4. Socket is gone → daemon is genuinely not running. Attempt restart. const restarted = await attemptRestart(); + let last = first; if (restarted) { // Retry once after short delay await Bun.sleep(300); - const retryResult = await trySocketQuery(cmd, payload, timeoutMs); - if (retryResult !== null) return retryResult; + const retry = await trySocketQuery(cmd, payload, timeoutMs); + if (retry.response !== null) return retry; + last = retry; } // 5. Restart failed → warn (once per session) warnDaemonDown(); - return null; + return last; +} + +/** + * Send a command to the daemon and return the response. + * + * Returns null if daemon is not available (either not installed or not running + * and can't be auto-restarted). Callers should fall back to direct execution. + */ +export async function daemonQuery( + cmd: string, + payload?: Record, + timeoutMs?: number, +): Promise { + return (await daemonQueryAttributed(cmd, payload, timeoutMs)).response; } /** * True if the last `daemonQuery` returned null because the request timed out - * (as opposed to the daemon being genuinely down). Used by action callers so - * they can surface "timed out — verify on GitLab" instead of the misleading - * "daemon unavailable". + * (as opposed to the daemon being genuinely down). Deprecated: still backed + * by shared module state, so it remains vulnerable to the exact + * cross-query attribution race `daemonQueryAttributed` closes — kept only + * for callers that predate that function. New callers should use + * `daemonQueryAttributed` and read `.timedOut` off their own result. */ export function lastQueryTimedOut(): boolean { return _lastQueryTimedOut; @@ -256,7 +291,7 @@ export function lastQueryTimedOut(): boolean { * Quick check: is the daemon reachable right now? */ export async function isDaemonRunning(): Promise { - const response = await trySocketQuery("ping"); + const { response } = await trySocketQuery("ping"); return response?.ok === true; } @@ -294,8 +329,8 @@ const MR_ACTION_TIMEOUT_MS = 30_000; export function mrActions(repoName: string, iid: number): DaemonMRActions { const fire = async (action: string, args: any[] = []): Promise => { - const res = await daemonQuery("mr:action", { repoName, iid, action, args }, MR_ACTION_TIMEOUT_MS); - if (!res) throw new Error(lastQueryTimedOut() ? `${action} timed out — verify on GitLab` : "daemon unavailable"); + const { response: res, timedOut } = await daemonQueryAttributed("mr:action", { repoName, iid, action, args }, MR_ACTION_TIMEOUT_MS); + if (!res) throw new Error(timedOut ? `${action} timed out — verify on GitLab` : "daemon unavailable"); if (!res.ok) throw new Error(res.error || `${action} failed`); }; @@ -312,14 +347,14 @@ export function mrActions(repoName: string, iid: number): DaemonMRActions { requestReReview: (uid) => fire("requestReReview", [uid]), fetchJobDetail: async (jobId, pipelineId) => { - const res = await daemonQuery("mr:fetch-job-detail", { repoName, iid, jobId, pipelineId }, MR_ACTION_TIMEOUT_MS); - if (!res) throw new Error(lastQueryTimedOut() ? "fetchJobDetail timed out" : "daemon unavailable"); + const { response: res, timedOut } = await daemonQueryAttributed("mr:fetch-job-detail", { repoName, iid, jobId, pipelineId }, MR_ACTION_TIMEOUT_MS); + if (!res) throw new Error(timedOut ? "fetchJobDetail timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "fetchJobDetail failed"); return res.data; }, fetchJobTrace: async (jobId) => { - const res = await daemonQuery("mr:fetch-job-trace", { repoName, iid, jobId }, MR_ACTION_TIMEOUT_MS); - if (!res) throw new Error(lastQueryTimedOut() ? "fetchJobTrace timed out" : "daemon unavailable"); + const { response: res, timedOut } = await daemonQueryAttributed("mr:fetch-job-trace", { repoName, iid, jobId }, MR_ACTION_TIMEOUT_MS); + if (!res) throw new Error(timedOut ? "fetchJobTrace timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "fetchJobTrace failed"); return res.data as string; }, @@ -352,12 +387,12 @@ export async function fetchDiscussions( iid: number, opts?: { force?: boolean }, ): Promise { - const res = await daemonQuery( + const { response: res, timedOut } = await daemonQueryAttributed( "discussions:read", { repoName, iid, force: opts?.force === true }, DISCUSSIONS_TIMEOUT_MS, ); - if (!res) throw new Error(lastQueryTimedOut() ? "discussions timed out" : "daemon unavailable"); + if (!res) throw new Error(timedOut ? "discussions timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "discussions:read failed"); return res.data as DiscussionsSnapshot; } @@ -369,12 +404,12 @@ export async function setDiscussionResolved( discussionId: string, resolved: boolean, ): Promise { - const res = await daemonQuery( + const { response: res, timedOut } = await daemonQueryAttributed( "discussions:resolve", { repoName, iid, discussionId, resolved }, DISCUSSIONS_TIMEOUT_MS, ); - if (!res) throw new Error(lastQueryTimedOut() ? "resolve timed out" : "daemon unavailable"); + if (!res) throw new Error(timedOut ? "resolve timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "discussions:resolve failed"); return res.data as DiscussionsSnapshot; } @@ -384,12 +419,12 @@ export async function fetchMRDiffs( repoName: string, iid: number, ): Promise> { - const res = await daemonQuery( + const { response: res, timedOut } = await daemonQueryAttributed( "discussions:diffs", { repoName, iid }, DISCUSSIONS_TIMEOUT_MS, ); - if (!res) throw new Error(lastQueryTimedOut() ? "diffs timed out" : "daemon unavailable"); + if (!res) throw new Error(timedOut ? "diffs timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "discussions:diffs failed"); return (res.data as { diffs: Array<{ newPath: string; diff: string }> }).diffs; } @@ -401,12 +436,12 @@ export async function replyToDiscussion( discussionId: string, body: string, ): Promise { - const res = await daemonQuery( + const { response: res, timedOut } = await daemonQueryAttributed( "discussions:reply", { repoName, iid, discussionId, body }, DISCUSSIONS_TIMEOUT_MS, ); - if (!res) throw new Error(lastQueryTimedOut() ? "reply timed out" : "daemon unavailable"); + if (!res) throw new Error(timedOut ? "reply timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "discussions:reply failed"); return res.data as DiscussionsSnapshot; } From 44485548f863a923362da28ef64f130af5eed6a1 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:15:48 -0500 Subject: [PATCH 033/106] daemon: default-deny CORS and gate /ws on origin/token (S005/S006) --- .../__tests__/api-server-cors-ws.test.ts | 26 +++++++++++ lib/daemon/api-server.ts | 43 +++++++++++++++---- 2 files changed, 61 insertions(+), 8 deletions(-) create mode 100644 lib/daemon/__tests__/api-server-cors-ws.test.ts diff --git a/lib/daemon/__tests__/api-server-cors-ws.test.ts b/lib/daemon/__tests__/api-server-cors-ws.test.ts new file mode 100644 index 00000000..58c5bf8a --- /dev/null +++ b/lib/daemon/__tests__/api-server-cors-ws.test.ts @@ -0,0 +1,26 @@ +import { describe, test, expect } from "bun:test"; +import { buildCorsHeaders } from "../api-server.ts"; + +describe("buildCorsHeaders", () => { + test("no Origin header: no Access-Control-Allow-Origin is set (non-browser request, CORS is irrelevant)", () => { + const headers = buildCorsHeaders(null, true); + expect(headers["Access-Control-Allow-Origin"]).toBeUndefined(); + }); + + test("an untrusted Origin gets no Access-Control-Allow-Origin (default-deny, S006)", () => { + const headers = buildCorsHeaders("http://evil.example", false); + expect(headers["Access-Control-Allow-Origin"]).toBeUndefined(); + }); + + test("a trusted Origin is echoed back with Vary: Origin", () => { + const headers = buildCorsHeaders("http://localhost:5544", true); + expect(headers["Access-Control-Allow-Origin"]).toBe("http://localhost:5544"); + expect(headers["Vary"]).toBe("Origin"); + }); + + test("always advertises the methods/headers a preflight needs, trusted or not", () => { + const headers = buildCorsHeaders("http://evil.example", false); + expect(headers["Access-Control-Allow-Methods"]).toContain("POST"); + expect(headers["Access-Control-Allow-Headers"]).toContain("X-RT-Token"); + }); +}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index d61909fd..3ac823f8 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -10,7 +10,7 @@ import type { Server, ServerWebSocket } from "bun"; import type { Logger } from "pino"; import { API_PORT } from "../daemon-config.ts"; -import { needsToken, tokenOk, getApiToken } from "./api-auth.ts"; +import { needsToken, tokenOk, getApiToken, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "./api-auth.ts"; import { getAggregatedConnection } from "./freshness.ts"; import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; @@ -92,6 +92,25 @@ export function clearWsClients(): void { wsClients.clear(); } +/** + * CORS default-deny (S006): a browser page on an untrusted Origin still gets + * its request served (127.0.0.1 loopback + the per-route token gate are the + * real defenses), but the response carries no Access-Control-Allow-Origin, + * so the page's own JavaScript cannot read the body. A request with no + * Origin at all (every non-browser consumer today) needs no CORS headers. + */ +export function buildCorsHeaders(origin: string | null, trusted: boolean): Record { + const headers: Record = { + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type, X-RT-Token", + }; + if (origin && trusted) { + headers["Access-Control-Allow-Origin"] = origin; + headers["Vary"] = "Origin"; + } + return headers; +} + export interface ApiServerDeps { handleCommand: (cmd: string, payload: any, signal?: AbortSignal) => Promise; log: Logger; @@ -144,19 +163,27 @@ export async function startApiServer(deps: ApiServerDeps): Promise> maxRequestBodySize: MAX_REQUEST_BODY_SIZE, async fetch(req, server) { const url = new URL(req.url); + const origin = req.headers.get("origin"); + const allowedOrigins = getTrustedBrowserOrigins(); - // WebSocket upgrade — broadcast channel + // WebSocket upgrade (broadcast channel). Browsers cannot set custom + // headers on a WS handshake, so the token (when a browser page wants + // to identify itself) travels as a ?token= query param instead of + // X-RT-Token (S005). if (url.pathname === "/ws") { + const wsToken = url.searchParams.get("token"); + if (!isBrowserRequestTrusted(origin, wsToken, apiToken, allowedOrigins)) { + return new Response("origin not allowed", { status: 403 }); + } if (server.upgrade(req, { data: { kind: "broadcast" } })) return undefined as any; return new Response("WebSocket upgrade failed", { status: 400 }); } - // CORS headers for local dev - const corsHeaders = { - "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Methods": "GET, POST, OPTIONS", - "Access-Control-Allow-Headers": "Content-Type", - }; + // CORS: default-deny. A trusted Origin (token or allowlist) gets its + // Origin echoed back; anything else gets no Access-Control-Allow-Origin + // at all, so a malicious page's own JS cannot read the response (S006). + const trusted = isBrowserRequestTrusted(origin, req.headers.get("x-rt-token"), apiToken, allowedOrigins); + const corsHeaders = buildCorsHeaders(origin, trusted); if (req.method === "OPTIONS") { return new Response(null, { status: 204, headers: corsHeaders }); From 6db63b8afe8e2910f66440063d0aff5ebb797023 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:16:42 -0500 Subject: [PATCH 034/106] runCapture: race reads against the deadline so a pipe-holding grandchild can't wedge it (S023, S024) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/__tests__/subprocess.test.ts | 38 ++++++++++++++++++ lib/subprocess.ts | 51 ++++++++++++++++++------- packages/rt-client/src/settings/exec.ts | 51 ++++++++++++++++++------- 3 files changed, 114 insertions(+), 26 deletions(-) diff --git a/lib/__tests__/subprocess.test.ts b/lib/__tests__/subprocess.test.ts index 9390076c..9fb2725d 100644 --- a/lib/__tests__/subprocess.test.ts +++ b/lib/__tests__/subprocess.test.ts @@ -29,6 +29,44 @@ describe("runCapture env", () => { }); }); +describe("runCapture timeout enforcement", () => { + test("resolves within the deadline even when a grandchild holds the pipe", async () => { + // zsh exits after ~0.2s, but backgrounds `sleep 20` which inherits stdout. + const t0 = Date.now(); + const r = await runCapture( + ["/bin/zsh", "-c", "sleep 20 & echo started; sleep 0.2"], + { timeoutMs: 1000 }, + ); + const elapsed = Date.now() - t0; + expect(elapsed).toBeLessThan(4000); // must NOT wait for the 20s grandchild + expect(r.timedOut).toBe(true); + expect(r.exitCode).toBe(-1); + }); + + test("a SIGTERM-ignoring child is bounded by SIGKILL escalation", async () => { + const t0 = Date.now(); + const r = await runCapture( + ["/bin/zsh", "-c", "trap '' TERM; sleep 20"], + { timeoutMs: 800 }, + ); + expect(Date.now() - t0).toBeLessThan(4000); + expect(r.timedOut).toBe(true); + }); + + test("normal fast command still returns real stdout and exitCode 0", async () => { + const r = await runCapture(["/bin/echo", "hello"], { timeoutMs: 5000 }); + expect(r.stdout.trim()).toBe("hello"); + expect(r.exitCode).toBe(0); + expect(r.timedOut).toBeUndefined(); + }); + + test("timed-out call reports exitCode -1 so callers treat it as failure", async () => { + const r = await runCapture(["/bin/sleep", "20"], { timeoutMs: 500 }); + expect(r.exitCode).toBe(-1); + expect(r.timedOut).toBe(true); + }); +}); + describe("outputTail", () => { test("passes short output through, trimmed", () => { expect(outputTail(" env: node: No such file or directory\n", 2000)) diff --git a/lib/subprocess.ts b/lib/subprocess.ts index afc62be9..9d3b23e2 100644 --- a/lib/subprocess.ts +++ b/lib/subprocess.ts @@ -10,6 +10,8 @@ export interface RunResult { stdout: string; stderr: string; exitCode: number; + /** Set true only when the deadline fired before the child settled. */ + timedOut?: boolean; } /** Longest slice of a failed step's output carried into a log line. */ @@ -57,21 +59,44 @@ export async function runCapture( return { stdout: "", stderr: "", exitCode: -1 }; } - const timer = setTimeout(() => { - try { proc.kill(); } catch { /* already exited */ } - }, opts.timeoutMs ?? 10_000); + const timeoutMs = opts.timeoutMs ?? 10_000; + // SIGTERM at the deadline, SIGKILL a short grace later. A child that ignores + // SIGTERM (or a D-state descendant) cannot be reaped in-band, so the read is + // raced against the deadline below rather than awaited unconditionally: that + // is what lets runCapture settle while a grandchild still holds the pipe. + let killTimer: ReturnType | undefined; + const term = setTimeout(() => { + try { proc.kill("SIGTERM"); } catch { /* already exited */ } + killTimer = setTimeout(() => { + try { proc.kill("SIGKILL"); } catch { /* already exited */ } + }, 2000); + }, timeoutMs); + + const captured: Promise = (async () => { + try { + const stdoutPromise = new Response(proc.stdout as ReadableStream).text(); + const stderrPromise = captureStderr + ? new Response(proc.stderr as ReadableStream).text() + : Promise.resolve(""); + const [stdout, stderr, exitCode] = await Promise.all([ + stdoutPromise, + stderrPromise, + proc.exited, + ]); + return { stdout, stderr, exitCode }; + } catch { + return { stdout: "", stderr: "", exitCode: -1 }; + } + })(); + + const deadline: Promise = new Promise((resolve) => { + setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); + }); try { - const stdoutPromise = new Response(proc.stdout as ReadableStream).text(); - const stderrPromise = captureStderr - ? new Response(proc.stderr as ReadableStream).text() - : Promise.resolve(""); - const [stdout, stderr] = await Promise.all([stdoutPromise, stderrPromise]); - const exitCode = await proc.exited; - return { stdout, stderr, exitCode }; - } catch { - return { stdout: "", stderr: "", exitCode: -1 }; + return await Promise.race([captured, deadline]); } finally { - clearTimeout(timer); + clearTimeout(term); + if (killTimer) clearTimeout(killTimer); } } diff --git a/packages/rt-client/src/settings/exec.ts b/packages/rt-client/src/settings/exec.ts index b5c68b07..cedc8e12 100644 --- a/packages/rt-client/src/settings/exec.ts +++ b/packages/rt-client/src/settings/exec.ts @@ -12,6 +12,8 @@ export interface RunResult { stdout: string; stderr: string; exitCode: number; + /** Set true only when the deadline fired before the child settled. */ + timedOut?: boolean; } /** @@ -47,21 +49,44 @@ export async function runCapture( return { stdout: "", stderr: "", exitCode: -1 }; } - const timer = setTimeout(() => { - try { proc.kill(); } catch { /* already exited */ } - }, opts.timeoutMs ?? 10_000); + const timeoutMs = opts.timeoutMs ?? 10_000; + // SIGTERM at the deadline, SIGKILL a short grace later. A child that ignores + // SIGTERM (or a D-state descendant) cannot be reaped in-band, so the read is + // raced against the deadline below rather than awaited unconditionally: that + // is what lets runCapture settle while a grandchild still holds the pipe. + let killTimer: ReturnType | undefined; + const term = setTimeout(() => { + try { proc.kill("SIGTERM"); } catch { /* already exited */ } + killTimer = setTimeout(() => { + try { proc.kill("SIGKILL"); } catch { /* already exited */ } + }, 2000); + }, timeoutMs); + + const captured: Promise = (async () => { + try { + const stdoutPromise = new Response(proc.stdout as ReadableStream).text(); + const stderrPromise = captureStderr + ? new Response(proc.stderr as ReadableStream).text() + : Promise.resolve(""); + const [stdout, stderr, exitCode] = await Promise.all([ + stdoutPromise, + stderrPromise, + proc.exited, + ]); + return { stdout, stderr, exitCode }; + } catch { + return { stdout: "", stderr: "", exitCode: -1 }; + } + })(); + + const deadline: Promise = new Promise((resolve) => { + setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); + }); try { - const stdoutPromise = new Response(proc.stdout as ReadableStream).text(); - const stderrPromise = captureStderr - ? new Response(proc.stderr as ReadableStream).text() - : Promise.resolve(""); - const [stdout, stderr] = await Promise.all([stdoutPromise, stderrPromise]); - const exitCode = await proc.exited; - return { stdout, stderr, exitCode }; - } catch { - return { stdout: "", stderr: "", exitCode: -1 }; + return await Promise.race([captured, deadline]); } finally { - clearTimeout(timer); + clearTimeout(term); + if (killTimer) clearTimeout(killTimer); } } From 59fc77a222df90d4ea4553a8c1093ccaff1a2ad1 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:16:45 -0500 Subject: [PATCH 035/106] daemon-client: bounded-poll rt.sock after a restart instead of one 300ms retry (S082) --- .../daemon-client-attribution.test.ts | 49 ++++++++++++++++++- lib/daemon-client.ts | 32 +++++++++++- 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/lib/__tests__/daemon-client-attribution.test.ts b/lib/__tests__/daemon-client-attribution.test.ts index 91fc5558..7d087d1e 100644 --- a/lib/__tests__/daemon-client-attribution.test.ts +++ b/lib/__tests__/daemon-client-attribution.test.ts @@ -10,7 +10,7 @@ * clobbered by an unrelated concurrent call. */ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { DAEMON_SOCK_PATH, markDaemonInstalled, markDaemonUninstalled } from "../daemon-config.ts"; +import { DAEMON_SOCK_PATH, TRAY_SOCK_PATH, markDaemonInstalled, markDaemonUninstalled } from "../daemon-config.ts"; import { daemonQueryAttributed } from "../daemon-client.ts"; describe("daemonQueryAttributed", () => { @@ -55,3 +55,50 @@ describe("daemonQueryAttributed", () => { await slow; // drain — this one times out at 200ms, don't leave it dangling }); }); + +// S082: auto-start retried the real query exactly once, 300ms after asking +// the tray to start the daemon — but parkUntilIntended's own socket probe, +// state.db open, and the identity migration routinely take longer than +// that, so a start that genuinely succeeds still gets reported as +// "installed but not running". +describe("auto-start bounded poll (S082)", () => { + const origRtAppSocket = process.env.RT_APP_SOCKET; + let daemonServer: ReturnType | undefined; + let traySock: string | undefined; + let trayServer: ReturnType | undefined; + let bindTimer: ReturnType | undefined; + + beforeEach(() => { + markDaemonInstalled(); + }); + + afterEach(() => { + if (bindTimer) clearTimeout(bindTimer); + daemonServer?.stop(true); + trayServer?.stop(true); + markDaemonUninstalled(); + if (origRtAppSocket === undefined) delete process.env.RT_APP_SOCKET; + else process.env.RT_APP_SOCKET = origRtAppSocket; + }); + + test("a restart that takes longer than 300ms to bind its socket is still picked up, not reported down", async () => { + // attemptRestart() reads the fixed TRAY_SOCK_PATH (not RT_APP_SOCKET), + // so the fake tray must listen there. + trayServer = Bun.serve({ + unix: TRAY_SOCK_PATH, + fetch: () => new Response(JSON.stringify({ ok: true }), { headers: { "Content-Type": "application/json" } }), + }); + + // No real daemon socket exists yet — simulates the gap between the tray + // accepting /daemon/start and the daemon actually binding rt.sock. + bindTimer = setTimeout(() => { + daemonServer = Bun.serve({ + unix: DAEMON_SOCK_PATH, + fetch: () => new Response(JSON.stringify({ ok: true, data: {} }), { headers: { "Content-Type": "application/json" } }), + }); + }, 600); + + const result = await daemonQueryAttributed("ping", {}, 200); + expect(result.response?.ok).toBe(true); + }, 5000); +}); diff --git a/lib/daemon-client.ts b/lib/daemon-client.ts index 0fca23a8..41b9739b 100644 --- a/lib/daemon-client.ts +++ b/lib/daemon-client.ts @@ -199,6 +199,31 @@ async function attemptRestart(): Promise { } } +const SOCKET_POLL_TOTAL_MS = 3_000; +const SOCKET_POLL_INTERVAL_MS = 150; + +/** + * Polls for rt.sock to exist and answer for up to ~3s after a restart + * request. parkUntilIntended's own socket probe, state.db open, and the + * identity migration routinely take longer than a single fixed delay, so a + * start that genuinely succeeds must not be reported as "installed but not + * running" just because the retry landed too early. + */ +async function waitForSocket( + totalMs: number = SOCKET_POLL_TOTAL_MS, + intervalMs: number = SOCKET_POLL_INTERVAL_MS, +): Promise { + const deadline = Date.now() + totalMs; + while (Date.now() < deadline) { + if (existsSync(DAEMON_SOCK_PATH)) { + const ping = await trySocketQuery("ping", {}, Math.min(intervalMs * 2, 1000)); + if (ping.response !== null) return true; + } + await Bun.sleep(intervalMs); + } + return existsSync(DAEMON_SOCK_PATH); +} + function warnDaemonDown(): void { if (hasWarnedThisSession || _warningSuppressed) return; hasWarnedThisSession = true; @@ -249,8 +274,11 @@ export async function daemonQueryAttributed( const restarted = await attemptRestart(); let last = first; if (restarted) { - // Retry once after short delay - await Bun.sleep(300); + // Bounded poll, not one fixed-delay retry: parkUntilIntended's own + // socket probe, state.db open, and the identity migration routinely + // take longer than 300ms, and a successful start must not be reported + // as "installed but not running" just because the retry landed early. + await waitForSocket(); const retry = await trySocketQuery(cmd, payload, timeoutMs); if (retry.response !== null) return retry; last = retry; From 0d816c013d87ca2c6e3d9b2a99ef76f37b38737d Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:18:55 -0500 Subject: [PATCH 036/106] runs prune: reap expired run dirs with a detached rm -rf, never a sync recursive rmSync (S100) --- lib/runs/__tests__/prune.test.ts | 40 ++++++++++++++++++++++++++++---- lib/runs/prune.ts | 24 +++++++++++++++++-- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/lib/runs/__tests__/prune.test.ts b/lib/runs/__tests__/prune.test.ts index 75adbbb2..86a7a815 100644 --- a/lib/runs/__tests__/prune.test.ts +++ b/lib/runs/__tests__/prune.test.ts @@ -35,6 +35,15 @@ function seedRun(dir: string, repo: string, id: string, startedAt: number, userV db.close(); } +/** Polls until the detached rm -rf a prune spawns has actually removed `path` (S100: the delete is no longer synchronous). */ +async function waitGone(path: string, timeoutMs = 5000): Promise { + const deadline = Date.now() + timeoutMs; + while (existsSync(path)) { + if (Date.now() > deadline) throw new Error(`timed out waiting for ${path} to be reaped`); + await new Promise((r) => setTimeout(r, 20)); + } +} + function seedRunEnded(dir: string, repo: string, id: string, startedAt: number, endedAt: number, status = "done"): void { const runDir = join(dir, repo, id); mkdirSync(runDir, { recursive: true }); @@ -55,7 +64,7 @@ function seedRunEnded(dir: string, repo: string, id: string, startedAt: number, } describe("pruneRuns", () => { - test("removes runs ended past the floor, keeps recent and running ones", () => { + test("removes runs ended past the floor, keeps recent and running ones", async () => { const dir = root(); const now = Date.now(); seedRunEnded(dir, "alpha", "old-done", now - 40 * DAY, now - 40 * DAY); // ended 40d ago -> pruned @@ -64,12 +73,12 @@ describe("pruneRuns", () => { // never-finished: age by state.db mtime (fresh in this test) -> kept const { removed } = pruneRuns(now); expect(removed).toEqual([join(dir, "alpha", "old-done")]); - expect(existsSync(join(dir, "alpha", "old-done"))).toBe(false); + await waitGone(join(dir, "alpha", "old-done")); // reaped by a detached rm -rf, not synchronously expect(existsSync(join(dir, "alpha", "new-done"))).toBe(true); expect(existsSync(join(dir, "alpha", "still-running"))).toBe(true); }); - test("a running run ages out once its state.db mtime crosses the floor", () => { + test("a running run ages out once its state.db mtime crosses the floor", async () => { const dir = root(); const now = Date.now(); seedRun(dir, "alpha", "stale-running", now - 40 * DAY); @@ -78,7 +87,7 @@ describe("pruneRuns", () => { utimesSync(dbPath, oldTime, oldTime); const { removed } = pruneRuns(now); expect(removed).toEqual([join(dir, "alpha", "stale-running")]); - expect(existsSync(join(dir, "alpha", "stale-running"))).toBe(false); + await waitGone(join(dir, "alpha", "stale-running")); }); test("a stale regular file inside a repo dir is not mistaken for a run and survives pruning", () => { @@ -110,4 +119,27 @@ describe("pruneRuns", () => { expect(() => assertPrunable(join(dir, "alpha", "victim-run"), dir)).toThrow(); expect(existsSync(join(outside, "victim-run"))).toBe(true); }); + + // S100: the boot-time prune (60s after start) previously unlinked every + // expired run tree synchronously, blocking the daemon's single thread for + // the full duration of each recursive rm — a tray poll or chat post + // arriving mid-sweep would time out. The delete must be off-thread. + test("prune spawns the delete asynchronously — the run dir is not synchronously unlinked (S100)", () => { + const dir = root(); + const now = Date.now(); + seedRunEnded(dir, "alpha", "old-done", now - 40 * DAY, now - 40 * DAY); + const { removed } = pruneRuns(now); + expect(removed).toEqual([join(dir, "alpha", "old-done")]); + // Not gone yet: the unlink runs in a detached child process, off this + // thread, so pruneRuns returning never means the disk is clean yet. + expect(existsSync(join(dir, "alpha", "old-done"))).toBe(true); + }); + + test("prune's detached delete eventually removes the run dir", async () => { + const dir = root(); + const now = Date.now(); + seedRunEnded(dir, "alpha", "old-done", now - 40 * DAY, now - 40 * DAY); + pruneRuns(now); + await waitGone(join(dir, "alpha", "old-done")); + }); }); diff --git a/lib/runs/prune.ts b/lib/runs/prune.ts index b42b05a2..e2b33638 100644 --- a/lib/runs/prune.ts +++ b/lib/runs/prune.ts @@ -5,7 +5,7 @@ * and must stay in front of every rmSync. */ import { Database } from "bun:sqlite"; -import { existsSync, readdirSync, realpathSync, rmSync, statSync, type Dirent } from "fs"; +import { existsSync, readdirSync, realpathSync, statSync, type Dirent } from "fs"; import { basename, dirname, join, sep } from "path"; import { getSetting } from "../settings/resolve.ts"; import { runsRoot } from "./store.ts"; @@ -56,6 +56,26 @@ function realDirNames(path: string): string[] { } } +/** + * Detached, unawaited `rm -rf` — mirrors lib/worktree/trash.ts's reap + * pattern. A recursive unlink of a large run tree must never block the + * daemon's single event-loop thread: a sync rmSync here froze every + * concurrent tray poll and chat post for the sweep's full duration. + */ +function reapAsync(path: string): void { + try { + const proc = Bun.spawn(["rm", "-rf", "--", path], { + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + detached: true, + }); + proc.unref(); + } catch { + // Best-effort: a run tree that survives a failed spawn costs disk, never correctness. + } +} + function floorDays(): number { try { const v = getSetting("rt.runsPruneDays").value; @@ -98,7 +118,7 @@ export function pruneRuns(now: number = Date.now()): { removed: string[] } { } if (stamp < cutoff) { assertPrunable(runDir, root); - rmSync(runDir, { recursive: true, force: true }); + reapAsync(runDir); removed.push(runDir); } } From cbba6b4f2caf2adbd7244d4c273dcea9a7ed77a3 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:22:12 -0500 Subject: [PATCH 037/106] daemon: broadcast() drops dead/backpressured ws clients instead of silently dropping frames (S042) --- .../__tests__/api-server-broadcast.test.ts | 75 +++++++++++++++++++ lib/daemon/api-server.ts | 64 +++++++++++++++- 2 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 lib/daemon/__tests__/api-server-broadcast.test.ts diff --git a/lib/daemon/__tests__/api-server-broadcast.test.ts b/lib/daemon/__tests__/api-server-broadcast.test.ts new file mode 100644 index 00000000..56faf914 --- /dev/null +++ b/lib/daemon/__tests__/api-server-broadcast.test.ts @@ -0,0 +1,75 @@ +import { describe, test, expect } from "bun:test"; +import { broadcastToClients, type BroadcastTarget } from "../api-server.ts"; + +function fakeClient(sendReturns: number[]): BroadcastTarget & { closed: boolean; sent: string[] } { + const sent: string[] = []; + let i = 0; + const client = { + closed: false, + sent, + send(data: string) { + sent.push(data); + const ret = sendReturns[Math.min(i, sendReturns.length - 1)] as number; + i++; + return ret; + }, + close() { client.closed = true; }, + }; + return client; +} + +function fakeLog() { + const warns: unknown[] = []; + return { warn: (o: unknown, _m: string) => { warns.push(o); }, warns }; +} + +describe("broadcastToClients", () => { + test("a healthy client (positive send return) is never closed", () => { + const client = fakeClient([42]); + broadcastToClients([client], "status", { ok: true }, fakeLog()); + expect(client.closed).toBe(false); + expect(client.sent.length).toBe(1); + }); + + test("a send() returning 0 (dropped frame) closes the client immediately and logs a warning", () => { + const client = fakeClient([0]); + const log = fakeLog(); + broadcastToClients([client], "status", { ok: true }, log); + expect(client.closed).toBe(true); + expect(log.warns.length).toBe(1); + }); + + test("a send() returning -1 (backpressure) is tolerated for a few sends before closing", () => { + const client = fakeClient([-1, -1, -1, -1]); + const log = fakeLog(); + broadcastToClients([client], "a", {}, log); + expect(client.closed).toBe(false); + broadcastToClients([client], "b", {}, log); + expect(client.closed).toBe(false); + broadcastToClients([client], "c", {}, log); + // third consecutive backpressure event closes the client + expect(client.closed).toBe(true); + }); + + test("a successful send resets the backpressure counter", () => { + const client = fakeClient([-1, -1, 99, -1, -1, -1]); + const log = fakeLog(); + broadcastToClients([client], "a", {}, log); // -1 (count=1) + broadcastToClients([client], "b", {}, log); // -1 (count=2) + broadcastToClients([client], "c", {}, log); // 99 -- resets to 0 + expect(client.closed).toBe(false); + broadcastToClients([client], "d", {}, log); // -1 (count=1) + broadcastToClients([client], "e", {}, log); // -1 (count=2) + expect(client.closed).toBe(false); + broadcastToClients([client], "f", {}, log); // -1 (count=3) -- closes + expect(client.closed).toBe(true); + }); + + test("a client whose send() throws is treated as gone: caught, not propagated", () => { + const client: BroadcastTarget = { + send() { throw new Error("ECONNRESET"); }, + close() { /* no-op */ }, + }; + expect(() => broadcastToClients([client], "a", {}, fakeLog())).not.toThrow(); + }); +}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index 3ac823f8..9670f861 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -78,13 +78,68 @@ interface ApiWSData { const wsClients = new Set>(); +let apiServerLog: { warn: (o: unknown, m: string) => void } = { warn: () => {} }; + +/** Consecutive Bun `ws.send()` backpressure (-1) returns tolerated before a + client is dropped as chronically stalled. */ +const BACKPRESSURE_CLOSE_THRESHOLD = 3; +const backpressureCounts = new WeakMap(); + +export interface BroadcastTarget { + send(data: string): number; + close(): void; +} + +/** + * Sends one frame to every client, dropping any that Bun's own send() return + * value marks as gone (S042). `ws.send()` never throws on a dead socket -- + * it returns 0 (this send silently failed) or -1 (backpressure) -- so a + * disconnected or stalled console/chat-viewer tab used to keep receiving a + * SUBSET of frames forever with nothing logged. 0 means Bun already dropped + * this exact frame for this client: closing immediately (rather than + * counting) is correct because the client's own reconnect logic is the only + * way it recovers a consistent stream. -1 means backpressure, which can be + * transient, so a few in a row are tolerated before giving up on the client. + */ +export function broadcastToClients( + clients: Iterable, + type: string, + data: any, + log: { warn: (o: unknown, m: string) => void }, +): void { + const msg = JSON.stringify({ type, data, timestamp: Date.now() }); + for (const client of clients) { + let result: number; + try { + result = client.send(msg); + } catch (err) { + log.warn({ err }, "ws client send threw; dropping"); + try { client.close(); } catch { /* already gone */ } + continue; + } + if (result === 0) { + log.warn({ type }, "ws client dropped a frame (send()=0); closing so its reconnect resyncs"); + backpressureCounts.delete(client); + try { client.close(); } catch { /* already gone */ } + } else if (result === -1) { + const count = (backpressureCounts.get(client) ?? 0) + 1; + if (count >= BACKPRESSURE_CLOSE_THRESHOLD) { + log.warn({ type, count }, "ws client chronically backpressured; closing"); + backpressureCounts.delete(client); + try { client.close(); } catch { /* already gone */ } + } else { + backpressureCounts.set(client, count); + } + } else { + backpressureCounts.delete(client); + } + } +} + /** Broadcast an event to all connected WebSocket clients. */ export function broadcast(type: string, data: any): void { if (wsClients.size === 0) return; - const msg = JSON.stringify({ type, data, timestamp: Date.now() }); - for (const ws of wsClients) { - try { ws.send(msg); } catch { /* client disconnected */ } - } + broadcastToClients(wsClients, type, data, apiServerLog); } /** Drop all broadcast clients (shutdown). */ @@ -150,6 +205,7 @@ export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDe export async function startApiServer(deps: ApiServerDeps): Promise> { const { handleCommand, log } = deps; + apiServerLog = log; const apiToken = getApiToken(); const server = await bindApiServerWithRetry(() => Bun.serve({ From 659ecee57c9c7bdc7ec8494fb748df03fecd0ecd Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:23:41 -0500 Subject: [PATCH 038/106] worktree reconciler: queue a follow-up pass for a kick arriving after the loop has started, instead of dropping it (S065) --- .../__tests__/worktree-reconciler.test.ts | 34 +++++++++++++++++++ lib/daemon/worktree-reconciler.ts | 25 +++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/lib/daemon/__tests__/worktree-reconciler.test.ts b/lib/daemon/__tests__/worktree-reconciler.test.ts index 30133a8c..87b17260 100644 --- a/lib/daemon/__tests__/worktree-reconciler.test.ts +++ b/lib/daemon/__tests__/worktree-reconciler.test.ts @@ -1348,6 +1348,40 @@ describe("detached trigger / latency", () => { expect(events.filter((e) => e.type === "worktree:created").length).toBe(1); }, 10_000); + + // S065: a kick() during an in-flight pass, once the pass has started + // working (its per-repo loop has begun — this repo's replenish may + // already have run), must not be silently dropped. Observed here via + // repoIndex() call count: it's read fresh once per runOnce() invocation, + // so a queued follow-up pass is externally visible as a second call. + test("a kick() arriving after the pass has started work triggers a follow-up pass, not a silent drop (S065)", async () => { + process.env.HOME = realpathSync(mkdtempSync(join(tmpdir(), "rtkick2-home-"))); + closeStateDb(); + __test__.createBackoff.clear(); + const repoName = "acme-kick2"; + const repo = makeRepo(); + addBareOrigin(repo); + writeFileSync(join(repo, "wip.txt"), "not idle\n"); + await declareWorktrees(repo, repoName, { onDeck: 1, root: join(repo, ".worktrees"), ready: [{ run: "sleep 2" }] }); + + let repoIndexCalls = 0; + const reconciler = createWorktreeReconciler({ + cache: { entries: {} }, + repoIndex: () => { repoIndexCalls++; return { [repoName]: repo }; }, + emit: () => {}, + log: fakeLog(), + }); + + reconciler.kick(); + await waitFor(() => reconciler.creationInFlight(repoName) !== null, 2000); + // Mid-pass: this repo's replenish step is already running, so this kick + // must queue a follow-up rather than being dropped. + reconciler.kick(); + + await waitFor(() => reconciler.creationInFlight(repoName) === null, 6000); + await waitFor(() => repoIndexCalls >= 2, 6000); // the queued follow-up pass actually ran + expect(repoIndexCalls).toBeGreaterThanOrEqual(2); + }, 15_000); }); describe("reapRepoTrash", () => { diff --git a/lib/daemon/worktree-reconciler.ts b/lib/daemon/worktree-reconciler.ts index d7af150d..6679dd70 100644 --- a/lib/daemon/worktree-reconciler.ts +++ b/lib/daemon/worktree-reconciler.ts @@ -1101,6 +1101,16 @@ export function createWorktreeReconciler(deps: ReconcilerDeps): { /** Non-null while a holder owns the reconciler. */ let hold: Promise | null = null; let kickQueued = false; + /** + * True once the current pass's per-repo loop has begun processing at + * least one repo. Two kicks that both land before this flips (the common + * "two synchronous kicks" case) still collapse to one pass — the + * upcoming loop reads fresh state regardless. A kick landing after it + * flips might be about a repo this pass has already stepped past (e.g. a + * provision claiming the last on-deck tree right after replenish ran for + * it), so it queues a follow-up instead of being silently dropped. + */ + let passStartedWork = false; const creationPromises = new Map>(); async function runOnce(): Promise { @@ -1118,6 +1128,7 @@ export function createWorktreeReconciler(deps: ReconcilerDeps): { const appConfig = loadWorktreeAppConfig(); for (const [repoName, repoPath] of Object.entries(repos)) { + passStartedWork = true; if (!(await repoHasWorktreeActivity(repoName, repoPath))) continue; try { await reconcileRepoRegistry({ repoName, repoPath, emit: deps.emit, log: deps.log }); @@ -1167,13 +1178,25 @@ export function createWorktreeReconciler(deps: ReconcilerDeps): { kickQueued = true; return; } - if (inFlight) return; + if (inFlight) { + // Two kicks landing before this pass has stepped into its per-repo + // loop still collapse to one pass; once it has, a kick might be about + // a repo already stepped past (its replenish already ran this pass), + // so queue a follow-up rather than dropping it silently. + if (passStartedWork) kickQueued = true; + return; + } + passStartedWork = false; const p = runOnce() .catch((err) => { deps.log.warn({ err }, "worktree reconciler: kick failed"); }) .finally(() => { if (inFlight === p) inFlight = null; + if (kickQueued) { + kickQueued = false; + kick(); + } }); inFlight = p; } From ba5e7fc40b965413011693d6d79e125e764d85c4 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:24:00 -0500 Subject: [PATCH 039/106] events.db: corruption quarantine + busy_timeout/synchronous pragmas; guard sweep timers createEventsBus now mirrors state/db.ts's corruption quarantine (rename to events.db.corrupt- + -wal/-shm sidecars, warn, recreate empty) and sets busy_timeout=250 / synchronous=NORMAL on open. events.db is a bounded-retention journal, so total loss on quarantine is harmless. Added safeInterval/safeTimeout (lib/daemon/safe-timers.ts) and wrapped the two eventsBus.sweep() timers in lib/daemon.ts with them, so a synchronous sqlite throw mid-tick (e.g. SQLITE_FULL) warns instead of becoming an uncaughtException that exits the daemon. Also fixes the near-vacuous e2e assertion carried from Task 3: the corrupt-events.db test now asserts the quarantine file exists AND the daemon actually boots and serves (a live rt events emit round trip), instead of existsSync(stderrLog) || quarantined, which was always true. --- e2e/tests/daemon.test.ts | 63 +++++++++++++++++++----- lib/daemon.ts | 9 ++-- lib/daemon/__tests__/events-bus.test.ts | 33 ++++++++++++- lib/daemon/__tests__/safe-timers.test.ts | 55 +++++++++++++++++++++ lib/daemon/events-bus.ts | 53 ++++++++++++++++++-- lib/daemon/safe-timers.ts | 42 ++++++++++++++++ lib/state/db.ts | 2 +- 7 files changed, 237 insertions(+), 20 deletions(-) create mode 100644 lib/daemon/__tests__/safe-timers.test.ts create mode 100644 lib/daemon/safe-timers.ts diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index 5bccbdfe..6b4af74d 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -1,7 +1,24 @@ import { describe, test, expect, beforeAll, afterAll } from "bun:test"; import { existsSync, mkdirSync, writeFileSync, readdirSync } from "fs"; import { join } from "path"; -import { createTestHome, rt } from "../harness.ts"; +import { createTestHome, rt, RT_BINARY } from "../harness.ts"; + +async function waitForSocket(sockPath: string, timeoutMs = 15_000): Promise { + const deadline = Date.now() + timeoutMs; + while (!existsSync(sockPath)) { + if (Date.now() > deadline) throw new Error(`daemon socket never appeared at ${sockPath}`); + await Bun.sleep(100); + } +} + +/** Grab a free TCP port by binding port 0 and releasing it. */ +function freePort(): number { + const srv = Bun.serve({ port: 0, fetch: () => new Response("") }); + const port = srv.port; + srv.stop(true); + if (!port) throw new Error("failed to allocate a free port"); + return port; +} describe("fatal boot", () => { test("daemon boot with API port already bound exits non-zero and leaves no stale rt.pid", async () => { @@ -20,25 +37,47 @@ describe("fatal boot", () => { } }, 60_000); - test("a corrupt events.db does not crash the daemon silently — error is captured", async () => { + test("a corrupt events.db self-heals — quarantined, and the daemon boots and serves", async () => { const { path: home, cleanup } = createTestHome(); + const bunDir = join(process.execPath, ".."); + let daemon: ReturnType | undefined; try { // Pre-create a corrupt events.db in the isolated HOME, before the - // daemon ever runs — createEventsBus (module scope) throws opening it. + // daemon ever runs — createEventsBus (module scope) opens it. const rtDir = join(home, ".mattstack", "rt"); mkdirSync(rtDir, { recursive: true }); writeFileSync(join(rtDir, "events.db"), "not a sqlite file at all"); - const result = await rt(["--daemon"], { home }); - - expect(result.exitCode).not.toBe(0); - // Task 4 makes this self-heal (quarantine + retry); for Task 3 we only - // require the failure is captured, not that boot recovers. - const stderrLog = join(rtDir, "logs", "daemon-stderr.log"); - const quarantined = existsSync(rtDir) && readdirSync(rtDir).some((f) => f.startsWith("events.db.corrupt-")); - const captured = existsSync(stderrLog) || quarantined; - expect(captured).toBe(true); + const apiPort = freePort(); + daemon = Bun.spawn([RT_BINARY, "--daemon"], { + env: { + HOME: home, + PATH: `${join(RT_BINARY, "..")}:${bunDir}:/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin`, + TERM: "xterm-256color", + RT_SKIP_SETUP: "1", + CI: "true", + RT_API_PORT: String(apiPort), + }, + stdout: "pipe", + stderr: "pipe", + }); + + await waitForSocket(join(rtDir, "rt.sock")); + expect(daemon.exitCode).toBeNull(); + + // (a) the corrupt events.db was quarantined, not just failed on. + expect(readdirSync(rtDir).some((f) => f.startsWith("events.db.corrupt-"))).toBe(true); + + // (b) the daemon actually boots and serves: a live round trip through + // the recreated events.db proves it, not just the socket's existence. + const served = await rt(["events", "emit", "e2e/corrupt-events-recover"], { + home, + env: { RT_API_PORT: String(apiPort) }, + }); + expect(served.exitCode).toBe(0); } finally { + try { daemon?.kill(); } catch { /* already gone */ } + await daemon?.exited; cleanup(); } }, 60_000); diff --git a/lib/daemon.ts b/lib/daemon.ts index 2dbd549c..ea49efa2 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -63,6 +63,7 @@ import { import { startDiscussionsPoller } from "./daemon/discussions-poller.ts"; import { createCleanup, installSignalHandlers } from "./daemon/shutdown.ts"; import { createEventsBus } from "./daemon/events-bus.ts"; +import { safeInterval, safeTimeout } from "./daemon/safe-timers.ts"; import { pruneRuns } from "./runs/prune.ts"; import { pruneLogs } from "./log-janitor.ts"; import { getSetting } from "./settings/resolve.ts"; @@ -208,10 +209,12 @@ const hooksGuard = createHooksGuard(log); // Pane-communication events bus (RT-44): SQLite journal + in-memory waiters. const eventsBus = createEventsBus({ dbPath: join(RT_DIR, "events.db"), log }); // Hourly retention sweep — cheap; rides its own interval rather than pollers -// because it needs no poller deps. -setInterval(() => eventsBus.sweep(), 60 * 60 * 1000); +// because it needs no poller deps. safeInterval/safeTimeout: a sync sqlite +// throw here (e.g. SQLITE_FULL) must warn, not become an uncaughtException +// that exits the daemon. +safeInterval(() => eventsBus.sweep(), 60 * 60 * 1000, "events-sweep", log); // Boot-time sweep to handle frequent daemon restarts that would otherwise starve the hourly interval. -setTimeout(() => eventsBus.sweep(), 30_000); +safeTimeout(() => eventsBus.sweep(), 30_000, "events-sweep-boot", log); // Age-floor prune of pipeline run dirs — daily; assertPrunable in prune.ts // guards every deletion against the runs root. diff --git a/lib/daemon/__tests__/events-bus.test.ts b/lib/daemon/__tests__/events-bus.test.ts index a1a85c03..c10e038f 100644 --- a/lib/daemon/__tests__/events-bus.test.ts +++ b/lib/daemon/__tests__/events-bus.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect, beforeEach, afterEach } from "bun:test"; -import { mkdtempSync, rmSync } from "fs"; +import { mkdtempSync, rmSync, writeFileSync, readdirSync } from "fs"; import { join } from "path"; import { tmpdir } from "os"; import pino from "pino"; @@ -7,6 +7,37 @@ import { createEventsBus, matchTopic, type EventsBus } from "../events-bus.ts"; const log = pino({ level: "silent" }); +describe("events bus corruption + pragmas", () => { + test("createEventsBus quarantines and recreates a corrupt events.db instead of throwing", () => { + const dir = mkdtempSync(join(tmpdir(), "events-corrupt-")); + try { + const dbPath = join(dir, "events.db"); + writeFileSync(dbPath, "garbage not sqlite"); + const bus = createEventsBus({ dbPath, log }); + expect(readdirSync(dir).some((f) => f.startsWith("events.db.corrupt-"))).toBe(true); + // fresh db works: + bus.emit("test", { hi: 1 }); + expect(bus.list({ pattern: "**" }).events.length).toBe(1); + bus.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("createEventsBus sets busy_timeout and synchronous=NORMAL", () => { + const dir = mkdtempSync(join(tmpdir(), "events-pragma-")); + try { + const bus = createEventsBus({ dbPath: join(dir, "events.db"), log }); + const handle = bus.__db!; + expect(handle.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); + expect(handle.query("PRAGMA synchronous").get()).toEqual({ synchronous: 1 }); // NORMAL + bus.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + describe("matchTopic", () => { test("bare topic matches itself only", () => { expect(matchTopic("job/x/question", "job/x/question")).toBe(true); diff --git a/lib/daemon/__tests__/safe-timers.test.ts b/lib/daemon/__tests__/safe-timers.test.ts new file mode 100644 index 00000000..13ab83b8 --- /dev/null +++ b/lib/daemon/__tests__/safe-timers.test.ts @@ -0,0 +1,55 @@ +import { describe, test, expect, mock } from "bun:test"; +import pino from "pino"; +import { safeInterval, safeTimeout } from "../safe-timers.ts"; + +const silentLog = pino({ level: "silent" }); + +describe("safeInterval / safeTimeout", () => { + test("safeInterval swallows a throwing tick and logs warn", async () => { + const warn = mock(() => {}); + const log = { ...silentLog, warn } as unknown as typeof silentLog; + let ticks = 0; + const handle = safeInterval( + () => { + ticks++; + throw new Error("SQLITE_FULL"); + }, + 10, + "test-sweep", + log, + ); + await new Promise((r) => setTimeout(r, 45)); + clearInterval(handle); + expect(ticks).toBeGreaterThan(0); + expect(warn).toHaveBeenCalled(); + }); + + test("safeTimeout swallows a throwing tick and logs warn", async () => { + const warn = mock(() => {}); + const log = { ...silentLog, warn } as unknown as typeof silentLog; + let ran = false; + safeTimeout( + () => { + ran = true; + throw new Error("SQLITE_FULL"); + }, + 10, + "test-timeout", + log, + ); + await new Promise((r) => setTimeout(r, 45)); + expect(ran).toBe(true); + expect(warn).toHaveBeenCalled(); + }); + + test("safeInterval does not swallow a non-throwing tick's normal operation", async () => { + const warn = mock(() => {}); + const log = { ...silentLog, warn } as unknown as typeof silentLog; + let ticks = 0; + const handle = safeInterval(() => { ticks++; }, 10, "test-sweep-ok", log); + await new Promise((r) => setTimeout(r, 45)); + clearInterval(handle); + expect(ticks).toBeGreaterThan(0); + expect(warn).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/daemon/events-bus.ts b/lib/daemon/events-bus.ts index aca1dd08..c44cd211 100644 --- a/lib/daemon/events-bus.ts +++ b/lib/daemon/events-bus.ts @@ -8,9 +8,10 @@ */ import { Database } from "bun:sqlite"; -import { mkdirSync } from "fs"; +import { mkdirSync, renameSync } from "fs"; import { dirname } from "path"; import type { Logger } from "pino"; +import { isCorruptionError } from "../state/db.ts"; export interface BusEvent { id: number; topic: string; payload: unknown; emittedAt: number } export interface WaitResult { events: BusEvent[]; cursor: number } @@ -31,6 +32,8 @@ export interface EventsBus { sweep(): number; waiterCount(): number; close(): void; + /** Test-only debug accessor for the underlying handle (e.g. pragma checks). Not for feature code. */ + __db?: Database; } // One matcher for wait AND list. Bun.Glob: `*` does not cross `/`, `**` does. @@ -61,6 +64,30 @@ function rowToEvent(row: EventRow): BusEvent { return { id: row.id, topic: row.topic, payload, emittedAt: row.emittedAt }; } +/** + * Renames a corrupt events.db out of the way and warns loudly, mirroring + * lib/state/db.ts's `quarantine`. events.db is a bounded-retention journal + * (sweep() already discards old rows), so losing it entirely on corruption + * is harmless — recreate empty rather than attempt any repair. WAL sidecars + * are best-effort cleaned since they are meaningless without the main file. + */ +function quarantineEventsDb(path: string, log: Logger): void { + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const quarantinedPath = `${path}.corrupt-${stamp}`; + log.warn( + { path, quarantinedPath }, + "events db could not be opened (corrupt), quarantining and recreating empty", + ); + renameSync(path, quarantinedPath); + for (const sidecar of [`${path}-wal`, `${path}-shm`]) { + try { + renameSync(sidecar, `${sidecar}.corrupt-${stamp}`); + } catch { + // sidecar absent — fine, WAL mode doesn't always leave one + } + } +} + export function createEventsBus(opts: { dbPath: string; log: Logger; @@ -73,8 +100,26 @@ export function createEventsBus(opts: { // Self-sufficient about its parent dir — daemon.ts constructs the bus at // module scope, before startDaemon()'s mkdirSync(RT_DIR) runs. mkdirSync(dirname(opts.dbPath), { recursive: true }); - const db = new Database(opts.dbPath, { create: true }); - db.exec("PRAGMA journal_mode = WAL;"); + // PRAGMA order matches lib/state/db.ts's applyPragmas: busy_timeout FIRST + // so the WAL conversion itself respects it, then journal_mode, then + // synchronous. + let db: Database; + try { + db = new Database(opts.dbPath, { create: true }); + db.exec("PRAGMA busy_timeout = 250;"); + db.exec("PRAGMA journal_mode = WAL;"); + db.exec("PRAGMA synchronous = NORMAL;"); + // Pragmas alone don't always force sqlite to validate the file header; + // touch it now so a corrupt file surfaces here, not mid-query later. + db.query("PRAGMA user_version").get(); + } catch (err) { + if (!isCorruptionError(err)) throw err; + quarantineEventsDb(opts.dbPath, log); + db = new Database(opts.dbPath, { create: true }); + db.exec("PRAGMA busy_timeout = 250;"); + db.exec("PRAGMA journal_mode = WAL;"); + db.exec("PRAGMA synchronous = NORMAL;"); + } db.exec(` CREATE TABLE IF NOT EXISTS events ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -215,6 +260,8 @@ export function createEventsBus(opts: { waiterCount() { return waiters.size; }, + __db: db, + close() { const head = maxId(); for (const w of [...waiters]) settle(w, { events: [], cursor: head }); diff --git a/lib/daemon/safe-timers.ts b/lib/daemon/safe-timers.ts new file mode 100644 index 00000000..6ba319c1 --- /dev/null +++ b/lib/daemon/safe-timers.ts @@ -0,0 +1,42 @@ +/** + * lib/daemon/safe-timers.ts — try/catch-wrapped setInterval/setTimeout. + * + * A bare `setInterval`/`setTimeout` callback that throws synchronously + * (e.g. a sqlite SQLITE_FULL on a WAL write) becomes an uncaughtException + * with no stack frame back to the timer that scheduled it — Node/Bun's + * event loop has nothing to attribute the throw to but the process itself, + * so installCrashHandlers treats it as fatal and exits the daemon. Wrapping + * the tick converts that crash into a logged warning. + */ + +import type { Logger } from "pino"; + +export function safeInterval( + fn: () => void, + ms: number, + label: string, + log: Logger, +): ReturnType { + return setInterval(() => { + try { + fn(); + } catch (err) { + log.warn({ err, label }, "timer tick failed"); + } + }, ms); +} + +export function safeTimeout( + fn: () => void, + ms: number, + label: string, + log: Logger, +): ReturnType { + return setTimeout(() => { + try { + fn(); + } catch (err) { + log.warn({ err, label }, "timer tick failed"); + } + }, ms); +} diff --git a/lib/state/db.ts b/lib/state/db.ts index 929a6a63..86cbeafc 100644 --- a/lib/state/db.ts +++ b/lib/state/db.ts @@ -304,7 +304,7 @@ function addArchivedAtColumnIfMissing(db: Database): void { } /** bun:sqlite error codes that mean "the file on disk is not a usable db". */ -function isCorruptionError(err: unknown): boolean { +export function isCorruptionError(err: unknown): boolean { const code = (err as { code?: string } | undefined)?.code; return code === "SQLITE_CORRUPT" || code === "SQLITE_NOTADB"; } From b86d4c2a89639c63c29c4848a0ff3eec707803ed Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:25:20 -0500 Subject: [PATCH 040/106] runCapture: stop clearing the SIGKILL timer before it fires, strengthen SIGKILL test Review found the finally block cleared killTimer in the same tick the deadline promise resolved, so a SIGTERM-ignoring child was never actually SIGKILLed even though runCapture's own promise settled on time. Leave killTimer running past the finally; it is already try/catch guarded so it is a no-op once the child has exited. Also capture and clear the deadline promise's own timer handle so it doesn't outlive the fast path. Strengthened the SIGKILL escalation test to assert the child process is actually gone (via its recorded pid) rather than only asserting runCapture's promise resolved on schedule, since the old assertion passed regardless of whether the kill worked. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/__tests__/subprocess.test.ts | 39 ++++++++++++++++++++----- lib/subprocess.ts | 9 ++++-- packages/rt-client/src/settings/exec.ts | 9 ++++-- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/lib/__tests__/subprocess.test.ts b/lib/__tests__/subprocess.test.ts index 9fb2725d..88eba03c 100644 --- a/lib/__tests__/subprocess.test.ts +++ b/lib/__tests__/subprocess.test.ts @@ -1,4 +1,7 @@ import { describe, test, expect, afterEach } from "bun:test"; +import { unlink } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import { runCapture, outputTail, MAX_LOGGED_OUTPUT } from "../subprocess.ts"; const SENTINEL = "RT_SPAWN_ENV_SENTINEL"; @@ -44,14 +47,36 @@ describe("runCapture timeout enforcement", () => { }); test("a SIGTERM-ignoring child is bounded by SIGKILL escalation", async () => { - const t0 = Date.now(); - const r = await runCapture( - ["/bin/zsh", "-c", "trap '' TERM; sleep 20"], - { timeoutMs: 800 }, + // Timing alone doesn't prove the kill worked: the deadline race resolves + // runCapture on schedule regardless of whether SIGKILL ever fires. So this + // asserts the process is actually gone, via its own recorded pid. + const pidFile = path.join( + os.tmpdir(), + `rt-subprocess-test-${Date.now()}-${Math.random().toString(36).slice(2)}.pid`, ); - expect(Date.now() - t0).toBeLessThan(4000); - expect(r.timedOut).toBe(true); - }); + try { + const t0 = Date.now(); + const r = await runCapture( + ["/bin/zsh", "-c", `echo $$ > ${pidFile}; trap '' TERM; sleep 20`], + { timeoutMs: 500 }, + ); + expect(Date.now() - t0).toBeLessThan(4000); + expect(r.timedOut).toBe(true); + + // Past the 2s SIGKILL grace, so the escalation has had time to land. + await new Promise((resolve) => setTimeout(resolve, 2500)); + const pid = Number((await Bun.file(pidFile).text()).trim()); + let alive = true; + try { + process.kill(pid, 0); + } catch { + alive = false; + } + expect(alive).toBe(false); + } finally { + await unlink(pidFile).catch(() => {}); + } + }, 8000); test("normal fast command still returns real stdout and exitCode 0", async () => { const r = await runCapture(["/bin/echo", "hello"], { timeoutMs: 5000 }); diff --git a/lib/subprocess.ts b/lib/subprocess.ts index 9d3b23e2..c6a286f2 100644 --- a/lib/subprocess.ts +++ b/lib/subprocess.ts @@ -89,14 +89,19 @@ export async function runCapture( } })(); + let deadlineTimer: ReturnType; const deadline: Promise = new Promise((resolve) => { - setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); + deadlineTimer = setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); }); try { return await Promise.race([captured, deadline]); } finally { clearTimeout(term); - if (killTimer) clearTimeout(killTimer); + clearTimeout(deadlineTimer!); + // killTimer intentionally NOT cleared here: on the timeout path it must + // survive this finally to fire SIGKILL against a child that ignored + // SIGTERM. proc.kill is already try/catch guarded, so it is a harmless + // no-op if the child exited before the 2s grace elapses. } } diff --git a/packages/rt-client/src/settings/exec.ts b/packages/rt-client/src/settings/exec.ts index cedc8e12..fae833ec 100644 --- a/packages/rt-client/src/settings/exec.ts +++ b/packages/rt-client/src/settings/exec.ts @@ -79,14 +79,19 @@ export async function runCapture( } })(); + let deadlineTimer: ReturnType; const deadline: Promise = new Promise((resolve) => { - setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); + deadlineTimer = setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); }); try { return await Promise.race([captured, deadline]); } finally { clearTimeout(term); - if (killTimer) clearTimeout(killTimer); + clearTimeout(deadlineTimer!); + // killTimer intentionally NOT cleared here: on the timeout path it must + // survive this finally to fire SIGKILL against a child that ignored + // SIGTERM. proc.kill is already try/catch guarded, so it is a harmless + // no-op if the child exited before the 2s grace elapses. } } From 456b8b8282e37233716a73e9f3ff80a3bedbde97 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:25:53 -0500 Subject: [PATCH 041/106] chat:post/chat:dm: validate room/handle/body shape and refuse an unknown room instead of a silent black hole (R010) --- lib/daemon/__tests__/chat-handlers.test.ts | 56 ++++++++++++++++++++++ lib/daemon/handlers/chat.ts | 26 ++++++++++ 2 files changed, 82 insertions(+) diff --git a/lib/daemon/__tests__/chat-handlers.test.ts b/lib/daemon/__tests__/chat-handlers.test.ts index fb75f284..399b80f1 100644 --- a/lib/daemon/__tests__/chat-handlers.test.ts +++ b/lib/daemon/__tests__/chat-handlers.test.ts @@ -99,6 +99,62 @@ test("chat:post rejects an invalid mentions element with a reason rather than st expect(res.error).toContain("handle"); }); +// R010: a typo'd room silently no-op'd through postMessage's REVIVE (a +// no-op for a room with no chat_rooms row) and returned {ok:true, +// recipients:[]} — no error, no listing, unreachable except by the exact +// typo'd name. It must fail loudly instead. +test("chat:post refuses a room nobody has joined instead of silently black-holing the message", async () => { + const h = freshHandlers(); + const res = await h["chat:post"]({ room: "typo-room", handle: "a", body: "hi" }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("typo-room"); +}); + +test("chat:post's unknown-room error names a close existing room", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "deck-main", handle: "a" }); + const res = await h["chat:post"]({ room: "deck", handle: "a", body: "hi" }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("deck-main"); +}); + +test("chat:post rejects a missing or empty body rather than routing it through unenforced", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + const missing = await h["chat:post"]({ room: "r", handle: "a" } as any); + expect(missing.ok).toBe(false); + const empty = await h["chat:post"]({ room: "r", handle: "a", body: "" }); + expect(empty.ok).toBe(false); +}); + +test("chat:post rejects a body over the size cap", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + const res = await h["chat:post"]({ room: "r", handle: "a", body: "x".repeat(64 * 1024 + 1) }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error.toLowerCase()).toContain("body"); +}); + +test("chat:post rejects a non-array mentions field", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + const res = await h["chat:post"]({ room: "r", handle: "a", body: "hi", mentions: "b" as any }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("mentions"); +}); + +test("chat:dm rejects a missing or empty body", async () => { + const h = freshHandlers(); + const missing = await h["chat:dm"]({ from: "a", to: "b" } as any); + expect(missing.ok).toBe(false); + const empty = await h["chat:dm"]({ from: "a", to: "b", body: "" }); + expect(empty.ok).toBe(false); +}); + test("chat:unread-waking reports what would wake a handle without advancing its cursor", async () => { const h = freshHandlers(); await h["chat:join"]({ room: "r", handle: "a" }); diff --git a/lib/daemon/handlers/chat.ts b/lib/daemon/handlers/chat.ts index 470e846d..80b05270 100644 --- a/lib/daemon/handlers/chat.ts +++ b/lib/daemon/handlers/chat.ts @@ -17,6 +17,7 @@ import { unreadWakingCount, listRooms, archiveRoom, + roomArchivedAt, roomDefaultWake, listMembers, armMember, @@ -83,6 +84,19 @@ function assertionError(fn: () => void): string | null { } } +/** Generous, not tight: bounds a single message body without constraining any real conversation. */ +const MAX_BODY_BYTES = 64 * 1024; + +function isValidBody(body: unknown): body is string { + return typeof body === "string" && body.length > 0 && Buffer.byteLength(body, "utf8") <= MAX_BODY_BYTES; +} + +/** Rooms `handle` already belongs to whose name is a prefix/suffix of the typo'd one — the common shape of a "deck" vs "deck-main" miss. */ +function closestRoomNames(typo: string, handle: string, db: Database): string[] { + const known = listRooms(handle, db, { includeArchived: true }).map((r) => r.room); + return known.filter((r) => r.startsWith(typo) || typo.startsWith(r)).slice(0, 3); +} + /** * The row must commit before either emit fires, or a woken agent reads the * wake pointer and finds no message yet. Shared by chat:post and chat:dm so @@ -202,8 +216,19 @@ export function createChatHandlers(opts: { "chat:post": async (payload: Commands["chat:post"]["payload"]): Promise> => { const { room, handle, body, mentions } = payload; + if (!isValidChatName(room)) return { ok: false, error: `invalid room "${room}"` }; + if (!isValidChatName(handle)) return { ok: false, error: `invalid handle "${handle}"` }; + if (!isValidBody(body)) return { ok: false, error: `body must be a non-empty string under ${MAX_BODY_BYTES} bytes` }; + if (mentions !== undefined && !Array.isArray(mentions)) return { ok: false, error: "mentions must be an array of handles" }; const invalidMention = mentions?.find((m) => !isValidChatName(m)); if (invalidMention !== undefined) return { ok: false, error: `invalid handle "${invalidMention}"` }; + // A typo'd room previously no-op'd through postMessage's REVIVE (a + // no-op for a room with no chat_rooms row) and returned ok with no + // recipients — unreachable except by the exact typo'd name. + if (roomArchivedAt(room, db) === undefined) { + const nearby = closestRoomNames(room, handle, db); + return { ok: false, error: `unknown room "${room}"${nearby.length ? ` — did you mean: ${nearby.join(", ")}` : ""}` }; + } const posted = postAndNotify(db, emitEvent, { room, handle, body, mentions }); if (!posted) return { ok: false, error: "chat: post failed (retry budget exhausted)" }; return { ok: true, data: posted }; @@ -354,6 +379,7 @@ export function createChatHandlers(opts: { const { from, to, body, sessionId } = payload; if (!isValidChatName(from)) return { ok: false, error: `invalid handle "${from}"` }; if (!isValidChatName(to)) return { ok: false, error: `invalid handle "${to}"` }; + if (!isValidBody(body)) return { ok: false, error: `body must be a non-empty string under ${MAX_BODY_BYTES} bytes` }; const err = assertionError(() => assertSessionOwnsHandle(from, sessionId, db)); if (err) return { ok: false, error: err }; const humanHandle = getSetting("chat.humanHandle").value; From 5657d276556ceaec680260895470bf093807d3c3 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:27:04 -0500 Subject: [PATCH 042/106] daemon: pathParam() helper -- malformed %-encoding is a 400, not a logged 500 (S083) Adds a shared pathParam() decode helper and wires it into the three parameterized routes (/api/cache/:branch, /api/hooks/:repo/repair, /api/runs/:repo/:runId), replacing each route's hand-rolled decodeURIComponent. A malformed path segment now returns a clean 400 instead of falling through to the outer catch's logged 500 (cache and hooks routes) or the generic 404 (runs route). --- .../__tests__/api-server-path-param.test.ts | 36 +++++++++++++ lib/daemon/api-server.ts | 50 ++++++++++++++----- 2 files changed, 73 insertions(+), 13 deletions(-) create mode 100644 lib/daemon/__tests__/api-server-path-param.test.ts diff --git a/lib/daemon/__tests__/api-server-path-param.test.ts b/lib/daemon/__tests__/api-server-path-param.test.ts new file mode 100644 index 00000000..353b9a0a --- /dev/null +++ b/lib/daemon/__tests__/api-server-path-param.test.ts @@ -0,0 +1,36 @@ +import { describe, test, expect } from "bun:test"; +import { pathParam } from "../api-server.ts"; + +describe("pathParam", () => { + test("decodes a clean prefix-only param", () => { + expect(pathParam("/api/cache/main", "/api/cache/")).toBe("main"); + }); + + test("decodes a URL-encoded segment", () => { + expect(pathParam("/api/cache/feature%2Ffoo", "/api/cache/")).toBe("feature/foo"); + }); + + test("returns undefined for malformed %-encoding instead of throwing", () => { + expect(pathParam("/api/cache/%E0%A4%A", "/api/cache/")).toBeUndefined(); + }); + + test("returns undefined when the pathname doesn't start with the prefix", () => { + expect(pathParam("/api/other/main", "/api/cache/")).toBeUndefined(); + }); + + test("handles a prefix+suffix pair (hooks repair shape)", () => { + expect(pathParam("/api/hooks/my-repo/repair", "/api/hooks/", "/repair")).toBe("my-repo"); + }); + + test("prefix+suffix: malformed encoding still returns undefined", () => { + expect(pathParam("/api/hooks/%E0%A4%A/repair", "/api/hooks/", "/repair")).toBeUndefined(); + }); + + test("prefix+suffix: wrong suffix returns undefined", () => { + expect(pathParam("/api/hooks/my-repo/other", "/api/hooks/", "/repair")).toBeUndefined(); + }); + + test("an empty captured segment returns undefined", () => { + expect(pathParam("/api/cache/", "/api/cache/")).toBeUndefined(); + }); +}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index 9670f861..baef3c04 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -166,6 +166,27 @@ export function buildCorsHeaders(origin: string | null, trusted: boolean): Recor return headers; } +/** + * Decodes one path segment between a fixed prefix (and optional suffix), + * returning `undefined` (never throwing) on any shape mismatch or malformed + * %-encoding (S083). Before this, each parameterized route hand-rolled its + * own decodeURIComponent inside the route's try block, so a malformed + * segment fell through to the OUTER catch and came back as a logged 500; + * every route using this helper instead gets a clean 400. + */ +export function pathParam(pathname: string, prefix: string, suffix = ""): string | undefined { + if (!pathname.startsWith(prefix)) return undefined; + if (suffix && !pathname.endsWith(suffix)) return undefined; + const end = suffix ? pathname.length - suffix.length : pathname.length; + if (end <= prefix.length) return undefined; + const raw = pathname.slice(prefix.length, end); + try { + return decodeURIComponent(raw); + } catch { + return undefined; + } +} + export interface ApiServerDeps { handleCommand: (cmd: string, payload: any, signal?: AbortSignal) => Promise; log: Logger; @@ -259,33 +280,36 @@ export async function startApiServer(deps: ApiServerDeps): Promise> // Single branch lookup: /api/cache/:branch if (url.pathname.startsWith("/api/cache/") && req.method === "GET") { - const branch = decodeURIComponent(url.pathname.slice("/api/cache/".length)); + const branch = pathParam(url.pathname, "/api/cache/"); + if (branch === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); + } const result = await handleCommand("cache:read", { branches: [branch] }, req.signal); return Response.json(result, { headers: corsHeaders }); } // Hooks repair: /api/hooks/:repo/repair if (url.pathname.startsWith("/api/hooks/") && url.pathname.endsWith("/repair") && req.method === "POST") { - const repo = decodeURIComponent(url.pathname.slice("/api/hooks/".length, -"/repair".length)); + const repo = pathParam(url.pathname, "/api/hooks/", "/repair"); + if (repo === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); + } const result = await handleCommand("hooks:repair", { repo }, req.signal); return Response.json(result, { headers: corsHeaders }); } // Runs detail: /api/runs/:repo/:runId if (url.pathname.startsWith("/api/runs/") && req.method === "GET") { - let rest: string | undefined; - try { - rest = decodeURIComponent(url.pathname.slice("/api/runs/".length)); - } catch { - rest = undefined; // malformed %-encoding -> fall through to the 404 path below + const rest = pathParam(url.pathname, "/api/runs/"); + if (rest === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); } - if (rest !== undefined) { - const slash = rest.indexOf("/"); - if (slash > 0 && slash < rest.length - 1) { - const result = await handleCommand("runs:get", { repo: rest.slice(0, slash), runId: rest.slice(slash + 1) }, req.signal); - return Response.json(result, { headers: corsHeaders }); - } + const slash = rest.indexOf("/"); + if (slash > 0 && slash < rest.length - 1) { + const result = await handleCommand("runs:get", { repo: rest.slice(0, slash), runId: rest.slice(slash + 1) }, req.signal); + return Response.json(result, { headers: corsHeaders }); } + // falls through to the 404 path below for a shape mismatch, e.g. "/api/runs/onlyonesegment" } // Secrets: forward the X-RT-Token header (already verified above by From e50895187324b7382f1e20ac060caf4db2832813 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:27:19 -0500 Subject: [PATCH 043/106] validate chat:join wakeOn and agent:start/resume surface against their enums (R033) --- lib/daemon/__tests__/agent-handlers.test.ts | 16 ++++++++++++++++ lib/daemon/__tests__/chat-handlers.test.ts | 15 +++++++++++++++ lib/daemon/handlers/agent.ts | 6 ++++++ lib/daemon/handlers/chat.ts | 10 ++++++++++ 4 files changed, 47 insertions(+) diff --git a/lib/daemon/__tests__/agent-handlers.test.ts b/lib/daemon/__tests__/agent-handlers.test.ts index 46f70fa6..848d77c9 100644 --- a/lib/daemon/__tests__/agent-handlers.test.ts +++ b/lib/daemon/__tests__/agent-handlers.test.ts @@ -119,6 +119,22 @@ test("agent:start headless refuses a missing prompt", async () => { expect(res.error).toMatch(/prompt/); }); +// R033: an unchecked surface value falls through to the headless spawn path +// with headless=false, spawning an interactive claude with stdin ignored +// and recording surface "bogus" — never a caller-visible error. +test("agent:start rejects a surface outside herdr/headless, naming the allowed values", async () => { + const h = fresh(); + const res = await h["agent:start"]({ repo: REPO, cwd: "/tmp/x", prompt: "hi", surface: "bogus" as any }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("surface"); + expect(res.error).toContain("herdr"); + expect(res.error).toContain("headless"); + const list = await h["agent:list"]({}); + if (!list.ok) throw new Error("unreachable"); + expect(list.data.agents).toHaveLength(0); +}); + test("agent:start headless finishes the record and emits agent/done", async () => { const emitted: string[] = []; let resolveExit!: (c: number) => void; diff --git a/lib/daemon/__tests__/chat-handlers.test.ts b/lib/daemon/__tests__/chat-handlers.test.ts index 399b80f1..9e8cfd31 100644 --- a/lib/daemon/__tests__/chat-handlers.test.ts +++ b/lib/daemon/__tests__/chat-handlers.test.ts @@ -44,6 +44,21 @@ test("chat:join rejects an invalid handle with a reason rather than normalizing expect(res.error).toContain("handle"); }); +// R033: an unchecked wakeOn value is stored on chat_members and, when the +// join creates the room, stamped into chat_room_defaults for every future +// joiner too — recipientsFromMembers treats it as neither "none" nor "all" +// (falls through to mention-only) and nothing ever reports the bad value. +test("chat:join rejects a wakeOn value outside mention/all/none, naming the allowed values", async () => { + const h = freshHandlers(); + const res = await h["chat:join"]({ room: "build", handle: "a", wakeOn: "sometimes" as any }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("wakeOn"); + expect(res.error).toContain("mention"); + expect(res.error).toContain("all"); + expect(res.error).toContain("none"); +}); + test("chat:post returns the recipients and emits one wake event per recipient", async () => { const emitted: string[] = []; const h = freshHandlers((topic) => { emitted.push(topic); return 0; }); diff --git a/lib/daemon/handlers/agent.ts b/lib/daemon/handlers/agent.ts index 9c548642..5273803c 100644 --- a/lib/daemon/handlers/agent.ts +++ b/lib/daemon/handlers/agent.ts @@ -135,6 +135,9 @@ export function createAgentHandlers(opts: { "agent:start": async (payload: Commands["agent:start"]["payload"]): Promise> => { const { repo, cwd } = payload; if (!repo || !cwd) return { ok: false, error: "agent:start requires repo (serialized identity) and cwd" }; + if (payload.surface !== undefined && payload.surface !== "herdr" && payload.surface !== "headless") { + return { ok: false, error: `invalid surface "${payload.surface}"; must be one of herdr, headless` }; + } const surface: AgentSurface = payload.surface ?? "herdr"; const prompt = payload.prompt; if (surface === "headless" && !prompt) { @@ -192,6 +195,9 @@ export function createAgentHandlers(opts: { "agent:resume": async (payload: Commands["agent:resume"]["payload"]): Promise> => { const rec = getAgent(payload.id, db); if (!rec) return { ok: false, error: `no agent record for "${payload.id}"` }; + if (payload.surface !== undefined && payload.surface !== "herdr" && payload.surface !== "headless") { + return { ok: false, error: `invalid surface "${payload.surface}"; must be one of herdr, headless` }; + } const surface: AgentSurface = payload.surface ?? rec.surface; if (surface === "headless" && !payload.prompt) { return { ok: false, error: "headless resume requires a prompt (claude -p with no prompt blocks on stdin)" }; diff --git a/lib/daemon/handlers/chat.ts b/lib/daemon/handlers/chat.ts index 80b05270..5d71c393 100644 --- a/lib/daemon/handlers/chat.ts +++ b/lib/daemon/handlers/chat.ts @@ -97,6 +97,13 @@ function closestRoomNames(typo: string, handle: string, db: Database): string[] return known.filter((r) => r.startsWith(typo) || typo.startsWith(r)).slice(0, 3); } +/** An unchecked value here lands on chat_members (and, on a join-creates, chat_room_defaults for every future joiner) and is silently treated as mention-only — never "none", never "all", never reported. */ +const VALID_WAKE_ON = ["mention", "all", "none"] as const; + +function isValidWakeOn(v: unknown): v is (typeof VALID_WAKE_ON)[number] { + return typeof v === "string" && (VALID_WAKE_ON as readonly string[]).includes(v); +} + /** * The row must commit before either emit fires, or a woken agent reads the * wake pointer and finds no message yet. Shared by chat:post and chat:dm so @@ -201,6 +208,9 @@ export function createChatHandlers(opts: { const { room, handle, wakeOn, cwd, pane } = payload; if (!isValidChatName(handle)) return { ok: false, error: `invalid handle "${handle}"` }; if (!isValidChatName(room)) return { ok: false, error: `invalid room "${room}"` }; + if (wakeOn !== undefined && !isValidWakeOn(wakeOn)) { + return { ok: false, error: `invalid wakeOn "${wakeOn}"; must be one of ${VALID_WAKE_ON.join(", ")}` }; + } try { const data = joinRoom({ room, handle, wakeOn, cwd, pane }, db); return { ok: true, data }; From 4b9afad5e7ed9c7481dbfe09738aecd35ceafd49 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:27:49 -0500 Subject: [PATCH 044/106] source-guards: update the boot-failure-exits guard for runDaemon owning the catch Review fix (round 2): the guard tested startDaemon's body for catch + process.exit(1), which fix-round-1 made stale by moving the catch-and-exit into runDaemon itself (startDaemon is now a thin await runDaemon()). Rewrote the guard to assert the invariant at its real location: runDaemon's body still owns catch + process.exit(1), and startDaemon just awaits it. Fixed the stale "startDaemon is a thin catch-and-exit wrapper" comment in the neighboring test to match. --- lib/state/__tests__/source-guards.test.ts | 38 ++++++++++++++--------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/lib/state/__tests__/source-guards.test.ts b/lib/state/__tests__/source-guards.test.ts index 8b0a6500..733a2965 100644 --- a/lib/state/__tests__/source-guards.test.ts +++ b/lib/state/__tests__/source-guards.test.ts @@ -92,9 +92,10 @@ describe("legacy JSON state files are retired", () => { describe("daemon startup opens state.db before serving", () => { test("openBranchCacheStore() precedes both server binds in runDaemon", () => { const src = readFileSync(join(REPO_ROOT, "lib", "daemon.ts"), "utf8"); - // startDaemon() itself is now a thin catch-and-exit wrapper (see - // "startDaemon exits on any runDaemon failure" below); the real ordered - // startup sequence this test asserts on lives in runDaemon(). + // startDaemon() itself is now just `await runDaemon()` (see "boot + // failure is fatal" below — the catch-and-exit lives in runDaemon + // itself); the real ordered startup sequence this test asserts on + // lives in runDaemon() too. const start = src.indexOf("async function runDaemon("); expect(start).toBeGreaterThan(-1); @@ -128,26 +129,33 @@ describe("daemon startup opens state.db before serving", () => { }); }); -describe("startDaemon exits on any runDaemon failure", () => { +describe("boot failure is fatal for both fire-and-forget callers", () => { test("runDaemon is not exported — startDaemon is the only safe entry point", () => { const src = readFileSync(join(REPO_ROOT, "lib", "daemon.ts"), "utf8"); expect(src).not.toContain("export async function runDaemon("); expect(src).not.toContain("export function runDaemon("); }); - test("startDaemon wraps runDaemon in try/catch and exits nonzero on failure", () => { + test("runDaemon wraps its own body in try/catch and exits nonzero on failure; startDaemon just awaits it", () => { const src = readFileSync(join(REPO_ROOT, "lib", "daemon.ts"), "utf8"); - const start = src.indexOf("export async function startDaemon("); - expect(start).toBeGreaterThan(-1); + const runStart = src.indexOf("async function runDaemon("); + const startDaemonStart = src.indexOf("export async function startDaemon("); + expect(runStart).toBeGreaterThan(-1); + expect(startDaemonStart).toBeGreaterThan(runStart); // Both real callers (cli.ts's --daemon entry, this file's own - // import.meta.main guard) invoke startDaemon() fire-and-forget — the - // catch-and-exit MUST live inside startDaemon itself, not at either - // call site, or an unhandledRejection silently leaves the daemon - // half-up (rt.sock possibly bound, nothing past the failure ever wired). - const body = src.slice(start, start + 400); - expect(body).toContain("await runDaemon()"); - expect(body).toContain("catch"); - expect(body).toContain("process.exit(1)"); + // import.meta.main guard) invoke startDaemon() fire-and-forget, and + // startDaemon() is now just `await runDaemon()` — so the catch-and-exit + // MUST live inside runDaemon() itself, or an unhandledRejection could + // silently leave the daemon half-up (rt.sock possibly bound, nothing + // past the failure ever wired). The booting-gated unhandledRejection + // handler (installCrashHandlers) is only the backstop for whatever + // still manages to slip past this try/catch. + const runBody = src.slice(runStart, startDaemonStart); + expect(runBody).toContain("catch"); + expect(runBody).toContain("process.exit(1)"); + + const startDaemonBody = src.slice(startDaemonStart, startDaemonStart + 200); + expect(startDaemonBody).toContain("await runDaemon()"); }); }); From fd935d752e5e32a6c91e7351140da2bc22ca5bb7 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:29:29 -0500 Subject: [PATCH 045/106] git-async: 5-min timeout for checkout/merge/stash/status so a large-repo checkout isn't killed half-applied (S104) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/__tests__/git-async-timeouts.test.ts | 14 +++++++++++++ lib/daemon/worktree-reconciler.ts | 9 ++++---- lib/worktree/git-async.ts | 26 +++++++++++++++++++----- 3 files changed, 40 insertions(+), 9 deletions(-) create mode 100644 lib/__tests__/git-async-timeouts.test.ts diff --git a/lib/__tests__/git-async-timeouts.test.ts b/lib/__tests__/git-async-timeouts.test.ts new file mode 100644 index 00000000..5ed9782c --- /dev/null +++ b/lib/__tests__/git-async-timeouts.test.ts @@ -0,0 +1,14 @@ +import { test, expect } from "bun:test"; +import { MUTATING_TIMEOUT_MS, stashChangesAsync, popStashAsync } from "../worktree/git-async.ts"; + +test("mutating timeout is 5 minutes", () => { + expect(MUTATING_TIMEOUT_MS).toBe(5 * 60_000); +}); + +test("stash helpers accept a timeout override", () => { + // Type-level: these must type-check with an opts arg. + const a: typeof stashChangesAsync = stashChangesAsync; + const b: typeof popStashAsync = popStashAsync; + expect(typeof a).toBe("function"); + expect(typeof b).toBe("function"); +}); diff --git a/lib/daemon/worktree-reconciler.ts b/lib/daemon/worktree-reconciler.ts index d427c942..8bf4c1b5 100644 --- a/lib/daemon/worktree-reconciler.ts +++ b/lib/daemon/worktree-reconciler.ts @@ -29,6 +29,7 @@ import { gitOk, headSha, listWorktreesAsync, + MUTATING_TIMEOUT_MS, remoteDefaultRef, runGit, stashChangesAsync, @@ -466,13 +467,13 @@ async function autoReturnMain( log.info({ ...fields }, `stashed uncommitted changes on "${mergedBranch}"`); } - const checkout = await runGit(rec.path, ["checkout", defaultBranch]); + const checkout = await runGit(rec.path, ["checkout", defaultBranch], { timeoutMs: MUTATING_TIMEOUT_MS }); if (checkout.exitCode !== 0) { log.warn({ ...fields, defaultBranch, output: checkout.stderr.trim() }, "auto-return: checkout failed"); return "retry"; } - const ff = await runGit(rec.path, ["merge", "--ff-only", defaultRef]); + const ff = await runGit(rec.path, ["merge", "--ff-only", defaultRef], { timeoutMs: MUTATING_TIMEOUT_MS }); if (ff.exitCode !== 0) { log.warn({ ...fields, defaultRef, output: ff.stderr.trim() }, "auto-return: fast-forward failed"); return "retry"; @@ -724,7 +725,7 @@ async function freshenOne(deps: FreshenDeps, rec: TreeRecord): Promise const classify = await classifyDirtyAsync(rec.path); if (classify.discard.length > 0) { - await runGit(rec.path, ["checkout", "--", ...classify.discard]); + await runGit(rec.path, ["checkout", "--", ...classify.discard], { timeoutMs: MUTATING_TIMEOUT_MS }); } // Blockers stashed under the tree's own branch name (Desktop-compatible @@ -753,7 +754,7 @@ async function freshenOne(deps: FreshenDeps, rec: TreeRecord): Promise } }; - const ff = await runGit(rec.path, ["merge", "--ff-only", defaultRef]); + const ff = await runGit(rec.path, ["merge", "--ff-only", defaultRef], { timeoutMs: MUTATING_TIMEOUT_MS }); if (ff.exitCode !== 0) { log.warn({ ...fields, defaultRef, output: ff.stderr.trim() }, "freshen: fast-forward failed"); await popStash(); diff --git a/lib/worktree/git-async.ts b/lib/worktree/git-async.ts index 1d994b66..7eea8562 100644 --- a/lib/worktree/git-async.ts +++ b/lib/worktree/git-async.ts @@ -32,6 +32,10 @@ const NO_HOOKS = ["-c", "core.hooksPath=/dev/null"]; const DEFAULT_TIMEOUT_MS = 60_000; +/** Checkout/merge/stash on a large tree can legitimately exceed a minute; a + * 60s SIGKILL leaves the working tree half-switched. Match fetch/worktree-add. */ +export const MUTATING_TIMEOUT_MS = 5 * 60_000; + const DESKTOP_STASH_RE = /!!GitHub_Desktop<(.+)>$/; /** Run a git command with hooks suppressed, capturing stdout+stderr. Never throws. */ @@ -64,7 +68,7 @@ export async function currentBranchAsync(cwd: string): Promise { } export async function statusPorcelainAsync(cwd: string): Promise { - const r = await runGit(cwd, ["status", "--porcelain"]); + const r = await runGit(cwd, ["status", "--porcelain"], { timeoutMs: MUTATING_TIMEOUT_MS }); return r.stdout; } @@ -174,13 +178,25 @@ export async function ensureInfoExclude(repoPath: string, pattern: string): Prom * Async port of git-ops.ts stashChanges — interoperable with GitHub Desktop * and worktree-context. */ -export async function stashChangesAsync(cwd: string, label: string): Promise { +export async function stashChangesAsync( + cwd: string, + label: string, + opts: { timeoutMs?: number } = {}, +): Promise { const message = `!!GitHub_Desktop<${label}>`; - await runGit(cwd, ["stash", "push", "-u", "-m", message]); + await runGit(cwd, ["stash", "push", "-u", "-m", message], { + timeoutMs: opts.timeoutMs ?? MUTATING_TIMEOUT_MS, + }); } -export async function popStashAsync(cwd: string, stashName: string): Promise { - await runGit(cwd, ["stash", "pop", stashName]); +export async function popStashAsync( + cwd: string, + stashName: string, + opts: { timeoutMs?: number } = {}, +): Promise { + await runGit(cwd, ["stash", "pop", stashName], { + timeoutMs: opts.timeoutMs ?? MUTATING_TIMEOUT_MS, + }); } /** From 2291068a362212fa2c4d0376aa620ecd9c3295c5 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:29:31 -0500 Subject: [PATCH 046/106] chat:read/chat:messages: clamp limit into [1,500] instead of reaching SQLite's unlimited negative LIMIT (R034) --- lib/daemon/__tests__/chat-handlers.test.ts | 42 +++++++++++++++++++++- lib/daemon/handlers/chat.ts | 20 +++++++++-- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/lib/daemon/__tests__/chat-handlers.test.ts b/lib/daemon/__tests__/chat-handlers.test.ts index 9e8cfd31..4dca43ee 100644 --- a/lib/daemon/__tests__/chat-handlers.test.ts +++ b/lib/daemon/__tests__/chat-handlers.test.ts @@ -2,7 +2,7 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; import type { Database } from "bun:sqlite"; import { tmpdir } from "os"; import { join } from "path"; -import { openStateDb } from "../../state/index.ts"; +import { openStateDb, postMessage } from "../../state/index.ts"; import { createChatHandlers, inviteText } from "../handlers/chat.ts"; import { herdrRequest } from "../../herdr/client.ts"; import { fakeHerdr, HerdrFakeError, type FakeHerdrHandler } from "../../herdr/__tests__/fake-herdr.ts"; @@ -187,6 +187,46 @@ test("chat:unread-waking reports what would wake a handle without advancing its expect(res2.data).toEqual(first); }); +// R034: `limit: -1` reaches `ORDER BY id ASC LIMIT ?`, where SQLite treats a +// negative LIMIT as unlimited, so a viewer/agent bug returns and +// JSON-serializes an entire (100k-row) room on the event loop. +test("chat:messages clamps a negative limit into [1,500] instead of reaching SQLite's unlimited LIMIT", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + for (let i = 0; i < 502; i++) postMessage({ room: "r", handle: "a", body: `msg ${i}` }, h.db); + const res = await h["chat:messages"]({ room: "r", limit: -1 }); + if (!res.ok) throw new Error("unreachable"); + expect(res.data.messages.length).toBeGreaterThanOrEqual(1); + expect(res.data.messages.length).toBeLessThanOrEqual(500); +}); + +test("chat:messages clamps an absurdly large limit to the cap", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + for (let i = 0; i < 502; i++) postMessage({ room: "r", handle: "a", body: `msg ${i}` }, h.db); + const res = await h["chat:messages"]({ room: "r", limit: 1_000_000 }); + if (!res.ok) throw new Error("unreachable"); + expect(res.data.messages.length).toBe(500); +}); + +test("chat:messages coerces a non-numeric limit to the default instead of a datatype 500", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + await h["chat:post"]({ room: "r", handle: "a", body: "hi" }); + const res = await h["chat:messages"]({ room: "r", limit: "lots" as any }); + expect(res.ok).toBe(true); +}); + +test("chat:read clamps a negative limit into [1,500] rather than reaching SQLite's unlimited LIMIT", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "b" }); // b's own cursor starts before every message below + for (let i = 0; i < 502; i++) postMessage({ room: "r", handle: "a", body: `msg ${i}` }, h.db); + const res = await h["chat:read"]({ handle: "b", limit: -1 } as any); + if (!res.ok) throw new Error("unreachable"); + expect(res.data.rooms[0]!.messages.length).toBeGreaterThanOrEqual(1); + expect(res.data.rooms[0]!.messages.length).toBeLessThanOrEqual(500); +}); + test("the read-only handlers mutate nothing", async () => { const h = freshHandlers(); await h["chat:join"]({ room: "r", handle: "a" }); diff --git a/lib/daemon/handlers/chat.ts b/lib/daemon/handlers/chat.ts index 5d71c393..3f4b9d52 100644 --- a/lib/daemon/handlers/chat.ts +++ b/lib/daemon/handlers/chat.ts @@ -97,6 +97,22 @@ function closestRoomNames(typo: string, handle: string, db: Database): string[] return known.filter((r) => r.startsWith(typo) || typo.startsWith(r)).slice(0, 3); } +/** + * `limit: -1` reaches `ORDER BY id ASC LIMIT ?`, where SQLite treats a + * negative LIMIT as unlimited, so a viewer/agent bug returns and + * JSON-serializes an entire (100k-row) room on the event loop; a + * non-numeric limit hits a datatype-mismatch SQLite error instead. + * Mirrors the events handler's `num()` coercion pattern. + */ +const MAX_CHAT_LIMIT = 500; + +function clampLimit(v: unknown, fallback: number): number { + if (v == null || v === "") return fallback; + const n = Number(v); + if (!Number.isFinite(n)) return fallback; + return Math.min(Math.max(Math.trunc(n), 1), MAX_CHAT_LIMIT); +} + /** An unchecked value here lands on chat_members (and, on a join-creates, chat_room_defaults for every future joiner) and is silently treated as mention-only — never "none", never "all", never reported. */ const VALID_WAKE_ON = ["mention", "all", "none"] as const; @@ -246,7 +262,7 @@ export function createChatHandlers(opts: { "chat:read": async (payload: Commands["chat:read"]["payload"]): Promise> => { const { handle, room, limit, sinceMs } = payload; - const rooms = readUnread({ handle, room, limit: limit ?? 20, sinceMs }, db); + const rooms = readUnread({ handle, room, limit: clampLimit(limit, 20), sinceMs }, db); return { ok: true, data: { rooms } }; }, @@ -298,7 +314,7 @@ export function createChatHandlers(opts: { "chat:messages": async (payload: Commands["chat:messages"]["payload"]): Promise> => { const { room, before, limit } = payload; - const messages = listMessages({ room, before, limit: limit ?? 50 }, db); + const messages = listMessages({ room, before, limit: clampLimit(limit, 50) }, db); return { ok: true, data: { messages } }; }, From 0bdb7e8f84afec0a79f45570127eb1afb17f3b17 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:31:26 -0500 Subject: [PATCH 047/106] daemon: coerce REST GET query params to number/boolean at the seam (S085) --- .../__tests__/api-server-query-coerce.test.ts | 35 +++++++++++++++++++ lib/daemon/api-server.ts | 23 +++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 lib/daemon/__tests__/api-server-query-coerce.test.ts diff --git a/lib/daemon/__tests__/api-server-query-coerce.test.ts b/lib/daemon/__tests__/api-server-query-coerce.test.ts new file mode 100644 index 00000000..f3f28cb3 --- /dev/null +++ b/lib/daemon/__tests__/api-server-query-coerce.test.ts @@ -0,0 +1,35 @@ +import { describe, test, expect } from "bun:test"; +import { coerceQueryParams } from "../api-server.ts"; + +describe("coerceQueryParams", () => { + test("coerces maxAgeMs to a number (the documented cache:read flag)", () => { + const out = coerceQueryParams(new URLSearchParams("maxAgeMs=60000")); + expect(out.maxAgeMs).toBe(60000); + expect(typeof out.maxAgeMs).toBe("number"); + }); + + test("coerces refresh=true to a boolean (the documented ports flag)", () => { + const out = coerceQueryParams(new URLSearchParams("refresh=true")); + expect(out.refresh).toBe(true); + }); + + test("coerces refresh=false to a boolean false, not a truthy string", () => { + const out = coerceQueryParams(new URLSearchParams("refresh=false")); + expect(out.refresh).toBe(false); + }); + + test("leaves a non-numeric, non-boolean string alone", () => { + const out = coerceQueryParams(new URLSearchParams("repo=my-repo-name")); + expect(out.repo).toBe("my-repo-name"); + }); + + test("leaves an empty string alone rather than coercing to 0", () => { + const out = coerceQueryParams(new URLSearchParams("q=")); + expect(out.q).toBe(""); + }); + + test("coerces a decimal number too", () => { + const out = coerceQueryParams(new URLSearchParams("ratio=1.5")); + expect(out.ratio).toBe(1.5); + }); +}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index baef3c04..1f0dc0f0 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -187,6 +187,27 @@ export function pathParam(pathname: string, prefix: string, suffix = ""): string } } +const PLAIN_NUMBER_RE = /^-?\d+(\.\d+)?$/; + +/** + * REST query strings arrive as strings no matter what the client meant + * (S085): "?maxAgeMs=60000" and "?refresh=true" reached handlers that do a + * strict `typeof x === "number"` or `x === true` check, so the documented + * flag silently no-op'd over HTTP while working over the socket (where + * payloads are real JSON). One coercion at the REST seam fixes every such + * flag at once instead of a per-handler private parser. + */ +export function coerceQueryParams(params: URLSearchParams): Record { + const out: Record = {}; + for (const [key, value] of params) { + if (value === "true") out[key] = true; + else if (value === "false") out[key] = false; + else if (value !== "" && PLAIN_NUMBER_RE.test(value)) out[key] = Number(value); + else out[key] = value; + } + return out; +} + export interface ApiServerDeps { handleCommand: (cmd: string, payload: any, signal?: AbortSignal) => Promise; log: Logger; @@ -336,7 +357,7 @@ export async function startApiServer(deps: ApiServerDeps): Promise> if (req.method === "POST") { try { payload = await req.json(); } catch { /* empty body */ } } else { - payload = Object.fromEntries(url.searchParams); + payload = coerceQueryParams(url.searchParams); } const result = await handleCommand(route.cmd, payload, req.signal); From d96051b2b207250b48e4590c357d5fcc2053b69f Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:34:20 -0500 Subject: [PATCH 048/106] git-async: bump the 3 inline runGit(status/stash pop) call sites to MUTATING_TIMEOUT_MS Closes the S104 gap left by fd935d75: freshenOne's stash-pop reapply and autoReturnMain's two status --porcelain checks called runGit directly rather than through the helpers, so they were still on the 60s default. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/daemon/worktree-reconciler.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/daemon/worktree-reconciler.ts b/lib/daemon/worktree-reconciler.ts index 8bf4c1b5..3147d73f 100644 --- a/lib/daemon/worktree-reconciler.ts +++ b/lib/daemon/worktree-reconciler.ts @@ -452,14 +452,14 @@ async function autoReturnMain( } } - const status = await runGit(rec.path, ["status", "--porcelain"]); + const status = await runGit(rec.path, ["status", "--porcelain"], { timeoutMs: MUTATING_TIMEOUT_MS }); if (status.exitCode !== 0) { log.warn({ ...fields, output: status.stderr.trim() }, "auto-return: git status failed"); return "retry"; } if (status.stdout.trim().length > 0) { await stashChangesAsync(rec.path, mergedBranch); - const after = await runGit(rec.path, ["status", "--porcelain"]); + const after = await runGit(rec.path, ["status", "--porcelain"], { timeoutMs: MUTATING_TIMEOUT_MS }); if (after.exitCode !== 0 || after.stdout.trim().length > 0) { log.warn({ ...fields }, "auto-return: stash did not clear the worktree"); return "retry"; @@ -745,7 +745,7 @@ async function freshenOne(deps: FreshenDeps, rec: TreeRecord): Promise const popStash = async (): Promise => { if (!stashName) return; - const pop = await runGit(rec.path, ["stash", "pop", stashName]); + const pop = await runGit(rec.path, ["stash", "pop", stashName], { timeoutMs: MUTATING_TIMEOUT_MS }); if (pop.exitCode !== 0) { log.warn( { ...fields, stashName }, From 746515df55ca89a91d7578f239f6983258914ede Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:34:21 -0500 Subject: [PATCH 049/106] discussions-diffs test: fix fetch cast to satisfy tsc --- lib/daemon/__tests__/discussions-diffs.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/daemon/__tests__/discussions-diffs.test.ts b/lib/daemon/__tests__/discussions-diffs.test.ts index 7ea75ddf..bfd945c1 100644 --- a/lib/daemon/__tests__/discussions-diffs.test.ts +++ b/lib/daemon/__tests__/discussions-diffs.test.ts @@ -12,27 +12,27 @@ function fakeDiffs(n: number) { } test("truncated is false when fewer than a full page comes back", async () => { - const fetchFn = (async () => new Response(JSON.stringify(fakeDiffs(3)), { status: 200 })) as typeof fetch; + const fetchFn = (async () => new Response(JSON.stringify(fakeDiffs(3)), { status: 200 })) as unknown as typeof fetch; const out = await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); expect(out.diffs).toHaveLength(3); expect(out.truncated).toBe(false); }); test("truncated is true when exactly a full page (100) comes back", async () => { - const fetchFn = (async () => new Response(JSON.stringify(fakeDiffs(100)), { status: 200 })) as typeof fetch; + const fetchFn = (async () => new Response(JSON.stringify(fakeDiffs(100)), { status: 200 })) as unknown as typeof fetch; const out = await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); expect(out.diffs).toHaveLength(100); expect(out.truncated).toBe(true); }); test("maps new_path/diff to newPath/diff", async () => { - const fetchFn = (async () => new Response(JSON.stringify([{ new_path: "a.ts", diff: "@@" }]), { status: 200 })) as typeof fetch; + const fetchFn = (async () => new Response(JSON.stringify([{ new_path: "a.ts", diff: "@@" }]), { status: 200 })) as unknown as typeof fetch; const out = await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); expect(out.diffs).toEqual([{ newPath: "a.ts", diff: "@@" }]); }); test("a non-ok response throws with the status", async () => { - const fetchFn = (async () => new Response("", { status: 502 })) as typeof fetch; + const fetchFn = (async () => new Response("", { status: 502 })) as unknown as typeof fetch; await expect(fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn })).rejects.toThrow(/502/); }); From 2bd75c66a3284c5142b0c9662464de8af7b3537e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:37:34 -0500 Subject: [PATCH 050/106] daemon: log the EADDRINUSE port holder and throw a typed ApiPortInUseError --- lib/daemon/__tests__/api-server-bind.test.ts | 66 +++++++++++++++++++- lib/daemon/api-server.ts | 38 ++++++++++- 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/lib/daemon/__tests__/api-server-bind.test.ts b/lib/daemon/__tests__/api-server-bind.test.ts index 3fc417f4..44139ac4 100644 --- a/lib/daemon/__tests__/api-server-bind.test.ts +++ b/lib/daemon/__tests__/api-server-bind.test.ts @@ -1,5 +1,6 @@ import { describe, test, expect } from "bun:test"; import { bindApiServerWithRetry, BIND_RETRY_ATTEMPTS, BIND_RETRY_DELAY_MS, type BindRetryDeps } from "../api-server.ts"; +import { ApiPortInUseError } from "../api-server.ts"; function eaddrinuse(): Error { return Object.assign(new Error("EADDRINUSE"), { code: "EADDRINUSE" }); @@ -11,6 +12,7 @@ function deps(overrides: Partial = {}): BindRetryDeps & { logs: s return { sleep: async (ms: number) => { sleeps.push(ms); }, log: { warn: (_o: unknown, m: string) => logs.push(`warn:${m}`) }, + probePortHolder: async () => "n/a", logs, sleeps, ...overrides, @@ -41,7 +43,7 @@ describe("bindApiServerWithRetry", () => { expect(d.logs.some((l) => l.includes("retrying"))).toBe(true); }); - test("exhausting retries rethrows the original error after exactly BIND_RETRY_ATTEMPTS calls", async () => { + test("exhausting retries rethrows as ApiPortInUseError after exactly BIND_RETRY_ATTEMPTS calls", async () => { const d = deps(); let calls = 0; await expect( @@ -61,3 +63,65 @@ describe("bindApiServerWithRetry", () => { expect(d.sleeps).toEqual([]); }); }); + +function depsWithProbe(overrides: Partial = {}) { + const logs: Array<{ o: unknown; m: string }> = []; + const sleeps: number[] = []; + const probeCalls: number[] = []; + return { + sleep: async (ms: number) => { sleeps.push(ms); }, + log: { warn: (o: unknown, m: string) => logs.push({ o, m }) }, + probePortHolder: async (port: number) => { probeCalls.push(port); return "COMMAND PID USER\nnode 123 matt"; }, + logs, + sleeps, + probeCalls, + ...overrides, + }; +} + +describe("bindApiServerWithRetry — exhausted retries (S043)", () => { + test("throws ApiPortInUseError (not the raw EADDRINUSE Error) once attempts are exhausted", async () => { + const d = depsWithProbe(); + let error: unknown; + try { + await bindApiServerWithRetry(() => { throw eaddrinuse(); }, d); + } catch (err) { + error = err; + } + expect(error).toBeInstanceOf(ApiPortInUseError); + expect((error as ApiPortInUseError).code).toBe("EADDRINUSE"); + expect((error as Error).message).toContain("EADDRINUSE"); + }); + + test("probes the port holder exactly once, only after the final attempt", async () => { + const d = depsWithProbe(); + let calls = 0; + await expect( + bindApiServerWithRetry(() => { calls++; throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + expect(calls).toBe(BIND_RETRY_ATTEMPTS); + expect(d.probeCalls.length).toBe(1); + }); + + test("logs the probe result at warn before throwing", async () => { + const d = depsWithProbe(); + await expect( + bindApiServerWithRetry(() => { throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + const finalWarn = d.logs.at(-1)!; + expect(finalWarn.o).toMatchObject({ holder: expect.stringContaining("node") }); + }); + + test("a probe failure does not prevent the ApiPortInUseError from being thrown", async () => { + const d = depsWithProbe({ probePortHolder: async () => { throw new Error("lsof: command not found"); } }); + await expect( + bindApiServerWithRetry(() => { throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + }); + + test("a successful bind never probes", async () => { + const d = depsWithProbe(); + await bindApiServerWithRetry(() => "server" as any, d); + expect(d.probeCalls.length).toBe(0); + }); +}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index 1f0dc0f0..6c5bd2c4 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -13,6 +13,7 @@ import { API_PORT } from "../daemon-config.ts"; import { needsToken, tokenOk, getApiToken, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "./api-auth.ts"; import { getAggregatedConnection } from "./freshness.ts"; import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; +import { runCapture } from "../subprocess.ts"; const API_INDEX = { name: "rt daemon", @@ -213,9 +214,34 @@ export interface ApiServerDeps { log: Logger; } +/** + * Thrown when every bind retry is exhausted with EADDRINUSE still held. A + * named error type rather than the raw EADDRINUSE Error, so lib/daemon.ts's + * caller can distinguish "the port is genuinely squatted" from any other + * startup failure and park-and-retry with backoff instead of crash-looping + * (that caller-side change belongs to a sibling job; this class is the + * contract it wires into). + */ +export class ApiPortInUseError extends Error { + readonly code = "EADDRINUSE" as const; + readonly port: number; + constructor(port: number) { + super(`EADDRINUSE: api server port ${port} is still in use after ${BIND_RETRY_ATTEMPTS} bind attempts`); + this.name = "ApiPortInUseError"; + this.port = port; + } +} + export interface BindRetryDeps { sleep: (ms: number) => Promise; log: { warn: (o: unknown, m: string) => void }; + /** Defaults to a real `lsof -i :` via runCapture; overridable so tests never shell out. */ + probePortHolder?: (port: number) => Promise; +} + +async function defaultProbePortHolder(port: number): Promise { + const result = await runCapture(["lsof", "-i", `:${port}`], { timeoutMs: 5_000, stderr: "pipe" }); + return result.stdout.trim(); } // evictStaleDaemon (lib/daemon/boot-reconcile.ts) already assumes a prior @@ -231,14 +257,24 @@ export const BIND_RETRY_DELAY_MS = 500; * daemon can reach this bind before the old one has actually released * 9401. Only EADDRINUSE is retried (bounded, ~3s total); anything else is a * real misconfiguration and fails on the first attempt, same as before. + * + * Once retries are exhausted, this logs who holds the port and throws + * ApiPortInUseError instead of the bare EADDRINUSE Error, so a caller can + * tell "give up cleanly" apart from "the bind function itself is broken". */ export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDeps): Promise { + const probe = deps.probePortHolder ?? defaultProbePortHolder; for (let attempt = 1; ; attempt++) { try { return bind(); } catch (err) { const isAddrInUse = err instanceof Error && (err as NodeJS.ErrnoException).code === "EADDRINUSE"; - if (!isAddrInUse || attempt >= BIND_RETRY_ATTEMPTS) throw err; + if (!isAddrInUse) throw err; + if (attempt >= BIND_RETRY_ATTEMPTS) { + const holder = await probe(API_PORT).catch((probeErr) => `lsof failed: ${String(probeErr)}`); + deps.log.warn({ port: API_PORT, holder }, "api port still in use after retries; giving up bind (the daemon should park and retry with backoff rather than crash-loop)"); + throw new ApiPortInUseError(API_PORT); + } deps.log.warn({ attempt, port: API_PORT }, "api port in use, retrying — another daemon is likely still shutting down"); await deps.sleep(BIND_RETRY_DELAY_MS); } From 150486f2f924525ab75d38eddf486791bd0f61b6 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:39:46 -0500 Subject: [PATCH 051/106] cache-refresh: async git + grant-gated doppler loop; add listWorktreeRootsAsync (S008, S045, S021) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../git-worktree-roots-async.test.ts | 20 +++++++++++ lib/daemon/__tests__/cache-refresh-gc.test.ts | 12 ++++--- lib/daemon/cache-refresh.ts | 36 ++++++++----------- lib/worktree/git-async.ts | 6 ++++ 4 files changed, 47 insertions(+), 27 deletions(-) create mode 100644 lib/__tests__/git-worktree-roots-async.test.ts diff --git a/lib/__tests__/git-worktree-roots-async.test.ts b/lib/__tests__/git-worktree-roots-async.test.ts new file mode 100644 index 00000000..3c933432 --- /dev/null +++ b/lib/__tests__/git-worktree-roots-async.test.ts @@ -0,0 +1,20 @@ +import { test, expect } from "bun:test"; +import { mkdtempSync } from "fs"; +import { realpathSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { listWorktreeRootsAsync } from "../worktree/git-async.ts"; +import { runGit } from "../worktree/git-async.ts"; + +test("listWorktreeRootsAsync returns the main worktree path", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-wt-"))); + await runGit(dir, ["init", "-q"]); + await runGit(dir, ["commit", "--allow-empty", "-m", "init", "-c", "user.email=a@b.c", "-c", "user.name=t"]); + const roots = await listWorktreeRootsAsync(dir); + expect(roots).toContain(dir); +}); + +test("listWorktreeRootsAsync returns [] on a non-repo", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-nonrepo-"))); + expect(await listWorktreeRootsAsync(dir)).toEqual([]); +}); diff --git a/lib/daemon/__tests__/cache-refresh-gc.test.ts b/lib/daemon/__tests__/cache-refresh-gc.test.ts index db295330..a959a4d1 100644 --- a/lib/daemon/__tests__/cache-refresh-gc.test.ts +++ b/lib/daemon/__tests__/cache-refresh-gc.test.ts @@ -31,7 +31,7 @@ import { join } from "path"; import type { Logger } from "pino"; import * as enrichModule from "../../enrich.ts"; -import * as gitWorktreesModule from "../../git-worktrees.ts"; +import * as gitAsync from "../../worktree/git-async.ts"; import * as notifierModule from "../../notifier.ts"; import * as repoTrackingModule from "../../repo-tracking.ts"; import * as discussionsModule from "../discussions-file-store.ts"; @@ -101,11 +101,13 @@ function wireCycle(): Wiring { }); // A real git tree is irrelevant to the GC claim; one branch per repo is - // enough to make the loop reach the enrichment call. - spyOn(gitWorktreesModule, "listWorktrees").mockImplementation((repoPath: string) => [ - { path: repoPath, branch: `wt-${repoPath}` } as any, + // enough to make the loop reach the enrichment call and let FLAKY's onError + // fire (an empty branch list would short-circuit refreshAllMRs entirely and + // make FLAKY look clean, defeating the test). + spyOn(gitAsync, "listWorktreesAsync").mockImplementation(async (repoPath: string) => [ + { path: repoPath, branch: `wt-${repoPath}` }, ]); - spyOn(gitWorktreesModule, "listWorktreeRoots").mockReturnValue([]); + spyOn(gitAsync, "listWorktreeRootsAsync").mockResolvedValue([]); spyOn(enrichModule, "refreshAllMRs").mockImplementation( async (_branches, _remoteUrl, onError, repoName) => { diff --git a/lib/daemon/cache-refresh.ts b/lib/daemon/cache-refresh.ts index 70194bd9..0ab58fc7 100644 --- a/lib/daemon/cache-refresh.ts +++ b/lib/daemon/cache-refresh.ts @@ -6,13 +6,12 @@ * * Concurrent callers are coalesced: the 5-minute timer and `cache:refresh` IPC * both fire-and-forget into the returned function. Without a guard they stack - * up, each running execSync across every repo + a batch GraphQL. If a refresh + * up, each running async git across every repo + a batch GraphQL. If a refresh * is already in flight, callers await the existing run instead of starting a * second one. */ import { existsSync } from "fs"; -import { execSync } from "child_process"; import type { Logger } from "pino"; import type { PortCacheRef, RepoIndex } from "./handlers/types.ts"; import type { BranchCacheStore } from "../state/index.ts"; @@ -24,7 +23,7 @@ import { getProjectMRs } from "./project-mrs-store.ts"; import { pruneDiscussionsStore } from "./discussions-file-store.ts"; import { reconcileForRepo } from "./doppler-sync.ts"; import { deriveRepoIdentity } from "../settings/identity.ts"; -import { listWorktreeRoots, listWorktrees } from "../git-worktrees.ts"; +import { listWorktreesAsync, listWorktreeRootsAsync, runGit } from "../worktree/git-async.ts"; export interface CacheRefresherDeps { log: Logger; @@ -109,37 +108,29 @@ export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise = listWorktrees(repoPath).filter( - (w) => w.branch && !w.branch.startsWith("on-deck/"), - ); + const branches: Array<{ path: string; branch: string }> = ((await listWorktreesAsync(repoPath)) ?? []) + .filter((w): w is { path: string; branch: string } => !!w.branch && !w.branch.startsWith("on-deck/")); // 2. Discover local branches (not just worktrees) const worktreeBranchSet = new Set(branches.map(b => b.branch)); - try { - const localBranchOutput = execSync( - "git for-each-ref --format='%(refname:short)' refs/heads/", - { cwd: repoPath, encoding: "utf8", stdio: "pipe" }, - ); - - for (const name of localBranchOutput.split("\n")) { - const trimmed = name.trim().replace(/^'|'$/g, ""); + const localBranches = await runGit(repoPath, ["for-each-ref", "--format=%(refname:short)", "refs/heads/"]); + if (localBranches.exitCode === 0) { + for (const name of localBranches.stdout.split("\n")) { + const trimmed = name.trim(); if (!trimmed || worktreeBranchSet.has(trimmed) || trimmed.startsWith("on-deck/")) continue; if (extractLinearId(trimmed)) { branches.push({ path: repoPath, branch: trimmed }); } } - } catch (err) { - log.warn({ err, repo: repoPath }, "local branch listing failed"); + } else { + log.warn({ repo: repoPath }, "local branch listing failed"); } if (branches.length > 0) { // Get remote URL let remoteUrl: string | undefined; - try { - remoteUrl = execSync("git config --get remote.origin.url", { - cwd: repoPath, encoding: "utf8", stdio: "pipe", - }).trim(); - } catch { /* no remote */ } + const remote = await runGit(repoPath, ["config", "--get", "remote.origin.url"]); + if (remote.exitCode === 0) remoteUrl = remote.stdout.trim() || undefined; // Optimized: 3 GraphQL calls for ALL open MRs + 1 Linear batch. // The onError callback fires on per-MR enrich failures (GitLab, @@ -205,8 +196,9 @@ export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise { + return (await listWorktreesAsync(repoPath) ?? []).map((w) => w.path); +} + /** * Idempotently append `pattern` to the common git dir's info/exclude, with a * "# rt worktree" marker comment written on first use of this file. Returns From f292058ec5648dc66e0e84842a0159ade7299440 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:40:46 -0500 Subject: [PATCH 052/106] daemon: bind API before socket; register rt.apiPort setting + resolveApiPort() Binding the API server first means a fatal API-bind failure never strands a socket-bound zombie behind it. rt.apiPort is the escape-hatch setting the api-server sibling consumes at bind time; resolveApiPort() resolves it lazily (env > setting > 9401) without disturbing the existing API_PORT const api-server.ts already imports. --- e2e/tests/daemon.test.ts | 20 +++++++++++++++++++ lib/__tests__/daemon-config.test.ts | 14 ++++++++++++- lib/daemon-config.ts | 17 ++++++++++++++++ lib/daemon.ts | 6 ++++-- .../src/settings/__tests__/registry.test.ts | 6 ++++-- .../rt-client/src/settings/registry-defs.ts | 9 +++++++++ 6 files changed, 67 insertions(+), 5 deletions(-) diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index 6b4af74d..e0403285 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -37,6 +37,26 @@ describe("fatal boot", () => { } }, 60_000); + test("API-bind failure leaves neither rt.sock nor rt.pid", async () => { + const { path: home, cleanup } = createTestHome(); + // A different port than the sibling squatter test above, so parallel + // test files can never collide on the same bound TCP port. + const port = 9412; + const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + try { + const result = await rt(["--daemon"], { home, env: { RT_API_PORT: String(port) } }); + + expect(result.exitCode).not.toBe(0); + // The API bind (now first) fails before the socket ever binds, so a + // fatal exit must strand neither file. + expect(existsSync(join(home, ".mattstack", "rt", "rt.sock"))).toBe(false); + expect(existsSync(join(home, ".mattstack", "rt", "rt.pid"))).toBe(false); + } finally { + squatter.stop(true); + cleanup(); + } + }, 60_000); + test("a corrupt events.db self-heals — quarantined, and the daemon boots and serves", async () => { const { path: home, cleanup } = createTestHome(); const bunDir = join(process.execPath, ".."); diff --git a/lib/__tests__/daemon-config.test.ts b/lib/__tests__/daemon-config.test.ts index 21fc45b5..32f06d15 100644 --- a/lib/__tests__/daemon-config.test.ts +++ b/lib/__tests__/daemon-config.test.ts @@ -9,7 +9,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { mkdirSync, rmSync, writeFileSync } from "fs"; import { join } from "path"; -import { activeLaunchdLabel } from "../daemon-config.ts"; +import { activeLaunchdLabel, resolveApiPort } from "../daemon-config.ts"; const WRAPPER_PATH = join(process.env.HOME!, ".local", "bin", "rt"); @@ -28,3 +28,15 @@ describe("activeLaunchdLabel", () => { expect(activeLaunchdLabel()).toBe("com.mattstack.daemon.dev"); }); }); + +describe("resolveApiPort", () => { + test("env wins, then setting, then 9401", () => { + const prev = process.env.RT_API_PORT; + process.env.RT_API_PORT = "12345"; + expect(resolveApiPort()).toBe(12345); + delete process.env.RT_API_PORT; + expect(resolveApiPort()).toBe(9401); // default setting value + if (prev !== undefined) process.env.RT_API_PORT = prev; + else delete process.env.RT_API_PORT; + }); +}); diff --git a/lib/daemon-config.ts b/lib/daemon-config.ts index 4164f10e..52f4a246 100644 --- a/lib/daemon-config.ts +++ b/lib/daemon-config.ts @@ -12,6 +12,7 @@ import { homedir } from "os"; import { dirname, join } from "path"; import { rtDir } from "./rt-paths.ts"; import { currentMode } from "./dev-mode.ts"; +import { getSetting } from "./settings/resolve.ts"; // ─── Types ─────────────────────────────────────────────────────────────────── @@ -71,6 +72,22 @@ export const TRAY_SOCK_PATH = join(RT_DIR, "tray.sock"); * daemon's hardcoded port (RT-45). */ export const API_PORT = Number(process.env.RT_API_PORT) || 9401; +/** + * Call-time API port resolution: RT_API_PORT env wins (e2e isolation, RT-45), + * then the rt.apiPort setting (escape hatch when 9401 is held), then 9401. + * A function, not a const — must never be evaluated at module load, since + * getSetting() reads the settings stores off ambient HOME. + */ +export function resolveApiPort(): number { + const env = Number(process.env.RT_API_PORT); + if (env) return env; + try { + return getSetting("rt.apiPort").value || 9401; + } catch { + return 9401; + } +} + // ─── Read / Write ──────────────────────────────────────────────────────────── // `home`, when passed, overrides the module-load `DAEMON_CONFIG_PATH` const diff --git a/lib/daemon.ts b/lib/daemon.ts index ea49efa2..d2997eb0 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -479,9 +479,11 @@ async function runDaemon(): Promise { persistOrWarn("daemon", () => { prunedPresence = prunePresence(Date.now()); }, { op: "prunePresence" }); if (prunedPresence > 0) log.info({ prunedPresence }, "chat: pruned stale presence rows at daemon startup"); - // Socket server (Unix socket for CLI/tray) + REST/WS server (external clients) - servers.socket = startSocketServer({ handleCommand, log }); + // API server first: a failed bind exits fatally (boot-phase catch below), + // and binding API before the unix socket means that fatal exit never + // strands a socket-bound zombie behind it. servers.api = await startApiServer({ handleCommand, log }); + servers.socket = startSocketServer({ handleCommand, log }); // Only write rt.pid once both servers are actually bound — a boot that // fails before this point must never leave a live-pid file with no diff --git a/packages/rt-client/src/settings/__tests__/registry.test.ts b/packages/rt-client/src/settings/__tests__/registry.test.ts index 1b534ea0..6500c111 100644 --- a/packages/rt-client/src/settings/__tests__/registry.test.ts +++ b/packages/rt-client/src/settings/__tests__/registry.test.ts @@ -51,7 +51,7 @@ describe("settings/registry", () => { } }); - test("exactly 22 keys are migrated:true", () => { + test("exactly 23 keys are migrated:true", () => { const migrated = allDefs().filter((d) => d.migrated); expect(migrated.map((d) => d.key).sort()).toEqual( @@ -60,6 +60,7 @@ describe("settings/registry", () => { "rt.notifications", "rt.cron", "rt.repoTracking", "rt.runsPruneDays", "rt.runaway", "rt.workspacePrefs", "rt.sync", "rt.branchNaming", "rt.variations", "rt.presets", "rt.dopplerTemplate", "rt.homeSnapshot", "rt.worktreeApp", "rt.sdmEnrichment", "rt.logRetentionDays", "rt.integrations", "rt.hooks", + "rt.apiPort", ].sort(), ); }); @@ -197,13 +198,14 @@ describe("settings/registry", () => { expect(def?.merge).toBe("replace"); }); - test("has exactly the 22 migrated:true keys and the 42 suite keys", () => { + test("has exactly the 23 migrated:true keys and the 42 suite keys", () => { const migratedFalseKeys: string[] = []; const migratedTrueKeys = [ "rt.roles", "rt.intercepts", "rt.worktrees", "rt.repoIdentityOverrides", "rt.repoRoots", "rt.notifications", "rt.cron", "rt.repoTracking", "rt.runsPruneDays", "rt.runaway", "rt.workspacePrefs", "rt.sync", "rt.branchNaming", "rt.variations", "rt.presets", "rt.dopplerTemplate", "rt.homeSnapshot", "rt.worktreeApp", "rt.sdmEnrichment", "rt.logRetentionDays", "rt.integrations", "rt.hooks", + "rt.apiPort", ]; const suiteKeys = [ "mattstack.integrations", diff --git a/packages/rt-client/src/settings/registry-defs.ts b/packages/rt-client/src/settings/registry-defs.ts index f0109ddd..9f37dfd4 100644 --- a/packages/rt-client/src/settings/registry-defs.ts +++ b/packages/rt-client/src/settings/registry-defs.ts @@ -196,6 +196,15 @@ export const REGISTRY: readonly SettingDef[] = [ migrated: true, description: "Age floor in days for the log janitor pruning every surface's rotated log files under ~/.mattstack/rt/logs (default 14). A fresh key, not an ownership-latch port, so a default is fine here.", }, + { + key: "rt.apiPort", + type: "number", + scopes: ["machine", "user"], + default: 9401, + merge: "replace", + migrated: true, + description: "TCP port the daemon's local HTTP/WS API binds (escape hatch when 9401 is held).", + }, { key: "rt.hooks", type: "object", From a458351d11dd08cf077a912738fefad54c08860e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:41:03 -0500 Subject: [PATCH 053/106] daemon: standalone git-ref validator for option-injection guard (worktree.ts wiring documented, not wired here) --- .../__tests__/git-ref-validation.test.ts | 36 +++++++++++++++++++ lib/daemon/git-ref-validation.ts | 24 +++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 lib/daemon/__tests__/git-ref-validation.test.ts create mode 100644 lib/daemon/git-ref-validation.ts diff --git a/lib/daemon/__tests__/git-ref-validation.test.ts b/lib/daemon/__tests__/git-ref-validation.test.ts new file mode 100644 index 00000000..27c9fc3d --- /dev/null +++ b/lib/daemon/__tests__/git-ref-validation.test.ts @@ -0,0 +1,36 @@ +import { describe, test, expect } from "bun:test"; +import { isSafeGitRef, validateGitRef } from "../git-ref-validation.ts"; + +describe("isSafeGitRef", () => { + test("a normal branch name is safe", () => { + expect(isSafeGitRef("feature/my-branch")).toBe(true); + }); + + test("a leading dash is unsafe (option injection)", () => { + expect(isSafeGitRef("--upload-pack=touch /tmp/x")).toBe(false); + }); + + test("a bare dash is unsafe", () => { + expect(isSafeGitRef("-")).toBe(false); + }); + + test("an empty string is unsafe", () => { + expect(isSafeGitRef("")).toBe(false); + }); + + test("a branch containing a dash mid-name is safe", () => { + expect(isSafeGitRef("job/p3-trust-boundary")).toBe(true); + }); +}); + +describe("validateGitRef", () => { + test("returns ok:true for a safe ref", () => { + expect(validateGitRef("main")).toEqual({ ok: true }); + }); + + test("returns ok:false with the offending ref named in the error for an unsafe one", () => { + const result = validateGitRef("--upload-pack=x"); + expect(result.ok).toBe(false); + expect((result as { ok: false; error: string }).error).toContain("--upload-pack=x"); + }); +}); diff --git a/lib/daemon/git-ref-validation.ts b/lib/daemon/git-ref-validation.ts new file mode 100644 index 00000000..809a733f --- /dev/null +++ b/lib/daemon/git-ref-validation.ts @@ -0,0 +1,24 @@ +/** + * Rejects a branch/ref string that git would parse as an OPTION rather than + * a ref. A caller-supplied `branch` like "--upload-pack=touch /tmp/x" reaches + * `git fetch origin ` and `git rev-list ......` unescaped, + * and git happily executes it as an option since nothing on that path + * validates the string first. This is the one guard both call sites need; + * any future caller (including a consumer app forwarding an untrusted string + * as `branch`) inherits the same hole without it. + * + * Deliberately narrow: reject a leading '-' rather than allowlisting a + * character set. `git check-ref-format --branch` accepts far more + * punctuation than is worth re-deriving here, and the vulnerable shape is + * specifically "parses as an option", not "contains an unusual character". + */ +export function isSafeGitRef(ref: string): boolean { + return ref.length > 0 && !ref.startsWith("-"); +} + +export function validateGitRef(ref: string): { ok: true } | { ok: false; error: string } { + if (!isSafeGitRef(ref)) { + return { ok: false, error: `unsafe git ref (starts with '-' or empty): ${ref}` }; + } + return { ok: true }; +} From 977c03df47fc60705f6c9d6dc2683d7cd6787fda Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:43:55 -0500 Subject: [PATCH 054/106] daemon: standalone credential-redaction utility (freshness.ts wiring documented, not wired here) --- .../__tests__/redact-credentials.test.ts | 35 +++++++++++++++++++ lib/daemon/redact-credentials.ts | 14 ++++++++ 2 files changed, 49 insertions(+) create mode 100644 lib/daemon/__tests__/redact-credentials.test.ts create mode 100644 lib/daemon/redact-credentials.ts diff --git a/lib/daemon/__tests__/redact-credentials.test.ts b/lib/daemon/__tests__/redact-credentials.test.ts new file mode 100644 index 00000000..57e5a5e2 --- /dev/null +++ b/lib/daemon/__tests__/redact-credentials.test.ts @@ -0,0 +1,35 @@ +import { describe, test, expect } from "bun:test"; +import { redactCredentials } from "../redact-credentials.ts"; + +describe("redactCredentials", () => { + test("redacts userinfo (user:token@) out of an https remote URL", () => { + const input = "https://oauth2:glpat-XXXXXXXXXXXXXXXXXXXX@gitlab.example.com/g/p.git"; + const out = redactCredentials(input); + expect(out).not.toContain("glpat-XXXXXXXXXXXXXXXXXXXX"); + expect(out).toContain("gitlab.example.com/g/p.git"); + }); + + test("redacts a GitHub PAT embedded in the URL", () => { + const input = "https://ghp_abcdefghijklmnopqrstuvwxyz012345@github.com/o/r.git"; + const out = redactCredentials(input); + expect(out).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz012345"); + expect(out).toContain("github.com/o/r.git"); + }); + + test("leaves a URL with no embedded credentials unchanged", () => { + const input = "https://gitlab.example.com/g/p.git"; + expect(redactCredentials(input)).toBe(input); + }); + + test("leaves plain text with no URL unchanged", () => { + const input = "local branch listing failed"; + expect(redactCredentials(input)).toBe(input); + }); + + test("redacts every match when more than one credential-bearing URL appears in the same string", () => { + const input = "tried https://oauth2:tok1@a.example/x then https://oauth2:tok2@b.example/y"; + const out = redactCredentials(input); + expect(out).not.toContain("tok1"); + expect(out).not.toContain("tok2"); + }); +}); diff --git a/lib/daemon/redact-credentials.ts b/lib/daemon/redact-credentials.ts new file mode 100644 index 00000000..2ac64c87 --- /dev/null +++ b/lib/daemon/redact-credentials.ts @@ -0,0 +1,14 @@ +/** + * Strips userinfo (user:token@ or user@) out of any http(s) URL embedded in + * a string. freshness.ts logs `remote.origin.url` verbatim on every + * reconcile and echoes it into thrown errors returned to callers; a repo + * cloned as `https://oauth2:glpat-XXXX@gitlab.example.com/...` (routine for + * dotfiles/CI-derived clones) puts that token into ~/.rt/logs/daemon.*.log + * and into any client-facing error message. Logs are the first thing a user + * pastes into a bug report. + */ +const CREDENTIAL_URL_RE = /(https?:\/\/)[^/@\s]+@/gi; + +export function redactCredentials(text: string): string { + return text.replace(CREDENTIAL_URL_RE, "$1[redacted]@"); +} From 7d1a2fad26a451cc5d22db220145f2965df8dfaf Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:44:49 -0500 Subject: [PATCH 055/106] freshness: async, cached getRemoteUrl via runCapture (R032) getRemoteUrl used execSync on the daemon thread inside every forge handler and the freshness reconcile loop. Switch to runCapture (5s timeout) and cache per repoPath for the process lifetime, since remotes rarely change. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/freshness-remote-url.test.ts | 9 +++++ lib/daemon/freshness.ts | 33 +++++++++++-------- 2 files changed, 29 insertions(+), 13 deletions(-) create mode 100644 lib/daemon/__tests__/freshness-remote-url.test.ts diff --git a/lib/daemon/__tests__/freshness-remote-url.test.ts b/lib/daemon/__tests__/freshness-remote-url.test.ts new file mode 100644 index 00000000..ec17a900 --- /dev/null +++ b/lib/daemon/__tests__/freshness-remote-url.test.ts @@ -0,0 +1,9 @@ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("freshness.ts no longer imports execSync/child_process", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + expect(src).not.toMatch(/from\s+["']child_process["']/); + expect(src).not.toMatch(/\bexecSync\b/); +}); diff --git a/lib/daemon/freshness.ts b/lib/daemon/freshness.ts index a5f7ceba..2b49b497 100644 --- a/lib/daemon/freshness.ts +++ b/lib/daemon/freshness.ts @@ -26,7 +26,6 @@ * disposeFreshness() — daemon shutdown */ -import { execSync } from "child_process"; import { GitLabProvider, type InvalidationKey, type MRApprovalRules, type PullRequest } from "@mattstack/glance"; import { loadRepoTracking, grants, type RepoGrants } from "../repo-tracking.ts"; import { loadSecrets } from "../linear.ts"; @@ -37,6 +36,7 @@ import { lazyChildLogger } from "../daemon-logger.ts"; import { getProjectMRs, type ProjectMRs } from "./project-mrs-store.ts"; import { getDiscussionsFileStore } from "./discussions-file-store.ts"; import { createCursorStore, type CursorStore } from "../state/index.ts"; +import { runCapture } from "../subprocess.ts"; const log = lazyChildLogger("freshness"); @@ -90,16 +90,23 @@ let userId: number | null = null; let userIdResolved = false; let selfUsername: string | null = null; +const remoteUrlCache = new Map(); + // ─── Helpers ───────────────────────────────────────────────────────────────── -function getRemoteUrl(repoPath: string): string | null { - try { - return execSync("git config --get remote.origin.url", { - cwd: repoPath, encoding: "utf8", stdio: "pipe", - }).trim(); - } catch { - return null; - } +/** remote.origin.url, cached per repoPath for the process lifetime (remotes + * rarely change). Async so it never blocks the event loop. */ +async function getRemoteUrl(repoPath: string): Promise { + const cached = remoteUrlCache.get(repoPath); + if (cached !== undefined) return cached; + const r = await runCapture(["git", "config", "--get", "remote.origin.url"], { + cwd: repoPath, + timeoutMs: 5000, + stderr: "ignore", + }); + const url = r.exitCode === 0 ? (r.stdout.trim() || null) : null; + remoteUrlCache.set(repoPath, url); + return url; } /** The same debug-accounting hook makeProvider wires, for SDK helpers rt constructs directly (NoteMutator). */ @@ -132,7 +139,7 @@ async function ensureProvider(repoName: string, repoPath: string): Promise { // startWatch rather than trust a check made before it. if (watches.has(repoName)) continue; - const remoteUrl = getRemoteUrl(repoPath); + const remoteUrl = await getRemoteUrl(repoPath); const remote = remoteUrl ? parseRemoteUrl(remoteUrl) : null; if (!remote) continue; From cf525960aee61099af1293a1e49122bcbebb614a Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:49:07 -0500 Subject: [PATCH 056/106] worktree-process-kill: async lsof/ps via runCapture (S015, S016) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/worktree-process-kill.test.ts | 8 +++ lib/daemon/worktree-process-kill.ts | 57 ++++++++----------- lib/daemon/worktree-reconciler.ts | 5 +- lib/worktree/dispose.ts | 2 +- 4 files changed, 35 insertions(+), 37 deletions(-) diff --git a/lib/daemon/__tests__/worktree-process-kill.test.ts b/lib/daemon/__tests__/worktree-process-kill.test.ts index faf05ed2..c86ebb8e 100644 --- a/lib/daemon/__tests__/worktree-process-kill.test.ts +++ b/lib/daemon/__tests__/worktree-process-kill.test.ts @@ -1,3 +1,5 @@ +import { readFileSync } from "fs"; +import { resolve } from "path"; import { describe, expect, test } from "bun:test"; import { selectKillTargets, type KillCandidate } from "../worktree-process-kill.ts"; @@ -5,6 +7,12 @@ function row(pid: number, ppid: number, command: string, fullCommand: string): K return { pid, ppid, command, fullCommand }; } +test("worktree-process-kill.ts imports no sync exec", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "worktree-process-kill.ts"), "utf8"); + expect(src).not.toMatch(/from\s+["']child_process["']/); + expect(src).not.toMatch(/\bexecSync\b/); +}); + describe("selectKillTargets", () => { test("kills package-script chains and orphaned compilers", () => { const rows = [ diff --git a/lib/daemon/worktree-process-kill.ts b/lib/daemon/worktree-process-kill.ts index ca499bc2..e41ee707 100644 --- a/lib/daemon/worktree-process-kill.ts +++ b/lib/daemon/worktree-process-kill.ts @@ -12,9 +12,8 @@ * the caller — parking proceeds while stragglers wind down. */ -import { execSync } from "child_process"; - import { lazyChildLogger } from "../daemon-logger.ts"; +import { runCapture } from "../subprocess.ts"; import { parseLsofCwdMap, parsePackageScripts } from "./system-process-scanner.ts"; const log = lazyChildLogger("worktree-kill"); @@ -126,18 +125,14 @@ export interface WorktreeKillResult { * Discovery is done fresh (not from the scanner's 10s-old snapshot) so we act * on ground truth at park time. */ -export function killWorktreeProcesses(worktreePath: string): WorktreeKillResult { - let lsofOut: string; - try { - lsofOut = execSync("lsof -d cwd -Fpn 2>/dev/null", { - encoding: "utf8", stdio: "pipe", timeout: 10000, - }); - } catch (err) { - log.warn({ err, worktreePath }, "lsof failed; skipping worktree process kill"); +export async function killWorktreeProcesses(worktreePath: string): Promise { + const lsof = await runCapture(["lsof", "-d", "cwd", "-Fpn"], { timeoutMs: 10_000 }); + if (lsof.exitCode !== 0 && !lsof.stdout) { + log.warn({ exitCode: lsof.exitCode, worktreePath }, "lsof failed; skipping worktree process kill"); return { terminated: [] }; } - const cwdMap = parseLsofCwdMap(lsofOut, [worktreePath]); + const cwdMap = parseLsofCwdMap(lsof.stdout, [worktreePath]); // Drop close-parent matches (parseLsofCwdMap keeps them for scanner display; // a process merely *above* the worktree must not be killed). for (const [pid, cwd] of cwdMap) { @@ -147,24 +142,21 @@ export function killWorktreeProcesses(worktreePath: string): WorktreeKillResult const pidList = [...cwdMap.keys()].join(","); const rows: KillCandidate[] = []; - try { - const psOut = execSync(`ps -p ${pidList} -o pid=,ppid=,comm=,args= 2>/dev/null`, { - encoding: "utf8", stdio: "pipe", timeout: 5000, - }); - for (const line of psOut.split("\n")) { - const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\S+)\s+(.*)/); - if (!match) continue; - rows.push({ - pid: parseInt(match[1]!, 10), - ppid: parseInt(match[2]!, 10), - command: basename(match[3]!), - fullCommand: match[4]!, - }); - } - } catch (err) { - log.warn({ err, worktreePath }, "ps failed; skipping worktree process kill"); + const ps = await runCapture(["ps", "-p", pidList, "-o", "pid=,ppid=,comm=,args="], { timeoutMs: 5000 }); + if (ps.exitCode !== 0 && !ps.stdout) { + log.warn({ exitCode: ps.exitCode, worktreePath }, "ps failed; skipping worktree process kill"); return { terminated: [] }; } + for (const line of ps.stdout.split("\n")) { + const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\S+)\s+(.*)/); + if (!match) continue; + rows.push({ + pid: parseInt(match[1]!, 10), + ppid: parseInt(match[2]!, 10), + command: basename(match[3]!), + fullCommand: match[4]!, + }); + } const targets = selectKillTargets(rows, { protectedPids: [process.pid, process.ppid], @@ -174,12 +166,11 @@ export function killWorktreeProcesses(worktreePath: string): WorktreeKillResult // Label with the package-script invocation when known — that's the name the // developer recognizes ("pnpm start:lite:watch", not "node"). let scripts = new Map(); - try { - const ewwOut = execSync(`ps eww -o pid=,command= -p ${targets.map(t => t.pid).join(",")} 2>/dev/null`, { - encoding: "utf8", stdio: "pipe", timeout: 5000, maxBuffer: 32 * 1024 * 1024, - }); - scripts = parsePackageScripts(ewwOut); - } catch { /* labels fall back to comm */ } + const eww = await runCapture( + ["ps", "eww", "-o", "pid=,command=", "-p", targets.map(t => t.pid).join(",")], + { timeoutMs: 5000 }, + ); + if (eww.exitCode === 0 || eww.stdout) scripts = parsePackageScripts(eww.stdout); const terminated: { pid: number; label: string }[] = []; for (const target of targets) { diff --git a/lib/daemon/worktree-reconciler.ts b/lib/daemon/worktree-reconciler.ts index 3147d73f..1746c735 100644 --- a/lib/daemon/worktree-reconciler.ts +++ b/lib/daemon/worktree-reconciler.ts @@ -442,10 +442,9 @@ async function autoReturnMain( } if (appConfig.killProcesses) { - // The ruled execSync exception (the process killer is sync by design); - // a failure here never blocks the return. + // A failure here never blocks the return. try { - const { terminated } = killWorktreeProcesses(rec.path); + const { terminated } = await killWorktreeProcesses(rec.path); if (terminated.length > 0) log.info({ ...fields, count: terminated.length }, "worktree processes terminated"); } catch (err) { log.warn({ err, ...fields }, "auto-return: process kill failed; returning anyway"); diff --git a/lib/worktree/dispose.ts b/lib/worktree/dispose.ts index 52e2bb1d..3f3efeb0 100644 --- a/lib/worktree/dispose.ts +++ b/lib/worktree/dispose.ts @@ -235,7 +235,7 @@ export async function disposeTree( } if (deps.killProcesses) { - const { terminated } = killWorktreeProcesses(rec.path); + const { terminated } = await killWorktreeProcesses(rec.path); if (terminated.length > 0) { log.info( { repo: repoName, tree: rec.name, count: terminated.length }, From 6d950e3624b5a725cb7e1573e819cf4749036115 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:50:28 -0500 Subject: [PATCH 057/106] home-snapshot: lazy daemon-flavored state.db; getStateDb re-applies a stronger flavor timeout startHomeSnapshot no longer resolves getStateDb() eagerly at construction with the default cli flavor (5000ms busy_timeout). db access is a thunk (resolveDb) defaulting to getStateDb("daemon"), first invoked inside init() after its await (past module-scope construction, so the daemon's own openBranchCacheStore() opens the singleton daemon-flavored first). getStateDb() also now re-applies PRAGMA busy_timeout when a caller requests a stronger (shorter) flavor than the already-open singleton holds, hardening against any other future eager-cli-then-daemon ordering bug. --- lib/daemon/__tests__/home-snapshot.test.ts | 37 ++++++++++++++++++++++ lib/daemon/home-snapshot.ts | 19 ++++++++--- lib/state/__tests__/db.test.ts | 7 ++++ lib/state/db.ts | 16 +++++++--- 4 files changed, 70 insertions(+), 9 deletions(-) diff --git a/lib/daemon/__tests__/home-snapshot.test.ts b/lib/daemon/__tests__/home-snapshot.test.ts index 569aa4c6..ba0bbba3 100644 --- a/lib/daemon/__tests__/home-snapshot.test.ts +++ b/lib/daemon/__tests__/home-snapshot.test.ts @@ -1005,6 +1005,43 @@ describe("startHomeSnapshot — state persistence", () => { }); }); +// ─── boot order: db must open daemon-flavored, never at construction ──────── + +describe("startHomeSnapshot — boot order", () => { + test("constructing startHomeSnapshot does not open the state.db singleton before the caller's next await", async () => { + const home = mkdtempSync(join(tmpdir(), "rt-home-snapshot-bootorder-")); + const origHome = process.env.HOME; + process.env.HOME = home; + closeStateDb(); + try { + const stateDbPath = join(home, ".mattstack", "rt", "state.db"); + const { fn: execFn } = makeFakeExec(defaultResponders({ statusZ: "?? a.txt\0" })); + // No `db` override: this exercises the real getStateDb() singleton, + // matching lib/daemon.ts's module-scope `startHomeSnapshot(...)` call — + // the exact call site that used to open state.db "cli"-flavored before + // startDaemon() ever got to openBranchCacheStore(). + const { deps } = baseDeps({ exec: execFn, db: undefined }); + + const handle = startHomeSnapshot(deps); + + // Synchronously, right after construction returns — mirroring the + // module-scope call in lib/daemon.ts, which runs to completion before + // startDaemon() (and its openBranchCacheStore() daemon-flavored open) + // is ever reached — no db file may exist yet. + expect(existsSync(stateDbPath)).toBe(false); + + await handle.ready; + // First real use (init()'s loadState, past its own `await deps.exec`) + // has by now opened it. + expect(existsSync(stateDbPath)).toBe(true); + } finally { + process.env.HOME = origHome; + closeStateDb(); + rmSync(home, { recursive: true, force: true }); + } + }); +}); + // ─── stop() ─────────────────────────────────────────────────────────────── describe("startHomeSnapshot — stop", () => { diff --git a/lib/daemon/home-snapshot.ts b/lib/daemon/home-snapshot.ts index 0d0fb56c..476fbee2 100644 --- a/lib/daemon/home-snapshot.ts +++ b/lib/daemon/home-snapshot.ts @@ -258,6 +258,11 @@ function persistPushRecord(db: Database, record: HomePushRecord, log: Logger): v export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle { const repoDir = rawDeps.repoDir ?? join(mattstackHome(), "user"); const rawReadSettings = rawDeps.readSettings ?? (() => getSetting("rt.homeSnapshot").value); + // Thunk, not a resolved value: module-scope construction (lib/daemon.ts) + // must not open state.db before startDaemon() has opened it daemon-flavored + // via openBranchCacheStore — see loadState's call site inside init() below, + // which is the first place this ever actually gets invoked. + const resolveDb = rawDeps.db ? (() => rawDeps.db!) : (() => getStateDb("daemon")); const deps = { log: rawDeps.log, broadcast: rawDeps.broadcast, @@ -269,7 +274,6 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle now: rawDeps.now ?? (() => Date.now()), readSettings: () => clampSettings(rawReadSettings()), readOwners: rawDeps.readOwners ?? readOwnersReal, - db: rawDeps.db ?? getStateDb(), }; const ownersPath = ownersPathFor(deps.repoDir); @@ -296,7 +300,8 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle let lastPushError: string | null = null; /** True once `home:push-failed` has been broadcast for the CURRENT unbroken run of push failures — reset to false the moment a push succeeds, so a retry storm broadcasts once, not on every attempt. */ let pushFailureBroadcast = false; - let firstSeenDirty: Record = loadState(deps.db, deps.log); + /** Populated in init(), after the is-inside-work-tree check — see resolveDb's comment for why this can't happen at construction time. */ + let firstSeenDirty: Record = {}; let lastLoggedOwnersError: string | null = null; /** Shared dedup key for every "deps.readSettings() itself threw" warn (armWatcher's debounce read, status()) — a settings store that broke after boot and stays broken must warn once, not on every fs event or every `rt home snapshot --status` poll. */ let lastLoggedSettingsError: string | null = null; @@ -370,6 +375,10 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle deps.log.warn({ repoDir: deps.repoDir }, "home-snapshot: repoDir is not a git repository; inert"); return; } + // First real db touch: this await already put us past the daemon's + // synchronous boot pass, so by now startDaemon() has opened state.db + // daemon-flavored via openBranchCacheStore (see resolveDb above). + firstSeenDirty = loadState(resolveDb(), deps.log); if (deps.readSettings().enabled !== false) { tryArm(); } @@ -525,7 +534,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle pushFailureBroadcast = false; lastPushAt = deps.now(); lastPushError = null; - persistPushRecord(deps.db, { at: lastPushAt, ok: true }, deps.log); + persistPushRecord(resolveDb(), { at: lastPushAt, ok: true }, deps.log); if (pushRetryTimer) { deps.clearTimeout(pushRetryTimer); pushRetryTimer = null; @@ -538,7 +547,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle const redactedStderr = redactCredentials(result.stderr); pushPending = true; lastPushError = redactedStderr; - persistPushRecord(deps.db, { at: deps.now(), ok: false, error: redactedStderr }, deps.log); + persistPushRecord(resolveDb(), { at: deps.now(), ok: false, error: redactedStderr }, deps.log); deps.log.warn({ stderr: redactedStderr }, "home-snapshot: push failed"); // Only the FIRST failure of an unbroken streak broadcasts — a retry // storm (schedulePushRetry firing every pushDelaySec*5) would @@ -642,7 +651,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle }); firstSeenDirty = plan.nextFirstSeenDirty; - persistState(deps.db, firstSeenDirty, deps.log); + persistState(resolveDb(), firstSeenDirty, deps.log); let committed = false; let sha: string | null = null; diff --git a/lib/state/__tests__/db.test.ts b/lib/state/__tests__/db.test.ts index 63777538..3d7f5f2d 100644 --- a/lib/state/__tests__/db.test.ts +++ b/lib/state/__tests__/db.test.ts @@ -351,6 +351,13 @@ describe("pragma values per flavor", () => { expect(timeout).toBe(5000); db.close(); }); + + test("getStateDb('daemon') reports busy_timeout 250 even after a default open", () => { + const cli = getStateDb(); // opens singleton, cli flavor + expect(cli.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 5000 }); + const daemon = getStateDb("daemon"); // same singleton — must not stay at 5000 + expect(daemon.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); + }); }); describe("startup busy budget — open+migrate blocks, it does not throw", () => { diff --git a/lib/state/db.ts b/lib/state/db.ts index 86cbeafc..99478f33 100644 --- a/lib/state/db.ts +++ b/lib/state/db.ts @@ -521,11 +521,19 @@ export function stateDbPath(): string { */ export function getStateDb(flavor: DbFlavor = "cli"): Database { const path = stateDbPath(); - if (!singleton || singletonPath !== path) { - singleton?.close(); - singleton = openStateDb(path, flavor); - singletonPath = path; + if (singleton && singletonPath === path) { + // A caller asking for a stronger (shorter) contention policy than the + // singleton currently holds must not silently inherit whatever flavor + // opened it first (e.g. a "cli" 5000ms opener beating the daemon's own + // "daemon" 250ms open) — re-tighten in place rather than reopening. + const want = BUSY_TIMEOUT_MS[flavor]; + const have = Number((singleton.query("PRAGMA busy_timeout").get() as { timeout?: number } | null)?.timeout ?? 0); + if (want < have) singleton.exec(`PRAGMA busy_timeout = ${want};`); + return singleton; } + singleton?.close(); + singleton = openStateDb(path, flavor); + singletonPath = path; return singleton; } From baaffc99919c19b49aefc3444230de0f5ed805f2 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:54:16 -0500 Subject: [PATCH 058/106] docs: the :9401 trust boundary model and the S010/S050/S043 sibling wiring notes --- docs/daemon-api-auth.md | 49 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/daemon-api-auth.md diff --git a/docs/daemon-api-auth.md b/docs/daemon-api-auth.md new file mode 100644 index 00000000..1eb33045 --- /dev/null +++ b/docs/daemon-api-auth.md @@ -0,0 +1,49 @@ +# The :9401 trust boundary + +How the daemon's REST/WS surface decides who to trust, and the follow-up +wiring two standalone modules from this phase still need in sibling-owned +files. + +## The model + +- **No `Origin` header at all** (the CLI, the Swift tray, rt-client from a + Bun/Node process, mr-board, gitq, the VS Code extension): unaffected, no + gate applies beyond what already existed. None of today's consumers send + an `Origin` header to `:9401`. +- **A browser `Origin` header is present**: trusted only if the request + presents the local `X-RT-Token` (`?token=` query param for `/ws`, since + browsers cannot set custom headers on a WS handshake) OR the Origin is on + the `rt.trustedBrowserOrigins` settings allowlist (see + `packages/rt-client/src/settings/registry-defs.ts`; `docs/settings-architecture.md` + is the settings-system contract). Otherwise: no `Access-Control-Allow-Origin` + on REST reads (default-deny CORS), and a 403 on `/ws`. +- **Mutating routes** (every method except GET/HEAD/OPTIONS, plus + `/api/secrets` and `/api/notifications` despite being GETs) require the + local `X-RT-Token` regardless of Origin... this is the CSRF defense against + a browser form/simple-request bypassing CORS preflight entirely, and it is + orthogonal to the Origin check above. + +See `lib/daemon/api-auth.ts` (`isBrowserRequestTrusted`, `needsToken`, +`getTrustedBrowserOrigins`) and `lib/daemon/api-server.ts` +(`buildCorsHeaders`, the `/ws` gate in `fetch()`) for the implementation. + +## Follow-up wiring for sibling-owned files (not done in this job) + +**S010** (`lib/daemon/handlers/worktree.ts`): `lib/daemon/git-ref-validation.ts` +exports `validateGitRef(ref)`. Call it right after `payload.branch` is read +(around `worktree.ts:282`) and return `{ ok: false, error }` on a rejection +BEFORE any `runGit` call reaches it... that single call site also covers the +weaker secondary instance in `divergence()` (`worktree.ts:211-213`), since +both read the same `branch` value. + +**S050** (`lib/daemon/freshness.ts`): `lib/daemon/redact-credentials.ts` +exports `redactCredentials(text)`. Wrap every log/error interpolation of a +remote URL with it... the audit names `freshness.ts:142, 148, 275, 279` as the +current call sites. + +**S043 caller side** (`lib/daemon.ts`): `lib/daemon/api-server.ts` exports +`ApiPortInUseError` (a named `Error` subclass with `.name === "ApiPortInUseError"` +and `.port`). Catch it around the `startApiServer()` call and park-and-retry +with backoff instead of letting it reach the top-level crash path; any other +error out of `startApiServer()` is a genuine misconfiguration and should keep +crashing as it does today. From fed0ce32a0041326e6df3524197804f7153f1722 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:58:30 -0500 Subject: [PATCH 059/106] repo-index: async observed-main-path on the endpoint:claim resolve path (S098) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/__tests__/repo-index-async.test.ts | 9 +++++++++ lib/repo-index.ts | 10 +++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 lib/__tests__/repo-index-async.test.ts diff --git a/lib/__tests__/repo-index-async.test.ts b/lib/__tests__/repo-index-async.test.ts new file mode 100644 index 00000000..fdcc423e --- /dev/null +++ b/lib/__tests__/repo-index-async.test.ts @@ -0,0 +1,9 @@ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("resolveIndexPathForIdentity no longer reaches a sync git via observedMainPath", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "repo-index.ts"), "utf8"); + // observedMainPath (sync execSync) must not be called from the async resolver path. + expect(src).toMatch(/observedMainPathAsync/); +}); diff --git a/lib/repo-index.ts b/lib/repo-index.ts index e5ddf614..648d6489 100644 --- a/lib/repo-index.ts +++ b/lib/repo-index.ts @@ -32,6 +32,7 @@ import { repoLabel, repoLabelFull, repoLabelQualified } from "./repo-label.ts"; import { dim } from "./ansi.ts"; import { getSetting } from "./settings/resolve.ts"; import { mergeRegistries, type TreeRecord } from "./worktree/registry.ts"; +import { listWorktreesAsync } from "./worktree/git-async.ts"; // ─── Types ─────────────────────────────────────────────────────────────────── @@ -151,6 +152,13 @@ function observedMainPath(repoRoot: string): string { } } +/** Async twin of observedMainPath: the repo's MAIN worktree path as git + * reports it, degrading to repoRoot. Safe on the daemon thread. */ +async function observedMainPathAsync(repoRoot: string): Promise { + const wts = await listWorktreesAsync(repoRoot); + return wts?.[0]?.path ?? repoRoot; +} + /** * The row's current path, read straight from the namespace rather than through * `loadRepoIndex()`: that function's legacy-repos.json import is a migration @@ -274,7 +282,7 @@ export async function resolveIndexPathForIdentity(serialized: string): Promise Date: Fri, 28 Aug 2026 11:02:01 -0500 Subject: [PATCH 060/106] fix rt-paths lint and registry test regressions on job/p3-trust-boundary redact-credentials.ts: reword the doc comment so it no longer spells out a literal .rt-prefixed path (rt-paths.test.ts bans that outside lib/rt-paths.ts); the rationale is unchanged. registry.test.ts: add the rt.trustedBrowserOrigins key (introduced by an earlier task on this branch) to the expected suiteKeys list and bump the length assertion from 42 to 43. Co-Authored-By: Claude Sonnet 5 --- lib/daemon/redact-credentials.ts | 6 +++--- packages/rt-client/src/settings/__tests__/registry.test.ts | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/daemon/redact-credentials.ts b/lib/daemon/redact-credentials.ts index 2ac64c87..b46d6f55 100644 --- a/lib/daemon/redact-credentials.ts +++ b/lib/daemon/redact-credentials.ts @@ -3,9 +3,9 @@ * a string. freshness.ts logs `remote.origin.url` verbatim on every * reconcile and echoes it into thrown errors returned to callers; a repo * cloned as `https://oauth2:glpat-XXXX@gitlab.example.com/...` (routine for - * dotfiles/CI-derived clones) puts that token into ~/.rt/logs/daemon.*.log - * and into any client-facing error message. Logs are the first thing a user - * pastes into a bug report. + * dotfiles/CI-derived clones) puts that token into the daemon's rotated log + * files and into any client-facing error message. Logs are the first thing + * a user pastes into a bug report. */ const CREDENTIAL_URL_RE = /(https?:\/\/)[^/@\s]+@/gi; diff --git a/packages/rt-client/src/settings/__tests__/registry.test.ts b/packages/rt-client/src/settings/__tests__/registry.test.ts index 1b534ea0..36305dec 100644 --- a/packages/rt-client/src/settings/__tests__/registry.test.ts +++ b/packages/rt-client/src/settings/__tests__/registry.test.ts @@ -248,8 +248,9 @@ describe("settings/registry", () => { "agent.effort", "agent.account", "agent.extraArgs", + "rt.trustedBrowserOrigins", ]; - expect(suiteKeys).toHaveLength(42); + expect(suiteKeys).toHaveLength(43); expect(allDefs().map((d) => d.key).sort()).toEqual( [...migratedFalseKeys, ...migratedTrueKeys, ...suiteKeys].sort(), From 3037b275df2f0556cf4e80d7a06098fd6c076a6c Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:02:15 -0500 Subject: [PATCH 061/106] state: isBusyError matches SQLITE_BUSY_*; read-then-write daemon txns use BEGIN IMMEDIATE joinRoom/archiveRoom/readUnread (chat-store.ts), dmRoomFor (dm-store.ts), and drainNotificationQueue (notifier-store.ts) read before they write inside a db.transaction; bun:sqlite's plain transaction() defers BEGIN, so a concurrent writer can produce SQLITE_BUSY_SNAPSHOT that busy_timeout cannot absorb. .immediate() takes the write lock up front instead. presence-store.ts:signIn has the same shape but is a sibling-owned write-fence file; left untouched as a documented follow-up. --- lib/state/__tests__/busy.test.ts | 23 ++++++++++++++++++++++- lib/state/busy.ts | 10 ++++++++-- lib/state/chat-store.ts | 7 ++++--- lib/state/dm-store.ts | 2 +- lib/state/notifier-store.ts | 2 +- 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/lib/state/__tests__/busy.test.ts b/lib/state/__tests__/busy.test.ts index 1f8eb70f..01791698 100644 --- a/lib/state/__tests__/busy.test.ts +++ b/lib/state/__tests__/busy.test.ts @@ -1,5 +1,9 @@ import { expect, test } from "bun:test"; -import { runCriticalWrite } from "../busy.ts"; +import { Database } from "bun:sqlite"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { isBusyError, runCriticalWrite } from "../busy.ts"; test("returns the value when fn succeeds", () => { expect(runCriticalWrite("t", () => 42, {})).toBe(42); @@ -26,3 +30,20 @@ test("returns undefined after exhausting attempts on a busy error", () => { test("rethrows a non-busy error rather than retrying", () => { expect(() => runCriticalWrite("t", () => { throw new Error("syntax error"); }, {})).toThrow("syntax error"); }); + +test("isBusyError matches SQLITE_BUSY_SNAPSHOT from a real conflict", () => { + const dir = mkdtempSync(join(tmpdir(), "busy-snap-")); + const path = join(dir, "t.db"); + const a = new Database(path); a.exec("PRAGMA journal_mode=WAL; CREATE TABLE t(id INTEGER PRIMARY KEY, v INTEGER);"); + a.exec("INSERT INTO t(id,v) VALUES(1,0);"); + const b = new Database(path); + a.exec("BEGIN;"); a.query("SELECT v FROM t WHERE id=1").get(); // pin snapshot on A + b.exec("UPDATE t SET v=1 WHERE id=1;"); // B commits (autocommit) + let caught: unknown; + try { a.exec("UPDATE t SET v=2 WHERE id=1;"); } catch (e) { caught = e; } + expect(caught).toBeDefined(); + expect((caught as any).code?.startsWith("SQLITE_BUSY")).toBe(true); + expect(isBusyError(caught)).toBe(true); + try { a.exec("ROLLBACK;"); } catch {} + a.close(); b.close(); +}); diff --git a/lib/state/busy.ts b/lib/state/busy.ts index 1124378c..c7de1f55 100644 --- a/lib/state/busy.ts +++ b/lib/state/busy.ts @@ -35,9 +35,15 @@ import type { DaemonLoggerHandle } from "../daemon-logger.ts"; let logHandle: Promise | null = null; -/** True for the bun:sqlite error thrown when a write can't get the lock inside busy_timeout. */ +/** + * True for the bun:sqlite error thrown when a write can't get the lock + * inside busy_timeout — including the SNAPSHOT/RECOVERY variants a + * deferred-BEGIN read-then-write transaction can throw, which busy_timeout + * does not retry the way it retries a plain SQLITE_BUSY. + */ export function isBusyError(err: unknown): boolean { - return (err as { code?: string } | undefined)?.code === "SQLITE_BUSY"; + const code = (err as { code?: string } | undefined)?.code; + return code === "SQLITE_BUSY" || (typeof code === "string" && code.startsWith("SQLITE_BUSY_")); } function warnBusy(module: string, context: Record): void { diff --git a/lib/state/chat-store.ts b/lib/state/chat-store.ts index dd4d569c..6177fa61 100644 --- a/lib/state/chat-store.ts +++ b/lib/state/chat-store.ts @@ -234,7 +234,8 @@ export function joinRoom( return { handle, memberCount, unread: Math.max(0, maxId - lastReadId) }; }); - return run(); + // BEGIN IMMEDIATE: read-then-write must lock up front or SQLITE_BUSY_SNAPSHOT bypasses busy_timeout. + return run.immediate(); } export function leaveRoom(room: string, handle: string, db: Database = getStateDb()): void { @@ -293,7 +294,7 @@ export function archiveRoom( if (current === null) db.query(UPDATE_ROOM_ARCHIVED_SQL).run(archivedAt, room); return { room, archivedAt }; }); - return run(); + return run.immediate(); } /** The wake mode stamped by whichever join created `room`; undefined for a room never stamped (including every DM room — dmRoomFor never stamps one). */ @@ -508,7 +509,7 @@ export function readUnread( return results; }); - return run(); + return run.immediate(); } export function listMessages( diff --git a/lib/state/dm-store.ts b/lib/state/dm-store.ts index 39c66b9f..f7d712f4 100644 --- a/lib/state/dm-store.ts +++ b/lib/state/dm-store.ts @@ -72,7 +72,7 @@ export function dmRoomFor( return { room, created: true }; }); - return run(); + return run.immediate(); } export function dmParticipants(room: string, db: Database = getStateDb()): { a: string; b: string } | null { diff --git a/lib/state/notifier-store.ts b/lib/state/notifier-store.ts index 682a822a..0b08238c 100644 --- a/lib/state/notifier-store.ts +++ b/lib/state/notifier-store.ts @@ -108,7 +108,7 @@ export function drainNotificationQueue(db: Database = getStateDb()): Notificatio db.exec(`DELETE FROM notify_queue;`); return rows.map(rowToEvent); }); - return runCriticalWrite("drain", () => run(), {}) ?? []; + return runCriticalWrite("drain", () => run.immediate(), {}) ?? []; } /** Peek reads without deleting — diagnostics, no mutation, no retry needed. */ From f76d9949084c4726f11affccbdb8dfbd88760853 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:06:02 -0500 Subject: [PATCH 062/106] repo-index: fix write-back regression in resolveIndexPathForIdentity setIndexPath is a bare KV write; it skipped the moved-repo guard, the repos.json compat mirror, and the unopenable-db degrade that writeIndexRow (and thus updateRepoIndex) provides. Swap to writeIndexRow, which takes the already-resolved async main path and preserves all three, with no sync git reintroduced. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/repo-index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/repo-index.ts b/lib/repo-index.ts index 648d6489..6a8a4f0f 100644 --- a/lib/repo-index.ts +++ b/lib/repo-index.ts @@ -282,7 +282,7 @@ export async function resolveIndexPathForIdentity(serialized: string): Promise Date: Fri, 28 Aug 2026 11:09:05 -0500 Subject: [PATCH 063/106] gate: fail on sync-exec anywhere in the daemon import graph (1.3) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/__tests__/no-daemon-sync-exec.test.ts | 83 +++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 lib/__tests__/no-daemon-sync-exec.test.ts diff --git a/lib/__tests__/no-daemon-sync-exec.test.ts b/lib/__tests__/no-daemon-sync-exec.test.ts new file mode 100644 index 00000000..2ba99d37 --- /dev/null +++ b/lib/__tests__/no-daemon-sync-exec.test.ts @@ -0,0 +1,83 @@ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { dirname, resolve } from "path"; + +// Files with sync-exec that Phase 1 does NOT remove. Each entry names the +// finding/phase that will delete it, so this list shrinks as later phases land. +// A regression that reintroduces sync-exec into any OTHER daemon-reachable +// module fails this gate (the rule has been re-broken twice). +const ALLOWLIST = new Set([ + "lib/daemon/user-path.ts", // Phase 6 PATH rebuild (S013/S014/S062) + "lib/daemon/boot-reconcile.ts", // Phase 0.6 / S044 (Bun.sleepSync) + "lib/state/db.ts", // Phase 0.7 / S072-S073 busy-retry + "lib/state/busy.ts", // Phase 0.7 / S072-S073 busy-retry + "lib/git-worktrees.ts", // S055: reached only via handlers/status.ts + "lib/daemon/handlers/status.ts", // S055: the edge into git-worktrees.ts + "lib/repo-index.ts", // Phase 5.3 dedup (heal/derive execSync) + "lib/repo.ts", // R050 / Phase 5.4 (via handlers/system-processes.ts) + "lib/git.ts", // R050 / Phase 5.4 (via repo.ts) + "lib/herdr-launch.ts", // Phase 5 herdr (via handlers/pane.ts) + "lib/rt-render.tsx", // R050 / Phase 5.4 (daemon carries the TUI) +]); + +const SYNC_EXEC = [ + /\bexecSync\s*\(/, + /\bspawnSync\s*\(/, + /\bBun\.spawnSync\s*\(/, + /\bBun\.sleepSync\s*\(/, +]; + +const REPO_ROOT = resolve(import.meta.dir, "..", ".."); +const stripShebang = (s: string) => s.replace(/^#!.*\n/, ""); +const tsT = new Bun.Transpiler({ loader: "ts" }); +const tsxT = new Bun.Transpiler({ loader: "tsx" }); +const loaderFor = (f: string) => (f.endsWith(".tsx") || f.endsWith(".jsx") ? tsxT : tsT); + +/** Files reachable from lib/daemon.ts via relative imports (the daemon graph). */ +function daemonClosure(): string[] { + const entry = resolve(REPO_ROOT, "lib/daemon.ts"); + const visited = new Set(); + const stack = [entry]; + while (stack.length) { + const file = stack.pop()!; + if (visited.has(file)) continue; + visited.add(file); + let src: string; + try { src = stripShebang(readFileSync(file, "utf8")); } catch { continue; } + let imports: { path: string }[]; + try { imports = loaderFor(file).scanImports(src); } catch { continue; } + for (const imp of imports) { + if (!imp.path.startsWith(".")) continue; // external package + stack.push(resolve(dirname(file), imp.path)); + } + } + return [...visited]; +} + +function hasSyncExec(source: string): boolean { + return SYNC_EXEC.some((re) => re.test(source)); +} + +test("no daemon-reachable module calls sync exec (outside the allowlist)", () => { + const offenders: string[] = []; + for (const file of daemonClosure()) { + const rel = file.replace(REPO_ROOT + "/", ""); + if (ALLOWLIST.has(rel)) continue; + let src: string; + try { src = readFileSync(file, "utf8"); } catch { continue; } + if (hasSyncExec(src)) offenders.push(rel); + } + expect(offenders).toEqual([]); +}); + +test("the checker flags a reintroduced sync-exec call (proves the gate bites)", () => { + // Permanent RED proof: the matcher must catch a fresh offense. + expect(hasSyncExec(`import { execSync } from "child_process";\nexecSync("true");`)).toBe(true); + expect(hasSyncExec(`await Bun.sleepSync(10);`)).toBe(true); + expect(hasSyncExec(`// a comment mentioning execSync without a call`)).toBe(false); +}); + +test("the daemon closure actually resolves (guards against a walker that finds nothing)", () => { + const closure = daemonClosure(); + expect(closure.length).toBeGreaterThan(50); // ~151 today; a collapse means the walk broke +}); From 56e7f146c2c17d2e667645668651d358e7359e42 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:10:17 -0500 Subject: [PATCH 064/106] state.db: isolate each legacy importer in a SAVEPOINT so one bad file cannot wedge migration A throwing legacy importer previously rolled back the whole v0->v1 migration, leaving user_version at 0 and repeating the identical throw on every later openStateDb call (daemon boot + every CLI command) with no self-heal. Wrap each LEGACY_IMPORTS entry in its own SAVEPOINT so one importer's throw only rolls back its own writes; warn with the file and error, still push it to consumed so it gets renamed .migrated, and let the schema DDL and every other importer land and reach SCHEMA_VERSION. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/state/__tests__/db.test.ts | 64 +++++++++++++++++++++++++++++++++- lib/state/db.ts | 30 ++++++++++++---- 2 files changed, 87 insertions(+), 7 deletions(-) diff --git a/lib/state/__tests__/db.test.ts b/lib/state/__tests__/db.test.ts index 3d7f5f2d..104a115c 100644 --- a/lib/state/__tests__/db.test.ts +++ b/lib/state/__tests__/db.test.ts @@ -6,7 +6,7 @@ * HOME isolation is handled by the repo-wide bun test preload * (test-setup.ts) — never removed here. */ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; import { Database } from "bun:sqlite"; import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "fs"; import { tmpdir } from "os"; @@ -18,6 +18,12 @@ import { openStateDb, SCHEMA_VERSION, } from "../db.ts"; +// Side-effect imports: registering the REAL project-mrs and discussions +// importers (module-load LEGACY_IMPORTS.push), not fakes, so the isolation +// test below exercises a genuine throw (duplicate-iid UNIQUE violation) +// and a genuine benign import, not a hand-rolled stand-in for either. +import "../../daemon/project-mrs-store.ts"; +import "../../daemon/discussions-file-store.ts"; const DB_TS_PATH = join(import.meta.dir, "..", "db.ts"); @@ -318,6 +324,62 @@ describe("legacy import seam", () => { }); expect(() => openStateDb(dbPath, "cli").close()).not.toThrow(); }); + + test("a throwing legacy importer is isolated: db reaches SCHEMA_VERSION, the other importer's rows land, and the offending file is still renamed", () => { + const warnSpy = spyOn(console, "warn").mockImplementation(() => {}); + try { + const dbPath = join(dir, "state.db"); + + // project-mrs-store's real importer: keys "5" and "05" both bind + // Number(iidStr) === 5, so the second INSERT hits project_mrs's + // (repo, iid) PRIMARY KEY and throws mid-transaction. + const projectMrsPath = join(dir, "project-mrs.json"); + writeFileSync( + projectMrsPath, + JSON.stringify({ + "host/repo": { + mrs: { + "5": { pr: { iid: 5, title: "first" }, fetchedAt: 111 }, + "05": { pr: { iid: 5, title: "duplicate" }, fetchedAt: 222 }, + }, + }, + }), + ); + + // discussions-file-store's real importer: a benign, unrelated file + // that must still land even though the importer above throws. + const discussionsPath = join(dir, "discussions.json"); + writeFileSync( + discussionsPath, + JSON.stringify({ + "host/repo:7": { discussions: [{ id: "d1" }], fetchedAt: 333 }, + }), + ); + + const db = openStateDb(dbPath, "cli"); + + expect(db.query("PRAGMA user_version;").get()).toEqual({ user_version: SCHEMA_VERSION }); + + const discussionsRow = db + .query("SELECT repo, iid, fetched_at FROM discussions WHERE repo = ? AND iid = ?;") + .get("host/repo", 7); + expect(discussionsRow).toEqual({ repo: "host/repo", iid: 7, fetched_at: 333 }); + + expect(existsSync(projectMrsPath)).toBe(false); + expect(existsSync(`${projectMrsPath}.migrated`)).toBe(true); + expect(existsSync(discussionsPath)).toBe(false); + expect(existsSync(`${discussionsPath}.migrated`)).toBe(true); + + const warnedAboutOffender = warnSpy.mock.calls.some((call) => + call.some((arg) => typeof arg === "string" && arg.includes("project-mrs.json")), + ); + expect(warnedAboutOffender).toBe(true); + + db.close(); + } finally { + warnSpy.mockRestore(); + } + }); }); describe("pragma values per flavor", () => { diff --git a/lib/state/db.ts b/lib/state/db.ts index 99478f33..34d53c68 100644 --- a/lib/state/db.ts +++ b/lib/state/db.ts @@ -391,11 +391,22 @@ function quarantine(path: string): void { /** * Runs each registered legacy importer whose source file exists, inside the * caller's transaction. Returns the list of source paths that were consumed - * (successfully imported OR corrupt-and-skipped) — both cases still rename - * per spec "Migration & contention" ("corrupt = warn + skip"; brief: "warn + - * skip + still rename"). Renaming itself happens AFTER COMMIT (the caller - * does it), since a filesystem rename cannot participate in the sqlite - * transaction. + * (successfully imported OR corrupt/throwing-and-skipped) — all three cases + * still rename per spec "Migration & contention" ("corrupt = warn + skip"; + * brief: "warn + skip + still rename"). Renaming itself happens AFTER COMMIT + * (the caller does it), since a filesystem rename cannot participate in the + * sqlite transaction. + * + * Each importer's `import(db, json)` runs inside its own SAVEPOINT, nested + * inside the caller's outer BEGIN IMMEDIATE. A throwing importer (e.g. two + * legacy keys that normalize to the same primary key, tripping a UNIQUE + * constraint) previously rolled back the WHOLE v0->v1 migration: user_version + * stayed 0, so every later openStateDb call replayed the identical throw + * forever with no self-heal. The savepoint confines that rollback to the one + * importer's own writes, so the schema DDL and every OTHER importer still + * commit and the db reaches SCHEMA_VERSION. This is deliberately narrower + * than the outer transaction's own error handling: schema/DDL failures are + * not wrapped here and still abort the whole migration loudly. */ function importLegacyStores(db: Database, dir: string): string[] { const consumed: string[] = []; @@ -410,7 +421,14 @@ function importLegacyStores(db: Database, dir: string): string[] { consumed.push(path); continue; } - entry.import(db, json); + db.exec("SAVEPOINT legacy_import;"); + try { + entry.import(db, json); + db.exec("RELEASE legacy_import;"); + } catch (err) { + db.exec("ROLLBACK TO legacy_import; RELEASE legacy_import;"); + console.warn(`rt: legacy import failed for ${path}, skipping (file will still be renamed): ${(err as Error).message}`); + } consumed.push(path); } return consumed; From 9783b1c55ea7569f06e24b92eadff7882a3fa2db Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:14:22 -0500 Subject: [PATCH 065/106] daemon: add supervision-state (boot attempts, failures, last-exit) kv + breadcrumb --- .../__tests__/supervision-state.test.ts | 71 +++++++++ lib/daemon/supervision-state.ts | 150 ++++++++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 lib/daemon/__tests__/supervision-state.test.ts create mode 100644 lib/daemon/supervision-state.ts diff --git a/lib/daemon/__tests__/supervision-state.test.ts b/lib/daemon/__tests__/supervision-state.test.ts new file mode 100644 index 00000000..6461af34 --- /dev/null +++ b/lib/daemon/__tests__/supervision-state.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, test } from "bun:test"; +import { + recordBootAttempt, + recordDaemonReady, + recordBootFailure, + recordCleanExit, + readSupervisionState, + isCrashLooping, + writeBreadcrumb, + readBreadcrumb, + clearBreadcrumb, +} from "../supervision-state.ts"; + +describe("supervision-state kv round-trip", () => { + test("boot attempts, ready stamp, failures and last-exit round-trip through kv", () => { + recordBootAttempt(); + recordBootAttempt(); + recordDaemonReady(); + recordBootFailure("api", "EADDRINUSE"); + const s = readSupervisionState(); + expect(s.bootAttempts).toBe(2); + expect(s.lastReadyAt).toBeGreaterThan(0); + expect(s.recentFailures.at(-1)).toMatchObject({ phase: "api", reason: "EADDRINUSE" }); + expect(s.lastExit).toMatchObject({ kind: "boot-failed", code: 1 }); + }); + + test("recordCleanExit sets last-exit with the given kind and code", () => { + recordCleanExit("shutdown", 0); + const s = readSupervisionState(); + expect(s.lastExit).toMatchObject({ kind: "shutdown", code: 0 }); + }); + + test("recent-failures is capped at 10 entries", () => { + for (let i = 0; i < 15; i++) recordBootFailure("api", `err-${i}`); + const s = readSupervisionState(); + expect(s.recentFailures.length).toBe(10); + expect(s.recentFailures.at(-1)).toMatchObject({ reason: "err-14" }); + }); +}); + +describe("isCrashLooping", () => { + test("true at >=3 failures within the window", () => { + const now = 1_000_000; + const fails = [now - 10, now - 20, now - 30].map((at) => ({ at, phase: "api" as const, reason: "x" })); + expect(isCrashLooping({ bootAttempts: 3, lastReadyAt: 0, recentFailures: fails, lastExit: null }, now)).toBe(true); + const old = [{ at: now - 10 * 60_000, phase: "api" as const, reason: "x" }]; + expect(isCrashLooping({ bootAttempts: 1, lastReadyAt: 0, recentFailures: old, lastExit: null }, now)).toBe(false); + }); +}); + +describe("breadcrumb file", () => { + test("writeBreadcrumb then readBreadcrumb round-trips phase, pid, flavor", () => { + writeBreadcrumb("api"); + const b = readBreadcrumb(); + expect(b).not.toBeNull(); + expect(b?.phase).toBe("api"); + expect(b?.pid).toBe(process.pid); + expect(typeof b?.at).toBe("number"); + }); + + test("clearBreadcrumb removes the file so readBreadcrumb returns null", () => { + writeBreadcrumb("ready"); + clearBreadcrumb(); + expect(readBreadcrumb()).toBeNull(); + }); + + test("readBreadcrumb returns null when no breadcrumb has been written", () => { + clearBreadcrumb(); + expect(readBreadcrumb()).toBeNull(); + }); +}); diff --git a/lib/daemon/supervision-state.ts b/lib/daemon/supervision-state.ts new file mode 100644 index 00000000..7fa12e7e --- /dev/null +++ b/lib/daemon/supervision-state.ts @@ -0,0 +1,150 @@ +/** + * lib/daemon/supervision-state.ts — daemon boot/crash history, so + * `rt daemon status` can report boot-failed/crash-looping and a stuck-phase + * breadcrumb for a live-but-silent daemon. + * + * Two tiers, deliberately not one: + * - The breadcrumb FILE (`writeBreadcrumb`/`readBreadcrumb`/`clearBreadcrumb`) + * opens no database, so it is safe to call at module scope, before + * state.db exists. It is the only tier a pre-db boot failure can reach. + * - The kv tier (`recordBootAttempt`, `recordDaemonReady`, + * `recordCleanExit`, and the kv half of `recordBootFailure`) goes through + * `getStateDb("daemon")`, so callers must not reach it until the daemon + * has opened its state.db (lib/daemon.ts's `openBranchCacheStore()`). + * `recordBootFailure` is safe to call at any point regardless — its kv + * write is try/catch'd and silently no-ops if the db isn't open yet. + */ + +import { existsSync, readFileSync, unlinkSync, writeFileSync } from "fs"; +import { join } from "path"; +import { RT_DIR } from "../daemon-config.ts"; +import { daemonFlavor } from "./park.ts"; +import { getKvValue, setKvValue } from "../state/kv-blob.ts"; +import { getStateDb } from "../state/db.ts"; + +export type BootPhase = "start" | "events-db" | "state-db" | "api" | "socket" | "ready"; + +const NS = "daemon-supervision"; +const KEY_BOOT_ATTEMPTS = "boot-attempts"; +const KEY_LAST_READY_AT = "last-ready-at"; +const KEY_RECENT_FAILURES = "recent-failures"; +const KEY_LAST_EXIT = "last-exit"; + +const RECENT_FAILURES_CAP = 10; + +export interface BootFailure { + at: number; + phase: BootPhase; + reason: string; +} + +export type LastExit = + | { at: number; kind: "boot-failed"; code: number; reason: string } + | { at: number; kind: "shutdown" | "signal"; code: number }; + +export interface SupervisionState { + bootAttempts: number; + lastReadyAt: number; + recentFailures: BootFailure[]; + lastExit: LastExit | null; +} + +function db() { + return getStateDb("daemon"); +} + +export function recordBootAttempt(): void { + const n = getKvValue(NS, KEY_BOOT_ATTEMPTS, 0, db()); + setKvValue(NS, KEY_BOOT_ATTEMPTS, n + 1, db()); +} + +export function recordDaemonReady(): void { + setKvValue(NS, KEY_LAST_READY_AT, Date.now(), db()); +} + +/** + * Always writes the breadcrumb file, which never needs state.db. The kv + * append is best-effort: a failure this early in boot may predate state.db + * being open at all, and that must not throw back into the caller's catch. + */ +export function recordBootFailure(phase: BootPhase, reason: string): void { + writeBreadcrumb(phase); + try { + const at = Date.now(); + const existing = getKvValue(NS, KEY_RECENT_FAILURES, [], db()); + const next = [...existing, { at, phase, reason }].slice(-RECENT_FAILURES_CAP); + setKvValue(NS, KEY_RECENT_FAILURES, next, db()); + setKvValue(NS, KEY_LAST_EXIT, { at, kind: "boot-failed", code: 1, reason }, db()); + } catch { + // Pre-db failure (or a busy/corrupt state.db) — the breadcrumb file above + // is the only record this failure gets, and that's fine. + } +} + +export function recordCleanExit(kind: "shutdown" | "signal", code: number): void { + try { + setKvValue(NS, KEY_LAST_EXIT, { at: Date.now(), kind, code }, db()); + } catch { + // Best-effort, same as recordBootFailure's kv half. + } +} + +export function readSupervisionState(): SupervisionState { + const store = db(); + return { + bootAttempts: getKvValue(NS, KEY_BOOT_ATTEMPTS, 0, store), + lastReadyAt: getKvValue(NS, KEY_LAST_READY_AT, 0, store), + recentFailures: getKvValue(NS, KEY_RECENT_FAILURES, [], store), + lastExit: getKvValue(NS, KEY_LAST_EXIT, null, store), + }; +} + +export function isCrashLooping( + state: SupervisionState, + now: number, + n = 3, + windowMs = 5 * 60_000, +): boolean { + const floor = now - windowMs; + return state.recentFailures.filter((f) => f.at > floor).length >= n; +} + +// ─── Breadcrumb file (db-free) ──────────────────────────────────────────── + +interface Breadcrumb { + at: number; + pid: number; + flavor: "dev" | "prod"; + phase: BootPhase; +} + +function breadcrumbPath(): string { + return join(RT_DIR, "daemon-boot.json"); +} + +/** Never fatal — a breadcrumb is a diagnostic aid, not something boot may fail over. */ +export function writeBreadcrumb(phase: BootPhase): void { + try { + const breadcrumb: Breadcrumb = { at: Date.now(), pid: process.pid, flavor: daemonFlavor(), phase }; + writeFileSync(breadcrumbPath(), JSON.stringify(breadcrumb)); + } catch { + // Best-effort. + } +} + +export function readBreadcrumb(): Breadcrumb | null { + try { + if (!existsSync(breadcrumbPath())) return null; + return JSON.parse(readFileSync(breadcrumbPath(), "utf8")) as Breadcrumb; + } catch { + return null; + } +} + +export function clearBreadcrumb(): void { + try { + if (existsSync(breadcrumbPath())) unlinkSync(breadcrumbPath()); + } catch { + // Best-effort. + } +} From 673cfcfa67f194e6cf2c3819e359c155a8acfb88 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:14:56 -0500 Subject: [PATCH 066/106] refresh: whole-cycle deadline clears the coalesce latch; cap RepoWatch.pending (S007) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/cache-refresh-coalesce.test.ts | 33 +++++++++++++++ .../__tests__/freshness-pending-cap.test.ts | 13 ++++++ lib/daemon/cache-refresh.ts | 40 +++++++++++++++---- lib/daemon/freshness.ts | 13 +++++- 4 files changed, 91 insertions(+), 8 deletions(-) create mode 100644 lib/daemon/__tests__/cache-refresh-coalesce.test.ts create mode 100644 lib/daemon/__tests__/freshness-pending-cap.test.ts diff --git a/lib/daemon/__tests__/cache-refresh-coalesce.test.ts b/lib/daemon/__tests__/cache-refresh-coalesce.test.ts new file mode 100644 index 00000000..32a609d0 --- /dev/null +++ b/lib/daemon/__tests__/cache-refresh-coalesce.test.ts @@ -0,0 +1,33 @@ +import { test, expect } from "bun:test"; +import { makeCoalescer } from "../cache-refresh.ts"; + +test("clears the in-flight latch after the deadline even if run never settles", async () => { + let starts = 0; + let timedOut = 0; + const coalesce = makeCoalescer( + () => { starts++; return new Promise(() => {}); }, // never resolves + 50, + () => { timedOut++; }, + ); + const t0 = Date.now(); + await coalesce(); // resolves at the deadline, not never + expect(Date.now() - t0).toBeLessThan(500); + expect(timedOut).toBe(1); + await coalesce(); // latch cleared, a new run can start + expect(starts).toBe(2); +}); + +test("coalesces concurrent callers onto one run", async () => { + let starts = 0; + let resolveRun!: () => void; + const coalesce = makeCoalescer( + () => { starts++; return new Promise((r) => { resolveRun = r; }); }, + 10_000, + () => {}, + ); + const a = coalesce(); + const b = coalesce(); + expect(starts).toBe(1); + resolveRun(); + await Promise.all([a, b]); +}); diff --git a/lib/daemon/__tests__/freshness-pending-cap.test.ts b/lib/daemon/__tests__/freshness-pending-cap.test.ts new file mode 100644 index 00000000..f488ba9e --- /dev/null +++ b/lib/daemon/__tests__/freshness-pending-cap.test.ts @@ -0,0 +1,13 @@ +import { test, expect } from "bun:test"; +import { applyInvalidationBatch, PENDING_CAP } from "../freshness.ts"; + +test("merged pending is deduped by kind:ref and capped", async () => { + const runner: any = { processing: true, pending: [] }; + // Push more distinct keys than the cap; plus duplicates. + const keys = Array.from({ length: PENDING_CAP + 500 }, (_, i) => ({ kind: "mr" as const, ref: String(i), cause: "test" })); + const dupes = [{ kind: "mr" as const, ref: "0", cause: "test" }, { kind: "mr" as const, ref: "0", cause: "test" }]; + await applyInvalidationBatch({} as any, {} as any, runner, [...keys, ...dupes], {}); + expect(runner.pending.length).toBeLessThanOrEqual(PENDING_CAP); + const ids = runner.pending.map((k: any) => `${k.kind}:${k.ref}`); + expect(new Set(ids).size).toBe(ids.length); // no duplicates +}); diff --git a/lib/daemon/cache-refresh.ts b/lib/daemon/cache-refresh.ts index 0ab58fc7..5fe14920 100644 --- a/lib/daemon/cache-refresh.ts +++ b/lib/daemon/cache-refresh.ts @@ -47,16 +47,42 @@ export interface CacheRefresherDeps { */ const BRANCH_CACHE_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; +/** Below the 5-min tick, above the slowest legitimate deep sync. */ +const REFRESH_CYCLE_DEADLINE_MS = 4 * 60 * 1000; + +/** + * Coalesce concurrent callers onto one in-flight run, but clear the latch after + * `deadlineMs` even if the run never settles, so a wedged cycle (a half-open + * GitLab socket that never rejects) cannot pin the latch forever. The wedged + * run's frame still leaks until the OS reaps the socket; this only frees the + * next tick. + */ +export function makeCoalescer( + run: () => Promise, + deadlineMs: number, + onTimeout: () => void, +): () => Promise { + let inFlight: Promise | null = null; + return () => { + if (inFlight) return inFlight; + const impl = run().catch(() => {}); // a rejected cycle still clears the latch + const guarded = Promise.race([ + impl, + new Promise((resolve) => setTimeout(() => { onTimeout(); resolve(); }, deadlineMs)), + ]).finally(() => { inFlight = null; }); + inFlight = guarded; + return guarded; + }; +} + export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise { const { log, cache, refreshStatusRef, portCacheRef, repoIndex, broadcast } = deps; - let refreshInFlight: Promise | null = null; - - function refreshCache(): Promise { - if (refreshInFlight) return refreshInFlight; - refreshInFlight = refreshCacheImpl().finally(() => { refreshInFlight = null; }); - return refreshInFlight; - } + const refreshCache = makeCoalescer( + refreshCacheImpl, + REFRESH_CYCLE_DEADLINE_MS, + () => log.warn("cache refresh timed out; cleared in-flight latch for next tick"), + ); async function refreshCacheImpl(): Promise { log.debug("cache: starting background refresh"); diff --git a/lib/daemon/freshness.ts b/lib/daemon/freshness.ts index 2b49b497..767feca0 100644 --- a/lib/daemon/freshness.ts +++ b/lib/daemon/freshness.ts @@ -85,6 +85,10 @@ interface RepoWatch { } const watches = new Map(); + +/** Bound merged pending so a wedged processKeys cannot grow memory unbounded. */ +export const PENDING_CAP = 1000; + const providers = new Map(); let userId: number | null = null; let userIdResolved = false; @@ -397,7 +401,14 @@ export async function applyInvalidationBatch( overrides: MappingOverrides = {}, ): Promise { if (runner.processing) { - runner.pending.push(...keys); + const seen = new Set(runner.pending.map((k) => `${k.kind}:${k.ref}`)); + for (const k of keys) { + if (runner.pending.length >= PENDING_CAP) break; + const id = `${k.kind}:${k.ref}`; + if (seen.has(id)) continue; + seen.add(id); + runner.pending.push(k); + } return; } runner.processing = true; From fcf8220fa4bd7e66322f64b9de9c6651c85b6c19 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:17:09 -0500 Subject: [PATCH 067/106] daemon: wire supervision-state breadcrumbs + kv records into boot/shutdown --- lib/daemon.ts | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/daemon.ts b/lib/daemon.ts index d2997eb0..e2a570fc 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -63,6 +63,14 @@ import { import { startDiscussionsPoller } from "./daemon/discussions-poller.ts"; import { createCleanup, installSignalHandlers } from "./daemon/shutdown.ts"; import { createEventsBus } from "./daemon/events-bus.ts"; +import { + writeBreadcrumb, + recordBootAttempt, + recordDaemonReady, + recordBootFailure, + recordCleanExit, + type BootPhase, +} from "./daemon/supervision-state.ts"; import { safeInterval, safeTimeout } from "./daemon/safe-timers.ts"; import { pruneRuns } from "./runs/prune.ts"; import { pruneLogs } from "./log-janitor.ts"; @@ -88,6 +96,16 @@ redirectNativeStderr(); // (no socket/API bound yet, nothing to recover), advisory-only once ready. let bootPhase: "booting" | "ready" = "booting"; +// Tracks the finer-grained boot phase for the breadcrumb file and for +// attributing a Task-2 fatal boot error to the phase it happened in. +// setPhase is db-free (writeBreadcrumb only writes a file), so it is safe +// to call from module scope, before state.db exists. +let currentPhase: BootPhase = "start"; +function setPhase(phase: BootPhase): void { + currentPhase = phase; + writeBreadcrumb(phase); +} + const rtMigration = migrateLegacyRtDir(); // ─── Logging ───────────────────────────────────────────────────────────────── @@ -104,6 +122,8 @@ const log = loggerHandle.logger; // must run BEFORE any of it does. installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" }); +setPhase("start"); + if (rtMigration === "migrated") { log.info(`migrated legacy ${LEGACY_RT_LABEL} state to ${RT_DIR_LABEL}`); } else if (rtMigration === "conflict") { @@ -208,6 +228,7 @@ const hooksGuard = createHooksGuard(log); // Pane-communication events bus (RT-44): SQLite journal + in-memory waiters. const eventsBus = createEventsBus({ dbPath: join(RT_DIR, "events.db"), log }); +setPhase("events-db"); // Hourly retention sweep — cheap; rides its own interval rather than pollers // because it needs no poller deps. safeInterval/safeTimeout: a sync sqlite // throw here (e.g. SQLITE_FULL) must warn, not become an uncaughtException @@ -374,6 +395,7 @@ async function routeCommand(cmd: string, payload: any, signal?: AbortSignal): Pr // force-closes all in-flight connections, including the one that // carried the shutdown request. setTimeout(() => { + recordCleanExit("shutdown", 0); cleanup(); loggerHandle.flush?.(); process.exit(0); @@ -435,6 +457,8 @@ async function runDaemon(): Promise { // legacy-JSON import, and it must never land inside the event loop. If a // CLI process is mid-import right now, we block here, in startup. openBranchCacheStore(); + setPhase("state-db"); + recordBootAttempt(); log.info({ count: Object.keys(cache.entries).length }, "branch cache loaded from state.db"); // one-shot re-key of every legacy NAME-keyed store row onto its @@ -482,7 +506,9 @@ async function runDaemon(): Promise { // API server first: a failed bind exits fatally (boot-phase catch below), // and binding API before the unix socket means that fatal exit never // strands a socket-bound zombie behind it. + setPhase("api"); servers.api = await startApiServer({ handleCommand, log }); + setPhase("socket"); servers.socket = startSocketServer({ handleCommand, log }); // Only write rt.pid once both servers are actually bound — a boot that @@ -533,10 +559,12 @@ async function runDaemon(): Promise { installSignalHandlers({ cleanup, flushLogs: () => loggerHandle.flush?.(), log }); bootPhase = "ready"; + recordDaemonReady(); + setPhase("ready"); log.info({ pid: process.pid }, "daemon ready"); } catch (err) { log.fatal({ err }, "daemon boot failed"); - // Task 9 adds recordBootFailure(currentPhase, err) here. + recordBootFailure(currentPhase, String(err)); try { loggerHandle.flush?.(); } catch { /* */ } process.exit(1); } From fa6523f32bc5e11f050c84c0086923bf7a6d573c Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:20:52 -0500 Subject: [PATCH 068/106] fix(refresh): clear the deadline timer on every settle path, not just on wedge Promise.race never cancels the losing branch. When impl won (every successful refresh), the deadline setTimeout still fired ~4 min later and called onTimeout, logging a spurious "cache refresh timed out" warn and leaking a ref'd timer per cycle. Capture the timer handle and clearTimeout it in the shared .finally, which runs on every settle path; clearing an already-fired timer is a no-op. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/cache-refresh-coalesce.test.ts | 12 ++++++++++++ lib/daemon/cache-refresh.ts | 15 +++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/lib/daemon/__tests__/cache-refresh-coalesce.test.ts b/lib/daemon/__tests__/cache-refresh-coalesce.test.ts index 32a609d0..fa3c3eb0 100644 --- a/lib/daemon/__tests__/cache-refresh-coalesce.test.ts +++ b/lib/daemon/__tests__/cache-refresh-coalesce.test.ts @@ -31,3 +31,15 @@ test("coalesces concurrent callers onto one run", async () => { resolveRun(); await Promise.all([a, b]); }); + +test("a fast success does not fire onTimeout after the deadline elapses", async () => { + let timedOut = 0; + const coalesce = makeCoalescer( + () => Promise.resolve(), // settles well before the deadline + 50, + () => { timedOut++; }, + ); + await coalesce(); + await new Promise((r) => setTimeout(r, 150)); // past the deadline + expect(timedOut).toBe(0); // the deadline timer must have been cleared, not just outraced +}); diff --git a/lib/daemon/cache-refresh.ts b/lib/daemon/cache-refresh.ts index 5fe14920..2771c28b 100644 --- a/lib/daemon/cache-refresh.ts +++ b/lib/daemon/cache-refresh.ts @@ -66,10 +66,17 @@ export function makeCoalescer( return () => { if (inFlight) return inFlight; const impl = run().catch(() => {}); // a rejected cycle still clears the latch - const guarded = Promise.race([ - impl, - new Promise((resolve) => setTimeout(() => { onTimeout(); resolve(); }, deadlineMs)), - ]).finally(() => { inFlight = null; }); + // Promise.race never cancels the losing branch, so the deadline timer must be + // captured and cleared on every settle path or a fast success still fires + // onTimeout deadlineMs later, misreported as a wedge. + let deadlineTimer: ReturnType; + const deadline = new Promise((resolve) => { + deadlineTimer = setTimeout(() => { onTimeout(); resolve(); }, deadlineMs); + }); + const guarded = Promise.race([impl, deadline]).finally(() => { + clearTimeout(deadlineTimer); + inFlight = null; + }); inFlight = guarded; return guarded; }; From e9cdbd88a89953bb641821bf7d472e42d9a40dc7 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:25:23 -0500 Subject: [PATCH 069/106] freshness: rebuild the provider cache when gitlabToken rotates (S048, S049) Key the providers map on a token fingerprint so a rotated gitlabToken rebuilds the GitLabProvider on the next ensureProvider/getRepoContext call instead of waiting for a daemon restart. A token mismatch drops the stale watch and resets the userIdResolved latch so the next reconcile re-authenticates. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../freshness-provider-rotation.test.ts | 11 ++++++++++ lib/daemon/freshness.ts | 22 ++++++++++++------- 2 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 lib/daemon/__tests__/freshness-provider-rotation.test.ts diff --git a/lib/daemon/__tests__/freshness-provider-rotation.test.ts b/lib/daemon/__tests__/freshness-provider-rotation.test.ts new file mode 100644 index 00000000..e0de5507 --- /dev/null +++ b/lib/daemon/__tests__/freshness-provider-rotation.test.ts @@ -0,0 +1,11 @@ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("ensureProvider compares the current token before reusing a cached provider", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + // The unconditional cache-hit return is the S049 bug; it must be gone. + expect(src).not.toMatch(/const cached = providers\.get\(repoName\);\s*\n\s*if \(cached\) return cached;/); + // A token fingerprint must be stored alongside the provider. + expect(src).toMatch(/providers\.set\(repoName,\s*\{\s*provider/); +}); diff --git a/lib/daemon/freshness.ts b/lib/daemon/freshness.ts index 767feca0..f3d0b43a 100644 --- a/lib/daemon/freshness.ts +++ b/lib/daemon/freshness.ts @@ -89,7 +89,7 @@ const watches = new Map(); /** Bound merged pending so a wedged processKeys cannot grow memory unbounded. */ export const PENDING_CAP = 1000; -const providers = new Map(); +const providers = new Map(); let userId: number | null = null; let userIdResolved = false; let selfUsername: string | null = null; @@ -134,15 +134,21 @@ function makeProvider(host: string, token: string): GitLabProvider { } async function ensureProvider(repoName: string, repoPath: string): Promise { - const cached = providers.get(repoName); - if (cached) return cached; - const secrets = await loadSecrets(); if (!secrets.gitlabToken) { log.info(`no gitlabToken; skipping ${repoName}`); return null; } + const cached = providers.get(repoName); + if (cached && cached.token === secrets.gitlabToken) return cached.provider; + if (cached) { + // Token rotated: drop the stale watch built on the old token and + // re-resolve userId against the new one on the next reconcile. + stopWatch(repoName); + userIdResolved = false; + } + const remoteUrl = await getRemoteUrl(repoPath); if (!remoteUrl) { log.info(`no origin remote for ${repoName}; skipping`); @@ -161,14 +167,14 @@ async function ensureProvider(repoName: string, repoPath: string): Promise { if (userIdResolved) return userId; // Resolve via any available provider. If none exist yet, defer until one does. - const anyProvider = providers.values().next().value as GitLabProvider | undefined; + const anyProvider = providers.values().next().value?.provider as GitLabProvider | undefined; if (!anyProvider) return null; try { @@ -259,7 +265,7 @@ export async function getRepoContext( projectPathOverride?: string, ): Promise<{ provider: GitLabProvider; projectPath: string; projectId: number }> { const watch = watches.get(repoName); - let provider = providers.get(repoName) ?? null; + let provider = providers.get(repoName)?.provider ?? null; // Live-watch fast path — but only when the caller didn't override projectPath. // If they did, fall through to the ephemeral path so we use the canonical path. @@ -290,7 +296,7 @@ export async function getRepoContext( throw new Error(`could not parse remote URL "${remoteUrl}"`); } provider = makeProvider(remote.host, secrets.gitlabToken); - providers.set(repoName, provider); + providers.set(repoName, { provider, token: secrets.gitlabToken }); } // Pick projectPath: explicit override > previously-cached ephemeral > git remote. From 1af0f282f6c734c0a6f9b053e53382689eb14ba0 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:26:17 -0500 Subject: [PATCH 070/106] daemon: lazy-resolve trusted-origin allowlist, drop finding-ID comments Fix A: getTrustedBrowserOrigins() did a synchronous settings-store read on every single :9401 request, even the ~100% with no Origin header (CLI, tray, rt-client from Bun/Node) that never needed it. Added resolveOriginTrust() in api-auth.ts, which only resolves the allowlist when an Origin header is present, and wired both api-server.ts call sites (the /ws gate and the CORS/token gate) through it. Fix B: stripped leftover audit finding-ID citations (S005/S006/S040/ S041/S042/S054/S083/S085) from production comments in api-auth.ts and api-server.ts, keeping the underlying technical rationale. Fix C: reworded two stale "CORS is *" comments to describe the actual current default-deny CORS model (a trusted Origin gets its response echoed back; the local token, not CORS, is what stops a page from firing a mutating request in the first place). Co-Authored-By: Claude Sonnet 5 --- lib/daemon/__tests__/api-auth.test.ts | 34 ++++++++++++++++++++++++++- lib/daemon/api-auth.ts | 33 ++++++++++++++++++++------ lib/daemon/api-server.ts | 29 +++++++++++++---------- 3 files changed, 75 insertions(+), 21 deletions(-) diff --git a/lib/daemon/__tests__/api-auth.test.ts b/lib/daemon/__tests__/api-auth.test.ts index 6abf9eab..b5113660 100644 --- a/lib/daemon/__tests__/api-auth.test.ts +++ b/lib/daemon/__tests__/api-auth.test.ts @@ -5,7 +5,7 @@ import { describe, test, expect } from "bun:test"; import { needsToken, tokenOk, getApiToken, reloadApiToken, loadOrCreateApiToken } from "../api-auth.ts"; -import { isOriginAllowed, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "../api-auth.ts"; +import { isOriginAllowed, isBrowserRequestTrusted, getTrustedBrowserOrigins, resolveOriginTrust } from "../api-auth.ts"; import { mkdtempSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; @@ -177,3 +177,35 @@ describe("getTrustedBrowserOrigins", () => { expect(Array.isArray(origins)).toBe(true); }); }); + +describe("resolveOriginTrust", () => { + const apiToken = "the-real-token"; + + test("no Origin header: never calls getAllowedOrigins (the settings read is disk I/O every non-browser request would otherwise pay for)", () => { + let calls = 0; + const getAllowedOrigins = () => { calls++; return []; }; + const trusted = resolveOriginTrust(null, null, apiToken, getAllowedOrigins); + expect(trusted).toBe(true); + expect(calls).toBe(0); + }); + + test("an Origin header present: does call getAllowedOrigins", () => { + let calls = 0; + const getAllowedOrigins = () => { calls++; return ["http://localhost:5544"]; }; + const trusted = resolveOriginTrust("http://localhost:5544", null, apiToken, getAllowedOrigins); + expect(trusted).toBe(true); + expect(calls).toBe(1); + }); + + test("an Origin header with the correct token is trusted without needing the allowlist call to matter", () => { + let calls = 0; + const getAllowedOrigins = () => { calls++; return []; }; + const trusted = resolveOriginTrust("http://evil.example", apiToken, apiToken, getAllowedOrigins); + expect(trusted).toBe(true); + expect(calls).toBe(1); + }); + + test("defaults to the real getTrustedBrowserOrigins when no override is passed", () => { + expect(resolveOriginTrust(null, null, apiToken)).toBe(true); + }); +}); diff --git a/lib/daemon/api-auth.ts b/lib/daemon/api-auth.ts index dae00efa..ac45a18f 100644 --- a/lib/daemon/api-auth.ts +++ b/lib/daemon/api-auth.ts @@ -1,8 +1,10 @@ /** * Local-token auth for the :9401 API server. * - * The server binds to 127.0.0.1, but CORS is `*`, so a malicious web page could - * still drive *mutating* endpoints via the browser. Requiring a custom header + * The server binds to 127.0.0.1, and CORS is default-deny (only a trusted + * Origin gets its response readable), but CORS alone does not stop a + * malicious web page from firing a mutating request in the first place -- it + * only stops the page from reading the reply. Requiring a custom header * (X-RT-Token) on those routes forces a CORS preflight the page can't satisfy * (it can't read the token), blocking cross-site control while leaving reads * open for convenience. @@ -44,7 +46,7 @@ export function loadOrCreateApiToken(tokenPath: string = API_TOKEN_PATH): string /** * `getApiToken`/`reloadApiToken` share ONE in-memory value between - * api-server.ts and the secrets handler (S054): before this, api-server + * api-server.ts and the secrets handler: before this, api-server * captured a token once at boot while the secrets handler called * `loadOrCreateApiToken()` fresh on every request, so an external rotation * (deleting api-token to force a new one) left the two permanently @@ -72,15 +74,15 @@ export function reloadApiToken(tokenPath: string = API_TOKEN_PATH): string { * drains something a GET should not silently consume, and must present the * local token. A CORS preflight (OPTIONS) can never present the custom * X-RT-Token header, so it is never gated, on any path. Otherwise - * default-gated for every method except GET/HEAD (S040: an allowlist-by-path - * guaranteed the next mutating route would ship unguarded), plus two + * default-gated for every method except GET/HEAD (an allowlist-by-path + * approach guarantees the next mutating route would ship unguarded), plus two * explicit GET exceptions whose verb lies about being a read. */ export function needsToken(method: string, pathname: string): boolean { if (method === "OPTIONS") return false; if (method === "GET" || method === "HEAD") { // Gated despite being a GET: /api/secrets's response body IS a - // credential (S054); /api/notifications DRAINS the queue (S041), so its + // credential; /api/notifications DRAINS the queue, so its // verb lies about being a read the way every other GET here is not. if (pathname === "/api/secrets") return true; if (pathname === "/api/notifications") return true; @@ -114,7 +116,7 @@ export function isOriginAllowed(origin: string, allowedOrigins: readonly string[ } /** - * The 127.0.0.1 trust boundary (S005/S006): the daemon binds loopback-only, + * The 127.0.0.1 trust boundary: the daemon binds loopback-only, * but any web page the user visits also runs on 127.0.0.1 and can send a * request. A request with NO Origin header at all is not a browser fetch -- * it is the CLI, the Swift tray, rt-client from a Bun/Node process, or the @@ -132,3 +134,20 @@ export function isBrowserRequestTrusted( if (tokenOk(token, apiToken)) return true; return isOriginAllowed(origin, allowedOrigins); } + +/** + * Same trust decision as isBrowserRequestTrusted, but resolves the allowlist + * lazily: getAllowedOrigins() only runs when an Origin header is present. + * getTrustedBrowserOrigins does synchronous disk I/O on the settings store, + * and the vast majority of :9401 traffic (the CLI, the tray, rt-client from a + * Bun/Node process) carries no Origin at all, so it must never pay that cost. + */ +export function resolveOriginTrust( + origin: string | null, + presentedToken: string | null, + apiToken: string, + getAllowedOrigins: () => readonly string[] = getTrustedBrowserOrigins, +): boolean { + if (!origin) return true; + return isBrowserRequestTrusted(origin, presentedToken, apiToken, getAllowedOrigins()); +} diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index 6c5bd2c4..cfe54790 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -10,7 +10,7 @@ import type { Server, ServerWebSocket } from "bun"; import type { Logger } from "pino"; import { API_PORT } from "../daemon-config.ts"; -import { needsToken, tokenOk, getApiToken, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "./api-auth.ts"; +import { needsToken, tokenOk, getApiToken, resolveOriginTrust } from "./api-auth.ts"; import { getAggregatedConnection } from "./freshness.ts"; import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; import { runCapture } from "../subprocess.ts"; @@ -93,7 +93,7 @@ export interface BroadcastTarget { /** * Sends one frame to every client, dropping any that Bun's own send() return - * value marks as gone (S042). `ws.send()` never throws on a dead socket -- + * value marks as gone. `ws.send()` never throws on a dead socket -- * it returns 0 (this send silently failed) or -1 (backpressure) -- so a * disconnected or stalled console/chat-viewer tab used to keep receiving a * SUBSET of frames forever with nothing logged. 0 means Bun already dropped @@ -149,7 +149,7 @@ export function clearWsClients(): void { } /** - * CORS default-deny (S006): a browser page on an untrusted Origin still gets + * CORS default-deny: a browser page on an untrusted Origin still gets * its request served (127.0.0.1 loopback + the per-route token gate are the * real defenses), but the response carries no Access-Control-Allow-Origin, * so the page's own JavaScript cannot read the body. A request with no @@ -170,7 +170,7 @@ export function buildCorsHeaders(origin: string | null, trusted: boolean): Recor /** * Decodes one path segment between a fixed prefix (and optional suffix), * returning `undefined` (never throwing) on any shape mismatch or malformed - * %-encoding (S083). Before this, each parameterized route hand-rolled its + * %-encoding. Before this, each parameterized route hand-rolled its * own decodeURIComponent inside the route's try block, so a malformed * segment fell through to the OUTER catch and came back as a logged 500; * every route using this helper instead gets a clean 400. @@ -191,8 +191,8 @@ export function pathParam(pathname: string, prefix: string, suffix = ""): string const PLAIN_NUMBER_RE = /^-?\d+(\.\d+)?$/; /** - * REST query strings arrive as strings no matter what the client meant - * (S085): "?maxAgeMs=60000" and "?refresh=true" reached handlers that do a + * REST query strings arrive as strings no matter what the client meant: + * "?maxAgeMs=60000" and "?refresh=true" reached handlers that do a * strict `typeof x === "number"` or `x === true` check, so the documented * flag silently no-op'd over HTTP while working over the socket (where * payloads are real JSON). One coercion at the REST seam fixes every such @@ -298,15 +298,14 @@ export async function startApiServer(deps: ApiServerDeps): Promise> async fetch(req, server) { const url = new URL(req.url); const origin = req.headers.get("origin"); - const allowedOrigins = getTrustedBrowserOrigins(); // WebSocket upgrade (broadcast channel). Browsers cannot set custom // headers on a WS handshake, so the token (when a browser page wants // to identify itself) travels as a ?token= query param instead of - // X-RT-Token (S005). + // X-RT-Token. if (url.pathname === "/ws") { const wsToken = url.searchParams.get("token"); - if (!isBrowserRequestTrusted(origin, wsToken, apiToken, allowedOrigins)) { + if (!resolveOriginTrust(origin, wsToken, apiToken)) { return new Response("origin not allowed", { status: 403 }); } if (server.upgrade(req, { data: { kind: "broadcast" } })) return undefined as any; @@ -315,16 +314,20 @@ export async function startApiServer(deps: ApiServerDeps): Promise> // CORS: default-deny. A trusted Origin (token or allowlist) gets its // Origin echoed back; anything else gets no Access-Control-Allow-Origin - // at all, so a malicious page's own JS cannot read the response (S006). - const trusted = isBrowserRequestTrusted(origin, req.headers.get("x-rt-token"), apiToken, allowedOrigins); + // at all, so a malicious page's own JS cannot read the response. + // resolveOriginTrust only resolves the allowlist when origin is set, + // since the settings read behind it is synchronous disk I/O. + const trusted = resolveOriginTrust(origin, req.headers.get("x-rt-token"), apiToken); const corsHeaders = buildCorsHeaders(origin, trusted); if (req.method === "OPTIONS") { return new Response(null, { status: 204, headers: corsHeaders }); } - // Gate mutating routes behind the local token (CORS is *, so this is the - // CSRF defense against a malicious page driving control endpoints). + // Gate mutating routes behind the local token. CORS default-deny only + // stops a malicious page from reading the response; it can still fire + // the request itself (a classic CSRF), so the token is the actual + // defense against a malicious page driving control endpoints. if (needsToken(req.method, url.pathname) && !tokenOk(req.headers.get("x-rt-token"), apiToken)) { return Response.json({ ok: false, error: "unauthorized" }, { status: 401, headers: corsHeaders }); } From fbf7ba88518aafd446dc54fdc57bc10f544b6462 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:29:39 -0500 Subject: [PATCH 071/106] docs: name the S084 GET-route known-gap ruling in the trust-boundary note --- docs/daemon-api-auth.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/daemon-api-auth.md b/docs/daemon-api-auth.md index 1eb33045..3e2b1584 100644 --- a/docs/daemon-api-auth.md +++ b/docs/daemon-api-auth.md @@ -47,3 +47,16 @@ and `.port`). Catch it around the `startApiServer()` call and park-and-retry with backoff instead of letting it reach the top-level crash path; any other error out of `startApiServer()` is a genuine misconfiguration and should keep crashing as it does today. + +## Known gap: GET routes that trigger real work stay ungated + +`GET /api/cache?maxAgeMs=` can force a full cache refresh, and +`GET /api/sdm/recents` spawns `sdm status`; neither requires the local token. +Default-deny CORS stops an untrusted browser Origin from reading the +response, but a plain cross-origin GET needs no preflight, so the request +still lands and the work still runs even when the response is unreadable. +This is deliberately not fixed here: gating these routes risks breaking +non-browser REST consumers that read them untokened today (the tray, +editor extensions), and there was no time in this pass to audit every such +consumer, which the audit's own fixer notes flag as the real risk of +tokening reads. Tracked as an open follow-up, not a silently-closed finding. From 88da40614f9c23bdc4344ef2c491ac2c1a3fbfa0 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:34:16 -0500 Subject: [PATCH 072/106] freshness: close the two remaining stale-token paths (S048/S049 review fix) getRepoContext served a cached provider without checking its token, so poll-mode repos and already-cached forge-handler providers kept a rotated token forever. reconcileFreshnessImpl skipped ensureProvider entirely for repos with a live watch, so a running watcher never rebuilt after rotation either. Both now compare the cached token against loadSecrets() before reuse and drop the stale watch/provider on a mismatch. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../freshness-provider-rotation.test.ts | 23 +++++++++++++++++++ lib/daemon/freshness.ts | 23 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/lib/daemon/__tests__/freshness-provider-rotation.test.ts b/lib/daemon/__tests__/freshness-provider-rotation.test.ts index e0de5507..8c5f59c3 100644 --- a/lib/daemon/__tests__/freshness-provider-rotation.test.ts +++ b/lib/daemon/__tests__/freshness-provider-rotation.test.ts @@ -9,3 +9,26 @@ test("ensureProvider compares the current token before reusing a cached provider // A token fingerprint must be stored alongside the provider. expect(src).toMatch(/providers\.set\(repoName,\s*\{\s*provider/); }); + +test("getRepoContext drops a stale-token provider before serving the cached one (S048/S049 fix 1)", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + const tokenCheckMatch = src.match(/cachedForToken\.token !== currentSecrets\.gitlabToken/); + expect(tokenCheckMatch).not.toBeNull(); + // The comparison must run before the cache is read for the fast path, so a + // rotated token can never reach a poll-mode repo or an already-cached + // forge-handler provider. + const fastPathIndex = src.indexOf("const watch = watches.get(repoName);"); + expect(fastPathIndex).toBeGreaterThan(-1); + expect(tokenCheckMatch!.index!).toBeLessThan(fastPathIndex); +}); + +test("reconcileFreshnessImpl drops a stale-token watch before skipping already-watched repos (S048/S049 fix 2)", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + const staleWatchDrop = src.match(/existing\.token !== secrets\.gitlabToken\)\s*stopWatch\(repoName\);/); + expect(staleWatchDrop).not.toBeNull(); + // The drop must happen before the "already watched, skip" short-circuit, + // or a live watcher built on a rotated token never rebuilds. + const skipIndex = src.indexOf("if (watches.has(repoName)) continue;"); + expect(skipIndex).toBeGreaterThan(-1); + expect(staleWatchDrop!.index!).toBeLessThan(skipIndex); +}); diff --git a/lib/daemon/freshness.ts b/lib/daemon/freshness.ts index f3d0b43a..e2bb2b2f 100644 --- a/lib/daemon/freshness.ts +++ b/lib/daemon/freshness.ts @@ -264,6 +264,20 @@ export async function getRepoContext( repoPath?: string, projectPathOverride?: string, ): Promise<{ provider: GitLabProvider; projectPath: string; projectId: number }> { + // A cached provider's token may have rotated since it was built; poll-mode + // repos and already-cached forge-handler providers never pass back through + // ensureProvider, so this is the only place that catches a stale token for + // them (S048/S049). + const cachedForToken = providers.get(repoName); + if (cachedForToken) { + const currentSecrets = await loadSecrets(); + if (currentSecrets.gitlabToken && cachedForToken.token !== currentSecrets.gitlabToken) { + stopWatch(repoName); + userIdResolved = false; + providers.delete(repoName); + } + } + const watch = watches.get(repoName); let provider = providers.get(repoName)?.provider ?? null; @@ -702,6 +716,15 @@ async function reconcileFreshnessImpl(env: FreshnessEnv): Promise { for (const [repoName, repoPath] of Object.entries(repoIndex)) { if (grants(tracking, repoName).mode !== "live") continue; + + // A live watch whose provider token has since rotated must be dropped so the + // ensureProvider/startWatch below rebuilds it with the current token (S048/S049). + const existing = providers.get(repoName); + if (existing && watches.has(repoName)) { + const secrets = await loadSecrets(); + if (secrets.gitlabToken && existing.token !== secrets.gitlabToken) stopWatch(repoName); + } + if (watches.has(repoName)) continue; const provider = await ensureProvider(repoName, repoPath); From 58ab7e45276fd5749d6f87d0ad77d6dc6908c064 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:40:03 -0500 Subject: [PATCH 073/106] system-process-scanner: a failed lsof preserves runaway windows (S061) gather() returned [] on both a real empty scan and an lsof/ps failure, so scan() pruned every tracked pid on a transient failure, resetting firstSeen and the runaway sample window. getAllRepoPids and gather now return null on scan failure (distinct from an empty result), and scan/refresh early-return the prior lastResult before touching tracked state. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../system-process-scanner-resilience.test.ts | 46 +++++++++++++++++++ lib/daemon/system-process-scanner.ts | 21 ++++++--- 2 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 lib/daemon/__tests__/system-process-scanner-resilience.test.ts diff --git a/lib/daemon/__tests__/system-process-scanner-resilience.test.ts b/lib/daemon/__tests__/system-process-scanner-resilience.test.ts new file mode 100644 index 00000000..c5db5939 --- /dev/null +++ b/lib/daemon/__tests__/system-process-scanner-resilience.test.ts @@ -0,0 +1,46 @@ +import { test, expect } from "bun:test"; +import { SystemProcessScanner } from "../system-process-scanner.ts"; + +/** Drives gather()'s return value directly so a null (failed lsof/ps tick) + * can be simulated without shelling out. */ +class FakeScanner extends SystemProcessScanner { + next: any[] | null = []; + protected override async gather(): Promise { + return this.next; + } +} + +function fakeProcess(pid: number, cpuPercent: number) { + return { + pid, + ppid: 1, + command: "node", + fullCommand: "node build.js", + cpuPercent, + rssKb: 10_000, + uptime: "01:00", + cwd: "/repo", + repo: "r", + worktree: null, + branch: null, + relativeDir: "", + port: null, + linearTicket: null, + packageScript: null, + }; +} + +test("a failed gather (null) keeps tracked windows and lastResult intact", async () => { + const s = new FakeScanner(); + + s.next = [fakeProcess(4242, 95)]; + const first = await s.scan(); + expect(first.find((p) => p.pid === 4242)).toBeTruthy(); + const firstSeen = s.getTracked(4242)?.firstSeen; + expect(firstSeen).toBeDefined(); + + s.next = null; + const during = await s.scan(); + expect(s.getTracked(4242)?.firstSeen).toBe(firstSeen); + expect(during.find((p) => p.pid === 4242)).toBeTruthy(); +}); diff --git a/lib/daemon/system-process-scanner.ts b/lib/daemon/system-process-scanner.ts index dfb121c2..2ac18941 100644 --- a/lib/daemon/system-process-scanner.ts +++ b/lib/daemon/system-process-scanner.ts @@ -222,7 +222,7 @@ async function getPackageScripts(pidList: string): Promise> return parsePackageScripts(stdout); } -async function getAllRepoPids(trackedPaths: string[]): Promise> { +async function getAllRepoPids(trackedPaths: string[]): Promise | null> { if (trackedPaths.length === 0) return new Map(); // Single lsof call to get cwds for ALL processes at once. @@ -232,8 +232,11 @@ async function getAllRepoPids(trackedPaths: string[]): Promise { + const gathered = await this.gather(portEntries); + if (gathered === null) return this.lastResult; // failed tick: preserve tracked/lastResult/lastScanAt try { - const gathered = await this.gather(portEntries); const now = Date.now(); const currentPids = new Set(); @@ -344,8 +348,9 @@ export class SystemProcessScanner { * alert. Also advances `lastScanAt` so freshness checks see the update. */ async refresh(portEntries: PortEntry[] = []): Promise { + const gathered = await this.gather(portEntries); + if (gathered === null) return this.lastResult; // failed tick: preserve tracked/lastResult/lastScanAt try { - const gathered = await this.gather(portEntries); const prev = new Map(this.lastResult.map(p => [p.pid, p])); const results = gathered.map(proc => { const before = prev.get(proc.pid); @@ -368,7 +373,7 @@ export class SystemProcessScanner { * decorate it with port + package-script, but no runaway/tracking state. * Shared by `scan` (adds tracking) and `refresh` (carries flags forward). */ - private async gather(portEntries: PortEntry[]): Promise { + protected async gather(portEntries: PortEntry[]): Promise { const repos = loadRepoIndex(); if (Object.keys(repos).length === 0) return []; @@ -378,7 +383,8 @@ export class SystemProcessScanner { const worktreeMap = await buildWorktreeMap(repos); const trackedPaths = [...new Set([...Object.values(repos), ...worktreeMap.keys()])]; const cwdMap = await getAllRepoPids(trackedPaths); - if (cwdMap.size === 0) return []; + if (cwdMap === null) return null; // lsof failed + if (cwdMap.size === 0) return []; // genuinely no tracked-cwd processes // Get CPU/memory for discovered PIDs const pidList = [...cwdMap.keys()].join(","); @@ -386,6 +392,7 @@ export class SystemProcessScanner { ["ps", "-p", pidList, "-o", "pid=,ppid=,pcpu=,rss=,etime=,comm=,args="], { timeoutMs: 5000 }, ); + if (psRes.exitCode !== 0 && !psRes.stdout) return null; // ps failed if (!psRes.stdout) return []; const parsed = parseProcessList(psRes.stdout, repos, cwdMap, worktreeMap); From 3c090d3daedca876376a24147d7312ab336e4275 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:45:09 -0500 Subject: [PATCH 074/106] runs: mtime-memoize finished-run summaries; back off herdr probe (S101, S038, S039) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/agent-status-poller.test.ts | 17 +++++++++++ lib/daemon/agent-status-poller.ts | 11 +++++++- lib/runs/__tests__/store-memo.test.ts | 28 +++++++++++++++++++ lib/runs/store.ts | 20 +++++++++++-- 4 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 lib/runs/__tests__/store-memo.test.ts diff --git a/lib/daemon/__tests__/agent-status-poller.test.ts b/lib/daemon/__tests__/agent-status-poller.test.ts index 57020710..a1507a8e 100644 --- a/lib/daemon/__tests__/agent-status-poller.test.ts +++ b/lib/daemon/__tests__/agent-status-poller.test.ts @@ -73,3 +73,20 @@ test("finished runs are ignored and dropped from tracking", async () => { await handle!.tick(); expect(events).toHaveLength(0); }); + +test("backs off the herdr probe after repeated failures", async () => { + let probeCalls = 0; + handle = startAgentStatusPoller({ + emitEvent: () => {}, + log: quietLog, + intervalMs: 3_600_000, // real timer never fires + probe: async () => { probeCalls++; return null; }, // herdr absent + list: () => [], + }); + for (let i = 0; i < 20; i++) await handle.tick(); + // Without backoff this would be 20; with backoff (threshold 3, 1-in-6) far fewer. + expect(probeCalls).toBeLessThan(10); + // Backoff never stops probing forever: 3 initial failures engage it, then + // one probe every BACKOFF_TICKS (ticks 9 and 15 of 20) keeps checking. + expect(probeCalls).toBe(5); +}); diff --git a/lib/daemon/agent-status-poller.ts b/lib/daemon/agent-status-poller.ts index faf30ed2..fb50a410 100644 --- a/lib/daemon/agent-status-poller.ts +++ b/lib/daemon/agent-status-poller.ts @@ -16,6 +16,8 @@ import type { RunLiveness } from "../runs/attention.ts"; import { listRuns } from "../runs/store.ts"; const POLL_MS = 10_000; +const FAILURE_THRESHOLD = 3; // consecutive null probes before backing off +const BACKOFF_TICKS = 6; // then probe once every 6 ticks (~60s at 10s cadence) interface Log { info(obj: unknown, msg?: string): void; @@ -39,10 +41,17 @@ export function startAgentStatusPoller(opts: { const list = opts.list ?? ((liveness: RunLiveness) => listRuns(undefined, liveness)); const last = new Map(); let seeded = false; + let consecutiveFailures = 0; + let ticksSkipped = 0; const tick = async (): Promise => { + if (consecutiveFailures >= FAILURE_THRESHOLD) { + if (++ticksSkipped < BACKOFF_TICKS) return; + ticksSkipped = 0; + } const entries = await probe(); - if (entries === null) return; + if (entries === null) { consecutiveFailures++; return; } + consecutiveFailures = 0; primeLivenessCache(entries); let runs: RunSummary[]; try { diff --git a/lib/runs/__tests__/store-memo.test.ts b/lib/runs/__tests__/store-memo.test.ts new file mode 100644 index 00000000..da1b9a46 --- /dev/null +++ b/lib/runs/__tests__/store-memo.test.ts @@ -0,0 +1,28 @@ +import { Database } from "bun:sqlite"; +import { afterEach, expect, spyOn, test } from "bun:test"; +import { listRuns } from "../store.ts"; +import { root, seedRun } from "./fixtures.ts"; + +afterEach(() => { delete process.env.RT_RUNS_ROOT; }); + +test("a finished run's db is opened once, then served from the mtime cache", () => { + const dir = root(); + seedRun(dir, "repoA", "run1", 1000, 1, { status: "done" }); + const openSpy = spyOn(Database.prototype, "query"); + listRuns(); // first call opens + reads + const afterFirst = openSpy.mock.calls.length; + listRuns(); // second call: mtime unchanged -> no reopen + expect(openSpy.mock.calls.length).toBe(afterFirst); // no additional queries for run1 + openSpy.mockRestore(); +}); + +test("a running run is never cached: it is reopened even when its mtime is unchanged", () => { + const dir = root(); + seedRun(dir, "repoA", "run2", 1000, 1, { status: "running" }); + const openSpy = spyOn(Database.prototype, "query"); + listRuns(); + const afterFirst = openSpy.mock.calls.length; + listRuns(); + expect(openSpy.mock.calls.length).toBeGreaterThan(afterFirst); + openSpy.mockRestore(); +}); diff --git a/lib/runs/store.ts b/lib/runs/store.ts index 3e290ac2..6d15e46d 100644 --- a/lib/runs/store.ts +++ b/lib/runs/store.ts @@ -4,7 +4,7 @@ * and per-call — no held connections, so a run dir can be pruned under us. */ import { Database } from "bun:sqlite"; -import { existsSync, readdirSync, type Dirent } from "fs"; +import { existsSync, readdirSync, statSync, type Dirent } from "fs"; import { homedir } from "os"; import { join } from "path"; import type { Attention, RunDetail, RunFieldRow, RunStageRow, RunSummary } from "../../packages/rt-client/src/commands.ts"; @@ -114,17 +114,33 @@ function withAttention(db: Database, row: RunSummary, liveness?: RunLiveness): R } } +// Finished runs never change; skip the open+PRAGMA+4-reads when the db mtime +// is unchanged. Running runs are never cached: their db still mutates and +// their liveness overlay is recomputed per call. +const summaryCache = new Map(); + export function listRuns(repo?: string, liveness?: RunLiveness): RunSummary[] { if (repo != null && !isPathComponent(repo)) return []; const repos = repo ? [repo] : dirs(runsRoot()); const out: RunSummary[] = []; for (const r of repos) { for (const id of dirs(join(runsRoot(), r))) { + const dbPath = join(runsRoot(), r, id, "state.db"); + let mtimeMs: number; + try { mtimeMs = statSync(dbPath).mtimeMs; } catch { continue; } + const key = `${r}/${id}`; + const hit = summaryCache.get(key); + if (hit && hit.mtimeMs === mtimeMs) { out.push(hit.summary); continue; } + const opened = openRun(r, id); if (!opened) continue; try { const row = runRow(opened.db); - if (row) out.push(withAttention(opened.db, row, liveness)); + if (row) { + const summary = withAttention(opened.db, row, liveness); + out.push(summary); + if (summary.status !== "running") summaryCache.set(key, { mtimeMs, summary }); + } } finally { opened.db.close(); } From e9313e0a9a52f249afd828ec6b57b4de5c9e456e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:46:40 -0500 Subject: [PATCH 075/106] daemon status: alive-not-serving / parked / boot-failed / crash-looping verdicts rt daemon status and the ping handler now read Task 9's breadcrumb file and supervision kv to distinguish a live-but-not-serving daemon (booting/wedged/ quarantined), a flavor standoff (parked), a single boot failure, and a crash loop, on top of the existing not-installed/running/degraded/not-running verdicts. pidAlive uses lsof scoped to RT_DIR instead of a system-wide pgrep, since the brief's suggested pgrep pattern false-positives against any real rt daemon running under a different HOME on the same machine. --- .../__tests__/daemon-status-render.test.ts | 37 +++++ commands/daemon.ts | 109 +++++++++++++- e2e/tests/daemon.test.ts | 24 ++++ lib/__tests__/daemon-status.test.ts | 133 +++++++++++++++++- lib/command-tree-def.ts | 4 +- lib/daemon-status.ts | 116 +++++++++++++-- lib/daemon/__tests__/status-identity.test.ts | 14 ++ lib/daemon/handlers/status.ts | 13 +- 8 files changed, 429 insertions(+), 21 deletions(-) diff --git a/commands/__tests__/daemon-status-render.test.ts b/commands/__tests__/daemon-status-render.test.ts index eb4ff2df..98e27b51 100644 --- a/commands/__tests__/daemon-status-render.test.ts +++ b/commands/__tests__/daemon-status-render.test.ts @@ -71,4 +71,41 @@ describe("statusLines", () => { expect(out).toContain("watching: 1 repo"); expect(out).not.toContain("1 repos"); }); + + // ── Task 10 ── + + test("a parked pid points at the flavor mismatch, not 'not running'", () => { + const out = plain({ state: "parked", pid: 42, holderFlavor: "prod" }); + expect(out).toContain("parked"); + expect(out).toContain("pid 42"); + expect(out).toContain("held by: prod"); + expect(out).not.toContain("installed but not running"); + }); + + test("alive-not-serving names the pid and the stuck detail", () => { + const out = plain({ state: "alive-not-serving", pid: 42, detail: "booting" }); + expect(out).toContain("process 42 is running but not answering rt.sock"); + expect(out).toContain("still booting"); + expect(out).not.toContain("installed but not running"); + }); + + test("alive-not-serving wedged/quarantined get their own detail lines", () => { + expect(plain({ state: "alive-not-serving", pid: 1, detail: "wedged" })).toContain("deadlocked"); + expect(plain({ state: "alive-not-serving", pid: 1, detail: "quarantined" })).toContain("recovered from a corrupt db"); + }); + + test("crash-looping surfaces the failure count and the last reason", () => { + const out = plain({ state: "crash-looping", failures: 4, reason: "EADDRINUSE" }); + expect(out).toContain("crash-looping"); + expect(out).toContain("4 failures"); + expect(out).toContain("EADDRINUSE"); + }); + + test("boot-failed surfaces the phase and reason, and points at rt daemon start", () => { + const out = plain({ state: "boot-failed", reason: "EADDRINUSE", phase: "api" }); + expect(out).toContain("boot failed"); + expect(out).toContain("phase: api"); + expect(out).toContain("EADDRINUSE"); + expect(out).toContain("rt daemon start"); + }); }); diff --git a/commands/daemon.ts b/commands/daemon.ts index ab571ad6..4a71a43a 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -35,6 +35,8 @@ import { daemonQuery, isDaemonRunning, trayQuery } from "../lib/daemon-client.ts import { classifyDaemonStatus, type DaemonStatusVerdict } from "../lib/daemon-status.ts"; import { resolveIntendedMode, currentMode, type IntendedMode } from "../lib/dev-mode.ts"; import { probeSocketHolder } from "../lib/daemon/park.ts"; +import { readBreadcrumb, readSupervisionState } from "../lib/daemon/supervision-state.ts"; +import { runCapture } from "../lib/subprocess.ts"; import { isGitLabRemote } from "../lib/enrich.ts"; import type { CacheKind, RepoTrackingEntry } from "../lib/repo-tracking.ts"; import { loadRepoTracking, loadMachineRepoTracking, loadMachineRepoTrackingRaw, saveRepoTrackingRaw, grants, parseCachesArg, CACHE_KINDS, DEFAULT_PROJECT_MRS_WINDOW_DAYS, teamNamesIdentity } from "../lib/repo-tracking.ts"; @@ -316,8 +318,41 @@ export async function restart(): Promise { // ─── Status ────────────────────────────────────────────────────────────────── -export async function showStatus(): Promise { +/** + * Raw OS-level liveness, independent of rt.sock. Tries a direct pid check + * first against every pid this HOME actually recorded — rt.pid, then the + * boot breadcrumb's pid (the breadcrumb survives failures rt.pid never gets + * written for, per Ruling P1) — before falling back to a last-resort scan. + * + * That scan is `lsof +D `, not the brief's suggested system-wide + * `pgrep -f 'rt --daemon|lib/daemon.ts'`: a raw pgrep matches ANY rt daemon + * on the machine regardless of which HOME started it, and on an ordinary dev + * workstation there usually IS one — the developer's own real daemon — so a + * pgrep-based check on an isolated/alternate HOME reliably misreports a dead + * boot attempt as alive-not-serving (verified live against this repo's own + * dev daemon while writing the e2e test below). `lsof +D` instead asks "does + * any process hold a file open under THIS HOME's rt dir" — home-scoped by + * construction, immune to that false positive and to pid-reuse. Only worth + * calling once both `status` and a plain ping have already failed. + */ +async function probePidAlive(recordedPid: number | null, breadcrumbPid?: number): Promise<{ alive: boolean; pid: number | null }> { + for (const candidate of [recordedPid, breadcrumbPid ?? null]) { + if (candidate === null) continue; + try { + process.kill(candidate, 0); + return { alive: true, pid: candidate }; + } catch { /* not this one — try the next candidate */ } + } + const { stdout } = await runCapture(["lsof", "-t", "+D", RT_DIR], { timeoutMs: 3000 }); + const pids = stdout.trim().split(/\s+/).filter(Boolean).map(Number).filter((n) => !isNaN(n)); + return pids.length > 0 ? { alive: true, pid: pids[0]! } : { alive: false, pid: recordedPid }; +} + +export async function showStatus(args: string[] = []): Promise { + const json = args.includes("--json"); + if (!isDaemonInstalled()) { + if (json) return void console.log(JSON.stringify({ ok: true, state: "not-installed" })); console.log(` ${dim}○${reset} not installed ${dim}(run rt daemon install)${reset}\n`); return; } @@ -326,14 +361,40 @@ export async function showStatus(): Promise { // A failed status query does NOT mean the daemon is down — it answers `ping` // in a fraction of the budget a loaded `status` needs. Establish liveness // before reporting, and only pay for the probe when nothing came back. - const alive = classifyDaemonStatus.needsLivenessProbe(response) ? await isDaemonRunning() : false; + const pingOk = classifyDaemonStatus.needsLivenessProbe(response) ? await isDaemonRunning() : false; + const recordedPid = readDaemonPid() ?? null; + + // Ping ALSO failed: the only remaining ground is the pid/breadcrumb/kv + // trail Task 9 left behind. Read it here, once, rather than on every status + // call — it's the uncommon path. + let pidAlive: boolean | undefined; + let pid = recordedPid; + let breadcrumb: ReturnType | undefined; + let supervision: ReturnType | undefined; + if (classifyDaemonStatus.needsPidProbe(response, pingOk)) { + breadcrumb = readBreadcrumb(); + // The kv tier can be legitimately empty (or reflect nothing useful) when + // a failure happened before state.db ever opened — Ruling P1. The + // breadcrumb read above is what classifyDaemonStatus falls back to then. + supervision = readSupervisionState(); + const probed = await probePidAlive(recordedPid, breadcrumb?.pid); + pidAlive = probed.alive; + pid = probed.pid; + } + const verdict = classifyDaemonStatus({ installed: true, response, - alive, - pid: readDaemonPid() ?? null, + pingOk, + pid, + pidAlive, + intendedFlavor: resolveIntendedMode().mode, + breadcrumb, + supervision, }); + if (json) return void console.log(JSON.stringify({ ok: true, ...verdict })); + for (const line of statusLines(verdict, Date.now())) console.log(line); if (verdict.state === "running") { @@ -417,6 +478,46 @@ export function statusLines(verdict: DaemonStatusVerdict, now: number): string[] return lines; } + if (verdict.state === "parked") { + const lines = [` ${yellow}◐${reset} parked ${dim}(pid ${verdict.pid} — another flavor owns rt.sock)${reset}`]; + lines.push( + verdict.holderFlavor + ? ` ${dim}held by: ${verdict.holderFlavor}${reset}` + : ` ${dim}waiting for the intended flavor to take rt.sock${reset}`, + ); + lines.push(` ${dim}check: rt settings dev-mode${reset}`); + return lines; + } + + if (verdict.state === "alive-not-serving") { + const detailLine = { + booting: "still booting", + wedged: "reached ready but stopped answering — likely deadlocked", + quarantined: "recovered from a corrupt db but still not answering", + }[verdict.detail]; + return [ + ` ${yellow}●${reset} process ${verdict.pid} is running but not answering rt.sock`, + ` ${dim}${detailLine}${reset}`, + ` ${dim}check: rt daemon logs -t${reset}`, + ]; + } + + if (verdict.state === "crash-looping") { + return [ + ` ${red}●${reset} crash-looping ${dim}(${verdict.failures} failures recently)${reset}`, + ` ${dim}last reason: ${verdict.reason}${reset}`, + ` ${dim}check: rt daemon logs -t${reset}`, + ]; + } + + if (verdict.state === "boot-failed") { + return [ + ` ${red}●${reset} boot failed ${dim}(phase: ${verdict.phase})${reset}`, + ` ${dim}reason: ${verdict.reason}${reset}`, + ` ${dim}run: rt daemon start${reset}`, + ]; + } + if (verdict.state === "not-running") { const lines = [` ${red}●${reset} installed but not running`]; if (verdict.pid) lines.push(` ${dim}last pid: ${verdict.pid}${reset}`); diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index e0403285..469d54a6 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -57,6 +57,30 @@ describe("fatal boot", () => { } }, 60_000); + test("API-bind failure surfaces as boot-failed/crash-looping via rt daemon status --json", async () => { + const { path: home, cleanup } = createTestHome(); + // A different port than the other API-bind-failure tests above, so + // parallel test files can never collide on the same bound TCP port. + const port = 9413; + const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + try { + // `rt daemon status` short-circuits to "not installed" before it ever + // reaches the boot-failed/crash-looping classification — install first. + await rt(["daemon", "install"], { home }); + + const boot = await rt(["--daemon"], { home, env: { RT_API_PORT: String(port) } }); + expect(boot.exitCode).not.toBe(0); + + const status = await rt(["daemon", "status", "--json"], { home }); + expect(status.exitCode).toBe(0); + const parsed = JSON.parse(status.stdout); + expect(["boot-failed", "crash-looping"]).toContain(parsed.state); + } finally { + squatter.stop(true); + cleanup(); + } + }, 60_000); + test("a corrupt events.db self-heals — quarantined, and the daemon boots and serves", async () => { const { path: home, cleanup } = createTestHome(); const bunDir = join(process.execPath, ".."); diff --git a/lib/__tests__/daemon-status.test.ts b/lib/__tests__/daemon-status.test.ts index 84242905..17187267 100644 --- a/lib/__tests__/daemon-status.test.ts +++ b/lib/__tests__/daemon-status.test.ts @@ -1,9 +1,35 @@ import { describe, expect, test } from "bun:test"; import { classifyDaemonStatus } from "../daemon-status.ts"; +import type { BootFailure, LastExit, SupervisionState } from "../daemon/supervision-state.ts"; + +// `alive` (Task 1's field name for "a plain ping succeeded") is renamed +// `pingOk` here: Task 10 adds a second, distinct liveness signal (`pidAlive`, +// a raw OS-level pid check independent of rt.sock), and keeping both named +// `alive` would make call sites ambiguous about which one they mean. +function emptySupervision(): SupervisionState { + return { bootAttempts: 0, lastReadyAt: 0, recentFailures: [], lastExit: null }; +} + +function oneFailure(phase: BootFailure["phase"], reason: string): BootFailure { + return { at: Date.now(), phase, reason }; +} + +function threeRecentFailures(): BootFailure[] { + const now = Date.now(); + return [ + { at: now - 3000, phase: "api", reason: "EADDRINUSE" }, + { at: now - 2000, phase: "api", reason: "EADDRINUSE" }, + { at: now - 1000, phase: "api", reason: "EADDRINUSE" }, + ]; +} + +function bootFailedExit(reason: string): LastExit { + return { at: Date.now(), kind: "boot-failed", code: 1, reason }; +} describe("classifyDaemonStatus", () => { test("not installed short-circuits everything else", () => { - const v = classifyDaemonStatus({ installed: false, response: null, alive: false, pid: null }); + const v = classifyDaemonStatus({ installed: false, response: null, pingOk: false, pid: null }); expect(v.state).toBe("not-installed"); }); @@ -11,7 +37,7 @@ describe("classifyDaemonStatus", () => { const v = classifyDaemonStatus({ installed: true, response: { ok: true, data: { pid: 42, uptime: 1000 } }, - alive: true, + pingOk: true, pid: 42, }); expect(v.state).toBe("running"); @@ -24,7 +50,7 @@ describe("classifyDaemonStatus", () => { const v = classifyDaemonStatus({ installed: true, response: { ok: false, error: "freshness store unreadable" }, - alive: false, // never consulted: the answer itself is proof of life + pingOk: false, // never consulted: the answer itself is proof of life pid: 89290, }); expect(v.state).toBe("degraded"); @@ -37,7 +63,7 @@ describe("classifyDaemonStatus", () => { // The other half of the bug: daemonQuery returns null on a 2s timeout even // when it has already established the socket is live (daemon-client.ts:149). test("a null response with a live ping means running but unresponsive", () => { - const v = classifyDaemonStatus({ installed: true, response: null, alive: true, pid: 89290 }); + const v = classifyDaemonStatus({ installed: true, response: null, pingOk: true, pid: 89290 }); expect(v.state).toBe("degraded"); if (v.state === "degraded") { expect(v.reason).toBe("unresponsive"); @@ -46,13 +72,13 @@ describe("classifyDaemonStatus", () => { }); test("a null response and a dead ping is genuinely not running", () => { - const v = classifyDaemonStatus({ installed: true, response: null, alive: false, pid: 123 }); + const v = classifyDaemonStatus({ installed: true, response: null, pingOk: false, pid: 123 }); expect(v.state).toBe("not-running"); if (v.state === "not-running") expect(v.pid).toBe(123); }); test("not running without a recorded pid is still not running", () => { - const v = classifyDaemonStatus({ installed: true, response: null, alive: false, pid: null }); + const v = classifyDaemonStatus({ installed: true, response: null, pingOk: false, pid: null }); expect(v.state).toBe("not-running"); if (v.state === "not-running") expect(v.pid).toBeNull(); }); @@ -61,11 +87,104 @@ describe("classifyDaemonStatus", () => { const v = classifyDaemonStatus({ installed: true, response: { ok: false, error: "boom" }, - alive: false, + pingOk: false, pid: null, }); expect(v.state).toBe("degraded"); }); + + // ── Task 10: alive-not-serving / parked / crash-looping / boot-failed ── + + test("alive pid + failed ping -> alive-not-serving with breadcrumb detail", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: true, + pid: 42, + breadcrumb: { phase: "socket" }, + supervision: emptySupervision(), + }); + expect(v).toMatchObject({ state: "alive-not-serving", pid: 42, detail: "booting" }); + }); + + test("alive pid stuck after reaching ready -> wedged", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: true, + pid: 42, + breadcrumb: { phase: "ready" }, + supervision: emptySupervision(), + }); + expect(v).toMatchObject({ state: "alive-not-serving", pid: 42, detail: "wedged" }); + }); + + test("alive pid at ready with a boot-failed exit on record -> quarantined", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: true, + pid: 42, + breadcrumb: { phase: "ready" }, + supervision: { ...emptySupervision(), lastExit: bootFailedExit("events.db corrupt") }, + }); + expect(v).toMatchObject({ state: "alive-not-serving", pid: 42, detail: "quarantined" }); + }); + + test("alive pid whose breadcrumb flavor disagrees with the intended flavor -> parked", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: true, + pid: 42, + intendedFlavor: "prod", + breadcrumb: { phase: "start", flavor: "dev" }, + supervision: emptySupervision(), + }); + expect(v).toMatchObject({ state: "parked", pid: 42 }); + }); + + test("a breadcrumb with no supervision (pre-state.db failure) still classifies from the file alone", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: true, + pid: 42, + breadcrumb: { phase: "events-db" }, + }); + expect(v).toMatchObject({ state: "alive-not-serving", pid: 42, detail: "booting" }); + }); + + test("no pid + >=3 recent failures -> crash-looping", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: false, + pid: null, + supervision: { ...emptySupervision(), recentFailures: threeRecentFailures(), lastExit: bootFailedExit("EADDRINUSE") }, + }); + expect(v).toMatchObject({ state: "crash-looping" }); + if (v.state === "crash-looping") { + expect(v.failures).toBeGreaterThanOrEqual(3); + expect(v.reason).toBe("EADDRINUSE"); + } + }); + + test("no pid + single boot-failed -> boot-failed with reason and phase", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: false, + pid: null, + supervision: { ...emptySupervision(), recentFailures: [oneFailure("api", "EADDRINUSE")], lastExit: bootFailedExit("EADDRINUSE") }, + }); + expect(v).toMatchObject({ state: "boot-failed", reason: "EADDRINUSE", phase: "api" }); + }); + + test("no pid + no supervision at all -> plain not-running (additive: old callers unaffected)", () => { + const v = classifyDaemonStatus({ installed: true, pingOk: false, pidAlive: false, pid: null }); + expect(v.state).toBe("not-running"); + }); }); describe("needsLivenessProbe", () => { diff --git a/lib/command-tree-def.ts b/lib/command-tree-def.ts index 6ed9c3c6..59779ff4 100644 --- a/lib/command-tree-def.ts +++ b/lib/command-tree-def.ts @@ -640,7 +640,9 @@ export const TREE: Record = { description: "Show daemon status", module: "./commands/daemon.ts", fn: "showStatus", - args: [], + args: [ + { name: "JSON", flag: "--json", type: "boolean", default: false, hint: "Emit the verdict as JSON instead of the formatted lines" }, + ], }, track: { description: "Per-repo background tracking (live/poll/off)", diff --git a/lib/daemon-status.ts b/lib/daemon-status.ts index c550985c..f9face8b 100644 --- a/lib/daemon-status.ts +++ b/lib/daemon-status.ts @@ -7,29 +7,98 @@ * attempting a restart). A caller that reads "no usable response" as "the * daemon is down" then tells the user to start a daemon that is already * running. This maps the raw outcome onto what is actually known. + * + * Below that, a second tier answers a harder question: rt.sock isn't + * answering at all, but is the daemon actually down, or alive-and-stuck? + * `pidAlive`/`breadcrumb`/`supervision` (Task 9's supervision-state.ts) are + * the only signals that can tell, and per Ruling P1 (2026-08-28 p0-supervision + * ledger) the breadcrumb FILE is the sole record a pre-state.db boot failure + * leaves — `supervision` (the kv tier) can be absent even when `breadcrumb` + * is present, and classification must still resolve to something useful from + * the breadcrumb alone. */ import type { DaemonResponse } from "./daemon-client.ts"; +import { isCrashLooping, type BootPhase, type SupervisionState } from "./daemon/supervision-state.ts"; export type DaemonStatusVerdict = | { state: "not-installed" } | { state: "running"; data: any } /** Up — proven by an answer or a ping — but `status` itself did not deliver. */ | { state: "degraded"; reason: "error" | "unresponsive"; detail?: string; pid: number | null } + /** Ping fails, a live pid exists, and it's parked waiting for a different + * flavor to hold rt.sock (park.ts) — a flavor standoff, not a stuck boot. */ + | { state: "parked"; pid: number; holderFlavor?: string } + /** Ping fails but the pid is alive: still mid-boot, stuck after reaching + * ready, or alive-but-quarantined (recovered from a corrupt db). */ + | { state: "alive-not-serving"; pid: number; detail: "booting" | "wedged" | "quarantined" } + /** No live pid, and the kv failure record shows >= N failures within the window. */ + | { state: "crash-looping"; failures: number; reason: string } + /** No live pid, and the most recent recorded exit was a boot throw (fewer than N failures). */ + | { state: "boot-failed"; reason: string; phase: string } | { state: "not-running"; pid: number | null }; +/** The boot breadcrumb (`daemon-boot.json`), as classifyDaemonStatus needs it. Not + * imported from supervision-state.ts — that module's `Breadcrumb` interface is + * intentionally unexported, and this shape only needs to be structurally + * compatible with it. */ +export interface DaemonBreadcrumbInput { + phase: BootPhase; + flavor?: "dev" | "prod"; + pid?: number; + at?: number; +} + export interface DaemonStatusInputs { installed: boolean; - /** The `status` reply, or null if the transport gave up. */ - response: DaemonResponse | null; - /** Result of a `ping` probe. Only meaningful when `response` is null. */ - alive: boolean; + /** The `status` reply, or null/absent if the transport gave up. */ + response?: DaemonResponse | null; + /** Result of a `ping` probe. Only meaningful when `response` is absent. */ + pingOk?: boolean; /** Last recorded pid, for the operator to act on. */ pid: number | null; + /** Raw OS-level liveness of `pid` (process.kill(pid,0), or a pgrep-found + * stand-in) — independent of rt.sock. Only worth gathering once `pingOk` + * has already come back false; see `classifyDaemonStatus.needsPidProbe`. */ + pidAlive?: boolean; + /** This machine's currently-intended flavor (`resolveIntendedMode().mode`). + * A live pid whose own breadcrumb flavor disagrees with this is parked + * (park.ts), not stuck — the same signal `parkUntilIntended` itself acts on. */ + intendedFlavor?: "dev" | "prod"; + /** The socket holder's flavor, when the caller managed to learn it (best + * effort — probing rt.sock again after a failed ping/status round rarely + * succeeds, since a parked pid never binds it). Display-only. */ + holderFlavor?: string | null; + breadcrumb?: DaemonBreadcrumbInput | null; + /** Task 9's kv tier. Can be absent even when `breadcrumb` is present — a + * pre-state.db failure leaves only the breadcrumb file (Ruling P1). */ + supervision?: SupervisionState; + /** Injected for deterministic crash-loop window checks under test; defaults to Date.now(). */ + now?: number; +} + +const PHASE_ORDER: BootPhase[] = ["start", "events-db", "state-db", "api", "socket", "ready"]; + +function classifyAliveNotServingDetail( + breadcrumb: DaemonBreadcrumbInput | null | undefined, + supervision: SupervisionState | undefined, +): "booting" | "wedged" | "quarantined" { + const phase = breadcrumb?.phase; + if (!phase || PHASE_ORDER.indexOf(phase) < PHASE_ORDER.indexOf("ready")) return "booting"; + // Reached ready this run, but a prior attempt is on record as boot-failed — + // most likely a corrupt-db quarantine (lib/state/db.ts, events-bus.ts) it + // recovered from and is now stuck behind for an unrelated reason. + if (supervision?.lastExit?.kind === "boot-failed") return "quarantined"; + return "wedged"; +} + +function countRecentFailures(supervision: SupervisionState, now: number, windowMs = 5 * 60_000): number { + const floor = now - windowMs; + return supervision.recentFailures.filter((f) => f.at > floor).length; } export function classifyDaemonStatus(opts: DaemonStatusInputs): DaemonStatusVerdict { - const { installed, response, alive, pid } = opts; + const { installed, response, pingOk, pid, pidAlive, intendedFlavor, holderFlavor, breadcrumb, supervision } = opts; if (!installed) return { state: "not-installed" }; @@ -42,9 +111,33 @@ export function classifyDaemonStatus(opts: DaemonStatusInputs): DaemonStatusVerd return { state: "degraded", reason: "error", detail: response.error, pid }; } - // No reply. Ping is the only ground truth left: a daemon busy enough to blow - // the status timeout still answers a trivial ping. - if (alive) return { state: "degraded", reason: "unresponsive", pid }; + // No reply. A plain ping is the next ground truth: a daemon busy enough to + // blow the status timeout still answers a trivial ping. + if (pingOk) return { state: "degraded", reason: "unresponsive", pid }; + + // Ping failed too. From here, only pidAlive/breadcrumb/supervision (new + // signals) can say more than "not running" — absent them, fall straight + // through to the pre-existing not-running verdict. + if (pidAlive && pid !== null) { + if (breadcrumb?.flavor && intendedFlavor && breadcrumb.flavor !== intendedFlavor) { + return { state: "parked", pid, ...(holderFlavor ? { holderFlavor } : {}) }; + } + return { state: "alive-not-serving", pid, detail: classifyAliveNotServingDetail(breadcrumb, supervision) }; + } + + if (supervision) { + const now = opts.now ?? Date.now(); + if (isCrashLooping(supervision, now)) { + const reason = supervision.lastExit?.kind === "boot-failed" + ? supervision.lastExit.reason + : (supervision.recentFailures.at(-1)?.reason ?? "unknown"); + return { state: "crash-looping", failures: countRecentFailures(supervision, now), reason }; + } + if (supervision.lastExit?.kind === "boot-failed") { + const phase = supervision.recentFailures.at(-1)?.phase ?? "unknown"; + return { state: "boot-failed", reason: supervision.lastExit.reason, phase }; + } + } return { state: "not-running", pid }; } @@ -55,3 +148,10 @@ export function classifyDaemonStatus(opts: DaemonStatusInputs): DaemonStatusVerd */ classifyDaemonStatus.needsLivenessProbe = (response: DaemonResponse | null): boolean => response === null; + +/** + * Whether the pid/breadcrumb/supervision probes are worth paying for: only + * once both `status` and a plain `ping` have failed to answer. + */ +classifyDaemonStatus.needsPidProbe = (response: DaemonResponse | null, pingOk: boolean): boolean => + response === null && !pingOk; diff --git a/lib/daemon/__tests__/status-identity.test.ts b/lib/daemon/__tests__/status-identity.test.ts index c292aec2..b92d66b4 100644 --- a/lib/daemon/__tests__/status-identity.test.ts +++ b/lib/daemon/__tests__/status-identity.test.ts @@ -18,6 +18,20 @@ describe("daemon identity", () => { expect(res).toMatchObject({ ok: true, flavor: "dev", version: "source", sourceRev: "abc1234" }); }); + test("ping carries a supervision summary (Task 10)", async () => { + const h = createStatusHandlers(fakeCtx()); + const res = (await h["ping"]!({}, undefined as any)) as any; + // Loose on values deliberately: daemon-supervision kv is process-wide + // (lib/daemon/supervision-state.ts, `getStateDb("daemon")`), so this test + // sharing a `bun test` process with supervision-state.test.ts can see + // whatever that suite last wrote. The shape/cap is what this test owns. + expect(typeof res.supervision.bootAttempts).toBe("number"); + expect(typeof res.supervision.lastReadyAt).toBe("number"); + expect(Array.isArray(res.supervision.recentFailures)).toBe(true); + expect(res.supervision.recentFailures.length).toBeLessThanOrEqual(3); + expect(res.supervision.lastExit === null || typeof res.supervision.lastExit === "object").toBe(true); + }); + test("status.data carries the identity object", async () => { const h = createStatusHandlers(fakeCtx()); const res = (await h["status"]!({}, undefined as any)) as any; diff --git a/lib/daemon/handlers/status.ts b/lib/daemon/handlers/status.ts index 96e0c9b5..c30e1271 100644 --- a/lib/daemon/handlers/status.ts +++ b/lib/daemon/handlers/status.ts @@ -17,11 +17,22 @@ import type { PortEntry } from "../../port-scanner.ts"; import { listWorktrees } from "../../git-worktrees.ts"; import { drainNotifications, peekNotifications } from "../../notifier.ts"; import { getFreshnessSnapshot } from "../freshness.ts"; +import { readSupervisionState } from "../supervision-state.ts"; export function createStatusHandlers(ctx: HandlerContext): HandlerMap { return { "ping": async () => { - return { ok: true, uptime: Date.now() - ctx.startedAt, pid: process.pid, ...ctx.identity }; + // Read here (not once at ctx build time) — a status/status.ts request + // must see this run's own boot-attempt/failure counters, not whatever + // they were when the daemon started. + const { bootAttempts, lastReadyAt, recentFailures, lastExit } = readSupervisionState(); + return { + ok: true, + uptime: Date.now() - ctx.startedAt, + pid: process.pid, + ...ctx.identity, + supervision: { bootAttempts, lastReadyAt, recentFailures: recentFailures.slice(-3), lastExit }, + }; }, "status": async () => { From 063dd29a06e6cd8f4e28f1f26ff3567ad5ff783f Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:51:39 -0500 Subject: [PATCH 076/106] runs: scope summary-cache key by runsRoot; poller: test the backoff recovery path Coordinator review follow-up on task 11: the mtime cache key could theoretically collide across different RT_RUNS_ROOT values sharing a process, and the backoff's consecutiveFailures reset on a successful probe had no test proving it re-enables per-tick probing. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/agent-status-poller.test.ts | 25 +++++++++++++++++++ lib/runs/store.ts | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/daemon/__tests__/agent-status-poller.test.ts b/lib/daemon/__tests__/agent-status-poller.test.ts index a1507a8e..7f9813b1 100644 --- a/lib/daemon/__tests__/agent-status-poller.test.ts +++ b/lib/daemon/__tests__/agent-status-poller.test.ts @@ -90,3 +90,28 @@ test("backs off the herdr probe after repeated failures", async () => { // one probe every BACKOFF_TICKS (ticks 9 and 15 of 20) keeps checking. expect(probeCalls).toBe(5); }); + +test("a successful probe after backoff resets consecutiveFailures and resumes per-tick probing", async () => { + let probeCalls = 0; + handle = startAgentStatusPoller({ + emitEvent: () => {}, + log: quietLog, + intervalMs: 3_600_000, + probe: async () => { + probeCalls++; + // Invocations 1-3 cross FAILURE_THRESHOLD and engage backoff; the + // backoff window then skips ticks 4-8 without invoking probe at all, + // so invocation 4 is the tick-9 retry ... make it succeed. + return probeCalls <= 3 ? null : []; + }, + list: () => [], + }); + for (let i = 0; i < 9; i++) await handle.tick(); + expect(probeCalls).toBe(4); // probed at ticks 1, 2, 3, then again at tick 9 + const callsBeforeRecovery = probeCalls; + await handle.tick(); // tick 10, immediately after the successful tick-9 probe + // A successful probe resets consecutiveFailures to 0, so the very next + // tick is not gated by backoff ... it probes right away instead of + // waiting out another BACKOFF_TICKS window. + expect(probeCalls).toBe(callsBeforeRecovery + 1); +}); diff --git a/lib/runs/store.ts b/lib/runs/store.ts index 6d15e46d..b9719ea6 100644 --- a/lib/runs/store.ts +++ b/lib/runs/store.ts @@ -128,7 +128,7 @@ export function listRuns(repo?: string, liveness?: RunLiveness): RunSummary[] { const dbPath = join(runsRoot(), r, id, "state.db"); let mtimeMs: number; try { mtimeMs = statSync(dbPath).mtimeMs; } catch { continue; } - const key = `${r}/${id}`; + const key = `${runsRoot()}/${r}/${id}`; const hit = summaryCache.get(key); if (hit && hit.mtimeMs === mtimeMs) { out.push(hit.summary); continue; } From 78803f5b3f414fd8ab1a73f2467a5bf8528bdb92 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:56:30 -0500 Subject: [PATCH 077/106] pollers: gate the 10s/30s scans on recent consumer demand (S058, S093) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/daemon/__tests__/demand-tracker.test.ts | 24 +++++++++++++++++++ lib/daemon/command-router.ts | 6 ++++- lib/daemon/demand-tracker.ts | 26 +++++++++++++++++++++ lib/daemon/pollers.ts | 5 ++++ 4 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 lib/daemon/__tests__/demand-tracker.test.ts create mode 100644 lib/daemon/demand-tracker.ts diff --git a/lib/daemon/__tests__/demand-tracker.test.ts b/lib/daemon/__tests__/demand-tracker.test.ts new file mode 100644 index 00000000..82c4ebc3 --- /dev/null +++ b/lib/daemon/__tests__/demand-tracker.test.ts @@ -0,0 +1,24 @@ +import { test, expect } from "bun:test"; +import { recordDemand, demandedWithin, wrapWithDemand } from "../demand-tracker.ts"; + +test("demandedWithin reflects a recent recordDemand", () => { + recordDemand(); + expect(demandedWithin(60_000)).toBe(true); + expect(demandedWithin(0)).toBe(false); // window of 0ms is never "recent" +}); + +test("wrapWithDemand records demand and delegates to the inner handler", async () => { + let called = false; + // Typed variadic, matching how buildRoutedHandlers' entries are actually invoked + // (some take a payload, some don't) — a fixed 0-arg signature would reject that call shape. + const handlers: Record Promise<{ ok: boolean }>> = { + "system-processes": async () => { called = true; return { ok: true }; }, + other: async () => ({ ok: true }), + }; + wrapWithDemand(handlers, ["system-processes"]); + const before = demandedWithin(50); + await handlers["system-processes"]!(undefined as any); + expect(called).toBe(true); + expect(demandedWithin(1000)).toBe(true); + void before; +}); diff --git a/lib/daemon/command-router.ts b/lib/daemon/command-router.ts index 568c7ec4..f3d55703 100644 --- a/lib/daemon/command-router.ts +++ b/lib/daemon/command-router.ts @@ -28,6 +28,7 @@ import { createSettingsHandlers } from "./handlers/settings.ts"; import { createHomeHandlers } from "./handlers/home.ts"; import { createReposHandlers } from "./handlers/repos.ts"; import { reconcileFreshness, getFreshnessSnapshot } from "./freshness.ts"; +import { wrapWithDemand } from "./demand-tracker.ts"; import type { SystemProcessScanner } from "./system-process-scanner.ts"; import type { EventsBus } from "./events-bus.ts"; import type { HomeSnapshotHandle } from "./home-snapshot.ts"; @@ -74,7 +75,7 @@ export function buildRoutedHandlers(opts: { // Same seam as chatHandlers above: createAgentHandlers exposes `db` for // test isolation only. const { db: _agentDb, ...agentHandlers } = createAgentHandlers({ db: opts.stateDb, emitEvent, log: ctx.log }); - return { + const handlers: TypedHandlers & HandlerMap = { ...createCacheHandlers(ctx), ...createHooksHandlers(ctx), ...createStatusHandlers(ctx), @@ -102,4 +103,7 @@ export function buildRoutedHandlers(opts: { return { ok: true, data: getFreshnessSnapshot() }; }, }; + // A tray/CLI/console read of any scan-backed command means "someone is + // watching", which un-gates the background scans (see pollers.ts, S058/S093). + return wrapWithDemand(handlers, ["ports", "system-processes", "tray:status"]); } diff --git a/lib/daemon/demand-tracker.ts b/lib/daemon/demand-tracker.ts new file mode 100644 index 00000000..29b6e617 --- /dev/null +++ b/lib/daemon/demand-tracker.ts @@ -0,0 +1,26 @@ +/** + * "A consumer is watching" signal for the background scans. The tray/CLI/console + * calling ports/system-processes/tray:status stamps demand here (via the + * command-router wrapper); pollers skip the 10s/30s scans when nothing has asked + * recently, so an idle machine stops paying the lsof/git tax (S058, S093). + */ +let lastDemandAt = 0; + +export function recordDemand(): void { + lastDemandAt = Date.now(); +} + +/** True when a consumer read a scan-backed command within `ms`. */ +export function demandedWithin(ms: number): boolean { + return lastDemandAt !== 0 && Date.now() - lastDemandAt < ms; +} + +/** Wrap the named handler entries so each call stamps demand, then delegates. */ +export function wrapWithDemand>(handlers: T, cmds: string[]): T { + for (const cmd of cmds) { + const inner = handlers[cmd]; + if (typeof inner !== "function") continue; + (handlers as any)[cmd] = (...args: any[]) => { recordDemand(); return inner(...args); }; + } + return handlers; +} diff --git a/lib/daemon/pollers.ts b/lib/daemon/pollers.ts index 91c82c53..c8a38475 100644 --- a/lib/daemon/pollers.ts +++ b/lib/daemon/pollers.ts @@ -12,11 +12,14 @@ import { checkRunawayProcesses } from "../notifier.ts"; import { primeTeamTrackingIdentityMap } from "../repo-tracking.ts"; import type { SystemProcessScanner } from "./system-process-scanner.ts"; import type { PortCacheRef, RepoIndex } from "./handlers/types.ts"; +import { demandedWithin } from "./demand-tracker.ts"; const MR_REFRESH_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes const PORT_SCAN_INTERVAL_MS = 30 * 1000; // 30 seconds const HOOKS_SCAN_INTERVAL_MS = 60 * 1000; // 60 seconds (fallback for stale watchers) const SYSTEM_PROCESS_SCAN_INTERVAL_MS = 10 * 1000; // 10 seconds +/** Consider a consumer "present" for 5 min after its last scan-backed read. */ +const DEMAND_WINDOW_MS = 5 * 60 * 1000; export interface PollerDeps { log: Logger; @@ -39,6 +42,7 @@ export function startPollers(deps: PollerDeps): void { async function refreshPortCache(): Promise { if (portScanInFlight) return; + if (!demandedWithin(DEMAND_WINDOW_MS)) return; // no consumer asked recently portScanInFlight = true; try { portCacheRef.ports = await scanListeningPorts(); @@ -55,6 +59,7 @@ export function startPollers(deps: PollerDeps): void { async function refreshSystemProcesses(): Promise { if (processScanInFlight) return; + if (!demandedWithin(DEMAND_WINDOW_MS)) return; processScanInFlight = true; try { const processes = await systemProcessScanner.scan(portCacheRef.ports); From 983020b6d09dfa9cd4c2afb80dfc9dd596d458f0 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 12:13:57 -0500 Subject: [PATCH 078/106] daemon status: exclude the calling process from probePidAlive's lsof fallback readSupervisionState() opens a bun:sqlite handle on state.db (inside RT_DIR) right before probePidAlive runs, so lsof +D RT_DIR was reporting the calling CLI process itself as a live holder of the directory -- a dead daemon with no rt.pid and no live breadcrumb pid could self-match and misclassify as alive-not-serving/parked. Filter process.pid out of the lsof result and add a regression test that opens state.db and asserts the fallback returns false with no daemon-related pids present. --- commands/__tests__/probe-pid-alive.test.ts | 19 +++++++++++++++++++ commands/daemon.ts | 16 ++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 commands/__tests__/probe-pid-alive.test.ts diff --git a/commands/__tests__/probe-pid-alive.test.ts b/commands/__tests__/probe-pid-alive.test.ts new file mode 100644 index 00000000..269318be --- /dev/null +++ b/commands/__tests__/probe-pid-alive.test.ts @@ -0,0 +1,19 @@ +import { describe, test, expect } from "bun:test"; +import { probePidAlive } from "../daemon.ts"; +import { readSupervisionState } from "../../lib/daemon/supervision-state.ts"; + +describe("probePidAlive", () => { + // Regression: the lsof fallback must exclude the CALLING process itself. + // showStatus opens a bun:sqlite handle on state.db (inside RT_DIR) via + // readSupervisionState() immediately before this probe runs — `lsof +D + // RT_DIR` then legitimately reports the calling CLI process as a live + // holder of the directory, with no daemon involved at all. Without the + // process.pid filter this self-matches and a genuinely dead daemon + // (no recorded pid, no breadcrumb pid) misclassifies as alive. + test("a state.db handle held by THIS process does not self-match as a live daemon", async () => { + readSupervisionState(); // opens (and keeps open) the isolated HOME's state.db + const result = await probePidAlive(null, undefined); + expect(result.alive).toBe(false); + expect(result.pid).toBeNull(); + }); +}); diff --git a/commands/daemon.ts b/commands/daemon.ts index 4a71a43a..7cfadef1 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -334,8 +334,19 @@ export async function restart(): Promise { * any process hold a file open under THIS HOME's rt dir" — home-scoped by * construction, immune to that false positive and to pid-reuse. Only worth * calling once both `status` and a plain ping have already failed. + * + * The CALLER itself is excluded from the lsof result: `showStatus` opens its + * own `bun:sqlite` handle on `state.db` (inside RT_DIR) via + * `readSupervisionState()` just before this probe runs, so `lsof +D RT_DIR` + * legitimately reports the calling CLI process as a live holder of the + * directory — with no daemon involved at all. Left unfiltered, a genuinely + * dead daemon self-matches and misclassifies as alive-not-serving/parked; + * this only failed to show up in manual testing because incidental work + * happened to separate the state.db open from the lsof call by enough time + * for state.db's own transient lock window to close — an accident of + * timing, not a guarantee. */ -async function probePidAlive(recordedPid: number | null, breadcrumbPid?: number): Promise<{ alive: boolean; pid: number | null }> { +export async function probePidAlive(recordedPid: number | null, breadcrumbPid?: number): Promise<{ alive: boolean; pid: number | null }> { for (const candidate of [recordedPid, breadcrumbPid ?? null]) { if (candidate === null) continue; try { @@ -344,7 +355,8 @@ async function probePidAlive(recordedPid: number | null, breadcrumbPid?: number) } catch { /* not this one — try the next candidate */ } } const { stdout } = await runCapture(["lsof", "-t", "+D", RT_DIR], { timeoutMs: 3000 }); - const pids = stdout.trim().split(/\s+/).filter(Boolean).map(Number).filter((n) => !isNaN(n)); + const pids = stdout.trim().split(/\s+/).filter(Boolean).map(Number) + .filter((n) => !isNaN(n) && n !== process.pid); return pids.length > 0 ? { alive: true, pid: pids[0]! } : { alive: false, pid: recordedPid }; } From fce330fef9733782ec4e32d739606b4ba5b60363 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 12:14:06 -0500 Subject: [PATCH 079/106] demand-tracker: document command-only demand-stamp contract; subprocess: unref SIGKILL timer Notes the reviewer-flagged gap that WS/SSE topic subscriptions don't stamp demand, only wrapped command handlers do. Unrefs the belt-and-suspenders SIGKILL timer in runCapture so a timed-out call can't hold a short-lived rt-client CLI process open for up to 2s; the daemon still delivers the SIGKILL since it stays alive. Applied identically to lib/subprocess.ts and its rt-client mirror, and rebuilt rt-client's dist/ to match. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/daemon/demand-tracker.ts | 8 ++++++++ lib/subprocess.ts | 4 ++++ packages/rt-client/src/settings/exec.ts | 4 ++++ 3 files changed, 16 insertions(+) diff --git a/lib/daemon/demand-tracker.ts b/lib/daemon/demand-tracker.ts index 29b6e617..1db1dd93 100644 --- a/lib/daemon/demand-tracker.ts +++ b/lib/daemon/demand-tracker.ts @@ -6,6 +6,14 @@ */ let lastDemandAt = 0; +/** + * Only the wrapped command handlers (ports / system-processes / tray:status, + * via wrapWithDemand) stamp demand. WS relay and SSE topic subscriptions do + * not: a push-only consumer that subscribes to a broadcast topic but never + * calls a command gets no demand credit and can starve past the demand + * window. A subscribe-side stamp would live in api-server.ts, out of scope + * this phase. + */ export function recordDemand(): void { lastDemandAt = Date.now(); } diff --git a/lib/subprocess.ts b/lib/subprocess.ts index c6a286f2..addc7eab 100644 --- a/lib/subprocess.ts +++ b/lib/subprocess.ts @@ -70,6 +70,10 @@ export async function runCapture( killTimer = setTimeout(() => { try { proc.kill("SIGKILL"); } catch { /* already exited */ } }, 2000); + // unref: a short-lived CLI process must not be held open by a timed-out + // call waiting on this timer; the daemon stays alive regardless, so its + // SIGKILL still fires. + killTimer.unref?.(); }, timeoutMs); const captured: Promise = (async () => { diff --git a/packages/rt-client/src/settings/exec.ts b/packages/rt-client/src/settings/exec.ts index fae833ec..5ecb3cb8 100644 --- a/packages/rt-client/src/settings/exec.ts +++ b/packages/rt-client/src/settings/exec.ts @@ -60,6 +60,10 @@ export async function runCapture( killTimer = setTimeout(() => { try { proc.kill("SIGKILL"); } catch { /* already exited */ } }, 2000); + // unref: a short-lived CLI process must not be held open by a timed-out + // call waiting on this timer; the daemon stays alive regardless, so its + // SIGKILL still fires. + killTimer.unref?.(); }, timeoutMs); const captured: Promise = (async () => { From f2dfbf0ddb5b447daeabaa8897b080a527867dfe Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 12:26:24 -0500 Subject: [PATCH 080/106] logs: rotate daemon-stderr.log on open; hide stale crash block by mtime daemon-stderr.log grew unbounded and rt daemon logs always showed its stale contents as "most recent crash". redirectNativeStderr now renames a non-empty file to a dated daemon-stderr..log (deduped with a .N suffix) before reopening, matching log-janitor's LOG_FILE_PATTERN so pruneLogs sweeps it for free. showLogs now gates the native-stderr block on mtime vs. the running daemon's startedAt (from ping), via a new pure nativeStderrDisplay helper, and stamps the mtime in the header. --- commands/__tests__/daemon-logs-render.test.ts | 38 ++++++++ commands/daemon.ts | 41 +++++++-- lib/__tests__/daemon-logger.test.ts | 88 ++++++++++++++++++- lib/daemon-logger.ts | 31 ++++++- 4 files changed, 190 insertions(+), 8 deletions(-) create mode 100644 commands/__tests__/daemon-logs-render.test.ts diff --git a/commands/__tests__/daemon-logs-render.test.ts b/commands/__tests__/daemon-logs-render.test.ts new file mode 100644 index 00000000..877e510b --- /dev/null +++ b/commands/__tests__/daemon-logs-render.test.ts @@ -0,0 +1,38 @@ +/** + * nativeStderrDisplay — showLogs' stale-crash mtime gate. + * + * daemon-stderr.log is rotated on daemon boot (lib/daemon-logger.ts), but a + * leftover file can still predate the *currently running* daemon (e.g. it was + * never rotated because the daemon has been up for days). This pins the + * show/hide + header decision without needing a live daemon or a real file. + */ + +import { describe, test, expect } from "bun:test"; +import { nativeStderrDisplay } from "../daemon.ts"; + +const NOW = 1_785_000_000_000; + +describe("nativeStderrDisplay", () => { + test("hides the block when the file predates the daemon's startedAt", () => { + const { show, header } = nativeStderrDisplay(NOW - 10_000, NOW); + expect(show).toBe(false); + expect(header).toBe("no crash since this daemon started"); + }); + + test("hides the block when the file mtime exactly equals startedAt", () => { + const { show } = nativeStderrDisplay(NOW, NOW); + expect(show).toBe(false); + }); + + test("shows the block, with the mtime in the header, when the file postdates startedAt", () => { + const mtimeMs = NOW + 5_000; + const { show, header } = nativeStderrDisplay(mtimeMs, NOW); + expect(show).toBe(true); + expect(header).toBe(`native stderr (captured ${new Date(mtimeMs).toISOString()})`); + }); + + test("fails open (shows) when the daemon's startedAt is unknown — nothing to compare against", () => { + const { show } = nativeStderrDisplay(NOW - 999_999, null); + expect(show).toBe(true); + }); +}); diff --git a/commands/daemon.ts b/commands/daemon.ts index 7cfadef1..2604e4f1 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -819,6 +819,25 @@ export async function manageTracking(args: string[] = []): Promise { // ─── Logs ──────────────────────────────────────────────────────────────────── +/** + * Decides whether showLogs' native-stderr block is worth printing, and its + * header. `daemon-stderr.log` is rotated on open (daemon-logger.ts) but the + * fresh file can still be non-empty from a crash that happened before *this* + * boot's rotation ran (e.g. a bun panic mid-startup) — so staleness is judged + * by mtime vs. the live daemon's startedAt, not by rotation alone. A `null` + * startedAt (daemon unreachable — nothing to compare against) fails open: + * show it, since a down daemon is exactly when the last crash matters most. + */ +export function nativeStderrDisplay( + mtimeMs: number, + daemonStartedAt: number | null, +): { show: boolean; header: string } { + if (daemonStartedAt !== null && mtimeMs <= daemonStartedAt) { + return { show: false, header: "no crash since this daemon started" }; + } + return { show: true, header: `native stderr (captured ${new Date(mtimeMs).toISOString()})` }; +} + /** * Show daemon logs. * @@ -836,16 +855,28 @@ export async function showLogs(args: string[] = []): Promise { // Surface captured native stderr first — these are bun panics/asserts that // bypassed the JS-side interceptor and were caught by the swift-shim's - // freopen of fd 2. If non-empty, the most recent crash leads the output. + // freopen of fd 2. Only shown when it postdates the running daemon's boot — + // otherwise it's a previous life's crash, not "the most recent crash". const stderrPath = join(LOG_DIR, "daemon-stderr.log"); if (existsSync(stderrPath)) { const content = readFileSync(stderrPath, "utf8").trim(); if (content) { - console.log(`\n ${red}${bold}native stderr${reset} ${dim}(${stderrPath})${reset}`); - for (const line of content.split("\n").slice(-20)) { - console.log(` ${red}${line}${reset}`); + const mtimeMs = statSync(stderrPath).mtimeMs; + const ping = await daemonQuery("ping"); + const daemonStartedAt = + ping && (ping as any).ok && typeof (ping as any).startedAt === "number" + ? ((ping as any).startedAt as number) + : null; + const { show, header } = nativeStderrDisplay(mtimeMs, daemonStartedAt); + if (show) { + console.log(`\n ${red}${bold}${header}${reset} ${dim}(${stderrPath})${reset}`); + for (const line of content.split("\n").slice(-20)) { + console.log(` ${red}${line}${reset}`); + } + console.log(""); + } else { + console.log(`\n ${dim}${header}${reset}\n`); } - console.log(""); } } diff --git a/lib/__tests__/daemon-logger.test.ts b/lib/__tests__/daemon-logger.test.ts index 69bd3a9e..c573f6aa 100644 --- a/lib/__tests__/daemon-logger.test.ts +++ b/lib/__tests__/daemon-logger.test.ts @@ -1,7 +1,8 @@ import { describe, it, expect, beforeEach, afterEach, mock } from "bun:test"; -import { mkdtempSync, rmSync, readFileSync, readdirSync, existsSync } from "node:fs"; +import { mkdtempSync, rmSync, readFileSync, readdirSync, existsSync, writeFileSync, statSync, unlinkSync, mkdirSync, closeSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { dlopen, suffix, FFIType } from "bun:ffi"; import type { Logger } from "pino"; // We import the factory (not the singleton) so each test gets isolation. @@ -10,6 +11,7 @@ import { lazyChildLogger, getDaemonLogger, installCrashHandlers, + redirectNativeStderr, __test__, type DaemonLoggerHandle, } from "../daemon-logger.ts"; @@ -134,6 +136,90 @@ describe("lazyChildLogger", () => { }); }); +describe("redirectNativeStderr — rotation", () => { + // redirectNativeStderr dup2's the REAL process fd 2 to the log file (that is + // the whole point of the function) — a bare call here would swallow this + // test process's own stderr for the rest of the run. Save/restore fd 2 + // around the call with the same dup/dup2 pair the implementation uses. + function withRealFd2Saved(fn: () => void): void { + const libc = dlopen(`libSystem.${suffix}`, { + dup: { args: [FFIType.i32], returns: FFIType.i32 }, + dup2: { args: [FFIType.i32, FFIType.i32], returns: FFIType.i32 }, + }); + const savedFd2 = libc.symbols.dup(2); + try { + fn(); + } finally { + libc.symbols.dup2(savedFd2, 2); + closeSync(savedFd2); + libc.close(); + } + } + + function stderrPath(): string { + return join(logsDir(), "daemon-stderr.log"); + } + + function clearRotatedFiles(): void { + const dir = logsDir(); + if (!existsSync(dir)) return; + for (const f of readdirSync(dir)) { + if (/^daemon-stderr\.\d{4}-\d{2}-\d{2}(\.\d+)?\.log$/.test(f)) unlinkSync(join(dir, f)); + } + } + + beforeEach(() => { + mkdirSync(logsDir(), { recursive: true }); + clearRotatedFiles(); + try { unlinkSync(stderrPath()); } catch { /* not present yet */ } + }); + + afterEach(() => { + clearRotatedFiles(); + try { unlinkSync(stderrPath()); } catch { /* already gone */ } + }); + + it("rotates a non-empty daemon-stderr.log before reopening, matching the janitor's dated pattern", () => { + if (process.platform !== "darwin") return; // redirectNativeStderr is a darwin-only no-op elsewhere + + writeFileSync(stderrPath(), "old panic\n"); + + withRealFd2Saved(() => redirectNativeStderr()); + + const rotated = readdirSync(logsDir()).filter((f) => /^daemon-stderr\.\d{4}-\d{2}-\d{2}\.log$/.test(f)); + expect(rotated.length).toBe(1); + expect(readFileSync(join(logsDir(), rotated[0]!), "utf8")).toBe("old panic\n"); + expect(statSync(stderrPath()).size).toBe(0); + }); + + it("dedupes with a .N suffix when the dated rotation name is already taken", () => { + if (process.platform !== "darwin") return; + + const d = new Date(); + const date = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`; + writeFileSync(join(logsDir(), `daemon-stderr.${date}.log`), "yesterday's rotation\n"); + writeFileSync(stderrPath(), "today's panic\n"); + + withRealFd2Saved(() => redirectNativeStderr()); + + const rotated = readdirSync(logsDir()).filter((f) => new RegExp(`^daemon-stderr\\.${date}(\\.\\d+)?\\.log$`).test(f)); + expect(rotated.length).toBe(2); + const suffixed = rotated.find((f) => f !== `daemon-stderr.${date}.log`); + expect(suffixed).toBeDefined(); + expect(readFileSync(join(logsDir(), suffixed!), "utf8")).toBe("today's panic\n"); + }); + + it("leaves an empty or missing daemon-stderr.log alone (nothing to rotate)", () => { + if (process.platform !== "darwin") return; + + withRealFd2Saved(() => redirectNativeStderr()); + + const rotated = readdirSync(logsDir()).filter((f) => /^daemon-stderr\.\d{4}-\d{2}-\d{2}(\.\d+)?\.log$/.test(f)); + expect(rotated.length).toBe(0); + expect(statSync(stderrPath()).size).toBe(0); + }); +}); + describe("getDaemonLogger — concurrency", () => { it("shares one in-flight initialization across concurrent callers", async () => { __test__.resetDaemonLoggerCache(); diff --git a/lib/daemon-logger.ts b/lib/daemon-logger.ts index bd4826e1..5c7bc0df 100644 --- a/lib/daemon-logger.ts +++ b/lib/daemon-logger.ts @@ -18,7 +18,7 @@ import pino, { type Logger } from "pino"; // @ts-ignore — no types shipped; the JS API is well-tested. import roll from "pino-roll"; import { dlopen, suffix, FFIType } from "bun:ffi"; -import { mkdirSync, openSync, closeSync } from "fs"; +import { mkdirSync, openSync, closeSync, existsSync, statSync, renameSync } from "fs"; import { join } from "path"; import { logsDir } from "./rt-paths.ts"; @@ -192,6 +192,26 @@ export const __test__ = { // ─── Native stderr capture ─────────────────────────────────────────────────── +/** Local `yyyy-MM-dd`, matching the janitor's dated-file convention (lib/cli-logger.ts's `today()`). */ +function todayDate(): string { + const d = new Date(); + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`; +} + +/** + * Picks the rotation target for `daemon-stderr.log`: `daemon-stderr..log`, + * or `..log` if that name is already taken (e.g. two boots same day) — both + * shapes match log-janitor's LOG_FILE_PATTERN, so pruneLogs sweeps them for free. + */ +function nextRotatedStderrPath(dir: string, date: string): string { + const base = join(dir, `daemon-stderr.${date}.log`); + if (!existsSync(base)) return base; + for (let n = 1; ; n++) { + const candidate = join(dir, `daemon-stderr.${date}.${n}.log`); + if (!existsSync(candidate)) return candidate; + } +} + /** * Point fd 2 at ~/.mattstack/rt/logs/daemon-stderr.log so native output that bypasses * JS entirely (bun panics, segfault reports, runtime asserts) is captured no @@ -206,7 +226,14 @@ export function redirectNativeStderr(): void { try { const dir = logsDir(); mkdirSync(dir, { recursive: true }); - const fd = openSync(join(dir, "daemon-stderr.log"), "a"); + const stderrPath = join(dir, "daemon-stderr.log"); + // Rotate any leftover content from a previous crash before reopening — + // otherwise `rt daemon logs` keeps showing yesterday's panic as "most + // recent". A rename here can never lose data (unlike truncation). + if (existsSync(stderrPath) && statSync(stderrPath).size > 0) { + renameSync(stderrPath, nextRotatedStderrPath(dir, todayDate())); + } + const fd = openSync(stderrPath, "a"); const libc = dlopen(`libSystem.${suffix}`, { dup2: { args: [FFIType.i32, FFIType.i32], returns: FFIType.i32 }, }); From 3366ff86c0afc40a44d037dce7c640f18fcb9d97 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 12:31:57 -0500 Subject: [PATCH 081/106] daemon: bare-signal exit is non-zero (launchd respawns); shutdown verb stays exit 0 launchd's KeepAlive.SuccessfulExit:false only respawns on a non-zero exit. Reserve exit 0 for the intentional shutdown verb; a bare SIGTERM/SIGINT/SIGHUP (external kill, memory pressure) now exits 1 so launchd relaunches. The sanctioned stop path (SMAppService.unregister) never goes through this signal path, so this does not affect intended stops. Refactors installSignalHandlers' inline gracefulExit into a testable makeGracefulExit(deps) that injects exit/recordCleanExit/wasVerbShutdown. --- lib/daemon.ts | 13 +++++++- lib/daemon/__tests__/shutdown.test.ts | 36 ++++++++++++++++++++++ lib/daemon/shutdown.ts | 43 ++++++++++++++++++++++----- 3 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 lib/daemon/__tests__/shutdown.test.ts diff --git a/lib/daemon.ts b/lib/daemon.ts index e2a570fc..d1e643e0 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -368,6 +368,11 @@ const freshnessEnv: FreshnessEnv = { ctx: handlerCtx, broadcast: emit }; // branch-cache facade above). let routedHandlers: ReturnType | undefined; +// Set by the `shutdown` verb before it exits, so a bare OS signal arriving +// mid-teardown is still distinguishable from the intentional stop (exit-code +// policy: docs/daemon-supervision-design.md). +let shuttingDownViaVerb = false; + async function handleCommand(cmd: string, payload: any, signal?: AbortSignal): Promise { const t0 = Date.now(); try { @@ -395,6 +400,7 @@ async function routeCommand(cmd: string, payload: any, signal?: AbortSignal): Pr // force-closes all in-flight connections, including the one that // carried the shutdown request. setTimeout(() => { + shuttingDownViaVerb = true; recordCleanExit("shutdown", 0); cleanup(); loggerHandle.flush?.(); @@ -556,7 +562,12 @@ async function runDaemon(): Promise { // typed-event → notification fan-out (no-op while no controller answers). // Graceful shutdown on all termination signals - installSignalHandlers({ cleanup, flushLogs: () => loggerHandle.flush?.(), log }); + installSignalHandlers({ + cleanup, + flushLogs: () => loggerHandle.flush?.(), + log, + wasVerbShutdown: () => shuttingDownViaVerb, + }); bootPhase = "ready"; recordDaemonReady(); diff --git a/lib/daemon/__tests__/shutdown.test.ts b/lib/daemon/__tests__/shutdown.test.ts new file mode 100644 index 00000000..cf3644a0 --- /dev/null +++ b/lib/daemon/__tests__/shutdown.test.ts @@ -0,0 +1,36 @@ +import { expect, test } from "bun:test"; +import type { Logger } from "pino"; +import { makeGracefulExit } from "../shutdown.ts"; + +const silentLog = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as unknown as Logger; + +test("gracefulExit exits 0 after the shutdown verb, 1 on a bare signal", () => { + const exits: number[] = []; + const exit = (c?: number) => { exits.push(c ?? 0); }; + let viaVerb = false; + const handlers = makeGracefulExit({ cleanup: () => {}, flushLogs: () => {}, log: silentLog, + wasVerbShutdown: () => viaVerb, exit, recordCleanExit: () => {} }); + handlers("SIGTERM"); + expect(exits).toEqual([1]); + viaVerb = true; + handlers("SIGTERM"); + expect(exits).toEqual([1, 0]); +}); + +test("gracefulExit records the exit kind and code via recordCleanExit", () => { + const recorded: Array<{ kind: string; code: number }> = []; + let viaVerb = false; + const handlers = makeGracefulExit({ + cleanup: () => {}, + flushLogs: () => {}, + log: silentLog, + wasVerbShutdown: () => viaVerb, + exit: () => {}, + recordCleanExit: (kind, code) => { recorded.push({ kind, code }); }, + }); + handlers("SIGINT"); + expect(recorded).toEqual([{ kind: "signal", code: 1 }]); + viaVerb = true; + handlers("SIGHUP"); + expect(recorded).toEqual([{ kind: "signal", code: 1 }, { kind: "shutdown", code: 0 }]); +}); diff --git a/lib/daemon/shutdown.ts b/lib/daemon/shutdown.ts index 4b427433..71ff5773 100644 --- a/lib/daemon/shutdown.ts +++ b/lib/daemon/shutdown.ts @@ -11,6 +11,7 @@ import { clearWsClients } from "./api-server.ts"; import { disposeFreshness } from "./freshness.ts"; import { stopDiscussionsPoller } from "./discussions-poller.ts"; import type { HooksGuard } from "./hooks-guard.ts"; +import { recordCleanExit } from "./supervision-state.ts"; export interface ShutdownDeps { /** Mutable holder — daemon.ts assigns the servers after boot. */ @@ -49,22 +50,50 @@ export function createCleanup(deps: ShutdownDeps): () => void { }; } +export interface GracefulExitDeps { + cleanup: () => void; + flushLogs: () => void; + log: Logger; + /** True once the `shutdown` verb has claimed this exit as intentional. */ + wasVerbShutdown: () => boolean; + exit: (code?: number) => void; + recordCleanExit: (kind: "shutdown" | "signal", code: number) => void; +} + /** * Graceful shutdown on all termination signals. SIGHUP is sent when the * parent process exits (e.g. launchd session ends, or a tray-spawned daemon's - * parent tray is killed) — treat it as a clean stop. + * parent tray is killed). + * + * launchd's KeepAlive.SuccessfulExit=false only respawns on a non-zero exit, + * so the code here must distinguish the intentional `shutdown` verb (exit 0, + * stay down) from a bare external signal — pkill, memory pressure, a stray + * script (exit 1, launchd respawns). The sanctioned stop path + * (SMAppService.unregister) doesn't go through this signal path at all, so + * exiting non-zero on a bare signal never fights an intended stop. */ +export function makeGracefulExit(deps: GracefulExitDeps): (signal: NodeJS.Signals) => void { + return (signal: NodeJS.Signals) => { + deps.log.info({ signal }, "received signal; shutting down"); + deps.cleanup(); + deps.flushLogs(); + if (deps.wasVerbShutdown()) { + deps.recordCleanExit("shutdown", 0); + deps.exit(0); + } else { + deps.recordCleanExit("signal", 1); + deps.exit(1); + } + }; +} + export function installSignalHandlers(opts: { cleanup: () => void; flushLogs: () => void; log: Logger; + wasVerbShutdown: () => boolean; }): void { - const gracefulExit = (signal: NodeJS.Signals) => { - opts.log.info({ signal }, "received signal; shutting down"); - opts.cleanup(); - opts.flushLogs(); - process.exit(0); - }; + const gracefulExit = makeGracefulExit({ ...opts, exit: process.exit, recordCleanExit }); process.on("SIGTERM", () => gracefulExit("SIGTERM")); process.on("SIGINT", () => gracefulExit("SIGINT")); process.on("SIGHUP", () => gracefulExit("SIGHUP")); From 33a8359e633cb2db6104193263f38ac7ca04dbaa Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 12:37:08 -0500 Subject: [PATCH 082/106] daemon: ownership-aware socket/pid unlink; eviction waits for pid death then SIGKILL --- lib/daemon.ts | 2 +- lib/daemon/__tests__/boot-reconcile.test.ts | 28 ++++++++++++++ lib/daemon/__tests__/shutdown.test.ts | 26 ++++++++++++- lib/daemon/boot-reconcile.ts | 42 +++++++++++++++++---- lib/daemon/shutdown.ts | 19 +++++++--- 5 files changed, 102 insertions(+), 15 deletions(-) create mode 100644 lib/daemon/__tests__/boot-reconcile.test.ts diff --git a/lib/daemon.ts b/lib/daemon.ts index d1e643e0..4e3ddfc3 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -437,7 +437,7 @@ async function runDaemon(): Promise { // If a previous daemon process is still alive (orphan from a failed // restart), evict it before we bind the socket. - evictStaleDaemon(log); + await evictStaleDaemon(log); // Auto-unlink any tagged tool link whose tool now has a genuine user copy // elsewhere on PATH (e.g. the user ran `brew install gh` after rt linked diff --git a/lib/daemon/__tests__/boot-reconcile.test.ts b/lib/daemon/__tests__/boot-reconcile.test.ts new file mode 100644 index 00000000..129a2d85 --- /dev/null +++ b/lib/daemon/__tests__/boot-reconcile.test.ts @@ -0,0 +1,28 @@ +import { mkdirSync, writeFileSync } from "fs"; +import { expect, test } from "bun:test"; +import type { Logger } from "pino"; +import { DAEMON_PID_PATH, RT_DIR } from "../../daemon-config.ts"; +import { evictStaleDaemon } from "../boot-reconcile.ts"; + +mkdirSync(RT_DIR, { recursive: true }); + +const silentLog = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as unknown as Logger; + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +test("evictStaleDaemon waits for the old pid to die, escalating to SIGKILL", async () => { + const child = Bun.spawn({ cmd: ["bash", "-c", "trap '' TERM; sleep 30"] }); + writeFileSync(DAEMON_PID_PATH, String(child.pid)); + const start = Date.now(); + await evictStaleDaemon(silentLog); + expect(isAlive(child.pid)).toBe(false); + expect(Date.now() - start).toBeLessThan(5000); + child.kill(); +}); diff --git a/lib/daemon/__tests__/shutdown.test.ts b/lib/daemon/__tests__/shutdown.test.ts index cf3644a0..139cd69f 100644 --- a/lib/daemon/__tests__/shutdown.test.ts +++ b/lib/daemon/__tests__/shutdown.test.ts @@ -1,9 +1,17 @@ +import { existsSync, writeFileSync } from "fs"; import { expect, test } from "bun:test"; import type { Logger } from "pino"; -import { makeGracefulExit } from "../shutdown.ts"; +import { DAEMON_PID_PATH, DAEMON_SOCK_PATH } from "../../daemon-config.ts"; +import { createCleanup, makeGracefulExit } from "../shutdown.ts"; const silentLog = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as unknown as Logger; +const cleanupDeps = { + servers: {}, + hooksGuard: { closeAll: () => {} } as any, + log: silentLog, +}; + test("gracefulExit exits 0 after the shutdown verb, 1 on a bare signal", () => { const exits: number[] = []; const exit = (c?: number) => { exits.push(c ?? 0); }; @@ -34,3 +42,19 @@ test("gracefulExit records the exit kind and code via recordCleanExit", () => { handlers("SIGHUP"); expect(recorded).toEqual([{ kind: "signal", code: 1 }, { kind: "shutdown", code: 0 }]); }); + +test("cleanup does not unlink rt.pid/rt.sock when the pid file belongs to another process", () => { + writeFileSync(DAEMON_PID_PATH, "999999"); + writeFileSync(DAEMON_SOCK_PATH, ""); + createCleanup({ ...cleanupDeps, pid: process.pid })(); + expect(existsSync(DAEMON_PID_PATH)).toBe(true); + expect(existsSync(DAEMON_SOCK_PATH)).toBe(true); +}); + +test("cleanup unlinks when the pid file is ours", () => { + writeFileSync(DAEMON_PID_PATH, String(process.pid)); + writeFileSync(DAEMON_SOCK_PATH, ""); + createCleanup({ ...cleanupDeps, pid: process.pid })(); + expect(existsSync(DAEMON_PID_PATH)).toBe(false); + expect(existsSync(DAEMON_SOCK_PATH)).toBe(false); +}); diff --git a/lib/daemon/boot-reconcile.ts b/lib/daemon/boot-reconcile.ts index 231fdec2..bcb106dc 100644 --- a/lib/daemon/boot-reconcile.ts +++ b/lib/daemon/boot-reconcile.ts @@ -8,19 +8,47 @@ import type { Logger } from "pino"; import { readDaemonPid } from "../daemon-config.ts"; +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); // signal 0 = existence check, throws if not alive + return true; + } catch { + return false; + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** Poll every ~100ms until `pid` is gone or `maxMs` elapses. */ +async function waitForDeath(pid: number, maxMs: number): Promise { + const deadline = Date.now() + maxMs; + while (isAlive(pid)) { + if (Date.now() >= deadline) return false; + await sleep(100); + } + return true; +} + /** * Evict a still-alive previous daemon. This is the last line of defence when * the `start` command's orphan-detection doesn't fire (e.g. launchd relaunches * us automatically without going through `rt daemon start`). + * + * Waits for the old process to actually die rather than a blind sleep — a + * daemon that survives the eviction window can still race the new one for + * rt.sock/rt.pid (S044). Escalates to SIGKILL if SIGTERM alone doesn't land. */ -export function evictStaleDaemon(log: Logger): void { +export async function evictStaleDaemon(log: Logger): Promise { const previousPid = readDaemonPid(); if (!previousPid || previousPid === process.pid) return; + if (!isAlive(previousPid)) return; + process.kill(previousPid, "SIGTERM"); + log.warn({ pid: previousPid }, "evicted stale daemon process"); + if (await waitForDeath(previousPid, 2500)) return; try { - process.kill(previousPid, 0); // throws if not alive - process.kill(previousPid, "SIGTERM"); - log.warn({ pid: previousPid }, "evicted stale daemon process"); - // Brief pause so the old process can exit and release any shared resources - Bun.sleepSync(300); - } catch { /* process not found — nothing to evict */ } + process.kill(previousPid, "SIGKILL"); + } catch { /* already gone */ } + await waitForDeath(previousPid, 500); } diff --git a/lib/daemon/shutdown.ts b/lib/daemon/shutdown.ts index 71ff5773..7f5993a0 100644 --- a/lib/daemon/shutdown.ts +++ b/lib/daemon/shutdown.ts @@ -3,7 +3,7 @@ * runtime files in an order that beats launchd's 5s ExitTimeOut. */ -import { existsSync, unlinkSync } from "fs"; +import { existsSync, readFileSync, unlinkSync } from "fs"; import type { Server } from "bun"; import type { Logger } from "pino"; import { DAEMON_SOCK_PATH, DAEMON_PID_PATH } from "../daemon-config.ts"; @@ -18,10 +18,12 @@ export interface ShutdownDeps { servers: { socket?: Server; api?: Server }; hooksGuard: HooksGuard; log: Logger; + /** This process's pid. Injected so cleanup's ownership check is testable. */ + pid?: number; } export function createCleanup(deps: ShutdownDeps): () => void { - const { servers, hooksGuard, log } = deps; + const { servers, hooksGuard, log, pid = process.pid } = deps; return function cleanup(): void { // Stop accepting new traffic first, and force-close all in-flight @@ -41,10 +43,15 @@ export function createCleanup(deps: ShutdownDeps): () => void { // through at every mutation site (lib/state/branch-cache.ts), so there // is nothing dirty in memory to race launchd's 5s ExitTimeOut. - // Remove runtime files - for (const path of [DAEMON_SOCK_PATH, DAEMON_PID_PATH]) { - try { if (existsSync(path)) unlinkSync(path); } catch { /* */ } - } + // Remove runtime files, but only if rt.pid still names THIS process. + // A shutting-down old daemon that unlinks unconditionally can delete a + // new daemon's rt.sock/rt.pid out from under it (S012/S044). + try { + if (existsSync(DAEMON_PID_PATH) && readFileSync(DAEMON_PID_PATH, "utf8").trim() === String(pid)) { + unlinkSync(DAEMON_PID_PATH); + if (existsSync(DAEMON_SOCK_PATH)) unlinkSync(DAEMON_SOCK_PATH); + } + } catch (err) { log.warn({ err }, "cleanup unlink skipped"); } log.info("daemon stopped"); }; From e84f6673f486f763c6ec507b1c1907d35390ff48 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 13:03:50 -0500 Subject: [PATCH 083/106] daemon CLI: uninstall guards on liveness; start escalates to kickstart; attemptRestart re-probes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit uninstall() no longer deletes rt.sock/rt.pid/daemon.json when a failed or absent tray /daemon/stop leaves the daemon actually still alive (isDaemonProcessRunning() or probeSocketHolder() says so) — it now prints the real remedy (launchctl bootout) and leaves the files in place instead of orphaning a live daemon. start() now escalates to the /daemon/restart (kickstart) route when the tray acks /daemon/start but the socket never comes up through the existing poll, since SMAppService can register a job that never actually launches. attemptRestart() in lib/daemon-client.ts re-probes isDaemonRunning() after the tray ack instead of trusting the POST response alone, so daemonQuery's retry logic stops treating a merely-accepted request as a real restart. Audited commands/settings.ts's dev-mode toggle for the same gap: it never calls cleanupDaemonFiles()/markDaemonUninstalled() at all (it goes through the tray's /flavor/retire route instead), so there's nothing to fix there. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/daemon-uninstall-start.test.ts | 145 ++++++++++++++++++ commands/daemon.ts | 36 ++++- lib/daemon-client.ts | 13 +- 3 files changed, 190 insertions(+), 4 deletions(-) create mode 100644 commands/__tests__/daemon-uninstall-start.test.ts diff --git a/commands/__tests__/daemon-uninstall-start.test.ts b/commands/__tests__/daemon-uninstall-start.test.ts new file mode 100644 index 00000000..1a3f58fd --- /dev/null +++ b/commands/__tests__/daemon-uninstall-start.test.ts @@ -0,0 +1,145 @@ +/** + * `rt daemon uninstall`/`start` — the CLI-side liveness guards (Task 14, + * S027/S030/S028-CLI). Fakes the tray over a real Bun.serve on + * TRAY_SOCK_PATH (same rig as commands/__tests__/settings-dev-mode.test.ts) + * and, where a scenario needs "the daemon is live", a real Bun.serve on + * DAEMON_SOCK_PATH answering /ping — isDaemonProcessRunning's pid check and + * probeSocketHolder/isDaemonRunning's socket ping are both exercised for + * real, never mocked module internals. + */ +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs"; +import { start, uninstall } from "../daemon.ts"; +import { + DAEMON_CONFIG_PATH, + DAEMON_PID_PATH, + DAEMON_SOCK_PATH, + RT_DIR, + TRAY_SOCK_PATH, + markDaemonInstalled, +} from "../../lib/daemon-config.ts"; +import { resolveIntendedMode } from "../../lib/dev-mode.ts"; + +let servers: ReturnType[] = []; +let logs: string[] = []; +const realLog = console.log; + +function captureLogs(): void { + logs = []; + console.log = (...args: unknown[]) => { logs.push(args.map(String).join(" ")); }; +} + +function serveTray(handlers: Record Response>): void { + servers.push(Bun.serve({ + unix: TRAY_SOCK_PATH, + fetch(req) { + const url = new URL(req.url); + const handler = handlers[url.pathname]; + return handler ? handler() : new Response("not found", { status: 404 }); + }, + })); +} + +/** A real listener on rt.sock that answers /ping — what both isDaemonRunning() + * (daemon-client.ts) and probeSocketHolder() (lib/daemon/park.ts) fetch. + * Flavor defaults to the CURRENT intended mode (not a hardcoded "prod") so + * start()'s post-liveness warnIfWrongFlavor() check never fires a spurious + * mismatch when this file runs after another test flips mattstack.mode in + * the shared isolated HOME `bun test` uses for the whole process. */ +function serveDaemonPing(body?: Record): void { + const resolvedBody = body ?? { ok: true, pid: 4242, flavor: resolveIntendedMode().mode }; + servers.push(Bun.serve({ + unix: DAEMON_SOCK_PATH, + fetch(req) { + const url = new URL(req.url); + if (url.pathname === "/ping") return Response.json(resolvedBody); + return new Response("not found", { status: 404 }); + }, + })); +} + +afterEach(() => { + console.log = realLog; + for (const s of servers) { try { s.stop(true); } catch { /* already stopped */ } } + servers = []; + for (const p of [DAEMON_SOCK_PATH, DAEMON_PID_PATH, TRAY_SOCK_PATH, DAEMON_CONFIG_PATH]) { + try { rmSync(p); } catch { /* absent */ } + } +}); + +describe("uninstall — liveness guard", () => { + test("leaves rt.pid/daemon.json when isDaemonProcessRunning() says the daemon is alive", async () => { + mkdirSync(RT_DIR, { recursive: true }); + markDaemonInstalled(); + writeFileSync(DAEMON_PID_PATH, String(process.pid)); // this test process is genuinely alive + // tray unreachable: trayQuery('/daemon/stop') resolves null (no server on TRAY_SOCK_PATH) + + captureLogs(); + await uninstall(); + + expect(existsSync(DAEMON_PID_PATH)).toBe(true); // cleanupDaemonFiles did NOT run + expect(JSON.parse(readFileSync(DAEMON_CONFIG_PATH, "utf8")).installed).toBe(true); // markDaemonUninstalled did NOT run + expect(logs.join("\n")).toContain("launchctl bootout"); + }); + + test("leaves rt.sock/daemon.json when probeSocketHolder() finds a live holder (no rt.pid at all)", async () => { + mkdirSync(RT_DIR, { recursive: true }); + markDaemonInstalled(); + serveDaemonPing(); + + captureLogs(); + await uninstall(); + + expect(JSON.parse(readFileSync(DAEMON_CONFIG_PATH, "utf8")).installed).toBe(true); + expect(logs.join("\n")).toContain("launchctl bootout"); + }); + + test("cleans up rt.sock/rt.pid/daemon.json when nothing is alive", async () => { + mkdirSync(RT_DIR, { recursive: true }); + markDaemonInstalled(); + writeFileSync(DAEMON_PID_PATH, "999999"); // no such pid + writeFileSync(DAEMON_SOCK_PATH, ""); // stale file, not a real listener — probeSocketHolder's fetch fails + + captureLogs(); + await uninstall(); + + expect(existsSync(DAEMON_PID_PATH)).toBe(false); + expect(JSON.parse(readFileSync(DAEMON_CONFIG_PATH, "utf8")).installed).toBe(false); + expect(logs.join("\n")).not.toContain("launchctl bootout"); + expect(logs.join("\n")).toContain("daemon fully uninstalled"); + }); +}); + +describe("start — kickstart escalation", () => { + test("falls back to /daemon/restart when the tray acks /daemon/start but the socket never comes up", async () => { + mkdirSync(RT_DIR, { recursive: true }); + markDaemonInstalled(); + let restartCalled = false; + serveTray({ + "/daemon/start": () => Response.json({ ok: true }), + "/daemon/restart": () => { restartCalled = true; return Response.json({ ok: true }); }, + }); + + captureLogs(); + await start(); + + expect(restartCalled).toBe(true); + }, 20_000); + + test("escalation succeeds once /daemon/restart actually brings the socket up", async () => { + mkdirSync(RT_DIR, { recursive: true }); + markDaemonInstalled(); + serveTray({ + "/daemon/start": () => Response.json({ ok: true }), + "/daemon/restart": () => { + serveDaemonPing(); + return Response.json({ ok: true }); + }, + }); + + captureLogs(); + await start(); + + expect(logs.join("\n")).toContain("daemon started"); + }, 20_000); +}); diff --git a/commands/daemon.ts b/commands/daemon.ts index 2604e4f1..43da6bd2 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -25,6 +25,8 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, unlinkSync, import { bold, dim, green, yellow, red, reset } from "../lib/tui.ts"; import { isDaemonInstalled, + isDaemonProcessRunning, + activeLaunchdLabel, markDaemonInstalled, markDaemonUninstalled, cleanupDaemonFiles, readDaemonPid, RT_DIR, @@ -224,7 +226,20 @@ export async function uninstall(): Promise { console.log(` ${green}✓${reset} removed legacy launchd plist`); } - // 3. Clear install flag + sock/pid files. + // 3. A failed/absent tray stop must never delete rt.sock/rt.pid/daemon.json + // out from under a daemon that's actually still alive — that would orphan + // it (still running, launchd-supervised, but rt's own bookkeeping says + // uninstalled). Check both liveness signals: the recorded pid, and whether + // anything still answers on rt.sock (a daemon can be alive with no + // matching rt.pid, e.g. after a crash-and-respawn under launchd). + const stillAlive = isDaemonProcessRunning() || (await probeSocketHolder()) !== null; + if (stillAlive) { + console.log(`\n ${yellow}⚠${reset} daemon is still running — leaving rt.sock/rt.pid/daemon.json in place`); + console.log(` ${dim}Fix: ${bold}launchctl bootout gui/$UID/${activeLaunchdLabel()}${reset}\n`); + return; + } + + // 4. Clear install flag + sock/pid files. markDaemonUninstalled(); cleanupDaemonFiles(); console.log(` ${green}✓${reset} cleared install flag`); @@ -258,16 +273,31 @@ export async function start(): Promise { } console.log(` ${dim}starting ${intended.mode} daemon via tray…${reset}`); + if (await pollForDaemonUp(intended)) return; + + // The tray acked /daemon/start, but SMAppService can register a job that + // never actually launches (still booting, crash-looping, etc.) — kick it + // via /daemon/restart, which forces launchd to invoke it, rather than + // leaving the operator staring at "check logs" for something a retry fixes. + console.log(` ${dim}not up yet — escalating to restart (kickstart)…${reset}`); + const restartResult = await trayQuery("/daemon/restart", "POST"); + if (restartResult?.ok && (await pollForDaemonUp(intended))) return; + + console.log(`\n ${yellow}daemon starting… check logs: rt daemon logs${reset}\n`); +} + +/** Shared poll loop for start()'s initial wait and its kickstart escalation. */ +async function pollForDaemonUp(intended: IntendedMode): Promise { for (let i = 0; i < 12; i++) { await Bun.sleep(250); if (await isDaemonRunning()) { if (!(await warnIfWrongFlavor("start", intended))) { console.log(`\n ${green}✓ daemon started${reset}\n`); } - return; + return true; } } - console.log(`\n ${yellow}daemon starting… check logs: rt daemon logs${reset}\n`); + return false; } export async function stop(): Promise { diff --git a/lib/daemon-client.ts b/lib/daemon-client.ts index 2a03a629..c29220d1 100644 --- a/lib/daemon-client.ts +++ b/lib/daemon-client.ts @@ -178,7 +178,18 @@ async function attemptRestart(): Promise { // null response (tray socket absent / request failed) means no restart // actually happened. const res = await trayQuery("/daemon/start", "POST"); - return res !== null; + if (res === null) return false; + + // The tray ack only proves the request was received, not that the + // daemon actually came up — re-probe rt.sock before reporting success, + // so daemonQuery's caller isn't told "restarted" while the daemon is + // still mid-boot and then misdirected into warnDaemonDown() on the very + // next query instead of actually waiting for it. + for (let i = 0; i < 12; i++) { + await Bun.sleep(250); + if (await isDaemonRunning()) return true; + } + return false; } catch { return false; } From 919100e09d74ef5a60ce030cf6e3c0f6dd46c52f Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 13:45:26 -0500 Subject: [PATCH 084/106] daemon: redirect stderr after legacy-dir migration; set shuttingDownViaVerb at verb receipt redirectNativeStderr() ran before migrateLegacyRtDir(), so its mkdirSync of the new rt dir made every real ~/.rt migration report a false "conflict". Move it after the migration check, still ahead of every other module-scope side effect. shuttingDownViaVerb was only set inside the shutdown verb's 100ms delayed cleanup, so a bare SIGTERM arriving in that window read it as unset and exited 1, causing launchd to respawn a daemon that was told to stop. Set it at verb receipt instead. --- lib/daemon.ts | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/daemon.ts b/lib/daemon.ts index 4e3ddfc3..64ea5fe1 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -85,13 +85,6 @@ import type { PortEntry } from "./port-scanner.ts"; // entry (cli.ts) also runs it, but `bun run lib/daemon.ts` skips cli.ts. import { migrateLegacyRtDir, LEGACY_RT_LABEL, RT_DIR_LABEL, logsDir } from "./rt-paths.ts"; -// Capture native panics (bypass JS entirely) at the fd level before anything -// else in this module runs, so a throw during any later module-scope -// construction (createEventsBus, cron, home-snapshot, …) lands in -// daemon-stderr.log instead of vanishing down whatever fd 2 the launcher gave -// us. Depends only on logsDir() and mkdirs its own dir; no logger needed yet. -redirectNativeStderr(); - // Gates installCrashHandlers' unhandledRejection handler: fatal during boot // (no socket/API bound yet, nothing to recover), advisory-only once ready. let bootPhase: "booting" | "ready" = "booting"; @@ -108,6 +101,16 @@ function setPhase(phase: BootPhase): void { const rtMigration = migrateLegacyRtDir(); +// Capture native panics (bypass JS entirely) at the fd level, so a throw +// during any later module-scope construction (createEventsBus, cron, +// home-snapshot, …) lands in daemon-stderr.log instead of vanishing down +// whatever fd 2 the launcher gave us. Depends only on logsDir() and mkdirs +// its own dir — this MUST run after migrateLegacyRtDir(): mkdirSync(logsDir()) +// creates the new rt dir, and migrateLegacyRtDir() treats that dir merely +// existing as a "conflict" with a real legacy tree, so redirecting first +// would defeat the migration. +redirectNativeStderr(); + // ─── Logging ───────────────────────────────────────────────────────────────── // Pino-backed structured logger. See lib/daemon-logger.ts. Top-level await // initializes the singleton before any other startup code runs, so `log` is @@ -396,11 +399,15 @@ async function routeCommand(cmd: string, payload: any, signal?: AbortSignal): Pr switch (cmd) { case "shutdown": log.info("received shutdown command"); + // Set before the delay, not inside the setTimeout callback: a bare + // SIGTERM arriving in the 100ms window must see this flag already + // true, or the signal handler treats an intentional stop as a crash + // (exit 1, launchd respawns). + shuttingDownViaVerb = true; // Delay cleanup so this response can be written first — cleanup() // force-closes all in-flight connections, including the one that // carried the shutdown request. setTimeout(() => { - shuttingDownViaVerb = true; recordCleanExit("shutdown", 0); cleanup(); loggerHandle.flush?.(); From 4e0af1e542280b86dc7532b3981a645345d6d31f Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 13:45:29 -0500 Subject: [PATCH 085/106] daemon: drop unused clearBreadcrumb Production only ever writes or reads the boot breadcrumb (it gets overwritten with "ready" on successful boot); clearBreadcrumb had no callers outside its own test. Removed the export and the now-unused unlinkSync import; the test that only covered clearBreadcrumb is gone, and the remaining "no breadcrumb written" test does its own file cleanup instead of relying on the removed API. --- lib/daemon/__tests__/supervision-state.test.ts | 18 ++++++++++-------- lib/daemon/supervision-state.ts | 12 ++---------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/lib/daemon/__tests__/supervision-state.test.ts b/lib/daemon/__tests__/supervision-state.test.ts index 6461af34..dbb6ea50 100644 --- a/lib/daemon/__tests__/supervision-state.test.ts +++ b/lib/daemon/__tests__/supervision-state.test.ts @@ -1,4 +1,6 @@ import { describe, expect, test } from "bun:test"; +import { rmSync } from "fs"; +import { join } from "path"; import { recordBootAttempt, recordDaemonReady, @@ -8,8 +10,14 @@ import { isCrashLooping, writeBreadcrumb, readBreadcrumb, - clearBreadcrumb, } from "../supervision-state.ts"; +import { RT_DIR } from "../../daemon-config.ts"; + +/** Test-only cleanup mirroring the breadcrumb file's path (production has + * no clear API — the daemon only ever writes or reads it). */ +function removeBreadcrumbFile(): void { + rmSync(join(RT_DIR, "daemon-boot.json"), { force: true }); +} describe("supervision-state kv round-trip", () => { test("boot attempts, ready stamp, failures and last-exit round-trip through kv", () => { @@ -58,14 +66,8 @@ describe("breadcrumb file", () => { expect(typeof b?.at).toBe("number"); }); - test("clearBreadcrumb removes the file so readBreadcrumb returns null", () => { - writeBreadcrumb("ready"); - clearBreadcrumb(); - expect(readBreadcrumb()).toBeNull(); - }); - test("readBreadcrumb returns null when no breadcrumb has been written", () => { - clearBreadcrumb(); + removeBreadcrumbFile(); expect(readBreadcrumb()).toBeNull(); }); }); diff --git a/lib/daemon/supervision-state.ts b/lib/daemon/supervision-state.ts index 7fa12e7e..5a753115 100644 --- a/lib/daemon/supervision-state.ts +++ b/lib/daemon/supervision-state.ts @@ -4,7 +4,7 @@ * breadcrumb for a live-but-silent daemon. * * Two tiers, deliberately not one: - * - The breadcrumb FILE (`writeBreadcrumb`/`readBreadcrumb`/`clearBreadcrumb`) + * - The breadcrumb FILE (`writeBreadcrumb`/`readBreadcrumb`) * opens no database, so it is safe to call at module scope, before * state.db exists. It is the only tier a pre-db boot failure can reach. * - The kv tier (`recordBootAttempt`, `recordDaemonReady`, @@ -15,7 +15,7 @@ * write is try/catch'd and silently no-ops if the db isn't open yet. */ -import { existsSync, readFileSync, unlinkSync, writeFileSync } from "fs"; +import { existsSync, readFileSync, writeFileSync } from "fs"; import { join } from "path"; import { RT_DIR } from "../daemon-config.ts"; import { daemonFlavor } from "./park.ts"; @@ -140,11 +140,3 @@ export function readBreadcrumb(): Breadcrumb | null { return null; } } - -export function clearBreadcrumb(): void { - try { - if (existsSync(breadcrumbPath())) unlinkSync(breadcrumbPath()); - } catch { - // Best-effort. - } -} From 2fa055cf6dde238678a0a06222d6cc534c74ae51 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 13:45:35 -0500 Subject: [PATCH 086/106] rt-client: soften the rt.apiPort description to match reality resolveApiPort() has no callers yet -- api-server.ts still binds the API_PORT const -- so the setting currently does nothing when set. The old description promised it as an "escape hatch when 9401 is held", which is not true until the sibling wiring lands. Rebuilt dist/ (not committed, gitignored). --- packages/rt-client/src/settings/registry-defs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rt-client/src/settings/registry-defs.ts b/packages/rt-client/src/settings/registry-defs.ts index 9f37dfd4..7d2f3bde 100644 --- a/packages/rt-client/src/settings/registry-defs.ts +++ b/packages/rt-client/src/settings/registry-defs.ts @@ -203,7 +203,7 @@ export const REGISTRY: readonly SettingDef[] = [ default: 9401, merge: "replace", migrated: true, - description: "TCP port the daemon's local HTTP/WS API binds (escape hatch when 9401 is held).", + description: "TCP port for the daemon's local HTTP/WS API. Not yet consumed at bind time (pending api-server wiring); today only the RT_API_PORT env var overrides the default 9401.", }, { key: "rt.hooks", From 0badca0ff808a863b44eca3a58726dc0c5d20e4e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 13:45:40 -0500 Subject: [PATCH 087/106] docs: scrub em/en dashes, fix daemon-supervision-design phase-order drift Owner rule forbids em/en dashes in committed text. Rephrased them out of daemon-supervision-design.md and the runner-health redirect title; mechanical ellipsis substitution for the large historical plan doc, where careful rephrase wasn't warranted. Also fixed two places daemon-supervision-design.md had drifted from the implementation: the breadcrumb BootPhase order is actually start -> events-db -> state-db -> api -> socket -> ready (no "crash-handlers" phase, API binds before socket), and the alive-not-serving liveness fallback is `lsof +D RT_DIR` scoped to this HOME and filtered to exclude the caller's own pid, not a system-wide `pgrep -f`. --- docs/daemon-runner-health.md | 2 +- docs/daemon-supervision-design.md | 46 ++-- .../plans/2026-08-28-p0-supervision.md | 222 +++++++++--------- 3 files changed, 136 insertions(+), 134 deletions(-) diff --git a/docs/daemon-runner-health.md b/docs/daemon-runner-health.md index 9d6db779..f9ee31b1 100644 --- a/docs/daemon-runner-health.md +++ b/docs/daemon-runner-health.md @@ -1,4 +1,4 @@ -# Daemon runner health — superseded +# Daemon runner health (superseded) This document audited subsystems (process-manager, remedy-engine, runner.tsx, workspace-sync) that no longer exist. It is retained only as diff --git a/docs/daemon-supervision-design.md b/docs/daemon-supervision-design.md index f672cafb..61b4f6c1 100644 --- a/docs/daemon-supervision-design.md +++ b/docs/daemon-supervision-design.md @@ -1,7 +1,7 @@ # Daemon supervision: status verdicts and exit-code semantics Phase 0 design anchor for the rt daemon stability roadmap (audit -2026-08). Tasks 9–14 of the Phase 0 plan implement this. +2026-08). Tasks 9-14 of the Phase 0 plan implement this. ## launchd contract @@ -19,51 +19,53 @@ down". Every exit-code decision below follows from that single fact. | Crash-loop guard trips (N in M minutes) | 0 (park) | Stop the flapping; surface `crash-looping` so a human intervenes instead of launchd hammering every ~10s. | Mechanism: a module-scope `shuttingDownViaVerb` flag is set true by the -`shutdown` verb before it calls cleanup; `gracefulExit(signal)` reads it -— set → exit(0), unset (bare signal) → exit(1). +`shutdown` verb before it calls cleanup; `gracefulExit(signal)` reads it: +set → exit(0), unset (bare signal) → exit(1). Boot-phase gate: a module-scope `bootPhase: "booting" | "ready"` flips to `"ready"` immediately before the `daemon ready` log. The `unhandledRejection` handler exits(1) while `bootPhase === "booting"` -and only logs (recovers) once ready — so a boot-time stray rejection is +and only logs (recovers) once ready, so a boot-time stray rejection is fatal but a steady-state one is not. ## Status verdicts `rt daemon status` and `/api/status` classify by first match: -1. `not-installed` — SMAppService not registered. -2. `serving` — ping on rt.sock succeeds. -3. `parked` — ping fails, a live rt pid exists, and the boot breadcrumb +1. `not-installed`: SMAppService not registered. +2. `serving`: ping on rt.sock succeeds. +3. `parked`: ping fails, a live rt pid exists, and the boot breadcrumb phase is a flavor standoff (park). Named distinctly so the user is told "another flavor owns the socket", not "wedged". -4. `alive-not-serving` — ping fails but a live rt pid exists - (`process.kill(pid,0)` on rt.pid, or `pgrep -f 'rt --daemon|lib/daemon.ts'`). +4. `alive-not-serving`: ping fails but a live rt pid exists + (`process.kill(pid,0)` on rt.pid, falling back to `lsof +D RT_DIR` + scoped to this HOME's rt dir and filtered to exclude the calling + process's own pid). Sub-detail from the breadcrumb phase: `booting` / `wedged`, or `quarantined` when a state.db/events.db boot-failed marker is present. - Prints "process is running but not answering rt.sock — rt daemon logs -t". -5. `crash-looping` — no live pid AND the kv failure record shows ≥ N + Prints "process is running but not answering rt.sock ... rt daemon logs -t". +5. `crash-looping`: no live pid AND the kv failure record shows ≥ N failures within the last M minutes (N=3, M=5). Prints the last reason. -6. `boot-failed` — no live pid AND the most recent kv exit record is a +6. `boot-failed`: no live pid AND the most recent kv exit record is a boot throw (fewer than N failures). Prints the last reason + phase. -7. `installed-not-running` — registered, no live pid, clean/again-absent +7. `installed-not-running`: registered, no live pid, clean/again-absent exit record. ## Persisted state (kv, ns `daemon-supervision`, no schema change) -- `boot-attempts` (number) — incremented at the top of `runDaemon()`. -- `last-ready-at` (number, epoch ms) — stamped just before `daemon ready`. -- `recent-failures` (array of `{ at, phase, reason }`, capped to 10) — +- `boot-attempts` (number): incremented at the top of `runDaemon()`. +- `last-ready-at` (number, epoch ms): stamped just before `daemon ready`. +- `recent-failures` (array of `{ at, phase, reason }`, capped to 10): appended by the boot fatal path and by state.db/events.db boot-failed markers. Crash-loop = ≥ N entries newer than now − M minutes. -- `last-exit` (`{ at, kind: "shutdown" | "signal" | "boot-failed", code, reason? }`) - — written by the shutdown verb, the signal handlers, and the boot +- `last-exit` (`{ at, kind: "shutdown" | "signal" | "boot-failed", code, reason? }`): + written by the shutdown verb, the signal handlers, and the boot fatal path. Lets status distinguish "cleanly stopped" from "died". ## Boot breadcrumb `~/.mattstack/rt/daemon-boot.json` = `{ at, pid, flavor, phase }`, -rewritten at each boot phase: `start` → `crash-handlers` → `events-db` -→ `state-db` → `socket` → `api` → `ready`. Lets `alive-not-serving` -name where a live-but-silent daemon is stuck even when the logs are -unreadable. Removed (or stamped `ready`) on successful boot. +rewritten at each boot phase: `start` → `events-db` → `state-db` → +`api` → `socket` → `ready`. Lets `alive-not-serving` name where a +live-but-silent daemon is stuck even when the logs are unreadable. +Removed (or stamped `ready`) on successful boot. diff --git a/docs/superpowers/plans/2026-08-28-p0-supervision.md b/docs/superpowers/plans/2026-08-28-p0-supervision.md index bbbd14d5..72453dda 100644 --- a/docs/superpowers/plans/2026-08-28-p0-supervision.md +++ b/docs/superpowers/plans/2026-08-28-p0-supervision.md @@ -8,13 +8,13 @@ **Tech Stack:** Bun, `bun:sqlite`, pino, TypeScript. Tests are `bun test` (unit) and `bun test --preload ./e2e/setup.ts` (e2e, isolated-HOME daemon spawns). -**Spec:** `/Users/matt/Documents/GitHub/repo-tools/.claude/worktrees/daemon-stability-audit/docs/daemon-stability-audit-2026-08.md` — "Roadmap › Phase 0" plus Appendix A/B entries S001, S003, S004, S009, S011, S012, S026, S027, S028, S029, S030, S035, S036, S037, S043, S044, S060, S072, S073, S074, R001, R002, R007, R017. Each carries a failure scenario, prescribed fix, and fixer notes; read the relevant entry before implementing its task. +**Spec:** `/Users/matt/Documents/GitHub/repo-tools/.claude/worktrees/daemon-stability-audit/docs/daemon-stability-audit-2026-08.md` ... "Roadmap › Phase 0" plus Appendix A/B entries S001, S003, S004, S009, S011, S012, S026, S027, S028, S029, S030, S035, S036, S037, S043, S044, S060, S072, S073, S074, R001, R002, R007, R017. Each carries a failure scenario, prescribed fix, and fixer notes; read the relevant entry before implementing its task. ## Global Constraints -- **No `SCHEMA_VERSION` bump and no new/edited `V*_SCHEMA` block.** Persist all supervision state (restart counters, last-exit reason, boot-failed markers) in the existing `kv` table under namespace `daemon-supervision`, via `setKvValue`/`getKvValue` from `lib/state/kv-blob.ts`. If any task appears to need a schema change, STOP and ask the user (per the job brief's question format) — do not proceed. +- **No `SCHEMA_VERSION` bump and no new/edited `V*_SCHEMA` block.** Persist all supervision state (restart counters, last-exit reason, boot-failed markers) in the existing `kv` table under namespace `daemon-supervision`, via `setKvValue`/`getKvValue` from `lib/state/kv-blob.ts`. If any task appears to need a schema change, STOP and ask the user (per the job brief's question format) ... do not proceed. - **Never start a daemon or run `rt` against the real machine.** Every daemon or `dist/rt` invocation in a test or check runs under `env -i HOME=` only (repo CLAUDE.md, "Operating on this machine"). e2e daemon spawns already do this via `e2e/setup.ts`; new e2e tests must follow the same isolation. -- **Write fence — do NOT modify these sibling-owned files** (ask the user if a task seems to need one): `lib/daemon/api-server.ts`, `lib/daemon/api-auth.ts`, `lib/daemon/socket-server.ts`, `lib/daemon/handlers/secrets.ts`, `lib/subprocess.ts`, `lib/daemon/cache-refresh.ts`, `lib/git-worktrees.ts`, `lib/daemon/freshness.ts`, `lib/daemon/pollers.ts`, `lib/daemon/worktree-process-kill.ts`, `lib/daemon/system-process-scanner.ts`, `lib/runs/store.ts`, `lib/notifier.ts`, `lib/daemon/handlers/discussions.ts`, `lib/daemon/handlers/chat.ts`, `lib/daemon/handlers/agent.ts`, `lib/daemon/handlers/pane.ts`, `lib/daemon/handlers/project-mrs.ts`, `lib/daemon/handlers/worktree.ts`, `lib/herdr/client.ts`, `lib/port-scanner.ts`, `lib/deps/links.ts`, `lib/worktree/trash.ts`, `lib/agent-herdr.ts`, `lib/daemon/cron.ts`, `lib/daemon/hooks-guard.ts`, `lib/home/age-key.ts`, `lib/daemon/discussions-store.ts`, `lib/state/presence-store.ts`. +- **Write fence ... do NOT modify these sibling-owned files** (ask the user if a task seems to need one): `lib/daemon/api-server.ts`, `lib/daemon/api-auth.ts`, `lib/daemon/socket-server.ts`, `lib/daemon/handlers/secrets.ts`, `lib/subprocess.ts`, `lib/daemon/cache-refresh.ts`, `lib/git-worktrees.ts`, `lib/daemon/freshness.ts`, `lib/daemon/pollers.ts`, `lib/daemon/worktree-process-kill.ts`, `lib/daemon/system-process-scanner.ts`, `lib/runs/store.ts`, `lib/notifier.ts`, `lib/daemon/handlers/discussions.ts`, `lib/daemon/handlers/chat.ts`, `lib/daemon/handlers/agent.ts`, `lib/daemon/handlers/pane.ts`, `lib/daemon/handlers/project-mrs.ts`, `lib/daemon/handlers/worktree.ts`, `lib/herdr/client.ts`, `lib/port-scanner.ts`, `lib/deps/links.ts`, `lib/worktree/trash.ts`, `lib/agent-herdr.ts`, `lib/daemon/cron.ts`, `lib/daemon/hooks-guard.ts`, `lib/home/age-key.ts`, `lib/daemon/discussions-store.ts`, `lib/state/presence-store.ts`. - **Every subagent dispatched during execution carries an explicit `model`** (`sonnet` for mechanical tasks, `haiku` for lookups). - **`packages/rt-client` is touched** (Task 5 edits `registry-defs.ts`). After that task and before the final whole-branch review, run `bun run build` inside `packages/rt-client` (keeps `dist/` and `dist-freshness.test.ts` green). - **Verification (must pass before the work is done):** @@ -24,15 +24,15 @@ ## Deferred / out-of-fence items (documented, not implemented here) -- **S073 · `presence-store.ts:signIn` → `.immediate()`** — `lib/state/presence-store.ts` is fenced. Task 7 converts the chat-store and notifier-store read-then-write transactions; the `signIn` caller is a follow-up for the presence-store owner. Not required for verification. -- **0.3 · `rt.apiPort` bind-time consumption** — `lib/daemon/api-server.ts` (the binder) is fenced. Task 5 registers the `rt.apiPort` setting and exposes `resolveApiPort()`; the bind-time read is the api-server sibling's hop. The escape hatch is therefore wired daemon-side but consumed sibling-side; do not claim it functions end-to-end until the sibling reads it. -- **Swift tray edits (S026 dot mapping, S028 `DaemonLifecycle` kickstart fallback, S029 `tray-crash.log` rotation, S060 `AppDelegate` comment)** — grouped as optional Task 16. They cannot be verified by the bun/tsc/e2e gate and the operating rules forbid rebuilding the blessed bundle. Scope confirmation is raised at the plan-review checkpoint. The verifiable CLI-side halves (S028 start→kickstart fallback, S060 exit-code policy) live in Tasks 14 and 12 and are done regardless. +- **S073 · `presence-store.ts:signIn` → `.immediate()`** ... `lib/state/presence-store.ts` is fenced. Task 7 converts the chat-store and notifier-store read-then-write transactions; the `signIn` caller is a follow-up for the presence-store owner. Not required for verification. +- **0.3 · `rt.apiPort` bind-time consumption** ... `lib/daemon/api-server.ts` (the binder) is fenced. Task 5 registers the `rt.apiPort` setting and exposes `resolveApiPort()`; the bind-time read is the api-server sibling's hop. The escape hatch is therefore wired daemon-side but consumed sibling-side; do not claim it functions end-to-end until the sibling reads it. +- **Swift tray edits (S026 dot mapping, S028 `DaemonLifecycle` kickstart fallback, S029 `tray-crash.log` rotation, S060 `AppDelegate` comment)** ... grouped as optional Task 16. They cannot be verified by the bun/tsc/e2e gate and the operating rules forbid rebuilding the blessed bundle. Scope confirmation is raised at the plan-review checkpoint. The verifiable CLI-side halves (S028 start→kickstart fallback, S060 exit-code policy) live in Tasks 14 and 12 and are done regardless. --- ## Task 1: Status-verdict + exit-code design sketch -The half-page design that Tasks 9–14 depend on. No production code; the deliverable is a committed design doc. (Retires nothing directly; anchors R001, R002, S036, S060, S026, S028.) +The half-page design that Tasks 9-14 depend on. No production code; the deliverable is a committed design doc. (Retires nothing directly; anchors R001, R002, S036, S060, S026, S028.) **Files:** - Create: `docs/daemon-supervision-design.md` @@ -46,7 +46,7 @@ The half-page design that Tasks 9–14 depend on. No production code; the delive # Daemon supervision: status verdicts and exit-code semantics Phase 0 design anchor for the rt daemon stability roadmap (audit -2026-08). Tasks 9–14 of the Phase 0 plan implement this. +2026-08). Tasks 9-14 of the Phase 0 plan implement this. ## launchd contract @@ -65,44 +65,44 @@ down". Every exit-code decision below follows from that single fact. Mechanism: a module-scope `shuttingDownViaVerb` flag is set true by the `shutdown` verb before it calls cleanup; `gracefulExit(signal)` reads it -— set → exit(0), unset (bare signal) → exit(1). +... set → exit(0), unset (bare signal) → exit(1). Boot-phase gate: a module-scope `bootPhase: "booting" | "ready"` flips to `"ready"` immediately before the `daemon ready` log. The `unhandledRejection` handler exits(1) while `bootPhase === "booting"` -and only logs (recovers) once ready — so a boot-time stray rejection is +and only logs (recovers) once ready ... so a boot-time stray rejection is fatal but a steady-state one is not. ## Status verdicts `rt daemon status` and `/api/status` classify by first match: -1. `not-installed` — SMAppService not registered. -2. `serving` — ping on rt.sock succeeds. -3. `parked` — ping fails, a live rt pid exists, and the boot breadcrumb +1. `not-installed` ... SMAppService not registered. +2. `serving` ... ping on rt.sock succeeds. +3. `parked` ... ping fails, a live rt pid exists, and the boot breadcrumb phase is a flavor standoff (park). Named distinctly so the user is told "another flavor owns the socket", not "wedged". -4. `alive-not-serving` — ping fails but a live rt pid exists +4. `alive-not-serving` ... ping fails but a live rt pid exists (`process.kill(pid,0)` on rt.pid, or `pgrep -f 'rt --daemon|lib/daemon.ts'`). Sub-detail from the breadcrumb phase: `booting` / `wedged`, or `quarantined` when a state.db/events.db boot-failed marker is present. - Prints "process is running but not answering rt.sock — rt daemon logs -t". -5. `crash-looping` — no live pid AND the kv failure record shows ≥ N + Prints "process is running but not answering rt.sock ... rt daemon logs -t". +5. `crash-looping` ... no live pid AND the kv failure record shows ≥ N failures within the last M minutes (N=3, M=5). Prints the last reason. -6. `boot-failed` — no live pid AND the most recent kv exit record is a +6. `boot-failed` ... no live pid AND the most recent kv exit record is a boot throw (fewer than N failures). Prints the last reason + phase. -7. `installed-not-running` — registered, no live pid, clean/again-absent +7. `installed-not-running` ... registered, no live pid, clean/again-absent exit record. ## Persisted state (kv, ns `daemon-supervision`, no schema change) -- `boot-attempts` (number) — incremented at the top of `runDaemon()`. -- `last-ready-at` (number, epoch ms) — stamped just before `daemon ready`. -- `recent-failures` (array of `{ at, phase, reason }`, capped to 10) — +- `boot-attempts` (number) ... incremented at the top of `runDaemon()`. +- `last-ready-at` (number, epoch ms) ... stamped just before `daemon ready`. +- `recent-failures` (array of `{ at, phase, reason }`, capped to 10) ... appended by the boot fatal path and by state.db/events.db boot-failed markers. Crash-loop = ≥ N entries newer than now − M minutes. - `last-exit` (`{ at, kind: "shutdown" | "signal" | "boot-failed", code, reason? }`) - — written by the shutdown verb, the signal handlers, and the boot + ... written by the shutdown verb, the signal handlers, and the boot fatal path. Lets status distinguish "cleanly stopped" from "died". ## Boot breadcrumb @@ -123,7 +123,7 @@ git commit -m "docs: sketch daemon supervision verdicts + exit-code semantics" --- -## Task 2: Fatal boot means exit (0.1 — S001, S037) +## Task 2: Fatal boot means exit (0.1 ... S001, S037) Boot failures on the prod path currently become an `unhandledRejection` that only logs, leaving a live-pid zombie with no socket/API. Make a `runDaemon()` throw fatal, gate the rejection handler on a boot-phase flag, and move the rt.pid write to after both binds. @@ -133,7 +133,7 @@ Boot failures on the prod path currently become an `unhandledRejection` that onl - Test: `lib/__tests__/daemon-logger.test.ts` (rejection handler), `e2e/tests/daemon.test.ts` (fatal boot) **Interfaces:** -- Produces: `installCrashHandlers(logger, opts?: { booting?: () => boolean })` — when `booting()` is true, `unhandledRejection` logs `fatal` and `process.exit(1)`; otherwise it logs `error` only (today's behavior). Default (no `booting`) preserves today's error-only behavior. +- Produces: `installCrashHandlers(logger, opts?: { booting?: () => boolean })` ... when `booting()` is true, `unhandledRejection` logs `fatal` and `process.exit(1)`; otherwise it logs `error` only (today's behavior). Default (no `booting`) preserves today's error-only behavior. - Produces: module-scope `let bootPhase: "booting" | "ready" = "booting"` in `lib/daemon.ts`, flipped to `"ready"` at line 513. - [ ] **Step 1: Write the failing unit test** in `lib/__tests__/daemon-logger.test.ts`: @@ -162,7 +162,7 @@ test("unhandledRejection exits(1) while booting, only logs once ready", () => { (If `makeFakeLogger` does not exist, build a minimal `{ info, warn, error, fatal }` of `mock(() => {})`. Remove the listeners this test adds in `afterEach` via `process.removeAllListeners("unhandledRejection")` scoped to the test, matching the file's existing cleanup convention.) -- [ ] **Step 2: Run it — expect FAIL** (`booting` option not supported): +- [ ] **Step 2: Run it ... expect FAIL** (`booting` option not supported): Run: `bun test lib/__tests__/daemon-logger.test.ts -t "unhandledRejection exits"` Expected: FAIL. @@ -187,7 +187,7 @@ export function installCrashHandlers( } ``` -- [ ] **Step 4: Run the unit test — expect PASS.** +- [ ] **Step 4: Run the unit test ... expect PASS.** - [ ] **Step 5: Wire the boot-phase flag + fatal wrap + rt.pid move in `lib/daemon.ts`.** - Add near the top of module scope (after imports, before line 78): `let bootPhase: "booting" | "ready" = "booting";` @@ -207,7 +207,7 @@ async function runDaemon() { } ``` - - **Move the rt.pid write** (currently `writeFileSync(DAEMON_PID_PATH, String(process.pid))` at line 416) to AFTER both binds — i.e. after the API bind at line 469 and the socket bind at line 468 (Task 5 will make API bind first; either way, rt.pid is written only once both `servers.socket` and `servers.api` are assigned). A failed boot then never leaves a live-pid file. + - **Move the rt.pid write** (currently `writeFileSync(DAEMON_PID_PATH, String(process.pid))` at line 416) to AFTER both binds ... i.e. after the API bind at line 469 and the socket bind at line 468 (Task 5 will make API bind first; either way, rt.pid is written only once both `servers.socket` and `servers.api` are assigned). A failed boot then never leaves a live-pid file. - Set `bootPhase = "ready";` immediately before `log.info({ pid }, "daemon ready")` at line 513. - [ ] **Step 6: Write the failing e2e test** in `e2e/tests/daemon.test.ts` (uses the isolated-HOME harness already in `e2e/setup.ts`): @@ -232,9 +232,9 @@ test("daemon boot with API port already bound exits non-zero and leaves no stale }); ``` -(Reuse the harness's existing `rtBinary`, `isolatedEnv`, `isolatedHome` fixtures — mirror `e2e/tests/daemon.test.ts`'s existing setup. The daemon's own `startApiServer` retries the bind 6× before throwing, so allow up to the 60s timeout.) +(Reuse the harness's existing `rtBinary`, `isolatedEnv`, `isolatedHome` fixtures ... mirror `e2e/tests/daemon.test.ts`'s existing setup. The daemon's own `startApiServer` retries the bind 6× before throwing, so allow up to the 60s timeout.) -- [ ] **Step 7: Run the e2e test — expect PASS.** Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "API port already bound"` +- [ ] **Step 7: Run the e2e test ... expect PASS.** Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "API port already bound"` - [ ] **Step 8: Commit** @@ -245,7 +245,7 @@ git commit -m "daemon: boot failure is fatal (exit 1), gated by boot-phase flag; --- -## Task 3: Crash handlers first (0.2 — S003, S004, R007) +## Task 3: Crash handlers first (0.2 ... S003, S004, R007) Move `redirectNativeStderr()` and `installCrashHandlers()` above every module-scope side effect so a pre-`startDaemon` failure lands in the crash log instead of a discarded stderr. @@ -256,16 +256,16 @@ Move `redirectNativeStderr()` and `installCrashHandlers()` above every module-sc **Interfaces:** - Consumes: `installCrashHandlers(logger, { booting })` from Task 2. -- [ ] **Step 1: Hoist `redirectNativeStderr()`** to the very first executable statement of `lib/daemon.ts` module scope — before `migrateLegacyRtDir()` at line 78. It depends only on `logsDir()` and has its own internal try/catch, so a subsequent module-scope throw's fd-2 output lands in `daemon-stderr.log`. +- [ ] **Step 1: Hoist `redirectNativeStderr()`** to the very first executable statement of `lib/daemon.ts` module scope ... before `migrateLegacyRtDir()` at line 78. It depends only on `logsDir()` and has its own internal try/catch, so a subsequent module-scope throw's fd-2 output lands in `daemon-stderr.log`. - [ ] **Step 2: Hoist `installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" })`** to immediately after `getDaemonLogger()` resolves (right after line ~95, before `parkUntilIntended` at 103 and before `createEventsBus` at 191). The logger must exist first (line 85), so this is the earliest correct point. -- [ ] **Step 3: Remove the now-duplicate `redirectNativeStderr()` and `installCrashHandlers()` calls** inside `runDaemon()` (lines 391-392). Keep `mkdirSync(RT_DIR, …)` at 386 (redirect needs the logs dir; `redirectNativeStderr` already mkdirs its own dir, and RT_DIR creation is idempotent — verify the hoisted `redirectNativeStderr` still finds/creates `logsDir()`). +- [ ] **Step 3: Remove the now-duplicate `redirectNativeStderr()` and `installCrashHandlers()` calls** inside `runDaemon()` (lines 391-392). Keep `mkdirSync(RT_DIR, …)` at 386 (redirect needs the logs dir; `redirectNativeStderr` already mkdirs its own dir, and RT_DIR creation is idempotent ... verify the hoisted `redirectNativeStderr` still finds/creates `logsDir()`). - [ ] **Step 4: Write the failing e2e test** in `e2e/tests/daemon.test.ts`: ```ts -test("a corrupt events.db does not crash the daemon silently — error is captured", async () => { +test("a corrupt events.db does not crash the daemon silently ... error is captured", async () => { // Pre-create a corrupt events.db in the isolated HOME. const rtDir = join(isolatedHome, ".mattstack/rt"); mkdirSync(rtDir, { recursive: true }); @@ -280,7 +280,7 @@ test("a corrupt events.db does not crash the daemon silently — error is captur }); ``` -- [ ] **Step 5: Run — expect PASS** (the redirect now runs before events.db construction, so a corruption throw is captured in `daemon-stderr.log`; after Task 4 it is quarantined instead). Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "corrupt events.db"` +- [ ] **Step 5: Run ... expect PASS** (the redirect now runs before events.db construction, so a corruption throw is captured in `daemon-stderr.log`; after Task 4 it is quarantined instead). Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "corrupt events.db"` - [ ] **Step 6: Commit** @@ -291,7 +291,7 @@ git commit -m "daemon: install stderr redirect + crash handlers before every mod --- -## Task 4: events.db joins the discipline (0.4 — S009, S035) +## Task 4: events.db joins the discipline (0.4 ... S009, S035) Give `events.db` the corruption quarantine + busy_timeout/synchronous pragmas state.db has, and guard the two sweep timers so a sync sqlite throw cannot exit the daemon. @@ -301,7 +301,7 @@ Give `events.db` the corruption quarantine + busy_timeout/synchronous pragmas st - Test: `lib/daemon/__tests__/events-bus.test.ts` **Interfaces:** -- Consumes: `isCorruptionError` and `quarantine` shape from `lib/state/db.ts` (reuse the `SQLITE_CORRUPT`/`SQLITE_NOTADB` detection; `events.db` is a bounded-retention journal, so total loss on quarantine is harmless — no migration concern). +- Consumes: `isCorruptionError` and `quarantine` shape from `lib/state/db.ts` (reuse the `SQLITE_CORRUPT`/`SQLITE_NOTADB` detection; `events.db` is a bounded-retention journal, so total loss on quarantine is harmless ... no migration concern). - [ ] **Step 1: Write the failing test** in `lib/daemon/__tests__/events-bus.test.ts`: @@ -331,9 +331,9 @@ test("createEventsBus sets busy_timeout and synchronous=NORMAL", () => { (If the bus does not expose its handle, add a minimal `__db` back-reference or a `busyTimeout()` debug accessor in `events-bus.ts` for the test; do not expose it beyond the module's test needs.) -- [ ] **Step 2: Run — expect FAIL.** Run: `bun test lib/daemon/__tests__/events-bus.test.ts -t "quarantine"` +- [ ] **Step 2: Run ... expect FAIL.** Run: `bun test lib/daemon/__tests__/events-bus.test.ts -t "quarantine"` -- [ ] **Step 3: Implement in `lib/daemon/events-bus.ts`** — wrap the open (lines 73-86): +- [ ] **Step 3: Implement in `lib/daemon/events-bus.ts`** ... wrap the open (lines 73-86): ```ts import { isCorruptionError } from "../state/db"; // export it if not already exported @@ -359,7 +359,7 @@ try { Write a local `quarantineEventsDb(path, log)` mirroring `state/db.ts`'s `quarantine` (rename the db + `-wal`/`-shm` sidecars to `path.corrupt-`, `log.warn`). If `isCorruptionError` is not exported from `state/db.ts`, add the export (it is a pure predicate, safe to export). -- [ ] **Step 4: Run the events-bus tests — expect PASS.** +- [ ] **Step 4: Run the events-bus tests ... expect PASS.** - [ ] **Step 5: Write the failing sweep-guard test** in `lib/daemon/__tests__/events-bus.test.ts` OR a small `lib/__tests__/daemon-sweep-guard.test.ts` for the helper: @@ -391,7 +391,7 @@ export function safeTimeout(fn: () => void, ms: number, label: string, log: Logg Replace the two bare sweep timers at `lib/daemon.ts:194` and `:196` with `safeInterval(() => eventsBus.sweep(), 60*60*1000, "events-sweep", log)` and `safeTimeout(() => eventsBus.sweep(), 30_000, "events-sweep-boot", log)`. (The `pruneRuns`/`pruneLogs` timers at 200-248 already wrap their bodies; leaving them is fine, but converting them to `safeInterval` is a welcome DRY cleanup if trivial.) -- [ ] **Step 7: Run — expect PASS.** Then `bun test lib/daemon/__tests__/events-bus.test.ts`. +- [ ] **Step 7: Run ... expect PASS.** Then `bun test lib/daemon/__tests__/events-bus.test.ts`. - [ ] **Step 8: Commit** @@ -402,7 +402,7 @@ git commit -m "events.db: corruption quarantine + busy_timeout/synchronous pragm --- -## Task 5: Bind order + rt.apiPort setting (0.3, my side — S043, S030 seam) +## Task 5: Bind order + rt.apiPort setting (0.3, my side ... S043, S030 seam) Bind the API server before the unix socket so a failed API bind never strands a socket-bound zombie, and register the `rt.apiPort` escape-hatch setting (the api-server sibling consumes it at bind time). @@ -446,7 +446,7 @@ test("resolveApiPort: env wins, then setting, then 9401", () => { }); ``` -- [ ] **Step 4: Run — expect FAIL** (`resolveApiPort` undefined). +- [ ] **Step 4: Run ... expect FAIL** (`resolveApiPort` undefined). - [ ] **Step 5: Implement in `lib/daemon-config.ts`** (leave `API_PORT` at line 72 untouched): @@ -459,13 +459,13 @@ export function resolveApiPort(): number { } ``` -- [ ] **Step 6: Run — expect PASS.** +- [ ] **Step 6: Run ... expect PASS.** - [ ] **Step 7: Swap the bind order in `lib/daemon.ts`.** Reorder so the API binds before the socket: ```ts -servers.api = await startApiServer({ handleCommand, log }); // was line 469 — now first -servers.socket = startSocketServer({ handleCommand, log }); // was line 468 — now second +servers.api = await startApiServer({ handleCommand, log }); // was line 469 ... now first +servers.socket = startSocketServer({ handleCommand, log }); // was line 468 ... now second // rt.pid write (moved by Task 2) stays after BOTH assignments ``` @@ -484,7 +484,7 @@ test("API-bind failure leaves neither rt.sock nor rt.pid", async () => { }); ``` -- [ ] **Step 9: Run — expect PASS.** Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "API-bind failure"` +- [ ] **Step 9: Run ... expect PASS.** Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "API-bind failure"` - [ ] **Step 10: Commit** @@ -495,7 +495,7 @@ git commit -m "daemon: bind API before socket; register rt.apiPort setting + res --- -## Task 6: home-snapshot opens state.db daemon-flavored (0.7 — S011) +## Task 6: home-snapshot opens state.db daemon-flavored (0.7 ... S011) `startHomeSnapshot` opens the state.db singleton at module scope with the default `cli` flavor (5000ms busy_timeout), so the daemon runs with the wrong contention policy forever. Make its db lazy and daemon-flavored, and harden `getStateDb` against a silent flavor mismatch. @@ -505,7 +505,7 @@ git commit -m "daemon: bind API before socket; register rt.apiPort setting + res - Test: `lib/state/__tests__/db.test.ts` **Interfaces:** -- Consumes: `getStateDb("daemon")` — 250ms busy_timeout. +- Consumes: `getStateDb("daemon")` ... 250ms busy_timeout. - [ ] **Step 1: Write the failing test** in `lib/state/__tests__/db.test.ts` (`describe("pragma values per flavor")`): @@ -513,16 +513,16 @@ git commit -m "daemon: bind API before socket; register rt.apiPort setting + res test("getStateDb('daemon') reports busy_timeout 250 even after a default open", () => { const cli = getStateDb(); // opens singleton, cli flavor expect(cli.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 5000 }); - const daemon = getStateDb("daemon"); // same singleton — must not stay at 5000 + const daemon = getStateDb("daemon"); // same singleton ... must not stay at 5000 expect(daemon.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); }); ``` (Use the file's existing isolated-HOME / `closeStateDb` setup so this does not leak into other tests.) -- [ ] **Step 2: Run — expect FAIL** (singleton keeps the cli 5000 timeout). +- [ ] **Step 2: Run ... expect FAIL** (singleton keeps the cli 5000 timeout). -- [ ] **Step 3: Harden `getStateDb` in `lib/state/db.ts:522-530`** — when the singleton is already open and a caller requests a stronger (shorter) flavor timeout, re-apply the pragma: +- [ ] **Step 3: Harden `getStateDb` in `lib/state/db.ts:522-530`** ... when the singleton is already open and a caller requests a stronger (shorter) flavor timeout, re-apply the pragma: ```ts export function getStateDb(flavor: DbFlavor = "cli"): Database { @@ -537,13 +537,13 @@ export function getStateDb(flavor: DbFlavor = "cli"): Database { } ``` -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Make home-snapshot's db lazy + daemon-flavored** in `lib/daemon/home-snapshot.ts`. Replace the eager `db: rawDeps.db ?? getStateDb()` (line 272) with a thunk defaulting to `() => getStateDb("daemon")`, and resolve it on first use inside `loadState`/`runNow`/`status` rather than at construction (line 299). Concretely: store `const resolveDb = rawDeps.db ? () => rawDeps.db! : () => getStateDb("daemon");` and call `resolveDb()` where `deps.db` was read, so no db opens until `startDaemon` has already opened it daemon-flavored via `openBranchCacheStore`. -- [ ] **Step 6: Add a boot-order regression test** in `lib/state/__tests__/db.test.ts` (or `home-snapshot.test.ts`) asserting that constructing `startHomeSnapshot` does NOT open the state.db singleton (call it, then assert `getStateDb` was not yet invoked — spy on the module or assert no `state.db` file exists until first use in an isolated HOME). +- [ ] **Step 6: Add a boot-order regression test** in `lib/state/__tests__/db.test.ts` (or `home-snapshot.test.ts`) asserting that constructing `startHomeSnapshot` does NOT open the state.db singleton (call it, then assert `getStateDb` was not yet invoked ... spy on the module or assert no `state.db` file exists until first use in an isolated HOME). -- [ ] **Step 7: Run the db tests — expect PASS.** Run: `bun test lib/state/__tests__/db.test.ts` +- [ ] **Step 7: Run the db tests ... expect PASS.** Run: `bun test lib/state/__tests__/db.test.ts` - [ ] **Step 8: Commit** @@ -554,7 +554,7 @@ git commit -m "home-snapshot: lazy daemon-flavored state.db; getStateDb re-appli --- -## Task 7: Extended busy codes + IMMEDIATE transactions (0.7 — S072, S073) +## Task 7: Extended busy codes + IMMEDIATE transactions (0.7 ... S072, S073) `isBusyError` misses `SQLITE_BUSY_SNAPSHOT`/`_RECOVERY`, and read-then-write daemon transactions use a deferred `BEGIN` that produces snapshot conflicts busy_timeout cannot absorb. Widen the match and take the write lock up front. @@ -563,12 +563,12 @@ git commit -m "home-snapshot: lazy daemon-flavored state.db; getStateDb re-appli - Modify: `lib/state/chat-store.ts` (`readUnread`, `joinRoom`, `archiveRoom`, `dmRoomFor` → `.immediate()`) - Modify: `lib/state/notifier-store.ts` (`drainNotificationQueue` → `.immediate()`) - Test: `lib/state/__tests__/busy.test.ts` -- **Deferred (fenced):** `lib/state/presence-store.ts:signIn` — documented follow-up, not done here. +- **Deferred (fenced):** `lib/state/presence-store.ts:signIn` ... documented follow-up, not done here. **Interfaces:** - Produces: `isBusyError` returns true for any `code` starting `SQLITE_BUSY`. -- [ ] **Step 1: Write the failing test** in `lib/state/__tests__/busy.test.ts` — a real two-connection snapshot conflict: +- [ ] **Step 1: Write the failing test** in `lib/state/__tests__/busy.test.ts` ... a real two-connection snapshot conflict: ```ts test("isBusyError matches SQLITE_BUSY_SNAPSHOT from a real conflict", () => { @@ -589,7 +589,7 @@ test("isBusyError matches SQLITE_BUSY_SNAPSHOT from a real conflict", () => { }); ``` -- [ ] **Step 2: Run — expect FAIL** (`isBusyError` returns false for `SQLITE_BUSY_SNAPSHOT`). +- [ ] **Step 2: Run ... expect FAIL** (`isBusyError` returns false for `SQLITE_BUSY_SNAPSHOT`). - [ ] **Step 3: Widen `isBusyError`** in `lib/state/busy.ts:39-41`: @@ -600,11 +600,11 @@ export function isBusyError(err: unknown): boolean { } ``` -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Convert read-then-write transactions to `.immediate()`.** In `lib/state/chat-store.ts`, change the four cited transaction builders (`readUnread` ~483, `joinRoom`, `archiveRoom`, `dmRoomFor`) from `db.transaction(fn)(...)` to `db.transaction(fn).immediate(...)` so the write lock is taken at `BEGIN IMMEDIATE`. Do the same for `drainNotificationQueue` in `lib/state/notifier-store.ts:111`. Leave a one-line comment at the first site: `// BEGIN IMMEDIATE: read-then-write must lock up front or SQLITE_BUSY_SNAPSHOT bypasses busy_timeout.` -- [ ] **Step 6: Verify no regression** — run the chat/notifier/state suites: `bun test lib/state` and any `chat` command tests. Expect PASS (behavior identical under no contention; the change only affects lock acquisition timing). +- [ ] **Step 6: Verify no regression** ... run the chat/notifier/state suites: `bun test lib/state` and any `chat` command tests. Expect PASS (behavior identical under no contention; the change only affects lock acquisition timing). - [ ] **Step 7: Commit** @@ -615,12 +615,12 @@ git commit -m "state: isBusyError matches SQLITE_BUSY_*; read-then-write daemon --- -## Task 8: state.db importer isolation (0.3 — S074) +## Task 8: state.db importer isolation (0.3 ... S074) A throwing legacy importer inside the v0 migration rolls back the whole migration, so `user_version` stays 0 and every subsequent open repeats the failure. Wrap each importer in a SAVEPOINT; on throw, roll back that one importer, warn, and still rename the file. **Files:** -- Modify: `lib/state/db.ts:400-417` (`importLegacyStores` — SAVEPOINT per importer) +- Modify: `lib/state/db.ts:400-417` (`importLegacyStores` ... SAVEPOINT per importer) - Test: `lib/state/__tests__/db.test.ts` **Interfaces:** @@ -644,9 +644,9 @@ test("a throwing legacy importer is isolated: db reaches SCHEMA_VERSION, other s }); ``` -(Fill ``/``/fixtures from an existing importer in `LEGACY_IMPORTS`; the db.test.ts legacy-import cases already build such fixtures — reuse one.) +(Fill ``/``/fixtures from an existing importer in `LEGACY_IMPORTS`; the db.test.ts legacy-import cases already build such fixtures ... reuse one.) -- [ ] **Step 2: Run — expect FAIL** (the throw rolls back the whole migration; `user_version` stays 0 / benign rows absent). +- [ ] **Step 2: Run ... expect FAIL** (the throw rolls back the whole migration; `user_version` stays 0 / benign rows absent). - [ ] **Step 3: Implement SAVEPOINT-per-importer** in `importLegacyStores` (`lib/state/db.ts:400-417`). For each `LEGACY_IMPORTS` entry, wrap its `run(db)` in a savepoint scoped to that importer only (do NOT loosen the surrounding schema-DDL migration, which must stay loud): @@ -666,9 +666,9 @@ for (const imp of LEGACY_IMPORTS) { } ``` -(Match the exact `LegacyImport` shape at db.ts:64-69 — `path()`/`run(db)` names may differ; adapt to the real fields.) +(Match the exact `LegacyImport` shape at db.ts:64-69 ... `path()`/`run(db)` names may differ; adapt to the real fields.) -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Commit** @@ -679,7 +679,7 @@ git commit -m "state.db: isolate each legacy importer in a SAVEPOINT so one bad --- -## Task 9: Restart counter + last-exit reason in kv (0.5 — R002, S037) +## Task 9: Restart counter + last-exit reason in kv (0.5 ... R002, S037) Persist boot attempts, last-ready stamp, recent failures, and last-exit reason in the `kv` table (ns `daemon-supervision`), plus a boot breadcrumb file. Wire the record calls into the boot path, the shutdown verb, the signal handlers, and the boot fatal path. No schema change. @@ -690,13 +690,13 @@ Persist boot attempts, last-ready stamp, recent failures, and last-exit reason i **Interfaces:** - Produces: - - `recordBootAttempt(): void` — increments `boot-attempts`, appends nothing. - - `recordDaemonReady(): void` — sets `last-ready-at = Date.now()`. - - `recordBootFailure(phase: BootPhase, reason: string): void` — appends `{ at, phase, reason }` to `recent-failures` (cap 10), sets `last-exit = { at, kind: "boot-failed", code: 1, reason }`. - - `recordCleanExit(kind: "shutdown" | "signal", code: number): void` — sets `last-exit = { at, kind, code }`. + - `recordBootAttempt(): void` ... increments `boot-attempts`, appends nothing. + - `recordDaemonReady(): void` ... sets `last-ready-at = Date.now()`. + - `recordBootFailure(phase: BootPhase, reason: string): void` ... appends `{ at, phase, reason }` to `recent-failures` (cap 10), sets `last-exit = { at, kind: "boot-failed", code: 1, reason }`. + - `recordCleanExit(kind: "shutdown" | "signal", code: number): void` ... sets `last-exit = { at, kind, code }`. - `readSupervisionState(): { bootAttempts, lastReadyAt, recentFailures, lastExit }`. - - `isCrashLooping(state, now, n = 3, windowMs = 5*60_000): boolean` — ≥ n failures newer than `now - windowMs`. - - `writeBreadcrumb(phase: BootPhase): void` / `clearBreadcrumb(): void` — `~/.mattstack/rt/daemon-boot.json`. + - `isCrashLooping(state, now, n = 3, windowMs = 5*60_000): boolean` ... ≥ n failures newer than `now - windowMs`. + - `writeBreadcrumb(phase: BootPhase): void` / `clearBreadcrumb(): void` ... `~/.mattstack/rt/daemon-boot.json`. - `type BootPhase = "start" | "crash-handlers" | "events-db" | "state-db" | "socket" | "api" | "ready"`. - Consumes: `getKvValue`/`setKvValue` from `lib/state/kv-blob.ts`; `getStateDb("daemon")`. @@ -723,26 +723,26 @@ test("isCrashLooping true at >=3 failures within the window", () => { }); ``` -(Use the file's isolated-HOME convention — `bunfig` preload already repoints HOME for `bun test`; `recordBootAttempt` writes to the test state.db.) +(Use the file's isolated-HOME convention ... `bunfig` preload already repoints HOME for `bun test`; `recordBootAttempt` writes to the test state.db.) -- [ ] **Step 2: Run — expect FAIL** (module does not exist). +- [ ] **Step 2: Run ... expect FAIL** (module does not exist). - [ ] **Step 3: Implement `lib/daemon/supervision-state.ts`** with the interfaces above. All reads/writes go through `getKvValue("daemon-supervision", key, fallback, getStateDb("daemon"))` / `setKvValue("daemon-supervision", key, value, getStateDb("daemon"))`. Cap `recent-failures` at 10 on append. Breadcrumb via `writeFileSync(join(RT_DIR, "daemon-boot.json"), JSON.stringify({ at, pid: process.pid, flavor: currentMode(), phase }))` inside a try/catch (never fatal). -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Wire into the boot path** (`lib/daemon.ts`): - `recordBootAttempt(); writeBreadcrumb("start");` at the top of `runDaemon()` (after Task 3's crash handlers are already at module scope; call these first thing inside `runDaemon`). - - `writeBreadcrumb("events-db" | "state-db" | "socket" | "api")` at each corresponding phase (events.db is module-scope — write that breadcrumb right after `createEventsBus`; state-db right after `openBranchCacheStore`; socket/api at the binds). + - `writeBreadcrumb("events-db" | "state-db" | "socket" | "api")` at each corresponding phase (events.db is module-scope ... write that breadcrumb right after `createEventsBus`; state-db right after `openBranchCacheStore`; socket/api at the binds). - In Task 2's catch: `recordBootFailure(currentPhase, String(err));` (track `currentPhase` in a module var updated alongside each `writeBreadcrumb`). - `recordDaemonReady(); writeBreadcrumb("ready");` right where `bootPhase = "ready"` is set (line 513). - Shutdown verb (349-364): before `process.exit(0)`, `recordCleanExit("shutdown", 0);` and set the Task 12 `shuttingDownViaVerb = true` flag (Task 12 adds the flag; here just add the record call). - [ ] **Step 6: Wire into signal exit** (`lib/daemon/shutdown.ts` `gracefulExit`): before exit, `recordCleanExit("signal", code)` (Task 12 sets `code` to 1 for bare signals; for now record with the code it exits with). -- [ ] **Step 7: Add an e2e assertion** in `e2e/tests/daemon.test.ts`: after the Task 5 API-bind-failure spawn, assert `daemon-boot.json` exists with `phase: "api"` and the kv `recent-failures` has an entry (read the state.db in the isolated HOME, or assert via `rt daemon status --json` once Task 10 lands — for Task 9, assert the breadcrumb file only). +- [ ] **Step 7: Add an e2e assertion** in `e2e/tests/daemon.test.ts`: after the Task 5 API-bind-failure spawn, assert `daemon-boot.json` exists with `phase: "api"` and the kv `recent-failures` has an entry (read the state.db in the isolated HOME, or assert via `rt daemon status --json` once Task 10 lands ... for Task 9, assert the breadcrumb file only). -- [ ] **Step 8: Run the unit + e2e tests — expect PASS.** +- [ ] **Step 8: Run the unit + e2e tests ... expect PASS.** - [ ] **Step 9: Commit** @@ -753,7 +753,7 @@ git commit -m "daemon: persist boot attempts, failures, last-exit in kv + boot b --- -## Task 10: Status verdicts (0.5 — R001, S026 daemon-side) +## Task 10: Status verdicts (0.5 ... R001, S026 daemon-side) Extend `DaemonStatusVerdict` and `classifyDaemonStatus` with `alive-not-serving`, `parked`, `boot-failed`, `crash-looping`, read from the liveness probe + supervision state + breadcrumb. Expose the fields in `ping`/`/api/status`, and print them in `rt daemon status`. @@ -787,17 +787,17 @@ test("no pid + single boot-failed -> boot-failed with reason", () => { }); ``` -- [ ] **Step 2: Run — expect FAIL.** +- [ ] **Step 2: Run ... expect FAIL.** - [ ] **Step 3: Extend the verdict type and `classifyDaemonStatus`** in `lib/daemon-status.ts` following the resolution order from the design doc (Task 1): not-installed → serving → parked → alive-not-serving → crash-looping → boot-failed → installed-not-running. `classifyDaemonStatus` takes the already-gathered inputs (`pingOk`, `pidAlive`, `pid`, `breadcrumb`, `supervision`); keep it a pure function (the liveness probe and kv/breadcrumb reads happen in `commands/daemon.ts`/the caller, matching the existing `needsLivenessProbe` split). `parked` when breadcrumb phase indicates a flavor standoff; `alive-not-serving` detail = `booting` (phase < ready), `wedged` (phase == ready), `quarantined` (a `*.boot-failed` marker present). -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Surface the data.** In `lib/daemon/handlers/status.ts` `ping`, add a `supervision: { bootAttempts, lastReadyAt, recentFailures: recentFailures.slice(-3), lastExit }` field (read via Task 9). In `commands/daemon.ts` `showStatus`/liveness probe, gather `pidAlive` (`process.kill(pid,0)` on rt.pid, fallback `pgrep -f 'rt --daemon|lib/daemon.ts'` via `runCapture`) and read the breadcrumb + supervision state (when ping fails), then pass to `classifyDaemonStatus`. In `statusLines`, print a line per new verdict (see the design doc's status strings). - [ ] **Step 6: Add a `--json` assertion e2e** in `e2e/tests/daemon.test.ts`: after an API-bind-failure spawn, `rt daemon status --json` under the same isolated HOME reports `boot-failed` or `crash-looping` (whichever the failure count yields). Run under `env -i HOME=`. -- [ ] **Step 7: Run the unit + e2e tests — expect PASS.** `bun test lib/__tests__/daemon-status.test.ts` +- [ ] **Step 7: Run the unit + e2e tests ... expect PASS.** `bun test lib/__tests__/daemon-status.test.ts` - [ ] **Step 8: Commit** @@ -808,7 +808,7 @@ git commit -m "daemon status: alive-not-serving / parked / boot-failed / crash-l --- -## Task 11: stderr log rotation + stale-crash stamp (0.5 — S029) +## Task 11: stderr log rotation + stale-crash stamp (0.5 ... S029) `daemon-stderr.log` is never rotated and its stale contents are shown as "most recent crash". Rotate on open and gate the crash block on mtime. @@ -833,13 +833,13 @@ test("redirectNativeStderr rotates a non-empty daemon-stderr.log before reopenin }); ``` -- [ ] **Step 2: Run — expect FAIL.** +- [ ] **Step 2: Run ... expect FAIL.** - [ ] **Step 3: Implement rotate-on-open** in `redirectNativeStderr` (`lib/daemon-logger.ts:204-219`): before `openSync(path, "a")`, if the file exists and is non-empty, `renameSync` it to `daemon-stderr..log` (dedupe with a `.N` suffix if that name already exists, matching the janitor's dated-file convention). Keep the existing swallow-on-failure behavior. -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** -- [ ] **Step 5: Write the failing `showLogs` test** — the native-stderr block is hidden when the file mtime predates the current daemon's start: +- [ ] **Step 5: Write the failing `showLogs` test** ... the native-stderr block is hidden when the file mtime predates the current daemon's start: ```ts test("showLogs hides the native-stderr block when the file is older than the daemon start", () => { @@ -849,7 +849,7 @@ test("showLogs hides the native-stderr block when the file is older than the dae - [ ] **Step 6: Implement in `showLogs`** (`commands/daemon.ts:724-737`): only print the native-stderr block when the file's `mtime` is newer than the current daemon's `startedAt` (from `ping`); include the mtime in the header (`native stderr (captured )`). When older, skip it silently (or print a one-line "no crash since this daemon started"). -- [ ] **Step 7: Run — expect PASS.** +- [ ] **Step 7: Run ... expect PASS.** - [ ] **Step 8: Commit** @@ -860,7 +860,7 @@ git commit -m "logs: rotate daemon-stderr.log on open; hide stale crash block by --- -## Task 12: Exit-code semantics (0.6 — S036, S060) +## Task 12: Exit-code semantics (0.6 ... S036, S060) The `shutdown` verb correctly exits 0, but bare OS signals also exit 0, so an externally-killed daemon stays down. Reserve exit 0 for the verb; exit non-zero on bare signals. @@ -892,11 +892,11 @@ test("gracefulExit exits 0 after the shutdown verb, 1 on a bare signal", () => { (Refactor `gracefulExit` into a testable `makeGracefulExit(deps)` that returns the handler, injecting `exit`/`recordCleanExit` so no real `process.exit` fires in the test.) -- [ ] **Step 2: Run — expect FAIL.** +- [ ] **Step 2: Run ... expect FAIL.** - [ ] **Step 3: Implement.** In `lib/daemon/shutdown.ts`, refactor `installSignalHandlers`/`gracefulExit` to `makeGracefulExit(deps)` reading `deps.wasVerbShutdown()`: true → `recordCleanExit("shutdown", 0)` + `exit(0)`; false → `recordCleanExit("signal", 1)` + `exit(1)`. In `lib/daemon.ts`, add module-scope `let shuttingDownViaVerb = false;`, set it `true` in the shutdown verb before `cleanup()`, and pass `wasVerbShutdown: () => shuttingDownViaVerb` into `installSignalHandlers` at line 511. The shutdown verb keeps `process.exit(0)`. -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Commit** @@ -907,7 +907,7 @@ git commit -m "daemon: bare-signal exit is non-zero (launchd respawns); shutdown --- -## Task 13: Ownership-aware cleanup + eviction death-confirmation (0.6 — S012, S044) +## Task 13: Ownership-aware cleanup + eviction death-confirmation (0.6 ... S012, S044) `cleanup()` unlinks rt.sock/rt.pid unconditionally, and eviction sleeps a blind 300ms. Make cleanup compare-and-delete, and make eviction wait for the old pid to actually die. @@ -940,7 +940,7 @@ test("cleanup unlinks when the pid file is ours", () => { (Inject `pid` into `createCleanup` deps for testability; default to `process.pid`.) -- [ ] **Step 2: Run — expect FAIL.** +- [ ] **Step 2: Run ... expect FAIL.** - [ ] **Step 3: Implement ownership-aware unlink** in `createCleanup` (`lib/daemon/shutdown.ts:44-46`): @@ -953,7 +953,7 @@ try { } catch (err) { deps.log.warn({ err }, "cleanup unlink skipped"); } ``` -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Write the failing eviction test** in `lib/daemon/__tests__/boot-reconcile.test.ts`: @@ -969,9 +969,9 @@ test("evictStaleDaemon waits for the old pid to die, escalating to SIGKILL", asy }); ``` -- [ ] **Step 6: Implement in `evictStaleDaemon`** (`lib/daemon/boot-reconcile.ts`): replace `Bun.sleepSync(300)` with an async poll — after SIGTERM, loop `process.kill(pid, 0)` every 100ms up to ~2.5s; if still alive, `process.kill(pid, "SIGKILL")` and poll another ~0.5s; return once `process.kill(pid,0)` throws (pid gone). Make the function `async` and `await` it at the call site (`lib/daemon.ts:396`). Keep the `previousPid === process.pid` self-guard. +- [ ] **Step 6: Implement in `evictStaleDaemon`** (`lib/daemon/boot-reconcile.ts`): replace `Bun.sleepSync(300)` with an async poll ... after SIGTERM, loop `process.kill(pid, 0)` every 100ms up to ~2.5s; if still alive, `process.kill(pid, "SIGKILL")` and poll another ~0.5s; return once `process.kill(pid,0)` throws (pid gone). Make the function `async` and `await` it at the call site (`lib/daemon.ts:396`). Keep the `previousPid === process.pid` self-guard. -- [ ] **Step 7: Run — expect PASS.** Then `bun test lib/daemon/__tests__/boot-reconcile.test.ts lib/daemon/__tests__/shutdown.test.ts` +- [ ] **Step 7: Run ... expect PASS.** Then `bun test lib/daemon/__tests__/boot-reconcile.test.ts lib/daemon/__tests__/shutdown.test.ts` - [ ] **Step 8: Commit** @@ -982,7 +982,7 @@ git commit -m "daemon: ownership-aware socket/pid unlink; eviction waits for pid --- -## Task 14: uninstall + start guards (0.6 — S027, S030, S028 CLI-side) +## Task 14: uninstall + start guards (0.6 ... S027, S030, S028 CLI-side) `rt daemon uninstall` deletes rt.sock/rt.pid from under a live daemon, and `rt daemon start` cannot revive a registered-but-exited-0 daemon. Guard uninstall on liveness; make start escalate to a kickstart route. @@ -1005,11 +1005,11 @@ test("uninstall leaves rt.sock/rt.pid and daemon.json when the daemon is still r }); ``` -- [ ] **Step 2: Run — expect FAIL.** +- [ ] **Step 2: Run ... expect FAIL.** - [ ] **Step 3: Implement the uninstall guard** (`commands/daemon.ts:210-231`): after a failed/absent `trayQuery("/daemon/stop")`, call `isDaemonProcessRunning()` (and `probeSocketHolder()` as a second signal). Only run `markDaemonUninstalled()` + `cleanupDaemonFiles()` when no live holder; otherwise print the remedy `launchctl bootout gui/$UID/${activeLaunchdLabel()}` and leave the files. Audit other callers of `cleanupDaemonFiles`/`markDaemonUninstalled` (e.g. the dev-mode toggle in `commands/settings.ts`) for the same missing guard and note any in the commit body. -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Write the failing start-escalation test**: @@ -1021,9 +1021,9 @@ test("start escalates to the restart/kickstart route when the tray acks but the }); ``` -- [ ] **Step 6: Implement start escalation** (`commands/daemon.ts:235-269`): when the tray acked `/daemon/start` but `isDaemonRunning()` stays false through the 12×250ms poll, fall back to the `/daemon/restart` route (kickstart). In `lib/daemon-client.ts:171-185`, make `attemptRestart` re-probe `isDaemonRunning()` after `trayQuery("/daemon/start")` and return `true` only when the daemon actually answers — so the `daemonQuery` nag stops misdirecting. Do NOT change signal-handler exit codes here (that is Task 12). +- [ ] **Step 6: Implement start escalation** (`commands/daemon.ts:235-269`): when the tray acked `/daemon/start` but `isDaemonRunning()` stays false through the 12×250ms poll, fall back to the `/daemon/restart` route (kickstart). In `lib/daemon-client.ts:171-185`, make `attemptRestart` re-probe `isDaemonRunning()` after `trayQuery("/daemon/start")` and return `true` only when the daemon actually answers ... so the `daemonQuery` nag stops misdirecting. Do NOT change signal-handler exit codes here (that is Task 12). -- [ ] **Step 7: Run — expect PASS.** +- [ ] **Step 7: Run ... expect PASS.** - [ ] **Step 8: Commit** @@ -1034,7 +1034,7 @@ git commit -m "daemon CLI: uninstall guards on liveness; start escalates to kick --- -## Task 15: Retire the stale audit doc (0.8 — R017) +## Task 15: Retire the stale audit doc (0.8 ... R017) Replace `docs/daemon-runner-health.md` (which audits deleted subsystems) with a pointer to the current audit and the new supervision design doc. @@ -1044,7 +1044,7 @@ Replace `docs/daemon-runner-health.md` (which audits deleted subsystems) with a - [ ] **Step 1: Replace the file's contents** with a short pointer: ```markdown -# Daemon runner health — superseded +# Daemon runner health ... superseded This document audited subsystems (process-manager, remedy-engine, runner.tsx, workspace-sync) that no longer exist. It is retained only as @@ -1066,14 +1066,14 @@ git commit -m "docs: retire stale daemon-runner-health.md, point at the current --- -## Task 16 (OPTIONAL — pending plan-review scope confirmation): Swift tray consumption +## Task 16 (OPTIONAL ... pending plan-review scope confirmation): Swift tray consumption **Do not start without the reviewer's go-ahead** (raised in the plan-milestone report). These edits cannot be verified by the bun/tsc/e2e gate, and the operating rules forbid rebuilding the blessed bundle, so they ship as source-only, unverified changes following the fixer notes: -- **S026** — `rt-tray/Sources/AppDelegate.swift`: give `.starting` an expiry (record `startingSince`; in `refreshStatus` treat `.starting` as expired after ~30s / 3 failed polls and fall through to `setHealth(.down)`), so the health dot stops sticking yellow; map the new daemon verdicts (Task 10) to dot colors. -- **S028** — `rt-tray/Sources/DaemonLifecycle.swift`: when `register()` returns already-registered but the socket stays unreachable, fall back to `launchctl kickstart` (Kickstart.arguments already exists). -- **S029** — `rt-tray/Sources/TrayLog.swift`: rotate `tray-crash.log` on open (rename-if-nonempty), matching Task 11's `daemon-stderr.log` treatment. -- **S060** — `rt-tray/Sources/AppDelegate.swift:625-627`: fix the stale comment to say `KeepAlive: SuccessfulExit=false` (not `KeepAlive=true`). +- **S026** ... `rt-tray/Sources/AppDelegate.swift`: give `.starting` an expiry (record `startingSince`; in `refreshStatus` treat `.starting` as expired after ~30s / 3 failed polls and fall through to `setHealth(.down)`), so the health dot stops sticking yellow; map the new daemon verdicts (Task 10) to dot colors. +- **S028** ... `rt-tray/Sources/DaemonLifecycle.swift`: when `register()` returns already-registered but the socket stays unreachable, fall back to `launchctl kickstart` (Kickstart.arguments already exists). +- **S029** ... `rt-tray/Sources/TrayLog.swift`: rotate `tray-crash.log` on open (rename-if-nonempty), matching Task 11's `daemon-stderr.log` treatment. +- **S060** ... `rt-tray/Sources/AppDelegate.swift:625-627`: fix the stale comment to say `KeepAlive: SuccessfulExit=false` (not `KeepAlive=true`). If confirmed, do the comment fix (S060) first (trivial), then S026/S028/S029, one commit each, each commit body noting "source-only, unverified: blessed bundle not rebuilt". @@ -1082,7 +1082,7 @@ If confirmed, do the comment fix (S060) first (trivial), then S026/S028/S029, on ## Final verification (before the whole-branch review) - [ ] `cd packages/rt-client && bun run build && cd -` (rt-client was touched in Task 5) -- [ ] `bunx tsc --noEmit` — zero errors -- [ ] `bun test lib commands packages scripts` — green -- [ ] `bun run test:e2e` (or `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts` — record which) +- [ ] `bunx tsc --noEmit` ... zero errors +- [ ] `bun test lib commands packages scripts` ... green +- [ ] `bun run test:e2e` (or `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts` ... record which) - [ ] Request the whole-branch code review (superpowers:requesting-code-review); address findings; re-run the gate. From 05a8f18e684fd8bda6d48e17a120b2eda8a22b5a Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 14:05:30 -0500 Subject: [PATCH 088/106] chore: scrub em/en dashes from branch-added comments and strings (owner rule) --- commands/__tests__/daemon-logs-render.test.ts | 4 +- .../__tests__/daemon-uninstall-start.test.ts | 16 ++++---- commands/__tests__/probe-pid-alive.test.ts | 2 +- commands/daemon.ts | 38 +++++++++---------- e2e/tests/daemon.test.ts | 6 +-- lib/__tests__/daemon-logger.test.ts | 8 ++-- lib/daemon-client.ts | 2 +- lib/daemon-config.ts | 2 +- lib/daemon-logger.ts | 4 +- lib/daemon-status.ts | 18 ++++----- lib/daemon.ts | 14 +++---- lib/daemon/__tests__/home-snapshot.test.ts | 8 ++-- .../__tests__/supervision-state.test.ts | 2 +- lib/daemon/boot-reconcile.ts | 2 +- lib/daemon/events-bus.ts | 4 +- lib/daemon/handlers/status.ts | 2 +- lib/daemon/home-snapshot.ts | 4 +- lib/daemon/safe-timers.ts | 4 +- lib/daemon/shutdown.ts | 4 +- lib/daemon/supervision-state.ts | 8 ++-- lib/state/__tests__/db.test.ts | 2 +- lib/state/__tests__/source-guards.test.ts | 4 +- lib/state/busy.ts | 2 +- lib/state/db.ts | 4 +- 24 files changed, 82 insertions(+), 82 deletions(-) diff --git a/commands/__tests__/daemon-logs-render.test.ts b/commands/__tests__/daemon-logs-render.test.ts index 877e510b..eb14e8b7 100644 --- a/commands/__tests__/daemon-logs-render.test.ts +++ b/commands/__tests__/daemon-logs-render.test.ts @@ -1,5 +1,5 @@ /** - * nativeStderrDisplay — showLogs' stale-crash mtime gate. + * nativeStderrDisplay (showLogs' stale-crash mtime gate). * * daemon-stderr.log is rotated on daemon boot (lib/daemon-logger.ts), but a * leftover file can still predate the *currently running* daemon (e.g. it was @@ -31,7 +31,7 @@ describe("nativeStderrDisplay", () => { expect(header).toBe(`native stderr (captured ${new Date(mtimeMs).toISOString()})`); }); - test("fails open (shows) when the daemon's startedAt is unknown — nothing to compare against", () => { + test("fails open (shows) when the daemon's startedAt is unknown (nothing to compare against)", () => { const { show } = nativeStderrDisplay(NOW - 999_999, null); expect(show).toBe(true); }); diff --git a/commands/__tests__/daemon-uninstall-start.test.ts b/commands/__tests__/daemon-uninstall-start.test.ts index 1a3f58fd..926f5dcc 100644 --- a/commands/__tests__/daemon-uninstall-start.test.ts +++ b/commands/__tests__/daemon-uninstall-start.test.ts @@ -1,11 +1,11 @@ /** - * `rt daemon uninstall`/`start` — the CLI-side liveness guards (Task 14, + * `rt daemon uninstall`/`start` (the CLI-side liveness guards, Task 14, * S027/S030/S028-CLI). Fakes the tray over a real Bun.serve on * TRAY_SOCK_PATH (same rig as commands/__tests__/settings-dev-mode.test.ts) * and, where a scenario needs "the daemon is live", a real Bun.serve on - * DAEMON_SOCK_PATH answering /ping — isDaemonProcessRunning's pid check and + * DAEMON_SOCK_PATH answering /ping (isDaemonProcessRunning's pid check and * probeSocketHolder/isDaemonRunning's socket ping are both exercised for - * real, never mocked module internals. + * real, never mocked module internals). */ import { afterEach, describe, expect, test } from "bun:test"; import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs"; @@ -40,12 +40,12 @@ function serveTray(handlers: Record Response>): void { })); } -/** A real listener on rt.sock that answers /ping — what both isDaemonRunning() +/** A real listener on rt.sock that answers /ping (what both isDaemonRunning() * (daemon-client.ts) and probeSocketHolder() (lib/daemon/park.ts) fetch. * Flavor defaults to the CURRENT intended mode (not a hardcoded "prod") so * start()'s post-liveness warnIfWrongFlavor() check never fires a spurious * mismatch when this file runs after another test flips mattstack.mode in - * the shared isolated HOME `bun test` uses for the whole process. */ + * the shared isolated HOME `bun test` uses for the whole process). */ function serveDaemonPing(body?: Record): void { const resolvedBody = body ?? { ok: true, pid: 4242, flavor: resolveIntendedMode().mode }; servers.push(Bun.serve({ @@ -67,7 +67,7 @@ afterEach(() => { } }); -describe("uninstall — liveness guard", () => { +describe("uninstall (liveness guard)", () => { test("leaves rt.pid/daemon.json when isDaemonProcessRunning() says the daemon is alive", async () => { mkdirSync(RT_DIR, { recursive: true }); markDaemonInstalled(); @@ -98,7 +98,7 @@ describe("uninstall — liveness guard", () => { mkdirSync(RT_DIR, { recursive: true }); markDaemonInstalled(); writeFileSync(DAEMON_PID_PATH, "999999"); // no such pid - writeFileSync(DAEMON_SOCK_PATH, ""); // stale file, not a real listener — probeSocketHolder's fetch fails + writeFileSync(DAEMON_SOCK_PATH, ""); // stale file, not a real listener (probeSocketHolder's fetch fails) captureLogs(); await uninstall(); @@ -110,7 +110,7 @@ describe("uninstall — liveness guard", () => { }); }); -describe("start — kickstart escalation", () => { +describe("start (kickstart escalation)", () => { test("falls back to /daemon/restart when the tray acks /daemon/start but the socket never comes up", async () => { mkdirSync(RT_DIR, { recursive: true }); markDaemonInstalled(); diff --git a/commands/__tests__/probe-pid-alive.test.ts b/commands/__tests__/probe-pid-alive.test.ts index 269318be..e699d456 100644 --- a/commands/__tests__/probe-pid-alive.test.ts +++ b/commands/__tests__/probe-pid-alive.test.ts @@ -5,7 +5,7 @@ import { readSupervisionState } from "../../lib/daemon/supervision-state.ts"; describe("probePidAlive", () => { // Regression: the lsof fallback must exclude the CALLING process itself. // showStatus opens a bun:sqlite handle on state.db (inside RT_DIR) via - // readSupervisionState() immediately before this probe runs — `lsof +D + // readSupervisionState() immediately before this probe runs ... `lsof +D // RT_DIR` then legitimately reports the calling CLI process as a live // holder of the directory, with no daemon involved at all. Without the // process.pid filter this self-matches and a genuinely dead daemon diff --git a/commands/daemon.ts b/commands/daemon.ts index 43da6bd2..8d492ab5 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -227,14 +227,14 @@ export async function uninstall(): Promise { } // 3. A failed/absent tray stop must never delete rt.sock/rt.pid/daemon.json - // out from under a daemon that's actually still alive — that would orphan - // it (still running, launchd-supervised, but rt's own bookkeeping says + // out from under a daemon that's actually still alive (that would orphan + // it, still running, launchd-supervised, but rt's own bookkeeping says // uninstalled). Check both liveness signals: the recorded pid, and whether // anything still answers on rt.sock (a daemon can be alive with no // matching rt.pid, e.g. after a crash-and-respawn under launchd). const stillAlive = isDaemonProcessRunning() || (await probeSocketHolder()) !== null; if (stillAlive) { - console.log(`\n ${yellow}⚠${reset} daemon is still running — leaving rt.sock/rt.pid/daemon.json in place`); + console.log(`\n ${yellow}⚠${reset} daemon is still running, leaving rt.sock/rt.pid/daemon.json in place`); console.log(` ${dim}Fix: ${bold}launchctl bootout gui/$UID/${activeLaunchdLabel()}${reset}\n`); return; } @@ -276,10 +276,10 @@ export async function start(): Promise { if (await pollForDaemonUp(intended)) return; // The tray acked /daemon/start, but SMAppService can register a job that - // never actually launches (still booting, crash-looping, etc.) — kick it + // never actually launches (still booting, crash-looping, etc.); kick it // via /daemon/restart, which forces launchd to invoke it, rather than // leaving the operator staring at "check logs" for something a retry fixes. - console.log(` ${dim}not up yet — escalating to restart (kickstart)…${reset}`); + console.log(` ${dim}not up yet, escalating to restart (kickstart)…${reset}`); const restartResult = await trayQuery("/daemon/restart", "POST"); if (restartResult?.ok && (await pollForDaemonUp(intended))) return; @@ -350,18 +350,18 @@ export async function restart(): Promise { /** * Raw OS-level liveness, independent of rt.sock. Tries a direct pid check - * first against every pid this HOME actually recorded — rt.pid, then the + * first against every pid this HOME actually recorded: rt.pid, then the * boot breadcrumb's pid (the breadcrumb survives failures rt.pid never gets - * written for, per Ruling P1) — before falling back to a last-resort scan. + * written for, per Ruling P1), before falling back to a last-resort scan. * * That scan is `lsof +D `, not the brief's suggested system-wide * `pgrep -f 'rt --daemon|lib/daemon.ts'`: a raw pgrep matches ANY rt daemon * on the machine regardless of which HOME started it, and on an ordinary dev - * workstation there usually IS one — the developer's own real daemon — so a + * workstation there usually IS one (the developer's own real daemon), so a * pgrep-based check on an isolated/alternate HOME reliably misreports a dead * boot attempt as alive-not-serving (verified live against this repo's own * dev daemon while writing the e2e test below). `lsof +D` instead asks "does - * any process hold a file open under THIS HOME's rt dir" — home-scoped by + * any process hold a file open under THIS HOME's rt dir", home-scoped by * construction, immune to that false positive and to pid-reuse. Only worth * calling once both `status` and a plain ping have already failed. * @@ -369,11 +369,11 @@ export async function restart(): Promise { * own `bun:sqlite` handle on `state.db` (inside RT_DIR) via * `readSupervisionState()` just before this probe runs, so `lsof +D RT_DIR` * legitimately reports the calling CLI process as a live holder of the - * directory — with no daemon involved at all. Left unfiltered, a genuinely + * directory, with no daemon involved at all. Left unfiltered, a genuinely * dead daemon self-matches and misclassifies as alive-not-serving/parked; * this only failed to show up in manual testing because incidental work * happened to separate the state.db open from the lsof call by enough time - * for state.db's own transient lock window to close — an accident of + * for state.db's own transient lock window to close, an accident of * timing, not a guarantee. */ export async function probePidAlive(recordedPid: number | null, breadcrumbPid?: number): Promise<{ alive: boolean; pid: number | null }> { @@ -382,7 +382,7 @@ export async function probePidAlive(recordedPid: number | null, breadcrumbPid?: try { process.kill(candidate, 0); return { alive: true, pid: candidate }; - } catch { /* not this one — try the next candidate */ } + } catch { /* not this one, try the next candidate */ } } const { stdout } = await runCapture(["lsof", "-t", "+D", RT_DIR], { timeoutMs: 3000 }); const pids = stdout.trim().split(/\s+/).filter(Boolean).map(Number) @@ -408,7 +408,7 @@ export async function showStatus(args: string[] = []): Promise { // Ping ALSO failed: the only remaining ground is the pid/breadcrumb/kv // trail Task 9 left behind. Read it here, once, rather than on every status - // call — it's the uncommon path. + // call, since it's the uncommon path. let pidAlive: boolean | undefined; let pid = recordedPid; let breadcrumb: ReturnType | undefined; @@ -416,7 +416,7 @@ export async function showStatus(args: string[] = []): Promise { if (classifyDaemonStatus.needsPidProbe(response, pingOk)) { breadcrumb = readBreadcrumb(); // The kv tier can be legitimately empty (or reflect nothing useful) when - // a failure happened before state.db ever opened — Ruling P1. The + // a failure happened before state.db ever opened (Ruling P1). The // breadcrumb read above is what classifyDaemonStatus falls back to then. supervision = readSupervisionState(); const probed = await probePidAlive(recordedPid, breadcrumb?.pid); @@ -521,7 +521,7 @@ export function statusLines(verdict: DaemonStatusVerdict, now: number): string[] } if (verdict.state === "parked") { - const lines = [` ${yellow}◐${reset} parked ${dim}(pid ${verdict.pid} — another flavor owns rt.sock)${reset}`]; + const lines = [` ${yellow}◐${reset} parked ${dim}(pid ${verdict.pid}, another flavor owns rt.sock)${reset}`]; lines.push( verdict.holderFlavor ? ` ${dim}held by: ${verdict.holderFlavor}${reset}` @@ -534,7 +534,7 @@ export function statusLines(verdict: DaemonStatusVerdict, now: number): string[] if (verdict.state === "alive-not-serving") { const detailLine = { booting: "still booting", - wedged: "reached ready but stopped answering — likely deadlocked", + wedged: "reached ready but stopped answering (likely deadlocked)", quarantined: "recovered from a corrupt db but still not answering", }[verdict.detail]; return [ @@ -853,9 +853,9 @@ export async function manageTracking(args: string[] = []): Promise { * Decides whether showLogs' native-stderr block is worth printing, and its * header. `daemon-stderr.log` is rotated on open (daemon-logger.ts) but the * fresh file can still be non-empty from a crash that happened before *this* - * boot's rotation ran (e.g. a bun panic mid-startup) — so staleness is judged + * boot's rotation ran (e.g. a bun panic mid-startup), so staleness is judged * by mtime vs. the live daemon's startedAt, not by rotation alone. A `null` - * startedAt (daemon unreachable — nothing to compare against) fails open: + * startedAt (daemon unreachable, nothing to compare against) fails open: * show it, since a down daemon is exactly when the last crash matters most. */ export function nativeStderrDisplay( @@ -885,7 +885,7 @@ export async function showLogs(args: string[] = []): Promise { // Surface captured native stderr first — these are bun panics/asserts that // bypassed the JS-side interceptor and were caught by the swift-shim's - // freopen of fd 2. Only shown when it postdates the running daemon's boot — + // freopen of fd 2. Only shown when it postdates the running daemon's boot, // otherwise it's a previous life's crash, not "the most recent crash". const stderrPath = join(LOG_DIR, "daemon-stderr.log"); if (existsSync(stderrPath)) { diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index 469d54a6..4f90640a 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -65,7 +65,7 @@ describe("fatal boot", () => { const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); try { // `rt daemon status` short-circuits to "not installed" before it ever - // reaches the boot-failed/crash-looping classification — install first. + // reaches the boot-failed/crash-looping classification, install first. await rt(["daemon", "install"], { home }); const boot = await rt(["--daemon"], { home, env: { RT_API_PORT: String(port) } }); @@ -81,13 +81,13 @@ describe("fatal boot", () => { } }, 60_000); - test("a corrupt events.db self-heals — quarantined, and the daemon boots and serves", async () => { + test("a corrupt events.db self-heals (quarantined), and the daemon boots and serves", async () => { const { path: home, cleanup } = createTestHome(); const bunDir = join(process.execPath, ".."); let daemon: ReturnType | undefined; try { // Pre-create a corrupt events.db in the isolated HOME, before the - // daemon ever runs — createEventsBus (module scope) opens it. + // daemon ever runs; createEventsBus (module scope) opens it. const rtDir = join(home, ".mattstack", "rt"); mkdirSync(rtDir, { recursive: true }); writeFileSync(join(rtDir, "events.db"), "not a sqlite file at all"); diff --git a/lib/__tests__/daemon-logger.test.ts b/lib/__tests__/daemon-logger.test.ts index c573f6aa..dd8e6b3b 100644 --- a/lib/__tests__/daemon-logger.test.ts +++ b/lib/__tests__/daemon-logger.test.ts @@ -136,9 +136,9 @@ describe("lazyChildLogger", () => { }); }); -describe("redirectNativeStderr — rotation", () => { +describe("redirectNativeStderr (rotation)", () => { // redirectNativeStderr dup2's the REAL process fd 2 to the log file (that is - // the whole point of the function) — a bare call here would swallow this + // the whole point of the function); a bare call here would swallow this // test process's own stderr for the rest of the run. Save/restore fd 2 // around the call with the same dup/dup2 pair the implementation uses. function withRealFd2Saved(fn: () => void): void { @@ -286,12 +286,12 @@ describe("lazyChildLogger — Proxy guard", () => { }); }); -describe("installCrashHandlers — boot-phase gating", () => { +describe("installCrashHandlers (boot-phase gating)", () => { it("unhandledRejection exits(1) while booting, only logs once ready", () => { const exits: number[] = []; const origExit = process.exit; const origStderrWrite = process.stderr.write.bind(process.stderr); - // @ts-expect-error test stub — captures the exit code instead of terminating + // @ts-expect-error test stub (captures the exit code instead of terminating) process.exit = (code?: number) => { exits.push(code ?? 0); }; const fatal = mock(() => {}); const error = mock(() => {}); diff --git a/lib/daemon-client.ts b/lib/daemon-client.ts index c29220d1..79e56e08 100644 --- a/lib/daemon-client.ts +++ b/lib/daemon-client.ts @@ -181,7 +181,7 @@ async function attemptRestart(): Promise { if (res === null) return false; // The tray ack only proves the request was received, not that the - // daemon actually came up — re-probe rt.sock before reporting success, + // daemon actually came up, so re-probe rt.sock before reporting success, // so daemonQuery's caller isn't told "restarted" while the daemon is // still mid-boot and then misdirected into warnDaemonDown() on the very // next query instead of actually waiting for it. diff --git a/lib/daemon-config.ts b/lib/daemon-config.ts index 52f4a246..6396a3a3 100644 --- a/lib/daemon-config.ts +++ b/lib/daemon-config.ts @@ -75,7 +75,7 @@ export const API_PORT = Number(process.env.RT_API_PORT) || 9401; /** * Call-time API port resolution: RT_API_PORT env wins (e2e isolation, RT-45), * then the rt.apiPort setting (escape hatch when 9401 is held), then 9401. - * A function, not a const — must never be evaluated at module load, since + * A function, not a const: it must never be evaluated at module load, since * getSetting() reads the settings stores off ambient HOME. */ export function resolveApiPort(): number { diff --git a/lib/daemon-logger.ts b/lib/daemon-logger.ts index 5c7bc0df..634e84b0 100644 --- a/lib/daemon-logger.ts +++ b/lib/daemon-logger.ts @@ -200,7 +200,7 @@ function todayDate(): string { /** * Picks the rotation target for `daemon-stderr.log`: `daemon-stderr..log`, - * or `..log` if that name is already taken (e.g. two boots same day) — both + * or `..log` if that name is already taken (e.g. two boots same day); both * shapes match log-janitor's LOG_FILE_PATTERN, so pruneLogs sweeps them for free. */ function nextRotatedStderrPath(dir: string, date: string): string { @@ -227,7 +227,7 @@ export function redirectNativeStderr(): void { const dir = logsDir(); mkdirSync(dir, { recursive: true }); const stderrPath = join(dir, "daemon-stderr.log"); - // Rotate any leftover content from a previous crash before reopening — + // Rotate any leftover content from a previous crash before reopening, // otherwise `rt daemon logs` keeps showing yesterday's panic as "most // recent". A rename here can never lose data (unlike truncation). if (existsSync(stderrPath) && statSync(stderrPath).size > 0) { diff --git a/lib/daemon-status.ts b/lib/daemon-status.ts index f9face8b..2cdbbcfc 100644 --- a/lib/daemon-status.ts +++ b/lib/daemon-status.ts @@ -13,7 +13,7 @@ * `pidAlive`/`breadcrumb`/`supervision` (Task 9's supervision-state.ts) are * the only signals that can tell, and per Ruling P1 (2026-08-28 p0-supervision * ledger) the breadcrumb FILE is the sole record a pre-state.db boot failure - * leaves — `supervision` (the kv tier) can be absent even when `breadcrumb` + * leaves. `supervision` (the kv tier) can be absent even when `breadcrumb` * is present, and classification must still resolve to something useful from * the breadcrumb alone. */ @@ -27,7 +27,7 @@ export type DaemonStatusVerdict = /** Up — proven by an answer or a ping — but `status` itself did not deliver. */ | { state: "degraded"; reason: "error" | "unresponsive"; detail?: string; pid: number | null } /** Ping fails, a live pid exists, and it's parked waiting for a different - * flavor to hold rt.sock (park.ts) — a flavor standoff, not a stuck boot. */ + * flavor to hold rt.sock (park.ts): a flavor standoff, not a stuck boot. */ | { state: "parked"; pid: number; holderFlavor?: string } /** Ping fails but the pid is alive: still mid-boot, stuck after reaching * ready, or alive-but-quarantined (recovered from a corrupt db). */ @@ -39,7 +39,7 @@ export type DaemonStatusVerdict = | { state: "not-running"; pid: number | null }; /** The boot breadcrumb (`daemon-boot.json`), as classifyDaemonStatus needs it. Not - * imported from supervision-state.ts — that module's `Breadcrumb` interface is + * imported from supervision-state.ts, since that module's `Breadcrumb` interface is * intentionally unexported, and this shape only needs to be structurally * compatible with it. */ export interface DaemonBreadcrumbInput { @@ -58,19 +58,19 @@ export interface DaemonStatusInputs { /** Last recorded pid, for the operator to act on. */ pid: number | null; /** Raw OS-level liveness of `pid` (process.kill(pid,0), or a pgrep-found - * stand-in) — independent of rt.sock. Only worth gathering once `pingOk` + * stand-in), independent of rt.sock. Only worth gathering once `pingOk` * has already come back false; see `classifyDaemonStatus.needsPidProbe`. */ pidAlive?: boolean; /** This machine's currently-intended flavor (`resolveIntendedMode().mode`). * A live pid whose own breadcrumb flavor disagrees with this is parked - * (park.ts), not stuck — the same signal `parkUntilIntended` itself acts on. */ + * (park.ts), not stuck: the same signal `parkUntilIntended` itself acts on. */ intendedFlavor?: "dev" | "prod"; /** The socket holder's flavor, when the caller managed to learn it (best - * effort — probing rt.sock again after a failed ping/status round rarely + * effort: probing rt.sock again after a failed ping/status round rarely * succeeds, since a parked pid never binds it). Display-only. */ holderFlavor?: string | null; breadcrumb?: DaemonBreadcrumbInput | null; - /** Task 9's kv tier. Can be absent even when `breadcrumb` is present — a + /** Task 9's kv tier. Can be absent even when `breadcrumb` is present: a * pre-state.db failure leaves only the breadcrumb file (Ruling P1). */ supervision?: SupervisionState; /** Injected for deterministic crash-loop window checks under test; defaults to Date.now(). */ @@ -85,7 +85,7 @@ function classifyAliveNotServingDetail( ): "booting" | "wedged" | "quarantined" { const phase = breadcrumb?.phase; if (!phase || PHASE_ORDER.indexOf(phase) < PHASE_ORDER.indexOf("ready")) return "booting"; - // Reached ready this run, but a prior attempt is on record as boot-failed — + // Reached ready this run, but a prior attempt is on record as boot-failed, // most likely a corrupt-db quarantine (lib/state/db.ts, events-bus.ts) it // recovered from and is now stuck behind for an unrelated reason. if (supervision?.lastExit?.kind === "boot-failed") return "quarantined"; @@ -116,7 +116,7 @@ export function classifyDaemonStatus(opts: DaemonStatusInputs): DaemonStatusVerd if (pingOk) return { state: "degraded", reason: "unresponsive", pid }; // Ping failed too. From here, only pidAlive/breadcrumb/supervision (new - // signals) can say more than "not running" — absent them, fall straight + // signals) can say more than "not running"; absent them, fall straight // through to the pre-existing not-running verdict. if (pidAlive && pid !== null) { if (breadcrumb?.flavor && intendedFlavor && breadcrumb.flavor !== intendedFlavor) { diff --git a/lib/daemon.ts b/lib/daemon.ts index 64ea5fe1..44af086f 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -105,10 +105,10 @@ const rtMigration = migrateLegacyRtDir(); // during any later module-scope construction (createEventsBus, cron, // home-snapshot, …) lands in daemon-stderr.log instead of vanishing down // whatever fd 2 the launcher gave us. Depends only on logsDir() and mkdirs -// its own dir — this MUST run after migrateLegacyRtDir(): mkdirSync(logsDir()) +// its own dir (this MUST run after migrateLegacyRtDir(): mkdirSync(logsDir()) // creates the new rt dir, and migrateLegacyRtDir() treats that dir merely // existing as a "conflict" with a real legacy tree, so redirecting first -// would defeat the migration. +// would defeat the migration). redirectNativeStderr(); // ─── Logging ───────────────────────────────────────────────────────────────── @@ -449,7 +449,7 @@ async function runDaemon(): Promise { // Auto-unlink any tagged tool link whose tool now has a genuine user copy // elsewhere on PATH (e.g. the user ran `brew install gh` after rt linked // the bundled one). reconcile() itself is synchronous (a ~/.local/bin - // readDir plus a handful of stats) — wrapping the call in `async` alone + // readDir plus a handful of stats); wrapping the call in `async` alone // would NOT defer it, since nothing inside actually awaits. setTimeout(0) // is what actually pushes it past the rest of this function: the PID // write, openBranchCacheStore, and both server binds below all run first, @@ -476,7 +476,7 @@ async function runDaemon(): Promise { // one-shot re-key of every legacy NAME-keyed store row onto its // serialized repo identity. Fire-and-forget (not awaited) like the PATH - // reconcile above — the ordering guarantee this depends on (running before + // reconcile above: the ordering guarantee this depends on (running before // anything prunes the repo index) only needs this to be on the boot path, // not blocking the socket bind; a prune only ever arrives as a command sent // to an already-running daemon. @@ -524,7 +524,7 @@ async function runDaemon(): Promise { setPhase("socket"); servers.socket = startSocketServer({ handleCommand, log }); - // Only write rt.pid once both servers are actually bound — a boot that + // Only write rt.pid once both servers are actually bound: a boot that // fails before this point must never leave a live-pid file with no // socket/API behind it. writeFileSync(DAEMON_PID_PATH, String(process.pid)); @@ -536,7 +536,7 @@ async function runDaemon(): Promise { hooksGuard.refreshWatchedRepos(); // Team tracking intent (mattstack.tracking) resolves through a primed - // identity→name map, not live derivation — loadRepoTracking is sync and + // identity→name map, not live derivation; loadRepoTracking is sync and // runs on every freshness tick. Team intent is inert until this completes. // The repo index moved into state.db (RT-50): there is no file to fs.watch // for new-repo changes any more, so the 60s hooks-scan poller (pollers.ts) @@ -588,7 +588,7 @@ async function runDaemon(): Promise { } } -// runDaemon() never rejects — it logs fatal and exit(1)s internally on any +// runDaemon() never rejects: it logs fatal and exit(1)s internally on any // boot failure, so this wrapper needs no catch of its own. export async function startDaemon(): Promise { await runDaemon(); diff --git a/lib/daemon/__tests__/home-snapshot.test.ts b/lib/daemon/__tests__/home-snapshot.test.ts index ba0bbba3..33cc9df4 100644 --- a/lib/daemon/__tests__/home-snapshot.test.ts +++ b/lib/daemon/__tests__/home-snapshot.test.ts @@ -1007,7 +1007,7 @@ describe("startHomeSnapshot — state persistence", () => { // ─── boot order: db must open daemon-flavored, never at construction ──────── -describe("startHomeSnapshot — boot order", () => { +describe("startHomeSnapshot (boot order)", () => { test("constructing startHomeSnapshot does not open the state.db singleton before the caller's next await", async () => { const home = mkdtempSync(join(tmpdir(), "rt-home-snapshot-bootorder-")); const origHome = process.env.HOME; @@ -1017,17 +1017,17 @@ describe("startHomeSnapshot — boot order", () => { const stateDbPath = join(home, ".mattstack", "rt", "state.db"); const { fn: execFn } = makeFakeExec(defaultResponders({ statusZ: "?? a.txt\0" })); // No `db` override: this exercises the real getStateDb() singleton, - // matching lib/daemon.ts's module-scope `startHomeSnapshot(...)` call — + // matching lib/daemon.ts's module-scope `startHomeSnapshot(...)` call, // the exact call site that used to open state.db "cli"-flavored before // startDaemon() ever got to openBranchCacheStore(). const { deps } = baseDeps({ exec: execFn, db: undefined }); const handle = startHomeSnapshot(deps); - // Synchronously, right after construction returns — mirroring the + // Synchronously, right after construction returns, mirroring the // module-scope call in lib/daemon.ts, which runs to completion before // startDaemon() (and its openBranchCacheStore() daemon-flavored open) - // is ever reached — no db file may exist yet. + // is ever reached, no db file may exist yet. expect(existsSync(stateDbPath)).toBe(false); await handle.ready; diff --git a/lib/daemon/__tests__/supervision-state.test.ts b/lib/daemon/__tests__/supervision-state.test.ts index dbb6ea50..828ead75 100644 --- a/lib/daemon/__tests__/supervision-state.test.ts +++ b/lib/daemon/__tests__/supervision-state.test.ts @@ -14,7 +14,7 @@ import { import { RT_DIR } from "../../daemon-config.ts"; /** Test-only cleanup mirroring the breadcrumb file's path (production has - * no clear API — the daemon only ever writes or reads it). */ + * no clear API; the daemon only ever writes or reads it). */ function removeBreadcrumbFile(): void { rmSync(join(RT_DIR, "daemon-boot.json"), { force: true }); } diff --git a/lib/daemon/boot-reconcile.ts b/lib/daemon/boot-reconcile.ts index bcb106dc..4010041c 100644 --- a/lib/daemon/boot-reconcile.ts +++ b/lib/daemon/boot-reconcile.ts @@ -36,7 +36,7 @@ async function waitForDeath(pid: number, maxMs: number): Promise { * the `start` command's orphan-detection doesn't fire (e.g. launchd relaunches * us automatically without going through `rt daemon start`). * - * Waits for the old process to actually die rather than a blind sleep — a + * Waits for the old process to actually die rather than a blind sleep: a * daemon that survives the eviction window can still race the new one for * rt.sock/rt.pid (S044). Escalates to SIGKILL if SIGTERM alone doesn't land. */ diff --git a/lib/daemon/events-bus.ts b/lib/daemon/events-bus.ts index c44cd211..bbc914a3 100644 --- a/lib/daemon/events-bus.ts +++ b/lib/daemon/events-bus.ts @@ -68,7 +68,7 @@ function rowToEvent(row: EventRow): BusEvent { * Renames a corrupt events.db out of the way and warns loudly, mirroring * lib/state/db.ts's `quarantine`. events.db is a bounded-retention journal * (sweep() already discards old rows), so losing it entirely on corruption - * is harmless — recreate empty rather than attempt any repair. WAL sidecars + * is harmless: recreate empty rather than attempt any repair. WAL sidecars * are best-effort cleaned since they are meaningless without the main file. */ function quarantineEventsDb(path: string, log: Logger): void { @@ -83,7 +83,7 @@ function quarantineEventsDb(path: string, log: Logger): void { try { renameSync(sidecar, `${sidecar}.corrupt-${stamp}`); } catch { - // sidecar absent — fine, WAL mode doesn't always leave one + // sidecar absent, fine: WAL mode doesn't always leave one } } } diff --git a/lib/daemon/handlers/status.ts b/lib/daemon/handlers/status.ts index c30e1271..fc450856 100644 --- a/lib/daemon/handlers/status.ts +++ b/lib/daemon/handlers/status.ts @@ -22,7 +22,7 @@ import { readSupervisionState } from "../supervision-state.ts"; export function createStatusHandlers(ctx: HandlerContext): HandlerMap { return { "ping": async () => { - // Read here (not once at ctx build time) — a status/status.ts request + // Read here (not once at ctx build time): a status/status.ts request // must see this run's own boot-attempt/failure counters, not whatever // they were when the daemon started. const { bootAttempts, lastReadyAt, recentFailures, lastExit } = readSupervisionState(); diff --git a/lib/daemon/home-snapshot.ts b/lib/daemon/home-snapshot.ts index 476fbee2..3c793ff9 100644 --- a/lib/daemon/home-snapshot.ts +++ b/lib/daemon/home-snapshot.ts @@ -260,7 +260,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle const rawReadSettings = rawDeps.readSettings ?? (() => getSetting("rt.homeSnapshot").value); // Thunk, not a resolved value: module-scope construction (lib/daemon.ts) // must not open state.db before startDaemon() has opened it daemon-flavored - // via openBranchCacheStore — see loadState's call site inside init() below, + // via openBranchCacheStore; see loadState's call site inside init() below, // which is the first place this ever actually gets invoked. const resolveDb = rawDeps.db ? (() => rawDeps.db!) : (() => getStateDb("daemon")); const deps = { @@ -300,7 +300,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle let lastPushError: string | null = null; /** True once `home:push-failed` has been broadcast for the CURRENT unbroken run of push failures — reset to false the moment a push succeeds, so a retry storm broadcasts once, not on every attempt. */ let pushFailureBroadcast = false; - /** Populated in init(), after the is-inside-work-tree check — see resolveDb's comment for why this can't happen at construction time. */ + /** Populated in init(), after the is-inside-work-tree check; see resolveDb's comment for why this can't happen at construction time. */ let firstSeenDirty: Record = {}; let lastLoggedOwnersError: string | null = null; /** Shared dedup key for every "deps.readSettings() itself threw" warn (armWatcher's debounce read, status()) — a settings store that broke after boot and stays broken must warn once, not on every fs event or every `rt home snapshot --status` poll. */ diff --git a/lib/daemon/safe-timers.ts b/lib/daemon/safe-timers.ts index 6ba319c1..b28a5de7 100644 --- a/lib/daemon/safe-timers.ts +++ b/lib/daemon/safe-timers.ts @@ -1,9 +1,9 @@ /** - * lib/daemon/safe-timers.ts — try/catch-wrapped setInterval/setTimeout. + * lib/daemon/safe-timers.ts: try/catch-wrapped setInterval/setTimeout. * * A bare `setInterval`/`setTimeout` callback that throws synchronously * (e.g. a sqlite SQLITE_FULL on a WAL write) becomes an uncaughtException - * with no stack frame back to the timer that scheduled it — Node/Bun's + * with no stack frame back to the timer that scheduled it; Node/Bun's * event loop has nothing to attribute the throw to but the process itself, * so installCrashHandlers treats it as fatal and exits the daemon. Wrapping * the tick converts that crash into a logged warning. diff --git a/lib/daemon/shutdown.ts b/lib/daemon/shutdown.ts index 7f5993a0..29015fcb 100644 --- a/lib/daemon/shutdown.ts +++ b/lib/daemon/shutdown.ts @@ -74,8 +74,8 @@ export interface GracefulExitDeps { * * launchd's KeepAlive.SuccessfulExit=false only respawns on a non-zero exit, * so the code here must distinguish the intentional `shutdown` verb (exit 0, - * stay down) from a bare external signal — pkill, memory pressure, a stray - * script (exit 1, launchd respawns). The sanctioned stop path + * stay down) from a bare external signal (pkill, memory pressure, a stray + * script, exit 1, launchd respawns). The sanctioned stop path * (SMAppService.unregister) doesn't go through this signal path at all, so * exiting non-zero on a bare signal never fights an intended stop. */ diff --git a/lib/daemon/supervision-state.ts b/lib/daemon/supervision-state.ts index 5a753115..7e364cbf 100644 --- a/lib/daemon/supervision-state.ts +++ b/lib/daemon/supervision-state.ts @@ -1,5 +1,5 @@ /** - * lib/daemon/supervision-state.ts — daemon boot/crash history, so + * lib/daemon/supervision-state.ts: daemon boot/crash history, so * `rt daemon status` can report boot-failed/crash-looping and a stuck-phase * breadcrumb for a live-but-silent daemon. * @@ -11,7 +11,7 @@ * `recordCleanExit`, and the kv half of `recordBootFailure`) goes through * `getStateDb("daemon")`, so callers must not reach it until the daemon * has opened its state.db (lib/daemon.ts's `openBranchCacheStore()`). - * `recordBootFailure` is safe to call at any point regardless — its kv + * `recordBootFailure` is safe to call at any point regardless: its kv * write is try/catch'd and silently no-ops if the db isn't open yet. */ @@ -76,7 +76,7 @@ export function recordBootFailure(phase: BootPhase, reason: string): void { setKvValue(NS, KEY_RECENT_FAILURES, next, db()); setKvValue(NS, KEY_LAST_EXIT, { at, kind: "boot-failed", code: 1, reason }, db()); } catch { - // Pre-db failure (or a busy/corrupt state.db) — the breadcrumb file above + // Pre-db failure (or a busy/corrupt state.db): the breadcrumb file above // is the only record this failure gets, and that's fine. } } @@ -122,7 +122,7 @@ function breadcrumbPath(): string { return join(RT_DIR, "daemon-boot.json"); } -/** Never fatal — a breadcrumb is a diagnostic aid, not something boot may fail over. */ +/** Never fatal: a breadcrumb is a diagnostic aid, not something boot may fail over. */ export function writeBreadcrumb(phase: BootPhase): void { try { const breadcrumb: Breadcrumb = { at: Date.now(), pid: process.pid, flavor: daemonFlavor(), phase }; diff --git a/lib/state/__tests__/db.test.ts b/lib/state/__tests__/db.test.ts index 104a115c..2167ec5f 100644 --- a/lib/state/__tests__/db.test.ts +++ b/lib/state/__tests__/db.test.ts @@ -417,7 +417,7 @@ describe("pragma values per flavor", () => { test("getStateDb('daemon') reports busy_timeout 250 even after a default open", () => { const cli = getStateDb(); // opens singleton, cli flavor expect(cli.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 5000 }); - const daemon = getStateDb("daemon"); // same singleton — must not stay at 5000 + const daemon = getStateDb("daemon"); // same singleton, must not stay at 5000 expect(daemon.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); }); }); diff --git a/lib/state/__tests__/source-guards.test.ts b/lib/state/__tests__/source-guards.test.ts index 733a2965..38e18d32 100644 --- a/lib/state/__tests__/source-guards.test.ts +++ b/lib/state/__tests__/source-guards.test.ts @@ -93,7 +93,7 @@ describe("daemon startup opens state.db before serving", () => { test("openBranchCacheStore() precedes both server binds in runDaemon", () => { const src = readFileSync(join(REPO_ROOT, "lib", "daemon.ts"), "utf8"); // startDaemon() itself is now just `await runDaemon()` (see "boot - // failure is fatal" below — the catch-and-exit lives in runDaemon + // failure is fatal" below; the catch-and-exit lives in runDaemon // itself); the real ordered startup sequence this test asserts on // lives in runDaemon() too. const start = src.indexOf("async function runDaemon("); @@ -145,7 +145,7 @@ describe("boot failure is fatal for both fire-and-forget callers", () => { // Both real callers (cli.ts's --daemon entry, this file's own // import.meta.main guard) invoke startDaemon() fire-and-forget, and - // startDaemon() is now just `await runDaemon()` — so the catch-and-exit + // startDaemon() is now just `await runDaemon()`, so the catch-and-exit // MUST live inside runDaemon() itself, or an unhandledRejection could // silently leave the daemon half-up (rt.sock possibly bound, nothing // past the failure ever wired). The booting-gated unhandledRejection diff --git a/lib/state/busy.ts b/lib/state/busy.ts index c7de1f55..fbcb2258 100644 --- a/lib/state/busy.ts +++ b/lib/state/busy.ts @@ -37,7 +37,7 @@ let logHandle: Promise | null = null; /** * True for the bun:sqlite error thrown when a write can't get the lock - * inside busy_timeout — including the SNAPSHOT/RECOVERY variants a + * inside busy_timeout, including the SNAPSHOT/RECOVERY variants a * deferred-BEGIN read-then-write transaction can throw, which busy_timeout * does not retry the way it retries a plain SQLITE_BUSY. */ diff --git a/lib/state/db.ts b/lib/state/db.ts index 34d53c68..eac6bc3d 100644 --- a/lib/state/db.ts +++ b/lib/state/db.ts @@ -391,7 +391,7 @@ function quarantine(path: string): void { /** * Runs each registered legacy importer whose source file exists, inside the * caller's transaction. Returns the list of source paths that were consumed - * (successfully imported OR corrupt/throwing-and-skipped) — all three cases + * (successfully imported OR corrupt/throwing-and-skipped): all three cases * still rename per spec "Migration & contention" ("corrupt = warn + skip"; * brief: "warn + skip + still rename"). Renaming itself happens AFTER COMMIT * (the caller does it), since a filesystem rename cannot participate in the @@ -543,7 +543,7 @@ export function getStateDb(flavor: DbFlavor = "cli"): Database { // A caller asking for a stronger (shorter) contention policy than the // singleton currently holds must not silently inherit whatever flavor // opened it first (e.g. a "cli" 5000ms opener beating the daemon's own - // "daemon" 250ms open) — re-tighten in place rather than reopening. + // "daemon" 250ms open); re-tighten in place rather than reopening. const want = BUSY_TIMEOUT_MS[flavor]; const have = Number((singleton.query("PRAGMA busy_timeout").get() as { timeout?: number } | null)?.timeout ?? 0); if (want < have) singleton.exec(`PRAGMA busy_timeout = ${want};`); From 2852e9d8f012441725d0039d4466ecda515bfecc Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 15:41:21 -0500 Subject: [PATCH 089/106] I5: wire the seams the four lanes documented but could not cross (a) daemon.ts catches ApiPortInUseError from startApiServer and parks with exponential backoff (withApiPortParkRetry); api-server.ts binds via resolveApiPort() instead of the hardcoded API_PORT constant. (b) S055: handlers/status.ts's "repos" command uses listWorktreesAsync instead of the sync execSync-based listWorktrees; git-worktrees.ts and handlers/status.ts drop out of the no-daemon-sync-exec allowlist. (c) S010: worktree:provision validates the resolved branch with validateGitRef before any runGit call (covers divergence() too). (d) S050: freshness.ts's three remote-URL log/error interpolations run through redactCredentials. (e) S022: resolveUserIdAcrossTracking resolves userId for any branches/project-mrs tracked repo regardless of mode, called from cache-refresh.ts before checkAndNotify so poll-only users get self-authored-transition notifications from cycle 1. (f) S073: presence-store.ts's signIn transaction uses .immediate(), matching the chat-store/dm-store/notifier-store siblings. (g) rt.apiPort's settings description now reflects that it's wired. --- lib/__tests__/no-daemon-sync-exec.test.ts | 2 - lib/daemon.ts | 13 +++- lib/daemon/__tests__/api-server-bind.test.ts | 35 +++++++++- .../__tests__/api-server-park-retry.test.ts | 48 ++++++++++++++ .../__tests__/freshness-poll-userid.test.ts | 39 +++++++++++ .../freshness-redact-credentials.test.ts | 22 +++++++ .../__tests__/status-repos-async.test.ts | 64 +++++++++++++++++++ .../__tests__/worktree-handlers.test.ts | 11 ++++ lib/daemon/api-server.ts | 55 +++++++++++++--- lib/daemon/cache-refresh.ts | 12 +++- lib/daemon/freshness.ts | 44 +++++++++++-- lib/daemon/handlers/status.ts | 6 +- lib/daemon/handlers/worktree.ts | 7 ++ lib/state/__tests__/presence-store.test.ts | 18 +++++- lib/state/presence-store.ts | 2 +- .../rt-client/src/settings/registry-defs.ts | 2 +- 16 files changed, 354 insertions(+), 26 deletions(-) create mode 100644 lib/daemon/__tests__/api-server-park-retry.test.ts create mode 100644 lib/daemon/__tests__/freshness-poll-userid.test.ts create mode 100644 lib/daemon/__tests__/freshness-redact-credentials.test.ts create mode 100644 lib/daemon/__tests__/status-repos-async.test.ts diff --git a/lib/__tests__/no-daemon-sync-exec.test.ts b/lib/__tests__/no-daemon-sync-exec.test.ts index 2ba99d37..f5d8c753 100644 --- a/lib/__tests__/no-daemon-sync-exec.test.ts +++ b/lib/__tests__/no-daemon-sync-exec.test.ts @@ -11,8 +11,6 @@ const ALLOWLIST = new Set([ "lib/daemon/boot-reconcile.ts", // Phase 0.6 / S044 (Bun.sleepSync) "lib/state/db.ts", // Phase 0.7 / S072-S073 busy-retry "lib/state/busy.ts", // Phase 0.7 / S072-S073 busy-retry - "lib/git-worktrees.ts", // S055: reached only via handlers/status.ts - "lib/daemon/handlers/status.ts", // S055: the edge into git-worktrees.ts "lib/repo-index.ts", // Phase 5.3 dedup (heal/derive execSync) "lib/repo.ts", // R050 / Phase 5.4 (via handlers/system-processes.ts) "lib/git.ts", // R050 / Phase 5.4 (via repo.ts) diff --git a/lib/daemon.ts b/lib/daemon.ts index 44af086f..3c3a40e9 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -50,7 +50,7 @@ import { runBootIdentityMigration } from "./daemon/boot-migrate.ts"; import { runCapture } from "./subprocess.ts"; import { buildRoutedHandlers } from "./daemon/command-router.ts"; import { startSocketServer } from "./daemon/socket-server.ts"; -import { startApiServer, broadcast } from "./daemon/api-server.ts"; +import { startApiServer, withApiPortParkRetry, broadcast } from "./daemon/api-server.ts"; import { loadCronConfig, startCron } from "./daemon/cron.ts"; import { startPollers } from "./daemon/pollers.ts"; import { startHomeSnapshot } from "./daemon/home-snapshot.ts"; @@ -518,9 +518,16 @@ async function runDaemon(): Promise { // API server first: a failed bind exits fatally (boot-phase catch below), // and binding API before the unix socket means that fatal exit never - // strands a socket-bound zombie behind it. + // strands a socket-bound zombie behind it. ApiPortInUseError is the one + // exception to "fatal": bindApiServerWithRetry has already exhausted its + // own ~3s inner retry, so the holder is a whole other process that may + // take much longer to exit — park-and-retry with backoff instead of + // crash-looping (S043 caller-side contract, docs/daemon-api-auth.md). setPhase("api"); - servers.api = await startApiServer({ handleCommand, log }); + servers.api = await withApiPortParkRetry( + () => startApiServer({ handleCommand, log }), + { sleep: (ms) => Bun.sleep(ms), log }, + ); setPhase("socket"); servers.socket = startSocketServer({ handleCommand, log }); diff --git a/lib/daemon/__tests__/api-server-bind.test.ts b/lib/daemon/__tests__/api-server-bind.test.ts index 44139ac4..f13ed6bf 100644 --- a/lib/daemon/__tests__/api-server-bind.test.ts +++ b/lib/daemon/__tests__/api-server-bind.test.ts @@ -1,6 +1,8 @@ -import { describe, test, expect } from "bun:test"; -import { bindApiServerWithRetry, BIND_RETRY_ATTEMPTS, BIND_RETRY_DELAY_MS, type BindRetryDeps } from "../api-server.ts"; +import { describe, test, expect, afterEach } from "bun:test"; +import type { Server } from "bun"; +import { bindApiServerWithRetry, BIND_RETRY_ATTEMPTS, BIND_RETRY_DELAY_MS, startApiServer, type BindRetryDeps } from "../api-server.ts"; import { ApiPortInUseError } from "../api-server.ts"; +import { setSetting } from "../../settings/write.ts"; function eaddrinuse(): Error { return Object.assign(new Error("EADDRINUSE"), { code: "EADDRINUSE" }); @@ -125,3 +127,32 @@ describe("bindApiServerWithRetry — exhausted retries (S043)", () => { expect(d.probeCalls.length).toBe(0); }); }); + +describe("startApiServer — binds via resolveApiPort() (S043 caller-side wiring)", () => { + let server: Server | undefined; + + afterEach(() => { + server?.stop(true); + server = undefined; + }); + + test("binds to the rt.apiPort setting value, not the hardcoded 9401 default", async () => { + const prevEnv = process.env.RT_API_PORT; + delete process.env.RT_API_PORT; + + // Measure a free port rather than hardcoding one, then release it + // immediately — startApiServer binds it back before anything else can. + const probe = Bun.serve({ port: 0, fetch: () => new Response() }); + const port = probe.port; + probe.stop(true); + + setSetting("rt.apiPort", port, "user"); + const log = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as any; + + server = await startApiServer({ handleCommand: async () => ({ ok: true }), log }); + + expect(server.port).toBe(port); + + if (prevEnv !== undefined) process.env.RT_API_PORT = prevEnv; + }); +}); diff --git a/lib/daemon/__tests__/api-server-park-retry.test.ts b/lib/daemon/__tests__/api-server-park-retry.test.ts new file mode 100644 index 00000000..413427b9 --- /dev/null +++ b/lib/daemon/__tests__/api-server-park-retry.test.ts @@ -0,0 +1,48 @@ +import { describe, test, expect } from "bun:test"; +import { withApiPortParkRetry, ApiPortInUseError } from "../api-server.ts"; + +describe("withApiPortParkRetry (S043 caller-side wiring)", () => { + test("retries with backoff on ApiPortInUseError and returns once start succeeds", async () => { + const sleeps: number[] = []; + const warns: unknown[] = []; + let attempts = 0; + const result = await withApiPortParkRetry( + async () => { + attempts++; + if (attempts < 3) throw new ApiPortInUseError(9401); + return "server"; + }, + { sleep: async (ms) => { sleeps.push(ms); }, log: { warn: (o) => warns.push(o) } }, + ); + expect(result).toBe("server"); + expect(attempts).toBe(3); + expect(sleeps.length).toBe(2); + expect(warns.length).toBe(2); + }); + + test("backs off with an increasing delay on each successive attempt", async () => { + const sleeps: number[] = []; + let attempts = 0; + await withApiPortParkRetry( + async () => { + attempts++; + if (attempts < 4) throw new ApiPortInUseError(9401); + return "server"; + }, + { sleep: async (ms) => { sleeps.push(ms); }, log: { warn: () => {} } }, + ); + expect(sleeps[1]).toBeGreaterThan(sleeps[0]!); + expect(sleeps[2]).toBeGreaterThan(sleeps[1]!); + }); + + test("a non-ApiPortInUseError error is never retried", async () => { + let attempts = 0; + await expect( + withApiPortParkRetry( + async () => { attempts++; throw new Error("state.db open failed"); }, + { sleep: async () => {}, log: { warn: () => {} } }, + ), + ).rejects.toThrow("state.db open failed"); + expect(attempts).toBe(1); + }); +}); diff --git a/lib/daemon/__tests__/freshness-poll-userid.test.ts b/lib/daemon/__tests__/freshness-poll-userid.test.ts new file mode 100644 index 00000000..5e3cbc55 --- /dev/null +++ b/lib/daemon/__tests__/freshness-poll-userid.test.ts @@ -0,0 +1,39 @@ +/** + * S022: userId used to resolve only inside reconcileFreshnessImpl's + * live-mode-only loop, so a poll-only tracked repo (mode: "poll") never + * built a provider, ensureUserId() never ran, getCurrentUserId() stayed + * null forever, and checkAndNotify silently suppressed every + * self-authored transition. Static source checks (matching this file's + * established S048/S049 test style — see freshness-provider-rotation.test.ts — + * since GitLabProvider is an external network client with no test seam). + */ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +const freshnessSrc = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); +const cacheRefreshSrc = readFileSync(resolve(import.meta.dir, "..", "cache-refresh.ts"), "utf8"); + +test("S022: a mode-independent userId resolver is exported from freshness.ts", () => { + expect(freshnessSrc).toMatch(/export async function resolveUserIdAcrossTracking\(/); +}); + +test("S022: the resolver gates on the branches/project-mrs grant, not on live mode", () => { + const fn = freshnessSrc.match(/export async function resolveUserIdAcrossTracking\([\s\S]*?\n\}\n/)?.[0]; + expect(fn).toBeTruthy(); + expect(fn).toMatch(/caches\.has\(["']branches["']\)/); + expect(fn).toMatch(/caches\.has\(["']project-mrs["']\)/); + expect(fn).not.toMatch(/mode\s*!==\s*["']live["']/); +}); + +test("S022: cache-refresh.ts resolves userId before checkAndNotify (cycle-1 fix)", () => { + const resolveIndex = cacheRefreshSrc.indexOf("resolveUserIdAcrossTracking("); + const checkAndNotifyIndex = cacheRefreshSrc.indexOf("checkAndNotify(cache.entries"); + expect(resolveIndex).toBeGreaterThan(-1); + expect(checkAndNotifyIndex).toBeGreaterThan(-1); + expect(resolveIndex).toBeLessThan(checkAndNotifyIndex); +}); + +test("S022: warns once when transitions are suppressed because userId never resolved", () => { + expect(freshnessSrc).toMatch(/userId is unresolved/); +}); diff --git a/lib/daemon/__tests__/freshness-redact-credentials.test.ts b/lib/daemon/__tests__/freshness-redact-credentials.test.ts new file mode 100644 index 00000000..c4228bb3 --- /dev/null +++ b/lib/daemon/__tests__/freshness-redact-credentials.test.ts @@ -0,0 +1,22 @@ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; +import { redactCredentials } from "../redact-credentials.ts"; + +const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + +test("S050: every log/error interpolation of a remote URL runs it through redactCredentials", () => { + expect(src).toMatch(/log\.info\(`remote "\$\{redactCredentials\(remoteUrl\)\}" for \$\{repoName\}/); + expect(src).toMatch(/log\.info\(`could not parse remote "\$\{redactCredentials\(remoteUrl\)\}"/); + expect(src).toMatch(/throw new Error\(`could not parse remote URL "\$\{redactCredentials\(remoteUrl\)\}"`\)/); +}); + +test("S050: no remaining bare ${remoteUrl} interpolation in freshness.ts", () => { + expect(src).not.toMatch(/\$\{remoteUrl\}/); +}); + +test("redactCredentials strips userinfo from a credentialed URL", () => { + expect(redactCredentials("https://oauth2:glpat-XXXX@gitlab.example.com/a/b.git")).toBe( + "https://[redacted]@gitlab.example.com/a/b.git", + ); +}); diff --git a/lib/daemon/__tests__/status-repos-async.test.ts b/lib/daemon/__tests__/status-repos-async.test.ts new file mode 100644 index 00000000..37283af0 --- /dev/null +++ b/lib/daemon/__tests__/status-repos-async.test.ts @@ -0,0 +1,64 @@ +/** + * S055: "repos" used the sync `listWorktrees` (lib/git-worktrees.ts, + * execSync) on the daemon's event loop. Swapped for the async + * `listWorktreesAsync` (lib/worktree/git-async.ts, already used elsewhere + * in the daemon). + */ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { execSync } from "child_process"; +import { mkdtempSync, realpathSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { createStatusHandlers } from "../handlers/status.ts"; + +let tmpRoot: string; + +beforeEach(() => { + tmpRoot = realpathSync(mkdtempSync(join(tmpdir(), "rt-status-repos-"))); +}); + +afterEach(() => { + try { rmSync(tmpRoot, { recursive: true, force: true }); } catch { /* */ } +}); + +function initRepo(path: string): void { + execSync(`git init -q "${path}"`); + writeFileSync(join(path, "README"), "x"); + execSync(`git -C "${path}" add . && git -C "${path}" -c user.email=t@t -c user.name=t commit -q -m init`); +} + +function fakeCtx(repos: Record): any { + return { + startedAt: 123, + identity: { flavor: "dev", version: "source", sourceRev: "abc1234", startedAt: 123 }, + watchedConfigs: new Map(), + cache: { entries: {} }, + portCacheRef: { ports: [], updatedAt: null }, + repoIndex: () => repos, + }; +} + +describe("status handlers — repos (S055 async worktree listing)", () => { + test("lists worktrees with a branch, omitting detached ones", async () => { + const repo = mkdtempSync(join(tmpRoot, "repo-")); + initRepo(repo); + const linked = join(tmpRoot, "linked"); + execSync(`git -C "${repo}" worktree add -q "${linked}" -b feat/x`); + + const h = createStatusHandlers(fakeCtx({ myrepo: repo })); + const res = (await h["repos"]!({}, undefined as any)) as any; + + expect(res.ok).toBe(true); + const worktrees = res.data.repos.myrepo.worktrees; + expect(worktrees.map((w: any) => w.path).sort()).toEqual([linked, repo].sort()); + expect(worktrees.every((w: any) => typeof w.branch === "string" && w.branch.length > 0)).toBe(true); + }); + + test("a repo whose git command fails (bad repoPath) yields no worktrees, not a throw", async () => { + const notARepo = mkdtempSync(join(tmpRoot, "not-a-repo-")); + const h = createStatusHandlers(fakeCtx({ broken: notARepo })); + const res = (await h["repos"]!({}, undefined as any)) as any; + expect(res.ok).toBe(true); + expect(res.data.repos.broken.worktrees).toEqual([]); + }); +}); diff --git a/lib/daemon/__tests__/worktree-handlers.test.ts b/lib/daemon/__tests__/worktree-handlers.test.ts index c95cfb7b..7b68a7a9 100644 --- a/lib/daemon/__tests__/worktree-handlers.test.ts +++ b/lib/daemon/__tests__/worktree-handlers.test.ts @@ -235,6 +235,17 @@ describe("worktree:provision", () => { expect(res.error).toBe("repo-unknown"); }); + test("S010: refuses a branch that git would parse as an option, before any git call", async () => { + const repo = makeRepo(); + const { h, events } = makeHandlers({ [repoName]: repo }); + + const res: any = await h["worktree:provision"]!({ repoName, branch: "--upload-pack=touch /tmp/x" }); + + expect(res.ok).toBe(false); + expect(res.error).toContain("unsafe git ref"); + expect(events.length).toBe(0); + }); + test("Hard cutover: a bare legacy name refuses even when it IS a registered repoIndex key", async () => { const repo = makeRepo(); // Registered under a non-identity key on purpose — proves the rejection diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index cfe54790..fd4b5d66 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -9,7 +9,7 @@ import type { Server, ServerWebSocket } from "bun"; import type { Logger } from "pino"; -import { API_PORT } from "../daemon-config.ts"; +import { API_PORT, resolveApiPort } from "../daemon-config.ts"; import { needsToken, tokenOk, getApiToken, resolveOriginTrust } from "./api-auth.ts"; import { getAggregatedConnection } from "./freshness.ts"; import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; @@ -262,7 +262,7 @@ export const BIND_RETRY_DELAY_MS = 500; * ApiPortInUseError instead of the bare EADDRINUSE Error, so a caller can * tell "give up cleanly" apart from "the bind function itself is broken". */ -export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDeps): Promise { +export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDeps, port: number = API_PORT): Promise { const probe = deps.probePortHolder ?? defaultProbePortHolder; for (let attempt = 1; ; attempt++) { try { @@ -271,23 +271,60 @@ export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDe const isAddrInUse = err instanceof Error && (err as NodeJS.ErrnoException).code === "EADDRINUSE"; if (!isAddrInUse) throw err; if (attempt >= BIND_RETRY_ATTEMPTS) { - const holder = await probe(API_PORT).catch((probeErr) => `lsof failed: ${String(probeErr)}`); - deps.log.warn({ port: API_PORT, holder }, "api port still in use after retries; giving up bind (the daemon should park and retry with backoff rather than crash-loop)"); - throw new ApiPortInUseError(API_PORT); + const holder = await probe(port).catch((probeErr) => `lsof failed: ${String(probeErr)}`); + deps.log.warn({ port, holder }, "api port still in use after retries; giving up bind (the daemon should park and retry with backoff rather than crash-loop)"); + throw new ApiPortInUseError(port); } - deps.log.warn({ attempt, port: API_PORT }, "api port in use, retrying — another daemon is likely still shutting down"); + deps.log.warn({ attempt, port }, "api port in use, retrying — another daemon is likely still shutting down"); await deps.sleep(BIND_RETRY_DELAY_MS); } } } +/** + * Backoff base/cap for {@link withApiPortParkRetry}'s outer loop. Distinct + * from BIND_RETRY_* (bindApiServerWithRetry's own ~3s inner retry, already + * exhausted before an ApiPortInUseError ever reaches here): this loop + * assumes the holder is a whole other process that may take much longer + * than 3s to exit, so it backs off further between each full re-attempt. + */ +const PARK_RETRY_BASE_MS = 3_000; +const PARK_RETRY_MAX_MS = 60_000; + +export interface ParkRetryDeps { + sleep: (ms: number) => Promise; + log: { warn: (o: unknown, m: string) => void }; +} + +/** + * Wraps a `startApiServer`-shaped call: on `ApiPortInUseError` (bind retries + * already exhausted), logs and waits with exponential backoff, then calls + * `start` again — indefinitely, never giving up — instead of letting the + * error reach the daemon's top-level crash path (S043 caller-side contract, + * docs/daemon-api-auth.md). Any other error propagates immediately: that is + * a genuine misconfiguration, not a transient port squat. + */ +export async function withApiPortParkRetry(start: () => Promise, deps: ParkRetryDeps): Promise { + for (let attempt = 1; ; attempt++) { + try { + return await start(); + } catch (err) { + if (!(err instanceof ApiPortInUseError)) throw err; + const delayMs = Math.min(PARK_RETRY_BASE_MS * 2 ** (attempt - 1), PARK_RETRY_MAX_MS); + deps.log.warn({ attempt, port: err.port, delayMs }, "api server port still in use; parked, retrying with backoff"); + await deps.sleep(delayMs); + } + } +} + export async function startApiServer(deps: ApiServerDeps): Promise> { const { handleCommand, log } = deps; apiServerLog = log; const apiToken = getApiToken(); + const port = resolveApiPort(); const server = await bindApiServerWithRetry(() => Bun.serve({ - port: API_PORT, + port, // Bind to loopback only — never expose the control surface on the LAN. hostname: "127.0.0.1", // Raise the request idle timeout off Bun's 10s default so long-lived @@ -429,8 +466,8 @@ export async function startApiServer(deps: ApiServerDeps): Promise> // Broadcast clients are read-only; inbound frames are ignored. }, }, - }), { sleep: (ms) => Bun.sleep(ms), log }); + }), { sleep: (ms) => Bun.sleep(ms), log }, port); - log.info({ port: API_PORT }, "api server listening"); + log.info({ port }, "api server listening"); return server; } diff --git a/lib/daemon/cache-refresh.ts b/lib/daemon/cache-refresh.ts index 2771c28b..93c9f7af 100644 --- a/lib/daemon/cache-refresh.ts +++ b/lib/daemon/cache-refresh.ts @@ -16,7 +16,7 @@ import type { Logger } from "pino"; import type { PortCacheRef, RepoIndex } from "./handlers/types.ts"; import type { BranchCacheStore } from "../state/index.ts"; import { checkAndNotify } from "../notifier.ts"; -import { getCurrentUserId } from "./freshness.ts"; +import { getCurrentUserId, resolveUserIdAcrossTracking } from "./freshness.ts"; import { loadRepoTracking, grants } from "../repo-tracking.ts"; import { syncProjectMRs } from "./project-sync.ts"; import { getProjectMRs } from "./project-mrs-store.ts"; @@ -221,6 +221,16 @@ export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise { return userId; } +let userIdSuppressedWarned = false; + +/** + * S022: reconcileFreshnessImpl only ever builds a provider (and thus only + * ever calls ensureUserId) for repos in live mode, so a poll-only tracked + * user's getCurrentUserId() stayed null forever and checkAndNotify silently + * suppressed every self-authored transition. Called from cache-refresh.ts + * BEFORE its checkAndNotify (reconcileSubscriptions/reconcileFreshnessImpl + * runs after, so relying on it alone would leave the first cache-refresh + * cycle still passing null). Gates on the `branches`/`project-mrs` grant, + * never on mode, and is a no-op once userIdResolved (ensureUserId's own + * guard) or once every candidate repo has been tried this cycle. + */ +export async function resolveUserIdAcrossTracking( + repoIndex: Record, + tracking: RepoTracking, +): Promise { + if (userIdResolved) return; + for (const [repoName, repoPath] of Object.entries(repoIndex)) { + const g = grants(tracking, repoName); + if (g.mode === "off") continue; + if (!g.caches.has("branches") && !g.caches.has("project-mrs")) continue; + if (!existsSync(repoPath)) continue; + const provider = await ensureProvider(repoName, repoPath); + if (!provider) continue; + await ensureUserId(); + if (userIdResolved) return; + } + if (!userIdResolved && !userIdSuppressedWarned) { + userIdSuppressedWarned = true; + log.warn("userId is unresolved; self-authored MR transitions are being suppressed (no gitlabToken, or token validation failed)"); + } +} + export function getSelfUsername(): string | null { return selfUsername; } export async function resolveSelfUsername(repoName: string, repoPath: string): Promise { @@ -307,7 +343,7 @@ export async function getRepoContext( } const remote = parseRemoteUrl(remoteUrl); if (!remote) { - throw new Error(`could not parse remote URL "${remoteUrl}"`); + throw new Error(`could not parse remote URL "${redactCredentials(remoteUrl)}"`); } provider = makeProvider(remote.host, secrets.gitlabToken); providers.set(repoName, { provider, token: secrets.gitlabToken }); diff --git a/lib/daemon/handlers/status.ts b/lib/daemon/handlers/status.ts index fc450856..19db8227 100644 --- a/lib/daemon/handlers/status.ts +++ b/lib/daemon/handlers/status.ts @@ -14,7 +14,7 @@ import { existsSync, readdirSync } from "fs"; import type { HandlerContext, HandlerMap } from "./types.ts"; import type { PortEntry } from "../../port-scanner.ts"; -import { listWorktrees } from "../../git-worktrees.ts"; +import { listWorktreesAsync } from "../../worktree/git-async.ts"; import { drainNotifications, peekNotifications } from "../../notifier.ts"; import { getFreshnessSnapshot } from "../freshness.ts"; import { readSupervisionState } from "../supervision-state.ts"; @@ -115,7 +115,9 @@ export function createStatusHandlers(ctx: HandlerContext): HandlerMap { for (const [repoName, repoPath] of Object.entries(repos)) { if (!existsSync(repoPath)) continue; // Detached worktrees have no branch — omit them from the listing. - const worktrees = listWorktrees(repoPath).filter((w) => w.branch); + const worktrees = ((await listWorktreesAsync(repoPath)) ?? []).filter( + (w): w is { path: string; branch: string } => Boolean(w.branch), + ); detailed[repoName] = { path: repoPath, worktrees }; } diff --git a/lib/daemon/handlers/worktree.ts b/lib/daemon/handlers/worktree.ts index bb98e53e..b4c3f223 100644 --- a/lib/daemon/handlers/worktree.ts +++ b/lib/daemon/handlers/worktree.ts @@ -32,6 +32,7 @@ import { realpathSync, rmSync } from "fs"; import { join } from "path"; import { parseIdentity } from "../../settings/identity.ts"; +import { validateGitRef } from "../git-ref-validation.ts"; import type { HandlerContext, HandlerMap } from "./types.ts"; import { findByBranch, @@ -295,6 +296,12 @@ export function createWorktreeHandlers( return { ok: false, error: "branch-unresolved" }; } + // S010: a branch that git would parse as an option (e.g. + // "--upload-pack=...") must never reach a runGit call, including + // divergence()'s below — both read this same `branch`. + const refCheck = validateGitRef(branch); + if (!refCheck.ok) return { ok: false, error: refCheck.error }; + const attached = findByBranch(trees, branch); if (attached.length > 1) return { ok: false, error: "branch-duplicated" }; if (attached.length === 1) return { ok: false, error: `branch-attached:${attached[0]!.name}` }; diff --git a/lib/state/__tests__/presence-store.test.ts b/lib/state/__tests__/presence-store.test.ts index a875d2b6..2d9e1521 100644 --- a/lib/state/__tests__/presence-store.test.ts +++ b/lib/state/__tests__/presence-store.test.ts @@ -6,8 +6,9 @@ * exercise the dual-write and room-default wiring those tests cover. */ import { expect, test } from "bun:test"; +import { readFileSync } from "fs"; import { tmpdir } from "os"; -import { join } from "path"; +import { join, resolve } from "path"; import { openStateDb } from "../db.ts"; import { armMember, clearAllArmed, disarmMember, joinRoom, listMembers, touchMember } from "../chat-store.ts"; import { assertSessionOwnsHandle, assertSessionSignedIn, buddyStatus, presenceForSession, presenceThresholds, prunePresence, pulseSession, signIn, signOut } from "../presence-store.ts"; @@ -100,6 +101,21 @@ test("assertSessionOwnsHandle throws only on a mismatched signed handle", () => expect(() => assertSessionOwnsHandle("x", undefined, db)).not.toThrow(); // no session id offered, no enforcement }); +test("S073: signIn's read-then-write transaction uses .immediate() (BEGIN IMMEDIATE), not a deferred BEGIN", () => { + // A plain db.transaction()'s deferred BEGIN lets signIn's own reads + // (prunePresence, SELECT_PRESENCE_BY_SESSION_SQL) open a snapshot before + // any write; a commit by another connection in that window turns the + // eventual write into an unretryable SQLITE_BUSY_SNAPSHOT that the + // flavor's busy_timeout cannot absorb. .immediate() takes the write lock + // at BEGIN, so contention surfaces as an ordinary, retryable SQLITE_BUSY + // instead (matching the chat-store.ts/dm-store.ts/notifier-store.ts + // siblings already converted for the same reason). + const src = readFileSync(resolve(import.meta.dir, "..", "presence-store.ts"), "utf8"); + const runIndex = src.indexOf("const run = db.transaction("); + expect(runIndex).toBeGreaterThan(-1); + expect(src.indexOf("return run.immediate();", runIndex)).toBeGreaterThan(runIndex); +}); + test("assertSessionSignedIn throws when the session's row is gone", () => { const db = fresh(); expect(() => assertSessionSignedIn("ghost", db)).toThrow(/handle reclaimed/); diff --git a/lib/state/presence-store.ts b/lib/state/presence-store.ts index c665b86f..530fcb53 100644 --- a/lib/state/presence-store.ts +++ b/lib/state/presence-store.ts @@ -308,7 +308,7 @@ export function signIn( return { handle, baseHandle, reclaimed: winnerRow !== null }; }); - return run(); + return run.immediate(); } const NAMES_KV_NS = "chat"; diff --git a/packages/rt-client/src/settings/registry-defs.ts b/packages/rt-client/src/settings/registry-defs.ts index 84002379..8a00016b 100644 --- a/packages/rt-client/src/settings/registry-defs.ts +++ b/packages/rt-client/src/settings/registry-defs.ts @@ -203,7 +203,7 @@ export const REGISTRY: readonly SettingDef[] = [ default: 9401, merge: "replace", migrated: true, - description: "TCP port for the daemon's local HTTP/WS API. Not yet consumed at bind time (pending api-server wiring); today only the RT_API_PORT env var overrides the default 9401.", + description: "TCP port for the daemon's local HTTP/WS API. Escape hatch when 9401 is held: RT_API_PORT env wins, then this setting, then 9401 (lib/daemon-config.ts resolveApiPort(), read at bind time by lib/daemon/api-server.ts).", }, { key: "rt.hooks", From f395575e00fe2548cb682cbd43580d1f693ffd39 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 15:51:59 -0500 Subject: [PATCH 090/106] e2e: rewrite the 3 API-port-squat daemon tests for I5(a)'s park-retry They asserted the pre-fix crash-on-EADDRINUSE behavior (fatal exit, boot-failed/crash-looping). I5(a) makes this recoverable instead: withApiPortParkRetry parks and retries with backoff rather than crashing, so the daemon now boots successfully once the squatted port frees. Rewritten to assert the new contract: alive (not exited) while parked, no rt.sock/rt.pid until bind succeeds, status never falsely reports "running" while parked, and both recover once the port frees. --- e2e/tests/daemon.test.ts | 106 ++++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 34 deletions(-) diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index 4f90640a..a7dc3cdb 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -21,62 +21,100 @@ function freePort(): number { } describe("fatal boot", () => { - test("daemon boot with API port already bound exits non-zero and leaves no stale rt.pid", async () => { + // These three used to assert a crash (S043 pre-fix: EADDRINUSE on the API + // port took the daemon down the fatal boot-failed path). The integration + // job's I5(a) wiring (lib/daemon.ts's withApiPortParkRetry around + // startApiServer, docs/daemon-api-auth.md's S043 caller-side contract) + // makes this recoverable instead: the daemon parks and retries with + // backoff rather than crashing, so it now boots successfully once the + // squatted port frees. + test("daemon parks (does not crash) while the API port is squatted, and boots once it frees", async () => { const { path: home, cleanup } = createTestHome(); - // Bind the API port inside the isolated HOME so the daemon cannot. + const bunDir = join(process.execPath, ".."); const port = 9411; + const rtDir = join(home, ".mattstack", "rt"); const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + let daemon: ReturnType | undefined; try { - const result = await rt(["--daemon"], { home, env: { RT_API_PORT: String(port) } }); + daemon = Bun.spawn([RT_BINARY, "--daemon"], { + env: { + HOME: home, + PATH: `${join(RT_BINARY, "..")}:${bunDir}:/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin`, + TERM: "xterm-256color", + RT_SKIP_SETUP: "1", + CI: "true", + RT_API_PORT: String(port), + }, + stdout: "pipe", + stderr: "pipe", + }); - expect(result.exitCode).not.toBe(0); - expect(existsSync(join(home, ".mattstack", "rt", "rt.pid"))).toBe(false); - } finally { - squatter.stop(true); - cleanup(); - } - }, 60_000); + // Give bindApiServerWithRetry's own ~3s inner retry a full cycle to + // exhaust and reach the outer park-retry loop; it must still be alive + // (parked, not crashed) and must not have written rt.sock/rt.pid yet + // (neither server has bound). + await Bun.sleep(4_000); + expect(daemon.exitCode).toBeNull(); + expect(existsSync(join(rtDir, "rt.sock"))).toBe(false); + expect(existsSync(join(rtDir, "rt.pid"))).toBe(false); - test("API-bind failure leaves neither rt.sock nor rt.pid", async () => { - const { path: home, cleanup } = createTestHome(); - // A different port than the sibling squatter test above, so parallel - // test files can never collide on the same bound TCP port. - const port = 9412; - const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); - try { - const result = await rt(["--daemon"], { home, env: { RT_API_PORT: String(port) } }); + squatter.stop(true); - expect(result.exitCode).not.toBe(0); - // The API bind (now first) fails before the socket ever binds, so a - // fatal exit must strand neither file. - expect(existsSync(join(home, ".mattstack", "rt", "rt.sock"))).toBe(false); - expect(existsSync(join(home, ".mattstack", "rt", "rt.pid"))).toBe(false); + await waitForSocket(join(rtDir, "rt.sock"), 40_000); + expect(daemon.exitCode).toBeNull(); + expect(existsSync(join(rtDir, "rt.pid"))).toBe(true); } finally { squatter.stop(true); + try { daemon?.kill(); } catch { /* already gone */ } + await daemon?.exited; cleanup(); } }, 60_000); - test("API-bind failure surfaces as boot-failed/crash-looping via rt daemon status --json", async () => { + test("daemon status --json never claims 'running' while parked on a squatted API port, and does once it recovers", async () => { const { path: home, cleanup } = createTestHome(); - // A different port than the other API-bind-failure tests above, so - // parallel test files can never collide on the same bound TCP port. - const port = 9413; + const bunDir = join(process.execPath, ".."); + // A different port than the sibling test above, so parallel test files + // can never collide on the same bound TCP port. + const port = 9412; + const rtDir = join(home, ".mattstack", "rt"); const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + let daemon: ReturnType | undefined; try { // `rt daemon status` short-circuits to "not installed" before it ever - // reaches the boot-failed/crash-looping classification, install first. + // reaches a liveness classification, install first. await rt(["daemon", "install"], { home }); - const boot = await rt(["--daemon"], { home, env: { RT_API_PORT: String(port) } }); - expect(boot.exitCode).not.toBe(0); + daemon = Bun.spawn([RT_BINARY, "--daemon"], { + env: { + HOME: home, + PATH: `${join(RT_BINARY, "..")}:${bunDir}:/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin`, + TERM: "xterm-256color", + RT_SKIP_SETUP: "1", + CI: "true", + RT_API_PORT: String(port), + }, + stdout: "pipe", + stderr: "pipe", + }); + + await Bun.sleep(4_000); + expect(daemon.exitCode).toBeNull(); + + const parked = await rt(["daemon", "status", "--json"], { home, env: { RT_API_PORT: String(port) } }); + expect(parked.exitCode).toBe(0); + expect(JSON.parse(parked.stdout).state).not.toBe("running"); + + squatter.stop(true); + await waitForSocket(join(rtDir, "rt.sock"), 40_000); - const status = await rt(["daemon", "status", "--json"], { home }); - expect(status.exitCode).toBe(0); - const parsed = JSON.parse(status.stdout); - expect(["boot-failed", "crash-looping"]).toContain(parsed.state); + const recovered = await rt(["daemon", "status", "--json"], { home, env: { RT_API_PORT: String(port) } }); + expect(recovered.exitCode).toBe(0); + expect(JSON.parse(recovered.stdout).state).toBe("running"); } finally { squatter.stop(true); + try { daemon?.kill(); } catch { /* already gone */ } + await daemon?.exited; cleanup(); } }, 60_000); From 1e79eb39ffa9c0a8bb4163e27508b92050abdfbd Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 19:50:41 -0500 Subject: [PATCH 091/106] docs: regenerate command reference for wave-1 daemon flags and verbs Co-Authored-By: Claude Fable 5 --- website/docs/reference/daemon/status.mdx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/website/docs/reference/daemon/status.mdx b/website/docs/reference/daemon/status.mdx index 0d7c2410..bfd689ac 100644 --- a/website/docs/reference/daemon/status.mdx +++ b/website/docs/reference/daemon/status.mdx @@ -12,9 +12,15 @@ Show daemon status ## Usage ```bash -rt daemon status +rt daemon status [flags] ``` +## Arguments & flags + +| Flag / Arg | Type | Default | Description | +| --- | --- | --- | --- | +| [`--json`](/guides/common-flags) | boolean | `false` | Emit the verdict as JSON instead of the formatted lines | + _See code: [commands/daemon.ts › showStatus](https://github.com/m4ttstack/rt/blob/main/commands/daemon.ts)_ {/* generated by scripts/gen-docs.ts; edit prose in _partials, not here */} \ No newline at end of file From e8d841706d0ead7e46a61e8dead20db6418ce896 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 20:14:12 -0500 Subject: [PATCH 092/106] ci: retrigger checks after docs regeneration From 68c2618f8db90211e57ffe18566660c0c18ffb61 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 20:52:05 -0500 Subject: [PATCH 093/106] chat-handlers.test: drop test for chat:unread-waking, a verb main's delivery-v2 removed --- lib/daemon/__tests__/chat-handlers.test.ts | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/lib/daemon/__tests__/chat-handlers.test.ts b/lib/daemon/__tests__/chat-handlers.test.ts index 93f4e9b9..f1941e82 100644 --- a/lib/daemon/__tests__/chat-handlers.test.ts +++ b/lib/daemon/__tests__/chat-handlers.test.ts @@ -174,23 +174,6 @@ test("chat:dm rejects a missing or empty body", async () => { expect(empty.ok).toBe(false); }); -test("chat:unread-waking reports what would wake a handle without advancing its cursor", async () => { - const h = freshHandlers(); - await h["chat:join"]({ room: "r", handle: "a" }); - await h["chat:join"]({ room: "r", handle: "b" }); - await h["chat:post"]({ room: "r", handle: "a", body: "@b hi" }); - const res1 = await h["chat:unread-waking"]({ handle: "b" }); - if (!res1.ok) throw new Error("unreachable"); - const first = res1.data; - expect(first).toMatchObject({ rooms: [{ room: "r", count: 1, mentions: 1 }] }); - // maxId is the watermark the tail's stream loop skips at or below; without - // it the tail cannot tell which wakes the catch-up already covered. - expect(first.rooms[0]!.maxId).toBeGreaterThan(0); - const res2 = await h["chat:unread-waking"]({ handle: "b" }); - if (!res2.ok) throw new Error("unreachable"); - expect(res2.data).toEqual(first); -}); - // R034: `limit: -1` reaches `ORDER BY id ASC LIMIT ?`, where SQLite treats a // negative LIMIT as unlimited, so a viewer/agent bug returns and // JSON-serializes an entire (100k-row) room on the event loop. From 161d5884486741c75223648684be8a2f8698dd68 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 23:08:55 -0500 Subject: [PATCH 094/106] api-auth: allow token-authenticated browser preflight for off-allowlist origins --- .../__tests__/api-server-cors-ws.test.ts | 46 ++++++++++++++++++- lib/daemon/api-auth.ts | 15 ++++++ lib/daemon/api-server.ts | 5 +- 3 files changed, 62 insertions(+), 4 deletions(-) diff --git a/lib/daemon/__tests__/api-server-cors-ws.test.ts b/lib/daemon/__tests__/api-server-cors-ws.test.ts index 58c5bf8a..39f2df0c 100644 --- a/lib/daemon/__tests__/api-server-cors-ws.test.ts +++ b/lib/daemon/__tests__/api-server-cors-ws.test.ts @@ -1,5 +1,8 @@ -import { describe, test, expect } from "bun:test"; -import { buildCorsHeaders } from "../api-server.ts"; +import { describe, test, expect, afterEach } from "bun:test"; +import type { Server } from "bun"; +import { buildCorsHeaders, startApiServer } from "../api-server.ts"; +import { getApiToken } from "../api-auth.ts"; +import { setSetting } from "../../settings/write.ts"; describe("buildCorsHeaders", () => { test("no Origin header: no Access-Control-Allow-Origin is set (non-browser request, CORS is irrelevant)", () => { @@ -24,3 +27,42 @@ describe("buildCorsHeaders", () => { expect(headers["Access-Control-Allow-Headers"]).toContain("X-RT-Token"); }); }); + +describe("token-authenticated browser preflight (S-C1: off-allowlist Origin, X-RT-Token preflight)", () => { + let server: Server | undefined; + + afterEach(() => { + server?.stop(true); + server = undefined; + }); + + test("an OPTIONS preflight requesting X-RT-Token from an off-allowlist Origin still gets Access-Control-Allow-Origin, so the browser proceeds to the real token-bearing request", async () => { + const probe = Bun.serve({ port: 0, fetch: () => new Response() }); + const port = probe.port; + probe.stop(true); + setSetting("rt.apiPort", port, "user"); + const log = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as any; + server = await startApiServer({ handleCommand: async () => ({ ok: true }), log }); + + const apiToken = getApiToken(); + const origin = "http://off-allowlist.example"; + + const preflight = await fetch(`http://127.0.0.1:${port}/api/refresh`, { + method: "OPTIONS", + headers: { + Origin: origin, + "Access-Control-Request-Method": "GET", + "Access-Control-Request-Headers": "x-rt-token", + }, + }); + expect(preflight.status).toBe(204); + expect(preflight.headers.get("access-control-allow-origin")).toBe(origin); + + const actual = await fetch(`http://127.0.0.1:${port}/api/refresh`, { + method: "POST", + headers: { Origin: origin, "X-RT-Token": apiToken }, + }); + expect(actual.status).not.toBe(401); + expect(actual.headers.get("access-control-allow-origin")).toBe(origin); + }); +}); diff --git a/lib/daemon/api-auth.ts b/lib/daemon/api-auth.ts index ac45a18f..45a058be 100644 --- a/lib/daemon/api-auth.ts +++ b/lib/daemon/api-auth.ts @@ -151,3 +151,18 @@ export function resolveOriginTrust( if (!origin) return true; return isBrowserRequestTrusted(origin, presentedToken, apiToken, getAllowedOrigins()); } + +/** + * A browser CORS preflight (OPTIONS) cannot carry the X-RT-Token value + * itself -- only Access-Control-Request-Headers names it as a header the + * follow-up request will use -- so an off-allowlist Origin that intends to + * authenticate with the token must be granted the preflight on trust alone. + * tokenOk() still gates the actual request; this only lets the browser send it. + */ +export function isTokenPreflight(method: string, requestHeaders: string | null): boolean { + if (method !== "OPTIONS" || !requestHeaders) return false; + return requestHeaders + .split(",") + .map((h) => h.trim().toLowerCase()) + .includes("x-rt-token"); +} diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index fd4b5d66..e17c3140 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -10,7 +10,7 @@ import type { Server, ServerWebSocket } from "bun"; import type { Logger } from "pino"; import { API_PORT, resolveApiPort } from "../daemon-config.ts"; -import { needsToken, tokenOk, getApiToken, resolveOriginTrust } from "./api-auth.ts"; +import { needsToken, tokenOk, getApiToken, resolveOriginTrust, isTokenPreflight } from "./api-auth.ts"; import { getAggregatedConnection } from "./freshness.ts"; import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; import { runCapture } from "../subprocess.ts"; @@ -354,7 +354,8 @@ export async function startApiServer(deps: ApiServerDeps): Promise> // at all, so a malicious page's own JS cannot read the response. // resolveOriginTrust only resolves the allowlist when origin is set, // since the settings read behind it is synchronous disk I/O. - const trusted = resolveOriginTrust(origin, req.headers.get("x-rt-token"), apiToken); + const trusted = resolveOriginTrust(origin, req.headers.get("x-rt-token"), apiToken) + || isTokenPreflight(req.method, req.headers.get("access-control-request-headers")); const corsHeaders = buildCorsHeaders(origin, trusted); if (req.method === "OPTIONS") { From ab211194372eacab5fa3daf9f3b5894a0a28ab33 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 23:09:29 -0500 Subject: [PATCH 095/106] boot-reconcile: treat SIGTERM ESRCH race as benign eviction, not boot failure --- lib/daemon/__tests__/boot-reconcile.test.ts | 22 +++++++++++++++++++++ lib/daemon/boot-reconcile.ts | 9 ++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/daemon/__tests__/boot-reconcile.test.ts b/lib/daemon/__tests__/boot-reconcile.test.ts index 129a2d85..30ff990a 100644 --- a/lib/daemon/__tests__/boot-reconcile.test.ts +++ b/lib/daemon/__tests__/boot-reconcile.test.ts @@ -26,3 +26,25 @@ test("evictStaleDaemon waits for the old pid to die, escalating to SIGKILL", asy expect(Date.now() - start).toBeLessThan(5000); child.kill(); }); + +test("evictStaleDaemon does not throw when the pid exits between the liveness check and the SIGTERM send (ESRCH)", async () => { + writeFileSync(DAEMON_PID_PATH, "999999"); + const originalKill = process.kill; + let sigtermSent = false; + (process as any).kill = (pid: number, signal?: string | number) => { + if (pid !== 999999) return originalKill(pid, signal as any); + if (signal === 0) return true; // liveness check still sees it alive + if (signal === "SIGTERM") { + sigtermSent = true; + const err = Object.assign(new Error("kill ESRCH"), { code: "ESRCH" }); + throw err; + } + throw new Error(`unexpected signal in test: ${String(signal)}`); + }; + try { + await expect(evictStaleDaemon(silentLog)).resolves.toBeUndefined(); + } finally { + process.kill = originalKill; + } + expect(sigtermSent).toBe(true); +}); diff --git a/lib/daemon/boot-reconcile.ts b/lib/daemon/boot-reconcile.ts index 4010041c..23aff773 100644 --- a/lib/daemon/boot-reconcile.ts +++ b/lib/daemon/boot-reconcile.ts @@ -44,7 +44,14 @@ export async function evictStaleDaemon(log: Logger): Promise { const previousPid = readDaemonPid(); if (!previousPid || previousPid === process.pid) return; if (!isAlive(previousPid)) return; - process.kill(previousPid, "SIGTERM"); + try { + process.kill(previousPid, "SIGTERM"); + } catch (err) { + // Exited between the liveness probe and this send (ESRCH), or is not + // ours (EPERM). Nothing left to evict either way. + log.warn({ err, pid: previousPid }, "stale daemon SIGTERM skipped"); + return; + } log.warn({ pid: previousPid }, "evicted stale daemon process"); if (await waitForDeath(previousPid, 2500)) return; try { From 2c4742955513556d80ff883f0197acbe8ba5a856 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 23:10:49 -0500 Subject: [PATCH 096/106] cache-refresh: bound stalled-cycle accumulation with an orphan cap --- .../__tests__/cache-refresh-coalesce.test.ts | 16 ++++++++ lib/daemon/cache-refresh.ts | 37 +++++++++++++++++-- 2 files changed, 50 insertions(+), 3 deletions(-) diff --git a/lib/daemon/__tests__/cache-refresh-coalesce.test.ts b/lib/daemon/__tests__/cache-refresh-coalesce.test.ts index fa3c3eb0..12191e6e 100644 --- a/lib/daemon/__tests__/cache-refresh-coalesce.test.ts +++ b/lib/daemon/__tests__/cache-refresh-coalesce.test.ts @@ -43,3 +43,19 @@ test("a fast success does not fire onTimeout after the deadline elapses", async await new Promise((r) => setTimeout(r, 150)); // past the deadline expect(timedOut).toBe(0); // the deadline timer must have been cleared, not just outraced }); + +test("refuses to admit a replacement cycle once maxOrphanCycles stalled runs are already stuck in the background", async () => { + let starts = 0; + let refused = 0; + const coalesce = makeCoalescer( + () => { starts++; return new Promise(() => {}); }, // never resolves — every cycle orphans + 10, + () => {}, + { maxOrphanCycles: 2, onRefused: () => { refused++; } }, + ); + await coalesce(); // orphans (1) + await coalesce(); // orphans (2) + await coalesce(); // cap hit — refused, no new socket work started + expect(starts).toBe(2); + expect(refused).toBe(1); +}); diff --git a/lib/daemon/cache-refresh.ts b/lib/daemon/cache-refresh.ts index 93c9f7af..8c1e5d03 100644 --- a/lib/daemon/cache-refresh.ts +++ b/lib/daemon/cache-refresh.ts @@ -50,28 +50,58 @@ const BRANCH_CACHE_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; /** Below the 5-min tick, above the slowest legitimate deep sync. */ const REFRESH_CYCLE_DEADLINE_MS = 4 * 60 * 1000; +export interface CoalescerOptions { + /** + * Cap on cycles still running in the background after their own deadline + * fired ("orphans"). `run` is never cancelled at the deadline (no + * AbortSignal threads through the GitLab/git calls it makes), so without a + * cap, repeated stalls could pile up half-open sockets and pending work + * without bound. Once the cap is hit, a new cycle is refused (not + * started) until an orphan settles. + */ + maxOrphanCycles?: number; + /** Called when a cycle is refused because the orphan cap is at capacity. */ + onRefused?: () => void; +} + /** * Coalesce concurrent callers onto one in-flight run, but clear the latch after * `deadlineMs` even if the run never settles, so a wedged cycle (a half-open * GitLab socket that never rejects) cannot pin the latch forever. The wedged * run's frame still leaks until the OS reaps the socket; this only frees the - * next tick. + * next tick. `maxOrphanCycles` bounds how many such wedged runs may be alive + * at once (see CoalescerOptions) — the cap is the mitigation; it does not + * cancel the wedged runs themselves. */ export function makeCoalescer( run: () => Promise, deadlineMs: number, onTimeout: () => void, + { maxOrphanCycles = 2, onRefused = () => {} }: CoalescerOptions = {}, ): () => Promise { let inFlight: Promise | null = null; + let orphanCycles = 0; return () => { if (inFlight) return inFlight; - const impl = run().catch(() => {}); // a rejected cycle still clears the latch + if (orphanCycles >= maxOrphanCycles) { + onRefused(); + return Promise.resolve(); + } + let timedOut = false; + const impl = run() + .catch(() => {}) // a rejected cycle still clears the latch + .finally(() => { if (timedOut) orphanCycles--; }); // Promise.race never cancels the losing branch, so the deadline timer must be // captured and cleared on every settle path or a fast success still fires // onTimeout deadlineMs later, misreported as a wedge. let deadlineTimer: ReturnType; const deadline = new Promise((resolve) => { - deadlineTimer = setTimeout(() => { onTimeout(); resolve(); }, deadlineMs); + deadlineTimer = setTimeout(() => { + timedOut = true; + orphanCycles++; + onTimeout(); + resolve(); + }, deadlineMs); }); const guarded = Promise.race([impl, deadline]).finally(() => { clearTimeout(deadlineTimer); @@ -89,6 +119,7 @@ export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise log.warn("cache refresh timed out; cleared in-flight latch for next tick"), + { onRefused: () => log.warn("cache refresh skipped; too many stalled cycles already running") }, ); async function refreshCacheImpl(): Promise { From f44f0b40962cdb97d3a90e1e530fab874ea235cc Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 23:11:56 -0500 Subject: [PATCH 097/106] freshness: invalidate cached provider when gitlabToken is removed, not just rotated --- .../__tests__/freshness-provider-rotation.test.ts | 10 ++++++++++ lib/daemon/freshness.ts | 2 +- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/lib/daemon/__tests__/freshness-provider-rotation.test.ts b/lib/daemon/__tests__/freshness-provider-rotation.test.ts index 8c5f59c3..bcda8181 100644 --- a/lib/daemon/__tests__/freshness-provider-rotation.test.ts +++ b/lib/daemon/__tests__/freshness-provider-rotation.test.ts @@ -22,6 +22,16 @@ test("getRepoContext drops a stale-token provider before serving the cached one expect(tokenCheckMatch!.index!).toBeLessThan(fastPathIndex); }); +test("getRepoContext invalidates the cached provider when gitlabToken is removed, not just rotated (C5)", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + // The buggy gate: invalidation only ran when a NEW token was present, so a + // caller whose secrets lost gitlabToken entirely kept getting served the + // stale authenticated provider. It must be gone. + expect(src).not.toMatch(/if \(currentSecrets\.gitlabToken && cachedForToken\.token !== currentSecrets\.gitlabToken\)/); + // The fixed condition must still invalidate on a plain mismatch, absent value included. + expect(src).toMatch(/if \(cachedForToken\.token !== currentSecrets\.gitlabToken\)/); +}); + test("reconcileFreshnessImpl drops a stale-token watch before skipping already-watched repos (S048/S049 fix 2)", () => { const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); const staleWatchDrop = src.match(/existing\.token !== secrets\.gitlabToken\)\s*stopWatch\(repoName\);/); diff --git a/lib/daemon/freshness.ts b/lib/daemon/freshness.ts index 91ce9f6e..8b4d0956 100644 --- a/lib/daemon/freshness.ts +++ b/lib/daemon/freshness.ts @@ -307,7 +307,7 @@ export async function getRepoContext( const cachedForToken = providers.get(repoName); if (cachedForToken) { const currentSecrets = await loadSecrets(); - if (currentSecrets.gitlabToken && cachedForToken.token !== currentSecrets.gitlabToken) { + if (cachedForToken.token !== currentSecrets.gitlabToken) { stopWatch(repoName); userIdResolved = false; providers.delete(repoName); From 855e872a22c264ded8ffe62600407ad8ce9421c6 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 23:13:51 -0500 Subject: [PATCH 098/106] chat handlers: thread ctx.log through instead of a module-private lazyChildLogger --- lib/daemon/__tests__/chat-handlers.test.ts | 18 ++++++++++++++++++ lib/daemon/command-router.ts | 2 +- lib/daemon/handlers/chat.ts | 12 +++++++++--- 3 files changed, 28 insertions(+), 4 deletions(-) diff --git a/lib/daemon/__tests__/chat-handlers.test.ts b/lib/daemon/__tests__/chat-handlers.test.ts index f1941e82..8b0de5f9 100644 --- a/lib/daemon/__tests__/chat-handlers.test.ts +++ b/lib/daemon/__tests__/chat-handlers.test.ts @@ -1051,3 +1051,21 @@ test("chat:dm-open refuses a reclaimed sender the same way chat:dm does", async const res = await h["chat:dm-open"]({ from: "a", to: "b", sessionId: "s1" }); expect(res.ok).toBe(false); }); + +test("chat:post warns through the injected logger (ctx.log), not a module-private lazyChildLogger (C6)", async () => { + const db = openStateDb(join(tmpdir(), `chat-h-log-${process.pid}-${n++}.db`)); + const warnCalls: unknown[] = []; + const log = { + info: () => {}, debug: () => {}, error: () => {}, + warn: (...args: unknown[]) => { warnCalls.push(args); }, + } as any; + const h = createChatHandlers({ + db, + emitEvent: () => { throw new Error("emit boom"); }, // postAndNotify's own warn path + log, + }); + await h["chat:join"]({ room: "r", handle: "a" }); + const res = await h["chat:post"]({ room: "r", handle: "a", body: "hi" }); + expect(res.ok).toBe(true); + expect(warnCalls.length).toBeGreaterThan(0); +}); diff --git a/lib/daemon/command-router.ts b/lib/daemon/command-router.ts index a5b10d98..c7e73410 100644 --- a/lib/daemon/command-router.ts +++ b/lib/daemon/command-router.ts @@ -70,7 +70,7 @@ export function buildRoutedHandlers(opts: { }; // createChatHandlers also exposes `db` (its test-isolation seam); dropped // here so it never lands as a bogus "db" entry in the command map below. - const { db: _chatDb, ...chatHandlers } = createChatHandlers({ db: opts.stateDb, emitEvent, repoIndex: ctx.repoIndex }); + const { db: _chatDb, ...chatHandlers } = createChatHandlers({ db: opts.stateDb, emitEvent, repoIndex: ctx.repoIndex, log: ctx.log }); const { db: _paneDb, ...paneHandlers } = createPaneHandlers({ db: opts.stateDb, repoIndex: ctx.repoIndex }); // Same seam as chatHandlers above: createAgentHandlers exposes `db` for // test isolation only. diff --git a/lib/daemon/handlers/chat.ts b/lib/daemon/handlers/chat.ts index 42c17ac1..ca6b6c50 100644 --- a/lib/daemon/handlers/chat.ts +++ b/lib/daemon/handlers/chat.ts @@ -5,6 +5,7 @@ */ import type { Database } from "bun:sqlite"; +import type { Logger } from "pino"; import { isValidChatName, joinRoom, @@ -56,7 +57,8 @@ import type { CommandResult, TypedHandlers } from "./types.ts"; export type InboxDeps = { resolve: typeof resolveInbox; deliver: typeof deliverToInbox }; const defaultInboxDeps: InboxDeps = { resolve: resolveInbox, deliver: deliverToInbox }; -const log = lazyChildLogger("chat"); +// Fallback only: real wiring threads ctx.log in from command-router.ts. +const defaultLog = lazyChildLogger("chat"); const CHAT_COMMANDS = [ "chat:join", @@ -378,6 +380,7 @@ function postAndNotify( inboxDeps: InboxDeps, herdr: typeof herdrRequest, deliveryChains: Map>, + log: Logger, ): { id: number; recipients: string[] } | undefined { const { room, handle, body, mentions } = args; const posted = postMessage({ room, handle, body, mentions }, db); @@ -444,8 +447,11 @@ export function createChatHandlers(opts: { /** `findPaneSessionRetrying`'s wall-clock budget/poll; overridable so a test whose fake herdr never resolves does not have to wait out the real production budget. */ paneSessionBudgetMs?: number; paneSessionPollMs?: number; + /** Request logger; wired from ctx.log by command-router.ts. */ + log?: Logger; }): Pick & { db: Database } { const { db, emitEvent } = opts; + const log = opts.log ?? defaultLog; const herdr = opts.herdr ?? herdrRequest; const inboxDeps = opts.inboxDeps ?? defaultInboxDeps; const registryDeps = opts.registryDeps; @@ -495,7 +501,7 @@ export function createChatHandlers(opts: { const nearby = closestRoomNames(room, handle, db); return { ok: false, error: `unknown room "${room}"${nearby.length ? ` — did you mean: ${nearby.join(", ")}` : ""}` }; } - const posted = postAndNotify(db, emitEvent, { room, handle, body, mentions }, inboxDeps, herdr, deliveryChains); + const posted = postAndNotify(db, emitEvent, { room, handle, body, mentions }, inboxDeps, herdr, deliveryChains, log); if (!posted) return { ok: false, error: "chat: post failed (retry budget exhausted)" }; return { ok: true, data: posted }; }, @@ -710,7 +716,7 @@ export function createChatHandlers(opts: { // Recipient travels in `mentions`, not the body, so the transcript // shows the text as typed and the desk still notifies when `to` is // the human. - const posted = postAndNotify(db, emitEvent, { room, handle: from, body, mentions: [to] }, inboxDeps, herdr, deliveryChains); + const posted = postAndNotify(db, emitEvent, { room, handle: from, body, mentions: [to] }, inboxDeps, herdr, deliveryChains, log); if (!posted) return { ok: false, error: "chat: dm failed (retry budget exhausted)" }; return { ok: true, data: { room, id: posted.id, recipients: posted.recipients } }; }, From aa82c3f3c567ad6558668b289286b37bf6c9c0e9 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 23:15:27 -0500 Subject: [PATCH 099/106] age-key: settle the seam.run deadline independently of proc.exited, escalate to SIGKILL --- lib/home/__tests__/age-key.test.ts | 11 ++++++++ lib/home/age-key.ts | 42 +++++++++++++++++++++--------- 2 files changed, 41 insertions(+), 12 deletions(-) diff --git a/lib/home/__tests__/age-key.test.ts b/lib/home/__tests__/age-key.test.ts index bc6ee48e..7615dab4 100644 --- a/lib/home/__tests__/age-key.test.ts +++ b/lib/home/__tests__/age-key.test.ts @@ -412,4 +412,15 @@ describe("createRealAgeKeySeam timeout (S070)", () => { expect(res.code).toBe(0); expect(res.stdout.trim()).toBe("hi"); }); + + test("a child that ignores SIGTERM still rejects promptly with AgeKeyTimeoutError (C7: the deadline settles independently of proc.exited)", async () => { + const seam = createRealAgeKeySeam(); + const start = Date.now(); + await expect( + seam.run(["bash", "-c", "trap '' TERM; sleep 30"], { timeoutMs: 50 }), + ).rejects.toThrow(AgeKeyTimeoutError); + // Must settle on the timeout deadline (plus the SIGKILL escalation grace), + // not wait out proc.exited for a child that never dies from SIGTERM alone. + expect(Date.now() - start).toBeLessThan(4000); + }); }); diff --git a/lib/home/age-key.ts b/lib/home/age-key.ts index 5629ce6d..fd1aa0fa 100644 --- a/lib/home/age-key.ts +++ b/lib/home/age-key.ts @@ -321,26 +321,44 @@ function createRawAgeKeySeam(): AgeKeySeam { proc.stdin.end(); const timeoutMs = opts?.timeoutMs ?? DEFAULT_AGE_KEY_TIMEOUT_MS; - let timedOut = false; - const timer = setTimeout(() => { - timedOut = true; - try { proc.kill(); } catch { /* already exited */ } - }, timeoutMs); - try { + // A child that ignores SIGTERM (or a locked-keychain dialog that never + // closes) would otherwise keep `proc.exited` pending forever, so the + // deadline is raced against the read independently rather than + // discovered only after Promise.all settles — mirrors lib/subprocess.ts. + const captured: Promise = (async () => { const [stdout, stderr, code] = await Promise.all([ new Response(proc.stdout).text(), new Response(proc.stderr).text(), proc.exited, ]); - if (timedOut) { - throw new AgeKeyTimeoutError( - `${redactArgv(cmd).join(" ")}: did not exit within ${timeoutMs}ms (keychain prompt pending?)`, - ); - } return { code, stdout, stderr }; + })(); + + let killTimer: ReturnType | undefined; + let deadlineTimer!: ReturnType; + const deadline: Promise = new Promise((_, reject) => { + deadlineTimer = setTimeout(() => { + try { proc.kill("SIGTERM"); } catch { /* already exited */ } + // SIGTERM alone is not guaranteed (a trapped or hung child can + // ignore it); escalate to SIGKILL after a short grace, unref'd so + // it never holds this process open past the caller's own use of it. + killTimer = setTimeout(() => { + try { proc.kill("SIGKILL"); } catch { /* already exited */ } + }, 2000); + killTimer.unref?.(); + reject(new AgeKeyTimeoutError( + `${redactArgv(cmd).join(" ")}: did not exit within ${timeoutMs}ms (keychain prompt pending?)`, + )); + }, timeoutMs); + }); + + try { + return await Promise.race([captured, deadline]); } finally { - clearTimeout(timer); + clearTimeout(deadlineTimer); + // killTimer intentionally NOT cleared here: on the timeout path it + // must survive to fire SIGKILL against a child that ignored SIGTERM. } }, }; From 8693c268beabcfb3246e460ebe619b060c43c524 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 23:16:48 -0500 Subject: [PATCH 100/106] api-server-bind.test: restore RT_API_PORT and unset rt.apiPort in afterEach --- lib/daemon/__tests__/api-server-bind.test.ts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/lib/daemon/__tests__/api-server-bind.test.ts b/lib/daemon/__tests__/api-server-bind.test.ts index f13ed6bf..d8ff5942 100644 --- a/lib/daemon/__tests__/api-server-bind.test.ts +++ b/lib/daemon/__tests__/api-server-bind.test.ts @@ -2,7 +2,8 @@ import { describe, test, expect, afterEach } from "bun:test"; import type { Server } from "bun"; import { bindApiServerWithRetry, BIND_RETRY_ATTEMPTS, BIND_RETRY_DELAY_MS, startApiServer, type BindRetryDeps } from "../api-server.ts"; import { ApiPortInUseError } from "../api-server.ts"; -import { setSetting } from "../../settings/write.ts"; +import { setSetting, unsetSetting } from "../../settings/write.ts"; +import { getSetting } from "../../settings/resolve.ts"; function eaddrinuse(): Error { return Object.assign(new Error("EADDRINUSE"), { code: "EADDRINUSE" }); @@ -130,14 +131,21 @@ describe("bindApiServerWithRetry — exhausted retries (S043)", () => { describe("startApiServer — binds via resolveApiPort() (S043 caller-side wiring)", () => { let server: Server | undefined; + let prevEnv: string | undefined; afterEach(() => { server?.stop(true); server = undefined; + // Runs even when an assertion above threw — restore both regardless of + // pass/fail, and regardless of test HOME being shared across the whole + // `bun test` run (test-setup.ts preloads it once, not per file). + if (prevEnv !== undefined) process.env.RT_API_PORT = prevEnv; + else delete process.env.RT_API_PORT; + unsetSetting("rt.apiPort", "user"); }); test("binds to the rt.apiPort setting value, not the hardcoded 9401 default", async () => { - const prevEnv = process.env.RT_API_PORT; + prevEnv = process.env.RT_API_PORT; delete process.env.RT_API_PORT; // Measure a free port rather than hardcoding one, then release it @@ -152,7 +160,11 @@ describe("startApiServer — binds via resolveApiPort() (S043 caller-side wiring server = await startApiServer({ handleCommand: async () => ({ ok: true }), log }); expect(server.port).toBe(port); + }); - if (prevEnv !== undefined) process.env.RT_API_PORT = prevEnv; + test("the rt.apiPort setting from the previous test does not leak into later tests (C8)", () => { + // 9401 is the registry default — proof the "user" scope value was + // actually removed, not just that some value happens to be present. + expect(getSetting("rt.apiPort").value).toBe(9401); }); }); From c4d43b28afc71cba8ec2f2ae523336a7860b90ab Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 23:18:00 -0500 Subject: [PATCH 101/106] api-server: build advertised docs/websocket URLs from the resolved bind port --- .../__tests__/api-server-cors-ws.test.ts | 30 ++++++++++++++++++- lib/daemon/api-server.ts | 14 +++++---- 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/lib/daemon/__tests__/api-server-cors-ws.test.ts b/lib/daemon/__tests__/api-server-cors-ws.test.ts index 39f2df0c..f7f6a0a1 100644 --- a/lib/daemon/__tests__/api-server-cors-ws.test.ts +++ b/lib/daemon/__tests__/api-server-cors-ws.test.ts @@ -2,7 +2,7 @@ import { describe, test, expect, afterEach } from "bun:test"; import type { Server } from "bun"; import { buildCorsHeaders, startApiServer } from "../api-server.ts"; import { getApiToken } from "../api-auth.ts"; -import { setSetting } from "../../settings/write.ts"; +import { setSetting, unsetSetting } from "../../settings/write.ts"; describe("buildCorsHeaders", () => { test("no Origin header: no Access-Control-Allow-Origin is set (non-browser request, CORS is irrelevant)", () => { @@ -34,6 +34,7 @@ describe("token-authenticated browser preflight (S-C1: off-allowlist Origin, X-R afterEach(() => { server?.stop(true); server = undefined; + unsetSetting("rt.apiPort", "user"); }); test("an OPTIONS preflight requesting X-RT-Token from an off-allowlist Origin still gets Access-Control-Allow-Origin, so the browser proceeds to the real token-bearing request", async () => { @@ -66,3 +67,30 @@ describe("token-authenticated browser preflight (S-C1: off-allowlist Origin, X-R expect(actual.headers.get("access-control-allow-origin")).toBe(origin); }); }); + +describe("advertised URLs use the resolved bind port, not the compile-time default (S9-a)", () => { + let server: Server | undefined; + + afterEach(() => { + server?.stop(true); + server = undefined; + unsetSetting("rt.apiPort", "user"); + }); + + test("GET / and a 404 both report the actually-bound port when it differs from 9401", async () => { + const probe = Bun.serve({ port: 0, fetch: () => new Response() }); + const port = probe.port; + probe.stop(true); + expect(port).not.toBe(9401); + setSetting("rt.apiPort", port, "user"); + const log = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as any; + server = await startApiServer({ handleCommand: async () => ({ ok: true }), log }); + + const root = await (await fetch(`http://127.0.0.1:${port}/`)).json(); + expect(root.docs).toBe(`http://localhost:${port}/`); + expect(root.websocket).toBe(`ws://localhost:${port}/ws`); + + const notFound = await (await fetch(`http://127.0.0.1:${port}/nope`)).json(); + expect(notFound.docs).toBe(`http://localhost:${port}/`); + }); +}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index e17c3140..073bf342 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -15,11 +15,12 @@ import { getAggregatedConnection } from "./freshness.ts"; import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; import { runCapture } from "../subprocess.ts"; -const API_INDEX = { +function buildApiIndex(port: number) { + return { name: "rt daemon", version: "1.0.0", - docs: `http://localhost:${API_PORT}/`, - websocket: `ws://localhost:${API_PORT}/ws`, + docs: `http://localhost:${port}/`, + websocket: `ws://localhost:${port}/ws`, endpoints: [ { method: "GET", path: "/api/status", description: "Daemon health, uptime, memory, cache stats" }, { method: "GET", path: "/api/ports", description: "Listening ports grouped by repo/worktree" }, @@ -50,7 +51,8 @@ const API_INDEX = { header: "X-RT-Token", description: "Required on mutating routes (shutdown, sdm reconnect, events emit) and /api/secrets. Token at ~/.mattstack/rt/api-token.", }, -}; + }; +} const REST_ROUTES: Record = { "/api/status": { cmd: "tray:status", method: "GET" }, @@ -373,7 +375,7 @@ export async function startApiServer(deps: ApiServerDeps): Promise> try { // Self-describing root if (url.pathname === "/" || url.pathname === "") { - return Response.json(API_INDEX, { headers: corsHeaders }); + return Response.json(buildApiIndex(port), { headers: corsHeaders }); } // Single branch lookup: /api/cache/:branch @@ -422,7 +424,7 @@ export async function startApiServer(deps: ApiServerDeps): Promise> // Static routes const route = REST_ROUTES[url.pathname]; if (!route) { - return Response.json({ ok: false, error: "not found", docs: `http://localhost:${API_PORT}/` }, { status: 404, headers: corsHeaders }); + return Response.json({ ok: false, error: "not found", docs: `http://localhost:${port}/` }, { status: 404, headers: corsHeaders }); } if (req.method !== route.method && req.method !== "OPTIONS") { From 65017f2c223f1f55c5444d5ea033f139f3c27c62 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 23:18:35 -0500 Subject: [PATCH 102/106] agent-herdr: merge the supplied env into the herdr child, not bare process.env --- lib/__tests__/agent-herdr.test.ts | 14 ++++++++++++++ lib/agent-herdr.ts | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/lib/__tests__/agent-herdr.test.ts b/lib/__tests__/agent-herdr.test.ts index a8bd035c..60b0f5f1 100644 --- a/lib/__tests__/agent-herdr.test.ts +++ b/lib/__tests__/agent-herdr.test.ts @@ -1,4 +1,7 @@ import { expect, test } from "bun:test"; +import { mkdtempSync, writeFileSync, chmodSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; import { defaultHerdrRunner, herdrAgentWait, launchInWorkspace, resolveHerdrBin, type HerdrRunner } from "../agent-herdr.ts"; function scripted(responses: Record) { @@ -110,3 +113,14 @@ test("defaultHerdrRunner throws a clear error when the resolved bin does not exi const runner = defaultHerdrRunner({ HERDR_BIN: "/nonexistent/herdr", HOME: "/home/x" }); await expect(runner(["workspace", "list"])).rejects.toThrow(/herdr not found/); }); + +test("defaultHerdrRunner passes the supplied env through to the child (C9: it must not fall back to bare process.env)", async () => { + const dir = mkdtempSync(join(tmpdir(), "agent-herdr-env-")); + const fakeHerdr = join(dir, "herdr"); + writeFileSync(fakeHerdr, "#!/bin/sh\necho \"$SENTINEL_VAR\"\n"); + chmodSync(fakeHerdr, 0o755); + + const runner = defaultHerdrRunner({ HERDR_BIN: fakeHerdr, HOME: dir, SENTINEL_VAR: "from-supplied-env" }); + const result = await runner(["workspace", "list"]); + expect(result.stdout.trim()).toBe("from-supplied-env"); +}); diff --git a/lib/agent-herdr.ts b/lib/agent-herdr.ts index 34761dec..9cb964ba 100644 --- a/lib/agent-herdr.ts +++ b/lib/agent-herdr.ts @@ -53,7 +53,7 @@ export function defaultHerdrRunner(env: NodeJS.ProcessEnv = process.env): HerdrR const r = await runCapture([bin, ...args], { timeoutMs: 15_000, stderr: "pipe", - env: { ...process.env, HERDR_SOCKET_PATH: socket }, + env: { ...env, HERDR_SOCKET_PATH: socket }, }); return { stdout: r.stdout || r.stderr, exitCode: r.exitCode }; }; From 2511977a8755198aabdf5041ddb3aecaf8187b1d Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 23:19:15 -0500 Subject: [PATCH 103/106] daemon: set state-db phase before opening the store; log boot flush failures No unit-test seam exists for lib/daemon.ts's runDaemon() (top-level await, side effects at module load; only e2e boots it), so these two ordering/ logging fixes ship without a new automated test, consistent with every other line in this function. --- lib/daemon.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/daemon.ts b/lib/daemon.ts index 43d96f93..e12db4fb 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -469,8 +469,8 @@ async function runDaemon(): Promise { // (spec "Migration & contention"): the one long transaction is the // legacy-JSON import, and it must never land inside the event loop. If a // CLI process is mid-import right now, we block here, in startup. - openBranchCacheStore(); setPhase("state-db"); + openBranchCacheStore(); recordBootAttempt(); log.info({ count: Object.keys(cache.entries).length }, "branch cache loaded from state.db"); @@ -584,7 +584,7 @@ async function runDaemon(): Promise { } catch (err) { log.fatal({ err }, "daemon boot failed"); recordBootFailure(currentPhase, String(err)); - try { loggerHandle.flush?.(); } catch { /* */ } + try { loggerHandle.flush?.(); } catch (flushErr) { log.warn({ err: flushErr }, "daemon boot log flush failed"); } process.exit(1); } } From b3a14f4fb382791ae6f584f1964305d8ca6dbea7 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 23:20:40 -0500 Subject: [PATCH 104/106] presence-store: fix prune SQL leg leak on signed-out rows; use immediate lock in reserveAgentHandle --- lib/state/__tests__/presence-store.test.ts | 19 +++++++++++++++++++ lib/state/presence-store.ts | 6 ++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/lib/state/__tests__/presence-store.test.ts b/lib/state/__tests__/presence-store.test.ts index a91f3a58..01146a47 100644 --- a/lib/state/__tests__/presence-store.test.ts +++ b/lib/state/__tests__/presence-store.test.ts @@ -196,6 +196,15 @@ test("S073: signIn's read-then-write transaction uses .immediate() (BEGIN IMMEDI expect(src.indexOf("return run.immediate();", runIndex)).toBeGreaterThan(runIndex); }); +test("C9: reserveAgentHandle's read-then-write transaction also uses .immediate(), same reason as signIn's S073 fix", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "presence-store.ts"), "utf8"); + const fnIndex = src.indexOf("export function reserveAgentHandle("); + expect(fnIndex).toBeGreaterThan(-1); + const runIndex = src.indexOf("const run = db.transaction(", fnIndex); + expect(runIndex).toBeGreaterThan(fnIndex); + expect(src.indexOf("return run.immediate();", runIndex)).toBeGreaterThan(runIndex); +}); + test("assertSessionSignedIn throws when the session's row is gone", () => { const db = fresh(); expect(() => assertSessionSignedIn("ghost", db)).toThrow(/handle reclaimed/); @@ -238,6 +247,16 @@ test("prune: a never-signed-out row past 24h with no live binding is deleted", ( expect(db.query("SELECT COUNT(*) c FROM chat_presence").get()).toMatchObject({ c: 0 }); }); +test("prune: a signed-out row within its 24h offline window survives even when last_seen_at is stale (C9: PRUNABLE_SQL must not let a signed-out row's last_seen_at leg bypass its own signed_out_at age bound)", () => { + const db = fresh(); + signIn({ sessionId: "s1", baseHandle: "x", now }, db); // last_seen_at pinned at `now`, no touches + signOut("s1", now + 30 * HOUR, db); // signed out well after last_seen_at went stale + // 1h after signing out: signed_out_at leg is nowhere near its 24h bound, + // but last_seen_at (still `now`, 31h stale) trips the OTHER leg. + expect(prunePresence(now + 31 * HOUR, db)).toBe(0); + expect(db.query("SELECT COUNT(*) c FROM chat_presence").get()).toMatchObject({ c: 1 }); +}); + test("touchLastSeen refreshes only last_seen_at -- the sole remaining route to it now that chat:pulse is gone", () => { const db = fresh(); signIn({ sessionId: "s1", baseHandle: "x", now }, db); diff --git a/lib/state/presence-store.ts b/lib/state/presence-store.ts index 615fdfa9..773844d4 100644 --- a/lib/state/presence-store.ts +++ b/lib/state/presence-store.ts @@ -111,7 +111,7 @@ function isReclaimable(row: PresenceRawRow, sessionStaleCutoff: number, deps: Re * SQL only narrows to "old enough to be worth a registry check", never the * final delete decision. Bind params in order: dayAgo, dayAgo (both legs). */ -const PRUNABLE_SQL = `(signed_out_at IS NOT NULL AND signed_out_at < ?) OR last_seen_at < ?`; +const PRUNABLE_SQL = `(signed_out_at IS NOT NULL AND signed_out_at < ?) OR (signed_out_at IS NULL AND last_seen_at < ?)`; const SELECT_PRESENCE_BY_HANDLE_SQL = `SELECT ${PRESENCE_COLUMNS} FROM chat_presence WHERE handle = ?;`; const SELECT_PRESENCE_BY_SESSION_SQL = `SELECT ${PRESENCE_COLUMNS} FROM chat_presence WHERE session_id = ?;`; @@ -359,7 +359,9 @@ export function reserveAgentHandle(db: Database = getStateDb(), now: number = Da recordPoolNameUse(name, now, db); return name; }); - return run(); + // BEGIN IMMEDIATE: read-then-write must lock up front or SQLITE_BUSY_SNAPSHOT + // bypasses busy_timeout (same reason as signIn's S073 fix above). + return run.immediate(); } export function signOut(sessionId: string, now: number = Date.now(), db: Database = getStateDb()): void { From 5af1f1ca40f7926d2dcda2f8647fdf37120ed3c5 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 23:26:30 -0500 Subject: [PATCH 105/106] api-server-cors-ws.test: fix tsc unknown-type errors on fetch().json() results --- lib/daemon/__tests__/api-server-cors-ws.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/daemon/__tests__/api-server-cors-ws.test.ts b/lib/daemon/__tests__/api-server-cors-ws.test.ts index f7f6a0a1..500e4546 100644 --- a/lib/daemon/__tests__/api-server-cors-ws.test.ts +++ b/lib/daemon/__tests__/api-server-cors-ws.test.ts @@ -86,11 +86,11 @@ describe("advertised URLs use the resolved bind port, not the compile-time defau const log = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as any; server = await startApiServer({ handleCommand: async () => ({ ok: true }), log }); - const root = await (await fetch(`http://127.0.0.1:${port}/`)).json(); + const root = await (await fetch(`http://127.0.0.1:${port}/`)).json() as { docs: string; websocket: string }; expect(root.docs).toBe(`http://localhost:${port}/`); expect(root.websocket).toBe(`ws://localhost:${port}/ws`); - const notFound = await (await fetch(`http://127.0.0.1:${port}/nope`)).json(); + const notFound = await (await fetch(`http://127.0.0.1:${port}/nope`)).json() as { docs: string }; expect(notFound.docs).toBe(`http://localhost:${port}/`); }); }); From 6f46b19b052852069dee23b3edad54443d566c6d Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 23:45:29 -0500 Subject: [PATCH 106/106] docs: regenerate reference for pane:focus --- website/docs/reference/pane/focus.mdx | 27 +++++++++++++++++++++++++++ website/docs/reference/pane/index.mdx | 1 + 2 files changed, 28 insertions(+) create mode 100644 website/docs/reference/pane/focus.mdx diff --git a/website/docs/reference/pane/focus.mdx b/website/docs/reference/pane/focus.mdx new file mode 100644 index 00000000..9f66c812 --- /dev/null +++ b/website/docs/reference/pane/focus.mdx @@ -0,0 +1,27 @@ +--- +title: rt pane focus +sidebar_label: focus +--- + +# rt pane focus + +`rt › pane › focus` + +Bring a herdr pane to the front (via the tray: workspace + tab focus and terminal window raise) + +## Usage + +```bash +rt pane focus [flags] +``` + +## Arguments & flags + +| Flag / Arg | Type | Default | Description | +| --- | --- | --- | --- | +| `` | text | | herdr pane id to focus | +| [`--json`](/guides/common-flags) | boolean | `false` | Emit the focus result as JSON instead of the plain line | + +_See code: [commands/pane.ts › paneFocus](https://github.com/m4ttstack/rt/blob/main/commands/pane.ts)_ + +{/* generated by scripts/gen-docs.ts; edit prose in _partials, not here */} \ No newline at end of file diff --git a/website/docs/reference/pane/index.mdx b/website/docs/reference/pane/index.mdx index 920ebecd..65abc8ca 100644 --- a/website/docs/reference/pane/index.mdx +++ b/website/docs/reference/pane/index.mdx @@ -23,6 +23,7 @@ rt pane | [`peek`](peek) | The last lines of a pane's visible screen | | [`spawn`](spawn) | Open a herdr tab in a directory and start claude in it, optionally under a cswap account | | [`send`](send) | Inject text into a pane as if typed and submitted (--text - reads stdin) | +| [`focus`](focus) | Bring a herdr pane to the front (via the tray: workspace + tab focus and terminal window raise) | | [`accounts`](accounts) | cswap accounts with rate-limit headroom, for spawn --account | | [`directories`](directories) | Repos and worktrees rt knows, as suggestions for spawn --cwd |