From bafe1b3aecc17d454e271061c4c403a2f093ba4a Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:29:30 -0500 Subject: [PATCH 001/142] agent-herdr: resolve herdr via PATH probe instead of hardcoded ~/.local/bin (S002) --- lib/__tests__/agent-herdr.test.ts | 22 +++++++++++++++++++++- lib/agent-herdr.ts | 30 ++++++++++++++++++++++++++---- 2 files changed, 47 insertions(+), 5 deletions(-) diff --git a/lib/__tests__/agent-herdr.test.ts b/lib/__tests__/agent-herdr.test.ts index 568fdc19..a8bd035c 100644 --- a/lib/__tests__/agent-herdr.test.ts +++ b/lib/__tests__/agent-herdr.test.ts @@ -1,5 +1,5 @@ import { expect, test } from "bun:test"; -import { herdrAgentWait, launchInWorkspace, type HerdrRunner } from "../agent-herdr.ts"; +import { defaultHerdrRunner, herdrAgentWait, launchInWorkspace, resolveHerdrBin, type HerdrRunner } from "../agent-herdr.ts"; function scripted(responses: Record) { const calls: string[][] = []; @@ -90,3 +90,23 @@ test("herdrAgentWait builds the current verb (agent wait --until)", async () => await herdrAgentWait("wA:p1", ["idle", "done"], 45000, runner); expect(calls[0]).toEqual(["agent", "wait", "wA:p1", "--until", "idle", "--until", "done", "--timeout", "45000"]); }); + +test("resolveHerdrBin prefers HERDR_BIN over everything else", () => { + const bin = resolveHerdrBin({ HERDR_BIN: "/custom/herdr", HOME: "/home/x" }, () => "/opt/homebrew/bin/herdr"); + expect(bin).toBe("/custom/herdr"); +}); + +test("resolveHerdrBin prefers a PATH-resolved herdr over the ~/.local/bin fallback", () => { + const bin = resolveHerdrBin({ HOME: "/home/x" }, () => "/opt/homebrew/bin/herdr"); + expect(bin).toBe("/opt/homebrew/bin/herdr"); +}); + +test("resolveHerdrBin falls back to ~/.local/bin/herdr when PATH resolution fails", () => { + const bin = resolveHerdrBin({ HOME: "/home/x" }, () => null); + expect(bin).toBe("/home/x/.local/bin/herdr"); +}); + +test("defaultHerdrRunner throws a clear error when the resolved bin does not exist", async () => { + const runner = defaultHerdrRunner({ HERDR_BIN: "/nonexistent/herdr", HOME: "/home/x" }); + await expect(runner(["workspace", "list"])).rejects.toThrow(/herdr not found/); +}); diff --git a/lib/agent-herdr.ts b/lib/agent-herdr.ts index ec3dccee..34761dec 100644 --- a/lib/agent-herdr.ts +++ b/lib/agent-herdr.ts @@ -16,6 +16,7 @@ * visible pane, rt verifies the result from real state afterward. */ +import { existsSync } from "fs"; import { homedir } from "os"; import { join } from "path"; import { runCapture } from "./subprocess.ts"; @@ -23,11 +24,32 @@ import { runCapture } from "./subprocess.ts"; export interface HerdrResult { stdout: string; exitCode: number } export type HerdrRunner = (args: string[]) => Promise; -export function defaultHerdrRunner(): HerdrRunner { - const home = process.env.HOME ?? homedir(); - const bin = process.env.HERDR_BIN ?? join(home, ".local", "bin", "herdr"); - const socket = process.env.HERDR_SOCKET_PATH ?? join(home, ".config", "herdr", "herdr.sock"); +/** + * Mirrors lib/cswap.ts's cswapBin(): Bun.which reads process.env.PATH at + * call time, and the daemon overlays the user's full login PATH onto + * process.env.PATH at boot (lib/daemon.ts resolveUserPath), so this resolves + * a brew-installed herdr even though the daemon's start-env PATH does not + * carry it. The ~/.local/bin fallback preserves the vendor-script install. + */ +export function resolveHerdrBin( + env: NodeJS.ProcessEnv = process.env, + which: (cmd: string) => string | null = (cmd) => Bun.which(cmd), +): string { + if (env.HERDR_BIN) return env.HERDR_BIN; + const onPath = which("herdr"); + if (onPath) return onPath; + const home = env.HOME ?? homedir(); + return join(home, ".local", "bin", "herdr"); +} + +export function defaultHerdrRunner(env: NodeJS.ProcessEnv = process.env): HerdrRunner { + const home = env.HOME ?? homedir(); + const bin = resolveHerdrBin(env); + const socket = env.HERDR_SOCKET_PATH ?? join(home, ".config", "herdr", "herdr.sock"); return async (args) => { + if (!existsSync(bin)) { + throw new Error(`herdr not found at ${bin} (install via \`rt setup\` / brew)`); + } const r = await runCapture([bin, ...args], { timeoutMs: 15_000, stderr: "pipe", From fa5f64b2c565626fc11e262fb7b7cee975b04b31 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:31:47 -0500 Subject: [PATCH 002/142] notifier: retry the post-push queue removal so a busy-swallowed delete can't cause a duplicate delivery (S096) --- lib/__tests__/notifier.test.ts | 30 ++++++++++++++++++++++++++++++ lib/notifier.ts | 27 +++++++++++++++++++++++++-- 2 files changed, 55 insertions(+), 2 deletions(-) diff --git a/lib/__tests__/notifier.test.ts b/lib/__tests__/notifier.test.ts index ed044d38..f9e5e103 100644 --- a/lib/__tests__/notifier.test.ts +++ b/lib/__tests__/notifier.test.ts @@ -280,3 +280,33 @@ describe("conflict-free streak", () => { expect(__test__.shouldRearmConflicts(conflicted, snap)).toBe(false); }); }); + +describe("removeFromQueueWithRetry (busy-swallowed post-push removal)", () => { + test("succeeds on the first attempt when the removal actually took", async () => { + let queued = true; + const removeFn = () => { queued = false; }; + const isQueuedFn = () => queued; + const ok = await __test__.removeFromQueueWithRetry("evt-1", 3, 1, removeFn, isQueuedFn); + expect(ok).toBe(true); + }); + + test("retries when the removal is busy-swallowed, then succeeds", async () => { + let queued = true; + let calls = 0; + const removeFn = () => { + calls++; + if (calls >= 3) queued = false; // the first two deletes are swallowed by SQLITE_BUSY + }; + const isQueuedFn = () => queued; + const ok = await __test__.removeFromQueueWithRetry("evt-2", 5, 1, removeFn, isQueuedFn); + expect(ok).toBe(true); + expect(calls).toBe(3); + }); + + test("gives up after the attempt budget and reports failure instead of hanging", async () => { + const removeFn = () => {}; // never actually removes it + const isQueuedFn = () => true; + const ok = await __test__.removeFromQueueWithRetry("evt-3", 3, 1, removeFn, isQueuedFn); + expect(ok).toBe(false); + }); +}); diff --git a/lib/notifier.ts b/lib/notifier.ts index 55db3c32..0c235c87 100644 --- a/lib/notifier.ts +++ b/lib/notifier.ts @@ -214,6 +214,29 @@ export function peekNotifications(): NotificationEvent[] { * Attempt to push a notification event to the tray app via its Unix socket. * Returns true if the push succeeded, false if tray is unavailable. */ +/** + * removeQueuedNotification's own write already retries on SQLITE_BUSY (3 x + * 20ms, lib/state/notifier-store.ts) and gives up silently. When that + * happens after a successful tray push, the row stays queued and the next + * drainNotifications()/peekNotifications() redelivers it as a duplicate. + * Retry the removal at this layer, bounded, before reporting success. + */ +async function removeFromQueueWithRetry( + eventId: string, + attempts = 3, + delayMs = 50, + removeFn: (id: string) => void = removeQueuedNotification, + isQueuedFn: (id: string) => boolean = isNotificationQueued, +): Promise { + for (let i = 0; i < attempts; i++) { + removeFn(eventId); + if (!isQueuedFn(eventId)) return true; + if (i < attempts - 1) await new Promise((r) => setTimeout(r, delayMs)); + } + log.warn({ eventId }, "notification queue removal kept failing after a successful push; it may redeliver on the next drain"); + return false; +} + async function pushToTray(event: NotificationEvent): Promise { if (!existsSync(TRAY_SOCK_PATH)) return false; @@ -227,8 +250,7 @@ async function pushToTray(event: NotificationEvent): Promise { } as any); if (response.ok) { - // Push succeeded — remove from queue - removeQueuedNotification(event.id); + await removeFromQueueWithRetry(event.id); return true; } return false; @@ -878,6 +900,7 @@ export const __test__ = { notifyFallback, firedKey, pruneFiredForEvictedBranches, + removeFromQueueWithRetry, setFallbackNotifier(path: string | null): void { fallbackNotifier = path ?? "osascript"; }, From ce9c8d6ad3c6923e0c5769260be4f76f4bd36f34 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:35:36 -0500 Subject: [PATCH 003/142] cron: pass the daemon's resolved process.env to spawn instead of the launchd-frozen snapshot (S046) --- lib/daemon/__tests__/cron.test.ts | 34 ++++++++++++++++++++++++++++++- lib/daemon/cron.ts | 8 +++++++- 2 files changed, 40 insertions(+), 2 deletions(-) diff --git a/lib/daemon/__tests__/cron.test.ts b/lib/daemon/__tests__/cron.test.ts index 1c535b19..b09ab97b 100644 --- a/lib/daemon/__tests__/cron.test.ts +++ b/lib/daemon/__tests__/cron.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { mkdtempSync, realpathSync, rmSync } from "fs"; +import { mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; import { setSetting } from "../../settings/write.ts"; @@ -121,4 +121,36 @@ describe("startCron", () => { await sleep(40); expect(runs).toHaveLength(0); }); + + // The real (non-overridden) spawn path: Bun.spawn without an explicit + // `env` key gives the child the env snapshot from when THIS bun process + // started, not process.env as mutated at runtime (verified directly: + // omitting `env` leaves a child reading a var reassigned post-startup at + // its ORIGINAL value; `env: { ...process.env }` gives it the live one). + // That is exactly what the daemon's boot-time PATH overlay does to + // process.env.PATH, so this exercises the same shape with a plain var — + // an absolute argv[0] sidesteps unrelated PATH-search mechanics. + test("cron spawns with the live process.env, not the frozen start snapshot", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-cron-env-"))); + const marker = join(dir, "ran.txt"); + const script = join(dir, "marker.sh"); + writeFileSync(script, `#!/bin/sh\necho "$CRON_TEST_VALUE" > "${marker}"\n`, { mode: 0o755 }); + + const varName = "CRON_TEST_VALUE"; + delete process.env[varName]; // absent at this bun process's own startup + process.env[varName] = "live-value"; // set only after startup, like resolveUserPath does to PATH + try { + const cron = startCron( + { triggers: [{ name: "t", event: "tick", run: [script], debounceMs: 5 }] }, + { log }, + ); + cron.onBroadcast("tick", null); + await sleep(500); // real process spawn + exit, not a mocked runCommand + cron.dispose(); + expect(readFileSync(marker, "utf8").trim()).toBe("live-value"); + } finally { + delete process.env[varName]; + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/lib/daemon/cron.ts b/lib/daemon/cron.ts index 3720afc9..ef9cc4ee 100644 --- a/lib/daemon/cron.ts +++ b/lib/daemon/cron.ts @@ -75,7 +75,13 @@ function defaultRunCommand(argv: string[], trigger: CronTrigger, log: CronLog): // so a daemon restart kills an in-flight command. Accepted (spec section // 5): invoked programs must be idempotent one-shot passes, and the next // matching event simply re-runs them. - const proc = Bun.spawn(argv, { stdin: "ignore", stdout: "ignore", stderr: "ignore" }); + // Bun.spawn ignores assignments made to process.env after startup unless + // `env` is passed explicitly (mirrors lib/subprocess.ts's runCapture and + // lib/daemon/handlers/agent.ts's defaultSpawnHeadless) — omitting it here + // strands the PATH the daemon overlays onto process.env at boot + // (lib/daemon.ts resolveUserPath) and leaves any non-absolute argv[0] + // (a "#!/usr/bin/env node" shebang, "pnpm", ...) unresolvable. + const proc = Bun.spawn(argv, { stdin: "ignore", stdout: "ignore", stderr: "ignore", env: { ...process.env } }); log.info(`cron ${trigger.name}: spawned "${argv.join(" ")}" (pid ${proc.pid})`); void proc.exited.then((code) => log.info(`cron ${trigger.name}: exited ${code}`)); } catch (err) { From 52be3d9ea06affb8d134a1c8be843338895af15e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:37:43 -0500 Subject: [PATCH 004/142] agent handler: return ok:false when herdr dedups the tab label instead of a phantom record (S051) --- lib/daemon/__tests__/agent-handlers.test.ts | 29 +++++++++++++++++++++ lib/daemon/handlers/agent.ts | 14 +++++++--- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/lib/daemon/__tests__/agent-handlers.test.ts b/lib/daemon/__tests__/agent-handlers.test.ts index 80643fff..46f70fa6 100644 --- a/lib/daemon/__tests__/agent-handlers.test.ts +++ b/lib/daemon/__tests__/agent-handlers.test.ts @@ -4,6 +4,7 @@ import { join } from "path"; import { openStateDb } from "../../state/index.ts"; import { createAgentHandlers, type HeadlessChild } from "../handlers/agent.ts"; import type { HerdrRunner } from "../../agent-herdr.ts"; +import { repoLabel } from "../../repo-arg.ts"; let n = 0; const REPO = "remote:example.com%2Fa%2Fb"; @@ -61,6 +62,34 @@ test("agent:start herdr rolls back the inserted record when launch fails", async expect(list.data.agents).toHaveLength(0); }); +// Pins S051: a tab-label dedup must never report success with a phantom +// record nothing is listening on (rt agent resume on it would run +// `claude --resume` for a session that never started). +test("agent:start herdr returns ok:false and rolls back when herdr dedups the tab label", async () => { + const label = "!7"; + const focusCalls: string[][] = []; + const runner: HerdrRunner = async (args) => { + focusCalls.push(args); + if (args[0] === "workspace" && args[1] === "list") { + return { stdout: JSON.stringify({ result: { workspaces: [{ workspace_id: "w1", label: repoLabel(REPO) }] } }), exitCode: 0 }; + } + if (args[0] === "tab" && args[1] === "list") { + return { stdout: JSON.stringify({ result: { tabs: [{ tab_id: "w1:t9", label } ] } }), exitCode: 0 }; + } + return { stdout: "{}", exitCode: 0 }; + }; + const h = fresh({ runner }); + const res = await h["agent:start"]({ repo: REPO, cwd: "/tmp/x", prompt: "hi", surface: "herdr", tab: label }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toMatch(/already open/); + expect(focusCalls.some((c) => c[0] === "tab" && c[1] === "focus")).toBe(true); + expect(focusCalls.some((c) => c[0] === "pane" && c[1] === "run")).toBe(false); + const list = await h["agent:list"]({}); + if (!list.ok) throw new Error("unreachable"); + expect(list.data.agents).toHaveLength(0); +}); + // Pins the guard: a no-op insert (standing in for runCriticalWrite giving up // after sustained SQLITE_BUSY) must block the launch, not just the record. test("agent:start refuses to launch when the insert did not persist", async () => { diff --git a/lib/daemon/handlers/agent.ts b/lib/daemon/handlers/agent.ts index 82fd61ff..9c548642 100644 --- a/lib/daemon/handlers/agent.ts +++ b/lib/daemon/handlers/agent.ts @@ -95,11 +95,17 @@ export function createAgentHandlers(opts: { { workspaceLabel, tabLabel, paneCommand: buildPaneCommand(rec.cwd, inv) }, runner, ); - if (!out.focusedExisting) { - rec.paneId = out.paneId; - rec.tabId = out.tabId; - rec.workspaceId = out.workspaceId; + if (out.focusedExisting) { + // A live tab already answers to this label: launchInWorkspace + // focused it and ran nothing. Reporting ok:true here would insert a + // record with a freshly minted sessionId nothing is listening on — + // rt agent resume against it would run `claude --resume` for a + // session that was never started. + return { ok: false, error: `tab "${tabLabel}" already open; focused it` }; } + rec.paneId = out.paneId; + rec.tabId = out.tabId; + rec.workspaceId = out.workspaceId; return { ok: true, data: rec }; } From 94d6218581b36963490711902290dad08d8cd11f Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:39:06 -0500 Subject: [PATCH 005/142] chat handlers: guard emit/notify after the message commit so a throw can't surface as a failed post (S052) --- lib/daemon/__tests__/chat-handlers.test.ts | 34 +++++++++++++++ lib/daemon/handlers/chat.ts | 48 +++++++++++++++------- 2 files changed, 68 insertions(+), 14 deletions(-) diff --git a/lib/daemon/__tests__/chat-handlers.test.ts b/lib/daemon/__tests__/chat-handlers.test.ts index cfa55e66..b54a0774 100644 --- a/lib/daemon/__tests__/chat-handlers.test.ts +++ b/lib/daemon/__tests__/chat-handlers.test.ts @@ -56,6 +56,40 @@ test("chat:post returns the recipients and emits one wake event per recipient", expect(emitted).toEqual(["chat/r/msg", "chat/wake/b"]); }); +test("chat:post still reports success when emitEvent throws after the message is durable", async () => { + const h = freshHandlers(() => { throw new Error("events.db locked"); }); + await h["chat:join"]({ room: "r", handle: "a" }); + const res = await h["chat:post"]({ room: "r", handle: "a", body: "hi" }); + expect(res.ok).toBe(true); + if (!res.ok) throw new Error("unreachable"); + expect(res.data.id).toBeGreaterThan(0); +}); + +test("chat:post continues waking the rest of the recipients when one recipient's emit throws", async () => { + const emitted: string[] = []; + const h = freshHandlers((topic) => { + emitted.push(topic); + if (topic === "chat/wake/a") throw new Error("boom for a"); + return 0; + }); + await h["chat:join"]({ room: "r", handle: "a" }); + await h["chat:join"]({ room: "r", handle: "b" }); + await h["chat:join"]({ room: "r", handle: "c" }); + const res = await h["chat:post"]({ room: "r", handle: "poster", body: "@a @b @c hi" }); + expect(res.ok).toBe(true); + expect(emitted).toContain("chat/wake/a"); + expect(emitted).toContain("chat/wake/b"); + expect(emitted).toContain("chat/wake/c"); +}); + +test("chat:dm still reports success when emitEvent throws after the message is durable", async () => { + const h = freshHandlers(() => { throw new Error("events.db locked"); }); + const res = await h["chat:dm"]({ from: "agent", to: "matt", body: "ping" }); + expect(res.ok).toBe(true); + if (!res.ok) throw new Error("unreachable"); + expect(res.data.id).toBeGreaterThan(0); +}); + test("chat:post rejects an invalid mentions element with a reason rather than storing it", async () => { const h = freshHandlers(); await h["chat:join"]({ room: "r", handle: "a" }); diff --git a/lib/daemon/handlers/chat.ts b/lib/daemon/handlers/chat.ts index f67d14ee..98101e8a 100644 --- a/lib/daemon/handlers/chat.ts +++ b/lib/daemon/handlers/chat.ts @@ -42,9 +42,12 @@ import { chatViewerUrl, readChatViewerUrlSetting } from "../../chat-viewer-url.t import { getSetting } from "../../settings/resolve.ts"; import { herdrRequest, waitTimeout } from "../../herdr/client.ts"; import { herdrError } from "./pane.ts"; +import { lazyChildLogger } from "../../daemon-logger.ts"; import type { Commands } from "../../../packages/rt-client/src/commands.ts"; import type { CommandResult, TypedHandlers } from "./types.ts"; +const log = lazyChildLogger("chat"); + const CHAT_COMMANDS = [ "chat:join", "chat:leave", @@ -94,9 +97,22 @@ function postAndNotify( const { room, handle, body, mentions } = args; const posted = postMessage({ room, handle, body, mentions }, db); if (!posted) return undefined; - emitEvent(`chat/${room}/msg`, { id: posted.id }); + // The row is durable at this point. Every step below is best-effort: a + // throw here (a full disk, an orphan daemon holding an events.db lock) + // must never surface as a failed post — the caller would retry and post + // the message twice — and one recipient's failure must not skip the wake + // for the rest. + try { + emitEvent(`chat/${room}/msg`, { id: posted.id }); + } catch (err) { + log.warn({ err, id: posted.id, room }, "chat: emit for the posted message threw; message is durable, this emit was not"); + } for (const recipient of posted.recipients) { - emitEvent(`chat/wake/${recipient}`, { id: posted.id, room }); + try { + emitEvent(`chat/wake/${recipient}`, { id: posted.id, room }); + } catch (err) { + log.warn({ err, id: posted.id, room, recipient }, "chat: wake emit threw for one recipient; continuing to the rest"); + } } // Independent of chat_members / wake_on: agents create rooms via // join-creates, so the human is typically not a member yet, and a @@ -104,18 +120,22 @@ function postAndNotify( const humanHandle = getSetting("chat.humanHandle").value; const allMentions = mergeMentions(body, mentions); if (humanHandle && allMentions.includes(humanHandle)) { - const dm = dmParticipants(room, db); - const title = dm ? `DM from ${handle}` : `#${room}`; - // The click target: the viewer at this exact message, when the viewer is - // configured. The tray opens `url` on a default click for any category. - notifyEnabled( - CHAT_NOTIFICATION_CATEGORY, - title, - `${handle}: ${body}`, - chatViewerUrl(readChatViewerUrlSetting(), room, posted.id), - undefined, - `chat:${posted.id}`, - ); + try { + const dm = dmParticipants(room, db); + const title = dm ? `DM from ${handle}` : `#${room}`; + // The click target: the viewer at this exact message, when the viewer is + // configured. The tray opens `url` on a default click for any category. + notifyEnabled( + CHAT_NOTIFICATION_CATEGORY, + title, + `${handle}: ${body}`, + chatViewerUrl(readChatViewerUrlSetting(), room, posted.id), + undefined, + `chat:${posted.id}`, + ); + } catch (err) { + log.warn({ err, id: posted.id, room }, "chat: desk notify threw after a successful post"); + } } return posted; } From 7b061da35ddc55805cf6d1235f6f01aa89726a59 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:40:39 -0500 Subject: [PATCH 006/142] discussions:diffs: bound the GitLab fetch with a timeout+abort signal and report truncation at 100 files (S053, S086) --- .../__tests__/discussions-diffs.test.ts | 62 +++++++++++++++++++ lib/daemon/handlers/discussions.ts | 52 +++++++++++++--- 2 files changed, 105 insertions(+), 9 deletions(-) create mode 100644 lib/daemon/__tests__/discussions-diffs.test.ts diff --git a/lib/daemon/__tests__/discussions-diffs.test.ts b/lib/daemon/__tests__/discussions-diffs.test.ts new file mode 100644 index 00000000..7ea75ddf --- /dev/null +++ b/lib/daemon/__tests__/discussions-diffs.test.ts @@ -0,0 +1,62 @@ +/** + * discussions:diffs (S053 + S086): the outbound GitLab fetch must carry a + * bound signal (so a stalled connection doesn't orphan the promise and the + * sops-decrypted token in its closure forever), and a full page (100 rows) + * must be reported as `truncated` rather than silently dropped. + */ +import { expect, test } from "bun:test"; +import { fetchMrDiffs } from "../handlers/discussions.ts"; + +function fakeDiffs(n: number) { + return Array.from({ length: n }, (_, i) => ({ new_path: `file${i}.ts`, diff: `@@ diff ${i}` })); +} + +test("truncated is false when fewer than a full page comes back", async () => { + const fetchFn = (async () => new Response(JSON.stringify(fakeDiffs(3)), { status: 200 })) as typeof fetch; + const out = await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); + expect(out.diffs).toHaveLength(3); + expect(out.truncated).toBe(false); +}); + +test("truncated is true when exactly a full page (100) comes back", async () => { + const fetchFn = (async () => new Response(JSON.stringify(fakeDiffs(100)), { status: 200 })) as typeof fetch; + const out = await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); + expect(out.diffs).toHaveLength(100); + expect(out.truncated).toBe(true); +}); + +test("maps new_path/diff to newPath/diff", async () => { + const fetchFn = (async () => new Response(JSON.stringify([{ new_path: "a.ts", diff: "@@" }]), { status: 200 })) as typeof fetch; + const out = await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); + expect(out.diffs).toEqual([{ newPath: "a.ts", diff: "@@" }]); +}); + +test("a non-ok response throws with the status", async () => { + const fetchFn = (async () => new Response("", { status: 502 })) as typeof fetch; + await expect(fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn })).rejects.toThrow(/502/); +}); + +test("the fetch call carries an AbortSignal", async () => { + let sawSignal: AbortSignal | undefined; + const fetchFn = (async (_url: any, init: any) => { + sawSignal = init?.signal; + return new Response(JSON.stringify([]), { status: 200 }); + }) as typeof fetch; + await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); + expect(sawSignal).toBeInstanceOf(AbortSignal); +}); + +test("aborting the caller's own request signal cancels the in-flight fetch", async () => { + const controller = new AbortController(); + let sawSignal: AbortSignal | undefined; + const fetchFn = (async (_url: any, init: any) => { + sawSignal = init?.signal; + return new Promise((_resolve, reject) => { + init.signal.addEventListener("abort", () => reject(new Error("aborted"))); + }); + }) as typeof fetch; + const promise = fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn, reqSignal: controller.signal }); + controller.abort(); + await expect(promise).rejects.toThrow(); + expect(sawSignal?.aborted).toBe(true); +}); diff --git a/lib/daemon/handlers/discussions.ts b/lib/daemon/handlers/discussions.ts index 86f5adab..5cbc3b3f 100644 --- a/lib/daemon/handlers/discussions.ts +++ b/lib/daemon/handlers/discussions.ts @@ -31,6 +31,40 @@ import type { Commands } from "../../../packages/rt-client/src/commands.ts"; /** Discussions are stable per push; 2min TTL keeps reads fast without going stale. */ const DISCUSSIONS_TTL_MS = 2 * 60 * 1000; +/** GitLab's page size for this endpoint; a full page means there may be more. */ +const DIFFS_PAGE_SIZE = 100; +const DIFFS_FETCH_TIMEOUT_MS = 30_000; + +/** + * Every other outbound fetch in the daemon carries a bound (linear.ts, + * notifier.ts, park.ts); this one previously had none, so a stalled GitLab + * connection left the promise (and the sops-decrypted token in its closure) + * pending indefinitely. `reqSignal` is the client's own request signal + * (handlers/types.ts's Handler(payload, signal?)) so a client giving up + * actually cancels the in-flight GitLab request instead of orphaning it. + * `truncated` reports a full page rather than silently dropping files past it. + */ +export async function fetchMrDiffs( + baseURL: string, + projectPath: string, + iid: number, + token: string, + opts: { reqSignal?: AbortSignal; fetchFn?: typeof fetch } = {}, +): Promise<{ diffs: Array<{ newPath: string; diff: string }>; truncated: boolean }> { + const fetchFn = opts.fetchFn ?? fetch; + const encoded = encodeURIComponent(projectPath); + const url = `${baseURL}/api/v4/projects/${encoded}/merge_requests/${iid}/diffs?per_page=${DIFFS_PAGE_SIZE}`; + const timeout = AbortSignal.timeout(DIFFS_FETCH_TIMEOUT_MS); + const signal = opts.reqSignal ? AbortSignal.any([timeout, opts.reqSignal]) : timeout; + const res = await fetchFn(url, { headers: { "PRIVATE-TOKEN": token }, signal }); + if (!res.ok) throw new Error(`GitLab diffs API: ${res.status}`); + const raw = (await res.json()) as Array<{ new_path: string; diff: string }>; + return { + diffs: raw.map((d) => ({ newPath: d.new_path, diff: d.diff })), + truncated: raw.length >= DIFFS_PAGE_SIZE, + }; +} + export function createDiscussionHandlers( ctx: HandlerContext, broadcast: BroadcastFn, @@ -122,7 +156,7 @@ export function createDiscussionHandlers( } }, - "discussions:diffs": async (payload) => { + "discussions:diffs": async (payload, signal) => { const repoName = payload?.repoName as string | undefined; const iid = payload?.iid as number | undefined; if (!repoName || typeof iid !== "number") { @@ -138,14 +172,14 @@ export function createDiscussionHandlers( const secrets = await loadSecrets(); if (!secrets.gitlabToken) return { ok: false, error: "no gitlabToken in secrets" }; - const encoded = encodeURIComponent(repoCtx.projectPath); - const url = `${repoCtx.provider.baseURL}/api/v4/projects/${encoded}/merge_requests/${iid}/diffs?per_page=100`; - const res = await fetch(url, { headers: { "PRIVATE-TOKEN": secrets.gitlabToken } }); - if (!res.ok) return { ok: false, error: `GitLab diffs API: ${res.status}` }; - - const raw = await res.json() as Array<{ new_path: string; diff: string }>; - const diffs = raw.map((d) => ({ newPath: d.new_path, diff: d.diff })); - return { ok: true, data: { diffs } }; + const { diffs, truncated } = await fetchMrDiffs( + repoCtx.provider.baseURL, + repoCtx.projectPath, + iid, + secrets.gitlabToken, + { reqSignal: signal }, + ); + return { ok: true, data: { diffs, truncated } }; } catch (err) { return { ok: false, error: String(err) }; } From cc924b5c84a13a1dc7abaa40144da2ad81cfe5bc Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:41:56 -0500 Subject: [PATCH 007/142] plan: Phase 0 honest-supervision implementation plan Co-Authored-By: Claude Opus 4.8 (1M context) --- .../plans/2026-08-28-p0-supervision.md | 1088 +++++++++++++++++ 1 file changed, 1088 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-28-p0-supervision.md diff --git a/docs/superpowers/plans/2026-08-28-p0-supervision.md b/docs/superpowers/plans/2026-08-28-p0-supervision.md new file mode 100644 index 00000000..bbbd14d5 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-p0-supervision.md @@ -0,0 +1,1088 @@ +# Phase 0 · Honest Supervision Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make rt daemon crashes visible and recoverable: every boot failure exits non-zero (or parks visibly) instead of becoming a silent zombie, crash/restart history is persisted and surfaced by `rt daemon status`, and start/stop/eviction stop deleting a live daemon's runtime files. + +**Architecture:** Retire the P0/P1 crash-and-hide class in `lib/daemon.ts` and its immediate collaborators. Boot failures become fatal (log + `process.exit(1)`) on the prod path, gated by a boot-phase flag so steady-state stray rejections still recover; crash handlers and stderr redirection move above every module-scope side effect; restart counters and last-exit reasons live in the existing `kv` table (no schema change); `rt daemon status` and `/api/status` gain `alive-not-serving` / `parked` / `boot-failed` / `crash-looping` verdicts; start/stop/eviction become ownership-aware. Two one-line correctness fixes (state.db flavor on the snapshot path, extended busy-code matching + `BEGIN IMMEDIATE`) make the contention policy real. + +**Tech Stack:** Bun, `bun:sqlite`, pino, TypeScript. Tests are `bun test` (unit) and `bun test --preload ./e2e/setup.ts` (e2e, isolated-HOME daemon spawns). + +**Spec:** `/Users/matt/Documents/GitHub/repo-tools/.claude/worktrees/daemon-stability-audit/docs/daemon-stability-audit-2026-08.md` — "Roadmap › Phase 0" plus Appendix A/B entries S001, S003, S004, S009, S011, S012, S026, S027, S028, S029, S030, S035, S036, S037, S043, S044, S060, S072, S073, S074, R001, R002, R007, R017. Each carries a failure scenario, prescribed fix, and fixer notes; read the relevant entry before implementing its task. + +## Global Constraints + +- **No `SCHEMA_VERSION` bump and no new/edited `V*_SCHEMA` block.** Persist all supervision state (restart counters, last-exit reason, boot-failed markers) in the existing `kv` table under namespace `daemon-supervision`, via `setKvValue`/`getKvValue` from `lib/state/kv-blob.ts`. If any task appears to need a schema change, STOP and ask the user (per the job brief's question format) — do not proceed. +- **Never start a daemon or run `rt` against the real machine.** Every daemon or `dist/rt` invocation in a test or check runs under `env -i HOME=` only (repo CLAUDE.md, "Operating on this machine"). e2e daemon spawns already do this via `e2e/setup.ts`; new e2e tests must follow the same isolation. +- **Write fence — do NOT modify these sibling-owned files** (ask the user if a task seems to need one): `lib/daemon/api-server.ts`, `lib/daemon/api-auth.ts`, `lib/daemon/socket-server.ts`, `lib/daemon/handlers/secrets.ts`, `lib/subprocess.ts`, `lib/daemon/cache-refresh.ts`, `lib/git-worktrees.ts`, `lib/daemon/freshness.ts`, `lib/daemon/pollers.ts`, `lib/daemon/worktree-process-kill.ts`, `lib/daemon/system-process-scanner.ts`, `lib/runs/store.ts`, `lib/notifier.ts`, `lib/daemon/handlers/discussions.ts`, `lib/daemon/handlers/chat.ts`, `lib/daemon/handlers/agent.ts`, `lib/daemon/handlers/pane.ts`, `lib/daemon/handlers/project-mrs.ts`, `lib/daemon/handlers/worktree.ts`, `lib/herdr/client.ts`, `lib/port-scanner.ts`, `lib/deps/links.ts`, `lib/worktree/trash.ts`, `lib/agent-herdr.ts`, `lib/daemon/cron.ts`, `lib/daemon/hooks-guard.ts`, `lib/home/age-key.ts`, `lib/daemon/discussions-store.ts`, `lib/state/presence-store.ts`. +- **Every subagent dispatched during execution carries an explicit `model`** (`sonnet` for mechanical tasks, `haiku` for lookups). +- **`packages/rt-client` is touched** (Task 5 edits `registry-defs.ts`). After that task and before the final whole-branch review, run `bun run build` inside `packages/rt-client` (keeps `dist/` and `dist-freshness.test.ts` green). +- **Verification (must pass before the work is done):** + - `bun test lib commands packages scripts` green (from the worktree root) + - `bunx tsc --noEmit` reports zero errors + - `bun run test:e2e` green, or at minimum `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts` (say which was run) + +## Deferred / out-of-fence items (documented, not implemented here) + +- **S073 · `presence-store.ts:signIn` → `.immediate()`** — `lib/state/presence-store.ts` is fenced. Task 7 converts the chat-store and notifier-store read-then-write transactions; the `signIn` caller is a follow-up for the presence-store owner. Not required for verification. +- **0.3 · `rt.apiPort` bind-time consumption** — `lib/daemon/api-server.ts` (the binder) is fenced. Task 5 registers the `rt.apiPort` setting and exposes `resolveApiPort()`; the bind-time read is the api-server sibling's hop. The escape hatch is therefore wired daemon-side but consumed sibling-side; do not claim it functions end-to-end until the sibling reads it. +- **Swift tray edits (S026 dot mapping, S028 `DaemonLifecycle` kickstart fallback, S029 `tray-crash.log` rotation, S060 `AppDelegate` comment)** — grouped as optional Task 16. They cannot be verified by the bun/tsc/e2e gate and the operating rules forbid rebuilding the blessed bundle. Scope confirmation is raised at the plan-review checkpoint. The verifiable CLI-side halves (S028 start→kickstart fallback, S060 exit-code policy) live in Tasks 14 and 12 and are done regardless. + +--- + +## Task 1: Status-verdict + exit-code design sketch + +The half-page design that Tasks 9–14 depend on. No production code; the deliverable is a committed design doc. (Retires nothing directly; anchors R001, R002, S036, S060, S026, S028.) + +**Files:** +- Create: `docs/daemon-supervision-design.md` + +**Interfaces:** +- Produces: the verdict names (`serving`, `alive-not-serving`, `parked`, `boot-failed`, `crash-looping`, `installed-not-running`, `not-installed`), the boot-phase names (`booting` → `ready`), the kv keys under ns `daemon-supervision`, the boot-breadcrumb shape, and the exit-code policy table. Tasks 9/10/12 consume these names verbatim. + +- [ ] **Step 1: Write the design doc** with exactly this content: + +```markdown +# Daemon supervision: status verdicts and exit-code semantics + +Phase 0 design anchor for the rt daemon stability roadmap (audit +2026-08). Tasks 9–14 of the Phase 0 plan implement this. + +## launchd contract + +The prod plist sets `KeepAlive = { SuccessfulExit: false }`: launchd +respawns the daemon ONLY on a non-zero exit. A zero exit means "stay +down". Every exit-code decision below follows from that single fact. + +## Exit-code policy + +| Path | Exit | Why | +|-----------------------------------------|------|-----| +| `startDaemon()` boot throw (prod path) | 1 | Visible + launchd relaunches. Paired with crash-loop detection so it cannot loop silently forever. | +| `shutdown` IPC/REST verb | 0 | Intentional stop; launchd must not respawn. Records `last-exit.kind = "shutdown"`. | +| Bare OS signal SIGTERM/SIGINT/SIGHUP | 1 | External kill (pkill, script, memory pressure); launchd SHOULD respawn. The sanctioned stop path goes through SMAppService.unregister, where the exit code is irrelevant, so exiting non-zero here does not break intended stops. | +| Crash-loop guard trips (N in M minutes) | 0 (park) | Stop the flapping; surface `crash-looping` so a human intervenes instead of launchd hammering every ~10s. | + +Mechanism: a module-scope `shuttingDownViaVerb` flag is set true by the +`shutdown` verb before it calls cleanup; `gracefulExit(signal)` reads it +— set → exit(0), unset (bare signal) → exit(1). + +Boot-phase gate: a module-scope `bootPhase: "booting" | "ready"` flips +to `"ready"` immediately before the `daemon ready` log. The +`unhandledRejection` handler exits(1) while `bootPhase === "booting"` +and only logs (recovers) once ready — so a boot-time stray rejection is +fatal but a steady-state one is not. + +## Status verdicts + +`rt daemon status` and `/api/status` classify by first match: + +1. `not-installed` — SMAppService not registered. +2. `serving` — ping on rt.sock succeeds. +3. `parked` — ping fails, a live rt pid exists, and the boot breadcrumb + phase is a flavor standoff (park). Named distinctly so the user is + told "another flavor owns the socket", not "wedged". +4. `alive-not-serving` — ping fails but a live rt pid exists + (`process.kill(pid,0)` on rt.pid, or `pgrep -f 'rt --daemon|lib/daemon.ts'`). + Sub-detail from the breadcrumb phase: `booting` / `wedged`, or + `quarantined` when a state.db/events.db boot-failed marker is present. + Prints "process is running but not answering rt.sock — rt daemon logs -t". +5. `crash-looping` — no live pid AND the kv failure record shows ≥ N + failures within the last M minutes (N=3, M=5). Prints the last reason. +6. `boot-failed` — no live pid AND the most recent kv exit record is a + boot throw (fewer than N failures). Prints the last reason + phase. +7. `installed-not-running` — registered, no live pid, clean/again-absent + exit record. + +## Persisted state (kv, ns `daemon-supervision`, no schema change) + +- `boot-attempts` (number) — incremented at the top of `runDaemon()`. +- `last-ready-at` (number, epoch ms) — stamped just before `daemon ready`. +- `recent-failures` (array of `{ at, phase, reason }`, capped to 10) — + appended by the boot fatal path and by state.db/events.db boot-failed + markers. Crash-loop = ≥ N entries newer than now − M minutes. +- `last-exit` (`{ at, kind: "shutdown" | "signal" | "boot-failed", code, reason? }`) + — written by the shutdown verb, the signal handlers, and the boot + fatal path. Lets status distinguish "cleanly stopped" from "died". + +## Boot breadcrumb + +`~/.mattstack/rt/daemon-boot.json` = `{ at, pid, flavor, phase }`, +rewritten at each boot phase: `start` → `crash-handlers` → `events-db` +→ `state-db` → `socket` → `api` → `ready`. Lets `alive-not-serving` +name where a live-but-silent daemon is stuck even when the logs are +unreadable. Removed (or stamped `ready`) on successful boot. +``` + +- [ ] **Step 2: Commit** + +```bash +git add docs/daemon-supervision-design.md +git commit -m "docs: sketch daemon supervision verdicts + exit-code semantics" +``` + +--- + +## Task 2: Fatal boot means exit (0.1 — S001, S037) + +Boot failures on the prod path currently become an `unhandledRejection` that only logs, leaving a live-pid zombie with no socket/API. Make a `runDaemon()` throw fatal, gate the rejection handler on a boot-phase flag, and move the rt.pid write to after both binds. + +**Files:** +- Modify: `lib/daemon-logger.ts:232-264` (`installCrashHandlers` gains a `booting` predicate) +- Modify: `lib/daemon.ts:385-514` (wrap `runDaemon()` body; boot-phase flag; move rt.pid write), `lib/daemon.ts:392` (pass predicate), `lib/daemon.ts:513` (flip flag) +- Test: `lib/__tests__/daemon-logger.test.ts` (rejection handler), `e2e/tests/daemon.test.ts` (fatal boot) + +**Interfaces:** +- Produces: `installCrashHandlers(logger, opts?: { booting?: () => boolean })` — when `booting()` is true, `unhandledRejection` logs `fatal` and `process.exit(1)`; otherwise it logs `error` only (today's behavior). Default (no `booting`) preserves today's error-only behavior. +- Produces: module-scope `let bootPhase: "booting" | "ready" = "booting"` in `lib/daemon.ts`, flipped to `"ready"` at line 513. + +- [ ] **Step 1: Write the failing unit test** in `lib/__tests__/daemon-logger.test.ts`: + +```ts +test("unhandledRejection exits(1) while booting, only logs once ready", () => { + const exits: number[] = []; + const origExit = process.exit; + // @ts-expect-error test stub + process.exit = (code?: number) => { exits.push(code ?? 0); }; + const fatal = mock(() => {}); + const error = mock(() => {}); + const logger = makeFakeLogger({ fatal, error }); // existing test helper + let booting = true; + installCrashHandlers(logger, { booting: () => booting }); + process.emit("unhandledRejection", new Error("boot boom"), Promise.resolve()); + expect(fatal).toHaveBeenCalledTimes(1); + expect(exits).toEqual([1]); + booting = false; + process.emit("unhandledRejection", new Error("steady boom"), Promise.resolve()); + expect(error).toHaveBeenCalledTimes(1); + expect(exits).toEqual([1]); // no second exit + process.exit = origExit; +}); +``` + +(If `makeFakeLogger` does not exist, build a minimal `{ info, warn, error, fatal }` of `mock(() => {})`. Remove the listeners this test adds in `afterEach` via `process.removeAllListeners("unhandledRejection")` scoped to the test, matching the file's existing cleanup convention.) + +- [ ] **Step 2: Run it — expect FAIL** (`booting` option not supported): + +Run: `bun test lib/__tests__/daemon-logger.test.ts -t "unhandledRejection exits"` +Expected: FAIL. + +- [ ] **Step 3: Implement in `lib/daemon-logger.ts`.** Change the signature and the `unhandledRejection` branch (currently lines 242-244): + +```ts +export function installCrashHandlers( + logger: Logger, + opts: { booting?: () => boolean } = {}, +): void { + // ... uncaughtException handler unchanged (still fatal + exit 1) ... + process.on("unhandledRejection", (reason) => { + if (opts.booting?.()) { + logger.fatal({ err: reason }, "unhandledRejection during boot"); + process.exit(1); + return; + } + logger.error({ err: reason }, "unhandledRejection"); + }); + // ... stderr interception unchanged ... +} +``` + +- [ ] **Step 4: Run the unit test — expect PASS.** + +- [ ] **Step 5: Wire the boot-phase flag + fatal wrap + rt.pid move in `lib/daemon.ts`.** + - Add near the top of module scope (after imports, before line 78): `let bootPhase: "booting" | "ready" = "booting";` + - At the `installCrashHandlers(loggerHandle)` call (line 392): `installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" });` + - Wrap the body of `runDaemon()` (385-514) in try/catch: + +```ts +async function runDaemon() { + try { + // ... existing body 386-513 ... + } catch (err) { + loggerHandle.fatal?.({ err }, "daemon boot failed"); + // Task 9 adds recordBootFailure(currentPhase, err) here. + try { loggerHandle.flush?.(); } catch {} + process.exit(1); + } +} +``` + + - **Move the rt.pid write** (currently `writeFileSync(DAEMON_PID_PATH, String(process.pid))` at line 416) to AFTER both binds — i.e. after the API bind at line 469 and the socket bind at line 468 (Task 5 will make API bind first; either way, rt.pid is written only once both `servers.socket` and `servers.api` are assigned). A failed boot then never leaves a live-pid file. + - Set `bootPhase = "ready";` immediately before `log.info({ pid }, "daemon ready")` at line 513. + +- [ ] **Step 6: Write the failing e2e test** in `e2e/tests/daemon.test.ts` (uses the isolated-HOME harness already in `e2e/setup.ts`): + +```ts +test("daemon boot with API port already bound exits non-zero and leaves no stale rt.pid", async () => { + // Bind the API port inside the isolated HOME so the daemon cannot. + const port = 9411; + const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + try { + const proc = Bun.spawn({ + cmd: [rtBinary, "--daemon"], + env: { ...isolatedEnv, RT_API_PORT: String(port) }, + stdout: "pipe", stderr: "pipe", + }); + const code = await proc.exited; + expect(code).not.toBe(0); + expect(existsSync(join(isolatedHome, ".mattstack/rt/rt.pid"))).toBe(false); + } finally { + squatter.stop(true); + } +}); +``` + +(Reuse the harness's existing `rtBinary`, `isolatedEnv`, `isolatedHome` fixtures — mirror `e2e/tests/daemon.test.ts`'s existing setup. The daemon's own `startApiServer` retries the bind 6× before throwing, so allow up to the 60s timeout.) + +- [ ] **Step 7: Run the e2e test — expect PASS.** Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "API port already bound"` + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon-logger.ts lib/daemon.ts lib/__tests__/daemon-logger.test.ts e2e/tests/daemon.test.ts +git commit -m "daemon: boot failure is fatal (exit 1), gated by boot-phase flag; rt.pid after binds" +``` + +--- + +## Task 3: Crash handlers first (0.2 — S003, S004, R007) + +Move `redirectNativeStderr()` and `installCrashHandlers()` above every module-scope side effect so a pre-`startDaemon` failure lands in the crash log instead of a discarded stderr. + +**Files:** +- Modify: `lib/daemon.ts` (hoist two calls to module scope; remove the duplicates inside `runDaemon()` at 391-392) +- Test: `e2e/tests/daemon.test.ts` + +**Interfaces:** +- Consumes: `installCrashHandlers(logger, { booting })` from Task 2. + +- [ ] **Step 1: Hoist `redirectNativeStderr()`** to the very first executable statement of `lib/daemon.ts` module scope — before `migrateLegacyRtDir()` at line 78. It depends only on `logsDir()` and has its own internal try/catch, so a subsequent module-scope throw's fd-2 output lands in `daemon-stderr.log`. + +- [ ] **Step 2: Hoist `installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" })`** to immediately after `getDaemonLogger()` resolves (right after line ~95, before `parkUntilIntended` at 103 and before `createEventsBus` at 191). The logger must exist first (line 85), so this is the earliest correct point. + +- [ ] **Step 3: Remove the now-duplicate `redirectNativeStderr()` and `installCrashHandlers()` calls** inside `runDaemon()` (lines 391-392). Keep `mkdirSync(RT_DIR, …)` at 386 (redirect needs the logs dir; `redirectNativeStderr` already mkdirs its own dir, and RT_DIR creation is idempotent — verify the hoisted `redirectNativeStderr` still finds/creates `logsDir()`). + +- [ ] **Step 4: Write the failing e2e test** in `e2e/tests/daemon.test.ts`: + +```ts +test("a corrupt events.db does not crash the daemon silently — error is captured", async () => { + // Pre-create a corrupt events.db in the isolated HOME. + const rtDir = join(isolatedHome, ".mattstack/rt"); + mkdirSync(rtDir, { recursive: true }); + writeFileSync(join(rtDir, "events.db"), "not a sqlite file at all"); + const proc = Bun.spawn({ cmd: [rtBinary, "--daemon"], env: isolatedEnv, stdout: "pipe", stderr: "pipe" }); + // Task 4 makes this self-heal; for Task 3 we only require the failure is captured, not /dev/null. + await Bun.sleep(3000); proc.kill(); + const stderrLog = join(rtDir, "logs", "daemon-stderr.log"); + const quarantined = readdirSync(rtDir).some((f) => f.startsWith("events.db.corrupt-")); + const captured = existsSync(stderrLog) || quarantined; + expect(captured).toBe(true); +}); +``` + +- [ ] **Step 5: Run — expect PASS** (the redirect now runs before events.db construction, so a corruption throw is captured in `daemon-stderr.log`; after Task 4 it is quarantined instead). Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "corrupt events.db"` + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon.ts e2e/tests/daemon.test.ts +git commit -m "daemon: install stderr redirect + crash handlers before every module-scope side effect" +``` + +--- + +## Task 4: events.db joins the discipline (0.4 — S009, S035) + +Give `events.db` the corruption quarantine + busy_timeout/synchronous pragmas state.db has, and guard the two sweep timers so a sync sqlite throw cannot exit the daemon. + +**Files:** +- Modify: `lib/daemon/events-bus.ts:64-224` (quarantine + pragmas in `createEventsBus`) +- Modify: `lib/daemon.ts:194,196` (wrap sweep timers; add a `safeInterval`/`safeTimeout` helper) +- Test: `lib/daemon/__tests__/events-bus.test.ts` + +**Interfaces:** +- Consumes: `isCorruptionError` and `quarantine` shape from `lib/state/db.ts` (reuse the `SQLITE_CORRUPT`/`SQLITE_NOTADB` detection; `events.db` is a bounded-retention journal, so total loss on quarantine is harmless — no migration concern). + +- [ ] **Step 1: Write the failing test** in `lib/daemon/__tests__/events-bus.test.ts`: + +```ts +test("createEventsBus quarantines and recreates a corrupt events.db instead of throwing", () => { + const dir = mkdtempSync(join(tmpdir(), "events-corrupt-")); + const dbPath = join(dir, "events.db"); + writeFileSync(dbPath, "garbage not sqlite"); + const bus = createEventsBus({ dbPath, log: silentLog }); + expect(readdirSync(dir).some((f) => f.startsWith("events.db.corrupt-"))).toBe(true); + // fresh db works: + bus.emit("test", { hi: 1 }); + expect(bus.list({ limit: 1 }).length).toBe(1); + bus.close(); +}); + +test("createEventsBus sets busy_timeout and synchronous=NORMAL", () => { + const dir = mkdtempSync(join(tmpdir(), "events-pragma-")); + const bus = createEventsBus({ dbPath: join(dir, "events.db"), log: silentLog }); + // @ts-expect-error internal handle access for the test, or expose a debug getter + const handle = bus.__db ?? getBusDb(bus); + expect(handle.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); + expect(handle.query("PRAGMA synchronous").get()).toEqual({ synchronous: 1 }); // NORMAL + bus.close(); +}); +``` + +(If the bus does not expose its handle, add a minimal `__db` back-reference or a `busyTimeout()` debug accessor in `events-bus.ts` for the test; do not expose it beyond the module's test needs.) + +- [ ] **Step 2: Run — expect FAIL.** Run: `bun test lib/daemon/__tests__/events-bus.test.ts -t "quarantine"` + +- [ ] **Step 3: Implement in `lib/daemon/events-bus.ts`** — wrap the open (lines 73-86): + +```ts +import { isCorruptionError } from "../state/db"; // export it if not already exported +// ... +mkdirSync(dirname(opts.dbPath), { recursive: true }); +let db: Database; +try { + db = new Database(opts.dbPath, { create: true }); + db.exec("PRAGMA busy_timeout = 250;"); // before journal_mode, matching state/db.ts ordering + db.exec("PRAGMA journal_mode = WAL;"); + db.exec("PRAGMA synchronous = NORMAL;"); + db.query("PRAGMA user_version").get(); // touch to force a real read that trips corruption +} catch (err) { + if (!isCorruptionError(err)) throw err; + quarantineEventsDb(opts.dbPath, opts.log); // rename to events.db.corrupt-, warn + db = new Database(opts.dbPath, { create: true }); + db.exec("PRAGMA busy_timeout = 250;"); + db.exec("PRAGMA journal_mode = WAL;"); + db.exec("PRAGMA synchronous = NORMAL;"); +} +// then the existing CREATE TABLE / CREATE INDEX (78-86) +``` + +Write a local `quarantineEventsDb(path, log)` mirroring `state/db.ts`'s `quarantine` (rename the db + `-wal`/`-shm` sidecars to `path.corrupt-`, `log.warn`). If `isCorruptionError` is not exported from `state/db.ts`, add the export (it is a pure predicate, safe to export). + +- [ ] **Step 4: Run the events-bus tests — expect PASS.** + +- [ ] **Step 5: Write the failing sweep-guard test** in `lib/daemon/__tests__/events-bus.test.ts` OR a small `lib/__tests__/daemon-sweep-guard.test.ts` for the helper: + +```ts +test("safeInterval swallows a throwing tick and logs warn", () => { + const warn = mock(() => {}); + const log = { ...silentLog, warn }; + let ticks = 0; + const handle = safeInterval(() => { ticks++; throw new Error("SQLITE_FULL"); }, 10, "test-sweep", log); + return new Promise((r) => setTimeout(() => { + clearInterval(handle); + expect(ticks).toBeGreaterThan(0); + expect(warn).toHaveBeenCalled(); + r(); + }, 45)); +}); +``` + +- [ ] **Step 6: Implement `safeInterval`/`safeTimeout`** in `lib/daemon.ts` (or a tiny `lib/daemon/safe-timers.ts` exporting both): + +```ts +export function safeInterval(fn: () => void, ms: number, label: string, log: Logger) { + return setInterval(() => { try { fn(); } catch (err) { log.warn({ err, label }, "timer tick failed"); } }, ms); +} +export function safeTimeout(fn: () => void, ms: number, label: string, log: Logger) { + return setTimeout(() => { try { fn(); } catch (err) { log.warn({ err, label }, "timer tick failed"); } }, ms); +} +``` + +Replace the two bare sweep timers at `lib/daemon.ts:194` and `:196` with `safeInterval(() => eventsBus.sweep(), 60*60*1000, "events-sweep", log)` and `safeTimeout(() => eventsBus.sweep(), 30_000, "events-sweep-boot", log)`. (The `pruneRuns`/`pruneLogs` timers at 200-248 already wrap their bodies; leaving them is fine, but converting them to `safeInterval` is a welcome DRY cleanup if trivial.) + +- [ ] **Step 7: Run — expect PASS.** Then `bun test lib/daemon/__tests__/events-bus.test.ts`. + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon/events-bus.ts lib/daemon.ts lib/state/db.ts lib/daemon/__tests__/events-bus.test.ts +git commit -m "events.db: corruption quarantine + busy_timeout/synchronous pragmas; guard sweep timers" +``` + +--- + +## Task 5: Bind order + rt.apiPort setting (0.3, my side — S043, S030 seam) + +Bind the API server before the unix socket so a failed API bind never strands a socket-bound zombie, and register the `rt.apiPort` escape-hatch setting (the api-server sibling consumes it at bind time). + +**Files:** +- Modify: `lib/daemon.ts:468-469` (swap: API bind first, then socket bind; rt.pid still written after both, per Task 2) +- Modify: `lib/daemon-config.ts:72` (add `resolveApiPort()`; keep `API_PORT` const non-breaking) +- Modify: `packages/rt-client/src/settings/registry-defs.ts` (register `rt.apiPort`) +- Test: `lib/__tests__/daemon-config.test.ts`, `packages/rt-client` settings test, `e2e/tests/daemon.test.ts` + +**Interfaces:** +- Produces: `resolveApiPort(): number` in `lib/daemon-config.ts` = `Number(process.env.RT_API_PORT) || getSetting("rt.apiPort").value || 9401`, resolved lazily (never at module load). `API_PORT` const stays as-is for `api-server.ts`'s existing import. + +- [ ] **Step 1: Register the setting.** Append to `packages/rt-client/src/settings/registry-defs.ts` (shaped like `rt.logRetentionDays` at 190-198): + +```ts +{ + key: "rt.apiPort", + type: "number", + scopes: ["machine", "user"], + default: 9401, + merge: "replace", + migrated: true, + description: "TCP port the daemon's local HTTP/WS API binds (escape hatch when 9401 is held).", +}, +``` + +- [ ] **Step 2: Rebuild rt-client** so the daemon and `dist-freshness` see the new key: + +Run: `cd packages/rt-client && bun run build && cd -` + +- [ ] **Step 3: Write the failing test** in `lib/__tests__/daemon-config.test.ts`: + +```ts +test("resolveApiPort: env wins, then setting, then 9401", () => { + const prev = process.env.RT_API_PORT; + process.env.RT_API_PORT = "12345"; + expect(resolveApiPort()).toBe(12345); + delete process.env.RT_API_PORT; + expect(resolveApiPort()).toBe(9401); // default setting value + if (prev !== undefined) process.env.RT_API_PORT = prev; +}); +``` + +- [ ] **Step 4: Run — expect FAIL** (`resolveApiPort` undefined). + +- [ ] **Step 5: Implement in `lib/daemon-config.ts`** (leave `API_PORT` at line 72 untouched): + +```ts +import { getSetting } from "./settings/resolve"; +export function resolveApiPort(): number { + const env = Number(process.env.RT_API_PORT); + if (env) return env; + try { return getSetting("rt.apiPort").value || 9401; } catch { return 9401; } +} +``` + +- [ ] **Step 6: Run — expect PASS.** + +- [ ] **Step 7: Swap the bind order in `lib/daemon.ts`.** Reorder so the API binds before the socket: + +```ts +servers.api = await startApiServer({ handleCommand, log }); // was line 469 — now first +servers.socket = startSocketServer({ handleCommand, log }); // was line 468 — now second +// rt.pid write (moved by Task 2) stays after BOTH assignments +``` + +- [ ] **Step 8: Write the failing e2e test** in `e2e/tests/daemon.test.ts` (extends the Task 2 squatter test): + +```ts +test("API-bind failure leaves neither rt.sock nor rt.pid", async () => { + const port = 9412; + const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + try { + const proc = Bun.spawn({ cmd: [rtBinary, "--daemon"], env: { ...isolatedEnv, RT_API_PORT: String(port) }, stdout: "pipe", stderr: "pipe" }); + await proc.exited; + expect(existsSync(join(isolatedHome, ".mattstack/rt/rt.sock"))).toBe(false); + expect(existsSync(join(isolatedHome, ".mattstack/rt/rt.pid"))).toBe(false); + } finally { squatter.stop(true); } +}); +``` + +- [ ] **Step 9: Run — expect PASS.** Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "API-bind failure"` + +- [ ] **Step 10: Commit** + +```bash +git add lib/daemon.ts lib/daemon-config.ts packages/rt-client/src/settings/registry-defs.ts packages/rt-client/dist lib/__tests__/daemon-config.test.ts e2e/tests/daemon.test.ts +git commit -m "daemon: bind API before socket; register rt.apiPort setting + resolveApiPort()" +``` + +--- + +## Task 6: home-snapshot opens state.db daemon-flavored (0.7 — S011) + +`startHomeSnapshot` opens the state.db singleton at module scope with the default `cli` flavor (5000ms busy_timeout), so the daemon runs with the wrong contention policy forever. Make its db lazy and daemon-flavored, and harden `getStateDb` against a silent flavor mismatch. + +**Files:** +- Modify: `lib/daemon/home-snapshot.ts:272,299` (lazy daemon-flavored db resolver) +- Modify: `lib/state/db.ts:522-530` (`getStateDb` mismatch hardening) +- Test: `lib/state/__tests__/db.test.ts` + +**Interfaces:** +- Consumes: `getStateDb("daemon")` — 250ms busy_timeout. + +- [ ] **Step 1: Write the failing test** in `lib/state/__tests__/db.test.ts` (`describe("pragma values per flavor")`): + +```ts +test("getStateDb('daemon') reports busy_timeout 250 even after a default open", () => { + const cli = getStateDb(); // opens singleton, cli flavor + expect(cli.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 5000 }); + const daemon = getStateDb("daemon"); // same singleton — must not stay at 5000 + expect(daemon.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); +}); +``` + +(Use the file's existing isolated-HOME / `closeStateDb` setup so this does not leak into other tests.) + +- [ ] **Step 2: Run — expect FAIL** (singleton keeps the cli 5000 timeout). + +- [ ] **Step 3: Harden `getStateDb` in `lib/state/db.ts:522-530`** — when the singleton is already open and a caller requests a stronger (shorter) flavor timeout, re-apply the pragma: + +```ts +export function getStateDb(flavor: DbFlavor = "cli"): Database { + const path = stateDbPath(); + if (singleton && singletonPath === path) { + const want = BUSY_TIMEOUT_MS[flavor]; + const have = Number((singleton.query("PRAGMA busy_timeout").get() as any)?.timeout ?? 0); + if (want < have) singleton.exec(`PRAGMA busy_timeout = ${want};`); + return singleton; + } + // ... existing (re)open path ... +} +``` + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Make home-snapshot's db lazy + daemon-flavored** in `lib/daemon/home-snapshot.ts`. Replace the eager `db: rawDeps.db ?? getStateDb()` (line 272) with a thunk defaulting to `() => getStateDb("daemon")`, and resolve it on first use inside `loadState`/`runNow`/`status` rather than at construction (line 299). Concretely: store `const resolveDb = rawDeps.db ? () => rawDeps.db! : () => getStateDb("daemon");` and call `resolveDb()` where `deps.db` was read, so no db opens until `startDaemon` has already opened it daemon-flavored via `openBranchCacheStore`. + +- [ ] **Step 6: Add a boot-order regression test** in `lib/state/__tests__/db.test.ts` (or `home-snapshot.test.ts`) asserting that constructing `startHomeSnapshot` does NOT open the state.db singleton (call it, then assert `getStateDb` was not yet invoked — spy on the module or assert no `state.db` file exists until first use in an isolated HOME). + +- [ ] **Step 7: Run the db tests — expect PASS.** Run: `bun test lib/state/__tests__/db.test.ts` + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon/home-snapshot.ts lib/state/db.ts lib/state/__tests__/db.test.ts +git commit -m "home-snapshot: lazy daemon-flavored state.db; getStateDb re-applies a stronger flavor timeout" +``` + +--- + +## Task 7: Extended busy codes + IMMEDIATE transactions (0.7 — S072, S073) + +`isBusyError` misses `SQLITE_BUSY_SNAPSHOT`/`_RECOVERY`, and read-then-write daemon transactions use a deferred `BEGIN` that produces snapshot conflicts busy_timeout cannot absorb. Widen the match and take the write lock up front. + +**Files:** +- Modify: `lib/state/busy.ts:39-41` (`isBusyError`) +- Modify: `lib/state/chat-store.ts` (`readUnread`, `joinRoom`, `archiveRoom`, `dmRoomFor` → `.immediate()`) +- Modify: `lib/state/notifier-store.ts` (`drainNotificationQueue` → `.immediate()`) +- Test: `lib/state/__tests__/busy.test.ts` +- **Deferred (fenced):** `lib/state/presence-store.ts:signIn` — documented follow-up, not done here. + +**Interfaces:** +- Produces: `isBusyError` returns true for any `code` starting `SQLITE_BUSY`. + +- [ ] **Step 1: Write the failing test** in `lib/state/__tests__/busy.test.ts` — a real two-connection snapshot conflict: + +```ts +test("isBusyError matches SQLITE_BUSY_SNAPSHOT from a real conflict", () => { + const dir = mkdtempSync(join(tmpdir(), "busy-snap-")); + const path = join(dir, "t.db"); + const a = new Database(path); a.exec("PRAGMA journal_mode=WAL; CREATE TABLE t(id INTEGER PRIMARY KEY, v INTEGER);"); + a.exec("INSERT INTO t(id,v) VALUES(1,0);"); + const b = new Database(path); + a.exec("BEGIN;"); a.query("SELECT v FROM t WHERE id=1").get(); // pin snapshot on A + b.exec("UPDATE t SET v=1 WHERE id=1;"); // B commits (autocommit) + let caught: unknown; + try { a.exec("UPDATE t SET v=2 WHERE id=1;"); } catch (e) { caught = e; } + expect(caught).toBeDefined(); + expect((caught as any).code?.startsWith("SQLITE_BUSY")).toBe(true); + expect(isBusyError(caught)).toBe(true); + try { a.exec("ROLLBACK;"); } catch {} + a.close(); b.close(); +}); +``` + +- [ ] **Step 2: Run — expect FAIL** (`isBusyError` returns false for `SQLITE_BUSY_SNAPSHOT`). + +- [ ] **Step 3: Widen `isBusyError`** in `lib/state/busy.ts:39-41`: + +```ts +export function isBusyError(err: unknown): boolean { + const code = (err as { code?: string })?.code; + return code === "SQLITE_BUSY" || (typeof code === "string" && code.startsWith("SQLITE_BUSY_")); +} +``` + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Convert read-then-write transactions to `.immediate()`.** In `lib/state/chat-store.ts`, change the four cited transaction builders (`readUnread` ~483, `joinRoom`, `archiveRoom`, `dmRoomFor`) from `db.transaction(fn)(...)` to `db.transaction(fn).immediate(...)` so the write lock is taken at `BEGIN IMMEDIATE`. Do the same for `drainNotificationQueue` in `lib/state/notifier-store.ts:111`. Leave a one-line comment at the first site: `// BEGIN IMMEDIATE: read-then-write must lock up front or SQLITE_BUSY_SNAPSHOT bypasses busy_timeout.` + +- [ ] **Step 6: Verify no regression** — run the chat/notifier/state suites: `bun test lib/state` and any `chat` command tests. Expect PASS (behavior identical under no contention; the change only affects lock acquisition timing). + +- [ ] **Step 7: Commit** + +```bash +git add lib/state/busy.ts lib/state/chat-store.ts lib/state/notifier-store.ts lib/state/__tests__/busy.test.ts +git commit -m "state: isBusyError matches SQLITE_BUSY_*; read-then-write daemon txns use BEGIN IMMEDIATE" +``` + +--- + +## Task 8: state.db importer isolation (0.3 — S074) + +A throwing legacy importer inside the v0 migration rolls back the whole migration, so `user_version` stays 0 and every subsequent open repeats the failure. Wrap each importer in a SAVEPOINT; on throw, roll back that one importer, warn, and still rename the file. + +**Files:** +- Modify: `lib/state/db.ts:400-417` (`importLegacyStores` — SAVEPOINT per importer) +- Test: `lib/state/__tests__/db.test.ts` + +**Interfaces:** +- Consumes: `LEGACY_IMPORTS` array (line 71), `runMigrations`'s `consumed` list (renames to `.migrated`). + +- [ ] **Step 1: Write the failing test** in `lib/state/__tests__/db.test.ts`: + +```ts +test("a throwing legacy importer is isolated: db reaches SCHEMA_VERSION, other stores import, file renamed", () => { + // Arrange an isolated HOME with a project-mrs.json whose keys "5" and "05" normalize to the same iid, + // plus one benign legacy file another importer consumes. + writeFileSync(join(rtDir, "project-mrs.json"), JSON.stringify({ "host/repo": { mrs: { "5": {...}, "05": {...} } } })); + writeFileSync(join(rtDir, ".json"), JSON.stringify(benignFixture)); + const db = openStateDb(join(rtDir, "state.db"), "daemon"); + expect(db.query("PRAGMA user_version").get()).toEqual({ user_version: SCHEMA_VERSION }); + // benign store's rows landed: + expect(db.query("SELECT COUNT(*) c FROM ").get()).toMatchObject({ c: benignCount }); + // offending file still renamed: + expect(existsSync(join(rtDir, "project-mrs.json"))).toBe(false); + expect(existsSync(join(rtDir, "project-mrs.json.migrated"))).toBe(true); +}); +``` + +(Fill ``/``/fixtures from an existing importer in `LEGACY_IMPORTS`; the db.test.ts legacy-import cases already build such fixtures — reuse one.) + +- [ ] **Step 2: Run — expect FAIL** (the throw rolls back the whole migration; `user_version` stays 0 / benign rows absent). + +- [ ] **Step 3: Implement SAVEPOINT-per-importer** in `importLegacyStores` (`lib/state/db.ts:400-417`). For each `LEGACY_IMPORTS` entry, wrap its `run(db)` in a savepoint scoped to that importer only (do NOT loosen the surrounding schema-DDL migration, which must stay loud): + +```ts +for (const imp of LEGACY_IMPORTS) { + if (!existsSync(imp.path())) continue; + db.exec("SAVEPOINT legacy_import;"); + try { + imp.run(db); + db.exec("RELEASE legacy_import;"); + consumed.push(imp.path()); + } catch (err) { + db.exec("ROLLBACK TO legacy_import; RELEASE legacy_import;"); + log?.warn?.({ err, file: imp.path() }, "legacy import failed; skipping (file will still be renamed)"); + consumed.push(imp.path()); // spec: corrupt = warn + skip, but still rename + } +} +``` + +(Match the exact `LegacyImport` shape at db.ts:64-69 — `path()`/`run(db)` names may differ; adapt to the real fields.) + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Commit** + +```bash +git add lib/state/db.ts lib/state/__tests__/db.test.ts +git commit -m "state.db: isolate each legacy importer in a SAVEPOINT so one bad file cannot wedge migration" +``` + +--- + +## Task 9: Restart counter + last-exit reason in kv (0.5 — R002, S037) + +Persist boot attempts, last-ready stamp, recent failures, and last-exit reason in the `kv` table (ns `daemon-supervision`), plus a boot breadcrumb file. Wire the record calls into the boot path, the shutdown verb, the signal handlers, and the boot fatal path. No schema change. + +**Files:** +- Create: `lib/daemon/supervision-state.ts` +- Create: `lib/daemon/__tests__/supervision-state.test.ts` +- Modify: `lib/daemon.ts` (record boot-attempt at `runDaemon` top; breadcrumb per phase; ready stamp; `recordBootFailure` in Task 2's catch; `recordCleanExit("shutdown")` in the shutdown verb), `lib/daemon/shutdown.ts` (record exit in `gracefulExit`) + +**Interfaces:** +- Produces: + - `recordBootAttempt(): void` — increments `boot-attempts`, appends nothing. + - `recordDaemonReady(): void` — sets `last-ready-at = Date.now()`. + - `recordBootFailure(phase: BootPhase, reason: string): void` — appends `{ at, phase, reason }` to `recent-failures` (cap 10), sets `last-exit = { at, kind: "boot-failed", code: 1, reason }`. + - `recordCleanExit(kind: "shutdown" | "signal", code: number): void` — sets `last-exit = { at, kind, code }`. + - `readSupervisionState(): { bootAttempts, lastReadyAt, recentFailures, lastExit }`. + - `isCrashLooping(state, now, n = 3, windowMs = 5*60_000): boolean` — ≥ n failures newer than `now - windowMs`. + - `writeBreadcrumb(phase: BootPhase): void` / `clearBreadcrumb(): void` — `~/.mattstack/rt/daemon-boot.json`. + - `type BootPhase = "start" | "crash-handlers" | "events-db" | "state-db" | "socket" | "api" | "ready"`. +- Consumes: `getKvValue`/`setKvValue` from `lib/state/kv-blob.ts`; `getStateDb("daemon")`. + +- [ ] **Step 1: Write the failing test** in `lib/daemon/__tests__/supervision-state.test.ts`: + +```ts +test("boot attempts, ready stamp, failures and last-exit round-trip through kv", () => { + recordBootAttempt(); recordBootAttempt(); + recordDaemonReady(); + recordBootFailure("api", "EADDRINUSE"); + const s = readSupervisionState(); + expect(s.bootAttempts).toBe(2); + expect(s.lastReadyAt).toBeGreaterThan(0); + expect(s.recentFailures.at(-1)).toMatchObject({ phase: "api", reason: "EADDRINUSE" }); + expect(s.lastExit).toMatchObject({ kind: "boot-failed", code: 1 }); +}); + +test("isCrashLooping true at >=3 failures within the window", () => { + const now = 1_000_000; + const fails = [now-10, now-20, now-30].map((at) => ({ at, phase: "api" as const, reason: "x" })); + expect(isCrashLooping({ bootAttempts: 3, lastReadyAt: 0, recentFailures: fails, lastExit: null }, now)).toBe(true); + const old = [{ at: now-10*60_000, phase: "api" as const, reason: "x" }]; + expect(isCrashLooping({ bootAttempts: 1, lastReadyAt: 0, recentFailures: old, lastExit: null }, now)).toBe(false); +}); +``` + +(Use the file's isolated-HOME convention — `bunfig` preload already repoints HOME for `bun test`; `recordBootAttempt` writes to the test state.db.) + +- [ ] **Step 2: Run — expect FAIL** (module does not exist). + +- [ ] **Step 3: Implement `lib/daemon/supervision-state.ts`** with the interfaces above. All reads/writes go through `getKvValue("daemon-supervision", key, fallback, getStateDb("daemon"))` / `setKvValue("daemon-supervision", key, value, getStateDb("daemon"))`. Cap `recent-failures` at 10 on append. Breadcrumb via `writeFileSync(join(RT_DIR, "daemon-boot.json"), JSON.stringify({ at, pid: process.pid, flavor: currentMode(), phase }))` inside a try/catch (never fatal). + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Wire into the boot path** (`lib/daemon.ts`): + - `recordBootAttempt(); writeBreadcrumb("start");` at the top of `runDaemon()` (after Task 3's crash handlers are already at module scope; call these first thing inside `runDaemon`). + - `writeBreadcrumb("events-db" | "state-db" | "socket" | "api")` at each corresponding phase (events.db is module-scope — write that breadcrumb right after `createEventsBus`; state-db right after `openBranchCacheStore`; socket/api at the binds). + - In Task 2's catch: `recordBootFailure(currentPhase, String(err));` (track `currentPhase` in a module var updated alongside each `writeBreadcrumb`). + - `recordDaemonReady(); writeBreadcrumb("ready");` right where `bootPhase = "ready"` is set (line 513). + - Shutdown verb (349-364): before `process.exit(0)`, `recordCleanExit("shutdown", 0);` and set the Task 12 `shuttingDownViaVerb = true` flag (Task 12 adds the flag; here just add the record call). + +- [ ] **Step 6: Wire into signal exit** (`lib/daemon/shutdown.ts` `gracefulExit`): before exit, `recordCleanExit("signal", code)` (Task 12 sets `code` to 1 for bare signals; for now record with the code it exits with). + +- [ ] **Step 7: Add an e2e assertion** in `e2e/tests/daemon.test.ts`: after the Task 5 API-bind-failure spawn, assert `daemon-boot.json` exists with `phase: "api"` and the kv `recent-failures` has an entry (read the state.db in the isolated HOME, or assert via `rt daemon status --json` once Task 10 lands — for Task 9, assert the breadcrumb file only). + +- [ ] **Step 8: Run the unit + e2e tests — expect PASS.** + +- [ ] **Step 9: Commit** + +```bash +git add lib/daemon/supervision-state.ts lib/daemon/__tests__/supervision-state.test.ts lib/daemon.ts lib/daemon/shutdown.ts e2e/tests/daemon.test.ts +git commit -m "daemon: persist boot attempts, failures, last-exit in kv + boot breadcrumb" +``` + +--- + +## Task 10: Status verdicts (0.5 — R001, S026 daemon-side) + +Extend `DaemonStatusVerdict` and `classifyDaemonStatus` with `alive-not-serving`, `parked`, `boot-failed`, `crash-looping`, read from the liveness probe + supervision state + breadcrumb. Expose the fields in `ping`/`/api/status`, and print them in `rt daemon status`. + +**Files:** +- Modify: `lib/daemon-status.ts:14-19` (verdict type), `31-…` (`classifyDaemonStatus`) +- Modify: `lib/daemon/handlers/status.ts:23` (`ping` includes supervision summary; NOT fenced) +- Modify: `commands/daemon.ts:379-…` (`statusLines` prints new verdicts) +- Test: `lib/__tests__/daemon-status.test.ts` + +**Interfaces:** +- Consumes: `readSupervisionState`, `isCrashLooping`, breadcrumb reader from Task 9. +- Produces: `DaemonStatusVerdict` extended with `{ state: "alive-not-serving"; pid: number; detail: "booting" | "wedged" | "quarantined" }`, `{ state: "parked"; pid: number; holderFlavor?: string }`, `{ state: "boot-failed"; reason: string; phase: string }`, `{ state: "crash-looping"; failures: number; reason: string }`. + +- [ ] **Step 1: Write the failing tests** in `lib/__tests__/daemon-status.test.ts`: + +```ts +test("alive pid + failed ping -> alive-not-serving with breadcrumb detail", () => { + const v = classifyDaemonStatus({ installed: true, pingOk: false, pidAlive: true, pid: 42, + breadcrumb: { phase: "socket" }, supervision: emptySupervision() }); + expect(v).toMatchObject({ state: "alive-not-serving", pid: 42, detail: "booting" }); +}); +test("no pid + >=3 recent failures -> crash-looping", () => { + const v = classifyDaemonStatus({ installed: true, pingOk: false, pidAlive: false, pid: null, + supervision: { recentFailures: threeRecentFailures(), lastExit: { kind: "boot-failed", reason: "EADDRINUSE" } } }); + expect(v).toMatchObject({ state: "crash-looping" }); +}); +test("no pid + single boot-failed -> boot-failed with reason", () => { + const v = classifyDaemonStatus({ installed: true, pingOk: false, pidAlive: false, pid: null, + supervision: { recentFailures: [oneFailure("api","EADDRINUSE")], lastExit: { kind: "boot-failed", reason: "EADDRINUSE", phase: "api" } } }); + expect(v).toMatchObject({ state: "boot-failed", reason: "EADDRINUSE", phase: "api" }); +}); +``` + +- [ ] **Step 2: Run — expect FAIL.** + +- [ ] **Step 3: Extend the verdict type and `classifyDaemonStatus`** in `lib/daemon-status.ts` following the resolution order from the design doc (Task 1): not-installed → serving → parked → alive-not-serving → crash-looping → boot-failed → installed-not-running. `classifyDaemonStatus` takes the already-gathered inputs (`pingOk`, `pidAlive`, `pid`, `breadcrumb`, `supervision`); keep it a pure function (the liveness probe and kv/breadcrumb reads happen in `commands/daemon.ts`/the caller, matching the existing `needsLivenessProbe` split). `parked` when breadcrumb phase indicates a flavor standoff; `alive-not-serving` detail = `booting` (phase < ready), `wedged` (phase == ready), `quarantined` (a `*.boot-failed` marker present). + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Surface the data.** In `lib/daemon/handlers/status.ts` `ping`, add a `supervision: { bootAttempts, lastReadyAt, recentFailures: recentFailures.slice(-3), lastExit }` field (read via Task 9). In `commands/daemon.ts` `showStatus`/liveness probe, gather `pidAlive` (`process.kill(pid,0)` on rt.pid, fallback `pgrep -f 'rt --daemon|lib/daemon.ts'` via `runCapture`) and read the breadcrumb + supervision state (when ping fails), then pass to `classifyDaemonStatus`. In `statusLines`, print a line per new verdict (see the design doc's status strings). + +- [ ] **Step 6: Add a `--json` assertion e2e** in `e2e/tests/daemon.test.ts`: after an API-bind-failure spawn, `rt daemon status --json` under the same isolated HOME reports `boot-failed` or `crash-looping` (whichever the failure count yields). Run under `env -i HOME=`. + +- [ ] **Step 7: Run the unit + e2e tests — expect PASS.** `bun test lib/__tests__/daemon-status.test.ts` + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon-status.ts lib/daemon/handlers/status.ts commands/daemon.ts lib/__tests__/daemon-status.test.ts e2e/tests/daemon.test.ts +git commit -m "daemon status: alive-not-serving / parked / boot-failed / crash-looping verdicts" +``` + +--- + +## Task 11: stderr log rotation + stale-crash stamp (0.5 — S029) + +`daemon-stderr.log` is never rotated and its stale contents are shown as "most recent crash". Rotate on open and gate the crash block on mtime. + +**Files:** +- Modify: `lib/daemon-logger.ts:204-219` (`redirectNativeStderr` rotate-on-open) +- Modify: `commands/daemon.ts:716-765` (`showLogs` mtime gate + label) +- Test: `lib/__tests__/daemon-logger.test.ts`, `commands/__tests__/daemon.test.ts` (or wherever `showLogs` is tested) + +**Interfaces:** +- Produces: rotated filename `daemon-stderr..log` (matches the janitor's `LOG_FILE_PATTERN`, so it prunes for free). + +- [ ] **Step 1: Write the failing test** in `lib/__tests__/daemon-logger.test.ts`: + +```ts +test("redirectNativeStderr rotates a non-empty daemon-stderr.log before reopening", () => { + // isolated logs dir with a pre-existing non-empty daemon-stderr.log + writeFileSync(join(logsDir(), "daemon-stderr.log"), "old panic\n"); + redirectNativeStderr(); + const rotated = readdirSync(logsDir()).filter((f) => /^daemon-stderr\.\d{4}-\d{2}-\d{2}\.log$/.test(f)); + expect(rotated.length).toBe(1); + expect(statSync(join(logsDir(), "daemon-stderr.log")).size).toBe(0); +}); +``` + +- [ ] **Step 2: Run — expect FAIL.** + +- [ ] **Step 3: Implement rotate-on-open** in `redirectNativeStderr` (`lib/daemon-logger.ts:204-219`): before `openSync(path, "a")`, if the file exists and is non-empty, `renameSync` it to `daemon-stderr..log` (dedupe with a `.N` suffix if that name already exists, matching the janitor's dated-file convention). Keep the existing swallow-on-failure behavior. + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Write the failing `showLogs` test** — the native-stderr block is hidden when the file mtime predates the current daemon's start: + +```ts +test("showLogs hides the native-stderr block when the file is older than the daemon start", () => { + // stub daemon startedAt newer than the file mtime; assert the red block is not printed +}); +``` + +- [ ] **Step 6: Implement in `showLogs`** (`commands/daemon.ts:724-737`): only print the native-stderr block when the file's `mtime` is newer than the current daemon's `startedAt` (from `ping`); include the mtime in the header (`native stderr (captured )`). When older, skip it silently (or print a one-line "no crash since this daemon started"). + +- [ ] **Step 7: Run — expect PASS.** + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon-logger.ts commands/daemon.ts lib/__tests__/daemon-logger.test.ts +git commit -m "logs: rotate daemon-stderr.log on open; hide stale crash block by mtime" +``` + +--- + +## Task 12: Exit-code semantics (0.6 — S036, S060) + +The `shutdown` verb correctly exits 0, but bare OS signals also exit 0, so an externally-killed daemon stays down. Reserve exit 0 for the verb; exit non-zero on bare signals. + +**Files:** +- Modify: `lib/daemon.ts:349-364` (shutdown verb sets `shuttingDownViaVerb = true`) +- Modify: `lib/daemon/shutdown.ts:57-71` (`gracefulExit` reads the flag) +- Test: `lib/daemon/__tests__/shutdown.test.ts` (new) + +**Interfaces:** +- Produces: a shared boolean the shutdown verb sets before cleanup; `installSignalHandlers` gains a `wasVerbShutdown: () => boolean` option (avoids a cross-module global). `gracefulExit` exits 0 when `wasVerbShutdown()` is true, else `process.exit(1)`. +- Consumes: `recordCleanExit` (Task 9). + +- [ ] **Step 1: Write the failing test** in `lib/daemon/__tests__/shutdown.test.ts`: + +```ts +test("gracefulExit exits 0 after the shutdown verb, 1 on a bare signal", () => { + const exits: number[] = []; + const exit = (c?: number) => { exits.push(c ?? 0); }; + let viaVerb = false; + const handlers = makeGracefulExit({ cleanup: () => {}, flushLogs: () => {}, log: silentLog, + wasVerbShutdown: () => viaVerb, exit, recordCleanExit: () => {} }); + handlers("SIGTERM"); + expect(exits).toEqual([1]); + viaVerb = true; + handlers("SIGTERM"); + expect(exits).toEqual([1, 0]); +}); +``` + +(Refactor `gracefulExit` into a testable `makeGracefulExit(deps)` that returns the handler, injecting `exit`/`recordCleanExit` so no real `process.exit` fires in the test.) + +- [ ] **Step 2: Run — expect FAIL.** + +- [ ] **Step 3: Implement.** In `lib/daemon/shutdown.ts`, refactor `installSignalHandlers`/`gracefulExit` to `makeGracefulExit(deps)` reading `deps.wasVerbShutdown()`: true → `recordCleanExit("shutdown", 0)` + `exit(0)`; false → `recordCleanExit("signal", 1)` + `exit(1)`. In `lib/daemon.ts`, add module-scope `let shuttingDownViaVerb = false;`, set it `true` in the shutdown verb before `cleanup()`, and pass `wasVerbShutdown: () => shuttingDownViaVerb` into `installSignalHandlers` at line 511. The shutdown verb keeps `process.exit(0)`. + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon.ts lib/daemon/shutdown.ts lib/daemon/__tests__/shutdown.test.ts +git commit -m "daemon: bare-signal exit is non-zero (launchd respawns); shutdown verb stays exit 0" +``` + +--- + +## Task 13: Ownership-aware cleanup + eviction death-confirmation (0.6 — S012, S044) + +`cleanup()` unlinks rt.sock/rt.pid unconditionally, and eviction sleeps a blind 300ms. Make cleanup compare-and-delete, and make eviction wait for the old pid to actually die. + +**Files:** +- Modify: `lib/daemon/shutdown.ts:44-46` (ownership-aware unlink) +- Modify: `lib/daemon/boot-reconcile.ts:16-26` (poll-to-death, escalate to SIGKILL) +- Test: `lib/daemon/__tests__/shutdown.test.ts`, `lib/daemon/__tests__/boot-reconcile.test.ts` (new) + +**Interfaces:** +- Produces: `cleanup()` unlinks rt.pid only when its content `=== String(process.pid)`, and gates the rt.sock unlink on that same check. `evictStaleDaemon` polls `process.kill(pid, 0)` up to a bound (e.g. 3s / 30×100ms), escalates to SIGKILL, and only returns once the pid is gone. + +- [ ] **Step 1: Write the failing cleanup test** in `lib/daemon/__tests__/shutdown.test.ts`: + +```ts +test("cleanup does not unlink rt.pid/rt.sock when the pid file belongs to another process", () => { + writeFileSync(DAEMON_PID_PATH, "999999"); // not our pid + writeFileSync(DAEMON_SOCK_PATH, ""); + createCleanup({ ...deps, pid: process.pid })(); + expect(existsSync(DAEMON_PID_PATH)).toBe(true); + expect(existsSync(DAEMON_SOCK_PATH)).toBe(true); +}); +test("cleanup unlinks when the pid file is ours", () => { + writeFileSync(DAEMON_PID_PATH, String(process.pid)); + writeFileSync(DAEMON_SOCK_PATH, ""); + createCleanup({ ...deps, pid: process.pid })(); + expect(existsSync(DAEMON_PID_PATH)).toBe(false); + expect(existsSync(DAEMON_SOCK_PATH)).toBe(false); +}); +``` + +(Inject `pid` into `createCleanup` deps for testability; default to `process.pid`.) + +- [ ] **Step 2: Run — expect FAIL.** + +- [ ] **Step 3: Implement ownership-aware unlink** in `createCleanup` (`lib/daemon/shutdown.ts:44-46`): + +```ts +try { + if (existsSync(DAEMON_PID_PATH) && readFileSync(DAEMON_PID_PATH, "utf8").trim() === String(deps.pid)) { + unlinkSync(DAEMON_PID_PATH); + if (existsSync(DAEMON_SOCK_PATH)) unlinkSync(DAEMON_SOCK_PATH); + } +} catch (err) { deps.log.warn({ err }, "cleanup unlink skipped"); } +``` + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Write the failing eviction test** in `lib/daemon/__tests__/boot-reconcile.test.ts`: + +```ts +test("evictStaleDaemon waits for the old pid to die, escalating to SIGKILL", async () => { + // Spawn a child that ignores SIGTERM; write its pid to rt.pid; assert evict SIGKILLs it and returns only once gone. + const child = Bun.spawn({ cmd: ["bash", "-c", "trap '' TERM; sleep 30"] }); + writeFileSync(DAEMON_PID_PATH, String(child.pid)); + const start = Date.now(); + await evictStaleDaemon(silentLog); // now async + expect(isAlive(child.pid)).toBe(false); + expect(Date.now() - start).toBeLessThan(5000); +}); +``` + +- [ ] **Step 6: Implement in `evictStaleDaemon`** (`lib/daemon/boot-reconcile.ts`): replace `Bun.sleepSync(300)` with an async poll — after SIGTERM, loop `process.kill(pid, 0)` every 100ms up to ~2.5s; if still alive, `process.kill(pid, "SIGKILL")` and poll another ~0.5s; return once `process.kill(pid,0)` throws (pid gone). Make the function `async` and `await` it at the call site (`lib/daemon.ts:396`). Keep the `previousPid === process.pid` self-guard. + +- [ ] **Step 7: Run — expect PASS.** Then `bun test lib/daemon/__tests__/boot-reconcile.test.ts lib/daemon/__tests__/shutdown.test.ts` + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon/shutdown.ts lib/daemon/boot-reconcile.ts lib/daemon.ts lib/daemon/__tests__/shutdown.test.ts lib/daemon/__tests__/boot-reconcile.test.ts +git commit -m "daemon: ownership-aware socket/pid unlink; eviction waits for pid death then SIGKILL" +``` + +--- + +## Task 14: uninstall + start guards (0.6 — S027, S030, S028 CLI-side) + +`rt daemon uninstall` deletes rt.sock/rt.pid from under a live daemon, and `rt daemon start` cannot revive a registered-but-exited-0 daemon. Guard uninstall on liveness; make start escalate to a kickstart route. + +**Files:** +- Modify: `commands/daemon.ts:210-231` (`uninstall` guard), `commands/daemon.ts:235-269` (`start` escalation) +- Modify: `lib/daemon-client.ts:171-185` (`attemptRestart` re-probes liveness) +- Test: `commands/__tests__/daemon.test.ts` (or the existing commands/daemon test location) + +**Interfaces:** +- Consumes: `isDaemonProcessRunning()` (daemon-config.ts:152), `probeSocketHolder()` (park.ts), `activeLaunchdLabel()` (daemon-config.ts:61), `isDaemonRunning()`. + +- [ ] **Step 1: Write the failing uninstall test**: + +```ts +test("uninstall leaves rt.sock/rt.pid and daemon.json when the daemon is still running", async () => { + // stub trayQuery('/daemon/stop') to fail, isDaemonProcessRunning -> true + await uninstall(); + expect(cleanupDaemonFilesSpy).not.toHaveBeenCalled(); + expect(printedRemedy).toContain("launchctl bootout"); +}); +``` + +- [ ] **Step 2: Run — expect FAIL.** + +- [ ] **Step 3: Implement the uninstall guard** (`commands/daemon.ts:210-231`): after a failed/absent `trayQuery("/daemon/stop")`, call `isDaemonProcessRunning()` (and `probeSocketHolder()` as a second signal). Only run `markDaemonUninstalled()` + `cleanupDaemonFiles()` when no live holder; otherwise print the remedy `launchctl bootout gui/$UID/${activeLaunchdLabel()}` and leave the files. Audit other callers of `cleanupDaemonFiles`/`markDaemonUninstalled` (e.g. the dev-mode toggle in `commands/settings.ts`) for the same missing guard and note any in the commit body. + +- [ ] **Step 4: Run — expect PASS.** + +- [ ] **Step 5: Write the failing start-escalation test**: + +```ts +test("start escalates to the restart/kickstart route when the tray acks but the socket stays absent", async () => { + // trayQuery('/daemon/start') returns ok, isDaemonRunning stays false through the poll + await start(); + expect(restartRouteSpy).toHaveBeenCalled(); +}); +``` + +- [ ] **Step 6: Implement start escalation** (`commands/daemon.ts:235-269`): when the tray acked `/daemon/start` but `isDaemonRunning()` stays false through the 12×250ms poll, fall back to the `/daemon/restart` route (kickstart). In `lib/daemon-client.ts:171-185`, make `attemptRestart` re-probe `isDaemonRunning()` after `trayQuery("/daemon/start")` and return `true` only when the daemon actually answers — so the `daemonQuery` nag stops misdirecting. Do NOT change signal-handler exit codes here (that is Task 12). + +- [ ] **Step 7: Run — expect PASS.** + +- [ ] **Step 8: Commit** + +```bash +git add commands/daemon.ts lib/daemon-client.ts commands/__tests__/daemon.test.ts +git commit -m "daemon CLI: uninstall guards on liveness; start escalates to kickstart; attemptRestart re-probes" +``` + +--- + +## Task 15: Retire the stale audit doc (0.8 — R017) + +Replace `docs/daemon-runner-health.md` (which audits deleted subsystems) with a pointer to the current audit and the new supervision design doc. + +**Files:** +- Modify: `docs/daemon-runner-health.md` + +- [ ] **Step 1: Replace the file's contents** with a short pointer: + +```markdown +# Daemon runner health — superseded + +This document audited subsystems (process-manager, remedy-engine, +runner.tsx, workspace-sync) that no longer exist. It is retained only as +a redirect. + +- Current stability audit + roadmap: `docs/daemon-stability-audit-2026-08.md` + (in the daemon-stability-audit worktree). +- Supervision verdicts + exit-code semantics: `docs/daemon-supervision-design.md`. +``` + +(If R058's comment sweep is trivially co-located, remove the one stale comment it names; otherwise leave it.) + +- [ ] **Step 2: Commit** + +```bash +git add docs/daemon-runner-health.md +git commit -m "docs: retire stale daemon-runner-health.md, point at the current audit + supervision design" +``` + +--- + +## Task 16 (OPTIONAL — pending plan-review scope confirmation): Swift tray consumption + +**Do not start without the reviewer's go-ahead** (raised in the plan-milestone report). These edits cannot be verified by the bun/tsc/e2e gate, and the operating rules forbid rebuilding the blessed bundle, so they ship as source-only, unverified changes following the fixer notes: + +- **S026** — `rt-tray/Sources/AppDelegate.swift`: give `.starting` an expiry (record `startingSince`; in `refreshStatus` treat `.starting` as expired after ~30s / 3 failed polls and fall through to `setHealth(.down)`), so the health dot stops sticking yellow; map the new daemon verdicts (Task 10) to dot colors. +- **S028** — `rt-tray/Sources/DaemonLifecycle.swift`: when `register()` returns already-registered but the socket stays unreachable, fall back to `launchctl kickstart` (Kickstart.arguments already exists). +- **S029** — `rt-tray/Sources/TrayLog.swift`: rotate `tray-crash.log` on open (rename-if-nonempty), matching Task 11's `daemon-stderr.log` treatment. +- **S060** — `rt-tray/Sources/AppDelegate.swift:625-627`: fix the stale comment to say `KeepAlive: SuccessfulExit=false` (not `KeepAlive=true`). + +If confirmed, do the comment fix (S060) first (trivial), then S026/S028/S029, one commit each, each commit body noting "source-only, unverified: blessed bundle not rebuilt". + +--- + +## Final verification (before the whole-branch review) + +- [ ] `cd packages/rt-client && bun run build && cd -` (rt-client was touched in Task 5) +- [ ] `bunx tsc --noEmit` — zero errors +- [ ] `bun test lib commands packages scripts` — green +- [ ] `bun run test:e2e` (or `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts` — record which) +- [ ] Request the whole-branch code review (superpowers:requesting-code-review); address findings; re-run the gate. From 71ca8a945c9cc5a201a2412abc95451203bd18fa Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:42:09 -0500 Subject: [PATCH 008/142] hooks-guard: handle fs.watch error events and reconcile stale watchers on refresh (S057) --- lib/daemon/__tests__/hooks-guard.test.ts | 57 ++++++++++++++++++++++++ lib/daemon/hooks-guard.ts | 30 ++++++++++++- 2 files changed, 85 insertions(+), 2 deletions(-) create mode 100644 lib/daemon/__tests__/hooks-guard.test.ts diff --git a/lib/daemon/__tests__/hooks-guard.test.ts b/lib/daemon/__tests__/hooks-guard.test.ts new file mode 100644 index 00000000..ee72af08 --- /dev/null +++ b/lib/daemon/__tests__/hooks-guard.test.ts @@ -0,0 +1,57 @@ +/** + * S057: fs.watch handles in hooks-guard have no 'error' listener (an + * emitted error with no listener is an uncaught exception, which + * installCrashHandlers turns into a daemon exit(1) and a launchd relaunch + * that re-arms the same watchers and hits the same limit); and + * refreshWatchedRepos is add-only, so a relocated or removed repo's stale + * watcher on a dead .git dir is kept forever. + */ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import pino from "pino"; +import { createHooksGuard } from "../hooks-guard.ts"; + +const log = pino({ level: "silent" }); + +let dir: string; +beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "rt-hooks-guard-")); }); +afterEach(() => { rmSync(dir, { recursive: true, force: true }); }); + +function makeRepo(name: string): string { + const repoPath = join(dir, name); + mkdirSync(join(repoPath, ".git"), { recursive: true }); + writeFileSync(join(repoPath, ".git", "config"), "[core]\n"); + return repoPath; +} + +test("an emitted 'error' event on the watcher does not throw, and self-heals the bookkeeping", async () => { + const repoPath = makeRepo("a"); + const guard = createHooksGuard(log); + guard.startWatchingRepo("a", repoPath); + expect(guard.watchedConfigs.size).toBe(1); + + const [configPath, watcher] = [...guard.watchedConfigs.entries()][0]!; + expect(() => watcher.emit("error", new Error("EMFILE"))).not.toThrow(); + // give the close/delete a tick if it's deferred + await Bun.sleep(0); + expect(guard.watchedConfigs.has(configPath)).toBe(false); + guard.closeAll(); +}); + +test("refreshWatchedRepos closes and drops a watcher whose repo left the index (relocated or removed)", () => { + const repoPathA = makeRepo("a"); + const repoPathB = makeRepo("b"); + let index: Record = { a: repoPathA, b: repoPathB }; + const guard = createHooksGuard(log, { loadRepoIndexFn: () => index }); + + guard.refreshWatchedRepos(); + expect(guard.watchedConfigs.size).toBe(2); + + // "b" is relocated/removed: the index no longer carries it. + index = { a: repoPathA }; + guard.refreshWatchedRepos(); + expect(guard.watchedConfigs.size).toBe(1); + guard.closeAll(); +}); diff --git a/lib/daemon/hooks-guard.ts b/lib/daemon/hooks-guard.ts index f0b30d53..758b1569 100644 --- a/lib/daemon/hooks-guard.ts +++ b/lib/daemon/hooks-guard.ts @@ -12,6 +12,7 @@ import type { Logger } from "pino"; import { repoDataDir } from "../rt-paths.ts"; import { runCapture } from "../subprocess.ts"; import { loadRepoIndex, resolveGitConfigPath } from "./repo-index.ts"; +import type { RepoIndex } from "./handlers/types.ts"; export interface HooksGuard { /** Live map of repo git-config watchers (configPath → FSWatcher). */ @@ -26,7 +27,11 @@ export interface HooksGuard { closeAll(): void; } -export function createHooksGuard(log: Logger): HooksGuard { +export function createHooksGuard( + log: Logger, + deps: { loadRepoIndexFn?: () => RepoIndex } = {}, +): HooksGuard { + const loadRepoIndexFn = deps.loadRepoIndexFn ?? loadRepoIndex; const watchedConfigs = new Map(); async function checkAndRepairHooksPath(repoName: string, repoPath: string): Promise { @@ -90,6 +95,16 @@ export function createHooksGuard(log: Logger): HooksGuard { }, 100); // slightly longer debounce: rename events can cluster }); + // FSWatcher is an EventEmitter: an 'error' with no listener is an + // uncaught exception, which installCrashHandlers turns into a daemon + // exit(1) and a launchd relaunch that re-arms the same watchers and + // hits the same limit (EMFILE, a watched dir unlinked, ...). + watcher.on("error", (err) => { + log.warn({ err, repo: repoName, configPath }, "hooks-guard watcher error; dropping this watch"); + watchedConfigs.delete(configPath); + try { watcher.close(); } catch { /* already gone */ } + }); + watchedConfigs.set(configPath, watcher); log.debug({ repo: repoName, file: `${gitDir}/${configFile}` }, "watching repo"); @@ -98,11 +113,22 @@ export function createHooksGuard(log: Logger): HooksGuard { } function refreshWatchedRepos(): void { - const repos = loadRepoIndex(); + const repos = loadRepoIndexFn(); + const liveConfigPaths = new Set(); for (const [repoName, repoPath] of Object.entries(repos)) { if (!existsSync(repoPath)) continue; + const configPath = resolveGitConfigPath(repoPath); + if (configPath) liveConfigPaths.add(configPath); startWatchingRepo(repoName, repoPath); } + // Reconcile, not just add: a repo relocated (rt repos locate) or removed + // from the index leaves its old watcher pointed at a dead .git dir, and + // status().watchedRepos over-reports forever without this. + for (const [configPath, watcher] of watchedConfigs) { + if (liveConfigPaths.has(configPath)) continue; + try { watcher.close(); } catch { /* already gone */ } + watchedConfigs.delete(configPath); + } } function closeAll(): void { From b944a2d3ed0860890a423da3208b85efce3dcdc1 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:42:18 -0500 Subject: [PATCH 009/142] plan: Phase 3 trust-boundary implementation plan (S005/S006/S010/S040-043/S050/S054/S083-085/S092) --- .../plans/2026-08-28-p3-trust-boundary.md | 1745 +++++++++++++++++ 1 file changed, 1745 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-28-p3-trust-boundary.md diff --git a/docs/superpowers/plans/2026-08-28-p3-trust-boundary.md b/docs/superpowers/plans/2026-08-28-p3-trust-boundary.md new file mode 100644 index 00000000..5c8ce0d9 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-p3-trust-boundary.md @@ -0,0 +1,1745 @@ +# Phase 3 — The 127.0.0.1 Trust Boundary — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the daemon's `:9401` trust boundary — WS/CORS origin auth, token-gate the ungated mutating/destructive routes, one shared api-token loader, input hygiene (body-size cap, path-param decode safety, query coercion) — plus the two standalone items S043 (EADDRINUSE diagnostics + typed failure) and S083 (`pathParam()` 400s). + +**Architecture:** All logic lands as small, pure, dependency-injected functions in `lib/daemon/api-auth.ts` and `lib/daemon/api-server.ts` (mirroring the existing `bindApiServerWithRetry`/`needsToken`/`tokenOk` style already in this codebase) so every gate is unit-testable without spinning a real socket. The one exception is the request-body-size cap, which is a Bun.serve runtime option — that gets one small live `Bun.serve({port:0})` integration test. Two findings (S010, S050) have their fix location in sibling-owned files (`lib/daemon/handlers/worktree.ts`, `lib/daemon/freshness.ts`); this plan creates standalone, tested validator/utility modules for them and documents the one-line call-site wiring for the owning job, per the job brief's explicit instruction for S010 (mirrored here for S050 since its fix location is equally out of this job's write fence). + +**Tech Stack:** Bun, TypeScript, `bun:test`, existing `@mattstack/rt-client` settings resolver. + +**Spec:** `/Users/matt/Documents/GitHub/repo-tools/.claude/worktrees/daemon-stability-audit/docs/daemon-stability-audit-2026-08.md` — "Roadmap > Phase 3" (lines 72-79) plus Appendix A findings S005, S006, S010, S040, S041, S042, S043, S050, S054, S083, S084, S085, S092 (read-only input; never modify). + +## Global Constraints + +- Write fence: only `lib/daemon/api-server.ts`, `lib/daemon/api-auth.ts`, `lib/daemon/socket-server.ts`, `lib/daemon/handlers/secrets.ts`, new files under `lib/daemon/` (+ their tests under `lib/daemon/__tests__/`), `e2e/tests/`, `packages/rt-client/src/` (only if the client must send a token), the settings registry file, and `docs/`. Never touch `lib/daemon/handlers/worktree.ts`, `lib/daemon/freshness.ts`, or `lib/daemon.ts` — those are sibling-owned. +- Never start a daemon or run `rt`/`dist/rt` against the real machine; any such invocation in a test must run under `env -i HOME=`. +- `bun test lib commands packages scripts` must stay green; `bunx tsc --noEmit` must report zero errors. +- If `packages/rt-client` is touched, run `bun run build` inside it before the final review (dist-freshness test enforces this). +- Non-browser clients (no `Origin` header: the Swift tray, rt-client from Bun processes, mr-board, gitq, the VS Code extension) must keep working completely unchanged. A browser `Origin` must present the `X-RT-Token` (or, for `/ws`, a `?token=` query param) OR match the new `rt.trustedBrowserOrigins` settings allowlist. +- Never use em dashes or en dashes in code comments, commit messages, or docs (project convention — use parens or "..."). +- Follow TDD: write the failing test first, watch it fail, then implement. + +--- + +## File Map + +| File | Change | +|---|---| +| `lib/daemon/request-limits.ts` | **new** — shared `MAX_REQUEST_BODY_SIZE` constant (S092) | +| `lib/daemon/api-auth.ts` | token singleton + warn (S054); settings-backed origin allowlist + `isBrowserRequestTrusted` (S005/S006); `needsToken` invert-default (S040/S041/S084) | +| `lib/daemon/api-server.ts` | CORS default-deny + `/ws` gate (S005/S006); `broadcastToClients` backpressure/dead-client handling (S042); `pathParam()` + 400 (S083); `coerceQueryParams` (S085); `ApiPortInUseError` + lsof probe (S043) | +| `lib/daemon/socket-server.ts` | wire `MAX_REQUEST_BODY_SIZE` (S092) | +| `lib/daemon/handlers/secrets.ts` | use the shared token singleton (S054) | +| `packages/rt-client/src/settings/registry-defs.ts` | new `rt.trustedBrowserOrigins` key | +| `lib/daemon/git-ref-validation.ts` | **new** — S010 validator (standalone; sibling wires into `worktree.ts`) | +| `lib/daemon/redact-credentials.ts` | **new** — S050 utility (standalone; sibling wires into `freshness.ts`) | +| `docs/daemon-api-auth.md` | **new** — short note on the auth model + the two wiring pointers | + +All new tests live under `lib/daemon/__tests__/`. + +--- + +## Task 1: Shared request body-size cap (S092) + +**Files:** +- Create: `lib/daemon/request-limits.ts` +- Modify: `lib/daemon/api-server.ts` (add `maxRequestBodySize` to the `Bun.serve` options) +- Modify: `lib/daemon/socket-server.ts` (add `maxRequestBodySize` to the `Bun.serve` options) +- Test: `lib/daemon/__tests__/request-body-size.test.ts` + +**Interfaces:** +- Produces: `export const MAX_REQUEST_BODY_SIZE: number` (1 MiB) from `lib/daemon/request-limits.ts`, imported by both servers. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/request-body-size.test.ts +/** + * Bun enforces `maxRequestBodySize` itself (413 before the handler runs) -- + * this is a live-server test, not a pure-function one, because there is no + * pure function to unit test: the cap is a Bun.serve runtime option. + */ +import { describe, test, expect, afterEach } from "bun:test"; +import type { Server } from "bun"; +import { MAX_REQUEST_BODY_SIZE } from "../request-limits.ts"; + +let server: Server | undefined; + +afterEach(() => { + server?.stop(true); + server = undefined; +}); + +describe("MAX_REQUEST_BODY_SIZE", () => { + test("is set to 1 MiB", () => { + expect(MAX_REQUEST_BODY_SIZE).toBe(1024 * 1024); + }); + + test("Bun rejects a body over the cap with a 4xx before the handler runs", async () => { + let handlerRan = false; + server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + maxRequestBodySize: 10, // tiny cap for a fast, deterministic test + async fetch(req) { + handlerRan = true; + await req.text(); + return new Response("ok"); + }, + }); + const res = await fetch(`http://127.0.0.1:${server.port}/`, { + method: "POST", + body: "x".repeat(1000), + }); + expect(res.status).toBeGreaterThanOrEqual(400); + expect(res.status).toBeLessThan(500); + expect(handlerRan).toBe(false); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/request-body-size.test.ts` +Expected: FAIL — `request-limits.ts` does not exist yet (module not found). + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/request-limits.ts +/** + * Shared cap on request body size for both daemon servers (api-server.ts's + * :9401 HTTP/WS surface and socket-server.ts's unix-socket IPC channel). + * Neither transport authenticates reads, so an unbounded body (Bun's + * default is 128 MB) lets any same-user process or a cross-origin browser + * request stall the daemon's single event loop parsing a giant payload. + * Real payloads on both transports are kilobytes; 1 MiB costs nothing and + * turns an oversized request into an immediate 413 instead. + */ +export const MAX_REQUEST_BODY_SIZE = 1024 * 1024; +``` + +Then in `lib/daemon/api-server.ts`, add the import and option: + +```ts +import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; +``` + +...and inside the `Bun.serve({ ... })` options object passed to `bindApiServerWithRetry`, add: + +```ts + maxRequestBodySize: MAX_REQUEST_BODY_SIZE, +``` + +(alongside the existing `port`, `hostname`, `idleTimeout` keys). + +In `lib/daemon/socket-server.ts`, add the same import and, inside the `Bun.serve({ ... })` call, add: + +```ts + maxRequestBodySize: MAX_REQUEST_BODY_SIZE, +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/request-body-size.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice for a quick regression check** + +Run: `bun test lib/daemon` +Expected: PASS (no existing test asserts a specific absence of `maxRequestBodySize`) + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/request-limits.ts lib/daemon/api-server.ts lib/daemon/socket-server.ts lib/daemon/__tests__/request-body-size.test.ts +git commit -m "daemon: cap request body size at 1 MiB on both servers (S092)" +``` + +--- + +## Task 2: One shared api-token loader, with a warn on persist failure (S054) + +**Files:** +- Modify: `lib/daemon/api-auth.ts` +- Modify: `lib/daemon/api-server.ts` (use the new getter instead of the raw loader) +- Modify: `lib/daemon/handlers/secrets.ts` (use the new getter as the default override) +- Test: `lib/daemon/__tests__/api-auth.test.ts` (extend) + +**Interfaces:** +- Produces: `export function getApiToken(tokenPath?: string): string` and `export function reloadApiToken(tokenPath?: string): string` from `lib/daemon/api-auth.ts`. `loadOrCreateApiToken` keeps its existing signature and export (still the underlying file I/O primitive; `getApiToken`/`reloadApiToken` wrap it with an in-memory cache). +- Consumes: `lazyChildLogger` from `../daemon-logger.ts` (already exported; see `getDaemonLogger`/`lazyChildLogger` in `lib/daemon-logger.ts`). + +- [ ] **Step 1: Write the failing test** + +Append to `lib/daemon/__tests__/api-auth.test.ts`: + +```ts +import { getApiToken, reloadApiToken, loadOrCreateApiToken } from "../api-auth.ts"; +import { mkdtempSync, rmSync, writeFileSync, chmodSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +describe("getApiToken / reloadApiToken singleton", () => { + test("getApiToken caches: a second call does not re-read the file even if it changes underneath", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + const first = reloadApiToken(tokenPath); // seed the cache with a known path + writeFileSync(tokenPath, "a-different-token", { mode: 0o600 }); + const second = getApiToken(tokenPath); // ignores the new file content -- cached + expect(second).toBe(first); + expect(second).not.toBe("a-different-token"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("reloadApiToken re-reads and updates the cache", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + reloadApiToken(tokenPath); + writeFileSync(tokenPath, "rotated-token", { mode: 0o600 }); + const reloaded = reloadApiToken(tokenPath); + expect(reloaded).toBe("rotated-token"); + expect(getApiToken(tokenPath)).toBe("rotated-token"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("loadOrCreateApiToken still works standalone (unchanged primitive)", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + const a = loadOrCreateApiToken(tokenPath); + const b = loadOrCreateApiToken(tokenPath); + expect(a).toBe(b); + expect(a.length).toBeGreaterThan(0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-auth.test.ts` +Expected: FAIL — `getApiToken`/`reloadApiToken` are not exported yet. + +- [ ] **Step 3: Write minimal implementation** + +In `lib/daemon/api-auth.ts`, add near the top (after the existing imports) and after `loadOrCreateApiToken`: + +```ts +import { lazyChildLogger } from "../daemon-logger.ts"; + +const log = lazyChildLogger("api-auth"); +``` + +Change the existing `loadOrCreateApiToken`'s silent write-failure catch to log a warning (this is the only edit to that function's body): + +```ts +export function loadOrCreateApiToken(tokenPath: string = API_TOKEN_PATH): string { + try { + if (existsSync(tokenPath)) { + const existing = readFileSync(tokenPath, "utf8").trim(); + if (existing) return existing; + } + } catch { /* fall through to regenerate */ } + const token = randomUUID(); + try { + mkdirSync(RT_DIR, { recursive: true }); + writeFileSync(tokenPath, token, { mode: 0o600 }); + } catch (err) { + log.warn({ err, tokenPath }, "could not persist api-token; enforced in-memory only this run, so a client reading the file will disagree until the daemon restarts"); + } + return token; +} + +/** + * `getApiToken`/`reloadApiToken` share ONE in-memory value between + * api-server.ts and the secrets handler (S054): before this, api-server + * captured a token once at boot while the secrets handler called + * `loadOrCreateApiToken()` fresh on every request, so an external rotation + * (deleting api-token to force a new one) left the two permanently + * disagreeing about which token is current -- and if the token dir was + * unwritable, the secrets handler regenerated a brand new random token on + * every single call, never matching anything a client could read from disk. + * Both consumers now read the SAME cached value; a rotation only takes + * effect for both after `reloadApiToken()` runs or the daemon restarts, + * either of which was already the closest thing to a happy path before. + */ +let cachedApiToken: string | null = null; + +export function getApiToken(tokenPath: string = API_TOKEN_PATH): string { + if (cachedApiToken === null) cachedApiToken = loadOrCreateApiToken(tokenPath); + return cachedApiToken; +} + +export function reloadApiToken(tokenPath: string = API_TOKEN_PATH): string { + cachedApiToken = loadOrCreateApiToken(tokenPath); + return cachedApiToken; +} +``` + +In `lib/daemon/api-server.ts`, change the import and the one call site: + +```ts +import { needsToken, tokenOk, getApiToken } from "./api-auth.ts"; +``` + +```ts + const apiToken = getApiToken(); +``` + +(replaces `const apiToken = loadOrCreateApiToken();`) + +In `lib/daemon/handlers/secrets.ts`, change the import and the one default-override: + +```ts +import { getApiToken, tokenOk } from "../api-auth.ts"; +``` + +```ts + /** Defaults to `getApiToken` (the real ~/.mattstack/rt/api-token, shared with api-auth.ts and api-server.ts). */ + apiToken?: () => string; +``` + +(only the doc comment line changes; the field name/type is unchanged) + +```ts + const apiToken = overrides.apiToken ?? (() => getApiToken()); +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-auth.test.ts lib/daemon/__tests__/secrets-handler.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice** + +Run: `bun test lib/daemon` +Expected: PASS + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/api-auth.ts lib/daemon/api-server.ts lib/daemon/handlers/secrets.ts lib/daemon/__tests__/api-auth.test.ts +git commit -m "daemon: one shared api-token cache for api-server and secrets handler (S054)" +``` + +--- + +## Task 3: Origin allowlist settings key + trust helper + needsToken invert-default (S005, S006, S040, S041, S084) + +**Files:** +- Modify: `packages/rt-client/src/settings/registry-defs.ts` +- Modify: `lib/daemon/api-auth.ts` +- Test: `lib/daemon/__tests__/api-auth.test.ts` (extend) + +**Interfaces:** +- Produces: + - `export function getTrustedBrowserOrigins(): readonly string[]` (reads the `rt.trustedBrowserOrigins` setting, `[]` on any error or when unset) + - `export function isOriginAllowed(origin: string, allowedOrigins: readonly string[]): boolean` + - `export function isBrowserRequestTrusted(origin: string | null, token: string | null, apiToken: string, allowedOrigins: readonly string[]): boolean` (the shared decision Task 4 wires into both the CORS header and the `/ws` gate: no `Origin` header at all -> trusted (non-browser); otherwise a valid token OR an allowlisted origin) + - `needsToken(method, pathname)` changes shape: default-gated for every method except `GET`/`HEAD`/`OPTIONS`, plus two explicit GET exceptions (`/api/secrets`, `/api/notifications`) that are gated despite being reads. +- Consumes: `getSetting` from `../settings/resolve.ts` (already re-exported from `@mattstack/rt-client`; see `lib/chat-viewer-url.ts` for the exact import/usage pattern), `tokenOk` (already in this file). + +- [ ] **Step 1: Add the registry key** + +In `packages/rt-client/src/settings/registry-defs.ts`, add a new entry after the `rt.hooks` block (around line 207, right before the `// --- mattstack (installer-lane) ---` comment): + +```ts + { + key: "rt.trustedBrowserOrigins", + type: "array", + scopes: ["user", "machine"], + default: [], + merge: "replace", + description: "Browser Origins (scheme://host:port, exact string match) trusted to read the :9401 daemon API and subscribe to /ws without presenting the local api-token -- e.g. a locally-hosted console or chat-viewer dev server. Empty by default: every current mattstack consumer (the CLI, the Swift tray, rt-client from Bun/Node processes, the VS Code extension) is a non-browser client (sends no Origin header at all) and is unaffected either way.", + }, +``` + +- [ ] **Step 2: Build rt-client so the new key is live for tests that resolve it** + +Run: `cd packages/rt-client && bun run build && cd -` +Expected: build succeeds; `dist/` picks up the new registry row. + +- [ ] **Step 3: Write the failing tests** + +Append to `lib/daemon/__tests__/api-auth.test.ts`: + +```ts +import { isOriginAllowed, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "../api-auth.ts"; + +describe("isOriginAllowed", () => { + test("exact match", () => { + expect(isOriginAllowed("http://localhost:5544", ["http://localhost:5544"])).toBe(true); + }); + test("no match", () => { + expect(isOriginAllowed("http://evil.example", ["http://localhost:5544"])).toBe(false); + }); + test("empty allowlist matches nothing", () => { + expect(isOriginAllowed("http://localhost:5544", [])).toBe(false); + }); +}); + +describe("isBrowserRequestTrusted", () => { + const apiToken = "the-real-token"; + + test("no Origin header at all -- a non-browser client -- is trusted regardless of token or allowlist", () => { + expect(isBrowserRequestTrusted(null, null, apiToken, [])).toBe(true); + }); + + test("a browser Origin with the correct token is trusted even off the allowlist", () => { + expect(isBrowserRequestTrusted("http://evil.example", apiToken, apiToken, [])).toBe(true); + }); + + test("a browser Origin with a wrong token and not on the allowlist is rejected", () => { + expect(isBrowserRequestTrusted("http://evil.example", "wrong", apiToken, [])).toBe(false); + }); + + test("a browser Origin with no token but on the allowlist is trusted", () => { + expect(isBrowserRequestTrusted("http://localhost:5544", null, apiToken, ["http://localhost:5544"])).toBe(true); + }); + + test("a browser Origin with no token and not on the allowlist is rejected", () => { + expect(isBrowserRequestTrusted("http://localhost:5544", null, apiToken, [])).toBe(false); + }); +}); + +describe("getTrustedBrowserOrigins", () => { + test("returns an array (empty by default in an isolated test HOME)", () => { + const origins = getTrustedBrowserOrigins(); + expect(Array.isArray(origins)).toBe(true); + }); +}); +``` + +Also extend the existing `needsToken` describe block in the same file with the new cases (add these `test`s inside the existing `describe("needsToken", ...)`): + +```ts + test("refresh requires a token now (S040)", () => { + expect(needsToken("POST", "/api/refresh")).toBe(true); + }); + + test("hooks repair requires a token now (S040/S084)", () => { + expect(needsToken("POST", "/api/hooks/my-repo/repair")).toBe(true); + }); + + test("notifications GET (destructive drain) requires a token now (S041)", () => { + expect(needsToken("GET", "/api/notifications")).toBe(true); + }); + + test("every non-GET/HEAD/OPTIONS method defaults to requiring a token", () => { + expect(needsToken("POST", "/api/some-future-mutating-route")).toBe(true); + expect(needsToken("PUT", "/api/anything")).toBe(true); + expect(needsToken("DELETE", "/api/anything")).toBe(true); + }); + + test("plain reads still do not require a token", () => { + expect(needsToken("GET", "/api/repos")).toBe(false); + expect(needsToken("GET", "/api/cache")).toBe(false); + expect(needsToken("HEAD", "/api/repos")).toBe(false); + }); +``` + +- [ ] **Step 4: Run tests to verify they fail** + +Run: `bun test lib/daemon/__tests__/api-auth.test.ts` +Expected: FAIL — `isOriginAllowed`/`isBrowserRequestTrusted`/`getTrustedBrowserOrigins` not exported; the two new `needsToken` cases for `/api/refresh` and `/api/hooks/.../repair` and `/api/notifications` fail against the current allowlist-based implementation. + +- [ ] **Step 5: Write minimal implementation** + +In `lib/daemon/api-auth.ts`, add the import (alongside the existing ones) and the new functions: + +```ts +import { getSetting } from "../settings/resolve.ts"; +``` + +```ts +/** + * `rt.trustedBrowserOrigins` -- see registry-defs.ts. Read fresh on every + * call (the settings resolver is deliberately unmemoized), wrapped in a + * try/catch since a request-path settings read must never 500 the daemon + * over a malformed store file. + */ +export function getTrustedBrowserOrigins(): readonly string[] { + try { + const resolved = getSetting("rt.trustedBrowserOrigins"); + return Array.isArray(resolved.value) ? resolved.value : []; + } catch { + return []; + } +} + +export function isOriginAllowed(origin: string, allowedOrigins: readonly string[]): boolean { + return allowedOrigins.includes(origin); +} + +/** + * The 127.0.0.1 trust boundary (S005/S006): the daemon binds loopback-only, + * but any web page the user visits also runs on 127.0.0.1 and can send a + * request. A request with NO Origin header at all is not a browser fetch -- + * it is the CLI, the Swift tray, rt-client from a Bun/Node process, or the + * VS Code extension, none of which send one -- so it is trusted unchanged. + * A request that DOES carry an Origin header is trusted only if it presents + * the local api-token or its Origin is on the explicit allowlist. + */ +export function isBrowserRequestTrusted( + origin: string | null, + token: string | null, + apiToken: string, + allowedOrigins: readonly string[], +): boolean { + if (!origin) return true; + if (tokenOk(token, apiToken)) return true; + return isOriginAllowed(origin, allowedOrigins); +} +``` + +Replace the whole `needsToken` function with: + +```ts +/** + * True when a request mutates state, or (secrets/notifications) returns or + * drains something a GET should not silently consume, and must present the + * local token. Default-gated for every method except GET/HEAD/OPTIONS (S040: + * an allowlist-by-path guaranteed the next mutating route would ship + * unguarded) plus two explicit GET exceptions whose verb lies about being a + * read. + */ +export function needsToken(method: string, pathname: string): boolean { + if (method === "GET" || method === "HEAD" || method === "OPTIONS") { + // Gated despite being a GET: /api/secrets's response body IS a + // credential (S054); /api/notifications DRAINS the queue (S041), so its + // verb lies about being a read the way every other GET here is not. + if (pathname === "/api/secrets") return true; + if (pathname === "/api/notifications") return true; + return false; + } + return true; +} +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `bun test lib/daemon/__tests__/api-auth.test.ts` +Expected: PASS + +- [ ] **Step 7: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. + +- [ ] **Step 8: Commit** + +```bash +git add packages/rt-client/src/settings/registry-defs.ts packages/rt-client/dist lib/daemon/api-auth.ts lib/daemon/__tests__/api-auth.test.ts +git commit -m "daemon: rt.trustedBrowserOrigins allowlist + needsToken invert-default (S005/S006/S040/S041/S084)" +``` + +--- + +## Task 4: Wire CORS default-deny and the `/ws` origin/token gate into api-server.ts (S005, S006) + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Test: `lib/daemon/__tests__/api-server-cors-ws.test.ts` (new) + +**Interfaces:** +- Consumes: `isBrowserRequestTrusted`, `getTrustedBrowserOrigins` from `./api-auth.ts` (Task 3). +- Produces: `export function buildCorsHeaders(origin: string | null, trusted: boolean): Record` — a pure function so the header-shape logic is unit-testable without a real server. The live `/ws` gate and the live CORS-header wiring inside `fetch()` are exercised indirectly through this same function plus the `isBrowserRequestTrusted` tests from Task 3 (this codebase's existing convention -- see `bindApiServerWithRetry` -- is to keep the decision logic in pure, tested functions and keep the `Bun.serve` wiring itself thin). + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/api-server-cors-ws.test.ts +import { describe, test, expect } from "bun:test"; +import { buildCorsHeaders } from "../api-server.ts"; + +describe("buildCorsHeaders", () => { + test("no Origin header: no Access-Control-Allow-Origin is set (non-browser request, CORS is irrelevant)", () => { + const headers = buildCorsHeaders(null, true); + expect(headers["Access-Control-Allow-Origin"]).toBeUndefined(); + }); + + test("an untrusted Origin gets no Access-Control-Allow-Origin (default-deny, S006)", () => { + const headers = buildCorsHeaders("http://evil.example", false); + expect(headers["Access-Control-Allow-Origin"]).toBeUndefined(); + }); + + test("a trusted Origin is echoed back with Vary: Origin", () => { + const headers = buildCorsHeaders("http://localhost:5544", true); + expect(headers["Access-Control-Allow-Origin"]).toBe("http://localhost:5544"); + expect(headers["Vary"]).toBe("Origin"); + }); + + test("always advertises the methods/headers a preflight needs, trusted or not", () => { + const headers = buildCorsHeaders("http://evil.example", false); + expect(headers["Access-Control-Allow-Methods"]).toContain("POST"); + expect(headers["Access-Control-Allow-Headers"]).toContain("X-RT-Token"); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-server-cors-ws.test.ts` +Expected: FAIL — `buildCorsHeaders` is not exported yet. + +- [ ] **Step 3: Write minimal implementation** + +In `lib/daemon/api-server.ts`, change the import line to add the two new helpers: + +```ts +import { needsToken, tokenOk, getApiToken, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "./api-auth.ts"; +``` + +Add this exported pure function near the top of the file (after the `wsClients`/`broadcast` block, before `startApiServer`): + +```ts +/** + * CORS default-deny (S006): a browser page on an untrusted Origin still gets + * its request served (127.0.0.1 loopback + the per-route token gate are the + * real defenses), but the response carries no Access-Control-Allow-Origin, + * so the page's own JavaScript cannot read the body. A request with no + * Origin at all (every non-browser consumer today) needs no CORS headers. + */ +export function buildCorsHeaders(origin: string | null, trusted: boolean): Record { + const headers: Record = { + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type, X-RT-Token", + }; + if (origin && trusted) { + headers["Access-Control-Allow-Origin"] = origin; + headers["Vary"] = "Origin"; + } + return headers; +} +``` + +Now replace the body of `fetch(req, server)` from the top through the old `corsHeaders` declaration. The current code (for reference) is: + +```ts + async fetch(req, server) { + const url = new URL(req.url); + + // WebSocket upgrade — broadcast channel + if (url.pathname === "/ws") { + if (server.upgrade(req, { data: { kind: "broadcast" } })) return undefined as any; + return new Response("WebSocket upgrade failed", { status: 400 }); + } + + // CORS headers for local dev + const corsHeaders = { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type", + }; +``` + +Replace it with: + +```ts + async fetch(req, server) { + const url = new URL(req.url); + const origin = req.headers.get("origin"); + const allowedOrigins = getTrustedBrowserOrigins(); + + // WebSocket upgrade — broadcast channel. Browsers cannot set custom + // headers on a WS handshake, so the token (when a browser page wants + // to identify itself) travels as a ?token= query param instead of + // X-RT-Token (S005). + if (url.pathname === "/ws") { + const wsToken = url.searchParams.get("token"); + if (!isBrowserRequestTrusted(origin, wsToken, apiToken, allowedOrigins)) { + return new Response("origin not allowed", { status: 403 }); + } + if (server.upgrade(req, { data: { kind: "broadcast" } })) return undefined as any; + return new Response("WebSocket upgrade failed", { status: 400 }); + } + + // CORS: default-deny. A trusted Origin (token or allowlist) gets its + // Origin echoed back; anything else gets no Access-Control-Allow-Origin + // at all, so a malicious page's own JS cannot read the response (S006). + const trusted = isBrowserRequestTrusted(origin, req.headers.get("x-rt-token"), apiToken, allowedOrigins); + const corsHeaders = buildCorsHeaders(origin, trusted); +``` + +No other lines in `fetch()` need to change — every later reference to `corsHeaders` in the function already reads from this same local binding. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-server-cors-ws.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/__tests__/api-server-cors-ws.test.ts +git commit -m "daemon: default-deny CORS and gate /ws on origin/token (S005/S006)" +``` + +--- + +## Task 5: broadcast() drops dead/backpressured WS clients (S042) + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Test: `lib/daemon/__tests__/api-server-broadcast.test.ts` (new) + +**Interfaces:** +- Produces: `export function broadcastToClients(clients: Iterable, type: string, data: any, log: { warn: (o: unknown, m: string) => void }): void` where `BroadcastTarget = { send(data: string): number; close(): void }` (exported type). `broadcast()` becomes a thin wrapper calling `broadcastToClients(wsClients, type, data, log)`. +- Consumes: nothing new. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/api-server-broadcast.test.ts +import { describe, test, expect } from "bun:test"; +import { broadcastToClients, type BroadcastTarget } from "../api-server.ts"; + +function fakeClient(sendReturns: number[]): BroadcastTarget & { closed: boolean; sent: string[] } { + const sent: string[] = []; + let i = 0; + const client = { + closed: false, + sent, + send(data: string) { + sent.push(data); + const ret = sendReturns[Math.min(i, sendReturns.length - 1)]; + i++; + return ret; + }, + close() { client.closed = true; }, + }; + return client; +} + +function fakeLog() { + const warns: unknown[] = []; + return { warn: (o: unknown, _m: string) => { warns.push(o); }, warns }; +} + +describe("broadcastToClients", () => { + test("a healthy client (positive send return) is never closed", () => { + const client = fakeClient([42]); + broadcastToClients([client], "status", { ok: true }, fakeLog()); + expect(client.closed).toBe(false); + expect(client.sent.length).toBe(1); + }); + + test("a send() returning 0 (dropped frame) closes the client immediately and logs a warning", () => { + const client = fakeClient([0]); + const log = fakeLog(); + broadcastToClients([client], "status", { ok: true }, log); + expect(client.closed).toBe(true); + expect(log.warns.length).toBe(1); + }); + + test("a send() returning -1 (backpressure) is tolerated for a few sends before closing", () => { + const client = fakeClient([-1, -1, -1, -1]); + const log = fakeLog(); + broadcastToClients([client], "a", {}, log); + expect(client.closed).toBe(false); + broadcastToClients([client], "b", {}, log); + expect(client.closed).toBe(false); + broadcastToClients([client], "c", {}, log); + // third consecutive backpressure event closes the client + expect(client.closed).toBe(true); + }); + + test("a successful send resets the backpressure counter", () => { + const client = fakeClient([-1, -1, 99, -1, -1, -1]); + const log = fakeLog(); + broadcastToClients([client], "a", {}, log); // -1 (count=1) + broadcastToClients([client], "b", {}, log); // -1 (count=2) + broadcastToClients([client], "c", {}, log); // 99 -- resets to 0 + expect(client.closed).toBe(false); + broadcastToClients([client], "d", {}, log); // -1 (count=1) + broadcastToClients([client], "e", {}, log); // -1 (count=2) + expect(client.closed).toBe(false); + broadcastToClients([client], "f", {}, log); // -1 (count=3) -- closes + expect(client.closed).toBe(true); + }); + + test("a client whose send() throws is treated as gone: caught, not propagated", () => { + const client: BroadcastTarget = { + send() { throw new Error("ECONNRESET"); }, + close() { /* no-op */ }, + }; + expect(() => broadcastToClients([client], "a", {}, fakeLog())).not.toThrow(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-server-broadcast.test.ts` +Expected: FAIL — `broadcastToClients`/`BroadcastTarget` not exported yet. + +- [ ] **Step 3: Write minimal implementation** + +Replace the existing `broadcast()` function and the `wsClients` declaration block in `lib/daemon/api-server.ts`: + +Current code (for reference): + +```ts +const wsClients = new Set>(); + +/** Broadcast an event to all connected WebSocket clients. */ +export function broadcast(type: string, data: any): void { + if (wsClients.size === 0) return; + const msg = JSON.stringify({ type, data, timestamp: Date.now() }); + for (const ws of wsClients) { + try { ws.send(msg); } catch { /* client disconnected */ } + } +} +``` + +Replace with: + +```ts +const wsClients = new Set>(); + +/** Consecutive Bun `ws.send()` backpressure (-1) returns tolerated before a + client is dropped as chronically stalled. */ +const BACKPRESSURE_CLOSE_THRESHOLD = 3; +const backpressureCounts = new WeakMap(); + +export interface BroadcastTarget { + send(data: string): number; + close(): void; +} + +/** + * Sends one frame to every client, dropping any that Bun's own send() return + * value marks as gone (S042). `ws.send()` never throws on a dead socket -- + * it returns 0 (this send silently failed) or -1 (backpressure) -- so a + * disconnected or stalled console/chat-viewer tab used to keep receiving a + * SUBSET of frames forever with nothing logged. 0 means Bun already dropped + * this exact frame for this client: closing immediately (rather than + * counting) is correct because the client's own reconnect logic is the only + * way it recovers a consistent stream. -1 means backpressure, which can be + * transient, so a few in a row are tolerated before giving up on the client. + */ +export function broadcastToClients( + clients: Iterable, + type: string, + data: any, + log: { warn: (o: unknown, m: string) => void }, +): void { + const msg = JSON.stringify({ type, data, timestamp: Date.now() }); + for (const client of clients) { + let result: number; + try { + result = client.send(msg); + } catch (err) { + log.warn({ err }, "ws client send threw; dropping"); + try { client.close(); } catch { /* already gone */ } + continue; + } + if (result === 0) { + log.warn({ type }, "ws client dropped a frame (send()=0); closing so its reconnect resyncs"); + backpressureCounts.delete(client); + try { client.close(); } catch { /* already gone */ } + } else if (result === -1) { + const count = (backpressureCounts.get(client) ?? 0) + 1; + if (count >= BACKPRESSURE_CLOSE_THRESHOLD) { + log.warn({ type, count }, "ws client chronically backpressured; closing"); + backpressureCounts.delete(client); + try { client.close(); } catch { /* already gone */ } + } else { + backpressureCounts.set(client, count); + } + } else { + backpressureCounts.delete(client); + } + } +} + +/** Broadcast an event to all connected WebSocket clients. */ +export function broadcast(type: string, data: any): void { + if (wsClients.size === 0) return; + broadcastToClients(wsClients, type, data, apiServerLog); +} +``` + +This introduces one new module-level binding, `apiServerLog`, since `broadcast()` previously had no logger in scope at all (it is called from many places across the daemon, not just from inside `startApiServer`). Add it right after the existing `wsClients`-adjacent declarations, and set it from `startApiServer`: + +```ts +let apiServerLog: { warn: (o: unknown, m: string) => void } = { warn: () => {} }; +``` + +...and inside `startApiServer`, right after `const { handleCommand, log } = deps;`, add: + +```ts + apiServerLog = log; +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-server-broadcast.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. (`ServerWebSocket` structurally satisfies `BroadcastTarget` since it has both `send(string): number` and `close(): void`, so `broadcastToClients(wsClients, ...)` type-checks with no cast.) + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/__tests__/api-server-broadcast.test.ts +git commit -m "daemon: broadcast() drops dead/backpressured ws clients instead of silently dropping frames (S042)" +``` + +--- + +## Task 6: pathParam() helper — malformed %-encoding returns 400, not a logged 500 (S083) + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Test: `lib/daemon/__tests__/api-server-path-param.test.ts` (new) + +**Interfaces:** +- Produces: `export function pathParam(pathname: string, prefix: string, suffix?: string): string | undefined` (returns `undefined` when the pathname doesn't match the prefix/suffix shape, the captured segment is empty, or `decodeURIComponent` throws). + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/api-server-path-param.test.ts +import { describe, test, expect } from "bun:test"; +import { pathParam } from "../api-server.ts"; + +describe("pathParam", () => { + test("decodes a clean prefix-only param", () => { + expect(pathParam("/api/cache/main", "/api/cache/")).toBe("main"); + }); + + test("decodes a URL-encoded segment", () => { + expect(pathParam("/api/cache/feature%2Ffoo", "/api/cache/")).toBe("feature/foo"); + }); + + test("returns undefined for malformed %-encoding instead of throwing", () => { + expect(pathParam("/api/cache/%E0%A4%A", "/api/cache/")).toBeUndefined(); + }); + + test("returns undefined when the pathname doesn't start with the prefix", () => { + expect(pathParam("/api/other/main", "/api/cache/")).toBeUndefined(); + }); + + test("handles a prefix+suffix pair (hooks repair shape)", () => { + expect(pathParam("/api/hooks/my-repo/repair", "/api/hooks/", "/repair")).toBe("my-repo"); + }); + + test("prefix+suffix: malformed encoding still returns undefined", () => { + expect(pathParam("/api/hooks/%E0%A4%A/repair", "/api/hooks/", "/repair")).toBeUndefined(); + }); + + test("prefix+suffix: wrong suffix returns undefined", () => { + expect(pathParam("/api/hooks/my-repo/other", "/api/hooks/", "/repair")).toBeUndefined(); + }); + + test("an empty captured segment returns undefined", () => { + expect(pathParam("/api/cache/", "/api/cache/")).toBeUndefined(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-server-path-param.test.ts` +Expected: FAIL — `pathParam` not exported yet. + +- [ ] **Step 3: Write minimal implementation** + +Add this exported function to `lib/daemon/api-server.ts` (near `buildCorsHeaders`, before `startApiServer`): + +```ts +/** + * Decodes one path segment between a fixed prefix (and optional suffix), + * returning `undefined` (never throwing) on any shape mismatch or malformed + * %-encoding (S083). Before this, each parameterized route hand-rolled its + * own decodeURIComponent inside the route's try block, so a malformed + * segment fell through to the OUTER catch and came back as a logged 500; + * every route using this helper instead gets a clean 400. + */ +export function pathParam(pathname: string, prefix: string, suffix = ""): string | undefined { + if (!pathname.startsWith(prefix)) return undefined; + if (suffix && !pathname.endsWith(suffix)) return undefined; + const end = suffix ? pathname.length - suffix.length : pathname.length; + if (end <= prefix.length) return undefined; + const raw = pathname.slice(prefix.length, end); + try { + return decodeURIComponent(raw); + } catch { + return undefined; + } +} +``` + +Now wire it into the three routes inside `fetch()`. Replace the `/api/cache/:branch` block: + +```ts + // Single branch lookup: /api/cache/:branch + if (url.pathname.startsWith("/api/cache/") && req.method === "GET") { + const branch = pathParam(url.pathname, "/api/cache/"); + if (branch === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); + } + const result = await handleCommand("cache:read", { branches: [branch] }, req.signal); + return Response.json(result, { headers: corsHeaders }); + } +``` + +Replace the `/api/hooks/:repo/repair` block: + +```ts + // Hooks repair: /api/hooks/:repo/repair + if (url.pathname.startsWith("/api/hooks/") && url.pathname.endsWith("/repair") && req.method === "POST") { + const repo = pathParam(url.pathname, "/api/hooks/", "/repair"); + if (repo === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); + } + const result = await handleCommand("hooks:repair", { repo }, req.signal); + return Response.json(result, { headers: corsHeaders }); + } +``` + +Replace the `/api/runs/:repo/:runId` block (this one already guarded against the URIError; simplify it onto the shared helper so there is one decode path in the file, not two): + +```ts + // Runs detail: /api/runs/:repo/:runId + if (url.pathname.startsWith("/api/runs/") && req.method === "GET") { + const rest = pathParam(url.pathname, "/api/runs/"); + if (rest === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); + } + const slash = rest.indexOf("/"); + if (slash > 0 && slash < rest.length - 1) { + const result = await handleCommand("runs:get", { repo: rest.slice(0, slash), runId: rest.slice(slash + 1) }, req.signal); + return Response.json(result, { headers: corsHeaders }); + } + // falls through to the 404 path below for a shape mismatch, e.g. "/api/runs/onlyonesegment" + } +``` + +Note the behavior change from before: a malformed `/api/runs/...` now returns 400 instead of falling through to the generic 404. This is intentional and matches S083's ask across all three routes uniformly (a malformed path parameter is a 400, a genuinely unknown route is a 404); update no other test, since no existing test in this repo asserts the old runs-route 500-vs-404 distinction (grep `lib/daemon/__tests__/` for `runs:get` before assuming otherwise, and if one exists, adjust it to expect 400 for the malformed case instead of removing coverage). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-server-path-param.test.ts` +Expected: PASS + +- [ ] **Step 5: Check for an existing runs-route test that might need updating** + +Run: `grep -rn "runs:get\|/api/runs/" lib/daemon/__tests__/` +If a test asserts the old 500-on-malformed or 404-on-malformed behavior for `/api/runs/`, update its expected status to 400 to match the new shared helper. + +- [ ] **Step 6: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. + +- [ ] **Step 7: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/__tests__/api-server-path-param.test.ts +git commit -m "daemon: pathParam() helper -- malformed %-encoding is a 400, not a logged 500 (S083)" +``` + +--- + +## Task 7: GET query param coercion (S085) + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Test: `lib/daemon/__tests__/api-server-query-coerce.test.ts` (new) + +**Interfaces:** +- Produces: `export function coerceQueryParams(params: URLSearchParams): Record` — converts `"true"`/`"false"` to booleans and plain-integer/decimal strings to numbers; everything else stays a string. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/api-server-query-coerce.test.ts +import { describe, test, expect } from "bun:test"; +import { coerceQueryParams } from "../api-server.ts"; + +describe("coerceQueryParams", () => { + test("coerces maxAgeMs to a number (the documented cache:read flag)", () => { + const out = coerceQueryParams(new URLSearchParams("maxAgeMs=60000")); + expect(out.maxAgeMs).toBe(60000); + expect(typeof out.maxAgeMs).toBe("number"); + }); + + test("coerces refresh=true to a boolean (the documented ports flag)", () => { + const out = coerceQueryParams(new URLSearchParams("refresh=true")); + expect(out.refresh).toBe(true); + }); + + test("coerces refresh=false to a boolean false, not a truthy string", () => { + const out = coerceQueryParams(new URLSearchParams("refresh=false")); + expect(out.refresh).toBe(false); + }); + + test("leaves a non-numeric, non-boolean string alone", () => { + const out = coerceQueryParams(new URLSearchParams("repo=my-repo-name")); + expect(out.repo).toBe("my-repo-name"); + }); + + test("leaves an empty string alone rather than coercing to 0", () => { + const out = coerceQueryParams(new URLSearchParams("q=")); + expect(out.q).toBe(""); + }); + + test("coerces a decimal number too", () => { + const out = coerceQueryParams(new URLSearchParams("ratio=1.5")); + expect(out.ratio).toBe(1.5); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-server-query-coerce.test.ts` +Expected: FAIL — `coerceQueryParams` not exported yet. + +- [ ] **Step 3: Write minimal implementation** + +Add to `lib/daemon/api-server.ts` (near `pathParam`): + +```ts +const PLAIN_NUMBER_RE = /^-?\d+(\.\d+)?$/; + +/** + * REST query strings arrive as strings no matter what the client meant + * (S085): `?maxAgeMs=60000` and `?refresh=true` reached handlers that do a + * strict `typeof x === "number"` or `x === true` check, so the documented + * flag silently no-op'd over HTTP while working over the socket (where + * payloads are real JSON). One coercion at the REST seam fixes every such + * flag at once instead of a per-handler private parser. + */ +export function coerceQueryParams(params: URLSearchParams): Record { + const out: Record = {}; + for (const [key, value] of params) { + if (value === "true") out[key] = true; + else if (value === "false") out[key] = false; + else if (value !== "" && PLAIN_NUMBER_RE.test(value)) out[key] = Number(value); + else out[key] = value; + } + return out; +} +``` + +Change the payload-building line inside `fetch()` from: + +```ts + } else { + payload = Object.fromEntries(url.searchParams); + } +``` + +to: + +```ts + } else { + payload = coerceQueryParams(url.searchParams); + } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-server-query-coerce.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/__tests__/api-server-query-coerce.test.ts +git commit -m "daemon: coerce REST GET query params to number/boolean at the seam (S085)" +``` + +--- + +## Task 8: ApiPortInUseError + lsof diagnostics when bind retries are exhausted (S043 addendum) + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Test: `lib/daemon/__tests__/api-server-bind.test.ts` (extend) + +**Interfaces:** +- Produces: `export class ApiPortInUseError extends Error { readonly code: "EADDRINUSE"; readonly port: number }`. `BindRetryDeps` grows an optional `probePortHolder?: (port: number) => Promise` field (defaults to a real `lsof -i :` via `runCapture`). `bindApiServerWithRetry` throws `ApiPortInUseError` (instead of the raw EADDRINUSE `Error`) once `BIND_RETRY_ATTEMPTS` is exhausted, after logging the probe result at `warn`. +- Consumes: `runCapture` from `../subprocess.ts`. +- **Contract for the sibling job owning `lib/daemon.ts`'s caller side:** catch `ApiPortInUseError` (check `err instanceof ApiPortInUseError`, or `err.name === "ApiPortInUseError"`) around the `startApiServer()` call and route to a park-and-retry-with-backoff loop instead of letting it propagate to the top-level crash/unhandledRejection path. Every other error out of `startApiServer()` is a real misconfiguration and should keep crashing as it does today. + +- [ ] **Step 1: Write the failing test** + +Append to `lib/daemon/__tests__/api-server-bind.test.ts`: + +```ts +import { ApiPortInUseError, BIND_RETRY_ATTEMPTS } from "../api-server.ts"; + +function depsWithProbe(overrides: Partial = {}) { + const logs: Array<{ o: unknown; m: string }> = []; + const sleeps: number[] = []; + const probeCalls: number[] = []; + return { + sleep: async (ms: number) => { sleeps.push(ms); }, + log: { warn: (o: unknown, m: string) => logs.push({ o, m }) }, + probePortHolder: async (port: number) => { probeCalls.push(port); return "COMMAND PID USER\nnode 123 matt"; }, + logs, + sleeps, + probeCalls, + ...overrides, + }; +} + +describe("bindApiServerWithRetry — exhausted retries (S043)", () => { + test("throws ApiPortInUseError (not the raw EADDRINUSE Error) once attempts are exhausted", async () => { + const d = depsWithProbe(); + let error: unknown; + try { + await bindApiServerWithRetry(() => { throw eaddrinuse(); }, d); + } catch (err) { + error = err; + } + expect(error).toBeInstanceOf(ApiPortInUseError); + expect((error as ApiPortInUseError).code).toBe("EADDRINUSE"); + expect((error as Error).message).toContain("EADDRINUSE"); + }); + + test("probes the port holder exactly once, only after the final attempt", async () => { + const d = depsWithProbe(); + let calls = 0; + await expect( + bindApiServerWithRetry(() => { calls++; throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + expect(calls).toBe(BIND_RETRY_ATTEMPTS); + expect(d.probeCalls.length).toBe(1); + }); + + test("logs the probe result at warn before throwing", async () => { + const d = depsWithProbe(); + await expect( + bindApiServerWithRetry(() => { throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + const finalWarn = d.logs.at(-1)!; + expect(finalWarn.o).toMatchObject({ holder: expect.stringContaining("node") }); + }); + + test("a probe failure does not prevent the ApiPortInUseError from being thrown", async () => { + const d = depsWithProbe({ probePortHolder: async () => { throw new Error("lsof: command not found"); } }); + await expect( + bindApiServerWithRetry(() => { throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + }); + + test("a successful bind never probes", async () => { + const d = depsWithProbe(); + await bindApiServerWithRetry(() => "server" as any, d); + expect(d.probeCalls.length).toBe(0); + }); +}); +``` + +Also update the OLD test that currently asserts the raw-error message (it now gets a differently-shaped, but still EADDRINUSE-mentioning, error): + +Find this existing test: + +```ts + test("exhausting retries rethrows the original error after exactly BIND_RETRY_ATTEMPTS calls", async () => { + const d = deps(); + let calls = 0; + await expect( + bindApiServerWithRetry(() => { calls++; throw eaddrinuse(); }, d), + ).rejects.toThrow("EADDRINUSE"); + expect(calls).toBe(BIND_RETRY_ATTEMPTS); + expect(d.sleeps.length).toBe(BIND_RETRY_ATTEMPTS - 1); + }); +``` + +Change its assertion to also cover the new type, since `deps()` (the original helper, with no `probePortHolder` override) must still work by falling back to a default real-`lsof` probe — which would actually shell out in a test. To keep this test hermetic, add `probePortHolder: async () => "n/a"` to the base `deps()` helper's return object (it is an optional field on `BindRetryDeps`, so this is a backward-compatible addition, not a signature break): + +```ts +function deps(overrides: Partial = {}): BindRetryDeps & { logs: string[]; sleeps: number[] } { + const logs: string[] = []; + const sleeps: number[] = []; + return { + sleep: async (ms: number) => { sleeps.push(ms); }, + log: { warn: (_o: unknown, m: string) => logs.push(`warn:${m}`) }, + probePortHolder: async () => "n/a", + logs, + sleeps, + ...overrides, + }; +} +``` + +Then the pre-existing test's assertion becomes: + +```ts + test("exhausting retries rethrows as ApiPortInUseError after exactly BIND_RETRY_ATTEMPTS calls", async () => { + const d = deps(); + let calls = 0; + await expect( + bindApiServerWithRetry(() => { calls++; throw eaddrinuse(); }, d), + ).rejects.toThrow("EADDRINUSE"); + expect(calls).toBe(BIND_RETRY_ATTEMPTS); + expect(d.sleeps.length).toBe(BIND_RETRY_ATTEMPTS - 1); + }); +``` + +(only the test name and this doc comment change; `.rejects.toThrow("EADDRINUSE")` still matches since `ApiPortInUseError`'s message contains that substring.) + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-server-bind.test.ts` +Expected: FAIL — `ApiPortInUseError` not exported; `probePortHolder` not a recognized field yet. + +- [ ] **Step 3: Write minimal implementation** + +In `lib/daemon/api-server.ts`, add the import: + +```ts +import { runCapture } from "../subprocess.ts"; +``` + +Add the error class and update `BindRetryDeps`/`bindApiServerWithRetry`. Replace the current block: + +```ts +export interface BindRetryDeps { + sleep: (ms: number) => Promise; + log: { warn: (o: unknown, m: string) => void }; +} + +// evictStaleDaemon (lib/daemon/boot-reconcile.ts) already assumes a prior +// holder is gone after a 300ms Bun.sleepSync — 6 attempts at 500ms (~3s +// worst case, exported so tests assert against these, not hardcoded copies) +// gives that same assumption room to be wrong once before this gives up too. +export const BIND_RETRY_ATTEMPTS = 6; +export const BIND_RETRY_DELAY_MS = 500; + +/** + * evictStaleDaemon() SIGTERMs the previous holder of this port before a new + * daemon binds, but the kill isn't synchronous with the exit — a fresh + * daemon can reach this bind before the old one has actually released + * 9401. Only EADDRINUSE is retried (bounded, ~3s total); anything else is a + * real misconfiguration and fails on the first attempt, same as before. + */ +export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDeps): Promise { + for (let attempt = 1; ; attempt++) { + try { + return bind(); + } catch (err) { + const isAddrInUse = err instanceof Error && (err as NodeJS.ErrnoException).code === "EADDRINUSE"; + if (!isAddrInUse || attempt >= BIND_RETRY_ATTEMPTS) throw err; + deps.log.warn({ attempt, port: API_PORT }, "api port in use, retrying — another daemon is likely still shutting down"); + await deps.sleep(BIND_RETRY_DELAY_MS); + } + } +} +``` + +with: + +```ts +/** + * Thrown when every bind retry is exhausted with EADDRINUSE still held. A + * NAMED error type (S043) rather than the raw EADDRINUSE Error, so + * lib/daemon.ts's caller can distinguish "the port is genuinely squatted" + * from any other startup failure and park-and-retry with backoff instead of + * crash-looping (that caller-side change belongs to a sibling job; this + * class is the contract it wires into). + */ +export class ApiPortInUseError extends Error { + readonly code = "EADDRINUSE" as const; + readonly port: number; + constructor(port: number) { + super(`EADDRINUSE: api server port ${port} is still in use after ${BIND_RETRY_ATTEMPTS} bind attempts`); + this.name = "ApiPortInUseError"; + this.port = port; + } +} + +export interface BindRetryDeps { + sleep: (ms: number) => Promise; + log: { warn: (o: unknown, m: string) => void }; + /** Defaults to a real `lsof -i :` via runCapture; overridable so tests never shell out. */ + probePortHolder?: (port: number) => Promise; +} + +async function defaultProbePortHolder(port: number): Promise { + const result = await runCapture(["lsof", "-i", `:${port}`], { timeoutMs: 5_000, stderr: "pipe" }); + return result.stdout.trim(); +} + +// evictStaleDaemon (lib/daemon/boot-reconcile.ts) already assumes a prior +// holder is gone after a 300ms Bun.sleepSync — 6 attempts at 500ms (~3s +// worst case, exported so tests assert against these, not hardcoded copies) +// gives that same assumption room to be wrong once before this gives up too. +export const BIND_RETRY_ATTEMPTS = 6; +export const BIND_RETRY_DELAY_MS = 500; + +/** + * evictStaleDaemon() SIGTERMs the previous holder of this port before a new + * daemon binds, but the kill isn't synchronous with the exit — a fresh + * daemon can reach this bind before the old one has actually released + * 9401. Only EADDRINUSE is retried (bounded, ~3s total); anything else is a + * real misconfiguration and fails on the first attempt, same as before. + * + * Once retries are exhausted, this logs WHO holds the port (S043's + * diagnostic ask) and throws ApiPortInUseError instead of the bare + * EADDRINUSE Error, so a caller can tell "give up cleanly" apart from "the + * bind function itself is broken". + */ +export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDeps): Promise { + const probe = deps.probePortHolder ?? defaultProbePortHolder; + for (let attempt = 1; ; attempt++) { + try { + return bind(); + } catch (err) { + const isAddrInUse = err instanceof Error && (err as NodeJS.ErrnoException).code === "EADDRINUSE"; + if (!isAddrInUse) throw err; + if (attempt >= BIND_RETRY_ATTEMPTS) { + const holder = await probe(API_PORT).catch((probeErr) => `lsof failed: ${String(probeErr)}`); + deps.log.warn({ port: API_PORT, holder }, "api port still in use after retries; giving up bind — the daemon should park and retry with backoff rather than crash-loop"); + throw new ApiPortInUseError(API_PORT); + } + deps.log.warn({ attempt, port: API_PORT }, "api port in use, retrying — another daemon is likely still shutting down"); + await deps.sleep(BIND_RETRY_DELAY_MS); + } + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-server-bind.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/__tests__/api-server-bind.test.ts +git commit -m "daemon: log the EADDRINUSE port holder and throw a typed ApiPortInUseError (S043)" +``` + +--- + +## Task 9: git-ref validation utility (S010 — standalone, sibling wires it into worktree.ts) + +**Files:** +- Create: `lib/daemon/git-ref-validation.ts` +- Test: `lib/daemon/__tests__/git-ref-validation.test.ts` + +**Interfaces:** +- Produces: `export function isSafeGitRef(ref: string): boolean` and `export function validateGitRef(ref: string): { ok: true } | { ok: false; error: string }`. + +This module is NOT wired into `lib/daemon/handlers/worktree.ts` by this job — that file is sibling-owned per the write fence. The wiring is documented in `docs/daemon-api-auth.md` (Task 11) and the job report's Notes. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/git-ref-validation.test.ts +import { describe, test, expect } from "bun:test"; +import { isSafeGitRef, validateGitRef } from "../git-ref-validation.ts"; + +describe("isSafeGitRef", () => { + test("a normal branch name is safe", () => { + expect(isSafeGitRef("feature/my-branch")).toBe(true); + }); + + test("a leading dash is unsafe (option injection, S010)", () => { + expect(isSafeGitRef("--upload-pack=touch /tmp/x")).toBe(false); + }); + + test("a bare dash is unsafe", () => { + expect(isSafeGitRef("-")).toBe(false); + }); + + test("an empty string is unsafe", () => { + expect(isSafeGitRef("")).toBe(false); + }); + + test("a branch containing a dash mid-name is safe", () => { + expect(isSafeGitRef("job/p3-trust-boundary")).toBe(true); + }); +}); + +describe("validateGitRef", () => { + test("returns ok:true for a safe ref", () => { + expect(validateGitRef("main")).toEqual({ ok: true }); + }); + + test("returns ok:false with the offending ref named in the error for an unsafe one", () => { + const result = validateGitRef("--upload-pack=x"); + expect(result.ok).toBe(false); + expect((result as { ok: false; error: string }).error).toContain("--upload-pack=x"); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/git-ref-validation.test.ts` +Expected: FAIL — module does not exist. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/git-ref-validation.ts +/** + * Rejects a branch/ref that could be parsed by git as an OPTION rather than + * a ref (S010): a caller-supplied `branch` like "--upload-pack=touch /tmp/x" + * reaches `git fetch origin ` and `git rev-list ......` + * unescaped in lib/daemon/handlers/worktree.ts, and git happily runs it as + * an option since nothing on that path validates the string first. This is + * the ONE guard both call sites need; a future caller (or a consumer app + * like mr-board/console/the chat viewer forwarding an untrusted string as + * `branch`) inherits the same hole without it. + * + * Deliberately narrow: reject a leading '-' rather than allowlisting a + * character set, since `git check-ref-format --branch` accepts far more + * punctuation than is worth re-deriving here, and the vulnerable shape is + * specifically "parses as an option", not "contains an unusual character". + */ +export function isSafeGitRef(ref: string): boolean { + return ref.length > 0 && !ref.startsWith("-"); +} + +export function validateGitRef(ref: string): { ok: true } | { ok: false; error: string } { + if (!isSafeGitRef(ref)) { + return { ok: false, error: `unsafe git ref (starts with '-' or empty): ${ref}` }; + } + return { ok: true }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/git-ref-validation.test.ts` +Expected: PASS + +- [ ] **Step 5: Run tsc** + +Run: `bunx tsc --noEmit` +Expected: clean. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/git-ref-validation.ts lib/daemon/__tests__/git-ref-validation.test.ts +git commit -m "daemon: standalone git-ref validator for S010 (worktree.ts wiring documented, not wired here)" +``` + +--- + +## Task 10: credential redaction utility (S050 — standalone, sibling wires it into freshness.ts) + +**Files:** +- Create: `lib/daemon/redact-credentials.ts` +- Test: `lib/daemon/__tests__/redact-credentials.test.ts` + +**Interfaces:** +- Produces: `export function redactCredentials(text: string): string`. + +Same treatment as Task 9: `lib/daemon/freshness.ts` (where the audit's fixer notes say this belongs, at lines ~142/148/275/279) is not in this job's write fence. This module is standalone and tested; the wiring is documented in `docs/daemon-api-auth.md` (Task 11) and the report's Notes. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/redact-credentials.test.ts +import { describe, test, expect } from "bun:test"; +import { redactCredentials } from "../redact-credentials.ts"; + +describe("redactCredentials", () => { + test("redacts userinfo (user:token@) out of an https remote URL", () => { + const input = "https://oauth2:glpat-XXXXXXXXXXXXXXXXXXXX@gitlab.example.com/g/p.git"; + const out = redactCredentials(input); + expect(out).not.toContain("glpat-XXXXXXXXXXXXXXXXXXXX"); + expect(out).toContain("gitlab.example.com/g/p.git"); + }); + + test("redacts a GitHub PAT embedded in the URL", () => { + const input = "https://ghp_abcdefghijklmnopqrstuvwxyz012345@github.com/o/r.git"; + const out = redactCredentials(input); + expect(out).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz012345"); + expect(out).toContain("github.com/o/r.git"); + }); + + test("leaves a URL with no embedded credentials unchanged", () => { + const input = "https://gitlab.example.com/g/p.git"; + expect(redactCredentials(input)).toBe(input); + }); + + test("leaves plain text with no URL unchanged", () => { + const input = "local branch listing failed"; + expect(redactCredentials(input)).toBe(input); + }); + + test("redacts every match when more than one credential-bearing URL appears in the same string", () => { + const input = "tried https://oauth2:tok1@a.example/x then https://oauth2:tok2@b.example/y"; + const out = redactCredentials(input); + expect(out).not.toContain("tok1"); + expect(out).not.toContain("tok2"); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/redact-credentials.test.ts` +Expected: FAIL — module does not exist. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/redact-credentials.ts +/** + * Strips userinfo (user:token@ or user@) out of any http(s) URL embedded in + * a string (S050): freshness.ts logs `remote.origin.url` verbatim at info + * on every reconcile, and echoes it into thrown errors every mr/discussions + * handler returns to callers. A repo cloned as + * `https://oauth2:glpat-XXXX@gitlab.example.com/...` (routine for + * dotfiles/CI-derived clones) puts that token into ~/.rt/logs/daemon.*.log + * and into any client-facing error message -- logs are the first thing a + * user pastes into a bug report. + */ +const CREDENTIAL_URL_RE = /(https?:\/\/)[^/@\s]+@/gi; + +export function redactCredentials(text: string): string { + return text.replace(CREDENTIAL_URL_RE, "$1[redacted]@"); +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/redact-credentials.test.ts` +Expected: PASS + +- [ ] **Step 5: Run tsc** + +Run: `bunx tsc --noEmit` +Expected: clean. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/redact-credentials.ts lib/daemon/__tests__/redact-credentials.test.ts +git commit -m "daemon: standalone credential-redaction utility for S050 (freshness.ts wiring documented, not wired here)" +``` + +--- + +## Task 11: docs note + final whole-branch verification + +**Files:** +- Create: `docs/daemon-api-auth.md` + +- [ ] **Step 1: Write the doc** + +```markdown +# The :9401 trust boundary + +How the daemon's REST/WS surface decides who to trust, and the follow-up +wiring two standalone modules from this phase still need in sibling-owned +files. + +## The model + +- **No `Origin` header at all** (the CLI, the Swift tray, rt-client from a + Bun/Node process, mr-board, gitq, the VS Code extension): unaffected, no + gate applies beyond what already existed. None of today's consumers send + an `Origin` header to `:9401`. +- **A browser `Origin` header is present**: trusted only if the request + presents the local `X-RT-Token` (`?token=` query param for `/ws`, since + browsers cannot set custom headers on a WS handshake) OR the Origin is on + the `rt.trustedBrowserOrigins` settings allowlist (see + `packages/rt-client/src/settings/registry-defs.ts`; `docs/settings-architecture.md` + is the settings-system contract). Otherwise: no `Access-Control-Allow-Origin` + on REST reads (default-deny CORS), and a 403 on `/ws`. +- **Mutating routes** (every method except GET/HEAD/OPTIONS, plus + `/api/secrets` and `/api/notifications` despite being GETs) require the + local `X-RT-Token` regardless of Origin — this is the CSRF defense against + a browser form/simple-request bypassing CORS preflight entirely, and it is + orthogonal to the Origin check above. + +See `lib/daemon/api-auth.ts` (`isBrowserRequestTrusted`, `needsToken`, +`getTrustedBrowserOrigins`) and `lib/daemon/api-server.ts` +(`buildCorsHeaders`, the `/ws` gate in `fetch()`) for the implementation. + +## Follow-up wiring for sibling-owned files (not done in this job) + +**S010** (`lib/daemon/handlers/worktree.ts`): `lib/daemon/git-ref-validation.ts` +exports `validateGitRef(ref)`. Call it right after `payload.branch` is read +(around `worktree.ts:282`) and return `{ ok: false, error }` on a rejection +BEFORE any `runGit` call reaches it — that single call site also covers the +weaker secondary instance in `divergence()` (`worktree.ts:211-213`), since +both read the same `branch` value. + +**S050** (`lib/daemon/freshness.ts`): `lib/daemon/redact-credentials.ts` +exports `redactCredentials(text)`. Wrap every log/error interpolation of a +remote URL with it — the audit names `freshness.ts:142, 148, 275, 279` as the +current call sites. + +**S043 caller side** (`lib/daemon.ts`): `lib/daemon/api-server.ts` exports +`ApiPortInUseError` (a named `Error` subclass with `.name === "ApiPortInUseError"` +and `.port`). Catch it around the `startApiServer()` call and park-and-retry +with backoff instead of letting it reach the top-level crash path; any other +error out of `startApiServer()` is a genuine misconfiguration and should keep +crashing as it does today. +``` + +- [ ] **Step 2: Run the full verification suite** + +Run: `bun test lib commands packages scripts` +Expected: all green. + +Run: `bunx tsc --noEmit` +Expected: zero errors. + +Run: `cd packages/rt-client && bun run build && cd -` +Expected: clean build (already run in Task 3, but re-run here as the final gate since later tasks may have touched files rt-client's dist-freshness test watches). + +- [ ] **Step 3: Commit** + +```bash +git add docs/daemon-api-auth.md +git commit -m "docs: the :9401 trust boundary model and the S010/S050/S043 sibling wiring notes" +``` + +--- + +## Self-Review Notes (already applied above) + +- **Spec coverage:** S092 (Task 1), S054 (Task 2), S005/S006/S040/S041/S084 (Tasks 3-4), S042 (Task 5), S083 (Task 6), S085 (Task 7), S043 (Task 8), S010 (Task 9), S050 (Task 10). All 13 cited findings have a task. +- **Write fence:** every modified/created file is inside the job's write fence; Tasks 9-10 deliberately stop short of editing `worktree.ts`/`freshness.ts` and document the wiring instead, mirroring the job brief's explicit instruction for S010 and extending the same treatment to S050 since its fix location is equally out of fence. +- **Type consistency:** `isBrowserRequestTrusted(origin, token, apiToken, allowedOrigins)` has the same parameter order and names everywhere it's used (Task 3 definition, Task 4 call sites). `BroadcastTarget`/`broadcastToClients` names match between Task 5's definition and its test. `pathParam`/`coerceQueryParams` signatures match between definition and test across Tasks 6-7. From 2246ec948f8367efa51df9e580114ce0550e9287 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:45:30 -0500 Subject: [PATCH 010/142] deps links: never auto-unlink a DEFAULT_EXPOSED tool (rt/fast-browser/gitq/deck) on a same-named PATH collision (S066) --- lib/deps/__tests__/links.test.ts | 20 ++++++++++++++++++++ lib/deps/links.ts | 11 ++++++++++- 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/lib/deps/__tests__/links.test.ts b/lib/deps/__tests__/links.test.ts index 5b9fef2e..63ea5048 100644 --- a/lib/deps/__tests__/links.test.ts +++ b/lib/deps/__tests__/links.test.ts @@ -289,6 +289,26 @@ describe("tagged PATH links", () => { expect(result).toEqual({ removed: ["gh"], kept: [] }); }); + // S066: a DEFAULT_EXPOSED tool (rt's own product surface, not a vendored + // third-party tool like "gh") is never auto-unlinked by a same-named PATH + // collision — an unrelated foreign tool of the same name (e.g. Kong's own + // "deck") must never shadow-remove mattstack's. + test("reconcile never auto-unlinks deck/gitq/rt even when a same-named binary appears elsewhere on PATH (S066)", () => { + const p = bundleProbe({ env: { PATH: "/opt/homebrew/bin" } }); + p.symlink(join(appRoot, HELPERS_DIR, "deck"), linkPath(home, "deck")); + p.symlink(join(appRoot, HELPERS_DIR, "gitq"), linkPath(home, "gitq")); + link(p, "rt", {}, { installRtBinary: (src, dest) => { p.symlink(src, dest); return dest; } }); + + // An unrelated foreign binary happens to share each name. + p.writeFile("/opt/homebrew/bin/deck", "kongs-deck-binary"); + p.writeFile("/opt/homebrew/bin/gitq", "some-other-gitq-binary"); + p.writeFile("/opt/homebrew/bin/rt", "some-other-rt-binary"); + + const result = reconcile(p); + expect(result.removed).toEqual([]); + expect(result.kept.sort()).toEqual(["deck", "gitq", "rt"]); + }); + test("real-fs repro (F1): after the daemon's boot-time PATH prepend, reconcile removes nothing", () => { // Real fs (existsSync/statSync/symlinkSync/readdirSync via createRealProbes) // is what actually matters for this proof — the file/directory distinction diff --git a/lib/deps/links.ts b/lib/deps/links.ts index e70d56a9..df570d16 100644 --- a/lib/deps/links.ts +++ b/lib/deps/links.ts @@ -147,11 +147,20 @@ export function unlink(p: Probes, tool: string): { removed: boolean } { export function reconcile(p: Probes): { removed: string[]; kept: string[] } { const dir = join(p.home, ".local", "bin"); const forced = new Set(readSetupState(p).forcedLinks); + const defaultExposed = new Set(DEFAULT_EXPOSED); const removed: string[] = []; const kept: string[] = []; for (const tool of p.readDir(dir)) { if (!isOurLink(p, tool)) continue; // not one of ours — nothing to reconcile - if (forced.has(tool) || !userCopyOnPath(p, tool)) { + // DEFAULT_EXPOSED (rt, fast-browser, gitq, deck) is mattstack's own + // product surface, not a vendored third-party tool like the bundled + // "gh" — userCopyOnPath matches by name only, so a same-named PATH + // entry is at least as likely to be an unrelated namesake (Kong's own + // "deck") as a genuine second copy of ours, and shadow-removing our own + // "rt" this way breaks every command. Only a vendored tool's step-aside + // (deferring to a real system copy the user installed) is safe to + // decide by name alone. + if (defaultExposed.has(tool) || forced.has(tool) || !userCopyOnPath(p, tool)) { kept.push(tool); } else { p.removeFile(linkPath(p.home, tool)); From def840fc8ed89d01c7af5a96047a92df0692ef9c Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:47:36 -0500 Subject: [PATCH 011/142] age-key: give the keychain spawn a bounded, distinguishable timeout (S070) --- lib/home/__tests__/age-key.test.ts | 35 ++++++++++++++++++++ lib/home/age-key.ts | 52 ++++++++++++++++++++++++++---- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/lib/home/__tests__/age-key.test.ts b/lib/home/__tests__/age-key.test.ts index 4aad547d..bc6ee48e 100644 --- a/lib/home/__tests__/age-key.test.ts +++ b/lib/home/__tests__/age-key.test.ts @@ -9,6 +9,8 @@ import { keyExport, withArgvRedaction, AgeKeyAbsentError, + AgeKeyTimeoutError, + createRealAgeKeySeam, type AgeExecResult, type AgeKeySeam, } from "../age-key.ts"; @@ -378,3 +380,36 @@ describe("keyExport", () => { expect('import { statfs } from "./statfs-helpers.ts";').not.toMatch(FS_IMPORT_RE); }); }); + +// S070: a locked keychain (screen-lock, or an ACL mismatch between the dev +// shim and mattstack.app) pops a GUI dialog that blocks the real spawn +// forever; every seam.run must have a bounded deadline instead. +describe("createRealAgeKeySeam timeout (S070)", () => { + test("a spawn that outlives its timeout rejects with a distinguished AgeKeyTimeoutError instead of hanging", async () => { + const seam = createRealAgeKeySeam(); + await expect(seam.run(["sleep", "5"], { timeoutMs: 50 })).rejects.toThrow(AgeKeyTimeoutError); + }); + + test("the timeout error names the command without leaking a sensitive -w value", async () => { + const seam = createRealAgeKeySeam(); + // sh -c's trailing args become unused positional params ($0, $1, ...) — + // this still sleeps for the script text alone, while still carrying a + // "-w " pair later in argv for redactArgv to find. + const cmd = ["sh", "-c", "sleep 5", "argv0", "-w", "AGE-SECRET-KEY-should-not-appear"]; + try { + await seam.run(cmd, { timeoutMs: 50, sensitive: true }); + throw new Error("expected a timeout"); + } catch (err) { + expect(err).toBeInstanceOf(AgeKeyTimeoutError); + expect((err as Error).message).not.toContain("AGE-SECRET-KEY-should-not-appear"); + expect((err as Error).message).toContain(""); + } + }); + + test("a spawn that finishes well within its timeout resolves normally", async () => { + const seam = createRealAgeKeySeam(); + const res = await seam.run(["echo", "hi"], { timeoutMs: 5000 }); + expect(res.code).toBe(0); + expect(res.stdout.trim()).toBe("hi"); + }); +}); diff --git a/lib/home/age-key.ts b/lib/home/age-key.ts index 1df556ab..5629ce6d 100644 --- a/lib/home/age-key.ts +++ b/lib/home/age-key.ts @@ -29,8 +29,26 @@ export interface AgeKeySeam { * the private key this way, keeping it out of argv). * `sensitive` marks a call whose argv or stdin carries key material, so a * logging wrapper (withArgvRedaction) knows to redact it. + * `timeoutMs` overrides the default kill-and-reject deadline (see + * AgeKeyTimeoutError) — a locked keychain or an unexpected access-control + * dialog otherwise blocks this call, and every caller behind it, forever. */ - run(cmd: string[], opts?: { input?: string; sensitive?: boolean }): Promise; + run(cmd: string[], opts?: { input?: string; sensitive?: boolean; timeoutMs?: number }): Promise; +} + +/** + * Thrown instead of resolving when the spawn outlives its timeout — a + * distinct class (not a generic Error, and never a resolved AgeExecResult) + * so a caller like loadSecrets's domainMemo can recognize "the keychain is + * probably showing a dialog right now" and avoid caching it as a permanent + * decrypt failure that would poison every later retry, including the one + * after the user dismisses the dialog. + */ +export class AgeKeyTimeoutError extends Error { + constructor(message: string) { + super(message); + this.name = "AgeKeyTimeoutError"; + } } const KEYCHAIN_ACCOUNT = "mattstack"; @@ -277,6 +295,9 @@ function debugLog(cmd: string[]): void { if (CLI_DEBUG) console.error(`[age-key] ${cmd.join(" ")}`); } +/** A locked keychain (screen-lock, or a keychain item whose ACL belongs to a different signed binary) pops a GUI dialog and blocks until clicked; this bounds every spawn against that. */ +const DEFAULT_AGE_KEY_TIMEOUT_MS = 30_000; + /** Bun.spawn-based capture, env passed live (PATH-snapshot gotcha). Unexported: only reachable wrapped, via createRealAgeKeySeam. */ function createRawAgeKeySeam(): AgeKeySeam { return { @@ -298,12 +319,29 @@ function createRawAgeKeySeam(): AgeKeySeam { // read it ignore the close, but age-keygen -y blocks on EOF otherwise. if (opts?.input !== undefined) proc.stdin.write(opts.input); proc.stdin.end(); - const [stdout, stderr, code] = await Promise.all([ - new Response(proc.stdout).text(), - new Response(proc.stderr).text(), - proc.exited, - ]); - return { code, stdout, stderr }; + + const timeoutMs = opts?.timeoutMs ?? DEFAULT_AGE_KEY_TIMEOUT_MS; + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + try { proc.kill(); } catch { /* already exited */ } + }, timeoutMs); + + try { + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + if (timedOut) { + throw new AgeKeyTimeoutError( + `${redactArgv(cmd).join(" ")}: did not exit within ${timeoutMs}ms (keychain prompt pending?)`, + ); + } + return { code, stdout, stderr }; + } finally { + clearTimeout(timer); + } }, }; } From c0c1ff00b0bf9c9de7012c53e54c1283cd5765ab Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:49:06 -0500 Subject: [PATCH 012/142] presence-store: honor the tail heartbeat in the offline rule and prune predicate (S075) --- lib/state/__tests__/presence-store.test.ts | 22 +++++++++++++++++++++ lib/state/presence-store.ts | 23 ++++++++++++++++------ 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/lib/state/__tests__/presence-store.test.ts b/lib/state/__tests__/presence-store.test.ts index f41f6cd2..c83e9c60 100644 --- a/lib/state/__tests__/presence-store.test.ts +++ b/lib/state/__tests__/presence-store.test.ts @@ -59,6 +59,28 @@ test("buddyStatus: table order, first match wins, tail heartbeat is COALESCE(tai expect(buddyStatus({ lastSeenAt: now }, now)).toBe("idle"); }); +// S075: a >24h autonomous agent (Monitor-driven, no user prompt) keeps its +// tail heartbeat fresh but its session heartbeat (last_seen_at, which only +// advances on a user prompt) goes stale past pruneMs. Both the offline rule +// and the prune predicate must honor the tail heartbeat, or the handle gets +// reclaimed out from under a session that never left. +test("buddyStatus: a fresh tail heartbeat keeps an armed row live past the 24h pruneMs boundary", () => { + expect(buddyStatus({ lastSeenAt: now - 25 * HOUR, armedAt: now, tailSeenAt: now }, now)).toBe("live"); +}); + +test("buddyStatus: an armed row is still offline when BOTH heartbeats are stale past pruneMs", () => { + expect(buddyStatus({ lastSeenAt: now - 25 * HOUR, armedAt: now - 25 * HOUR, tailSeenAt: now - 25 * HOUR }, now)).toBe("offline"); +}); + +test("prune: an armed row with a stale session heartbeat but a fresh tail heartbeat survives past 24h", () => { + const db = fresh(); + signIn({ sessionId: "s1", baseHandle: "x", now }, db); // last_seen_at stays at `now`, 25h stale by the prune call below + const pruneNow = now + 25 * HOUR; + db.run("UPDATE chat_presence SET armed_at = ?, tail_seen_at = ? WHERE session_id = 's1'", [pruneNow, pruneNow]); // still touching + expect(prunePresence(pruneNow, db)).toBe(0); + expect(db.query("SELECT COUNT(*) c FROM chat_presence").get()).toMatchObject({ c: 1 }); +}); + test("pulse writes last_seen_at and deets only", () => { const db = fresh(); signIn({ sessionId: "s1", baseHandle: "x", now }, db); diff --git a/lib/state/presence-store.ts b/lib/state/presence-store.ts index 3b4401a6..0d9c235a 100644 --- a/lib/state/presence-store.ts +++ b/lib/state/presence-store.ts @@ -83,10 +83,13 @@ const RECLAIMABLE_SQL = `signed_out_at IS NOT NULL OR (last_seen_at < ? AND COAL /** * Prune's own predicate, deliberately never RECLAIMABLE_SQL: that fragment's * bare `signed_out_at IS NOT NULL` leg would delete every signed-out row at - * daemon startup and empty the offline window. Bind params in order: - * dayAgo, dayAgo (same cutoff, both legs). + * daemon startup and empty the offline window. The second leg also honors + * the tail heartbeat (COALESCE(tail_seen_at, armed_at, 0)), the same fold + * buddyStatus's offline check uses — an armed row a long autonomous turn is + * still touching must survive even once last_seen_at alone looks stale. + * Bind params in order: dayAgo, dayAgo, dayAgo (same cutoff, all three legs). */ -const PRUNABLE_SQL = `(signed_out_at IS NOT NULL AND signed_out_at < ?) OR last_seen_at < ?`; +const PRUNABLE_SQL = `(signed_out_at IS NOT NULL AND signed_out_at < ?) OR (last_seen_at < ? AND COALESCE(tail_seen_at, armed_at, 0) < ?)`; const SELECT_PRESENCE_BY_HANDLE_SQL = `SELECT ${PRESENCE_COLUMNS} FROM chat_presence WHERE handle = ?;`; const SELECT_PRESENCE_BY_SESSION_SQL = `SELECT ${PRESENCE_COLUMNS} FROM chat_presence WHERE session_id = ?;`; @@ -147,7 +150,15 @@ export function buddyStatus( ): BuddyStatus { if (row.signedOutAt !== undefined) return "offline"; const lastSeenAt = row.lastSeenAt ?? 0; - if (now - lastSeenAt > th.pruneMs) return "offline"; + // A >24h autonomous agent (Monitor-driven, no user prompt) keeps only its + // tail heartbeat fresh — last_seen_at advances on a user prompt alone, so + // it starves for hours while the tail keeps touching. The offline check + // must honor whichever heartbeat is newer, or the handle reads offline + // (and the next chat:sign-in by any session prunes and reclaims it) out + // from under a session that never left. + const tailLiveness = row.armedAt !== undefined ? (row.tailSeenAt ?? row.armedAt) : 0; + const liveness = Math.max(lastSeenAt, tailLiveness); + if (now - liveness > th.pruneMs) return "offline"; if (row.armedAt !== undefined) { const tailHeartbeat = row.tailSeenAt ?? row.armedAt; return now - tailHeartbeat <= th.tailStaleMs ? "live" : "deaf"; @@ -309,7 +320,7 @@ export function pulseSession( export function listBuddies(now: number, db: Database = getStateDb()): Array { const th = presenceThresholds(); const dayAgo = now - th.pruneMs; - const rows = db.query(SELECT_NON_PRUNABLE_PRESENCE_SQL).all(dayAgo, dayAgo) as PresenceRawRow[]; + const rows = db.query(SELECT_NON_PRUNABLE_PRESENCE_SQL).all(dayAgo, dayAgo, dayAgo) as PresenceRawRow[]; return rows.map((raw) => { const presence = rowToPresence(raw); return { ...presence, status: buddyStatus(presence, now, th) }; @@ -358,7 +369,7 @@ export function assertSessionSignedIn(sessionId: string, db: Database = getState export function prunePresence(now: number, db: Database = getStateDb()): number { const th = presenceThresholds(); const dayAgo = now - th.pruneMs; - return db.query(DELETE_PRUNABLE_PRESENCE_SQL).run(dayAgo, dayAgo).changes; + return db.query(DELETE_PRUNABLE_PRESENCE_SQL).run(dayAgo, dayAgo, dayAgo).changes; } // --- Internal wiring for chat-store.ts's dual-write (arm/touch/disarm). --- From fd626d8b4f4290c65311c0979371c7656c155fd7 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:49:52 -0500 Subject: [PATCH 013/142] chat:sign-in: reject a missing/empty sessionId instead of storing a NULL-keyed presence row (S076) --- lib/daemon/__tests__/chat-handlers.test.ts | 29 ++++++++++++++++++++++ lib/daemon/handlers/chat.ts | 8 ++++++ lib/state/presence-store.ts | 6 +++++ 3 files changed, 43 insertions(+) diff --git a/lib/daemon/__tests__/chat-handlers.test.ts b/lib/daemon/__tests__/chat-handlers.test.ts index b54a0774..fb75f284 100644 --- a/lib/daemon/__tests__/chat-handlers.test.ts +++ b/lib/daemon/__tests__/chat-handlers.test.ts @@ -221,6 +221,35 @@ test("sign-in rejects an invalid baseHandle with a reason rather than normalizin expect(res.error).toContain("handle"); }); +// S076: a missing/empty sessionId binds as NULL against session_id TEXT +// PRIMARY KEY, which SQLite accepts — the row then holds the UNIQUE handle +// but the reclaim-by-session_id path can never match it, wedging every +// later sign-in under that base handle with a UNIQUE constraint failure. +test("sign-in rejects a missing sessionId rather than storing a NULL-keyed row", async () => { + const h = freshHandlers(); + const res = await h["chat:sign-in"]({ baseHandle: "x" } as any); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("sessionId"); +}); + +test("sign-in rejects an empty-string sessionId the same way", async () => { + const h = freshHandlers(); + const res = await h["chat:sign-in"]({ sessionId: "", baseHandle: "x" }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("sessionId"); +}); + +test("a rejected sign-in never wedges the next sign-in under the same base handle", async () => { + const h = freshHandlers(); + await h["chat:sign-in"]({ baseHandle: "x" } as any); // rejected, must not persist a row + const ok = await h["chat:sign-in"]({ sessionId: "s1", baseHandle: "x" }); + expect(ok.ok).toBe(true); + if (!ok.ok) throw new Error("unreachable"); + expect(ok.data).toMatchObject({ handle: "x" }); +}); + test("a reclaimed handle refuses the old session's pulse with the reason", async () => { const h = freshHandlers(); await h["chat:sign-in"]({ sessionId: "s1", baseHandle: "x" }); diff --git a/lib/daemon/handlers/chat.ts b/lib/daemon/handlers/chat.ts index 98101e8a..470e846d 100644 --- a/lib/daemon/handlers/chat.ts +++ b/lib/daemon/handlers/chat.ts @@ -297,6 +297,14 @@ export function createChatHandlers(opts: { "chat:sign-in": async (payload: Commands["chat:sign-in"]["payload"]): Promise> => { const { sessionId, baseHandle, cwd, repo, branch, pane, statusText } = payload; + // A missing/empty sessionId binds as NULL against session_id TEXT + // PRIMARY KEY, which SQLite accepts silently: the row then holds the + // UNIQUE handle, but the reclaim-by-session_id path can never match + // it, so every later sign-in under this base handle 500s with a + // UNIQUE constraint failure until prunePresence eventually removes it. + if (typeof sessionId !== "string" || sessionId.length === 0) { + return { ok: false, error: "chat:sign-in requires a non-empty sessionId" }; + } if (!isValidChatName(baseHandle)) return { ok: false, error: `invalid handle "${baseHandle}"` }; const data = signIn({ sessionId, baseHandle, cwd, repo, branch, pane, statusText }, db); return { ok: true, data }; diff --git a/lib/state/presence-store.ts b/lib/state/presence-store.ts index 0d9c235a..0d052aac 100644 --- a/lib/state/presence-store.ts +++ b/lib/state/presence-store.ts @@ -224,6 +224,12 @@ export function signIn( db: Database = getStateDb(), ): { handle: string; reclaimed: boolean } { const { sessionId, baseHandle, statusText } = args; + // Defense in depth: the handler (lib/daemon/handlers/chat.ts) is the + // root-cause guard, but session_id is a bare TEXT PRIMARY KEY with no + // NOT NULL/CHECK constraint (bun:sqlite binds undefined as NULL, which + // SQLite accepts), so any future caller of this store function directly + // must not be able to wedge the same NULL-keyed-row failure mode. + if (!sessionId) throw new Error("signIn: sessionId is required"); const cwd = args.cwd ?? null; const repo = args.repo ?? null; const branch = args.branch ?? null; From 5651d9a16ed53300c86934d2fca9256d028a54c9 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:50:45 -0500 Subject: [PATCH 014/142] worktree trash: exclude .worktrees/ from the repo's git status on every retire, not just createTree's path (S078) --- lib/worktree/__tests__/trash.test.ts | 25 +++++++++++++++++++++++++ lib/worktree/trash.ts | 8 ++++++++ 2 files changed, 33 insertions(+) diff --git a/lib/worktree/__tests__/trash.test.ts b/lib/worktree/__tests__/trash.test.ts index c4ed2348..73c77411 100644 --- a/lib/worktree/__tests__/trash.test.ts +++ b/lib/worktree/__tests__/trash.test.ts @@ -164,6 +164,31 @@ describe("worktree trash", () => { expect(result.ok).toBe(false); expect(existsSync(tree)).toBe(true); }); + + // S078: `.worktrees/` was never added to info/exclude unless the tree + // went through createTree first (e.g. `rt worktree adopt`'s disposals, + // or any repo whose worktrees root differs from the default, skip that + // call entirely) — the retention store then shows up as `?? .worktrees/` + // in the user's own `git status`, and `git add -A` stages a whole second + // copy of the source tree into it. + test("retireTree excludes .worktrees/ from the repo's own git status, even without going through createTree first", async () => { + Bun.spawnSync(["git", "init", "-q", repo]); + Bun.spawnSync(["git", "-C", repo, "config", "user.email", "test@example.com"]); + Bun.spawnSync(["git", "-C", repo, "config", "user.name", "Test"]); + writeFileSync(join(repo, "README.md"), "hi\n"); + Bun.spawnSync(["git", "-C", repo, "add", "README.md"]); + Bun.spawnSync(["git", "-C", repo, "commit", "-q", "-m", "init"]); + + const tree = makeTree(root, "hotel"); + const result = await retireTree(tree, "hotel", repo); + expect(result.ok).toBe(true); + + const exclude = readFileSync(join(repo, ".git", "info", "exclude"), "utf8"); + expect(exclude).toContain(".worktrees/"); + + const status = Bun.spawnSync(["git", "-C", repo, "status", "--porcelain"]).stdout.toString(); + expect(status).not.toContain(".worktrees"); + }); }); describe("stripTrashDir", () => { diff --git a/lib/worktree/trash.ts b/lib/worktree/trash.ts index 9b39f72a..80292078 100644 --- a/lib/worktree/trash.ts +++ b/lib/worktree/trash.ts @@ -22,6 +22,7 @@ import { mkdir, readdir, rename } from "fs/promises"; import { basename, dirname, join } from "path"; +import { ensureInfoExclude } from "./git-async.ts"; /** Marks a directory as rt's to delete. Nothing without this prefix is ever reaped. */ export const TRASH_PREFIX = ".trash-"; @@ -110,6 +111,13 @@ export async function retireTree( if (!name || name.includes("/") || name.includes("\\")) { throw new Error(`worktree trash name must be a single path component: ${JSON.stringify(name)}`); } + // retainedTrashRoot is always /.worktrees/.trash regardless of + // cfg.root, so this exclude pattern is correct in every case — including + // a tree disposed without ever going through createTree first (e.g. `rt + // worktree adopt`'s disposals), whose repo may never have had + // ".worktrees/" excluded at all. Without it the retention store shows up + // as `?? .worktrees/` in the user's own `git status`. + await ensureInfoExclude(repoPath, ".worktrees/"); const root = retainedTrashRoot(repoPath); await mkdir(root, { recursive: true }); trashPath = join(root, `${name}-${Date.now()}`); From b64dac756f378e6f0e880ef113c9ee393b4eff0e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:50:52 -0500 Subject: [PATCH 015/142] docs: sketch daemon supervision verdicts + exit-code semantics --- docs/daemon-supervision-design.md | 69 +++++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 docs/daemon-supervision-design.md diff --git a/docs/daemon-supervision-design.md b/docs/daemon-supervision-design.md new file mode 100644 index 00000000..f672cafb --- /dev/null +++ b/docs/daemon-supervision-design.md @@ -0,0 +1,69 @@ +# Daemon supervision: status verdicts and exit-code semantics + +Phase 0 design anchor for the rt daemon stability roadmap (audit +2026-08). Tasks 9–14 of the Phase 0 plan implement this. + +## launchd contract + +The prod plist sets `KeepAlive = { SuccessfulExit: false }`: launchd +respawns the daemon ONLY on a non-zero exit. A zero exit means "stay +down". Every exit-code decision below follows from that single fact. + +## Exit-code policy + +| Path | Exit | Why | +|-----------------------------------------|------|-----| +| `startDaemon()` boot throw (prod path) | 1 | Visible + launchd relaunches. Paired with crash-loop detection so it cannot loop silently forever. | +| `shutdown` IPC/REST verb | 0 | Intentional stop; launchd must not respawn. Records `last-exit.kind = "shutdown"`. | +| Bare OS signal SIGTERM/SIGINT/SIGHUP | 1 | External kill (pkill, script, memory pressure); launchd SHOULD respawn. The sanctioned stop path goes through SMAppService.unregister, where the exit code is irrelevant, so exiting non-zero here does not break intended stops. | +| Crash-loop guard trips (N in M minutes) | 0 (park) | Stop the flapping; surface `crash-looping` so a human intervenes instead of launchd hammering every ~10s. | + +Mechanism: a module-scope `shuttingDownViaVerb` flag is set true by the +`shutdown` verb before it calls cleanup; `gracefulExit(signal)` reads it +— set → exit(0), unset (bare signal) → exit(1). + +Boot-phase gate: a module-scope `bootPhase: "booting" | "ready"` flips +to `"ready"` immediately before the `daemon ready` log. The +`unhandledRejection` handler exits(1) while `bootPhase === "booting"` +and only logs (recovers) once ready — so a boot-time stray rejection is +fatal but a steady-state one is not. + +## Status verdicts + +`rt daemon status` and `/api/status` classify by first match: + +1. `not-installed` — SMAppService not registered. +2. `serving` — ping on rt.sock succeeds. +3. `parked` — ping fails, a live rt pid exists, and the boot breadcrumb + phase is a flavor standoff (park). Named distinctly so the user is + told "another flavor owns the socket", not "wedged". +4. `alive-not-serving` — ping fails but a live rt pid exists + (`process.kill(pid,0)` on rt.pid, or `pgrep -f 'rt --daemon|lib/daemon.ts'`). + Sub-detail from the breadcrumb phase: `booting` / `wedged`, or + `quarantined` when a state.db/events.db boot-failed marker is present. + Prints "process is running but not answering rt.sock — rt daemon logs -t". +5. `crash-looping` — no live pid AND the kv failure record shows ≥ N + failures within the last M minutes (N=3, M=5). Prints the last reason. +6. `boot-failed` — no live pid AND the most recent kv exit record is a + boot throw (fewer than N failures). Prints the last reason + phase. +7. `installed-not-running` — registered, no live pid, clean/again-absent + exit record. + +## Persisted state (kv, ns `daemon-supervision`, no schema change) + +- `boot-attempts` (number) — incremented at the top of `runDaemon()`. +- `last-ready-at` (number, epoch ms) — stamped just before `daemon ready`. +- `recent-failures` (array of `{ at, phase, reason }`, capped to 10) — + appended by the boot fatal path and by state.db/events.db boot-failed + markers. Crash-loop = ≥ N entries newer than now − M minutes. +- `last-exit` (`{ at, kind: "shutdown" | "signal" | "boot-failed", code, reason? }`) + — written by the shutdown verb, the signal handlers, and the boot + fatal path. Lets status distinguish "cleanly stopped" from "died". + +## Boot breadcrumb + +`~/.mattstack/rt/daemon-boot.json` = `{ at, pid, flavor, phase }`, +rewritten at each boot phase: `start` → `crash-handlers` → `events-db` +→ `state-db` → `socket` → `api` → `ready`. Lets `alive-not-serving` +name where a live-but-silent daemon is stuck even when the logs are +unreadable. Removed (or stamped `ready`) on successful boot. From f0ffdbbc384df978c1df0c500c37a47a80078b92 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:50:53 -0500 Subject: [PATCH 016/142] docs: retire stale daemon-runner-health.md, point at the current audit + supervision design --- docs/daemon-runner-health.md | 189 ++--------------------------------- 1 file changed, 7 insertions(+), 182 deletions(-) diff --git a/docs/daemon-runner-health.md b/docs/daemon-runner-health.md index 95b5ae62..9d6db779 100644 --- a/docs/daemon-runner-health.md +++ b/docs/daemon-runner-health.md @@ -1,184 +1,9 @@ -# Daemon & Runner Code-Health Audit +# Daemon runner health — superseded -Reference analysis for the `rt` daemon ([lib/daemon.ts](../lib/daemon.ts) and [lib/daemon/](../lib/daemon/)) and the runner TUI ([commands/runner.tsx](../commands/runner.tsx)). +This document audited subsystems (process-manager, remedy-engine, +runner.tsx, workspace-sync) that no longer exist. It is retained only as +a redirect. -Both subsystems are central — the daemon owns all process lifecycle and long-lived state; the runner is our flagship feature and the primary surface that touches that state. Defects here leak processes, corrupt UI state, or silently drop features without any obvious error. This document exists to catalog known issues so they can be fixed deliberately rather than re-discovered. - -## Legend - -- **P0** — Active correctness bug or resource leak with user-visible impact. -- **P1** — Reliability issue, silent feature drop, or unbounded growth under realistic use. -- **P2** — Edge case or latent bug that will bite under specific conditions. -- **P3** — Code quality / testability — not a bug, but the surrounding code will rot without it. - -Each item is scoped by file:line-link so a fix PR can jump straight to the site. - ---- - -## 1. Orphaned processes - -### ~~P0 — Child process groups are not reaped on kill~~ ✅ FIXED -[lib/daemon/process-manager.ts:34-54](../lib/daemon/process-manager.ts#L34-L54), [lib/daemon/process-manager.ts:140-147](../lib/daemon/process-manager.ts#L140-L147), [lib/daemon/process-manager.ts:182-190](../lib/daemon/process-manager.ts#L182-L190) - -`Bun.spawn` now passes `detached: true` so the child is a session/pgroup leader (pgid == pid). A new `killGroup(pid, signal)` helper sends signals to `-pid` with guards against pid ≤ 1 and ESRCH. Both the kill() path and the existing-process eviction path now use it. New regression test [`process-manager.test.ts`](../lib/daemon/__tests__/process-manager.test.ts) `"kill reaps grandchildren (detached pgroup)"` verifies the fix by spawning backgrounded sleepers, killing the parent, and asserting every grandchild pid is gone. - -### ~~P0 — `process:respawn` drops the remedy subscription~~ ✅ FIXED -[lib/daemon.ts:720-726](../lib/daemon.ts#L720-L726) - -Added `remedyEngine.onSpawn(id)` after `processManager.respawn(id)`. Re-uses stored `processMeta` (cwd/cmd unchanged across respawn). - -### ~~P1 — Warm (SIGSTOP) processes survive daemon death as stopped zombies~~ ✅ FIXED -[lib/daemon/state-store.ts:28-104](../lib/daemon/state-store.ts#L28-L104), [lib/daemon.ts:1398-1411](../lib/daemon.ts#L1398-L1411), [lib/daemon/process-manager.ts:168-174](../lib/daemon/process-manager.ts#L168-L174) - -`StateStore` now persists `{state, pid}` per id. `ProcessManager.spawn` records pid on spawn; terminal transitions (`stopped`/`crashed`) clear it. `reconcileAfterRestart()` returns orphans (non-stopped with a pid), which the daemon reaps via `killGroup(pid, SIGCONT)` followed by `killGroup(pid, SIGKILL)`. Legacy `{id: state}` persistence format is still accepted on load. Regression coverage in [state-store.test.ts](../lib/daemon/__tests__/state-store.test.ts#L200-L230). - -### ~~P1 — `subscribeToOutput` unsubscribers leave empty Sets behind~~ ✅ FIXED -[lib/daemon/process-manager.ts:100-109](../lib/daemon/process-manager.ts#L100-L109) - -Unsubscribe now deletes the Set from `outputHooks` once its size reaches zero. - ---- - -## 2. Memory leaks - -### ~~P0 — `:wt` git watcher is torn down every reconcile tick~~ ✅ FIXED -[commands/runner.tsx:1580-1584](../commands/runner.tsx#L1580-L1584) - -Reconcile loop now strips a trailing `:wt` from the key before checking `activeLaneIds`, so the companion watcher lives and dies with its parent lane instead of being torn down every tick. - -### ~~P1 — Global-remedy `fs.watch` has no handle and no cleanup~~ ✅ FIXED -[lib/daemon.ts:132-163](../lib/daemon.ts#L132-L163), [lib/daemon.ts:1320](../lib/daemon.ts#L1320) - -Handle retained in `globalRemedyWatcher` and closed in `cleanup()`. - -### ~~P1 — `ProcessManager.spawnConfigs` and `outputHooks` grow unbounded~~ ✅ FIXED -[lib/daemon.ts:770-779](../lib/daemon.ts#L770-L779), [commands/runner.tsx](../commands/runner.tsx) - -Added `process:remove` IPC handler that tears down all five daemon-side maps (remedy state, attach socket, log buffer, process config, state-store entry). The runner now calls it from every entry-deletion path (remove-entry, remove-lane, reset). Entries with fresh ids no longer leak daemon-side state. - -### ~~P2 — `RemedyEngine` can create orphan state for globals-only matches~~ ✅ FIXED -[lib/daemon.ts:773](../lib/daemon.ts#L773) - -`process:remove` handler (added in Session B) calls `remedyEngine.unregister(id)` before tearing down the other maps, so globals-only state is cleaned up when the runner deletes an entry. - -### ~~P2 — `refreshCache` has no in-flight guard~~ ✅ FIXED -[lib/daemon.ts:390-407](../lib/daemon.ts#L390-L407) - -Split into a coalescing `refreshCache()` wrapper and the original logic as `refreshCacheImpl()`. Concurrent callers await the same promise. - -### ~~P2 — outputHooks Map retains empty Sets~~ ✅ FIXED -Covered by the subscribeToOutput unsubscribe cleanup above. - ---- - -## 3. Bugs - -### ~~P1 — Invalid JSON during save wipes all globals~~ ✅ FIXED -[lib/runner-store.ts:154-168](../lib/runner-store.ts#L154-L168), [lib/daemon.ts:137-158](../lib/daemon.ts#L137-L158) - -`loadGlobalRemedies` now throws on parse failure / non-array shape; missing file still returns `[]`. Daemon watcher callback catches and logs without calling `reloadGlobals`, so live rules persist through transient invalid states. - -### ~~P1 — Global-remedy watcher is not debounced~~ ✅ FIXED -[lib/daemon.ts:147-161](../lib/daemon.ts#L147-L161) - -100ms settle timer collapses the rename+change burst into a single reload. - -### ~~P1 — Concurrent `cache:refresh` via REST API~~ ✅ FIXED -Resolved by the `refreshCache` in-flight guard above. - -### ~~P2 — `entry.id` basename collision~~ ✅ FIXED -[lib/runner-store.ts:410-420](../lib/runner-store.ts#L410-L420) - -`normalizeLane` now detects within-lane entry id duplicates and appends a 6-char sha1 of the worktree path to the loser (`~a1b2c3`). Collisions no longer silently alias two processes' PTY/state. - -### ~~P2 — `onSpawn` re-merge uses stale meta for callers that pass neither cwd nor cmd~~ ✅ FIXED -[lib/daemon.ts:753-761](../lib/daemon.ts#L753-L761), [lib/daemon/process-manager.ts:245-247](../lib/daemon/process-manager.ts#L245-L247) - -`process:respawn` — the only caller that used to pass no cwd/cmd — now pulls them from `processManager.getSpawnConfig(id)` so globals-only matches survive respawn. Other `onSpawn` callers already pass cwd/cmd from their handler payload. - -### ~~P2 — `reloadGlobals` didn't subscribe for processes registered with empty remedies~~ ✅ FIXED -[lib/daemon/remedy-engine.ts:108-127](../lib/daemon/remedy-engine.ts#L108-L127) - -If `register()` was called with no per-entry remedies and `onSpawn` early-returned (no subscription because merged was empty), a later `reloadGlobals` that added matching rules updated `s.remedies` but never subscribed to output — so nothing fired. Found while writing the RemedyEngine test suite. Fix: `reloadGlobals` now subscribes on empty→non-empty and unsubscribes on non-empty→empty. Regression test in [remedy-engine.test.ts "reloadGlobals mid-flight re-merges active states"](../lib/daemon/__tests__/remedy-engine.test.ts#L300-L319). - -### ~~P3 — Dead constant~~ ✅ FIXED -[lib/daemon.ts:48](../lib/daemon.ts#L48) - -`LINEAR_REFRESH_INTERVAL_MS` removed. - -### ~~P3 — Compaction ordering is interleaving-sensitive~~ ✅ FIXED -[lib/runner-store.ts:338-348](../lib/runner-store.ts#L338-L348) - -Each entry now gets its absolute input index as its position. Groups inherit the position of their first member. Output order follows input order deterministically. - -### ~~P3 — State-store permits invalid transitions silently~~ ✅ FIXED -[lib/daemon/state-store.ts:111-119](../lib/daemon/state-store.ts#L111-L119), [lib/daemon.ts:1403-1407](../lib/daemon.ts#L1403-L1407) - -Forced transitions remain permitted (needed for kill-of-warm, reconcile, etc.) but `StateStore` now exposes `onInvalidTransition(cb)` and fires it when the move isn't in `VALID_TRANSITIONS`. The daemon wires this to its log so drift surfaces in the daemon log instead of being silently swallowed. - ---- - -## 4. Code organization / testability - -### ~~P1 — `lib/daemon.ts` is 1400+ lines with no seams~~ ✅ FIXED -[lib/daemon.ts](../lib/daemon.ts), [lib/daemon/handlers/](../lib/daemon/handlers/) - -`handleCommand` split into four domain modules behind a routed-lookup-first-then-switch dispatch: - -- [lib/daemon/handlers/process.ts](../lib/daemon/handlers/process.ts) — `process:spawn|kill|respawn|remove|start|stop|restart|list|state|states|logs|attach-info|suspend|resume` -- [lib/daemon/handlers/cache.ts](../lib/daemon/handlers/cache.ts) — `cache:read|refresh` + `branch:enrich` -- [lib/daemon/handlers/remedy.ts](../lib/daemon/handlers/remedy.ts) — `remedy:set|clear|drain` -- [lib/daemon/handlers/proxy.ts](../lib/daemon/handlers/proxy.ts) — `proxy:start|stop|set-upstream|status|list` - -Each module is a factory that takes a `HandlerContext` and returns a `HandlerMap`. Daemon.ts constructs the ctx once and merges the maps into `routedHandlers`; `handleCommand` does `routedHandlers[cmd] ?? switch` so non-extracted commands (ping, hooks:*, repos, ports, status, tcc:check, notifications*, tray:status, group:*, port:*, shutdown) remain inline because they read daemon-local state (watchers, repos index, notifications, port allocator, groups) that wouldn't benefit from being pushed out. - -Live cache access goes through `ctx.cache.entries` — `loadCache()` now mutates `cache.entries` in place instead of reassigning, so handlers see disk reloads without plumbing getters. - -### ~~P1 — `lib/runner-store.ts` doubled in size with no tests~~ ✅ FIXED -[lib/runner-store/compact.ts](../lib/runner-store/compact.ts), [lib/__tests__/runner-store-compact.test.ts](../lib/__tests__/runner-store-compact.test.ts) - -Added 7 round-trip tests and physically extracted `normalizeRemedy`, `normalizeEntry`, `compactEntries` (and their helpers) to `lib/runner-store/compact.ts`. runner-store.ts now re-exports `compactEntries`/`normalizeEntry` and delegates all compact↔expand logic. The extracted module imports only types from `runner-store.ts`, so there's no runtime cycle. - -### ~~P1 — `commands/runner.tsx` is 2500+ lines~~ ✅ PARTIALLY FIXED -[lib/runner/git-watchers.ts](../lib/runner/git-watchers.ts) - -Git watcher setup, per-lane debounce, and linked-worktree `.git/worktrees//HEAD` handling extracted into `createGitWatcherPool(onChange) → { sync, dispose }`. `repoGitDir`, `readCurrentBranch`, `readCurrentBranchAsync` now live there too. runner.tsx drops from 2500 → ~2430 lines and no longer imports `fs.watch` or `FSWatcher` at all. Remaining deferrals (keymap-handler split, `reconcileWatchers` module) are cosmetic and not bugs. - -### ~~P2 — `ProcessManager.spawn` does too many things~~ ✅ PARTIALLY FIXED -[lib/daemon/process-manager.ts:31-44](../lib/daemon/process-manager.ts#L31-L44) - -Port eviction is now an async `evictPort(port)` helper — no more synchronous `Bun.spawnSync(["sh","-c","lsof..."])` stalling the event loop. The broader decomposition of `spawn()` is deferred with the other structural refactors. - -### ~~P2 — No unit tests for `RemedyEngine` lifecycle invariants~~ ✅ FIXED -[lib/daemon/__tests__/remedy-engine.test.ts](../lib/daemon/__tests__/remedy-engine.test.ts) - -Added 13 tests driving `register`/`onSpawn`/`unregister` against a `FakeProcessManager` stub. Covers all five invariants (hook accumulation, orphan on removal, concurrent triggers, cooldown gating, double-respawn guard) plus pattern handling (array OR, invalid regex tolerance, ANSI stripping) and global remedies (cwdContains match/non-match, reload mid-flight re-merge). Writing the tests uncovered the latent `reloadGlobals`-doesn't-subscribe bug fixed above. - -### ~~P3 — Repeated sync git reads on runner startup~~ ✅ FIXED -[commands/runner.tsx:483-498](../commands/runner.tsx#L483-L498), [commands/runner.tsx:2527-2545](../commands/runner.tsx#L2527-L2545) - -Added async `readCurrentBranchAsync` (uses `Bun.spawn` + `proc.exited`) and parallelized the startup fan-out with `Promise.all`. Cost is now ~max-spawn-time, not N×spawn-time. - ---- - -## Status summary (audit sweep) - -All correctness, leak, and resource-safety items from the audit have been addressed: - -- **P0 / P1 bugs & leaks:** all fixed — pgroup reaping, remedy respawn subscription, `:wt` watcher, global-remedy hardening, refreshCache coalescing, warm-process crash recovery, empty-Set cleanup, map growth. -- **P2 correctness:** all fixed — entry.id collision salting, onSpawn meta passthrough, reloadGlobals missed-subscribe, RemedyEngine lifecycle tests, state-store invalid-transition observability. -- **P3:** all fixed — dead constant removed, compaction ordering deterministic, async parallel git reads on startup. - -**Structural refactors completed:** -- `handleCommand` split into `lib/daemon/handlers/{process,cache,remedy,proxy}.ts` — routed-lookup dispatch in daemon.ts, 14 `process:*` + 3 `cache:*`/`branch:*` + 3 `remedy:*` + 5 `proxy:*` commands all flow through factory-injected `HandlerContext`. -- compact/expand physically extracted to `lib/runner-store/compact.ts`; runner-store.ts re-exports for back-compat. -- Git watcher pool extracted to `lib/runner/git-watchers.ts` — `createGitWatcherPool(onChange) → { sync, dispose }` with built-in 150ms per-lane debounce. - -**Remaining cosmetic deferrals:** -- `commands/runner.tsx` keymap handlers could be split into sibling files for testability, but that's pure reorg — no correctness signal behind it. - -## Not covered here - -- `SuspendManager`, `ExclusiveGroup`, `AttachServer`, `WorkspaceSync`, `PortAllocator` — not re-read this pass. Worth a follow-up audit on the same four axes. -- Test coverage itself — there is very little for the daemon. Any of the extraction work above should ship with the matching test file. -- Observability — `diag()` is sprinkled throughout but there's no sampling, no correlation id, no structured reader. A shared `makeChildLogger(component, id)` helper would clean up call sites and produce greppable logs. +- Current stability audit + roadmap: `docs/daemon-stability-audit-2026-08.md` + (in the daemon-stability-audit worktree). +- Supervision verdicts + exit-code semantics: `docs/daemon-supervision-design.md`. From a15ed3e00905543bb0218c5cf252015ee0db6d0e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:51:22 -0500 Subject: [PATCH 017/142] daemon: cap request body size at 1 MiB on both servers (S092) --- .../__tests__/request-body-size.test.ts | 42 +++++++++++++++++++ lib/daemon/api-server.ts | 2 + lib/daemon/request-limits.ts | 10 +++++ lib/daemon/socket-server.ts | 2 + 4 files changed, 56 insertions(+) create mode 100644 lib/daemon/__tests__/request-body-size.test.ts create mode 100644 lib/daemon/request-limits.ts diff --git a/lib/daemon/__tests__/request-body-size.test.ts b/lib/daemon/__tests__/request-body-size.test.ts new file mode 100644 index 00000000..b6ee04a5 --- /dev/null +++ b/lib/daemon/__tests__/request-body-size.test.ts @@ -0,0 +1,42 @@ +/** + * Bun enforces `maxRequestBodySize` itself (413 before the handler runs) -- + * this is a live-server test, not a pure-function one, because there is no + * pure function to unit test: the cap is a Bun.serve runtime option. + */ +import { describe, test, expect, afterEach } from "bun:test"; +import type { Server } from "bun"; +import { MAX_REQUEST_BODY_SIZE } from "../request-limits.ts"; + +let server: Server | undefined; + +afterEach(() => { + server?.stop(true); + server = undefined; +}); + +describe("MAX_REQUEST_BODY_SIZE", () => { + test("is set to 1 MiB", () => { + expect(MAX_REQUEST_BODY_SIZE).toBe(1024 * 1024); + }); + + test("Bun rejects a body over the cap with a 4xx before the handler runs", async () => { + let handlerRan = false; + server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + maxRequestBodySize: 10, // tiny cap for a fast, deterministic test + async fetch(req) { + handlerRan = true; + await req.text(); + return new Response("ok"); + }, + }); + const res = await fetch(`http://127.0.0.1:${server.port}/`, { + method: "POST", + body: "x".repeat(1000), + }); + expect(res.status).toBeGreaterThanOrEqual(400); + expect(res.status).toBeLessThan(500); + expect(handlerRan).toBe(false); + }); +}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index 7dab9b6c..39558743 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -12,6 +12,7 @@ import type { Logger } from "pino"; import { API_PORT } from "../daemon-config.ts"; import { needsToken, tokenOk, loadOrCreateApiToken } from "./api-auth.ts"; import { getAggregatedConnection } from "./freshness.ts"; +import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; const API_INDEX = { name: "rt daemon", @@ -140,6 +141,7 @@ export async function startApiServer(deps: ApiServerDeps): Promise> // clients aren't reaped every 10s ("[Bun.serve]: request timed out"). // 255 is the server max; the websocket block below sets its own larger one. idleTimeout: 255, + maxRequestBodySize: MAX_REQUEST_BODY_SIZE, async fetch(req, server) { const url = new URL(req.url); diff --git a/lib/daemon/request-limits.ts b/lib/daemon/request-limits.ts new file mode 100644 index 00000000..d8448818 --- /dev/null +++ b/lib/daemon/request-limits.ts @@ -0,0 +1,10 @@ +/** + * Shared cap on request body size for both daemon servers (api-server.ts's + * :9401 HTTP/WS surface and socket-server.ts's unix-socket IPC channel). + * Neither transport authenticates reads, so an unbounded body (Bun's + * default is 128 MB) lets any same-user process or a cross-origin browser + * request stall the daemon's single event loop parsing a giant payload. + * Real payloads on both transports are kilobytes; 1 MiB costs nothing and + * turns an oversized request into an immediate 413 instead. + */ +export const MAX_REQUEST_BODY_SIZE = 1024 * 1024; diff --git a/lib/daemon/socket-server.ts b/lib/daemon/socket-server.ts index 7ab37fb8..fee4c27e 100644 --- a/lib/daemon/socket-server.ts +++ b/lib/daemon/socket-server.ts @@ -9,6 +9,7 @@ import { existsSync, unlinkSync } from "fs"; import type { Server } from "bun"; import type { Logger } from "pino"; import { DAEMON_SOCK_PATH } from "../daemon-config.ts"; +import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; export function startSocketServer(opts: { handleCommand: (cmd: string, payload: any, signal?: AbortSignal) => Promise; @@ -33,6 +34,7 @@ export function startSocketServer(opts: { // on how long an idle connection may sit before Bun kills it — it never // holds connections open on its own. idleTimeout: 255, + maxRequestBodySize: MAX_REQUEST_BODY_SIZE, async fetch(req) { try { const url = new URL(req.url); From cd20722ce829db604e3b852e300d5a489bbf8eca Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:55:23 -0500 Subject: [PATCH 018/142] worktree trash reap: require a plausible rt-written epoch, and refuse a configured root that is an ancestor of the repo (S079) --- .../__tests__/worktree-reconciler.test.ts | 21 +++++++++++++++- lib/daemon/worktree-reconciler.ts | 24 +++++++++++++++++-- lib/worktree/__tests__/trash.test.ts | 13 ++++++++++ lib/worktree/trash.ts | 17 ++++++++++++- 4 files changed, 71 insertions(+), 4 deletions(-) diff --git a/lib/daemon/__tests__/worktree-reconciler.test.ts b/lib/daemon/__tests__/worktree-reconciler.test.ts index 8f30830a..8d2bca47 100644 --- a/lib/daemon/__tests__/worktree-reconciler.test.ts +++ b/lib/daemon/__tests__/worktree-reconciler.test.ts @@ -2,7 +2,7 @@ import { describe, test, expect, beforeEach, spyOn } from "bun:test"; import { execSync } from "child_process"; import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; -import { basename, join } from "path"; +import { basename, dirname, join } from "path"; import type { Logger } from "pino"; import { readJson, writeJson } from "../../json-store.ts"; import { closeStateDb, listKvValues, setKvValue } from "../../state/index.ts"; @@ -1380,4 +1380,23 @@ describe("reapRepoTrash", () => { await waitFor(() => !existsSync(leftover) && !existsSync(expired)); expect(existsSync(fresh)).toBe(true); }); + + // S079: sanitizeRoot (lib/worktree/config.ts) has no ancestor check, so a + // repo configured with e.g. `root: "${repoRoot}/.."` makes the crash sweep + // walk the parent directory of every sibling repo for `.trash-*` names. + test("refuses a configured root outside the repo and warns instead of sweeping it", async () => { + const parent = dirname(repo); + const siblingLeftover = join(parent, ".trash-should-survive-123"); + mkdirSync(siblingLeftover, { recursive: true }); + await declareWorktrees(repo, "acme", { root: parent }); + + const warns: unknown[][] = []; + const log = { info: () => {}, warn: (...a: unknown[]) => warns.push(a), error: () => {}, debug: () => {} } as unknown as Logger; + + await __test__.reapRepoTrash({ repoName: "acme", repoPath: repo, log }); + await new Promise((r) => setTimeout(r, 300)); // give a wrongly-spawned detached rm time to run + + expect(existsSync(siblingLeftover)).toBe(true); + expect(warns.some((w) => JSON.stringify(w).includes(parent))).toBe(true); + }); }); diff --git a/lib/daemon/worktree-reconciler.ts b/lib/daemon/worktree-reconciler.ts index d427c942..f2e20065 100644 --- a/lib/daemon/worktree-reconciler.ts +++ b/lib/daemon/worktree-reconciler.ts @@ -8,7 +8,7 @@ * `creationInFlight`). */ -import { basename, join } from "path"; +import { basename, isAbsolute, join, relative, resolve } from "path"; import { realpathSync } from "fs"; import type { Logger } from "pino"; import { rtDir } from "../rt-paths.ts"; @@ -1002,10 +1002,30 @@ async function replenishAndShrink( * parks trees stripped-but-recoverable (RT-51) — are reaped only past the * retention window. */ +/** + * Whether `root` is repoPath itself or a strict ancestor of it — + * sanitizeRoot (lib/worktree/config.ts) has no such check, so a value like + * `${repoRoot}/..` sweeps the parent directory shared by every sibling repo + * for `.trash-*` names. An unrelated, dedicated external root (the + * documented `root: "~/wt"` case) is fine to sweep — it's a repo-specific + * destination nothing else shares — so this only refuses the ancestor + * shape, not "root lies outside repoPath" in general. + */ +function isRootAnAncestorOfRepo(repoPath: string, root: string): boolean { + const rel = relative(resolve(root), resolve(repoPath)); + return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel)); +} + async function reapRepoTrash(deps: { repoName: string; repoPath: string; log: Logger }): Promise { const { repoName, repoPath, log } = deps; const cfg = await loadWorktreeRepoConfig(repoName, repoPath); - const reaped = await reapTrashInRoots([join(repoPath, ".worktrees"), cfg.root], log); + const roots = [join(repoPath, ".worktrees")]; + if (isRootAnAncestorOfRepo(repoPath, cfg.root)) { + log.warn({ repo: repoName, root: cfg.root, repoPath }, "worktree trash sweep refused a configured root that is an ancestor of the repo"); + } else { + roots.push(cfg.root); + } + const reaped = await reapTrashInRoots(roots, log); if (reaped > 0) log.info({ repo: repoName, count: reaped }, "worktree trash reaped"); const expired = await reapExpiredTrash(repoPath, log); if (expired > 0) log.info({ repo: repoName, count: expired }, "worktree retention trash reaped"); diff --git a/lib/worktree/__tests__/trash.test.ts b/lib/worktree/__tests__/trash.test.ts index 73c77411..f732e550 100644 --- a/lib/worktree/__tests__/trash.test.ts +++ b/lib/worktree/__tests__/trash.test.ts @@ -267,6 +267,19 @@ describe("worktree trash", () => { expect(existsSync(stray)).toBe(true); expect(warns.length).toBe(1); }); + + // S079: a trailing small integer (a manual "backup-3", "notes-42" dropped + // "with the other trash") parses as a number just fine and, taken at face + // value as a ms-epoch, is always ancient — rm -rf'd on the very next pass + // despite the doc comment promising non-rt entries are kept. This is a + // distinct failure shape from "not-rt-made" above (no digits at all). + test("an entry whose trailing digits are not a plausible rt epoch is kept and warned about", async () => { + const stray = makeTree(retainedTrashRoot(repo), "backup-3"); + const { log, warns } = capturingLog(); + expect(await reapExpiredTrash(repo, log, Date.now())).toBe(0); + expect(existsSync(stray)).toBe(true); + expect(warns.length).toBe(1); + }); }); describe("reapTrashDir on retained entries", () => { diff --git a/lib/worktree/trash.ts b/lib/worktree/trash.ts index 80292078..70921734 100644 --- a/lib/worktree/trash.ts +++ b/lib/worktree/trash.ts @@ -39,6 +39,21 @@ const RETAIN_DIR = ".trash"; /** How long a retained tree survives before the reconciler reaps it. */ export const RETENTION_MS = 14 * 24 * 60 * 60 * 1000; +/** + * A ms-epoch rt actually wrote (`${name}-${Date.now()}`) is always after + * this. A trailing small integer — a manual "backup-3" or "notes-42" + * dropped "with the other trash" — parses as a number just fine and, taken + * at face value as an epoch, is always ancient; without this floor it gets + * reaped on the very next pass despite not being rt's to delete. + */ +const EPOCH_FLOOR_MS = Date.UTC(2020, 0, 1); + +/** Whether `raw` looks like an epoch rt itself would have written, not merely any integer. */ +function looksLikeRtEpoch(raw: string): boolean { + const n = Number(raw); + return Number.isInteger(n) && n >= EPOCH_FLOOR_MS; +} + /** * Top-level dirs inside a retained tree that are reinstallable and deleted at * dispose time (exact names, plus the `dist-*` family). Mispredicting here @@ -202,7 +217,7 @@ export async function reapExpiredTrash( let reaped = 0; for (const entry of entries) { const epoch = /-(\d+)$/.exec(entry)?.[1]; - if (!epoch) { + if (!epoch || !looksLikeRtEpoch(epoch)) { log.warn({ root, entry }, "worktree retention sweep skipped an entry it did not write"); continue; } From 32fd1973f9d3f49e6c8594a5d407987ac6d14983 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:56:47 -0500 Subject: [PATCH 019/142] daemon: one shared api-token cache for api-server and secrets handler (S054) api-server captured the token once at boot while the secrets handler called loadOrCreateApiToken() fresh on every request; an external rotation or an unwritable token dir left the two permanently disagreeing about the current token. getApiToken/reloadApiToken share one in-memory cache between both consumers, and a persist failure now logs a warning instead of failing silently. --- lib/daemon/__tests__/api-auth.test.ts | 48 ++++++++++++++++++++++++++- lib/daemon/api-auth.ts | 32 +++++++++++++++++- lib/daemon/api-server.ts | 4 +-- lib/daemon/handlers/secrets.ts | 6 ++-- 4 files changed, 83 insertions(+), 7 deletions(-) diff --git a/lib/daemon/__tests__/api-auth.test.ts b/lib/daemon/__tests__/api-auth.test.ts index d9cda0b1..ad97eef9 100644 --- a/lib/daemon/__tests__/api-auth.test.ts +++ b/lib/daemon/__tests__/api-auth.test.ts @@ -4,7 +4,10 @@ */ import { describe, test, expect } from "bun:test"; -import { needsToken, tokenOk } from "../api-auth.ts"; +import { needsToken, tokenOk, getApiToken, reloadApiToken, loadOrCreateApiToken } from "../api-auth.ts"; +import { mkdtempSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; describe("needsToken", () => { test("shutdown requires a token", () => { @@ -58,3 +61,46 @@ describe("tokenOk", () => { expect(tokenOk("anything", "")).toBe(false); }); }); + +describe("getApiToken / reloadApiToken singleton", () => { + test("getApiToken caches: a second call does not re-read the file even if it changes underneath", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + const first = reloadApiToken(tokenPath); // seed the cache with a known path + writeFileSync(tokenPath, "a-different-token", { mode: 0o600 }); + const second = getApiToken(tokenPath); // ignores the new file content -- cached + expect(second).toBe(first); + expect(second).not.toBe("a-different-token"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("reloadApiToken re-reads and updates the cache", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + reloadApiToken(tokenPath); + writeFileSync(tokenPath, "rotated-token", { mode: 0o600 }); + const reloaded = reloadApiToken(tokenPath); + expect(reloaded).toBe("rotated-token"); + expect(getApiToken(tokenPath)).toBe("rotated-token"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("loadOrCreateApiToken still works standalone (unchanged primitive)", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + const a = loadOrCreateApiToken(tokenPath); + const b = loadOrCreateApiToken(tokenPath); + expect(a).toBe(b); + expect(a.length).toBeGreaterThan(0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); diff --git a/lib/daemon/api-auth.ts b/lib/daemon/api-auth.ts index e41b923b..89d07980 100644 --- a/lib/daemon/api-auth.ts +++ b/lib/daemon/api-auth.ts @@ -12,6 +12,9 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; import { join } from "path"; import { randomUUID } from "crypto"; import { RT_DIR } from "../daemon-config.ts"; +import { lazyChildLogger } from "../daemon-logger.ts"; + +const log = lazyChildLogger("api-auth"); /** Where the local API token is persisted (0600) for trusted local clients. */ export const API_TOKEN_PATH = join(RT_DIR, "api-token"); @@ -32,10 +35,37 @@ export function loadOrCreateApiToken(tokenPath: string = API_TOKEN_PATH): string try { mkdirSync(RT_DIR, { recursive: true }); writeFileSync(tokenPath, token, { mode: 0o600 }); - } catch { /* best-effort; token still enforced in-memory this run */ } + } catch (err) { + log.warn({ err, tokenPath }, "could not persist api-token; enforced in-memory only this run, so a client reading the file will disagree until the daemon restarts"); + } return token; } +/** + * `getApiToken`/`reloadApiToken` share ONE in-memory value between + * api-server.ts and the secrets handler (S054): before this, api-server + * captured a token once at boot while the secrets handler called + * `loadOrCreateApiToken()` fresh on every request, so an external rotation + * (deleting api-token to force a new one) left the two permanently + * disagreeing about which token is current -- and if the token dir was + * unwritable, the secrets handler regenerated a brand new random token on + * every single call, never matching anything a client could read from disk. + * Both consumers now read the SAME cached value; a rotation only takes + * effect for both after `reloadApiToken()` runs or the daemon restarts, + * either of which was already the closest thing to a happy path before. + */ +let cachedApiToken: string | null = null; + +export function getApiToken(tokenPath: string = API_TOKEN_PATH): string { + if (cachedApiToken === null) cachedApiToken = loadOrCreateApiToken(tokenPath); + return cachedApiToken; +} + +export function reloadApiToken(tokenPath: string = API_TOKEN_PATH): string { + cachedApiToken = loadOrCreateApiToken(tokenPath); + return cachedApiToken; +} + /** True when a request mutates state, or (secrets) returns raw credential values, and must present the local token. */ export function needsToken(method: string, pathname: string): boolean { if (method === "OPTIONS") return false; diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index 39558743..d61909fd 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -10,7 +10,7 @@ import type { Server, ServerWebSocket } from "bun"; import type { Logger } from "pino"; import { API_PORT } from "../daemon-config.ts"; -import { needsToken, tokenOk, loadOrCreateApiToken } from "./api-auth.ts"; +import { needsToken, tokenOk, getApiToken } from "./api-auth.ts"; import { getAggregatedConnection } from "./freshness.ts"; import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; @@ -131,7 +131,7 @@ export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDe export async function startApiServer(deps: ApiServerDeps): Promise> { const { handleCommand, log } = deps; - const apiToken = loadOrCreateApiToken(); + const apiToken = getApiToken(); const server = await bindApiServerWithRetry(() => Bun.serve({ port: API_PORT, diff --git a/lib/daemon/handlers/secrets.ts b/lib/daemon/handlers/secrets.ts index a0e33021..2608e603 100644 --- a/lib/daemon/handlers/secrets.ts +++ b/lib/daemon/handlers/secrets.ts @@ -46,7 +46,7 @@ import { loadSecrets } from "../../linear.ts"; import { parseIdentity } from "../../settings/identity.ts"; import { loadMachineRepoTracking, grants, type RepoTracking } from "../../repo-tracking.ts"; -import { loadOrCreateApiToken, tokenOk } from "../api-auth.ts"; +import { getApiToken, tokenOk } from "../api-auth.ts"; import { readSecret, createRealSecretsExecSeam, type SecretsSeams } from "../../secrets/store.ts"; import { createRealAgeKeySeam } from "../../home/age-key.ts"; import type { Commands, ForgeSlug } from "../../../packages/rt-client/src/commands.ts"; @@ -125,7 +125,7 @@ export interface SecretsHandlerOverrides { deckSecrets?: () => Promise<{ cfApiToken?: string; cfZoneId?: string }>; /** Defaults to `loadBoardSecrets` (cross-domain: `board` + `rt`) for secrets:read's "board" scope. */ boardSecrets?: () => Promise; - /** Defaults to `loadOrCreateApiToken` (the real ~/.mattstack/rt/api-token, shared with api-auth.ts). */ + /** Defaults to `getApiToken` (the real ~/.mattstack/rt/api-token, shared with api-auth.ts and api-server.ts). */ apiToken?: () => string; } @@ -142,7 +142,7 @@ export function createSecretsHandlers( const extensionSecrets = overrides.extensionSecrets ?? loadSecrets; const deckSecrets = overrides.deckSecrets ?? loadDeckSecrets; const boardSecrets = overrides.boardSecrets ?? loadBoardSecrets; - const apiToken = overrides.apiToken ?? (() => loadOrCreateApiToken()); + const apiToken = overrides.apiToken ?? (() => getApiToken()); return { "secrets:forge-token": async (payload: Commands["secrets:forge-token"]["payload"]) => { From 421d67b6ec4ae1338fa4bf2a62a039db6a637315 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:57:46 -0500 Subject: [PATCH 020/142] daemon: boot failure is fatal (exit 1), gated by boot-phase flag; rt.pid after binds Boot failures on the prod path used to leave a live-pid zombie: a stale rt.pid could get written before the socket/API binds even attempted, so a failed boot exited without ever removing it. runDaemon() now wraps its body in try/catch (log.fatal + flush + exit 1 on any failure), and rt.pid is only written once both servers.socket and servers.api are assigned. installCrashHandlers gains an opts.booting predicate: unhandledRejection is fatal + exit 1 while true (boot phase, nothing worth staying up for), and logs only (today's behavior) once bootPhase flips to "ready" right before "daemon ready". --- e2e/tests/daemon.test.ts | 18 ++ lib/__tests__/daemon-logger.test.ts | 42 ++++- lib/daemon-logger.ts | 16 +- lib/daemon.ts | 263 +++++++++++++++------------- 4 files changed, 212 insertions(+), 127 deletions(-) diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index 48345d0f..37982958 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -3,6 +3,24 @@ import { existsSync } from "fs"; import { join } from "path"; import { createTestHome, rt } from "../harness.ts"; +describe("fatal boot", () => { + test("daemon boot with API port already bound exits non-zero and leaves no stale rt.pid", async () => { + const { path: home, cleanup } = createTestHome(); + // Bind the API port inside the isolated HOME so the daemon cannot. + const port = 9411; + const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + try { + const result = await rt(["--daemon"], { home, env: { RT_API_PORT: String(port) } }); + + expect(result.exitCode).not.toBe(0); + expect(existsSync(join(home, ".mattstack", "rt", "rt.pid"))).toBe(false); + } finally { + squatter.stop(true); + cleanup(); + } + }, 60_000); +}); + describe("daemon", () => { describe("install creates config", () => { let home: string; diff --git a/lib/__tests__/daemon-logger.test.ts b/lib/__tests__/daemon-logger.test.ts index 1370be57..69bd3a9e 100644 --- a/lib/__tests__/daemon-logger.test.ts +++ b/lib/__tests__/daemon-logger.test.ts @@ -1,10 +1,18 @@ -import { describe, it, expect, beforeEach, afterEach } from "bun:test"; +import { describe, it, expect, beforeEach, afterEach, mock } from "bun:test"; import { mkdtempSync, rmSync, readFileSync, readdirSync, existsSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import type { Logger } from "pino"; // We import the factory (not the singleton) so each test gets isolation. -import { createDaemonLogger, lazyChildLogger, getDaemonLogger, __test__ } from "../daemon-logger.ts"; +import { + createDaemonLogger, + lazyChildLogger, + getDaemonLogger, + installCrashHandlers, + __test__, + type DaemonLoggerHandle, +} from "../daemon-logger.ts"; import { logsDir } from "../rt-paths.ts"; let logDir: string; @@ -191,3 +199,33 @@ describe("lazyChildLogger — Proxy guard", () => { expect(() => (log as any).level).toThrow(); }); }); + +describe("installCrashHandlers — boot-phase gating", () => { + it("unhandledRejection exits(1) while booting, only logs once ready", () => { + const exits: number[] = []; + const origExit = process.exit; + const origStderrWrite = process.stderr.write.bind(process.stderr); + // @ts-expect-error test stub — captures the exit code instead of terminating + process.exit = (code?: number) => { exits.push(code ?? 0); }; + const fatal = mock(() => {}); + const error = mock(() => {}); + const logger = { fatal, error } as unknown as Logger; + let booting = true; + try { + installCrashHandlers({ logger } as DaemonLoggerHandle, { booting: () => booting }); + process.emit("unhandledRejection", new Error("boot boom"), Promise.resolve()); + expect(fatal).toHaveBeenCalledTimes(1); + expect(exits).toEqual([1]); + + booting = false; + process.emit("unhandledRejection", new Error("steady boom"), Promise.resolve()); + expect(error).toHaveBeenCalledTimes(1); + expect(exits).toEqual([1]); // no second exit + } finally { + process.exit = origExit; + process.stderr.write = origStderrWrite; + process.removeAllListeners("unhandledRejection"); + process.removeAllListeners("uncaughtException"); + } + }); +}); diff --git a/lib/daemon-logger.ts b/lib/daemon-logger.ts index ecca0df9..bd4826e1 100644 --- a/lib/daemon-logger.ts +++ b/lib/daemon-logger.ts @@ -225,11 +225,18 @@ export function redirectNativeStderr(): void { * logger before exit. Call ONCE during daemon startup, AFTER logger init. * * - uncaughtException: log as fatal (sync), exit 1 - * - unhandledRejection: log as error, do NOT exit (let the daemon recover) + * - unhandledRejection: fatal + exit 1 while `opts.booting()` is true (no + * socket/API bound yet, nothing worth staying up for); log as error and + * stay alive once booted, so a stray steady-state rejection doesn't kill a + * daemon that's already serving. No `booting` given preserves the old + * always-log, never-exit behavior. * - process.stderr.write: intercept so console.error / anything writing to * stderr lands in the JSON log instead of vanishing. */ -export function installCrashHandlers(handle: DaemonLoggerHandle): void { +export function installCrashHandlers( + handle: DaemonLoggerHandle, + opts: { booting?: () => boolean } = {}, +): void { const { logger } = handle; // Because the pino-roll stream is opened with sync:true, logger.fatal() @@ -240,6 +247,11 @@ export function installCrashHandlers(handle: DaemonLoggerHandle): void { }); process.on("unhandledRejection", (reason) => { + if (opts.booting?.()) { + logger.fatal({ err: reason }, "unhandledRejection during boot"); + process.exit(1); + return; + } logger.error({ err: reason }, "unhandledRejection"); }); diff --git a/lib/daemon.ts b/lib/daemon.ts index fc2ba5ed..fcfe6ca5 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -75,6 +75,11 @@ import type { PortEntry } from "./port-scanner.ts"; // a clean rename of a real legacy tree into a conflict. Idempotent — the CLI // entry (cli.ts) also runs it, but `bun run lib/daemon.ts` skips cli.ts. import { migrateLegacyRtDir, LEGACY_RT_LABEL, RT_DIR_LABEL, logsDir } from "./rt-paths.ts"; + +// Gates installCrashHandlers' unhandledRejection handler: fatal during boot +// (no socket/API bound yet, nothing to recover), advisory-only once ready. +let bootPhase: "booting" | "ready" = "booting"; + const rtMigration = migrateLegacyRtDir(); // ─── Logging ───────────────────────────────────────────────────────────────── @@ -383,134 +388,146 @@ const cleanup = (): void => { // ─── Entry ─────────────────────────────────────────────────────────────────── async function runDaemon(): Promise { - mkdirSync(RT_DIR, { recursive: true }); - - // Capture native panics (bypass JS entirely) at the fd level, then wire - // uncaughtException + unhandledRejection through pino. Must run BEFORE - // any async work that could throw uncaught. - redirectNativeStderr(); - installCrashHandlers(loggerHandle); - - // If a previous daemon process is still alive (orphan from a failed - // restart), evict it before we bind the socket. - evictStaleDaemon(log); - - // Auto-unlink any tagged tool link whose tool now has a genuine user copy - // elsewhere on PATH (e.g. the user ran `brew install gh` after rt linked - // the bundled one). reconcile() itself is synchronous (a ~/.local/bin - // readDir plus a handful of stats) — wrapping the call in `async` alone - // would NOT defer it, since nothing inside actually awaits. setTimeout(0) - // is what actually pushes it past the rest of this function: the PID - // write, openBranchCacheStore, and both server binds below all run first, - // on this same synchronous pass, before the timer callback ever fires. - setTimeout(() => { - try { - const { removed } = reconcileLinks(createRealProbes()); - if (removed.length > 0) log.info({ removed }, "deps: auto-unlinked tools now shadowed by a user copy"); - } catch (err) { - log.warn({ err }, "deps: link reconcile failed"); - } - }, 0); - - log.info("daemon starting"); - writeFileSync(DAEMON_PID_PATH, String(process.pid)); - - // Open state.db and build the in-memory branch-cache map BEFORE serving - // (spec "Migration & contention"): the one long transaction is the - // legacy-JSON import, and it must never land inside the event loop. If a - // CLI process is mid-import right now, we block here, in startup. - openBranchCacheStore(); - log.info({ count: Object.keys(cache.entries).length }, "branch cache loaded from state.db"); - - // one-shot re-key of every legacy NAME-keyed store row onto its - // serialized repo identity. Fire-and-forget (not awaited) like the PATH - // reconcile above — the ordering guarantee this depends on (running before - // anything prunes the repo index) only needs this to be on the boot path, - // not blocking the socket bind; a prune only ever arrives as a command sent - // to an already-running daemon. - runBootIdentityMigration(log).catch((err) => { - log.warn({ err }, "boot identity migration failed"); - }); - - routedHandlers = buildRoutedHandlers({ - ctx: handlerCtx, - broadcast: emit, - systemProcessScanner, - worktree: { - emit, - kick: worktreeReconciler.kick, - creationInFlight: worktreeReconciler.creationInFlight, - }, - eventsBus, - homeSnapshot, - repos: { - withReconcilerHeld: worktreeReconciler.withReconcilerHeld, - refreshWatchedRepos: hooksGuard.refreshWatchedRepos, - }, - stateDb: getStateDb("daemon"), - }); - - // No waiter outlives the daemon, so every armed_at set at boot is stale; - // clearing must finish before the socket listens, or an agent that arms - // in the gap has its fresh armed_at wiped. - const clearedArmed = clearAllArmed(); - if (clearedArmed > 0) log.info({ clearedArmed }, "chat: cleared stale armed_at from previous daemon run"); - - // Daemon startup is one of the two moments a handle is about to be - // needed (spec "Pruning"); sign-in is the other, inside signIn itself. - // Pruning is best-effort cleanup, so a concurrent CLI writer's - // SQLITE_BUSY must not abort startup before the socket binds. - let prunedPresence = 0; - persistOrWarn("daemon", () => { prunedPresence = prunePresence(Date.now()); }, { op: "prunePresence" }); - if (prunedPresence > 0) log.info({ prunedPresence }, "chat: pruned stale presence rows at daemon startup"); - - // Socket server (Unix socket for CLI/tray) + REST/WS server (external clients) - servers.socket = startSocketServer({ handleCommand, log }); - servers.api = await startApiServer({ handleCommand, log }); - - // Wire notification broadcasts to WebSocket clients - onNotification(emit); - - // Discover and watch repos - hooksGuard.refreshWatchedRepos(); - - // Team tracking intent (mattstack.tracking) resolves through a primed - // identity→name map, not live derivation — loadRepoTracking is sync and - // runs on every freshness tick. Team intent is inert until this completes. - // The repo index moved into state.db (RT-50): there is no file to fs.watch - // for new-repo changes any more, so the 60s hooks-scan poller (pollers.ts) - // is the only re-prime mechanism now, not just the reliable one. - primeTeamTrackingIdentityMap(loadRepoIndex()).catch((err) => { - log.warn({ err }, "repo-tracking: failed to prime team-intent identity map"); - }); - - // Periodic background work: cache refresh, port scan, system-process scan, - // hooks-guard fallback rescan. - startPollers({ - log, refreshCache, portCacheRef, broadcast: emit, systemProcessScanner, - repoIndex: loadRepoIndex, - checkAndRepairHooksPath: hooksGuard.checkAndRepairHooksPath, - }); - - // Kick off the events watchers once the first refresh has populated the - // cache with repoName stamps. reconcileFreshness inside the cache refresher - // follows repo-index changes from there. - setTimeout(() => { - initFreshness(freshnessEnv).catch((err) => { - log.error({ err }, "freshness: init failed"); + try { + mkdirSync(RT_DIR, { recursive: true }); + + // Capture native panics (bypass JS entirely) at the fd level, then wire + // uncaughtException + unhandledRejection through pino. Must run BEFORE + // any async work that could throw uncaught. + redirectNativeStderr(); + installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" }); + + // If a previous daemon process is still alive (orphan from a failed + // restart), evict it before we bind the socket. + evictStaleDaemon(log); + + // Auto-unlink any tagged tool link whose tool now has a genuine user copy + // elsewhere on PATH (e.g. the user ran `brew install gh` after rt linked + // the bundled one). reconcile() itself is synchronous (a ~/.local/bin + // readDir plus a handful of stats) — wrapping the call in `async` alone + // would NOT defer it, since nothing inside actually awaits. setTimeout(0) + // is what actually pushes it past the rest of this function: the PID + // write, openBranchCacheStore, and both server binds below all run first, + // on this same synchronous pass, before the timer callback ever fires. + setTimeout(() => { + try { + const { removed } = reconcileLinks(createRealProbes()); + if (removed.length > 0) log.info({ removed }, "deps: auto-unlinked tools now shadowed by a user copy"); + } catch (err) { + log.warn({ err }, "deps: link reconcile failed"); + } + }, 0); + + log.info("daemon starting"); + + // Open state.db and build the in-memory branch-cache map BEFORE serving + // (spec "Migration & contention"): the one long transaction is the + // legacy-JSON import, and it must never land inside the event loop. If a + // CLI process is mid-import right now, we block here, in startup. + openBranchCacheStore(); + log.info({ count: Object.keys(cache.entries).length }, "branch cache loaded from state.db"); + + // one-shot re-key of every legacy NAME-keyed store row onto its + // serialized repo identity. Fire-and-forget (not awaited) like the PATH + // reconcile above — the ordering guarantee this depends on (running before + // anything prunes the repo index) only needs this to be on the boot path, + // not blocking the socket bind; a prune only ever arrives as a command sent + // to an already-running daemon. + runBootIdentityMigration(log).catch((err) => { + log.warn({ err }, "boot identity migration failed"); + }); + + routedHandlers = buildRoutedHandlers({ + ctx: handlerCtx, + broadcast: emit, + systemProcessScanner, + worktree: { + emit, + kick: worktreeReconciler.kick, + creationInFlight: worktreeReconciler.creationInFlight, + }, + eventsBus, + homeSnapshot, + repos: { + withReconcilerHeld: worktreeReconciler.withReconcilerHeld, + refreshWatchedRepos: hooksGuard.refreshWatchedRepos, + }, + stateDb: getStateDb("daemon"), + }); + + // No waiter outlives the daemon, so every armed_at set at boot is stale; + // clearing must finish before the socket listens, or an agent that arms + // in the gap has its fresh armed_at wiped. + const clearedArmed = clearAllArmed(); + if (clearedArmed > 0) log.info({ clearedArmed }, "chat: cleared stale armed_at from previous daemon run"); + + // Daemon startup is one of the two moments a handle is about to be + // needed (spec "Pruning"); sign-in is the other, inside signIn itself. + // Pruning is best-effort cleanup, so a concurrent CLI writer's + // SQLITE_BUSY must not abort startup before the socket binds. + let prunedPresence = 0; + persistOrWarn("daemon", () => { prunedPresence = prunePresence(Date.now()); }, { op: "prunePresence" }); + if (prunedPresence > 0) log.info({ prunedPresence }, "chat: pruned stale presence rows at daemon startup"); + + // Socket server (Unix socket for CLI/tray) + REST/WS server (external clients) + servers.socket = startSocketServer({ handleCommand, log }); + servers.api = await startApiServer({ handleCommand, log }); + + // Only write rt.pid once both servers are actually bound — a boot that + // fails before this point must never leave a live-pid file with no + // socket/API behind it. + writeFileSync(DAEMON_PID_PATH, String(process.pid)); + + // Wire notification broadcasts to WebSocket clients + onNotification(emit); + + // Discover and watch repos + hooksGuard.refreshWatchedRepos(); + + // Team tracking intent (mattstack.tracking) resolves through a primed + // identity→name map, not live derivation — loadRepoTracking is sync and + // runs on every freshness tick. Team intent is inert until this completes. + // The repo index moved into state.db (RT-50): there is no file to fs.watch + // for new-repo changes any more, so the 60s hooks-scan poller (pollers.ts) + // is the only re-prime mechanism now, not just the reliable one. + primeTeamTrackingIdentityMap(loadRepoIndex()).catch((err) => { + log.warn({ err }, "repo-tracking: failed to prime team-intent identity map"); + }); + + // Periodic background work: cache refresh, port scan, system-process scan, + // hooks-guard fallback rescan. + startPollers({ + log, refreshCache, portCacheRef, broadcast: emit, systemProcessScanner, + repoIndex: loadRepoIndex, + checkAndRepairHooksPath: hooksGuard.checkAndRepairHooksPath, }); - }, 7000); - // Background sweep for new MR comments → `discussions:new-comments` events. - startDiscussionsPoller({ ctx: handlerCtx, broadcast: emit }); + // Kick off the events watchers once the first refresh has populated the + // cache with repoName stamps. reconcileFreshness inside the cache refresher + // follows repo-index changes from there. + setTimeout(() => { + initFreshness(freshnessEnv).catch((err) => { + log.error({ err }, "freshness: init failed"); + }); + }, 7000); + + // Background sweep for new MR comments → `discussions:new-comments` events. + startDiscussionsPoller({ ctx: handlerCtx, broadcast: emit }); - // Sandbox ground-truth reconcile: port-forwards, dev-ports mirroring, and - // typed-event → notification fan-out (no-op while no controller answers). + // Sandbox ground-truth reconcile: port-forwards, dev-ports mirroring, and + // typed-event → notification fan-out (no-op while no controller answers). - // Graceful shutdown on all termination signals - installSignalHandlers({ cleanup, flushLogs: () => loggerHandle.flush?.(), log }); + // Graceful shutdown on all termination signals + installSignalHandlers({ cleanup, flushLogs: () => loggerHandle.flush?.(), log }); - log.info({ pid: process.pid }, "daemon ready"); + bootPhase = "ready"; + log.info({ pid: process.pid }, "daemon ready"); + } catch (err) { + log.fatal({ err }, "daemon boot failed"); + // Task 9 adds recordBootFailure(currentPhase, err) here. + try { loggerHandle.flush?.(); } catch { /* */ } + process.exit(1); + } } /** From 8be75f0d093d019105ac649384f1a3c3967cfb5e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 09:58:45 -0500 Subject: [PATCH 021/142] pane:spawn: check the caller's abort signal between steps and stop early instead of racing a retry into a second pane (S087) --- lib/daemon/__tests__/pane-handlers.test.ts | 22 ++++++++++++++++++ lib/daemon/handlers/pane.ts | 27 +++++++++++++++++++--- 2 files changed, 46 insertions(+), 3 deletions(-) diff --git a/lib/daemon/__tests__/pane-handlers.test.ts b/lib/daemon/__tests__/pane-handlers.test.ts index 6f09a1ee..2da4bbbf 100644 --- a/lib/daemon/__tests__/pane-handlers.test.ts +++ b/lib/daemon/__tests__/pane-handlers.test.ts @@ -279,6 +279,28 @@ test("pane:spawn stops polling for registration at the wall-clock budget, not a expect(calls.filter((c) => c === "agent.get").length).toBeLessThanOrEqual(2); }); +// S087: pane:spawn's worst-case daemon budget (register + idle wait + trust +// retry + prompt) exceeds rt-client's client-side timeout — the client +// gives up and reports failure while the daemon keeps working, so a retry +// spawns a second claude pane in the same cwd. Once the caller's own +// AbortSignal fires, the handler must stop spending further budget and +// return the pane it already created instead of continuing the full flow. +test("pane:spawn stops after tab creation once the caller's AbortSignal fires, returning the pane already created", async () => { + const controller = new AbortController(); + const { handler, calls } = spawnFake({ statuses: ["idle"] }); + const abortingHandler: typeof handler = (method, params) => { + if (method === "pane.send_input") controller.abort(); + return handler(method, params); + }; + const { pane } = harness(abortingHandler); + const res = await pane["pane:spawn"]({ cwd: "/repos/chat" }, controller.signal); + if (!res.ok) throw new Error(res.error); + expect(res.data.pane.paneId).toBe("w2:p7"); + expect(res.data.ready).toBe(false); + expect(calls).not.toContain("agent.get"); + expect(calls).not.toContain("agent.wait"); +}); + test("pane:spawn refuses an unknown cswap account before touching herdr", async () => { const { handler, calls } = spawnFake({ statuses: ["idle"] }); const { pane } = harness(handler); diff --git a/lib/daemon/handlers/pane.ts b/lib/daemon/handlers/pane.ts index 3eb2db2f..317ac067 100644 --- a/lib/daemon/handlers/pane.ts +++ b/lib/daemon/handlers/pane.ts @@ -124,7 +124,9 @@ export function createPaneHandlers(opts: { exec?: typeof runCapture; now?: () => number; registry?: (repoName: string) => Array<{ path: string; branch: string | null | undefined }>; -}): Pick & { db: Database } { +}): Pick + & { "pane:spawn": (payload: Commands["pane:spawn"]["payload"], signal?: AbortSignal) => Promise> } + & { db: Database } { const { db, repoIndex } = opts; const herdr = opts.herdr ?? herdrRequest; const exec = opts.exec ?? runCapture; @@ -190,7 +192,7 @@ export function createPaneHandlers(opts: { return { ok: true, data: { directories: out } }; }, - "pane:spawn": async (payload: Commands["pane:spawn"]["payload"]): Promise> => { + "pane:spawn": async (payload: Commands["pane:spawn"]["payload"], signal?: AbortSignal): Promise> => { const { cwd, account, model, effort, prompt } = payload; if (!cwd || !cwd.startsWith("/")) return { ok: false, error: "cwd must be an absolute path" }; if (account) { @@ -213,8 +215,23 @@ export function createPaneHandlers(opts: { if (!tab.ok) return herdrError(tab); const paneId = tab.result.root_pane.pane_id; + // pane:spawn's summed worst-case budget (register + idle wait + a + // blocked-trust retry + the opening prompt) can run longer than + // rt-client's own client-side timeout for this call. Once the caller + // has given up, continuing to spend the daemon's budget only risks a + // retry racing a second claude pane into the same cwd — so every step + // past tab creation checks the signal first and returns the pane + // already created (not-ready) rather than pressing on for a client + // that is no longer listening. + const earlyReturn = async (status: AgentStatus): Promise> => { + const ctx: PaneRowContext = { db, repoIndex, exec, now, workspaces: new Map([[workspaceId!, label]]), ...presenceMaps(db, now()) }; + const pane = await paneRow({ ...tab.result.root_pane, agent: "claude", agent_status: status }, ctx); + return { ok: true, data: { pane, ready: false } }; + }; + const sent = await herdr("pane.send_input", { pane_id: paneId, text: launchCommand({ cwd, account, model, effort }), keys: ["enter"] }); if (!sent.ok) return herdrError(sent); + if (signal?.aborted) return earlyReturn("unknown"); // herdr registers the agent a few hundred ms after the shell starts claude. // Bound the wait by wall-clock, not a fixed attempt count: a slow-but-alive @@ -223,7 +240,7 @@ export function createPaneHandlers(opts: { // holds its caller. let registered = false; const registerDeadline = now() + REGISTER_BUDGET_MS; - while (now() < registerDeadline) { + while (now() < registerDeadline && !signal?.aborted) { const got = await herdr("agent.get", { target: paneId }); if (got.ok) { registered = true; @@ -231,22 +248,26 @@ export function createPaneHandlers(opts: { } await Bun.sleep(REGISTER_POLL_MS); } + if (signal?.aborted) return earlyReturn("unknown"); let status: AgentStatus = "unknown"; let ready = false; if (registered) { const settled = await herdr<{ agent: HerdrAgent }>("agent.wait", { target: paneId, until: SETTLED, timeout_ms: IDLE_BUDGET_MS }, { timeoutMs: waitTimeout(IDLE_BUDGET_MS) }); if (settled.ok) status = settled.result.agent.agent_status; + if (signal?.aborted) return earlyReturn(status); if (status === "blocked") { const screen = await herdr<{ read: { text: string } }>("pane.read", { pane_id: paneId, source: "visible" }); if (screen.ok && /trust/i.test(screen.result.read.text)) { await herdr("pane.send_keys", { pane_id: paneId, keys: ["enter"] }); + if (signal?.aborted) return earlyReturn(status); const again = await herdr<{ agent: HerdrAgent }>("agent.wait", { target: paneId, until: SETTLED, timeout_ms: TRUST_BUDGET_MS }, { timeoutMs: waitTimeout(TRUST_BUDGET_MS) }); if (again.ok) status = again.result.agent.agent_status; } } ready = status === "idle" || status === "done"; } + if (signal?.aborted) return earlyReturn(status); if (ready && prompt) { await herdr("agent.prompt", { target: paneId, text: prompt, wait: { until: ["working"], timeout_ms: PROMPT_BUDGET_MS } }, { timeoutMs: waitTimeout(PROMPT_BUDGET_MS) }); From eea6451095898c8b359c1741344ba18b5dd80074 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:00:07 -0500 Subject: [PATCH 022/142] mr:by-branch: apply the same demand-scope gate to the forge write-back that the sync path enforces (S088) --- lib/daemon/__tests__/project-sync.test.ts | 29 +++++++++++++++++++++++ lib/daemon/handlers/project-mrs.ts | 13 +++++++++- 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/lib/daemon/__tests__/project-sync.test.ts b/lib/daemon/__tests__/project-sync.test.ts index 2437da6f..57612384 100644 --- a/lib/daemon/__tests__/project-sync.test.ts +++ b/lib/daemon/__tests__/project-sync.test.ts @@ -494,6 +494,35 @@ describe("project-mrs:read handler", async () => { expect(forgeCalls).toBe(1); }); + // S088: mr:by-branch's forge write-back upserted unconditionally, bypassing + // the same scope/tagged gate upsertProject (lib/daemon/freshness.ts) + // enforces for every other write path — a demand-scoped repo could pick up + // a stranger's MR that the next delta sync filters right back out, then + // reappears on the next by-branch call. The caller still gets the PR (it + // asked for this exact branch); it just must not land in the store. + test("by-branch: forge write-back respects the same demand scope the sync path enforces", async () => { + const store = tmpStore(); + store.fullSync("remote:repo", "g/p", [], Date.now()); + store.setScope("remote:repo", { authors: ["alice"], windowDays: 30 }); + const h = createProjectMRsHandlers(fakeCtx, () => {}, { store, sync: async () => {}, tracking: grantedTracking, + fetchByBranch: async (_r, branch) => ({ pr: pr(9, { sourceBranch: branch, author: { id: 2, username: "stranger" } as any }), projectPath: "g/p" }) }); + const res = await h["mr:by-branch"]!({ repoName: "remote:repo", branches: ["feat-x"] }); + expect(res.ok).toBe(true); + expect((dataOf(res) as any).byBranch["feat-x"]).toMatchObject({ source: "forge", pr: { iid: 9 } }); + expect(store.read("remote:repo")!.mrs[9]).toBeUndefined(); + }); + + test("by-branch: forge write-back still stores an in-scope author's MR", async () => { + const store = tmpStore(); + store.fullSync("remote:repo", "g/p", [], Date.now()); + store.setScope("remote:repo", { authors: ["alice"], windowDays: 30 }); + const h = createProjectMRsHandlers(fakeCtx, () => {}, { store, sync: async () => {}, tracking: grantedTracking, + fetchByBranch: async (_r, branch) => ({ pr: pr(10, { sourceBranch: branch, author: { id: 1, username: "alice" } as any }), projectPath: "g/p" }) }); + const res = await h["mr:by-branch"]!({ repoName: "remote:repo", branches: ["feat-y"] }); + expect(res.ok).toBe(true); + expect(store.read("remote:repo")!.mrs[10]).toBeDefined(); + }); + test("by-branch: no MR anywhere is null; a per-branch forge failure is null with a warn, not a batch failure", async () => { const store = tmpStore(); store.fullSync("remote:repo", "g/p", [pr(1, { sourceBranch: "ok" })], Date.now()); diff --git a/lib/daemon/handlers/project-mrs.ts b/lib/daemon/handlers/project-mrs.ts index c9441a78..0395ecdc 100644 --- a/lib/daemon/handlers/project-mrs.ts +++ b/lib/daemon/handlers/project-mrs.ts @@ -245,7 +245,18 @@ export function createProjectMRsHandlers( try { const { pr, projectPath } = await fetchByBranch(repoName, branch); if (pr) { - store().upsert(repoName, projectPath, pr, "events"); + // Same scope/tagged gate upsertProject (lib/daemon/freshness.ts) + // enforces for every other write path: a demand-scoped repo + // must not pick up a stranger's MR here just because a client + // happened to ask for their branch by name — the next delta + // sync would filter it right back out, and it would reappear + // on the next by-branch call. The caller still gets the PR + // (it asked for this exact branch); it just isn't stored. + const rec = store().read(repoName); + const scope = rec?.scope; + const tagged = (rec?.mrs[pr.iid]?.codeownerSections?.length ?? 0) > 0; + const inScope = !scope || !pr.author?.username || scope.authors.includes(pr.author.username) || tagged; + if (inScope) store().upsert(repoName, projectPath, pr, "events"); byBranch[branch] = { pr, source: "forge" }; } else { byBranch[branch] = null; From 3bd511ba8d42c3a8af7636b6a193a6a76c1b1149 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:01:50 -0500 Subject: [PATCH 023/142] plan: Phase 1 event-loop implementation plan (RT-78, items 1.1-1.5) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../plans/2026-08-28-p1-event-loop.md | 1490 +++++++++++++++++ 1 file changed, 1490 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-28-p1-event-loop.md diff --git a/docs/superpowers/plans/2026-08-28-p1-event-loop.md b/docs/superpowers/plans/2026-08-28-p1-event-loop.md new file mode 100644 index 00000000..c8d35527 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-p1-event-loop.md @@ -0,0 +1,1490 @@ +# Phase 1 · Event-Loop Sacred Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Excise every synchronous subprocess call from daemon-reachable code, make `runCapture`'s timeout actually enforceable, add an import-graph gate that keeps sync-exec dead, and stop the coalesced refresh and background scans from wedging or taxing the event loop. + +**Architecture:** Route all daemon-thread subprocess work through the existing async helpers (`runCapture` in `lib/subprocess.ts`, `runGit`/`listWorktreesAsync` in `lib/worktree/git-async.ts`). Harden `runCapture` so a pipe-holding grandchild can no longer hold its promise open. Add a source-graph test that walks the daemon's import tree and fails on any sync-exec import. Bound the refresh cycle and the provider cache, and gate/cheapen the periodic scans. + +**Tech Stack:** Bun 1.3.13, TypeScript (strict), `bun:test`, `bun:sqlite`, pino. `@mattstack/glance` provides `GitLabProvider`. + +**Spec:** `/Users/matt/Documents/GitHub/repo-tools/.claude/worktrees/daemon-stability-audit/docs/daemon-stability-audit-2026-08.md` ... "Roadmap > Phase 1" (items 1.1-1.5) plus Appendix A/B for each finding (S007, S008, S015, S016, S021, S023, S024, S038, S039, S045, S048, S049, S055, S058, S061, S093, S098, S101, S104, R032). + +## Global Constraints + +- **Isolated HOME for any binary run.** Never start a daemon or run `dist/rt` against the real machine. Any daemon or `dist/rt` invocation runs under `env -i HOME=` only. Tests use `bun:test` with tmp HOME/`RT_RUNS_ROOT` fixtures ... never touch the developer's `~/.mattstack` or `~/.rt`. +- **Write fence.** These sibling-owned files must NOT be modified this run: `cli.ts`, `lib/daemon.ts`, `lib/daemon/park.ts`, `lib/daemon/boot-reconcile.ts`, `lib/daemon/boot-migrate.ts`, `lib/daemon/shutdown.ts`, `lib/daemon-logger.ts`, `lib/daemon-config.ts`, `lib/daemon-status.ts`, `commands/daemon.ts`, `lib/daemon/events-bus.ts`, `lib/daemon/home-snapshot.ts`, `lib/daemon/handlers/status.ts`, `lib/state/*`, `rt-tray/**`, `lib/daemon/api-server.ts`, `lib/daemon/api-auth.ts`, `lib/daemon/socket-server.ts`, `lib/daemon/handlers/secrets.ts`, `lib/notifier.ts`, `lib/daemon/handlers/discussions.ts`, `lib/daemon/handlers/chat.ts`, `lib/daemon/handlers/agent.ts`, `lib/daemon/handlers/pane.ts`, `lib/daemon/handlers/project-mrs.ts`, `lib/daemon/handlers/worktree.ts`, `lib/herdr/client.ts`, `lib/port-scanner.ts`, `lib/deps/links.ts`, `lib/worktree/trash.ts`, `lib/agent-herdr.ts`, `lib/daemon/cron.ts`, `lib/daemon/hooks-guard.ts`, `lib/home/age-key.ts`, `lib/daemon/discussions-store.ts`. +- **S055 carve-out.** `lib/daemon/handlers/status.ts` is sibling-owned. Do NOT edit it. The async replacement it needs (`listWorktreesAsync`) already lives in a file we own (`lib/worktree/git-async.ts`). The one-line call-site swap is documented in the report Notes, not applied here. The import-graph gate (Task 7) allowlists `status.ts` with a comment naming S055. +- **rt-client mirror.** `packages/rt-client/src/settings/exec.ts` is a byte-for-byte mirror of `lib/subprocess.ts`'s `runCapture`. Change `lib/subprocess.ts` first, mirror there, then run `bun run build` inside `packages/rt-client` (its `dist/` is gitignored and copied verbatim by `file:` consumers; `packages/rt-client/test/dist-freshness.test.ts` fails otherwise). +- **Canonical fixtures.** `listWorktreesAsync` returns git's canonicalized paths (`/private/var/...` on macOS tmpdirs). Tests that build temp repos must compare against `realpathSync`'d paths. +- **Subagent models.** Every subagent dispatched during execution carries an explicit `model` (`sonnet` for mechanical tasks, `haiku` for lookups). +- **Verification (all must pass before done):** `bun test lib commands packages scripts` green; `bunx tsc --noEmit` zero errors; the Task 7 gate passes AND is proven to fail when a forbidden import is reintroduced (show the RED run in the report). +- **Comments:** clean-code rules. A comment states a constraint the code cannot show (a parity anchor, an ordering trap, a non-obvious invariant). No narration, no finding IDs in source. No em dashes. + +--- + +### Task 1: Make `runCapture`'s timeout enforceable (S023, S024) + +`runCapture` awaits `new Response(proc.stdout).text()` unconditionally after the kill timer fires; a grandchild that inherited the pipe keeps that read pending forever, so the promise never settles and every in-flight guard latches. Fix: race the reads against the deadline so `runCapture` always settles within `timeoutMs`, and escalate SIGTERM→SIGKILL on the child. (Group-kill via `process.kill(-pid)` is NOT reliable on Bun 1.3.13 ... verified ESRCH once the direct child exits ... so this uses the race as the load-critical fix, per the S023 fixer note that racing the reads is mandatory and kill escalation is belt-and-suspenders. A surviving grandchild leaks its fd until the OS reaps it, which the audit accepts.) + +**Files:** +- Modify: `lib/subprocess.ts` (the `runCapture` body and `RunResult`) +- Mirror: `packages/rt-client/src/settings/exec.ts` (byte-for-byte) +- Test: `lib/__tests__/subprocess.test.ts` (create if absent) + +**Interfaces:** +- Produces: `runCapture(argv, opts) => Promise` where `RunResult` gains `timedOut?: boolean`. Contract unchanged otherwise: never throws; on spawn failure/timeout/read error returns `exitCode: -1` (callers already branch on `!== 0`). `timedOut: true` is additive and set only on the deadline path. + +- [ ] **Step 1: Write the failing test** + +Create `lib/__tests__/subprocess.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { runCapture } from "../subprocess.ts"; + +test("resolves within the deadline even when a grandchild holds the pipe", async () => { + // zsh exits after ~0.2s, but backgrounds `sleep 20` which inherits stdout. + const t0 = Date.now(); + const r = await runCapture( + ["/bin/zsh", "-c", "sleep 20 & echo started; sleep 0.2"], + { timeoutMs: 1000 }, + ); + const elapsed = Date.now() - t0; + expect(elapsed).toBeLessThan(4000); // must NOT wait for the 20s grandchild + expect(r.timedOut).toBe(true); + expect(r.exitCode).toBe(-1); +}); + +test("a SIGTERM-ignoring child is bounded by SIGKILL escalation", async () => { + const t0 = Date.now(); + const r = await runCapture( + ["/bin/zsh", "-c", "trap '' TERM; sleep 20"], + { timeoutMs: 800 }, + ); + expect(Date.now() - t0).toBeLessThan(4000); + expect(r.timedOut).toBe(true); +}); + +test("normal fast command still returns real stdout and exitCode 0", async () => { + const r = await runCapture(["/bin/echo", "hello"], { timeoutMs: 5000 }); + expect(r.stdout.trim()).toBe("hello"); + expect(r.exitCode).toBe(0); + expect(r.timedOut).toBeUndefined(); +}); + +test("timed-out call reports exitCode -1 so callers treat it as failure", async () => { + const r = await runCapture(["/bin/sleep", "20"], { timeoutMs: 500 }); + expect(r.exitCode).toBe(-1); + expect(r.timedOut).toBe(true); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/__tests__/subprocess.test.ts` +Expected: the grandchild test FAILS (elapsed ~20s, exceeds 4000ms) or times out ... proving the current unconditional-await bug. + +- [ ] **Step 3: Rewrite `runCapture` to race the reads against the deadline** + +In `lib/subprocess.ts`, add `timedOut` to `RunResult` and replace the timer + read block: + +```ts +export interface RunResult { + stdout: string; + stderr: string; + exitCode: number; + /** Set true only when the deadline fired before the child settled. */ + timedOut?: boolean; +} +``` + +Replace the body from the `const timer = setTimeout(...)` line through the closing of the `try/catch/finally` (lines 60-76) with: + +```ts + const timeoutMs = opts.timeoutMs ?? 10_000; + // SIGTERM at the deadline, SIGKILL a short grace later. A child that ignores + // SIGTERM (or a D-state descendant) cannot be reaped in-band, so the read is + // raced against the deadline below rather than awaited unconditionally: that + // is what lets runCapture settle while a grandchild still holds the pipe. + let killTimer: ReturnType | undefined; + const term = setTimeout(() => { + try { proc.kill("SIGTERM"); } catch { /* already exited */ } + killTimer = setTimeout(() => { + try { proc.kill("SIGKILL"); } catch { /* already exited */ } + }, 2000); + }, timeoutMs); + + const captured: Promise = (async () => { + try { + const stdoutPromise = new Response(proc.stdout as ReadableStream).text(); + const stderrPromise = captureStderr + ? new Response(proc.stderr as ReadableStream).text() + : Promise.resolve(""); + const [stdout, stderr, exitCode] = await Promise.all([ + stdoutPromise, + stderrPromise, + proc.exited, + ]); + return { stdout, stderr, exitCode }; + } catch { + return { stdout: "", stderr: "", exitCode: -1 }; + } + })(); + + const deadline: Promise = new Promise((resolve) => { + setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); + }); + + try { + return await Promise.race([captured, deadline]); + } finally { + clearTimeout(term); + if (killTimer) clearTimeout(killTimer); + } +``` + +(Keep the `Bun.spawn` block above it unchanged, including the env comment.) + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/__tests__/subprocess.test.ts` +Expected: PASS (all four). + +- [ ] **Step 5: Mirror into rt-client and rebuild** + +Apply the identical `RunResult`/body change to `packages/rt-client/src/settings/exec.ts` (its `runCapture` is the same shape; keep its own env comment). Then: + +Run: `bun run build` (from `packages/rt-client/`) +Then: `bun test packages/rt-client` +Expected: `dist-freshness.test.ts` PASS. + +- [ ] **Step 6: Commit** + +```bash +git add lib/subprocess.ts packages/rt-client/src/settings/exec.ts packages/rt-client/dist lib/__tests__/subprocess.test.ts +git commit -m "runCapture: race reads against the deadline so a pipe-holding grandchild can't wedge it (S023, S024)" +``` + +--- + +### Task 2: Long timeouts for mutating git verbs (S104) + +Mutating git verbs (checkout, merge, stash push/pop) and `status` share the 60s `DEFAULT_TIMEOUT_MS`; a large-repo checkout gets SIGKILLed half-applied at 60s. Give them the same 5-minute budget that fetch/`worktree add` already have. + +**Files:** +- Modify: `lib/worktree/git-async.ts` (add constant; thread `timeoutMs` into stash helpers and `statusPorcelainAsync`) +- Modify: `lib/daemon/worktree-reconciler.ts` (checkout/merge call sites in `freshenOne` and `autoReturnMain`) +- Test: `lib/__tests__/git-async-timeouts.test.ts` (create) + +**Interfaces:** +- Produces: `export const MUTATING_TIMEOUT_MS = 5 * 60_000;` in `git-async.ts`. `stashChangesAsync(cwd, label, opts?: { timeoutMs?: number })` and `popStashAsync(cwd, stashName, opts?: { timeoutMs?: number })` now accept an optional timeout, defaulting to `MUTATING_TIMEOUT_MS`. + +- [ ] **Step 1: Write the failing test** + +Create `lib/__tests__/git-async-timeouts.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { MUTATING_TIMEOUT_MS } from "../worktree/git-async.ts"; + +test("mutating timeout is 5 minutes", () => { + expect(MUTATING_TIMEOUT_MS).toBe(5 * 60_000); +}); +``` + +Also assert the stash helper signature accepts an override (compile-time guard): + +```ts +import { stashChangesAsync, popStashAsync } from "../worktree/git-async.ts"; +test("stash helpers accept a timeout override", () => { + // Type-level: these must type-check with an opts arg. + const a: typeof stashChangesAsync = stashChangesAsync; + const b: typeof popStashAsync = popStashAsync; + expect(typeof a).toBe("function"); + expect(typeof b).toBe("function"); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/__tests__/git-async-timeouts.test.ts` +Expected: FAIL ... `MUTATING_TIMEOUT_MS` not exported. + +- [ ] **Step 3: Add the constant and thread it** + +In `lib/worktree/git-async.ts`, below `const DEFAULT_TIMEOUT_MS = 60_000;`: + +```ts +/** Checkout/merge/stash on a large tree can legitimately exceed a minute; a + * 60s SIGKILL leaves the working tree half-switched. Match fetch/worktree-add. */ +export const MUTATING_TIMEOUT_MS = 5 * 60_000; +``` + +Update the helpers: + +```ts +export async function statusPorcelainAsync(cwd: string): Promise { + const r = await runGit(cwd, ["status", "--porcelain"], { timeoutMs: MUTATING_TIMEOUT_MS }); + return r.stdout; +} + +export async function stashChangesAsync( + cwd: string, + label: string, + opts: { timeoutMs?: number } = {}, +): Promise { + const message = `!!GitHub_Desktop<${label}>`; + await runGit(cwd, ["stash", "push", "-u", "-m", message], { + timeoutMs: opts.timeoutMs ?? MUTATING_TIMEOUT_MS, + }); +} + +export async function popStashAsync( + cwd: string, + stashName: string, + opts: { timeoutMs?: number } = {}, +): Promise { + await runGit(cwd, ["stash", "pop", stashName], { + timeoutMs: opts.timeoutMs ?? MUTATING_TIMEOUT_MS, + }); +} +``` + +- [ ] **Step 4: Bump the reconciler checkout/merge call sites** + +In `lib/daemon/worktree-reconciler.ts`, add `MUTATING_TIMEOUT_MS` to the existing `../worktree/git-async.ts` import (the file already imports `runGit` from there). Add `{ timeoutMs: MUTATING_TIMEOUT_MS }` to these `runGit` calls: +- `freshenOne`: `["checkout", "--", ...classify.discard]` (~line 727); `["merge", "--ff-only", defaultRef]` (~line 756). +- `autoReturnMain`: `["checkout", defaultBranch]` (~line 469); `["merge", "--ff-only", defaultRef]` (~line 475). + +Example: + +```ts +const merge = await runGit(rec.path, ["merge", "--ff-only", defaultRef], { timeoutMs: MUTATING_TIMEOUT_MS }); +``` + +The stash push/pop calls in these functions inherit the new default automatically. (Leave `worktree prune` and `branch -D` at the 60s default; they are not in S104's named set and are fast.) + +- [ ] **Step 5: Run tests** + +Run: `bun test lib/__tests__/git-async-timeouts.test.ts && bun test lib/daemon/__tests__/worktree-reconciler` +Expected: PASS. If a reconciler test asserts an exact `runGit` argv without opts, update it to allow the opts arg. + +- [ ] **Step 6: Commit** + +```bash +git add lib/worktree/git-async.ts lib/daemon/worktree-reconciler.ts lib/__tests__/git-async-timeouts.test.ts +git commit -m "git-async: 5-min timeout for checkout/merge/stash/status so a large-repo checkout isn't killed half-applied (S104)" +``` + +--- + +### Task 3: Excise sync exec from cache-refresh and git-worktrees (S008, S045, S021) + +The 5-minute refresh runs `execSync` (`for-each-ref`, `git config`) and the sync `listWorktrees`/`listWorktreeRoots` per repo on the daemon thread. Swap to the async helpers. Add the missing `listWorktreeRootsAsync` twin. Gate the doppler loop on grants (honors the "off = zero background work" contract). + +**Files:** +- Modify: `lib/worktree/git-async.ts` (add `listWorktreeRootsAsync`) +- Modify: `lib/daemon/cache-refresh.ts` (three swaps + doppler grant gate) +- Modify: `lib/daemon/__tests__/cache-refresh-gc.test.ts` (retarget the `listWorktrees`/`listWorktreeRoots` spies) +- Test: `lib/__tests__/git-async.test.ts` or add to existing git-async coverage for `listWorktreeRootsAsync` + +**Interfaces:** +- Consumes: `listWorktreesAsync(path) => Promise`, `runGit(cwd, args) => Promise` (Task 1/2 hardened). +- Produces: `listWorktreeRootsAsync(repoPath) => Promise` (returns `[]` on git failure, matching the sync contract). + +- [ ] **Step 1: Write the failing test for `listWorktreeRootsAsync`** + +Add to a git-async test file (create `lib/__tests__/git-worktree-roots-async.test.ts`): + +```ts +import { test, expect } from "bun:test"; +import { mkdtempSync } from "fs"; +import { realpathSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { listWorktreeRootsAsync } from "../worktree/git-async.ts"; +import { runGit } from "../worktree/git-async.ts"; + +test("listWorktreeRootsAsync returns the main worktree path", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-wt-"))); + await runGit(dir, ["init", "-q"]); + await runGit(dir, ["commit", "--allow-empty", "-m", "init", "-c", "user.email=a@b.c", "-c", "user.name=t"]); + const roots = await listWorktreeRootsAsync(dir); + expect(roots).toContain(dir); +}); + +test("listWorktreeRootsAsync returns [] on a non-repo", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-nonrepo-"))); + expect(await listWorktreeRootsAsync(dir)).toEqual([]); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/__tests__/git-worktree-roots-async.test.ts` +Expected: FAIL ... `listWorktreeRootsAsync` not exported. + +- [ ] **Step 3: Add `listWorktreeRootsAsync`** + +In `lib/worktree/git-async.ts`, after `listWorktreesAsync`: + +```ts +/** Worktree root paths (main + linked), existing-on-disk only. `[]` on git + * failure ... the async twin of git-worktrees.ts listWorktreeRoots. */ +export async function listWorktreeRootsAsync(repoPath: string): Promise { + return (await listWorktreesAsync(repoPath) ?? []).map((w) => w.path); +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/__tests__/git-worktree-roots-async.test.ts` +Expected: PASS. + +- [ ] **Step 5: Swap the cache-refresh call sites** + +In `lib/daemon/cache-refresh.ts`: + +Imports ... remove `import { execSync } from "child_process";` and `import { listWorktreeRoots, listWorktrees } from "../git-worktrees.ts";`; add `import { listWorktreesAsync, listWorktreeRootsAsync, runGit } from "../worktree/git-async.ts";`. + +Branch listing (line ~112): + +```ts +const branches: Array<{ path: string; branch: string }> = ((await listWorktreesAsync(repoPath)) ?? []) + .filter((w): w is { path: string; branch: string } => !!w.branch && !w.branch.startsWith("on-deck/")); +``` + +`for-each-ref` (lines ~118-133): + +```ts +const worktreeBranchSet = new Set(branches.map((b) => b.branch)); +const localBranches = await runGit(repoPath, ["for-each-ref", "--format=%(refname:short)", "refs/heads/"]); +if (localBranches.exitCode === 0) { + for (const name of localBranches.stdout.split("\n")) { + const trimmed = name.trim(); + if (!trimmed || worktreeBranchSet.has(trimmed) || trimmed.startsWith("on-deck/")) continue; + if (extractLinearId(trimmed)) branches.push({ path: repoPath, branch: trimmed }); + } +} else { + log.warn({ repo: repoPath }, "local branch listing failed"); +} +``` + +(argv form removes the shell, so the old `replace(/^'|'$/g, "")` quote-stripping is no longer needed ... the `--format=%(refname:short)` has no surrounding quotes without a shell.) + +Remote URL (lines ~138-142): + +```ts +let remoteUrl: string | undefined; +const remote = await runGit(repoPath, ["config", "--get", "remote.origin.url"]); +if (remote.exitCode === 0) remoteUrl = remote.stdout.trim() || undefined; +``` + +Doppler loop (line ~209): replace `listWorktreeRoots(repoPath)` with `await listWorktreeRootsAsync(repoPath)`, and gate the loop body on grants. Load tracking once above the loop and skip repos with no cache grant: + +```ts +const tracking = loadRepoTracking(); +for (const [repoName, repoPath] of Object.entries(repoIndex())) { + if (!existsSync(repoPath)) continue; + if (grants(tracking, repoName).caches.size === 0) continue; // off = zero background work + try { + const worktreeRoots = await listWorktreeRootsAsync(repoPath); + // ...unchanged body... +``` + +(`loadRepoTracking` and `grants` are already imported in this file.) + +- [ ] **Step 6: Retarget the GC test spies** + +In `lib/daemon/__tests__/cache-refresh-gc.test.ts` (~lines 105, 108), the test spies on `gitWorktreesModule.listWorktrees`/`listWorktreeRoots`. Retarget to the async twins: + +```ts +import * as gitAsync from "../../worktree/git-async.ts"; +// ... +spyOn(gitAsync, "listWorktreesAsync").mockResolvedValue([]); +spyOn(gitAsync, "listWorktreeRootsAsync").mockResolvedValue([]); +``` + +(Remove the now-unused `gitWorktreesModule` import if nothing else uses it.) + +- [ ] **Step 7: Run the cache-refresh tests** + +Run: `bun test lib/daemon/__tests__/cache-refresh-gc.test.ts` +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add lib/worktree/git-async.ts lib/daemon/cache-refresh.ts lib/daemon/__tests__/cache-refresh-gc.test.ts lib/__tests__/git-worktree-roots-async.test.ts +git commit -m "cache-refresh: async git + grant-gated doppler loop; add listWorktreeRootsAsync (S008, S045, S021)" +``` + +--- + +### Task 4: Async, cached `getRemoteUrl` in freshness (R032) + +`getRemoteUrl` uses `execSync` on the daemon thread inside every forge handler (mr:action, discussions:*, project sync) and the freshness reconcile loop. Make it async via `runCapture` with a 5s timeout, cached per repoPath for the process lifetime. + +**Files:** +- Modify: `lib/daemon/freshness.ts` +- Test: `lib/daemon/__tests__/freshness-remote-url.test.ts` (create) + +**Interfaces:** +- Produces: `getRemoteUrl(repoPath) => Promise` (was sync). All four call sites become `await getRemoteUrl(...)`. + +- [ ] **Step 1: Write the failing test** + +Create `lib/daemon/__tests__/freshness-remote-url.test.ts` ... since `getRemoteUrl` is module-private, test through a real repo via a tiny exported probe is not available; instead assert no `execSync`/`child_process` import remains in freshness.ts (the observable contract for R032) plus a behavioral cache test using a real temp repo through the public `getRepoContext` is heavy. Keep it to the source guard, which the Task 7 gate will also enforce, plus a targeted unit if a seam exists: + +```ts +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("freshness.ts no longer imports execSync/child_process", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + expect(src).not.toMatch(/from\s+["']child_process["']/); + expect(src).not.toMatch(/\bexecSync\b/); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/freshness-remote-url.test.ts` +Expected: FAIL ... `child_process`/`execSync` still present. + +- [ ] **Step 3: Convert `getRemoteUrl` and add the cache** + +In `lib/daemon/freshness.ts`: remove `import { execSync } from "child_process";`; add `import { runCapture } from "../subprocess.ts";`. Add a module-level cache near the other module state (below the `providers`/`userId` block ~line 91): + +```ts +const remoteUrlCache = new Map(); +``` + +Replace `getRemoteUrl` (lines 95-103): + +```ts +/** remote.origin.url, cached per repoPath for the process lifetime (remotes + * rarely change). Async so it never blocks the event loop. */ +async function getRemoteUrl(repoPath: string): Promise { + const cached = remoteUrlCache.get(repoPath); + if (cached !== undefined) return cached; + const r = await runCapture(["git", "config", "--get", "remote.origin.url"], { + cwd: repoPath, + timeoutMs: 5000, + stderr: "ignore", + }); + const url = r.exitCode === 0 ? (r.stdout.trim() || null) : null; + remoteUrlCache.set(repoPath, url); + return url; +} +``` + +Add `await` at the four call sites: `ensureProvider` (~135), `getRepoContext` (~273 and ~292), `reconcileFreshnessImpl` (~692). All three enclosing functions are already `async`. + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/freshness-remote-url.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc zero errors (the `await` additions type-check because callers are async). + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/freshness.ts lib/daemon/__tests__/freshness-remote-url.test.ts +git commit -m "freshness: async, cached getRemoteUrl via runCapture (R032)" +``` + +--- + +### Task 5: Async `killWorktreeProcesses` (S015, S016) + +`killWorktreeProcesses` runs three `execSync` calls (system-wide `lsof`, two `ps`) on the daemon thread per disposal and on the unattended reactor path; a stuck `lsof` wedges the loop. Port to `runCapture` (the identical async `lsof` already exists in `system-process-scanner.ts`). + +**Files:** +- Modify: `lib/daemon/worktree-process-kill.ts` (make `killWorktreeProcesses` async; three `runCapture` swaps) +- Modify: `lib/worktree/dispose.ts:238` (add `await`) +- Modify: `lib/daemon/worktree-reconciler.ts:447` (add `await`; update the "sync by design" comment) +- Test: `lib/daemon/__tests__/worktree-process-kill.test.ts` (add a source guard + keep `selectKillTargets` coverage) + +**Interfaces:** +- Produces: `killWorktreeProcesses(worktreePath) => Promise` (was sync). `WorktreeKillResult` unchanged. + +- [ ] **Step 1: Write the failing test** + +Add to `lib/daemon/__tests__/worktree-process-kill.test.ts`: + +```ts +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("worktree-process-kill.ts imports no sync exec", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "worktree-process-kill.ts"), "utf8"); + expect(src).not.toMatch(/from\s+["']child_process["']/); + expect(src).not.toMatch(/\bexecSync\b/); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/worktree-process-kill.test.ts` +Expected: FAIL ... `execSync` still imported. + +- [ ] **Step 3: Port the three calls to `runCapture`** + +In `lib/daemon/worktree-process-kill.ts`: remove `import { execSync } from "child_process";`; add `import { runCapture } from "../subprocess.ts";`. Make the function async and swap each call (argv form, no shell, no `2>/dev/null`): + +```ts +export async function killWorktreeProcesses(worktreePath: string): Promise { + const lsof = await runCapture(["lsof", "-d", "cwd", "-Fpn"], { timeoutMs: 10_000 }); + if (lsof.exitCode !== 0 && !lsof.stdout) { + log.warn({ exitCode: lsof.exitCode, worktreePath }, "lsof failed; skipping worktree process kill"); + return { terminated: [] }; + } + const lsofOut = lsof.stdout; + // ...existing parse of lsofOut into candidate pids (unchanged)... +``` + +Then the `ps -p -o pid=,ppid=,comm=,args=` call: + +```ts + const ps = await runCapture(["ps", "-p", pidList, "-o", "pid=,ppid=,comm=,args="], { timeoutMs: 5000 }); + if (ps.exitCode !== 0 && !ps.stdout) { + log.warn({ exitCode: ps.exitCode, worktreePath }, "ps failed; skipping worktree process kill"); + return { terminated: [] }; + } + for (const line of ps.stdout.split("\n")) { /* unchanged parse */ } +``` + +Then the `ps eww` label call: + +```ts + const eww = await runCapture(["ps", "eww", "-o", "pid=,command=", "-p", targets.map((t) => t.pid).join(",")], { timeoutMs: 5000 }); + if (eww.exitCode === 0 || eww.stdout) scripts = parsePackageScripts(eww.stdout); +``` + +(`pidList` is the same comma/space-joined pid string the old `ps` used; pass it as one argv element ... `ps -p` accepts a comma-separated list.) + +- [ ] **Step 4: Await at the two call sites** + +`lib/worktree/dispose.ts:238`: `const { terminated } = await killWorktreeProcesses(rec.path);` +`lib/daemon/worktree-reconciler.ts:447`: `const { terminated } = await killWorktreeProcesses(rec.path);` ... and replace the "ruled execSync exception (the process killer is sync by design)" comment with one that no longer claims sync (e.g. drop it; the try/catch keeps its "a failure here never blocks the return" rationale). + +- [ ] **Step 5: Run tests** + +Run: `bun test lib/daemon/__tests__/worktree-process-kill.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean (both call sites already sit in `async` functions). + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/worktree-process-kill.ts lib/worktree/dispose.ts lib/daemon/worktree-reconciler.ts lib/daemon/__tests__/worktree-process-kill.test.ts +git commit -m "worktree-process-kill: async lsof/ps via runCapture (S015, S016)" +``` + +--- + +### Task 6: Async index write in `resolveIndexPathForIdentity` (S098) + +`endpoint:claim`/`endpoint:lookup` can reach `resolveIndexPathForIdentity`, which on a legacy-key match calls `updateRepoIndex` → `observedMainPath` → a synchronous `git worktree list` on the daemon thread. Resolve the observed main path asynchronously. + +**Files:** +- Modify: `lib/repo-index.ts` (add `observedMainPathAsync`; use it on the async path) +- Test: `lib/__tests__/repo-index-async.test.ts` (create) + +**Interfaces:** +- Consumes: `listWorktreesAsync(path) => Promise`, `setIndexPath(key, mainPath)` (existing sync KV write, no subprocess). +- Produces: `observedMainPathAsync(repoRoot) => Promise` (degrades to `repoRoot`). + +- [ ] **Step 1: Write the failing test** + +Create `lib/__tests__/repo-index-async.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("resolveIndexPathForIdentity no longer reaches a sync git via observedMainPath", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "repo-index.ts"), "utf8"); + // observedMainPath (sync execSync) must not be called from the async resolver path. + expect(src).toMatch(/observedMainPathAsync/); +}); +``` + +(The Task 7 import-graph gate is the real enforcement that no sync git reaches the daemon graph; this test just anchors the async twin's existence.) + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/__tests__/repo-index-async.test.ts` +Expected: FAIL ... `observedMainPathAsync` not present. + +- [ ] **Step 3: Add `observedMainPathAsync` and use it** + +In `lib/repo-index.ts`, add `import { listWorktreesAsync } from "./worktree/git-async.ts";` (top imports). Add next to `observedMainPath`: + +```ts +/** Async twin of observedMainPath: the repo's MAIN worktree path as git + * reports it, degrading to repoRoot. Safe on the daemon thread. */ +async function observedMainPathAsync(repoRoot: string): Promise { + const wts = await listWorktreesAsync(repoRoot); + return wts?.[0]?.path ?? repoRoot; +} +``` + +In `resolveIndexPathForIdentity` (line ~277), replace `updateRepoIndex(serialized, path);` with: + +```ts +setIndexPath(serialized, await observedMainPathAsync(path)); +``` + +(`setIndexPath` is the raw KV write already exported in this file; it does the same persistence `updateRepoIndex` does, minus the sync git probe, which `observedMainPathAsync` now supplies.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/__tests__/repo-index-async.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean (`resolveIndexPathForIdentity` is already async). + +- [ ] **Step 5: Commit** + +```bash +git add lib/repo-index.ts lib/__tests__/repo-index-async.test.ts +git commit -m "repo-index: async observed-main-path on the endpoint:claim resolve path (S098)" +``` + +--- + +### Task 7: The import-graph gate (1.3) + +A test that walks the daemon's import graph from `lib/daemon.ts` and fails if any daemon-reachable module has a sync-exec call site (`execSync(`, `spawnSync(`, `Bun.spawnSync(`, `Bun.sleepSync(`). The rule has been re-broken twice; only a gate keeps it dead. Ships GREEN with an honest allowlist of every current out-of-Phase-1 offender, each commented with the finding/phase that removes it (decision: full-closure walk with honest allowlist). + +**Verified facts (from a throwaway walker run against the live tree):** +- Closure from `lib/daemon.ts` = 151 files. `lib/daemon.ts` and other entry files carry a `#!/usr/bin/env bun` shebang that makes `Bun.Transpiler.scanImports` throw ... strip a leading shebang before scanning. +- `.tsx` files (e.g. `lib/rt-render.tsx`, in the closure) need the `tsx` loader, not `ts`. +- Relative imports are spelled with explicit `.ts`/`.tsx` extensions throughout, so `resolve(dirname(file), importPath)` needs no extension guessing. +- Matching call sites (`\bexecSync\s*\(`) rather than bare `execSync` avoids false positives: `api-server.ts:105` mentions `Bun.sleepSync` in a comment with no paren, correctly NOT flagged. Raw-source matching yields the same 13 offenders as comment-stripped matching, so the test uses raw matching. +- After Tasks 3/4/5 remove cache-refresh.ts, freshness.ts, and worktree-process-kill.ts from the offender set, exactly these 10 remain (the allowlist): + +| Allowlist entry | Removing finding / phase | +|---|---| +| `lib/daemon/user-path.ts` | Phase 6 PATH rebuild (S013/S014/S062) | +| `lib/daemon/boot-reconcile.ts` | Phase 0.6 / S044 (`Bun.sleepSync`) | +| `lib/state/db.ts` | Phase 0.7 / S072-S073 busy-retry (`Bun.sleepSync`) | +| `lib/state/busy.ts` | Phase 0.7 / S072-S073 busy-retry (`Bun.sleepSync`) | +| `lib/git-worktrees.ts` | S055 (reached only via `handlers/status.ts`; that swap to `listWorktreesAsync` removes the edge) | +| `lib/daemon/handlers/status.ts` | S055 (the edge that pulls in git-worktrees.ts; own source has no sync-exec, listed per the brief) | +| `lib/repo-index.ts` | Phase 5.3 dedup (retains `execSync` at heal/derive paths; Task 6's endpoint path is covered by `repo-index-async.test.ts`) | +| `lib/repo.ts` | R050 / Phase 5.4 (reached via `handlers/system-processes.ts` → `repo-arg.ts`) | +| `lib/git.ts` | R050 / Phase 5.4 (via `repo.ts`) | +| `lib/herdr-launch.ts` | Phase 5 herdr (reached via `handlers/pane.ts`) | +| `lib/rt-render.tsx` | R050 / Phase 5.4 (daemon carries the TUI; `no-eager-tui` extension breaks the chain) | + +**Files:** +- Create: `lib/__tests__/no-daemon-sync-exec.test.ts` + +**Interfaces:** +- Self-contained test; consumes nothing from other tasks. Must run AFTER Tasks 3/4/5 land (else the three fixed files fail it). + +- [ ] **Step 1: Write the gate test (RED against current tree, before Tasks 3/4/5, or GREEN after)** + +Create `lib/__tests__/no-daemon-sync-exec.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { dirname, resolve } from "path"; + +// Files with sync-exec that Phase 1 does NOT remove. Each entry names the +// finding/phase that will delete it, so this list shrinks as later phases land. +// A regression that reintroduces sync-exec into any OTHER daemon-reachable +// module fails this gate (the rule has been re-broken twice). +const ALLOWLIST = new Set([ + "lib/daemon/user-path.ts", // Phase 6 PATH rebuild (S013/S014/S062) + "lib/daemon/boot-reconcile.ts", // Phase 0.6 / S044 (Bun.sleepSync) + "lib/state/db.ts", // Phase 0.7 / S072-S073 busy-retry + "lib/state/busy.ts", // Phase 0.7 / S072-S073 busy-retry + "lib/git-worktrees.ts", // S055: reached only via handlers/status.ts + "lib/daemon/handlers/status.ts", // S055: the edge into git-worktrees.ts + "lib/repo-index.ts", // Phase 5.3 dedup (heal/derive execSync) + "lib/repo.ts", // R050 / Phase 5.4 (via handlers/system-processes.ts) + "lib/git.ts", // R050 / Phase 5.4 (via repo.ts) + "lib/herdr-launch.ts", // Phase 5 herdr (via handlers/pane.ts) + "lib/rt-render.tsx", // R050 / Phase 5.4 (daemon carries the TUI) +]); + +const SYNC_EXEC = [ + /\bexecSync\s*\(/, + /\bspawnSync\s*\(/, + /\bBun\.spawnSync\s*\(/, + /\bBun\.sleepSync\s*\(/, +]; + +const REPO_ROOT = resolve(import.meta.dir, "..", ".."); +const stripShebang = (s: string) => s.replace(/^#!.*\n/, ""); +const tsT = new Bun.Transpiler({ loader: "ts" }); +const tsxT = new Bun.Transpiler({ loader: "tsx" }); +const loaderFor = (f: string) => (f.endsWith(".tsx") || f.endsWith(".jsx") ? tsxT : tsT); + +/** Files reachable from lib/daemon.ts via relative imports (the daemon graph). */ +function daemonClosure(): string[] { + const entry = resolve(REPO_ROOT, "lib/daemon.ts"); + const visited = new Set(); + const stack = [entry]; + while (stack.length) { + const file = stack.pop()!; + if (visited.has(file)) continue; + visited.add(file); + let src: string; + try { src = stripShebang(readFileSync(file, "utf8")); } catch { continue; } + let imports: { path: string }[]; + try { imports = loaderFor(file).scanImports(src); } catch { continue; } + for (const imp of imports) { + if (!imp.path.startsWith(".")) continue; // external package + stack.push(resolve(dirname(file), imp.path)); + } + } + return [...visited]; +} + +function hasSyncExec(source: string): boolean { + return SYNC_EXEC.some((re) => re.test(source)); +} + +test("no daemon-reachable module calls sync exec (outside the allowlist)", () => { + const offenders: string[] = []; + for (const file of daemonClosure()) { + const rel = file.replace(REPO_ROOT + "/", ""); + if (ALLOWLIST.has(rel)) continue; + let src: string; + try { src = readFileSync(file, "utf8"); } catch { continue; } + if (hasSyncExec(src)) offenders.push(rel); + } + expect(offenders).toEqual([]); +}); + +test("the checker flags a reintroduced sync-exec call (proves the gate bites)", () => { + // Permanent RED proof: the matcher must catch a fresh offense. + expect(hasSyncExec(`import { execSync } from "child_process";\nexecSync("true");`)).toBe(true); + expect(hasSyncExec(`await Bun.sleepSync(10);`)).toBe(true); + expect(hasSyncExec(`// a comment mentioning execSync without a call`)).toBe(false); +}); + +test("the daemon closure actually resolves (guards against a walker that finds nothing)", () => { + const closure = daemonClosure(); + expect(closure.length).toBeGreaterThan(50); // ~151 today; a collapse means the walk broke +}); +``` + +- [ ] **Step 2: Run the gate** + +Run: `bun test lib/__tests__/no-daemon-sync-exec.test.ts` +Expected: GREEN (Tasks 3/4/5 already removed cache-refresh, freshness, worktree-process-kill). If it lists an offender not in the allowlist, that file is either a Phase-1 target that regressed (fix it) or a daemon-reachable sync-exec outside Phase 1 (add it to the allowlist with a finding/phase comment). The closure-size guard and the checker RED-proof tests must also pass. + +- [ ] **Step 3: Demonstrate the gate fails on reintroduction (report evidence)** + +Temporarily add to `lib/daemon/pollers.ts` (a Phase-1-clean, non-allowlisted daemon file): `import { execSync } from "child_process";` and a call `execSync("true");`. Run: + +Run: `bun test lib/__tests__/no-daemon-sync-exec.test.ts` +Expected: FAIL ... offenders `["lib/daemon/pollers.ts"]`. Capture this RED output for the report, then revert both lines. + +- [ ] **Step 4: Commit** + +```bash +git add lib/__tests__/no-daemon-sync-exec.test.ts +git commit -m "gate: fail on sync-exec anywhere in the daemon import graph (1.3)" +``` + +--- + +### Task 8: Refresh cannot wedge (S007) + +One hung GitLab call latches `refreshInFlight` forever, freezing the coalesced refresh, and merged `runner.pending` grows unbounded. Add a whole-cycle deadline that clears the latch so the next tick can start, and cap the pending queue. + +**Files:** +- Modify: `lib/daemon/cache-refresh.ts` (extract a coalescer with a deadline) +- Modify: `lib/daemon/freshness.ts` (`applyInvalidationBatch` pending cap) +- Test: `lib/daemon/__tests__/cache-refresh-coalesce.test.ts` (create), `lib/daemon/__tests__/freshness-pending-cap.test.ts` (create) + +**Interfaces:** +- Produces: `makeCoalescer(run, deadlineMs, onTimeout) => () => Promise` exported from `cache-refresh.ts`. +- Produces: `PENDING_CAP` const in `freshness.ts`; `applyInvalidationBatch` dedupes-and-caps pending. + +- [ ] **Step 1: Write the failing coalescer test** + +Create `lib/daemon/__tests__/cache-refresh-coalesce.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { makeCoalescer } from "../cache-refresh.ts"; + +test("clears the in-flight latch after the deadline even if run never settles", async () => { + let starts = 0; + let timedOut = 0; + const coalesce = makeCoalescer( + () => { starts++; return new Promise(() => {}); }, // never resolves + 50, + () => { timedOut++; }, + ); + const t0 = Date.now(); + await coalesce(); // resolves at the deadline, not never + expect(Date.now() - t0).toBeLessThan(500); + expect(timedOut).toBe(1); + await coalesce(); // latch cleared, a new run can start + expect(starts).toBe(2); +}); + +test("coalesces concurrent callers onto one run", async () => { + let starts = 0; + let resolveRun!: () => void; + const coalesce = makeCoalescer( + () => { starts++; return new Promise((r) => { resolveRun = r; }); }, + 10_000, + () => {}, + ); + const a = coalesce(); + const b = coalesce(); + expect(starts).toBe(1); + resolveRun(); + await Promise.all([a, b]); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/cache-refresh-coalesce.test.ts` +Expected: FAIL ... `makeCoalescer` not exported. + +- [ ] **Step 3: Add `makeCoalescer` and use it** + +In `lib/daemon/cache-refresh.ts`, add above `createCacheRefresher`: + +```ts +/** Below the 5-min tick, above the slowest legitimate deep sync. */ +const REFRESH_CYCLE_DEADLINE_MS = 4 * 60 * 1000; + +/** + * Coalesce concurrent callers onto one in-flight run, but clear the latch after + * `deadlineMs` even if the run never settles, so a wedged cycle (a half-open + * GitLab socket that never rejects) cannot pin the latch forever. The wedged + * run's frame still leaks until the OS reaps the socket; this only frees the + * next tick. + */ +export function makeCoalescer( + run: () => Promise, + deadlineMs: number, + onTimeout: () => void, +): () => Promise { + let inFlight: Promise | null = null; + return () => { + if (inFlight) return inFlight; + const impl = run().catch(() => {}); // a rejected cycle still clears the latch + const guarded = Promise.race([ + impl, + new Promise((resolve) => setTimeout(() => { onTimeout(); resolve(); }, deadlineMs)), + ]).finally(() => { inFlight = null; }); + inFlight = guarded; + return guarded; + }; +} +``` + +Replace the `refreshInFlight` coalescer (lines 54-60) inside `createCacheRefresher`: + +```ts + const refreshCache = makeCoalescer( + refreshCacheImpl, + REFRESH_CYCLE_DEADLINE_MS, + () => log.warn("cache refresh timed out; cleared in-flight latch for next tick"), + ); +``` + +(Delete `let refreshInFlight` and the old `function refreshCache`. `refreshCacheImpl` stays as the async body; `return refreshCache;` at the end is unchanged.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/cache-refresh-coalesce.test.ts` +Expected: PASS. + +- [ ] **Step 5: Add the pending-cap test** + +Create `lib/daemon/__tests__/freshness-pending-cap.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { applyInvalidationBatch, PENDING_CAP } from "../freshness.ts"; + +test("merged pending is deduped by kind:ref and capped", async () => { + const runner: any = { processing: true, pending: [] }; + // Push more distinct keys than the cap; plus duplicates. + const keys = Array.from({ length: PENDING_CAP + 500 }, (_, i) => ({ kind: "mr", ref: String(i) })); + const dupes = [{ kind: "mr", ref: "0" }, { kind: "mr", ref: "0" }]; + await applyInvalidationBatch({} as any, {} as any, runner, [...keys, ...dupes], {}); + expect(runner.pending.length).toBeLessThanOrEqual(PENDING_CAP); + const ids = runner.pending.map((k: any) => `${k.kind}:${k.ref}`); + expect(new Set(ids).size).toBe(ids.length); // no duplicates +}); +``` + +- [ ] **Step 6: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/freshness-pending-cap.test.ts` +Expected: FAIL ... `PENDING_CAP` not exported (and current code pushes unbounded). + +- [ ] **Step 7: Cap the pending queue** + +In `lib/daemon/freshness.ts`, add near the module state (below `const watches`): + +```ts +/** Bound merged pending so a wedged processKeys cannot grow memory unbounded. */ +export const PENDING_CAP = 1000; +``` + +Replace the `if (runner.processing)` block in `applyInvalidationBatch` (lines 392-395): + +```ts + if (runner.processing) { + const seen = new Set(runner.pending.map((k) => `${k.kind}:${k.ref}`)); + for (const k of keys) { + if (runner.pending.length >= PENDING_CAP) break; + const id = `${k.kind}:${k.ref}`; + if (seen.has(id)) continue; + seen.add(id); + runner.pending.push(k); + } + return; + } +``` + +- [ ] **Step 8: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/freshness-pending-cap.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean. + +- [ ] **Step 9: Commit** + +```bash +git add lib/daemon/cache-refresh.ts lib/daemon/freshness.ts lib/daemon/__tests__/cache-refresh-coalesce.test.ts lib/daemon/__tests__/freshness-pending-cap.test.ts +git commit -m "refresh: whole-cycle deadline clears the coalesce latch; cap RepoWatch.pending (S007)" +``` + +--- + +### Task 9: Provider cache invalidation on token rotation (S048, S049) + +`providers` caches `GitLabProvider` instances by repoName with no token check, so a rotated `gitlabToken` never reaches watchers or forge handlers until a daemon restart. Key the cache on a token fingerprint and rebuild on mismatch; reset the `userIdResolved` latch. + +**Files:** +- Modify: `lib/daemon/freshness.ts` +- Test: `lib/daemon/__tests__/freshness-provider-rotation.test.ts` (create) + +**Interfaces:** +- Consumes: `loadSecrets()` (from `../linear.ts`, returns `{ gitlabToken }`), `makeProvider(host, token)`. +- Internal: `providers` map value becomes `{ provider: GitLabProvider; token: string }`. + +- [ ] **Step 1: Write the failing test** + +Create `lib/daemon/__tests__/freshness-provider-rotation.test.ts`. Since `ensureProvider` is module-private, test the observable contract: the source no longer returns a cached provider without comparing the current token. Assert the fingerprint mechanism exists and the `if (cached) return cached;` short-circuit is gone: + +```ts +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("ensureProvider compares the current token before reusing a cached provider", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + // The unconditional cache-hit return is the S049 bug; it must be gone. + expect(src).not.toMatch(/const cached = providers\.get\(repoName\);\s*\n\s*if \(cached\) return cached;/); + // A token fingerprint must be stored alongside the provider. + expect(src).toMatch(/providers\.set\(repoName,\s*\{\s*provider/); +}); +``` + +(A behavioral test would need a real GitLabProvider + secrets seam; the freshness module has no injection point for `loadSecrets`. This source-contract test plus `bunx tsc` is the achievable guard; note the limitation in the report.) + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/freshness-provider-rotation.test.ts` +Expected: FAIL ... the unconditional cache-hit return still present. + +- [ ] **Step 3: Key the provider cache on a token fingerprint** + +In `lib/daemon/freshness.ts`, change the `providers` map type (line 88): + +```ts +const providers = new Map(); +``` + +Rewrite `ensureProvider` so `loadSecrets()` runs before the cache decision and a token mismatch rebuilds (lines 125-155): + +```ts +async function ensureProvider(repoName: string, repoPath: string): Promise { + const secrets = await loadSecrets(); + if (!secrets.gitlabToken) { + log.info(`no gitlabToken; skipping ${repoName}`); + return null; + } + const cached = providers.get(repoName); + if (cached && cached.token === secrets.gitlabToken) return cached.provider; + if (cached) { + // Token rotated: drop the stale provider and any live watch built on it so + // the next reconcile rebuilds with the new token, and re-resolve userId. + stopWatch(repoName); + userIdResolved = false; + } + + const remoteUrl = await getRemoteUrl(repoPath); + if (!remoteUrl) { log.info(`no origin remote for ${repoName}; skipping`); return null; } + if (!isGitLabRemote(remoteUrl)) { log.info(`remote "${remoteUrl}" for ${repoName} is not GitLab; skipping events watch`); return null; } + const remote = parseRemoteUrl(remoteUrl); + if (!remote) { log.info(`could not parse remote "${remoteUrl}" for ${repoName}; skipping`); return null; } + + const provider = makeProvider(remote.host, secrets.gitlabToken); + providers.set(repoName, { provider, token: secrets.gitlabToken }); + return provider; +} +``` + +Update the other cache reads to the new value shape: +- `getRepoContext` line 251: `let provider = providers.get(repoName)?.provider ?? null;` +- `getRepoContext` line 282 (after building): `providers.set(repoName, { provider, token: secrets.gitlabToken });` +- `ensureUserId` line 160: `const anyProvider = providers.values().next().value?.provider as GitLabProvider | undefined;` +- `disposeFreshness` `providers.clear()` unchanged. + +(`stopWatch` is already defined in this module; calling it for a repo with no watch is a no-op ... verify its guard, add an early return if absent.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/freshness-provider-rotation.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean (all `providers.get/set/values` sites updated to the new shape). + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/freshness.ts lib/daemon/__tests__/freshness-provider-rotation.test.ts +git commit -m "freshness: rebuild the provider cache when gitlabToken rotates (S048, S049)" +``` + +--- + +### Task 10: A single failed lsof no longer resets runaway detection (S061) + +`gather()` returns `[]` on both a real empty scan and an `lsof`/`ps` failure; `scan()` then prunes every tracked pid, resetting `firstSeen` and the runaway sample window so a machine where `lsof` fails intermittently can never fire a runaway notification. Distinguish failure (null) from empty. + +**Files:** +- Modify: `lib/daemon/system-process-scanner.ts` (`getAllRepoPids`, `gather`, `scan`, `refresh`) +- Test: `lib/daemon/__tests__/system-process-scanner-resilience.test.ts` (create) + +**Interfaces:** +- Internal: `getAllRepoPids` and `gather` return `... | null` (null = the underlying `lsof`/`ps` scan failed). `scan`/`refresh` preserve `tracked`/`lastResult`/`lastScanAt` when `gather` returns null. +- Change `gather` from `private` to `protected` so a test subclass can drive its return value. + +- [ ] **Step 1: Write the failing test** + +Create `lib/daemon/__tests__/system-process-scanner-resilience.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { SystemProcessScanner } from "../system-process-scanner.ts"; + +class FakeScanner extends SystemProcessScanner { + next: any[] | null = []; + protected async gather(): Promise { return this.next; } +} + +test("a failed gather (null) keeps tracked windows and lastResult intact", async () => { + const s = new FakeScanner(); + // Seed a tracked process across enough scans that firstSeen is established. + s.next = [{ pid: 4242, cpuPercent: 95, command: "node", args: "x", ppid: 1, port: null, memoryMB: 10, etime: "01:00", repo: "r", linearTicket: null, packageScript: null }]; + const first = await s.scan(); + expect(first.find((p) => p.pid === 4242)).toBeTruthy(); + const firstSeen = s.getTracked().get(4242)?.firstSeen; + + // Now lsof fails: gather returns null. tracked and lastResult must survive. + s.next = null; + const during = await s.scan(); + expect(s.getTracked().get(4242)?.firstSeen).toBe(firstSeen); + expect(during.find((p) => p.pid === 4242)).toBeTruthy(); // lastResult carried forward +}); +``` + +(Field names on the fake process object follow `GatheredProcess`; adjust to the exact shape when implementing. `getTracked()` is an existing accessor.) + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/system-process-scanner-resilience.test.ts` +Expected: FAIL ... `gather` is `private` (can't override) and/or `scan` wipes `tracked` on the null tick. + +- [ ] **Step 3: Thread the failure signal** + +In `lib/daemon/system-process-scanner.ts`: + +`getAllRepoPids` ... return null on lsof failure instead of an empty Map: + +```ts +async function getAllRepoPids(trackedPaths: string[]): Promise | null> { + if (trackedPaths.length === 0) return new Map(); + const { stdout, exitCode } = await runCapture(["lsof", "-d", "cwd", "-Fpn"], { timeoutMs: 10_000 }); + if (exitCode !== 0 && !stdout) { + log.warn({ exitCode }, "lsof scan failed; preserving prior process state"); + return null; + } + return parseLsofCwdMap(stdout, trackedPaths); +} +``` + +`gather` ... change signature to `protected async gather(...): Promise` and propagate null (lines 380, 389): + +```ts + const cwdMap = await getAllRepoPids(trackedPaths); + if (cwdMap === null) return null; // lsof failed + if (cwdMap.size === 0) return []; // genuinely no tracked-cwd processes + // ... + const psRes = await runCapture([...], { timeoutMs: 5000 }); + if (psRes.exitCode !== 0 && !psRes.stdout) return null; // ps failed + if (!psRes.stdout) return []; +``` + +`scan` ... early-return on null before the try/finally so `tracked`/`lastResult`/`lastScanAt` are untouched: + +```ts + async scan(portEntries: PortEntry[] = []): Promise { + const gathered = await this.gather(portEntries); + if (gathered === null) return this.lastResult; // failed tick: preserve everything + try { + // ...existing loop over `gathered`, prune, this.lastResult = results... + return results; + } finally { + this.lastScanAt = Date.now(); + } + } +``` + +`refresh` ... same null guard: + +```ts + async refresh(portEntries: PortEntry[] = []): Promise { + const gathered = await this.gather(portEntries); + if (gathered === null) return this.lastResult; + try { + // ...existing map... + return results; + } finally { + this.lastScanAt = Date.now(); + } + } +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/system-process-scanner-resilience.test.ts && bun test lib/daemon/__tests__/system-process-scanner.test.ts && bunx tsc --noEmit` +Expected: PASS (new + existing scanner tests); tsc clean. + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/system-process-scanner.ts lib/daemon/__tests__/system-process-scanner-resilience.test.ts +git commit -m "system-process-scanner: a failed lsof preserves runaway windows (S061)" +``` + +--- + +### Task 11: Cheapen run-DB opens: mtime memoization + herdr backoff (S101, S038, S039) + +The agent-status poller and `runs:list` open every retained run's SQLite db every 10s (cost scales with retained runs, not live runs), and the poller spawns herdr every 10s forever on a herdr-less machine. Memoize finished-run summaries by state.db mtime, and back off the herdr probe after repeated failures. + +**Files:** +- Modify: `lib/runs/store.ts` (mtime cache in `listRuns`) +- Modify: `lib/daemon/agent-status-poller.ts` (consecutive-failure backoff) +- Test: `lib/runs/__tests__/store-memo.test.ts` (create), add to `lib/daemon/__tests__/agent-status-poller.test.ts` + +**Interfaces:** +- Internal: `listRuns` caches finished-run `RunSummary` keyed by `${repo}/${id}` → `{ mtimeMs, summary }`; running runs are never cached (their db still mutates and their liveness overlay changes). +- Internal: poller gains `FAILURE_THRESHOLD` / `BACKOFF_TICKS` backoff. + +- [ ] **Step 1: Write the failing memoization test** + +Create `lib/runs/__tests__/store-memo.test.ts`. Use the existing `seedRun`/`root` fixtures and a spy to prove a finished run's db is not reopened when its mtime is unchanged: + +```ts +import { test, expect, afterEach, spyOn } from "bun:test"; +import { Database } from "bun:sqlite"; +import { listRuns } from "../store.ts"; +import { root, seedRun } from "./fixtures.ts"; + +afterEach(() => { delete process.env.RT_RUNS_ROOT; }); + +test("a finished run's db is opened once, then served from the mtime cache", () => { + root(); // sets RT_RUNS_ROOT to a temp dir + seedRun("repoA", "run1", { status: "done" }); + const openSpy = spyOn(Database.prototype, "query"); + listRuns(); // first call opens + reads + const afterFirst = openSpy.mock.calls.length; + listRuns(); // second call: mtime unchanged -> no reopen + expect(openSpy.mock.calls.length).toBe(afterFirst); // no additional queries for run1 + openSpy.mockRestore(); +}); +``` + +(If `seedRun`'s signature differs, match it; the point is a finished run plus a proof the second `listRuns` does not re-query it.) + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/runs/__tests__/store-memo.test.ts` +Expected: FAIL ... every `listRuns` reopens and re-queries every run. + +- [ ] **Step 3: Add the mtime cache** + +In `lib/runs/store.ts`, add `import { statSync } from "fs";` (if absent) and a module cache: + +```ts +// Finished runs never change; skip the open+PRAGMA+4-reads when the db mtime +// is unchanged. Running runs are never cached: their db still mutates and their +// liveness overlay is recomputed per call. +const summaryCache = new Map(); +``` + +Rewrite the per-run body of `listRuns` (lines 122-130): + +```ts + for (const id of dirs(join(runsRoot(), r))) { + const dbPath = join(runsRoot(), r, id, "state.db"); + let mtimeMs: number; + try { mtimeMs = statSync(dbPath).mtimeMs; } catch { continue; } + const key = `${r}/${id}`; + const hit = summaryCache.get(key); + if (hit && hit.mtimeMs === mtimeMs) { out.push(hit.summary); continue; } + + const opened = openRun(r, id); + if (!opened) continue; + try { + const row = runRow(opened.db); + if (row) { + const summary = withAttention(opened.db, row, liveness); + out.push(summary); + if (summary.status !== "running") summaryCache.set(key, { mtimeMs, summary }); + } + } finally { + opened.db.close(); + } + } +``` + +(Confirm the field is `summary.status`; the poller reads `run.status`. If `RunSummary` names it `state`, use that.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/runs/__tests__/store-memo.test.ts && bun test lib/runs/__tests__/store.test.ts` +Expected: PASS (new + existing store tests, including corrupt/missing-db cases). + +- [ ] **Step 5: Write the failing herdr-backoff test** + +Add to `lib/daemon/__tests__/agent-status-poller.test.ts` (using the existing `probe`/`list` DI seams and manual `tick()`): + +```ts +test("backs off the herdr probe after repeated failures", async () => { + let probeCalls = 0; + const handle = startAgentStatusPoller({ + intervalMs: 3_600_000, // real timer never fires + probe: async () => { probeCalls++; return null; }, // herdr absent + list: () => [], + }); + for (let i = 0; i < 20; i++) await handle.tick(); + handle.stop(); + // Without backoff this would be 20; with backoff (threshold 3, 1-in-6) far fewer. + expect(probeCalls).toBeLessThan(10); +}); +``` + +- [ ] **Step 6: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/agent-status-poller.test.ts` +Expected: FAIL ... `probeCalls` is 20 (probes every tick). + +- [ ] **Step 7: Add the backoff** + +In `lib/daemon/agent-status-poller.ts`, add constants and counter state in the closure around `tick`: + +```ts +const FAILURE_THRESHOLD = 3; // consecutive null probes before backing off +const BACKOFF_TICKS = 6; // then probe once every 6 ticks (~60s at 10s cadence) +``` + +```ts + let consecutiveFailures = 0; + let ticksSkipped = 0; + async function tick() { + if (consecutiveFailures >= FAILURE_THRESHOLD) { + if (++ticksSkipped < BACKOFF_TICKS) return; + ticksSkipped = 0; + } + const entries = await probe(); + if (entries === null) { consecutiveFailures++; return; } + consecutiveFailures = 0; + // ...existing tick body... + } +``` + +- [ ] **Step 8: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/agent-status-poller.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean. + +- [ ] **Step 9: Commit** + +```bash +git add lib/runs/store.ts lib/daemon/agent-status-poller.ts lib/runs/__tests__/store-memo.test.ts lib/daemon/__tests__/agent-status-poller.test.ts +git commit -m "runs: mtime-memoize finished-run summaries; back off herdr probe (S101, S038, S039)" +``` + +--- + +### Task 12: Gate background scans on demand (S058, S093) + +The 10s system-process scan and 30s port scan run at full cadence with zero consumers (idle tray, battery). Gate them on a recent consumer read (tray/CLI/REST hit of `ports`/`system-processes`/`tray:status`). Wired via a new tracker read from `pollers.ts` and set by wrapping the demand commands in `command-router.ts` (no forbidden-file edit; verified reachable). + +**Files:** +- Create: `lib/daemon/demand-tracker.ts` +- Modify: `lib/daemon/command-router.ts` (wrap the demand-command entries) +- Modify: `lib/daemon/pollers.ts` (skip scans when no recent demand) +- Test: `lib/daemon/__tests__/demand-tracker.test.ts` (create) + +**Interfaces:** +- Produces: `recordDemand()`, `demandedWithin(ms) => boolean`, `wrapWithDemand(handlers, cmds) => handlers` in `demand-tracker.ts`. + +- [ ] **Step 1: Write the failing test** + +Create `lib/daemon/__tests__/demand-tracker.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { recordDemand, demandedWithin, wrapWithDemand } from "../demand-tracker.ts"; + +test("demandedWithin reflects a recent recordDemand", () => { + recordDemand(); + expect(demandedWithin(60_000)).toBe(true); + expect(demandedWithin(0)).toBe(false); // window of 0ms is never "recent" +}); + +test("wrapWithDemand records demand and delegates to the inner handler", async () => { + let called = false; + const handlers = { "system-processes": async () => { called = true; return { ok: true }; }, other: async () => ({ ok: true }) }; + wrapWithDemand(handlers, ["system-processes"]); + const before = demandedWithin(50); + await handlers["system-processes"](undefined as any); + expect(called).toBe(true); + expect(demandedWithin(1000)).toBe(true); + void before; +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/demand-tracker.test.ts` +Expected: FAIL ... module does not exist. + +- [ ] **Step 3: Create the tracker** + +Create `lib/daemon/demand-tracker.ts`: + +```ts +/** + * "A consumer is watching" signal for the background scans. The tray/CLI/console + * calling ports/system-processes/tray:status stamps demand here (via the + * command-router wrapper); pollers skip the 10s/30s scans when nothing has asked + * recently, so an idle machine stops paying the lsof/git tax (S058, S093). + */ +let lastDemandAt = 0; + +export function recordDemand(): void { + lastDemandAt = Date.now(); +} + +/** True when a consumer read a scan-backed command within `ms`. */ +export function demandedWithin(ms: number): boolean { + return lastDemandAt !== 0 && Date.now() - lastDemandAt < ms; +} + +/** Wrap the named handler entries so each call stamps demand, then delegates. */ +export function wrapWithDemand>(handlers: T, cmds: string[]): T { + for (const cmd of cmds) { + const inner = handlers[cmd]; + if (typeof inner !== "function") continue; + (handlers as any)[cmd] = (...args: any[]) => { recordDemand(); return inner(...args); }; + } + return handlers; +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/demand-tracker.test.ts` +Expected: PASS. + +- [ ] **Step 5: Wrap the demand commands in the router** + +In `lib/daemon/command-router.ts`: `import { wrapWithDemand } from "./demand-tracker.ts";`. Change the trailing `return { ...handlers... }` of `buildRoutedHandlers` to assign then wrap: + +```ts + const handlers: TypedHandlers & HandlerMap = { + ...createCacheHandlers(ctx), + // ...unchanged spreads... + "freshness:reconcile": async () => { + await reconcileFreshness({ ctx, broadcast }); + return { ok: true, data: getFreshnessSnapshot() }; + }, + }; + // A tray/CLI/console read of any scan-backed command means "someone is + // watching", which un-gates the background scans (see pollers.ts, S058/S093). + return wrapWithDemand(handlers, ["ports", "system-processes", "tray:status"]); +``` + +- [ ] **Step 6: Gate the scans in pollers** + +In `lib/daemon/pollers.ts`: `import { demandedWithin } from "./demand-tracker.ts";` and add: + +```ts +/** Consider a consumer "present" for 5 min after its last scan-backed read. */ +const DEMAND_WINDOW_MS = 5 * 60 * 1000; +``` + +Add the gate as the first line inside both scan bodies (after the in-flight guard): + +```ts + async function refreshPortCache(): Promise { + if (portScanInFlight) return; + if (!demandedWithin(DEMAND_WINDOW_MS)) return; // no consumer asked recently + portScanInFlight = true; + // ...unchanged... + } + + async function refreshSystemProcesses(): Promise { + if (processScanInFlight) return; + if (!demandedWithin(DEMAND_WINDOW_MS)) return; + processScanInFlight = true; + // ...unchanged... + } +``` + +(The tray's on-demand `system-processes` handler already calls `scanner.refresh()` when its cache is stale, so a freshly-connecting consumer gets immediate data and its read stamps demand, resuming the poller on the next tick. The hooks-scan and cache-refresh intervals are unaffected.) + +- [ ] **Step 7: Run tests + typecheck** + +Run: `bun test lib/daemon/__tests__/demand-tracker.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean. + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon/demand-tracker.ts lib/daemon/command-router.ts lib/daemon/pollers.ts lib/daemon/__tests__/demand-tracker.test.ts +git commit -m "pollers: gate the 10s/30s scans on recent consumer demand (S058, S093)" +``` + +--- + +### Final verification (run before the whole-branch review) + +- [ ] `bun run build` inside `packages/rt-client` (rt-client's `runCapture` mirror changed in Task 1). +- [ ] `bun test lib commands packages scripts` ... all green. +- [ ] `bunx tsc --noEmit` ... zero errors. +- [ ] `bun test lib/__tests__/no-daemon-sync-exec.test.ts` ... green; capture the Task 7 Step 3 RED demonstration for the report. +- [ ] Confirm no sibling-owned file in the write fence was modified (`git diff --name-only main...` reviewed against the fence). From b7694dd0b70d5bde7530307dd7f60173701a3752 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:02:31 -0500 Subject: [PATCH 024/142] daemon: drop startDaemon's dead-code catch and its now-stale JSDoc Review fix (round 1): runDaemon() now catches and exit(1)s internally on boot failure, so startDaemon()'s outer try/catch never runs, and its JSDoc claiming otherwise was misleading. Replaced with a one-line comment stating the real constraint. --- lib/daemon.ts | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/lib/daemon.ts b/lib/daemon.ts index fcfe6ca5..d06c8a51 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -530,25 +530,10 @@ async function runDaemon(): Promise { } } -/** - * Both real callers (cli.ts's `--daemon` entry, and this file's own - * import.meta.main guard below) invoke this fire-and-forget — neither awaits - * or catches. Left as a bare async function, ANY failure inside runDaemon() - * (this bind, openBranchCacheStore, anything else awaited) becomes an - * unhandledRejection, whose handler (installCrashHandlers, above) logs and - * deliberately does NOT exit — the daemon would stay alive with rt.sock - * possibly bound but startup never having reached signal handlers, pollers, - * or "daemon ready". Catching here and exiting explicitly gives every - * startup failure the same outcome a synchronous one already had: the - * process dies and whatever restarts a crashed daemon restarts this one too. - */ +// runDaemon() never rejects — it logs fatal and exit(1)s internally on any +// boot failure, so this wrapper needs no catch of its own. export async function startDaemon(): Promise { - try { - await runDaemon(); - } catch (err) { - log.fatal({ err }, "daemon startup failed — exiting"); - process.exit(1); - } + await runDaemon(); } // Auto-run when executed directly (source mode: bun run lib/daemon.ts) From b24bb2b1d057cc4713c32ce7404a26ced6edb59d Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:04:05 -0500 Subject: [PATCH 025/142] worktree create: serialize createTree per repoPath so provision and replenish don't race git ref locks (S089) --- .../__tests__/worktree-reconciler.test.ts | 54 ++++++++++++++++++- lib/daemon/handlers/worktree.ts | 12 +++-- lib/daemon/worktree-reconciler.ts | 26 ++++++++- 3 files changed, 86 insertions(+), 6 deletions(-) diff --git a/lib/daemon/__tests__/worktree-reconciler.test.ts b/lib/daemon/__tests__/worktree-reconciler.test.ts index 8d2bca47..30133a8c 100644 --- a/lib/daemon/__tests__/worktree-reconciler.test.ts +++ b/lib/daemon/__tests__/worktree-reconciler.test.ts @@ -19,7 +19,7 @@ import { import { createTree } from "../../worktree/create.ts"; import type { WorktreeAppConfig } from "../../worktree/config.ts"; import { RETENTION_MS } from "../../worktree/trash.ts"; -import { reconcileRepoRegistry, createWorktreeReconciler, __test__ } from "../worktree-reconciler.ts"; +import { reconcileRepoRegistry, createWorktreeReconciler, withCreateLock, __test__ } from "../worktree-reconciler.ts"; function makeRepo(): string { // realpathSync: git canonicalizes /var -> /private/var on macOS (Global Constraints) @@ -1400,3 +1400,55 @@ describe("reapRepoTrash", () => { expect(warns.some((w) => JSON.stringify(w).includes(parent))).toBe(true); }); }); + +// S089: a provision's cold createTree and the reconciler's own replenish +// createTree can run concurrently for the same repo, both `git fetch origin +// ` against the same repoPath — the loser fails to lock +// refs/remotes/origin/, and that failure gets charged to +// createBackoff (a 5-to-30-minute replenish hold) for what was really just +// contention, not a genuine failure. Serializing createTree per repoPath +// closes the race at its root. +describe("withCreateLock", () => { + test("serializes concurrent calls for the same repoPath — never two holders at once", async () => { + const order: string[] = []; + let active = 0; + let maxActive = 0; + const run = (id: string) => withCreateLock("/repo/a", async () => { + active++; + maxActive = Math.max(maxActive, active); + order.push(`start-${id}`); + await new Promise((r) => setTimeout(r, 10)); + order.push(`end-${id}`); + active--; + }); + await Promise.all([run("1"), run("2"), run("3")]); + expect(maxActive).toBe(1); + expect(order).toEqual(["start-1", "end-1", "start-2", "end-2", "start-3", "end-3"]); + }); + + test("different repoPaths are not serialized against each other", async () => { + let active = 0; + let maxActive = 0; + const run = (path: string) => withCreateLock(path, async () => { + active++; + maxActive = Math.max(maxActive, active); + await new Promise((r) => setTimeout(r, 10)); + active--; + }); + await Promise.all([run("/repo/b"), run("/repo/c")]); + expect(maxActive).toBe(2); + }); + + test("a holder that throws still releases the lock for the next caller", async () => { + await expect(withCreateLock("/repo/d", async () => { throw new Error("boom"); })).rejects.toThrow("boom"); + let ran = false; + await withCreateLock("/repo/d", async () => { ran = true; }); + expect(ran).toBe(true); + }); +}); + +test("both cold-create call sites in handlers/worktree.ts serialize createTree through the shared per-repo lock (S089)", () => { + const source = readFileSync(new URL("../handlers/worktree.ts", import.meta.url), "utf8"); + const matches = source.match(/withCreateLock\(/g) ?? []; + expect(matches.length).toBeGreaterThanOrEqual(2); +}); diff --git a/lib/daemon/handlers/worktree.ts b/lib/daemon/handlers/worktree.ts index b65f672a..bb98e53e 100644 --- a/lib/daemon/handlers/worktree.ts +++ b/lib/daemon/handlers/worktree.ts @@ -57,7 +57,7 @@ import { resolveReadySteps, } from "../../worktree/config.ts"; import { changedSince, runReadySteps, stepsToRun } from "../../worktree/ready.ts"; -import { freshenRepo, reconcileRepoRegistry } from "../worktree-reconciler.ts"; +import { freshenRepo, reconcileRepoRegistry, withCreateLock } from "../worktree-reconciler.ts"; import { repoDataDir, rtDir } from "../../rt-paths.ts"; const PROVISION_FETCH_TIMEOUT_MS = 5 * 60_000; @@ -313,9 +313,13 @@ export function createWorktreeHandlers( } } if (!rec) { - const created = await createTree({ + // Serialized against the reconciler's own replenish createTree for + // this repo (S089): both `git fetch origin ` against the + // same repoPath, and an unserialized race charges the loser's + // ref-lock failure to createBackoff for what was just contention. + const created = await withCreateLock(repoPath, () => createTree({ repoName, repoPath, emit: opts.emit, log: ctx.log, - }); + })); if (!created.ok) { if (created.error === "busy") return { ok: false, error: "busy" }; return { @@ -467,7 +471,7 @@ export function createWorktreeHandlers( const repoPath = repoName ? ctx.repoIndex()[repoName] : undefined; if (!repoName || !repoPath || parseIdentity(repoName) === null) return { ok: false, error: "repo-unknown" }; - const created = await createTree({ repoName, repoPath, emit: opts.emit, log: ctx.log }); + const created = await withCreateLock(repoPath, () => createTree({ repoName, repoPath, emit: opts.emit, log: ctx.log })); if (!created.ok) { if (created.error === "busy") return { ok: false, error: "busy" }; return { ok: false, error: createFailedError(created) }; diff --git a/lib/daemon/worktree-reconciler.ts b/lib/daemon/worktree-reconciler.ts index f2e20065..d7af150d 100644 --- a/lib/daemon/worktree-reconciler.ts +++ b/lib/daemon/worktree-reconciler.ts @@ -851,6 +851,30 @@ export async function freshenRepo( */ const createBackoff = new Map(); +/** + * S089: a provision's cold `createTree` (handlers/worktree.ts) and this + * reconciler's own replenish `createTree` can run concurrently for the same + * repo, both `git fetch origin ` against the same repoPath — the + * loser fails to lock refs/remotes/origin/, and that failure gets + * charged to createBackoff (a 5-to-30-minute replenish hold) for what was + * really just contention, not a genuine failure. Chained per repoPath so + * concurrent callers queue instead of racing; a rejected holder still + * releases the lock for the next one. + */ +const createLocks = new Map>(); + +export function withCreateLock(repoPath: string, fn: () => Promise): Promise { + const prior = createLocks.get(repoPath) ?? Promise.resolve(); + const ready = prior.catch(() => {}); // a previous holder's rejection must not block the next one + const result = ready.then(fn); + const tracked: Promise = result.then(() => undefined, () => undefined); + createLocks.set(repoPath, tracked); + void tracked.finally(() => { + if (createLocks.get(repoPath) === tracked) createLocks.delete(repoPath); + }); + return result; +} + /** The active backoff deadline for a repo, or null when creates may run now. */ function createBlockedUntil(repoName: string): string | null { const entry = createBackoff.get(repoName); @@ -920,7 +944,7 @@ async function replenishAndShrink( break; } budget--; - const p: Promise = createTree({ repoName, repoPath, emit, log }) + const p: Promise = withCreateLock(repoPath, () => createTree({ repoName, repoPath, emit, log })) .then((result) => { if (result.ok) { createBackoff.delete(repoName); From 8da0e048f4cf8e89cdfd800d39cee19492ee88cf Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:04:45 -0500 Subject: [PATCH 026/142] daemon: rt.trustedBrowserOrigins allowlist + needsToken invert-default (S005/S006/S040/S041/S084) --- lib/daemon/__tests__/api-auth.test.ts | 68 +++++++++++++++++++ lib/daemon/api-auth.ts | 67 +++++++++++++++--- .../rt-client/src/settings/registry-defs.ts | 8 +++ 3 files changed, 133 insertions(+), 10 deletions(-) diff --git a/lib/daemon/__tests__/api-auth.test.ts b/lib/daemon/__tests__/api-auth.test.ts index ad97eef9..8f8e102e 100644 --- a/lib/daemon/__tests__/api-auth.test.ts +++ b/lib/daemon/__tests__/api-auth.test.ts @@ -5,6 +5,7 @@ import { describe, test, expect } from "bun:test"; import { needsToken, tokenOk, getApiToken, reloadApiToken, loadOrCreateApiToken } from "../api-auth.ts"; +import { isOriginAllowed, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "../api-auth.ts"; import { mkdtempSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; @@ -42,6 +43,30 @@ describe("needsToken", () => { test("secrets requires a token even though it's a GET — the response body is a credential, not metadata", () => { expect(needsToken("GET", "/api/secrets")).toBe(true); }); + + test("refresh requires a token now (S040)", () => { + expect(needsToken("POST", "/api/refresh")).toBe(true); + }); + + test("hooks repair requires a token now (S040/S084)", () => { + expect(needsToken("POST", "/api/hooks/my-repo/repair")).toBe(true); + }); + + test("notifications GET (destructive drain) requires a token now (S041)", () => { + expect(needsToken("GET", "/api/notifications")).toBe(true); + }); + + test("every non-GET/HEAD/OPTIONS method defaults to requiring a token", () => { + expect(needsToken("POST", "/api/some-future-mutating-route")).toBe(true); + expect(needsToken("PUT", "/api/anything")).toBe(true); + expect(needsToken("DELETE", "/api/anything")).toBe(true); + }); + + test("plain reads still do not require a token", () => { + expect(needsToken("GET", "/api/repos")).toBe(false); + expect(needsToken("GET", "/api/cache")).toBe(false); + expect(needsToken("HEAD", "/api/repos")).toBe(false); + }); }); describe("tokenOk", () => { @@ -104,3 +129,46 @@ describe("getApiToken / reloadApiToken singleton", () => { } }); }); + +describe("isOriginAllowed", () => { + test("exact match", () => { + expect(isOriginAllowed("http://localhost:5544", ["http://localhost:5544"])).toBe(true); + }); + test("no match", () => { + expect(isOriginAllowed("http://evil.example", ["http://localhost:5544"])).toBe(false); + }); + test("empty allowlist matches nothing", () => { + expect(isOriginAllowed("http://localhost:5544", [])).toBe(false); + }); +}); + +describe("isBrowserRequestTrusted", () => { + const apiToken = "the-real-token"; + + test("no Origin header at all -- a non-browser client -- is trusted regardless of token or allowlist", () => { + expect(isBrowserRequestTrusted(null, null, apiToken, [])).toBe(true); + }); + + test("a browser Origin with the correct token is trusted even off the allowlist", () => { + expect(isBrowserRequestTrusted("http://evil.example", apiToken, apiToken, [])).toBe(true); + }); + + test("a browser Origin with a wrong token and not on the allowlist is rejected", () => { + expect(isBrowserRequestTrusted("http://evil.example", "wrong", apiToken, [])).toBe(false); + }); + + test("a browser Origin with no token but on the allowlist is trusted", () => { + expect(isBrowserRequestTrusted("http://localhost:5544", null, apiToken, ["http://localhost:5544"])).toBe(true); + }); + + test("a browser Origin with no token and not on the allowlist is rejected", () => { + expect(isBrowserRequestTrusted("http://localhost:5544", null, apiToken, [])).toBe(false); + }); +}); + +describe("getTrustedBrowserOrigins", () => { + test("returns an array (empty by default in an isolated test HOME)", () => { + const origins = getTrustedBrowserOrigins(); + expect(Array.isArray(origins)).toBe(true); + }); +}); diff --git a/lib/daemon/api-auth.ts b/lib/daemon/api-auth.ts index 89d07980..acc67076 100644 --- a/lib/daemon/api-auth.ts +++ b/lib/daemon/api-auth.ts @@ -13,6 +13,7 @@ import { join } from "path"; import { randomUUID } from "crypto"; import { RT_DIR } from "../daemon-config.ts"; import { lazyChildLogger } from "../daemon-logger.ts"; +import { getSetting } from "../settings/resolve.ts"; const log = lazyChildLogger("api-auth"); @@ -66,20 +67,66 @@ export function reloadApiToken(tokenPath: string = API_TOKEN_PATH): string { return cachedApiToken; } -/** True when a request mutates state, or (secrets) returns raw credential values, and must present the local token. */ +/** + * True when a request mutates state, or (secrets/notifications) returns or + * drains something a GET should not silently consume, and must present the + * local token. Default-gated for every method except GET/HEAD/OPTIONS (S040: + * an allowlist-by-path guaranteed the next mutating route would ship + * unguarded) plus two explicit GET exceptions whose verb lies about being a + * read. + */ export function needsToken(method: string, pathname: string): boolean { - if (method === "OPTIONS") return false; - if (pathname === "/api/shutdown") return true; - if (pathname === "/api/sdm/reconnect") return true; - if (pathname === "/api/events/emit") return true; - // Gated despite being a GET: every other read-only route returns metadata - // (branch names, MR titles, ports) safe under the open-CORS "reads are - // free" policy above; this one's response body IS the credential. - if (pathname === "/api/secrets") return true; - return false; + if (method === "GET" || method === "HEAD" || method === "OPTIONS") { + // Gated despite being a GET: /api/secrets's response body IS a + // credential (S054); /api/notifications DRAINS the queue (S041), so its + // verb lies about being a read the way every other GET here is not. + if (pathname === "/api/secrets") return true; + if (pathname === "/api/notifications") return true; + return false; + } + return true; } /** True when the presented token matches the configured one (and one exists). */ export function tokenOk(provided: string | null, expected: string): boolean { return expected.length > 0 && provided === expected; } + +/** + * `rt.trustedBrowserOrigins` -- see registry-defs.ts. Read fresh on every + * call (the settings resolver is deliberately unmemoized), wrapped in a + * try/catch since a request-path settings read must never 500 the daemon + * over a malformed store file. + */ +export function getTrustedBrowserOrigins(): readonly string[] { + try { + const resolved = getSetting("rt.trustedBrowserOrigins"); + return Array.isArray(resolved.value) ? resolved.value : []; + } catch { + return []; + } +} + +export function isOriginAllowed(origin: string, allowedOrigins: readonly string[]): boolean { + return allowedOrigins.includes(origin); +} + +/** + * The 127.0.0.1 trust boundary (S005/S006): the daemon binds loopback-only, + * but any web page the user visits also runs on 127.0.0.1 and can send a + * request. A request with NO Origin header at all is not a browser fetch -- + * it is the CLI, the Swift tray, rt-client from a Bun/Node process, or the + * VS Code extension, none of which send one -- so it is trusted unchanged. + * A request that DOES carry an Origin header is trusted only if it presents + * the local api-token or its Origin is on the explicit allowlist. + */ +export function isBrowserRequestTrusted( + origin: string | null, + token: string | null, + apiToken: string, + allowedOrigins: readonly string[], +): boolean { + if (!origin) return true; + if (tokenOk(token, apiToken)) return true; + return isOriginAllowed(origin, allowedOrigins); +} diff --git a/packages/rt-client/src/settings/registry-defs.ts b/packages/rt-client/src/settings/registry-defs.ts index f0109ddd..198a3d84 100644 --- a/packages/rt-client/src/settings/registry-defs.ts +++ b/packages/rt-client/src/settings/registry-defs.ts @@ -205,6 +205,14 @@ export const REGISTRY: readonly SettingDef[] = [ migrated: true, description: "Per-repo git hook enable/disable state ({enabled, hooks: {: boolean}}); ownership-latch port of repos//hooks.json, store wins per field once it owns the key — including per-hook-name entries inside the nested hooks map, each defaulting to enabled when absent. The installed git-hook shim still greps repos//hooks.json with zero process spawns (a hook fires on every git operation); that file is now a DERIVED CACHE this key writes through, kept current by commands/hooks.ts's regenerateHooksCache at every write seam.", }, + { + key: "rt.trustedBrowserOrigins", + type: "array", + scopes: ["user", "machine"], + default: [], + merge: "replace", + description: "Browser Origins (scheme://host:port, exact string match) trusted to read the :9401 daemon API and subscribe to /ws without presenting the local api-token -- e.g. a locally-hosted console or chat-viewer dev server. Empty by default: every current mattstack consumer (the CLI, the Swift tray, rt-client from Bun/Node processes, the VS Code extension) is a non-browser client (sends no Origin header at all) and is unaffected either way.", + }, // --- mattstack (installer-lane) ----------------------------------------- { From 40e1e35bf074b4b6f5726f1e625465315afe9237 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:05:30 -0500 Subject: [PATCH 027/142] herdr client: accumulate raw socket bytes and decode once instead of per-chunk, so a split multibyte char survives (S095) --- lib/herdr/__tests__/client.test.ts | 30 ++++++++++++++++++++++++++++++ lib/herdr/client.ts | 10 ++++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/lib/herdr/__tests__/client.test.ts b/lib/herdr/__tests__/client.test.ts index 431b7132..952601ab 100644 --- a/lib/herdr/__tests__/client.test.ts +++ b/lib/herdr/__tests__/client.test.ts @@ -66,3 +66,33 @@ test("herdrAvailable probes session.snapshot", async () => { test("waitTimeout adds the 5s margin herdr needs to answer at its own budget", () => { expect(waitTimeout(60_000)).toBe(65_000); }); + +// S095: herdrRequest decoded each socket chunk independently — a multibyte +// character (box-drawing, emoji, ...) split across a chunk boundary became +// two partial byte sequences, each decoding to U+FFFD, corrupting the reply +// even though JSON.parse still succeeded on the well-formed-but-wrong text. +test("a multibyte character split across two socket chunks decodes correctly", async () => { + const sock = join(tmpdir(), `herdr-split-${process.pid}.sock`); + const text = "a─b"; // U+2500 BOX DRAWINGS LIGHT HORIZONTAL: E2 94 80, 3 bytes + const payload = JSON.stringify({ id: "x", result: { text } }) + "\n"; + const bytes = Buffer.from(payload, "utf8"); + const charStart = bytes.indexOf(Buffer.from("─", "utf8")); + const splitAt = charStart + 1; // split inside the multibyte sequence, after its first byte + + const server = Bun.listen({ + unix: sock, + socket: { + open(socket) { + socket.write(bytes.subarray(0, splitAt)); + setTimeout(() => socket.write(bytes.subarray(splitAt)), 20); + }, + data() {}, + close() {}, + error() {}, + }, + }); + stops.push(() => server.stop(true)); + + const res = await herdrRequest<{ text: string }>("whatever", {}, { sockPath: sock }); + expect(res).toEqual({ ok: true, result: { text: "a─b" } }); +}); diff --git a/lib/herdr/client.ts b/lib/herdr/client.ts index eb1dc373..c252b406 100644 --- a/lib/herdr/client.ts +++ b/lib/herdr/client.ts @@ -37,7 +37,12 @@ export function herdrRequest( return new Promise((resolve) => { let settled = false; - let buf = ""; + // A multibyte UTF-8 character (box-drawing, emoji, ...) can straddle a + // socket chunk boundary; decoding each chunk independently turns each + // half into its own U+FFFD replacement character, corrupting the reply + // (JSON.parse still succeeds — the text is just wrong). Accumulating raw + // bytes and decoding the whole buffer together avoids that entirely. + const chunks: Buffer[] = []; let conn: { end(): void } | undefined; const finish = (r: HerdrResult) => { if (settled) return; @@ -55,6 +60,7 @@ export function herdrRequest( } const tryParseBuffered = (): HerdrResult | undefined => { + const buf = Buffer.concat(chunks).toString("utf8"); const nl = buf.indexOf("\n"); if (nl < 0) return undefined; const text = buf.slice(0, nl); @@ -81,7 +87,7 @@ export function herdrRequest( socket.write(line); }, data(_socket, chunk) { - buf += chunk.toString(); + chunks.push(chunk); const parsed = tryParseBuffered(); if (parsed) finish(parsed); }, From d34dfbf4cf275097938fe7774707b4b74fe3a246 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:07:30 -0500 Subject: [PATCH 028/142] daemon: install stderr redirect + crash handlers before every module-scope side effect Hoists redirectNativeStderr() to the first executable statement and installCrashHandlers() to right after the logger resolves, both before createEventsBus, cron, sweep timers, and home-snapshot construction. A pre-startDaemon throw (e.g. a corrupt events.db) now lands in daemon-stderr.log instead of vanishing down a discarded fd 2. Co-Authored-By: Claude Opus 4.8 (1M context) --- e2e/tests/daemon.test.ts | 25 ++++++++++++++++++++++++- lib/daemon.ts | 19 +++++++++++++------ 2 files changed, 37 insertions(+), 7 deletions(-) diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index 37982958..5bccbdfe 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from "bun:test"; -import { existsSync } from "fs"; +import { existsSync, mkdirSync, writeFileSync, readdirSync } from "fs"; import { join } from "path"; import { createTestHome, rt } from "../harness.ts"; @@ -19,6 +19,29 @@ describe("fatal boot", () => { cleanup(); } }, 60_000); + + test("a corrupt events.db does not crash the daemon silently — error is captured", async () => { + const { path: home, cleanup } = createTestHome(); + try { + // Pre-create a corrupt events.db in the isolated HOME, before the + // daemon ever runs — createEventsBus (module scope) throws opening it. + const rtDir = join(home, ".mattstack", "rt"); + mkdirSync(rtDir, { recursive: true }); + writeFileSync(join(rtDir, "events.db"), "not a sqlite file at all"); + + const result = await rt(["--daemon"], { home }); + + expect(result.exitCode).not.toBe(0); + // Task 4 makes this self-heal (quarantine + retry); for Task 3 we only + // require the failure is captured, not that boot recovers. + const stderrLog = join(rtDir, "logs", "daemon-stderr.log"); + const quarantined = existsSync(rtDir) && readdirSync(rtDir).some((f) => f.startsWith("events.db.corrupt-")); + const captured = existsSync(stderrLog) || quarantined; + expect(captured).toBe(true); + } finally { + cleanup(); + } + }, 60_000); }); describe("daemon", () => { diff --git a/lib/daemon.ts b/lib/daemon.ts index d06c8a51..2dbd549c 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -76,6 +76,13 @@ import type { PortEntry } from "./port-scanner.ts"; // entry (cli.ts) also runs it, but `bun run lib/daemon.ts` skips cli.ts. import { migrateLegacyRtDir, LEGACY_RT_LABEL, RT_DIR_LABEL, logsDir } from "./rt-paths.ts"; +// Capture native panics (bypass JS entirely) at the fd level before anything +// else in this module runs, so a throw during any later module-scope +// construction (createEventsBus, cron, home-snapshot, …) lands in +// daemon-stderr.log instead of vanishing down whatever fd 2 the launcher gave +// us. Depends only on logsDir() and mkdirs its own dir; no logger needed yet. +redirectNativeStderr(); + // Gates installCrashHandlers' unhandledRejection handler: fatal during boot // (no socket/API bound yet, nothing to recover), advisory-only once ready. let bootPhase: "booting" | "ready" = "booting"; @@ -90,6 +97,12 @@ const rtMigration = migrateLegacyRtDir(); const loggerHandle = await getDaemonLogger(); const log = loggerHandle.logger; +// Wire uncaughtException + unhandledRejection through pino as early as the +// logger allows: every module-scope side effect below this point +// (createEventsBus, cron, home-snapshot, sweep timers) can throw, and this +// must run BEFORE any of it does. +installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" }); + if (rtMigration === "migrated") { log.info(`migrated legacy ${LEGACY_RT_LABEL} state to ${RT_DIR_LABEL}`); } else if (rtMigration === "conflict") { @@ -391,12 +404,6 @@ async function runDaemon(): Promise { try { mkdirSync(RT_DIR, { recursive: true }); - // Capture native panics (bypass JS entirely) at the fd level, then wire - // uncaughtException + unhandledRejection through pino. Must run BEFORE - // any async work that could throw uncaught. - redirectNativeStderr(); - installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" }); - // If a previous daemon process is still alive (orphan from a failed // restart), evict it before we bind the socket. evictStaleDaemon(log); From 43c5486c8bfa248a00b26ab8679de771bd89ea07 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:07:37 -0500 Subject: [PATCH 029/142] port-scanner: canonicalize repo/worktree paths once per scan so a symlinked or case-variant root matches lsof (S097) --- lib/__tests__/port-scanner.test.ts | 64 ++++++++++++++++++++++++++++++ lib/port-scanner.ts | 47 +++++++++++++++++++--- 2 files changed, 106 insertions(+), 5 deletions(-) diff --git a/lib/__tests__/port-scanner.test.ts b/lib/__tests__/port-scanner.test.ts index 750c574b..033c0ada 100644 --- a/lib/__tests__/port-scanner.test.ts +++ b/lib/__tests__/port-scanner.test.ts @@ -1,10 +1,16 @@ import { describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; import { parseListeningLsof, parsePidValueMap, parseCwdMap, parseWorktreePorcelain, parseEtimeMs, + matchCwdToRepo, + canonicalizeRepoIndex, + canonicalizeWorktreeMap, } from "../port-scanner.ts"; describe("parseListeningLsof", () => { @@ -120,3 +126,61 @@ describe("parseEtimeMs", () => { expect(parseEtimeMs("")).toBeNull(); }); }); + +// S097: lsof reports the kernel's resolved (real) cwd; the repo index and +// worktree map carry whatever path the user cd'd through when registering +// a repo. A symlinked repo root (~/code -> /Volumes/Dev/code) previously +// matched nothing at all — no ports, no runaway detection, and stale dev +// servers kept running past dispose because killWorktreeProcesses found +// nothing either. +describe("canonicalizeRepoIndex / canonicalizeWorktreeMap (S097)", () => { + let dir: string; + let real: string; + let link: string; + + const setup = () => { + dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-port-scanner-"))); + real = join(dir, "real-repo"); + link = join(dir, "linked-repo"); + mkdirSync(real, { recursive: true }); + symlinkSync(real, link); + }; + const teardown = () => { rmSync(dir, { recursive: true, force: true }); }; + + test("a symlinked repo root resolves to the real path lsof would report", () => { + setup(); + try { + const repos = canonicalizeRepoIndex({ acme: link }); + expect(repos.acme).toBe(realpathSync(real)); + } finally { + teardown(); + } + }); + + test("a nonexistent path (deleted mid-scan) falls back to the literal rather than throwing", () => { + const repos = canonicalizeRepoIndex({ acme: "/does/not/exist/anywhere" }); + expect(repos.acme).toBe("/does/not/exist/anywhere"); + }); + + test("worktree map keys are canonicalized the same way", () => { + setup(); + try { + const wt = canonicalizeWorktreeMap(new Map([[link, { repo: "acme", branch: "main" }]])); + expect([...wt.keys()]).toEqual([realpathSync(real)]); + } finally { + teardown(); + } + }); + + test("end-to-end: matchCwdToRepo finds a repo registered under a symlinked root once canonicalized", () => { + setup(); + try { + const cwd = realpathSync(real); // what lsof would report + const repos = canonicalizeRepoIndex({ acme: link }); // what the index has + const match = matchCwdToRepo(cwd, repos, new Map()); + expect(match.repo).toBe("acme"); + } finally { + teardown(); + } + }); +}); diff --git a/lib/port-scanner.ts b/lib/port-scanner.ts index c437b620..8ff5b4d9 100644 --- a/lib/port-scanner.ts +++ b/lib/port-scanner.ts @@ -10,7 +10,7 @@ * would freeze the event loop long enough to time out status polls. */ -import { existsSync } from "fs"; +import { existsSync, realpathSync } from "fs"; import { homedir } from "os"; import { loadRepoIndex as loadRepoIndexFromStore } from "./repo-index.ts"; import { runCapture } from "./subprocess.ts"; @@ -167,6 +167,39 @@ export async function buildWorktreeMap( return new Map(perRepo.flat()); } +/** + * lsof reports the kernel's resolved (real, canonically-cased) path for a + * process's cwd. The repo index and worktree map instead carry whatever + * path the user cd'd through when the repo was registered (behind a + * symlink, under a case-insensitive APFS volume's non-canonical spelling, + * ...), so a raw string comparison against lsof's cwd can miss even a repo + * that is genuinely running. Falls back to the literal path when it does + * not (yet) exist on disk — mirrors lib/runs/prune.ts's canon(). + */ +function canon(path: string): string { + try { + return realpathSync(path); + } catch { + return path; + } +} + +/** Canonicalizes every path in the repo index once per scan (not once per matched port). */ +export function canonicalizeRepoIndex(repos: Record): Record { + const out: Record = {}; + for (const [name, path] of Object.entries(repos)) out[name] = canon(path); + return out; +} + +/** Canonicalizes every worktree path's key once per scan (not once per matched port). */ +export function canonicalizeWorktreeMap( + worktreeMap: Map, +): Map { + const out = new Map(); + for (const [path, info] of worktreeMap) out.set(canon(path), info); + return out; +} + export function matchCwdToRepo( cwd: string, repos: Record, @@ -215,13 +248,17 @@ export function matchCwdToRepo( * ports whose process CWD matches a known repo. */ export async function scanListeningPorts(): Promise { - const repos = loadRepoIndex(); - if (Object.keys(repos).length === 0) return []; + const rawRepos = loadRepoIndex(); + if (Object.keys(rawRepos).length === 0) return []; - const [worktreeMap, listenersRes] = await Promise.all([ - buildWorktreeMap(repos), + const [rawWorktreeMap, listenersRes] = await Promise.all([ + buildWorktreeMap(rawRepos), runCapture(["lsof", "-iTCP", "-sTCP:LISTEN", "-P", "-n"], { timeoutMs: 10_000 }), ]); + // Once per scan, not once per matched port: lsof's cwd is already real, + // so the index/worktree side is what needs canonicalizing to match it. + const repos = canonicalizeRepoIndex(rawRepos); + const worktreeMap = canonicalizeWorktreeMap(rawWorktreeMap); const listeners = parseListeningLsof(listenersRes.stdout); if (listeners.length === 0) return []; From ff751ee5ac657fecf8a4b54d42cda9543951504e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:09:38 -0500 Subject: [PATCH 030/142] rt-paths: only migrate a legacy ~/.rt that carries an actual rt signature (S099) --- lib/__tests__/rt-paths.test.ts | 48 ++++++++++++++++++++++++++++++++++ lib/rt-paths.ts | 24 ++++++++++++++--- 2 files changed, 69 insertions(+), 3 deletions(-) diff --git a/lib/__tests__/rt-paths.test.ts b/lib/__tests__/rt-paths.test.ts index 50cf38a6..665eafdb 100644 --- a/lib/__tests__/rt-paths.test.ts +++ b/lib/__tests__/rt-paths.test.ts @@ -228,6 +228,7 @@ describe("rt-paths", () => { const home = makeHome(); process.env.HOME = home; mkdirSync(join(home, ".rt"), { recursive: true }); + writeFileSync(join(home, ".rt", "repos.json"), "{}"); // rt signature marker (S099) writeFileSync(join(home, ".rt", "old.json"), "old"); mkdirSync(join(home, ".mattstack", "rt"), { recursive: true }); writeFileSync(join(home, ".mattstack", "rt", "new.json"), "new"); @@ -241,11 +242,58 @@ describe("rt-paths", () => { const home = makeHome(); process.env.HOME = home; mkdirSync(join(home, ".rt"), { recursive: true }); + writeFileSync(join(home, ".rt", "repos.json"), "{}"); // rt signature marker (S099) expect(migrateLegacyRtDir()).toBe("migrated"); expect(migrateLegacyRtDir()).toBe("none"); rmSync(home, { recursive: true, force: true }); }); + // S099: a new user who has never run rt, but has another tool that also + // uses ~/.rt as its config dir, must not have that directory silently + // annexed as rt state (parsed, quarantined, or renamed) — only a + // directory carrying an actual rt signature is ever touched. + test("migrate: a foreign ~/.rt with no rt signature is left alone, no ~/.mattstack/rt materializes (S099)", () => { + const home = makeHome(); + process.env.HOME = home; + mkdirSync(join(home, ".rt"), { recursive: true }); + writeFileSync(join(home, ".rt", "config.toml"), "some-other-tools-config"); + const result = migrateLegacyRtDir(); + expect(result).not.toBe("migrated"); + expect(result).not.toBe("conflict"); + expect(readFileSync(join(home, ".rt", "config.toml"), "utf8")).toBe("some-other-tools-config"); + expect(existsSync(join(home, ".mattstack", "rt"))).toBe(false); + rmSync(home, { recursive: true, force: true }); + }); + + test("migrate: a foreign ~/.rt is left alone even when ~/.mattstack/rt already exists — not reported as a conflict (S099)", () => { + const home = makeHome(); + process.env.HOME = home; + mkdirSync(join(home, ".rt"), { recursive: true }); + writeFileSync(join(home, ".rt", "config.toml"), "some-other-tools-config"); + mkdirSync(join(home, ".mattstack", "rt"), { recursive: true }); + const result = migrateLegacyRtDir(); + expect(result).not.toBe("conflict"); + expect(readFileSync(join(home, ".rt", "config.toml"), "utf8")).toBe("some-other-tools-config"); + rmSync(home, { recursive: true, force: true }); + }); + + test("migrate: a bare 'logs' dir alone is a sufficient rt signature", () => { + const home = makeHome(); + process.env.HOME = home; + mkdirSync(join(home, ".rt", "logs"), { recursive: true }); + expect(migrateLegacyRtDir()).toBe("migrated"); + rmSync(home, { recursive: true, force: true }); + }); + + test("migrate: a bare 'state.db' file alone is a sufficient rt signature", () => { + const home = makeHome(); + process.env.HOME = home; + mkdirSync(join(home, ".rt"), { recursive: true }); + writeFileSync(join(home, ".rt", "state.db"), ""); + expect(migrateLegacyRtDir()).toBe("migrated"); + rmSync(home, { recursive: true, force: true }); + }); + // ── legacyDirsPresent (the canary probe) ──────────────────────────────────── test("canary: reports real legacy dirs, ignores symlinks, skips absent", () => { diff --git a/lib/rt-paths.ts b/lib/rt-paths.ts index 3fc07fa5..ce92b121 100644 --- a/lib/rt-paths.ts +++ b/lib/rt-paths.ts @@ -280,6 +280,20 @@ export type LegacyMigrationResult = "none" | "migrated" | "conflict"; export const LEGACY_RT_LABEL = "~/.rt"; export const RT_DIR_LABEL = "~/.mattstack/rt"; +/** + * Entries whose presence at the top of ~/.rt proves rt itself wrote it — + * not merely that something happens to live at that path. A new user who + * has never run an old rt but has an unrelated tool using ~/.rt as ITS + * config dir must never have that directory silently annexed as rt state + * (renamed, then parsed/quarantined as daemon.json/repos.json/state.db). + */ +const RT_SIGNATURE_ENTRIES = ["state.db", "logs", "repos.json"]; + +/** Whether `dir` carries an actual rt signature, not merely a directory of the same name. */ +function hasRtSignature(dir: string): boolean { + return RT_SIGNATURE_ENTRIES.some((name) => existsSync(join(dir, name))); +} + /** * One-shot migration of a real legacy ~/.rt directory to ~/.mattstack/rt. * Called early from the CLI entry and daemon boot — BEFORE anything (loggers @@ -287,9 +301,12 @@ export const RT_DIR_LABEL = "~/.mattstack/rt"; * ~/.rt would land in "conflict" instead of migrating. * * - ~/.rt absent, or a symlink (the RT-33 compat shim): "none", untouched. - * - real ~/.rt, no ~/.mattstack/rt: rename it into place → "migrated". - * - real ~/.rt AND ~/.mattstack/rt both exist: "conflict" — state is split - * and a human must merge; nothing is touched. + * - ~/.rt real but carries no rt signature (another tool's config dir of + * the same name): "none", untouched — never renamed, parsed, or reported + * as a conflict against ~/.mattstack/rt. + * - real ~/.rt (rt's), no ~/.mattstack/rt: rename it into place → "migrated". + * - real ~/.rt (rt's) AND ~/.mattstack/rt both exist: "conflict" — state is + * split and a human must merge; nothing is touched. */ export function migrateLegacyRtDir(): LegacyMigrationResult { const legacy = legacyRtDir(); @@ -300,6 +317,7 @@ export function migrateLegacyRtDir(): LegacyMigrationResult { return "none"; // no ~/.rt at all } if (legacyStat.isSymbolicLink() || !legacyStat.isDirectory()) return "none"; + if (!hasRtSignature(legacy)) return "none"; // real directory, but not rt's — never touch it const target = rtDir(); try { From 8e689b06f0c52b9f45f04720af75e6ec393e5fac Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:11:25 -0500 Subject: [PATCH 031/142] daemon: fix needsToken to never gate OPTIONS preflight (review fix) Splitting OPTIONS out of the GET/HEAD bucket restores the pre-existing guarantee that a CORS preflight is never gated, regardless of path, matching this file's own docblock and existing test coverage. --- lib/daemon/__tests__/api-auth.test.ts | 5 +++++ lib/daemon/api-auth.ts | 12 +++++++----- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/lib/daemon/__tests__/api-auth.test.ts b/lib/daemon/__tests__/api-auth.test.ts index 8f8e102e..6abf9eab 100644 --- a/lib/daemon/__tests__/api-auth.test.ts +++ b/lib/daemon/__tests__/api-auth.test.ts @@ -67,6 +67,11 @@ describe("needsToken", () => { expect(needsToken("GET", "/api/cache")).toBe(false); expect(needsToken("HEAD", "/api/repos")).toBe(false); }); + + test("OPTIONS never requires a token even for secrets/notifications (preflight must never be gated)", () => { + expect(needsToken("OPTIONS", "/api/secrets")).toBe(false); + expect(needsToken("OPTIONS", "/api/notifications")).toBe(false); + }); }); describe("tokenOk", () => { diff --git a/lib/daemon/api-auth.ts b/lib/daemon/api-auth.ts index acc67076..dae00efa 100644 --- a/lib/daemon/api-auth.ts +++ b/lib/daemon/api-auth.ts @@ -70,13 +70,15 @@ export function reloadApiToken(tokenPath: string = API_TOKEN_PATH): string { /** * True when a request mutates state, or (secrets/notifications) returns or * drains something a GET should not silently consume, and must present the - * local token. Default-gated for every method except GET/HEAD/OPTIONS (S040: - * an allowlist-by-path guaranteed the next mutating route would ship - * unguarded) plus two explicit GET exceptions whose verb lies about being a - * read. + * local token. A CORS preflight (OPTIONS) can never present the custom + * X-RT-Token header, so it is never gated, on any path. Otherwise + * default-gated for every method except GET/HEAD (S040: an allowlist-by-path + * guaranteed the next mutating route would ship unguarded), plus two + * explicit GET exceptions whose verb lies about being a read. */ export function needsToken(method: string, pathname: string): boolean { - if (method === "GET" || method === "HEAD" || method === "OPTIONS") { + if (method === "OPTIONS") return false; + if (method === "GET" || method === "HEAD") { // Gated despite being a GET: /api/secrets's response body IS a // credential (S054); /api/notifications DRAINS the queue (S041), so its // verb lies about being a read the way every other GET here is not. From e8d0a7c87c93b7f16ec00fd86f72b2e7ef1469bd Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:15:15 -0500 Subject: [PATCH 032/142] daemon-client: return timed-out/refused attribution per query instead of shared module flags (S081) --- .../daemon-client-attribution.test.ts | 57 +++++++++ lib/daemon-client.ts | 119 +++++++++++------- 2 files changed, 134 insertions(+), 42 deletions(-) create mode 100644 lib/__tests__/daemon-client-attribution.test.ts diff --git a/lib/__tests__/daemon-client-attribution.test.ts b/lib/__tests__/daemon-client-attribution.test.ts new file mode 100644 index 00000000..91fc5558 --- /dev/null +++ b/lib/__tests__/daemon-client-attribution.test.ts @@ -0,0 +1,57 @@ +/** + * S081: daemon-client's timed-out/refused attribution used to live in + * module-level flags (`_lastQueryTimedOut`/`_lastQueryWasRefused`) shared + * across every concurrent query. A fast query resolving in between a slow + * query's own trySocketQuery call and the moment its caller reads + * lastQueryTimedOut() could reset those flags out from under it — the slow + * query's caller would then see "daemon unavailable" for a call that + * actually just exceeded its own window. daemonQueryAttributed returns the + * failure kind alongside the response instead, so it can never be + * clobbered by an unrelated concurrent call. + */ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { DAEMON_SOCK_PATH, markDaemonInstalled, markDaemonUninstalled } from "../daemon-config.ts"; +import { daemonQueryAttributed } from "../daemon-client.ts"; + +describe("daemonQueryAttributed", () => { + let server: ReturnType; + + beforeEach(() => { + markDaemonInstalled(); + server = Bun.serve({ + unix: DAEMON_SOCK_PATH, + async fetch(req) { + const url = new URL(req.url); + if (url.pathname === "/slow") { + await new Promise(() => {}); // never resolves — the client's own AbortSignal.timeout fires + } + return new Response(JSON.stringify({ ok: true, data: {} }), { headers: { "Content-Type": "application/json" } }); + }, + }); + }); + + afterEach(() => { + server.stop(true); + markDaemonUninstalled(); + }); + + test("a slow query that times out reports timedOut:true even resolved after a concurrent fast success", async () => { + const slow = daemonQueryAttributed("slow", {}, 50); + const fast = await daemonQueryAttributed("fast", {}); + expect(fast.response?.ok).toBe(true); + expect(fast.timedOut).toBe(false); + + const slowResult = await slow; + expect(slowResult.response).toBeNull(); + expect(slowResult.timedOut).toBe(true); + expect(slowResult.refused).toBe(false); + }); + + test("a fast query's own success is reported correctly even started after a slow one is already in flight", async () => { + const slow = daemonQueryAttributed("slow", {}, 200); + const fast = await daemonQueryAttributed("fast", {}); + expect(fast.response?.ok).toBe(true); + expect(fast.timedOut).toBe(false); + await slow; // drain — this one times out at 200ms, don't leave it dangling + }); +}); diff --git a/lib/daemon-client.ts b/lib/daemon-client.ts index 2a03a629..0fca23a8 100644 --- a/lib/daemon-client.ts +++ b/lib/daemon-client.ts @@ -30,19 +30,32 @@ export interface DaemonResponse { const REQUEST_TIMEOUT_MS = 2000; +// Deprecated shim state (see lastQueryTimedOut's doc): kept in sync so an +// external caller still reading it via the module-level accessor keeps +// working, but nothing in this file reads these two vars anymore — every +// internal caller carries its own attribution on its own return value +// instead, which a concurrent query can never clobber. let _lastQueryWasRefused = false; let _lastQueryTimedOut = false; +interface SocketQueryAttempt { + response: DaemonResponse | null; + /** True on ECONNREFUSED — the socket is stale, not merely slow. */ + refused: boolean; + /** True when the request itself exceeded timeoutMs. */ + timedOut: boolean; +} + async function trySocketQuery( cmd: string, payload?: Record, timeoutMs: number = REQUEST_TIMEOUT_MS, -): Promise { - // Reset per-query flags up front so a missing-socket early return doesn't - // leave a previous query's timeout/refused state visible to callers. - _lastQueryWasRefused = false; - _lastQueryTimedOut = false; - if (!existsSync(DAEMON_SOCK_PATH)) return null; +): Promise { + if (!existsSync(DAEMON_SOCK_PATH)) { + _lastQueryWasRefused = false; + _lastQueryTimedOut = false; + return { response: null, refused: false, timedOut: false }; + } try { const hasBody = payload && Object.keys(payload).length > 0; @@ -57,14 +70,16 @@ async function trySocketQuery( _lastQueryWasRefused = false; _lastQueryTimedOut = false; - return (await response.json()) as DaemonResponse; + return { response: (await response.json()) as DaemonResponse, refused: false, timedOut: false }; } catch (err) { const code = (err as any)?.code ?? ""; const name = (err as any)?.name ?? ""; const msg = err instanceof Error ? err.message : ""; - _lastQueryWasRefused = code === "ECONNREFUSED" || msg.includes("ECONNREFUSED") || msg.includes("Connection refused"); - _lastQueryTimedOut = name === "TimeoutError" || name === "AbortError" || msg.includes("timed out"); - return null; + const refused = code === "ECONNREFUSED" || msg.includes("ECONNREFUSED") || msg.includes("Connection refused"); + const timedOut = name === "TimeoutError" || name === "AbortError" || msg.includes("timed out"); + _lastQueryWasRefused = refused; + _lastQueryTimedOut = timedOut; + return { response: null, refused, timedOut }; } } @@ -102,7 +117,7 @@ export async function daemonSocketQuery( payload?: Record, timeoutMs?: number, ): Promise { - return trySocketQuery(cmd, payload, timeoutMs); + return (await trySocketQuery(cmd, payload, timeoutMs)).response; } // ─── Tray request client (MAT-383 setup verbs) ─────────────────────────────── @@ -205,48 +220,68 @@ export function suppressDaemonDownWarning(): void { // ─── Public API ────────────────────────────────────────────────────────────── /** - * Send a command to the daemon and return the response. - * - * Returns null if daemon is not available (either not installed or not running - * and can't be auto-restarted). Callers should fall back to direct execution. + * Same contract as `daemonQuery`, but the failure kind travels on the + * return value instead of the module-level `_lastQuery*` flags — so a + * concurrent query (a 30s mr:action merge racing a 2s status poll, say) + * can never clobber this call's own attribution between its trySocketQuery + * resolving and its caller reading it. Prefer this over + * `daemonQuery` + `lastQueryTimedOut()` for any new caller. */ -export async function daemonQuery( +export async function daemonQueryAttributed( cmd: string, payload?: Record, timeoutMs?: number, -): Promise { +): Promise { // 1. Try HTTP request over Unix socket - const result = await trySocketQuery(cmd, payload, timeoutMs); - if (result !== null) return result; + const first = await trySocketQuery(cmd, payload, timeoutMs); + if (first.response !== null) return first; // 2. Check if user opted in - if (!isDaemonInstalled()) return null; // not installed → silent fallback + if (!isDaemonInstalled()) return { response: null, refused: false, timedOut: false }; // not installed → silent fallback // 3. If the socket file still exists AND the connection wasn't refused, // the daemon IS running — this query timed out or hit a transient error. // Return null silently. But if the connection was refused, the socket is // stale (daemon died without cleaning up) — fall through to attempt restart. - if (existsSync(DAEMON_SOCK_PATH) && !_lastQueryWasRefused) return null; + if (existsSync(DAEMON_SOCK_PATH) && !first.refused) return first; // 4. Socket is gone → daemon is genuinely not running. Attempt restart. const restarted = await attemptRestart(); + let last = first; if (restarted) { // Retry once after short delay await Bun.sleep(300); - const retryResult = await trySocketQuery(cmd, payload, timeoutMs); - if (retryResult !== null) return retryResult; + const retry = await trySocketQuery(cmd, payload, timeoutMs); + if (retry.response !== null) return retry; + last = retry; } // 5. Restart failed → warn (once per session) warnDaemonDown(); - return null; + return last; +} + +/** + * Send a command to the daemon and return the response. + * + * Returns null if daemon is not available (either not installed or not running + * and can't be auto-restarted). Callers should fall back to direct execution. + */ +export async function daemonQuery( + cmd: string, + payload?: Record, + timeoutMs?: number, +): Promise { + return (await daemonQueryAttributed(cmd, payload, timeoutMs)).response; } /** * True if the last `daemonQuery` returned null because the request timed out - * (as opposed to the daemon being genuinely down). Used by action callers so - * they can surface "timed out — verify on GitLab" instead of the misleading - * "daemon unavailable". + * (as opposed to the daemon being genuinely down). Deprecated: still backed + * by shared module state, so it remains vulnerable to the exact + * cross-query attribution race `daemonQueryAttributed` closes — kept only + * for callers that predate that function. New callers should use + * `daemonQueryAttributed` and read `.timedOut` off their own result. */ export function lastQueryTimedOut(): boolean { return _lastQueryTimedOut; @@ -256,7 +291,7 @@ export function lastQueryTimedOut(): boolean { * Quick check: is the daemon reachable right now? */ export async function isDaemonRunning(): Promise { - const response = await trySocketQuery("ping"); + const { response } = await trySocketQuery("ping"); return response?.ok === true; } @@ -294,8 +329,8 @@ const MR_ACTION_TIMEOUT_MS = 30_000; export function mrActions(repoName: string, iid: number): DaemonMRActions { const fire = async (action: string, args: any[] = []): Promise => { - const res = await daemonQuery("mr:action", { repoName, iid, action, args }, MR_ACTION_TIMEOUT_MS); - if (!res) throw new Error(lastQueryTimedOut() ? `${action} timed out — verify on GitLab` : "daemon unavailable"); + const { response: res, timedOut } = await daemonQueryAttributed("mr:action", { repoName, iid, action, args }, MR_ACTION_TIMEOUT_MS); + if (!res) throw new Error(timedOut ? `${action} timed out — verify on GitLab` : "daemon unavailable"); if (!res.ok) throw new Error(res.error || `${action} failed`); }; @@ -312,14 +347,14 @@ export function mrActions(repoName: string, iid: number): DaemonMRActions { requestReReview: (uid) => fire("requestReReview", [uid]), fetchJobDetail: async (jobId, pipelineId) => { - const res = await daemonQuery("mr:fetch-job-detail", { repoName, iid, jobId, pipelineId }, MR_ACTION_TIMEOUT_MS); - if (!res) throw new Error(lastQueryTimedOut() ? "fetchJobDetail timed out" : "daemon unavailable"); + const { response: res, timedOut } = await daemonQueryAttributed("mr:fetch-job-detail", { repoName, iid, jobId, pipelineId }, MR_ACTION_TIMEOUT_MS); + if (!res) throw new Error(timedOut ? "fetchJobDetail timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "fetchJobDetail failed"); return res.data; }, fetchJobTrace: async (jobId) => { - const res = await daemonQuery("mr:fetch-job-trace", { repoName, iid, jobId }, MR_ACTION_TIMEOUT_MS); - if (!res) throw new Error(lastQueryTimedOut() ? "fetchJobTrace timed out" : "daemon unavailable"); + const { response: res, timedOut } = await daemonQueryAttributed("mr:fetch-job-trace", { repoName, iid, jobId }, MR_ACTION_TIMEOUT_MS); + if (!res) throw new Error(timedOut ? "fetchJobTrace timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "fetchJobTrace failed"); return res.data as string; }, @@ -352,12 +387,12 @@ export async function fetchDiscussions( iid: number, opts?: { force?: boolean }, ): Promise { - const res = await daemonQuery( + const { response: res, timedOut } = await daemonQueryAttributed( "discussions:read", { repoName, iid, force: opts?.force === true }, DISCUSSIONS_TIMEOUT_MS, ); - if (!res) throw new Error(lastQueryTimedOut() ? "discussions timed out" : "daemon unavailable"); + if (!res) throw new Error(timedOut ? "discussions timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "discussions:read failed"); return res.data as DiscussionsSnapshot; } @@ -369,12 +404,12 @@ export async function setDiscussionResolved( discussionId: string, resolved: boolean, ): Promise { - const res = await daemonQuery( + const { response: res, timedOut } = await daemonQueryAttributed( "discussions:resolve", { repoName, iid, discussionId, resolved }, DISCUSSIONS_TIMEOUT_MS, ); - if (!res) throw new Error(lastQueryTimedOut() ? "resolve timed out" : "daemon unavailable"); + if (!res) throw new Error(timedOut ? "resolve timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "discussions:resolve failed"); return res.data as DiscussionsSnapshot; } @@ -384,12 +419,12 @@ export async function fetchMRDiffs( repoName: string, iid: number, ): Promise> { - const res = await daemonQuery( + const { response: res, timedOut } = await daemonQueryAttributed( "discussions:diffs", { repoName, iid }, DISCUSSIONS_TIMEOUT_MS, ); - if (!res) throw new Error(lastQueryTimedOut() ? "diffs timed out" : "daemon unavailable"); + if (!res) throw new Error(timedOut ? "diffs timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "discussions:diffs failed"); return (res.data as { diffs: Array<{ newPath: string; diff: string }> }).diffs; } @@ -401,12 +436,12 @@ export async function replyToDiscussion( discussionId: string, body: string, ): Promise { - const res = await daemonQuery( + const { response: res, timedOut } = await daemonQueryAttributed( "discussions:reply", { repoName, iid, discussionId, body }, DISCUSSIONS_TIMEOUT_MS, ); - if (!res) throw new Error(lastQueryTimedOut() ? "reply timed out" : "daemon unavailable"); + if (!res) throw new Error(timedOut ? "reply timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "discussions:reply failed"); return res.data as DiscussionsSnapshot; } From 44485548f863a923362da28ef64f130af5eed6a1 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:15:48 -0500 Subject: [PATCH 033/142] daemon: default-deny CORS and gate /ws on origin/token (S005/S006) --- .../__tests__/api-server-cors-ws.test.ts | 26 +++++++++++ lib/daemon/api-server.ts | 43 +++++++++++++++---- 2 files changed, 61 insertions(+), 8 deletions(-) create mode 100644 lib/daemon/__tests__/api-server-cors-ws.test.ts diff --git a/lib/daemon/__tests__/api-server-cors-ws.test.ts b/lib/daemon/__tests__/api-server-cors-ws.test.ts new file mode 100644 index 00000000..58c5bf8a --- /dev/null +++ b/lib/daemon/__tests__/api-server-cors-ws.test.ts @@ -0,0 +1,26 @@ +import { describe, test, expect } from "bun:test"; +import { buildCorsHeaders } from "../api-server.ts"; + +describe("buildCorsHeaders", () => { + test("no Origin header: no Access-Control-Allow-Origin is set (non-browser request, CORS is irrelevant)", () => { + const headers = buildCorsHeaders(null, true); + expect(headers["Access-Control-Allow-Origin"]).toBeUndefined(); + }); + + test("an untrusted Origin gets no Access-Control-Allow-Origin (default-deny, S006)", () => { + const headers = buildCorsHeaders("http://evil.example", false); + expect(headers["Access-Control-Allow-Origin"]).toBeUndefined(); + }); + + test("a trusted Origin is echoed back with Vary: Origin", () => { + const headers = buildCorsHeaders("http://localhost:5544", true); + expect(headers["Access-Control-Allow-Origin"]).toBe("http://localhost:5544"); + expect(headers["Vary"]).toBe("Origin"); + }); + + test("always advertises the methods/headers a preflight needs, trusted or not", () => { + const headers = buildCorsHeaders("http://evil.example", false); + expect(headers["Access-Control-Allow-Methods"]).toContain("POST"); + expect(headers["Access-Control-Allow-Headers"]).toContain("X-RT-Token"); + }); +}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index d61909fd..3ac823f8 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -10,7 +10,7 @@ import type { Server, ServerWebSocket } from "bun"; import type { Logger } from "pino"; import { API_PORT } from "../daemon-config.ts"; -import { needsToken, tokenOk, getApiToken } from "./api-auth.ts"; +import { needsToken, tokenOk, getApiToken, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "./api-auth.ts"; import { getAggregatedConnection } from "./freshness.ts"; import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; @@ -92,6 +92,25 @@ export function clearWsClients(): void { wsClients.clear(); } +/** + * CORS default-deny (S006): a browser page on an untrusted Origin still gets + * its request served (127.0.0.1 loopback + the per-route token gate are the + * real defenses), but the response carries no Access-Control-Allow-Origin, + * so the page's own JavaScript cannot read the body. A request with no + * Origin at all (every non-browser consumer today) needs no CORS headers. + */ +export function buildCorsHeaders(origin: string | null, trusted: boolean): Record { + const headers: Record = { + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type, X-RT-Token", + }; + if (origin && trusted) { + headers["Access-Control-Allow-Origin"] = origin; + headers["Vary"] = "Origin"; + } + return headers; +} + export interface ApiServerDeps { handleCommand: (cmd: string, payload: any, signal?: AbortSignal) => Promise; log: Logger; @@ -144,19 +163,27 @@ export async function startApiServer(deps: ApiServerDeps): Promise> maxRequestBodySize: MAX_REQUEST_BODY_SIZE, async fetch(req, server) { const url = new URL(req.url); + const origin = req.headers.get("origin"); + const allowedOrigins = getTrustedBrowserOrigins(); - // WebSocket upgrade — broadcast channel + // WebSocket upgrade (broadcast channel). Browsers cannot set custom + // headers on a WS handshake, so the token (when a browser page wants + // to identify itself) travels as a ?token= query param instead of + // X-RT-Token (S005). if (url.pathname === "/ws") { + const wsToken = url.searchParams.get("token"); + if (!isBrowserRequestTrusted(origin, wsToken, apiToken, allowedOrigins)) { + return new Response("origin not allowed", { status: 403 }); + } if (server.upgrade(req, { data: { kind: "broadcast" } })) return undefined as any; return new Response("WebSocket upgrade failed", { status: 400 }); } - // CORS headers for local dev - const corsHeaders = { - "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Methods": "GET, POST, OPTIONS", - "Access-Control-Allow-Headers": "Content-Type", - }; + // CORS: default-deny. A trusted Origin (token or allowlist) gets its + // Origin echoed back; anything else gets no Access-Control-Allow-Origin + // at all, so a malicious page's own JS cannot read the response (S006). + const trusted = isBrowserRequestTrusted(origin, req.headers.get("x-rt-token"), apiToken, allowedOrigins); + const corsHeaders = buildCorsHeaders(origin, trusted); if (req.method === "OPTIONS") { return new Response(null, { status: 204, headers: corsHeaders }); From 6db63b8afe8e2910f66440063d0aff5ebb797023 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:16:42 -0500 Subject: [PATCH 034/142] runCapture: race reads against the deadline so a pipe-holding grandchild can't wedge it (S023, S024) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/__tests__/subprocess.test.ts | 38 ++++++++++++++++++ lib/subprocess.ts | 51 ++++++++++++++++++------- packages/rt-client/src/settings/exec.ts | 51 ++++++++++++++++++------- 3 files changed, 114 insertions(+), 26 deletions(-) diff --git a/lib/__tests__/subprocess.test.ts b/lib/__tests__/subprocess.test.ts index 9390076c..9fb2725d 100644 --- a/lib/__tests__/subprocess.test.ts +++ b/lib/__tests__/subprocess.test.ts @@ -29,6 +29,44 @@ describe("runCapture env", () => { }); }); +describe("runCapture timeout enforcement", () => { + test("resolves within the deadline even when a grandchild holds the pipe", async () => { + // zsh exits after ~0.2s, but backgrounds `sleep 20` which inherits stdout. + const t0 = Date.now(); + const r = await runCapture( + ["/bin/zsh", "-c", "sleep 20 & echo started; sleep 0.2"], + { timeoutMs: 1000 }, + ); + const elapsed = Date.now() - t0; + expect(elapsed).toBeLessThan(4000); // must NOT wait for the 20s grandchild + expect(r.timedOut).toBe(true); + expect(r.exitCode).toBe(-1); + }); + + test("a SIGTERM-ignoring child is bounded by SIGKILL escalation", async () => { + const t0 = Date.now(); + const r = await runCapture( + ["/bin/zsh", "-c", "trap '' TERM; sleep 20"], + { timeoutMs: 800 }, + ); + expect(Date.now() - t0).toBeLessThan(4000); + expect(r.timedOut).toBe(true); + }); + + test("normal fast command still returns real stdout and exitCode 0", async () => { + const r = await runCapture(["/bin/echo", "hello"], { timeoutMs: 5000 }); + expect(r.stdout.trim()).toBe("hello"); + expect(r.exitCode).toBe(0); + expect(r.timedOut).toBeUndefined(); + }); + + test("timed-out call reports exitCode -1 so callers treat it as failure", async () => { + const r = await runCapture(["/bin/sleep", "20"], { timeoutMs: 500 }); + expect(r.exitCode).toBe(-1); + expect(r.timedOut).toBe(true); + }); +}); + describe("outputTail", () => { test("passes short output through, trimmed", () => { expect(outputTail(" env: node: No such file or directory\n", 2000)) diff --git a/lib/subprocess.ts b/lib/subprocess.ts index afc62be9..9d3b23e2 100644 --- a/lib/subprocess.ts +++ b/lib/subprocess.ts @@ -10,6 +10,8 @@ export interface RunResult { stdout: string; stderr: string; exitCode: number; + /** Set true only when the deadline fired before the child settled. */ + timedOut?: boolean; } /** Longest slice of a failed step's output carried into a log line. */ @@ -57,21 +59,44 @@ export async function runCapture( return { stdout: "", stderr: "", exitCode: -1 }; } - const timer = setTimeout(() => { - try { proc.kill(); } catch { /* already exited */ } - }, opts.timeoutMs ?? 10_000); + const timeoutMs = opts.timeoutMs ?? 10_000; + // SIGTERM at the deadline, SIGKILL a short grace later. A child that ignores + // SIGTERM (or a D-state descendant) cannot be reaped in-band, so the read is + // raced against the deadline below rather than awaited unconditionally: that + // is what lets runCapture settle while a grandchild still holds the pipe. + let killTimer: ReturnType | undefined; + const term = setTimeout(() => { + try { proc.kill("SIGTERM"); } catch { /* already exited */ } + killTimer = setTimeout(() => { + try { proc.kill("SIGKILL"); } catch { /* already exited */ } + }, 2000); + }, timeoutMs); + + const captured: Promise = (async () => { + try { + const stdoutPromise = new Response(proc.stdout as ReadableStream).text(); + const stderrPromise = captureStderr + ? new Response(proc.stderr as ReadableStream).text() + : Promise.resolve(""); + const [stdout, stderr, exitCode] = await Promise.all([ + stdoutPromise, + stderrPromise, + proc.exited, + ]); + return { stdout, stderr, exitCode }; + } catch { + return { stdout: "", stderr: "", exitCode: -1 }; + } + })(); + + const deadline: Promise = new Promise((resolve) => { + setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); + }); try { - const stdoutPromise = new Response(proc.stdout as ReadableStream).text(); - const stderrPromise = captureStderr - ? new Response(proc.stderr as ReadableStream).text() - : Promise.resolve(""); - const [stdout, stderr] = await Promise.all([stdoutPromise, stderrPromise]); - const exitCode = await proc.exited; - return { stdout, stderr, exitCode }; - } catch { - return { stdout: "", stderr: "", exitCode: -1 }; + return await Promise.race([captured, deadline]); } finally { - clearTimeout(timer); + clearTimeout(term); + if (killTimer) clearTimeout(killTimer); } } diff --git a/packages/rt-client/src/settings/exec.ts b/packages/rt-client/src/settings/exec.ts index b5c68b07..cedc8e12 100644 --- a/packages/rt-client/src/settings/exec.ts +++ b/packages/rt-client/src/settings/exec.ts @@ -12,6 +12,8 @@ export interface RunResult { stdout: string; stderr: string; exitCode: number; + /** Set true only when the deadline fired before the child settled. */ + timedOut?: boolean; } /** @@ -47,21 +49,44 @@ export async function runCapture( return { stdout: "", stderr: "", exitCode: -1 }; } - const timer = setTimeout(() => { - try { proc.kill(); } catch { /* already exited */ } - }, opts.timeoutMs ?? 10_000); + const timeoutMs = opts.timeoutMs ?? 10_000; + // SIGTERM at the deadline, SIGKILL a short grace later. A child that ignores + // SIGTERM (or a D-state descendant) cannot be reaped in-band, so the read is + // raced against the deadline below rather than awaited unconditionally: that + // is what lets runCapture settle while a grandchild still holds the pipe. + let killTimer: ReturnType | undefined; + const term = setTimeout(() => { + try { proc.kill("SIGTERM"); } catch { /* already exited */ } + killTimer = setTimeout(() => { + try { proc.kill("SIGKILL"); } catch { /* already exited */ } + }, 2000); + }, timeoutMs); + + const captured: Promise = (async () => { + try { + const stdoutPromise = new Response(proc.stdout as ReadableStream).text(); + const stderrPromise = captureStderr + ? new Response(proc.stderr as ReadableStream).text() + : Promise.resolve(""); + const [stdout, stderr, exitCode] = await Promise.all([ + stdoutPromise, + stderrPromise, + proc.exited, + ]); + return { stdout, stderr, exitCode }; + } catch { + return { stdout: "", stderr: "", exitCode: -1 }; + } + })(); + + const deadline: Promise = new Promise((resolve) => { + setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); + }); try { - const stdoutPromise = new Response(proc.stdout as ReadableStream).text(); - const stderrPromise = captureStderr - ? new Response(proc.stderr as ReadableStream).text() - : Promise.resolve(""); - const [stdout, stderr] = await Promise.all([stdoutPromise, stderrPromise]); - const exitCode = await proc.exited; - return { stdout, stderr, exitCode }; - } catch { - return { stdout: "", stderr: "", exitCode: -1 }; + return await Promise.race([captured, deadline]); } finally { - clearTimeout(timer); + clearTimeout(term); + if (killTimer) clearTimeout(killTimer); } } From 59fc77a222df90d4ea4553a8c1093ccaff1a2ad1 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:16:45 -0500 Subject: [PATCH 035/142] daemon-client: bounded-poll rt.sock after a restart instead of one 300ms retry (S082) --- .../daemon-client-attribution.test.ts | 49 ++++++++++++++++++- lib/daemon-client.ts | 32 +++++++++++- 2 files changed, 78 insertions(+), 3 deletions(-) diff --git a/lib/__tests__/daemon-client-attribution.test.ts b/lib/__tests__/daemon-client-attribution.test.ts index 91fc5558..7d087d1e 100644 --- a/lib/__tests__/daemon-client-attribution.test.ts +++ b/lib/__tests__/daemon-client-attribution.test.ts @@ -10,7 +10,7 @@ * clobbered by an unrelated concurrent call. */ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { DAEMON_SOCK_PATH, markDaemonInstalled, markDaemonUninstalled } from "../daemon-config.ts"; +import { DAEMON_SOCK_PATH, TRAY_SOCK_PATH, markDaemonInstalled, markDaemonUninstalled } from "../daemon-config.ts"; import { daemonQueryAttributed } from "../daemon-client.ts"; describe("daemonQueryAttributed", () => { @@ -55,3 +55,50 @@ describe("daemonQueryAttributed", () => { await slow; // drain — this one times out at 200ms, don't leave it dangling }); }); + +// S082: auto-start retried the real query exactly once, 300ms after asking +// the tray to start the daemon — but parkUntilIntended's own socket probe, +// state.db open, and the identity migration routinely take longer than +// that, so a start that genuinely succeeds still gets reported as +// "installed but not running". +describe("auto-start bounded poll (S082)", () => { + const origRtAppSocket = process.env.RT_APP_SOCKET; + let daemonServer: ReturnType | undefined; + let traySock: string | undefined; + let trayServer: ReturnType | undefined; + let bindTimer: ReturnType | undefined; + + beforeEach(() => { + markDaemonInstalled(); + }); + + afterEach(() => { + if (bindTimer) clearTimeout(bindTimer); + daemonServer?.stop(true); + trayServer?.stop(true); + markDaemonUninstalled(); + if (origRtAppSocket === undefined) delete process.env.RT_APP_SOCKET; + else process.env.RT_APP_SOCKET = origRtAppSocket; + }); + + test("a restart that takes longer than 300ms to bind its socket is still picked up, not reported down", async () => { + // attemptRestart() reads the fixed TRAY_SOCK_PATH (not RT_APP_SOCKET), + // so the fake tray must listen there. + trayServer = Bun.serve({ + unix: TRAY_SOCK_PATH, + fetch: () => new Response(JSON.stringify({ ok: true }), { headers: { "Content-Type": "application/json" } }), + }); + + // No real daemon socket exists yet — simulates the gap between the tray + // accepting /daemon/start and the daemon actually binding rt.sock. + bindTimer = setTimeout(() => { + daemonServer = Bun.serve({ + unix: DAEMON_SOCK_PATH, + fetch: () => new Response(JSON.stringify({ ok: true, data: {} }), { headers: { "Content-Type": "application/json" } }), + }); + }, 600); + + const result = await daemonQueryAttributed("ping", {}, 200); + expect(result.response?.ok).toBe(true); + }, 5000); +}); diff --git a/lib/daemon-client.ts b/lib/daemon-client.ts index 0fca23a8..41b9739b 100644 --- a/lib/daemon-client.ts +++ b/lib/daemon-client.ts @@ -199,6 +199,31 @@ async function attemptRestart(): Promise { } } +const SOCKET_POLL_TOTAL_MS = 3_000; +const SOCKET_POLL_INTERVAL_MS = 150; + +/** + * Polls for rt.sock to exist and answer for up to ~3s after a restart + * request. parkUntilIntended's own socket probe, state.db open, and the + * identity migration routinely take longer than a single fixed delay, so a + * start that genuinely succeeds must not be reported as "installed but not + * running" just because the retry landed too early. + */ +async function waitForSocket( + totalMs: number = SOCKET_POLL_TOTAL_MS, + intervalMs: number = SOCKET_POLL_INTERVAL_MS, +): Promise { + const deadline = Date.now() + totalMs; + while (Date.now() < deadline) { + if (existsSync(DAEMON_SOCK_PATH)) { + const ping = await trySocketQuery("ping", {}, Math.min(intervalMs * 2, 1000)); + if (ping.response !== null) return true; + } + await Bun.sleep(intervalMs); + } + return existsSync(DAEMON_SOCK_PATH); +} + function warnDaemonDown(): void { if (hasWarnedThisSession || _warningSuppressed) return; hasWarnedThisSession = true; @@ -249,8 +274,11 @@ export async function daemonQueryAttributed( const restarted = await attemptRestart(); let last = first; if (restarted) { - // Retry once after short delay - await Bun.sleep(300); + // Bounded poll, not one fixed-delay retry: parkUntilIntended's own + // socket probe, state.db open, and the identity migration routinely + // take longer than 300ms, and a successful start must not be reported + // as "installed but not running" just because the retry landed early. + await waitForSocket(); const retry = await trySocketQuery(cmd, payload, timeoutMs); if (retry.response !== null) return retry; last = retry; From 0d816c013d87ca2c6e3d9b2a99ef76f37b38737d Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:18:55 -0500 Subject: [PATCH 036/142] runs prune: reap expired run dirs with a detached rm -rf, never a sync recursive rmSync (S100) --- lib/runs/__tests__/prune.test.ts | 40 ++++++++++++++++++++++++++++---- lib/runs/prune.ts | 24 +++++++++++++++++-- 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/lib/runs/__tests__/prune.test.ts b/lib/runs/__tests__/prune.test.ts index 75adbbb2..86a7a815 100644 --- a/lib/runs/__tests__/prune.test.ts +++ b/lib/runs/__tests__/prune.test.ts @@ -35,6 +35,15 @@ function seedRun(dir: string, repo: string, id: string, startedAt: number, userV db.close(); } +/** Polls until the detached rm -rf a prune spawns has actually removed `path` (S100: the delete is no longer synchronous). */ +async function waitGone(path: string, timeoutMs = 5000): Promise { + const deadline = Date.now() + timeoutMs; + while (existsSync(path)) { + if (Date.now() > deadline) throw new Error(`timed out waiting for ${path} to be reaped`); + await new Promise((r) => setTimeout(r, 20)); + } +} + function seedRunEnded(dir: string, repo: string, id: string, startedAt: number, endedAt: number, status = "done"): void { const runDir = join(dir, repo, id); mkdirSync(runDir, { recursive: true }); @@ -55,7 +64,7 @@ function seedRunEnded(dir: string, repo: string, id: string, startedAt: number, } describe("pruneRuns", () => { - test("removes runs ended past the floor, keeps recent and running ones", () => { + test("removes runs ended past the floor, keeps recent and running ones", async () => { const dir = root(); const now = Date.now(); seedRunEnded(dir, "alpha", "old-done", now - 40 * DAY, now - 40 * DAY); // ended 40d ago -> pruned @@ -64,12 +73,12 @@ describe("pruneRuns", () => { // never-finished: age by state.db mtime (fresh in this test) -> kept const { removed } = pruneRuns(now); expect(removed).toEqual([join(dir, "alpha", "old-done")]); - expect(existsSync(join(dir, "alpha", "old-done"))).toBe(false); + await waitGone(join(dir, "alpha", "old-done")); // reaped by a detached rm -rf, not synchronously expect(existsSync(join(dir, "alpha", "new-done"))).toBe(true); expect(existsSync(join(dir, "alpha", "still-running"))).toBe(true); }); - test("a running run ages out once its state.db mtime crosses the floor", () => { + test("a running run ages out once its state.db mtime crosses the floor", async () => { const dir = root(); const now = Date.now(); seedRun(dir, "alpha", "stale-running", now - 40 * DAY); @@ -78,7 +87,7 @@ describe("pruneRuns", () => { utimesSync(dbPath, oldTime, oldTime); const { removed } = pruneRuns(now); expect(removed).toEqual([join(dir, "alpha", "stale-running")]); - expect(existsSync(join(dir, "alpha", "stale-running"))).toBe(false); + await waitGone(join(dir, "alpha", "stale-running")); }); test("a stale regular file inside a repo dir is not mistaken for a run and survives pruning", () => { @@ -110,4 +119,27 @@ describe("pruneRuns", () => { expect(() => assertPrunable(join(dir, "alpha", "victim-run"), dir)).toThrow(); expect(existsSync(join(outside, "victim-run"))).toBe(true); }); + + // S100: the boot-time prune (60s after start) previously unlinked every + // expired run tree synchronously, blocking the daemon's single thread for + // the full duration of each recursive rm — a tray poll or chat post + // arriving mid-sweep would time out. The delete must be off-thread. + test("prune spawns the delete asynchronously — the run dir is not synchronously unlinked (S100)", () => { + const dir = root(); + const now = Date.now(); + seedRunEnded(dir, "alpha", "old-done", now - 40 * DAY, now - 40 * DAY); + const { removed } = pruneRuns(now); + expect(removed).toEqual([join(dir, "alpha", "old-done")]); + // Not gone yet: the unlink runs in a detached child process, off this + // thread, so pruneRuns returning never means the disk is clean yet. + expect(existsSync(join(dir, "alpha", "old-done"))).toBe(true); + }); + + test("prune's detached delete eventually removes the run dir", async () => { + const dir = root(); + const now = Date.now(); + seedRunEnded(dir, "alpha", "old-done", now - 40 * DAY, now - 40 * DAY); + pruneRuns(now); + await waitGone(join(dir, "alpha", "old-done")); + }); }); diff --git a/lib/runs/prune.ts b/lib/runs/prune.ts index b42b05a2..e2b33638 100644 --- a/lib/runs/prune.ts +++ b/lib/runs/prune.ts @@ -5,7 +5,7 @@ * and must stay in front of every rmSync. */ import { Database } from "bun:sqlite"; -import { existsSync, readdirSync, realpathSync, rmSync, statSync, type Dirent } from "fs"; +import { existsSync, readdirSync, realpathSync, statSync, type Dirent } from "fs"; import { basename, dirname, join, sep } from "path"; import { getSetting } from "../settings/resolve.ts"; import { runsRoot } from "./store.ts"; @@ -56,6 +56,26 @@ function realDirNames(path: string): string[] { } } +/** + * Detached, unawaited `rm -rf` — mirrors lib/worktree/trash.ts's reap + * pattern. A recursive unlink of a large run tree must never block the + * daemon's single event-loop thread: a sync rmSync here froze every + * concurrent tray poll and chat post for the sweep's full duration. + */ +function reapAsync(path: string): void { + try { + const proc = Bun.spawn(["rm", "-rf", "--", path], { + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + detached: true, + }); + proc.unref(); + } catch { + // Best-effort: a run tree that survives a failed spawn costs disk, never correctness. + } +} + function floorDays(): number { try { const v = getSetting("rt.runsPruneDays").value; @@ -98,7 +118,7 @@ export function pruneRuns(now: number = Date.now()): { removed: string[] } { } if (stamp < cutoff) { assertPrunable(runDir, root); - rmSync(runDir, { recursive: true, force: true }); + reapAsync(runDir); removed.push(runDir); } } From cbba6b4f2caf2adbd7244d4c273dcea9a7ed77a3 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:22:12 -0500 Subject: [PATCH 037/142] daemon: broadcast() drops dead/backpressured ws clients instead of silently dropping frames (S042) --- .../__tests__/api-server-broadcast.test.ts | 75 +++++++++++++++++++ lib/daemon/api-server.ts | 64 +++++++++++++++- 2 files changed, 135 insertions(+), 4 deletions(-) create mode 100644 lib/daemon/__tests__/api-server-broadcast.test.ts diff --git a/lib/daemon/__tests__/api-server-broadcast.test.ts b/lib/daemon/__tests__/api-server-broadcast.test.ts new file mode 100644 index 00000000..56faf914 --- /dev/null +++ b/lib/daemon/__tests__/api-server-broadcast.test.ts @@ -0,0 +1,75 @@ +import { describe, test, expect } from "bun:test"; +import { broadcastToClients, type BroadcastTarget } from "../api-server.ts"; + +function fakeClient(sendReturns: number[]): BroadcastTarget & { closed: boolean; sent: string[] } { + const sent: string[] = []; + let i = 0; + const client = { + closed: false, + sent, + send(data: string) { + sent.push(data); + const ret = sendReturns[Math.min(i, sendReturns.length - 1)] as number; + i++; + return ret; + }, + close() { client.closed = true; }, + }; + return client; +} + +function fakeLog() { + const warns: unknown[] = []; + return { warn: (o: unknown, _m: string) => { warns.push(o); }, warns }; +} + +describe("broadcastToClients", () => { + test("a healthy client (positive send return) is never closed", () => { + const client = fakeClient([42]); + broadcastToClients([client], "status", { ok: true }, fakeLog()); + expect(client.closed).toBe(false); + expect(client.sent.length).toBe(1); + }); + + test("a send() returning 0 (dropped frame) closes the client immediately and logs a warning", () => { + const client = fakeClient([0]); + const log = fakeLog(); + broadcastToClients([client], "status", { ok: true }, log); + expect(client.closed).toBe(true); + expect(log.warns.length).toBe(1); + }); + + test("a send() returning -1 (backpressure) is tolerated for a few sends before closing", () => { + const client = fakeClient([-1, -1, -1, -1]); + const log = fakeLog(); + broadcastToClients([client], "a", {}, log); + expect(client.closed).toBe(false); + broadcastToClients([client], "b", {}, log); + expect(client.closed).toBe(false); + broadcastToClients([client], "c", {}, log); + // third consecutive backpressure event closes the client + expect(client.closed).toBe(true); + }); + + test("a successful send resets the backpressure counter", () => { + const client = fakeClient([-1, -1, 99, -1, -1, -1]); + const log = fakeLog(); + broadcastToClients([client], "a", {}, log); // -1 (count=1) + broadcastToClients([client], "b", {}, log); // -1 (count=2) + broadcastToClients([client], "c", {}, log); // 99 -- resets to 0 + expect(client.closed).toBe(false); + broadcastToClients([client], "d", {}, log); // -1 (count=1) + broadcastToClients([client], "e", {}, log); // -1 (count=2) + expect(client.closed).toBe(false); + broadcastToClients([client], "f", {}, log); // -1 (count=3) -- closes + expect(client.closed).toBe(true); + }); + + test("a client whose send() throws is treated as gone: caught, not propagated", () => { + const client: BroadcastTarget = { + send() { throw new Error("ECONNRESET"); }, + close() { /* no-op */ }, + }; + expect(() => broadcastToClients([client], "a", {}, fakeLog())).not.toThrow(); + }); +}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index 3ac823f8..9670f861 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -78,13 +78,68 @@ interface ApiWSData { const wsClients = new Set>(); +let apiServerLog: { warn: (o: unknown, m: string) => void } = { warn: () => {} }; + +/** Consecutive Bun `ws.send()` backpressure (-1) returns tolerated before a + client is dropped as chronically stalled. */ +const BACKPRESSURE_CLOSE_THRESHOLD = 3; +const backpressureCounts = new WeakMap(); + +export interface BroadcastTarget { + send(data: string): number; + close(): void; +} + +/** + * Sends one frame to every client, dropping any that Bun's own send() return + * value marks as gone (S042). `ws.send()` never throws on a dead socket -- + * it returns 0 (this send silently failed) or -1 (backpressure) -- so a + * disconnected or stalled console/chat-viewer tab used to keep receiving a + * SUBSET of frames forever with nothing logged. 0 means Bun already dropped + * this exact frame for this client: closing immediately (rather than + * counting) is correct because the client's own reconnect logic is the only + * way it recovers a consistent stream. -1 means backpressure, which can be + * transient, so a few in a row are tolerated before giving up on the client. + */ +export function broadcastToClients( + clients: Iterable, + type: string, + data: any, + log: { warn: (o: unknown, m: string) => void }, +): void { + const msg = JSON.stringify({ type, data, timestamp: Date.now() }); + for (const client of clients) { + let result: number; + try { + result = client.send(msg); + } catch (err) { + log.warn({ err }, "ws client send threw; dropping"); + try { client.close(); } catch { /* already gone */ } + continue; + } + if (result === 0) { + log.warn({ type }, "ws client dropped a frame (send()=0); closing so its reconnect resyncs"); + backpressureCounts.delete(client); + try { client.close(); } catch { /* already gone */ } + } else if (result === -1) { + const count = (backpressureCounts.get(client) ?? 0) + 1; + if (count >= BACKPRESSURE_CLOSE_THRESHOLD) { + log.warn({ type, count }, "ws client chronically backpressured; closing"); + backpressureCounts.delete(client); + try { client.close(); } catch { /* already gone */ } + } else { + backpressureCounts.set(client, count); + } + } else { + backpressureCounts.delete(client); + } + } +} + /** Broadcast an event to all connected WebSocket clients. */ export function broadcast(type: string, data: any): void { if (wsClients.size === 0) return; - const msg = JSON.stringify({ type, data, timestamp: Date.now() }); - for (const ws of wsClients) { - try { ws.send(msg); } catch { /* client disconnected */ } - } + broadcastToClients(wsClients, type, data, apiServerLog); } /** Drop all broadcast clients (shutdown). */ @@ -150,6 +205,7 @@ export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDe export async function startApiServer(deps: ApiServerDeps): Promise> { const { handleCommand, log } = deps; + apiServerLog = log; const apiToken = getApiToken(); const server = await bindApiServerWithRetry(() => Bun.serve({ From 659ecee57c9c7bdc7ec8494fb748df03fecd0ecd Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:23:41 -0500 Subject: [PATCH 038/142] worktree reconciler: queue a follow-up pass for a kick arriving after the loop has started, instead of dropping it (S065) --- .../__tests__/worktree-reconciler.test.ts | 34 +++++++++++++++++++ lib/daemon/worktree-reconciler.ts | 25 +++++++++++++- 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/lib/daemon/__tests__/worktree-reconciler.test.ts b/lib/daemon/__tests__/worktree-reconciler.test.ts index 30133a8c..87b17260 100644 --- a/lib/daemon/__tests__/worktree-reconciler.test.ts +++ b/lib/daemon/__tests__/worktree-reconciler.test.ts @@ -1348,6 +1348,40 @@ describe("detached trigger / latency", () => { expect(events.filter((e) => e.type === "worktree:created").length).toBe(1); }, 10_000); + + // S065: a kick() during an in-flight pass, once the pass has started + // working (its per-repo loop has begun — this repo's replenish may + // already have run), must not be silently dropped. Observed here via + // repoIndex() call count: it's read fresh once per runOnce() invocation, + // so a queued follow-up pass is externally visible as a second call. + test("a kick() arriving after the pass has started work triggers a follow-up pass, not a silent drop (S065)", async () => { + process.env.HOME = realpathSync(mkdtempSync(join(tmpdir(), "rtkick2-home-"))); + closeStateDb(); + __test__.createBackoff.clear(); + const repoName = "acme-kick2"; + const repo = makeRepo(); + addBareOrigin(repo); + writeFileSync(join(repo, "wip.txt"), "not idle\n"); + await declareWorktrees(repo, repoName, { onDeck: 1, root: join(repo, ".worktrees"), ready: [{ run: "sleep 2" }] }); + + let repoIndexCalls = 0; + const reconciler = createWorktreeReconciler({ + cache: { entries: {} }, + repoIndex: () => { repoIndexCalls++; return { [repoName]: repo }; }, + emit: () => {}, + log: fakeLog(), + }); + + reconciler.kick(); + await waitFor(() => reconciler.creationInFlight(repoName) !== null, 2000); + // Mid-pass: this repo's replenish step is already running, so this kick + // must queue a follow-up rather than being dropped. + reconciler.kick(); + + await waitFor(() => reconciler.creationInFlight(repoName) === null, 6000); + await waitFor(() => repoIndexCalls >= 2, 6000); // the queued follow-up pass actually ran + expect(repoIndexCalls).toBeGreaterThanOrEqual(2); + }, 15_000); }); describe("reapRepoTrash", () => { diff --git a/lib/daemon/worktree-reconciler.ts b/lib/daemon/worktree-reconciler.ts index d7af150d..6679dd70 100644 --- a/lib/daemon/worktree-reconciler.ts +++ b/lib/daemon/worktree-reconciler.ts @@ -1101,6 +1101,16 @@ export function createWorktreeReconciler(deps: ReconcilerDeps): { /** Non-null while a holder owns the reconciler. */ let hold: Promise | null = null; let kickQueued = false; + /** + * True once the current pass's per-repo loop has begun processing at + * least one repo. Two kicks that both land before this flips (the common + * "two synchronous kicks" case) still collapse to one pass — the + * upcoming loop reads fresh state regardless. A kick landing after it + * flips might be about a repo this pass has already stepped past (e.g. a + * provision claiming the last on-deck tree right after replenish ran for + * it), so it queues a follow-up instead of being silently dropped. + */ + let passStartedWork = false; const creationPromises = new Map>(); async function runOnce(): Promise { @@ -1118,6 +1128,7 @@ export function createWorktreeReconciler(deps: ReconcilerDeps): { const appConfig = loadWorktreeAppConfig(); for (const [repoName, repoPath] of Object.entries(repos)) { + passStartedWork = true; if (!(await repoHasWorktreeActivity(repoName, repoPath))) continue; try { await reconcileRepoRegistry({ repoName, repoPath, emit: deps.emit, log: deps.log }); @@ -1167,13 +1178,25 @@ export function createWorktreeReconciler(deps: ReconcilerDeps): { kickQueued = true; return; } - if (inFlight) return; + if (inFlight) { + // Two kicks landing before this pass has stepped into its per-repo + // loop still collapse to one pass; once it has, a kick might be about + // a repo already stepped past (its replenish already ran this pass), + // so queue a follow-up rather than dropping it silently. + if (passStartedWork) kickQueued = true; + return; + } + passStartedWork = false; const p = runOnce() .catch((err) => { deps.log.warn({ err }, "worktree reconciler: kick failed"); }) .finally(() => { if (inFlight === p) inFlight = null; + if (kickQueued) { + kickQueued = false; + kick(); + } }); inFlight = p; } From ba5e7fc40b965413011693d6d79e125e764d85c4 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:24:00 -0500 Subject: [PATCH 039/142] events.db: corruption quarantine + busy_timeout/synchronous pragmas; guard sweep timers createEventsBus now mirrors state/db.ts's corruption quarantine (rename to events.db.corrupt- + -wal/-shm sidecars, warn, recreate empty) and sets busy_timeout=250 / synchronous=NORMAL on open. events.db is a bounded-retention journal, so total loss on quarantine is harmless. Added safeInterval/safeTimeout (lib/daemon/safe-timers.ts) and wrapped the two eventsBus.sweep() timers in lib/daemon.ts with them, so a synchronous sqlite throw mid-tick (e.g. SQLITE_FULL) warns instead of becoming an uncaughtException that exits the daemon. Also fixes the near-vacuous e2e assertion carried from Task 3: the corrupt-events.db test now asserts the quarantine file exists AND the daemon actually boots and serves (a live rt events emit round trip), instead of existsSync(stderrLog) || quarantined, which was always true. --- e2e/tests/daemon.test.ts | 63 +++++++++++++++++++----- lib/daemon.ts | 9 ++-- lib/daemon/__tests__/events-bus.test.ts | 33 ++++++++++++- lib/daemon/__tests__/safe-timers.test.ts | 55 +++++++++++++++++++++ lib/daemon/events-bus.ts | 53 ++++++++++++++++++-- lib/daemon/safe-timers.ts | 42 ++++++++++++++++ lib/state/db.ts | 2 +- 7 files changed, 237 insertions(+), 20 deletions(-) create mode 100644 lib/daemon/__tests__/safe-timers.test.ts create mode 100644 lib/daemon/safe-timers.ts diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index 5bccbdfe..6b4af74d 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -1,7 +1,24 @@ import { describe, test, expect, beforeAll, afterAll } from "bun:test"; import { existsSync, mkdirSync, writeFileSync, readdirSync } from "fs"; import { join } from "path"; -import { createTestHome, rt } from "../harness.ts"; +import { createTestHome, rt, RT_BINARY } from "../harness.ts"; + +async function waitForSocket(sockPath: string, timeoutMs = 15_000): Promise { + const deadline = Date.now() + timeoutMs; + while (!existsSync(sockPath)) { + if (Date.now() > deadline) throw new Error(`daemon socket never appeared at ${sockPath}`); + await Bun.sleep(100); + } +} + +/** Grab a free TCP port by binding port 0 and releasing it. */ +function freePort(): number { + const srv = Bun.serve({ port: 0, fetch: () => new Response("") }); + const port = srv.port; + srv.stop(true); + if (!port) throw new Error("failed to allocate a free port"); + return port; +} describe("fatal boot", () => { test("daemon boot with API port already bound exits non-zero and leaves no stale rt.pid", async () => { @@ -20,25 +37,47 @@ describe("fatal boot", () => { } }, 60_000); - test("a corrupt events.db does not crash the daemon silently — error is captured", async () => { + test("a corrupt events.db self-heals — quarantined, and the daemon boots and serves", async () => { const { path: home, cleanup } = createTestHome(); + const bunDir = join(process.execPath, ".."); + let daemon: ReturnType | undefined; try { // Pre-create a corrupt events.db in the isolated HOME, before the - // daemon ever runs — createEventsBus (module scope) throws opening it. + // daemon ever runs — createEventsBus (module scope) opens it. const rtDir = join(home, ".mattstack", "rt"); mkdirSync(rtDir, { recursive: true }); writeFileSync(join(rtDir, "events.db"), "not a sqlite file at all"); - const result = await rt(["--daemon"], { home }); - - expect(result.exitCode).not.toBe(0); - // Task 4 makes this self-heal (quarantine + retry); for Task 3 we only - // require the failure is captured, not that boot recovers. - const stderrLog = join(rtDir, "logs", "daemon-stderr.log"); - const quarantined = existsSync(rtDir) && readdirSync(rtDir).some((f) => f.startsWith("events.db.corrupt-")); - const captured = existsSync(stderrLog) || quarantined; - expect(captured).toBe(true); + const apiPort = freePort(); + daemon = Bun.spawn([RT_BINARY, "--daemon"], { + env: { + HOME: home, + PATH: `${join(RT_BINARY, "..")}:${bunDir}:/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin`, + TERM: "xterm-256color", + RT_SKIP_SETUP: "1", + CI: "true", + RT_API_PORT: String(apiPort), + }, + stdout: "pipe", + stderr: "pipe", + }); + + await waitForSocket(join(rtDir, "rt.sock")); + expect(daemon.exitCode).toBeNull(); + + // (a) the corrupt events.db was quarantined, not just failed on. + expect(readdirSync(rtDir).some((f) => f.startsWith("events.db.corrupt-"))).toBe(true); + + // (b) the daemon actually boots and serves: a live round trip through + // the recreated events.db proves it, not just the socket's existence. + const served = await rt(["events", "emit", "e2e/corrupt-events-recover"], { + home, + env: { RT_API_PORT: String(apiPort) }, + }); + expect(served.exitCode).toBe(0); } finally { + try { daemon?.kill(); } catch { /* already gone */ } + await daemon?.exited; cleanup(); } }, 60_000); diff --git a/lib/daemon.ts b/lib/daemon.ts index 2dbd549c..ea49efa2 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -63,6 +63,7 @@ import { import { startDiscussionsPoller } from "./daemon/discussions-poller.ts"; import { createCleanup, installSignalHandlers } from "./daemon/shutdown.ts"; import { createEventsBus } from "./daemon/events-bus.ts"; +import { safeInterval, safeTimeout } from "./daemon/safe-timers.ts"; import { pruneRuns } from "./runs/prune.ts"; import { pruneLogs } from "./log-janitor.ts"; import { getSetting } from "./settings/resolve.ts"; @@ -208,10 +209,12 @@ const hooksGuard = createHooksGuard(log); // Pane-communication events bus (RT-44): SQLite journal + in-memory waiters. const eventsBus = createEventsBus({ dbPath: join(RT_DIR, "events.db"), log }); // Hourly retention sweep — cheap; rides its own interval rather than pollers -// because it needs no poller deps. -setInterval(() => eventsBus.sweep(), 60 * 60 * 1000); +// because it needs no poller deps. safeInterval/safeTimeout: a sync sqlite +// throw here (e.g. SQLITE_FULL) must warn, not become an uncaughtException +// that exits the daemon. +safeInterval(() => eventsBus.sweep(), 60 * 60 * 1000, "events-sweep", log); // Boot-time sweep to handle frequent daemon restarts that would otherwise starve the hourly interval. -setTimeout(() => eventsBus.sweep(), 30_000); +safeTimeout(() => eventsBus.sweep(), 30_000, "events-sweep-boot", log); // Age-floor prune of pipeline run dirs — daily; assertPrunable in prune.ts // guards every deletion against the runs root. diff --git a/lib/daemon/__tests__/events-bus.test.ts b/lib/daemon/__tests__/events-bus.test.ts index a1a85c03..c10e038f 100644 --- a/lib/daemon/__tests__/events-bus.test.ts +++ b/lib/daemon/__tests__/events-bus.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect, beforeEach, afterEach } from "bun:test"; -import { mkdtempSync, rmSync } from "fs"; +import { mkdtempSync, rmSync, writeFileSync, readdirSync } from "fs"; import { join } from "path"; import { tmpdir } from "os"; import pino from "pino"; @@ -7,6 +7,37 @@ import { createEventsBus, matchTopic, type EventsBus } from "../events-bus.ts"; const log = pino({ level: "silent" }); +describe("events bus corruption + pragmas", () => { + test("createEventsBus quarantines and recreates a corrupt events.db instead of throwing", () => { + const dir = mkdtempSync(join(tmpdir(), "events-corrupt-")); + try { + const dbPath = join(dir, "events.db"); + writeFileSync(dbPath, "garbage not sqlite"); + const bus = createEventsBus({ dbPath, log }); + expect(readdirSync(dir).some((f) => f.startsWith("events.db.corrupt-"))).toBe(true); + // fresh db works: + bus.emit("test", { hi: 1 }); + expect(bus.list({ pattern: "**" }).events.length).toBe(1); + bus.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("createEventsBus sets busy_timeout and synchronous=NORMAL", () => { + const dir = mkdtempSync(join(tmpdir(), "events-pragma-")); + try { + const bus = createEventsBus({ dbPath: join(dir, "events.db"), log }); + const handle = bus.__db!; + expect(handle.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); + expect(handle.query("PRAGMA synchronous").get()).toEqual({ synchronous: 1 }); // NORMAL + bus.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + describe("matchTopic", () => { test("bare topic matches itself only", () => { expect(matchTopic("job/x/question", "job/x/question")).toBe(true); diff --git a/lib/daemon/__tests__/safe-timers.test.ts b/lib/daemon/__tests__/safe-timers.test.ts new file mode 100644 index 00000000..13ab83b8 --- /dev/null +++ b/lib/daemon/__tests__/safe-timers.test.ts @@ -0,0 +1,55 @@ +import { describe, test, expect, mock } from "bun:test"; +import pino from "pino"; +import { safeInterval, safeTimeout } from "../safe-timers.ts"; + +const silentLog = pino({ level: "silent" }); + +describe("safeInterval / safeTimeout", () => { + test("safeInterval swallows a throwing tick and logs warn", async () => { + const warn = mock(() => {}); + const log = { ...silentLog, warn } as unknown as typeof silentLog; + let ticks = 0; + const handle = safeInterval( + () => { + ticks++; + throw new Error("SQLITE_FULL"); + }, + 10, + "test-sweep", + log, + ); + await new Promise((r) => setTimeout(r, 45)); + clearInterval(handle); + expect(ticks).toBeGreaterThan(0); + expect(warn).toHaveBeenCalled(); + }); + + test("safeTimeout swallows a throwing tick and logs warn", async () => { + const warn = mock(() => {}); + const log = { ...silentLog, warn } as unknown as typeof silentLog; + let ran = false; + safeTimeout( + () => { + ran = true; + throw new Error("SQLITE_FULL"); + }, + 10, + "test-timeout", + log, + ); + await new Promise((r) => setTimeout(r, 45)); + expect(ran).toBe(true); + expect(warn).toHaveBeenCalled(); + }); + + test("safeInterval does not swallow a non-throwing tick's normal operation", async () => { + const warn = mock(() => {}); + const log = { ...silentLog, warn } as unknown as typeof silentLog; + let ticks = 0; + const handle = safeInterval(() => { ticks++; }, 10, "test-sweep-ok", log); + await new Promise((r) => setTimeout(r, 45)); + clearInterval(handle); + expect(ticks).toBeGreaterThan(0); + expect(warn).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/daemon/events-bus.ts b/lib/daemon/events-bus.ts index aca1dd08..c44cd211 100644 --- a/lib/daemon/events-bus.ts +++ b/lib/daemon/events-bus.ts @@ -8,9 +8,10 @@ */ import { Database } from "bun:sqlite"; -import { mkdirSync } from "fs"; +import { mkdirSync, renameSync } from "fs"; import { dirname } from "path"; import type { Logger } from "pino"; +import { isCorruptionError } from "../state/db.ts"; export interface BusEvent { id: number; topic: string; payload: unknown; emittedAt: number } export interface WaitResult { events: BusEvent[]; cursor: number } @@ -31,6 +32,8 @@ export interface EventsBus { sweep(): number; waiterCount(): number; close(): void; + /** Test-only debug accessor for the underlying handle (e.g. pragma checks). Not for feature code. */ + __db?: Database; } // One matcher for wait AND list. Bun.Glob: `*` does not cross `/`, `**` does. @@ -61,6 +64,30 @@ function rowToEvent(row: EventRow): BusEvent { return { id: row.id, topic: row.topic, payload, emittedAt: row.emittedAt }; } +/** + * Renames a corrupt events.db out of the way and warns loudly, mirroring + * lib/state/db.ts's `quarantine`. events.db is a bounded-retention journal + * (sweep() already discards old rows), so losing it entirely on corruption + * is harmless — recreate empty rather than attempt any repair. WAL sidecars + * are best-effort cleaned since they are meaningless without the main file. + */ +function quarantineEventsDb(path: string, log: Logger): void { + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const quarantinedPath = `${path}.corrupt-${stamp}`; + log.warn( + { path, quarantinedPath }, + "events db could not be opened (corrupt), quarantining and recreating empty", + ); + renameSync(path, quarantinedPath); + for (const sidecar of [`${path}-wal`, `${path}-shm`]) { + try { + renameSync(sidecar, `${sidecar}.corrupt-${stamp}`); + } catch { + // sidecar absent — fine, WAL mode doesn't always leave one + } + } +} + export function createEventsBus(opts: { dbPath: string; log: Logger; @@ -73,8 +100,26 @@ export function createEventsBus(opts: { // Self-sufficient about its parent dir — daemon.ts constructs the bus at // module scope, before startDaemon()'s mkdirSync(RT_DIR) runs. mkdirSync(dirname(opts.dbPath), { recursive: true }); - const db = new Database(opts.dbPath, { create: true }); - db.exec("PRAGMA journal_mode = WAL;"); + // PRAGMA order matches lib/state/db.ts's applyPragmas: busy_timeout FIRST + // so the WAL conversion itself respects it, then journal_mode, then + // synchronous. + let db: Database; + try { + db = new Database(opts.dbPath, { create: true }); + db.exec("PRAGMA busy_timeout = 250;"); + db.exec("PRAGMA journal_mode = WAL;"); + db.exec("PRAGMA synchronous = NORMAL;"); + // Pragmas alone don't always force sqlite to validate the file header; + // touch it now so a corrupt file surfaces here, not mid-query later. + db.query("PRAGMA user_version").get(); + } catch (err) { + if (!isCorruptionError(err)) throw err; + quarantineEventsDb(opts.dbPath, log); + db = new Database(opts.dbPath, { create: true }); + db.exec("PRAGMA busy_timeout = 250;"); + db.exec("PRAGMA journal_mode = WAL;"); + db.exec("PRAGMA synchronous = NORMAL;"); + } db.exec(` CREATE TABLE IF NOT EXISTS events ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -215,6 +260,8 @@ export function createEventsBus(opts: { waiterCount() { return waiters.size; }, + __db: db, + close() { const head = maxId(); for (const w of [...waiters]) settle(w, { events: [], cursor: head }); diff --git a/lib/daemon/safe-timers.ts b/lib/daemon/safe-timers.ts new file mode 100644 index 00000000..6ba319c1 --- /dev/null +++ b/lib/daemon/safe-timers.ts @@ -0,0 +1,42 @@ +/** + * lib/daemon/safe-timers.ts — try/catch-wrapped setInterval/setTimeout. + * + * A bare `setInterval`/`setTimeout` callback that throws synchronously + * (e.g. a sqlite SQLITE_FULL on a WAL write) becomes an uncaughtException + * with no stack frame back to the timer that scheduled it — Node/Bun's + * event loop has nothing to attribute the throw to but the process itself, + * so installCrashHandlers treats it as fatal and exits the daemon. Wrapping + * the tick converts that crash into a logged warning. + */ + +import type { Logger } from "pino"; + +export function safeInterval( + fn: () => void, + ms: number, + label: string, + log: Logger, +): ReturnType { + return setInterval(() => { + try { + fn(); + } catch (err) { + log.warn({ err, label }, "timer tick failed"); + } + }, ms); +} + +export function safeTimeout( + fn: () => void, + ms: number, + label: string, + log: Logger, +): ReturnType { + return setTimeout(() => { + try { + fn(); + } catch (err) { + log.warn({ err, label }, "timer tick failed"); + } + }, ms); +} diff --git a/lib/state/db.ts b/lib/state/db.ts index 929a6a63..86cbeafc 100644 --- a/lib/state/db.ts +++ b/lib/state/db.ts @@ -304,7 +304,7 @@ function addArchivedAtColumnIfMissing(db: Database): void { } /** bun:sqlite error codes that mean "the file on disk is not a usable db". */ -function isCorruptionError(err: unknown): boolean { +export function isCorruptionError(err: unknown): boolean { const code = (err as { code?: string } | undefined)?.code; return code === "SQLITE_CORRUPT" || code === "SQLITE_NOTADB"; } From b86d4c2a89639c63c29c4848a0ff3eec707803ed Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:25:20 -0500 Subject: [PATCH 040/142] runCapture: stop clearing the SIGKILL timer before it fires, strengthen SIGKILL test Review found the finally block cleared killTimer in the same tick the deadline promise resolved, so a SIGTERM-ignoring child was never actually SIGKILLed even though runCapture's own promise settled on time. Leave killTimer running past the finally; it is already try/catch guarded so it is a no-op once the child has exited. Also capture and clear the deadline promise's own timer handle so it doesn't outlive the fast path. Strengthened the SIGKILL escalation test to assert the child process is actually gone (via its recorded pid) rather than only asserting runCapture's promise resolved on schedule, since the old assertion passed regardless of whether the kill worked. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/__tests__/subprocess.test.ts | 39 ++++++++++++++++++++----- lib/subprocess.ts | 9 ++++-- packages/rt-client/src/settings/exec.ts | 9 ++++-- 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/lib/__tests__/subprocess.test.ts b/lib/__tests__/subprocess.test.ts index 9fb2725d..88eba03c 100644 --- a/lib/__tests__/subprocess.test.ts +++ b/lib/__tests__/subprocess.test.ts @@ -1,4 +1,7 @@ import { describe, test, expect, afterEach } from "bun:test"; +import { unlink } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import { runCapture, outputTail, MAX_LOGGED_OUTPUT } from "../subprocess.ts"; const SENTINEL = "RT_SPAWN_ENV_SENTINEL"; @@ -44,14 +47,36 @@ describe("runCapture timeout enforcement", () => { }); test("a SIGTERM-ignoring child is bounded by SIGKILL escalation", async () => { - const t0 = Date.now(); - const r = await runCapture( - ["/bin/zsh", "-c", "trap '' TERM; sleep 20"], - { timeoutMs: 800 }, + // Timing alone doesn't prove the kill worked: the deadline race resolves + // runCapture on schedule regardless of whether SIGKILL ever fires. So this + // asserts the process is actually gone, via its own recorded pid. + const pidFile = path.join( + os.tmpdir(), + `rt-subprocess-test-${Date.now()}-${Math.random().toString(36).slice(2)}.pid`, ); - expect(Date.now() - t0).toBeLessThan(4000); - expect(r.timedOut).toBe(true); - }); + try { + const t0 = Date.now(); + const r = await runCapture( + ["/bin/zsh", "-c", `echo $$ > ${pidFile}; trap '' TERM; sleep 20`], + { timeoutMs: 500 }, + ); + expect(Date.now() - t0).toBeLessThan(4000); + expect(r.timedOut).toBe(true); + + // Past the 2s SIGKILL grace, so the escalation has had time to land. + await new Promise((resolve) => setTimeout(resolve, 2500)); + const pid = Number((await Bun.file(pidFile).text()).trim()); + let alive = true; + try { + process.kill(pid, 0); + } catch { + alive = false; + } + expect(alive).toBe(false); + } finally { + await unlink(pidFile).catch(() => {}); + } + }, 8000); test("normal fast command still returns real stdout and exitCode 0", async () => { const r = await runCapture(["/bin/echo", "hello"], { timeoutMs: 5000 }); diff --git a/lib/subprocess.ts b/lib/subprocess.ts index 9d3b23e2..c6a286f2 100644 --- a/lib/subprocess.ts +++ b/lib/subprocess.ts @@ -89,14 +89,19 @@ export async function runCapture( } })(); + let deadlineTimer: ReturnType; const deadline: Promise = new Promise((resolve) => { - setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); + deadlineTimer = setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); }); try { return await Promise.race([captured, deadline]); } finally { clearTimeout(term); - if (killTimer) clearTimeout(killTimer); + clearTimeout(deadlineTimer!); + // killTimer intentionally NOT cleared here: on the timeout path it must + // survive this finally to fire SIGKILL against a child that ignored + // SIGTERM. proc.kill is already try/catch guarded, so it is a harmless + // no-op if the child exited before the 2s grace elapses. } } diff --git a/packages/rt-client/src/settings/exec.ts b/packages/rt-client/src/settings/exec.ts index cedc8e12..fae833ec 100644 --- a/packages/rt-client/src/settings/exec.ts +++ b/packages/rt-client/src/settings/exec.ts @@ -79,14 +79,19 @@ export async function runCapture( } })(); + let deadlineTimer: ReturnType; const deadline: Promise = new Promise((resolve) => { - setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); + deadlineTimer = setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); }); try { return await Promise.race([captured, deadline]); } finally { clearTimeout(term); - if (killTimer) clearTimeout(killTimer); + clearTimeout(deadlineTimer!); + // killTimer intentionally NOT cleared here: on the timeout path it must + // survive this finally to fire SIGKILL against a child that ignored + // SIGTERM. proc.kill is already try/catch guarded, so it is a harmless + // no-op if the child exited before the 2s grace elapses. } } From 456b8b8282e37233716a73e9f3ff80a3bedbde97 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:25:53 -0500 Subject: [PATCH 041/142] chat:post/chat:dm: validate room/handle/body shape and refuse an unknown room instead of a silent black hole (R010) --- lib/daemon/__tests__/chat-handlers.test.ts | 56 ++++++++++++++++++++++ lib/daemon/handlers/chat.ts | 26 ++++++++++ 2 files changed, 82 insertions(+) diff --git a/lib/daemon/__tests__/chat-handlers.test.ts b/lib/daemon/__tests__/chat-handlers.test.ts index fb75f284..399b80f1 100644 --- a/lib/daemon/__tests__/chat-handlers.test.ts +++ b/lib/daemon/__tests__/chat-handlers.test.ts @@ -99,6 +99,62 @@ test("chat:post rejects an invalid mentions element with a reason rather than st expect(res.error).toContain("handle"); }); +// R010: a typo'd room silently no-op'd through postMessage's REVIVE (a +// no-op for a room with no chat_rooms row) and returned {ok:true, +// recipients:[]} — no error, no listing, unreachable except by the exact +// typo'd name. It must fail loudly instead. +test("chat:post refuses a room nobody has joined instead of silently black-holing the message", async () => { + const h = freshHandlers(); + const res = await h["chat:post"]({ room: "typo-room", handle: "a", body: "hi" }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("typo-room"); +}); + +test("chat:post's unknown-room error names a close existing room", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "deck-main", handle: "a" }); + const res = await h["chat:post"]({ room: "deck", handle: "a", body: "hi" }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("deck-main"); +}); + +test("chat:post rejects a missing or empty body rather than routing it through unenforced", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + const missing = await h["chat:post"]({ room: "r", handle: "a" } as any); + expect(missing.ok).toBe(false); + const empty = await h["chat:post"]({ room: "r", handle: "a", body: "" }); + expect(empty.ok).toBe(false); +}); + +test("chat:post rejects a body over the size cap", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + const res = await h["chat:post"]({ room: "r", handle: "a", body: "x".repeat(64 * 1024 + 1) }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error.toLowerCase()).toContain("body"); +}); + +test("chat:post rejects a non-array mentions field", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + const res = await h["chat:post"]({ room: "r", handle: "a", body: "hi", mentions: "b" as any }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("mentions"); +}); + +test("chat:dm rejects a missing or empty body", async () => { + const h = freshHandlers(); + const missing = await h["chat:dm"]({ from: "a", to: "b" } as any); + expect(missing.ok).toBe(false); + const empty = await h["chat:dm"]({ from: "a", to: "b", body: "" }); + expect(empty.ok).toBe(false); +}); + test("chat:unread-waking reports what would wake a handle without advancing its cursor", async () => { const h = freshHandlers(); await h["chat:join"]({ room: "r", handle: "a" }); diff --git a/lib/daemon/handlers/chat.ts b/lib/daemon/handlers/chat.ts index 470e846d..80b05270 100644 --- a/lib/daemon/handlers/chat.ts +++ b/lib/daemon/handlers/chat.ts @@ -17,6 +17,7 @@ import { unreadWakingCount, listRooms, archiveRoom, + roomArchivedAt, roomDefaultWake, listMembers, armMember, @@ -83,6 +84,19 @@ function assertionError(fn: () => void): string | null { } } +/** Generous, not tight: bounds a single message body without constraining any real conversation. */ +const MAX_BODY_BYTES = 64 * 1024; + +function isValidBody(body: unknown): body is string { + return typeof body === "string" && body.length > 0 && Buffer.byteLength(body, "utf8") <= MAX_BODY_BYTES; +} + +/** Rooms `handle` already belongs to whose name is a prefix/suffix of the typo'd one — the common shape of a "deck" vs "deck-main" miss. */ +function closestRoomNames(typo: string, handle: string, db: Database): string[] { + const known = listRooms(handle, db, { includeArchived: true }).map((r) => r.room); + return known.filter((r) => r.startsWith(typo) || typo.startsWith(r)).slice(0, 3); +} + /** * The row must commit before either emit fires, or a woken agent reads the * wake pointer and finds no message yet. Shared by chat:post and chat:dm so @@ -202,8 +216,19 @@ export function createChatHandlers(opts: { "chat:post": async (payload: Commands["chat:post"]["payload"]): Promise> => { const { room, handle, body, mentions } = payload; + if (!isValidChatName(room)) return { ok: false, error: `invalid room "${room}"` }; + if (!isValidChatName(handle)) return { ok: false, error: `invalid handle "${handle}"` }; + if (!isValidBody(body)) return { ok: false, error: `body must be a non-empty string under ${MAX_BODY_BYTES} bytes` }; + if (mentions !== undefined && !Array.isArray(mentions)) return { ok: false, error: "mentions must be an array of handles" }; const invalidMention = mentions?.find((m) => !isValidChatName(m)); if (invalidMention !== undefined) return { ok: false, error: `invalid handle "${invalidMention}"` }; + // A typo'd room previously no-op'd through postMessage's REVIVE (a + // no-op for a room with no chat_rooms row) and returned ok with no + // recipients — unreachable except by the exact typo'd name. + if (roomArchivedAt(room, db) === undefined) { + const nearby = closestRoomNames(room, handle, db); + return { ok: false, error: `unknown room "${room}"${nearby.length ? ` — did you mean: ${nearby.join(", ")}` : ""}` }; + } const posted = postAndNotify(db, emitEvent, { room, handle, body, mentions }); if (!posted) return { ok: false, error: "chat: post failed (retry budget exhausted)" }; return { ok: true, data: posted }; @@ -354,6 +379,7 @@ export function createChatHandlers(opts: { const { from, to, body, sessionId } = payload; if (!isValidChatName(from)) return { ok: false, error: `invalid handle "${from}"` }; if (!isValidChatName(to)) return { ok: false, error: `invalid handle "${to}"` }; + if (!isValidBody(body)) return { ok: false, error: `body must be a non-empty string under ${MAX_BODY_BYTES} bytes` }; const err = assertionError(() => assertSessionOwnsHandle(from, sessionId, db)); if (err) return { ok: false, error: err }; const humanHandle = getSetting("chat.humanHandle").value; From 5657d276556ceaec680260895470bf093807d3c3 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:27:04 -0500 Subject: [PATCH 042/142] daemon: pathParam() helper -- malformed %-encoding is a 400, not a logged 500 (S083) Adds a shared pathParam() decode helper and wires it into the three parameterized routes (/api/cache/:branch, /api/hooks/:repo/repair, /api/runs/:repo/:runId), replacing each route's hand-rolled decodeURIComponent. A malformed path segment now returns a clean 400 instead of falling through to the outer catch's logged 500 (cache and hooks routes) or the generic 404 (runs route). --- .../__tests__/api-server-path-param.test.ts | 36 +++++++++++++ lib/daemon/api-server.ts | 50 ++++++++++++++----- 2 files changed, 73 insertions(+), 13 deletions(-) create mode 100644 lib/daemon/__tests__/api-server-path-param.test.ts diff --git a/lib/daemon/__tests__/api-server-path-param.test.ts b/lib/daemon/__tests__/api-server-path-param.test.ts new file mode 100644 index 00000000..353b9a0a --- /dev/null +++ b/lib/daemon/__tests__/api-server-path-param.test.ts @@ -0,0 +1,36 @@ +import { describe, test, expect } from "bun:test"; +import { pathParam } from "../api-server.ts"; + +describe("pathParam", () => { + test("decodes a clean prefix-only param", () => { + expect(pathParam("/api/cache/main", "/api/cache/")).toBe("main"); + }); + + test("decodes a URL-encoded segment", () => { + expect(pathParam("/api/cache/feature%2Ffoo", "/api/cache/")).toBe("feature/foo"); + }); + + test("returns undefined for malformed %-encoding instead of throwing", () => { + expect(pathParam("/api/cache/%E0%A4%A", "/api/cache/")).toBeUndefined(); + }); + + test("returns undefined when the pathname doesn't start with the prefix", () => { + expect(pathParam("/api/other/main", "/api/cache/")).toBeUndefined(); + }); + + test("handles a prefix+suffix pair (hooks repair shape)", () => { + expect(pathParam("/api/hooks/my-repo/repair", "/api/hooks/", "/repair")).toBe("my-repo"); + }); + + test("prefix+suffix: malformed encoding still returns undefined", () => { + expect(pathParam("/api/hooks/%E0%A4%A/repair", "/api/hooks/", "/repair")).toBeUndefined(); + }); + + test("prefix+suffix: wrong suffix returns undefined", () => { + expect(pathParam("/api/hooks/my-repo/other", "/api/hooks/", "/repair")).toBeUndefined(); + }); + + test("an empty captured segment returns undefined", () => { + expect(pathParam("/api/cache/", "/api/cache/")).toBeUndefined(); + }); +}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index 9670f861..baef3c04 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -166,6 +166,27 @@ export function buildCorsHeaders(origin: string | null, trusted: boolean): Recor return headers; } +/** + * Decodes one path segment between a fixed prefix (and optional suffix), + * returning `undefined` (never throwing) on any shape mismatch or malformed + * %-encoding (S083). Before this, each parameterized route hand-rolled its + * own decodeURIComponent inside the route's try block, so a malformed + * segment fell through to the OUTER catch and came back as a logged 500; + * every route using this helper instead gets a clean 400. + */ +export function pathParam(pathname: string, prefix: string, suffix = ""): string | undefined { + if (!pathname.startsWith(prefix)) return undefined; + if (suffix && !pathname.endsWith(suffix)) return undefined; + const end = suffix ? pathname.length - suffix.length : pathname.length; + if (end <= prefix.length) return undefined; + const raw = pathname.slice(prefix.length, end); + try { + return decodeURIComponent(raw); + } catch { + return undefined; + } +} + export interface ApiServerDeps { handleCommand: (cmd: string, payload: any, signal?: AbortSignal) => Promise; log: Logger; @@ -259,33 +280,36 @@ export async function startApiServer(deps: ApiServerDeps): Promise> // Single branch lookup: /api/cache/:branch if (url.pathname.startsWith("/api/cache/") && req.method === "GET") { - const branch = decodeURIComponent(url.pathname.slice("/api/cache/".length)); + const branch = pathParam(url.pathname, "/api/cache/"); + if (branch === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); + } const result = await handleCommand("cache:read", { branches: [branch] }, req.signal); return Response.json(result, { headers: corsHeaders }); } // Hooks repair: /api/hooks/:repo/repair if (url.pathname.startsWith("/api/hooks/") && url.pathname.endsWith("/repair") && req.method === "POST") { - const repo = decodeURIComponent(url.pathname.slice("/api/hooks/".length, -"/repair".length)); + const repo = pathParam(url.pathname, "/api/hooks/", "/repair"); + if (repo === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); + } const result = await handleCommand("hooks:repair", { repo }, req.signal); return Response.json(result, { headers: corsHeaders }); } // Runs detail: /api/runs/:repo/:runId if (url.pathname.startsWith("/api/runs/") && req.method === "GET") { - let rest: string | undefined; - try { - rest = decodeURIComponent(url.pathname.slice("/api/runs/".length)); - } catch { - rest = undefined; // malformed %-encoding -> fall through to the 404 path below + const rest = pathParam(url.pathname, "/api/runs/"); + if (rest === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); } - if (rest !== undefined) { - const slash = rest.indexOf("/"); - if (slash > 0 && slash < rest.length - 1) { - const result = await handleCommand("runs:get", { repo: rest.slice(0, slash), runId: rest.slice(slash + 1) }, req.signal); - return Response.json(result, { headers: corsHeaders }); - } + const slash = rest.indexOf("/"); + if (slash > 0 && slash < rest.length - 1) { + const result = await handleCommand("runs:get", { repo: rest.slice(0, slash), runId: rest.slice(slash + 1) }, req.signal); + return Response.json(result, { headers: corsHeaders }); } + // falls through to the 404 path below for a shape mismatch, e.g. "/api/runs/onlyonesegment" } // Secrets: forward the X-RT-Token header (already verified above by From e50895187324b7382f1e20ac060caf4db2832813 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:27:19 -0500 Subject: [PATCH 043/142] validate chat:join wakeOn and agent:start/resume surface against their enums (R033) --- lib/daemon/__tests__/agent-handlers.test.ts | 16 ++++++++++++++++ lib/daemon/__tests__/chat-handlers.test.ts | 15 +++++++++++++++ lib/daemon/handlers/agent.ts | 6 ++++++ lib/daemon/handlers/chat.ts | 10 ++++++++++ 4 files changed, 47 insertions(+) diff --git a/lib/daemon/__tests__/agent-handlers.test.ts b/lib/daemon/__tests__/agent-handlers.test.ts index 46f70fa6..848d77c9 100644 --- a/lib/daemon/__tests__/agent-handlers.test.ts +++ b/lib/daemon/__tests__/agent-handlers.test.ts @@ -119,6 +119,22 @@ test("agent:start headless refuses a missing prompt", async () => { expect(res.error).toMatch(/prompt/); }); +// R033: an unchecked surface value falls through to the headless spawn path +// with headless=false, spawning an interactive claude with stdin ignored +// and recording surface "bogus" — never a caller-visible error. +test("agent:start rejects a surface outside herdr/headless, naming the allowed values", async () => { + const h = fresh(); + const res = await h["agent:start"]({ repo: REPO, cwd: "/tmp/x", prompt: "hi", surface: "bogus" as any }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("surface"); + expect(res.error).toContain("herdr"); + expect(res.error).toContain("headless"); + const list = await h["agent:list"]({}); + if (!list.ok) throw new Error("unreachable"); + expect(list.data.agents).toHaveLength(0); +}); + test("agent:start headless finishes the record and emits agent/done", async () => { const emitted: string[] = []; let resolveExit!: (c: number) => void; diff --git a/lib/daemon/__tests__/chat-handlers.test.ts b/lib/daemon/__tests__/chat-handlers.test.ts index 399b80f1..9e8cfd31 100644 --- a/lib/daemon/__tests__/chat-handlers.test.ts +++ b/lib/daemon/__tests__/chat-handlers.test.ts @@ -44,6 +44,21 @@ test("chat:join rejects an invalid handle with a reason rather than normalizing expect(res.error).toContain("handle"); }); +// R033: an unchecked wakeOn value is stored on chat_members and, when the +// join creates the room, stamped into chat_room_defaults for every future +// joiner too — recipientsFromMembers treats it as neither "none" nor "all" +// (falls through to mention-only) and nothing ever reports the bad value. +test("chat:join rejects a wakeOn value outside mention/all/none, naming the allowed values", async () => { + const h = freshHandlers(); + const res = await h["chat:join"]({ room: "build", handle: "a", wakeOn: "sometimes" as any }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("wakeOn"); + expect(res.error).toContain("mention"); + expect(res.error).toContain("all"); + expect(res.error).toContain("none"); +}); + test("chat:post returns the recipients and emits one wake event per recipient", async () => { const emitted: string[] = []; const h = freshHandlers((topic) => { emitted.push(topic); return 0; }); diff --git a/lib/daemon/handlers/agent.ts b/lib/daemon/handlers/agent.ts index 9c548642..5273803c 100644 --- a/lib/daemon/handlers/agent.ts +++ b/lib/daemon/handlers/agent.ts @@ -135,6 +135,9 @@ export function createAgentHandlers(opts: { "agent:start": async (payload: Commands["agent:start"]["payload"]): Promise> => { const { repo, cwd } = payload; if (!repo || !cwd) return { ok: false, error: "agent:start requires repo (serialized identity) and cwd" }; + if (payload.surface !== undefined && payload.surface !== "herdr" && payload.surface !== "headless") { + return { ok: false, error: `invalid surface "${payload.surface}"; must be one of herdr, headless` }; + } const surface: AgentSurface = payload.surface ?? "herdr"; const prompt = payload.prompt; if (surface === "headless" && !prompt) { @@ -192,6 +195,9 @@ export function createAgentHandlers(opts: { "agent:resume": async (payload: Commands["agent:resume"]["payload"]): Promise> => { const rec = getAgent(payload.id, db); if (!rec) return { ok: false, error: `no agent record for "${payload.id}"` }; + if (payload.surface !== undefined && payload.surface !== "herdr" && payload.surface !== "headless") { + return { ok: false, error: `invalid surface "${payload.surface}"; must be one of herdr, headless` }; + } const surface: AgentSurface = payload.surface ?? rec.surface; if (surface === "headless" && !payload.prompt) { return { ok: false, error: "headless resume requires a prompt (claude -p with no prompt blocks on stdin)" }; diff --git a/lib/daemon/handlers/chat.ts b/lib/daemon/handlers/chat.ts index 80b05270..5d71c393 100644 --- a/lib/daemon/handlers/chat.ts +++ b/lib/daemon/handlers/chat.ts @@ -97,6 +97,13 @@ function closestRoomNames(typo: string, handle: string, db: Database): string[] return known.filter((r) => r.startsWith(typo) || typo.startsWith(r)).slice(0, 3); } +/** An unchecked value here lands on chat_members (and, on a join-creates, chat_room_defaults for every future joiner) and is silently treated as mention-only — never "none", never "all", never reported. */ +const VALID_WAKE_ON = ["mention", "all", "none"] as const; + +function isValidWakeOn(v: unknown): v is (typeof VALID_WAKE_ON)[number] { + return typeof v === "string" && (VALID_WAKE_ON as readonly string[]).includes(v); +} + /** * The row must commit before either emit fires, or a woken agent reads the * wake pointer and finds no message yet. Shared by chat:post and chat:dm so @@ -201,6 +208,9 @@ export function createChatHandlers(opts: { const { room, handle, wakeOn, cwd, pane } = payload; if (!isValidChatName(handle)) return { ok: false, error: `invalid handle "${handle}"` }; if (!isValidChatName(room)) return { ok: false, error: `invalid room "${room}"` }; + if (wakeOn !== undefined && !isValidWakeOn(wakeOn)) { + return { ok: false, error: `invalid wakeOn "${wakeOn}"; must be one of ${VALID_WAKE_ON.join(", ")}` }; + } try { const data = joinRoom({ room, handle, wakeOn, cwd, pane }, db); return { ok: true, data }; From 4b9afad5e7ed9c7481dbfe09738aecd35ceafd49 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:27:49 -0500 Subject: [PATCH 044/142] source-guards: update the boot-failure-exits guard for runDaemon owning the catch Review fix (round 2): the guard tested startDaemon's body for catch + process.exit(1), which fix-round-1 made stale by moving the catch-and-exit into runDaemon itself (startDaemon is now a thin await runDaemon()). Rewrote the guard to assert the invariant at its real location: runDaemon's body still owns catch + process.exit(1), and startDaemon just awaits it. Fixed the stale "startDaemon is a thin catch-and-exit wrapper" comment in the neighboring test to match. --- lib/state/__tests__/source-guards.test.ts | 38 ++++++++++++++--------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/lib/state/__tests__/source-guards.test.ts b/lib/state/__tests__/source-guards.test.ts index 8b0a6500..733a2965 100644 --- a/lib/state/__tests__/source-guards.test.ts +++ b/lib/state/__tests__/source-guards.test.ts @@ -92,9 +92,10 @@ describe("legacy JSON state files are retired", () => { describe("daemon startup opens state.db before serving", () => { test("openBranchCacheStore() precedes both server binds in runDaemon", () => { const src = readFileSync(join(REPO_ROOT, "lib", "daemon.ts"), "utf8"); - // startDaemon() itself is now a thin catch-and-exit wrapper (see - // "startDaemon exits on any runDaemon failure" below); the real ordered - // startup sequence this test asserts on lives in runDaemon(). + // startDaemon() itself is now just `await runDaemon()` (see "boot + // failure is fatal" below — the catch-and-exit lives in runDaemon + // itself); the real ordered startup sequence this test asserts on + // lives in runDaemon() too. const start = src.indexOf("async function runDaemon("); expect(start).toBeGreaterThan(-1); @@ -128,26 +129,33 @@ describe("daemon startup opens state.db before serving", () => { }); }); -describe("startDaemon exits on any runDaemon failure", () => { +describe("boot failure is fatal for both fire-and-forget callers", () => { test("runDaemon is not exported — startDaemon is the only safe entry point", () => { const src = readFileSync(join(REPO_ROOT, "lib", "daemon.ts"), "utf8"); expect(src).not.toContain("export async function runDaemon("); expect(src).not.toContain("export function runDaemon("); }); - test("startDaemon wraps runDaemon in try/catch and exits nonzero on failure", () => { + test("runDaemon wraps its own body in try/catch and exits nonzero on failure; startDaemon just awaits it", () => { const src = readFileSync(join(REPO_ROOT, "lib", "daemon.ts"), "utf8"); - const start = src.indexOf("export async function startDaemon("); - expect(start).toBeGreaterThan(-1); + const runStart = src.indexOf("async function runDaemon("); + const startDaemonStart = src.indexOf("export async function startDaemon("); + expect(runStart).toBeGreaterThan(-1); + expect(startDaemonStart).toBeGreaterThan(runStart); // Both real callers (cli.ts's --daemon entry, this file's own - // import.meta.main guard) invoke startDaemon() fire-and-forget — the - // catch-and-exit MUST live inside startDaemon itself, not at either - // call site, or an unhandledRejection silently leaves the daemon - // half-up (rt.sock possibly bound, nothing past the failure ever wired). - const body = src.slice(start, start + 400); - expect(body).toContain("await runDaemon()"); - expect(body).toContain("catch"); - expect(body).toContain("process.exit(1)"); + // import.meta.main guard) invoke startDaemon() fire-and-forget, and + // startDaemon() is now just `await runDaemon()` — so the catch-and-exit + // MUST live inside runDaemon() itself, or an unhandledRejection could + // silently leave the daemon half-up (rt.sock possibly bound, nothing + // past the failure ever wired). The booting-gated unhandledRejection + // handler (installCrashHandlers) is only the backstop for whatever + // still manages to slip past this try/catch. + const runBody = src.slice(runStart, startDaemonStart); + expect(runBody).toContain("catch"); + expect(runBody).toContain("process.exit(1)"); + + const startDaemonBody = src.slice(startDaemonStart, startDaemonStart + 200); + expect(startDaemonBody).toContain("await runDaemon()"); }); }); From fd935d752e5e32a6c91e7351140da2bc22ca5bb7 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:29:29 -0500 Subject: [PATCH 045/142] git-async: 5-min timeout for checkout/merge/stash/status so a large-repo checkout isn't killed half-applied (S104) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/__tests__/git-async-timeouts.test.ts | 14 +++++++++++++ lib/daemon/worktree-reconciler.ts | 9 ++++---- lib/worktree/git-async.ts | 26 +++++++++++++++++++----- 3 files changed, 40 insertions(+), 9 deletions(-) create mode 100644 lib/__tests__/git-async-timeouts.test.ts diff --git a/lib/__tests__/git-async-timeouts.test.ts b/lib/__tests__/git-async-timeouts.test.ts new file mode 100644 index 00000000..5ed9782c --- /dev/null +++ b/lib/__tests__/git-async-timeouts.test.ts @@ -0,0 +1,14 @@ +import { test, expect } from "bun:test"; +import { MUTATING_TIMEOUT_MS, stashChangesAsync, popStashAsync } from "../worktree/git-async.ts"; + +test("mutating timeout is 5 minutes", () => { + expect(MUTATING_TIMEOUT_MS).toBe(5 * 60_000); +}); + +test("stash helpers accept a timeout override", () => { + // Type-level: these must type-check with an opts arg. + const a: typeof stashChangesAsync = stashChangesAsync; + const b: typeof popStashAsync = popStashAsync; + expect(typeof a).toBe("function"); + expect(typeof b).toBe("function"); +}); diff --git a/lib/daemon/worktree-reconciler.ts b/lib/daemon/worktree-reconciler.ts index d427c942..8bf4c1b5 100644 --- a/lib/daemon/worktree-reconciler.ts +++ b/lib/daemon/worktree-reconciler.ts @@ -29,6 +29,7 @@ import { gitOk, headSha, listWorktreesAsync, + MUTATING_TIMEOUT_MS, remoteDefaultRef, runGit, stashChangesAsync, @@ -466,13 +467,13 @@ async function autoReturnMain( log.info({ ...fields }, `stashed uncommitted changes on "${mergedBranch}"`); } - const checkout = await runGit(rec.path, ["checkout", defaultBranch]); + const checkout = await runGit(rec.path, ["checkout", defaultBranch], { timeoutMs: MUTATING_TIMEOUT_MS }); if (checkout.exitCode !== 0) { log.warn({ ...fields, defaultBranch, output: checkout.stderr.trim() }, "auto-return: checkout failed"); return "retry"; } - const ff = await runGit(rec.path, ["merge", "--ff-only", defaultRef]); + const ff = await runGit(rec.path, ["merge", "--ff-only", defaultRef], { timeoutMs: MUTATING_TIMEOUT_MS }); if (ff.exitCode !== 0) { log.warn({ ...fields, defaultRef, output: ff.stderr.trim() }, "auto-return: fast-forward failed"); return "retry"; @@ -724,7 +725,7 @@ async function freshenOne(deps: FreshenDeps, rec: TreeRecord): Promise const classify = await classifyDirtyAsync(rec.path); if (classify.discard.length > 0) { - await runGit(rec.path, ["checkout", "--", ...classify.discard]); + await runGit(rec.path, ["checkout", "--", ...classify.discard], { timeoutMs: MUTATING_TIMEOUT_MS }); } // Blockers stashed under the tree's own branch name (Desktop-compatible @@ -753,7 +754,7 @@ async function freshenOne(deps: FreshenDeps, rec: TreeRecord): Promise } }; - const ff = await runGit(rec.path, ["merge", "--ff-only", defaultRef]); + const ff = await runGit(rec.path, ["merge", "--ff-only", defaultRef], { timeoutMs: MUTATING_TIMEOUT_MS }); if (ff.exitCode !== 0) { log.warn({ ...fields, defaultRef, output: ff.stderr.trim() }, "freshen: fast-forward failed"); await popStash(); diff --git a/lib/worktree/git-async.ts b/lib/worktree/git-async.ts index 1d994b66..7eea8562 100644 --- a/lib/worktree/git-async.ts +++ b/lib/worktree/git-async.ts @@ -32,6 +32,10 @@ const NO_HOOKS = ["-c", "core.hooksPath=/dev/null"]; const DEFAULT_TIMEOUT_MS = 60_000; +/** Checkout/merge/stash on a large tree can legitimately exceed a minute; a + * 60s SIGKILL leaves the working tree half-switched. Match fetch/worktree-add. */ +export const MUTATING_TIMEOUT_MS = 5 * 60_000; + const DESKTOP_STASH_RE = /!!GitHub_Desktop<(.+)>$/; /** Run a git command with hooks suppressed, capturing stdout+stderr. Never throws. */ @@ -64,7 +68,7 @@ export async function currentBranchAsync(cwd: string): Promise { } export async function statusPorcelainAsync(cwd: string): Promise { - const r = await runGit(cwd, ["status", "--porcelain"]); + const r = await runGit(cwd, ["status", "--porcelain"], { timeoutMs: MUTATING_TIMEOUT_MS }); return r.stdout; } @@ -174,13 +178,25 @@ export async function ensureInfoExclude(repoPath: string, pattern: string): Prom * Async port of git-ops.ts stashChanges — interoperable with GitHub Desktop * and worktree-context. */ -export async function stashChangesAsync(cwd: string, label: string): Promise { +export async function stashChangesAsync( + cwd: string, + label: string, + opts: { timeoutMs?: number } = {}, +): Promise { const message = `!!GitHub_Desktop<${label}>`; - await runGit(cwd, ["stash", "push", "-u", "-m", message]); + await runGit(cwd, ["stash", "push", "-u", "-m", message], { + timeoutMs: opts.timeoutMs ?? MUTATING_TIMEOUT_MS, + }); } -export async function popStashAsync(cwd: string, stashName: string): Promise { - await runGit(cwd, ["stash", "pop", stashName]); +export async function popStashAsync( + cwd: string, + stashName: string, + opts: { timeoutMs?: number } = {}, +): Promise { + await runGit(cwd, ["stash", "pop", stashName], { + timeoutMs: opts.timeoutMs ?? MUTATING_TIMEOUT_MS, + }); } /** From 2291068a362212fa2c4d0376aa620ecd9c3295c5 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:29:31 -0500 Subject: [PATCH 046/142] chat:read/chat:messages: clamp limit into [1,500] instead of reaching SQLite's unlimited negative LIMIT (R034) --- lib/daemon/__tests__/chat-handlers.test.ts | 42 +++++++++++++++++++++- lib/daemon/handlers/chat.ts | 20 +++++++++-- 2 files changed, 59 insertions(+), 3 deletions(-) diff --git a/lib/daemon/__tests__/chat-handlers.test.ts b/lib/daemon/__tests__/chat-handlers.test.ts index 9e8cfd31..4dca43ee 100644 --- a/lib/daemon/__tests__/chat-handlers.test.ts +++ b/lib/daemon/__tests__/chat-handlers.test.ts @@ -2,7 +2,7 @@ import { afterEach, beforeEach, expect, test } from "bun:test"; import type { Database } from "bun:sqlite"; import { tmpdir } from "os"; import { join } from "path"; -import { openStateDb } from "../../state/index.ts"; +import { openStateDb, postMessage } from "../../state/index.ts"; import { createChatHandlers, inviteText } from "../handlers/chat.ts"; import { herdrRequest } from "../../herdr/client.ts"; import { fakeHerdr, HerdrFakeError, type FakeHerdrHandler } from "../../herdr/__tests__/fake-herdr.ts"; @@ -187,6 +187,46 @@ test("chat:unread-waking reports what would wake a handle without advancing its expect(res2.data).toEqual(first); }); +// R034: `limit: -1` reaches `ORDER BY id ASC LIMIT ?`, where SQLite treats a +// negative LIMIT as unlimited, so a viewer/agent bug returns and +// JSON-serializes an entire (100k-row) room on the event loop. +test("chat:messages clamps a negative limit into [1,500] instead of reaching SQLite's unlimited LIMIT", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + for (let i = 0; i < 502; i++) postMessage({ room: "r", handle: "a", body: `msg ${i}` }, h.db); + const res = await h["chat:messages"]({ room: "r", limit: -1 }); + if (!res.ok) throw new Error("unreachable"); + expect(res.data.messages.length).toBeGreaterThanOrEqual(1); + expect(res.data.messages.length).toBeLessThanOrEqual(500); +}); + +test("chat:messages clamps an absurdly large limit to the cap", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + for (let i = 0; i < 502; i++) postMessage({ room: "r", handle: "a", body: `msg ${i}` }, h.db); + const res = await h["chat:messages"]({ room: "r", limit: 1_000_000 }); + if (!res.ok) throw new Error("unreachable"); + expect(res.data.messages.length).toBe(500); +}); + +test("chat:messages coerces a non-numeric limit to the default instead of a datatype 500", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + await h["chat:post"]({ room: "r", handle: "a", body: "hi" }); + const res = await h["chat:messages"]({ room: "r", limit: "lots" as any }); + expect(res.ok).toBe(true); +}); + +test("chat:read clamps a negative limit into [1,500] rather than reaching SQLite's unlimited LIMIT", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "b" }); // b's own cursor starts before every message below + for (let i = 0; i < 502; i++) postMessage({ room: "r", handle: "a", body: `msg ${i}` }, h.db); + const res = await h["chat:read"]({ handle: "b", limit: -1 } as any); + if (!res.ok) throw new Error("unreachable"); + expect(res.data.rooms[0]!.messages.length).toBeGreaterThanOrEqual(1); + expect(res.data.rooms[0]!.messages.length).toBeLessThanOrEqual(500); +}); + test("the read-only handlers mutate nothing", async () => { const h = freshHandlers(); await h["chat:join"]({ room: "r", handle: "a" }); diff --git a/lib/daemon/handlers/chat.ts b/lib/daemon/handlers/chat.ts index 5d71c393..3f4b9d52 100644 --- a/lib/daemon/handlers/chat.ts +++ b/lib/daemon/handlers/chat.ts @@ -97,6 +97,22 @@ function closestRoomNames(typo: string, handle: string, db: Database): string[] return known.filter((r) => r.startsWith(typo) || typo.startsWith(r)).slice(0, 3); } +/** + * `limit: -1` reaches `ORDER BY id ASC LIMIT ?`, where SQLite treats a + * negative LIMIT as unlimited, so a viewer/agent bug returns and + * JSON-serializes an entire (100k-row) room on the event loop; a + * non-numeric limit hits a datatype-mismatch SQLite error instead. + * Mirrors the events handler's `num()` coercion pattern. + */ +const MAX_CHAT_LIMIT = 500; + +function clampLimit(v: unknown, fallback: number): number { + if (v == null || v === "") return fallback; + const n = Number(v); + if (!Number.isFinite(n)) return fallback; + return Math.min(Math.max(Math.trunc(n), 1), MAX_CHAT_LIMIT); +} + /** An unchecked value here lands on chat_members (and, on a join-creates, chat_room_defaults for every future joiner) and is silently treated as mention-only — never "none", never "all", never reported. */ const VALID_WAKE_ON = ["mention", "all", "none"] as const; @@ -246,7 +262,7 @@ export function createChatHandlers(opts: { "chat:read": async (payload: Commands["chat:read"]["payload"]): Promise> => { const { handle, room, limit, sinceMs } = payload; - const rooms = readUnread({ handle, room, limit: limit ?? 20, sinceMs }, db); + const rooms = readUnread({ handle, room, limit: clampLimit(limit, 20), sinceMs }, db); return { ok: true, data: { rooms } }; }, @@ -298,7 +314,7 @@ export function createChatHandlers(opts: { "chat:messages": async (payload: Commands["chat:messages"]["payload"]): Promise> => { const { room, before, limit } = payload; - const messages = listMessages({ room, before, limit: limit ?? 50 }, db); + const messages = listMessages({ room, before, limit: clampLimit(limit, 50) }, db); return { ok: true, data: { messages } }; }, From 0bdb7e8f84afec0a79f45570127eb1afb17f3b17 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:31:26 -0500 Subject: [PATCH 047/142] daemon: coerce REST GET query params to number/boolean at the seam (S085) --- .../__tests__/api-server-query-coerce.test.ts | 35 +++++++++++++++++++ lib/daemon/api-server.ts | 23 +++++++++++- 2 files changed, 57 insertions(+), 1 deletion(-) create mode 100644 lib/daemon/__tests__/api-server-query-coerce.test.ts diff --git a/lib/daemon/__tests__/api-server-query-coerce.test.ts b/lib/daemon/__tests__/api-server-query-coerce.test.ts new file mode 100644 index 00000000..f3f28cb3 --- /dev/null +++ b/lib/daemon/__tests__/api-server-query-coerce.test.ts @@ -0,0 +1,35 @@ +import { describe, test, expect } from "bun:test"; +import { coerceQueryParams } from "../api-server.ts"; + +describe("coerceQueryParams", () => { + test("coerces maxAgeMs to a number (the documented cache:read flag)", () => { + const out = coerceQueryParams(new URLSearchParams("maxAgeMs=60000")); + expect(out.maxAgeMs).toBe(60000); + expect(typeof out.maxAgeMs).toBe("number"); + }); + + test("coerces refresh=true to a boolean (the documented ports flag)", () => { + const out = coerceQueryParams(new URLSearchParams("refresh=true")); + expect(out.refresh).toBe(true); + }); + + test("coerces refresh=false to a boolean false, not a truthy string", () => { + const out = coerceQueryParams(new URLSearchParams("refresh=false")); + expect(out.refresh).toBe(false); + }); + + test("leaves a non-numeric, non-boolean string alone", () => { + const out = coerceQueryParams(new URLSearchParams("repo=my-repo-name")); + expect(out.repo).toBe("my-repo-name"); + }); + + test("leaves an empty string alone rather than coercing to 0", () => { + const out = coerceQueryParams(new URLSearchParams("q=")); + expect(out.q).toBe(""); + }); + + test("coerces a decimal number too", () => { + const out = coerceQueryParams(new URLSearchParams("ratio=1.5")); + expect(out.ratio).toBe(1.5); + }); +}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index baef3c04..1f0dc0f0 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -187,6 +187,27 @@ export function pathParam(pathname: string, prefix: string, suffix = ""): string } } +const PLAIN_NUMBER_RE = /^-?\d+(\.\d+)?$/; + +/** + * REST query strings arrive as strings no matter what the client meant + * (S085): "?maxAgeMs=60000" and "?refresh=true" reached handlers that do a + * strict `typeof x === "number"` or `x === true` check, so the documented + * flag silently no-op'd over HTTP while working over the socket (where + * payloads are real JSON). One coercion at the REST seam fixes every such + * flag at once instead of a per-handler private parser. + */ +export function coerceQueryParams(params: URLSearchParams): Record { + const out: Record = {}; + for (const [key, value] of params) { + if (value === "true") out[key] = true; + else if (value === "false") out[key] = false; + else if (value !== "" && PLAIN_NUMBER_RE.test(value)) out[key] = Number(value); + else out[key] = value; + } + return out; +} + export interface ApiServerDeps { handleCommand: (cmd: string, payload: any, signal?: AbortSignal) => Promise; log: Logger; @@ -336,7 +357,7 @@ export async function startApiServer(deps: ApiServerDeps): Promise> if (req.method === "POST") { try { payload = await req.json(); } catch { /* empty body */ } } else { - payload = Object.fromEntries(url.searchParams); + payload = coerceQueryParams(url.searchParams); } const result = await handleCommand(route.cmd, payload, req.signal); From d96051b2b207250b48e4590c357d5fcc2053b69f Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:34:20 -0500 Subject: [PATCH 048/142] git-async: bump the 3 inline runGit(status/stash pop) call sites to MUTATING_TIMEOUT_MS Closes the S104 gap left by fd935d75: freshenOne's stash-pop reapply and autoReturnMain's two status --porcelain checks called runGit directly rather than through the helpers, so they were still on the 60s default. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/daemon/worktree-reconciler.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/daemon/worktree-reconciler.ts b/lib/daemon/worktree-reconciler.ts index 8bf4c1b5..3147d73f 100644 --- a/lib/daemon/worktree-reconciler.ts +++ b/lib/daemon/worktree-reconciler.ts @@ -452,14 +452,14 @@ async function autoReturnMain( } } - const status = await runGit(rec.path, ["status", "--porcelain"]); + const status = await runGit(rec.path, ["status", "--porcelain"], { timeoutMs: MUTATING_TIMEOUT_MS }); if (status.exitCode !== 0) { log.warn({ ...fields, output: status.stderr.trim() }, "auto-return: git status failed"); return "retry"; } if (status.stdout.trim().length > 0) { await stashChangesAsync(rec.path, mergedBranch); - const after = await runGit(rec.path, ["status", "--porcelain"]); + const after = await runGit(rec.path, ["status", "--porcelain"], { timeoutMs: MUTATING_TIMEOUT_MS }); if (after.exitCode !== 0 || after.stdout.trim().length > 0) { log.warn({ ...fields }, "auto-return: stash did not clear the worktree"); return "retry"; @@ -745,7 +745,7 @@ async function freshenOne(deps: FreshenDeps, rec: TreeRecord): Promise const popStash = async (): Promise => { if (!stashName) return; - const pop = await runGit(rec.path, ["stash", "pop", stashName]); + const pop = await runGit(rec.path, ["stash", "pop", stashName], { timeoutMs: MUTATING_TIMEOUT_MS }); if (pop.exitCode !== 0) { log.warn( { ...fields, stashName }, From 746515df55ca89a91d7578f239f6983258914ede Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:34:21 -0500 Subject: [PATCH 049/142] discussions-diffs test: fix fetch cast to satisfy tsc --- lib/daemon/__tests__/discussions-diffs.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/daemon/__tests__/discussions-diffs.test.ts b/lib/daemon/__tests__/discussions-diffs.test.ts index 7ea75ddf..bfd945c1 100644 --- a/lib/daemon/__tests__/discussions-diffs.test.ts +++ b/lib/daemon/__tests__/discussions-diffs.test.ts @@ -12,27 +12,27 @@ function fakeDiffs(n: number) { } test("truncated is false when fewer than a full page comes back", async () => { - const fetchFn = (async () => new Response(JSON.stringify(fakeDiffs(3)), { status: 200 })) as typeof fetch; + const fetchFn = (async () => new Response(JSON.stringify(fakeDiffs(3)), { status: 200 })) as unknown as typeof fetch; const out = await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); expect(out.diffs).toHaveLength(3); expect(out.truncated).toBe(false); }); test("truncated is true when exactly a full page (100) comes back", async () => { - const fetchFn = (async () => new Response(JSON.stringify(fakeDiffs(100)), { status: 200 })) as typeof fetch; + const fetchFn = (async () => new Response(JSON.stringify(fakeDiffs(100)), { status: 200 })) as unknown as typeof fetch; const out = await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); expect(out.diffs).toHaveLength(100); expect(out.truncated).toBe(true); }); test("maps new_path/diff to newPath/diff", async () => { - const fetchFn = (async () => new Response(JSON.stringify([{ new_path: "a.ts", diff: "@@" }]), { status: 200 })) as typeof fetch; + const fetchFn = (async () => new Response(JSON.stringify([{ new_path: "a.ts", diff: "@@" }]), { status: 200 })) as unknown as typeof fetch; const out = await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); expect(out.diffs).toEqual([{ newPath: "a.ts", diff: "@@" }]); }); test("a non-ok response throws with the status", async () => { - const fetchFn = (async () => new Response("", { status: 502 })) as typeof fetch; + const fetchFn = (async () => new Response("", { status: 502 })) as unknown as typeof fetch; await expect(fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn })).rejects.toThrow(/502/); }); From 2bd75c66a3284c5142b0c9662464de8af7b3537e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:37:34 -0500 Subject: [PATCH 050/142] daemon: log the EADDRINUSE port holder and throw a typed ApiPortInUseError --- lib/daemon/__tests__/api-server-bind.test.ts | 66 +++++++++++++++++++- lib/daemon/api-server.ts | 38 ++++++++++- 2 files changed, 102 insertions(+), 2 deletions(-) diff --git a/lib/daemon/__tests__/api-server-bind.test.ts b/lib/daemon/__tests__/api-server-bind.test.ts index 3fc417f4..44139ac4 100644 --- a/lib/daemon/__tests__/api-server-bind.test.ts +++ b/lib/daemon/__tests__/api-server-bind.test.ts @@ -1,5 +1,6 @@ import { describe, test, expect } from "bun:test"; import { bindApiServerWithRetry, BIND_RETRY_ATTEMPTS, BIND_RETRY_DELAY_MS, type BindRetryDeps } from "../api-server.ts"; +import { ApiPortInUseError } from "../api-server.ts"; function eaddrinuse(): Error { return Object.assign(new Error("EADDRINUSE"), { code: "EADDRINUSE" }); @@ -11,6 +12,7 @@ function deps(overrides: Partial = {}): BindRetryDeps & { logs: s return { sleep: async (ms: number) => { sleeps.push(ms); }, log: { warn: (_o: unknown, m: string) => logs.push(`warn:${m}`) }, + probePortHolder: async () => "n/a", logs, sleeps, ...overrides, @@ -41,7 +43,7 @@ describe("bindApiServerWithRetry", () => { expect(d.logs.some((l) => l.includes("retrying"))).toBe(true); }); - test("exhausting retries rethrows the original error after exactly BIND_RETRY_ATTEMPTS calls", async () => { + test("exhausting retries rethrows as ApiPortInUseError after exactly BIND_RETRY_ATTEMPTS calls", async () => { const d = deps(); let calls = 0; await expect( @@ -61,3 +63,65 @@ describe("bindApiServerWithRetry", () => { expect(d.sleeps).toEqual([]); }); }); + +function depsWithProbe(overrides: Partial = {}) { + const logs: Array<{ o: unknown; m: string }> = []; + const sleeps: number[] = []; + const probeCalls: number[] = []; + return { + sleep: async (ms: number) => { sleeps.push(ms); }, + log: { warn: (o: unknown, m: string) => logs.push({ o, m }) }, + probePortHolder: async (port: number) => { probeCalls.push(port); return "COMMAND PID USER\nnode 123 matt"; }, + logs, + sleeps, + probeCalls, + ...overrides, + }; +} + +describe("bindApiServerWithRetry — exhausted retries (S043)", () => { + test("throws ApiPortInUseError (not the raw EADDRINUSE Error) once attempts are exhausted", async () => { + const d = depsWithProbe(); + let error: unknown; + try { + await bindApiServerWithRetry(() => { throw eaddrinuse(); }, d); + } catch (err) { + error = err; + } + expect(error).toBeInstanceOf(ApiPortInUseError); + expect((error as ApiPortInUseError).code).toBe("EADDRINUSE"); + expect((error as Error).message).toContain("EADDRINUSE"); + }); + + test("probes the port holder exactly once, only after the final attempt", async () => { + const d = depsWithProbe(); + let calls = 0; + await expect( + bindApiServerWithRetry(() => { calls++; throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + expect(calls).toBe(BIND_RETRY_ATTEMPTS); + expect(d.probeCalls.length).toBe(1); + }); + + test("logs the probe result at warn before throwing", async () => { + const d = depsWithProbe(); + await expect( + bindApiServerWithRetry(() => { throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + const finalWarn = d.logs.at(-1)!; + expect(finalWarn.o).toMatchObject({ holder: expect.stringContaining("node") }); + }); + + test("a probe failure does not prevent the ApiPortInUseError from being thrown", async () => { + const d = depsWithProbe({ probePortHolder: async () => { throw new Error("lsof: command not found"); } }); + await expect( + bindApiServerWithRetry(() => { throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + }); + + test("a successful bind never probes", async () => { + const d = depsWithProbe(); + await bindApiServerWithRetry(() => "server" as any, d); + expect(d.probeCalls.length).toBe(0); + }); +}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index 1f0dc0f0..6c5bd2c4 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -13,6 +13,7 @@ import { API_PORT } from "../daemon-config.ts"; import { needsToken, tokenOk, getApiToken, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "./api-auth.ts"; import { getAggregatedConnection } from "./freshness.ts"; import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; +import { runCapture } from "../subprocess.ts"; const API_INDEX = { name: "rt daemon", @@ -213,9 +214,34 @@ export interface ApiServerDeps { log: Logger; } +/** + * Thrown when every bind retry is exhausted with EADDRINUSE still held. A + * named error type rather than the raw EADDRINUSE Error, so lib/daemon.ts's + * caller can distinguish "the port is genuinely squatted" from any other + * startup failure and park-and-retry with backoff instead of crash-looping + * (that caller-side change belongs to a sibling job; this class is the + * contract it wires into). + */ +export class ApiPortInUseError extends Error { + readonly code = "EADDRINUSE" as const; + readonly port: number; + constructor(port: number) { + super(`EADDRINUSE: api server port ${port} is still in use after ${BIND_RETRY_ATTEMPTS} bind attempts`); + this.name = "ApiPortInUseError"; + this.port = port; + } +} + export interface BindRetryDeps { sleep: (ms: number) => Promise; log: { warn: (o: unknown, m: string) => void }; + /** Defaults to a real `lsof -i :` via runCapture; overridable so tests never shell out. */ + probePortHolder?: (port: number) => Promise; +} + +async function defaultProbePortHolder(port: number): Promise { + const result = await runCapture(["lsof", "-i", `:${port}`], { timeoutMs: 5_000, stderr: "pipe" }); + return result.stdout.trim(); } // evictStaleDaemon (lib/daemon/boot-reconcile.ts) already assumes a prior @@ -231,14 +257,24 @@ export const BIND_RETRY_DELAY_MS = 500; * daemon can reach this bind before the old one has actually released * 9401. Only EADDRINUSE is retried (bounded, ~3s total); anything else is a * real misconfiguration and fails on the first attempt, same as before. + * + * Once retries are exhausted, this logs who holds the port and throws + * ApiPortInUseError instead of the bare EADDRINUSE Error, so a caller can + * tell "give up cleanly" apart from "the bind function itself is broken". */ export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDeps): Promise { + const probe = deps.probePortHolder ?? defaultProbePortHolder; for (let attempt = 1; ; attempt++) { try { return bind(); } catch (err) { const isAddrInUse = err instanceof Error && (err as NodeJS.ErrnoException).code === "EADDRINUSE"; - if (!isAddrInUse || attempt >= BIND_RETRY_ATTEMPTS) throw err; + if (!isAddrInUse) throw err; + if (attempt >= BIND_RETRY_ATTEMPTS) { + const holder = await probe(API_PORT).catch((probeErr) => `lsof failed: ${String(probeErr)}`); + deps.log.warn({ port: API_PORT, holder }, "api port still in use after retries; giving up bind (the daemon should park and retry with backoff rather than crash-loop)"); + throw new ApiPortInUseError(API_PORT); + } deps.log.warn({ attempt, port: API_PORT }, "api port in use, retrying — another daemon is likely still shutting down"); await deps.sleep(BIND_RETRY_DELAY_MS); } From 150486f2f924525ab75d38eddf486791bd0f61b6 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:39:46 -0500 Subject: [PATCH 051/142] cache-refresh: async git + grant-gated doppler loop; add listWorktreeRootsAsync (S008, S045, S021) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../git-worktree-roots-async.test.ts | 20 +++++++++++ lib/daemon/__tests__/cache-refresh-gc.test.ts | 12 ++++--- lib/daemon/cache-refresh.ts | 36 ++++++++----------- lib/worktree/git-async.ts | 6 ++++ 4 files changed, 47 insertions(+), 27 deletions(-) create mode 100644 lib/__tests__/git-worktree-roots-async.test.ts diff --git a/lib/__tests__/git-worktree-roots-async.test.ts b/lib/__tests__/git-worktree-roots-async.test.ts new file mode 100644 index 00000000..3c933432 --- /dev/null +++ b/lib/__tests__/git-worktree-roots-async.test.ts @@ -0,0 +1,20 @@ +import { test, expect } from "bun:test"; +import { mkdtempSync } from "fs"; +import { realpathSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { listWorktreeRootsAsync } from "../worktree/git-async.ts"; +import { runGit } from "../worktree/git-async.ts"; + +test("listWorktreeRootsAsync returns the main worktree path", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-wt-"))); + await runGit(dir, ["init", "-q"]); + await runGit(dir, ["commit", "--allow-empty", "-m", "init", "-c", "user.email=a@b.c", "-c", "user.name=t"]); + const roots = await listWorktreeRootsAsync(dir); + expect(roots).toContain(dir); +}); + +test("listWorktreeRootsAsync returns [] on a non-repo", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-nonrepo-"))); + expect(await listWorktreeRootsAsync(dir)).toEqual([]); +}); diff --git a/lib/daemon/__tests__/cache-refresh-gc.test.ts b/lib/daemon/__tests__/cache-refresh-gc.test.ts index db295330..a959a4d1 100644 --- a/lib/daemon/__tests__/cache-refresh-gc.test.ts +++ b/lib/daemon/__tests__/cache-refresh-gc.test.ts @@ -31,7 +31,7 @@ import { join } from "path"; import type { Logger } from "pino"; import * as enrichModule from "../../enrich.ts"; -import * as gitWorktreesModule from "../../git-worktrees.ts"; +import * as gitAsync from "../../worktree/git-async.ts"; import * as notifierModule from "../../notifier.ts"; import * as repoTrackingModule from "../../repo-tracking.ts"; import * as discussionsModule from "../discussions-file-store.ts"; @@ -101,11 +101,13 @@ function wireCycle(): Wiring { }); // A real git tree is irrelevant to the GC claim; one branch per repo is - // enough to make the loop reach the enrichment call. - spyOn(gitWorktreesModule, "listWorktrees").mockImplementation((repoPath: string) => [ - { path: repoPath, branch: `wt-${repoPath}` } as any, + // enough to make the loop reach the enrichment call and let FLAKY's onError + // fire (an empty branch list would short-circuit refreshAllMRs entirely and + // make FLAKY look clean, defeating the test). + spyOn(gitAsync, "listWorktreesAsync").mockImplementation(async (repoPath: string) => [ + { path: repoPath, branch: `wt-${repoPath}` }, ]); - spyOn(gitWorktreesModule, "listWorktreeRoots").mockReturnValue([]); + spyOn(gitAsync, "listWorktreeRootsAsync").mockResolvedValue([]); spyOn(enrichModule, "refreshAllMRs").mockImplementation( async (_branches, _remoteUrl, onError, repoName) => { diff --git a/lib/daemon/cache-refresh.ts b/lib/daemon/cache-refresh.ts index 70194bd9..0ab58fc7 100644 --- a/lib/daemon/cache-refresh.ts +++ b/lib/daemon/cache-refresh.ts @@ -6,13 +6,12 @@ * * Concurrent callers are coalesced: the 5-minute timer and `cache:refresh` IPC * both fire-and-forget into the returned function. Without a guard they stack - * up, each running execSync across every repo + a batch GraphQL. If a refresh + * up, each running async git across every repo + a batch GraphQL. If a refresh * is already in flight, callers await the existing run instead of starting a * second one. */ import { existsSync } from "fs"; -import { execSync } from "child_process"; import type { Logger } from "pino"; import type { PortCacheRef, RepoIndex } from "./handlers/types.ts"; import type { BranchCacheStore } from "../state/index.ts"; @@ -24,7 +23,7 @@ import { getProjectMRs } from "./project-mrs-store.ts"; import { pruneDiscussionsStore } from "./discussions-file-store.ts"; import { reconcileForRepo } from "./doppler-sync.ts"; import { deriveRepoIdentity } from "../settings/identity.ts"; -import { listWorktreeRoots, listWorktrees } from "../git-worktrees.ts"; +import { listWorktreesAsync, listWorktreeRootsAsync, runGit } from "../worktree/git-async.ts"; export interface CacheRefresherDeps { log: Logger; @@ -109,37 +108,29 @@ export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise = listWorktrees(repoPath).filter( - (w) => w.branch && !w.branch.startsWith("on-deck/"), - ); + const branches: Array<{ path: string; branch: string }> = ((await listWorktreesAsync(repoPath)) ?? []) + .filter((w): w is { path: string; branch: string } => !!w.branch && !w.branch.startsWith("on-deck/")); // 2. Discover local branches (not just worktrees) const worktreeBranchSet = new Set(branches.map(b => b.branch)); - try { - const localBranchOutput = execSync( - "git for-each-ref --format='%(refname:short)' refs/heads/", - { cwd: repoPath, encoding: "utf8", stdio: "pipe" }, - ); - - for (const name of localBranchOutput.split("\n")) { - const trimmed = name.trim().replace(/^'|'$/g, ""); + const localBranches = await runGit(repoPath, ["for-each-ref", "--format=%(refname:short)", "refs/heads/"]); + if (localBranches.exitCode === 0) { + for (const name of localBranches.stdout.split("\n")) { + const trimmed = name.trim(); if (!trimmed || worktreeBranchSet.has(trimmed) || trimmed.startsWith("on-deck/")) continue; if (extractLinearId(trimmed)) { branches.push({ path: repoPath, branch: trimmed }); } } - } catch (err) { - log.warn({ err, repo: repoPath }, "local branch listing failed"); + } else { + log.warn({ repo: repoPath }, "local branch listing failed"); } if (branches.length > 0) { // Get remote URL let remoteUrl: string | undefined; - try { - remoteUrl = execSync("git config --get remote.origin.url", { - cwd: repoPath, encoding: "utf8", stdio: "pipe", - }).trim(); - } catch { /* no remote */ } + const remote = await runGit(repoPath, ["config", "--get", "remote.origin.url"]); + if (remote.exitCode === 0) remoteUrl = remote.stdout.trim() || undefined; // Optimized: 3 GraphQL calls for ALL open MRs + 1 Linear batch. // The onError callback fires on per-MR enrich failures (GitLab, @@ -205,8 +196,9 @@ export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise { + return (await listWorktreesAsync(repoPath) ?? []).map((w) => w.path); +} + /** * Idempotently append `pattern` to the common git dir's info/exclude, with a * "# rt worktree" marker comment written on first use of this file. Returns From f292058ec5648dc66e0e84842a0159ade7299440 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:40:46 -0500 Subject: [PATCH 052/142] daemon: bind API before socket; register rt.apiPort setting + resolveApiPort() Binding the API server first means a fatal API-bind failure never strands a socket-bound zombie behind it. rt.apiPort is the escape-hatch setting the api-server sibling consumes at bind time; resolveApiPort() resolves it lazily (env > setting > 9401) without disturbing the existing API_PORT const api-server.ts already imports. --- e2e/tests/daemon.test.ts | 20 +++++++++++++++++++ lib/__tests__/daemon-config.test.ts | 14 ++++++++++++- lib/daemon-config.ts | 17 ++++++++++++++++ lib/daemon.ts | 6 ++++-- .../src/settings/__tests__/registry.test.ts | 6 ++++-- .../rt-client/src/settings/registry-defs.ts | 9 +++++++++ 6 files changed, 67 insertions(+), 5 deletions(-) diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index 6b4af74d..e0403285 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -37,6 +37,26 @@ describe("fatal boot", () => { } }, 60_000); + test("API-bind failure leaves neither rt.sock nor rt.pid", async () => { + const { path: home, cleanup } = createTestHome(); + // A different port than the sibling squatter test above, so parallel + // test files can never collide on the same bound TCP port. + const port = 9412; + const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + try { + const result = await rt(["--daemon"], { home, env: { RT_API_PORT: String(port) } }); + + expect(result.exitCode).not.toBe(0); + // The API bind (now first) fails before the socket ever binds, so a + // fatal exit must strand neither file. + expect(existsSync(join(home, ".mattstack", "rt", "rt.sock"))).toBe(false); + expect(existsSync(join(home, ".mattstack", "rt", "rt.pid"))).toBe(false); + } finally { + squatter.stop(true); + cleanup(); + } + }, 60_000); + test("a corrupt events.db self-heals — quarantined, and the daemon boots and serves", async () => { const { path: home, cleanup } = createTestHome(); const bunDir = join(process.execPath, ".."); diff --git a/lib/__tests__/daemon-config.test.ts b/lib/__tests__/daemon-config.test.ts index 21fc45b5..32f06d15 100644 --- a/lib/__tests__/daemon-config.test.ts +++ b/lib/__tests__/daemon-config.test.ts @@ -9,7 +9,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { mkdirSync, rmSync, writeFileSync } from "fs"; import { join } from "path"; -import { activeLaunchdLabel } from "../daemon-config.ts"; +import { activeLaunchdLabel, resolveApiPort } from "../daemon-config.ts"; const WRAPPER_PATH = join(process.env.HOME!, ".local", "bin", "rt"); @@ -28,3 +28,15 @@ describe("activeLaunchdLabel", () => { expect(activeLaunchdLabel()).toBe("com.mattstack.daemon.dev"); }); }); + +describe("resolveApiPort", () => { + test("env wins, then setting, then 9401", () => { + const prev = process.env.RT_API_PORT; + process.env.RT_API_PORT = "12345"; + expect(resolveApiPort()).toBe(12345); + delete process.env.RT_API_PORT; + expect(resolveApiPort()).toBe(9401); // default setting value + if (prev !== undefined) process.env.RT_API_PORT = prev; + else delete process.env.RT_API_PORT; + }); +}); diff --git a/lib/daemon-config.ts b/lib/daemon-config.ts index 4164f10e..52f4a246 100644 --- a/lib/daemon-config.ts +++ b/lib/daemon-config.ts @@ -12,6 +12,7 @@ import { homedir } from "os"; import { dirname, join } from "path"; import { rtDir } from "./rt-paths.ts"; import { currentMode } from "./dev-mode.ts"; +import { getSetting } from "./settings/resolve.ts"; // ─── Types ─────────────────────────────────────────────────────────────────── @@ -71,6 +72,22 @@ export const TRAY_SOCK_PATH = join(RT_DIR, "tray.sock"); * daemon's hardcoded port (RT-45). */ export const API_PORT = Number(process.env.RT_API_PORT) || 9401; +/** + * Call-time API port resolution: RT_API_PORT env wins (e2e isolation, RT-45), + * then the rt.apiPort setting (escape hatch when 9401 is held), then 9401. + * A function, not a const — must never be evaluated at module load, since + * getSetting() reads the settings stores off ambient HOME. + */ +export function resolveApiPort(): number { + const env = Number(process.env.RT_API_PORT); + if (env) return env; + try { + return getSetting("rt.apiPort").value || 9401; + } catch { + return 9401; + } +} + // ─── Read / Write ──────────────────────────────────────────────────────────── // `home`, when passed, overrides the module-load `DAEMON_CONFIG_PATH` const diff --git a/lib/daemon.ts b/lib/daemon.ts index ea49efa2..d2997eb0 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -479,9 +479,11 @@ async function runDaemon(): Promise { persistOrWarn("daemon", () => { prunedPresence = prunePresence(Date.now()); }, { op: "prunePresence" }); if (prunedPresence > 0) log.info({ prunedPresence }, "chat: pruned stale presence rows at daemon startup"); - // Socket server (Unix socket for CLI/tray) + REST/WS server (external clients) - servers.socket = startSocketServer({ handleCommand, log }); + // API server first: a failed bind exits fatally (boot-phase catch below), + // and binding API before the unix socket means that fatal exit never + // strands a socket-bound zombie behind it. servers.api = await startApiServer({ handleCommand, log }); + servers.socket = startSocketServer({ handleCommand, log }); // Only write rt.pid once both servers are actually bound — a boot that // fails before this point must never leave a live-pid file with no diff --git a/packages/rt-client/src/settings/__tests__/registry.test.ts b/packages/rt-client/src/settings/__tests__/registry.test.ts index 1b534ea0..6500c111 100644 --- a/packages/rt-client/src/settings/__tests__/registry.test.ts +++ b/packages/rt-client/src/settings/__tests__/registry.test.ts @@ -51,7 +51,7 @@ describe("settings/registry", () => { } }); - test("exactly 22 keys are migrated:true", () => { + test("exactly 23 keys are migrated:true", () => { const migrated = allDefs().filter((d) => d.migrated); expect(migrated.map((d) => d.key).sort()).toEqual( @@ -60,6 +60,7 @@ describe("settings/registry", () => { "rt.notifications", "rt.cron", "rt.repoTracking", "rt.runsPruneDays", "rt.runaway", "rt.workspacePrefs", "rt.sync", "rt.branchNaming", "rt.variations", "rt.presets", "rt.dopplerTemplate", "rt.homeSnapshot", "rt.worktreeApp", "rt.sdmEnrichment", "rt.logRetentionDays", "rt.integrations", "rt.hooks", + "rt.apiPort", ].sort(), ); }); @@ -197,13 +198,14 @@ describe("settings/registry", () => { expect(def?.merge).toBe("replace"); }); - test("has exactly the 22 migrated:true keys and the 42 suite keys", () => { + test("has exactly the 23 migrated:true keys and the 42 suite keys", () => { const migratedFalseKeys: string[] = []; const migratedTrueKeys = [ "rt.roles", "rt.intercepts", "rt.worktrees", "rt.repoIdentityOverrides", "rt.repoRoots", "rt.notifications", "rt.cron", "rt.repoTracking", "rt.runsPruneDays", "rt.runaway", "rt.workspacePrefs", "rt.sync", "rt.branchNaming", "rt.variations", "rt.presets", "rt.dopplerTemplate", "rt.homeSnapshot", "rt.worktreeApp", "rt.sdmEnrichment", "rt.logRetentionDays", "rt.integrations", "rt.hooks", + "rt.apiPort", ]; const suiteKeys = [ "mattstack.integrations", diff --git a/packages/rt-client/src/settings/registry-defs.ts b/packages/rt-client/src/settings/registry-defs.ts index f0109ddd..9f37dfd4 100644 --- a/packages/rt-client/src/settings/registry-defs.ts +++ b/packages/rt-client/src/settings/registry-defs.ts @@ -196,6 +196,15 @@ export const REGISTRY: readonly SettingDef[] = [ migrated: true, description: "Age floor in days for the log janitor pruning every surface's rotated log files under ~/.mattstack/rt/logs (default 14). A fresh key, not an ownership-latch port, so a default is fine here.", }, + { + key: "rt.apiPort", + type: "number", + scopes: ["machine", "user"], + default: 9401, + merge: "replace", + migrated: true, + description: "TCP port the daemon's local HTTP/WS API binds (escape hatch when 9401 is held).", + }, { key: "rt.hooks", type: "object", From a458351d11dd08cf077a912738fefad54c08860e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:41:03 -0500 Subject: [PATCH 053/142] daemon: standalone git-ref validator for option-injection guard (worktree.ts wiring documented, not wired here) --- .../__tests__/git-ref-validation.test.ts | 36 +++++++++++++++++++ lib/daemon/git-ref-validation.ts | 24 +++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 lib/daemon/__tests__/git-ref-validation.test.ts create mode 100644 lib/daemon/git-ref-validation.ts diff --git a/lib/daemon/__tests__/git-ref-validation.test.ts b/lib/daemon/__tests__/git-ref-validation.test.ts new file mode 100644 index 00000000..27c9fc3d --- /dev/null +++ b/lib/daemon/__tests__/git-ref-validation.test.ts @@ -0,0 +1,36 @@ +import { describe, test, expect } from "bun:test"; +import { isSafeGitRef, validateGitRef } from "../git-ref-validation.ts"; + +describe("isSafeGitRef", () => { + test("a normal branch name is safe", () => { + expect(isSafeGitRef("feature/my-branch")).toBe(true); + }); + + test("a leading dash is unsafe (option injection)", () => { + expect(isSafeGitRef("--upload-pack=touch /tmp/x")).toBe(false); + }); + + test("a bare dash is unsafe", () => { + expect(isSafeGitRef("-")).toBe(false); + }); + + test("an empty string is unsafe", () => { + expect(isSafeGitRef("")).toBe(false); + }); + + test("a branch containing a dash mid-name is safe", () => { + expect(isSafeGitRef("job/p3-trust-boundary")).toBe(true); + }); +}); + +describe("validateGitRef", () => { + test("returns ok:true for a safe ref", () => { + expect(validateGitRef("main")).toEqual({ ok: true }); + }); + + test("returns ok:false with the offending ref named in the error for an unsafe one", () => { + const result = validateGitRef("--upload-pack=x"); + expect(result.ok).toBe(false); + expect((result as { ok: false; error: string }).error).toContain("--upload-pack=x"); + }); +}); diff --git a/lib/daemon/git-ref-validation.ts b/lib/daemon/git-ref-validation.ts new file mode 100644 index 00000000..809a733f --- /dev/null +++ b/lib/daemon/git-ref-validation.ts @@ -0,0 +1,24 @@ +/** + * Rejects a branch/ref string that git would parse as an OPTION rather than + * a ref. A caller-supplied `branch` like "--upload-pack=touch /tmp/x" reaches + * `git fetch origin ` and `git rev-list ......` unescaped, + * and git happily executes it as an option since nothing on that path + * validates the string first. This is the one guard both call sites need; + * any future caller (including a consumer app forwarding an untrusted string + * as `branch`) inherits the same hole without it. + * + * Deliberately narrow: reject a leading '-' rather than allowlisting a + * character set. `git check-ref-format --branch` accepts far more + * punctuation than is worth re-deriving here, and the vulnerable shape is + * specifically "parses as an option", not "contains an unusual character". + */ +export function isSafeGitRef(ref: string): boolean { + return ref.length > 0 && !ref.startsWith("-"); +} + +export function validateGitRef(ref: string): { ok: true } | { ok: false; error: string } { + if (!isSafeGitRef(ref)) { + return { ok: false, error: `unsafe git ref (starts with '-' or empty): ${ref}` }; + } + return { ok: true }; +} From 977c03df47fc60705f6c9d6dc2683d7cd6787fda Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:43:55 -0500 Subject: [PATCH 054/142] daemon: standalone credential-redaction utility (freshness.ts wiring documented, not wired here) --- .../__tests__/redact-credentials.test.ts | 35 +++++++++++++++++++ lib/daemon/redact-credentials.ts | 14 ++++++++ 2 files changed, 49 insertions(+) create mode 100644 lib/daemon/__tests__/redact-credentials.test.ts create mode 100644 lib/daemon/redact-credentials.ts diff --git a/lib/daemon/__tests__/redact-credentials.test.ts b/lib/daemon/__tests__/redact-credentials.test.ts new file mode 100644 index 00000000..57e5a5e2 --- /dev/null +++ b/lib/daemon/__tests__/redact-credentials.test.ts @@ -0,0 +1,35 @@ +import { describe, test, expect } from "bun:test"; +import { redactCredentials } from "../redact-credentials.ts"; + +describe("redactCredentials", () => { + test("redacts userinfo (user:token@) out of an https remote URL", () => { + const input = "https://oauth2:glpat-XXXXXXXXXXXXXXXXXXXX@gitlab.example.com/g/p.git"; + const out = redactCredentials(input); + expect(out).not.toContain("glpat-XXXXXXXXXXXXXXXXXXXX"); + expect(out).toContain("gitlab.example.com/g/p.git"); + }); + + test("redacts a GitHub PAT embedded in the URL", () => { + const input = "https://ghp_abcdefghijklmnopqrstuvwxyz012345@github.com/o/r.git"; + const out = redactCredentials(input); + expect(out).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz012345"); + expect(out).toContain("github.com/o/r.git"); + }); + + test("leaves a URL with no embedded credentials unchanged", () => { + const input = "https://gitlab.example.com/g/p.git"; + expect(redactCredentials(input)).toBe(input); + }); + + test("leaves plain text with no URL unchanged", () => { + const input = "local branch listing failed"; + expect(redactCredentials(input)).toBe(input); + }); + + test("redacts every match when more than one credential-bearing URL appears in the same string", () => { + const input = "tried https://oauth2:tok1@a.example/x then https://oauth2:tok2@b.example/y"; + const out = redactCredentials(input); + expect(out).not.toContain("tok1"); + expect(out).not.toContain("tok2"); + }); +}); diff --git a/lib/daemon/redact-credentials.ts b/lib/daemon/redact-credentials.ts new file mode 100644 index 00000000..2ac64c87 --- /dev/null +++ b/lib/daemon/redact-credentials.ts @@ -0,0 +1,14 @@ +/** + * Strips userinfo (user:token@ or user@) out of any http(s) URL embedded in + * a string. freshness.ts logs `remote.origin.url` verbatim on every + * reconcile and echoes it into thrown errors returned to callers; a repo + * cloned as `https://oauth2:glpat-XXXX@gitlab.example.com/...` (routine for + * dotfiles/CI-derived clones) puts that token into ~/.rt/logs/daemon.*.log + * and into any client-facing error message. Logs are the first thing a user + * pastes into a bug report. + */ +const CREDENTIAL_URL_RE = /(https?:\/\/)[^/@\s]+@/gi; + +export function redactCredentials(text: string): string { + return text.replace(CREDENTIAL_URL_RE, "$1[redacted]@"); +} From 7d1a2fad26a451cc5d22db220145f2965df8dfaf Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:44:49 -0500 Subject: [PATCH 055/142] freshness: async, cached getRemoteUrl via runCapture (R032) getRemoteUrl used execSync on the daemon thread inside every forge handler and the freshness reconcile loop. Switch to runCapture (5s timeout) and cache per repoPath for the process lifetime, since remotes rarely change. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/freshness-remote-url.test.ts | 9 +++++ lib/daemon/freshness.ts | 33 +++++++++++-------- 2 files changed, 29 insertions(+), 13 deletions(-) create mode 100644 lib/daemon/__tests__/freshness-remote-url.test.ts diff --git a/lib/daemon/__tests__/freshness-remote-url.test.ts b/lib/daemon/__tests__/freshness-remote-url.test.ts new file mode 100644 index 00000000..ec17a900 --- /dev/null +++ b/lib/daemon/__tests__/freshness-remote-url.test.ts @@ -0,0 +1,9 @@ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("freshness.ts no longer imports execSync/child_process", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + expect(src).not.toMatch(/from\s+["']child_process["']/); + expect(src).not.toMatch(/\bexecSync\b/); +}); diff --git a/lib/daemon/freshness.ts b/lib/daemon/freshness.ts index a5f7ceba..2b49b497 100644 --- a/lib/daemon/freshness.ts +++ b/lib/daemon/freshness.ts @@ -26,7 +26,6 @@ * disposeFreshness() — daemon shutdown */ -import { execSync } from "child_process"; import { GitLabProvider, type InvalidationKey, type MRApprovalRules, type PullRequest } from "@mattstack/glance"; import { loadRepoTracking, grants, type RepoGrants } from "../repo-tracking.ts"; import { loadSecrets } from "../linear.ts"; @@ -37,6 +36,7 @@ import { lazyChildLogger } from "../daemon-logger.ts"; import { getProjectMRs, type ProjectMRs } from "./project-mrs-store.ts"; import { getDiscussionsFileStore } from "./discussions-file-store.ts"; import { createCursorStore, type CursorStore } from "../state/index.ts"; +import { runCapture } from "../subprocess.ts"; const log = lazyChildLogger("freshness"); @@ -90,16 +90,23 @@ let userId: number | null = null; let userIdResolved = false; let selfUsername: string | null = null; +const remoteUrlCache = new Map(); + // ─── Helpers ───────────────────────────────────────────────────────────────── -function getRemoteUrl(repoPath: string): string | null { - try { - return execSync("git config --get remote.origin.url", { - cwd: repoPath, encoding: "utf8", stdio: "pipe", - }).trim(); - } catch { - return null; - } +/** remote.origin.url, cached per repoPath for the process lifetime (remotes + * rarely change). Async so it never blocks the event loop. */ +async function getRemoteUrl(repoPath: string): Promise { + const cached = remoteUrlCache.get(repoPath); + if (cached !== undefined) return cached; + const r = await runCapture(["git", "config", "--get", "remote.origin.url"], { + cwd: repoPath, + timeoutMs: 5000, + stderr: "ignore", + }); + const url = r.exitCode === 0 ? (r.stdout.trim() || null) : null; + remoteUrlCache.set(repoPath, url); + return url; } /** The same debug-accounting hook makeProvider wires, for SDK helpers rt constructs directly (NoteMutator). */ @@ -132,7 +139,7 @@ async function ensureProvider(repoName: string, repoPath: string): Promise { // startWatch rather than trust a check made before it. if (watches.has(repoName)) continue; - const remoteUrl = getRemoteUrl(repoPath); + const remoteUrl = await getRemoteUrl(repoPath); const remote = remoteUrl ? parseRemoteUrl(remoteUrl) : null; if (!remote) continue; From cf525960aee61099af1293a1e49122bcbebb614a Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:49:07 -0500 Subject: [PATCH 056/142] worktree-process-kill: async lsof/ps via runCapture (S015, S016) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/worktree-process-kill.test.ts | 8 +++ lib/daemon/worktree-process-kill.ts | 57 ++++++++----------- lib/daemon/worktree-reconciler.ts | 5 +- lib/worktree/dispose.ts | 2 +- 4 files changed, 35 insertions(+), 37 deletions(-) diff --git a/lib/daemon/__tests__/worktree-process-kill.test.ts b/lib/daemon/__tests__/worktree-process-kill.test.ts index faf05ed2..c86ebb8e 100644 --- a/lib/daemon/__tests__/worktree-process-kill.test.ts +++ b/lib/daemon/__tests__/worktree-process-kill.test.ts @@ -1,3 +1,5 @@ +import { readFileSync } from "fs"; +import { resolve } from "path"; import { describe, expect, test } from "bun:test"; import { selectKillTargets, type KillCandidate } from "../worktree-process-kill.ts"; @@ -5,6 +7,12 @@ function row(pid: number, ppid: number, command: string, fullCommand: string): K return { pid, ppid, command, fullCommand }; } +test("worktree-process-kill.ts imports no sync exec", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "worktree-process-kill.ts"), "utf8"); + expect(src).not.toMatch(/from\s+["']child_process["']/); + expect(src).not.toMatch(/\bexecSync\b/); +}); + describe("selectKillTargets", () => { test("kills package-script chains and orphaned compilers", () => { const rows = [ diff --git a/lib/daemon/worktree-process-kill.ts b/lib/daemon/worktree-process-kill.ts index ca499bc2..e41ee707 100644 --- a/lib/daemon/worktree-process-kill.ts +++ b/lib/daemon/worktree-process-kill.ts @@ -12,9 +12,8 @@ * the caller — parking proceeds while stragglers wind down. */ -import { execSync } from "child_process"; - import { lazyChildLogger } from "../daemon-logger.ts"; +import { runCapture } from "../subprocess.ts"; import { parseLsofCwdMap, parsePackageScripts } from "./system-process-scanner.ts"; const log = lazyChildLogger("worktree-kill"); @@ -126,18 +125,14 @@ export interface WorktreeKillResult { * Discovery is done fresh (not from the scanner's 10s-old snapshot) so we act * on ground truth at park time. */ -export function killWorktreeProcesses(worktreePath: string): WorktreeKillResult { - let lsofOut: string; - try { - lsofOut = execSync("lsof -d cwd -Fpn 2>/dev/null", { - encoding: "utf8", stdio: "pipe", timeout: 10000, - }); - } catch (err) { - log.warn({ err, worktreePath }, "lsof failed; skipping worktree process kill"); +export async function killWorktreeProcesses(worktreePath: string): Promise { + const lsof = await runCapture(["lsof", "-d", "cwd", "-Fpn"], { timeoutMs: 10_000 }); + if (lsof.exitCode !== 0 && !lsof.stdout) { + log.warn({ exitCode: lsof.exitCode, worktreePath }, "lsof failed; skipping worktree process kill"); return { terminated: [] }; } - const cwdMap = parseLsofCwdMap(lsofOut, [worktreePath]); + const cwdMap = parseLsofCwdMap(lsof.stdout, [worktreePath]); // Drop close-parent matches (parseLsofCwdMap keeps them for scanner display; // a process merely *above* the worktree must not be killed). for (const [pid, cwd] of cwdMap) { @@ -147,24 +142,21 @@ export function killWorktreeProcesses(worktreePath: string): WorktreeKillResult const pidList = [...cwdMap.keys()].join(","); const rows: KillCandidate[] = []; - try { - const psOut = execSync(`ps -p ${pidList} -o pid=,ppid=,comm=,args= 2>/dev/null`, { - encoding: "utf8", stdio: "pipe", timeout: 5000, - }); - for (const line of psOut.split("\n")) { - const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\S+)\s+(.*)/); - if (!match) continue; - rows.push({ - pid: parseInt(match[1]!, 10), - ppid: parseInt(match[2]!, 10), - command: basename(match[3]!), - fullCommand: match[4]!, - }); - } - } catch (err) { - log.warn({ err, worktreePath }, "ps failed; skipping worktree process kill"); + const ps = await runCapture(["ps", "-p", pidList, "-o", "pid=,ppid=,comm=,args="], { timeoutMs: 5000 }); + if (ps.exitCode !== 0 && !ps.stdout) { + log.warn({ exitCode: ps.exitCode, worktreePath }, "ps failed; skipping worktree process kill"); return { terminated: [] }; } + for (const line of ps.stdout.split("\n")) { + const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\S+)\s+(.*)/); + if (!match) continue; + rows.push({ + pid: parseInt(match[1]!, 10), + ppid: parseInt(match[2]!, 10), + command: basename(match[3]!), + fullCommand: match[4]!, + }); + } const targets = selectKillTargets(rows, { protectedPids: [process.pid, process.ppid], @@ -174,12 +166,11 @@ export function killWorktreeProcesses(worktreePath: string): WorktreeKillResult // Label with the package-script invocation when known — that's the name the // developer recognizes ("pnpm start:lite:watch", not "node"). let scripts = new Map(); - try { - const ewwOut = execSync(`ps eww -o pid=,command= -p ${targets.map(t => t.pid).join(",")} 2>/dev/null`, { - encoding: "utf8", stdio: "pipe", timeout: 5000, maxBuffer: 32 * 1024 * 1024, - }); - scripts = parsePackageScripts(ewwOut); - } catch { /* labels fall back to comm */ } + const eww = await runCapture( + ["ps", "eww", "-o", "pid=,command=", "-p", targets.map(t => t.pid).join(",")], + { timeoutMs: 5000 }, + ); + if (eww.exitCode === 0 || eww.stdout) scripts = parsePackageScripts(eww.stdout); const terminated: { pid: number; label: string }[] = []; for (const target of targets) { diff --git a/lib/daemon/worktree-reconciler.ts b/lib/daemon/worktree-reconciler.ts index 3147d73f..1746c735 100644 --- a/lib/daemon/worktree-reconciler.ts +++ b/lib/daemon/worktree-reconciler.ts @@ -442,10 +442,9 @@ async function autoReturnMain( } if (appConfig.killProcesses) { - // The ruled execSync exception (the process killer is sync by design); - // a failure here never blocks the return. + // A failure here never blocks the return. try { - const { terminated } = killWorktreeProcesses(rec.path); + const { terminated } = await killWorktreeProcesses(rec.path); if (terminated.length > 0) log.info({ ...fields, count: terminated.length }, "worktree processes terminated"); } catch (err) { log.warn({ err, ...fields }, "auto-return: process kill failed; returning anyway"); diff --git a/lib/worktree/dispose.ts b/lib/worktree/dispose.ts index 52e2bb1d..3f3efeb0 100644 --- a/lib/worktree/dispose.ts +++ b/lib/worktree/dispose.ts @@ -235,7 +235,7 @@ export async function disposeTree( } if (deps.killProcesses) { - const { terminated } = killWorktreeProcesses(rec.path); + const { terminated } = await killWorktreeProcesses(rec.path); if (terminated.length > 0) { log.info( { repo: repoName, tree: rec.name, count: terminated.length }, From 6d950e3624b5a725cb7e1573e819cf4749036115 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:50:28 -0500 Subject: [PATCH 057/142] home-snapshot: lazy daemon-flavored state.db; getStateDb re-applies a stronger flavor timeout startHomeSnapshot no longer resolves getStateDb() eagerly at construction with the default cli flavor (5000ms busy_timeout). db access is a thunk (resolveDb) defaulting to getStateDb("daemon"), first invoked inside init() after its await (past module-scope construction, so the daemon's own openBranchCacheStore() opens the singleton daemon-flavored first). getStateDb() also now re-applies PRAGMA busy_timeout when a caller requests a stronger (shorter) flavor than the already-open singleton holds, hardening against any other future eager-cli-then-daemon ordering bug. --- lib/daemon/__tests__/home-snapshot.test.ts | 37 ++++++++++++++++++++++ lib/daemon/home-snapshot.ts | 19 ++++++++--- lib/state/__tests__/db.test.ts | 7 ++++ lib/state/db.ts | 16 +++++++--- 4 files changed, 70 insertions(+), 9 deletions(-) diff --git a/lib/daemon/__tests__/home-snapshot.test.ts b/lib/daemon/__tests__/home-snapshot.test.ts index 569aa4c6..ba0bbba3 100644 --- a/lib/daemon/__tests__/home-snapshot.test.ts +++ b/lib/daemon/__tests__/home-snapshot.test.ts @@ -1005,6 +1005,43 @@ describe("startHomeSnapshot — state persistence", () => { }); }); +// ─── boot order: db must open daemon-flavored, never at construction ──────── + +describe("startHomeSnapshot — boot order", () => { + test("constructing startHomeSnapshot does not open the state.db singleton before the caller's next await", async () => { + const home = mkdtempSync(join(tmpdir(), "rt-home-snapshot-bootorder-")); + const origHome = process.env.HOME; + process.env.HOME = home; + closeStateDb(); + try { + const stateDbPath = join(home, ".mattstack", "rt", "state.db"); + const { fn: execFn } = makeFakeExec(defaultResponders({ statusZ: "?? a.txt\0" })); + // No `db` override: this exercises the real getStateDb() singleton, + // matching lib/daemon.ts's module-scope `startHomeSnapshot(...)` call — + // the exact call site that used to open state.db "cli"-flavored before + // startDaemon() ever got to openBranchCacheStore(). + const { deps } = baseDeps({ exec: execFn, db: undefined }); + + const handle = startHomeSnapshot(deps); + + // Synchronously, right after construction returns — mirroring the + // module-scope call in lib/daemon.ts, which runs to completion before + // startDaemon() (and its openBranchCacheStore() daemon-flavored open) + // is ever reached — no db file may exist yet. + expect(existsSync(stateDbPath)).toBe(false); + + await handle.ready; + // First real use (init()'s loadState, past its own `await deps.exec`) + // has by now opened it. + expect(existsSync(stateDbPath)).toBe(true); + } finally { + process.env.HOME = origHome; + closeStateDb(); + rmSync(home, { recursive: true, force: true }); + } + }); +}); + // ─── stop() ─────────────────────────────────────────────────────────────── describe("startHomeSnapshot — stop", () => { diff --git a/lib/daemon/home-snapshot.ts b/lib/daemon/home-snapshot.ts index 0d0fb56c..476fbee2 100644 --- a/lib/daemon/home-snapshot.ts +++ b/lib/daemon/home-snapshot.ts @@ -258,6 +258,11 @@ function persistPushRecord(db: Database, record: HomePushRecord, log: Logger): v export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle { const repoDir = rawDeps.repoDir ?? join(mattstackHome(), "user"); const rawReadSettings = rawDeps.readSettings ?? (() => getSetting("rt.homeSnapshot").value); + // Thunk, not a resolved value: module-scope construction (lib/daemon.ts) + // must not open state.db before startDaemon() has opened it daemon-flavored + // via openBranchCacheStore — see loadState's call site inside init() below, + // which is the first place this ever actually gets invoked. + const resolveDb = rawDeps.db ? (() => rawDeps.db!) : (() => getStateDb("daemon")); const deps = { log: rawDeps.log, broadcast: rawDeps.broadcast, @@ -269,7 +274,6 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle now: rawDeps.now ?? (() => Date.now()), readSettings: () => clampSettings(rawReadSettings()), readOwners: rawDeps.readOwners ?? readOwnersReal, - db: rawDeps.db ?? getStateDb(), }; const ownersPath = ownersPathFor(deps.repoDir); @@ -296,7 +300,8 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle let lastPushError: string | null = null; /** True once `home:push-failed` has been broadcast for the CURRENT unbroken run of push failures — reset to false the moment a push succeeds, so a retry storm broadcasts once, not on every attempt. */ let pushFailureBroadcast = false; - let firstSeenDirty: Record = loadState(deps.db, deps.log); + /** Populated in init(), after the is-inside-work-tree check — see resolveDb's comment for why this can't happen at construction time. */ + let firstSeenDirty: Record = {}; let lastLoggedOwnersError: string | null = null; /** Shared dedup key for every "deps.readSettings() itself threw" warn (armWatcher's debounce read, status()) — a settings store that broke after boot and stays broken must warn once, not on every fs event or every `rt home snapshot --status` poll. */ let lastLoggedSettingsError: string | null = null; @@ -370,6 +375,10 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle deps.log.warn({ repoDir: deps.repoDir }, "home-snapshot: repoDir is not a git repository; inert"); return; } + // First real db touch: this await already put us past the daemon's + // synchronous boot pass, so by now startDaemon() has opened state.db + // daemon-flavored via openBranchCacheStore (see resolveDb above). + firstSeenDirty = loadState(resolveDb(), deps.log); if (deps.readSettings().enabled !== false) { tryArm(); } @@ -525,7 +534,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle pushFailureBroadcast = false; lastPushAt = deps.now(); lastPushError = null; - persistPushRecord(deps.db, { at: lastPushAt, ok: true }, deps.log); + persistPushRecord(resolveDb(), { at: lastPushAt, ok: true }, deps.log); if (pushRetryTimer) { deps.clearTimeout(pushRetryTimer); pushRetryTimer = null; @@ -538,7 +547,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle const redactedStderr = redactCredentials(result.stderr); pushPending = true; lastPushError = redactedStderr; - persistPushRecord(deps.db, { at: deps.now(), ok: false, error: redactedStderr }, deps.log); + persistPushRecord(resolveDb(), { at: deps.now(), ok: false, error: redactedStderr }, deps.log); deps.log.warn({ stderr: redactedStderr }, "home-snapshot: push failed"); // Only the FIRST failure of an unbroken streak broadcasts — a retry // storm (schedulePushRetry firing every pushDelaySec*5) would @@ -642,7 +651,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle }); firstSeenDirty = plan.nextFirstSeenDirty; - persistState(deps.db, firstSeenDirty, deps.log); + persistState(resolveDb(), firstSeenDirty, deps.log); let committed = false; let sha: string | null = null; diff --git a/lib/state/__tests__/db.test.ts b/lib/state/__tests__/db.test.ts index 63777538..3d7f5f2d 100644 --- a/lib/state/__tests__/db.test.ts +++ b/lib/state/__tests__/db.test.ts @@ -351,6 +351,13 @@ describe("pragma values per flavor", () => { expect(timeout).toBe(5000); db.close(); }); + + test("getStateDb('daemon') reports busy_timeout 250 even after a default open", () => { + const cli = getStateDb(); // opens singleton, cli flavor + expect(cli.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 5000 }); + const daemon = getStateDb("daemon"); // same singleton — must not stay at 5000 + expect(daemon.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); + }); }); describe("startup busy budget — open+migrate blocks, it does not throw", () => { diff --git a/lib/state/db.ts b/lib/state/db.ts index 86cbeafc..99478f33 100644 --- a/lib/state/db.ts +++ b/lib/state/db.ts @@ -521,11 +521,19 @@ export function stateDbPath(): string { */ export function getStateDb(flavor: DbFlavor = "cli"): Database { const path = stateDbPath(); - if (!singleton || singletonPath !== path) { - singleton?.close(); - singleton = openStateDb(path, flavor); - singletonPath = path; + if (singleton && singletonPath === path) { + // A caller asking for a stronger (shorter) contention policy than the + // singleton currently holds must not silently inherit whatever flavor + // opened it first (e.g. a "cli" 5000ms opener beating the daemon's own + // "daemon" 250ms open) — re-tighten in place rather than reopening. + const want = BUSY_TIMEOUT_MS[flavor]; + const have = Number((singleton.query("PRAGMA busy_timeout").get() as { timeout?: number } | null)?.timeout ?? 0); + if (want < have) singleton.exec(`PRAGMA busy_timeout = ${want};`); + return singleton; } + singleton?.close(); + singleton = openStateDb(path, flavor); + singletonPath = path; return singleton; } From baaffc99919c19b49aefc3444230de0f5ed805f2 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:54:16 -0500 Subject: [PATCH 058/142] docs: the :9401 trust boundary model and the S010/S050/S043 sibling wiring notes --- docs/daemon-api-auth.md | 49 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 docs/daemon-api-auth.md diff --git a/docs/daemon-api-auth.md b/docs/daemon-api-auth.md new file mode 100644 index 00000000..1eb33045 --- /dev/null +++ b/docs/daemon-api-auth.md @@ -0,0 +1,49 @@ +# The :9401 trust boundary + +How the daemon's REST/WS surface decides who to trust, and the follow-up +wiring two standalone modules from this phase still need in sibling-owned +files. + +## The model + +- **No `Origin` header at all** (the CLI, the Swift tray, rt-client from a + Bun/Node process, mr-board, gitq, the VS Code extension): unaffected, no + gate applies beyond what already existed. None of today's consumers send + an `Origin` header to `:9401`. +- **A browser `Origin` header is present**: trusted only if the request + presents the local `X-RT-Token` (`?token=` query param for `/ws`, since + browsers cannot set custom headers on a WS handshake) OR the Origin is on + the `rt.trustedBrowserOrigins` settings allowlist (see + `packages/rt-client/src/settings/registry-defs.ts`; `docs/settings-architecture.md` + is the settings-system contract). Otherwise: no `Access-Control-Allow-Origin` + on REST reads (default-deny CORS), and a 403 on `/ws`. +- **Mutating routes** (every method except GET/HEAD/OPTIONS, plus + `/api/secrets` and `/api/notifications` despite being GETs) require the + local `X-RT-Token` regardless of Origin... this is the CSRF defense against + a browser form/simple-request bypassing CORS preflight entirely, and it is + orthogonal to the Origin check above. + +See `lib/daemon/api-auth.ts` (`isBrowserRequestTrusted`, `needsToken`, +`getTrustedBrowserOrigins`) and `lib/daemon/api-server.ts` +(`buildCorsHeaders`, the `/ws` gate in `fetch()`) for the implementation. + +## Follow-up wiring for sibling-owned files (not done in this job) + +**S010** (`lib/daemon/handlers/worktree.ts`): `lib/daemon/git-ref-validation.ts` +exports `validateGitRef(ref)`. Call it right after `payload.branch` is read +(around `worktree.ts:282`) and return `{ ok: false, error }` on a rejection +BEFORE any `runGit` call reaches it... that single call site also covers the +weaker secondary instance in `divergence()` (`worktree.ts:211-213`), since +both read the same `branch` value. + +**S050** (`lib/daemon/freshness.ts`): `lib/daemon/redact-credentials.ts` +exports `redactCredentials(text)`. Wrap every log/error interpolation of a +remote URL with it... the audit names `freshness.ts:142, 148, 275, 279` as the +current call sites. + +**S043 caller side** (`lib/daemon.ts`): `lib/daemon/api-server.ts` exports +`ApiPortInUseError` (a named `Error` subclass with `.name === "ApiPortInUseError"` +and `.port`). Catch it around the `startApiServer()` call and park-and-retry +with backoff instead of letting it reach the top-level crash path; any other +error out of `startApiServer()` is a genuine misconfiguration and should keep +crashing as it does today. From fed0ce32a0041326e6df3524197804f7153f1722 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 10:58:30 -0500 Subject: [PATCH 059/142] repo-index: async observed-main-path on the endpoint:claim resolve path (S098) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/__tests__/repo-index-async.test.ts | 9 +++++++++ lib/repo-index.ts | 10 +++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) create mode 100644 lib/__tests__/repo-index-async.test.ts diff --git a/lib/__tests__/repo-index-async.test.ts b/lib/__tests__/repo-index-async.test.ts new file mode 100644 index 00000000..fdcc423e --- /dev/null +++ b/lib/__tests__/repo-index-async.test.ts @@ -0,0 +1,9 @@ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("resolveIndexPathForIdentity no longer reaches a sync git via observedMainPath", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "repo-index.ts"), "utf8"); + // observedMainPath (sync execSync) must not be called from the async resolver path. + expect(src).toMatch(/observedMainPathAsync/); +}); diff --git a/lib/repo-index.ts b/lib/repo-index.ts index e5ddf614..648d6489 100644 --- a/lib/repo-index.ts +++ b/lib/repo-index.ts @@ -32,6 +32,7 @@ import { repoLabel, repoLabelFull, repoLabelQualified } from "./repo-label.ts"; import { dim } from "./ansi.ts"; import { getSetting } from "./settings/resolve.ts"; import { mergeRegistries, type TreeRecord } from "./worktree/registry.ts"; +import { listWorktreesAsync } from "./worktree/git-async.ts"; // ─── Types ─────────────────────────────────────────────────────────────────── @@ -151,6 +152,13 @@ function observedMainPath(repoRoot: string): string { } } +/** Async twin of observedMainPath: the repo's MAIN worktree path as git + * reports it, degrading to repoRoot. Safe on the daemon thread. */ +async function observedMainPathAsync(repoRoot: string): Promise { + const wts = await listWorktreesAsync(repoRoot); + return wts?.[0]?.path ?? repoRoot; +} + /** * The row's current path, read straight from the namespace rather than through * `loadRepoIndex()`: that function's legacy-repos.json import is a migration @@ -274,7 +282,7 @@ export async function resolveIndexPathForIdentity(serialized: string): Promise Date: Fri, 28 Aug 2026 11:02:01 -0500 Subject: [PATCH 060/142] fix rt-paths lint and registry test regressions on job/p3-trust-boundary redact-credentials.ts: reword the doc comment so it no longer spells out a literal .rt-prefixed path (rt-paths.test.ts bans that outside lib/rt-paths.ts); the rationale is unchanged. registry.test.ts: add the rt.trustedBrowserOrigins key (introduced by an earlier task on this branch) to the expected suiteKeys list and bump the length assertion from 42 to 43. Co-Authored-By: Claude Sonnet 5 --- lib/daemon/redact-credentials.ts | 6 +++--- packages/rt-client/src/settings/__tests__/registry.test.ts | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/daemon/redact-credentials.ts b/lib/daemon/redact-credentials.ts index 2ac64c87..b46d6f55 100644 --- a/lib/daemon/redact-credentials.ts +++ b/lib/daemon/redact-credentials.ts @@ -3,9 +3,9 @@ * a string. freshness.ts logs `remote.origin.url` verbatim on every * reconcile and echoes it into thrown errors returned to callers; a repo * cloned as `https://oauth2:glpat-XXXX@gitlab.example.com/...` (routine for - * dotfiles/CI-derived clones) puts that token into ~/.rt/logs/daemon.*.log - * and into any client-facing error message. Logs are the first thing a user - * pastes into a bug report. + * dotfiles/CI-derived clones) puts that token into the daemon's rotated log + * files and into any client-facing error message. Logs are the first thing + * a user pastes into a bug report. */ const CREDENTIAL_URL_RE = /(https?:\/\/)[^/@\s]+@/gi; diff --git a/packages/rt-client/src/settings/__tests__/registry.test.ts b/packages/rt-client/src/settings/__tests__/registry.test.ts index 1b534ea0..36305dec 100644 --- a/packages/rt-client/src/settings/__tests__/registry.test.ts +++ b/packages/rt-client/src/settings/__tests__/registry.test.ts @@ -248,8 +248,9 @@ describe("settings/registry", () => { "agent.effort", "agent.account", "agent.extraArgs", + "rt.trustedBrowserOrigins", ]; - expect(suiteKeys).toHaveLength(42); + expect(suiteKeys).toHaveLength(43); expect(allDefs().map((d) => d.key).sort()).toEqual( [...migratedFalseKeys, ...migratedTrueKeys, ...suiteKeys].sort(), From 3037b275df2f0556cf4e80d7a06098fd6c076a6c Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:02:15 -0500 Subject: [PATCH 061/142] state: isBusyError matches SQLITE_BUSY_*; read-then-write daemon txns use BEGIN IMMEDIATE joinRoom/archiveRoom/readUnread (chat-store.ts), dmRoomFor (dm-store.ts), and drainNotificationQueue (notifier-store.ts) read before they write inside a db.transaction; bun:sqlite's plain transaction() defers BEGIN, so a concurrent writer can produce SQLITE_BUSY_SNAPSHOT that busy_timeout cannot absorb. .immediate() takes the write lock up front instead. presence-store.ts:signIn has the same shape but is a sibling-owned write-fence file; left untouched as a documented follow-up. --- lib/state/__tests__/busy.test.ts | 23 ++++++++++++++++++++++- lib/state/busy.ts | 10 ++++++++-- lib/state/chat-store.ts | 7 ++++--- lib/state/dm-store.ts | 2 +- lib/state/notifier-store.ts | 2 +- 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/lib/state/__tests__/busy.test.ts b/lib/state/__tests__/busy.test.ts index 1f8eb70f..01791698 100644 --- a/lib/state/__tests__/busy.test.ts +++ b/lib/state/__tests__/busy.test.ts @@ -1,5 +1,9 @@ import { expect, test } from "bun:test"; -import { runCriticalWrite } from "../busy.ts"; +import { Database } from "bun:sqlite"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { isBusyError, runCriticalWrite } from "../busy.ts"; test("returns the value when fn succeeds", () => { expect(runCriticalWrite("t", () => 42, {})).toBe(42); @@ -26,3 +30,20 @@ test("returns undefined after exhausting attempts on a busy error", () => { test("rethrows a non-busy error rather than retrying", () => { expect(() => runCriticalWrite("t", () => { throw new Error("syntax error"); }, {})).toThrow("syntax error"); }); + +test("isBusyError matches SQLITE_BUSY_SNAPSHOT from a real conflict", () => { + const dir = mkdtempSync(join(tmpdir(), "busy-snap-")); + const path = join(dir, "t.db"); + const a = new Database(path); a.exec("PRAGMA journal_mode=WAL; CREATE TABLE t(id INTEGER PRIMARY KEY, v INTEGER);"); + a.exec("INSERT INTO t(id,v) VALUES(1,0);"); + const b = new Database(path); + a.exec("BEGIN;"); a.query("SELECT v FROM t WHERE id=1").get(); // pin snapshot on A + b.exec("UPDATE t SET v=1 WHERE id=1;"); // B commits (autocommit) + let caught: unknown; + try { a.exec("UPDATE t SET v=2 WHERE id=1;"); } catch (e) { caught = e; } + expect(caught).toBeDefined(); + expect((caught as any).code?.startsWith("SQLITE_BUSY")).toBe(true); + expect(isBusyError(caught)).toBe(true); + try { a.exec("ROLLBACK;"); } catch {} + a.close(); b.close(); +}); diff --git a/lib/state/busy.ts b/lib/state/busy.ts index 1124378c..c7de1f55 100644 --- a/lib/state/busy.ts +++ b/lib/state/busy.ts @@ -35,9 +35,15 @@ import type { DaemonLoggerHandle } from "../daemon-logger.ts"; let logHandle: Promise | null = null; -/** True for the bun:sqlite error thrown when a write can't get the lock inside busy_timeout. */ +/** + * True for the bun:sqlite error thrown when a write can't get the lock + * inside busy_timeout — including the SNAPSHOT/RECOVERY variants a + * deferred-BEGIN read-then-write transaction can throw, which busy_timeout + * does not retry the way it retries a plain SQLITE_BUSY. + */ export function isBusyError(err: unknown): boolean { - return (err as { code?: string } | undefined)?.code === "SQLITE_BUSY"; + const code = (err as { code?: string } | undefined)?.code; + return code === "SQLITE_BUSY" || (typeof code === "string" && code.startsWith("SQLITE_BUSY_")); } function warnBusy(module: string, context: Record): void { diff --git a/lib/state/chat-store.ts b/lib/state/chat-store.ts index dd4d569c..6177fa61 100644 --- a/lib/state/chat-store.ts +++ b/lib/state/chat-store.ts @@ -234,7 +234,8 @@ export function joinRoom( return { handle, memberCount, unread: Math.max(0, maxId - lastReadId) }; }); - return run(); + // BEGIN IMMEDIATE: read-then-write must lock up front or SQLITE_BUSY_SNAPSHOT bypasses busy_timeout. + return run.immediate(); } export function leaveRoom(room: string, handle: string, db: Database = getStateDb()): void { @@ -293,7 +294,7 @@ export function archiveRoom( if (current === null) db.query(UPDATE_ROOM_ARCHIVED_SQL).run(archivedAt, room); return { room, archivedAt }; }); - return run(); + return run.immediate(); } /** The wake mode stamped by whichever join created `room`; undefined for a room never stamped (including every DM room — dmRoomFor never stamps one). */ @@ -508,7 +509,7 @@ export function readUnread( return results; }); - return run(); + return run.immediate(); } export function listMessages( diff --git a/lib/state/dm-store.ts b/lib/state/dm-store.ts index 39c66b9f..f7d712f4 100644 --- a/lib/state/dm-store.ts +++ b/lib/state/dm-store.ts @@ -72,7 +72,7 @@ export function dmRoomFor( return { room, created: true }; }); - return run(); + return run.immediate(); } export function dmParticipants(room: string, db: Database = getStateDb()): { a: string; b: string } | null { diff --git a/lib/state/notifier-store.ts b/lib/state/notifier-store.ts index 682a822a..0b08238c 100644 --- a/lib/state/notifier-store.ts +++ b/lib/state/notifier-store.ts @@ -108,7 +108,7 @@ export function drainNotificationQueue(db: Database = getStateDb()): Notificatio db.exec(`DELETE FROM notify_queue;`); return rows.map(rowToEvent); }); - return runCriticalWrite("drain", () => run(), {}) ?? []; + return runCriticalWrite("drain", () => run.immediate(), {}) ?? []; } /** Peek reads without deleting — diagnostics, no mutation, no retry needed. */ From f76d9949084c4726f11affccbdb8dfbd88760853 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:06:02 -0500 Subject: [PATCH 062/142] repo-index: fix write-back regression in resolveIndexPathForIdentity setIndexPath is a bare KV write; it skipped the moved-repo guard, the repos.json compat mirror, and the unopenable-db degrade that writeIndexRow (and thus updateRepoIndex) provides. Swap to writeIndexRow, which takes the already-resolved async main path and preserves all three, with no sync git reintroduced. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/repo-index.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/repo-index.ts b/lib/repo-index.ts index 648d6489..6a8a4f0f 100644 --- a/lib/repo-index.ts +++ b/lib/repo-index.ts @@ -282,7 +282,7 @@ export async function resolveIndexPathForIdentity(serialized: string): Promise Date: Fri, 28 Aug 2026 11:09:05 -0500 Subject: [PATCH 063/142] gate: fail on sync-exec anywhere in the daemon import graph (1.3) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/__tests__/no-daemon-sync-exec.test.ts | 83 +++++++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 lib/__tests__/no-daemon-sync-exec.test.ts diff --git a/lib/__tests__/no-daemon-sync-exec.test.ts b/lib/__tests__/no-daemon-sync-exec.test.ts new file mode 100644 index 00000000..2ba99d37 --- /dev/null +++ b/lib/__tests__/no-daemon-sync-exec.test.ts @@ -0,0 +1,83 @@ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { dirname, resolve } from "path"; + +// Files with sync-exec that Phase 1 does NOT remove. Each entry names the +// finding/phase that will delete it, so this list shrinks as later phases land. +// A regression that reintroduces sync-exec into any OTHER daemon-reachable +// module fails this gate (the rule has been re-broken twice). +const ALLOWLIST = new Set([ + "lib/daemon/user-path.ts", // Phase 6 PATH rebuild (S013/S014/S062) + "lib/daemon/boot-reconcile.ts", // Phase 0.6 / S044 (Bun.sleepSync) + "lib/state/db.ts", // Phase 0.7 / S072-S073 busy-retry + "lib/state/busy.ts", // Phase 0.7 / S072-S073 busy-retry + "lib/git-worktrees.ts", // S055: reached only via handlers/status.ts + "lib/daemon/handlers/status.ts", // S055: the edge into git-worktrees.ts + "lib/repo-index.ts", // Phase 5.3 dedup (heal/derive execSync) + "lib/repo.ts", // R050 / Phase 5.4 (via handlers/system-processes.ts) + "lib/git.ts", // R050 / Phase 5.4 (via repo.ts) + "lib/herdr-launch.ts", // Phase 5 herdr (via handlers/pane.ts) + "lib/rt-render.tsx", // R050 / Phase 5.4 (daemon carries the TUI) +]); + +const SYNC_EXEC = [ + /\bexecSync\s*\(/, + /\bspawnSync\s*\(/, + /\bBun\.spawnSync\s*\(/, + /\bBun\.sleepSync\s*\(/, +]; + +const REPO_ROOT = resolve(import.meta.dir, "..", ".."); +const stripShebang = (s: string) => s.replace(/^#!.*\n/, ""); +const tsT = new Bun.Transpiler({ loader: "ts" }); +const tsxT = new Bun.Transpiler({ loader: "tsx" }); +const loaderFor = (f: string) => (f.endsWith(".tsx") || f.endsWith(".jsx") ? tsxT : tsT); + +/** Files reachable from lib/daemon.ts via relative imports (the daemon graph). */ +function daemonClosure(): string[] { + const entry = resolve(REPO_ROOT, "lib/daemon.ts"); + const visited = new Set(); + const stack = [entry]; + while (stack.length) { + const file = stack.pop()!; + if (visited.has(file)) continue; + visited.add(file); + let src: string; + try { src = stripShebang(readFileSync(file, "utf8")); } catch { continue; } + let imports: { path: string }[]; + try { imports = loaderFor(file).scanImports(src); } catch { continue; } + for (const imp of imports) { + if (!imp.path.startsWith(".")) continue; // external package + stack.push(resolve(dirname(file), imp.path)); + } + } + return [...visited]; +} + +function hasSyncExec(source: string): boolean { + return SYNC_EXEC.some((re) => re.test(source)); +} + +test("no daemon-reachable module calls sync exec (outside the allowlist)", () => { + const offenders: string[] = []; + for (const file of daemonClosure()) { + const rel = file.replace(REPO_ROOT + "/", ""); + if (ALLOWLIST.has(rel)) continue; + let src: string; + try { src = readFileSync(file, "utf8"); } catch { continue; } + if (hasSyncExec(src)) offenders.push(rel); + } + expect(offenders).toEqual([]); +}); + +test("the checker flags a reintroduced sync-exec call (proves the gate bites)", () => { + // Permanent RED proof: the matcher must catch a fresh offense. + expect(hasSyncExec(`import { execSync } from "child_process";\nexecSync("true");`)).toBe(true); + expect(hasSyncExec(`await Bun.sleepSync(10);`)).toBe(true); + expect(hasSyncExec(`// a comment mentioning execSync without a call`)).toBe(false); +}); + +test("the daemon closure actually resolves (guards against a walker that finds nothing)", () => { + const closure = daemonClosure(); + expect(closure.length).toBeGreaterThan(50); // ~151 today; a collapse means the walk broke +}); From 56e7f146c2c17d2e667645668651d358e7359e42 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:10:17 -0500 Subject: [PATCH 064/142] state.db: isolate each legacy importer in a SAVEPOINT so one bad file cannot wedge migration A throwing legacy importer previously rolled back the whole v0->v1 migration, leaving user_version at 0 and repeating the identical throw on every later openStateDb call (daemon boot + every CLI command) with no self-heal. Wrap each LEGACY_IMPORTS entry in its own SAVEPOINT so one importer's throw only rolls back its own writes; warn with the file and error, still push it to consumed so it gets renamed .migrated, and let the schema DDL and every other importer land and reach SCHEMA_VERSION. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/state/__tests__/db.test.ts | 64 +++++++++++++++++++++++++++++++++- lib/state/db.ts | 30 ++++++++++++---- 2 files changed, 87 insertions(+), 7 deletions(-) diff --git a/lib/state/__tests__/db.test.ts b/lib/state/__tests__/db.test.ts index 3d7f5f2d..104a115c 100644 --- a/lib/state/__tests__/db.test.ts +++ b/lib/state/__tests__/db.test.ts @@ -6,7 +6,7 @@ * HOME isolation is handled by the repo-wide bun test preload * (test-setup.ts) — never removed here. */ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; import { Database } from "bun:sqlite"; import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "fs"; import { tmpdir } from "os"; @@ -18,6 +18,12 @@ import { openStateDb, SCHEMA_VERSION, } from "../db.ts"; +// Side-effect imports: registering the REAL project-mrs and discussions +// importers (module-load LEGACY_IMPORTS.push), not fakes, so the isolation +// test below exercises a genuine throw (duplicate-iid UNIQUE violation) +// and a genuine benign import, not a hand-rolled stand-in for either. +import "../../daemon/project-mrs-store.ts"; +import "../../daemon/discussions-file-store.ts"; const DB_TS_PATH = join(import.meta.dir, "..", "db.ts"); @@ -318,6 +324,62 @@ describe("legacy import seam", () => { }); expect(() => openStateDb(dbPath, "cli").close()).not.toThrow(); }); + + test("a throwing legacy importer is isolated: db reaches SCHEMA_VERSION, the other importer's rows land, and the offending file is still renamed", () => { + const warnSpy = spyOn(console, "warn").mockImplementation(() => {}); + try { + const dbPath = join(dir, "state.db"); + + // project-mrs-store's real importer: keys "5" and "05" both bind + // Number(iidStr) === 5, so the second INSERT hits project_mrs's + // (repo, iid) PRIMARY KEY and throws mid-transaction. + const projectMrsPath = join(dir, "project-mrs.json"); + writeFileSync( + projectMrsPath, + JSON.stringify({ + "host/repo": { + mrs: { + "5": { pr: { iid: 5, title: "first" }, fetchedAt: 111 }, + "05": { pr: { iid: 5, title: "duplicate" }, fetchedAt: 222 }, + }, + }, + }), + ); + + // discussions-file-store's real importer: a benign, unrelated file + // that must still land even though the importer above throws. + const discussionsPath = join(dir, "discussions.json"); + writeFileSync( + discussionsPath, + JSON.stringify({ + "host/repo:7": { discussions: [{ id: "d1" }], fetchedAt: 333 }, + }), + ); + + const db = openStateDb(dbPath, "cli"); + + expect(db.query("PRAGMA user_version;").get()).toEqual({ user_version: SCHEMA_VERSION }); + + const discussionsRow = db + .query("SELECT repo, iid, fetched_at FROM discussions WHERE repo = ? AND iid = ?;") + .get("host/repo", 7); + expect(discussionsRow).toEqual({ repo: "host/repo", iid: 7, fetched_at: 333 }); + + expect(existsSync(projectMrsPath)).toBe(false); + expect(existsSync(`${projectMrsPath}.migrated`)).toBe(true); + expect(existsSync(discussionsPath)).toBe(false); + expect(existsSync(`${discussionsPath}.migrated`)).toBe(true); + + const warnedAboutOffender = warnSpy.mock.calls.some((call) => + call.some((arg) => typeof arg === "string" && arg.includes("project-mrs.json")), + ); + expect(warnedAboutOffender).toBe(true); + + db.close(); + } finally { + warnSpy.mockRestore(); + } + }); }); describe("pragma values per flavor", () => { diff --git a/lib/state/db.ts b/lib/state/db.ts index 99478f33..34d53c68 100644 --- a/lib/state/db.ts +++ b/lib/state/db.ts @@ -391,11 +391,22 @@ function quarantine(path: string): void { /** * Runs each registered legacy importer whose source file exists, inside the * caller's transaction. Returns the list of source paths that were consumed - * (successfully imported OR corrupt-and-skipped) — both cases still rename - * per spec "Migration & contention" ("corrupt = warn + skip"; brief: "warn + - * skip + still rename"). Renaming itself happens AFTER COMMIT (the caller - * does it), since a filesystem rename cannot participate in the sqlite - * transaction. + * (successfully imported OR corrupt/throwing-and-skipped) — all three cases + * still rename per spec "Migration & contention" ("corrupt = warn + skip"; + * brief: "warn + skip + still rename"). Renaming itself happens AFTER COMMIT + * (the caller does it), since a filesystem rename cannot participate in the + * sqlite transaction. + * + * Each importer's `import(db, json)` runs inside its own SAVEPOINT, nested + * inside the caller's outer BEGIN IMMEDIATE. A throwing importer (e.g. two + * legacy keys that normalize to the same primary key, tripping a UNIQUE + * constraint) previously rolled back the WHOLE v0->v1 migration: user_version + * stayed 0, so every later openStateDb call replayed the identical throw + * forever with no self-heal. The savepoint confines that rollback to the one + * importer's own writes, so the schema DDL and every OTHER importer still + * commit and the db reaches SCHEMA_VERSION. This is deliberately narrower + * than the outer transaction's own error handling: schema/DDL failures are + * not wrapped here and still abort the whole migration loudly. */ function importLegacyStores(db: Database, dir: string): string[] { const consumed: string[] = []; @@ -410,7 +421,14 @@ function importLegacyStores(db: Database, dir: string): string[] { consumed.push(path); continue; } - entry.import(db, json); + db.exec("SAVEPOINT legacy_import;"); + try { + entry.import(db, json); + db.exec("RELEASE legacy_import;"); + } catch (err) { + db.exec("ROLLBACK TO legacy_import; RELEASE legacy_import;"); + console.warn(`rt: legacy import failed for ${path}, skipping (file will still be renamed): ${(err as Error).message}`); + } consumed.push(path); } return consumed; From 9783b1c55ea7569f06e24b92eadff7882a3fa2db Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:14:22 -0500 Subject: [PATCH 065/142] daemon: add supervision-state (boot attempts, failures, last-exit) kv + breadcrumb --- .../__tests__/supervision-state.test.ts | 71 +++++++++ lib/daemon/supervision-state.ts | 150 ++++++++++++++++++ 2 files changed, 221 insertions(+) create mode 100644 lib/daemon/__tests__/supervision-state.test.ts create mode 100644 lib/daemon/supervision-state.ts diff --git a/lib/daemon/__tests__/supervision-state.test.ts b/lib/daemon/__tests__/supervision-state.test.ts new file mode 100644 index 00000000..6461af34 --- /dev/null +++ b/lib/daemon/__tests__/supervision-state.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, test } from "bun:test"; +import { + recordBootAttempt, + recordDaemonReady, + recordBootFailure, + recordCleanExit, + readSupervisionState, + isCrashLooping, + writeBreadcrumb, + readBreadcrumb, + clearBreadcrumb, +} from "../supervision-state.ts"; + +describe("supervision-state kv round-trip", () => { + test("boot attempts, ready stamp, failures and last-exit round-trip through kv", () => { + recordBootAttempt(); + recordBootAttempt(); + recordDaemonReady(); + recordBootFailure("api", "EADDRINUSE"); + const s = readSupervisionState(); + expect(s.bootAttempts).toBe(2); + expect(s.lastReadyAt).toBeGreaterThan(0); + expect(s.recentFailures.at(-1)).toMatchObject({ phase: "api", reason: "EADDRINUSE" }); + expect(s.lastExit).toMatchObject({ kind: "boot-failed", code: 1 }); + }); + + test("recordCleanExit sets last-exit with the given kind and code", () => { + recordCleanExit("shutdown", 0); + const s = readSupervisionState(); + expect(s.lastExit).toMatchObject({ kind: "shutdown", code: 0 }); + }); + + test("recent-failures is capped at 10 entries", () => { + for (let i = 0; i < 15; i++) recordBootFailure("api", `err-${i}`); + const s = readSupervisionState(); + expect(s.recentFailures.length).toBe(10); + expect(s.recentFailures.at(-1)).toMatchObject({ reason: "err-14" }); + }); +}); + +describe("isCrashLooping", () => { + test("true at >=3 failures within the window", () => { + const now = 1_000_000; + const fails = [now - 10, now - 20, now - 30].map((at) => ({ at, phase: "api" as const, reason: "x" })); + expect(isCrashLooping({ bootAttempts: 3, lastReadyAt: 0, recentFailures: fails, lastExit: null }, now)).toBe(true); + const old = [{ at: now - 10 * 60_000, phase: "api" as const, reason: "x" }]; + expect(isCrashLooping({ bootAttempts: 1, lastReadyAt: 0, recentFailures: old, lastExit: null }, now)).toBe(false); + }); +}); + +describe("breadcrumb file", () => { + test("writeBreadcrumb then readBreadcrumb round-trips phase, pid, flavor", () => { + writeBreadcrumb("api"); + const b = readBreadcrumb(); + expect(b).not.toBeNull(); + expect(b?.phase).toBe("api"); + expect(b?.pid).toBe(process.pid); + expect(typeof b?.at).toBe("number"); + }); + + test("clearBreadcrumb removes the file so readBreadcrumb returns null", () => { + writeBreadcrumb("ready"); + clearBreadcrumb(); + expect(readBreadcrumb()).toBeNull(); + }); + + test("readBreadcrumb returns null when no breadcrumb has been written", () => { + clearBreadcrumb(); + expect(readBreadcrumb()).toBeNull(); + }); +}); diff --git a/lib/daemon/supervision-state.ts b/lib/daemon/supervision-state.ts new file mode 100644 index 00000000..7fa12e7e --- /dev/null +++ b/lib/daemon/supervision-state.ts @@ -0,0 +1,150 @@ +/** + * lib/daemon/supervision-state.ts — daemon boot/crash history, so + * `rt daemon status` can report boot-failed/crash-looping and a stuck-phase + * breadcrumb for a live-but-silent daemon. + * + * Two tiers, deliberately not one: + * - The breadcrumb FILE (`writeBreadcrumb`/`readBreadcrumb`/`clearBreadcrumb`) + * opens no database, so it is safe to call at module scope, before + * state.db exists. It is the only tier a pre-db boot failure can reach. + * - The kv tier (`recordBootAttempt`, `recordDaemonReady`, + * `recordCleanExit`, and the kv half of `recordBootFailure`) goes through + * `getStateDb("daemon")`, so callers must not reach it until the daemon + * has opened its state.db (lib/daemon.ts's `openBranchCacheStore()`). + * `recordBootFailure` is safe to call at any point regardless — its kv + * write is try/catch'd and silently no-ops if the db isn't open yet. + */ + +import { existsSync, readFileSync, unlinkSync, writeFileSync } from "fs"; +import { join } from "path"; +import { RT_DIR } from "../daemon-config.ts"; +import { daemonFlavor } from "./park.ts"; +import { getKvValue, setKvValue } from "../state/kv-blob.ts"; +import { getStateDb } from "../state/db.ts"; + +export type BootPhase = "start" | "events-db" | "state-db" | "api" | "socket" | "ready"; + +const NS = "daemon-supervision"; +const KEY_BOOT_ATTEMPTS = "boot-attempts"; +const KEY_LAST_READY_AT = "last-ready-at"; +const KEY_RECENT_FAILURES = "recent-failures"; +const KEY_LAST_EXIT = "last-exit"; + +const RECENT_FAILURES_CAP = 10; + +export interface BootFailure { + at: number; + phase: BootPhase; + reason: string; +} + +export type LastExit = + | { at: number; kind: "boot-failed"; code: number; reason: string } + | { at: number; kind: "shutdown" | "signal"; code: number }; + +export interface SupervisionState { + bootAttempts: number; + lastReadyAt: number; + recentFailures: BootFailure[]; + lastExit: LastExit | null; +} + +function db() { + return getStateDb("daemon"); +} + +export function recordBootAttempt(): void { + const n = getKvValue(NS, KEY_BOOT_ATTEMPTS, 0, db()); + setKvValue(NS, KEY_BOOT_ATTEMPTS, n + 1, db()); +} + +export function recordDaemonReady(): void { + setKvValue(NS, KEY_LAST_READY_AT, Date.now(), db()); +} + +/** + * Always writes the breadcrumb file, which never needs state.db. The kv + * append is best-effort: a failure this early in boot may predate state.db + * being open at all, and that must not throw back into the caller's catch. + */ +export function recordBootFailure(phase: BootPhase, reason: string): void { + writeBreadcrumb(phase); + try { + const at = Date.now(); + const existing = getKvValue(NS, KEY_RECENT_FAILURES, [], db()); + const next = [...existing, { at, phase, reason }].slice(-RECENT_FAILURES_CAP); + setKvValue(NS, KEY_RECENT_FAILURES, next, db()); + setKvValue(NS, KEY_LAST_EXIT, { at, kind: "boot-failed", code: 1, reason }, db()); + } catch { + // Pre-db failure (or a busy/corrupt state.db) — the breadcrumb file above + // is the only record this failure gets, and that's fine. + } +} + +export function recordCleanExit(kind: "shutdown" | "signal", code: number): void { + try { + setKvValue(NS, KEY_LAST_EXIT, { at: Date.now(), kind, code }, db()); + } catch { + // Best-effort, same as recordBootFailure's kv half. + } +} + +export function readSupervisionState(): SupervisionState { + const store = db(); + return { + bootAttempts: getKvValue(NS, KEY_BOOT_ATTEMPTS, 0, store), + lastReadyAt: getKvValue(NS, KEY_LAST_READY_AT, 0, store), + recentFailures: getKvValue(NS, KEY_RECENT_FAILURES, [], store), + lastExit: getKvValue(NS, KEY_LAST_EXIT, null, store), + }; +} + +export function isCrashLooping( + state: SupervisionState, + now: number, + n = 3, + windowMs = 5 * 60_000, +): boolean { + const floor = now - windowMs; + return state.recentFailures.filter((f) => f.at > floor).length >= n; +} + +// ─── Breadcrumb file (db-free) ──────────────────────────────────────────── + +interface Breadcrumb { + at: number; + pid: number; + flavor: "dev" | "prod"; + phase: BootPhase; +} + +function breadcrumbPath(): string { + return join(RT_DIR, "daemon-boot.json"); +} + +/** Never fatal — a breadcrumb is a diagnostic aid, not something boot may fail over. */ +export function writeBreadcrumb(phase: BootPhase): void { + try { + const breadcrumb: Breadcrumb = { at: Date.now(), pid: process.pid, flavor: daemonFlavor(), phase }; + writeFileSync(breadcrumbPath(), JSON.stringify(breadcrumb)); + } catch { + // Best-effort. + } +} + +export function readBreadcrumb(): Breadcrumb | null { + try { + if (!existsSync(breadcrumbPath())) return null; + return JSON.parse(readFileSync(breadcrumbPath(), "utf8")) as Breadcrumb; + } catch { + return null; + } +} + +export function clearBreadcrumb(): void { + try { + if (existsSync(breadcrumbPath())) unlinkSync(breadcrumbPath()); + } catch { + // Best-effort. + } +} From 673cfcfa67f194e6cf2c3819e359c155a8acfb88 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:14:56 -0500 Subject: [PATCH 066/142] refresh: whole-cycle deadline clears the coalesce latch; cap RepoWatch.pending (S007) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/cache-refresh-coalesce.test.ts | 33 +++++++++++++++ .../__tests__/freshness-pending-cap.test.ts | 13 ++++++ lib/daemon/cache-refresh.ts | 40 +++++++++++++++---- lib/daemon/freshness.ts | 13 +++++- 4 files changed, 91 insertions(+), 8 deletions(-) create mode 100644 lib/daemon/__tests__/cache-refresh-coalesce.test.ts create mode 100644 lib/daemon/__tests__/freshness-pending-cap.test.ts diff --git a/lib/daemon/__tests__/cache-refresh-coalesce.test.ts b/lib/daemon/__tests__/cache-refresh-coalesce.test.ts new file mode 100644 index 00000000..32a609d0 --- /dev/null +++ b/lib/daemon/__tests__/cache-refresh-coalesce.test.ts @@ -0,0 +1,33 @@ +import { test, expect } from "bun:test"; +import { makeCoalescer } from "../cache-refresh.ts"; + +test("clears the in-flight latch after the deadline even if run never settles", async () => { + let starts = 0; + let timedOut = 0; + const coalesce = makeCoalescer( + () => { starts++; return new Promise(() => {}); }, // never resolves + 50, + () => { timedOut++; }, + ); + const t0 = Date.now(); + await coalesce(); // resolves at the deadline, not never + expect(Date.now() - t0).toBeLessThan(500); + expect(timedOut).toBe(1); + await coalesce(); // latch cleared, a new run can start + expect(starts).toBe(2); +}); + +test("coalesces concurrent callers onto one run", async () => { + let starts = 0; + let resolveRun!: () => void; + const coalesce = makeCoalescer( + () => { starts++; return new Promise((r) => { resolveRun = r; }); }, + 10_000, + () => {}, + ); + const a = coalesce(); + const b = coalesce(); + expect(starts).toBe(1); + resolveRun(); + await Promise.all([a, b]); +}); diff --git a/lib/daemon/__tests__/freshness-pending-cap.test.ts b/lib/daemon/__tests__/freshness-pending-cap.test.ts new file mode 100644 index 00000000..f488ba9e --- /dev/null +++ b/lib/daemon/__tests__/freshness-pending-cap.test.ts @@ -0,0 +1,13 @@ +import { test, expect } from "bun:test"; +import { applyInvalidationBatch, PENDING_CAP } from "../freshness.ts"; + +test("merged pending is deduped by kind:ref and capped", async () => { + const runner: any = { processing: true, pending: [] }; + // Push more distinct keys than the cap; plus duplicates. + const keys = Array.from({ length: PENDING_CAP + 500 }, (_, i) => ({ kind: "mr" as const, ref: String(i), cause: "test" })); + const dupes = [{ kind: "mr" as const, ref: "0", cause: "test" }, { kind: "mr" as const, ref: "0", cause: "test" }]; + await applyInvalidationBatch({} as any, {} as any, runner, [...keys, ...dupes], {}); + expect(runner.pending.length).toBeLessThanOrEqual(PENDING_CAP); + const ids = runner.pending.map((k: any) => `${k.kind}:${k.ref}`); + expect(new Set(ids).size).toBe(ids.length); // no duplicates +}); diff --git a/lib/daemon/cache-refresh.ts b/lib/daemon/cache-refresh.ts index 0ab58fc7..5fe14920 100644 --- a/lib/daemon/cache-refresh.ts +++ b/lib/daemon/cache-refresh.ts @@ -47,16 +47,42 @@ export interface CacheRefresherDeps { */ const BRANCH_CACHE_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; +/** Below the 5-min tick, above the slowest legitimate deep sync. */ +const REFRESH_CYCLE_DEADLINE_MS = 4 * 60 * 1000; + +/** + * Coalesce concurrent callers onto one in-flight run, but clear the latch after + * `deadlineMs` even if the run never settles, so a wedged cycle (a half-open + * GitLab socket that never rejects) cannot pin the latch forever. The wedged + * run's frame still leaks until the OS reaps the socket; this only frees the + * next tick. + */ +export function makeCoalescer( + run: () => Promise, + deadlineMs: number, + onTimeout: () => void, +): () => Promise { + let inFlight: Promise | null = null; + return () => { + if (inFlight) return inFlight; + const impl = run().catch(() => {}); // a rejected cycle still clears the latch + const guarded = Promise.race([ + impl, + new Promise((resolve) => setTimeout(() => { onTimeout(); resolve(); }, deadlineMs)), + ]).finally(() => { inFlight = null; }); + inFlight = guarded; + return guarded; + }; +} + export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise { const { log, cache, refreshStatusRef, portCacheRef, repoIndex, broadcast } = deps; - let refreshInFlight: Promise | null = null; - - function refreshCache(): Promise { - if (refreshInFlight) return refreshInFlight; - refreshInFlight = refreshCacheImpl().finally(() => { refreshInFlight = null; }); - return refreshInFlight; - } + const refreshCache = makeCoalescer( + refreshCacheImpl, + REFRESH_CYCLE_DEADLINE_MS, + () => log.warn("cache refresh timed out; cleared in-flight latch for next tick"), + ); async function refreshCacheImpl(): Promise { log.debug("cache: starting background refresh"); diff --git a/lib/daemon/freshness.ts b/lib/daemon/freshness.ts index 2b49b497..767feca0 100644 --- a/lib/daemon/freshness.ts +++ b/lib/daemon/freshness.ts @@ -85,6 +85,10 @@ interface RepoWatch { } const watches = new Map(); + +/** Bound merged pending so a wedged processKeys cannot grow memory unbounded. */ +export const PENDING_CAP = 1000; + const providers = new Map(); let userId: number | null = null; let userIdResolved = false; @@ -397,7 +401,14 @@ export async function applyInvalidationBatch( overrides: MappingOverrides = {}, ): Promise { if (runner.processing) { - runner.pending.push(...keys); + const seen = new Set(runner.pending.map((k) => `${k.kind}:${k.ref}`)); + for (const k of keys) { + if (runner.pending.length >= PENDING_CAP) break; + const id = `${k.kind}:${k.ref}`; + if (seen.has(id)) continue; + seen.add(id); + runner.pending.push(k); + } return; } runner.processing = true; From fcf8220fa4bd7e66322f64b9de9c6651c85b6c19 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:17:09 -0500 Subject: [PATCH 067/142] daemon: wire supervision-state breadcrumbs + kv records into boot/shutdown --- lib/daemon.ts | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/daemon.ts b/lib/daemon.ts index d2997eb0..e2a570fc 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -63,6 +63,14 @@ import { import { startDiscussionsPoller } from "./daemon/discussions-poller.ts"; import { createCleanup, installSignalHandlers } from "./daemon/shutdown.ts"; import { createEventsBus } from "./daemon/events-bus.ts"; +import { + writeBreadcrumb, + recordBootAttempt, + recordDaemonReady, + recordBootFailure, + recordCleanExit, + type BootPhase, +} from "./daemon/supervision-state.ts"; import { safeInterval, safeTimeout } from "./daemon/safe-timers.ts"; import { pruneRuns } from "./runs/prune.ts"; import { pruneLogs } from "./log-janitor.ts"; @@ -88,6 +96,16 @@ redirectNativeStderr(); // (no socket/API bound yet, nothing to recover), advisory-only once ready. let bootPhase: "booting" | "ready" = "booting"; +// Tracks the finer-grained boot phase for the breadcrumb file and for +// attributing a Task-2 fatal boot error to the phase it happened in. +// setPhase is db-free (writeBreadcrumb only writes a file), so it is safe +// to call from module scope, before state.db exists. +let currentPhase: BootPhase = "start"; +function setPhase(phase: BootPhase): void { + currentPhase = phase; + writeBreadcrumb(phase); +} + const rtMigration = migrateLegacyRtDir(); // ─── Logging ───────────────────────────────────────────────────────────────── @@ -104,6 +122,8 @@ const log = loggerHandle.logger; // must run BEFORE any of it does. installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" }); +setPhase("start"); + if (rtMigration === "migrated") { log.info(`migrated legacy ${LEGACY_RT_LABEL} state to ${RT_DIR_LABEL}`); } else if (rtMigration === "conflict") { @@ -208,6 +228,7 @@ const hooksGuard = createHooksGuard(log); // Pane-communication events bus (RT-44): SQLite journal + in-memory waiters. const eventsBus = createEventsBus({ dbPath: join(RT_DIR, "events.db"), log }); +setPhase("events-db"); // Hourly retention sweep — cheap; rides its own interval rather than pollers // because it needs no poller deps. safeInterval/safeTimeout: a sync sqlite // throw here (e.g. SQLITE_FULL) must warn, not become an uncaughtException @@ -374,6 +395,7 @@ async function routeCommand(cmd: string, payload: any, signal?: AbortSignal): Pr // force-closes all in-flight connections, including the one that // carried the shutdown request. setTimeout(() => { + recordCleanExit("shutdown", 0); cleanup(); loggerHandle.flush?.(); process.exit(0); @@ -435,6 +457,8 @@ async function runDaemon(): Promise { // legacy-JSON import, and it must never land inside the event loop. If a // CLI process is mid-import right now, we block here, in startup. openBranchCacheStore(); + setPhase("state-db"); + recordBootAttempt(); log.info({ count: Object.keys(cache.entries).length }, "branch cache loaded from state.db"); // one-shot re-key of every legacy NAME-keyed store row onto its @@ -482,7 +506,9 @@ async function runDaemon(): Promise { // API server first: a failed bind exits fatally (boot-phase catch below), // and binding API before the unix socket means that fatal exit never // strands a socket-bound zombie behind it. + setPhase("api"); servers.api = await startApiServer({ handleCommand, log }); + setPhase("socket"); servers.socket = startSocketServer({ handleCommand, log }); // Only write rt.pid once both servers are actually bound — a boot that @@ -533,10 +559,12 @@ async function runDaemon(): Promise { installSignalHandlers({ cleanup, flushLogs: () => loggerHandle.flush?.(), log }); bootPhase = "ready"; + recordDaemonReady(); + setPhase("ready"); log.info({ pid: process.pid }, "daemon ready"); } catch (err) { log.fatal({ err }, "daemon boot failed"); - // Task 9 adds recordBootFailure(currentPhase, err) here. + recordBootFailure(currentPhase, String(err)); try { loggerHandle.flush?.(); } catch { /* */ } process.exit(1); } From fa6523f32bc5e11f050c84c0086923bf7a6d573c Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:20:52 -0500 Subject: [PATCH 068/142] fix(refresh): clear the deadline timer on every settle path, not just on wedge Promise.race never cancels the losing branch. When impl won (every successful refresh), the deadline setTimeout still fired ~4 min later and called onTimeout, logging a spurious "cache refresh timed out" warn and leaking a ref'd timer per cycle. Capture the timer handle and clearTimeout it in the shared .finally, which runs on every settle path; clearing an already-fired timer is a no-op. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/cache-refresh-coalesce.test.ts | 12 ++++++++++++ lib/daemon/cache-refresh.ts | 15 +++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/lib/daemon/__tests__/cache-refresh-coalesce.test.ts b/lib/daemon/__tests__/cache-refresh-coalesce.test.ts index 32a609d0..fa3c3eb0 100644 --- a/lib/daemon/__tests__/cache-refresh-coalesce.test.ts +++ b/lib/daemon/__tests__/cache-refresh-coalesce.test.ts @@ -31,3 +31,15 @@ test("coalesces concurrent callers onto one run", async () => { resolveRun(); await Promise.all([a, b]); }); + +test("a fast success does not fire onTimeout after the deadline elapses", async () => { + let timedOut = 0; + const coalesce = makeCoalescer( + () => Promise.resolve(), // settles well before the deadline + 50, + () => { timedOut++; }, + ); + await coalesce(); + await new Promise((r) => setTimeout(r, 150)); // past the deadline + expect(timedOut).toBe(0); // the deadline timer must have been cleared, not just outraced +}); diff --git a/lib/daemon/cache-refresh.ts b/lib/daemon/cache-refresh.ts index 5fe14920..2771c28b 100644 --- a/lib/daemon/cache-refresh.ts +++ b/lib/daemon/cache-refresh.ts @@ -66,10 +66,17 @@ export function makeCoalescer( return () => { if (inFlight) return inFlight; const impl = run().catch(() => {}); // a rejected cycle still clears the latch - const guarded = Promise.race([ - impl, - new Promise((resolve) => setTimeout(() => { onTimeout(); resolve(); }, deadlineMs)), - ]).finally(() => { inFlight = null; }); + // Promise.race never cancels the losing branch, so the deadline timer must be + // captured and cleared on every settle path or a fast success still fires + // onTimeout deadlineMs later, misreported as a wedge. + let deadlineTimer: ReturnType; + const deadline = new Promise((resolve) => { + deadlineTimer = setTimeout(() => { onTimeout(); resolve(); }, deadlineMs); + }); + const guarded = Promise.race([impl, deadline]).finally(() => { + clearTimeout(deadlineTimer); + inFlight = null; + }); inFlight = guarded; return guarded; }; From e9cdbd88a89953bb641821bf7d472e42d9a40dc7 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:25:23 -0500 Subject: [PATCH 069/142] freshness: rebuild the provider cache when gitlabToken rotates (S048, S049) Key the providers map on a token fingerprint so a rotated gitlabToken rebuilds the GitLabProvider on the next ensureProvider/getRepoContext call instead of waiting for a daemon restart. A token mismatch drops the stale watch and resets the userIdResolved latch so the next reconcile re-authenticates. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../freshness-provider-rotation.test.ts | 11 ++++++++++ lib/daemon/freshness.ts | 22 ++++++++++++------- 2 files changed, 25 insertions(+), 8 deletions(-) create mode 100644 lib/daemon/__tests__/freshness-provider-rotation.test.ts diff --git a/lib/daemon/__tests__/freshness-provider-rotation.test.ts b/lib/daemon/__tests__/freshness-provider-rotation.test.ts new file mode 100644 index 00000000..e0de5507 --- /dev/null +++ b/lib/daemon/__tests__/freshness-provider-rotation.test.ts @@ -0,0 +1,11 @@ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("ensureProvider compares the current token before reusing a cached provider", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + // The unconditional cache-hit return is the S049 bug; it must be gone. + expect(src).not.toMatch(/const cached = providers\.get\(repoName\);\s*\n\s*if \(cached\) return cached;/); + // A token fingerprint must be stored alongside the provider. + expect(src).toMatch(/providers\.set\(repoName,\s*\{\s*provider/); +}); diff --git a/lib/daemon/freshness.ts b/lib/daemon/freshness.ts index 767feca0..f3d0b43a 100644 --- a/lib/daemon/freshness.ts +++ b/lib/daemon/freshness.ts @@ -89,7 +89,7 @@ const watches = new Map(); /** Bound merged pending so a wedged processKeys cannot grow memory unbounded. */ export const PENDING_CAP = 1000; -const providers = new Map(); +const providers = new Map(); let userId: number | null = null; let userIdResolved = false; let selfUsername: string | null = null; @@ -134,15 +134,21 @@ function makeProvider(host: string, token: string): GitLabProvider { } async function ensureProvider(repoName: string, repoPath: string): Promise { - const cached = providers.get(repoName); - if (cached) return cached; - const secrets = await loadSecrets(); if (!secrets.gitlabToken) { log.info(`no gitlabToken; skipping ${repoName}`); return null; } + const cached = providers.get(repoName); + if (cached && cached.token === secrets.gitlabToken) return cached.provider; + if (cached) { + // Token rotated: drop the stale watch built on the old token and + // re-resolve userId against the new one on the next reconcile. + stopWatch(repoName); + userIdResolved = false; + } + const remoteUrl = await getRemoteUrl(repoPath); if (!remoteUrl) { log.info(`no origin remote for ${repoName}; skipping`); @@ -161,14 +167,14 @@ async function ensureProvider(repoName: string, repoPath: string): Promise { if (userIdResolved) return userId; // Resolve via any available provider. If none exist yet, defer until one does. - const anyProvider = providers.values().next().value as GitLabProvider | undefined; + const anyProvider = providers.values().next().value?.provider as GitLabProvider | undefined; if (!anyProvider) return null; try { @@ -259,7 +265,7 @@ export async function getRepoContext( projectPathOverride?: string, ): Promise<{ provider: GitLabProvider; projectPath: string; projectId: number }> { const watch = watches.get(repoName); - let provider = providers.get(repoName) ?? null; + let provider = providers.get(repoName)?.provider ?? null; // Live-watch fast path — but only when the caller didn't override projectPath. // If they did, fall through to the ephemeral path so we use the canonical path. @@ -290,7 +296,7 @@ export async function getRepoContext( throw new Error(`could not parse remote URL "${remoteUrl}"`); } provider = makeProvider(remote.host, secrets.gitlabToken); - providers.set(repoName, provider); + providers.set(repoName, { provider, token: secrets.gitlabToken }); } // Pick projectPath: explicit override > previously-cached ephemeral > git remote. From 1af0f282f6c734c0a6f9b053e53382689eb14ba0 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:26:17 -0500 Subject: [PATCH 070/142] daemon: lazy-resolve trusted-origin allowlist, drop finding-ID comments Fix A: getTrustedBrowserOrigins() did a synchronous settings-store read on every single :9401 request, even the ~100% with no Origin header (CLI, tray, rt-client from Bun/Node) that never needed it. Added resolveOriginTrust() in api-auth.ts, which only resolves the allowlist when an Origin header is present, and wired both api-server.ts call sites (the /ws gate and the CORS/token gate) through it. Fix B: stripped leftover audit finding-ID citations (S005/S006/S040/ S041/S042/S054/S083/S085) from production comments in api-auth.ts and api-server.ts, keeping the underlying technical rationale. Fix C: reworded two stale "CORS is *" comments to describe the actual current default-deny CORS model (a trusted Origin gets its response echoed back; the local token, not CORS, is what stops a page from firing a mutating request in the first place). Co-Authored-By: Claude Sonnet 5 --- lib/daemon/__tests__/api-auth.test.ts | 34 ++++++++++++++++++++++++++- lib/daemon/api-auth.ts | 33 ++++++++++++++++++++------ lib/daemon/api-server.ts | 29 +++++++++++++---------- 3 files changed, 75 insertions(+), 21 deletions(-) diff --git a/lib/daemon/__tests__/api-auth.test.ts b/lib/daemon/__tests__/api-auth.test.ts index 6abf9eab..b5113660 100644 --- a/lib/daemon/__tests__/api-auth.test.ts +++ b/lib/daemon/__tests__/api-auth.test.ts @@ -5,7 +5,7 @@ import { describe, test, expect } from "bun:test"; import { needsToken, tokenOk, getApiToken, reloadApiToken, loadOrCreateApiToken } from "../api-auth.ts"; -import { isOriginAllowed, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "../api-auth.ts"; +import { isOriginAllowed, isBrowserRequestTrusted, getTrustedBrowserOrigins, resolveOriginTrust } from "../api-auth.ts"; import { mkdtempSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; @@ -177,3 +177,35 @@ describe("getTrustedBrowserOrigins", () => { expect(Array.isArray(origins)).toBe(true); }); }); + +describe("resolveOriginTrust", () => { + const apiToken = "the-real-token"; + + test("no Origin header: never calls getAllowedOrigins (the settings read is disk I/O every non-browser request would otherwise pay for)", () => { + let calls = 0; + const getAllowedOrigins = () => { calls++; return []; }; + const trusted = resolveOriginTrust(null, null, apiToken, getAllowedOrigins); + expect(trusted).toBe(true); + expect(calls).toBe(0); + }); + + test("an Origin header present: does call getAllowedOrigins", () => { + let calls = 0; + const getAllowedOrigins = () => { calls++; return ["http://localhost:5544"]; }; + const trusted = resolveOriginTrust("http://localhost:5544", null, apiToken, getAllowedOrigins); + expect(trusted).toBe(true); + expect(calls).toBe(1); + }); + + test("an Origin header with the correct token is trusted without needing the allowlist call to matter", () => { + let calls = 0; + const getAllowedOrigins = () => { calls++; return []; }; + const trusted = resolveOriginTrust("http://evil.example", apiToken, apiToken, getAllowedOrigins); + expect(trusted).toBe(true); + expect(calls).toBe(1); + }); + + test("defaults to the real getTrustedBrowserOrigins when no override is passed", () => { + expect(resolveOriginTrust(null, null, apiToken)).toBe(true); + }); +}); diff --git a/lib/daemon/api-auth.ts b/lib/daemon/api-auth.ts index dae00efa..ac45a18f 100644 --- a/lib/daemon/api-auth.ts +++ b/lib/daemon/api-auth.ts @@ -1,8 +1,10 @@ /** * Local-token auth for the :9401 API server. * - * The server binds to 127.0.0.1, but CORS is `*`, so a malicious web page could - * still drive *mutating* endpoints via the browser. Requiring a custom header + * The server binds to 127.0.0.1, and CORS is default-deny (only a trusted + * Origin gets its response readable), but CORS alone does not stop a + * malicious web page from firing a mutating request in the first place -- it + * only stops the page from reading the reply. Requiring a custom header * (X-RT-Token) on those routes forces a CORS preflight the page can't satisfy * (it can't read the token), blocking cross-site control while leaving reads * open for convenience. @@ -44,7 +46,7 @@ export function loadOrCreateApiToken(tokenPath: string = API_TOKEN_PATH): string /** * `getApiToken`/`reloadApiToken` share ONE in-memory value between - * api-server.ts and the secrets handler (S054): before this, api-server + * api-server.ts and the secrets handler: before this, api-server * captured a token once at boot while the secrets handler called * `loadOrCreateApiToken()` fresh on every request, so an external rotation * (deleting api-token to force a new one) left the two permanently @@ -72,15 +74,15 @@ export function reloadApiToken(tokenPath: string = API_TOKEN_PATH): string { * drains something a GET should not silently consume, and must present the * local token. A CORS preflight (OPTIONS) can never present the custom * X-RT-Token header, so it is never gated, on any path. Otherwise - * default-gated for every method except GET/HEAD (S040: an allowlist-by-path - * guaranteed the next mutating route would ship unguarded), plus two + * default-gated for every method except GET/HEAD (an allowlist-by-path + * approach guarantees the next mutating route would ship unguarded), plus two * explicit GET exceptions whose verb lies about being a read. */ export function needsToken(method: string, pathname: string): boolean { if (method === "OPTIONS") return false; if (method === "GET" || method === "HEAD") { // Gated despite being a GET: /api/secrets's response body IS a - // credential (S054); /api/notifications DRAINS the queue (S041), so its + // credential; /api/notifications DRAINS the queue, so its // verb lies about being a read the way every other GET here is not. if (pathname === "/api/secrets") return true; if (pathname === "/api/notifications") return true; @@ -114,7 +116,7 @@ export function isOriginAllowed(origin: string, allowedOrigins: readonly string[ } /** - * The 127.0.0.1 trust boundary (S005/S006): the daemon binds loopback-only, + * The 127.0.0.1 trust boundary: the daemon binds loopback-only, * but any web page the user visits also runs on 127.0.0.1 and can send a * request. A request with NO Origin header at all is not a browser fetch -- * it is the CLI, the Swift tray, rt-client from a Bun/Node process, or the @@ -132,3 +134,20 @@ export function isBrowserRequestTrusted( if (tokenOk(token, apiToken)) return true; return isOriginAllowed(origin, allowedOrigins); } + +/** + * Same trust decision as isBrowserRequestTrusted, but resolves the allowlist + * lazily: getAllowedOrigins() only runs when an Origin header is present. + * getTrustedBrowserOrigins does synchronous disk I/O on the settings store, + * and the vast majority of :9401 traffic (the CLI, the tray, rt-client from a + * Bun/Node process) carries no Origin at all, so it must never pay that cost. + */ +export function resolveOriginTrust( + origin: string | null, + presentedToken: string | null, + apiToken: string, + getAllowedOrigins: () => readonly string[] = getTrustedBrowserOrigins, +): boolean { + if (!origin) return true; + return isBrowserRequestTrusted(origin, presentedToken, apiToken, getAllowedOrigins()); +} diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index 6c5bd2c4..cfe54790 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -10,7 +10,7 @@ import type { Server, ServerWebSocket } from "bun"; import type { Logger } from "pino"; import { API_PORT } from "../daemon-config.ts"; -import { needsToken, tokenOk, getApiToken, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "./api-auth.ts"; +import { needsToken, tokenOk, getApiToken, resolveOriginTrust } from "./api-auth.ts"; import { getAggregatedConnection } from "./freshness.ts"; import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; import { runCapture } from "../subprocess.ts"; @@ -93,7 +93,7 @@ export interface BroadcastTarget { /** * Sends one frame to every client, dropping any that Bun's own send() return - * value marks as gone (S042). `ws.send()` never throws on a dead socket -- + * value marks as gone. `ws.send()` never throws on a dead socket -- * it returns 0 (this send silently failed) or -1 (backpressure) -- so a * disconnected or stalled console/chat-viewer tab used to keep receiving a * SUBSET of frames forever with nothing logged. 0 means Bun already dropped @@ -149,7 +149,7 @@ export function clearWsClients(): void { } /** - * CORS default-deny (S006): a browser page on an untrusted Origin still gets + * CORS default-deny: a browser page on an untrusted Origin still gets * its request served (127.0.0.1 loopback + the per-route token gate are the * real defenses), but the response carries no Access-Control-Allow-Origin, * so the page's own JavaScript cannot read the body. A request with no @@ -170,7 +170,7 @@ export function buildCorsHeaders(origin: string | null, trusted: boolean): Recor /** * Decodes one path segment between a fixed prefix (and optional suffix), * returning `undefined` (never throwing) on any shape mismatch or malformed - * %-encoding (S083). Before this, each parameterized route hand-rolled its + * %-encoding. Before this, each parameterized route hand-rolled its * own decodeURIComponent inside the route's try block, so a malformed * segment fell through to the OUTER catch and came back as a logged 500; * every route using this helper instead gets a clean 400. @@ -191,8 +191,8 @@ export function pathParam(pathname: string, prefix: string, suffix = ""): string const PLAIN_NUMBER_RE = /^-?\d+(\.\d+)?$/; /** - * REST query strings arrive as strings no matter what the client meant - * (S085): "?maxAgeMs=60000" and "?refresh=true" reached handlers that do a + * REST query strings arrive as strings no matter what the client meant: + * "?maxAgeMs=60000" and "?refresh=true" reached handlers that do a * strict `typeof x === "number"` or `x === true` check, so the documented * flag silently no-op'd over HTTP while working over the socket (where * payloads are real JSON). One coercion at the REST seam fixes every such @@ -298,15 +298,14 @@ export async function startApiServer(deps: ApiServerDeps): Promise> async fetch(req, server) { const url = new URL(req.url); const origin = req.headers.get("origin"); - const allowedOrigins = getTrustedBrowserOrigins(); // WebSocket upgrade (broadcast channel). Browsers cannot set custom // headers on a WS handshake, so the token (when a browser page wants // to identify itself) travels as a ?token= query param instead of - // X-RT-Token (S005). + // X-RT-Token. if (url.pathname === "/ws") { const wsToken = url.searchParams.get("token"); - if (!isBrowserRequestTrusted(origin, wsToken, apiToken, allowedOrigins)) { + if (!resolveOriginTrust(origin, wsToken, apiToken)) { return new Response("origin not allowed", { status: 403 }); } if (server.upgrade(req, { data: { kind: "broadcast" } })) return undefined as any; @@ -315,16 +314,20 @@ export async function startApiServer(deps: ApiServerDeps): Promise> // CORS: default-deny. A trusted Origin (token or allowlist) gets its // Origin echoed back; anything else gets no Access-Control-Allow-Origin - // at all, so a malicious page's own JS cannot read the response (S006). - const trusted = isBrowserRequestTrusted(origin, req.headers.get("x-rt-token"), apiToken, allowedOrigins); + // at all, so a malicious page's own JS cannot read the response. + // resolveOriginTrust only resolves the allowlist when origin is set, + // since the settings read behind it is synchronous disk I/O. + const trusted = resolveOriginTrust(origin, req.headers.get("x-rt-token"), apiToken); const corsHeaders = buildCorsHeaders(origin, trusted); if (req.method === "OPTIONS") { return new Response(null, { status: 204, headers: corsHeaders }); } - // Gate mutating routes behind the local token (CORS is *, so this is the - // CSRF defense against a malicious page driving control endpoints). + // Gate mutating routes behind the local token. CORS default-deny only + // stops a malicious page from reading the response; it can still fire + // the request itself (a classic CSRF), so the token is the actual + // defense against a malicious page driving control endpoints. if (needsToken(req.method, url.pathname) && !tokenOk(req.headers.get("x-rt-token"), apiToken)) { return Response.json({ ok: false, error: "unauthorized" }, { status: 401, headers: corsHeaders }); } From fbf7ba88518aafd446dc54fdc57bc10f544b6462 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:29:39 -0500 Subject: [PATCH 071/142] docs: name the S084 GET-route known-gap ruling in the trust-boundary note --- docs/daemon-api-auth.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/docs/daemon-api-auth.md b/docs/daemon-api-auth.md index 1eb33045..3e2b1584 100644 --- a/docs/daemon-api-auth.md +++ b/docs/daemon-api-auth.md @@ -47,3 +47,16 @@ and `.port`). Catch it around the `startApiServer()` call and park-and-retry with backoff instead of letting it reach the top-level crash path; any other error out of `startApiServer()` is a genuine misconfiguration and should keep crashing as it does today. + +## Known gap: GET routes that trigger real work stay ungated + +`GET /api/cache?maxAgeMs=` can force a full cache refresh, and +`GET /api/sdm/recents` spawns `sdm status`; neither requires the local token. +Default-deny CORS stops an untrusted browser Origin from reading the +response, but a plain cross-origin GET needs no preflight, so the request +still lands and the work still runs even when the response is unreadable. +This is deliberately not fixed here: gating these routes risks breaking +non-browser REST consumers that read them untokened today (the tray, +editor extensions), and there was no time in this pass to audit every such +consumer, which the audit's own fixer notes flag as the real risk of +tokening reads. Tracked as an open follow-up, not a silently-closed finding. From 88da40614f9c23bdc4344ef2c491ac2c1a3fbfa0 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:34:16 -0500 Subject: [PATCH 072/142] freshness: close the two remaining stale-token paths (S048/S049 review fix) getRepoContext served a cached provider without checking its token, so poll-mode repos and already-cached forge-handler providers kept a rotated token forever. reconcileFreshnessImpl skipped ensureProvider entirely for repos with a live watch, so a running watcher never rebuilt after rotation either. Both now compare the cached token against loadSecrets() before reuse and drop the stale watch/provider on a mismatch. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../freshness-provider-rotation.test.ts | 23 +++++++++++++++++++ lib/daemon/freshness.ts | 23 +++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/lib/daemon/__tests__/freshness-provider-rotation.test.ts b/lib/daemon/__tests__/freshness-provider-rotation.test.ts index e0de5507..8c5f59c3 100644 --- a/lib/daemon/__tests__/freshness-provider-rotation.test.ts +++ b/lib/daemon/__tests__/freshness-provider-rotation.test.ts @@ -9,3 +9,26 @@ test("ensureProvider compares the current token before reusing a cached provider // A token fingerprint must be stored alongside the provider. expect(src).toMatch(/providers\.set\(repoName,\s*\{\s*provider/); }); + +test("getRepoContext drops a stale-token provider before serving the cached one (S048/S049 fix 1)", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + const tokenCheckMatch = src.match(/cachedForToken\.token !== currentSecrets\.gitlabToken/); + expect(tokenCheckMatch).not.toBeNull(); + // The comparison must run before the cache is read for the fast path, so a + // rotated token can never reach a poll-mode repo or an already-cached + // forge-handler provider. + const fastPathIndex = src.indexOf("const watch = watches.get(repoName);"); + expect(fastPathIndex).toBeGreaterThan(-1); + expect(tokenCheckMatch!.index!).toBeLessThan(fastPathIndex); +}); + +test("reconcileFreshnessImpl drops a stale-token watch before skipping already-watched repos (S048/S049 fix 2)", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + const staleWatchDrop = src.match(/existing\.token !== secrets\.gitlabToken\)\s*stopWatch\(repoName\);/); + expect(staleWatchDrop).not.toBeNull(); + // The drop must happen before the "already watched, skip" short-circuit, + // or a live watcher built on a rotated token never rebuilds. + const skipIndex = src.indexOf("if (watches.has(repoName)) continue;"); + expect(skipIndex).toBeGreaterThan(-1); + expect(staleWatchDrop!.index!).toBeLessThan(skipIndex); +}); diff --git a/lib/daemon/freshness.ts b/lib/daemon/freshness.ts index f3d0b43a..e2bb2b2f 100644 --- a/lib/daemon/freshness.ts +++ b/lib/daemon/freshness.ts @@ -264,6 +264,20 @@ export async function getRepoContext( repoPath?: string, projectPathOverride?: string, ): Promise<{ provider: GitLabProvider; projectPath: string; projectId: number }> { + // A cached provider's token may have rotated since it was built; poll-mode + // repos and already-cached forge-handler providers never pass back through + // ensureProvider, so this is the only place that catches a stale token for + // them (S048/S049). + const cachedForToken = providers.get(repoName); + if (cachedForToken) { + const currentSecrets = await loadSecrets(); + if (currentSecrets.gitlabToken && cachedForToken.token !== currentSecrets.gitlabToken) { + stopWatch(repoName); + userIdResolved = false; + providers.delete(repoName); + } + } + const watch = watches.get(repoName); let provider = providers.get(repoName)?.provider ?? null; @@ -702,6 +716,15 @@ async function reconcileFreshnessImpl(env: FreshnessEnv): Promise { for (const [repoName, repoPath] of Object.entries(repoIndex)) { if (grants(tracking, repoName).mode !== "live") continue; + + // A live watch whose provider token has since rotated must be dropped so the + // ensureProvider/startWatch below rebuilds it with the current token (S048/S049). + const existing = providers.get(repoName); + if (existing && watches.has(repoName)) { + const secrets = await loadSecrets(); + if (secrets.gitlabToken && existing.token !== secrets.gitlabToken) stopWatch(repoName); + } + if (watches.has(repoName)) continue; const provider = await ensureProvider(repoName, repoPath); From 58ab7e45276fd5749d6f87d0ad77d6dc6908c064 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:40:03 -0500 Subject: [PATCH 073/142] system-process-scanner: a failed lsof preserves runaway windows (S061) gather() returned [] on both a real empty scan and an lsof/ps failure, so scan() pruned every tracked pid on a transient failure, resetting firstSeen and the runaway sample window. getAllRepoPids and gather now return null on scan failure (distinct from an empty result), and scan/refresh early-return the prior lastResult before touching tracked state. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../system-process-scanner-resilience.test.ts | 46 +++++++++++++++++++ lib/daemon/system-process-scanner.ts | 21 ++++++--- 2 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 lib/daemon/__tests__/system-process-scanner-resilience.test.ts diff --git a/lib/daemon/__tests__/system-process-scanner-resilience.test.ts b/lib/daemon/__tests__/system-process-scanner-resilience.test.ts new file mode 100644 index 00000000..c5db5939 --- /dev/null +++ b/lib/daemon/__tests__/system-process-scanner-resilience.test.ts @@ -0,0 +1,46 @@ +import { test, expect } from "bun:test"; +import { SystemProcessScanner } from "../system-process-scanner.ts"; + +/** Drives gather()'s return value directly so a null (failed lsof/ps tick) + * can be simulated without shelling out. */ +class FakeScanner extends SystemProcessScanner { + next: any[] | null = []; + protected override async gather(): Promise { + return this.next; + } +} + +function fakeProcess(pid: number, cpuPercent: number) { + return { + pid, + ppid: 1, + command: "node", + fullCommand: "node build.js", + cpuPercent, + rssKb: 10_000, + uptime: "01:00", + cwd: "/repo", + repo: "r", + worktree: null, + branch: null, + relativeDir: "", + port: null, + linearTicket: null, + packageScript: null, + }; +} + +test("a failed gather (null) keeps tracked windows and lastResult intact", async () => { + const s = new FakeScanner(); + + s.next = [fakeProcess(4242, 95)]; + const first = await s.scan(); + expect(first.find((p) => p.pid === 4242)).toBeTruthy(); + const firstSeen = s.getTracked(4242)?.firstSeen; + expect(firstSeen).toBeDefined(); + + s.next = null; + const during = await s.scan(); + expect(s.getTracked(4242)?.firstSeen).toBe(firstSeen); + expect(during.find((p) => p.pid === 4242)).toBeTruthy(); +}); diff --git a/lib/daemon/system-process-scanner.ts b/lib/daemon/system-process-scanner.ts index dfb121c2..2ac18941 100644 --- a/lib/daemon/system-process-scanner.ts +++ b/lib/daemon/system-process-scanner.ts @@ -222,7 +222,7 @@ async function getPackageScripts(pidList: string): Promise> return parsePackageScripts(stdout); } -async function getAllRepoPids(trackedPaths: string[]): Promise> { +async function getAllRepoPids(trackedPaths: string[]): Promise | null> { if (trackedPaths.length === 0) return new Map(); // Single lsof call to get cwds for ALL processes at once. @@ -232,8 +232,11 @@ async function getAllRepoPids(trackedPaths: string[]): Promise { + const gathered = await this.gather(portEntries); + if (gathered === null) return this.lastResult; // failed tick: preserve tracked/lastResult/lastScanAt try { - const gathered = await this.gather(portEntries); const now = Date.now(); const currentPids = new Set(); @@ -344,8 +348,9 @@ export class SystemProcessScanner { * alert. Also advances `lastScanAt` so freshness checks see the update. */ async refresh(portEntries: PortEntry[] = []): Promise { + const gathered = await this.gather(portEntries); + if (gathered === null) return this.lastResult; // failed tick: preserve tracked/lastResult/lastScanAt try { - const gathered = await this.gather(portEntries); const prev = new Map(this.lastResult.map(p => [p.pid, p])); const results = gathered.map(proc => { const before = prev.get(proc.pid); @@ -368,7 +373,7 @@ export class SystemProcessScanner { * decorate it with port + package-script, but no runaway/tracking state. * Shared by `scan` (adds tracking) and `refresh` (carries flags forward). */ - private async gather(portEntries: PortEntry[]): Promise { + protected async gather(portEntries: PortEntry[]): Promise { const repos = loadRepoIndex(); if (Object.keys(repos).length === 0) return []; @@ -378,7 +383,8 @@ export class SystemProcessScanner { const worktreeMap = await buildWorktreeMap(repos); const trackedPaths = [...new Set([...Object.values(repos), ...worktreeMap.keys()])]; const cwdMap = await getAllRepoPids(trackedPaths); - if (cwdMap.size === 0) return []; + if (cwdMap === null) return null; // lsof failed + if (cwdMap.size === 0) return []; // genuinely no tracked-cwd processes // Get CPU/memory for discovered PIDs const pidList = [...cwdMap.keys()].join(","); @@ -386,6 +392,7 @@ export class SystemProcessScanner { ["ps", "-p", pidList, "-o", "pid=,ppid=,pcpu=,rss=,etime=,comm=,args="], { timeoutMs: 5000 }, ); + if (psRes.exitCode !== 0 && !psRes.stdout) return null; // ps failed if (!psRes.stdout) return []; const parsed = parseProcessList(psRes.stdout, repos, cwdMap, worktreeMap); From 3c090d3daedca876376a24147d7312ab336e4275 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:45:09 -0500 Subject: [PATCH 074/142] runs: mtime-memoize finished-run summaries; back off herdr probe (S101, S038, S039) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/agent-status-poller.test.ts | 17 +++++++++++ lib/daemon/agent-status-poller.ts | 11 +++++++- lib/runs/__tests__/store-memo.test.ts | 28 +++++++++++++++++++ lib/runs/store.ts | 20 +++++++++++-- 4 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 lib/runs/__tests__/store-memo.test.ts diff --git a/lib/daemon/__tests__/agent-status-poller.test.ts b/lib/daemon/__tests__/agent-status-poller.test.ts index 57020710..a1507a8e 100644 --- a/lib/daemon/__tests__/agent-status-poller.test.ts +++ b/lib/daemon/__tests__/agent-status-poller.test.ts @@ -73,3 +73,20 @@ test("finished runs are ignored and dropped from tracking", async () => { await handle!.tick(); expect(events).toHaveLength(0); }); + +test("backs off the herdr probe after repeated failures", async () => { + let probeCalls = 0; + handle = startAgentStatusPoller({ + emitEvent: () => {}, + log: quietLog, + intervalMs: 3_600_000, // real timer never fires + probe: async () => { probeCalls++; return null; }, // herdr absent + list: () => [], + }); + for (let i = 0; i < 20; i++) await handle.tick(); + // Without backoff this would be 20; with backoff (threshold 3, 1-in-6) far fewer. + expect(probeCalls).toBeLessThan(10); + // Backoff never stops probing forever: 3 initial failures engage it, then + // one probe every BACKOFF_TICKS (ticks 9 and 15 of 20) keeps checking. + expect(probeCalls).toBe(5); +}); diff --git a/lib/daemon/agent-status-poller.ts b/lib/daemon/agent-status-poller.ts index faf30ed2..fb50a410 100644 --- a/lib/daemon/agent-status-poller.ts +++ b/lib/daemon/agent-status-poller.ts @@ -16,6 +16,8 @@ import type { RunLiveness } from "../runs/attention.ts"; import { listRuns } from "../runs/store.ts"; const POLL_MS = 10_000; +const FAILURE_THRESHOLD = 3; // consecutive null probes before backing off +const BACKOFF_TICKS = 6; // then probe once every 6 ticks (~60s at 10s cadence) interface Log { info(obj: unknown, msg?: string): void; @@ -39,10 +41,17 @@ export function startAgentStatusPoller(opts: { const list = opts.list ?? ((liveness: RunLiveness) => listRuns(undefined, liveness)); const last = new Map(); let seeded = false; + let consecutiveFailures = 0; + let ticksSkipped = 0; const tick = async (): Promise => { + if (consecutiveFailures >= FAILURE_THRESHOLD) { + if (++ticksSkipped < BACKOFF_TICKS) return; + ticksSkipped = 0; + } const entries = await probe(); - if (entries === null) return; + if (entries === null) { consecutiveFailures++; return; } + consecutiveFailures = 0; primeLivenessCache(entries); let runs: RunSummary[]; try { diff --git a/lib/runs/__tests__/store-memo.test.ts b/lib/runs/__tests__/store-memo.test.ts new file mode 100644 index 00000000..da1b9a46 --- /dev/null +++ b/lib/runs/__tests__/store-memo.test.ts @@ -0,0 +1,28 @@ +import { Database } from "bun:sqlite"; +import { afterEach, expect, spyOn, test } from "bun:test"; +import { listRuns } from "../store.ts"; +import { root, seedRun } from "./fixtures.ts"; + +afterEach(() => { delete process.env.RT_RUNS_ROOT; }); + +test("a finished run's db is opened once, then served from the mtime cache", () => { + const dir = root(); + seedRun(dir, "repoA", "run1", 1000, 1, { status: "done" }); + const openSpy = spyOn(Database.prototype, "query"); + listRuns(); // first call opens + reads + const afterFirst = openSpy.mock.calls.length; + listRuns(); // second call: mtime unchanged -> no reopen + expect(openSpy.mock.calls.length).toBe(afterFirst); // no additional queries for run1 + openSpy.mockRestore(); +}); + +test("a running run is never cached: it is reopened even when its mtime is unchanged", () => { + const dir = root(); + seedRun(dir, "repoA", "run2", 1000, 1, { status: "running" }); + const openSpy = spyOn(Database.prototype, "query"); + listRuns(); + const afterFirst = openSpy.mock.calls.length; + listRuns(); + expect(openSpy.mock.calls.length).toBeGreaterThan(afterFirst); + openSpy.mockRestore(); +}); diff --git a/lib/runs/store.ts b/lib/runs/store.ts index 3e290ac2..6d15e46d 100644 --- a/lib/runs/store.ts +++ b/lib/runs/store.ts @@ -4,7 +4,7 @@ * and per-call — no held connections, so a run dir can be pruned under us. */ import { Database } from "bun:sqlite"; -import { existsSync, readdirSync, type Dirent } from "fs"; +import { existsSync, readdirSync, statSync, type Dirent } from "fs"; import { homedir } from "os"; import { join } from "path"; import type { Attention, RunDetail, RunFieldRow, RunStageRow, RunSummary } from "../../packages/rt-client/src/commands.ts"; @@ -114,17 +114,33 @@ function withAttention(db: Database, row: RunSummary, liveness?: RunLiveness): R } } +// Finished runs never change; skip the open+PRAGMA+4-reads when the db mtime +// is unchanged. Running runs are never cached: their db still mutates and +// their liveness overlay is recomputed per call. +const summaryCache = new Map(); + export function listRuns(repo?: string, liveness?: RunLiveness): RunSummary[] { if (repo != null && !isPathComponent(repo)) return []; const repos = repo ? [repo] : dirs(runsRoot()); const out: RunSummary[] = []; for (const r of repos) { for (const id of dirs(join(runsRoot(), r))) { + const dbPath = join(runsRoot(), r, id, "state.db"); + let mtimeMs: number; + try { mtimeMs = statSync(dbPath).mtimeMs; } catch { continue; } + const key = `${r}/${id}`; + const hit = summaryCache.get(key); + if (hit && hit.mtimeMs === mtimeMs) { out.push(hit.summary); continue; } + const opened = openRun(r, id); if (!opened) continue; try { const row = runRow(opened.db); - if (row) out.push(withAttention(opened.db, row, liveness)); + if (row) { + const summary = withAttention(opened.db, row, liveness); + out.push(summary); + if (summary.status !== "running") summaryCache.set(key, { mtimeMs, summary }); + } } finally { opened.db.close(); } From e9313e0a9a52f249afd828ec6b57b4de5c9e456e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:46:40 -0500 Subject: [PATCH 075/142] daemon status: alive-not-serving / parked / boot-failed / crash-looping verdicts rt daemon status and the ping handler now read Task 9's breadcrumb file and supervision kv to distinguish a live-but-not-serving daemon (booting/wedged/ quarantined), a flavor standoff (parked), a single boot failure, and a crash loop, on top of the existing not-installed/running/degraded/not-running verdicts. pidAlive uses lsof scoped to RT_DIR instead of a system-wide pgrep, since the brief's suggested pgrep pattern false-positives against any real rt daemon running under a different HOME on the same machine. --- .../__tests__/daemon-status-render.test.ts | 37 +++++ commands/daemon.ts | 109 +++++++++++++- e2e/tests/daemon.test.ts | 24 ++++ lib/__tests__/daemon-status.test.ts | 133 +++++++++++++++++- lib/command-tree-def.ts | 4 +- lib/daemon-status.ts | 116 +++++++++++++-- lib/daemon/__tests__/status-identity.test.ts | 14 ++ lib/daemon/handlers/status.ts | 13 +- 8 files changed, 429 insertions(+), 21 deletions(-) diff --git a/commands/__tests__/daemon-status-render.test.ts b/commands/__tests__/daemon-status-render.test.ts index eb4ff2df..98e27b51 100644 --- a/commands/__tests__/daemon-status-render.test.ts +++ b/commands/__tests__/daemon-status-render.test.ts @@ -71,4 +71,41 @@ describe("statusLines", () => { expect(out).toContain("watching: 1 repo"); expect(out).not.toContain("1 repos"); }); + + // ── Task 10 ── + + test("a parked pid points at the flavor mismatch, not 'not running'", () => { + const out = plain({ state: "parked", pid: 42, holderFlavor: "prod" }); + expect(out).toContain("parked"); + expect(out).toContain("pid 42"); + expect(out).toContain("held by: prod"); + expect(out).not.toContain("installed but not running"); + }); + + test("alive-not-serving names the pid and the stuck detail", () => { + const out = plain({ state: "alive-not-serving", pid: 42, detail: "booting" }); + expect(out).toContain("process 42 is running but not answering rt.sock"); + expect(out).toContain("still booting"); + expect(out).not.toContain("installed but not running"); + }); + + test("alive-not-serving wedged/quarantined get their own detail lines", () => { + expect(plain({ state: "alive-not-serving", pid: 1, detail: "wedged" })).toContain("deadlocked"); + expect(plain({ state: "alive-not-serving", pid: 1, detail: "quarantined" })).toContain("recovered from a corrupt db"); + }); + + test("crash-looping surfaces the failure count and the last reason", () => { + const out = plain({ state: "crash-looping", failures: 4, reason: "EADDRINUSE" }); + expect(out).toContain("crash-looping"); + expect(out).toContain("4 failures"); + expect(out).toContain("EADDRINUSE"); + }); + + test("boot-failed surfaces the phase and reason, and points at rt daemon start", () => { + const out = plain({ state: "boot-failed", reason: "EADDRINUSE", phase: "api" }); + expect(out).toContain("boot failed"); + expect(out).toContain("phase: api"); + expect(out).toContain("EADDRINUSE"); + expect(out).toContain("rt daemon start"); + }); }); diff --git a/commands/daemon.ts b/commands/daemon.ts index ab571ad6..4a71a43a 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -35,6 +35,8 @@ import { daemonQuery, isDaemonRunning, trayQuery } from "../lib/daemon-client.ts import { classifyDaemonStatus, type DaemonStatusVerdict } from "../lib/daemon-status.ts"; import { resolveIntendedMode, currentMode, type IntendedMode } from "../lib/dev-mode.ts"; import { probeSocketHolder } from "../lib/daemon/park.ts"; +import { readBreadcrumb, readSupervisionState } from "../lib/daemon/supervision-state.ts"; +import { runCapture } from "../lib/subprocess.ts"; import { isGitLabRemote } from "../lib/enrich.ts"; import type { CacheKind, RepoTrackingEntry } from "../lib/repo-tracking.ts"; import { loadRepoTracking, loadMachineRepoTracking, loadMachineRepoTrackingRaw, saveRepoTrackingRaw, grants, parseCachesArg, CACHE_KINDS, DEFAULT_PROJECT_MRS_WINDOW_DAYS, teamNamesIdentity } from "../lib/repo-tracking.ts"; @@ -316,8 +318,41 @@ export async function restart(): Promise { // ─── Status ────────────────────────────────────────────────────────────────── -export async function showStatus(): Promise { +/** + * Raw OS-level liveness, independent of rt.sock. Tries a direct pid check + * first against every pid this HOME actually recorded — rt.pid, then the + * boot breadcrumb's pid (the breadcrumb survives failures rt.pid never gets + * written for, per Ruling P1) — before falling back to a last-resort scan. + * + * That scan is `lsof +D `, not the brief's suggested system-wide + * `pgrep -f 'rt --daemon|lib/daemon.ts'`: a raw pgrep matches ANY rt daemon + * on the machine regardless of which HOME started it, and on an ordinary dev + * workstation there usually IS one — the developer's own real daemon — so a + * pgrep-based check on an isolated/alternate HOME reliably misreports a dead + * boot attempt as alive-not-serving (verified live against this repo's own + * dev daemon while writing the e2e test below). `lsof +D` instead asks "does + * any process hold a file open under THIS HOME's rt dir" — home-scoped by + * construction, immune to that false positive and to pid-reuse. Only worth + * calling once both `status` and a plain ping have already failed. + */ +async function probePidAlive(recordedPid: number | null, breadcrumbPid?: number): Promise<{ alive: boolean; pid: number | null }> { + for (const candidate of [recordedPid, breadcrumbPid ?? null]) { + if (candidate === null) continue; + try { + process.kill(candidate, 0); + return { alive: true, pid: candidate }; + } catch { /* not this one — try the next candidate */ } + } + const { stdout } = await runCapture(["lsof", "-t", "+D", RT_DIR], { timeoutMs: 3000 }); + const pids = stdout.trim().split(/\s+/).filter(Boolean).map(Number).filter((n) => !isNaN(n)); + return pids.length > 0 ? { alive: true, pid: pids[0]! } : { alive: false, pid: recordedPid }; +} + +export async function showStatus(args: string[] = []): Promise { + const json = args.includes("--json"); + if (!isDaemonInstalled()) { + if (json) return void console.log(JSON.stringify({ ok: true, state: "not-installed" })); console.log(` ${dim}○${reset} not installed ${dim}(run rt daemon install)${reset}\n`); return; } @@ -326,14 +361,40 @@ export async function showStatus(): Promise { // A failed status query does NOT mean the daemon is down — it answers `ping` // in a fraction of the budget a loaded `status` needs. Establish liveness // before reporting, and only pay for the probe when nothing came back. - const alive = classifyDaemonStatus.needsLivenessProbe(response) ? await isDaemonRunning() : false; + const pingOk = classifyDaemonStatus.needsLivenessProbe(response) ? await isDaemonRunning() : false; + const recordedPid = readDaemonPid() ?? null; + + // Ping ALSO failed: the only remaining ground is the pid/breadcrumb/kv + // trail Task 9 left behind. Read it here, once, rather than on every status + // call — it's the uncommon path. + let pidAlive: boolean | undefined; + let pid = recordedPid; + let breadcrumb: ReturnType | undefined; + let supervision: ReturnType | undefined; + if (classifyDaemonStatus.needsPidProbe(response, pingOk)) { + breadcrumb = readBreadcrumb(); + // The kv tier can be legitimately empty (or reflect nothing useful) when + // a failure happened before state.db ever opened — Ruling P1. The + // breadcrumb read above is what classifyDaemonStatus falls back to then. + supervision = readSupervisionState(); + const probed = await probePidAlive(recordedPid, breadcrumb?.pid); + pidAlive = probed.alive; + pid = probed.pid; + } + const verdict = classifyDaemonStatus({ installed: true, response, - alive, - pid: readDaemonPid() ?? null, + pingOk, + pid, + pidAlive, + intendedFlavor: resolveIntendedMode().mode, + breadcrumb, + supervision, }); + if (json) return void console.log(JSON.stringify({ ok: true, ...verdict })); + for (const line of statusLines(verdict, Date.now())) console.log(line); if (verdict.state === "running") { @@ -417,6 +478,46 @@ export function statusLines(verdict: DaemonStatusVerdict, now: number): string[] return lines; } + if (verdict.state === "parked") { + const lines = [` ${yellow}◐${reset} parked ${dim}(pid ${verdict.pid} — another flavor owns rt.sock)${reset}`]; + lines.push( + verdict.holderFlavor + ? ` ${dim}held by: ${verdict.holderFlavor}${reset}` + : ` ${dim}waiting for the intended flavor to take rt.sock${reset}`, + ); + lines.push(` ${dim}check: rt settings dev-mode${reset}`); + return lines; + } + + if (verdict.state === "alive-not-serving") { + const detailLine = { + booting: "still booting", + wedged: "reached ready but stopped answering — likely deadlocked", + quarantined: "recovered from a corrupt db but still not answering", + }[verdict.detail]; + return [ + ` ${yellow}●${reset} process ${verdict.pid} is running but not answering rt.sock`, + ` ${dim}${detailLine}${reset}`, + ` ${dim}check: rt daemon logs -t${reset}`, + ]; + } + + if (verdict.state === "crash-looping") { + return [ + ` ${red}●${reset} crash-looping ${dim}(${verdict.failures} failures recently)${reset}`, + ` ${dim}last reason: ${verdict.reason}${reset}`, + ` ${dim}check: rt daemon logs -t${reset}`, + ]; + } + + if (verdict.state === "boot-failed") { + return [ + ` ${red}●${reset} boot failed ${dim}(phase: ${verdict.phase})${reset}`, + ` ${dim}reason: ${verdict.reason}${reset}`, + ` ${dim}run: rt daemon start${reset}`, + ]; + } + if (verdict.state === "not-running") { const lines = [` ${red}●${reset} installed but not running`]; if (verdict.pid) lines.push(` ${dim}last pid: ${verdict.pid}${reset}`); diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index e0403285..469d54a6 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -57,6 +57,30 @@ describe("fatal boot", () => { } }, 60_000); + test("API-bind failure surfaces as boot-failed/crash-looping via rt daemon status --json", async () => { + const { path: home, cleanup } = createTestHome(); + // A different port than the other API-bind-failure tests above, so + // parallel test files can never collide on the same bound TCP port. + const port = 9413; + const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + try { + // `rt daemon status` short-circuits to "not installed" before it ever + // reaches the boot-failed/crash-looping classification — install first. + await rt(["daemon", "install"], { home }); + + const boot = await rt(["--daemon"], { home, env: { RT_API_PORT: String(port) } }); + expect(boot.exitCode).not.toBe(0); + + const status = await rt(["daemon", "status", "--json"], { home }); + expect(status.exitCode).toBe(0); + const parsed = JSON.parse(status.stdout); + expect(["boot-failed", "crash-looping"]).toContain(parsed.state); + } finally { + squatter.stop(true); + cleanup(); + } + }, 60_000); + test("a corrupt events.db self-heals — quarantined, and the daemon boots and serves", async () => { const { path: home, cleanup } = createTestHome(); const bunDir = join(process.execPath, ".."); diff --git a/lib/__tests__/daemon-status.test.ts b/lib/__tests__/daemon-status.test.ts index 84242905..17187267 100644 --- a/lib/__tests__/daemon-status.test.ts +++ b/lib/__tests__/daemon-status.test.ts @@ -1,9 +1,35 @@ import { describe, expect, test } from "bun:test"; import { classifyDaemonStatus } from "../daemon-status.ts"; +import type { BootFailure, LastExit, SupervisionState } from "../daemon/supervision-state.ts"; + +// `alive` (Task 1's field name for "a plain ping succeeded") is renamed +// `pingOk` here: Task 10 adds a second, distinct liveness signal (`pidAlive`, +// a raw OS-level pid check independent of rt.sock), and keeping both named +// `alive` would make call sites ambiguous about which one they mean. +function emptySupervision(): SupervisionState { + return { bootAttempts: 0, lastReadyAt: 0, recentFailures: [], lastExit: null }; +} + +function oneFailure(phase: BootFailure["phase"], reason: string): BootFailure { + return { at: Date.now(), phase, reason }; +} + +function threeRecentFailures(): BootFailure[] { + const now = Date.now(); + return [ + { at: now - 3000, phase: "api", reason: "EADDRINUSE" }, + { at: now - 2000, phase: "api", reason: "EADDRINUSE" }, + { at: now - 1000, phase: "api", reason: "EADDRINUSE" }, + ]; +} + +function bootFailedExit(reason: string): LastExit { + return { at: Date.now(), kind: "boot-failed", code: 1, reason }; +} describe("classifyDaemonStatus", () => { test("not installed short-circuits everything else", () => { - const v = classifyDaemonStatus({ installed: false, response: null, alive: false, pid: null }); + const v = classifyDaemonStatus({ installed: false, response: null, pingOk: false, pid: null }); expect(v.state).toBe("not-installed"); }); @@ -11,7 +37,7 @@ describe("classifyDaemonStatus", () => { const v = classifyDaemonStatus({ installed: true, response: { ok: true, data: { pid: 42, uptime: 1000 } }, - alive: true, + pingOk: true, pid: 42, }); expect(v.state).toBe("running"); @@ -24,7 +50,7 @@ describe("classifyDaemonStatus", () => { const v = classifyDaemonStatus({ installed: true, response: { ok: false, error: "freshness store unreadable" }, - alive: false, // never consulted: the answer itself is proof of life + pingOk: false, // never consulted: the answer itself is proof of life pid: 89290, }); expect(v.state).toBe("degraded"); @@ -37,7 +63,7 @@ describe("classifyDaemonStatus", () => { // The other half of the bug: daemonQuery returns null on a 2s timeout even // when it has already established the socket is live (daemon-client.ts:149). test("a null response with a live ping means running but unresponsive", () => { - const v = classifyDaemonStatus({ installed: true, response: null, alive: true, pid: 89290 }); + const v = classifyDaemonStatus({ installed: true, response: null, pingOk: true, pid: 89290 }); expect(v.state).toBe("degraded"); if (v.state === "degraded") { expect(v.reason).toBe("unresponsive"); @@ -46,13 +72,13 @@ describe("classifyDaemonStatus", () => { }); test("a null response and a dead ping is genuinely not running", () => { - const v = classifyDaemonStatus({ installed: true, response: null, alive: false, pid: 123 }); + const v = classifyDaemonStatus({ installed: true, response: null, pingOk: false, pid: 123 }); expect(v.state).toBe("not-running"); if (v.state === "not-running") expect(v.pid).toBe(123); }); test("not running without a recorded pid is still not running", () => { - const v = classifyDaemonStatus({ installed: true, response: null, alive: false, pid: null }); + const v = classifyDaemonStatus({ installed: true, response: null, pingOk: false, pid: null }); expect(v.state).toBe("not-running"); if (v.state === "not-running") expect(v.pid).toBeNull(); }); @@ -61,11 +87,104 @@ describe("classifyDaemonStatus", () => { const v = classifyDaemonStatus({ installed: true, response: { ok: false, error: "boom" }, - alive: false, + pingOk: false, pid: null, }); expect(v.state).toBe("degraded"); }); + + // ── Task 10: alive-not-serving / parked / crash-looping / boot-failed ── + + test("alive pid + failed ping -> alive-not-serving with breadcrumb detail", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: true, + pid: 42, + breadcrumb: { phase: "socket" }, + supervision: emptySupervision(), + }); + expect(v).toMatchObject({ state: "alive-not-serving", pid: 42, detail: "booting" }); + }); + + test("alive pid stuck after reaching ready -> wedged", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: true, + pid: 42, + breadcrumb: { phase: "ready" }, + supervision: emptySupervision(), + }); + expect(v).toMatchObject({ state: "alive-not-serving", pid: 42, detail: "wedged" }); + }); + + test("alive pid at ready with a boot-failed exit on record -> quarantined", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: true, + pid: 42, + breadcrumb: { phase: "ready" }, + supervision: { ...emptySupervision(), lastExit: bootFailedExit("events.db corrupt") }, + }); + expect(v).toMatchObject({ state: "alive-not-serving", pid: 42, detail: "quarantined" }); + }); + + test("alive pid whose breadcrumb flavor disagrees with the intended flavor -> parked", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: true, + pid: 42, + intendedFlavor: "prod", + breadcrumb: { phase: "start", flavor: "dev" }, + supervision: emptySupervision(), + }); + expect(v).toMatchObject({ state: "parked", pid: 42 }); + }); + + test("a breadcrumb with no supervision (pre-state.db failure) still classifies from the file alone", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: true, + pid: 42, + breadcrumb: { phase: "events-db" }, + }); + expect(v).toMatchObject({ state: "alive-not-serving", pid: 42, detail: "booting" }); + }); + + test("no pid + >=3 recent failures -> crash-looping", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: false, + pid: null, + supervision: { ...emptySupervision(), recentFailures: threeRecentFailures(), lastExit: bootFailedExit("EADDRINUSE") }, + }); + expect(v).toMatchObject({ state: "crash-looping" }); + if (v.state === "crash-looping") { + expect(v.failures).toBeGreaterThanOrEqual(3); + expect(v.reason).toBe("EADDRINUSE"); + } + }); + + test("no pid + single boot-failed -> boot-failed with reason and phase", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: false, + pid: null, + supervision: { ...emptySupervision(), recentFailures: [oneFailure("api", "EADDRINUSE")], lastExit: bootFailedExit("EADDRINUSE") }, + }); + expect(v).toMatchObject({ state: "boot-failed", reason: "EADDRINUSE", phase: "api" }); + }); + + test("no pid + no supervision at all -> plain not-running (additive: old callers unaffected)", () => { + const v = classifyDaemonStatus({ installed: true, pingOk: false, pidAlive: false, pid: null }); + expect(v.state).toBe("not-running"); + }); }); describe("needsLivenessProbe", () => { diff --git a/lib/command-tree-def.ts b/lib/command-tree-def.ts index 6ed9c3c6..59779ff4 100644 --- a/lib/command-tree-def.ts +++ b/lib/command-tree-def.ts @@ -640,7 +640,9 @@ export const TREE: Record = { description: "Show daemon status", module: "./commands/daemon.ts", fn: "showStatus", - args: [], + args: [ + { name: "JSON", flag: "--json", type: "boolean", default: false, hint: "Emit the verdict as JSON instead of the formatted lines" }, + ], }, track: { description: "Per-repo background tracking (live/poll/off)", diff --git a/lib/daemon-status.ts b/lib/daemon-status.ts index c550985c..f9face8b 100644 --- a/lib/daemon-status.ts +++ b/lib/daemon-status.ts @@ -7,29 +7,98 @@ * attempting a restart). A caller that reads "no usable response" as "the * daemon is down" then tells the user to start a daemon that is already * running. This maps the raw outcome onto what is actually known. + * + * Below that, a second tier answers a harder question: rt.sock isn't + * answering at all, but is the daemon actually down, or alive-and-stuck? + * `pidAlive`/`breadcrumb`/`supervision` (Task 9's supervision-state.ts) are + * the only signals that can tell, and per Ruling P1 (2026-08-28 p0-supervision + * ledger) the breadcrumb FILE is the sole record a pre-state.db boot failure + * leaves — `supervision` (the kv tier) can be absent even when `breadcrumb` + * is present, and classification must still resolve to something useful from + * the breadcrumb alone. */ import type { DaemonResponse } from "./daemon-client.ts"; +import { isCrashLooping, type BootPhase, type SupervisionState } from "./daemon/supervision-state.ts"; export type DaemonStatusVerdict = | { state: "not-installed" } | { state: "running"; data: any } /** Up — proven by an answer or a ping — but `status` itself did not deliver. */ | { state: "degraded"; reason: "error" | "unresponsive"; detail?: string; pid: number | null } + /** Ping fails, a live pid exists, and it's parked waiting for a different + * flavor to hold rt.sock (park.ts) — a flavor standoff, not a stuck boot. */ + | { state: "parked"; pid: number; holderFlavor?: string } + /** Ping fails but the pid is alive: still mid-boot, stuck after reaching + * ready, or alive-but-quarantined (recovered from a corrupt db). */ + | { state: "alive-not-serving"; pid: number; detail: "booting" | "wedged" | "quarantined" } + /** No live pid, and the kv failure record shows >= N failures within the window. */ + | { state: "crash-looping"; failures: number; reason: string } + /** No live pid, and the most recent recorded exit was a boot throw (fewer than N failures). */ + | { state: "boot-failed"; reason: string; phase: string } | { state: "not-running"; pid: number | null }; +/** The boot breadcrumb (`daemon-boot.json`), as classifyDaemonStatus needs it. Not + * imported from supervision-state.ts — that module's `Breadcrumb` interface is + * intentionally unexported, and this shape only needs to be structurally + * compatible with it. */ +export interface DaemonBreadcrumbInput { + phase: BootPhase; + flavor?: "dev" | "prod"; + pid?: number; + at?: number; +} + export interface DaemonStatusInputs { installed: boolean; - /** The `status` reply, or null if the transport gave up. */ - response: DaemonResponse | null; - /** Result of a `ping` probe. Only meaningful when `response` is null. */ - alive: boolean; + /** The `status` reply, or null/absent if the transport gave up. */ + response?: DaemonResponse | null; + /** Result of a `ping` probe. Only meaningful when `response` is absent. */ + pingOk?: boolean; /** Last recorded pid, for the operator to act on. */ pid: number | null; + /** Raw OS-level liveness of `pid` (process.kill(pid,0), or a pgrep-found + * stand-in) — independent of rt.sock. Only worth gathering once `pingOk` + * has already come back false; see `classifyDaemonStatus.needsPidProbe`. */ + pidAlive?: boolean; + /** This machine's currently-intended flavor (`resolveIntendedMode().mode`). + * A live pid whose own breadcrumb flavor disagrees with this is parked + * (park.ts), not stuck — the same signal `parkUntilIntended` itself acts on. */ + intendedFlavor?: "dev" | "prod"; + /** The socket holder's flavor, when the caller managed to learn it (best + * effort — probing rt.sock again after a failed ping/status round rarely + * succeeds, since a parked pid never binds it). Display-only. */ + holderFlavor?: string | null; + breadcrumb?: DaemonBreadcrumbInput | null; + /** Task 9's kv tier. Can be absent even when `breadcrumb` is present — a + * pre-state.db failure leaves only the breadcrumb file (Ruling P1). */ + supervision?: SupervisionState; + /** Injected for deterministic crash-loop window checks under test; defaults to Date.now(). */ + now?: number; +} + +const PHASE_ORDER: BootPhase[] = ["start", "events-db", "state-db", "api", "socket", "ready"]; + +function classifyAliveNotServingDetail( + breadcrumb: DaemonBreadcrumbInput | null | undefined, + supervision: SupervisionState | undefined, +): "booting" | "wedged" | "quarantined" { + const phase = breadcrumb?.phase; + if (!phase || PHASE_ORDER.indexOf(phase) < PHASE_ORDER.indexOf("ready")) return "booting"; + // Reached ready this run, but a prior attempt is on record as boot-failed — + // most likely a corrupt-db quarantine (lib/state/db.ts, events-bus.ts) it + // recovered from and is now stuck behind for an unrelated reason. + if (supervision?.lastExit?.kind === "boot-failed") return "quarantined"; + return "wedged"; +} + +function countRecentFailures(supervision: SupervisionState, now: number, windowMs = 5 * 60_000): number { + const floor = now - windowMs; + return supervision.recentFailures.filter((f) => f.at > floor).length; } export function classifyDaemonStatus(opts: DaemonStatusInputs): DaemonStatusVerdict { - const { installed, response, alive, pid } = opts; + const { installed, response, pingOk, pid, pidAlive, intendedFlavor, holderFlavor, breadcrumb, supervision } = opts; if (!installed) return { state: "not-installed" }; @@ -42,9 +111,33 @@ export function classifyDaemonStatus(opts: DaemonStatusInputs): DaemonStatusVerd return { state: "degraded", reason: "error", detail: response.error, pid }; } - // No reply. Ping is the only ground truth left: a daemon busy enough to blow - // the status timeout still answers a trivial ping. - if (alive) return { state: "degraded", reason: "unresponsive", pid }; + // No reply. A plain ping is the next ground truth: a daemon busy enough to + // blow the status timeout still answers a trivial ping. + if (pingOk) return { state: "degraded", reason: "unresponsive", pid }; + + // Ping failed too. From here, only pidAlive/breadcrumb/supervision (new + // signals) can say more than "not running" — absent them, fall straight + // through to the pre-existing not-running verdict. + if (pidAlive && pid !== null) { + if (breadcrumb?.flavor && intendedFlavor && breadcrumb.flavor !== intendedFlavor) { + return { state: "parked", pid, ...(holderFlavor ? { holderFlavor } : {}) }; + } + return { state: "alive-not-serving", pid, detail: classifyAliveNotServingDetail(breadcrumb, supervision) }; + } + + if (supervision) { + const now = opts.now ?? Date.now(); + if (isCrashLooping(supervision, now)) { + const reason = supervision.lastExit?.kind === "boot-failed" + ? supervision.lastExit.reason + : (supervision.recentFailures.at(-1)?.reason ?? "unknown"); + return { state: "crash-looping", failures: countRecentFailures(supervision, now), reason }; + } + if (supervision.lastExit?.kind === "boot-failed") { + const phase = supervision.recentFailures.at(-1)?.phase ?? "unknown"; + return { state: "boot-failed", reason: supervision.lastExit.reason, phase }; + } + } return { state: "not-running", pid }; } @@ -55,3 +148,10 @@ export function classifyDaemonStatus(opts: DaemonStatusInputs): DaemonStatusVerd */ classifyDaemonStatus.needsLivenessProbe = (response: DaemonResponse | null): boolean => response === null; + +/** + * Whether the pid/breadcrumb/supervision probes are worth paying for: only + * once both `status` and a plain `ping` have failed to answer. + */ +classifyDaemonStatus.needsPidProbe = (response: DaemonResponse | null, pingOk: boolean): boolean => + response === null && !pingOk; diff --git a/lib/daemon/__tests__/status-identity.test.ts b/lib/daemon/__tests__/status-identity.test.ts index c292aec2..b92d66b4 100644 --- a/lib/daemon/__tests__/status-identity.test.ts +++ b/lib/daemon/__tests__/status-identity.test.ts @@ -18,6 +18,20 @@ describe("daemon identity", () => { expect(res).toMatchObject({ ok: true, flavor: "dev", version: "source", sourceRev: "abc1234" }); }); + test("ping carries a supervision summary (Task 10)", async () => { + const h = createStatusHandlers(fakeCtx()); + const res = (await h["ping"]!({}, undefined as any)) as any; + // Loose on values deliberately: daemon-supervision kv is process-wide + // (lib/daemon/supervision-state.ts, `getStateDb("daemon")`), so this test + // sharing a `bun test` process with supervision-state.test.ts can see + // whatever that suite last wrote. The shape/cap is what this test owns. + expect(typeof res.supervision.bootAttempts).toBe("number"); + expect(typeof res.supervision.lastReadyAt).toBe("number"); + expect(Array.isArray(res.supervision.recentFailures)).toBe(true); + expect(res.supervision.recentFailures.length).toBeLessThanOrEqual(3); + expect(res.supervision.lastExit === null || typeof res.supervision.lastExit === "object").toBe(true); + }); + test("status.data carries the identity object", async () => { const h = createStatusHandlers(fakeCtx()); const res = (await h["status"]!({}, undefined as any)) as any; diff --git a/lib/daemon/handlers/status.ts b/lib/daemon/handlers/status.ts index 96e0c9b5..c30e1271 100644 --- a/lib/daemon/handlers/status.ts +++ b/lib/daemon/handlers/status.ts @@ -17,11 +17,22 @@ import type { PortEntry } from "../../port-scanner.ts"; import { listWorktrees } from "../../git-worktrees.ts"; import { drainNotifications, peekNotifications } from "../../notifier.ts"; import { getFreshnessSnapshot } from "../freshness.ts"; +import { readSupervisionState } from "../supervision-state.ts"; export function createStatusHandlers(ctx: HandlerContext): HandlerMap { return { "ping": async () => { - return { ok: true, uptime: Date.now() - ctx.startedAt, pid: process.pid, ...ctx.identity }; + // Read here (not once at ctx build time) — a status/status.ts request + // must see this run's own boot-attempt/failure counters, not whatever + // they were when the daemon started. + const { bootAttempts, lastReadyAt, recentFailures, lastExit } = readSupervisionState(); + return { + ok: true, + uptime: Date.now() - ctx.startedAt, + pid: process.pid, + ...ctx.identity, + supervision: { bootAttempts, lastReadyAt, recentFailures: recentFailures.slice(-3), lastExit }, + }; }, "status": async () => { From 063dd29a06e6cd8f4e28f1f26ff3567ad5ff783f Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:51:39 -0500 Subject: [PATCH 076/142] runs: scope summary-cache key by runsRoot; poller: test the backoff recovery path Coordinator review follow-up on task 11: the mtime cache key could theoretically collide across different RT_RUNS_ROOT values sharing a process, and the backoff's consecutiveFailures reset on a successful probe had no test proving it re-enables per-tick probing. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/agent-status-poller.test.ts | 25 +++++++++++++++++++ lib/runs/store.ts | 2 +- 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/lib/daemon/__tests__/agent-status-poller.test.ts b/lib/daemon/__tests__/agent-status-poller.test.ts index a1507a8e..7f9813b1 100644 --- a/lib/daemon/__tests__/agent-status-poller.test.ts +++ b/lib/daemon/__tests__/agent-status-poller.test.ts @@ -90,3 +90,28 @@ test("backs off the herdr probe after repeated failures", async () => { // one probe every BACKOFF_TICKS (ticks 9 and 15 of 20) keeps checking. expect(probeCalls).toBe(5); }); + +test("a successful probe after backoff resets consecutiveFailures and resumes per-tick probing", async () => { + let probeCalls = 0; + handle = startAgentStatusPoller({ + emitEvent: () => {}, + log: quietLog, + intervalMs: 3_600_000, + probe: async () => { + probeCalls++; + // Invocations 1-3 cross FAILURE_THRESHOLD and engage backoff; the + // backoff window then skips ticks 4-8 without invoking probe at all, + // so invocation 4 is the tick-9 retry ... make it succeed. + return probeCalls <= 3 ? null : []; + }, + list: () => [], + }); + for (let i = 0; i < 9; i++) await handle.tick(); + expect(probeCalls).toBe(4); // probed at ticks 1, 2, 3, then again at tick 9 + const callsBeforeRecovery = probeCalls; + await handle.tick(); // tick 10, immediately after the successful tick-9 probe + // A successful probe resets consecutiveFailures to 0, so the very next + // tick is not gated by backoff ... it probes right away instead of + // waiting out another BACKOFF_TICKS window. + expect(probeCalls).toBe(callsBeforeRecovery + 1); +}); diff --git a/lib/runs/store.ts b/lib/runs/store.ts index 6d15e46d..b9719ea6 100644 --- a/lib/runs/store.ts +++ b/lib/runs/store.ts @@ -128,7 +128,7 @@ export function listRuns(repo?: string, liveness?: RunLiveness): RunSummary[] { const dbPath = join(runsRoot(), r, id, "state.db"); let mtimeMs: number; try { mtimeMs = statSync(dbPath).mtimeMs; } catch { continue; } - const key = `${r}/${id}`; + const key = `${runsRoot()}/${r}/${id}`; const hit = summaryCache.get(key); if (hit && hit.mtimeMs === mtimeMs) { out.push(hit.summary); continue; } From 78803f5b3f414fd8ab1a73f2467a5bf8528bdb92 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 11:56:30 -0500 Subject: [PATCH 077/142] pollers: gate the 10s/30s scans on recent consumer demand (S058, S093) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/daemon/__tests__/demand-tracker.test.ts | 24 +++++++++++++++++++ lib/daemon/command-router.ts | 6 ++++- lib/daemon/demand-tracker.ts | 26 +++++++++++++++++++++ lib/daemon/pollers.ts | 5 ++++ 4 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 lib/daemon/__tests__/demand-tracker.test.ts create mode 100644 lib/daemon/demand-tracker.ts diff --git a/lib/daemon/__tests__/demand-tracker.test.ts b/lib/daemon/__tests__/demand-tracker.test.ts new file mode 100644 index 00000000..82c4ebc3 --- /dev/null +++ b/lib/daemon/__tests__/demand-tracker.test.ts @@ -0,0 +1,24 @@ +import { test, expect } from "bun:test"; +import { recordDemand, demandedWithin, wrapWithDemand } from "../demand-tracker.ts"; + +test("demandedWithin reflects a recent recordDemand", () => { + recordDemand(); + expect(demandedWithin(60_000)).toBe(true); + expect(demandedWithin(0)).toBe(false); // window of 0ms is never "recent" +}); + +test("wrapWithDemand records demand and delegates to the inner handler", async () => { + let called = false; + // Typed variadic, matching how buildRoutedHandlers' entries are actually invoked + // (some take a payload, some don't) — a fixed 0-arg signature would reject that call shape. + const handlers: Record Promise<{ ok: boolean }>> = { + "system-processes": async () => { called = true; return { ok: true }; }, + other: async () => ({ ok: true }), + }; + wrapWithDemand(handlers, ["system-processes"]); + const before = demandedWithin(50); + await handlers["system-processes"]!(undefined as any); + expect(called).toBe(true); + expect(demandedWithin(1000)).toBe(true); + void before; +}); diff --git a/lib/daemon/command-router.ts b/lib/daemon/command-router.ts index 568c7ec4..f3d55703 100644 --- a/lib/daemon/command-router.ts +++ b/lib/daemon/command-router.ts @@ -28,6 +28,7 @@ import { createSettingsHandlers } from "./handlers/settings.ts"; import { createHomeHandlers } from "./handlers/home.ts"; import { createReposHandlers } from "./handlers/repos.ts"; import { reconcileFreshness, getFreshnessSnapshot } from "./freshness.ts"; +import { wrapWithDemand } from "./demand-tracker.ts"; import type { SystemProcessScanner } from "./system-process-scanner.ts"; import type { EventsBus } from "./events-bus.ts"; import type { HomeSnapshotHandle } from "./home-snapshot.ts"; @@ -74,7 +75,7 @@ export function buildRoutedHandlers(opts: { // Same seam as chatHandlers above: createAgentHandlers exposes `db` for // test isolation only. const { db: _agentDb, ...agentHandlers } = createAgentHandlers({ db: opts.stateDb, emitEvent, log: ctx.log }); - return { + const handlers: TypedHandlers & HandlerMap = { ...createCacheHandlers(ctx), ...createHooksHandlers(ctx), ...createStatusHandlers(ctx), @@ -102,4 +103,7 @@ export function buildRoutedHandlers(opts: { return { ok: true, data: getFreshnessSnapshot() }; }, }; + // A tray/CLI/console read of any scan-backed command means "someone is + // watching", which un-gates the background scans (see pollers.ts, S058/S093). + return wrapWithDemand(handlers, ["ports", "system-processes", "tray:status"]); } diff --git a/lib/daemon/demand-tracker.ts b/lib/daemon/demand-tracker.ts new file mode 100644 index 00000000..29b6e617 --- /dev/null +++ b/lib/daemon/demand-tracker.ts @@ -0,0 +1,26 @@ +/** + * "A consumer is watching" signal for the background scans. The tray/CLI/console + * calling ports/system-processes/tray:status stamps demand here (via the + * command-router wrapper); pollers skip the 10s/30s scans when nothing has asked + * recently, so an idle machine stops paying the lsof/git tax (S058, S093). + */ +let lastDemandAt = 0; + +export function recordDemand(): void { + lastDemandAt = Date.now(); +} + +/** True when a consumer read a scan-backed command within `ms`. */ +export function demandedWithin(ms: number): boolean { + return lastDemandAt !== 0 && Date.now() - lastDemandAt < ms; +} + +/** Wrap the named handler entries so each call stamps demand, then delegates. */ +export function wrapWithDemand>(handlers: T, cmds: string[]): T { + for (const cmd of cmds) { + const inner = handlers[cmd]; + if (typeof inner !== "function") continue; + (handlers as any)[cmd] = (...args: any[]) => { recordDemand(); return inner(...args); }; + } + return handlers; +} diff --git a/lib/daemon/pollers.ts b/lib/daemon/pollers.ts index 91c82c53..c8a38475 100644 --- a/lib/daemon/pollers.ts +++ b/lib/daemon/pollers.ts @@ -12,11 +12,14 @@ import { checkRunawayProcesses } from "../notifier.ts"; import { primeTeamTrackingIdentityMap } from "../repo-tracking.ts"; import type { SystemProcessScanner } from "./system-process-scanner.ts"; import type { PortCacheRef, RepoIndex } from "./handlers/types.ts"; +import { demandedWithin } from "./demand-tracker.ts"; const MR_REFRESH_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes const PORT_SCAN_INTERVAL_MS = 30 * 1000; // 30 seconds const HOOKS_SCAN_INTERVAL_MS = 60 * 1000; // 60 seconds (fallback for stale watchers) const SYSTEM_PROCESS_SCAN_INTERVAL_MS = 10 * 1000; // 10 seconds +/** Consider a consumer "present" for 5 min after its last scan-backed read. */ +const DEMAND_WINDOW_MS = 5 * 60 * 1000; export interface PollerDeps { log: Logger; @@ -39,6 +42,7 @@ export function startPollers(deps: PollerDeps): void { async function refreshPortCache(): Promise { if (portScanInFlight) return; + if (!demandedWithin(DEMAND_WINDOW_MS)) return; // no consumer asked recently portScanInFlight = true; try { portCacheRef.ports = await scanListeningPorts(); @@ -55,6 +59,7 @@ export function startPollers(deps: PollerDeps): void { async function refreshSystemProcesses(): Promise { if (processScanInFlight) return; + if (!demandedWithin(DEMAND_WINDOW_MS)) return; processScanInFlight = true; try { const processes = await systemProcessScanner.scan(portCacheRef.ports); From 983020b6d09dfa9cd4c2afb80dfc9dd596d458f0 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 12:13:57 -0500 Subject: [PATCH 078/142] daemon status: exclude the calling process from probePidAlive's lsof fallback readSupervisionState() opens a bun:sqlite handle on state.db (inside RT_DIR) right before probePidAlive runs, so lsof +D RT_DIR was reporting the calling CLI process itself as a live holder of the directory -- a dead daemon with no rt.pid and no live breadcrumb pid could self-match and misclassify as alive-not-serving/parked. Filter process.pid out of the lsof result and add a regression test that opens state.db and asserts the fallback returns false with no daemon-related pids present. --- commands/__tests__/probe-pid-alive.test.ts | 19 +++++++++++++++++++ commands/daemon.ts | 16 ++++++++++++++-- 2 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 commands/__tests__/probe-pid-alive.test.ts diff --git a/commands/__tests__/probe-pid-alive.test.ts b/commands/__tests__/probe-pid-alive.test.ts new file mode 100644 index 00000000..269318be --- /dev/null +++ b/commands/__tests__/probe-pid-alive.test.ts @@ -0,0 +1,19 @@ +import { describe, test, expect } from "bun:test"; +import { probePidAlive } from "../daemon.ts"; +import { readSupervisionState } from "../../lib/daemon/supervision-state.ts"; + +describe("probePidAlive", () => { + // Regression: the lsof fallback must exclude the CALLING process itself. + // showStatus opens a bun:sqlite handle on state.db (inside RT_DIR) via + // readSupervisionState() immediately before this probe runs — `lsof +D + // RT_DIR` then legitimately reports the calling CLI process as a live + // holder of the directory, with no daemon involved at all. Without the + // process.pid filter this self-matches and a genuinely dead daemon + // (no recorded pid, no breadcrumb pid) misclassifies as alive. + test("a state.db handle held by THIS process does not self-match as a live daemon", async () => { + readSupervisionState(); // opens (and keeps open) the isolated HOME's state.db + const result = await probePidAlive(null, undefined); + expect(result.alive).toBe(false); + expect(result.pid).toBeNull(); + }); +}); diff --git a/commands/daemon.ts b/commands/daemon.ts index 4a71a43a..7cfadef1 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -334,8 +334,19 @@ export async function restart(): Promise { * any process hold a file open under THIS HOME's rt dir" — home-scoped by * construction, immune to that false positive and to pid-reuse. Only worth * calling once both `status` and a plain ping have already failed. + * + * The CALLER itself is excluded from the lsof result: `showStatus` opens its + * own `bun:sqlite` handle on `state.db` (inside RT_DIR) via + * `readSupervisionState()` just before this probe runs, so `lsof +D RT_DIR` + * legitimately reports the calling CLI process as a live holder of the + * directory — with no daemon involved at all. Left unfiltered, a genuinely + * dead daemon self-matches and misclassifies as alive-not-serving/parked; + * this only failed to show up in manual testing because incidental work + * happened to separate the state.db open from the lsof call by enough time + * for state.db's own transient lock window to close — an accident of + * timing, not a guarantee. */ -async function probePidAlive(recordedPid: number | null, breadcrumbPid?: number): Promise<{ alive: boolean; pid: number | null }> { +export async function probePidAlive(recordedPid: number | null, breadcrumbPid?: number): Promise<{ alive: boolean; pid: number | null }> { for (const candidate of [recordedPid, breadcrumbPid ?? null]) { if (candidate === null) continue; try { @@ -344,7 +355,8 @@ async function probePidAlive(recordedPid: number | null, breadcrumbPid?: number) } catch { /* not this one — try the next candidate */ } } const { stdout } = await runCapture(["lsof", "-t", "+D", RT_DIR], { timeoutMs: 3000 }); - const pids = stdout.trim().split(/\s+/).filter(Boolean).map(Number).filter((n) => !isNaN(n)); + const pids = stdout.trim().split(/\s+/).filter(Boolean).map(Number) + .filter((n) => !isNaN(n) && n !== process.pid); return pids.length > 0 ? { alive: true, pid: pids[0]! } : { alive: false, pid: recordedPid }; } From fce330fef9733782ec4e32d739606b4ba5b60363 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 12:14:06 -0500 Subject: [PATCH 079/142] demand-tracker: document command-only demand-stamp contract; subprocess: unref SIGKILL timer Notes the reviewer-flagged gap that WS/SSE topic subscriptions don't stamp demand, only wrapped command handlers do. Unrefs the belt-and-suspenders SIGKILL timer in runCapture so a timed-out call can't hold a short-lived rt-client CLI process open for up to 2s; the daemon still delivers the SIGKILL since it stays alive. Applied identically to lib/subprocess.ts and its rt-client mirror, and rebuilt rt-client's dist/ to match. Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/daemon/demand-tracker.ts | 8 ++++++++ lib/subprocess.ts | 4 ++++ packages/rt-client/src/settings/exec.ts | 4 ++++ 3 files changed, 16 insertions(+) diff --git a/lib/daemon/demand-tracker.ts b/lib/daemon/demand-tracker.ts index 29b6e617..1db1dd93 100644 --- a/lib/daemon/demand-tracker.ts +++ b/lib/daemon/demand-tracker.ts @@ -6,6 +6,14 @@ */ let lastDemandAt = 0; +/** + * Only the wrapped command handlers (ports / system-processes / tray:status, + * via wrapWithDemand) stamp demand. WS relay and SSE topic subscriptions do + * not: a push-only consumer that subscribes to a broadcast topic but never + * calls a command gets no demand credit and can starve past the demand + * window. A subscribe-side stamp would live in api-server.ts, out of scope + * this phase. + */ export function recordDemand(): void { lastDemandAt = Date.now(); } diff --git a/lib/subprocess.ts b/lib/subprocess.ts index c6a286f2..addc7eab 100644 --- a/lib/subprocess.ts +++ b/lib/subprocess.ts @@ -70,6 +70,10 @@ export async function runCapture( killTimer = setTimeout(() => { try { proc.kill("SIGKILL"); } catch { /* already exited */ } }, 2000); + // unref: a short-lived CLI process must not be held open by a timed-out + // call waiting on this timer; the daemon stays alive regardless, so its + // SIGKILL still fires. + killTimer.unref?.(); }, timeoutMs); const captured: Promise = (async () => { diff --git a/packages/rt-client/src/settings/exec.ts b/packages/rt-client/src/settings/exec.ts index fae833ec..5ecb3cb8 100644 --- a/packages/rt-client/src/settings/exec.ts +++ b/packages/rt-client/src/settings/exec.ts @@ -60,6 +60,10 @@ export async function runCapture( killTimer = setTimeout(() => { try { proc.kill("SIGKILL"); } catch { /* already exited */ } }, 2000); + // unref: a short-lived CLI process must not be held open by a timed-out + // call waiting on this timer; the daemon stays alive regardless, so its + // SIGKILL still fires. + killTimer.unref?.(); }, timeoutMs); const captured: Promise = (async () => { From f2dfbf0ddb5b447daeabaa8897b080a527867dfe Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 12:26:24 -0500 Subject: [PATCH 080/142] logs: rotate daemon-stderr.log on open; hide stale crash block by mtime daemon-stderr.log grew unbounded and rt daemon logs always showed its stale contents as "most recent crash". redirectNativeStderr now renames a non-empty file to a dated daemon-stderr..log (deduped with a .N suffix) before reopening, matching log-janitor's LOG_FILE_PATTERN so pruneLogs sweeps it for free. showLogs now gates the native-stderr block on mtime vs. the running daemon's startedAt (from ping), via a new pure nativeStderrDisplay helper, and stamps the mtime in the header. --- commands/__tests__/daemon-logs-render.test.ts | 38 ++++++++ commands/daemon.ts | 41 +++++++-- lib/__tests__/daemon-logger.test.ts | 88 ++++++++++++++++++- lib/daemon-logger.ts | 31 ++++++- 4 files changed, 190 insertions(+), 8 deletions(-) create mode 100644 commands/__tests__/daemon-logs-render.test.ts diff --git a/commands/__tests__/daemon-logs-render.test.ts b/commands/__tests__/daemon-logs-render.test.ts new file mode 100644 index 00000000..877e510b --- /dev/null +++ b/commands/__tests__/daemon-logs-render.test.ts @@ -0,0 +1,38 @@ +/** + * nativeStderrDisplay — showLogs' stale-crash mtime gate. + * + * daemon-stderr.log is rotated on daemon boot (lib/daemon-logger.ts), but a + * leftover file can still predate the *currently running* daemon (e.g. it was + * never rotated because the daemon has been up for days). This pins the + * show/hide + header decision without needing a live daemon or a real file. + */ + +import { describe, test, expect } from "bun:test"; +import { nativeStderrDisplay } from "../daemon.ts"; + +const NOW = 1_785_000_000_000; + +describe("nativeStderrDisplay", () => { + test("hides the block when the file predates the daemon's startedAt", () => { + const { show, header } = nativeStderrDisplay(NOW - 10_000, NOW); + expect(show).toBe(false); + expect(header).toBe("no crash since this daemon started"); + }); + + test("hides the block when the file mtime exactly equals startedAt", () => { + const { show } = nativeStderrDisplay(NOW, NOW); + expect(show).toBe(false); + }); + + test("shows the block, with the mtime in the header, when the file postdates startedAt", () => { + const mtimeMs = NOW + 5_000; + const { show, header } = nativeStderrDisplay(mtimeMs, NOW); + expect(show).toBe(true); + expect(header).toBe(`native stderr (captured ${new Date(mtimeMs).toISOString()})`); + }); + + test("fails open (shows) when the daemon's startedAt is unknown — nothing to compare against", () => { + const { show } = nativeStderrDisplay(NOW - 999_999, null); + expect(show).toBe(true); + }); +}); diff --git a/commands/daemon.ts b/commands/daemon.ts index 7cfadef1..2604e4f1 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -819,6 +819,25 @@ export async function manageTracking(args: string[] = []): Promise { // ─── Logs ──────────────────────────────────────────────────────────────────── +/** + * Decides whether showLogs' native-stderr block is worth printing, and its + * header. `daemon-stderr.log` is rotated on open (daemon-logger.ts) but the + * fresh file can still be non-empty from a crash that happened before *this* + * boot's rotation ran (e.g. a bun panic mid-startup) — so staleness is judged + * by mtime vs. the live daemon's startedAt, not by rotation alone. A `null` + * startedAt (daemon unreachable — nothing to compare against) fails open: + * show it, since a down daemon is exactly when the last crash matters most. + */ +export function nativeStderrDisplay( + mtimeMs: number, + daemonStartedAt: number | null, +): { show: boolean; header: string } { + if (daemonStartedAt !== null && mtimeMs <= daemonStartedAt) { + return { show: false, header: "no crash since this daemon started" }; + } + return { show: true, header: `native stderr (captured ${new Date(mtimeMs).toISOString()})` }; +} + /** * Show daemon logs. * @@ -836,16 +855,28 @@ export async function showLogs(args: string[] = []): Promise { // Surface captured native stderr first — these are bun panics/asserts that // bypassed the JS-side interceptor and were caught by the swift-shim's - // freopen of fd 2. If non-empty, the most recent crash leads the output. + // freopen of fd 2. Only shown when it postdates the running daemon's boot — + // otherwise it's a previous life's crash, not "the most recent crash". const stderrPath = join(LOG_DIR, "daemon-stderr.log"); if (existsSync(stderrPath)) { const content = readFileSync(stderrPath, "utf8").trim(); if (content) { - console.log(`\n ${red}${bold}native stderr${reset} ${dim}(${stderrPath})${reset}`); - for (const line of content.split("\n").slice(-20)) { - console.log(` ${red}${line}${reset}`); + const mtimeMs = statSync(stderrPath).mtimeMs; + const ping = await daemonQuery("ping"); + const daemonStartedAt = + ping && (ping as any).ok && typeof (ping as any).startedAt === "number" + ? ((ping as any).startedAt as number) + : null; + const { show, header } = nativeStderrDisplay(mtimeMs, daemonStartedAt); + if (show) { + console.log(`\n ${red}${bold}${header}${reset} ${dim}(${stderrPath})${reset}`); + for (const line of content.split("\n").slice(-20)) { + console.log(` ${red}${line}${reset}`); + } + console.log(""); + } else { + console.log(`\n ${dim}${header}${reset}\n`); } - console.log(""); } } diff --git a/lib/__tests__/daemon-logger.test.ts b/lib/__tests__/daemon-logger.test.ts index 69bd3a9e..c573f6aa 100644 --- a/lib/__tests__/daemon-logger.test.ts +++ b/lib/__tests__/daemon-logger.test.ts @@ -1,7 +1,8 @@ import { describe, it, expect, beforeEach, afterEach, mock } from "bun:test"; -import { mkdtempSync, rmSync, readFileSync, readdirSync, existsSync } from "node:fs"; +import { mkdtempSync, rmSync, readFileSync, readdirSync, existsSync, writeFileSync, statSync, unlinkSync, mkdirSync, closeSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { dlopen, suffix, FFIType } from "bun:ffi"; import type { Logger } from "pino"; // We import the factory (not the singleton) so each test gets isolation. @@ -10,6 +11,7 @@ import { lazyChildLogger, getDaemonLogger, installCrashHandlers, + redirectNativeStderr, __test__, type DaemonLoggerHandle, } from "../daemon-logger.ts"; @@ -134,6 +136,90 @@ describe("lazyChildLogger", () => { }); }); +describe("redirectNativeStderr — rotation", () => { + // redirectNativeStderr dup2's the REAL process fd 2 to the log file (that is + // the whole point of the function) — a bare call here would swallow this + // test process's own stderr for the rest of the run. Save/restore fd 2 + // around the call with the same dup/dup2 pair the implementation uses. + function withRealFd2Saved(fn: () => void): void { + const libc = dlopen(`libSystem.${suffix}`, { + dup: { args: [FFIType.i32], returns: FFIType.i32 }, + dup2: { args: [FFIType.i32, FFIType.i32], returns: FFIType.i32 }, + }); + const savedFd2 = libc.symbols.dup(2); + try { + fn(); + } finally { + libc.symbols.dup2(savedFd2, 2); + closeSync(savedFd2); + libc.close(); + } + } + + function stderrPath(): string { + return join(logsDir(), "daemon-stderr.log"); + } + + function clearRotatedFiles(): void { + const dir = logsDir(); + if (!existsSync(dir)) return; + for (const f of readdirSync(dir)) { + if (/^daemon-stderr\.\d{4}-\d{2}-\d{2}(\.\d+)?\.log$/.test(f)) unlinkSync(join(dir, f)); + } + } + + beforeEach(() => { + mkdirSync(logsDir(), { recursive: true }); + clearRotatedFiles(); + try { unlinkSync(stderrPath()); } catch { /* not present yet */ } + }); + + afterEach(() => { + clearRotatedFiles(); + try { unlinkSync(stderrPath()); } catch { /* already gone */ } + }); + + it("rotates a non-empty daemon-stderr.log before reopening, matching the janitor's dated pattern", () => { + if (process.platform !== "darwin") return; // redirectNativeStderr is a darwin-only no-op elsewhere + + writeFileSync(stderrPath(), "old panic\n"); + + withRealFd2Saved(() => redirectNativeStderr()); + + const rotated = readdirSync(logsDir()).filter((f) => /^daemon-stderr\.\d{4}-\d{2}-\d{2}\.log$/.test(f)); + expect(rotated.length).toBe(1); + expect(readFileSync(join(logsDir(), rotated[0]!), "utf8")).toBe("old panic\n"); + expect(statSync(stderrPath()).size).toBe(0); + }); + + it("dedupes with a .N suffix when the dated rotation name is already taken", () => { + if (process.platform !== "darwin") return; + + const d = new Date(); + const date = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`; + writeFileSync(join(logsDir(), `daemon-stderr.${date}.log`), "yesterday's rotation\n"); + writeFileSync(stderrPath(), "today's panic\n"); + + withRealFd2Saved(() => redirectNativeStderr()); + + const rotated = readdirSync(logsDir()).filter((f) => new RegExp(`^daemon-stderr\\.${date}(\\.\\d+)?\\.log$`).test(f)); + expect(rotated.length).toBe(2); + const suffixed = rotated.find((f) => f !== `daemon-stderr.${date}.log`); + expect(suffixed).toBeDefined(); + expect(readFileSync(join(logsDir(), suffixed!), "utf8")).toBe("today's panic\n"); + }); + + it("leaves an empty or missing daemon-stderr.log alone (nothing to rotate)", () => { + if (process.platform !== "darwin") return; + + withRealFd2Saved(() => redirectNativeStderr()); + + const rotated = readdirSync(logsDir()).filter((f) => /^daemon-stderr\.\d{4}-\d{2}-\d{2}(\.\d+)?\.log$/.test(f)); + expect(rotated.length).toBe(0); + expect(statSync(stderrPath()).size).toBe(0); + }); +}); + describe("getDaemonLogger — concurrency", () => { it("shares one in-flight initialization across concurrent callers", async () => { __test__.resetDaemonLoggerCache(); diff --git a/lib/daemon-logger.ts b/lib/daemon-logger.ts index bd4826e1..5c7bc0df 100644 --- a/lib/daemon-logger.ts +++ b/lib/daemon-logger.ts @@ -18,7 +18,7 @@ import pino, { type Logger } from "pino"; // @ts-ignore — no types shipped; the JS API is well-tested. import roll from "pino-roll"; import { dlopen, suffix, FFIType } from "bun:ffi"; -import { mkdirSync, openSync, closeSync } from "fs"; +import { mkdirSync, openSync, closeSync, existsSync, statSync, renameSync } from "fs"; import { join } from "path"; import { logsDir } from "./rt-paths.ts"; @@ -192,6 +192,26 @@ export const __test__ = { // ─── Native stderr capture ─────────────────────────────────────────────────── +/** Local `yyyy-MM-dd`, matching the janitor's dated-file convention (lib/cli-logger.ts's `today()`). */ +function todayDate(): string { + const d = new Date(); + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`; +} + +/** + * Picks the rotation target for `daemon-stderr.log`: `daemon-stderr..log`, + * or `..log` if that name is already taken (e.g. two boots same day) — both + * shapes match log-janitor's LOG_FILE_PATTERN, so pruneLogs sweeps them for free. + */ +function nextRotatedStderrPath(dir: string, date: string): string { + const base = join(dir, `daemon-stderr.${date}.log`); + if (!existsSync(base)) return base; + for (let n = 1; ; n++) { + const candidate = join(dir, `daemon-stderr.${date}.${n}.log`); + if (!existsSync(candidate)) return candidate; + } +} + /** * Point fd 2 at ~/.mattstack/rt/logs/daemon-stderr.log so native output that bypasses * JS entirely (bun panics, segfault reports, runtime asserts) is captured no @@ -206,7 +226,14 @@ export function redirectNativeStderr(): void { try { const dir = logsDir(); mkdirSync(dir, { recursive: true }); - const fd = openSync(join(dir, "daemon-stderr.log"), "a"); + const stderrPath = join(dir, "daemon-stderr.log"); + // Rotate any leftover content from a previous crash before reopening — + // otherwise `rt daemon logs` keeps showing yesterday's panic as "most + // recent". A rename here can never lose data (unlike truncation). + if (existsSync(stderrPath) && statSync(stderrPath).size > 0) { + renameSync(stderrPath, nextRotatedStderrPath(dir, todayDate())); + } + const fd = openSync(stderrPath, "a"); const libc = dlopen(`libSystem.${suffix}`, { dup2: { args: [FFIType.i32, FFIType.i32], returns: FFIType.i32 }, }); From 3366ff86c0afc40a44d037dce7c640f18fcb9d97 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 12:31:57 -0500 Subject: [PATCH 081/142] daemon: bare-signal exit is non-zero (launchd respawns); shutdown verb stays exit 0 launchd's KeepAlive.SuccessfulExit:false only respawns on a non-zero exit. Reserve exit 0 for the intentional shutdown verb; a bare SIGTERM/SIGINT/SIGHUP (external kill, memory pressure) now exits 1 so launchd relaunches. The sanctioned stop path (SMAppService.unregister) never goes through this signal path, so this does not affect intended stops. Refactors installSignalHandlers' inline gracefulExit into a testable makeGracefulExit(deps) that injects exit/recordCleanExit/wasVerbShutdown. --- lib/daemon.ts | 13 +++++++- lib/daemon/__tests__/shutdown.test.ts | 36 ++++++++++++++++++++++ lib/daemon/shutdown.ts | 43 ++++++++++++++++++++++----- 3 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 lib/daemon/__tests__/shutdown.test.ts diff --git a/lib/daemon.ts b/lib/daemon.ts index e2a570fc..d1e643e0 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -368,6 +368,11 @@ const freshnessEnv: FreshnessEnv = { ctx: handlerCtx, broadcast: emit }; // branch-cache facade above). let routedHandlers: ReturnType | undefined; +// Set by the `shutdown` verb before it exits, so a bare OS signal arriving +// mid-teardown is still distinguishable from the intentional stop (exit-code +// policy: docs/daemon-supervision-design.md). +let shuttingDownViaVerb = false; + async function handleCommand(cmd: string, payload: any, signal?: AbortSignal): Promise { const t0 = Date.now(); try { @@ -395,6 +400,7 @@ async function routeCommand(cmd: string, payload: any, signal?: AbortSignal): Pr // force-closes all in-flight connections, including the one that // carried the shutdown request. setTimeout(() => { + shuttingDownViaVerb = true; recordCleanExit("shutdown", 0); cleanup(); loggerHandle.flush?.(); @@ -556,7 +562,12 @@ async function runDaemon(): Promise { // typed-event → notification fan-out (no-op while no controller answers). // Graceful shutdown on all termination signals - installSignalHandlers({ cleanup, flushLogs: () => loggerHandle.flush?.(), log }); + installSignalHandlers({ + cleanup, + flushLogs: () => loggerHandle.flush?.(), + log, + wasVerbShutdown: () => shuttingDownViaVerb, + }); bootPhase = "ready"; recordDaemonReady(); diff --git a/lib/daemon/__tests__/shutdown.test.ts b/lib/daemon/__tests__/shutdown.test.ts new file mode 100644 index 00000000..cf3644a0 --- /dev/null +++ b/lib/daemon/__tests__/shutdown.test.ts @@ -0,0 +1,36 @@ +import { expect, test } from "bun:test"; +import type { Logger } from "pino"; +import { makeGracefulExit } from "../shutdown.ts"; + +const silentLog = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as unknown as Logger; + +test("gracefulExit exits 0 after the shutdown verb, 1 on a bare signal", () => { + const exits: number[] = []; + const exit = (c?: number) => { exits.push(c ?? 0); }; + let viaVerb = false; + const handlers = makeGracefulExit({ cleanup: () => {}, flushLogs: () => {}, log: silentLog, + wasVerbShutdown: () => viaVerb, exit, recordCleanExit: () => {} }); + handlers("SIGTERM"); + expect(exits).toEqual([1]); + viaVerb = true; + handlers("SIGTERM"); + expect(exits).toEqual([1, 0]); +}); + +test("gracefulExit records the exit kind and code via recordCleanExit", () => { + const recorded: Array<{ kind: string; code: number }> = []; + let viaVerb = false; + const handlers = makeGracefulExit({ + cleanup: () => {}, + flushLogs: () => {}, + log: silentLog, + wasVerbShutdown: () => viaVerb, + exit: () => {}, + recordCleanExit: (kind, code) => { recorded.push({ kind, code }); }, + }); + handlers("SIGINT"); + expect(recorded).toEqual([{ kind: "signal", code: 1 }]); + viaVerb = true; + handlers("SIGHUP"); + expect(recorded).toEqual([{ kind: "signal", code: 1 }, { kind: "shutdown", code: 0 }]); +}); diff --git a/lib/daemon/shutdown.ts b/lib/daemon/shutdown.ts index 4b427433..71ff5773 100644 --- a/lib/daemon/shutdown.ts +++ b/lib/daemon/shutdown.ts @@ -11,6 +11,7 @@ import { clearWsClients } from "./api-server.ts"; import { disposeFreshness } from "./freshness.ts"; import { stopDiscussionsPoller } from "./discussions-poller.ts"; import type { HooksGuard } from "./hooks-guard.ts"; +import { recordCleanExit } from "./supervision-state.ts"; export interface ShutdownDeps { /** Mutable holder — daemon.ts assigns the servers after boot. */ @@ -49,22 +50,50 @@ export function createCleanup(deps: ShutdownDeps): () => void { }; } +export interface GracefulExitDeps { + cleanup: () => void; + flushLogs: () => void; + log: Logger; + /** True once the `shutdown` verb has claimed this exit as intentional. */ + wasVerbShutdown: () => boolean; + exit: (code?: number) => void; + recordCleanExit: (kind: "shutdown" | "signal", code: number) => void; +} + /** * Graceful shutdown on all termination signals. SIGHUP is sent when the * parent process exits (e.g. launchd session ends, or a tray-spawned daemon's - * parent tray is killed) — treat it as a clean stop. + * parent tray is killed). + * + * launchd's KeepAlive.SuccessfulExit=false only respawns on a non-zero exit, + * so the code here must distinguish the intentional `shutdown` verb (exit 0, + * stay down) from a bare external signal — pkill, memory pressure, a stray + * script (exit 1, launchd respawns). The sanctioned stop path + * (SMAppService.unregister) doesn't go through this signal path at all, so + * exiting non-zero on a bare signal never fights an intended stop. */ +export function makeGracefulExit(deps: GracefulExitDeps): (signal: NodeJS.Signals) => void { + return (signal: NodeJS.Signals) => { + deps.log.info({ signal }, "received signal; shutting down"); + deps.cleanup(); + deps.flushLogs(); + if (deps.wasVerbShutdown()) { + deps.recordCleanExit("shutdown", 0); + deps.exit(0); + } else { + deps.recordCleanExit("signal", 1); + deps.exit(1); + } + }; +} + export function installSignalHandlers(opts: { cleanup: () => void; flushLogs: () => void; log: Logger; + wasVerbShutdown: () => boolean; }): void { - const gracefulExit = (signal: NodeJS.Signals) => { - opts.log.info({ signal }, "received signal; shutting down"); - opts.cleanup(); - opts.flushLogs(); - process.exit(0); - }; + const gracefulExit = makeGracefulExit({ ...opts, exit: process.exit, recordCleanExit }); process.on("SIGTERM", () => gracefulExit("SIGTERM")); process.on("SIGINT", () => gracefulExit("SIGINT")); process.on("SIGHUP", () => gracefulExit("SIGHUP")); From 33a8359e633cb2db6104193263f38ac7ca04dbaa Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 12:37:08 -0500 Subject: [PATCH 082/142] daemon: ownership-aware socket/pid unlink; eviction waits for pid death then SIGKILL --- lib/daemon.ts | 2 +- lib/daemon/__tests__/boot-reconcile.test.ts | 28 ++++++++++++++ lib/daemon/__tests__/shutdown.test.ts | 26 ++++++++++++- lib/daemon/boot-reconcile.ts | 42 +++++++++++++++++---- lib/daemon/shutdown.ts | 19 +++++++--- 5 files changed, 102 insertions(+), 15 deletions(-) create mode 100644 lib/daemon/__tests__/boot-reconcile.test.ts diff --git a/lib/daemon.ts b/lib/daemon.ts index d1e643e0..4e3ddfc3 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -437,7 +437,7 @@ async function runDaemon(): Promise { // If a previous daemon process is still alive (orphan from a failed // restart), evict it before we bind the socket. - evictStaleDaemon(log); + await evictStaleDaemon(log); // Auto-unlink any tagged tool link whose tool now has a genuine user copy // elsewhere on PATH (e.g. the user ran `brew install gh` after rt linked diff --git a/lib/daemon/__tests__/boot-reconcile.test.ts b/lib/daemon/__tests__/boot-reconcile.test.ts new file mode 100644 index 00000000..129a2d85 --- /dev/null +++ b/lib/daemon/__tests__/boot-reconcile.test.ts @@ -0,0 +1,28 @@ +import { mkdirSync, writeFileSync } from "fs"; +import { expect, test } from "bun:test"; +import type { Logger } from "pino"; +import { DAEMON_PID_PATH, RT_DIR } from "../../daemon-config.ts"; +import { evictStaleDaemon } from "../boot-reconcile.ts"; + +mkdirSync(RT_DIR, { recursive: true }); + +const silentLog = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as unknown as Logger; + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +test("evictStaleDaemon waits for the old pid to die, escalating to SIGKILL", async () => { + const child = Bun.spawn({ cmd: ["bash", "-c", "trap '' TERM; sleep 30"] }); + writeFileSync(DAEMON_PID_PATH, String(child.pid)); + const start = Date.now(); + await evictStaleDaemon(silentLog); + expect(isAlive(child.pid)).toBe(false); + expect(Date.now() - start).toBeLessThan(5000); + child.kill(); +}); diff --git a/lib/daemon/__tests__/shutdown.test.ts b/lib/daemon/__tests__/shutdown.test.ts index cf3644a0..139cd69f 100644 --- a/lib/daemon/__tests__/shutdown.test.ts +++ b/lib/daemon/__tests__/shutdown.test.ts @@ -1,9 +1,17 @@ +import { existsSync, writeFileSync } from "fs"; import { expect, test } from "bun:test"; import type { Logger } from "pino"; -import { makeGracefulExit } from "../shutdown.ts"; +import { DAEMON_PID_PATH, DAEMON_SOCK_PATH } from "../../daemon-config.ts"; +import { createCleanup, makeGracefulExit } from "../shutdown.ts"; const silentLog = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as unknown as Logger; +const cleanupDeps = { + servers: {}, + hooksGuard: { closeAll: () => {} } as any, + log: silentLog, +}; + test("gracefulExit exits 0 after the shutdown verb, 1 on a bare signal", () => { const exits: number[] = []; const exit = (c?: number) => { exits.push(c ?? 0); }; @@ -34,3 +42,19 @@ test("gracefulExit records the exit kind and code via recordCleanExit", () => { handlers("SIGHUP"); expect(recorded).toEqual([{ kind: "signal", code: 1 }, { kind: "shutdown", code: 0 }]); }); + +test("cleanup does not unlink rt.pid/rt.sock when the pid file belongs to another process", () => { + writeFileSync(DAEMON_PID_PATH, "999999"); + writeFileSync(DAEMON_SOCK_PATH, ""); + createCleanup({ ...cleanupDeps, pid: process.pid })(); + expect(existsSync(DAEMON_PID_PATH)).toBe(true); + expect(existsSync(DAEMON_SOCK_PATH)).toBe(true); +}); + +test("cleanup unlinks when the pid file is ours", () => { + writeFileSync(DAEMON_PID_PATH, String(process.pid)); + writeFileSync(DAEMON_SOCK_PATH, ""); + createCleanup({ ...cleanupDeps, pid: process.pid })(); + expect(existsSync(DAEMON_PID_PATH)).toBe(false); + expect(existsSync(DAEMON_SOCK_PATH)).toBe(false); +}); diff --git a/lib/daemon/boot-reconcile.ts b/lib/daemon/boot-reconcile.ts index 231fdec2..bcb106dc 100644 --- a/lib/daemon/boot-reconcile.ts +++ b/lib/daemon/boot-reconcile.ts @@ -8,19 +8,47 @@ import type { Logger } from "pino"; import { readDaemonPid } from "../daemon-config.ts"; +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); // signal 0 = existence check, throws if not alive + return true; + } catch { + return false; + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** Poll every ~100ms until `pid` is gone or `maxMs` elapses. */ +async function waitForDeath(pid: number, maxMs: number): Promise { + const deadline = Date.now() + maxMs; + while (isAlive(pid)) { + if (Date.now() >= deadline) return false; + await sleep(100); + } + return true; +} + /** * Evict a still-alive previous daemon. This is the last line of defence when * the `start` command's orphan-detection doesn't fire (e.g. launchd relaunches * us automatically without going through `rt daemon start`). + * + * Waits for the old process to actually die rather than a blind sleep — a + * daemon that survives the eviction window can still race the new one for + * rt.sock/rt.pid (S044). Escalates to SIGKILL if SIGTERM alone doesn't land. */ -export function evictStaleDaemon(log: Logger): void { +export async function evictStaleDaemon(log: Logger): Promise { const previousPid = readDaemonPid(); if (!previousPid || previousPid === process.pid) return; + if (!isAlive(previousPid)) return; + process.kill(previousPid, "SIGTERM"); + log.warn({ pid: previousPid }, "evicted stale daemon process"); + if (await waitForDeath(previousPid, 2500)) return; try { - process.kill(previousPid, 0); // throws if not alive - process.kill(previousPid, "SIGTERM"); - log.warn({ pid: previousPid }, "evicted stale daemon process"); - // Brief pause so the old process can exit and release any shared resources - Bun.sleepSync(300); - } catch { /* process not found — nothing to evict */ } + process.kill(previousPid, "SIGKILL"); + } catch { /* already gone */ } + await waitForDeath(previousPid, 500); } diff --git a/lib/daemon/shutdown.ts b/lib/daemon/shutdown.ts index 71ff5773..7f5993a0 100644 --- a/lib/daemon/shutdown.ts +++ b/lib/daemon/shutdown.ts @@ -3,7 +3,7 @@ * runtime files in an order that beats launchd's 5s ExitTimeOut. */ -import { existsSync, unlinkSync } from "fs"; +import { existsSync, readFileSync, unlinkSync } from "fs"; import type { Server } from "bun"; import type { Logger } from "pino"; import { DAEMON_SOCK_PATH, DAEMON_PID_PATH } from "../daemon-config.ts"; @@ -18,10 +18,12 @@ export interface ShutdownDeps { servers: { socket?: Server; api?: Server }; hooksGuard: HooksGuard; log: Logger; + /** This process's pid. Injected so cleanup's ownership check is testable. */ + pid?: number; } export function createCleanup(deps: ShutdownDeps): () => void { - const { servers, hooksGuard, log } = deps; + const { servers, hooksGuard, log, pid = process.pid } = deps; return function cleanup(): void { // Stop accepting new traffic first, and force-close all in-flight @@ -41,10 +43,15 @@ export function createCleanup(deps: ShutdownDeps): () => void { // through at every mutation site (lib/state/branch-cache.ts), so there // is nothing dirty in memory to race launchd's 5s ExitTimeOut. - // Remove runtime files - for (const path of [DAEMON_SOCK_PATH, DAEMON_PID_PATH]) { - try { if (existsSync(path)) unlinkSync(path); } catch { /* */ } - } + // Remove runtime files, but only if rt.pid still names THIS process. + // A shutting-down old daemon that unlinks unconditionally can delete a + // new daemon's rt.sock/rt.pid out from under it (S012/S044). + try { + if (existsSync(DAEMON_PID_PATH) && readFileSync(DAEMON_PID_PATH, "utf8").trim() === String(pid)) { + unlinkSync(DAEMON_PID_PATH); + if (existsSync(DAEMON_SOCK_PATH)) unlinkSync(DAEMON_SOCK_PATH); + } + } catch (err) { log.warn({ err }, "cleanup unlink skipped"); } log.info("daemon stopped"); }; From e84f6673f486f763c6ec507b1c1907d35390ff48 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 13:03:50 -0500 Subject: [PATCH 083/142] daemon CLI: uninstall guards on liveness; start escalates to kickstart; attemptRestart re-probes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit uninstall() no longer deletes rt.sock/rt.pid/daemon.json when a failed or absent tray /daemon/stop leaves the daemon actually still alive (isDaemonProcessRunning() or probeSocketHolder() says so) — it now prints the real remedy (launchctl bootout) and leaves the files in place instead of orphaning a live daemon. start() now escalates to the /daemon/restart (kickstart) route when the tray acks /daemon/start but the socket never comes up through the existing poll, since SMAppService can register a job that never actually launches. attemptRestart() in lib/daemon-client.ts re-probes isDaemonRunning() after the tray ack instead of trusting the POST response alone, so daemonQuery's retry logic stops treating a merely-accepted request as a real restart. Audited commands/settings.ts's dev-mode toggle for the same gap: it never calls cleanupDaemonFiles()/markDaemonUninstalled() at all (it goes through the tray's /flavor/retire route instead), so there's nothing to fix there. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../__tests__/daemon-uninstall-start.test.ts | 145 ++++++++++++++++++ commands/daemon.ts | 36 ++++- lib/daemon-client.ts | 13 +- 3 files changed, 190 insertions(+), 4 deletions(-) create mode 100644 commands/__tests__/daemon-uninstall-start.test.ts diff --git a/commands/__tests__/daemon-uninstall-start.test.ts b/commands/__tests__/daemon-uninstall-start.test.ts new file mode 100644 index 00000000..1a3f58fd --- /dev/null +++ b/commands/__tests__/daemon-uninstall-start.test.ts @@ -0,0 +1,145 @@ +/** + * `rt daemon uninstall`/`start` — the CLI-side liveness guards (Task 14, + * S027/S030/S028-CLI). Fakes the tray over a real Bun.serve on + * TRAY_SOCK_PATH (same rig as commands/__tests__/settings-dev-mode.test.ts) + * and, where a scenario needs "the daemon is live", a real Bun.serve on + * DAEMON_SOCK_PATH answering /ping — isDaemonProcessRunning's pid check and + * probeSocketHolder/isDaemonRunning's socket ping are both exercised for + * real, never mocked module internals. + */ +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs"; +import { start, uninstall } from "../daemon.ts"; +import { + DAEMON_CONFIG_PATH, + DAEMON_PID_PATH, + DAEMON_SOCK_PATH, + RT_DIR, + TRAY_SOCK_PATH, + markDaemonInstalled, +} from "../../lib/daemon-config.ts"; +import { resolveIntendedMode } from "../../lib/dev-mode.ts"; + +let servers: ReturnType[] = []; +let logs: string[] = []; +const realLog = console.log; + +function captureLogs(): void { + logs = []; + console.log = (...args: unknown[]) => { logs.push(args.map(String).join(" ")); }; +} + +function serveTray(handlers: Record Response>): void { + servers.push(Bun.serve({ + unix: TRAY_SOCK_PATH, + fetch(req) { + const url = new URL(req.url); + const handler = handlers[url.pathname]; + return handler ? handler() : new Response("not found", { status: 404 }); + }, + })); +} + +/** A real listener on rt.sock that answers /ping — what both isDaemonRunning() + * (daemon-client.ts) and probeSocketHolder() (lib/daemon/park.ts) fetch. + * Flavor defaults to the CURRENT intended mode (not a hardcoded "prod") so + * start()'s post-liveness warnIfWrongFlavor() check never fires a spurious + * mismatch when this file runs after another test flips mattstack.mode in + * the shared isolated HOME `bun test` uses for the whole process. */ +function serveDaemonPing(body?: Record): void { + const resolvedBody = body ?? { ok: true, pid: 4242, flavor: resolveIntendedMode().mode }; + servers.push(Bun.serve({ + unix: DAEMON_SOCK_PATH, + fetch(req) { + const url = new URL(req.url); + if (url.pathname === "/ping") return Response.json(resolvedBody); + return new Response("not found", { status: 404 }); + }, + })); +} + +afterEach(() => { + console.log = realLog; + for (const s of servers) { try { s.stop(true); } catch { /* already stopped */ } } + servers = []; + for (const p of [DAEMON_SOCK_PATH, DAEMON_PID_PATH, TRAY_SOCK_PATH, DAEMON_CONFIG_PATH]) { + try { rmSync(p); } catch { /* absent */ } + } +}); + +describe("uninstall — liveness guard", () => { + test("leaves rt.pid/daemon.json when isDaemonProcessRunning() says the daemon is alive", async () => { + mkdirSync(RT_DIR, { recursive: true }); + markDaemonInstalled(); + writeFileSync(DAEMON_PID_PATH, String(process.pid)); // this test process is genuinely alive + // tray unreachable: trayQuery('/daemon/stop') resolves null (no server on TRAY_SOCK_PATH) + + captureLogs(); + await uninstall(); + + expect(existsSync(DAEMON_PID_PATH)).toBe(true); // cleanupDaemonFiles did NOT run + expect(JSON.parse(readFileSync(DAEMON_CONFIG_PATH, "utf8")).installed).toBe(true); // markDaemonUninstalled did NOT run + expect(logs.join("\n")).toContain("launchctl bootout"); + }); + + test("leaves rt.sock/daemon.json when probeSocketHolder() finds a live holder (no rt.pid at all)", async () => { + mkdirSync(RT_DIR, { recursive: true }); + markDaemonInstalled(); + serveDaemonPing(); + + captureLogs(); + await uninstall(); + + expect(JSON.parse(readFileSync(DAEMON_CONFIG_PATH, "utf8")).installed).toBe(true); + expect(logs.join("\n")).toContain("launchctl bootout"); + }); + + test("cleans up rt.sock/rt.pid/daemon.json when nothing is alive", async () => { + mkdirSync(RT_DIR, { recursive: true }); + markDaemonInstalled(); + writeFileSync(DAEMON_PID_PATH, "999999"); // no such pid + writeFileSync(DAEMON_SOCK_PATH, ""); // stale file, not a real listener — probeSocketHolder's fetch fails + + captureLogs(); + await uninstall(); + + expect(existsSync(DAEMON_PID_PATH)).toBe(false); + expect(JSON.parse(readFileSync(DAEMON_CONFIG_PATH, "utf8")).installed).toBe(false); + expect(logs.join("\n")).not.toContain("launchctl bootout"); + expect(logs.join("\n")).toContain("daemon fully uninstalled"); + }); +}); + +describe("start — kickstart escalation", () => { + test("falls back to /daemon/restart when the tray acks /daemon/start but the socket never comes up", async () => { + mkdirSync(RT_DIR, { recursive: true }); + markDaemonInstalled(); + let restartCalled = false; + serveTray({ + "/daemon/start": () => Response.json({ ok: true }), + "/daemon/restart": () => { restartCalled = true; return Response.json({ ok: true }); }, + }); + + captureLogs(); + await start(); + + expect(restartCalled).toBe(true); + }, 20_000); + + test("escalation succeeds once /daemon/restart actually brings the socket up", async () => { + mkdirSync(RT_DIR, { recursive: true }); + markDaemonInstalled(); + serveTray({ + "/daemon/start": () => Response.json({ ok: true }), + "/daemon/restart": () => { + serveDaemonPing(); + return Response.json({ ok: true }); + }, + }); + + captureLogs(); + await start(); + + expect(logs.join("\n")).toContain("daemon started"); + }, 20_000); +}); diff --git a/commands/daemon.ts b/commands/daemon.ts index 2604e4f1..43da6bd2 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -25,6 +25,8 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, unlinkSync, import { bold, dim, green, yellow, red, reset } from "../lib/tui.ts"; import { isDaemonInstalled, + isDaemonProcessRunning, + activeLaunchdLabel, markDaemonInstalled, markDaemonUninstalled, cleanupDaemonFiles, readDaemonPid, RT_DIR, @@ -224,7 +226,20 @@ export async function uninstall(): Promise { console.log(` ${green}✓${reset} removed legacy launchd plist`); } - // 3. Clear install flag + sock/pid files. + // 3. A failed/absent tray stop must never delete rt.sock/rt.pid/daemon.json + // out from under a daemon that's actually still alive — that would orphan + // it (still running, launchd-supervised, but rt's own bookkeeping says + // uninstalled). Check both liveness signals: the recorded pid, and whether + // anything still answers on rt.sock (a daemon can be alive with no + // matching rt.pid, e.g. after a crash-and-respawn under launchd). + const stillAlive = isDaemonProcessRunning() || (await probeSocketHolder()) !== null; + if (stillAlive) { + console.log(`\n ${yellow}⚠${reset} daemon is still running — leaving rt.sock/rt.pid/daemon.json in place`); + console.log(` ${dim}Fix: ${bold}launchctl bootout gui/$UID/${activeLaunchdLabel()}${reset}\n`); + return; + } + + // 4. Clear install flag + sock/pid files. markDaemonUninstalled(); cleanupDaemonFiles(); console.log(` ${green}✓${reset} cleared install flag`); @@ -258,16 +273,31 @@ export async function start(): Promise { } console.log(` ${dim}starting ${intended.mode} daemon via tray…${reset}`); + if (await pollForDaemonUp(intended)) return; + + // The tray acked /daemon/start, but SMAppService can register a job that + // never actually launches (still booting, crash-looping, etc.) — kick it + // via /daemon/restart, which forces launchd to invoke it, rather than + // leaving the operator staring at "check logs" for something a retry fixes. + console.log(` ${dim}not up yet — escalating to restart (kickstart)…${reset}`); + const restartResult = await trayQuery("/daemon/restart", "POST"); + if (restartResult?.ok && (await pollForDaemonUp(intended))) return; + + console.log(`\n ${yellow}daemon starting… check logs: rt daemon logs${reset}\n`); +} + +/** Shared poll loop for start()'s initial wait and its kickstart escalation. */ +async function pollForDaemonUp(intended: IntendedMode): Promise { for (let i = 0; i < 12; i++) { await Bun.sleep(250); if (await isDaemonRunning()) { if (!(await warnIfWrongFlavor("start", intended))) { console.log(`\n ${green}✓ daemon started${reset}\n`); } - return; + return true; } } - console.log(`\n ${yellow}daemon starting… check logs: rt daemon logs${reset}\n`); + return false; } export async function stop(): Promise { diff --git a/lib/daemon-client.ts b/lib/daemon-client.ts index 2a03a629..c29220d1 100644 --- a/lib/daemon-client.ts +++ b/lib/daemon-client.ts @@ -178,7 +178,18 @@ async function attemptRestart(): Promise { // null response (tray socket absent / request failed) means no restart // actually happened. const res = await trayQuery("/daemon/start", "POST"); - return res !== null; + if (res === null) return false; + + // The tray ack only proves the request was received, not that the + // daemon actually came up — re-probe rt.sock before reporting success, + // so daemonQuery's caller isn't told "restarted" while the daemon is + // still mid-boot and then misdirected into warnDaemonDown() on the very + // next query instead of actually waiting for it. + for (let i = 0; i < 12; i++) { + await Bun.sleep(250); + if (await isDaemonRunning()) return true; + } + return false; } catch { return false; } From 919100e09d74ef5a60ce030cf6e3c0f6dd46c52f Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 13:45:26 -0500 Subject: [PATCH 084/142] daemon: redirect stderr after legacy-dir migration; set shuttingDownViaVerb at verb receipt redirectNativeStderr() ran before migrateLegacyRtDir(), so its mkdirSync of the new rt dir made every real ~/.rt migration report a false "conflict". Move it after the migration check, still ahead of every other module-scope side effect. shuttingDownViaVerb was only set inside the shutdown verb's 100ms delayed cleanup, so a bare SIGTERM arriving in that window read it as unset and exited 1, causing launchd to respawn a daemon that was told to stop. Set it at verb receipt instead. --- lib/daemon.ts | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/daemon.ts b/lib/daemon.ts index 4e3ddfc3..64ea5fe1 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -85,13 +85,6 @@ import type { PortEntry } from "./port-scanner.ts"; // entry (cli.ts) also runs it, but `bun run lib/daemon.ts` skips cli.ts. import { migrateLegacyRtDir, LEGACY_RT_LABEL, RT_DIR_LABEL, logsDir } from "./rt-paths.ts"; -// Capture native panics (bypass JS entirely) at the fd level before anything -// else in this module runs, so a throw during any later module-scope -// construction (createEventsBus, cron, home-snapshot, …) lands in -// daemon-stderr.log instead of vanishing down whatever fd 2 the launcher gave -// us. Depends only on logsDir() and mkdirs its own dir; no logger needed yet. -redirectNativeStderr(); - // Gates installCrashHandlers' unhandledRejection handler: fatal during boot // (no socket/API bound yet, nothing to recover), advisory-only once ready. let bootPhase: "booting" | "ready" = "booting"; @@ -108,6 +101,16 @@ function setPhase(phase: BootPhase): void { const rtMigration = migrateLegacyRtDir(); +// Capture native panics (bypass JS entirely) at the fd level, so a throw +// during any later module-scope construction (createEventsBus, cron, +// home-snapshot, …) lands in daemon-stderr.log instead of vanishing down +// whatever fd 2 the launcher gave us. Depends only on logsDir() and mkdirs +// its own dir — this MUST run after migrateLegacyRtDir(): mkdirSync(logsDir()) +// creates the new rt dir, and migrateLegacyRtDir() treats that dir merely +// existing as a "conflict" with a real legacy tree, so redirecting first +// would defeat the migration. +redirectNativeStderr(); + // ─── Logging ───────────────────────────────────────────────────────────────── // Pino-backed structured logger. See lib/daemon-logger.ts. Top-level await // initializes the singleton before any other startup code runs, so `log` is @@ -396,11 +399,15 @@ async function routeCommand(cmd: string, payload: any, signal?: AbortSignal): Pr switch (cmd) { case "shutdown": log.info("received shutdown command"); + // Set before the delay, not inside the setTimeout callback: a bare + // SIGTERM arriving in the 100ms window must see this flag already + // true, or the signal handler treats an intentional stop as a crash + // (exit 1, launchd respawns). + shuttingDownViaVerb = true; // Delay cleanup so this response can be written first — cleanup() // force-closes all in-flight connections, including the one that // carried the shutdown request. setTimeout(() => { - shuttingDownViaVerb = true; recordCleanExit("shutdown", 0); cleanup(); loggerHandle.flush?.(); From 4e0af1e542280b86dc7532b3981a645345d6d31f Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 13:45:29 -0500 Subject: [PATCH 085/142] daemon: drop unused clearBreadcrumb Production only ever writes or reads the boot breadcrumb (it gets overwritten with "ready" on successful boot); clearBreadcrumb had no callers outside its own test. Removed the export and the now-unused unlinkSync import; the test that only covered clearBreadcrumb is gone, and the remaining "no breadcrumb written" test does its own file cleanup instead of relying on the removed API. --- lib/daemon/__tests__/supervision-state.test.ts | 18 ++++++++++-------- lib/daemon/supervision-state.ts | 12 ++---------- 2 files changed, 12 insertions(+), 18 deletions(-) diff --git a/lib/daemon/__tests__/supervision-state.test.ts b/lib/daemon/__tests__/supervision-state.test.ts index 6461af34..dbb6ea50 100644 --- a/lib/daemon/__tests__/supervision-state.test.ts +++ b/lib/daemon/__tests__/supervision-state.test.ts @@ -1,4 +1,6 @@ import { describe, expect, test } from "bun:test"; +import { rmSync } from "fs"; +import { join } from "path"; import { recordBootAttempt, recordDaemonReady, @@ -8,8 +10,14 @@ import { isCrashLooping, writeBreadcrumb, readBreadcrumb, - clearBreadcrumb, } from "../supervision-state.ts"; +import { RT_DIR } from "../../daemon-config.ts"; + +/** Test-only cleanup mirroring the breadcrumb file's path (production has + * no clear API — the daemon only ever writes or reads it). */ +function removeBreadcrumbFile(): void { + rmSync(join(RT_DIR, "daemon-boot.json"), { force: true }); +} describe("supervision-state kv round-trip", () => { test("boot attempts, ready stamp, failures and last-exit round-trip through kv", () => { @@ -58,14 +66,8 @@ describe("breadcrumb file", () => { expect(typeof b?.at).toBe("number"); }); - test("clearBreadcrumb removes the file so readBreadcrumb returns null", () => { - writeBreadcrumb("ready"); - clearBreadcrumb(); - expect(readBreadcrumb()).toBeNull(); - }); - test("readBreadcrumb returns null when no breadcrumb has been written", () => { - clearBreadcrumb(); + removeBreadcrumbFile(); expect(readBreadcrumb()).toBeNull(); }); }); diff --git a/lib/daemon/supervision-state.ts b/lib/daemon/supervision-state.ts index 7fa12e7e..5a753115 100644 --- a/lib/daemon/supervision-state.ts +++ b/lib/daemon/supervision-state.ts @@ -4,7 +4,7 @@ * breadcrumb for a live-but-silent daemon. * * Two tiers, deliberately not one: - * - The breadcrumb FILE (`writeBreadcrumb`/`readBreadcrumb`/`clearBreadcrumb`) + * - The breadcrumb FILE (`writeBreadcrumb`/`readBreadcrumb`) * opens no database, so it is safe to call at module scope, before * state.db exists. It is the only tier a pre-db boot failure can reach. * - The kv tier (`recordBootAttempt`, `recordDaemonReady`, @@ -15,7 +15,7 @@ * write is try/catch'd and silently no-ops if the db isn't open yet. */ -import { existsSync, readFileSync, unlinkSync, writeFileSync } from "fs"; +import { existsSync, readFileSync, writeFileSync } from "fs"; import { join } from "path"; import { RT_DIR } from "../daemon-config.ts"; import { daemonFlavor } from "./park.ts"; @@ -140,11 +140,3 @@ export function readBreadcrumb(): Breadcrumb | null { return null; } } - -export function clearBreadcrumb(): void { - try { - if (existsSync(breadcrumbPath())) unlinkSync(breadcrumbPath()); - } catch { - // Best-effort. - } -} From 2fa055cf6dde238678a0a06222d6cc534c74ae51 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 13:45:35 -0500 Subject: [PATCH 086/142] rt-client: soften the rt.apiPort description to match reality resolveApiPort() has no callers yet -- api-server.ts still binds the API_PORT const -- so the setting currently does nothing when set. The old description promised it as an "escape hatch when 9401 is held", which is not true until the sibling wiring lands. Rebuilt dist/ (not committed, gitignored). --- packages/rt-client/src/settings/registry-defs.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/rt-client/src/settings/registry-defs.ts b/packages/rt-client/src/settings/registry-defs.ts index 9f37dfd4..7d2f3bde 100644 --- a/packages/rt-client/src/settings/registry-defs.ts +++ b/packages/rt-client/src/settings/registry-defs.ts @@ -203,7 +203,7 @@ export const REGISTRY: readonly SettingDef[] = [ default: 9401, merge: "replace", migrated: true, - description: "TCP port the daemon's local HTTP/WS API binds (escape hatch when 9401 is held).", + description: "TCP port for the daemon's local HTTP/WS API. Not yet consumed at bind time (pending api-server wiring); today only the RT_API_PORT env var overrides the default 9401.", }, { key: "rt.hooks", From 0badca0ff808a863b44eca3a58726dc0c5d20e4e Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 13:45:40 -0500 Subject: [PATCH 087/142] docs: scrub em/en dashes, fix daemon-supervision-design phase-order drift Owner rule forbids em/en dashes in committed text. Rephrased them out of daemon-supervision-design.md and the runner-health redirect title; mechanical ellipsis substitution for the large historical plan doc, where careful rephrase wasn't warranted. Also fixed two places daemon-supervision-design.md had drifted from the implementation: the breadcrumb BootPhase order is actually start -> events-db -> state-db -> api -> socket -> ready (no "crash-handlers" phase, API binds before socket), and the alive-not-serving liveness fallback is `lsof +D RT_DIR` scoped to this HOME and filtered to exclude the caller's own pid, not a system-wide `pgrep -f`. --- docs/daemon-runner-health.md | 2 +- docs/daemon-supervision-design.md | 46 ++-- .../plans/2026-08-28-p0-supervision.md | 222 +++++++++--------- 3 files changed, 136 insertions(+), 134 deletions(-) diff --git a/docs/daemon-runner-health.md b/docs/daemon-runner-health.md index 9d6db779..f9ee31b1 100644 --- a/docs/daemon-runner-health.md +++ b/docs/daemon-runner-health.md @@ -1,4 +1,4 @@ -# Daemon runner health — superseded +# Daemon runner health (superseded) This document audited subsystems (process-manager, remedy-engine, runner.tsx, workspace-sync) that no longer exist. It is retained only as diff --git a/docs/daemon-supervision-design.md b/docs/daemon-supervision-design.md index f672cafb..61b4f6c1 100644 --- a/docs/daemon-supervision-design.md +++ b/docs/daemon-supervision-design.md @@ -1,7 +1,7 @@ # Daemon supervision: status verdicts and exit-code semantics Phase 0 design anchor for the rt daemon stability roadmap (audit -2026-08). Tasks 9–14 of the Phase 0 plan implement this. +2026-08). Tasks 9-14 of the Phase 0 plan implement this. ## launchd contract @@ -19,51 +19,53 @@ down". Every exit-code decision below follows from that single fact. | Crash-loop guard trips (N in M minutes) | 0 (park) | Stop the flapping; surface `crash-looping` so a human intervenes instead of launchd hammering every ~10s. | Mechanism: a module-scope `shuttingDownViaVerb` flag is set true by the -`shutdown` verb before it calls cleanup; `gracefulExit(signal)` reads it -— set → exit(0), unset (bare signal) → exit(1). +`shutdown` verb before it calls cleanup; `gracefulExit(signal)` reads it: +set → exit(0), unset (bare signal) → exit(1). Boot-phase gate: a module-scope `bootPhase: "booting" | "ready"` flips to `"ready"` immediately before the `daemon ready` log. The `unhandledRejection` handler exits(1) while `bootPhase === "booting"` -and only logs (recovers) once ready — so a boot-time stray rejection is +and only logs (recovers) once ready, so a boot-time stray rejection is fatal but a steady-state one is not. ## Status verdicts `rt daemon status` and `/api/status` classify by first match: -1. `not-installed` — SMAppService not registered. -2. `serving` — ping on rt.sock succeeds. -3. `parked` — ping fails, a live rt pid exists, and the boot breadcrumb +1. `not-installed`: SMAppService not registered. +2. `serving`: ping on rt.sock succeeds. +3. `parked`: ping fails, a live rt pid exists, and the boot breadcrumb phase is a flavor standoff (park). Named distinctly so the user is told "another flavor owns the socket", not "wedged". -4. `alive-not-serving` — ping fails but a live rt pid exists - (`process.kill(pid,0)` on rt.pid, or `pgrep -f 'rt --daemon|lib/daemon.ts'`). +4. `alive-not-serving`: ping fails but a live rt pid exists + (`process.kill(pid,0)` on rt.pid, falling back to `lsof +D RT_DIR` + scoped to this HOME's rt dir and filtered to exclude the calling + process's own pid). Sub-detail from the breadcrumb phase: `booting` / `wedged`, or `quarantined` when a state.db/events.db boot-failed marker is present. - Prints "process is running but not answering rt.sock — rt daemon logs -t". -5. `crash-looping` — no live pid AND the kv failure record shows ≥ N + Prints "process is running but not answering rt.sock ... rt daemon logs -t". +5. `crash-looping`: no live pid AND the kv failure record shows ≥ N failures within the last M minutes (N=3, M=5). Prints the last reason. -6. `boot-failed` — no live pid AND the most recent kv exit record is a +6. `boot-failed`: no live pid AND the most recent kv exit record is a boot throw (fewer than N failures). Prints the last reason + phase. -7. `installed-not-running` — registered, no live pid, clean/again-absent +7. `installed-not-running`: registered, no live pid, clean/again-absent exit record. ## Persisted state (kv, ns `daemon-supervision`, no schema change) -- `boot-attempts` (number) — incremented at the top of `runDaemon()`. -- `last-ready-at` (number, epoch ms) — stamped just before `daemon ready`. -- `recent-failures` (array of `{ at, phase, reason }`, capped to 10) — +- `boot-attempts` (number): incremented at the top of `runDaemon()`. +- `last-ready-at` (number, epoch ms): stamped just before `daemon ready`. +- `recent-failures` (array of `{ at, phase, reason }`, capped to 10): appended by the boot fatal path and by state.db/events.db boot-failed markers. Crash-loop = ≥ N entries newer than now − M minutes. -- `last-exit` (`{ at, kind: "shutdown" | "signal" | "boot-failed", code, reason? }`) - — written by the shutdown verb, the signal handlers, and the boot +- `last-exit` (`{ at, kind: "shutdown" | "signal" | "boot-failed", code, reason? }`): + written by the shutdown verb, the signal handlers, and the boot fatal path. Lets status distinguish "cleanly stopped" from "died". ## Boot breadcrumb `~/.mattstack/rt/daemon-boot.json` = `{ at, pid, flavor, phase }`, -rewritten at each boot phase: `start` → `crash-handlers` → `events-db` -→ `state-db` → `socket` → `api` → `ready`. Lets `alive-not-serving` -name where a live-but-silent daemon is stuck even when the logs are -unreadable. Removed (or stamped `ready`) on successful boot. +rewritten at each boot phase: `start` → `events-db` → `state-db` → +`api` → `socket` → `ready`. Lets `alive-not-serving` name where a +live-but-silent daemon is stuck even when the logs are unreadable. +Removed (or stamped `ready`) on successful boot. diff --git a/docs/superpowers/plans/2026-08-28-p0-supervision.md b/docs/superpowers/plans/2026-08-28-p0-supervision.md index bbbd14d5..72453dda 100644 --- a/docs/superpowers/plans/2026-08-28-p0-supervision.md +++ b/docs/superpowers/plans/2026-08-28-p0-supervision.md @@ -8,13 +8,13 @@ **Tech Stack:** Bun, `bun:sqlite`, pino, TypeScript. Tests are `bun test` (unit) and `bun test --preload ./e2e/setup.ts` (e2e, isolated-HOME daemon spawns). -**Spec:** `/Users/matt/Documents/GitHub/repo-tools/.claude/worktrees/daemon-stability-audit/docs/daemon-stability-audit-2026-08.md` — "Roadmap › Phase 0" plus Appendix A/B entries S001, S003, S004, S009, S011, S012, S026, S027, S028, S029, S030, S035, S036, S037, S043, S044, S060, S072, S073, S074, R001, R002, R007, R017. Each carries a failure scenario, prescribed fix, and fixer notes; read the relevant entry before implementing its task. +**Spec:** `/Users/matt/Documents/GitHub/repo-tools/.claude/worktrees/daemon-stability-audit/docs/daemon-stability-audit-2026-08.md` ... "Roadmap › Phase 0" plus Appendix A/B entries S001, S003, S004, S009, S011, S012, S026, S027, S028, S029, S030, S035, S036, S037, S043, S044, S060, S072, S073, S074, R001, R002, R007, R017. Each carries a failure scenario, prescribed fix, and fixer notes; read the relevant entry before implementing its task. ## Global Constraints -- **No `SCHEMA_VERSION` bump and no new/edited `V*_SCHEMA` block.** Persist all supervision state (restart counters, last-exit reason, boot-failed markers) in the existing `kv` table under namespace `daemon-supervision`, via `setKvValue`/`getKvValue` from `lib/state/kv-blob.ts`. If any task appears to need a schema change, STOP and ask the user (per the job brief's question format) — do not proceed. +- **No `SCHEMA_VERSION` bump and no new/edited `V*_SCHEMA` block.** Persist all supervision state (restart counters, last-exit reason, boot-failed markers) in the existing `kv` table under namespace `daemon-supervision`, via `setKvValue`/`getKvValue` from `lib/state/kv-blob.ts`. If any task appears to need a schema change, STOP and ask the user (per the job brief's question format) ... do not proceed. - **Never start a daemon or run `rt` against the real machine.** Every daemon or `dist/rt` invocation in a test or check runs under `env -i HOME=` only (repo CLAUDE.md, "Operating on this machine"). e2e daemon spawns already do this via `e2e/setup.ts`; new e2e tests must follow the same isolation. -- **Write fence — do NOT modify these sibling-owned files** (ask the user if a task seems to need one): `lib/daemon/api-server.ts`, `lib/daemon/api-auth.ts`, `lib/daemon/socket-server.ts`, `lib/daemon/handlers/secrets.ts`, `lib/subprocess.ts`, `lib/daemon/cache-refresh.ts`, `lib/git-worktrees.ts`, `lib/daemon/freshness.ts`, `lib/daemon/pollers.ts`, `lib/daemon/worktree-process-kill.ts`, `lib/daemon/system-process-scanner.ts`, `lib/runs/store.ts`, `lib/notifier.ts`, `lib/daemon/handlers/discussions.ts`, `lib/daemon/handlers/chat.ts`, `lib/daemon/handlers/agent.ts`, `lib/daemon/handlers/pane.ts`, `lib/daemon/handlers/project-mrs.ts`, `lib/daemon/handlers/worktree.ts`, `lib/herdr/client.ts`, `lib/port-scanner.ts`, `lib/deps/links.ts`, `lib/worktree/trash.ts`, `lib/agent-herdr.ts`, `lib/daemon/cron.ts`, `lib/daemon/hooks-guard.ts`, `lib/home/age-key.ts`, `lib/daemon/discussions-store.ts`, `lib/state/presence-store.ts`. +- **Write fence ... do NOT modify these sibling-owned files** (ask the user if a task seems to need one): `lib/daemon/api-server.ts`, `lib/daemon/api-auth.ts`, `lib/daemon/socket-server.ts`, `lib/daemon/handlers/secrets.ts`, `lib/subprocess.ts`, `lib/daemon/cache-refresh.ts`, `lib/git-worktrees.ts`, `lib/daemon/freshness.ts`, `lib/daemon/pollers.ts`, `lib/daemon/worktree-process-kill.ts`, `lib/daemon/system-process-scanner.ts`, `lib/runs/store.ts`, `lib/notifier.ts`, `lib/daemon/handlers/discussions.ts`, `lib/daemon/handlers/chat.ts`, `lib/daemon/handlers/agent.ts`, `lib/daemon/handlers/pane.ts`, `lib/daemon/handlers/project-mrs.ts`, `lib/daemon/handlers/worktree.ts`, `lib/herdr/client.ts`, `lib/port-scanner.ts`, `lib/deps/links.ts`, `lib/worktree/trash.ts`, `lib/agent-herdr.ts`, `lib/daemon/cron.ts`, `lib/daemon/hooks-guard.ts`, `lib/home/age-key.ts`, `lib/daemon/discussions-store.ts`, `lib/state/presence-store.ts`. - **Every subagent dispatched during execution carries an explicit `model`** (`sonnet` for mechanical tasks, `haiku` for lookups). - **`packages/rt-client` is touched** (Task 5 edits `registry-defs.ts`). After that task and before the final whole-branch review, run `bun run build` inside `packages/rt-client` (keeps `dist/` and `dist-freshness.test.ts` green). - **Verification (must pass before the work is done):** @@ -24,15 +24,15 @@ ## Deferred / out-of-fence items (documented, not implemented here) -- **S073 · `presence-store.ts:signIn` → `.immediate()`** — `lib/state/presence-store.ts` is fenced. Task 7 converts the chat-store and notifier-store read-then-write transactions; the `signIn` caller is a follow-up for the presence-store owner. Not required for verification. -- **0.3 · `rt.apiPort` bind-time consumption** — `lib/daemon/api-server.ts` (the binder) is fenced. Task 5 registers the `rt.apiPort` setting and exposes `resolveApiPort()`; the bind-time read is the api-server sibling's hop. The escape hatch is therefore wired daemon-side but consumed sibling-side; do not claim it functions end-to-end until the sibling reads it. -- **Swift tray edits (S026 dot mapping, S028 `DaemonLifecycle` kickstart fallback, S029 `tray-crash.log` rotation, S060 `AppDelegate` comment)** — grouped as optional Task 16. They cannot be verified by the bun/tsc/e2e gate and the operating rules forbid rebuilding the blessed bundle. Scope confirmation is raised at the plan-review checkpoint. The verifiable CLI-side halves (S028 start→kickstart fallback, S060 exit-code policy) live in Tasks 14 and 12 and are done regardless. +- **S073 · `presence-store.ts:signIn` → `.immediate()`** ... `lib/state/presence-store.ts` is fenced. Task 7 converts the chat-store and notifier-store read-then-write transactions; the `signIn` caller is a follow-up for the presence-store owner. Not required for verification. +- **0.3 · `rt.apiPort` bind-time consumption** ... `lib/daemon/api-server.ts` (the binder) is fenced. Task 5 registers the `rt.apiPort` setting and exposes `resolveApiPort()`; the bind-time read is the api-server sibling's hop. The escape hatch is therefore wired daemon-side but consumed sibling-side; do not claim it functions end-to-end until the sibling reads it. +- **Swift tray edits (S026 dot mapping, S028 `DaemonLifecycle` kickstart fallback, S029 `tray-crash.log` rotation, S060 `AppDelegate` comment)** ... grouped as optional Task 16. They cannot be verified by the bun/tsc/e2e gate and the operating rules forbid rebuilding the blessed bundle. Scope confirmation is raised at the plan-review checkpoint. The verifiable CLI-side halves (S028 start→kickstart fallback, S060 exit-code policy) live in Tasks 14 and 12 and are done regardless. --- ## Task 1: Status-verdict + exit-code design sketch -The half-page design that Tasks 9–14 depend on. No production code; the deliverable is a committed design doc. (Retires nothing directly; anchors R001, R002, S036, S060, S026, S028.) +The half-page design that Tasks 9-14 depend on. No production code; the deliverable is a committed design doc. (Retires nothing directly; anchors R001, R002, S036, S060, S026, S028.) **Files:** - Create: `docs/daemon-supervision-design.md` @@ -46,7 +46,7 @@ The half-page design that Tasks 9–14 depend on. No production code; the delive # Daemon supervision: status verdicts and exit-code semantics Phase 0 design anchor for the rt daemon stability roadmap (audit -2026-08). Tasks 9–14 of the Phase 0 plan implement this. +2026-08). Tasks 9-14 of the Phase 0 plan implement this. ## launchd contract @@ -65,44 +65,44 @@ down". Every exit-code decision below follows from that single fact. Mechanism: a module-scope `shuttingDownViaVerb` flag is set true by the `shutdown` verb before it calls cleanup; `gracefulExit(signal)` reads it -— set → exit(0), unset (bare signal) → exit(1). +... set → exit(0), unset (bare signal) → exit(1). Boot-phase gate: a module-scope `bootPhase: "booting" | "ready"` flips to `"ready"` immediately before the `daemon ready` log. The `unhandledRejection` handler exits(1) while `bootPhase === "booting"` -and only logs (recovers) once ready — so a boot-time stray rejection is +and only logs (recovers) once ready ... so a boot-time stray rejection is fatal but a steady-state one is not. ## Status verdicts `rt daemon status` and `/api/status` classify by first match: -1. `not-installed` — SMAppService not registered. -2. `serving` — ping on rt.sock succeeds. -3. `parked` — ping fails, a live rt pid exists, and the boot breadcrumb +1. `not-installed` ... SMAppService not registered. +2. `serving` ... ping on rt.sock succeeds. +3. `parked` ... ping fails, a live rt pid exists, and the boot breadcrumb phase is a flavor standoff (park). Named distinctly so the user is told "another flavor owns the socket", not "wedged". -4. `alive-not-serving` — ping fails but a live rt pid exists +4. `alive-not-serving` ... ping fails but a live rt pid exists (`process.kill(pid,0)` on rt.pid, or `pgrep -f 'rt --daemon|lib/daemon.ts'`). Sub-detail from the breadcrumb phase: `booting` / `wedged`, or `quarantined` when a state.db/events.db boot-failed marker is present. - Prints "process is running but not answering rt.sock — rt daemon logs -t". -5. `crash-looping` — no live pid AND the kv failure record shows ≥ N + Prints "process is running but not answering rt.sock ... rt daemon logs -t". +5. `crash-looping` ... no live pid AND the kv failure record shows ≥ N failures within the last M minutes (N=3, M=5). Prints the last reason. -6. `boot-failed` — no live pid AND the most recent kv exit record is a +6. `boot-failed` ... no live pid AND the most recent kv exit record is a boot throw (fewer than N failures). Prints the last reason + phase. -7. `installed-not-running` — registered, no live pid, clean/again-absent +7. `installed-not-running` ... registered, no live pid, clean/again-absent exit record. ## Persisted state (kv, ns `daemon-supervision`, no schema change) -- `boot-attempts` (number) — incremented at the top of `runDaemon()`. -- `last-ready-at` (number, epoch ms) — stamped just before `daemon ready`. -- `recent-failures` (array of `{ at, phase, reason }`, capped to 10) — +- `boot-attempts` (number) ... incremented at the top of `runDaemon()`. +- `last-ready-at` (number, epoch ms) ... stamped just before `daemon ready`. +- `recent-failures` (array of `{ at, phase, reason }`, capped to 10) ... appended by the boot fatal path and by state.db/events.db boot-failed markers. Crash-loop = ≥ N entries newer than now − M minutes. - `last-exit` (`{ at, kind: "shutdown" | "signal" | "boot-failed", code, reason? }`) - — written by the shutdown verb, the signal handlers, and the boot + ... written by the shutdown verb, the signal handlers, and the boot fatal path. Lets status distinguish "cleanly stopped" from "died". ## Boot breadcrumb @@ -123,7 +123,7 @@ git commit -m "docs: sketch daemon supervision verdicts + exit-code semantics" --- -## Task 2: Fatal boot means exit (0.1 — S001, S037) +## Task 2: Fatal boot means exit (0.1 ... S001, S037) Boot failures on the prod path currently become an `unhandledRejection` that only logs, leaving a live-pid zombie with no socket/API. Make a `runDaemon()` throw fatal, gate the rejection handler on a boot-phase flag, and move the rt.pid write to after both binds. @@ -133,7 +133,7 @@ Boot failures on the prod path currently become an `unhandledRejection` that onl - Test: `lib/__tests__/daemon-logger.test.ts` (rejection handler), `e2e/tests/daemon.test.ts` (fatal boot) **Interfaces:** -- Produces: `installCrashHandlers(logger, opts?: { booting?: () => boolean })` — when `booting()` is true, `unhandledRejection` logs `fatal` and `process.exit(1)`; otherwise it logs `error` only (today's behavior). Default (no `booting`) preserves today's error-only behavior. +- Produces: `installCrashHandlers(logger, opts?: { booting?: () => boolean })` ... when `booting()` is true, `unhandledRejection` logs `fatal` and `process.exit(1)`; otherwise it logs `error` only (today's behavior). Default (no `booting`) preserves today's error-only behavior. - Produces: module-scope `let bootPhase: "booting" | "ready" = "booting"` in `lib/daemon.ts`, flipped to `"ready"` at line 513. - [ ] **Step 1: Write the failing unit test** in `lib/__tests__/daemon-logger.test.ts`: @@ -162,7 +162,7 @@ test("unhandledRejection exits(1) while booting, only logs once ready", () => { (If `makeFakeLogger` does not exist, build a minimal `{ info, warn, error, fatal }` of `mock(() => {})`. Remove the listeners this test adds in `afterEach` via `process.removeAllListeners("unhandledRejection")` scoped to the test, matching the file's existing cleanup convention.) -- [ ] **Step 2: Run it — expect FAIL** (`booting` option not supported): +- [ ] **Step 2: Run it ... expect FAIL** (`booting` option not supported): Run: `bun test lib/__tests__/daemon-logger.test.ts -t "unhandledRejection exits"` Expected: FAIL. @@ -187,7 +187,7 @@ export function installCrashHandlers( } ``` -- [ ] **Step 4: Run the unit test — expect PASS.** +- [ ] **Step 4: Run the unit test ... expect PASS.** - [ ] **Step 5: Wire the boot-phase flag + fatal wrap + rt.pid move in `lib/daemon.ts`.** - Add near the top of module scope (after imports, before line 78): `let bootPhase: "booting" | "ready" = "booting";` @@ -207,7 +207,7 @@ async function runDaemon() { } ``` - - **Move the rt.pid write** (currently `writeFileSync(DAEMON_PID_PATH, String(process.pid))` at line 416) to AFTER both binds — i.e. after the API bind at line 469 and the socket bind at line 468 (Task 5 will make API bind first; either way, rt.pid is written only once both `servers.socket` and `servers.api` are assigned). A failed boot then never leaves a live-pid file. + - **Move the rt.pid write** (currently `writeFileSync(DAEMON_PID_PATH, String(process.pid))` at line 416) to AFTER both binds ... i.e. after the API bind at line 469 and the socket bind at line 468 (Task 5 will make API bind first; either way, rt.pid is written only once both `servers.socket` and `servers.api` are assigned). A failed boot then never leaves a live-pid file. - Set `bootPhase = "ready";` immediately before `log.info({ pid }, "daemon ready")` at line 513. - [ ] **Step 6: Write the failing e2e test** in `e2e/tests/daemon.test.ts` (uses the isolated-HOME harness already in `e2e/setup.ts`): @@ -232,9 +232,9 @@ test("daemon boot with API port already bound exits non-zero and leaves no stale }); ``` -(Reuse the harness's existing `rtBinary`, `isolatedEnv`, `isolatedHome` fixtures — mirror `e2e/tests/daemon.test.ts`'s existing setup. The daemon's own `startApiServer` retries the bind 6× before throwing, so allow up to the 60s timeout.) +(Reuse the harness's existing `rtBinary`, `isolatedEnv`, `isolatedHome` fixtures ... mirror `e2e/tests/daemon.test.ts`'s existing setup. The daemon's own `startApiServer` retries the bind 6× before throwing, so allow up to the 60s timeout.) -- [ ] **Step 7: Run the e2e test — expect PASS.** Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "API port already bound"` +- [ ] **Step 7: Run the e2e test ... expect PASS.** Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "API port already bound"` - [ ] **Step 8: Commit** @@ -245,7 +245,7 @@ git commit -m "daemon: boot failure is fatal (exit 1), gated by boot-phase flag; --- -## Task 3: Crash handlers first (0.2 — S003, S004, R007) +## Task 3: Crash handlers first (0.2 ... S003, S004, R007) Move `redirectNativeStderr()` and `installCrashHandlers()` above every module-scope side effect so a pre-`startDaemon` failure lands in the crash log instead of a discarded stderr. @@ -256,16 +256,16 @@ Move `redirectNativeStderr()` and `installCrashHandlers()` above every module-sc **Interfaces:** - Consumes: `installCrashHandlers(logger, { booting })` from Task 2. -- [ ] **Step 1: Hoist `redirectNativeStderr()`** to the very first executable statement of `lib/daemon.ts` module scope — before `migrateLegacyRtDir()` at line 78. It depends only on `logsDir()` and has its own internal try/catch, so a subsequent module-scope throw's fd-2 output lands in `daemon-stderr.log`. +- [ ] **Step 1: Hoist `redirectNativeStderr()`** to the very first executable statement of `lib/daemon.ts` module scope ... before `migrateLegacyRtDir()` at line 78. It depends only on `logsDir()` and has its own internal try/catch, so a subsequent module-scope throw's fd-2 output lands in `daemon-stderr.log`. - [ ] **Step 2: Hoist `installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" })`** to immediately after `getDaemonLogger()` resolves (right after line ~95, before `parkUntilIntended` at 103 and before `createEventsBus` at 191). The logger must exist first (line 85), so this is the earliest correct point. -- [ ] **Step 3: Remove the now-duplicate `redirectNativeStderr()` and `installCrashHandlers()` calls** inside `runDaemon()` (lines 391-392). Keep `mkdirSync(RT_DIR, …)` at 386 (redirect needs the logs dir; `redirectNativeStderr` already mkdirs its own dir, and RT_DIR creation is idempotent — verify the hoisted `redirectNativeStderr` still finds/creates `logsDir()`). +- [ ] **Step 3: Remove the now-duplicate `redirectNativeStderr()` and `installCrashHandlers()` calls** inside `runDaemon()` (lines 391-392). Keep `mkdirSync(RT_DIR, …)` at 386 (redirect needs the logs dir; `redirectNativeStderr` already mkdirs its own dir, and RT_DIR creation is idempotent ... verify the hoisted `redirectNativeStderr` still finds/creates `logsDir()`). - [ ] **Step 4: Write the failing e2e test** in `e2e/tests/daemon.test.ts`: ```ts -test("a corrupt events.db does not crash the daemon silently — error is captured", async () => { +test("a corrupt events.db does not crash the daemon silently ... error is captured", async () => { // Pre-create a corrupt events.db in the isolated HOME. const rtDir = join(isolatedHome, ".mattstack/rt"); mkdirSync(rtDir, { recursive: true }); @@ -280,7 +280,7 @@ test("a corrupt events.db does not crash the daemon silently — error is captur }); ``` -- [ ] **Step 5: Run — expect PASS** (the redirect now runs before events.db construction, so a corruption throw is captured in `daemon-stderr.log`; after Task 4 it is quarantined instead). Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "corrupt events.db"` +- [ ] **Step 5: Run ... expect PASS** (the redirect now runs before events.db construction, so a corruption throw is captured in `daemon-stderr.log`; after Task 4 it is quarantined instead). Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "corrupt events.db"` - [ ] **Step 6: Commit** @@ -291,7 +291,7 @@ git commit -m "daemon: install stderr redirect + crash handlers before every mod --- -## Task 4: events.db joins the discipline (0.4 — S009, S035) +## Task 4: events.db joins the discipline (0.4 ... S009, S035) Give `events.db` the corruption quarantine + busy_timeout/synchronous pragmas state.db has, and guard the two sweep timers so a sync sqlite throw cannot exit the daemon. @@ -301,7 +301,7 @@ Give `events.db` the corruption quarantine + busy_timeout/synchronous pragmas st - Test: `lib/daemon/__tests__/events-bus.test.ts` **Interfaces:** -- Consumes: `isCorruptionError` and `quarantine` shape from `lib/state/db.ts` (reuse the `SQLITE_CORRUPT`/`SQLITE_NOTADB` detection; `events.db` is a bounded-retention journal, so total loss on quarantine is harmless — no migration concern). +- Consumes: `isCorruptionError` and `quarantine` shape from `lib/state/db.ts` (reuse the `SQLITE_CORRUPT`/`SQLITE_NOTADB` detection; `events.db` is a bounded-retention journal, so total loss on quarantine is harmless ... no migration concern). - [ ] **Step 1: Write the failing test** in `lib/daemon/__tests__/events-bus.test.ts`: @@ -331,9 +331,9 @@ test("createEventsBus sets busy_timeout and synchronous=NORMAL", () => { (If the bus does not expose its handle, add a minimal `__db` back-reference or a `busyTimeout()` debug accessor in `events-bus.ts` for the test; do not expose it beyond the module's test needs.) -- [ ] **Step 2: Run — expect FAIL.** Run: `bun test lib/daemon/__tests__/events-bus.test.ts -t "quarantine"` +- [ ] **Step 2: Run ... expect FAIL.** Run: `bun test lib/daemon/__tests__/events-bus.test.ts -t "quarantine"` -- [ ] **Step 3: Implement in `lib/daemon/events-bus.ts`** — wrap the open (lines 73-86): +- [ ] **Step 3: Implement in `lib/daemon/events-bus.ts`** ... wrap the open (lines 73-86): ```ts import { isCorruptionError } from "../state/db"; // export it if not already exported @@ -359,7 +359,7 @@ try { Write a local `quarantineEventsDb(path, log)` mirroring `state/db.ts`'s `quarantine` (rename the db + `-wal`/`-shm` sidecars to `path.corrupt-`, `log.warn`). If `isCorruptionError` is not exported from `state/db.ts`, add the export (it is a pure predicate, safe to export). -- [ ] **Step 4: Run the events-bus tests — expect PASS.** +- [ ] **Step 4: Run the events-bus tests ... expect PASS.** - [ ] **Step 5: Write the failing sweep-guard test** in `lib/daemon/__tests__/events-bus.test.ts` OR a small `lib/__tests__/daemon-sweep-guard.test.ts` for the helper: @@ -391,7 +391,7 @@ export function safeTimeout(fn: () => void, ms: number, label: string, log: Logg Replace the two bare sweep timers at `lib/daemon.ts:194` and `:196` with `safeInterval(() => eventsBus.sweep(), 60*60*1000, "events-sweep", log)` and `safeTimeout(() => eventsBus.sweep(), 30_000, "events-sweep-boot", log)`. (The `pruneRuns`/`pruneLogs` timers at 200-248 already wrap their bodies; leaving them is fine, but converting them to `safeInterval` is a welcome DRY cleanup if trivial.) -- [ ] **Step 7: Run — expect PASS.** Then `bun test lib/daemon/__tests__/events-bus.test.ts`. +- [ ] **Step 7: Run ... expect PASS.** Then `bun test lib/daemon/__tests__/events-bus.test.ts`. - [ ] **Step 8: Commit** @@ -402,7 +402,7 @@ git commit -m "events.db: corruption quarantine + busy_timeout/synchronous pragm --- -## Task 5: Bind order + rt.apiPort setting (0.3, my side — S043, S030 seam) +## Task 5: Bind order + rt.apiPort setting (0.3, my side ... S043, S030 seam) Bind the API server before the unix socket so a failed API bind never strands a socket-bound zombie, and register the `rt.apiPort` escape-hatch setting (the api-server sibling consumes it at bind time). @@ -446,7 +446,7 @@ test("resolveApiPort: env wins, then setting, then 9401", () => { }); ``` -- [ ] **Step 4: Run — expect FAIL** (`resolveApiPort` undefined). +- [ ] **Step 4: Run ... expect FAIL** (`resolveApiPort` undefined). - [ ] **Step 5: Implement in `lib/daemon-config.ts`** (leave `API_PORT` at line 72 untouched): @@ -459,13 +459,13 @@ export function resolveApiPort(): number { } ``` -- [ ] **Step 6: Run — expect PASS.** +- [ ] **Step 6: Run ... expect PASS.** - [ ] **Step 7: Swap the bind order in `lib/daemon.ts`.** Reorder so the API binds before the socket: ```ts -servers.api = await startApiServer({ handleCommand, log }); // was line 469 — now first -servers.socket = startSocketServer({ handleCommand, log }); // was line 468 — now second +servers.api = await startApiServer({ handleCommand, log }); // was line 469 ... now first +servers.socket = startSocketServer({ handleCommand, log }); // was line 468 ... now second // rt.pid write (moved by Task 2) stays after BOTH assignments ``` @@ -484,7 +484,7 @@ test("API-bind failure leaves neither rt.sock nor rt.pid", async () => { }); ``` -- [ ] **Step 9: Run — expect PASS.** Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "API-bind failure"` +- [ ] **Step 9: Run ... expect PASS.** Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "API-bind failure"` - [ ] **Step 10: Commit** @@ -495,7 +495,7 @@ git commit -m "daemon: bind API before socket; register rt.apiPort setting + res --- -## Task 6: home-snapshot opens state.db daemon-flavored (0.7 — S011) +## Task 6: home-snapshot opens state.db daemon-flavored (0.7 ... S011) `startHomeSnapshot` opens the state.db singleton at module scope with the default `cli` flavor (5000ms busy_timeout), so the daemon runs with the wrong contention policy forever. Make its db lazy and daemon-flavored, and harden `getStateDb` against a silent flavor mismatch. @@ -505,7 +505,7 @@ git commit -m "daemon: bind API before socket; register rt.apiPort setting + res - Test: `lib/state/__tests__/db.test.ts` **Interfaces:** -- Consumes: `getStateDb("daemon")` — 250ms busy_timeout. +- Consumes: `getStateDb("daemon")` ... 250ms busy_timeout. - [ ] **Step 1: Write the failing test** in `lib/state/__tests__/db.test.ts` (`describe("pragma values per flavor")`): @@ -513,16 +513,16 @@ git commit -m "daemon: bind API before socket; register rt.apiPort setting + res test("getStateDb('daemon') reports busy_timeout 250 even after a default open", () => { const cli = getStateDb(); // opens singleton, cli flavor expect(cli.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 5000 }); - const daemon = getStateDb("daemon"); // same singleton — must not stay at 5000 + const daemon = getStateDb("daemon"); // same singleton ... must not stay at 5000 expect(daemon.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); }); ``` (Use the file's existing isolated-HOME / `closeStateDb` setup so this does not leak into other tests.) -- [ ] **Step 2: Run — expect FAIL** (singleton keeps the cli 5000 timeout). +- [ ] **Step 2: Run ... expect FAIL** (singleton keeps the cli 5000 timeout). -- [ ] **Step 3: Harden `getStateDb` in `lib/state/db.ts:522-530`** — when the singleton is already open and a caller requests a stronger (shorter) flavor timeout, re-apply the pragma: +- [ ] **Step 3: Harden `getStateDb` in `lib/state/db.ts:522-530`** ... when the singleton is already open and a caller requests a stronger (shorter) flavor timeout, re-apply the pragma: ```ts export function getStateDb(flavor: DbFlavor = "cli"): Database { @@ -537,13 +537,13 @@ export function getStateDb(flavor: DbFlavor = "cli"): Database { } ``` -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Make home-snapshot's db lazy + daemon-flavored** in `lib/daemon/home-snapshot.ts`. Replace the eager `db: rawDeps.db ?? getStateDb()` (line 272) with a thunk defaulting to `() => getStateDb("daemon")`, and resolve it on first use inside `loadState`/`runNow`/`status` rather than at construction (line 299). Concretely: store `const resolveDb = rawDeps.db ? () => rawDeps.db! : () => getStateDb("daemon");` and call `resolveDb()` where `deps.db` was read, so no db opens until `startDaemon` has already opened it daemon-flavored via `openBranchCacheStore`. -- [ ] **Step 6: Add a boot-order regression test** in `lib/state/__tests__/db.test.ts` (or `home-snapshot.test.ts`) asserting that constructing `startHomeSnapshot` does NOT open the state.db singleton (call it, then assert `getStateDb` was not yet invoked — spy on the module or assert no `state.db` file exists until first use in an isolated HOME). +- [ ] **Step 6: Add a boot-order regression test** in `lib/state/__tests__/db.test.ts` (or `home-snapshot.test.ts`) asserting that constructing `startHomeSnapshot` does NOT open the state.db singleton (call it, then assert `getStateDb` was not yet invoked ... spy on the module or assert no `state.db` file exists until first use in an isolated HOME). -- [ ] **Step 7: Run the db tests — expect PASS.** Run: `bun test lib/state/__tests__/db.test.ts` +- [ ] **Step 7: Run the db tests ... expect PASS.** Run: `bun test lib/state/__tests__/db.test.ts` - [ ] **Step 8: Commit** @@ -554,7 +554,7 @@ git commit -m "home-snapshot: lazy daemon-flavored state.db; getStateDb re-appli --- -## Task 7: Extended busy codes + IMMEDIATE transactions (0.7 — S072, S073) +## Task 7: Extended busy codes + IMMEDIATE transactions (0.7 ... S072, S073) `isBusyError` misses `SQLITE_BUSY_SNAPSHOT`/`_RECOVERY`, and read-then-write daemon transactions use a deferred `BEGIN` that produces snapshot conflicts busy_timeout cannot absorb. Widen the match and take the write lock up front. @@ -563,12 +563,12 @@ git commit -m "home-snapshot: lazy daemon-flavored state.db; getStateDb re-appli - Modify: `lib/state/chat-store.ts` (`readUnread`, `joinRoom`, `archiveRoom`, `dmRoomFor` → `.immediate()`) - Modify: `lib/state/notifier-store.ts` (`drainNotificationQueue` → `.immediate()`) - Test: `lib/state/__tests__/busy.test.ts` -- **Deferred (fenced):** `lib/state/presence-store.ts:signIn` — documented follow-up, not done here. +- **Deferred (fenced):** `lib/state/presence-store.ts:signIn` ... documented follow-up, not done here. **Interfaces:** - Produces: `isBusyError` returns true for any `code` starting `SQLITE_BUSY`. -- [ ] **Step 1: Write the failing test** in `lib/state/__tests__/busy.test.ts` — a real two-connection snapshot conflict: +- [ ] **Step 1: Write the failing test** in `lib/state/__tests__/busy.test.ts` ... a real two-connection snapshot conflict: ```ts test("isBusyError matches SQLITE_BUSY_SNAPSHOT from a real conflict", () => { @@ -589,7 +589,7 @@ test("isBusyError matches SQLITE_BUSY_SNAPSHOT from a real conflict", () => { }); ``` -- [ ] **Step 2: Run — expect FAIL** (`isBusyError` returns false for `SQLITE_BUSY_SNAPSHOT`). +- [ ] **Step 2: Run ... expect FAIL** (`isBusyError` returns false for `SQLITE_BUSY_SNAPSHOT`). - [ ] **Step 3: Widen `isBusyError`** in `lib/state/busy.ts:39-41`: @@ -600,11 +600,11 @@ export function isBusyError(err: unknown): boolean { } ``` -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Convert read-then-write transactions to `.immediate()`.** In `lib/state/chat-store.ts`, change the four cited transaction builders (`readUnread` ~483, `joinRoom`, `archiveRoom`, `dmRoomFor`) from `db.transaction(fn)(...)` to `db.transaction(fn).immediate(...)` so the write lock is taken at `BEGIN IMMEDIATE`. Do the same for `drainNotificationQueue` in `lib/state/notifier-store.ts:111`. Leave a one-line comment at the first site: `// BEGIN IMMEDIATE: read-then-write must lock up front or SQLITE_BUSY_SNAPSHOT bypasses busy_timeout.` -- [ ] **Step 6: Verify no regression** — run the chat/notifier/state suites: `bun test lib/state` and any `chat` command tests. Expect PASS (behavior identical under no contention; the change only affects lock acquisition timing). +- [ ] **Step 6: Verify no regression** ... run the chat/notifier/state suites: `bun test lib/state` and any `chat` command tests. Expect PASS (behavior identical under no contention; the change only affects lock acquisition timing). - [ ] **Step 7: Commit** @@ -615,12 +615,12 @@ git commit -m "state: isBusyError matches SQLITE_BUSY_*; read-then-write daemon --- -## Task 8: state.db importer isolation (0.3 — S074) +## Task 8: state.db importer isolation (0.3 ... S074) A throwing legacy importer inside the v0 migration rolls back the whole migration, so `user_version` stays 0 and every subsequent open repeats the failure. Wrap each importer in a SAVEPOINT; on throw, roll back that one importer, warn, and still rename the file. **Files:** -- Modify: `lib/state/db.ts:400-417` (`importLegacyStores` — SAVEPOINT per importer) +- Modify: `lib/state/db.ts:400-417` (`importLegacyStores` ... SAVEPOINT per importer) - Test: `lib/state/__tests__/db.test.ts` **Interfaces:** @@ -644,9 +644,9 @@ test("a throwing legacy importer is isolated: db reaches SCHEMA_VERSION, other s }); ``` -(Fill ``/``/fixtures from an existing importer in `LEGACY_IMPORTS`; the db.test.ts legacy-import cases already build such fixtures — reuse one.) +(Fill ``/``/fixtures from an existing importer in `LEGACY_IMPORTS`; the db.test.ts legacy-import cases already build such fixtures ... reuse one.) -- [ ] **Step 2: Run — expect FAIL** (the throw rolls back the whole migration; `user_version` stays 0 / benign rows absent). +- [ ] **Step 2: Run ... expect FAIL** (the throw rolls back the whole migration; `user_version` stays 0 / benign rows absent). - [ ] **Step 3: Implement SAVEPOINT-per-importer** in `importLegacyStores` (`lib/state/db.ts:400-417`). For each `LEGACY_IMPORTS` entry, wrap its `run(db)` in a savepoint scoped to that importer only (do NOT loosen the surrounding schema-DDL migration, which must stay loud): @@ -666,9 +666,9 @@ for (const imp of LEGACY_IMPORTS) { } ``` -(Match the exact `LegacyImport` shape at db.ts:64-69 — `path()`/`run(db)` names may differ; adapt to the real fields.) +(Match the exact `LegacyImport` shape at db.ts:64-69 ... `path()`/`run(db)` names may differ; adapt to the real fields.) -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Commit** @@ -679,7 +679,7 @@ git commit -m "state.db: isolate each legacy importer in a SAVEPOINT so one bad --- -## Task 9: Restart counter + last-exit reason in kv (0.5 — R002, S037) +## Task 9: Restart counter + last-exit reason in kv (0.5 ... R002, S037) Persist boot attempts, last-ready stamp, recent failures, and last-exit reason in the `kv` table (ns `daemon-supervision`), plus a boot breadcrumb file. Wire the record calls into the boot path, the shutdown verb, the signal handlers, and the boot fatal path. No schema change. @@ -690,13 +690,13 @@ Persist boot attempts, last-ready stamp, recent failures, and last-exit reason i **Interfaces:** - Produces: - - `recordBootAttempt(): void` — increments `boot-attempts`, appends nothing. - - `recordDaemonReady(): void` — sets `last-ready-at = Date.now()`. - - `recordBootFailure(phase: BootPhase, reason: string): void` — appends `{ at, phase, reason }` to `recent-failures` (cap 10), sets `last-exit = { at, kind: "boot-failed", code: 1, reason }`. - - `recordCleanExit(kind: "shutdown" | "signal", code: number): void` — sets `last-exit = { at, kind, code }`. + - `recordBootAttempt(): void` ... increments `boot-attempts`, appends nothing. + - `recordDaemonReady(): void` ... sets `last-ready-at = Date.now()`. + - `recordBootFailure(phase: BootPhase, reason: string): void` ... appends `{ at, phase, reason }` to `recent-failures` (cap 10), sets `last-exit = { at, kind: "boot-failed", code: 1, reason }`. + - `recordCleanExit(kind: "shutdown" | "signal", code: number): void` ... sets `last-exit = { at, kind, code }`. - `readSupervisionState(): { bootAttempts, lastReadyAt, recentFailures, lastExit }`. - - `isCrashLooping(state, now, n = 3, windowMs = 5*60_000): boolean` — ≥ n failures newer than `now - windowMs`. - - `writeBreadcrumb(phase: BootPhase): void` / `clearBreadcrumb(): void` — `~/.mattstack/rt/daemon-boot.json`. + - `isCrashLooping(state, now, n = 3, windowMs = 5*60_000): boolean` ... ≥ n failures newer than `now - windowMs`. + - `writeBreadcrumb(phase: BootPhase): void` / `clearBreadcrumb(): void` ... `~/.mattstack/rt/daemon-boot.json`. - `type BootPhase = "start" | "crash-handlers" | "events-db" | "state-db" | "socket" | "api" | "ready"`. - Consumes: `getKvValue`/`setKvValue` from `lib/state/kv-blob.ts`; `getStateDb("daemon")`. @@ -723,26 +723,26 @@ test("isCrashLooping true at >=3 failures within the window", () => { }); ``` -(Use the file's isolated-HOME convention — `bunfig` preload already repoints HOME for `bun test`; `recordBootAttempt` writes to the test state.db.) +(Use the file's isolated-HOME convention ... `bunfig` preload already repoints HOME for `bun test`; `recordBootAttempt` writes to the test state.db.) -- [ ] **Step 2: Run — expect FAIL** (module does not exist). +- [ ] **Step 2: Run ... expect FAIL** (module does not exist). - [ ] **Step 3: Implement `lib/daemon/supervision-state.ts`** with the interfaces above. All reads/writes go through `getKvValue("daemon-supervision", key, fallback, getStateDb("daemon"))` / `setKvValue("daemon-supervision", key, value, getStateDb("daemon"))`. Cap `recent-failures` at 10 on append. Breadcrumb via `writeFileSync(join(RT_DIR, "daemon-boot.json"), JSON.stringify({ at, pid: process.pid, flavor: currentMode(), phase }))` inside a try/catch (never fatal). -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Wire into the boot path** (`lib/daemon.ts`): - `recordBootAttempt(); writeBreadcrumb("start");` at the top of `runDaemon()` (after Task 3's crash handlers are already at module scope; call these first thing inside `runDaemon`). - - `writeBreadcrumb("events-db" | "state-db" | "socket" | "api")` at each corresponding phase (events.db is module-scope — write that breadcrumb right after `createEventsBus`; state-db right after `openBranchCacheStore`; socket/api at the binds). + - `writeBreadcrumb("events-db" | "state-db" | "socket" | "api")` at each corresponding phase (events.db is module-scope ... write that breadcrumb right after `createEventsBus`; state-db right after `openBranchCacheStore`; socket/api at the binds). - In Task 2's catch: `recordBootFailure(currentPhase, String(err));` (track `currentPhase` in a module var updated alongside each `writeBreadcrumb`). - `recordDaemonReady(); writeBreadcrumb("ready");` right where `bootPhase = "ready"` is set (line 513). - Shutdown verb (349-364): before `process.exit(0)`, `recordCleanExit("shutdown", 0);` and set the Task 12 `shuttingDownViaVerb = true` flag (Task 12 adds the flag; here just add the record call). - [ ] **Step 6: Wire into signal exit** (`lib/daemon/shutdown.ts` `gracefulExit`): before exit, `recordCleanExit("signal", code)` (Task 12 sets `code` to 1 for bare signals; for now record with the code it exits with). -- [ ] **Step 7: Add an e2e assertion** in `e2e/tests/daemon.test.ts`: after the Task 5 API-bind-failure spawn, assert `daemon-boot.json` exists with `phase: "api"` and the kv `recent-failures` has an entry (read the state.db in the isolated HOME, or assert via `rt daemon status --json` once Task 10 lands — for Task 9, assert the breadcrumb file only). +- [ ] **Step 7: Add an e2e assertion** in `e2e/tests/daemon.test.ts`: after the Task 5 API-bind-failure spawn, assert `daemon-boot.json` exists with `phase: "api"` and the kv `recent-failures` has an entry (read the state.db in the isolated HOME, or assert via `rt daemon status --json` once Task 10 lands ... for Task 9, assert the breadcrumb file only). -- [ ] **Step 8: Run the unit + e2e tests — expect PASS.** +- [ ] **Step 8: Run the unit + e2e tests ... expect PASS.** - [ ] **Step 9: Commit** @@ -753,7 +753,7 @@ git commit -m "daemon: persist boot attempts, failures, last-exit in kv + boot b --- -## Task 10: Status verdicts (0.5 — R001, S026 daemon-side) +## Task 10: Status verdicts (0.5 ... R001, S026 daemon-side) Extend `DaemonStatusVerdict` and `classifyDaemonStatus` with `alive-not-serving`, `parked`, `boot-failed`, `crash-looping`, read from the liveness probe + supervision state + breadcrumb. Expose the fields in `ping`/`/api/status`, and print them in `rt daemon status`. @@ -787,17 +787,17 @@ test("no pid + single boot-failed -> boot-failed with reason", () => { }); ``` -- [ ] **Step 2: Run — expect FAIL.** +- [ ] **Step 2: Run ... expect FAIL.** - [ ] **Step 3: Extend the verdict type and `classifyDaemonStatus`** in `lib/daemon-status.ts` following the resolution order from the design doc (Task 1): not-installed → serving → parked → alive-not-serving → crash-looping → boot-failed → installed-not-running. `classifyDaemonStatus` takes the already-gathered inputs (`pingOk`, `pidAlive`, `pid`, `breadcrumb`, `supervision`); keep it a pure function (the liveness probe and kv/breadcrumb reads happen in `commands/daemon.ts`/the caller, matching the existing `needsLivenessProbe` split). `parked` when breadcrumb phase indicates a flavor standoff; `alive-not-serving` detail = `booting` (phase < ready), `wedged` (phase == ready), `quarantined` (a `*.boot-failed` marker present). -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Surface the data.** In `lib/daemon/handlers/status.ts` `ping`, add a `supervision: { bootAttempts, lastReadyAt, recentFailures: recentFailures.slice(-3), lastExit }` field (read via Task 9). In `commands/daemon.ts` `showStatus`/liveness probe, gather `pidAlive` (`process.kill(pid,0)` on rt.pid, fallback `pgrep -f 'rt --daemon|lib/daemon.ts'` via `runCapture`) and read the breadcrumb + supervision state (when ping fails), then pass to `classifyDaemonStatus`. In `statusLines`, print a line per new verdict (see the design doc's status strings). - [ ] **Step 6: Add a `--json` assertion e2e** in `e2e/tests/daemon.test.ts`: after an API-bind-failure spawn, `rt daemon status --json` under the same isolated HOME reports `boot-failed` or `crash-looping` (whichever the failure count yields). Run under `env -i HOME=`. -- [ ] **Step 7: Run the unit + e2e tests — expect PASS.** `bun test lib/__tests__/daemon-status.test.ts` +- [ ] **Step 7: Run the unit + e2e tests ... expect PASS.** `bun test lib/__tests__/daemon-status.test.ts` - [ ] **Step 8: Commit** @@ -808,7 +808,7 @@ git commit -m "daemon status: alive-not-serving / parked / boot-failed / crash-l --- -## Task 11: stderr log rotation + stale-crash stamp (0.5 — S029) +## Task 11: stderr log rotation + stale-crash stamp (0.5 ... S029) `daemon-stderr.log` is never rotated and its stale contents are shown as "most recent crash". Rotate on open and gate the crash block on mtime. @@ -833,13 +833,13 @@ test("redirectNativeStderr rotates a non-empty daemon-stderr.log before reopenin }); ``` -- [ ] **Step 2: Run — expect FAIL.** +- [ ] **Step 2: Run ... expect FAIL.** - [ ] **Step 3: Implement rotate-on-open** in `redirectNativeStderr` (`lib/daemon-logger.ts:204-219`): before `openSync(path, "a")`, if the file exists and is non-empty, `renameSync` it to `daemon-stderr..log` (dedupe with a `.N` suffix if that name already exists, matching the janitor's dated-file convention). Keep the existing swallow-on-failure behavior. -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** -- [ ] **Step 5: Write the failing `showLogs` test** — the native-stderr block is hidden when the file mtime predates the current daemon's start: +- [ ] **Step 5: Write the failing `showLogs` test** ... the native-stderr block is hidden when the file mtime predates the current daemon's start: ```ts test("showLogs hides the native-stderr block when the file is older than the daemon start", () => { @@ -849,7 +849,7 @@ test("showLogs hides the native-stderr block when the file is older than the dae - [ ] **Step 6: Implement in `showLogs`** (`commands/daemon.ts:724-737`): only print the native-stderr block when the file's `mtime` is newer than the current daemon's `startedAt` (from `ping`); include the mtime in the header (`native stderr (captured )`). When older, skip it silently (or print a one-line "no crash since this daemon started"). -- [ ] **Step 7: Run — expect PASS.** +- [ ] **Step 7: Run ... expect PASS.** - [ ] **Step 8: Commit** @@ -860,7 +860,7 @@ git commit -m "logs: rotate daemon-stderr.log on open; hide stale crash block by --- -## Task 12: Exit-code semantics (0.6 — S036, S060) +## Task 12: Exit-code semantics (0.6 ... S036, S060) The `shutdown` verb correctly exits 0, but bare OS signals also exit 0, so an externally-killed daemon stays down. Reserve exit 0 for the verb; exit non-zero on bare signals. @@ -892,11 +892,11 @@ test("gracefulExit exits 0 after the shutdown verb, 1 on a bare signal", () => { (Refactor `gracefulExit` into a testable `makeGracefulExit(deps)` that returns the handler, injecting `exit`/`recordCleanExit` so no real `process.exit` fires in the test.) -- [ ] **Step 2: Run — expect FAIL.** +- [ ] **Step 2: Run ... expect FAIL.** - [ ] **Step 3: Implement.** In `lib/daemon/shutdown.ts`, refactor `installSignalHandlers`/`gracefulExit` to `makeGracefulExit(deps)` reading `deps.wasVerbShutdown()`: true → `recordCleanExit("shutdown", 0)` + `exit(0)`; false → `recordCleanExit("signal", 1)` + `exit(1)`. In `lib/daemon.ts`, add module-scope `let shuttingDownViaVerb = false;`, set it `true` in the shutdown verb before `cleanup()`, and pass `wasVerbShutdown: () => shuttingDownViaVerb` into `installSignalHandlers` at line 511. The shutdown verb keeps `process.exit(0)`. -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Commit** @@ -907,7 +907,7 @@ git commit -m "daemon: bare-signal exit is non-zero (launchd respawns); shutdown --- -## Task 13: Ownership-aware cleanup + eviction death-confirmation (0.6 — S012, S044) +## Task 13: Ownership-aware cleanup + eviction death-confirmation (0.6 ... S012, S044) `cleanup()` unlinks rt.sock/rt.pid unconditionally, and eviction sleeps a blind 300ms. Make cleanup compare-and-delete, and make eviction wait for the old pid to actually die. @@ -940,7 +940,7 @@ test("cleanup unlinks when the pid file is ours", () => { (Inject `pid` into `createCleanup` deps for testability; default to `process.pid`.) -- [ ] **Step 2: Run — expect FAIL.** +- [ ] **Step 2: Run ... expect FAIL.** - [ ] **Step 3: Implement ownership-aware unlink** in `createCleanup` (`lib/daemon/shutdown.ts:44-46`): @@ -953,7 +953,7 @@ try { } catch (err) { deps.log.warn({ err }, "cleanup unlink skipped"); } ``` -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Write the failing eviction test** in `lib/daemon/__tests__/boot-reconcile.test.ts`: @@ -969,9 +969,9 @@ test("evictStaleDaemon waits for the old pid to die, escalating to SIGKILL", asy }); ``` -- [ ] **Step 6: Implement in `evictStaleDaemon`** (`lib/daemon/boot-reconcile.ts`): replace `Bun.sleepSync(300)` with an async poll — after SIGTERM, loop `process.kill(pid, 0)` every 100ms up to ~2.5s; if still alive, `process.kill(pid, "SIGKILL")` and poll another ~0.5s; return once `process.kill(pid,0)` throws (pid gone). Make the function `async` and `await` it at the call site (`lib/daemon.ts:396`). Keep the `previousPid === process.pid` self-guard. +- [ ] **Step 6: Implement in `evictStaleDaemon`** (`lib/daemon/boot-reconcile.ts`): replace `Bun.sleepSync(300)` with an async poll ... after SIGTERM, loop `process.kill(pid, 0)` every 100ms up to ~2.5s; if still alive, `process.kill(pid, "SIGKILL")` and poll another ~0.5s; return once `process.kill(pid,0)` throws (pid gone). Make the function `async` and `await` it at the call site (`lib/daemon.ts:396`). Keep the `previousPid === process.pid` self-guard. -- [ ] **Step 7: Run — expect PASS.** Then `bun test lib/daemon/__tests__/boot-reconcile.test.ts lib/daemon/__tests__/shutdown.test.ts` +- [ ] **Step 7: Run ... expect PASS.** Then `bun test lib/daemon/__tests__/boot-reconcile.test.ts lib/daemon/__tests__/shutdown.test.ts` - [ ] **Step 8: Commit** @@ -982,7 +982,7 @@ git commit -m "daemon: ownership-aware socket/pid unlink; eviction waits for pid --- -## Task 14: uninstall + start guards (0.6 — S027, S030, S028 CLI-side) +## Task 14: uninstall + start guards (0.6 ... S027, S030, S028 CLI-side) `rt daemon uninstall` deletes rt.sock/rt.pid from under a live daemon, and `rt daemon start` cannot revive a registered-but-exited-0 daemon. Guard uninstall on liveness; make start escalate to a kickstart route. @@ -1005,11 +1005,11 @@ test("uninstall leaves rt.sock/rt.pid and daemon.json when the daemon is still r }); ``` -- [ ] **Step 2: Run — expect FAIL.** +- [ ] **Step 2: Run ... expect FAIL.** - [ ] **Step 3: Implement the uninstall guard** (`commands/daemon.ts:210-231`): after a failed/absent `trayQuery("/daemon/stop")`, call `isDaemonProcessRunning()` (and `probeSocketHolder()` as a second signal). Only run `markDaemonUninstalled()` + `cleanupDaemonFiles()` when no live holder; otherwise print the remedy `launchctl bootout gui/$UID/${activeLaunchdLabel()}` and leave the files. Audit other callers of `cleanupDaemonFiles`/`markDaemonUninstalled` (e.g. the dev-mode toggle in `commands/settings.ts`) for the same missing guard and note any in the commit body. -- [ ] **Step 4: Run — expect PASS.** +- [ ] **Step 4: Run ... expect PASS.** - [ ] **Step 5: Write the failing start-escalation test**: @@ -1021,9 +1021,9 @@ test("start escalates to the restart/kickstart route when the tray acks but the }); ``` -- [ ] **Step 6: Implement start escalation** (`commands/daemon.ts:235-269`): when the tray acked `/daemon/start` but `isDaemonRunning()` stays false through the 12×250ms poll, fall back to the `/daemon/restart` route (kickstart). In `lib/daemon-client.ts:171-185`, make `attemptRestart` re-probe `isDaemonRunning()` after `trayQuery("/daemon/start")` and return `true` only when the daemon actually answers — so the `daemonQuery` nag stops misdirecting. Do NOT change signal-handler exit codes here (that is Task 12). +- [ ] **Step 6: Implement start escalation** (`commands/daemon.ts:235-269`): when the tray acked `/daemon/start` but `isDaemonRunning()` stays false through the 12×250ms poll, fall back to the `/daemon/restart` route (kickstart). In `lib/daemon-client.ts:171-185`, make `attemptRestart` re-probe `isDaemonRunning()` after `trayQuery("/daemon/start")` and return `true` only when the daemon actually answers ... so the `daemonQuery` nag stops misdirecting. Do NOT change signal-handler exit codes here (that is Task 12). -- [ ] **Step 7: Run — expect PASS.** +- [ ] **Step 7: Run ... expect PASS.** - [ ] **Step 8: Commit** @@ -1034,7 +1034,7 @@ git commit -m "daemon CLI: uninstall guards on liveness; start escalates to kick --- -## Task 15: Retire the stale audit doc (0.8 — R017) +## Task 15: Retire the stale audit doc (0.8 ... R017) Replace `docs/daemon-runner-health.md` (which audits deleted subsystems) with a pointer to the current audit and the new supervision design doc. @@ -1044,7 +1044,7 @@ Replace `docs/daemon-runner-health.md` (which audits deleted subsystems) with a - [ ] **Step 1: Replace the file's contents** with a short pointer: ```markdown -# Daemon runner health — superseded +# Daemon runner health ... superseded This document audited subsystems (process-manager, remedy-engine, runner.tsx, workspace-sync) that no longer exist. It is retained only as @@ -1066,14 +1066,14 @@ git commit -m "docs: retire stale daemon-runner-health.md, point at the current --- -## Task 16 (OPTIONAL — pending plan-review scope confirmation): Swift tray consumption +## Task 16 (OPTIONAL ... pending plan-review scope confirmation): Swift tray consumption **Do not start without the reviewer's go-ahead** (raised in the plan-milestone report). These edits cannot be verified by the bun/tsc/e2e gate, and the operating rules forbid rebuilding the blessed bundle, so they ship as source-only, unverified changes following the fixer notes: -- **S026** — `rt-tray/Sources/AppDelegate.swift`: give `.starting` an expiry (record `startingSince`; in `refreshStatus` treat `.starting` as expired after ~30s / 3 failed polls and fall through to `setHealth(.down)`), so the health dot stops sticking yellow; map the new daemon verdicts (Task 10) to dot colors. -- **S028** — `rt-tray/Sources/DaemonLifecycle.swift`: when `register()` returns already-registered but the socket stays unreachable, fall back to `launchctl kickstart` (Kickstart.arguments already exists). -- **S029** — `rt-tray/Sources/TrayLog.swift`: rotate `tray-crash.log` on open (rename-if-nonempty), matching Task 11's `daemon-stderr.log` treatment. -- **S060** — `rt-tray/Sources/AppDelegate.swift:625-627`: fix the stale comment to say `KeepAlive: SuccessfulExit=false` (not `KeepAlive=true`). +- **S026** ... `rt-tray/Sources/AppDelegate.swift`: give `.starting` an expiry (record `startingSince`; in `refreshStatus` treat `.starting` as expired after ~30s / 3 failed polls and fall through to `setHealth(.down)`), so the health dot stops sticking yellow; map the new daemon verdicts (Task 10) to dot colors. +- **S028** ... `rt-tray/Sources/DaemonLifecycle.swift`: when `register()` returns already-registered but the socket stays unreachable, fall back to `launchctl kickstart` (Kickstart.arguments already exists). +- **S029** ... `rt-tray/Sources/TrayLog.swift`: rotate `tray-crash.log` on open (rename-if-nonempty), matching Task 11's `daemon-stderr.log` treatment. +- **S060** ... `rt-tray/Sources/AppDelegate.swift:625-627`: fix the stale comment to say `KeepAlive: SuccessfulExit=false` (not `KeepAlive=true`). If confirmed, do the comment fix (S060) first (trivial), then S026/S028/S029, one commit each, each commit body noting "source-only, unverified: blessed bundle not rebuilt". @@ -1082,7 +1082,7 @@ If confirmed, do the comment fix (S060) first (trivial), then S026/S028/S029, on ## Final verification (before the whole-branch review) - [ ] `cd packages/rt-client && bun run build && cd -` (rt-client was touched in Task 5) -- [ ] `bunx tsc --noEmit` — zero errors -- [ ] `bun test lib commands packages scripts` — green -- [ ] `bun run test:e2e` (or `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts` — record which) +- [ ] `bunx tsc --noEmit` ... zero errors +- [ ] `bun test lib commands packages scripts` ... green +- [ ] `bun run test:e2e` (or `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts` ... record which) - [ ] Request the whole-branch code review (superpowers:requesting-code-review); address findings; re-run the gate. From 05a8f18e684fd8bda6d48e17a120b2eda8a22b5a Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 14:05:30 -0500 Subject: [PATCH 088/142] chore: scrub em/en dashes from branch-added comments and strings (owner rule) --- commands/__tests__/daemon-logs-render.test.ts | 4 +- .../__tests__/daemon-uninstall-start.test.ts | 16 ++++---- commands/__tests__/probe-pid-alive.test.ts | 2 +- commands/daemon.ts | 38 +++++++++---------- e2e/tests/daemon.test.ts | 6 +-- lib/__tests__/daemon-logger.test.ts | 8 ++-- lib/daemon-client.ts | 2 +- lib/daemon-config.ts | 2 +- lib/daemon-logger.ts | 4 +- lib/daemon-status.ts | 18 ++++----- lib/daemon.ts | 14 +++---- lib/daemon/__tests__/home-snapshot.test.ts | 8 ++-- .../__tests__/supervision-state.test.ts | 2 +- lib/daemon/boot-reconcile.ts | 2 +- lib/daemon/events-bus.ts | 4 +- lib/daemon/handlers/status.ts | 2 +- lib/daemon/home-snapshot.ts | 4 +- lib/daemon/safe-timers.ts | 4 +- lib/daemon/shutdown.ts | 4 +- lib/daemon/supervision-state.ts | 8 ++-- lib/state/__tests__/db.test.ts | 2 +- lib/state/__tests__/source-guards.test.ts | 4 +- lib/state/busy.ts | 2 +- lib/state/db.ts | 4 +- 24 files changed, 82 insertions(+), 82 deletions(-) diff --git a/commands/__tests__/daemon-logs-render.test.ts b/commands/__tests__/daemon-logs-render.test.ts index 877e510b..eb14e8b7 100644 --- a/commands/__tests__/daemon-logs-render.test.ts +++ b/commands/__tests__/daemon-logs-render.test.ts @@ -1,5 +1,5 @@ /** - * nativeStderrDisplay — showLogs' stale-crash mtime gate. + * nativeStderrDisplay (showLogs' stale-crash mtime gate). * * daemon-stderr.log is rotated on daemon boot (lib/daemon-logger.ts), but a * leftover file can still predate the *currently running* daemon (e.g. it was @@ -31,7 +31,7 @@ describe("nativeStderrDisplay", () => { expect(header).toBe(`native stderr (captured ${new Date(mtimeMs).toISOString()})`); }); - test("fails open (shows) when the daemon's startedAt is unknown — nothing to compare against", () => { + test("fails open (shows) when the daemon's startedAt is unknown (nothing to compare against)", () => { const { show } = nativeStderrDisplay(NOW - 999_999, null); expect(show).toBe(true); }); diff --git a/commands/__tests__/daemon-uninstall-start.test.ts b/commands/__tests__/daemon-uninstall-start.test.ts index 1a3f58fd..926f5dcc 100644 --- a/commands/__tests__/daemon-uninstall-start.test.ts +++ b/commands/__tests__/daemon-uninstall-start.test.ts @@ -1,11 +1,11 @@ /** - * `rt daemon uninstall`/`start` — the CLI-side liveness guards (Task 14, + * `rt daemon uninstall`/`start` (the CLI-side liveness guards, Task 14, * S027/S030/S028-CLI). Fakes the tray over a real Bun.serve on * TRAY_SOCK_PATH (same rig as commands/__tests__/settings-dev-mode.test.ts) * and, where a scenario needs "the daemon is live", a real Bun.serve on - * DAEMON_SOCK_PATH answering /ping — isDaemonProcessRunning's pid check and + * DAEMON_SOCK_PATH answering /ping (isDaemonProcessRunning's pid check and * probeSocketHolder/isDaemonRunning's socket ping are both exercised for - * real, never mocked module internals. + * real, never mocked module internals). */ import { afterEach, describe, expect, test } from "bun:test"; import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs"; @@ -40,12 +40,12 @@ function serveTray(handlers: Record Response>): void { })); } -/** A real listener on rt.sock that answers /ping — what both isDaemonRunning() +/** A real listener on rt.sock that answers /ping (what both isDaemonRunning() * (daemon-client.ts) and probeSocketHolder() (lib/daemon/park.ts) fetch. * Flavor defaults to the CURRENT intended mode (not a hardcoded "prod") so * start()'s post-liveness warnIfWrongFlavor() check never fires a spurious * mismatch when this file runs after another test flips mattstack.mode in - * the shared isolated HOME `bun test` uses for the whole process. */ + * the shared isolated HOME `bun test` uses for the whole process). */ function serveDaemonPing(body?: Record): void { const resolvedBody = body ?? { ok: true, pid: 4242, flavor: resolveIntendedMode().mode }; servers.push(Bun.serve({ @@ -67,7 +67,7 @@ afterEach(() => { } }); -describe("uninstall — liveness guard", () => { +describe("uninstall (liveness guard)", () => { test("leaves rt.pid/daemon.json when isDaemonProcessRunning() says the daemon is alive", async () => { mkdirSync(RT_DIR, { recursive: true }); markDaemonInstalled(); @@ -98,7 +98,7 @@ describe("uninstall — liveness guard", () => { mkdirSync(RT_DIR, { recursive: true }); markDaemonInstalled(); writeFileSync(DAEMON_PID_PATH, "999999"); // no such pid - writeFileSync(DAEMON_SOCK_PATH, ""); // stale file, not a real listener — probeSocketHolder's fetch fails + writeFileSync(DAEMON_SOCK_PATH, ""); // stale file, not a real listener (probeSocketHolder's fetch fails) captureLogs(); await uninstall(); @@ -110,7 +110,7 @@ describe("uninstall — liveness guard", () => { }); }); -describe("start — kickstart escalation", () => { +describe("start (kickstart escalation)", () => { test("falls back to /daemon/restart when the tray acks /daemon/start but the socket never comes up", async () => { mkdirSync(RT_DIR, { recursive: true }); markDaemonInstalled(); diff --git a/commands/__tests__/probe-pid-alive.test.ts b/commands/__tests__/probe-pid-alive.test.ts index 269318be..e699d456 100644 --- a/commands/__tests__/probe-pid-alive.test.ts +++ b/commands/__tests__/probe-pid-alive.test.ts @@ -5,7 +5,7 @@ import { readSupervisionState } from "../../lib/daemon/supervision-state.ts"; describe("probePidAlive", () => { // Regression: the lsof fallback must exclude the CALLING process itself. // showStatus opens a bun:sqlite handle on state.db (inside RT_DIR) via - // readSupervisionState() immediately before this probe runs — `lsof +D + // readSupervisionState() immediately before this probe runs ... `lsof +D // RT_DIR` then legitimately reports the calling CLI process as a live // holder of the directory, with no daemon involved at all. Without the // process.pid filter this self-matches and a genuinely dead daemon diff --git a/commands/daemon.ts b/commands/daemon.ts index 43da6bd2..8d492ab5 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -227,14 +227,14 @@ export async function uninstall(): Promise { } // 3. A failed/absent tray stop must never delete rt.sock/rt.pid/daemon.json - // out from under a daemon that's actually still alive — that would orphan - // it (still running, launchd-supervised, but rt's own bookkeeping says + // out from under a daemon that's actually still alive (that would orphan + // it, still running, launchd-supervised, but rt's own bookkeeping says // uninstalled). Check both liveness signals: the recorded pid, and whether // anything still answers on rt.sock (a daemon can be alive with no // matching rt.pid, e.g. after a crash-and-respawn under launchd). const stillAlive = isDaemonProcessRunning() || (await probeSocketHolder()) !== null; if (stillAlive) { - console.log(`\n ${yellow}⚠${reset} daemon is still running — leaving rt.sock/rt.pid/daemon.json in place`); + console.log(`\n ${yellow}⚠${reset} daemon is still running, leaving rt.sock/rt.pid/daemon.json in place`); console.log(` ${dim}Fix: ${bold}launchctl bootout gui/$UID/${activeLaunchdLabel()}${reset}\n`); return; } @@ -276,10 +276,10 @@ export async function start(): Promise { if (await pollForDaemonUp(intended)) return; // The tray acked /daemon/start, but SMAppService can register a job that - // never actually launches (still booting, crash-looping, etc.) — kick it + // never actually launches (still booting, crash-looping, etc.); kick it // via /daemon/restart, which forces launchd to invoke it, rather than // leaving the operator staring at "check logs" for something a retry fixes. - console.log(` ${dim}not up yet — escalating to restart (kickstart)…${reset}`); + console.log(` ${dim}not up yet, escalating to restart (kickstart)…${reset}`); const restartResult = await trayQuery("/daemon/restart", "POST"); if (restartResult?.ok && (await pollForDaemonUp(intended))) return; @@ -350,18 +350,18 @@ export async function restart(): Promise { /** * Raw OS-level liveness, independent of rt.sock. Tries a direct pid check - * first against every pid this HOME actually recorded — rt.pid, then the + * first against every pid this HOME actually recorded: rt.pid, then the * boot breadcrumb's pid (the breadcrumb survives failures rt.pid never gets - * written for, per Ruling P1) — before falling back to a last-resort scan. + * written for, per Ruling P1), before falling back to a last-resort scan. * * That scan is `lsof +D `, not the brief's suggested system-wide * `pgrep -f 'rt --daemon|lib/daemon.ts'`: a raw pgrep matches ANY rt daemon * on the machine regardless of which HOME started it, and on an ordinary dev - * workstation there usually IS one — the developer's own real daemon — so a + * workstation there usually IS one (the developer's own real daemon), so a * pgrep-based check on an isolated/alternate HOME reliably misreports a dead * boot attempt as alive-not-serving (verified live against this repo's own * dev daemon while writing the e2e test below). `lsof +D` instead asks "does - * any process hold a file open under THIS HOME's rt dir" — home-scoped by + * any process hold a file open under THIS HOME's rt dir", home-scoped by * construction, immune to that false positive and to pid-reuse. Only worth * calling once both `status` and a plain ping have already failed. * @@ -369,11 +369,11 @@ export async function restart(): Promise { * own `bun:sqlite` handle on `state.db` (inside RT_DIR) via * `readSupervisionState()` just before this probe runs, so `lsof +D RT_DIR` * legitimately reports the calling CLI process as a live holder of the - * directory — with no daemon involved at all. Left unfiltered, a genuinely + * directory, with no daemon involved at all. Left unfiltered, a genuinely * dead daemon self-matches and misclassifies as alive-not-serving/parked; * this only failed to show up in manual testing because incidental work * happened to separate the state.db open from the lsof call by enough time - * for state.db's own transient lock window to close — an accident of + * for state.db's own transient lock window to close, an accident of * timing, not a guarantee. */ export async function probePidAlive(recordedPid: number | null, breadcrumbPid?: number): Promise<{ alive: boolean; pid: number | null }> { @@ -382,7 +382,7 @@ export async function probePidAlive(recordedPid: number | null, breadcrumbPid?: try { process.kill(candidate, 0); return { alive: true, pid: candidate }; - } catch { /* not this one — try the next candidate */ } + } catch { /* not this one, try the next candidate */ } } const { stdout } = await runCapture(["lsof", "-t", "+D", RT_DIR], { timeoutMs: 3000 }); const pids = stdout.trim().split(/\s+/).filter(Boolean).map(Number) @@ -408,7 +408,7 @@ export async function showStatus(args: string[] = []): Promise { // Ping ALSO failed: the only remaining ground is the pid/breadcrumb/kv // trail Task 9 left behind. Read it here, once, rather than on every status - // call — it's the uncommon path. + // call, since it's the uncommon path. let pidAlive: boolean | undefined; let pid = recordedPid; let breadcrumb: ReturnType | undefined; @@ -416,7 +416,7 @@ export async function showStatus(args: string[] = []): Promise { if (classifyDaemonStatus.needsPidProbe(response, pingOk)) { breadcrumb = readBreadcrumb(); // The kv tier can be legitimately empty (or reflect nothing useful) when - // a failure happened before state.db ever opened — Ruling P1. The + // a failure happened before state.db ever opened (Ruling P1). The // breadcrumb read above is what classifyDaemonStatus falls back to then. supervision = readSupervisionState(); const probed = await probePidAlive(recordedPid, breadcrumb?.pid); @@ -521,7 +521,7 @@ export function statusLines(verdict: DaemonStatusVerdict, now: number): string[] } if (verdict.state === "parked") { - const lines = [` ${yellow}◐${reset} parked ${dim}(pid ${verdict.pid} — another flavor owns rt.sock)${reset}`]; + const lines = [` ${yellow}◐${reset} parked ${dim}(pid ${verdict.pid}, another flavor owns rt.sock)${reset}`]; lines.push( verdict.holderFlavor ? ` ${dim}held by: ${verdict.holderFlavor}${reset}` @@ -534,7 +534,7 @@ export function statusLines(verdict: DaemonStatusVerdict, now: number): string[] if (verdict.state === "alive-not-serving") { const detailLine = { booting: "still booting", - wedged: "reached ready but stopped answering — likely deadlocked", + wedged: "reached ready but stopped answering (likely deadlocked)", quarantined: "recovered from a corrupt db but still not answering", }[verdict.detail]; return [ @@ -853,9 +853,9 @@ export async function manageTracking(args: string[] = []): Promise { * Decides whether showLogs' native-stderr block is worth printing, and its * header. `daemon-stderr.log` is rotated on open (daemon-logger.ts) but the * fresh file can still be non-empty from a crash that happened before *this* - * boot's rotation ran (e.g. a bun panic mid-startup) — so staleness is judged + * boot's rotation ran (e.g. a bun panic mid-startup), so staleness is judged * by mtime vs. the live daemon's startedAt, not by rotation alone. A `null` - * startedAt (daemon unreachable — nothing to compare against) fails open: + * startedAt (daemon unreachable, nothing to compare against) fails open: * show it, since a down daemon is exactly when the last crash matters most. */ export function nativeStderrDisplay( @@ -885,7 +885,7 @@ export async function showLogs(args: string[] = []): Promise { // Surface captured native stderr first — these are bun panics/asserts that // bypassed the JS-side interceptor and were caught by the swift-shim's - // freopen of fd 2. Only shown when it postdates the running daemon's boot — + // freopen of fd 2. Only shown when it postdates the running daemon's boot, // otherwise it's a previous life's crash, not "the most recent crash". const stderrPath = join(LOG_DIR, "daemon-stderr.log"); if (existsSync(stderrPath)) { diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index 469d54a6..4f90640a 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -65,7 +65,7 @@ describe("fatal boot", () => { const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); try { // `rt daemon status` short-circuits to "not installed" before it ever - // reaches the boot-failed/crash-looping classification — install first. + // reaches the boot-failed/crash-looping classification, install first. await rt(["daemon", "install"], { home }); const boot = await rt(["--daemon"], { home, env: { RT_API_PORT: String(port) } }); @@ -81,13 +81,13 @@ describe("fatal boot", () => { } }, 60_000); - test("a corrupt events.db self-heals — quarantined, and the daemon boots and serves", async () => { + test("a corrupt events.db self-heals (quarantined), and the daemon boots and serves", async () => { const { path: home, cleanup } = createTestHome(); const bunDir = join(process.execPath, ".."); let daemon: ReturnType | undefined; try { // Pre-create a corrupt events.db in the isolated HOME, before the - // daemon ever runs — createEventsBus (module scope) opens it. + // daemon ever runs; createEventsBus (module scope) opens it. const rtDir = join(home, ".mattstack", "rt"); mkdirSync(rtDir, { recursive: true }); writeFileSync(join(rtDir, "events.db"), "not a sqlite file at all"); diff --git a/lib/__tests__/daemon-logger.test.ts b/lib/__tests__/daemon-logger.test.ts index c573f6aa..dd8e6b3b 100644 --- a/lib/__tests__/daemon-logger.test.ts +++ b/lib/__tests__/daemon-logger.test.ts @@ -136,9 +136,9 @@ describe("lazyChildLogger", () => { }); }); -describe("redirectNativeStderr — rotation", () => { +describe("redirectNativeStderr (rotation)", () => { // redirectNativeStderr dup2's the REAL process fd 2 to the log file (that is - // the whole point of the function) — a bare call here would swallow this + // the whole point of the function); a bare call here would swallow this // test process's own stderr for the rest of the run. Save/restore fd 2 // around the call with the same dup/dup2 pair the implementation uses. function withRealFd2Saved(fn: () => void): void { @@ -286,12 +286,12 @@ describe("lazyChildLogger — Proxy guard", () => { }); }); -describe("installCrashHandlers — boot-phase gating", () => { +describe("installCrashHandlers (boot-phase gating)", () => { it("unhandledRejection exits(1) while booting, only logs once ready", () => { const exits: number[] = []; const origExit = process.exit; const origStderrWrite = process.stderr.write.bind(process.stderr); - // @ts-expect-error test stub — captures the exit code instead of terminating + // @ts-expect-error test stub (captures the exit code instead of terminating) process.exit = (code?: number) => { exits.push(code ?? 0); }; const fatal = mock(() => {}); const error = mock(() => {}); diff --git a/lib/daemon-client.ts b/lib/daemon-client.ts index c29220d1..79e56e08 100644 --- a/lib/daemon-client.ts +++ b/lib/daemon-client.ts @@ -181,7 +181,7 @@ async function attemptRestart(): Promise { if (res === null) return false; // The tray ack only proves the request was received, not that the - // daemon actually came up — re-probe rt.sock before reporting success, + // daemon actually came up, so re-probe rt.sock before reporting success, // so daemonQuery's caller isn't told "restarted" while the daemon is // still mid-boot and then misdirected into warnDaemonDown() on the very // next query instead of actually waiting for it. diff --git a/lib/daemon-config.ts b/lib/daemon-config.ts index 52f4a246..6396a3a3 100644 --- a/lib/daemon-config.ts +++ b/lib/daemon-config.ts @@ -75,7 +75,7 @@ export const API_PORT = Number(process.env.RT_API_PORT) || 9401; /** * Call-time API port resolution: RT_API_PORT env wins (e2e isolation, RT-45), * then the rt.apiPort setting (escape hatch when 9401 is held), then 9401. - * A function, not a const — must never be evaluated at module load, since + * A function, not a const: it must never be evaluated at module load, since * getSetting() reads the settings stores off ambient HOME. */ export function resolveApiPort(): number { diff --git a/lib/daemon-logger.ts b/lib/daemon-logger.ts index 5c7bc0df..634e84b0 100644 --- a/lib/daemon-logger.ts +++ b/lib/daemon-logger.ts @@ -200,7 +200,7 @@ function todayDate(): string { /** * Picks the rotation target for `daemon-stderr.log`: `daemon-stderr..log`, - * or `..log` if that name is already taken (e.g. two boots same day) — both + * or `..log` if that name is already taken (e.g. two boots same day); both * shapes match log-janitor's LOG_FILE_PATTERN, so pruneLogs sweeps them for free. */ function nextRotatedStderrPath(dir: string, date: string): string { @@ -227,7 +227,7 @@ export function redirectNativeStderr(): void { const dir = logsDir(); mkdirSync(dir, { recursive: true }); const stderrPath = join(dir, "daemon-stderr.log"); - // Rotate any leftover content from a previous crash before reopening — + // Rotate any leftover content from a previous crash before reopening, // otherwise `rt daemon logs` keeps showing yesterday's panic as "most // recent". A rename here can never lose data (unlike truncation). if (existsSync(stderrPath) && statSync(stderrPath).size > 0) { diff --git a/lib/daemon-status.ts b/lib/daemon-status.ts index f9face8b..2cdbbcfc 100644 --- a/lib/daemon-status.ts +++ b/lib/daemon-status.ts @@ -13,7 +13,7 @@ * `pidAlive`/`breadcrumb`/`supervision` (Task 9's supervision-state.ts) are * the only signals that can tell, and per Ruling P1 (2026-08-28 p0-supervision * ledger) the breadcrumb FILE is the sole record a pre-state.db boot failure - * leaves — `supervision` (the kv tier) can be absent even when `breadcrumb` + * leaves. `supervision` (the kv tier) can be absent even when `breadcrumb` * is present, and classification must still resolve to something useful from * the breadcrumb alone. */ @@ -27,7 +27,7 @@ export type DaemonStatusVerdict = /** Up — proven by an answer or a ping — but `status` itself did not deliver. */ | { state: "degraded"; reason: "error" | "unresponsive"; detail?: string; pid: number | null } /** Ping fails, a live pid exists, and it's parked waiting for a different - * flavor to hold rt.sock (park.ts) — a flavor standoff, not a stuck boot. */ + * flavor to hold rt.sock (park.ts): a flavor standoff, not a stuck boot. */ | { state: "parked"; pid: number; holderFlavor?: string } /** Ping fails but the pid is alive: still mid-boot, stuck after reaching * ready, or alive-but-quarantined (recovered from a corrupt db). */ @@ -39,7 +39,7 @@ export type DaemonStatusVerdict = | { state: "not-running"; pid: number | null }; /** The boot breadcrumb (`daemon-boot.json`), as classifyDaemonStatus needs it. Not - * imported from supervision-state.ts — that module's `Breadcrumb` interface is + * imported from supervision-state.ts, since that module's `Breadcrumb` interface is * intentionally unexported, and this shape only needs to be structurally * compatible with it. */ export interface DaemonBreadcrumbInput { @@ -58,19 +58,19 @@ export interface DaemonStatusInputs { /** Last recorded pid, for the operator to act on. */ pid: number | null; /** Raw OS-level liveness of `pid` (process.kill(pid,0), or a pgrep-found - * stand-in) — independent of rt.sock. Only worth gathering once `pingOk` + * stand-in), independent of rt.sock. Only worth gathering once `pingOk` * has already come back false; see `classifyDaemonStatus.needsPidProbe`. */ pidAlive?: boolean; /** This machine's currently-intended flavor (`resolveIntendedMode().mode`). * A live pid whose own breadcrumb flavor disagrees with this is parked - * (park.ts), not stuck — the same signal `parkUntilIntended` itself acts on. */ + * (park.ts), not stuck: the same signal `parkUntilIntended` itself acts on. */ intendedFlavor?: "dev" | "prod"; /** The socket holder's flavor, when the caller managed to learn it (best - * effort — probing rt.sock again after a failed ping/status round rarely + * effort: probing rt.sock again after a failed ping/status round rarely * succeeds, since a parked pid never binds it). Display-only. */ holderFlavor?: string | null; breadcrumb?: DaemonBreadcrumbInput | null; - /** Task 9's kv tier. Can be absent even when `breadcrumb` is present — a + /** Task 9's kv tier. Can be absent even when `breadcrumb` is present: a * pre-state.db failure leaves only the breadcrumb file (Ruling P1). */ supervision?: SupervisionState; /** Injected for deterministic crash-loop window checks under test; defaults to Date.now(). */ @@ -85,7 +85,7 @@ function classifyAliveNotServingDetail( ): "booting" | "wedged" | "quarantined" { const phase = breadcrumb?.phase; if (!phase || PHASE_ORDER.indexOf(phase) < PHASE_ORDER.indexOf("ready")) return "booting"; - // Reached ready this run, but a prior attempt is on record as boot-failed — + // Reached ready this run, but a prior attempt is on record as boot-failed, // most likely a corrupt-db quarantine (lib/state/db.ts, events-bus.ts) it // recovered from and is now stuck behind for an unrelated reason. if (supervision?.lastExit?.kind === "boot-failed") return "quarantined"; @@ -116,7 +116,7 @@ export function classifyDaemonStatus(opts: DaemonStatusInputs): DaemonStatusVerd if (pingOk) return { state: "degraded", reason: "unresponsive", pid }; // Ping failed too. From here, only pidAlive/breadcrumb/supervision (new - // signals) can say more than "not running" — absent them, fall straight + // signals) can say more than "not running"; absent them, fall straight // through to the pre-existing not-running verdict. if (pidAlive && pid !== null) { if (breadcrumb?.flavor && intendedFlavor && breadcrumb.flavor !== intendedFlavor) { diff --git a/lib/daemon.ts b/lib/daemon.ts index 64ea5fe1..44af086f 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -105,10 +105,10 @@ const rtMigration = migrateLegacyRtDir(); // during any later module-scope construction (createEventsBus, cron, // home-snapshot, …) lands in daemon-stderr.log instead of vanishing down // whatever fd 2 the launcher gave us. Depends only on logsDir() and mkdirs -// its own dir — this MUST run after migrateLegacyRtDir(): mkdirSync(logsDir()) +// its own dir (this MUST run after migrateLegacyRtDir(): mkdirSync(logsDir()) // creates the new rt dir, and migrateLegacyRtDir() treats that dir merely // existing as a "conflict" with a real legacy tree, so redirecting first -// would defeat the migration. +// would defeat the migration). redirectNativeStderr(); // ─── Logging ───────────────────────────────────────────────────────────────── @@ -449,7 +449,7 @@ async function runDaemon(): Promise { // Auto-unlink any tagged tool link whose tool now has a genuine user copy // elsewhere on PATH (e.g. the user ran `brew install gh` after rt linked // the bundled one). reconcile() itself is synchronous (a ~/.local/bin - // readDir plus a handful of stats) — wrapping the call in `async` alone + // readDir plus a handful of stats); wrapping the call in `async` alone // would NOT defer it, since nothing inside actually awaits. setTimeout(0) // is what actually pushes it past the rest of this function: the PID // write, openBranchCacheStore, and both server binds below all run first, @@ -476,7 +476,7 @@ async function runDaemon(): Promise { // one-shot re-key of every legacy NAME-keyed store row onto its // serialized repo identity. Fire-and-forget (not awaited) like the PATH - // reconcile above — the ordering guarantee this depends on (running before + // reconcile above: the ordering guarantee this depends on (running before // anything prunes the repo index) only needs this to be on the boot path, // not blocking the socket bind; a prune only ever arrives as a command sent // to an already-running daemon. @@ -524,7 +524,7 @@ async function runDaemon(): Promise { setPhase("socket"); servers.socket = startSocketServer({ handleCommand, log }); - // Only write rt.pid once both servers are actually bound — a boot that + // Only write rt.pid once both servers are actually bound: a boot that // fails before this point must never leave a live-pid file with no // socket/API behind it. writeFileSync(DAEMON_PID_PATH, String(process.pid)); @@ -536,7 +536,7 @@ async function runDaemon(): Promise { hooksGuard.refreshWatchedRepos(); // Team tracking intent (mattstack.tracking) resolves through a primed - // identity→name map, not live derivation — loadRepoTracking is sync and + // identity→name map, not live derivation; loadRepoTracking is sync and // runs on every freshness tick. Team intent is inert until this completes. // The repo index moved into state.db (RT-50): there is no file to fs.watch // for new-repo changes any more, so the 60s hooks-scan poller (pollers.ts) @@ -588,7 +588,7 @@ async function runDaemon(): Promise { } } -// runDaemon() never rejects — it logs fatal and exit(1)s internally on any +// runDaemon() never rejects: it logs fatal and exit(1)s internally on any // boot failure, so this wrapper needs no catch of its own. export async function startDaemon(): Promise { await runDaemon(); diff --git a/lib/daemon/__tests__/home-snapshot.test.ts b/lib/daemon/__tests__/home-snapshot.test.ts index ba0bbba3..33cc9df4 100644 --- a/lib/daemon/__tests__/home-snapshot.test.ts +++ b/lib/daemon/__tests__/home-snapshot.test.ts @@ -1007,7 +1007,7 @@ describe("startHomeSnapshot — state persistence", () => { // ─── boot order: db must open daemon-flavored, never at construction ──────── -describe("startHomeSnapshot — boot order", () => { +describe("startHomeSnapshot (boot order)", () => { test("constructing startHomeSnapshot does not open the state.db singleton before the caller's next await", async () => { const home = mkdtempSync(join(tmpdir(), "rt-home-snapshot-bootorder-")); const origHome = process.env.HOME; @@ -1017,17 +1017,17 @@ describe("startHomeSnapshot — boot order", () => { const stateDbPath = join(home, ".mattstack", "rt", "state.db"); const { fn: execFn } = makeFakeExec(defaultResponders({ statusZ: "?? a.txt\0" })); // No `db` override: this exercises the real getStateDb() singleton, - // matching lib/daemon.ts's module-scope `startHomeSnapshot(...)` call — + // matching lib/daemon.ts's module-scope `startHomeSnapshot(...)` call, // the exact call site that used to open state.db "cli"-flavored before // startDaemon() ever got to openBranchCacheStore(). const { deps } = baseDeps({ exec: execFn, db: undefined }); const handle = startHomeSnapshot(deps); - // Synchronously, right after construction returns — mirroring the + // Synchronously, right after construction returns, mirroring the // module-scope call in lib/daemon.ts, which runs to completion before // startDaemon() (and its openBranchCacheStore() daemon-flavored open) - // is ever reached — no db file may exist yet. + // is ever reached, no db file may exist yet. expect(existsSync(stateDbPath)).toBe(false); await handle.ready; diff --git a/lib/daemon/__tests__/supervision-state.test.ts b/lib/daemon/__tests__/supervision-state.test.ts index dbb6ea50..828ead75 100644 --- a/lib/daemon/__tests__/supervision-state.test.ts +++ b/lib/daemon/__tests__/supervision-state.test.ts @@ -14,7 +14,7 @@ import { import { RT_DIR } from "../../daemon-config.ts"; /** Test-only cleanup mirroring the breadcrumb file's path (production has - * no clear API — the daemon only ever writes or reads it). */ + * no clear API; the daemon only ever writes or reads it). */ function removeBreadcrumbFile(): void { rmSync(join(RT_DIR, "daemon-boot.json"), { force: true }); } diff --git a/lib/daemon/boot-reconcile.ts b/lib/daemon/boot-reconcile.ts index bcb106dc..4010041c 100644 --- a/lib/daemon/boot-reconcile.ts +++ b/lib/daemon/boot-reconcile.ts @@ -36,7 +36,7 @@ async function waitForDeath(pid: number, maxMs: number): Promise { * the `start` command's orphan-detection doesn't fire (e.g. launchd relaunches * us automatically without going through `rt daemon start`). * - * Waits for the old process to actually die rather than a blind sleep — a + * Waits for the old process to actually die rather than a blind sleep: a * daemon that survives the eviction window can still race the new one for * rt.sock/rt.pid (S044). Escalates to SIGKILL if SIGTERM alone doesn't land. */ diff --git a/lib/daemon/events-bus.ts b/lib/daemon/events-bus.ts index c44cd211..bbc914a3 100644 --- a/lib/daemon/events-bus.ts +++ b/lib/daemon/events-bus.ts @@ -68,7 +68,7 @@ function rowToEvent(row: EventRow): BusEvent { * Renames a corrupt events.db out of the way and warns loudly, mirroring * lib/state/db.ts's `quarantine`. events.db is a bounded-retention journal * (sweep() already discards old rows), so losing it entirely on corruption - * is harmless — recreate empty rather than attempt any repair. WAL sidecars + * is harmless: recreate empty rather than attempt any repair. WAL sidecars * are best-effort cleaned since they are meaningless without the main file. */ function quarantineEventsDb(path: string, log: Logger): void { @@ -83,7 +83,7 @@ function quarantineEventsDb(path: string, log: Logger): void { try { renameSync(sidecar, `${sidecar}.corrupt-${stamp}`); } catch { - // sidecar absent — fine, WAL mode doesn't always leave one + // sidecar absent, fine: WAL mode doesn't always leave one } } } diff --git a/lib/daemon/handlers/status.ts b/lib/daemon/handlers/status.ts index c30e1271..fc450856 100644 --- a/lib/daemon/handlers/status.ts +++ b/lib/daemon/handlers/status.ts @@ -22,7 +22,7 @@ import { readSupervisionState } from "../supervision-state.ts"; export function createStatusHandlers(ctx: HandlerContext): HandlerMap { return { "ping": async () => { - // Read here (not once at ctx build time) — a status/status.ts request + // Read here (not once at ctx build time): a status/status.ts request // must see this run's own boot-attempt/failure counters, not whatever // they were when the daemon started. const { bootAttempts, lastReadyAt, recentFailures, lastExit } = readSupervisionState(); diff --git a/lib/daemon/home-snapshot.ts b/lib/daemon/home-snapshot.ts index 476fbee2..3c793ff9 100644 --- a/lib/daemon/home-snapshot.ts +++ b/lib/daemon/home-snapshot.ts @@ -260,7 +260,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle const rawReadSettings = rawDeps.readSettings ?? (() => getSetting("rt.homeSnapshot").value); // Thunk, not a resolved value: module-scope construction (lib/daemon.ts) // must not open state.db before startDaemon() has opened it daemon-flavored - // via openBranchCacheStore — see loadState's call site inside init() below, + // via openBranchCacheStore; see loadState's call site inside init() below, // which is the first place this ever actually gets invoked. const resolveDb = rawDeps.db ? (() => rawDeps.db!) : (() => getStateDb("daemon")); const deps = { @@ -300,7 +300,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle let lastPushError: string | null = null; /** True once `home:push-failed` has been broadcast for the CURRENT unbroken run of push failures — reset to false the moment a push succeeds, so a retry storm broadcasts once, not on every attempt. */ let pushFailureBroadcast = false; - /** Populated in init(), after the is-inside-work-tree check — see resolveDb's comment for why this can't happen at construction time. */ + /** Populated in init(), after the is-inside-work-tree check; see resolveDb's comment for why this can't happen at construction time. */ let firstSeenDirty: Record = {}; let lastLoggedOwnersError: string | null = null; /** Shared dedup key for every "deps.readSettings() itself threw" warn (armWatcher's debounce read, status()) — a settings store that broke after boot and stays broken must warn once, not on every fs event or every `rt home snapshot --status` poll. */ diff --git a/lib/daemon/safe-timers.ts b/lib/daemon/safe-timers.ts index 6ba319c1..b28a5de7 100644 --- a/lib/daemon/safe-timers.ts +++ b/lib/daemon/safe-timers.ts @@ -1,9 +1,9 @@ /** - * lib/daemon/safe-timers.ts — try/catch-wrapped setInterval/setTimeout. + * lib/daemon/safe-timers.ts: try/catch-wrapped setInterval/setTimeout. * * A bare `setInterval`/`setTimeout` callback that throws synchronously * (e.g. a sqlite SQLITE_FULL on a WAL write) becomes an uncaughtException - * with no stack frame back to the timer that scheduled it — Node/Bun's + * with no stack frame back to the timer that scheduled it; Node/Bun's * event loop has nothing to attribute the throw to but the process itself, * so installCrashHandlers treats it as fatal and exits the daemon. Wrapping * the tick converts that crash into a logged warning. diff --git a/lib/daemon/shutdown.ts b/lib/daemon/shutdown.ts index 7f5993a0..29015fcb 100644 --- a/lib/daemon/shutdown.ts +++ b/lib/daemon/shutdown.ts @@ -74,8 +74,8 @@ export interface GracefulExitDeps { * * launchd's KeepAlive.SuccessfulExit=false only respawns on a non-zero exit, * so the code here must distinguish the intentional `shutdown` verb (exit 0, - * stay down) from a bare external signal — pkill, memory pressure, a stray - * script (exit 1, launchd respawns). The sanctioned stop path + * stay down) from a bare external signal (pkill, memory pressure, a stray + * script, exit 1, launchd respawns). The sanctioned stop path * (SMAppService.unregister) doesn't go through this signal path at all, so * exiting non-zero on a bare signal never fights an intended stop. */ diff --git a/lib/daemon/supervision-state.ts b/lib/daemon/supervision-state.ts index 5a753115..7e364cbf 100644 --- a/lib/daemon/supervision-state.ts +++ b/lib/daemon/supervision-state.ts @@ -1,5 +1,5 @@ /** - * lib/daemon/supervision-state.ts — daemon boot/crash history, so + * lib/daemon/supervision-state.ts: daemon boot/crash history, so * `rt daemon status` can report boot-failed/crash-looping and a stuck-phase * breadcrumb for a live-but-silent daemon. * @@ -11,7 +11,7 @@ * `recordCleanExit`, and the kv half of `recordBootFailure`) goes through * `getStateDb("daemon")`, so callers must not reach it until the daemon * has opened its state.db (lib/daemon.ts's `openBranchCacheStore()`). - * `recordBootFailure` is safe to call at any point regardless — its kv + * `recordBootFailure` is safe to call at any point regardless: its kv * write is try/catch'd and silently no-ops if the db isn't open yet. */ @@ -76,7 +76,7 @@ export function recordBootFailure(phase: BootPhase, reason: string): void { setKvValue(NS, KEY_RECENT_FAILURES, next, db()); setKvValue(NS, KEY_LAST_EXIT, { at, kind: "boot-failed", code: 1, reason }, db()); } catch { - // Pre-db failure (or a busy/corrupt state.db) — the breadcrumb file above + // Pre-db failure (or a busy/corrupt state.db): the breadcrumb file above // is the only record this failure gets, and that's fine. } } @@ -122,7 +122,7 @@ function breadcrumbPath(): string { return join(RT_DIR, "daemon-boot.json"); } -/** Never fatal — a breadcrumb is a diagnostic aid, not something boot may fail over. */ +/** Never fatal: a breadcrumb is a diagnostic aid, not something boot may fail over. */ export function writeBreadcrumb(phase: BootPhase): void { try { const breadcrumb: Breadcrumb = { at: Date.now(), pid: process.pid, flavor: daemonFlavor(), phase }; diff --git a/lib/state/__tests__/db.test.ts b/lib/state/__tests__/db.test.ts index 104a115c..2167ec5f 100644 --- a/lib/state/__tests__/db.test.ts +++ b/lib/state/__tests__/db.test.ts @@ -417,7 +417,7 @@ describe("pragma values per flavor", () => { test("getStateDb('daemon') reports busy_timeout 250 even after a default open", () => { const cli = getStateDb(); // opens singleton, cli flavor expect(cli.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 5000 }); - const daemon = getStateDb("daemon"); // same singleton — must not stay at 5000 + const daemon = getStateDb("daemon"); // same singleton, must not stay at 5000 expect(daemon.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); }); }); diff --git a/lib/state/__tests__/source-guards.test.ts b/lib/state/__tests__/source-guards.test.ts index 733a2965..38e18d32 100644 --- a/lib/state/__tests__/source-guards.test.ts +++ b/lib/state/__tests__/source-guards.test.ts @@ -93,7 +93,7 @@ describe("daemon startup opens state.db before serving", () => { test("openBranchCacheStore() precedes both server binds in runDaemon", () => { const src = readFileSync(join(REPO_ROOT, "lib", "daemon.ts"), "utf8"); // startDaemon() itself is now just `await runDaemon()` (see "boot - // failure is fatal" below — the catch-and-exit lives in runDaemon + // failure is fatal" below; the catch-and-exit lives in runDaemon // itself); the real ordered startup sequence this test asserts on // lives in runDaemon() too. const start = src.indexOf("async function runDaemon("); @@ -145,7 +145,7 @@ describe("boot failure is fatal for both fire-and-forget callers", () => { // Both real callers (cli.ts's --daemon entry, this file's own // import.meta.main guard) invoke startDaemon() fire-and-forget, and - // startDaemon() is now just `await runDaemon()` — so the catch-and-exit + // startDaemon() is now just `await runDaemon()`, so the catch-and-exit // MUST live inside runDaemon() itself, or an unhandledRejection could // silently leave the daemon half-up (rt.sock possibly bound, nothing // past the failure ever wired). The booting-gated unhandledRejection diff --git a/lib/state/busy.ts b/lib/state/busy.ts index c7de1f55..fbcb2258 100644 --- a/lib/state/busy.ts +++ b/lib/state/busy.ts @@ -37,7 +37,7 @@ let logHandle: Promise | null = null; /** * True for the bun:sqlite error thrown when a write can't get the lock - * inside busy_timeout — including the SNAPSHOT/RECOVERY variants a + * inside busy_timeout, including the SNAPSHOT/RECOVERY variants a * deferred-BEGIN read-then-write transaction can throw, which busy_timeout * does not retry the way it retries a plain SQLITE_BUSY. */ diff --git a/lib/state/db.ts b/lib/state/db.ts index 34d53c68..eac6bc3d 100644 --- a/lib/state/db.ts +++ b/lib/state/db.ts @@ -391,7 +391,7 @@ function quarantine(path: string): void { /** * Runs each registered legacy importer whose source file exists, inside the * caller's transaction. Returns the list of source paths that were consumed - * (successfully imported OR corrupt/throwing-and-skipped) — all three cases + * (successfully imported OR corrupt/throwing-and-skipped): all three cases * still rename per spec "Migration & contention" ("corrupt = warn + skip"; * brief: "warn + skip + still rename"). Renaming itself happens AFTER COMMIT * (the caller does it), since a filesystem rename cannot participate in the @@ -543,7 +543,7 @@ export function getStateDb(flavor: DbFlavor = "cli"): Database { // A caller asking for a stronger (shorter) contention policy than the // singleton currently holds must not silently inherit whatever flavor // opened it first (e.g. a "cli" 5000ms opener beating the daemon's own - // "daemon" 250ms open) — re-tighten in place rather than reopening. + // "daemon" 250ms open); re-tighten in place rather than reopening. const want = BUSY_TIMEOUT_MS[flavor]; const have = Number((singleton.query("PRAGMA busy_timeout").get() as { timeout?: number } | null)?.timeout ?? 0); if (want < have) singleton.exec(`PRAGMA busy_timeout = ${want};`); From 2852e9d8f012441725d0039d4466ecda515bfecc Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 15:41:21 -0500 Subject: [PATCH 089/142] I5: wire the seams the four lanes documented but could not cross (a) daemon.ts catches ApiPortInUseError from startApiServer and parks with exponential backoff (withApiPortParkRetry); api-server.ts binds via resolveApiPort() instead of the hardcoded API_PORT constant. (b) S055: handlers/status.ts's "repos" command uses listWorktreesAsync instead of the sync execSync-based listWorktrees; git-worktrees.ts and handlers/status.ts drop out of the no-daemon-sync-exec allowlist. (c) S010: worktree:provision validates the resolved branch with validateGitRef before any runGit call (covers divergence() too). (d) S050: freshness.ts's three remote-URL log/error interpolations run through redactCredentials. (e) S022: resolveUserIdAcrossTracking resolves userId for any branches/project-mrs tracked repo regardless of mode, called from cache-refresh.ts before checkAndNotify so poll-only users get self-authored-transition notifications from cycle 1. (f) S073: presence-store.ts's signIn transaction uses .immediate(), matching the chat-store/dm-store/notifier-store siblings. (g) rt.apiPort's settings description now reflects that it's wired. --- lib/__tests__/no-daemon-sync-exec.test.ts | 2 - lib/daemon.ts | 13 +++- lib/daemon/__tests__/api-server-bind.test.ts | 35 +++++++++- .../__tests__/api-server-park-retry.test.ts | 48 ++++++++++++++ .../__tests__/freshness-poll-userid.test.ts | 39 +++++++++++ .../freshness-redact-credentials.test.ts | 22 +++++++ .../__tests__/status-repos-async.test.ts | 64 +++++++++++++++++++ .../__tests__/worktree-handlers.test.ts | 11 ++++ lib/daemon/api-server.ts | 55 +++++++++++++--- lib/daemon/cache-refresh.ts | 12 +++- lib/daemon/freshness.ts | 44 +++++++++++-- lib/daemon/handlers/status.ts | 6 +- lib/daemon/handlers/worktree.ts | 7 ++ lib/state/__tests__/presence-store.test.ts | 18 +++++- lib/state/presence-store.ts | 2 +- .../rt-client/src/settings/registry-defs.ts | 2 +- 16 files changed, 354 insertions(+), 26 deletions(-) create mode 100644 lib/daemon/__tests__/api-server-park-retry.test.ts create mode 100644 lib/daemon/__tests__/freshness-poll-userid.test.ts create mode 100644 lib/daemon/__tests__/freshness-redact-credentials.test.ts create mode 100644 lib/daemon/__tests__/status-repos-async.test.ts diff --git a/lib/__tests__/no-daemon-sync-exec.test.ts b/lib/__tests__/no-daemon-sync-exec.test.ts index 2ba99d37..f5d8c753 100644 --- a/lib/__tests__/no-daemon-sync-exec.test.ts +++ b/lib/__tests__/no-daemon-sync-exec.test.ts @@ -11,8 +11,6 @@ const ALLOWLIST = new Set([ "lib/daemon/boot-reconcile.ts", // Phase 0.6 / S044 (Bun.sleepSync) "lib/state/db.ts", // Phase 0.7 / S072-S073 busy-retry "lib/state/busy.ts", // Phase 0.7 / S072-S073 busy-retry - "lib/git-worktrees.ts", // S055: reached only via handlers/status.ts - "lib/daemon/handlers/status.ts", // S055: the edge into git-worktrees.ts "lib/repo-index.ts", // Phase 5.3 dedup (heal/derive execSync) "lib/repo.ts", // R050 / Phase 5.4 (via handlers/system-processes.ts) "lib/git.ts", // R050 / Phase 5.4 (via repo.ts) diff --git a/lib/daemon.ts b/lib/daemon.ts index 44af086f..3c3a40e9 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -50,7 +50,7 @@ import { runBootIdentityMigration } from "./daemon/boot-migrate.ts"; import { runCapture } from "./subprocess.ts"; import { buildRoutedHandlers } from "./daemon/command-router.ts"; import { startSocketServer } from "./daemon/socket-server.ts"; -import { startApiServer, broadcast } from "./daemon/api-server.ts"; +import { startApiServer, withApiPortParkRetry, broadcast } from "./daemon/api-server.ts"; import { loadCronConfig, startCron } from "./daemon/cron.ts"; import { startPollers } from "./daemon/pollers.ts"; import { startHomeSnapshot } from "./daemon/home-snapshot.ts"; @@ -518,9 +518,16 @@ async function runDaemon(): Promise { // API server first: a failed bind exits fatally (boot-phase catch below), // and binding API before the unix socket means that fatal exit never - // strands a socket-bound zombie behind it. + // strands a socket-bound zombie behind it. ApiPortInUseError is the one + // exception to "fatal": bindApiServerWithRetry has already exhausted its + // own ~3s inner retry, so the holder is a whole other process that may + // take much longer to exit — park-and-retry with backoff instead of + // crash-looping (S043 caller-side contract, docs/daemon-api-auth.md). setPhase("api"); - servers.api = await startApiServer({ handleCommand, log }); + servers.api = await withApiPortParkRetry( + () => startApiServer({ handleCommand, log }), + { sleep: (ms) => Bun.sleep(ms), log }, + ); setPhase("socket"); servers.socket = startSocketServer({ handleCommand, log }); diff --git a/lib/daemon/__tests__/api-server-bind.test.ts b/lib/daemon/__tests__/api-server-bind.test.ts index 44139ac4..f13ed6bf 100644 --- a/lib/daemon/__tests__/api-server-bind.test.ts +++ b/lib/daemon/__tests__/api-server-bind.test.ts @@ -1,6 +1,8 @@ -import { describe, test, expect } from "bun:test"; -import { bindApiServerWithRetry, BIND_RETRY_ATTEMPTS, BIND_RETRY_DELAY_MS, type BindRetryDeps } from "../api-server.ts"; +import { describe, test, expect, afterEach } from "bun:test"; +import type { Server } from "bun"; +import { bindApiServerWithRetry, BIND_RETRY_ATTEMPTS, BIND_RETRY_DELAY_MS, startApiServer, type BindRetryDeps } from "../api-server.ts"; import { ApiPortInUseError } from "../api-server.ts"; +import { setSetting } from "../../settings/write.ts"; function eaddrinuse(): Error { return Object.assign(new Error("EADDRINUSE"), { code: "EADDRINUSE" }); @@ -125,3 +127,32 @@ describe("bindApiServerWithRetry — exhausted retries (S043)", () => { expect(d.probeCalls.length).toBe(0); }); }); + +describe("startApiServer — binds via resolveApiPort() (S043 caller-side wiring)", () => { + let server: Server | undefined; + + afterEach(() => { + server?.stop(true); + server = undefined; + }); + + test("binds to the rt.apiPort setting value, not the hardcoded 9401 default", async () => { + const prevEnv = process.env.RT_API_PORT; + delete process.env.RT_API_PORT; + + // Measure a free port rather than hardcoding one, then release it + // immediately — startApiServer binds it back before anything else can. + const probe = Bun.serve({ port: 0, fetch: () => new Response() }); + const port = probe.port; + probe.stop(true); + + setSetting("rt.apiPort", port, "user"); + const log = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as any; + + server = await startApiServer({ handleCommand: async () => ({ ok: true }), log }); + + expect(server.port).toBe(port); + + if (prevEnv !== undefined) process.env.RT_API_PORT = prevEnv; + }); +}); diff --git a/lib/daemon/__tests__/api-server-park-retry.test.ts b/lib/daemon/__tests__/api-server-park-retry.test.ts new file mode 100644 index 00000000..413427b9 --- /dev/null +++ b/lib/daemon/__tests__/api-server-park-retry.test.ts @@ -0,0 +1,48 @@ +import { describe, test, expect } from "bun:test"; +import { withApiPortParkRetry, ApiPortInUseError } from "../api-server.ts"; + +describe("withApiPortParkRetry (S043 caller-side wiring)", () => { + test("retries with backoff on ApiPortInUseError and returns once start succeeds", async () => { + const sleeps: number[] = []; + const warns: unknown[] = []; + let attempts = 0; + const result = await withApiPortParkRetry( + async () => { + attempts++; + if (attempts < 3) throw new ApiPortInUseError(9401); + return "server"; + }, + { sleep: async (ms) => { sleeps.push(ms); }, log: { warn: (o) => warns.push(o) } }, + ); + expect(result).toBe("server"); + expect(attempts).toBe(3); + expect(sleeps.length).toBe(2); + expect(warns.length).toBe(2); + }); + + test("backs off with an increasing delay on each successive attempt", async () => { + const sleeps: number[] = []; + let attempts = 0; + await withApiPortParkRetry( + async () => { + attempts++; + if (attempts < 4) throw new ApiPortInUseError(9401); + return "server"; + }, + { sleep: async (ms) => { sleeps.push(ms); }, log: { warn: () => {} } }, + ); + expect(sleeps[1]).toBeGreaterThan(sleeps[0]!); + expect(sleeps[2]).toBeGreaterThan(sleeps[1]!); + }); + + test("a non-ApiPortInUseError error is never retried", async () => { + let attempts = 0; + await expect( + withApiPortParkRetry( + async () => { attempts++; throw new Error("state.db open failed"); }, + { sleep: async () => {}, log: { warn: () => {} } }, + ), + ).rejects.toThrow("state.db open failed"); + expect(attempts).toBe(1); + }); +}); diff --git a/lib/daemon/__tests__/freshness-poll-userid.test.ts b/lib/daemon/__tests__/freshness-poll-userid.test.ts new file mode 100644 index 00000000..5e3cbc55 --- /dev/null +++ b/lib/daemon/__tests__/freshness-poll-userid.test.ts @@ -0,0 +1,39 @@ +/** + * S022: userId used to resolve only inside reconcileFreshnessImpl's + * live-mode-only loop, so a poll-only tracked repo (mode: "poll") never + * built a provider, ensureUserId() never ran, getCurrentUserId() stayed + * null forever, and checkAndNotify silently suppressed every + * self-authored transition. Static source checks (matching this file's + * established S048/S049 test style — see freshness-provider-rotation.test.ts — + * since GitLabProvider is an external network client with no test seam). + */ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +const freshnessSrc = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); +const cacheRefreshSrc = readFileSync(resolve(import.meta.dir, "..", "cache-refresh.ts"), "utf8"); + +test("S022: a mode-independent userId resolver is exported from freshness.ts", () => { + expect(freshnessSrc).toMatch(/export async function resolveUserIdAcrossTracking\(/); +}); + +test("S022: the resolver gates on the branches/project-mrs grant, not on live mode", () => { + const fn = freshnessSrc.match(/export async function resolveUserIdAcrossTracking\([\s\S]*?\n\}\n/)?.[0]; + expect(fn).toBeTruthy(); + expect(fn).toMatch(/caches\.has\(["']branches["']\)/); + expect(fn).toMatch(/caches\.has\(["']project-mrs["']\)/); + expect(fn).not.toMatch(/mode\s*!==\s*["']live["']/); +}); + +test("S022: cache-refresh.ts resolves userId before checkAndNotify (cycle-1 fix)", () => { + const resolveIndex = cacheRefreshSrc.indexOf("resolveUserIdAcrossTracking("); + const checkAndNotifyIndex = cacheRefreshSrc.indexOf("checkAndNotify(cache.entries"); + expect(resolveIndex).toBeGreaterThan(-1); + expect(checkAndNotifyIndex).toBeGreaterThan(-1); + expect(resolveIndex).toBeLessThan(checkAndNotifyIndex); +}); + +test("S022: warns once when transitions are suppressed because userId never resolved", () => { + expect(freshnessSrc).toMatch(/userId is unresolved/); +}); diff --git a/lib/daemon/__tests__/freshness-redact-credentials.test.ts b/lib/daemon/__tests__/freshness-redact-credentials.test.ts new file mode 100644 index 00000000..c4228bb3 --- /dev/null +++ b/lib/daemon/__tests__/freshness-redact-credentials.test.ts @@ -0,0 +1,22 @@ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; +import { redactCredentials } from "../redact-credentials.ts"; + +const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + +test("S050: every log/error interpolation of a remote URL runs it through redactCredentials", () => { + expect(src).toMatch(/log\.info\(`remote "\$\{redactCredentials\(remoteUrl\)\}" for \$\{repoName\}/); + expect(src).toMatch(/log\.info\(`could not parse remote "\$\{redactCredentials\(remoteUrl\)\}"/); + expect(src).toMatch(/throw new Error\(`could not parse remote URL "\$\{redactCredentials\(remoteUrl\)\}"`\)/); +}); + +test("S050: no remaining bare ${remoteUrl} interpolation in freshness.ts", () => { + expect(src).not.toMatch(/\$\{remoteUrl\}/); +}); + +test("redactCredentials strips userinfo from a credentialed URL", () => { + expect(redactCredentials("https://oauth2:glpat-XXXX@gitlab.example.com/a/b.git")).toBe( + "https://[redacted]@gitlab.example.com/a/b.git", + ); +}); diff --git a/lib/daemon/__tests__/status-repos-async.test.ts b/lib/daemon/__tests__/status-repos-async.test.ts new file mode 100644 index 00000000..37283af0 --- /dev/null +++ b/lib/daemon/__tests__/status-repos-async.test.ts @@ -0,0 +1,64 @@ +/** + * S055: "repos" used the sync `listWorktrees` (lib/git-worktrees.ts, + * execSync) on the daemon's event loop. Swapped for the async + * `listWorktreesAsync` (lib/worktree/git-async.ts, already used elsewhere + * in the daemon). + */ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { execSync } from "child_process"; +import { mkdtempSync, realpathSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { createStatusHandlers } from "../handlers/status.ts"; + +let tmpRoot: string; + +beforeEach(() => { + tmpRoot = realpathSync(mkdtempSync(join(tmpdir(), "rt-status-repos-"))); +}); + +afterEach(() => { + try { rmSync(tmpRoot, { recursive: true, force: true }); } catch { /* */ } +}); + +function initRepo(path: string): void { + execSync(`git init -q "${path}"`); + writeFileSync(join(path, "README"), "x"); + execSync(`git -C "${path}" add . && git -C "${path}" -c user.email=t@t -c user.name=t commit -q -m init`); +} + +function fakeCtx(repos: Record): any { + return { + startedAt: 123, + identity: { flavor: "dev", version: "source", sourceRev: "abc1234", startedAt: 123 }, + watchedConfigs: new Map(), + cache: { entries: {} }, + portCacheRef: { ports: [], updatedAt: null }, + repoIndex: () => repos, + }; +} + +describe("status handlers — repos (S055 async worktree listing)", () => { + test("lists worktrees with a branch, omitting detached ones", async () => { + const repo = mkdtempSync(join(tmpRoot, "repo-")); + initRepo(repo); + const linked = join(tmpRoot, "linked"); + execSync(`git -C "${repo}" worktree add -q "${linked}" -b feat/x`); + + const h = createStatusHandlers(fakeCtx({ myrepo: repo })); + const res = (await h["repos"]!({}, undefined as any)) as any; + + expect(res.ok).toBe(true); + const worktrees = res.data.repos.myrepo.worktrees; + expect(worktrees.map((w: any) => w.path).sort()).toEqual([linked, repo].sort()); + expect(worktrees.every((w: any) => typeof w.branch === "string" && w.branch.length > 0)).toBe(true); + }); + + test("a repo whose git command fails (bad repoPath) yields no worktrees, not a throw", async () => { + const notARepo = mkdtempSync(join(tmpRoot, "not-a-repo-")); + const h = createStatusHandlers(fakeCtx({ broken: notARepo })); + const res = (await h["repos"]!({}, undefined as any)) as any; + expect(res.ok).toBe(true); + expect(res.data.repos.broken.worktrees).toEqual([]); + }); +}); diff --git a/lib/daemon/__tests__/worktree-handlers.test.ts b/lib/daemon/__tests__/worktree-handlers.test.ts index c95cfb7b..7b68a7a9 100644 --- a/lib/daemon/__tests__/worktree-handlers.test.ts +++ b/lib/daemon/__tests__/worktree-handlers.test.ts @@ -235,6 +235,17 @@ describe("worktree:provision", () => { expect(res.error).toBe("repo-unknown"); }); + test("S010: refuses a branch that git would parse as an option, before any git call", async () => { + const repo = makeRepo(); + const { h, events } = makeHandlers({ [repoName]: repo }); + + const res: any = await h["worktree:provision"]!({ repoName, branch: "--upload-pack=touch /tmp/x" }); + + expect(res.ok).toBe(false); + expect(res.error).toContain("unsafe git ref"); + expect(events.length).toBe(0); + }); + test("Hard cutover: a bare legacy name refuses even when it IS a registered repoIndex key", async () => { const repo = makeRepo(); // Registered under a non-identity key on purpose — proves the rejection diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index cfe54790..fd4b5d66 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -9,7 +9,7 @@ import type { Server, ServerWebSocket } from "bun"; import type { Logger } from "pino"; -import { API_PORT } from "../daemon-config.ts"; +import { API_PORT, resolveApiPort } from "../daemon-config.ts"; import { needsToken, tokenOk, getApiToken, resolveOriginTrust } from "./api-auth.ts"; import { getAggregatedConnection } from "./freshness.ts"; import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; @@ -262,7 +262,7 @@ export const BIND_RETRY_DELAY_MS = 500; * ApiPortInUseError instead of the bare EADDRINUSE Error, so a caller can * tell "give up cleanly" apart from "the bind function itself is broken". */ -export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDeps): Promise { +export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDeps, port: number = API_PORT): Promise { const probe = deps.probePortHolder ?? defaultProbePortHolder; for (let attempt = 1; ; attempt++) { try { @@ -271,23 +271,60 @@ export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDe const isAddrInUse = err instanceof Error && (err as NodeJS.ErrnoException).code === "EADDRINUSE"; if (!isAddrInUse) throw err; if (attempt >= BIND_RETRY_ATTEMPTS) { - const holder = await probe(API_PORT).catch((probeErr) => `lsof failed: ${String(probeErr)}`); - deps.log.warn({ port: API_PORT, holder }, "api port still in use after retries; giving up bind (the daemon should park and retry with backoff rather than crash-loop)"); - throw new ApiPortInUseError(API_PORT); + const holder = await probe(port).catch((probeErr) => `lsof failed: ${String(probeErr)}`); + deps.log.warn({ port, holder }, "api port still in use after retries; giving up bind (the daemon should park and retry with backoff rather than crash-loop)"); + throw new ApiPortInUseError(port); } - deps.log.warn({ attempt, port: API_PORT }, "api port in use, retrying — another daemon is likely still shutting down"); + deps.log.warn({ attempt, port }, "api port in use, retrying — another daemon is likely still shutting down"); await deps.sleep(BIND_RETRY_DELAY_MS); } } } +/** + * Backoff base/cap for {@link withApiPortParkRetry}'s outer loop. Distinct + * from BIND_RETRY_* (bindApiServerWithRetry's own ~3s inner retry, already + * exhausted before an ApiPortInUseError ever reaches here): this loop + * assumes the holder is a whole other process that may take much longer + * than 3s to exit, so it backs off further between each full re-attempt. + */ +const PARK_RETRY_BASE_MS = 3_000; +const PARK_RETRY_MAX_MS = 60_000; + +export interface ParkRetryDeps { + sleep: (ms: number) => Promise; + log: { warn: (o: unknown, m: string) => void }; +} + +/** + * Wraps a `startApiServer`-shaped call: on `ApiPortInUseError` (bind retries + * already exhausted), logs and waits with exponential backoff, then calls + * `start` again — indefinitely, never giving up — instead of letting the + * error reach the daemon's top-level crash path (S043 caller-side contract, + * docs/daemon-api-auth.md). Any other error propagates immediately: that is + * a genuine misconfiguration, not a transient port squat. + */ +export async function withApiPortParkRetry(start: () => Promise, deps: ParkRetryDeps): Promise { + for (let attempt = 1; ; attempt++) { + try { + return await start(); + } catch (err) { + if (!(err instanceof ApiPortInUseError)) throw err; + const delayMs = Math.min(PARK_RETRY_BASE_MS * 2 ** (attempt - 1), PARK_RETRY_MAX_MS); + deps.log.warn({ attempt, port: err.port, delayMs }, "api server port still in use; parked, retrying with backoff"); + await deps.sleep(delayMs); + } + } +} + export async function startApiServer(deps: ApiServerDeps): Promise> { const { handleCommand, log } = deps; apiServerLog = log; const apiToken = getApiToken(); + const port = resolveApiPort(); const server = await bindApiServerWithRetry(() => Bun.serve({ - port: API_PORT, + port, // Bind to loopback only — never expose the control surface on the LAN. hostname: "127.0.0.1", // Raise the request idle timeout off Bun's 10s default so long-lived @@ -429,8 +466,8 @@ export async function startApiServer(deps: ApiServerDeps): Promise> // Broadcast clients are read-only; inbound frames are ignored. }, }, - }), { sleep: (ms) => Bun.sleep(ms), log }); + }), { sleep: (ms) => Bun.sleep(ms), log }, port); - log.info({ port: API_PORT }, "api server listening"); + log.info({ port }, "api server listening"); return server; } diff --git a/lib/daemon/cache-refresh.ts b/lib/daemon/cache-refresh.ts index 2771c28b..93c9f7af 100644 --- a/lib/daemon/cache-refresh.ts +++ b/lib/daemon/cache-refresh.ts @@ -16,7 +16,7 @@ import type { Logger } from "pino"; import type { PortCacheRef, RepoIndex } from "./handlers/types.ts"; import type { BranchCacheStore } from "../state/index.ts"; import { checkAndNotify } from "../notifier.ts"; -import { getCurrentUserId } from "./freshness.ts"; +import { getCurrentUserId, resolveUserIdAcrossTracking } from "./freshness.ts"; import { loadRepoTracking, grants } from "../repo-tracking.ts"; import { syncProjectMRs } from "./project-sync.ts"; import { getProjectMRs } from "./project-mrs-store.ts"; @@ -221,6 +221,16 @@ export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise { return userId; } +let userIdSuppressedWarned = false; + +/** + * S022: reconcileFreshnessImpl only ever builds a provider (and thus only + * ever calls ensureUserId) for repos in live mode, so a poll-only tracked + * user's getCurrentUserId() stayed null forever and checkAndNotify silently + * suppressed every self-authored transition. Called from cache-refresh.ts + * BEFORE its checkAndNotify (reconcileSubscriptions/reconcileFreshnessImpl + * runs after, so relying on it alone would leave the first cache-refresh + * cycle still passing null). Gates on the `branches`/`project-mrs` grant, + * never on mode, and is a no-op once userIdResolved (ensureUserId's own + * guard) or once every candidate repo has been tried this cycle. + */ +export async function resolveUserIdAcrossTracking( + repoIndex: Record, + tracking: RepoTracking, +): Promise { + if (userIdResolved) return; + for (const [repoName, repoPath] of Object.entries(repoIndex)) { + const g = grants(tracking, repoName); + if (g.mode === "off") continue; + if (!g.caches.has("branches") && !g.caches.has("project-mrs")) continue; + if (!existsSync(repoPath)) continue; + const provider = await ensureProvider(repoName, repoPath); + if (!provider) continue; + await ensureUserId(); + if (userIdResolved) return; + } + if (!userIdResolved && !userIdSuppressedWarned) { + userIdSuppressedWarned = true; + log.warn("userId is unresolved; self-authored MR transitions are being suppressed (no gitlabToken, or token validation failed)"); + } +} + export function getSelfUsername(): string | null { return selfUsername; } export async function resolveSelfUsername(repoName: string, repoPath: string): Promise { @@ -307,7 +343,7 @@ export async function getRepoContext( } const remote = parseRemoteUrl(remoteUrl); if (!remote) { - throw new Error(`could not parse remote URL "${remoteUrl}"`); + throw new Error(`could not parse remote URL "${redactCredentials(remoteUrl)}"`); } provider = makeProvider(remote.host, secrets.gitlabToken); providers.set(repoName, { provider, token: secrets.gitlabToken }); diff --git a/lib/daemon/handlers/status.ts b/lib/daemon/handlers/status.ts index fc450856..19db8227 100644 --- a/lib/daemon/handlers/status.ts +++ b/lib/daemon/handlers/status.ts @@ -14,7 +14,7 @@ import { existsSync, readdirSync } from "fs"; import type { HandlerContext, HandlerMap } from "./types.ts"; import type { PortEntry } from "../../port-scanner.ts"; -import { listWorktrees } from "../../git-worktrees.ts"; +import { listWorktreesAsync } from "../../worktree/git-async.ts"; import { drainNotifications, peekNotifications } from "../../notifier.ts"; import { getFreshnessSnapshot } from "../freshness.ts"; import { readSupervisionState } from "../supervision-state.ts"; @@ -115,7 +115,9 @@ export function createStatusHandlers(ctx: HandlerContext): HandlerMap { for (const [repoName, repoPath] of Object.entries(repos)) { if (!existsSync(repoPath)) continue; // Detached worktrees have no branch — omit them from the listing. - const worktrees = listWorktrees(repoPath).filter((w) => w.branch); + const worktrees = ((await listWorktreesAsync(repoPath)) ?? []).filter( + (w): w is { path: string; branch: string } => Boolean(w.branch), + ); detailed[repoName] = { path: repoPath, worktrees }; } diff --git a/lib/daemon/handlers/worktree.ts b/lib/daemon/handlers/worktree.ts index bb98e53e..b4c3f223 100644 --- a/lib/daemon/handlers/worktree.ts +++ b/lib/daemon/handlers/worktree.ts @@ -32,6 +32,7 @@ import { realpathSync, rmSync } from "fs"; import { join } from "path"; import { parseIdentity } from "../../settings/identity.ts"; +import { validateGitRef } from "../git-ref-validation.ts"; import type { HandlerContext, HandlerMap } from "./types.ts"; import { findByBranch, @@ -295,6 +296,12 @@ export function createWorktreeHandlers( return { ok: false, error: "branch-unresolved" }; } + // S010: a branch that git would parse as an option (e.g. + // "--upload-pack=...") must never reach a runGit call, including + // divergence()'s below — both read this same `branch`. + const refCheck = validateGitRef(branch); + if (!refCheck.ok) return { ok: false, error: refCheck.error }; + const attached = findByBranch(trees, branch); if (attached.length > 1) return { ok: false, error: "branch-duplicated" }; if (attached.length === 1) return { ok: false, error: `branch-attached:${attached[0]!.name}` }; diff --git a/lib/state/__tests__/presence-store.test.ts b/lib/state/__tests__/presence-store.test.ts index a875d2b6..2d9e1521 100644 --- a/lib/state/__tests__/presence-store.test.ts +++ b/lib/state/__tests__/presence-store.test.ts @@ -6,8 +6,9 @@ * exercise the dual-write and room-default wiring those tests cover. */ import { expect, test } from "bun:test"; +import { readFileSync } from "fs"; import { tmpdir } from "os"; -import { join } from "path"; +import { join, resolve } from "path"; import { openStateDb } from "../db.ts"; import { armMember, clearAllArmed, disarmMember, joinRoom, listMembers, touchMember } from "../chat-store.ts"; import { assertSessionOwnsHandle, assertSessionSignedIn, buddyStatus, presenceForSession, presenceThresholds, prunePresence, pulseSession, signIn, signOut } from "../presence-store.ts"; @@ -100,6 +101,21 @@ test("assertSessionOwnsHandle throws only on a mismatched signed handle", () => expect(() => assertSessionOwnsHandle("x", undefined, db)).not.toThrow(); // no session id offered, no enforcement }); +test("S073: signIn's read-then-write transaction uses .immediate() (BEGIN IMMEDIATE), not a deferred BEGIN", () => { + // A plain db.transaction()'s deferred BEGIN lets signIn's own reads + // (prunePresence, SELECT_PRESENCE_BY_SESSION_SQL) open a snapshot before + // any write; a commit by another connection in that window turns the + // eventual write into an unretryable SQLITE_BUSY_SNAPSHOT that the + // flavor's busy_timeout cannot absorb. .immediate() takes the write lock + // at BEGIN, so contention surfaces as an ordinary, retryable SQLITE_BUSY + // instead (matching the chat-store.ts/dm-store.ts/notifier-store.ts + // siblings already converted for the same reason). + const src = readFileSync(resolve(import.meta.dir, "..", "presence-store.ts"), "utf8"); + const runIndex = src.indexOf("const run = db.transaction("); + expect(runIndex).toBeGreaterThan(-1); + expect(src.indexOf("return run.immediate();", runIndex)).toBeGreaterThan(runIndex); +}); + test("assertSessionSignedIn throws when the session's row is gone", () => { const db = fresh(); expect(() => assertSessionSignedIn("ghost", db)).toThrow(/handle reclaimed/); diff --git a/lib/state/presence-store.ts b/lib/state/presence-store.ts index c665b86f..530fcb53 100644 --- a/lib/state/presence-store.ts +++ b/lib/state/presence-store.ts @@ -308,7 +308,7 @@ export function signIn( return { handle, baseHandle, reclaimed: winnerRow !== null }; }); - return run(); + return run.immediate(); } const NAMES_KV_NS = "chat"; diff --git a/packages/rt-client/src/settings/registry-defs.ts b/packages/rt-client/src/settings/registry-defs.ts index 84002379..8a00016b 100644 --- a/packages/rt-client/src/settings/registry-defs.ts +++ b/packages/rt-client/src/settings/registry-defs.ts @@ -203,7 +203,7 @@ export const REGISTRY: readonly SettingDef[] = [ default: 9401, merge: "replace", migrated: true, - description: "TCP port for the daemon's local HTTP/WS API. Not yet consumed at bind time (pending api-server wiring); today only the RT_API_PORT env var overrides the default 9401.", + description: "TCP port for the daemon's local HTTP/WS API. Escape hatch when 9401 is held: RT_API_PORT env wins, then this setting, then 9401 (lib/daemon-config.ts resolveApiPort(), read at bind time by lib/daemon/api-server.ts).", }, { key: "rt.hooks", From f395575e00fe2548cb682cbd43580d1f693ffd39 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 15:51:59 -0500 Subject: [PATCH 090/142] e2e: rewrite the 3 API-port-squat daemon tests for I5(a)'s park-retry They asserted the pre-fix crash-on-EADDRINUSE behavior (fatal exit, boot-failed/crash-looping). I5(a) makes this recoverable instead: withApiPortParkRetry parks and retries with backoff rather than crashing, so the daemon now boots successfully once the squatted port frees. Rewritten to assert the new contract: alive (not exited) while parked, no rt.sock/rt.pid until bind succeeds, status never falsely reports "running" while parked, and both recover once the port frees. --- e2e/tests/daemon.test.ts | 106 ++++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 34 deletions(-) diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index 4f90640a..a7dc3cdb 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -21,62 +21,100 @@ function freePort(): number { } describe("fatal boot", () => { - test("daemon boot with API port already bound exits non-zero and leaves no stale rt.pid", async () => { + // These three used to assert a crash (S043 pre-fix: EADDRINUSE on the API + // port took the daemon down the fatal boot-failed path). The integration + // job's I5(a) wiring (lib/daemon.ts's withApiPortParkRetry around + // startApiServer, docs/daemon-api-auth.md's S043 caller-side contract) + // makes this recoverable instead: the daemon parks and retries with + // backoff rather than crashing, so it now boots successfully once the + // squatted port frees. + test("daemon parks (does not crash) while the API port is squatted, and boots once it frees", async () => { const { path: home, cleanup } = createTestHome(); - // Bind the API port inside the isolated HOME so the daemon cannot. + const bunDir = join(process.execPath, ".."); const port = 9411; + const rtDir = join(home, ".mattstack", "rt"); const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + let daemon: ReturnType | undefined; try { - const result = await rt(["--daemon"], { home, env: { RT_API_PORT: String(port) } }); + daemon = Bun.spawn([RT_BINARY, "--daemon"], { + env: { + HOME: home, + PATH: `${join(RT_BINARY, "..")}:${bunDir}:/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin`, + TERM: "xterm-256color", + RT_SKIP_SETUP: "1", + CI: "true", + RT_API_PORT: String(port), + }, + stdout: "pipe", + stderr: "pipe", + }); - expect(result.exitCode).not.toBe(0); - expect(existsSync(join(home, ".mattstack", "rt", "rt.pid"))).toBe(false); - } finally { - squatter.stop(true); - cleanup(); - } - }, 60_000); + // Give bindApiServerWithRetry's own ~3s inner retry a full cycle to + // exhaust and reach the outer park-retry loop; it must still be alive + // (parked, not crashed) and must not have written rt.sock/rt.pid yet + // (neither server has bound). + await Bun.sleep(4_000); + expect(daemon.exitCode).toBeNull(); + expect(existsSync(join(rtDir, "rt.sock"))).toBe(false); + expect(existsSync(join(rtDir, "rt.pid"))).toBe(false); - test("API-bind failure leaves neither rt.sock nor rt.pid", async () => { - const { path: home, cleanup } = createTestHome(); - // A different port than the sibling squatter test above, so parallel - // test files can never collide on the same bound TCP port. - const port = 9412; - const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); - try { - const result = await rt(["--daemon"], { home, env: { RT_API_PORT: String(port) } }); + squatter.stop(true); - expect(result.exitCode).not.toBe(0); - // The API bind (now first) fails before the socket ever binds, so a - // fatal exit must strand neither file. - expect(existsSync(join(home, ".mattstack", "rt", "rt.sock"))).toBe(false); - expect(existsSync(join(home, ".mattstack", "rt", "rt.pid"))).toBe(false); + await waitForSocket(join(rtDir, "rt.sock"), 40_000); + expect(daemon.exitCode).toBeNull(); + expect(existsSync(join(rtDir, "rt.pid"))).toBe(true); } finally { squatter.stop(true); + try { daemon?.kill(); } catch { /* already gone */ } + await daemon?.exited; cleanup(); } }, 60_000); - test("API-bind failure surfaces as boot-failed/crash-looping via rt daemon status --json", async () => { + test("daemon status --json never claims 'running' while parked on a squatted API port, and does once it recovers", async () => { const { path: home, cleanup } = createTestHome(); - // A different port than the other API-bind-failure tests above, so - // parallel test files can never collide on the same bound TCP port. - const port = 9413; + const bunDir = join(process.execPath, ".."); + // A different port than the sibling test above, so parallel test files + // can never collide on the same bound TCP port. + const port = 9412; + const rtDir = join(home, ".mattstack", "rt"); const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + let daemon: ReturnType | undefined; try { // `rt daemon status` short-circuits to "not installed" before it ever - // reaches the boot-failed/crash-looping classification, install first. + // reaches a liveness classification, install first. await rt(["daemon", "install"], { home }); - const boot = await rt(["--daemon"], { home, env: { RT_API_PORT: String(port) } }); - expect(boot.exitCode).not.toBe(0); + daemon = Bun.spawn([RT_BINARY, "--daemon"], { + env: { + HOME: home, + PATH: `${join(RT_BINARY, "..")}:${bunDir}:/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin`, + TERM: "xterm-256color", + RT_SKIP_SETUP: "1", + CI: "true", + RT_API_PORT: String(port), + }, + stdout: "pipe", + stderr: "pipe", + }); + + await Bun.sleep(4_000); + expect(daemon.exitCode).toBeNull(); + + const parked = await rt(["daemon", "status", "--json"], { home, env: { RT_API_PORT: String(port) } }); + expect(parked.exitCode).toBe(0); + expect(JSON.parse(parked.stdout).state).not.toBe("running"); + + squatter.stop(true); + await waitForSocket(join(rtDir, "rt.sock"), 40_000); - const status = await rt(["daemon", "status", "--json"], { home }); - expect(status.exitCode).toBe(0); - const parsed = JSON.parse(status.stdout); - expect(["boot-failed", "crash-looping"]).toContain(parsed.state); + const recovered = await rt(["daemon", "status", "--json"], { home, env: { RT_API_PORT: String(port) } }); + expect(recovered.exitCode).toBe(0); + expect(JSON.parse(recovered.stdout).state).toBe("running"); } finally { squatter.stop(true); + try { daemon?.kill(); } catch { /* already gone */ } + await daemon?.exited; cleanup(); } }, 60_000); From 17927204efec8ebc123806480cc4ae314608eca1 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 16:20:22 -0500 Subject: [PATCH 091/142] spec: p2-health daemon health model design Co-Authored-By: Claude Opus 4.8 (1M context) --- .../specs/2026-08-28-p2-health-design.md | 213 ++++++++++++++++++ 1 file changed, 213 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-28-p2-health-design.md diff --git a/docs/superpowers/specs/2026-08-28-p2-health-design.md b/docs/superpowers/specs/2026-08-28-p2-health-design.md new file mode 100644 index 00000000..299bcdee --- /dev/null +++ b/docs/superpowers/specs/2026-08-28-p2-health-design.md @@ -0,0 +1,213 @@ +# Daemon health you can see (Phase 2 / RT-79) + +Design record for Phase 2 of the rt daemon stability roadmap. Makes a +non-author user able to tell, from `rt daemon status`, the tray dot, and +`/api/status`, whether the daemon is serving, degraded, stalled, or dead, and +why. Builds on Phase 0 (the status classifier in `lib/daemon-status.ts`, the +`daemon-supervision` kv namespace, the pre-db breadcrumb file). Covers audit +findings R011, R012, R003, R004, S031, S032, S033, R005, R008, R021. + +## Health model + +A single server-computed verdict every surface reads, replacing three +independent client-side classifications (CLI, Swift tray, none in `/api/status`). + +`lib/daemon/health.ts` exports a pure `computeHealth(inputs): HealthSnapshot`: + +``` +HealthSnapshot = { + level: "ok" | "degraded" | "unhealthy", + reasons: string[], // one subsystem-prefixed line per trigger + metrics: { rss, heapUsed, external, uptimeMs, wsClients, watchers }, + // watchers = fs.watch handle count (watchedConfigs.size) + eventLoop: { maxLagMs, lastStallAt, lastStallCmd, stalls }, +} +``` + +Level is severity-ordered; unhealthy wins over degraded. + +- **unhealthy** if any: logger degraded (ENOSPC, from S032); event loop currently + stalled (heartbeat/monitor); restart storm (Phase 0 `isCrashLooping`, or >= N + supervision failures in the last hour); disk free under a hard floor. +- **degraded** if any: a freshness watcher is `degraded` + (`getFreshnessSnapshot()`); the last refresh cycle had `failedRepos > 0` or + `enrichErrors > 0`; last successful refresh age > 2x the refresh interval; rss + over a soft threshold or grown > 50% in the last hour; event-loop `maxLagMs` + over the lag threshold within the window; recovered-error rate over a small + threshold in the window; disk free under a soft floor. +- **ok** otherwise. + +`reasons` name the failing subsystem so the operator knows where to look, e.g. +`"refresh: 3 repos failing (auth?)"`, `"event-loop: stalled 8s"`, +`"logging: disabled (ENOSPC)"`, `"disk: 180MB free"` (R011). `metrics` gives the +memory/handle/uptime numbers a leak hunt needs (R012). + +A daemon-side adapter (`buildHealthSnapshot(ctx)`) gathers the inputs from `ctx`, +the loop monitor, the logger handle, Phase 0 supervision, and an optional +`fs.statfs` probe, then calls the pure function. Inputs that need new tracking: +`refreshStatusRef` grows from `{ lastRefreshAt }` to also carry the last cycle's +`{ lastSuccessAt, failedRepos, enrichErrors }` (populated in `cache-refresh.ts`); +`wsClients.size` is exposed from `api-server.ts` to the adapter. **Deferred behind +a typed hook** (V1 does not wire them): SQLITE_BUSY-skip and critical-write-failure +counters. `HealthInputs` declares the fields so they slot in later without a shape +change. + +### Where each surface reads it + +- `status` and `tray:status` verbs (`lib/daemon/handlers/status.ts`) gain + additive `health`, `metrics`, `eventLoop` blocks. `/api/status` aliases + `tray:status`, so it carries the verdict. +- `ping` gains `health.level`, `version` (`ctx.identity.version`), and the + heartbeat `seq` (all cheap). +- `rt daemon status` (`commands/daemon.ts` `statusLines`) renders `level` + + `reasons` and metrics/eventLoop lines on the running branch; on the + alive-but-not-serving branch it prints `maxLag` / last stall instead of today's + "likely mid-sync" guess (R003). +- **Swift tray is deferred** (brief constraint): no `rt-tray/` edits. Documented + contract for the follow-up: read `data.health.level` -> green/orange/red and + `data.health.reasons[0]` as `statusText`; the current client derivation + (pendingNotifications + two-miss) becomes the fallback when `health` is absent. + +## Heartbeat and stall detection (R003) + +`lib/daemon/loop-monitor.ts`: a ~250ms `setInterval` measuring drift +(`actualElapsed - expected`) into a preallocated `LoopStats` object. The tick is +allocation-free (no per-tick closures/objects) and the timer is `unref()`'d so it +never keeps the process alive. It maintains `{ lagMs, maxLagMs, stalls, +lastStallAt, lastStallCmd }`; on drift > 1s it increments `stalls`, records +`lastStallCmd` (a module `currentCmd` set by `handleCommand`), and logs one warn. + +**Heartbeat is a file, not kv** (ratified): every ~2s the monitor writes +`{ at, seq }` (seq monotonic) to `RT_DIR/daemon-heartbeat.json` via atomic rename +(write temp + `renameSync`), the same db-free pattern as Phase 0's breadcrumb. +Rationale: state.db is the WAL every CLI contends on, and a stalled or +lock-wedged daemon is exactly when it is least readable. `lib/daemon/heartbeat-file.ts` +owns `writeHeartbeat`/`readHeartbeat` (missing/corrupt -> null). + +Cross-process detection extends `lib/daemon-status.ts`: `classifyDaemonStatus` +takes an optional `heartbeat: { at, seq } | null` plus a stale threshold. In the +alive-not-serving branch, when `breadcrumb.phase === "ready"` and +`now - heartbeat.at` exceeds the threshold, the detail becomes a new `"stalled"` +(with age) instead of `"wedged"`. The `alive-not-serving` detail union grows to +`"booting" | "wedged" | "quarantined" | "stalled"`. `commands/daemon.ts` reads the +heartbeat file only when the pid probe is already needed (`needsPidProbe`), and +`statusLines` prints "event loop stalled Ns ago". + +## Log level and growth policy + +- **rt.logLevel** (R004): new registry row, `type: "string"`, + `scopes: ["machine", "user"]`, `default: "info"`, `merge: "replace"`, following + `docs/settings-architecture.md`'s checklist exactly as `rt.apiPort` did (add the + row, `cd packages/rt-client && bun run build` so the dist-freshness test stays + green). `getDaemonLogger` resolves `level = RT_LOG_LEVEL env ?? getSetting("rt.logLevel") ?? "info"` + (env wins, mirroring `resolveApiPort`). +- **Live control** (R004): a `daemon:log-level` IPC verb sets `logger.level` + at runtime and logs the change; `rt daemon log-level ` dispatches it. The + new command registers in `command-tree-def.ts` and `lib/module-registry.ts`, and + its required positional `level` (a select over pino levels) declares + `omitBehavior: "picker"` so `bun run picker:check` stays green. +- **Slow-command visibility** (R004): `handleCommand` logs successful commands at + `info` when `durationMs > 2s` (else `debug`, as today), so latency outliers are + visible at the default level. +- **Growth cap** (S031): pino-roll gets `size: "50m"` beside `limit: { count: 14 }` + (bounds within-day growth independent of the daily/age sweep). Per-(cmd,error) + suppression in `handleCommand`: a Map keyed `${cmd}|${errorKey}` tracking + `{ count, lastLoggedAt }`. **Guardrail:** always log the first occurrence + immediately; within 60s of the last logged line, increment silently; at >= 60s + emit one line carrying `suppressed: ` and reset. `pruneLogs` takes an + `onError` callback so the janitor's readdir/unlink failures log at warn instead + of being swallowed. + +## Logger resilience and stderr noise + +- **Stream error listener** (S032): `createDaemonLogger` adds + `stream.on("error", ...)` that sets a `loggerDegraded` flag and does a raw + `fs.writeSync(2, ...)`, so a full-disk write never throws out of a log call. The + handle exposes `loggerDegraded` (feeds health -> unhealthy). The + `uncaughtException` / `unhandledRejection` handler bodies are wrapped in + try/catch with a raw-write fallback that still calls `process.exit(1)` (Phase 0's + boot-vs-steady-state semantics preserved). +- **stderr demotion** (S033, R005): the stderr interceptor logs at `warn` with + `source: "stderr"`, escalating to `error` only for known panic/exception + prefixes. `unhandledRejection` and recovered errors increment a process-wide + counter exposed in `health`/`metrics`. +- **Resolver warn sink** (S033): `packages/rt-client`'s resolver + (`resolve.ts` `warnInvalid` and siblings) takes an injectable warn sink + defaulting to `console.warn` (CLI/test behavior unchanged). The daemon binds a + sink that dedupes per `(key, scope, reason)` to `log.warn`, so a hot-path + `getSetting` on a disallowed-scope key warns once, not every tick. + +## Request attribution + +- **reqId + caller** (R008): `handleCommand` mints a short request id per request + and logs `{ reqId, cmd, caller, durationMs }` on every seam line; `ok:false` + envelopes echo `reqId`. Caller comes from an `X-RT-Client` header (REST) or a + `_client` field on the socket frame, formatted `/` (default + `unknown`). On reject/fail, log a redacted payload digest: top-level keys plus + the whitelisted `repo`/`branch`/`iid`/`room` when present. rt's own transport + (`lib/daemon-client.ts`) and `packages/rt-client`'s transport both send the tag. +- **Unknown-command envelope** (R021): the `routeCommand` default returns + `{ ok: false, code: "unknown-command", error, version }`. Both transports map + `code === "unknown-command"` to distinct text ("daemon at version X does not know + ; restart or upgrade rt"). `ping` optionally exposes the command-name list + for pre-checks. + +**rt-client blast radius:** the `packages/rt-client` edits (registry row, warn +sink, caller tag, unknown-command text) ship as source + a `dist` rebuild; the +version is **not** bumped and the package is **not** published (publishing is +release-class, from `main` only). The estate-wide rollout to board/gitq/console +rides the next release from `main`. + +## Constraints and invariants + +- No `SCHEMA_VERSION` bump. The only new persisted state is the heartbeat file + (`RT_DIR/daemon-heartbeat.json`); everything else is computed live or reuses the + Phase 0 `daemon-supervision` kv namespace. +- `rt.logLevel` goes through the settings registry per the checklist; rebuild + rt-client `dist`; no version bump, no publish. +- No `rt-tray/` edits; the tray read contract above is a documented follow-up. +- Never start a daemon or run `dist/rt` except under `env -i HOME=`. +- Do not touch the p6-portability-owned files, nor the module-scope + `resolveUserPath()` call in `lib/daemon.ts` (p6 makes it awaited-async). + +## Components + +**New:** `lib/daemon/health.ts` (pure `computeHealth` + `HealthInputs`), +`lib/daemon/loop-monitor.ts`, `lib/daemon/heartbeat-file.ts`, the +`buildHealthSnapshot` adapter, the `rt daemon log-level` command handler. + +**Changed:** `lib/daemon.ts` (handleCommand reqId/caller/suppression/currentCmd, +loop-monitor + metrics-logger wiring), `lib/daemon/handlers/status.ts` +(health/metrics/eventLoop in status + tray:status + ping; unknown-command code), +`lib/daemon/handlers/types.ts` (extend `refreshStatusRef`, expose health inputs), +`lib/daemon/cache-refresh.ts` (populate the extended ref), +`lib/daemon/api-server.ts` (read `X-RT-Client`, expose `wsClients`), +`lib/daemon-logger.ts` (level from setting, stream error listener, stderr +demotion, crash-handler wrap, size cap, `loggerDegraded`), `lib/daemon-status.ts` +(heartbeat input + `stalled` detail), `commands/daemon.ts` (render health + read +heartbeat), `lib/log-janitor.ts` (`onError`), `lib/daemon-client.ts` (send caller +tag, surface unknown-command), `packages/rt-client` (registry row, warn sink, +transport caller tag + unknown-command text), `lib/command-tree-def.ts` + +`lib/module-registry.ts` (log-level command). + +## Testing + +- `health.ts`: each level transition and reason string (pure, table-driven). +- `loop-monitor.ts`: drift math with injected clock; unref'd; no per-tick + allocation. +- `heartbeat-file.ts`: write/read round trip via atomic rename; missing/corrupt + -> null. +- `daemon-status.ts`: `stalled` detail when heartbeat is stale + pid alive + boot + reached ready; every existing verdict unchanged. +- `daemon-logger.ts`: stream error -> `info()` does not throw and `loggerDegraded` + set; crash handler still exits under a throwing logger; a non-panic stderr line + logs at warn not error; the size cap option is present. +- `handleCommand`: reqId minted and echoed in `ok:false`; caller logged; a burst + of identical `ok:false` produces a bounded number of lines with a suppressed + count. +- unknown-command envelope carries `code` + `version`; transport surfaces the + distinct text. +- resolver warn sink dedupes once per `(key, scope, reason)`. +- settings: `rt.logLevel` row + settings-paths parity + dist freshness. +- E2E `e2e/tests/daemon.test.ts`: status/tray:status/ping additive fields; + `/api/status` shape stays additive. From e1300623dcf6c4038b1be2d85d3d7da81f861b53 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 16:25:14 -0500 Subject: [PATCH 092/142] spec: p6-portability design (Phase 6 portability) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../specs/2026-08-28-p6-portability-design.md | 434 ++++++++++++++++++ 1 file changed, 434 insertions(+) create mode 100644 docs/superpowers/specs/2026-08-28-p6-portability-design.md diff --git a/docs/superpowers/specs/2026-08-28-p6-portability-design.md b/docs/superpowers/specs/2026-08-28-p6-portability-design.md new file mode 100644 index 00000000..5f982a4f --- /dev/null +++ b/docs/superpowers/specs/2026-08-28-p6-portability-design.md @@ -0,0 +1,434 @@ +# Phase 6 · Someone else's Mac (p6-portability) ... design + +**Status:** proposed +**Date:** 2026-08-28 +**Branch:** `job/p6-portability` (stacked on `job/integration`, wave 1) +**Roadmap:** daemon-stability-audit-2026-08 §"Phase 6 · Someone else's Mac" (RT-83) + +Phase 6 makes the daemon survive a machine that is not the author's: a +teammate whose login shell is fish, whose `.zshrc` blocks or execs into tmux, +whose Mac has a different hostname, who has no home repo yet, whose git has no +identity, or who is on an Intel Mac. The audit lists ~8 items across four +sub-themes. The `superpowers` chain runs a full spec for 6.1 and treats the +rest as bounded, plan-sized units. + +## Verification pass ... what survived, what wave 1 already closed + +Every Phase 6 finding was re-verified against the merged wave-1 code before +scoping this spec. Result: + +**Open, in scope:** + +| Item | Finding(s) | One-line defect | +|---|---|---| +| 6.1 PATH rebuild | S013, S014, S062 | fish emits space-separated PATH; boot hangs forever on a blocking `.zshrc`; silent fallback to launchd's bare PATH when `.zshrc` execs fish/tmux | +| 6.2 machine key | S071 | machine settings key derives from the mutable hostname | +| 6.2 dev wrapper | S020, S067 | a foreign `~/.local/bin/rt` `#!` script is misread as our dev-mode wrapper, parking prod | +| 6.3 platform | R051 | no Intel / unsupported-arch warning at setup | +| 6.4 first-run | S090 | missing `~/.mattstack/user` diagnosed as "git missing" not "not provisioned" | +| 6.4 first-run | R043 | no git `user.name`/`user.email` check with an actionable message | +| 6.4 first-run | S070 (sops half) | age-key spawn got a timeout in wave 1; the sops spawn in `lib/secrets/store.ts` still has none, so a locked keychain hangs `loadSecrets()` | +| 6.4 first-run | S069 | `branch_cache` keys on the bare branch name; a same-name branch in a second repo overwrites the first | + +**Already closed by wave 1, dropped from scope** (verified in code): + +- **S046** ... `lib/daemon/cron.ts:84` now passes `env: { ...process.env }`. +- **S099** ... `lib/rt-paths.ts` gates the `~/.rt` rename behind `hasRtSignature()` (`RT_SIGNATURE_ENTRIES`). +- **S066** ... `lib/deps/links.ts` keeps every `DEFAULT_EXPOSED` tool; reconcile never unlinks our own product surface by name. +- **S002** ... `lib/agent-herdr.ts` resolves herdr via `resolveHerdrBin()` (`HERDR_BIN` ?? `Bun.which("herdr")` ?? `~/.local/bin/herdr`) with a clear error. +- **S039** ... `agent-status-poller.ts` backs the herdr probe off after 3 null probes; `lib/runs/store.ts` memoizes run summaries by db mtime. +- **S051** ... `handlers/agent.ts` returns `ok:false` and rolls back the record when herdr focuses an existing tab. +- **S022** ... `lib/daemon/freshness.ts resolveUserIdAcrossTracking()` gates on grant, not live-vs-poll, so poll-only users get notifications. +- **S070 (age-key half)** ... `lib/home/age-key.ts` has the 30s timeout + `AgeKeyTimeoutError` already. + +The one correction to the brief: the brief listed **S070 as done**. Only the +age-key half is; the sops spawn in `lib/secrets/store.ts` still has no timeout. +That half is kept in scope (6.4). + +No change here requires a `SCHEMA_VERSION` bump (S069 reuses the existing +`branch TEXT PRIMARY KEY` column ... see 6.4). `packages/rt-client` is touched +(one new registry key), so `bun run build` runs in it before the final review. + +--- + +## 6.1 · PATH resolution rebuilt (S013, S014, S062) + +### The problem + +`lib/daemon/user-path.ts resolveUserPath()` scrapes the user's PATH with +`execSync($SHELL -ilc 'echo $PATH')` at daemon boot (called synchronously at +`lib/daemon.ts:163`). Three failure classes on someone else's Mac: + +- **S013 (fish):** `fish -ilc 'echo $PATH'` prints a *space-separated* list. + The daemon splits on `:`, so every real dir lands inside one bogus entry; + `git`/`node`/`pnpm` vanish from every child's PATH. `entries: 1` is logged; + nothing warns. +- **S014 (hang):** `-i` sources `.zshrc`. Under launchd (no TTY, no network + yet) a plugin, `gpg-agent`/pinentry, `direnv`, or a `read` can block + forever. `execSync`'s timeout only SIGTERMs; an interactive shell ignores + SIGTERM, so the daemon hangs before it binds anything ... the exact "starts, + binds nothing, logs nothing" symptom CLAUDE.md warns gets misdiagnosed. +- **S062 (silent fallback):** `.zshrc` ending in `exec fish` / `exec tmux` + replaces zsh before `-c` runs; stdout is empty, `|| resolvedPath` silently + keeps launchd's `/usr/bin:/bin:/usr/sbin:/sbin`. The pool then wedges with + `env: node: No such file or directory` (the 2026-08-21 spawn-env incident), + now for any common `.zshrc` idiom. + +### Decisions + +1. **Drop the interactive shell (`-i`).** The probe uses a *non-interactive + login* shell (`-lc`), which sources `.zprofile`/`.zshenv` (zsh) or + `.bash_profile` (bash) but never the interactive rc files (`.zshrc`, + `.bashrc`). This removes both the hang (S014: interactive plugins live in + `.zshrc`) and the exec-into-tmux fallback (S062: those idioms live in + `.zshrc`) at the source. `.zshenv`'s absolute-path fnm bootstrap and + `.zprofile`'s `brew shellenv` (both fixed by the 2026-08-21 spawn-env + contract) are still sourced, so a standard Homebrew+fnm Mac resolves fully. + + **Accepted cost:** PATH exports that a user put *only* in `.zshrc` (classic + `nvm` installs, hand-rolled `export PATH=` lines) are no longer picked up by + the shell probe. This is the exact trade-off S062's fixer notes flagged. + It is covered by (a) an explicit `rt.daemonPath` override (decision 4), + (b) an explicit `nvm.sh` overlay inside the probe (decision 3), and (c) a + loud missing-tools warning (decision 6). This reverses the current file's + header preference for `-ilc`; the reversal is deliberate and is the crux of + Phase 6, so it is called out here for the spec-review gate. + +2. **Hard timeout in a killable process group.** The probe spawns via + `Bun.spawn([...], { detached: true })` (a new session/process group; the + same option `lib/worktree/trash.ts:183` already uses) and `proc.unref()`s + it. A `setTimeout` pair escalates `process.kill(-proc.pid, "SIGTERM")` then, + after a short grace, `process.kill(-proc.pid, "SIGKILL")` ... the negative + pid targets the whole group, so a hung grandchild (pinentry, a stuck + `direnv`) is reaped too, not just the shell. The result is a + `Promise.race([captured, deadline])` so `resolveUserPath` always resolves + within the timeout regardless of what the child does. `detached: true` is + what makes `-pid` safe: without it, `-pid` would signal the daemon's own + group. Default timeout 5000ms (a login shell resolves in well under 1s), + overridable via `RT_PATH_PROBE_TIMEOUT_MS` and via an injected seam for + tests. + +3. **fish-aware, colon-joined output, nvm overlay.** + - `shellName = basename($SHELL || "/bin/zsh")`. + - fish: `[$SHELL, "-lc", "string join : $PATH"]` ... emits a colon-joined + list (fixes S013). + - everything else: `[$SHELL, "-lc", '{ [ -s "${NVM_DIR:-$HOME/.nvm}/nvm.sh" ] && . "${NVM_DIR:-$HOME/.nvm}/nvm.sh" >/dev/null 2>&1; }; printf %s "$PATH"']` + ... `printf %s "$PATH"` is already colon-joined; the nvm overlay replaces + the current file's second `execSync` and is now the only way an + `.zshrc`-only nvm node reaches PATH under `-lc`. + +4. **Explicit `rt.daemonPath` override (settings registry).** A new + machine-scoped key. When set to a non-empty value, `resolveUserPath` uses it + verbatim and skips the shell probe entirely ... instant, deterministic, and + the honest replacement for scraping an exotic shell. Registry row (in + `packages/rt-client/src/settings/registry-defs.ts`, mirrored nowhere else): + + ```ts + { + key: "rt.daemonPath", + type: "string", + scopes: ["machine"], + merge: "replace", + // no `default`: absent means "resolve via the shell probe below". + // no `pathGuardFields`: the value IS a PATH literal, and machine scope + // is exempt from the path-literal guard anyway (write.ts). + description: "Absolute colon-separated PATH the daemon uses for every child it spawns, instead of probing your login shell. Set this when the daemon can't find node/git/bun/pnpm (e.g. a fish shell, a blocking .zshrc, or PATH exports that live only in .zshrc). Machine-scoped: it never travels to another machine.", + } + ``` + + Read synchronously via `getSetting("rt.daemonPath")` (getSetting is + sync and throws only on an *unregistered* key; an unset registered key + resolves to `undefined`). + +5. **Validate before trusting the probe.** Trim the output; reject it (keep the + baseline `process.env.PATH`, warn) when it is empty, contains whitespace + (a space/tab means a fish-unsplit or corrupt value), splits into fewer than + two colon segments, or equals the launchd baseline verbatim (the S062 + silent-fallback signature). Acceptance is the only path that overwrites the + baseline. + +6. **Observability.** The probe's tool set grows to `node`, `git`, `bun`, + `pnpm`. After resolution (override, probe, or baseline), if any are missing + from the resolved PATH, log one `warn` naming the remedy ("set + `rt.daemonPath`"). Distinguish, in the log, the three outcomes: override + used / probe accepted / fell back to baseline (with the reason: killed, + empty, invalid). + +7. **Async integration (fence-granted).** `resolveUserPath` becomes + `async (log) => Promise`. `lib/daemon.ts:163` changes the one + statement to `const resolvedPath = await resolveUserPath(log);` (shepherd + granted this single-statement exception to the p2-health lane's ownership of + `daemon.ts`; the surrounding block and everything else in the file stay + p2's). `daemon.ts` already uses top-level `await` (lines 119, 145, ...), so + this is a literal one-line change. Boot waits at most the hard timeout and + can never hang. The bundle-Helpers + `~/.local/bin` prefix block that + follows (`daemon.ts:167-183`) is unchanged and still runs after the await. + +8. **Remove the sync-exec allowlist entry.** With both `execSync` calls gone + (the code now uses async `Bun.spawn` only), delete + `"lib/daemon/user-path.ts", // Phase 6 PATH rebuild (S013/S014/S062)` from + the `ALLOWLIST` in `lib/__tests__/no-daemon-sync-exec.test.ts`. The gate's + static regex scans for `execSync(`/`spawnSync(`/`Bun.spawnSync(`/ + `Bun.sleepSync(`; `Bun.spawn(` + `setTimeout` + `process.kill` match none. + +### Shape + +``` +resolveUserPath(log): // async, Promise + override = getSetting("rt.daemonPath") // sync read + if override non-empty: + result = override; source = "override" + else: + raw = await probe(shell, timeout) // Bun.spawn detached, pgroup kill, race + result = validate(raw) ? raw : baseline // reject fish/empty/launchd-baseline + source = accepted ? "probe" : "baseline" + warnIfMissing(result, [node, git, bun, pnpm]) // one warn line, names rt.daemonPath + log.info({ source, entries, hasNode, hasGit, hasBun, hasPnpm }, "PATH resolved") + return result +``` + +`probe` is an injectable seam (default: the real `Bun.spawn`) so tests never +spawn a real shell. + +### Tests (`lib/daemon/__tests__/user-path.test.ts`) + +- fish-style space-separated output is rejected → baseline kept + warn. +- a hanging probe (injected seam that never settles) → resolves within the + timeout, returns baseline, logs the killed/fallback reason. +- an `exec`-into-empty `.zshrc` (probe returns "") → baseline kept, distinguishable in the log. +- `rt.daemonPath` set → probe never called, value used verbatim. +- a valid colon PATH → accepted; `hasNode` etc. reflected; no warn. +- missing-tool warn fires once when node/git absent. +- `probeTools` existing coverage retained. + +--- + +## 6.2 · Machine key from a stable identifier (S071) + +### The problem + +`machineKey()` (`lib/rt-paths.ts:140`, mirrored byte-for-byte in +`packages/rt-client/src/settings/paths.ts`) reads the `~/.mattstack/machine-key` +pin file, and *falls back to a slug of `os.hostname()`* when no pin exists. The +machine settings store lives at `user/local//settings.local.jsonc`. +Rename the Mac and the key changes; the machine's settings silently vanish. +`machineKey()` is on a hot synchronous path (every `getSetting` calls +`readStores()` → `machineSettingsPath()` → `machineKey()`), so it must stay +sync and subprocess-free. + +Today the only pin writer is `rt home init` (`lib/home/init-exec.ts:103` +`writeMachineKey`), and it writes `config.machineKey`, which *defaults to +`machineKey()` itself* (`commands/home.ts:552`) ... i.e. the hostname slug. So +even a set-up machine self-pins its hostname slug rather than a stable id. + +### Decision + +Keep `machineKey()` exactly as-is (sync, pin-first, hostname fallback ... no +subprocess, so rt-paths ↔ rt-client parity is preserved). Establish a *stable* +pin at setup time, data-preservingly: + +- New `async stableMachineId(): Promise` (rt-side, e.g. + `lib/home/machine-id.ts`): `Bun.spawn(["ioreg", "-rd1", "-c", + "IOPlatformExpertDevice"])` (async, hard timeout, detached), parse + `"IOPlatformUUID" = ""`, slug it through `isSafeMachineKeySegment`. + Returns `null` on any failure (non-mac, CI, parse miss). +- `rt home init`'s default key becomes `seams.key ?? (await resolveInitialMachineKey())`: + 1. pin file already exists → return `machineKey()` (its current value; **no change**). + 2. else the hostname-slug machine store already has data on disk → return the + hostname slug (freeze the current key; this is the "migrate the + hostname-keyed section" step, done with zero data movement). + 3. else (truly fresh) → `(await stableMachineId()) ?? machineKey()` (hardware + UUID for new installs; hostname slug as the last resort). + The interactive picker's explicit key still wins (`seams.key`). + +**Data-preserving + idempotent:** an existing pin is never rewritten; a machine +with existing data keeps its current key (frozen); only a genuinely fresh +machine gets the hardware UUID. `machineKey()` reads the same value before and +after, so there is no within-boot key drift on any machine that has data. + +**Deliberately out of this item:** no daemon-boot pin write (the write fence +grants only the one `resolveUserPath` statement in `daemon.ts`; adding a call +there is out of bounds, and setup is the correct owner of the pin anyway). No +hot-path warn (it would diverge the rt-paths ↔ rt-client mirror). A machine run +without `rt home init` therefore keeps the live hostname slug; this is a +dev-only residual, and the spec-review gate can add a setup-row surface for it +if wanted. + +### Tests + +- fresh (no pin, no data) + injected `stableMachineId` → pin written with the + stable id. +- existing pin → `resolveInitialMachineKey` returns it unchanged. +- existing hostname-slug data, no pin → pin written with the hostname slug (frozen). +- `stableMachineId` parses a real `ioreg` fixture; returns null on a failing/empty probe. + +--- + +## 6.2 · Dev-mode wrapper marker (S020, S067) + +### The problem + +`currentMode()` (`lib/dev-mode.ts:76`) classifies `~/.local/bin/rt` as "dev" +whenever its first two bytes are `#!`. Any foreign `#!` script parked there +reads as dev and the prod daemon parks forever. Its companion +`isDevModeWrapper()` (`lib/deps/links.ts:46`) treats any `#!` file whose line 2 +does not start with `LINK_TAG` as our dev wrapper, so `rt deps link rt --force` +refuses to replace a foreign script (`dev-mode-owns-rt`). The current wrapper +(`renderDevModeWrapper()`, `commands/settings.ts:510`) carries no marker ... its +line 2 is a real `export PATH=...`. + +### Decision + +Mirror the `LINK_TAG` pattern (`lib/deps/resolve.ts:113`, +`# mattstack-link:`). Add a marker line 2 to new wrappers and centralize +detection so the two call sites cannot diverge (the audit's "fix both +together"). + +- `renderDevModeWrapper()` emits `# mattstack-dev-mode` as line 2 (after the + shebang, before the `export PATH`). +- New shared `isDevModeWrapperContent(content): boolean` (in `lib/dev-mode.ts`, + imported by `lib/deps/links.ts`): true iff `content` starts with `#!` **and** + either line 2 starts with `# mattstack-dev-mode` (new wrappers) **or** + `content.includes("RT_LAUNCH_CWD")` (the legacy markerless body's unique + tell). A foreign `#!` script has neither → false → classified prod / eligible + for replacement. +- `currentMode()` reads the whole (small) file and delegates to + `isDevModeWrapperContent`; `isDevModeWrapper()` in `links.ts` delegates too. + +**Backward-compatible:** existing dev machines whose wrapper predates the +marker still classify as dev (via the `RT_LAUNCH_CWD` tell), so no re-link is +needed and no dev machine flips to prod. This matters: a wrong flip is the +dev/prod standoff that `rt` daemon verbs cannot themselves repair. + +### Tests + +- a foreign `#!/bin/sh\necho hi` at the wrapper path → `currentMode()` prod, + `isDevModeWrapper()` false. +- a legacy markerless wrapper (`RT_LAUNCH_CWD` body) → dev / true. +- a new marked wrapper → dev / true. +- a `LINK_TAG` link → not a dev wrapper. + +--- + +## 6.3 · Unsupported platform at setup (R051) + +### Decision + +Add an architecture row to `lib/setup/validators/mac.ts`, mirroring +`macosVersionRow`'s honesty ruling: + +```ts +async function archRow(p: Probes): Promise { + const base = { id: "tool.arch", kind: "tool" as const, title: "Processor", + why: "mattstack ships an Apple-silicon (arm64) build; Intel Macs are not supported.", required: true }; + const res = await p.exec(["uname", "-m"]); + const arch = res.stdout.trim(); + if (res.code !== 0 || !arch) return row({ ...base, status: "error", detail: "Could not determine your processor" }); + if (arch === "arm64") return row({ ...base, status: "ready", detail: "Apple silicon (arm64)" }); + return row({ ...base, status: "invalid", detail: `${arch}: Apple silicon (arm64) required` }); +} +``` + +`macRows()` returns `[macos, clt, archRow, pathRow]` (arch and macos/clt probe +in the same `Promise.all`). A failed probe reports `error` ("couldn't +determine"), never `invalid` ... same ruling as the macOS-version row. + +### Tests + +- `uname -m` = `arm64` → ready. +- `= x86_64` → invalid with an arm64 message. +- probe fails (code !== 0) → error, not invalid. + +--- + +## 6.4 · First-run honesty + +### S090 · "not provisioned" vs "git missing" + +In `lib/daemon/home-snapshot.ts init()` (line 357), before the +`git rev-parse --is-inside-work-tree` spawn, `existsSync(deps.repoDir)`. When +the dir is absent set a distinct `disabledReason` (`"not-provisioned"`) and a +`warn` naming `rt home init`. The existing `exitCode === -1` branch stays for a +genuine spawn failure (git truly missing from PATH). Pure code change. + +### R043 · git identity checked once + +Before the first `home-snapshot` commit (and in +`lib/home/init-exec.ts commitInitialUserRepo`, which has the same gap), check +identity once: `git config user.name` and `git config user.email` via +`deps.exec`. If either is empty, set a distinct `disabledReason` +(`"no-git-identity"`), log one actionable `warn` (`git config --global +user.name/…user.email`), and skip committing (a commit would fail anyway). +Checking config directly is cleaner than parsing "Author identity unknown" out +of stderr and gives an actionable message once, not per cycle. + +### S070 (sops half) · timeout on the secrets spawn + +`createRealSecretsExecSeam` in `lib/secrets/store.ts` (the sops `Bun.spawn`, +awaited via `Promise.all([..., proc.exited])`) gains the exact pattern +`lib/home/age-key.ts` already uses: a `DEFAULT_SECRETS_TIMEOUT_MS` (30s), a +`setTimeout` → `proc.kill()` (SIGTERM then SIGKILL grace), and a distinguished +error (`SecretsTimeoutError`) so a locked-keychain hang surfaces as a timeout +rather than poisoning any cache with a generic failure. Async already; just add +the timer + distinguished error. + +### S069 · branch_cache keyed by repo + branch + +`branch_cache` has `branch TEXT PRIMARY KEY` with `repo` as a nullable +attribute column (already holding the serialized repo identity post the wave-1 +`rekeyBranchCacheTable` migration). A same-name branch in a second repo +overwrites the first. Per `docs/repo-identity.md`, a state.db table keys on the +**serialized wire identity** (`remote:host%2Fpath` / `path:%2F…`). + +**Fix without a schema bump:** make the primary-key *value* the composite +`${serializedIdentity}:${branch}`, reusing the existing `branch TEXT PRIMARY +KEY` column (no DDL change). This is safe to parse because a git branch name +cannot contain `:` (git `check-ref-format`), so the bare branch is always +`key.slice(key.lastIndexOf(":") + 1)` and the identity is everything before it. + +- Store API takes `(identity, branch)` and composes internally; a shared + `composeKey(identity, branch)` / `branchOf(key)` pair lives in + `lib/state/branch-cache.ts`. +- Writers (`lib/enrich.ts writeEnriched`/`fetchAndCache`/`refreshAllMRs`, and + the standalone `import.meta.main` entry) already hold the repo identity + (via `serializeIdentity(await deriveRepoIdentity(...))` from the local + `lib/settings/identity.ts` barrel); they pass it to `put`. +- Readers: `enrich.ts` composes the key for its `in store.entries` / lookup + checks; `commands/status/data.ts` (raw SELECT for display) shows + `branchOf(row.branch)` and reads identity from the `repo` column. +- `boot-migrate.ts`'s existing `repo`-column rekey is untouched and coexists. + +**Migration:** none. Old bare-branch rows become unused and age out via the +existing GC; the cache self-heals. Idempotent, no backfill, no schema bump. + +### Tests + +- S090: missing `repoDir` → `disabledReason "not-provisioned"`, message names `rt home init`; present-but-not-a-repo and git-missing branches unchanged. +- R043: empty `user.name`/`user.email` → `no-git-identity`, one warn, no commit; identity present → commits normally. +- S070: an injected hanging sops seam → `SecretsTimeoutError` within the timeout; the daemon/caller never blocks. +- S069: two repos, same branch name → two distinct rows; lookups resolve per repo; `branchOf` recovers the display name; a branch containing no `:` round-trips. + +--- + +## Task decomposition (preview for the plan) + +Independent enough to parallelize; 6.1 is the spine. + +1. **6.1a** ... `rt.daemonPath` registry key + `bun run build` in rt-client (unblocks 6.1b's override read). +2. **6.1b** ... rewrite `resolveUserPath` (async probe, detached pgroup kill, fish-aware, nvm overlay, validation, override, missing-tools warn) + its tests. +3. **6.1c** ... `daemon.ts:163` one-line `await`; remove the `user-path.ts` allowlist entry; gate stays green. +4. **6.2a** ... `stableMachineId` + `resolveInitialMachineKey` at `rt home init` + tests. +5. **6.2b** ... dev-mode marker: `renderDevModeWrapper` + shared `isDevModeWrapperContent` + both call sites + tests. +6. **6.3** ... `archRow` in `mac.ts` + tests. +7. **6.4a** ... S090 `existsSync` + R043 identity check in home-snapshot / init-exec + tests. +8. **6.4b** ... S070 sops timeout in `lib/secrets/store.ts` + test. +9. **6.4c** ... S069 composite branch-cache key across store + enrich + status + tests. + +## Verification (must pass) + +- `bun test lib commands packages scripts` green (worktree root). +- `bunx tsc --noEmit` zero errors. +- `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts e2e/tests/setup.test.ts e2e/tests/first-run.test.ts` green. +- `lib/__tests__/no-daemon-sync-exec.test.ts` green with the `user-path.ts` allowlist entry removed. +- `packages/rt-client`: `bun run build` before the final review (registry touched). +- Never start a daemon or run `dist/rt` against the real machine; any such run uses `env -i HOME=`. From 4ed21e041b6a668a4ef6dd8532a23adc8195eb14 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 16:29:13 -0500 Subject: [PATCH 093/142] spec: fix degraded-vs-alive-not-serving branch, pin thresholds, scope R012 watcher-close out Co-Authored-By: Claude Opus 4.8 (1M context) --- .../specs/2026-08-28-p2-health-design.md | 49 ++++++++++++++++--- 1 file changed, 43 insertions(+), 6 deletions(-) diff --git a/docs/superpowers/specs/2026-08-28-p2-health-design.md b/docs/superpowers/specs/2026-08-28-p2-health-design.md index 299bcdee..1f296251 100644 --- a/docs/superpowers/specs/2026-08-28-p2-health-design.md +++ b/docs/superpowers/specs/2026-08-28-p2-health-design.md @@ -40,7 +40,10 @@ Level is severity-ordered; unhealthy wins over degraded. `reasons` name the failing subsystem so the operator knows where to look, e.g. `"refresh: 3 repos failing (auth?)"`, `"event-loop: stalled 8s"`, `"logging: disabled (ENOSPC)"`, `"disk: 180MB free"` (R011). `metrics` gives the -memory/handle/uptime numbers a leak hunt needs (R012). +memory/handle/uptime numbers a leak hunt needs (R012). R012's watcher-close +remediation (closing fs.watch handles for repos no longer indexed) is **out of +Phase 2 scope**: Phase 2 ships the metrics + growth-alarm half only, and the +leak-close rides the later watcher-lifecycle work. A daemon-side adapter (`buildHealthSnapshot(ctx)`) gathers the inputs from `ctx`, the loop monitor, the logger handle, Phase 0 supervision, and an optional @@ -57,12 +60,17 @@ change. - `status` and `tray:status` verbs (`lib/daemon/handlers/status.ts`) gain additive `health`, `metrics`, `eventLoop` blocks. `/api/status` aliases `tray:status`, so it carries the verdict. -- `ping` gains `health.level`, `version` (`ctx.identity.version`), and the - heartbeat `seq` (all cheap). +- `ping` gains `health.level`, `version` (`ctx.identity.version`), the heartbeat + `seq`, and the `eventLoop` summary (`maxLagMs`, `lastStallAt`, `lastStallCmd`) so + a caller that only got a ping through can still show lag (all cheap). - `rt daemon status` (`commands/daemon.ts` `statusLines`) renders `level` + - `reasons` and metrics/eventLoop lines on the running branch; on the - alive-but-not-serving branch it prints `maxLag` / last stall instead of today's - "likely mid-sync" guess (R003). + `reasons` and the metrics/eventLoop lines on the running branch. Two corrections + to today's guesswork, on two different verdicts (R003): + - **degraded / unresponsive** (ping answered but the `status` verb timed out): + print the ping-carried `maxLagMs` / last stall instead of "likely mid-sync". + - **alive-not-serving** (ping failed, pid alive): print the new "stalled Ns ago" + detail from `now - heartbeat.at`. The heartbeat file is the only signal + reachable here and carries `{ at, seq }`, no `maxLag`. - **Swift tray is deferred** (brief constraint): no `rt-tray/` edits. Documented contract for the follow-up: read `data.health.level` -> green/orange/red and `data.health.reasons[0]` as `statusText`; the current client derivation @@ -93,6 +101,35 @@ alive-not-serving branch, when `breadcrumb.phase === "ready"` and heartbeat file only when the pid probe is already needed (`needsPidProbe`), and `statusLines` prints "event loop stalled Ns ago". +## Default thresholds + +`computeHealth` stays pure; these are the defaults the daemon-side adapter feeds +it (and the classifier's heartbeat threshold). Named constants, tunable later. + +| Constant | Default | Drives | +|---|---|---| +| `loopTickMs` | 250 ms | loop-monitor tick cadence | +| `loopLagDegradedMs` | 500 ms | degraded: `maxLagMs` in the window exceeds this | +| `loopStallLogMs` | 1000 ms | warn + `stalls++` when a single tick's drift exceeds this (R003's "> 1s") | +| `loopStallUnhealthyMs` | 2000 ms | "currently stalled" -> unhealthy: the most recent tick's drift exceeded this within the last `stallRecentMs` | +| `stallRecentMs` | 10 s | window in which a large recent drift still counts as "currently stalled" | +| `heartbeatIntervalMs` | 2000 ms | heartbeat file write cadence | +| `heartbeatStaleMs` | 6000 ms | classifier: alive-not-serving + heartbeat age over this -> "stalled Ns ago" (3 missed writes) | +| `refreshStaleMultiplier` | 2x the refresh interval | degraded: last successful refresh older than this | +| `rssSoftThresholdMB` | 1024 MB | degraded: rss over this | +| `rssGrowthPct` / `rssGrowthWindow` | 50% over 1 h | degraded: rss grew this much in the window | +| `diskSoftFloorMB` | 500 MB | degraded: free space under RT_DIR below this | +| `diskHardFloorMB` | 100 MB | unhealthy: free space below this | +| `restartsPerHourUnhealthy` | 5 (or Phase 0 `isCrashLooping`, >= 3 / 5 min) | unhealthy: restart storm | +| `recoveredErrorRate` / window | > 10 in 5 min | degraded: stderr/rejection error churn | +| `slowCommandMs` | 2000 ms | log a successful command at `info` above this | + +"Currently stalled" is necessarily an in-process near-miss (a daemon answering +`status` is not stalled at that instant): the adapter sets it when the last tick's +drift exceeded `loopStallUnhealthyMs` within `stallRecentMs`, catching a daemon +that just unstuck. An ongoing stall is caught cross-process instead, by the +classifier's stale-heartbeat path above. + ## Log level and growth policy - **rt.logLevel** (R004): new registry row, `type: "string"`, From 90ce759742e7e78e4637925ba0b066fd077c7b4c Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 16:41:17 -0500 Subject: [PATCH 094/142] spec: apply reviewer edits (S069 read contract + 3 sites, bounded-prefix wrapper read, interactive PATH overlay, S071/doppler notes) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../specs/2026-08-28-p6-portability-design.md | 200 +++++++++++++----- 1 file changed, 143 insertions(+), 57 deletions(-) diff --git a/docs/superpowers/specs/2026-08-28-p6-portability-design.md b/docs/superpowers/specs/2026-08-28-p6-portability-design.md index 5f982a4f..393bc2fd 100644 --- a/docs/superpowers/specs/2026-08-28-p6-portability-design.md +++ b/docs/superpowers/specs/2026-08-28-p6-portability-design.md @@ -76,23 +76,30 @@ No change here requires a `SCHEMA_VERSION` bump (S069 reuses the existing ### Decisions -1. **Drop the interactive shell (`-i`).** The probe uses a *non-interactive - login* shell (`-lc`), which sources `.zprofile`/`.zshenv` (zsh) or - `.bash_profile` (bash) but never the interactive rc files (`.zshrc`, - `.bashrc`). This removes both the hang (S014: interactive plugins live in - `.zshrc`) and the exec-into-tmux fallback (S062: those idioms live in - `.zshrc`) at the source. `.zshenv`'s absolute-path fnm bootstrap and - `.zprofile`'s `brew shellenv` (both fixed by the 2026-08-21 spawn-env - contract) are still sourced, so a standard Homebrew+fnm Mac resolves fully. - - **Accepted cost:** PATH exports that a user put *only* in `.zshrc` (classic - `nvm` installs, hand-rolled `export PATH=` lines) are no longer picked up by - the shell probe. This is the exact trade-off S062's fixer notes flagged. - It is covered by (a) an explicit `rt.daemonPath` override (decision 4), - (b) an explicit `nvm.sh` overlay inside the probe (decision 3), and (c) a - loud missing-tools warning (decision 6). This reverses the current file's - header preference for `-ilc`; the reversal is deliberate and is the crux of - Phase 6, so it is called out here for the spec-review gate. +1. **Non-interactive login base probe (`-lc`), interactive overlay unioned on + top.** The base probe uses a *non-interactive login* shell (`-lc`), which + sources `.zprofile`/`.zshenv` (zsh) or `.bash_profile` (bash) but never the + interactive rc files (`.zshrc`, `.bashrc`). This is the safe floor: it + cannot hang on an interactive plugin (S014) or exec into tmux/fish (S062), + because those idioms live in `.zshrc`. `.zshenv`'s absolute-path fnm + bootstrap and `.zprofile`'s `brew shellenv` (both fixed by the 2026-08-21 + spawn-env contract) are still sourced, so a standard Homebrew+fnm Mac + resolves fully from the base alone. + + **Interactive overlay (best-effort), per the shepherd ruling.** After the + base resolves, run a best-effort `$SHELL -ilc 'echo $PATH'` (fish: + `-ilc 'string join : $PATH'`) with **stdin from `/dev/null`**, **`TERM=dumb`** + in the child env, and a **3s hard timeout in the same killable process + group** (decision 2). Validate its output as a colon-separated list of + *absolute* dirs; **union its unique dirs after the base entries** (append, + never prepend, so the base and the daemon's own prefix keep priority). On + timeout or garbage, skip it with one `warn` line and keep the base result. + This recovers the common `.zshrc`-only PATH exports (`nvm`, `pyenv`, + `cargo`) without reintroducing the hang: the interactive shell can block or + exec, but the base has already resolved and the overlay is bounded and + killable, so a bad `.zshrc` only costs the overlay, never the daemon. The + `rt.daemonPath` override (decision 4) and the missing-tools warning + (decision 6) remain the backstops when both probes fall short. 2. **Hard timeout in a killable process group.** The probe spawns via `Bun.spawn([...], { detached: true })` (a new session/process group; the @@ -114,8 +121,10 @@ No change here requires a `SCHEMA_VERSION` bump (S069 reuses the existing list (fixes S013). - everything else: `[$SHELL, "-lc", '{ [ -s "${NVM_DIR:-$HOME/.nvm}/nvm.sh" ] && . "${NVM_DIR:-$HOME/.nvm}/nvm.sh" >/dev/null 2>&1; }; printf %s "$PATH"']` ... `printf %s "$PATH"` is already colon-joined; the nvm overlay replaces - the current file's second `execSync` and is now the only way an - `.zshrc`-only nvm node reaches PATH under `-lc`. + the current file's second `execSync`. It is the fast, safe recovery of an + nvm node in the base probe (so nvm resolves even when the interactive + overlay in decision 1 is skipped on timeout); the interactive overlay is + the broader net for pyenv/cargo/hand-rolled `.zshrc` exports. 4. **Explicit `rt.daemonPath` override (settings registry).** A new machine-scoped key. When set to a non-empty value, `resolveUserPath` uses it @@ -147,12 +156,15 @@ No change here requires a `SCHEMA_VERSION` bump (S069 reuses the existing silent-fallback signature). Acceptance is the only path that overwrites the baseline. -6. **Observability.** The probe's tool set grows to `node`, `git`, `bun`, - `pnpm`. After resolution (override, probe, or baseline), if any are missing - from the resolved PATH, log one `warn` naming the remedy ("set - `rt.daemonPath`"). Distinguish, in the log, the three outcomes: override - used / probe accepted / fell back to baseline (with the reason: killed, - empty, invalid). +6. **Observability.** The probe's tool set becomes `node`, `git`, `bun`, + `pnpm`. `doppler` is intentionally dropped from the logged tool set: it is + an optional integration, not a toolchain prerequisite, and its absence is + not a portability failure worth a boot-time signal. After resolution + (override, base, base+overlay, or baseline), if any of node/git/bun/pnpm are + missing from the resolved PATH, log one `warn` naming the remedy ("set + `rt.daemonPath`"). Distinguish, in the log, the outcomes: override used / + base accepted / overlay unioned / fell back to baseline (with the reason: + killed, empty, invalid). 7. **Async integration (fence-granted).** `resolveUserPath` becomes `async (log) => Promise`. `lib/daemon.ts:163` changes the one @@ -174,30 +186,39 @@ No change here requires a `SCHEMA_VERSION` bump (S069 reuses the existing ### Shape ``` -resolveUserPath(log): // async, Promise - override = getSetting("rt.daemonPath") // sync read +resolveUserPath(log): // async, Promise + override = getSetting("rt.daemonPath") // sync read if override non-empty: result = override; source = "override" else: - raw = await probe(shell, timeout) // Bun.spawn detached, pgroup kill, race - result = validate(raw) ? raw : baseline // reject fish/empty/launchd-baseline - source = accepted ? "probe" : "baseline" - warnIfMissing(result, [node, git, bun, pnpm]) // one warn line, names rt.daemonPath + raw = await probe(shell, "-lc", 5000) // base: detached pgroup, hard timeout, race + base = validate(raw) ? raw : baseline // reject fish-space / empty / launchd-baseline + ov = await probe(shell, "-ilc", 3000, // overlay: stdin=/dev/null, TERM=dumb, + { stdin: "/dev/null", env: { TERM: "dumb" } }) // same detached pgroup kill + result = union(base, absoluteDirsOf(ov)) // append overlay's unique absolute dirs + source = base-rejected ? "baseline" : ("base" + overlay-unioned? "+overlay" : "") + warnIfMissing(result, [node, git, bun, pnpm]) // one warn line, names rt.daemonPath log.info({ source, entries, hasNode, hasGit, hasBun, hasPnpm }, "PATH resolved") return result ``` -`probe` is an injectable seam (default: the real `Bun.spawn`) so tests never -spawn a real shell. +`probe` is a single injectable seam (default: the real detached `Bun.spawn` + +pgroup-kill + deadline-race), used for both the base and the overlay, so tests +never spawn a real shell. `absoluteDirsOf` rejects a non-colon / whitespace / +non-absolute overlay value (returns `[]`, logs the skip). ### Tests (`lib/daemon/__tests__/user-path.test.ts`) -- fish-style space-separated output is rejected → baseline kept + warn. -- a hanging probe (injected seam that never settles) → resolves within the +- fish-style space-separated base output is rejected → baseline kept + warn. +- a hanging base probe (injected seam that never settles) → resolves within the timeout, returns baseline, logs the killed/fallback reason. -- an `exec`-into-empty `.zshrc` (probe returns "") → baseline kept, distinguishable in the log. -- `rt.daemonPath` set → probe never called, value used verbatim. -- a valid colon PATH → accepted; `hasNode` etc. reflected; no warn. +- an `exec`-into-empty `.zshrc` (base returns "") → baseline kept, distinguishable in the log. +- `rt.daemonPath` set → neither probe called, value used verbatim. +- a valid colon base PATH → accepted; `hasNode` etc. reflected; no warn. +- interactive overlay contributes a `.zshrc`-only dir (e.g. an nvm/pyenv dir) → + it is appended after the base entries, unique-only, order preserved. +- a hanging or garbage overlay → skipped with one warn; the base result is kept + unchanged (overlay never regresses the base). - missing-tool warn fires once when node/git absent. - `probeTools` existing coverage retained. @@ -236,7 +257,13 @@ pin at setup time, data-preservingly: 1. pin file already exists → return `machineKey()` (its current value; **no change**). 2. else the hostname-slug machine store already has data on disk → return the hostname slug (freeze the current key; this is the "migrate the - hostname-keyed section" step, done with zero data movement). + hostname-keyed section" step, done with zero data movement). The predicate + is the one `gatherHomeState` already uses ... `profileDirPresent` + (`commands/home.ts:152`, `probes.exists(join(home, "user", "local", + ))`) ... except the freeze guard requires the dir to be **non-empty** + (an empty stub is a fresh machine, not data to preserve), so it checks + existence AND at least one entry (e.g. `settings.local.jsonc`), not bare + existence. 3. else (truly fresh) → `(await stableMachineId()) ?? machineKey()` (hardware UUID for new installs; hostname slug as the last resort). The interactive picker's explicit key still wins (`seams.key`). @@ -286,14 +313,21 @@ together"). - `renderDevModeWrapper()` emits `# mattstack-dev-mode` as line 2 (after the shebang, before the `export PATH`). -- New shared `isDevModeWrapperContent(content): boolean` (in `lib/dev-mode.ts`, - imported by `lib/deps/links.ts`): true iff `content` starts with `#!` **and** +- New shared `isDevModeWrapperContent(prefix): boolean` (in `lib/dev-mode.ts`, + imported by `lib/deps/links.ts`): true iff `prefix` starts with `#!` **and** either line 2 starts with `# mattstack-dev-mode` (new wrappers) **or** - `content.includes("RT_LAUNCH_CWD")` (the legacy markerless body's unique - tell). A foreign `#!` script has neither → false → classified prod / eligible - for replacement. -- `currentMode()` reads the whole (small) file and delegates to - `isDevModeWrapperContent`; `isDevModeWrapper()` in `links.ts` delegates too. + `prefix.includes("RT_LAUNCH_CWD")` (the legacy markerless body's unique tell, + which is line 3). A foreign `#!` script has neither → false → classified + prod / eligible for replacement. +- **Read a bounded prefix, never the whole file.** In prod, `~/.local/bin/rt` + is a symlink to the compiled binary inside the app bundle, and `readFileSync` + follows the symlink ... reading the whole file would slurp a multi-MB binary. + Both detectors read only the first few KB (e.g. an `openSync` + + `readSync(4096)`, extending the current `currentMode()` 2-byte read), which + is more than enough for the marker on line 2 and the `RT_LAUNCH_CWD` tell on + line 3. `currentMode()` reads that prefix and delegates to + `isDevModeWrapperContent`; `isDevModeWrapper()` in `links.ts` reads a bounded + prefix (not `p.readFile`'s full read) and delegates too. **Backward-compatible:** existing dev machines whose wrapper predates the marker still classify as dev (via the `RT_LAUNCH_CWD` tell), so no re-link is @@ -307,6 +341,8 @@ dev/prod standoff that `rt` daemon verbs cannot themselves repair. - a legacy markerless wrapper (`RT_LAUNCH_CWD` body) → dev / true. - a new marked wrapper → dev / true. - a `LINK_TAG` link → not a dev wrapper. +- the wrapper path is a symlink to a large (>4KB) binary-shaped file → prod, + and only the bounded prefix is read (no whole-file slurp). --- @@ -386,16 +422,59 @@ KEY` column (no DDL change). This is safe to parse because a git branch name cannot contain `:` (git `check-ref-format`), so the bare branch is always `key.slice(key.lastIndexOf(":") + 1)` and the identity is everything before it. -- Store API takes `(identity, branch)` and composes internally; a shared - `composeKey(identity, branch)` / `branchOf(key)` pair lives in - `lib/state/branch-cache.ts`. -- Writers (`lib/enrich.ts writeEnriched`/`fetchAndCache`/`refreshAllMRs`, and - the standalone `import.meta.main` entry) already hold the repo identity - (via `serializeIdentity(await deriveRepoIdentity(...))` from the local - `lib/settings/identity.ts` barrel); they pass it to `put`. -- Readers: `enrich.ts` composes the key for its `in store.entries` / lookup - checks; `commands/status/data.ts` (raw SELECT for display) shows - `branchOf(row.branch)` and reads identity from the `repo` column. +**Read contract (unchanged externally).** `cache:read` and every by-branch +lookup keep resolving a **bare branch name**: scoped to the caller's repo when +the repo is known (compose the exact `${identity}:${branch}` key), falling back +to a **suffix match** across repos (`key.endsWith(":" + branch)`) when it is +not. The CLI, board, and tray therefore see bare branch names exactly as today +and never regress; the composite key is an internal storage detail. + +**Store API** (`lib/state/branch-cache.ts`): shared `composeKey(identity, +branch)`, `branchOf(key)`, `identityOf(key)` helpers (split on the LAST `:`, +safe because branches contain none). `put(identity, branch, entry)` / +`get(identity, branch)` compose the exact key; `getByBranch(branch)` does the +suffix-match fallback for callers without an identity. `entries` stays a map, +now keyed by the composite; iterating consumers use `branchOf`/`identityOf`. + +**Consumer sites** (each gets a test): + +- **Writers** ... `lib/enrich.ts` (`writeEnriched`/`fetchAndCache`/ + `refreshAllMRs` and the standalone `import.meta.main` entry) already hold the + identity (`serializeIdentity(await deriveRepoIdentity(...))` via the local + `lib/settings/identity.ts` barrel); they call `put(identity, branch, …)` and + compose keys for their own `branch in store.entries` / lookup checks. +- **`lib/notifier.ts`** ... `state.branches` and the `fired` set key off the + same map keys as `cacheEntries` (`ctx.cache.entries`), and + `pruneFiredForEvictedBranches(fired, Object.keys(cacheEntries))` (line 887) + compares them directly. Carrying the composite key through + `state.branches`/`fired`/`detectBranchTransitions` makes the fired-state + correctly repo-scoped for free (two repos' same-named branch no longer share + one fired entry); `branchOf(key)` is used only where a human-readable branch + name is shown in the notification. **Test:** two repos, same branch name → + independent fired-state; evicting one repo's branch does not prune the + other's. +- **`lib/daemon/worktree-reconciler.ts`** ... `for (const [branch, entry] of + Object.entries(cacheEntries))` (line 594) treats the map key as a bare branch + and builds `mrState` keyed `:` (comment line 262). It switches + to `branchOf(key)` for the branch and scopes to the repo being reconciled via + `identityOf(key)` (the reconciler always knows its repo). **Test:** the + reactor builds `mrState` only from the reconciled repo's entries; a same-named + branch in another repo does not leak in. +- **`lib/daemon/freshness.ts`** ... direct lookups + `ctx.cache.entries[pr.sourceBranch]` (545), `[k.ref]` (579), `[branch]` (639) + and the `Object.entries(ctx.cache.entries)` iterations (505, 657, 703) run + per repo (the enclosing loop carries `repoName`/`repoPath` → identity). Each + direct lookup composes the exact key; each iteration filters by + `identityOf(key)` and uses `branchOf(key)`. **Test:** a branch present in two + repos resolves to the correct repo's entry. +- **`lib/daemon/handlers/cache.ts` (`cache:read`)** ... returns bare-branch- + keyed data per the read contract: exact-key when the request names a repo, + suffix-match otherwise. **Test:** `cache:read` returns bare branch names, and + a repo-scoped read never returns another repo's same-named branch. +- **`commands/status/data.ts`** (raw `SELECT branch,…` for display) ... shows + `branchOf(row.branch)`, taking identity from the row's `repo` column. +- **`lib/daemon/discussions-poller.ts`** ... iterates `Object.values(entries)` + (line 73), never keying by branch, so it **self-heals** and needs no change. - `boot-migrate.ts`'s existing `repo`-column rekey is untouched and coexists. **Migration:** none. Old bare-branch rows become unused and age out via the @@ -422,7 +501,14 @@ Independent enough to parallelize; 6.1 is the spine. 6. **6.3** ... `archRow` in `mac.ts` + tests. 7. **6.4a** ... S090 `existsSync` + R043 identity check in home-snapshot / init-exec + tests. 8. **6.4b** ... S070 sops timeout in `lib/secrets/store.ts` + test. -9. **6.4c** ... S069 composite branch-cache key across store + enrich + status + tests. +9. **6.4c** ... S069 composite branch-cache key. Wider blast radius (each with + a test): store API (`composeKey`/`branchOf`/`identityOf`/`getByBranch`) → + writers (`enrich.ts`) → `notifier.ts` (fired-state + prune) → + `worktree-reconciler.ts` (mrState) → `freshness.ts` (direct lookups + + iterations) → `handlers/cache.ts` (`cache:read` read contract) → + `status/data.ts` (display). `discussions-poller.ts` self-heals (no change). + Land the store + helpers first, then the consumers; keep the read contract + (bare branch out) intact at each step. ## Verification (must pass) From 52f450e25bc50ed440b44ca49ecf6dc326e62894 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 16:43:00 -0500 Subject: [PATCH 095/142] plan: p2-health implementation plan (16 TDD tasks) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../superpowers/plans/2026-08-28-p2-health.md | 1899 +++++++++++++++++ 1 file changed, 1899 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-28-p2-health.md diff --git a/docs/superpowers/plans/2026-08-28-p2-health.md b/docs/superpowers/plans/2026-08-28-p2-health.md new file mode 100644 index 00000000..360d9f9d --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-p2-health.md @@ -0,0 +1,1899 @@ +# Daemon Health You Can See (Phase 2 / RT-79) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give the rt daemon a single server-computed health verdict (ok / degraded / unhealthy) that names the failing subsystem, add event-loop stall detection, and fix log ergonomics and request attribution, so a non-author can tell from `rt daemon status`, `/api/status`, and the tray whether the daemon is serving, degraded, stalled, or dead, and why. + +**Architecture:** A pure `computeHealth` plus a daemon-side adapter feed a `health`/`metrics`/`eventLoop` block into the `status`, `tray:status`, and `ping` verbs. A 250ms unref'd loop monitor measures event-loop drift and writes a monotonic heartbeat file (atomic rename, db-free); the cross-process status classifier reads that file to report a stall when the pid is alive but ping fails. Logger gains a level-from-setting, a stream error listener, stderr demotion, and a size cap; handleCommand gains a request id, caller tag, and per-(cmd,error) suppression; unknown commands gain a distinct envelope. + +**Tech Stack:** Bun, TypeScript, pino + pino-roll, bun:sqlite (kv only, no schema change), `@mattstack/rt-client` settings registry. + +**Spec:** `docs/superpowers/specs/2026-08-28-p2-health-design.md` (read it alongside this plan; the plan argues from it). + +## Global Constraints + +- **No `SCHEMA_VERSION` bump.** The only new persisted state is the heartbeat file `RT_DIR/daemon-heartbeat.json`; everything else is computed live or reuses the Phase 0 `daemon-supervision` kv namespace. +- **`rt.logLevel` goes through the settings registry** per `docs/settings-architecture.md`: add the row in `registry-defs.ts`, then `cd packages/rt-client && bun run build` so the dist-freshness test stays green. **Do not bump the rt-client version and do not publish** (publishing is release-class, from `main` only). Its estate rollout rides the next release. +- **No `rt-tray/` edits.** Document the tray read contract only. +- **Never start a daemon or run `dist/rt` except under `env -i HOME=`.** Tests use isolated HOME via the bunfig preload; never touch the real `~/.mattstack`. +- **Do not modify these p6-portability-owned files:** `lib/daemon/user-path.ts`, `lib/rt-paths.ts`, `lib/deps/links.ts`, `lib/agent-herdr.ts`, `lib/dev-mode.ts`, `lib/setup/**`, `lib/daemon/agent-status-poller.ts`, `lib/enrich.ts`, `lib/daemon/home-snapshot.ts`, `lib/home/**`, `lib/daemon/cron.ts`, `lib/daemon/handlers/agent.ts`. Also do not touch the module-scope `resolveUserPath()` call in `lib/daemon.ts` (p6 makes it awaited-async). Everything else in `lib/daemon.ts` is in scope. +- **Default thresholds** (spec's table): `loopTickMs`=250, `loopLagDegradedMs`=500, `loopStallLogMs`=1000, `loopStallUnhealthyMs`=2000, `stallRecentMs`=10_000, `heartbeatIntervalMs`=2000, `heartbeatStaleMs`=6000, `refreshStaleMultiplier`=2, `rssSoftThresholdMB`=1024, `rssGrowthPct`=50 over 1h, `diskSoftFloorMB`=500, `diskHardFloorMB`=100, `restartsPerHourUnhealthy`=5 (or Phase 0 `isCrashLooping`), `recoveredErrorRate`=10 per 5min, `slowCommandMs`=2000. +- **Commit after every task.** Run `bunx tsc --noEmit` (0 errors) before each commit that touches TS. + +--- + +## File Structure + +**New files:** +- `lib/daemon/health.ts` — pure `computeHealth(inputs): HealthSnapshot`, the `HealthInputs`/`HealthSnapshot`/threshold types and constants. +- `lib/daemon/heartbeat-file.ts` — `writeHeartbeat`/`readHeartbeat` (atomic rename, db-free). +- `lib/daemon/loop-monitor.ts` — `startLoopMonitor` + the pure `applyTick` drift function. +- Tests colocated under `lib/daemon/__tests__/` and `commands/__tests__/` following the repo pattern. + +**Modified files:** `lib/daemon-status.ts`, `commands/daemon.ts`, `lib/daemon-client.ts`, `lib/daemon-logger.ts`, `lib/log-janitor.ts`, `lib/daemon.ts`, `lib/daemon/handlers/status.ts`, `lib/daemon/handlers/types.ts`, `lib/daemon/cache-refresh.ts`, `lib/daemon/api-server.ts`, `lib/daemon/socket-server.ts`, `lib/command-tree-def.ts`, `packages/rt-client/src/settings/registry-defs.ts`, `packages/rt-client/src/settings/resolve.ts`, `packages/rt-client/src/settings/registry-machinery.ts` (ResolveOpts only), `packages/rt-client/src/transport.ts`, `packages/rt-client/src/index.ts`. + +--- + +## Task 1: `lib/daemon/health.ts` — pure health computation + +**Files:** +- Create: `lib/daemon/health.ts` +- Test: `lib/daemon/__tests__/health.test.ts` + +**Interfaces:** +- Produces: `computeHealth(inputs: HealthInputs): HealthSnapshot`; `HealthInputs`, `HealthSnapshot`, `HealthMetrics`, `HealthEventLoop`, `HEALTH_THRESHOLDS`. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/health.test.ts +import { test, expect } from "bun:test"; +import { computeHealth, type HealthInputs } from "../health.ts"; + +function base(): HealthInputs { + return { + now: 1_000_000, + uptimeMs: 60_000, + mem: { rss: 200 * 1024 * 1024, heapUsed: 50 * 1024 * 1024, external: 1 * 1024 * 1024 }, + rssBaseline: null, + wsClients: 0, + watchers: 3, + freshness: { "remote:gitlab/acme": { state: "live" } }, + refresh: { lastSuccessAt: 1_000_000 - 60_000, failedRepos: 0, enrichErrors: 0 }, + refreshIntervalMs: 5 * 60_000, + eventLoop: { maxLagMs: 20, lastStallAt: null, lastStallCmd: null, stalls: 0, currentlyStalled: false }, + supervisionFailuresLastHour: 0, + crashLooping: false, + loggerDegraded: false, + recoveredErrorRateLastWindow: 0, + freeBytes: 50 * 1024 * 1024 * 1024, + }; +} + +test("all-nominal inputs are ok with no reasons", () => { + const h = computeHealth(base()); + expect(h.level).toBe("ok"); + expect(h.reasons).toEqual([]); + expect(h.metrics.watchers).toBe(3); + expect(h.eventLoop.maxLagMs).toBe(20); +}); + +test("a degraded freshness watcher flips degraded and names refresh", () => { + const i = base(); + i.freshness = { "remote:gitlab/acme": { state: "degraded" } }; + const h = computeHealth(i); + expect(h.level).toBe("degraded"); + expect(h.reasons.some((r) => r.startsWith("refresh:"))).toBe(true); +}); + +test("failed repos in the last cycle flip degraded", () => { + const i = base(); + i.refresh = { lastSuccessAt: i.now - 60_000, failedRepos: 3, enrichErrors: 5 }; + expect(computeHealth(i).level).toBe("degraded"); +}); + +test("logger degraded flips unhealthy and names logging", () => { + const i = base(); + i.loggerDegraded = true; + const h = computeHealth(i); + expect(h.level).toBe("unhealthy"); + expect(h.reasons.some((r) => r.startsWith("logging:"))).toBe(true); +}); + +test("currently stalled event loop is unhealthy; unhealthy wins over a degraded signal", () => { + const i = base(); + i.eventLoop.currentlyStalled = true; + i.freshness = { r: { state: "degraded" } }; // also degraded + const h = computeHealth(i); + expect(h.level).toBe("unhealthy"); + expect(h.reasons[0].startsWith("event-loop:")).toBe(true); // unhealthy reasons first +}); + +test("critical disk is unhealthy; low disk is degraded", () => { + const crit = base(); crit.freeBytes = 50 * 1024 * 1024; + expect(computeHealth(crit).level).toBe("unhealthy"); + const low = base(); low.freeBytes = 300 * 1024 * 1024; + expect(computeHealth(low).level).toBe("degraded"); +}); + +test("stale refresh (older than 2 intervals) is degraded", () => { + const i = base(); + i.refresh = { lastSuccessAt: i.now - 11 * 60_000, failedRepos: 0, enrichErrors: 0 }; + expect(computeHealth(i).level).toBe("degraded"); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/health.test.ts` +Expected: FAIL, `Cannot find module '../health.ts'`. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/health.ts +/** + * Pure daemon health verdict. computeHealth takes a fully-gathered input + * struct (the daemon-side adapter does all I/O) and returns the level, the + * named reasons, and the metrics/eventLoop blocks the surfaces echo. + */ + +export const HEALTH_THRESHOLDS = { + refreshStaleMultiplier: 2, + rssSoftThresholdBytes: 1024 * 1024 * 1024, + rssGrowthPct: 50, + diskSoftFloorBytes: 500 * 1024 * 1024, + diskHardFloorBytes: 100 * 1024 * 1024, + restartsPerHourUnhealthy: 5, + recoveredErrorRate: 10, +} as const; + +export interface HealthMetrics { + rss: number; + heapUsed: number; + external: number; + uptimeMs: number; + wsClients: number; + watchers: number; +} + +export interface HealthEventLoop { + maxLagMs: number; + lastStallAt: number | null; + lastStallCmd: string | null; + stalls: number; +} + +export interface HealthInputs { + now: number; + uptimeMs: number; + mem: { rss: number; heapUsed: number; external: number }; + /** rss + timestamp from ~1h ago, for growth detection; null if not yet sampled. */ + rssBaseline: { rss: number; at: number } | null; + wsClients: number; + watchers: number; + freshness: Record; + refresh: { lastSuccessAt: number; failedRepos: number; enrichErrors: number }; + refreshIntervalMs: number; + eventLoop: HealthEventLoop & { currentlyStalled: boolean }; + supervisionFailuresLastHour: number; + crashLooping: boolean; + loggerDegraded: boolean; + recoveredErrorRateLastWindow: number; + freeBytes: number | null; + /** Deferred inputs (spec): wired in a later phase, ignored today. */ + busySkips?: number; + criticalWriteFailures?: number; +} + +export interface HealthSnapshot { + level: "ok" | "degraded" | "unhealthy"; + reasons: string[]; + metrics: HealthMetrics; + eventLoop: HealthEventLoop; +} + +function mb(bytes: number): number { + return Math.round(bytes / (1024 * 1024)); +} + +export function computeHealth(i: HealthInputs): HealthSnapshot { + const T = HEALTH_THRESHOLDS; + const unhealthy: string[] = []; + const degraded: string[] = []; + + // --- unhealthy --- + if (i.loggerDegraded) unhealthy.push("logging: disabled (ENOSPC)"); + if (i.eventLoop.currentlyStalled) unhealthy.push("event-loop: currently stalled"); + if (i.crashLooping || i.supervisionFailuresLastHour >= T.restartsPerHourUnhealthy) { + unhealthy.push(`restarts: ${i.supervisionFailuresLastHour} in the last hour`); + } + if (i.freeBytes !== null && i.freeBytes < T.diskHardFloorBytes) { + unhealthy.push(`disk: ${mb(i.freeBytes)}MB free (critical)`); + } + + // --- degraded --- + const degradedRepos = Object.values(i.freshness).filter((f) => f.state === "degraded").length; + if (degradedRepos > 0) degraded.push(`refresh: ${degradedRepos} watcher${degradedRepos !== 1 ? "s" : ""} degraded`); + if (i.refresh.failedRepos > 0 || i.refresh.enrichErrors > 0) { + degraded.push(`refresh: ${i.refresh.failedRepos} repos failing (auth?)`); + } + const refreshAge = i.now - i.refresh.lastSuccessAt; + if (i.refresh.lastSuccessAt > 0 && refreshAge > T.refreshStaleMultiplier * i.refreshIntervalMs) { + degraded.push(`refresh: last success ${Math.round(refreshAge / 1000)}s ago`); + } + if (i.mem.rss > T.rssSoftThresholdBytes) degraded.push(`memory: rss ${mb(i.mem.rss)}MB`); + if (i.rssBaseline && i.mem.rss > i.rssBaseline.rss * (1 + T.rssGrowthPct / 100)) { + degraded.push(`memory: rss grew >${T.rssGrowthPct}% in the last hour`); + } + if (i.eventLoop.maxLagMs > 500) degraded.push(`event-loop: lag ${i.eventLoop.maxLagMs}ms`); + if (i.recoveredErrorRateLastWindow > T.recoveredErrorRate) { + degraded.push(`errors: ${i.recoveredErrorRateLastWindow} recovered in 5min`); + } + if (i.freeBytes !== null && i.freeBytes >= T.diskHardFloorBytes && i.freeBytes < T.diskSoftFloorBytes) { + degraded.push(`disk: ${mb(i.freeBytes)}MB free`); + } + + const level = unhealthy.length > 0 ? "unhealthy" : degraded.length > 0 ? "degraded" : "ok"; + return { + level, + reasons: level === "ok" ? [] : [...unhealthy, ...degraded], + metrics: { + rss: i.mem.rss, + heapUsed: i.mem.heapUsed, + external: i.mem.external, + uptimeMs: i.uptimeMs, + wsClients: i.wsClients, + watchers: i.watchers, + }, + eventLoop: { + maxLagMs: i.eventLoop.maxLagMs, + lastStallAt: i.eventLoop.lastStallAt, + lastStallCmd: i.eventLoop.lastStallCmd, + stalls: i.eventLoop.stalls, + }, + }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/health.test.ts` +Expected: PASS (all 7). + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/health.ts lib/daemon/__tests__/health.test.ts +git commit -m "add lib/daemon/health.ts: pure computeHealth + thresholds" +``` + +--- + +## Task 2: `lib/daemon/heartbeat-file.ts` — atomic-rename heartbeat + +**Files:** +- Create: `lib/daemon/heartbeat-file.ts` +- Test: `lib/daemon/__tests__/heartbeat-file.test.ts` + +**Interfaces:** +- Produces: `writeHeartbeat(dir: string, hb: Heartbeat): void`, `readHeartbeat(dir: string): Heartbeat | null`, `interface Heartbeat { at: number; seq: number }`. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/heartbeat-file.test.ts +import { test, expect } from "bun:test"; +import { mkdtempSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { writeHeartbeat, readHeartbeat } from "../heartbeat-file.ts"; + +test("write then read round-trips", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + writeHeartbeat(dir, { at: 123, seq: 7 }); + expect(readHeartbeat(dir)).toEqual({ at: 123, seq: 7 }); +}); + +test("missing file reads as null", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + expect(readHeartbeat(dir)).toBeNull(); +}); + +test("corrupt file reads as null", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + writeFileSync(join(dir, "daemon-heartbeat.json"), "{not json"); + expect(readHeartbeat(dir)).toBeNull(); +}); + +test("a second write overwrites atomically", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + writeHeartbeat(dir, { at: 1, seq: 1 }); + writeHeartbeat(dir, { at: 2, seq: 2 }); + expect(readHeartbeat(dir)).toEqual({ at: 2, seq: 2 }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/heartbeat-file.test.ts` +Expected: FAIL, module not found. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/heartbeat-file.ts +/** + * Monotonic liveness heartbeat, written to a small file via atomic rename so + * it never opens state.db. A stalled/lock-wedged daemon is exactly when the + * WAL is least readable, so the cross-process classifier reads THIS, not kv. + * Same db-free pattern as the Phase 0 breadcrumb. + */ +import { existsSync, readFileSync, renameSync, writeFileSync } from "fs"; +import { join } from "path"; + +export interface Heartbeat { + at: number; + seq: number; +} + +function heartbeatPath(dir: string): string { + return join(dir, "daemon-heartbeat.json"); +} + +/** Never fatal: a heartbeat is a diagnostic aid, not something a tick may fail over. */ +export function writeHeartbeat(dir: string, hb: Heartbeat): void { + try { + const tmp = `${heartbeatPath(dir)}.${process.pid}.tmp`; + writeFileSync(tmp, JSON.stringify(hb)); + renameSync(tmp, heartbeatPath(dir)); + } catch { + // best-effort + } +} + +export function readHeartbeat(dir: string): Heartbeat | null { + try { + const p = heartbeatPath(dir); + if (!existsSync(p)) return null; + return JSON.parse(readFileSync(p, "utf8")) as Heartbeat; + } catch { + return null; + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/heartbeat-file.test.ts` +Expected: PASS (4). + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/heartbeat-file.ts lib/daemon/__tests__/heartbeat-file.test.ts +git commit -m "add lib/daemon/heartbeat-file.ts: atomic-rename heartbeat" +``` + +--- + +## Task 3: `lib/daemon/loop-monitor.ts` — event-loop drift + heartbeat + +**Files:** +- Create: `lib/daemon/loop-monitor.ts` +- Test: `lib/daemon/__tests__/loop-monitor.test.ts` + +**Interfaces:** +- Consumes: nothing (pure `applyTick` plus a thin timer wrapper). +- Produces: `applyTick(stats, expected, now, opts): void` (pure), `startLoopMonitor(opts): { stats: LoopStats; stop: () => void }`, `interface LoopStats { lagMs; maxLagMs; stalls; lastStallAt; lastStallCmd; currentlyStalled }`. + +- [ ] **Step 1: Write the failing test** (test the pure tick math; the timer wrapper is thin) + +```ts +// lib/daemon/__tests__/loop-monitor.test.ts +import { test, expect } from "bun:test"; +import { applyTick, newLoopStats, type LoopStats } from "../loop-monitor.ts"; + +const OPTS = { stallLogMs: 1000, stallUnhealthyMs: 2000, stallRecentMs: 10_000 }; + +test("an on-time tick records small lag and no stall", () => { + const s = newLoopStats(); + applyTick(s, /*expected*/ 1000, /*now*/ 1010, "cache:refresh", OPTS, () => {}); + expect(s.lagMs).toBe(10); + expect(s.maxLagMs).toBe(10); + expect(s.stalls).toBe(0); + expect(s.currentlyStalled).toBe(false); +}); + +test("a >1s drift counts a stall, records the in-flight cmd, and warns", () => { + const s = newLoopStats(); + let warned = 0; + applyTick(s, 1000, 2500, "mr:action", OPTS, () => { warned++; }); + expect(s.stalls).toBe(1); + expect(s.lastStallCmd).toBe("mr:action"); + expect(s.lastStallAt).toBe(2500); + expect(s.maxLagMs).toBe(1500); + expect(warned).toBe(1); +}); + +test("currentlyStalled is true when the last big drift is within stallRecentMs", () => { + const s = newLoopStats(); + applyTick(s, 1000, 3500, "x", OPTS, () => {}); // 2500ms drift >= 2000 unhealthy + expect(s.currentlyStalled).toBe(true); + // a later on-time tick outside the recent window clears it + applyTick(s, 3500 + 250, 3500 + 250 + 20_000, null, OPTS, () => {}); + expect(s.currentlyStalled).toBe(false); +}); + +test("maxLagMs is a high-water mark", () => { + const s: LoopStats = newLoopStats(); + applyTick(s, 1000, 1300, null, OPTS, () => {}); + applyTick(s, 1550, 1600, null, OPTS, () => {}); + expect(s.maxLagMs).toBe(300); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/loop-monitor.test.ts` +Expected: FAIL, module not found. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/loop-monitor.ts +/** + * Event-loop drift monitor. A ~250ms unref'd interval measures how late each + * tick fires vs its scheduled time; a large drift means the loop was blocked. + * The interval callback is created once and the stats object is preallocated, + * so the hot tick allocates nothing. Every ~2s it also writes the heartbeat + * file the cross-process classifier reads. + */ +import type { Logger } from "pino"; + +export interface LoopStats { + lagMs: number; + maxLagMs: number; + stalls: number; + lastStallAt: number | null; + lastStallCmd: string | null; + currentlyStalled: boolean; +} + +export function newLoopStats(): LoopStats { + return { lagMs: 0, maxLagMs: 0, stalls: 0, lastStallAt: null, lastStallCmd: null, currentlyStalled: false }; +} + +interface TickOpts { + stallLogMs: number; + stallUnhealthyMs: number; + stallRecentMs: number; +} + +/** Pure: fold one tick into `stats`. `onStall` fires once per stall (warn sink). */ +export function applyTick( + stats: LoopStats, + expected: number, + now: number, + currentCmd: string | null, + opts: TickOpts, + onStall: (drift: number, cmd: string | null) => void, +): void { + const drift = now - expected; + stats.lagMs = drift > 0 ? drift : 0; + if (stats.lagMs > stats.maxLagMs) stats.maxLagMs = stats.lagMs; + if (drift > opts.stallLogMs) { + stats.stalls += 1; + stats.lastStallAt = now; + stats.lastStallCmd = currentCmd; + onStall(drift, currentCmd); + } + stats.currentlyStalled = + stats.lastStallAt !== null && + now - stats.lastStallAt <= opts.stallRecentMs && + (drift > opts.stallUnhealthyMs || stats.maxLagMs > opts.stallUnhealthyMs && now - stats.lastStallAt <= opts.stallRecentMs); +} + +export interface LoopMonitorOpts { + log: Logger; + tickMs?: number; + stallLogMs?: number; + stallUnhealthyMs?: number; + stallRecentMs?: number; + heartbeatMs?: number; + currentCmd: () => string | null; + onHeartbeat: (at: number, seq: number) => void; +} + +export function startLoopMonitor(opts: LoopMonitorOpts): { stats: LoopStats; stop: () => void } { + const tickMs = opts.tickMs ?? 250; + const tickOpts: TickOpts = { + stallLogMs: opts.stallLogMs ?? 1000, + stallUnhealthyMs: opts.stallUnhealthyMs ?? 2000, + stallRecentMs: opts.stallRecentMs ?? 10_000, + }; + const heartbeatMs = opts.heartbeatMs ?? 2000; + const stats = newLoopStats(); + let expected = Date.now() + tickMs; + let lastHeartbeat = 0; + let seq = 0; + let warnedThisStall = false; + + const timer = setInterval(() => { + const now = Date.now(); + applyTick(stats, expected, now, opts.currentCmd(), tickOpts, (drift, cmd) => { + if (!warnedThisStall) { + opts.log.warn({ driftMs: drift, cmd }, "event loop stalled"); + warnedThisStall = true; + } + }); + if (stats.lagMs <= tickOpts.stallLogMs) warnedThisStall = false; + expected = now + tickMs; + if (now - lastHeartbeat >= heartbeatMs) { + lastHeartbeat = now; + opts.onHeartbeat(now, ++seq); + } + }, tickMs); + timer.unref(); + + return { stats, stop: () => clearInterval(timer) }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/loop-monitor.test.ts` +Expected: PASS (4). + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/loop-monitor.ts lib/daemon/__tests__/loop-monitor.test.ts +git commit -m "add lib/daemon/loop-monitor.ts: drift monitor + heartbeat cadence" +``` + +--- + +## Task 4: `lib/daemon-status.ts` — heartbeat input + `stalled` detail + +**Files:** +- Modify: `lib/daemon-status.ts` +- Test: extend `lib/__tests__/daemon-status.test.ts` (add cases; do not rewrite existing ones). + +**Interfaces:** +- Consumes: `Heartbeat` shape `{ at, seq }` (structural; do not import to avoid a cycle), `HealthEventLoop` shape for the degraded passthrough. +- Produces: `DaemonStatusInputs` gains `heartbeat?`, `heartbeatStaleMs?`, `pingEventLoop?`; the `alive-not-serving` verdict gains `detail: "...|stalled"` and optional `stalledForMs`; the `degraded` verdict gains optional `eventLoop`. + +- [ ] **Step 1: Write the failing test** + +```ts +// add to lib/__tests__/daemon-status.test.ts +import { classifyDaemonStatus } from "../daemon-status.ts"; + +test("alive + ping-fail + ready + stale heartbeat => alive-not-serving 'stalled'", () => { + const now = 1_000_000; + const v = classifyDaemonStatus({ + installed: true, response: null, pingOk: false, pid: 42, pidAlive: true, + breadcrumb: { phase: "ready" }, + heartbeat: { at: now - 8000, seq: 3 }, heartbeatStaleMs: 6000, + now, + }); + expect(v.state).toBe("alive-not-serving"); + if (v.state === "alive-not-serving") { + expect(v.detail).toBe("stalled"); + expect(v.stalledForMs).toBe(8000); + } +}); + +test("alive + ready + FRESH heartbeat => 'wedged', not 'stalled'", () => { + const now = 1_000_000; + const v = classifyDaemonStatus({ + installed: true, response: null, pingOk: false, pid: 42, pidAlive: true, + breadcrumb: { phase: "ready" }, + heartbeat: { at: now - 500, seq: 9 }, heartbeatStaleMs: 6000, + now, + }); + expect(v.state === "alive-not-serving" && v.detail).toBe("wedged"); +}); + +test("degraded/unresponsive carries the ping-supplied eventLoop", () => { + const v = classifyDaemonStatus({ + installed: true, response: null, pingOk: true, pid: 42, + pingEventLoop: { maxLagMs: 1400, lastStallAt: 123, lastStallCmd: "mr:action", stalls: 2 }, + }); + expect(v.state).toBe("degraded"); + if (v.state === "degraded") expect(v.eventLoop?.maxLagMs).toBe(1400); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/__tests__/daemon-status.test.ts` +Expected: FAIL (unknown properties `heartbeat`/`pingEventLoop`; `stalled` not in union). + +- [ ] **Step 3: Write minimal implementation** + +In `lib/daemon-status.ts`: + +Add a structural type near the top: +```ts +export interface HeartbeatInput { at: number; seq: number } +export interface StatusEventLoop { maxLagMs: number; lastStallAt: number | null; lastStallCmd: string | null; stalls: number } +``` + +Extend the verdict union (the two affected members): +```ts + | { state: "degraded"; reason: "error" | "unresponsive"; detail?: string; pid: number | null; eventLoop?: StatusEventLoop } + | { state: "alive-not-serving"; pid: number; detail: "booting" | "wedged" | "quarantined" | "stalled"; stalledForMs?: number } +``` + +Extend `DaemonStatusInputs`: +```ts + heartbeat?: HeartbeatInput | null; + heartbeatStaleMs?: number; + pingEventLoop?: StatusEventLoop; +``` + +Change `classifyAliveNotServingDetail` to also detect stall, and return the age: +```ts +function classifyAliveNotServingDetail( + breadcrumb: DaemonBreadcrumbInput | null | undefined, + supervision: SupervisionState | undefined, + heartbeat: HeartbeatInput | null | undefined, + heartbeatStaleMs: number, + now: number, +): { detail: "booting" | "wedged" | "quarantined" | "stalled"; stalledForMs?: number } { + const phase = breadcrumb?.phase; + if (!phase || PHASE_ORDER.indexOf(phase) < PHASE_ORDER.indexOf("ready")) return { detail: "booting" }; + if (heartbeat && now - heartbeat.at > heartbeatStaleMs) { + return { detail: "stalled", stalledForMs: now - heartbeat.at }; + } + if (supervision?.lastExit?.kind === "boot-failed") return { detail: "quarantined" }; + return { detail: "wedged" }; +} +``` + +In `classifyDaemonStatus`, thread `now` into the alive-not-serving branch and pass eventLoop into degraded/unresponsive: +```ts + if (response) { + return { state: "degraded", reason: "error", detail: response.error, pid }; + } + if (pingOk) return { state: "degraded", reason: "unresponsive", pid, eventLoop: opts.pingEventLoop }; + ... + if (pidAlive && pid !== null) { + if (breadcrumb?.flavor && intendedFlavor && breadcrumb.flavor !== intendedFlavor) { + return { state: "parked", pid, ...(holderFlavor ? { holderFlavor } : {}) }; + } + const now = opts.now ?? Date.now(); + const d = classifyAliveNotServingDetail(breadcrumb, supervision, opts.heartbeat, opts.heartbeatStaleMs ?? 6000, now); + return { state: "alive-not-serving", pid, detail: d.detail, ...(d.stalledForMs ? { stalledForMs: d.stalledForMs } : {}) }; + } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/__tests__/daemon-status.test.ts` +Expected: PASS (new + existing). + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon-status.ts lib/__tests__/daemon-status.test.ts +git commit -m "daemon-status: heartbeat-stale 'stalled' detail + degraded eventLoop" +``` + +--- + +## Task 5: `commands/daemon.ts` + `lib/daemon-client.ts` — render health, read heartbeat, non-restarting ping + +**Files:** +- Modify: `commands/daemon.ts` (`statusLines`, `showStatus`) +- Modify: `lib/daemon-client.ts` (add `pingDaemon`) +- Test: extend `commands/__tests__/daemon-status-lines.test.ts` (or the existing statusLines test file; if none, create `commands/__tests__/status-lines.test.ts`). + +**Interfaces:** +- Consumes: `classifyDaemonStatus` (Task 4 shape), `readHeartbeat` (Task 2), `RT_DIR`. +- Produces: `pingDaemon(timeoutMs?): Promise` (non-restarting); `statusLines` renders the new degraded/stalled/health lines. + +- [ ] **Step 1: Write the failing test** + +```ts +// commands/__tests__/status-lines.test.ts +import { test, expect } from "bun:test"; +import { statusLines } from "../daemon.ts"; + +const strip = (s: string) => s.replace(/\[[0-9;]*m/g, ""); + +test("degraded/unresponsive prints ping-carried maxLag, not 'likely mid-sync'", () => { + const lines = statusLines( + { state: "degraded", reason: "unresponsive", pid: 42, eventLoop: { maxLagMs: 1400, lastStallAt: 1, lastStallCmd: "mr:action", stalls: 2 } } as any, + 2000, + ).map(strip).join("\n"); + expect(lines).not.toContain("likely mid-sync"); + expect(lines).toContain("1400ms"); + expect(lines).toContain("mr:action"); +}); + +test("alive-not-serving 'stalled' prints stalled Ns ago", () => { + const lines = statusLines( + { state: "alive-not-serving", pid: 42, detail: "stalled", stalledForMs: 8000 } as any, + 0, + ).map(strip).join("\n"); + expect(lines).toContain("event loop stalled"); + expect(lines).toContain("8s"); +}); + +test("running prints the health level and reasons when present", () => { + const lines = statusLines( + { state: "running", data: { pid: 42, uptime: 60000, watchedRepos: 3, cacheEntries: 10, + health: { level: "degraded", reasons: ["refresh: 3 repos failing (auth?)"] } } } as any, + 0, + ).map(strip).join("\n"); + expect(lines).toContain("degraded"); + expect(lines).toContain("refresh: 3 repos failing"); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test commands/__tests__/status-lines.test.ts` +Expected: FAIL (old text / missing health line). + +- [ ] **Step 3: Write minimal implementation** + +In `commands/daemon.ts` `statusLines`: + +Replace the degraded ternary (currently line ~514-518) so the `unresponsive` branch shows lag when present: +```ts + if (verdict.reason === "error") { + lines.push(` ${dim}status command failed: ${verdict.detail ?? "unknown error"}${reset}`); + } else if (verdict.eventLoop && verdict.eventLoop.maxLagMs > 0) { + const el = verdict.eventLoop; + lines.push(` ${dim}answers ping, status timed out — event loop maxLag ${el.maxLagMs}ms${el.lastStallCmd ? ` (last stall in ${el.lastStallCmd})` : ""}${reset}`); + } else { + lines.push(` ${dim}answers ping, but status timed out — likely mid-sync${reset}`); + } +``` + +In the `alive-not-serving` branch, add the `stalled` case and render the age: +```ts + const detailLine = { + booting: "still booting", + wedged: "reached ready but stopped answering (likely deadlocked)", + quarantined: "recovered from a corrupt db but still not answering", + stalled: `event loop stalled ${Math.round((verdict.stalledForMs ?? 0) / 1000)}s ago (no heartbeat)`, + }[verdict.detail]; +``` + +In the `running` branch, after the cache line, append health + metrics/eventLoop when present: +```ts + const health = verdict.data.health as { level: string; reasons: string[] } | undefined; + if (health && health.level !== "ok") { + const dot = health.level === "unhealthy" ? red : yellow; + lines.push(` ${dot}health: ${health.level}${reset}`); + for (const r of health.reasons) lines.push(` ${dim}- ${r}${reset}`); + } + const el = verdict.data.eventLoop as { maxLagMs: number } | undefined; + if (el && el.maxLagMs >= 500) lines.push(` ${dim}event loop: maxLag ${el.maxLagMs}ms${reset}`); +``` + +In `lib/daemon-client.ts`, add a non-restarting ping (uses the existing single-attempt `trySocketQuery`): +```ts +/** Single-attempt ping that never triggers the restart machinery, so + * `rt daemon status` can probe liveness and read the daemon's eventLoop + * summary without spawning a daemon as a side effect. */ +export async function pingDaemon(timeoutMs?: number): Promise { + return (await trySocketQuery("ping", undefined, timeoutMs)).response; +} +``` + +In `commands/daemon.ts` `showStatus`, capture the ping payload and heartbeat, and pass them in. Replace the `pingOk` line and the `needsPidProbe` block: +```ts + const pingResp = classifyDaemonStatus.needsLivenessProbe(response) ? await pingDaemon() : null; + const pingOk = pingResp?.ok === true; + ... + let heartbeat: ReturnType | undefined; + if (classifyDaemonStatus.needsPidProbe(response, pingOk)) { + breadcrumb = readBreadcrumb(); + supervision = readSupervisionState(); + heartbeat = readHeartbeat(RT_DIR); + const probed = await probePidAlive(recordedPid, breadcrumb?.pid); + pidAlive = probed.alive; + pid = probed.pid; + } + + const verdict = classifyDaemonStatus({ + installed: true, response, pingOk, pid, pidAlive, + intendedFlavor: resolveIntendedMode().mode, + breadcrumb, supervision, + heartbeat, pingEventLoop: (pingResp as any)?.eventLoop, + }); +``` +Add imports: `import { pingDaemon } from "../lib/daemon-client.ts"` (or extend the existing daemon-client import), `import { readHeartbeat } from "../lib/daemon/heartbeat-file.ts"`, `import { RT_DIR } from "../lib/daemon-config.ts"`. (Confirm exact relative paths against the file's existing imports.) + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test commands/__tests__/status-lines.test.ts` +Expected: PASS. Then `bunx tsc --noEmit` (0 errors). + +- [ ] **Step 5: Commit** + +```bash +git add commands/daemon.ts lib/daemon-client.ts commands/__tests__/status-lines.test.ts +git commit -m "daemon status: render health/stall lines; add non-restarting pingDaemon" +``` + +--- + +## Task 6: `lib/daemon-logger.ts` — stream error listener + crash-handler resilience + +**Files:** +- Modify: `lib/daemon-logger.ts` +- Test: `lib/__tests__/daemon-logger-resilience.test.ts` + +**Interfaces:** +- Produces: `DaemonLoggerHandle` gains `loggerDegraded(): boolean`; `createDaemonLogger` installs a stream `error` listener; the crash handlers fall back to a raw write and still `process.exit(1)`. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/__tests__/daemon-logger-resilience.test.ts +import { test, expect } from "bun:test"; +import { createDaemonLogger } from "../daemon-logger.ts"; +import { mkdtempSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +test("a stream write error does not throw out of log.info and flips loggerDegraded", async () => { + const dir = mkdtempSync(join(tmpdir(), "logres-")); + const handle = await createDaemonLogger({ logDir: dir, level: "info" }); + // Simulate a write failure by emitting 'error' on the underlying stream. + handle.stream.emit("error", Object.assign(new Error("no space"), { code: "ENOSPC" })); + expect(() => handle.logger.info("after enospc")).not.toThrow(); + expect(handle.loggerDegraded()).toBe(true); +}); +``` + +Note: this requires `createDaemonLogger` to expose the `stream` on the returned handle (add it to the handle type). If exposing `stream` is undesirable, the test may instead construct the logger against an injected stream; adjust `createDaemonLogger`'s options to accept an optional `stream` for testing. Prefer exposing `stream` on the handle (smallest change). + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/__tests__/daemon-logger-resilience.test.ts` +Expected: FAIL (`loggerDegraded` undefined / `stream` undefined). + +- [ ] **Step 3: Write minimal implementation** + +In `createDaemonLogger` (after the `roll(...)` stream is created, before building the pino logger): +```ts + let degraded = false; + stream.on("error", (err: any) => { + degraded = true; + try { + require("fs").writeSync(2, `daemon-logger: ${err?.code ?? ""} ${err?.message ?? err}\n`); + } catch { /* nothing left to do */ } + }); +``` +Return the handle with the new members: +```ts + return { + logger, + stream, + loggerDegraded: () => degraded, + childLogger: (module: string) => logger.child({ module }), + flush: () => { try { logger.flush(); } catch {} }, + }; +``` +Update the `DaemonLoggerHandle` interface to include `stream: NodeJS.WritableStream` and `loggerDegraded(): boolean`. + +In `installCrashHandlers`, wrap each handler body so a throwing logger cannot abort the handler (keep the existing exit semantics — boot-vs-steady per Phase 0): +```ts + process.on("uncaughtException", (err) => { + try { + handle.logger.fatal({ err }, "uncaughtException"); + } catch { + try { require("fs").writeSync(2, `uncaughtException (logger failed): ${err?.stack ?? err}\n`); } catch {} + } + handle.flush?.(); + process.exit(1); + }); +``` +Apply the identical try/catch + raw-write fallback to the `unhandledRejection` handler, preserving its current boot-phase-aware exit decision (do not change whether it exits; only guard the logging). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/__tests__/daemon-logger-resilience.test.ts` +Expected: PASS. `bunx tsc --noEmit` clean. + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon-logger.ts lib/__tests__/daemon-logger-resilience.test.ts +git commit -m "daemon-logger: stream error listener + loggerDegraded + crash-handler raw-write fallback" +``` + +--- + +## Task 7: `lib/daemon-logger.ts` — level from setting, stderr demotion, size cap, recovered-error counter + +**Files:** +- Modify: `lib/daemon-logger.ts` +- Test: `lib/__tests__/daemon-logger-level.test.ts` + +**Interfaces:** +- Consumes: `getSetting` from `@mattstack/rt-client`. +- Produces: `getDaemonLogger` resolves level `RT_LOG_LEVEL env ?? getSetting("rt.logLevel") ?? "info"`; `createDaemonLogger` adds `size: "50m"` to pino-roll; the stderr interceptor logs at `warn` unless a panic prefix; a `recoveredErrorCount()` getter on the handle. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/__tests__/daemon-logger-level.test.ts +import { test, expect } from "bun:test"; +import { resolveDaemonLogLevel, isPanicLine } from "../daemon-logger.ts"; + +test("RT_LOG_LEVEL env wins over the setting", () => { + expect(resolveDaemonLogLevel("debug", () => "warn")).toBe("debug"); +}); +test("setting is used when env is unset", () => { + expect(resolveDaemonLogLevel(undefined, () => "warn")).toBe("warn"); +}); +test("falls back to info when neither is set", () => { + expect(resolveDaemonLogLevel(undefined, () => undefined)).toBe("info"); +}); +test("a panic-looking stderr line is escalated; ordinary noise is not", () => { + expect(isPanicLine("panic: runtime error")).toBe(true); + expect(isPanicLine("Uncaught Error: boom")).toBe(true); + expect(isPanicLine("rt: ignoring \"x\" from the team scope")).toBe(false); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/__tests__/daemon-logger-level.test.ts` +Expected: FAIL (functions not exported). + +- [ ] **Step 3: Write minimal implementation** + +Add pure helpers: +```ts +export function resolveDaemonLogLevel( + env: string | undefined, + fromSetting: () => string | undefined, +): string { + if (env) return env; + try { + const v = fromSetting(); + if (v) return v; + } catch { /* resolver may be unavailable pre-boot */ } + return "info"; +} + +const PANIC_PREFIXES = ["panic:", "fatal error:", "Uncaught ", "UnhandledPromiseRejection"]; +export function isPanicLine(text: string): boolean { + return PANIC_PREFIXES.some((p) => text.startsWith(p)); +} +``` +In `getDaemonLogger`, use it: +```ts + cachedPromise = createDaemonLogger({ + logDir: logsDir(), + level: resolveDaemonLogLevel(process.env.RT_LOG_LEVEL, () => { + return getSetting("rt.logLevel").value; + }) as pino.LevelWithSilent, + })... +``` +Add `import { getSetting } from "@mattstack/rt-client"` (match the existing import style in the repo; if daemon-logger must stay dependency-light, inject the getter from `lib/daemon.ts` instead — but the settings resolver is in-process and cheap, so a direct import is fine). + +In `createDaemonLogger`, add the size cap to the `roll(...)` options: +```ts + const stream = await roll({ + file: `${opts.logDir}/daemon`, + extension: ".log", + frequency: "daily", + dateFormat: "yyyy-MM-dd", + mkdir: true, + size: "50m", + limit: { count: 14 }, + sync: true, + }); +``` + +In the stderr interceptor (`redirectNativeStderr`/the `process.stderr.write` override that routes into pino), route at `warn` with `source: "stderr"` unless `isPanicLine`, and increment a recovered-error counter that the handle exposes: +```ts + // inside the intercept, `text` is the stderr chunk: + if (isPanicLine(text)) { + handleLogger.error({ source: "stderr" }, text.trimEnd()); + } else { + recovered += 1; + handleLogger.warn({ source: "stderr" }, text.trimEnd()); + } +``` +Expose `recoveredErrorCount: () => recovered` on the handle (module-scope `let recovered = 0`). Also increment `recovered` in the `unhandledRejection` recovered path (steady-state). Add `recoveredErrorCount(): number` to `DaemonLoggerHandle`. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/__tests__/daemon-logger-level.test.ts` +Expected: PASS. `bunx tsc --noEmit` clean. + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon-logger.ts lib/__tests__/daemon-logger-level.test.ts +git commit -m "daemon-logger: rt.logLevel resolution, stderr->warn demotion, 50m size cap, recovered-error counter" +``` + +--- + +## Task 8: `lib/log-janitor.ts` — `onError` callback + +**Files:** +- Modify: `lib/log-janitor.ts` +- Test: extend `lib/__tests__/log-janitor.test.ts` (or create if absent). + +**Interfaces:** +- Produces: `pruneLogs(dir, retentionDays, now, onError?): { removed }` where `onError?: (phase: "readdir" | "unlink", err: unknown, file?: string) => void`. + +- [ ] **Step 1: Write the failing test** + +```ts +// add to lib/__tests__/log-janitor.test.ts +import { test, expect } from "bun:test"; +import { pruneLogs } from "../log-janitor.ts"; +import { join } from "path"; + +test("readdir failure reports via onError instead of swallowing", () => { + const calls: string[] = []; + const bogus = join("/nonexistent-xyz", "rt", "logs"); + pruneLogs(bogus, 14, Date.now(), (phase) => calls.push(phase)); + expect(calls).toContain("readdir"); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/__tests__/log-janitor.test.ts` +Expected: FAIL (`onError` not a parameter). + +- [ ] **Step 3: Write minimal implementation** + +Add the optional param and call it in both catches: +```ts +export function pruneLogs( + dir: string, + retentionDays: number, + now: number, + onError?: (phase: "readdir" | "unlink", err: unknown, file?: string) => void, +): { removed: string[] } { + ... + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch (err) { + onError?.("readdir", err); + return { removed }; + } + ... + try { + unlinkSync(full); + removed.push(entry.name); + } catch (err) { + onError?.("unlink", err, entry.name); + } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/__tests__/log-janitor.test.ts` +Expected: PASS. + +- [ ] **Step 5: Wire the daemon's callers to log at warn, then commit** + +In `lib/daemon.ts`, both `pruneLogs(logsDir(), logRetentionDays(), Date.now())` call sites (the daily interval and the boot timeout) pass an onError that warns: +```ts + const { removed } = pruneLogs(logsDir(), logRetentionDays(), Date.now(), + (phase, err, file) => log.warn({ err, phase, file }, "log prune step failed")); +``` + +```bash +git add lib/log-janitor.ts lib/daemon.ts lib/__tests__/log-janitor.test.ts +git commit -m "log-janitor: onError callback; daemon logs prune failures at warn" +``` + +--- + +## Task 9: Settings — `rt.logLevel` row, injectable resolver warn sink, rebuild dist + +**Files:** +- Modify: `packages/rt-client/src/settings/registry-defs.ts` +- Modify: `packages/rt-client/src/settings/resolve.ts` +- Modify: `packages/rt-client/src/settings/registry-machinery.ts` (ResolveOpts) OR add a module-level sink (chosen below) +- Modify: `packages/rt-client/src/index.ts` (export the sink setter) +- Test: `packages/rt-client/test/settings-warn-sink.test.ts`; existing `settings-paths-parity` and `dist-freshness` tests must stay green. + +**Interfaces:** +- Produces: registry key `rt.logLevel`; `setSettingsWarnSink(sink: ((msg: string) => void) | null): void` exported from the package (default `console.warn`). + +- [ ] **Step 1: Add the registry row** + +In `registry-defs.ts`, insert directly after the `rt.logRetentionDays` row (before `rt.apiPort`): +```ts + { + key: "rt.logLevel", + type: "string", + scopes: ["machine", "user"], + default: "info", + merge: "replace", + migrated: true, + description: "Daemon log level (trace|debug|info|warn|error). RT_LOG_LEVEL env wins, then this setting, then info (lib/daemon-logger.ts resolveDaemonLogLevel). A fresh key, not an ownership-latch port, so a default is fine here.", + }, +``` + +- [ ] **Step 2: Write the failing warn-sink test** + +```ts +// packages/rt-client/test/settings-warn-sink.test.ts +import { test, expect } from "bun:test"; +import { setSettingsWarnSink } from "../src/index.ts"; +import { emitSettingsWarning } from "../src/settings/resolve.ts"; + +test("a bound sink receives warnings and dedupes on identical messages", () => { + const seen: string[] = []; + setSettingsWarnSink((m) => seen.push(m)); + emitSettingsWarning("rt: sample warning"); + emitSettingsWarning("rt: sample warning"); + expect(seen).toEqual(["rt: sample warning"]); // deduped + setSettingsWarnSink(null); // restore default +}); +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `cd packages/rt-client && bun test test/settings-warn-sink.test.ts` +Expected: FAIL (exports missing). + +- [ ] **Step 4: Implement the sink in `resolve.ts`** + +Add a module-level, deduping sink and a public emit used by the 3 warn sites: +```ts +let warnSink: ((msg: string) => void) | null = null; +const warnedOnce = new Set(); + +/** The daemon binds a deduped log.warn here so a hot-path getSetting on a + * disallowed-scope key warns once, not every tick. Default: console.warn + * (CLI/test behavior unchanged). null restores the default. */ +export function setSettingsWarnSink(sink: ((msg: string) => void) | null): void { + warnSink = sink; + warnedOnce.clear(); +} + +export function emitSettingsWarning(msg: string): void { + if (warnSink) { + if (warnedOnce.has(msg)) return; + warnedOnce.add(msg); + warnSink(msg); + return; + } + console.warn(msg); +} +``` +Replace the three `console.warn(...)` calls (`warnInvalid` line ~491, `listSettings` line ~546, `listUnregistered` line ~586) with `emitSettingsWarning(...)` passing the same message string. + +Export `setSettingsWarnSink` from `packages/rt-client/src/index.ts` alongside the other settings exports: +```ts +export { getSetting, listSettings, explainSetting, expandVariables, SCOPE_ORDER, setSettingsWarnSink } from "./settings/resolve.ts"; +``` + +- [ ] **Step 5: Run tests, rebuild dist** + +Run: `cd packages/rt-client && bun test test/settings-warn-sink.test.ts && bun run build` +Expected: PASS; `bun run build` regenerates `dist/` so `test/dist-freshness.test.ts` stays green. **Do not bump the version; do not publish.** + +- [ ] **Step 6: Commit** + +```bash +git add packages/rt-client/src/settings/registry-defs.ts packages/rt-client/src/settings/resolve.ts packages/rt-client/src/index.ts packages/rt-client/test/settings-warn-sink.test.ts packages/rt-client/dist +git commit -m "rt-client: rt.logLevel registry row + injectable deduped settings warn sink (dist rebuilt, no bump)" +``` + +--- + +## Task 10: `lib/daemon.ts` `handleCommand` — reqId, caller, suppression, slow-command, currentCmd + +**Files:** +- Modify: `lib/daemon.ts` +- Create: `lib/daemon/command-attribution.ts` (pure helpers: reqId, suppression bookkeeping) +- Test: `lib/daemon/__tests__/command-attribution.test.ts` + +**Interfaces:** +- Produces: `shortReqId(): string`; `shouldLogSuppressed(map, key, now, windowMs): { emit: boolean; suppressed: number }`; module-scope `currentCmd` ref that `handleCommand` sets; `handleCommand` logs `{ reqId, cmd, caller, durationMs }` and echoes `reqId` in `ok:false` envelopes. + +- [ ] **Step 1: Write the failing test (pure helpers)** + +```ts +// lib/daemon/__tests__/command-attribution.test.ts +import { test, expect } from "bun:test"; +import { shortReqId, makeSuppressor } from "../command-attribution.ts"; + +test("shortReqId is short and unique-ish", () => { + const a = shortReqId(); const b = shortReqId(); + expect(a).toMatch(/^[a-z0-9]{6}$/); + expect(a).not.toBe(b); +}); + +test("suppressor logs first, then throttles with a running suppressed count", () => { + const s = makeSuppressor(60_000); + expect(s.check("mr:action|boom", 0)).toEqual({ emit: true, suppressed: 0 }); // first: log + expect(s.check("mr:action|boom", 1_000)).toEqual({ emit: false, suppressed: 1 }); // within window: silent + expect(s.check("mr:action|boom", 2_000)).toEqual({ emit: false, suppressed: 2 }); + expect(s.check("mr:action|boom", 61_000)).toEqual({ emit: true, suppressed: 2 }); // window elapsed: log with count + expect(s.check("mr:action|boom", 61_500)).toEqual({ emit: false, suppressed: 1 }); // count resets after an emit +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/command-attribution.test.ts` +Expected: FAIL (module not found). + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/command-attribution.ts +/** Short request id for tying a daemon log line to the invocation. */ +export function shortReqId(): string { + return Math.random().toString(36).slice(2, 8).padEnd(6, "0"); +} + +interface SuppressEntry { lastEmitAt: number; suppressed: number } + +/** Per-(cmd,error) suppression: always emit the first occurrence and, once per + * window, emit again carrying the count suppressed since the last emit. */ +export function makeSuppressor(windowMs: number) { + const map = new Map(); + return { + check(key: string, now: number): { emit: boolean; suppressed: number } { + const e = map.get(key); + if (!e) { + map.set(key, { lastEmitAt: now, suppressed: 0 }); + return { emit: true, suppressed: 0 }; + } + if (now - e.lastEmitAt >= windowMs) { + const suppressed = e.suppressed; + e.lastEmitAt = now; + e.suppressed = 0; + return { emit: true, suppressed }; + } + e.suppressed += 1; + return { emit: false, suppressed: e.suppressed }; + }, + }; +} +``` + +In `lib/daemon.ts`, add module-scope state near `handleCommand`: +```ts +import { shortReqId, makeSuppressor } from "./daemon/command-attribution.ts"; +const currentCmd: { cmd: string | null } = { cmd: null }; +const rejectSuppressor = makeSuppressor(60_000); +const SLOW_COMMAND_MS = 2000; +``` +Rewrite `handleCommand` (keep the throw-on-exception contract): +```ts +async function handleCommand(cmd: string, payload: any, signal?: AbortSignal): Promise { + const t0 = Date.now(); + const reqId = shortReqId(); + const caller = (payload && typeof payload._client === "string" ? payload._client : "unknown"); + currentCmd.cmd = cmd; + try { + const result = await routeCommand(cmd, payload, signal); + const durationMs = Date.now() - t0; + if (result && result.ok === false) { + const key = `${cmd}|${result.error ?? ""}`; + const { emit, suppressed } = rejectSuppressor.check(key, Date.now()); + if (emit) log.warn({ reqId, cmd, caller, error: result.error, durationMs, digest: redactDigest(payload), ...(suppressed ? { suppressed } : {}) }, "command rejected"); + return { ...result, reqId }; + } + if (durationMs > SLOW_COMMAND_MS) log.info({ reqId, cmd, caller, durationMs }, "command handled (slow)"); + else log.debug({ reqId, cmd, caller, durationMs }, "command handled"); + return result; + } catch (err) { + log.error({ err, reqId, cmd, caller, durationMs: Date.now() - t0, digest: redactDigest(payload) }, "command failed"); + throw err; + } finally { + currentCmd.cmd = null; + } +} + +function redactDigest(payload: any): Record { + if (!payload || typeof payload !== "object") return {}; + const keys = Object.keys(payload); + const pick = (k: string) => (payload[k] !== undefined ? { [k]: payload[k] } : {}); + return { keys, ...pick("repo"), ...pick("repoName"), ...pick("branch"), ...pick("iid"), ...pick("room") }; +} +``` +Wire `currentCmd.cmd` into the loop monitor in Task 13 (the monitor's `currentCmd: () => currentCmd.cmd`). + +- [ ] **Step 4: Run tests** + +Run: `bun test lib/daemon/__tests__/command-attribution.test.ts` then `bunx tsc --noEmit`. +Expected: PASS; 0 type errors. + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/command-attribution.ts lib/daemon.ts lib/daemon/__tests__/command-attribution.test.ts +git commit -m "handleCommand: reqId + caller tag + per-(cmd,error) suppression + slow-command info + currentCmd" +``` + +--- + +## Task 11: Unknown-command envelope + `X-RT-Client` in both transports + +**Files:** +- Modify: `lib/daemon.ts` (`routeCommand` default) +- Modify: `lib/daemon-client.ts` (send `X-RT-Client` on GET+POST) +- Modify: `packages/rt-client/src/transport.ts` (send `X-RT-Client`) +- Test: `lib/daemon/__tests__/unknown-command.test.ts` + +**Interfaces:** +- Produces: unknown-command returns `{ ok: false, code: "unknown-command", error, version }`. Both transports set `X-RT-Client: /`. + +- [ ] **Step 1: Write the failing test** + +Extract the default-branch shape into a tiny pure helper so it's testable: +```ts +// lib/daemon/__tests__/unknown-command.test.ts +import { test, expect } from "bun:test"; +import { unknownCommandReply } from "../unknown-command.ts"; + +test("unknown command carries a code, version, and actionable text", () => { + const r = unknownCommandReply("chat:archive", "v0.9.0"); + expect(r.ok).toBe(false); + expect(r.code).toBe("unknown-command"); + expect(r.version).toBe("v0.9.0"); + expect(r.error).toContain("v0.9.0"); + expect(r.error).toContain("chat:archive"); + expect(r.error.toLowerCase()).toContain("restart"); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/unknown-command.test.ts` +Expected: FAIL (module not found). + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/unknown-command.ts +export function unknownCommandReply(cmd: string, version: string) { + return { + ok: false as const, + code: "unknown-command" as const, + version, + error: `daemon at version ${version} does not know "${cmd}"; restart or upgrade rt (rt daemon restart)`, + }; +} +``` +In `lib/daemon.ts` `routeCommand`, replace the default branch: +```ts + default: + return unknownCommandReply(cmd, typeof RT_VERSION !== "undefined" ? RT_VERSION : "source"); +``` +(add `import { unknownCommandReply } from "./daemon/unknown-command.ts"`). + +In `lib/daemon-client.ts` `trySocketQuery`, always send the client header (restructure the `hasBody` ternary so headers fire on GET too): +```ts + const headers: Record = { "X-RT-Client": `rt-cli/${process.pid}` }; + if (hasBody) headers["Content-Type"] = "application/json"; + const response = await fetch(`http://localhost/${cmd}`, { + unix: DAEMON_SOCK_PATH, + method: hasBody ? "POST" : "GET", + headers, + body: hasBody ? JSON.stringify(payload) : undefined, + signal: AbortSignal.timeout(timeoutMs), + } as any); +``` + +In `packages/rt-client/src/transport.ts` `rtCommand`, add the header (the caller label defaults to the package’s consumer; use a generic tag): +```ts + headers: { "Content-Type": "application/json", "X-RT-Client": `rt-client/${process.pid}` }, +``` + +- [ ] **Step 4: Run tests + rebuild dist (rt-client touched)** + +Run: `bun test lib/daemon/__tests__/unknown-command.test.ts`, then `cd packages/rt-client && bun run build`. `bunx tsc --noEmit` clean. +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/unknown-command.ts lib/daemon.ts lib/daemon-client.ts packages/rt-client/src/transport.ts packages/rt-client/dist lib/daemon/__tests__/unknown-command.test.ts +git commit -m "unknown-command envelope (code+version); transports send X-RT-Client (dist rebuilt, no bump)" +``` + +--- + +## Task 12: Servers read `X-RT-Client` into `payload._client`; CORS allow-header + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Modify: `lib/daemon/socket-server.ts` +- Test: `lib/daemon/__tests__/caller-tag.test.ts` + +**Interfaces:** +- Consumes: `handleCommand(cmd, payload)` reading `payload._client` (Task 10). +- Produces: both servers merge `req.headers.get("x-rt-client")` into `payload._client` before dispatch; `buildCorsHeaders` allows `X-RT-Client`. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/caller-tag.test.ts +import { test, expect } from "bun:test"; +import { buildCorsHeaders } from "../api-server.ts"; + +test("CORS allow-headers advertises X-RT-Client so browser preflight passes", () => { + const h = buildCorsHeaders("https://example.com", true); + expect(h["Access-Control-Allow-Headers"]).toContain("X-RT-Client"); +}); +``` +(The header-merge behavior is covered end-to-end by the e2e daemon test in Task 15; the unit test guards the CORS regression, which is otherwise silent.) + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/caller-tag.test.ts` +Expected: FAIL (no `X-RT-Client` in allow-headers). + +- [ ] **Step 3: Write minimal implementation** + +In `buildCorsHeaders`: +```ts + "Access-Control-Allow-Headers": "Content-Type, X-RT-Token, X-RT-Client", +``` +In `api-server.ts`, in the generic dispatch block (after `payload` is built from query/body, before `handleCommand(route.cmd, ...)`): +```ts + const client = req.headers.get("x-rt-client"); + if (client) payload._client = client; + const result = await handleCommand(route.cmd, payload, req.signal); +``` +In `socket-server.ts`, after the payload parse (line ~46), before dispatch: +```ts + const client = req.headers.get("x-rt-client"); + if (client) (payload as any)._client = client; + const result = await handleCommand(cmd, payload, req.signal); +``` + +- [ ] **Step 4: Run test + typecheck** + +Run: `bun test lib/daemon/__tests__/caller-tag.test.ts`; `bunx tsc --noEmit`. +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/socket-server.ts lib/daemon/__tests__/caller-tag.test.ts +git commit -m "servers: thread X-RT-Client into payload._client; advertise it in CORS" +``` + +--- + +## Task 13: `HandlerContext` extension + `cache-refresh` populates the refresh ref + wsClient count + +**Files:** +- Modify: `lib/daemon/handlers/types.ts` (extend `refreshStatusRef`, add `getHealth`) +- Modify: `lib/daemon/cache-refresh.ts` (populate the extended ref) +- Modify: `lib/daemon/api-server.ts` (export `apiWsClientCount`) +- Test: `lib/daemon/__tests__/refresh-status-ref.test.ts` (a focused unit around the ref update helper) + +**Interfaces:** +- Produces: `refreshStatusRef: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }`; `HandlerContext.getHealth: () => HealthSnapshot`; `apiWsClientCount(): number`. + +- [ ] **Step 1: Extend the type** + +In `lib/daemon/handlers/types.ts`, change the `refreshStatusRef` field and add `getHealth`: +```ts + refreshStatusRef: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }; + getHealth: () => import("../health.ts").HealthSnapshot; +``` + +- [ ] **Step 2: Update the init site and the refresher** + +In `lib/daemon.ts` line ~210: +```ts +const refreshStatusRef = { lastRefreshAt: 0, lastSuccessAt: 0, failedRepos: 0, enrichErrors: 0 }; +``` +In `lib/daemon/cache-refresh.ts`, where the cycle finishes (currently sets `refreshStatusRef.lastRefreshAt = Date.now()` at ~line 195), also record the cycle outcome from the `failedRepos`/`enrichErrors` locals already computed in `refreshCacheImpl`: +```ts + refreshStatusRef.lastRefreshAt = Date.now(); + refreshStatusRef.failedRepos = failedRepos.size; + refreshStatusRef.enrichErrors = enrichErrors; + if (failedRepos.size === 0 && enrichErrors === 0) refreshStatusRef.lastSuccessAt = refreshStatusRef.lastRefreshAt; +``` +(Confirm the exact local names `failedRepos`/`enrichErrors` and that `refreshStatusRef` is in scope there; both were confirmed present in `cache-refresh.ts`.) + +- [ ] **Step 3: Export the ws-client count** + +In `lib/daemon/api-server.ts`, add a module-scope accessor next to `wsClients`: +```ts +export function apiWsClientCount(): number { + return wsClients.size; +} +``` + +- [ ] **Step 4: Focused test** + +```ts +// lib/daemon/__tests__/refresh-status-ref.test.ts +import { test, expect } from "bun:test"; +import { applyRefreshOutcome } from "../cache-refresh.ts"; + +test("a clean cycle advances lastSuccessAt; a failing cycle does not", () => { + const ref = { lastRefreshAt: 0, lastSuccessAt: 0, failedRepos: 0, enrichErrors: 0 }; + applyRefreshOutcome(ref, 1000, 0, 0); + expect(ref.lastSuccessAt).toBe(1000); + applyRefreshOutcome(ref, 2000, 2, 5); + expect(ref.lastRefreshAt).toBe(2000); + expect(ref.lastSuccessAt).toBe(1000); // unchanged on failure + expect(ref.failedRepos).toBe(2); +}); +``` +Extract the four-line update into an exported pure helper `applyRefreshOutcome(ref, at, failedReposCount, enrichErrors)` in `cache-refresh.ts` and call it from the cycle end, so the logic is unit-tested without running a real refresh. + +- [ ] **Step 5: Run tests, typecheck, commit** + +Run: `bun test lib/daemon/__tests__/refresh-status-ref.test.ts`; `bunx tsc --noEmit`. +```bash +git add lib/daemon/handlers/types.ts lib/daemon.ts lib/daemon/cache-refresh.ts lib/daemon/api-server.ts lib/daemon/__tests__/refresh-status-ref.test.ts +git commit -m "ctx: extend refreshStatusRef with cycle outcome + getHealth; export apiWsClientCount" +``` + +--- + +## Task 14: Daemon wiring — loop monitor, metrics sampler, heartbeat, `getHealth`; surface health in status/tray:status/ping + +**Files:** +- Modify: `lib/daemon.ts` (start the monitor + sampler; build `getHealth`; put it on `handlerCtx`) +- Modify: `lib/daemon/handlers/status.ts` (add `health`/`metrics`/`eventLoop` to `status` + `tray:status`; `health.level` + `eventLoop` to `ping`) +- Create: `lib/daemon/health-sampler.ts` (5-min metrics log + rss baseline + disk-free cache) +- Test: `lib/daemon/__tests__/health-sampler.test.ts` + +**Interfaces:** +- Consumes: `startLoopMonitor` (Task 3), `computeHealth` (Task 1), `writeHeartbeat` (Task 2), `apiWsClientCount` (Task 13), `readSupervisionState`/`isCrashLooping` (Phase 0), `refreshStatusRef` (Task 13), logger handle's `loggerDegraded`/`recoveredErrorCount` (Tasks 6/7). +- Produces: `createHealthSampler(opts)` returning `{ sample(): void; freeBytes(): number | null; rssBaseline(): {rss;at}|null; recoveredRateLastWindow(): number }`; `handlerCtx.getHealth` closure. + +- [ ] **Step 1: Write the sampler test** + +```ts +// lib/daemon/__tests__/health-sampler.test.ts +import { test, expect } from "bun:test"; +import { rollRssBaseline } from "../health-sampler.ts"; + +test("rss baseline rolls forward only after the window elapses", () => { + // baseline null -> set on first sample + let b = rollRssBaseline(null, { rss: 100, at: 0 }, 60 * 60_000); + expect(b).toEqual({ rss: 100, at: 0 }); + // within the hour: unchanged + b = rollRssBaseline(b, { rss: 200, at: 30 * 60_000 }, 60 * 60_000); + expect(b).toEqual({ rss: 100, at: 0 }); + // after the hour: rolls to the new sample + b = rollRssBaseline(b, { rss: 250, at: 61 * 60_000 }, 60 * 60_000); + expect(b).toEqual({ rss: 250, at: 61 * 60_000 }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/health-sampler.test.ts` +Expected: FAIL (module not found). + +- [ ] **Step 3: Implement the sampler** + +```ts +// lib/daemon/health-sampler.ts +/** Periodic (5-min) metrics logging + the two cached signals health needs that + * are too costly to compute per ping: the 1h rss baseline (growth) and free + * disk under RT_DIR. Pure helpers are unit-tested; the timer just calls sample. */ +import type { Logger } from "pino"; + +export function rollRssBaseline( + prev: { rss: number; at: number } | null, + now: { rss: number; at: number }, + windowMs: number, +): { rss: number; at: number } { + if (!prev) return now; + if (now.at - prev.at >= windowMs) return now; + return prev; +} + +export interface HealthSampler { + sample(): void; + freeBytes(): number | null; + rssBaseline(): { rss: number; at: number } | null; +} + +export function createHealthSampler(opts: { + log: Logger; + rtDir: string; + wsClients: () => number; + watchers: () => number; + startedAt: number; +}): HealthSampler { + let baseline: { rss: number; at: number } | null = null; + let free: number | null = null; + + function statfsFree(dir: string): number | null { + try { + // Node/Bun fs.statfsSync where available; guarded so an unsupported + // platform leaves free=null and disk checks are simply skipped. + const { statfsSync } = require("fs"); + const s = statfsSync(dir); + return s.bavail * s.bsize; + } catch { + return null; + } + } + + return { + freeBytes: () => free, + rssBaseline: () => baseline, + sample() { + const mem = process.memoryUsage(); + const now = Date.now(); + baseline = rollRssBaseline(baseline, { rss: mem.rss, at: now }, 60 * 60_000); + free = statfsFree(opts.rtDir); + opts.log.info( + { rss: mem.rss, heapUsed: mem.heapUsed, external: mem.external, wsClients: opts.wsClients(), watchers: opts.watchers(), uptimeMs: now - opts.startedAt }, + "daemon metrics", + ); + }, + }; +} +``` + +- [ ] **Step 4: Wire it all in `lib/daemon.ts`** (module scope, after `log` and after `handlerCtx` fields are available; the monitor/sampler timers mirror the existing events-sweep timers) + +```ts +import { startLoopMonitor } from "./daemon/loop-monitor.ts"; +import { createHealthSampler } from "./daemon/health-sampler.ts"; +import { writeHeartbeat } from "./daemon/heartbeat-file.ts"; +import { computeHealth } from "./daemon/health.ts"; +import { apiWsClientCount } from "./daemon/api-server.ts"; +import { isCrashLooping, readSupervisionState } from "./daemon/supervision-state.ts"; +import { setSettingsWarnSink } from "@mattstack/rt-client"; + +// Bind the resolver's warn sink to a deduped daemon log.warn (S033/R005): a +// hot-path getSetting on a disallowed-scope key warns once, not every tick. +// The sink dedupes internally; here we only route it into structured logging. +setSettingsWarnSink((m) => log.warn({ src: "settings" }, m)); + +const healthSampler = createHealthSampler({ + log, rtDir: RT_DIR, wsClients: apiWsClientCount, watchers: () => watchedConfigs.size, startedAt, +}); +healthSampler.sample(); // seed baseline/free immediately +safeInterval(() => healthSampler.sample(), 5 * 60_000, "health-sample", log); + +const loopMon = startLoopMonitor({ + log, + currentCmd: () => currentCmd.cmd, + onHeartbeat: (at, seq) => writeHeartbeat(RT_DIR, { at, seq }), +}); + +function buildHealthSnapshot() { + const now = Date.now(); + const sup = readSupervisionState(); + const failuresLastHour = sup.recentFailures.filter((f) => f.at > now - 60 * 60_000).length; + return computeHealth({ + now, + uptimeMs: now - startedAt, + mem: process.memoryUsage(), + rssBaseline: healthSampler.rssBaseline(), + wsClients: apiWsClientCount(), + watchers: watchedConfigs.size, + freshness: getFreshnessSnapshot(), + refresh: { lastSuccessAt: refreshStatusRef.lastSuccessAt, failedRepos: refreshStatusRef.failedRepos, enrichErrors: refreshStatusRef.enrichErrors }, + refreshIntervalMs: 5 * 60_000, + eventLoop: { ...loopMon.stats }, + supervisionFailuresLastHour: failuresLastHour, + crashLooping: isCrashLooping(sup, now), + loggerDegraded: loggerHandle.loggerDegraded?.() ?? false, + recoveredErrorRateLastWindow: loggerHandle.recoveredErrorCount?.() ?? 0, + freeBytes: healthSampler.freeBytes(), + }); +} +``` +Add `getHealth: buildHealthSnapshot` to the `handlerCtx` object literal (lines ~353-364). Import `getFreshnessSnapshot` if not already in `daemon.ts` scope (it lives in `lib/daemon/freshness.ts`). Ensure `loopMon.stop()` is called in `cleanup()`. + +- [ ] **Step 5: Surface the snapshot in the handlers** + +In `lib/daemon/handlers/status.ts`: +- `status` handler `data`: add `health: ctx.getHealth().health` — but `getHealth()` returns the whole snapshot; splice the three blocks: +```ts + "status": async () => { + const h = ctx.getHealth(); + return { ok: true, data: { + pid: process.pid, + uptime: Date.now() - ctx.startedAt, + watchedRepos: ctx.watchedConfigs.size, + cacheEntries: Object.keys(ctx.cache.entries).length, + portsCached: ctx.portCacheRef.ports.length, + portCacheAge: ctx.portCacheRef.updatedAt ? Date.now() - ctx.portCacheRef.updatedAt : null, + freshness: getFreshnessSnapshot(), + identity: ctx.identity, + health: { level: h.level, reasons: h.reasons }, + metrics: h.metrics, + eventLoop: h.eventLoop, + } }; + }, +``` +- `tray:status` handler `data`: add the same `health`/`metrics`/`eventLoop` three (keep the existing fields). +- `ping` handler: add `health: h.level` and `eventLoop: h.eventLoop` (cheap): +```ts + "ping": async () => { + const { bootAttempts, lastReadyAt, recentFailures, lastExit } = readSupervisionState(); + const h = ctx.getHealth(); + return { ok: true, uptime: Date.now() - ctx.startedAt, pid: process.pid, ...ctx.identity, + health: h.level, eventLoop: h.eventLoop, + supervision: { bootAttempts, lastReadyAt, recentFailures: recentFailures.slice(-3), lastExit } }; + }, +``` + +- [ ] **Step 6: Run tests, typecheck, commit** + +Run: `bun test lib/daemon/__tests__/health-sampler.test.ts`; `bunx tsc --noEmit`. +```bash +git add lib/daemon.ts lib/daemon/handlers/status.ts lib/daemon/health-sampler.ts lib/daemon/__tests__/health-sampler.test.ts +git commit -m "daemon: wire loop monitor + heartbeat + health sampler; surface health/metrics/eventLoop in status/tray:status/ping" +``` + +--- + +## Task 15: `rt daemon log-level` command + `daemon:log-level` verb + +**Files:** +- Modify: `commands/daemon.ts` (add `setLogLevel`) +- Modify: `lib/command-tree-def.ts` (add the `log-level` leaf in the `daemon` subtree) +- Modify: `lib/daemon/handlers/status.ts` (add `daemon:log-level` handler) OR a small dedicated handler module registered in the router +- Test: `commands/__tests__/log-level.test.ts` (the pure format/parse), and picker conformance. + +**Interfaces:** +- Consumes: `daemonQuery`, the logger handle's live level setter. +- Produces: `daemon:log-level` verb: payload `{ level? }` → sets `logger.level` live and returns `{ ok, level }`; with no `level`, returns the current `{ ok, level }`. `setLogLevel(args)` CLI handler. + +- [ ] **Step 1: Add the command-tree leaf** (after `logs:` in the `daemon` subtree; select with `omitBehavior: "list"` so omitting shows the current level, matching `settings.runaway`) + +```ts + "log-level": { + description: "Show or set the daemon's live log level", + module: "./commands/daemon.ts", + fn: "setLogLevel", + omitBehavior: "list", + args: [ + { name: "Level", type: "select", hint: "Omit to show the current level", + options: [ + { value: "trace", label: "trace" }, { value: "debug", label: "debug" }, + { value: "info", label: "info" }, { value: "warn", label: "warn" }, + { value: "error", label: "error" }, + ] }, + ], + }, +``` + +- [ ] **Step 2: Write the failing test** + +```ts +// commands/__tests__/log-level.test.ts +import { test, expect } from "bun:test"; +import { formatLogLevelResult } from "../daemon.ts"; + +test("formats a set result", () => { + expect(formatLogLevelResult({ ok: true, level: "debug" }, true)).toContain("debug"); +}); +test("formats a show result", () => { + expect(formatLogLevelResult({ ok: true, level: "info" }, false)).toContain("info"); +}); +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `bun test commands/__tests__/log-level.test.ts` +Expected: FAIL (function missing). + +- [ ] **Step 4: Implement handler + verb** + +In `commands/daemon.ts`: +```ts +export function formatLogLevelResult(res: { ok: boolean; level?: string; error?: string }, wasSet: boolean): string { + if (!res.ok) return ` ${red}●${reset} ${res.error ?? "failed"}`; + return ` ${green}●${reset} daemon log level ${wasSet ? "set to" : "is"} ${res.level}`; +} + +export async function setLogLevel(args: string[] = []): Promise { + const json = args.includes("--json"); + const level = args.find((a) => !a.startsWith("--")); + const res = await daemonQuery("daemon:log-level", level ? { level } : {}); + if (!res) { console.log(` ${red}●${reset} daemon not reachable`); return; } + if (json) { console.log(JSON.stringify(res)); return; } + console.log(formatLogLevelResult(res as any, Boolean(level))); +} +``` +The `daemon:log-level` handler needs to set the live pino level on the singleton logger. Add it where the router is built (it needs `loggerHandle`); the cleanest spot is a handler that closes over `log`/`loggerHandle` in `lib/daemon.ts`’s routed map, or add it to `createStatusHandlers` by passing a `setLevel`/`getLevel` accessor on `ctx`. Minimal approach: extend `HandlerContext` with `setLogLevel: (l: string) => void` and `getLogLevel: () => string`, wired in `daemon.ts`: +```ts +// in daemon.ts handlerCtx: + setLogLevel: (l: string) => { log.level = l; log.info({ level: l }, "log level changed"); }, + getLogLevel: () => log.level, +``` +and the handler: +```ts + "daemon:log-level": async (payload?: { level?: string }) => { + const VALID = ["trace", "debug", "info", "warn", "error"]; + if (payload?.level) { + if (!VALID.includes(payload.level)) return { ok: false, error: `invalid level: ${payload.level}` }; + ctx.setLogLevel(payload.level); + } + return { ok: true, level: ctx.getLogLevel() }; + }, +``` + +- [ ] **Step 5: Run picker conformance + tests, commit** + +Run: `bun run picker:check` (must pass — the leaf declares `omitBehavior`), `bun test commands/__tests__/log-level.test.ts`, `bunx tsc --noEmit`. +```bash +git add commands/daemon.ts lib/command-tree-def.ts lib/daemon/handlers/status.ts lib/daemon.ts commands/__tests__/log-level.test.ts +git commit -m "add rt daemon log-level: live level set/show via daemon:log-level verb" +``` + +--- + +## Task 16: E2E surface assertions + full verification + +**Files:** +- Modify: `e2e/tests/daemon.test.ts` (assert the new fields are additive and present) +- Test: the whole suite. + +- [ ] **Step 1: Add e2e assertions** + +In `e2e/tests/daemon.test.ts`, after the daemon is up, assert `/api/status` (tray:status) and the `status` verb carry the new blocks and that `ping` carries `health`: +```ts + const status = await rtJson(["daemon", "status", "--json"]); + // additive: pre-existing fields still present, new blocks present when running + // (exact assertions match the harness's existing patterns in this file) +``` +Follow the file's existing helper conventions (do not invent a new harness). Assert: `health.level` is one of ok/degraded/unhealthy; `metrics.rss` is a number; `eventLoop.maxLagMs` is a number; and a heartbeat file exists under the isolated HOME's RT_DIR after ~3s. + +- [ ] **Step 2: Run the daemon e2e** + +Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts` +Expected: PASS. (This harness starts a daemon under an isolated HOME via the preload — never against the real machine.) + +- [ ] **Step 3: Full verification gate** + +Run, in order, and record results: +```bash +bunx tsc --noEmit +bun test lib commands packages scripts +bun run picker:check +cd packages/rt-client && bun run build && cd - +bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts +``` +Expected: `tsc` 0 errors; unit suites green; picker:check green; rt-client dist fresh; daemon e2e green. If practical, run the full `bun run test:e2e` and note which was run. + +- [ ] **Step 4: Commit** + +```bash +git add e2e/tests/daemon.test.ts +git commit -m "e2e: assert additive health/metrics/eventLoop + heartbeat file" +``` + +--- + +## Documentation deliverable (tray read contract) + +Not a code task, but part of the spec's scope: in the spec file (already committed) the tray read contract is documented. If a `docs/` note for the Swift-owning follow-up is wanted, add one line to `docs/daemon-supervision-design.md` or a new `docs/daemon-health.md` pointing the tray at `data.health.level` (green/orange/red) and `data.health.reasons[0]`. Keep it to a short paragraph; no `rt-tray/` edits. + +--- + +## Self-Review (completed by the plan author) + +**Spec coverage:** R011 → Tasks 1,14 (health level + reasons in surfaces). R012 → Tasks 1,14 (metrics block + 5-min sampler + growth); watcher-close explicitly out of scope per spec. R003 → Tasks 3,4,5,14 (loop monitor + heartbeat + classifier stall + rendering). R004 → Tasks 7,9,15 (rt.logLevel setting + live verb + slow-command info in Task 10). S031 → Tasks 7,8,10 (size cap + pruneLogs onError + suppression). S032 → Task 6 (stream error listener + crash-handler wrap + loggerDegraded). S033/R005 → Tasks 7,9 (stderr demotion + resolver warn sink + recovered-error counter). R008 → Tasks 10,11,12 (reqId + caller tag + digest). R021 → Task 11 (unknown-command envelope). Constraints (no schema bump, dist rebuild no publish, no tray edits, isolated HOME) → Global Constraints + Tasks 9/11/16. + +**Placeholder scan:** every code step carries real test + impl code; the two spots that say "confirm exact names against the file" (cache-refresh locals, showStatus imports) are verification instructions, not placeholders, and the names were confirmed present by investigation. + +**Type consistency:** `HealthSnapshot`/`HealthInputs` (Task 1) are consumed unchanged in Tasks 13/14; `refreshStatusRef` fields defined in Task 13 match their use in Task 14's `buildHealthSnapshot`; `LoopStats` (Task 3) is spread into the classifier-shaped `eventLoop` in Task 14; `pingDaemon` (Task 5) return type matches `showStatus`'s use; `setSettingsWarnSink` (Task 9) exported name matches the daemon bind (Task 14 note: bind it at daemon boot — add `setSettingsWarnSink((m) => log.warn({ src: "settings" }, m))` near the logger setup, deduped by the sink itself). From 803c96002592643ff1e3c1dcfabf180255793990 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 16:53:45 -0500 Subject: [PATCH 096/142] plan: p6-portability implementation plan (10 tasks) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../plans/2026-08-28-p6-portability.md | 1289 +++++++++++++++++ 1 file changed, 1289 insertions(+) create mode 100644 docs/superpowers/plans/2026-08-28-p6-portability.md diff --git a/docs/superpowers/plans/2026-08-28-p6-portability.md b/docs/superpowers/plans/2026-08-28-p6-portability.md new file mode 100644 index 00000000..3b957c5f --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-p6-portability.md @@ -0,0 +1,1289 @@ +# Phase 6 · Someone else's Mac (p6-portability) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the rt daemon survive a machine that is not the author's ... a fish shell, a blocking `.zshrc`, a renamed Mac, a foreign `~/.local/bin/rt`, no home repo, no git identity, an Intel Mac, and a locked keychain. + +**Architecture:** Nine bounded units over three subsystems: (1) rebuild `resolveUserPath` as an async, killable, fish-aware PATH probe with an `rt.daemonPath` override; (2) stabilize machine identity and dev-mode detection; (3) first-run honesty in home-snapshot, secrets, setup, and the branch cache. Each unit is independently testable; task 10 (the branch-cache key flip) is the one atomic multi-file change. + +**Tech Stack:** Bun, TypeScript, `bun:sqlite`, `bun test`, `@mattstack/rt-client` settings registry, pino logger. + +**Spec:** `docs/superpowers/specs/2026-08-28-p6-portability-design.md` (read it alongside this plan; the plan argues from it). + +## Global Constraints + +- **No `SCHEMA_VERSION` bump.** Every fix here is code-only; S069 reuses the existing `branch TEXT PRIMARY KEY` column with no DDL change. If a bump ever looks unavoidable, STOP and ask through the shepherd channel first. +- **Never start a daemon or run `dist/rt` against the real machine.** Any daemon or compiled-binary invocation runs under `env -i HOME=`. Tests use injected seams and never spawn a real login shell. +- **Do not edit `rt-tray/`.** +- **Write fence:** work only inside this worktree. These files are the sibling p2-health lane's and MUST NOT be modified: `lib/daemon.ts` (EXCEPT the single `resolveUserPath` call statement at `lib/daemon.ts:163`, which the shepherd granted for Task 3 ... await the async result, change nothing else in the file), `lib/daemon-logger.ts`, `lib/daemon-status.ts`, `lib/daemon/supervision-state.ts`, `lib/daemon/handlers/status.ts`, `commands/daemon.ts`, `lib/daemon/command-router.ts`, `lib/daemon/api-server.ts`, `lib/daemon/socket-server.ts`, `lib/log-janitor.ts`, `lib/daemon/safe-timers.ts`. +- **`packages/rt-client` is touched (Task 1).** After any change under it, run `bun run build` inside `packages/rt-client` (the `dist/` that `file:` consumers copy). `packages/rt-client/test/dist-freshness.test.ts` is the guard. +- **The daemon sync-exec gate.** `lib/__tests__/no-daemon-sync-exec.test.ts` forbids `execSync(`/`spawnSync(`/`Bun.spawnSync(`/`Bun.sleepSync(` in any daemon-reachable module. All new subprocess use is async `Bun.spawn`. Remove the `user-path.ts` allowlist entry once Task 2 lands (Task 3). +- **Comments:** clean-code only (state a constraint the code cannot show; no narration, no task numbers in source). Never use em dashes; use "..." or rephrase. +- **Serialized repo identity:** per `docs/repo-identity.md`, state.db tables (branch_cache) key on the serialized wire identity (`remote:host%2Fpath` / `path:%2F…`). In the daemon, the variable `repoName` and `CacheEntry.repoName` already hold that serialized identity. + +--- + +## Task 1: `rt.daemonPath` settings registry key + +**Files:** +- Modify: `packages/rt-client/src/settings/registry-defs.ts` (add one row to the `REGISTRY` array) +- Test: `packages/rt-client/test/registry-defs.test.ts` (or the existing registry test file; add a case) +- Build: `packages/rt-client` (`bun run build`) + +**Interfaces:** +- Produces: the registered key `"rt.daemonPath"` (type `string`, scope `machine`), readable via `getSetting("rt.daemonPath")` (sync; returns `undefined` when unset; throws only on an unregistered key). + +- [ ] **Step 1: Write the failing test** + +Add to the registry test (mirror how existing keys are asserted): + +```ts +import { getDef } from "../src/settings/registry-machinery.ts"; + +test("rt.daemonPath is a machine-scoped string key with no default", () => { + const def = getDef("rt.daemonPath"); + expect(def).toBeDefined(); + expect(def!.type).toBe("string"); + expect(def!.scopes).toEqual(["machine"]); + expect(def!.default).toBeUndefined(); + expect(def!.pathGuardFields).toBeUndefined(); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd packages/rt-client && bun test test/registry-defs.test.ts -t "rt.daemonPath"` +Expected: FAIL (`def` is undefined). + +- [ ] **Step 3: Add the registry row** + +In `packages/rt-client/src/settings/registry-defs.ts`, add to the `REGISTRY` array (place it near `rt.apiPort`, the other machine-scoped daemon key): + +```ts +{ + key: "rt.daemonPath", + type: "string", + scopes: ["machine"], + merge: "replace", + description: + "Absolute colon-separated PATH the daemon uses for every child it spawns, instead of probing your login shell. Set this when the daemon can't find node/git/bun/pnpm (e.g. a fish shell, a blocking .zshrc, or PATH exports that live only in .zshrc). Machine-scoped: it never travels to another machine.", +}, +``` + +No `default` (absent means "probe the shell"); no `pathGuardFields` (the value is itself a PATH literal, and machine scope is exempt from the path-literal guard). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd packages/rt-client && bun test test/registry-defs.test.ts -t "rt.daemonPath"` +Expected: PASS. + +- [ ] **Step 5: Rebuild rt-client dist and verify freshness** + +Run: `cd packages/rt-client && bun run build && bun test test/dist-freshness.test.ts` +Expected: PASS (dist regenerated). + +- [ ] **Step 6: Commit** + +```bash +git add packages/rt-client/src/settings/registry-defs.ts packages/rt-client/test packages/rt-client/dist +git commit -m "rt-client: register rt.daemonPath machine setting (6.1)" +``` + +--- + +## Task 2: Rebuild `resolveUserPath` (S013, S014, S062) + +**Files:** +- Modify: `lib/daemon/user-path.ts` (rewrite `resolveUserPath`; keep `probeTools` exported; drop both `execSync` calls) +- Test: `lib/daemon/__tests__/user-path.test.ts` + +**Interfaces:** +- Consumes: `getSetting("rt.daemonPath")` from Task 1. +- Produces: `export async function resolveUserPath(log: Logger, probe?: ProbeFn): Promise` (was sync). `ProbeFn = (argv: [string, ...string[]], opts: { timeoutMs: number; env?: Record }) => Promise` (resolves the child's stdout, or `null` on spawn failure / timeout / kill). `probeTools(pathValue, names)` unchanged. + +- [ ] **Step 1: Write the failing tests** + +Replace/extend `lib/daemon/__tests__/user-path.test.ts`. Use an injected `probe` seam so no real shell is spawned. `makeLog()` returns a pino-shaped stub capturing `warn`/`info` calls. + +```ts +import { resolveUserPath } from "../user-path.ts"; + +function makeLog() { + const warns: any[] = []; const infos: any[] = []; + return { log: { warn: (...a: any[]) => warns.push(a), info: (...a: any[]) => infos.push(a) } as any, warns, infos }; +} + +test("fish-style space-separated base output is rejected, baseline kept + warn", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async () => "/opt/homebrew/bin /usr/bin /bin"; // spaces = fish-unsplit + const out = await resolveUserPath(log, probe); + expect(out).toBe("/usr/bin:/bin"); + expect(warns.some((w) => JSON.stringify(w).includes("whitespace"))).toBe(true); +}); + +test("a hanging probe returns baseline within the timeout", async () => { + const { log } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async () => null; // seam models kill/timeout as null + const out = await resolveUserPath(log, probe); + expect(out).toBe("/usr/bin:/bin"); +}); + +test("base equal to launchd baseline is treated as silent fallback (S062)", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin:/usr/sbin:/sbin"; + const probe = async (argv: any) => (argv[1] === "-lc" ? "/usr/bin:/bin:/usr/sbin:/sbin" : null); + const out = await resolveUserPath(log, probe); + expect(out).toBe("/usr/bin:/bin:/usr/sbin:/sbin"); + expect(warns.some((w) => JSON.stringify(w).includes("equals-baseline"))).toBe(true); +}); + +test("rt.daemonPath override skips both probes", async () => { + const { log } = makeLog(); + let called = false; + const probe = async () => { called = true; return "x"; }; + // Point HOME at a scratch machine store that sets rt.daemonPath, OR stub getSetting. + // (Executor: use the repo's settings test harness to set rt.daemonPath = "/over/bin:/x/bin" at machine scope.) + const out = await resolveUserPath(log, probe); + expect(out).toBe("/over/bin:/x/bin"); + expect(called).toBe(false); +}); + +test("valid base accepted; interactive overlay appends a .zshrc-only dir after base", async () => { + const { log } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async (argv: any) => + argv[1] === "-lc" ? "/opt/homebrew/bin:/usr/bin:/bin" : "/opt/homebrew/bin:/usr/bin:/bin:/Users/x/.nvm/versions/node/v22/bin"; + const out = await resolveUserPath(log, probe); + expect(out).toBe("/opt/homebrew/bin:/usr/bin:/bin:/Users/x/.nvm/versions/node/v22/bin"); +}); + +test("overlay timeout is skipped with a warn; base kept unchanged", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async (argv: any) => (argv[1] === "-lc" ? "/opt/homebrew/bin:/usr/bin:/bin" : null); + const out = await resolveUserPath(log, probe); + expect(out).toBe("/opt/homebrew/bin:/usr/bin:/bin"); + expect(warns.some((w) => JSON.stringify(w).includes("overlay"))).toBe(true); +}); + +test("missing-tool warn fires when node is absent", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async () => "/usr/bin:/bin"; // no node + await resolveUserPath(log, probe); + expect(warns.some((w) => JSON.stringify(w).includes("missing"))).toBe(true); +}); +``` + +Keep the existing `probeTools` tests. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test lib/daemon/__tests__/user-path.test.ts` +Expected: FAIL (resolveUserPath is still sync / no override / no overlay). + +- [ ] **Step 3: Rewrite `lib/daemon/user-path.ts`** + +Replace the module (keep `probeTools` as-is; remove `import { execSync }`): + +```ts +import { basename } from "path"; +import type { Logger } from "pino"; +import { getSetting } from "@mattstack/rt-client"; + +export type ProbeFn = ( + argv: [string, ...string[]], + opts: { timeoutMs: number; env?: Record }, +) => Promise; + +const BASE_TIMEOUT_MS = 5_000; +const OVERLAY_TIMEOUT_MS = 3_000; +const KILL_GRACE_MS = 500; + +/** Default probe: a detached (own process-group) Bun.spawn whose whole group is + * SIGTERM'd then SIGKILL'd at the deadline, raced so a hung shell (or a hung + * grandchild it spawned) can never block boot past the timeout. */ +const runProbe: ProbeFn = async (argv, opts) => { + let proc: ReturnType; + try { + proc = Bun.spawn(argv, { + detached: true, + env: opts.env ?? { ...process.env }, + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + }); + } catch { + return null; + } + proc.unref(); + const pid = proc.pid; + let killTimer: ReturnType | undefined; + const term = setTimeout(() => { + try { process.kill(-pid, "SIGTERM"); } catch { /* group already gone */ } + killTimer = setTimeout(() => { try { process.kill(-pid, "SIGKILL"); } catch { /* gone */ } }, KILL_GRACE_MS); + killTimer.unref?.(); + }, opts.timeoutMs); + const captured: Promise = (async () => { + try { + const [out] = await Promise.all([new Response(proc.stdout as ReadableStream).text(), proc.exited]); + return out; + } catch { + return null; + } + })(); + let deadlineTimer: ReturnType; + const deadline: Promise = new Promise((resolve) => { + deadlineTimer = setTimeout(() => resolve(null), opts.timeoutMs + KILL_GRACE_MS + 250); + }); + try { + return await Promise.race([captured, deadline]); + } finally { + clearTimeout(term); + if (killTimer) clearTimeout(killTimer); + clearTimeout(deadlineTimer!); + } +}; + +function validateBase(raw: string | null, baseline: string): { path: string; source: "probe" | "baseline"; reason?: string } { + if (raw === null) return { path: baseline, source: "baseline", reason: "killed-or-empty" }; + const v = raw.trim(); + if (v.length === 0) return { path: baseline, source: "baseline", reason: "empty" }; + if (/\s/.test(v)) return { path: baseline, source: "baseline", reason: "whitespace" }; + if (v.split(":").filter(Boolean).length < 2) return { path: baseline, source: "baseline", reason: "too-few-segments" }; + if (v === baseline) return { path: baseline, source: "baseline", reason: "equals-baseline" }; + return { path: v, source: "probe" }; +} + +/** Overlay contributes only well-formed absolute dirs; anything else yields []. */ +function absoluteDirsOf(raw: string | null): string[] { + if (raw === null) return []; + const v = raw.trim(); + if (v.length === 0 || /\s/.test(v)) return []; + return v.split(":").filter((d) => d.startsWith("/")); +} + +function unionAppend(base: string, extra: string[]): string { + const have = new Set(base.split(":").filter(Boolean)); + const add = extra.filter((d) => !have.has(d)); + return add.length === 0 ? base : [base, ...add].join(":"); +} + +export function probeTools(pathValue: string, names: string[]): Record { + const entries = pathValue.split(":").filter((p) => p.length > 0); + const probed: Record = {}; + for (const name of names) { + probed[`has${name[0]!.toUpperCase()}${name.slice(1)}`] = entries.some((p) => { + try { return Bun.file(`${p}/${name}`).size > 0; } catch { return false; } + }); + } + return probed; +} + +export async function resolveUserPath(log: Logger, probe: ProbeFn = runProbe): Promise { + const baseline = process.env.PATH ?? ""; + + const override = getSetting("rt.daemonPath"); + let result: string; + let source: string; + + if (typeof override === "string" && override.trim().length > 0) { + result = override.trim(); + source = "override"; + } else { + const shell = process.env.SHELL ?? "/bin/zsh"; + const isFish = basename(shell) === "fish"; + const baseArgv: [string, ...string[]] = isFish + ? [shell, "-lc", "string join : $PATH"] + : [shell, "-lc", `{ [ -s "${'${NVM_DIR:-$HOME/.nvm}'}/nvm.sh" ] && . "${'${NVM_DIR:-$HOME/.nvm}'}/nvm.sh" >/dev/null 2>&1; }; printf %s "$PATH"`]; + const base = validateBase(await probe(baseArgv, { timeoutMs: BASE_TIMEOUT_MS }), baseline); + result = base.path; + source = base.source; + if (base.reason) log.warn({ reason: base.reason }, "PATH base probe unusable; kept baseline"); + + const ovArgv: [string, ...string[]] = isFish + ? [shell, "-ilc", "string join : $PATH"] + : [shell, "-ilc", "echo $PATH"]; + const ovRaw = await probe(ovArgv, { timeoutMs: OVERLAY_TIMEOUT_MS, env: { ...process.env, TERM: "dumb" } }); + const extra = absoluteDirsOf(ovRaw); + if (extra.length > 0) { + const before = result; + result = unionAppend(result, extra); + if (result !== before) source += "+overlay"; + } else if (ovRaw === null) { + log.warn("PATH interactive overlay skipped (timed out or empty)"); + } + } + + const probed = probeTools(result, ["node", "git", "bun", "pnpm"]); + const missing = Object.entries(probed).filter(([, v]) => !v).map(([k]) => k.replace(/^has/, "").toLowerCase()); + if (missing.length > 0) log.warn({ missing }, "PATH missing required tools; set rt.daemonPath to override"); + log.info({ source, entries: result.split(":").length, ...probed }, "PATH resolved"); + return result; +} +``` + +(The `${'${NVM_DIR...}'}` fragments above are a template-literal escape so the literal `${NVM_DIR:-$HOME/.nvm}` survives into the shell body ... the executor writes the shell string so `$NVM_DIR`/`$HOME` expand in the child shell, not in TS.) + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test lib/daemon/__tests__/user-path.test.ts` +Expected: PASS. + +- [ ] **Step 5: Type-check** + +Run: `bunx tsc --noEmit` +Expected: zero errors. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/user-path.ts lib/daemon/__tests__/user-path.test.ts +git commit -m "user-path: async fish-aware killable PATH probe + rt.daemonPath override (S013/S014/S062)" +``` + +--- + +## Task 3: Await the async resolver in `daemon.ts`; drop the gate allowlist entry + +**Files:** +- Modify: `lib/daemon.ts:163` (the single granted statement only) +- Modify: `lib/__tests__/no-daemon-sync-exec.test.ts` (remove the `user-path.ts` allowlist line) + +**Interfaces:** +- Consumes: `resolveUserPath` (now async) from Task 2. + +- [ ] **Step 1: Make the one-line daemon.ts change** + +At `lib/daemon.ts:163`, change only: + +```ts + const resolvedPath = resolveUserPath(log); +``` +to: +```ts + const resolvedPath = await resolveUserPath(log); +``` + +Do not touch anything else in `lib/daemon.ts` (the surrounding block, line 164's `if (resolvedPath) process.env.PATH = resolvedPath;`, and the prefix block at 167-183 stay exactly as they are). Module-scope `await` is already used in this file (lines 119, 145). + +- [ ] **Step 2: Remove the allowlist entry** + +In `lib/__tests__/no-daemon-sync-exec.test.ts`, delete this line from the `ALLOWLIST` set: + +```ts + "lib/daemon/user-path.ts", // Phase 6 PATH rebuild (S013/S014/S062) +``` + +- [ ] **Step 3: Run the gate + type-check** + +Run: `bun test lib/__tests__/no-daemon-sync-exec.test.ts && bunx tsc --noEmit` +Expected: PASS, zero errors. (If the gate fails naming `user-path.ts`, a stray sync-exec remains in Task 2 ... fix there.) + +- [ ] **Step 4: Commit** + +```bash +git add lib/daemon.ts lib/__tests__/no-daemon-sync-exec.test.ts +git commit -m "daemon: await async resolveUserPath; drop user-path sync-exec allowlist (6.1)" +``` + +--- + +## Task 4: Stable machine-key at setup (S071) + +**Files:** +- Create: `lib/home/machine-id.ts` (`stableMachineId`, `resolveInitialMachineKey`) +- Modify: `commands/home.ts:552` (use `resolveInitialMachineKey`) +- Test: `lib/home/__tests__/machine-id.test.ts` + +**Interfaces:** +- Consumes: `machineKey()`, `isSafeMachineKeySegment` from `lib/rt-paths.ts`; `HomeProbes` (has `listProfiles(userLocalDir)`, `exists`). +- Produces: `export async function stableMachineId(exec?: (argv: string[]) => Promise): Promise`; `export async function resolveInitialMachineKey(home: string, probes: HomeProbes, deps?: {...}): Promise`. + +- [ ] **Step 1: Write the failing tests** + +```ts +import { stableMachineId, resolveInitialMachineKey } from "../machine-id.ts"; + +const IOREG_FIXTURE = ` "IOPlatformUUID" = "D9E8F7A6-1234-5678-9ABC-DEF012345678"`; + +test("stableMachineId parses IOPlatformUUID and slugs it", async () => { + const id = await stableMachineId(async () => IOREG_FIXTURE); + expect(id).toBe("d9e8f7a6-1234-5678-9abc-def012345678"); +}); + +test("stableMachineId returns null when ioreg fails", async () => { + expect(await stableMachineId(async () => null)).toBeNull(); + expect(await stableMachineId(async () => "no uuid here")).toBeNull(); +}); + +test("resolveInitialMachineKey: existing pin file is returned unchanged", async () => { + const probes = { exists: (p: string) => p.endsWith("machine-key"), listProfiles: () => [] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => "pinned-key", stableId: async () => "uuid-x" }); + expect(key).toBe("pinned-key"); +}); + +test("resolveInitialMachineKey: existing non-empty hostname-slug store freezes the slug", async () => { + const probes = { exists: () => false, listProfiles: () => ["myhost"] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => null, hostnameSlug: () => "myhost", stableId: async () => "uuid-x" }); + expect(key).toBe("myhost"); // frozen, data preserved, no move +}); + +test("resolveInitialMachineKey: fresh machine gets the stable id", async () => { + const probes = { exists: () => false, listProfiles: () => [] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => null, hostnameSlug: () => "myhost", stableId: async () => "uuid-x" }); + expect(key).toBe("uuid-x"); +}); + +test("resolveInitialMachineKey: fresh machine, ioreg fails -> hostname slug", async () => { + const probes = { exists: () => false, listProfiles: () => [] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => null, hostnameSlug: () => "myhost", stableId: async () => null }); + expect(key).toBe("myhost"); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test lib/home/__tests__/machine-id.test.ts` +Expected: FAIL (module missing). + +- [ ] **Step 3: Implement `lib/home/machine-id.ts`** + +```ts +import { hostname } from "os"; +import { join } from "path"; +import { readFileSync } from "fs"; +import { isSafeMachineKeySegment, machineKey } from "../rt-paths.ts"; +import type { HomeProbes } from "../../commands/home.ts"; + +/** IOPlatformUUID via ioreg, slugged; null on any failure (non-mac, CI, no match). */ +export async function stableMachineId( + exec: (argv: string[]) => Promise = defaultIoreg, +): Promise { + const out = await exec(["ioreg", "-rd1", "-c", "IOPlatformExpertDevice"]); + if (!out) return null; + const m = out.match(/"IOPlatformUUID"\s*=\s*"([^"]+)"/); + if (!m) return null; + const slug = m[1]!.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, ""); + return isSafeMachineKeySegment(slug) ? slug : null; +} + +const defaultIoreg = async (argv: [string, ...string[]] | string[]): Promise => { + try { + const proc = Bun.spawn(argv as string[], { stdin: "ignore", stdout: "pipe", stderr: "ignore" }); + const term = setTimeout(() => { try { proc.kill("SIGKILL"); } catch { /* gone */ } }, 3_000); + try { + const [out, code] = await Promise.all([new Response(proc.stdout as ReadableStream).text(), proc.exited]); + return code === 0 ? out : null; + } finally { clearTimeout(term); } + } catch { return null; } +}; + +interface InitKeyDeps { + readPin?: () => string | null; + hostnameSlug?: () => string; + stableId?: () => Promise; +} + +/** Establishes the machine key at `rt home init`. Data-preserving + idempotent: + * an existing pin is kept; a machine with existing data freezes its current + * slug (zero move); only a genuinely fresh machine gets the stable id. */ +export async function resolveInitialMachineKey(home: string, probes: HomeProbes, deps: InitKeyDeps = {}): Promise { + const readPin = deps.readPin ?? (() => { try { const v = readFileSync(join(home, "machine-key"), "utf8").trim(); return v || null; } catch { return null; } }); + const hostnameSlug = deps.hostnameSlug ?? (() => machineKey()); // machineKey() with no pin returns the hostname slug + const stableId = deps.stableId ?? (() => stableMachineId()); + + const pinned = readPin(); + if (pinned && isSafeMachineKeySegment(pinned)) return pinned; + + const slug = hostnameSlug(); + const profiles = probes.listProfiles(join(home, "user", "local")); // dirs carrying settings.local.jsonc + if (profiles.includes(slug)) return slug; // freeze existing non-empty store + + return (await stableId()) ?? slug; +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test lib/home/__tests__/machine-id.test.ts` +Expected: PASS. + +- [ ] **Step 5: Wire it into `rt home init`** + +In `commands/home.ts:552`, change: + +```ts + const key = seams.key ?? machineKey(); +``` +to: +```ts + const key = seams.key ?? (await resolveInitialMachineKey(mattstackHome(), probes)); +``` + +Add `import { resolveInitialMachineKey } from "../lib/home/machine-id.ts";` at the top. `homeInit` is already `async`. + +- [ ] **Step 6: Run the home command tests + type-check** + +Run: `bun test commands/__tests__/home.test.ts && bunx tsc --noEmit` +Expected: PASS (existing tests pass `seams.key`, so they bypass the new path; zero type errors). + +- [ ] **Step 7: Commit** + +```bash +git add lib/home/machine-id.ts lib/home/__tests__/machine-id.test.ts commands/home.ts +git commit -m "home: stable machine-key at init, data-preserving freeze of existing stores (S071)" +``` + +--- + +## Task 5: Dev-mode wrapper marker (S020, S067) + +**Files:** +- Modify: `commands/settings.ts` (`renderDevModeWrapper`: add marker line 2) +- Modify: `lib/dev-mode.ts` (`currentMode`: bounded-prefix read + delegate; export `isDevModeWrapperContent`, `DEV_MODE_TAG`) +- Modify: `lib/deps/links.ts` (`isDevModeWrapper`: bounded-prefix read + delegate) +- Test: `lib/__tests__/dev-mode.test.ts` (or the existing dev-mode test file) + +**Interfaces:** +- Produces: `export const DEV_MODE_TAG = "# mattstack-dev-mode";` and `export function isDevModeWrapperContent(prefix: string): boolean` in `lib/dev-mode.ts`. + +- [ ] **Step 1: Write the failing tests** + +```ts +import { isDevModeWrapperContent, DEV_MODE_TAG } from "../dev-mode.ts"; + +test("new marked wrapper is recognized", () => { + expect(isDevModeWrapperContent(`#!/bin/zsh\n${DEV_MODE_TAG}\nexport PATH=...\n`)).toBe(true); +}); +test("legacy markerless wrapper (RT_LAUNCH_CWD tell) is recognized", () => { + expect(isDevModeWrapperContent(`#!/bin/zsh\nexport PATH="x"\nexport RT_LAUNCH_CWD="$PWD"\n`)).toBe(true); +}); +test("foreign #! script is not a dev wrapper", () => { + expect(isDevModeWrapperContent(`#!/bin/sh\necho hi\n`)).toBe(false); +}); +test("a mattstack-link file is not a dev wrapper", () => { + expect(isDevModeWrapperContent(`#!/bin/sh\n# mattstack-link: rt\nexec ...\n`)).toBe(false); +}); +test("non-shebang content is not a dev wrapper", () => { + expect(isDevModeWrapperContent(`ELF\x00binary`)).toBe(false); +}); +``` + +Plus a `currentMode()` test: write a symlink at `devModeWrapperPath()` to a >4KB binary-shaped file and assert `currentMode() === "prod"` (and that it does not throw / read the whole file). Use the existing dev-mode test's HOME-scratch pattern. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test lib/__tests__/dev-mode.test.ts` +Expected: FAIL (`isDevModeWrapperContent` missing). + +- [ ] **Step 3: Add the marker to the emitted wrapper** + +In `commands/settings.ts renderDevModeWrapper`, insert the marker as line 2: + +```ts + return [ + `#!/bin/zsh`, + `# mattstack-dev-mode`, + `export PATH="${bunDir}:/opt/homebrew/bin:/usr/local/bin:$PATH"`, + `export RT_LAUNCH_CWD="$PWD"`, + `cd "${sourcePath}" || { echo "rt: dev-mode source checkout missing: ${sourcePath}" >&2; exit 1; }`, + `exec "${bunPath}" run --preload="${DEV_MODE_PRELOAD}" "${sourcePath}/cli.ts" "$@"`, + ].join("\n") + "\n"; +``` + +- [ ] **Step 4: Add the shared detector + bounded read in `lib/dev-mode.ts`** + +Add: + +```ts +export const DEV_MODE_TAG = "# mattstack-dev-mode"; + +/** A recognized dev-mode wrapper: our new marker on line 2, OR a legacy + * markerless wrapper (its RT_LAUNCH_CWD line is our unique tell). A foreign + * #! script has neither. `prefix` is a bounded head of the file, never the + * whole file: in prod this path is a symlink to the compiled binary. */ +export function isDevModeWrapperContent(prefix: string): boolean { + if (!prefix.startsWith("#!")) return false; + const line2 = prefix.split("\n")[1] ?? ""; + return line2.startsWith(DEV_MODE_TAG) || prefix.includes("RT_LAUNCH_CWD"); +} + +function readWrapperPrefix(path: string): string | null { + try { + const fd = openSync(path, "r"); + try { + const buf = Buffer.alloc(4096); + const n = readSync(fd, buf, 0, 4096, 0); + return buf.toString("latin1", 0, n); + } finally { closeSync(fd); } + } catch { return null; } +} +``` + +Rewrite `currentMode()` to use them: + +```ts +export function currentMode(): "dev" | "prod" { + const path = devModeWrapperPath(); + if (!existsSync(path)) return "prod"; + const prefix = readWrapperPrefix(path); + return prefix !== null && isDevModeWrapperContent(prefix) ? "dev" : "prod"; +} +``` + +(Keep the existing `openSync`/`readSync`/`closeSync` imports; add `Buffer` if not already available via global.) + +- [ ] **Step 5: Delegate from `lib/deps/links.ts`** + +Replace `isDevModeWrapper`: + +```ts +import { isDevModeWrapperContent } from "../dev-mode.ts"; + +function isDevModeWrapper(p: Pick, path: string): boolean { + const content = p.readFile(path); + if (content === null) return false; + return isDevModeWrapperContent(content.slice(0, 4096)); +} +``` + +(`p.readFile` returns the whole file here; slicing to 4096 keeps the detector bounded and consistent with `currentMode`. If a `Probes` bounded-read seam exists, prefer it; otherwise the slice suffices for the string comparison.) + +- [ ] **Step 6: Run tests + type-check** + +Run: `bun test lib/__tests__/dev-mode.test.ts lib/deps/__tests__/links.test.ts && bunx tsc --noEmit` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add commands/settings.ts lib/dev-mode.ts lib/deps/links.ts lib/__tests__/dev-mode.test.ts +git commit -m "dev-mode: marker-based wrapper detection, bounded read, legacy fallback (S020/S067)" +``` + +--- + +## Task 6: Unsupported-platform row at setup (R051) + +**Files:** +- Modify: `lib/setup/validators/mac.ts` (add `archRow`, include in `macRows`) +- Test: `lib/setup/validators/__tests__/mac.test.ts` (or the existing mac validator test) + +**Interfaces:** +- Consumes: `Probes` (`p.exec(argv)` → `{ stdout, code }`), `row()` from `lib/setup/contract.ts`. + +- [ ] **Step 1: Write the failing tests** + +```ts +import { macRows } from "../mac.ts"; + +function probes(uname: { stdout: string; code: number }) { + return { exec: async (argv: string[]) => (argv[0] === "uname" ? uname : { stdout: "", code: 0 }), exists: () => false, readFile: () => null, env: {}, home: "/h" } as any; +} + +test("arm64 -> ready", async () => { + const rows = await macRows(probes({ stdout: "arm64", code: 0 })); + const arch = rows.find((r) => r.id === "tool.arch")!; + expect(arch.status).toBe("ready"); +}); +test("x86_64 -> invalid", async () => { + const rows = await macRows(probes({ stdout: "x86_64", code: 0 })); + expect(rows.find((r) => r.id === "tool.arch")!.status).toBe("invalid"); +}); +test("probe failure -> error, not invalid", async () => { + const rows = await macRows(probes({ stdout: "", code: 127 })); + expect(rows.find((r) => r.id === "tool.arch")!.status).toBe("error"); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test lib/setup/validators/__tests__/mac.test.ts -t arch` +Expected: FAIL (no `tool.arch` row). + +- [ ] **Step 3: Add `archRow` and include it** + +In `lib/setup/validators/mac.ts`: + +```ts +async function archRow(p: Probes): Promise { + const base = { id: "tool.arch", kind: "tool" as const, title: "Processor", + why: "mattstack ships an Apple-silicon (arm64) build; Intel Macs are not supported.", required: true }; + const res = await p.exec(["uname", "-m"]); + const arch = res.stdout.trim(); + if (res.code !== 0 || !arch) return row({ ...base, status: "error", detail: "Could not determine your processor" }); + if (arch === "arm64") return row({ ...base, status: "ready", detail: "Apple silicon (arm64)" }); + return row({ ...base, status: "invalid", detail: `${arch}: Apple silicon (arm64) required` }); +} +``` + +And change `macRows`: + +```ts +export async function macRows(p: Probes): Promise { + const [macos, clt, arch] = await Promise.all([macosVersionRow(p), cltRow(p), archRow(p)]); + return [macos, clt, arch, pathRow(p)]; +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test lib/setup/validators/__tests__/mac.test.ts` +Expected: PASS. (Note: an existing "macRows returns N rows" count assertion may need +1 ... update it.) + +- [ ] **Step 5: Commit** + +```bash +git add lib/setup/validators/mac.ts lib/setup/validators/__tests__/mac.test.ts +git commit -m "setup: arm64/unsupported-arch row at setup (R051)" +``` + +--- + +## Task 7: Home-repo first-run honesty (S090, R043) + +**Files:** +- Modify: `lib/daemon/home-snapshot.ts` (init `existsSync` check; identity check before commit; new SkipReason values) +- Modify: `lib/home/init-exec.ts` (identity check before the initial commit) +- Test: `lib/daemon/__tests__/home-snapshot.test.ts` (or the existing home-snapshot test) + +**Interfaces:** +- Consumes: `deps.exec` (async runCapture-shaped: `{ exitCode, stdout, stderr }`), `deps.repoDir`, `deps.log`. + +- [ ] **Step 1: Write the failing tests** + +For S090 (init) and R043 (identity), use the home-snapshot test's fake `exec`/`deps`: + +```ts +test("S090: missing repoDir is diagnosed not-provisioned, names rt home init", async () => { + // deps.repoDir points at a path that does not exist; exec is never reached for rev-parse. + const snap = makeSnapshot({ repoDir: "/does/not/exist" }); + await snap.init(); + expect(snap.__disabledReason()).toBe("not-provisioned"); + expect(warnsInclude(snap, "rt home init")).toBe(true); +}); + +test("R043: missing git identity blocks the commit with an actionable reason", async () => { + const exec = fakeExec({ + "git rev-parse --is-inside-work-tree": { exitCode: 0, stdout: "true" }, + "git config user.name": { exitCode: 1, stdout: "" }, + "git config user.email": { exitCode: 1, stdout: "" }, + }); + const snap = makeSnapshot({ repoDir: existingRepoDir, exec }); + await snap.init(); + const r = await snap.snapshot("watch"); + expect(r.skipped ?? snap.__disabledReason()).toBe("no-git-identity"); + expect(execCalled(exec, "git ... commit")).toBe(false); // never attempted +}); +``` + +(Executor: adapt to the test file's real seam names; the assertions ... `not-provisioned`, the `rt home init` string, `no-git-identity`, and "commit never attempted" ... are the contract.) + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test lib/daemon/__tests__/home-snapshot.test.ts -t "S090\|R043"` +Expected: FAIL. + +- [ ] **Step 3: Add the new SkipReason values** + +In `lib/daemon/home-snapshot.ts`, extend the `SkipReason` union (lines 46-55): + +```ts +export type SkipReason = + | "disabled" + | "not-a-repo" + | "not-provisioned" + | "no-git-identity" + | "init-failed" + | "detached" + | "merge-in-progress" + | "owners-read-error" + | "index-locked" + | "add-failed" + | "no-changes"; +``` + +- [ ] **Step 4: S090 ... existsSync guard in `init()`** + +In `init()` (around line 357), before the `git rev-parse` spawn: + +```ts + async function init(): Promise { + try { + if (!existsSync(deps.repoDir)) { + disabledReason = "not-provisioned"; + deps.log.warn({ repoDir: deps.repoDir }, "home-snapshot: home repo not provisioned; run `rt home init`; inert"); + return; + } + const check = await deps.exec(["git", "rev-parse", "--is-inside-work-tree"], { /* unchanged */ }); + // ... existing exitCode === -1 / not-a-repo branches unchanged ... +``` + +(`existsSync` is already imported at line 24.) + +- [ ] **Step 5: R043 ... identity check before the snapshot commit** + +In the snapshot path, immediately before the commit spawn (line 692), gate once: + +```ts + const name = await deps.exec(["git", "config", "user.name"], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, stderr: "pipe" }); + const email = await deps.exec(["git", "config", "user.email"], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, stderr: "pipe" }); + if (name.exitCode !== 0 || !name.stdout.trim() || email.exitCode !== 0 || !email.stdout.trim()) { + disabledReason = "no-git-identity"; + if (lastLoggedCommitError !== "no-git-identity") { + deps.log.warn("home-snapshot: no git identity; run `git config --global user.name` and `git config --global user.email`; snapshots inert"); + lastLoggedCommitError = "no-git-identity"; + } + return { committed: false, sha: null, paths: [], reason, skipped: "no-git-identity" }; + } + const message = /* unchanged */; + const commitResult = await deps.exec(["git", "-c", "commit.gpgsign=false", "commit", ...]); +``` + +(Return shape mirrors the existing skipped-return objects in this function ... executor matches the actual local return type; the contract is: identity missing → skip with `no-git-identity`, commit never attempted, logged once.) + +- [ ] **Step 6: R043 companion ... initial commit in `init-exec.ts`** + +In `lib/home/init-exec.ts commitInitialUserRepo` (line 82), before the `commit`: + +```ts + case "commitInitialUserRepo": { + log("committing the initial user/ tree"); + await run(exec, ["git", "-C", "user", "add", "-A"]); + const name = await exec.run(["git", "-C", "user", "config", "user.name"]); + const email = await exec.run(["git", "-C", "user", "config", "user.email"]); + if (name.code !== 0 || !name.stdout.trim() || email.code !== 0 || !email.stdout.trim()) { + throw new StepFailed("no git identity: run `git config --global user.name` and `git config --global user.email`, then re-run `rt home init`"); + } + const result = await exec.run(["git", "-c", "commit.gpgsign=false", "-C", "user", "commit", "-m", "initial home repo"]); + // ... existing nothing-to-commit tolerance unchanged ... +``` + +- [ ] **Step 7: Run tests + type-check** + +Run: `bun test lib/daemon/__tests__/home-snapshot.test.ts lib/home/__tests__/init-exec.test.ts && bunx tsc --noEmit` +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon/home-snapshot.ts lib/home/init-exec.ts lib/daemon/__tests__/home-snapshot.test.ts lib/home/__tests__ +git commit -m "home-snapshot: diagnose not-provisioned and missing git identity (S090/R043)" +``` + +--- + +## Task 8: Timeout on the sops secrets spawn (S070, sops half) + +**Files:** +- Modify: `lib/secrets/store.ts` (`createRealSecretsExecSeam`: add timeout/kill + `SecretsTimeoutError`) +- Test: `lib/secrets/__tests__/store.test.ts` + +**Interfaces:** +- Produces: `export class SecretsTimeoutError extends Error`. + +- [ ] **Step 1: Write the failing test** + +```ts +import { createRealSecretsExecSeam, SecretsTimeoutError } from "../store.ts"; + +test("a sops spawn that never exits times out with SecretsTimeoutError, does not hang", async () => { + const seam = createRealSecretsExecSeam(); + // Inject a hanging command via the seam's spawn boundary; use a fixture like + // ["sh", "-c", "trap '' TERM; sleep 60"] with a short timeout override. + await expect(seam.run(["sh", "-c", "trap '' TERM; sleep 60"], { timeoutMs: 200 } as any)) + .rejects.toBeInstanceOf(SecretsTimeoutError); +}, 5_000); +``` + +(Executor: the real seam resolves argv[0] via `resolveBundledTool`; for the test, either pass a plain command that resolves to itself, or add a spawn-injection seam mirroring age-key's testability. The contract: a non-exiting child rejects with `SecretsTimeoutError` within the timeout, and the killable child is terminated.) + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/secrets/__tests__/store.test.ts -t "times out"` +Expected: FAIL (`SecretsTimeoutError` undefined; the call hangs). + +- [ ] **Step 3: Add the error + timeout, mirroring `age-key.ts`** + +In `lib/secrets/store.ts`, add near the other error classes (after `InvalidSecretsSegmentError`): + +```ts +/** Thrown when a sops/keychain spawn does not exit in time (a locked keychain pops a GUI dialog and blocks until clicked). */ +export class SecretsTimeoutError extends Error {} + +const DEFAULT_SECRETS_TIMEOUT_MS = 30_000; +``` + +In `createRealSecretsExecSeam`'s `run` (lines 470-484), wrap the await with the same timer pattern `age-key.ts` uses: + +```ts + async run(cmd, opts) { + debugLog(cmd, opts?.sensitive); + const [bin, ...args] = cmd; + const resolved = bin === undefined ? cmd : [resolveBundledTool(bin), ...args]; + const proc = Bun.spawn(resolved, buildSecretsSpawnOptions({ env: opts?.env, cwd })); + const timeoutMs = (opts as { timeoutMs?: number } | undefined)?.timeoutMs ?? DEFAULT_SECRETS_TIMEOUT_MS; + let timedOut = false; + const timer = setTimeout(() => { timedOut = true; try { proc.kill(); } catch { /* already exited */ } }, timeoutMs); + try { + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + if (timedOut) throw new SecretsTimeoutError(`${cmd[0]}: did not exit within ${timeoutMs}ms (keychain prompt pending?)`); + return { code, stdout, stderr }; + } finally { clearTimeout(timer); } + }, +``` + +(If `SecretsExecSeam.run`'s opts type has no `timeoutMs`, add it to the interface as optional.) + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/secrets/__tests__/store.test.ts -t "times out"` +Expected: PASS. + +- [ ] **Step 5: Type-check + commit** + +Run: `bunx tsc --noEmit` + +```bash +git add lib/secrets/store.ts lib/secrets/__tests__/store.test.ts +git commit -m "secrets: timeout + SecretsTimeoutError on the sops spawn (S070 sops half)" +``` + +--- + +## Task 9: Branch-cache key helpers (S069, part 1 of 2) + +**Files:** +- Modify: `lib/state/branch-cache.ts` (add pure helpers + `get`/`getByBranch`; store still bare-keyed here) +- Test: `lib/state/__tests__/branch-cache.test.ts` + +**Interfaces:** +- Produces: `export function composeKey(identity: string | undefined, branch: string): string`; `export function branchOf(key: string): string`; `export function identityOf(key: string): string | undefined`; store methods `get(identity: string | undefined, branch: string): CacheEntry | undefined` and `getByBranch(branch: string): CacheEntry | undefined`. + +- [ ] **Step 1: Write the failing tests** + +```ts +import { composeKey, branchOf, identityOf } from "../branch-cache.ts"; + +test("composeKey/branchOf/identityOf round-trip with a serialized identity", () => { + const id = "remote:gitlab.com%2Facme%2Facme-dev"; + const k = composeKey(id, "feature/x"); + expect(k).toBe(`${id}:feature/x`); + expect(branchOf(k)).toBe("feature/x"); + expect(identityOf(k)).toBe(id); +}); +test("bare key (no identity) degrades gracefully", () => { + expect(composeKey(undefined, "main")).toBe("main"); + expect(branchOf("main")).toBe("main"); + expect(identityOf("main")).toBeUndefined(); +}); +test("branch never contains a colon, so lastIndexOf split is unambiguous", () => { + const k = composeKey("path:%2FUsers%2Fdev%2Fscratch", "release"); + expect(branchOf(k)).toBe("release"); + expect(identityOf(k)).toBe("path:%2FUsers%2Fdev%2Fscratch"); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test lib/state/__tests__/branch-cache.test.ts -t "composeKey"` +Expected: FAIL (helpers missing). + +- [ ] **Step 3: Add the helpers + accessors** + +In `lib/state/branch-cache.ts`, add module-level: + +```ts +/** state.db keys the branch cache on `${serializedIdentity}:${branch}`. Split + * on the LAST colon: git branch names contain none, serialized identities + * always carry their own (remote:/path:), so this is unambiguous. */ +export function composeKey(identity: string | undefined, branch: string): string { + return identity ? `${identity}:${branch}` : branch; +} +export function branchOf(key: string): string { + const i = key.lastIndexOf(":"); + return i < 0 ? key : key.slice(i + 1); +} +export function identityOf(key: string): string | undefined { + const i = key.lastIndexOf(":"); + return i < 0 ? undefined : key.slice(0, i); +} +``` + +In `createStore`, add to the returned object: + +```ts + function get(identity: string | undefined, branch: string): CacheEntry | undefined { + return entries[composeKey(identity, branch)]; + } + function getByBranch(branch: string): CacheEntry | undefined { + const suffix = `:${branch}`; + for (const [k, v] of Object.entries(entries)) if (k === branch || k.endsWith(suffix)) return v; + return undefined; + } +``` + +and include `get, getByBranch` in the returned store object and in the `BranchCacheStore` interface. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test lib/state/__tests__/branch-cache.test.ts && bunx tsc --noEmit` +Expected: PASS (helpers are additive; store behavior unchanged this task). + +- [ ] **Step 5: Commit** + +```bash +git add lib/state/branch-cache.ts lib/state/__tests__/branch-cache.test.ts +git commit -m "branch-cache: add composeKey/branchOf/identityOf + get/getByBranch (S069 part 1)" +``` + +--- + +## Task 10: Flip branch-cache to the composite key (S069, part 2 of 2) ... ATOMIC + +This is the one multi-file atomic change: the store key becomes composite and every direct-lookup consumer switches in the same commit. Intermediate states are not green, so land it as one commit after the whole suite passes. Read contract: every by-branch lookup still resolves a bare branch, scoped to the caller's repo (exact key) or suffix-matched across repos when the repo is unknown; `cache:read`, CLI, board, and tray see bare branch names exactly as before. + +**Files:** +- Modify: `lib/state/branch-cache.ts` (`put` keys off `entry.repoName`) +- Modify: `lib/enrich.ts` (cold-start sets `repoName` from identity; `allCached`/lookup use composeKey) +- Modify: `lib/notifier.ts` (loop var is the composite key; `branchOf` only for display) +- Modify: `lib/daemon/worktree-reconciler.ts` (`branchOf(key)` for the bare branch) +- Modify: `lib/daemon/freshness.ts` (direct lookups compose; iterations use `branchOf`) +- Modify: `lib/daemon/handlers/cache.ts` (`cache:read` returns bare-branch keys via suffix-match; optional `repoIdentity`) +- Modify: `commands/status/data.ts` (display `branchOf(row.branch)`) +- Test: add cases to `branch-cache.test.ts`, `enrich` test, `notifier` test, `worktree-reconciler` test, `freshness` test, `cache` handler test. + +**Interfaces:** +- Consumes: `composeKey/branchOf/identityOf/getByBranch` (Task 9); `serializeIdentity`, `identityFromRemote` from `lib/settings/identity.ts`. + +- [ ] **Step 1: Write the failing tests (collision-safety across all sites)** + +```ts +// branch-cache: two repos, same branch, coexist +test("put keys by entry.repoName so same-name branches in two repos coexist", () => { + const s = makeStore(); // over a temp db + s.put("main", { repoName: "remote:host%2Fa", ticket: null, linearId: "", mr: null, fetchedAt: 1 }); + s.put("main", { repoName: "remote:host%2Fb", ticket: null, linearId: "", mr: null, fetchedAt: 2 }); + expect(s.get("remote:host%2Fa", "main")!.fetchedAt).toBe(1); + expect(s.get("remote:host%2Fb", "main")!.fetchedAt).toBe(2); + expect(Object.keys(s.entries).length).toBe(2); +}); + +// cache:read returns bare-branch keys +test("cache:read returns bare branch names (suffix match), never composite keys", async () => { + // seed ctx.cache with a composite-keyed entry, call the handler with branches:["main"] + const res = await handler["cache:read"]({ branches: ["main"] }); + expect(Object.keys(res.data)).toEqual(["main"]); +}); + +// notifier: same branch, two repos, independent fired-state +test("evicting one repo's branch does not prune the other repo's fired key", () => { + // build cacheEntries with composeKey("remote:host%2Fa","main") and ...b/main + // fire on a's main, then run checkAndNotify with only b/main present + // assert a's fired key survives (it is keyed by the composite branch var) +}); + +// reconciler: mrState only from the reconciled repo +test("reactor builds mrState only from the reconciled repo's entries", async () => { + // cacheEntries has ${A}:main (opened->merged) and ${B}:main (opened) + // run for repo A; assert only A:main transitions, B untouched +}); + +// freshness: a branch in two repos resolves to the right repo's entry +test("updateEntry composes the repo-scoped key", () => { + // seed ${A}:main and ${B}:main; updateEntry(env, A, "main", pr) + // assert only ${A}:main.mr changed +}); +``` + +(These are the acceptance contracts; the executor fills fixtures using each test file's existing helpers.) + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test lib/state/__tests__/branch-cache.test.ts lib/daemon/__tests__ lib/__tests__/notifier.test.ts` +Expected: FAIL (collisions overwrite; composite keys not yet used). + +- [ ] **Step 3: `branch-cache.ts` ... `put` keys off `entry.repoName`** + +In `createStore.put`, derive the key and use it for BOTH the row PK and the map (the `branch` column now stores the composite key; `repo` still stores the identity, so `gc`/`reload`/`delete` keep working transparently since they operate on the `branch` column value): + +```ts + function put(branch: string, entry: CacheEntry): void { + const key = composeKey(entry.repoName, branch); + persistOrWarn("branch-cache", () => { + db.query(UPSERT_SQL).run( + key, + entry.repoName ?? null, + entry.ticket !== null ? JSON.stringify(entry.ticket) : null, + entry.linearId, + entry.mr !== null ? JSON.stringify(entry.mr) : null, + entry.fetchedAt, + ); + }, { op: "put", branch: key }); + entries[key] = entry; + } +``` + +`delete(branch)` callers pass a key that is already the map key; if any caller passes a bare branch, route it through `getByBranch`/`composeKey` at that call site. `gc` is unchanged (it deletes by the `branch` column value, which is now the composite key, and gates on `row.repo` = identity). + +- [ ] **Step 4: `enrich.ts` ... cold-start sets identity; lookups compose** + +In `fetchAndCache`, compute the identity once and set it on every entry it writes, and in `enrichBranches`'s cached path compose the key: + +```ts +import { composeKey } from "./state/branch-cache.ts"; +import { serializeIdentity, identityFromRemote } from "./settings/identity.ts"; + +// inside fetchAndCache: derive identity from remoteUrl (best-effort; undefined if no remote) +const identity = remoteUrl ? serializeIdentity(identityFromRemote(remoteUrl)) : undefined; +// ...when building each CacheEntry, set repoName: identity +// ...store.put(branch, { ...entry, repoName: identity }) + +// inside enrichBranches cached path: +const allCached = !options?.forceRefresh && willFetch + && branches.every((b) => composeKey(identity, b.branch) in store.entries); +// ... +const entry = store.entries[composeKey(identity, b.branch)]!; +``` + +(`enrichBranches` must compute `identity` from its `remoteUrl` the same way, before the cached check.) + +- [ ] **Step 5: `notifier.ts` ... composite key through, `branchOf` for display** + +`detectBranchTransitions` and the `checkAndNotify` snapshot loop already key `state.branches`, `newBranches`, `firedKey`, and `pruneFiredForEvictedBranches` off the `cacheEntries` map keys. With composite keys those become repo-scoped automatically. The only change: wherever a human-readable branch name is put into a notification message, use `branchOf(key)`. Add `import { branchOf } from "../state/branch-cache.ts";` (adjust path) and apply it at the message-construction sites inside `detectBranchTransitions`. + +- [ ] **Step 6: `worktree-reconciler.ts` ... `branchOf(key)` for the bare branch** + +In the reactor loop (line 594 onward), the loop key is now the composite key. Derive the bare branch for registry lookups; keep the repo filter: + +```ts + for (const [key, entry] of Object.entries(cacheEntries)) { + if (entry.repoName && entry.repoName !== repoName) continue; + if (!entry.mr) continue; + const branch = branchOf(key); + // ... `const mrKey = prefix + branch;` (rename the local `key` used for mrState to `mrKey` + // to avoid colliding with the composite map key) ... + // findByBranch(loadRegistry(repoName), branch) and resumeTrees(deps, branch) use the bare branch. + } +``` + +Add `import { branchOf } from "../state/branch-cache.ts";`. Rename the existing local `const key = prefix + branch;` to `mrKey` and update its uses (`nextMrState[mrKey]`, `state.mrState[mrKey]`). + +- [ ] **Step 7: `freshness.ts` ... compose direct lookups, `branchOf` iterations** + +`repoName` here is the serialized identity, so: + +```ts +import { composeKey, branchOf } from "../state/branch-cache.ts"; + +// line 505 branchByIid: iterate, filter entry.repoName !== repoName, store bare branch: +for (const [key, entry] of Object.entries(ctx.cache.entries)) { + if (entry.repoName !== repoName) continue; + if (typeof entry.mr?.iid === "number") branchByIid.set(entry.mr.iid, branchOf(key)); +} + +// line 545: `ctx.cache.entries[pr.sourceBranch]?.repoName === repoName` +// -> `ctx.cache.entries[composeKey(repoName, pr.sourceBranch)] !== undefined` + +// line 579: `const entry = ctx.cache.entries[k.ref];` then `entry.repoName === repoName` +// -> `const entry = ctx.cache.entries[composeKey(repoName, k.ref)];` (the repoName check is then redundant) + +// updateEntry (line 637): compose for both read and write +function updateEntry(env, repoName, branch, pr) { + const key = composeKey(repoName, branch); + const existing = env.ctx.cache.entries[key]; + if (!existing) return false; + env.ctx.cache.put(branch, { ...existing, mr: pr ? toMRInfo(pr) : null, fetchedAt: Date.now(), repoName }); + // ... +} + +// applyMRWriteback (657) + runGapFill (703): iterate, filter by entry.repoName, map keys via branchOf, +// pass bare branch to updateEntry (which recomposes). +``` + +- [ ] **Step 8: `handlers/cache.ts` ... bare-branch output via suffix-match** + +`cache:read` must return bare-branch keys. Accept an optional `repoIdentity` for exact scoping; otherwise suffix-match. Import `branchOf`/`getByBranch` semantics: + +```ts +"cache:read": async (payload) => { + const branches = payload?.branches as string[] | undefined; + const repoIdentity = payload?.repoIdentity as string | undefined; + const maxAgeMs = payload?.maxAgeMs as number | undefined; + + const lookup = (b: string): CacheEntry | undefined => + repoIdentity ? ctx.cache.entries[`${repoIdentity}:${b}`] + : Object.entries(ctx.cache.entries).find(([k]) => k === b || k.endsWith(`:${b}`))?.[1]; + + if (typeof maxAgeMs === "number") { + const pool = branches ?? Object.keys(ctx.cache.entries).map(branchOf); + let oldest = 0; + if (pool.length > 0) oldest = Math.min(...pool.map((b) => lookup(b)?.fetchedAt ?? 0)); + if (Date.now() - oldest >= maxAgeMs) await ctx.refreshCache(); + } + + if (!branches) { + const out: Record = {}; + for (const [k, v] of Object.entries(ctx.cache.entries)) out[branchOf(k)] = v; // bare-branch keyed + return { ok: true, data: out }; + } + const filtered: Record = {}; + for (const b of branches) { const e = lookup(b); if (e) filtered[b] = e; } + return { ok: true, data: filtered }; +}, +``` + +Add `import { branchOf } from "../../state/branch-cache.ts";` (adjust path). `branch:enrich`'s `ctx.cache.entries[branch]` lookups: compose with the payload's repo identity when present, else `getByBranch`-style suffix match. + +- [ ] **Step 9: `commands/status/data.ts` ... display bare branch** + +In `readBranchesFromStateDb`, key the returned dict by the bare branch: + +```ts +import { branchOf } from "../../lib/state/branch-cache.ts"; +// ... +for (const row of rows) { + branches[branchOf(row.branch)] = { + ticket: row.ticket !== null ? JSON.parse(row.ticket) : null, + linearId: row.linear_id, + mr: row.mr !== null ? JSON.parse(row.mr) : null, + fetchedAt: row.fetched_at, + repoName: row.repo ?? undefined, + }; +} +``` + +- [ ] **Step 10: Run the whole affected suite + type-check** + +Run: `bun test lib commands packages scripts && bunx tsc --noEmit` +Expected: PASS, zero errors. Confirm `lib/daemon/discussions-poller.ts` needs no change (it iterates `Object.values`, self-healing). + +- [ ] **Step 11: Commit (single atomic commit)** + +```bash +git add lib/state/branch-cache.ts lib/enrich.ts lib/notifier.ts lib/daemon/worktree-reconciler.ts lib/daemon/freshness.ts lib/daemon/handlers/cache.ts commands/status/data.ts lib/state/__tests__ lib/daemon/__tests__ lib/__tests__/notifier.test.ts commands/__tests__ +git commit -m "branch-cache: flip to composite ${identity}:${branch} key; scope all consumers (S069 part 2)" +``` + +--- + +## Final verification (run before the whole-branch review) + +- [ ] `bun test lib commands packages scripts` green (worktree root). +- [ ] `bunx tsc --noEmit` reports zero errors. +- [ ] `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts e2e/tests/setup.test.ts e2e/tests/first-run.test.ts` green (run full `bun run test:e2e` if practical; record which was run). +- [ ] `lib/__tests__/no-daemon-sync-exec.test.ts` green with the `user-path.ts` allowlist entry removed. +- [ ] `cd packages/rt-client && bun run build && bun test test/dist-freshness.test.ts` green. + +## Self-review (author checklist ... completed before saving) + +- **Spec coverage:** every spec item maps to a task ... 6.1 → Tasks 1-3; S071 → Task 4; S020/S067 → Task 5; R051 → Task 6; S090/R043 → Task 7; S070 sops → Task 8; S069 → Tasks 9-10. +- **Placeholder scan:** no TBD/TODO; new code is inlined; edit sites carry before/after snippets and exact anchors. +- **Type consistency:** `composeKey/branchOf/identityOf` used identically in Tasks 9-10; `ProbeFn` signature consistent in Task 2; `SkipReason` additions consistent in Task 7. From dcca3ffeb060fe378b71bf1279c112eaac53e7bf Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 16:54:17 -0500 Subject: [PATCH 097/142] plan: apply reviewer fixes (loop-monitor test/alloc, CacheRefresherDeps widen, seq in ping, second fetch tag) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../superpowers/plans/2026-08-28-p2-health.md | 60 +++++++++++++------ 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/docs/superpowers/plans/2026-08-28-p2-health.md b/docs/superpowers/plans/2026-08-28-p2-health.md index 360d9f9d..22834bd9 100644 --- a/docs/superpowers/plans/2026-08-28-p2-health.md +++ b/docs/superpowers/plans/2026-08-28-p2-health.md @@ -385,7 +385,7 @@ git commit -m "add lib/daemon/heartbeat-file.ts: atomic-rename heartbeat" **Interfaces:** - Consumes: nothing (pure `applyTick` plus a thin timer wrapper). -- Produces: `applyTick(stats, expected, now, opts): void` (pure), `startLoopMonitor(opts): { stats: LoopStats; stop: () => void }`, `interface LoopStats { lagMs; maxLagMs; stalls; lastStallAt; lastStallCmd; currentlyStalled }`. +- Produces: `applyTick(stats, expected, now, cmd, opts, onStall): void` (pure), `startLoopMonitor(opts): { stats: LoopStats; seq: () => number; stop: () => void }`, `interface LoopStats { lagMs; maxLagMs; stalls; lastStallAt; lastStallCmd; currentlyStalled }`. - [ ] **Step 1: Write the failing test** (test the pure tick math; the timer wrapper is thin) @@ -418,10 +418,11 @@ test("a >1s drift counts a stall, records the in-flight cmd, and warns", () => { test("currentlyStalled is true when the last big drift is within stallRecentMs", () => { const s = newLoopStats(); - applyTick(s, 1000, 3500, "x", OPTS, () => {}); // 2500ms drift >= 2000 unhealthy + applyTick(s, 1000, 3500, "x", OPTS, () => {}); // 2500ms drift >= 2000 unhealthy, lastStallAt=3500 expect(s.currentlyStalled).toBe(true); - // a later on-time tick outside the recent window clears it - applyTick(s, 3500 + 250, 3500 + 250 + 20_000, null, OPTS, () => {}); + // a small-drift tick whose `now` is past lastStallAt + stallRecentMs clears it + // (10ms drift, so no new stall; now-lastStallAt = 10500 > 10000 recent window) + applyTick(s, 13990, 14000, null, OPTS, () => {}); expect(s.currentlyStalled).toBe(false); }); @@ -491,7 +492,7 @@ export function applyTick( stats.currentlyStalled = stats.lastStallAt !== null && now - stats.lastStallAt <= opts.stallRecentMs && - (drift > opts.stallUnhealthyMs || stats.maxLagMs > opts.stallUnhealthyMs && now - stats.lastStallAt <= opts.stallRecentMs); + (drift > opts.stallUnhealthyMs || stats.maxLagMs > opts.stallUnhealthyMs); } export interface LoopMonitorOpts { @@ -519,14 +520,18 @@ export function startLoopMonitor(opts: LoopMonitorOpts): { stats: LoopStats; sto let seq = 0; let warnedThisStall = false; + // Hoisted once (allocation-free ruling): the tick must not build a fresh + // closure every 250ms. onStall closes over warnedThisStall by reference. + const onStall = (drift: number, cmd: string | null): void => { + if (!warnedThisStall) { + opts.log.warn({ driftMs: drift, cmd }, "event loop stalled"); + warnedThisStall = true; + } + }; + const timer = setInterval(() => { const now = Date.now(); - applyTick(stats, expected, now, opts.currentCmd(), tickOpts, (drift, cmd) => { - if (!warnedThisStall) { - opts.log.warn({ driftMs: drift, cmd }, "event loop stalled"); - warnedThisStall = true; - } - }); + applyTick(stats, expected, now, opts.currentCmd(), tickOpts, onStall); if (stats.lagMs <= tickOpts.stallLogMs) warnedThisStall = false; expected = now + tickMs; if (now - lastHeartbeat >= heartbeatMs) { @@ -536,7 +541,7 @@ export function startLoopMonitor(opts: LoopMonitorOpts): { stats: LoopStats; sto }, tickMs); timer.unref(); - return { stats, stop: () => clearInterval(timer) }; + return { stats, seq: () => seq, stop: () => clearInterval(timer) }; } ``` @@ -1174,7 +1179,7 @@ export function emitSettingsWarning(msg: string): void { console.warn(msg); } ``` -Replace the three `console.warn(...)` calls (`warnInvalid` line ~491, `listSettings` line ~546, `listUnregistered` line ~586) with `emitSettingsWarning(...)` passing the same message string. +Replace the three `console.warn(...)` calls (`warnInvalid` line ~491, `listSettings` line ~546, `listUnregistered` line ~586) with `emitSettingsWarning(...)` passing the same message string. Note: dedupe-by-message collapses to the spec's per-`(key, scope, reason)` tuple only because each message string is deterministic in exactly those fields (`warnInvalid` interpolates key + scope + file + reason). That holds for all three sites today; keep it true if you edit the message text. Export `setSettingsWarnSink` from `packages/rt-client/src/index.ts` alongside the other settings exports: ```ts @@ -1392,6 +1397,7 @@ In `lib/daemon-client.ts` `trySocketQuery`, always send the client header (restr signal: AbortSignal.timeout(timeoutMs), } as any); ``` +`lib/daemon-client.ts` has a **second** header-building `fetch(...)` path (around lines 155-160, a separate request helper); add the identical `X-RT-Client` header there too so every rt-CLI request is attributed, not just `trySocketQuery`'s. In `packages/rt-client/src/transport.ts` `rtCommand`, add the header (the caller label defaults to the package’s consumer; use a generic tag): ```ts @@ -1488,10 +1494,13 @@ git commit -m "servers: thread X-RT-Client into payload._client; advertise it in - [ ] **Step 1: Extend the type** -In `lib/daemon/handlers/types.ts`, change the `refreshStatusRef` field and add `getHealth`: +In `lib/daemon/handlers/types.ts`, change the `refreshStatusRef` field and add `getHealth` + `heartbeatSeq` (the ping handler in Task 14 echoes the seq; declaring it here keeps `ctx` typed): ```ts refreshStatusRef: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }; getHealth: () => import("../health.ts").HealthSnapshot; + heartbeatSeq: () => number; + setLogLevel: (l: string) => void; // Task 15 uses these; declare now to avoid a second types.ts edit + getLogLevel: () => string; ``` - [ ] **Step 2: Update the init site and the refresher** @@ -1500,7 +1509,12 @@ In `lib/daemon.ts` line ~210: ```ts const refreshStatusRef = { lastRefreshAt: 0, lastSuccessAt: 0, failedRepos: 0, enrichErrors: 0 }; ``` -In `lib/daemon/cache-refresh.ts`, where the cycle finishes (currently sets `refreshStatusRef.lastRefreshAt = Date.now()` at ~line 195), also record the cycle outcome from the `failedRepos`/`enrichErrors` locals already computed in `refreshCacheImpl`: +In `lib/daemon/cache-refresh.ts`, widen the `CacheRefresherDeps` type's own `refreshStatusRef` field (declared around line 32) to match, or `tsc` fails at the pass into `createCacheRefresher`: +```ts + // CacheRefresherDeps (cache-refresh.ts ~line 32): + refreshStatusRef: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }; +``` +Then, where the cycle finishes (currently sets `refreshStatusRef.lastRefreshAt = Date.now()` at ~line 195), also record the cycle outcome from the `failedRepos`/`enrichErrors` locals already computed in `refreshCacheImpl`: ```ts refreshStatusRef.lastRefreshAt = Date.now(); refreshStatusRef.failedRepos = failedRepos.size; @@ -1665,7 +1679,10 @@ import { setSettingsWarnSink } from "@mattstack/rt-client"; setSettingsWarnSink((m) => log.warn({ src: "settings" }, m)); const healthSampler = createHealthSampler({ - log, rtDir: RT_DIR, wsClients: apiWsClientCount, watchers: () => watchedConfigs.size, startedAt, + // Source the watched-configs map exactly as handlerCtx.watchedConfigs is + // sourced today: it is hooksGuard.watchedConfigs (there is no bare + // `watchedConfigs` alias in daemon.ts scope). + log, rtDir: RT_DIR, wsClients: apiWsClientCount, watchers: () => hooksGuard.watchedConfigs.size, startedAt, }); healthSampler.sample(); // seed baseline/free immediately safeInterval(() => healthSampler.sample(), 5 * 60_000, "health-sample", log); @@ -1699,6 +1716,12 @@ function buildHealthSnapshot() { }); } ``` + +Add `heartbeatSeq: loopMon.seq` to the `handlerCtx` object literal (and `heartbeatSeq: () => number` to `HandlerContext` in `types.ts`) so `ping` can echo the current heartbeat sequence per the spec: +```ts + getHealth: buildHealthSnapshot, + heartbeatSeq: loopMon.seq, +``` Add `getHealth: buildHealthSnapshot` to the `handlerCtx` object literal (lines ~353-364). Import `getFreshnessSnapshot` if not already in `daemon.ts` scope (it lives in `lib/daemon/freshness.ts`). Ensure `loopMon.stop()` is called in `cleanup()`. - [ ] **Step 5: Surface the snapshot in the handlers** @@ -1730,7 +1753,7 @@ In `lib/daemon/handlers/status.ts`: const { bootAttempts, lastReadyAt, recentFailures, lastExit } = readSupervisionState(); const h = ctx.getHealth(); return { ok: true, uptime: Date.now() - ctx.startedAt, pid: process.pid, ...ctx.identity, - health: h.level, eventLoop: h.eventLoop, + health: h.level, eventLoop: h.eventLoop, heartbeatSeq: ctx.heartbeatSeq(), supervision: { bootAttempts, lastReadyAt, recentFailures: recentFailures.slice(-3), lastExit } }; }, ``` @@ -1814,12 +1837,13 @@ export async function setLogLevel(args: string[] = []): Promise { console.log(formatLogLevelResult(res as any, Boolean(level))); } ``` -The `daemon:log-level` handler needs to set the live pino level on the singleton logger. Add it where the router is built (it needs `loggerHandle`); the cleanest spot is a handler that closes over `log`/`loggerHandle` in `lib/daemon.ts`’s routed map, or add it to `createStatusHandlers` by passing a `setLevel`/`getLevel` accessor on `ctx`. Minimal approach: extend `HandlerContext` with `setLogLevel: (l: string) => void` and `getLogLevel: () => string`, wired in `daemon.ts`: +The `daemon:log-level` handler sets the live pino level on the singleton logger. The `setLogLevel`/`getLogLevel` accessors are already declared on `HandlerContext` (Task 13); wire them in `daemon.ts`'s `handlerCtx` object literal: ```ts // in daemon.ts handlerCtx: setLogLevel: (l: string) => { log.level = l; log.info({ level: l }, "log level changed"); }, getLogLevel: () => log.level, ``` +The `daemon:log-level` verb lives in `createStatusHandlers` (it already closes over `ctx`). and the handler: ```ts "daemon:log-level": async (payload?: { level?: string }) => { From 3dcbfe7a8998679a600ad9ed5c8e637b7dceebfc Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 16:58:03 -0500 Subject: [PATCH 098/142] add lib/daemon/health.ts: pure computeHealth + thresholds --- lib/daemon/__tests__/health.test.ts | 75 +++++++++++++++++ lib/daemon/health.ts | 123 ++++++++++++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 lib/daemon/__tests__/health.test.ts create mode 100644 lib/daemon/health.ts diff --git a/lib/daemon/__tests__/health.test.ts b/lib/daemon/__tests__/health.test.ts new file mode 100644 index 00000000..387ef901 --- /dev/null +++ b/lib/daemon/__tests__/health.test.ts @@ -0,0 +1,75 @@ +// lib/daemon/__tests__/health.test.ts +import { test, expect } from "bun:test"; +import { computeHealth, type HealthInputs } from "../health.ts"; + +function base(): HealthInputs { + return { + now: 1_000_000, + uptimeMs: 60_000, + mem: { rss: 200 * 1024 * 1024, heapUsed: 50 * 1024 * 1024, external: 1 * 1024 * 1024 }, + rssBaseline: null, + wsClients: 0, + watchers: 3, + freshness: { "remote:gitlab/acme": { state: "live" } }, + refresh: { lastSuccessAt: 1_000_000 - 60_000, failedRepos: 0, enrichErrors: 0 }, + refreshIntervalMs: 5 * 60_000, + eventLoop: { maxLagMs: 20, lastStallAt: null, lastStallCmd: null, stalls: 0, currentlyStalled: false }, + supervisionFailuresLastHour: 0, + crashLooping: false, + loggerDegraded: false, + recoveredErrorRateLastWindow: 0, + freeBytes: 50 * 1024 * 1024 * 1024, + }; +} + +test("all-nominal inputs are ok with no reasons", () => { + const h = computeHealth(base()); + expect(h.level).toBe("ok"); + expect(h.reasons).toEqual([]); + expect(h.metrics.watchers).toBe(3); + expect(h.eventLoop.maxLagMs).toBe(20); +}); + +test("a degraded freshness watcher flips degraded and names refresh", () => { + const i = base(); + i.freshness = { "remote:gitlab/acme": { state: "degraded" } }; + const h = computeHealth(i); + expect(h.level).toBe("degraded"); + expect(h.reasons.some((r) => r.startsWith("refresh:"))).toBe(true); +}); + +test("failed repos in the last cycle flip degraded", () => { + const i = base(); + i.refresh = { lastSuccessAt: i.now - 60_000, failedRepos: 3, enrichErrors: 5 }; + expect(computeHealth(i).level).toBe("degraded"); +}); + +test("logger degraded flips unhealthy and names logging", () => { + const i = base(); + i.loggerDegraded = true; + const h = computeHealth(i); + expect(h.level).toBe("unhealthy"); + expect(h.reasons.some((r) => r.startsWith("logging:"))).toBe(true); +}); + +test("currently stalled event loop is unhealthy; unhealthy wins over a degraded signal", () => { + const i = base(); + i.eventLoop.currentlyStalled = true; + i.freshness = { r: { state: "degraded" } }; // also degraded + const h = computeHealth(i); + expect(h.level).toBe("unhealthy"); + expect(h.reasons[0]?.startsWith("event-loop:")).toBe(true); // unhealthy reasons first +}); + +test("critical disk is unhealthy; low disk is degraded", () => { + const crit = base(); crit.freeBytes = 50 * 1024 * 1024; + expect(computeHealth(crit).level).toBe("unhealthy"); + const low = base(); low.freeBytes = 300 * 1024 * 1024; + expect(computeHealth(low).level).toBe("degraded"); +}); + +test("stale refresh (older than 2 intervals) is degraded", () => { + const i = base(); + i.refresh = { lastSuccessAt: i.now - 11 * 60_000, failedRepos: 0, enrichErrors: 0 }; + expect(computeHealth(i).level).toBe("degraded"); +}); diff --git a/lib/daemon/health.ts b/lib/daemon/health.ts new file mode 100644 index 00000000..57c687a3 --- /dev/null +++ b/lib/daemon/health.ts @@ -0,0 +1,123 @@ +// lib/daemon/health.ts +/** + * Pure daemon health verdict. computeHealth takes a fully-gathered input + * struct (the daemon-side adapter does all I/O) and returns the level, the + * named reasons, and the metrics/eventLoop blocks the surfaces echo. + */ + +export const HEALTH_THRESHOLDS = { + refreshStaleMultiplier: 2, + rssSoftThresholdBytes: 1024 * 1024 * 1024, + rssGrowthPct: 50, + diskSoftFloorBytes: 500 * 1024 * 1024, + diskHardFloorBytes: 100 * 1024 * 1024, + restartsPerHourUnhealthy: 5, + recoveredErrorRate: 10, +} as const; + +export interface HealthMetrics { + rss: number; + heapUsed: number; + external: number; + uptimeMs: number; + wsClients: number; + watchers: number; +} + +export interface HealthEventLoop { + maxLagMs: number; + lastStallAt: number | null; + lastStallCmd: string | null; + stalls: number; +} + +export interface HealthInputs { + now: number; + uptimeMs: number; + mem: { rss: number; heapUsed: number; external: number }; + /** rss + timestamp from ~1h ago, for growth detection; null if not yet sampled. */ + rssBaseline: { rss: number; at: number } | null; + wsClients: number; + watchers: number; + freshness: Record; + refresh: { lastSuccessAt: number; failedRepos: number; enrichErrors: number }; + refreshIntervalMs: number; + eventLoop: HealthEventLoop & { currentlyStalled: boolean }; + supervisionFailuresLastHour: number; + crashLooping: boolean; + loggerDegraded: boolean; + recoveredErrorRateLastWindow: number; + freeBytes: number | null; + /** Deferred inputs (spec): wired in a later phase, ignored today. */ + busySkips?: number; + criticalWriteFailures?: number; +} + +export interface HealthSnapshot { + level: "ok" | "degraded" | "unhealthy"; + reasons: string[]; + metrics: HealthMetrics; + eventLoop: HealthEventLoop; +} + +function mb(bytes: number): number { + return Math.round(bytes / (1024 * 1024)); +} + +export function computeHealth(i: HealthInputs): HealthSnapshot { + const T = HEALTH_THRESHOLDS; + const unhealthy: string[] = []; + const degraded: string[] = []; + + // --- unhealthy --- + if (i.loggerDegraded) unhealthy.push("logging: disabled (ENOSPC)"); + if (i.eventLoop.currentlyStalled) unhealthy.push("event-loop: currently stalled"); + if (i.crashLooping || i.supervisionFailuresLastHour >= T.restartsPerHourUnhealthy) { + unhealthy.push(`restarts: ${i.supervisionFailuresLastHour} in the last hour`); + } + if (i.freeBytes !== null && i.freeBytes < T.diskHardFloorBytes) { + unhealthy.push(`disk: ${mb(i.freeBytes)}MB free (critical)`); + } + + // --- degraded --- + const degradedRepos = Object.values(i.freshness).filter((f) => f.state === "degraded").length; + if (degradedRepos > 0) degraded.push(`refresh: ${degradedRepos} watcher${degradedRepos !== 1 ? "s" : ""} degraded`); + if (i.refresh.failedRepos > 0 || i.refresh.enrichErrors > 0) { + degraded.push(`refresh: ${i.refresh.failedRepos} repos failing (auth?)`); + } + const refreshAge = i.now - i.refresh.lastSuccessAt; + if (i.refresh.lastSuccessAt > 0 && refreshAge > T.refreshStaleMultiplier * i.refreshIntervalMs) { + degraded.push(`refresh: last success ${Math.round(refreshAge / 1000)}s ago`); + } + if (i.mem.rss > T.rssSoftThresholdBytes) degraded.push(`memory: rss ${mb(i.mem.rss)}MB`); + if (i.rssBaseline && i.mem.rss > i.rssBaseline.rss * (1 + T.rssGrowthPct / 100)) { + degraded.push(`memory: rss grew >${T.rssGrowthPct}% in the last hour`); + } + if (i.eventLoop.maxLagMs > 500) degraded.push(`event-loop: lag ${i.eventLoop.maxLagMs}ms`); + if (i.recoveredErrorRateLastWindow > T.recoveredErrorRate) { + degraded.push(`errors: ${i.recoveredErrorRateLastWindow} recovered in 5min`); + } + if (i.freeBytes !== null && i.freeBytes >= T.diskHardFloorBytes && i.freeBytes < T.diskSoftFloorBytes) { + degraded.push(`disk: ${mb(i.freeBytes)}MB free`); + } + + const level = unhealthy.length > 0 ? "unhealthy" : degraded.length > 0 ? "degraded" : "ok"; + return { + level, + reasons: level === "ok" ? [] : [...unhealthy, ...degraded], + metrics: { + rss: i.mem.rss, + heapUsed: i.mem.heapUsed, + external: i.mem.external, + uptimeMs: i.uptimeMs, + wsClients: i.wsClients, + watchers: i.watchers, + }, + eventLoop: { + maxLagMs: i.eventLoop.maxLagMs, + lastStallAt: i.eventLoop.lastStallAt, + lastStallCmd: i.eventLoop.lastStallCmd, + stalls: i.eventLoop.stalls, + }, + }; +} From b8f1572f5f43d8c1641f2fb718fe1e5310fc4945 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:01:52 -0500 Subject: [PATCH 099/142] add lib/daemon/heartbeat-file.ts: atomic-rename heartbeat --- lib/daemon/__tests__/heartbeat-file.test.ts | 29 ++++++++++++++++ lib/daemon/heartbeat-file.ts | 38 +++++++++++++++++++++ 2 files changed, 67 insertions(+) create mode 100644 lib/daemon/__tests__/heartbeat-file.test.ts create mode 100644 lib/daemon/heartbeat-file.ts diff --git a/lib/daemon/__tests__/heartbeat-file.test.ts b/lib/daemon/__tests__/heartbeat-file.test.ts new file mode 100644 index 00000000..1a4fc93d --- /dev/null +++ b/lib/daemon/__tests__/heartbeat-file.test.ts @@ -0,0 +1,29 @@ +import { test, expect } from "bun:test"; +import { mkdtempSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { writeHeartbeat, readHeartbeat } from "../heartbeat-file.ts"; + +test("write then read round-trips", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + writeHeartbeat(dir, { at: 123, seq: 7 }); + expect(readHeartbeat(dir)).toEqual({ at: 123, seq: 7 }); +}); + +test("missing file reads as null", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + expect(readHeartbeat(dir)).toBeNull(); +}); + +test("corrupt file reads as null", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + writeFileSync(join(dir, "daemon-heartbeat.json"), "{not json"); + expect(readHeartbeat(dir)).toBeNull(); +}); + +test("a second write overwrites atomically", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + writeHeartbeat(dir, { at: 1, seq: 1 }); + writeHeartbeat(dir, { at: 2, seq: 2 }); + expect(readHeartbeat(dir)).toEqual({ at: 2, seq: 2 }); +}); diff --git a/lib/daemon/heartbeat-file.ts b/lib/daemon/heartbeat-file.ts new file mode 100644 index 00000000..77206798 --- /dev/null +++ b/lib/daemon/heartbeat-file.ts @@ -0,0 +1,38 @@ +/** + * Monotonic liveness heartbeat, written to a small file via atomic rename so + * it never opens state.db. A stalled/lock-wedged daemon is exactly when the + * WAL is least readable, so the cross-process classifier reads THIS, not kv. + * Same db-free pattern as the Phase 0 breadcrumb. + */ +import { existsSync, readFileSync, renameSync, writeFileSync } from "fs"; +import { join } from "path"; + +export interface Heartbeat { + at: number; + seq: number; +} + +function heartbeatPath(dir: string): string { + return join(dir, "daemon-heartbeat.json"); +} + +/** Never fatal: a heartbeat is a diagnostic aid, not something a tick may fail over. */ +export function writeHeartbeat(dir: string, hb: Heartbeat): void { + try { + const tmp = `${heartbeatPath(dir)}.${process.pid}.tmp`; + writeFileSync(tmp, JSON.stringify(hb)); + renameSync(tmp, heartbeatPath(dir)); + } catch { + // best-effort + } +} + +export function readHeartbeat(dir: string): Heartbeat | null { + try { + const p = heartbeatPath(dir); + if (!existsSync(p)) return null; + return JSON.parse(readFileSync(p, "utf8")) as Heartbeat; + } catch { + return null; + } +} From e9b22f232a3b6e1a36da0860270f079de7b69054 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:04:22 -0500 Subject: [PATCH 100/142] plan: apply reviewer fixes (bounded read in links.ts, widen disabledReason, overlay garbage warn, sops spawn-injection test) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../plans/2026-08-28-p6-portability.md | 99 ++++++++++++++----- 1 file changed, 77 insertions(+), 22 deletions(-) diff --git a/docs/superpowers/plans/2026-08-28-p6-portability.md b/docs/superpowers/plans/2026-08-28-p6-portability.md index 3b957c5f..35dd9f98 100644 --- a/docs/superpowers/plans/2026-08-28-p6-portability.md +++ b/docs/superpowers/plans/2026-08-28-p6-portability.md @@ -168,6 +168,15 @@ test("overlay timeout is skipped with a warn; base kept unchanged", async () => expect(warns.some((w) => JSON.stringify(w).includes("overlay"))).toBe(true); }); +test("garbage overlay (non-null, no absolute dirs) is skipped with a warn", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async (argv: any) => (argv[1] === "-lc" ? "/opt/homebrew/bin:/usr/bin:/bin" : "not-a-path:also-not"); + const out = await resolveUserPath(log, probe); + expect(out).toBe("/opt/homebrew/bin:/usr/bin:/bin"); + expect(warns.some((w) => JSON.stringify(w).includes("overlay"))).toBe(true); +}); + test("missing-tool warn fires when node is absent", async () => { const { log, warns } = makeLog(); process.env.PATH = "/usr/bin:/bin"; @@ -308,12 +317,14 @@ export async function resolveUserPath(log: Logger, probe: ProbeFn = runProbe): P : [shell, "-ilc", "echo $PATH"]; const ovRaw = await probe(ovArgv, { timeoutMs: OVERLAY_TIMEOUT_MS, env: { ...process.env, TERM: "dumb" } }); const extra = absoluteDirsOf(ovRaw); - if (extra.length > 0) { + if (extra.length === 0) { + // Warn on BOTH timeout (null) and garbage (non-null but no usable + // absolute dirs) ... the ruling says timeout OR garbage. + log.warn("PATH interactive overlay skipped (timed out or no usable dirs)"); + } else { const before = result; result = unionAppend(result, extra); if (result !== before) source += "+overlay"; - } else if (ovRaw === null) { - log.warn("PATH interactive overlay skipped (timed out or empty)"); } } @@ -611,7 +622,10 @@ export function isDevModeWrapperContent(prefix: string): boolean { return line2.startsWith(DEV_MODE_TAG) || prefix.includes("RT_LAUNCH_CWD"); } -function readWrapperPrefix(path: string): string | null { +/** A bounded head of the file (never the whole file: in prod the wrapper path + * is a symlink to the multi-MB compiled binary). Exported so links.ts shares + * the same real bounded read. */ +export function readWrapperPrefix(path: string): string | null { try { const fd = openSync(path, "r"); try { @@ -636,21 +650,28 @@ export function currentMode(): "dev" | "prod" { (Keep the existing `openSync`/`readSync`/`closeSync` imports; add `Buffer` if not already available via global.) -- [ ] **Step 5: Delegate from `lib/deps/links.ts`** +- [ ] **Step 5: Delegate from `lib/deps/links.ts` via a REAL bounded read** -Replace `isDevModeWrapper`: +`p.readFile` is `readFileSync`, which in prod follows the `~/.local/bin/rt` +symlink into the multi-MB compiled binary ... a whole-file read. Slicing its +result does NOT deliver the bounded-read ruling (the whole file is already in +memory). Use the exported bounded `readWrapperPrefix` (openSync + readSync +4096) instead, so no whole-file read ever happens: ```ts -import { isDevModeWrapperContent } from "../dev-mode.ts"; +import { isDevModeWrapperContent, readWrapperPrefix } from "../dev-mode.ts"; -function isDevModeWrapper(p: Pick, path: string): boolean { - const content = p.readFile(path); - if (content === null) return false; - return isDevModeWrapperContent(content.slice(0, 4096)); +function isDevModeWrapper(path: string): boolean { + const prefix = readWrapperPrefix(path); + return prefix !== null && isDevModeWrapperContent(prefix); } ``` -(`p.readFile` returns the whole file here; slicing to 4096 keeps the detector bounded and consistent with `currentMode`. If a `Probes` bounded-read seam exists, prefer it; otherwise the slice suffices for the string comparison.) +Drop the now-unused `p: Pick` parameter and update +`isDevModeWrapper`'s single call site in `links.ts` to pass just the path. +Any `links.ts` test that drove this through an injected `readFile` switches to +writing a real temp file at `path` (the detector reads the actual symlink +target's head on the real fs, by design). - [ ] **Step 6: Run tests + type-check** @@ -806,6 +827,15 @@ export type SkipReason = | "no-changes"; ``` +Also widen the local `disabledReason` declaration ... it is currently narrowed +(`let disabledReason: "not-a-repo" | "init-failed" | null;` around +`lib/daemon/home-snapshot.ts:281`), so assigning `"not-provisioned"` / +`"no-git-identity"` fails `tsc`. Change it to: + +```ts + let disabledReason: SkipReason | null = null; +``` + - [ ] **Step 4: S090 ... existsSync guard in `init()`** In `init()` (around line 357), before the `git rev-parse` spawn: @@ -887,19 +917,33 @@ git commit -m "home-snapshot: diagnose not-provisioned and missing git identity - [ ] **Step 1: Write the failing test** +Do NOT spawn a real `trap '' TERM; sleep 60` process: the seam's kill is +SIGTERM-only (mirroring age-key), so a SIGTERM-immune child would hang the +test (`proc.exited` never resolves). Instead inject a fake spawn whose child +resolves `exited` only when `kill()` is called (a killable process), so the +timeout timer fires, kills it, and the seam throws: + ```ts import { createRealSecretsExecSeam, SecretsTimeoutError } from "../store.ts"; -test("a sops spawn that never exits times out with SecretsTimeoutError, does not hang", async () => { - const seam = createRealSecretsExecSeam(); - // Inject a hanging command via the seam's spawn boundary; use a fixture like - // ["sh", "-c", "trap '' TERM; sleep 60"] with a short timeout override. - await expect(seam.run(["sh", "-c", "trap '' TERM; sleep 60"], { timeoutMs: 200 } as any)) +test("a hanging sops spawn times out with SecretsTimeoutError, does not hang", async () => { + let resolveExit: (code: number) => void = () => {}; + const fakeProc = { + pid: 1, + stdout: new Response("").body, + stderr: new Response("").body, + exited: new Promise((r) => { resolveExit = r; }), + kill: () => resolveExit(143), // killable: kill resolves exit, no real process + }; + const seam = createRealSecretsExecSeam(undefined, () => fakeProc as any); + await expect(seam.run(["sops", "-d", "x"], { timeoutMs: 50 } as any)) .rejects.toBeInstanceOf(SecretsTimeoutError); -}, 5_000); +}, 2_000); ``` -(Executor: the real seam resolves argv[0] via `resolveBundledTool`; for the test, either pass a plain command that resolves to itself, or add a spawn-injection seam mirroring age-key's testability. The contract: a non-exiting child rejects with `SecretsTimeoutError` within the timeout, and the killable child is terminated.) +(The contract: a child that does not exit on its own rejects with +`SecretsTimeoutError` within the timeout, and the timer's `kill()` terminates +it. The fake models a real killable process without one.) - [ ] **Step 2: Run test to verify it fails** @@ -917,14 +961,22 @@ export class SecretsTimeoutError extends Error {} const DEFAULT_SECRETS_TIMEOUT_MS = 30_000; ``` -In `createRealSecretsExecSeam`'s `run` (lines 470-484), wrap the await with the same timer pattern `age-key.ts` uses: +Make the spawn injectable (mirroring how age-key isolates its raw seam for +testability) and wrap the await with the same timer pattern `age-key.ts` uses. +Change the factory signature to accept an optional spawn seam: ```ts +type SecretsSpawn = (argv: string[], opts: any) => { + stdout: ReadableStream; stderr: ReadableStream; exited: Promise; kill: (sig?: number | string) => void; +}; + +export function createRealSecretsExecSeam(cwd?: string, spawn: SecretsSpawn = Bun.spawn as unknown as SecretsSpawn): SecretsExecSeam { + return { async run(cmd, opts) { debugLog(cmd, opts?.sensitive); const [bin, ...args] = cmd; const resolved = bin === undefined ? cmd : [resolveBundledTool(bin), ...args]; - const proc = Bun.spawn(resolved, buildSecretsSpawnOptions({ env: opts?.env, cwd })); + const proc = spawn(resolved, buildSecretsSpawnOptions({ env: opts?.env, cwd })); const timeoutMs = (opts as { timeoutMs?: number } | undefined)?.timeoutMs ?? DEFAULT_SECRETS_TIMEOUT_MS; let timedOut = false; const timer = setTimeout(() => { timedOut = true; try { proc.kill(); } catch { /* already exited */ } }, timeoutMs); @@ -938,9 +990,12 @@ In `createRealSecretsExecSeam`'s `run` (lines 470-484), wrap the await with the return { code, stdout, stderr }; } finally { clearTimeout(timer); } }, + // ... fileExists / listDir / the rest of the seam unchanged ... + }; +} ``` -(If `SecretsExecSeam.run`'s opts type has no `timeoutMs`, add it to the interface as optional.) +(If `SecretsExecSeam.run`'s opts type has no `timeoutMs`, add it to the interface as optional. The default `spawn` is the real `Bun.spawn`, so production behavior is unchanged; only tests inject a fake.) - [ ] **Step 4: Run test to verify it passes** From 25c73c525af5ac3903de62904122c171c1b8bf5a Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:04:33 -0500 Subject: [PATCH 101/142] add lib/daemon/loop-monitor.ts: drift monitor + heartbeat cadence --- lib/daemon/__tests__/loop-monitor.test.ts | 41 +++++++++ lib/daemon/loop-monitor.ts | 102 ++++++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 lib/daemon/__tests__/loop-monitor.test.ts create mode 100644 lib/daemon/loop-monitor.ts diff --git a/lib/daemon/__tests__/loop-monitor.test.ts b/lib/daemon/__tests__/loop-monitor.test.ts new file mode 100644 index 00000000..7ca34211 --- /dev/null +++ b/lib/daemon/__tests__/loop-monitor.test.ts @@ -0,0 +1,41 @@ +import { test, expect } from "bun:test"; +import { applyTick, newLoopStats, type LoopStats } from "../loop-monitor.ts"; + +const OPTS = { stallLogMs: 1000, stallUnhealthyMs: 2000, stallRecentMs: 10_000 }; + +test("an on-time tick records small lag and no stall", () => { + const s = newLoopStats(); + applyTick(s, /*expected*/ 1000, /*now*/ 1010, "cache:refresh", OPTS, () => {}); + expect(s.lagMs).toBe(10); + expect(s.maxLagMs).toBe(10); + expect(s.stalls).toBe(0); + expect(s.currentlyStalled).toBe(false); +}); + +test("a >1s drift counts a stall, records the in-flight cmd, and warns", () => { + const s = newLoopStats(); + let warned = 0; + applyTick(s, 1000, 2500, "mr:action", OPTS, () => { warned++; }); + expect(s.stalls).toBe(1); + expect(s.lastStallCmd).toBe("mr:action"); + expect(s.lastStallAt).toBe(2500); + expect(s.maxLagMs).toBe(1500); + expect(warned).toBe(1); +}); + +test("currentlyStalled is true when the last big drift is within stallRecentMs", () => { + const s = newLoopStats(); + applyTick(s, 1000, 3500, "x", OPTS, () => {}); // 2500ms drift >= 2000 unhealthy, lastStallAt=3500 + expect(s.currentlyStalled).toBe(true); + // a small-drift tick whose `now` is past lastStallAt + stallRecentMs clears it + // (10ms drift, so no new stall; now-lastStallAt = 10500 > 10000 recent window) + applyTick(s, 13990, 14000, null, OPTS, () => {}); + expect(s.currentlyStalled).toBe(false); +}); + +test("maxLagMs is a high-water mark", () => { + const s: LoopStats = newLoopStats(); + applyTick(s, 1000, 1300, null, OPTS, () => {}); + applyTick(s, 1550, 1600, null, OPTS, () => {}); + expect(s.maxLagMs).toBe(300); +}); diff --git a/lib/daemon/loop-monitor.ts b/lib/daemon/loop-monitor.ts new file mode 100644 index 00000000..78f03046 --- /dev/null +++ b/lib/daemon/loop-monitor.ts @@ -0,0 +1,102 @@ +/** + * Event-loop drift monitor. A ~250ms unref'd interval measures how late each + * tick fires vs its scheduled time; a large drift means the loop was blocked. + * The interval callback is created once and the stats object is preallocated, + * so the hot tick allocates nothing. Every ~2s it also writes the heartbeat + * file the cross-process classifier reads. + */ +import type { Logger } from "pino"; + +export interface LoopStats { + lagMs: number; + maxLagMs: number; + stalls: number; + lastStallAt: number | null; + lastStallCmd: string | null; + currentlyStalled: boolean; +} + +export function newLoopStats(): LoopStats { + return { lagMs: 0, maxLagMs: 0, stalls: 0, lastStallAt: null, lastStallCmd: null, currentlyStalled: false }; +} + +interface TickOpts { + stallLogMs: number; + stallUnhealthyMs: number; + stallRecentMs: number; +} + +/** Pure: fold one tick into `stats`. `onStall` fires once per stall (warn sink). */ +export function applyTick( + stats: LoopStats, + expected: number, + now: number, + currentCmd: string | null, + opts: TickOpts, + onStall: (drift: number, cmd: string | null) => void, +): void { + const drift = now - expected; + stats.lagMs = drift > 0 ? drift : 0; + if (stats.lagMs > stats.maxLagMs) stats.maxLagMs = stats.lagMs; + if (drift > opts.stallLogMs) { + stats.stalls += 1; + stats.lastStallAt = now; + stats.lastStallCmd = currentCmd; + onStall(drift, currentCmd); + } + stats.currentlyStalled = + stats.lastStallAt !== null && + now - stats.lastStallAt <= opts.stallRecentMs && + (drift > opts.stallUnhealthyMs || stats.maxLagMs > opts.stallUnhealthyMs); +} + +export interface LoopMonitorOpts { + log: Logger; + tickMs?: number; + stallLogMs?: number; + stallUnhealthyMs?: number; + stallRecentMs?: number; + heartbeatMs?: number; + currentCmd: () => string | null; + onHeartbeat: (at: number, seq: number) => void; +} + +export function startLoopMonitor( + opts: LoopMonitorOpts, +): { stats: LoopStats; seq: () => number; stop: () => void } { + const tickMs = opts.tickMs ?? 250; + const tickOpts: TickOpts = { + stallLogMs: opts.stallLogMs ?? 1000, + stallUnhealthyMs: opts.stallUnhealthyMs ?? 2000, + stallRecentMs: opts.stallRecentMs ?? 10_000, + }; + const heartbeatMs = opts.heartbeatMs ?? 2000; + const stats = newLoopStats(); + let expected = Date.now() + tickMs; + let lastHeartbeat = 0; + let seq = 0; + let warnedThisStall = false; + + // Hoisted once (allocation-free ruling): the tick must not build a fresh + // closure every 250ms. onStall closes over warnedThisStall by reference. + const onStall = (drift: number, cmd: string | null): void => { + if (!warnedThisStall) { + opts.log.warn({ driftMs: drift, cmd }, "event loop stalled"); + warnedThisStall = true; + } + }; + + const timer = setInterval(() => { + const now = Date.now(); + applyTick(stats, expected, now, opts.currentCmd(), tickOpts, onStall); + if (stats.lagMs <= tickOpts.stallLogMs) warnedThisStall = false; + expected = now + tickMs; + if (now - lastHeartbeat >= heartbeatMs) { + lastHeartbeat = now; + opts.onHeartbeat(now, ++seq); + } + }, tickMs); + timer.unref(); + + return { stats, seq: () => seq, stop: () => clearInterval(timer) }; +} From cc5374ba39e4b9733984a632e49cddd6a02a2efb Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:08:05 -0500 Subject: [PATCH 102/142] rt-client: register rt.daemonPath machine setting (6.1) --- .../src/settings/__tests__/registry.test.ts | 12 +++++++++++- packages/rt-client/src/settings/registry-defs.ts | 8 ++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/packages/rt-client/src/settings/__tests__/registry.test.ts b/packages/rt-client/src/settings/__tests__/registry.test.ts index e84146b4..09e286e8 100644 --- a/packages/rt-client/src/settings/__tests__/registry.test.ts +++ b/packages/rt-client/src/settings/__tests__/registry.test.ts @@ -74,6 +74,15 @@ describe("settings/registry", () => { expect(def?.default).toBe(14); }); + test("rt.daemonPath is a machine-scoped string key with no default", () => { + const def = getDef("rt.daemonPath"); + expect(def).toBeDefined(); + expect(def!.type).toBe("string"); + expect(def!.scopes).toEqual(["machine"]); + expect(def!.default).toBeUndefined(); + expect(def!.pathGuardFields).toBeUndefined(); + }); + test("rt.worktreeApp is a machine-only field-bag object with no default (ownership latch)", () => { const def = getDef("rt.worktreeApp"); @@ -251,8 +260,9 @@ describe("settings/registry", () => { "agent.account", "agent.extraArgs", "rt.trustedBrowserOrigins", + "rt.daemonPath", ]; - expect(suiteKeys).toHaveLength(43); + expect(suiteKeys).toHaveLength(44); expect(allDefs().map((d) => d.key).sort()).toEqual( [...migratedFalseKeys, ...migratedTrueKeys, ...suiteKeys].sort(), diff --git a/packages/rt-client/src/settings/registry-defs.ts b/packages/rt-client/src/settings/registry-defs.ts index 8a00016b..f9a588da 100644 --- a/packages/rt-client/src/settings/registry-defs.ts +++ b/packages/rt-client/src/settings/registry-defs.ts @@ -214,6 +214,14 @@ export const REGISTRY: readonly SettingDef[] = [ migrated: true, description: "Per-repo git hook enable/disable state ({enabled, hooks: {: boolean}}); ownership-latch port of repos//hooks.json, store wins per field once it owns the key — including per-hook-name entries inside the nested hooks map, each defaulting to enabled when absent. The installed git-hook shim still greps repos//hooks.json with zero process spawns (a hook fires on every git operation); that file is now a DERIVED CACHE this key writes through, kept current by commands/hooks.ts's regenerateHooksCache at every write seam.", }, + { + key: "rt.daemonPath", + type: "string", + scopes: ["machine"], + merge: "replace", + description: + "Absolute colon-separated PATH the daemon uses for every child it spawns, instead of probing your login shell. Set this when the daemon can't find node/git/bun/pnpm (e.g. a fish shell, a blocking .zshrc, or PATH exports that live only in .zshrc). Machine-scoped: it never travels to another machine.", + }, { key: "rt.trustedBrowserOrigins", type: "array", From d2b3e21790b0918c026ddec5cad97c099ea330e4 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:09:04 -0500 Subject: [PATCH 103/142] daemon-status: heartbeat-stale 'stalled' detail + degraded eventLoop --- lib/__tests__/daemon-status.test.ts | 37 ++++++++++++++++++++ lib/daemon-status.ts | 52 ++++++++++++++++++++++++----- 2 files changed, 80 insertions(+), 9 deletions(-) diff --git a/lib/__tests__/daemon-status.test.ts b/lib/__tests__/daemon-status.test.ts index 17187267..f403782e 100644 --- a/lib/__tests__/daemon-status.test.ts +++ b/lib/__tests__/daemon-status.test.ts @@ -185,6 +185,43 @@ describe("classifyDaemonStatus", () => { const v = classifyDaemonStatus({ installed: true, pingOk: false, pidAlive: false, pid: null }); expect(v.state).toBe("not-running"); }); + + // ── Task 4: heartbeat-stale "stalled" detail + degraded eventLoop ── + + test("alive + ping-fail + ready + stale heartbeat => alive-not-serving 'stalled'", () => { + const now = 1_000_000; + const v = classifyDaemonStatus({ + installed: true, response: null, pingOk: false, pid: 42, pidAlive: true, + breadcrumb: { phase: "ready" }, + heartbeat: { at: now - 8000, seq: 3 }, heartbeatStaleMs: 6000, + now, + }); + expect(v.state).toBe("alive-not-serving"); + if (v.state === "alive-not-serving") { + expect(v.detail).toBe("stalled"); + expect(v.stalledForMs).toBe(8000); + } + }); + + test("alive + ready + FRESH heartbeat => 'wedged', not 'stalled'", () => { + const now = 1_000_000; + const v = classifyDaemonStatus({ + installed: true, response: null, pingOk: false, pid: 42, pidAlive: true, + breadcrumb: { phase: "ready" }, + heartbeat: { at: now - 500, seq: 9 }, heartbeatStaleMs: 6000, + now, + }); + expect(v.state === "alive-not-serving" && v.detail).toBe("wedged"); + }); + + test("degraded/unresponsive carries the ping-supplied eventLoop", () => { + const v = classifyDaemonStatus({ + installed: true, response: null, pingOk: true, pid: 42, + pingEventLoop: { maxLagMs: 1400, lastStallAt: 123, lastStallCmd: "mr:action", stalls: 2 }, + }); + expect(v.state).toBe("degraded"); + if (v.state === "degraded") expect(v.eventLoop?.maxLagMs).toBe(1400); + }); }); describe("needsLivenessProbe", () => { diff --git a/lib/daemon-status.ts b/lib/daemon-status.ts index 2cdbbcfc..8179cde0 100644 --- a/lib/daemon-status.ts +++ b/lib/daemon-status.ts @@ -25,19 +25,36 @@ export type DaemonStatusVerdict = | { state: "not-installed" } | { state: "running"; data: any } /** Up — proven by an answer or a ping — but `status` itself did not deliver. */ - | { state: "degraded"; reason: "error" | "unresponsive"; detail?: string; pid: number | null } + | { state: "degraded"; reason: "error" | "unresponsive"; detail?: string; pid: number | null; eventLoop?: StatusEventLoop } /** Ping fails, a live pid exists, and it's parked waiting for a different * flavor to hold rt.sock (park.ts): a flavor standoff, not a stuck boot. */ | { state: "parked"; pid: number; holderFlavor?: string } /** Ping fails but the pid is alive: still mid-boot, stuck after reaching - * ready, or alive-but-quarantined (recovered from a corrupt db). */ - | { state: "alive-not-serving"; pid: number; detail: "booting" | "wedged" | "quarantined" } + * ready, alive-but-quarantined (recovered from a corrupt db), or stalled + * (reached ready but the heartbeat file has gone stale). */ + | { state: "alive-not-serving"; pid: number; detail: "booting" | "wedged" | "quarantined" | "stalled"; stalledForMs?: number } /** No live pid, and the kv failure record shows >= N failures within the window. */ | { state: "crash-looping"; failures: number; reason: string } /** No live pid, and the most recent recorded exit was a boot throw (fewer than N failures). */ | { state: "boot-failed"; reason: string; phase: string } | { state: "not-running"; pid: number | null }; +/** Structural match for the daemon's heartbeat-file record; not imported from + * its owning module to avoid a cycle. */ +export interface HeartbeatInput { + at: number; + seq: number; +} + +/** Structural match for the ping-supplied event-loop summary, passed through + * on the degraded verdict for display. */ +export interface StatusEventLoop { + maxLagMs: number; + lastStallAt: number | null; + lastStallCmd: string | null; + stalls: number; +} + /** The boot breadcrumb (`daemon-boot.json`), as classifyDaemonStatus needs it. Not * imported from supervision-state.ts, since that module's `Breadcrumb` interface is * intentionally unexported, and this shape only needs to be structurally @@ -75,6 +92,12 @@ export interface DaemonStatusInputs { supervision?: SupervisionState; /** Injected for deterministic crash-loop window checks under test; defaults to Date.now(). */ now?: number; + /** The daemon's heartbeat-file record, when the caller read one. */ + heartbeat?: HeartbeatInput | null; + /** How old `heartbeat` must be to count as stale. Defaults to 6000ms. */ + heartbeatStaleMs?: number; + /** Ping's event-loop summary, passed through onto a `degraded` verdict. */ + pingEventLoop?: StatusEventLoop; } const PHASE_ORDER: BootPhase[] = ["start", "events-db", "state-db", "api", "socket", "ready"]; @@ -82,14 +105,23 @@ const PHASE_ORDER: BootPhase[] = ["start", "events-db", "state-db", "api", "sock function classifyAliveNotServingDetail( breadcrumb: DaemonBreadcrumbInput | null | undefined, supervision: SupervisionState | undefined, -): "booting" | "wedged" | "quarantined" { + heartbeat: HeartbeatInput | null | undefined, + heartbeatStaleMs: number, + now: number, +): { detail: "booting" | "wedged" | "quarantined" | "stalled"; stalledForMs?: number } { const phase = breadcrumb?.phase; - if (!phase || PHASE_ORDER.indexOf(phase) < PHASE_ORDER.indexOf("ready")) return "booting"; + if (!phase || PHASE_ORDER.indexOf(phase) < PHASE_ORDER.indexOf("ready")) return { detail: "booting" }; + // A live heartbeat gone stale outranks the boot-failed check below: it is + // ground truth that the process stopped ticking, not a record of a past + // recovery it may be running fine behind. + if (heartbeat && now - heartbeat.at > heartbeatStaleMs) { + return { detail: "stalled", stalledForMs: now - heartbeat.at }; + } // Reached ready this run, but a prior attempt is on record as boot-failed, // most likely a corrupt-db quarantine (lib/state/db.ts, events-bus.ts) it // recovered from and is now stuck behind for an unrelated reason. - if (supervision?.lastExit?.kind === "boot-failed") return "quarantined"; - return "wedged"; + if (supervision?.lastExit?.kind === "boot-failed") return { detail: "quarantined" }; + return { detail: "wedged" }; } function countRecentFailures(supervision: SupervisionState, now: number, windowMs = 5 * 60_000): number { @@ -113,7 +145,7 @@ export function classifyDaemonStatus(opts: DaemonStatusInputs): DaemonStatusVerd // No reply. A plain ping is the next ground truth: a daemon busy enough to // blow the status timeout still answers a trivial ping. - if (pingOk) return { state: "degraded", reason: "unresponsive", pid }; + if (pingOk) return { state: "degraded", reason: "unresponsive", pid, eventLoop: opts.pingEventLoop }; // Ping failed too. From here, only pidAlive/breadcrumb/supervision (new // signals) can say more than "not running"; absent them, fall straight @@ -122,7 +154,9 @@ export function classifyDaemonStatus(opts: DaemonStatusInputs): DaemonStatusVerd if (breadcrumb?.flavor && intendedFlavor && breadcrumb.flavor !== intendedFlavor) { return { state: "parked", pid, ...(holderFlavor ? { holderFlavor } : {}) }; } - return { state: "alive-not-serving", pid, detail: classifyAliveNotServingDetail(breadcrumb, supervision) }; + const now = opts.now ?? Date.now(); + const d = classifyAliveNotServingDetail(breadcrumb, supervision, opts.heartbeat, opts.heartbeatStaleMs ?? 6000, now); + return { state: "alive-not-serving", pid, detail: d.detail, ...(d.stalledForMs !== undefined ? { stalledForMs: d.stalledForMs } : {}) }; } if (supervision) { From 38ffaa81eeca362648f768cea48a4767a47e33fd Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:15:02 -0500 Subject: [PATCH 104/142] daemon status: render health/stall lines; add non-restarting pingDaemon --- commands/__tests__/status-lines.test.ts | 33 +++++++++++++++++++++ commands/daemon.ts | 39 +++++++++++++++++++------ lib/daemon-client.ts | 7 +++++ 3 files changed, 70 insertions(+), 9 deletions(-) create mode 100644 commands/__tests__/status-lines.test.ts diff --git a/commands/__tests__/status-lines.test.ts b/commands/__tests__/status-lines.test.ts new file mode 100644 index 00000000..85c81df8 --- /dev/null +++ b/commands/__tests__/status-lines.test.ts @@ -0,0 +1,33 @@ +import { test, expect } from "bun:test"; +import { statusLines } from "../daemon.ts"; + +const strip = (s: string) => s.replace(/\[[0-9;]*m/g, ""); + +test("degraded/unresponsive prints ping-carried maxLag, not 'likely mid-sync'", () => { + const lines = statusLines( + { state: "degraded", reason: "unresponsive", pid: 42, eventLoop: { maxLagMs: 1400, lastStallAt: 1, lastStallCmd: "mr:action", stalls: 2 } } as any, + 2000, + ).map(strip).join("\n"); + expect(lines).not.toContain("likely mid-sync"); + expect(lines).toContain("1400ms"); + expect(lines).toContain("mr:action"); +}); + +test("alive-not-serving 'stalled' prints stalled Ns ago", () => { + const lines = statusLines( + { state: "alive-not-serving", pid: 42, detail: "stalled", stalledForMs: 8000 } as any, + 0, + ).map(strip).join("\n"); + expect(lines).toContain("event loop stalled"); + expect(lines).toContain("8s"); +}); + +test("running prints the health level and reasons when present", () => { + const lines = statusLines( + { state: "running", data: { pid: 42, uptime: 60000, watchedRepos: 3, cacheEntries: 10, + health: { level: "degraded", reasons: ["refresh: 3 repos failing (auth?)"] } } } as any, + 0, + ).map(strip).join("\n"); + expect(lines).toContain("degraded"); + expect(lines).toContain("refresh: 3 repos failing"); +}); diff --git a/commands/daemon.ts b/commands/daemon.ts index 8d492ab5..2c1374ec 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -33,11 +33,12 @@ import { LOG_DIR, LAUNCHD_PLIST_PATH, } from "../lib/daemon-config.ts"; -import { daemonQuery, isDaemonRunning, trayQuery } from "../lib/daemon-client.ts"; +import { daemonQuery, isDaemonRunning, pingDaemon, trayQuery } from "../lib/daemon-client.ts"; import { classifyDaemonStatus, type DaemonStatusVerdict } from "../lib/daemon-status.ts"; import { resolveIntendedMode, currentMode, type IntendedMode } from "../lib/dev-mode.ts"; import { probeSocketHolder } from "../lib/daemon/park.ts"; import { readBreadcrumb, readSupervisionState } from "../lib/daemon/supervision-state.ts"; +import { readHeartbeat } from "../lib/daemon/heartbeat-file.ts"; import { runCapture } from "../lib/subprocess.ts"; import { isGitLabRemote } from "../lib/enrich.ts"; import type { CacheKind, RepoTrackingEntry } from "../lib/repo-tracking.ts"; @@ -403,22 +404,27 @@ export async function showStatus(args: string[] = []): Promise { // A failed status query does NOT mean the daemon is down — it answers `ping` // in a fraction of the budget a loaded `status` needs. Establish liveness // before reporting, and only pay for the probe when nothing came back. - const pingOk = classifyDaemonStatus.needsLivenessProbe(response) ? await isDaemonRunning() : false; + // pingDaemon (not isDaemonRunning) so the raw reply's eventLoop summary is + // still on hand to render, and so this probe never risks a restart. + const pingResp = classifyDaemonStatus.needsLivenessProbe(response) ? await pingDaemon() : null; + const pingOk = pingResp?.ok === true; const recordedPid = readDaemonPid() ?? null; - // Ping ALSO failed: the only remaining ground is the pid/breadcrumb/kv - // trail Task 9 left behind. Read it here, once, rather than on every status + // Ping ALSO failed: the only remaining ground is the pid/breadcrumb/kv/heartbeat + // trail Task 9/2 left behind. Read it here, once, rather than on every status // call, since it's the uncommon path. let pidAlive: boolean | undefined; let pid = recordedPid; let breadcrumb: ReturnType | undefined; let supervision: ReturnType | undefined; + let heartbeat: ReturnType | undefined; if (classifyDaemonStatus.needsPidProbe(response, pingOk)) { breadcrumb = readBreadcrumb(); // The kv tier can be legitimately empty (or reflect nothing useful) when // a failure happened before state.db ever opened (Ruling P1). The // breadcrumb read above is what classifyDaemonStatus falls back to then. supervision = readSupervisionState(); + heartbeat = readHeartbeat(RT_DIR); const probed = await probePidAlive(recordedPid, breadcrumb?.pid); pidAlive = probed.alive; pid = probed.pid; @@ -433,6 +439,8 @@ export async function showStatus(args: string[] = []): Promise { intendedFlavor: resolveIntendedMode().mode, breadcrumb, supervision, + heartbeat, + pingEventLoop: (pingResp as any)?.eventLoop, }); if (json) return void console.log(JSON.stringify({ ok: true, ...verdict })); @@ -503,6 +511,15 @@ export function statusLines(verdict: DaemonStatusVerdict, now: number): string[] }); lines.push(` ${dim}events: ${parts.join(" · ")}${reset}`); } + + const health = verdict.data.health as { level: string; reasons: string[] } | undefined; + if (health && health.level !== "ok") { + const dot = health.level === "unhealthy" ? red : yellow; + lines.push(` ${dot}health: ${health.level}${reset}`); + for (const r of health.reasons) lines.push(` ${dim}- ${r}${reset}`); + } + const el = verdict.data.eventLoop as { maxLagMs: number } | undefined; + if (el && el.maxLagMs >= 500) lines.push(` ${dim}event loop: maxLag ${el.maxLagMs}ms${reset}`); return lines; } @@ -511,11 +528,14 @@ export function statusLines(verdict: DaemonStatusVerdict, now: number): string[] // the operator to `rt daemon start` against a daemon that is already up. const lines = [` ${yellow}●${reset} running, but not reporting status`]; if (verdict.pid) lines.push(` ${dim}pid: ${verdict.pid}${reset}`); - lines.push( - verdict.reason === "error" - ? ` ${dim}status command failed: ${verdict.detail ?? "unknown error"}${reset}` - : ` ${dim}answers ping, but status timed out — likely mid-sync${reset}`, - ); + if (verdict.reason === "error") { + lines.push(` ${dim}status command failed: ${verdict.detail ?? "unknown error"}${reset}`); + } else if (verdict.eventLoop && verdict.eventLoop.maxLagMs > 0) { + const el = verdict.eventLoop; + lines.push(` ${dim}answers ping, status timed out — event loop maxLag ${el.maxLagMs}ms${el.lastStallCmd ? ` (last stall in ${el.lastStallCmd})` : ""}${reset}`); + } else { + lines.push(` ${dim}answers ping, but status timed out — likely mid-sync${reset}`); + } lines.push(` ${dim}check: rt daemon logs${reset}`); return lines; } @@ -536,6 +556,7 @@ export function statusLines(verdict: DaemonStatusVerdict, now: number): string[] booting: "still booting", wedged: "reached ready but stopped answering (likely deadlocked)", quarantined: "recovered from a corrupt db but still not answering", + stalled: `event loop stalled ${Math.round((verdict.stalledForMs ?? 0) / 1000)}s ago (no heartbeat)`, }[verdict.detail]; return [ ` ${yellow}●${reset} process ${verdict.pid} is running but not answering rt.sock`, diff --git a/lib/daemon-client.ts b/lib/daemon-client.ts index da89c789..aff0ad11 100644 --- a/lib/daemon-client.ts +++ b/lib/daemon-client.ts @@ -334,6 +334,13 @@ export async function isDaemonRunning(): Promise { return response?.ok === true; } +/** Single-attempt ping that never triggers the restart machinery, so + * `rt daemon status` can probe liveness and read the daemon's eventLoop + * summary without spawning a daemon as a side effect. */ +export async function pingDaemon(timeoutMs?: number): Promise { + return (await trySocketQuery("ping", undefined, timeoutMs)).response; +} + // ─── MR action facade ──────────────────────────────────────────────────────── /** From 319849a2086206a923c09c37849db947f4a8c3fc Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:15:17 -0500 Subject: [PATCH 105/142] user-path: async fish-aware killable PATH probe + rt.daemonPath override (S013/S014/S062) --- lib/daemon/__tests__/user-path.test.ts | 92 +++++++++++- lib/daemon/user-path.ts | 186 ++++++++++++++++++++----- 2 files changed, 239 insertions(+), 39 deletions(-) diff --git a/lib/daemon/__tests__/user-path.test.ts b/lib/daemon/__tests__/user-path.test.ts index ceb38719..984fc7af 100644 --- a/lib/daemon/__tests__/user-path.test.ts +++ b/lib/daemon/__tests__/user-path.test.ts @@ -2,7 +2,8 @@ import { describe, test, expect, beforeEach } from "bun:test"; import { mkdtempSync, writeFileSync, chmodSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; -import { probeTools } from "../user-path.ts"; +import { setSetting } from "../../settings/write.ts"; +import { resolveUserPath, probeTools } from "../user-path.ts"; describe("probeTools", () => { let binDir: string; @@ -32,3 +33,92 @@ describe("probeTools", () => { expect(probeTools("", ["node"])).toEqual({ hasNode: false }); }); }); + +function makeLog() { + const warns: any[] = []; + const infos: any[] = []; + return { log: { warn: (...a: any[]) => warns.push(a), info: (...a: any[]) => infos.push(a) } as any, warns, infos }; +} + +describe("resolveUserPath", () => { + test("fish-style space-separated base output is rejected, baseline kept + warn", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async () => "/opt/homebrew/bin /usr/bin /bin"; // spaces = fish-unsplit + const out = await resolveUserPath(log, probe); + expect(out).toBe("/usr/bin:/bin"); + expect(warns.some((w) => JSON.stringify(w).includes("whitespace"))).toBe(true); + }); + + test("a hanging probe returns baseline within the timeout", async () => { + const { log } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async () => null; // seam models kill/timeout as null + const out = await resolveUserPath(log, probe); + expect(out).toBe("/usr/bin:/bin"); + }); + + test("base equal to launchd baseline is treated as silent fallback (S062)", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin:/usr/sbin:/sbin"; + const probe = async (argv: any) => (argv[1] === "-lc" ? "/usr/bin:/bin:/usr/sbin:/sbin" : null); + const out = await resolveUserPath(log, probe); + expect(out).toBe("/usr/bin:/bin:/usr/sbin:/sbin"); + expect(warns.some((w) => JSON.stringify(w).includes("equals-baseline"))).toBe(true); + }); + + test("rt.daemonPath override skips both probes", async () => { + const { log } = makeLog(); + let called = false; + const probe = async () => { + called = true; + return "x"; + }; + const scratchHome = mkdtempSync(join(tmpdir(), "rt-daemonpath-override-")); + const originalHome = process.env.HOME; + process.env.HOME = scratchHome; + try { + setSetting("rt.daemonPath", "/over/bin:/x/bin", "machine"); + const out = await resolveUserPath(log, probe); + expect(out).toBe("/over/bin:/x/bin"); + expect(called).toBe(false); + } finally { + process.env.HOME = originalHome; + } + }); + + test("valid base accepted; interactive overlay appends a .zshrc-only dir after base", async () => { + const { log } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async (argv: any) => + argv[1] === "-lc" ? "/opt/homebrew/bin:/usr/bin:/bin" : "/opt/homebrew/bin:/usr/bin:/bin:/Users/x/.nvm/versions/node/v22/bin"; + const out = await resolveUserPath(log, probe); + expect(out).toBe("/opt/homebrew/bin:/usr/bin:/bin:/Users/x/.nvm/versions/node/v22/bin"); + }); + + test("overlay timeout is skipped with a warn; base kept unchanged", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async (argv: any) => (argv[1] === "-lc" ? "/opt/homebrew/bin:/usr/bin:/bin" : null); + const out = await resolveUserPath(log, probe); + expect(out).toBe("/opt/homebrew/bin:/usr/bin:/bin"); + expect(warns.some((w) => JSON.stringify(w).includes("overlay"))).toBe(true); + }); + + test("garbage overlay (non-null, no absolute dirs) is skipped with a warn", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async (argv: any) => (argv[1] === "-lc" ? "/opt/homebrew/bin:/usr/bin:/bin" : "not-a-path:also-not"); + const out = await resolveUserPath(log, probe); + expect(out).toBe("/opt/homebrew/bin:/usr/bin:/bin"); + expect(warns.some((w) => JSON.stringify(w).includes("overlay"))).toBe(true); + }); + + test("missing-tool warn fires when node is absent", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async () => "/usr/bin:/bin"; // no node + await resolveUserPath(log, probe); + expect(warns.some((w) => JSON.stringify(w).includes("missing"))).toBe(true); + }); +}); diff --git a/lib/daemon/user-path.ts b/lib/daemon/user-path.ts index a28a66d4..ccafb309 100644 --- a/lib/daemon/user-path.ts +++ b/lib/daemon/user-path.ts @@ -1,16 +1,112 @@ /** * Resolve the user's full PATH once at daemon startup. * - * Strategy: use `$SHELL -ilc` (interactive login). Sources .zprofile AND - * .zshrc, which is where most users actually put their PATH exports - * (bun, ~/.local/bin, etc.). Slower than `-lc` due to compinit/OMZ, but - * the daemon is long-running so the one-time cost is irrelevant. - * Then layer in an explicit NVM resolution so nvm-managed tools (node, pnpm, - * etc.) are included regardless of how the daemon was launched. + * Strategy: a fast non-interactive `-lc` login shell (sources .zprofile, plus + * an explicit NVM fallback) establishes the base PATH quickly and can't hang + * on interactive-shell setup (compinit, OMZ, etc). A separate `-ilc` + * interactive probe then layers in whatever only .zshrc/.bashrc export + * (nvm's own PATH lines, ~/.local/bin, etc), unioned onto the base rather + * than replacing it, so a slow or misbehaving interactive probe can never + * regress the base PATH ... it can only fail to add to it. + * + * Both probes run through the injected `ProbeFn` seam so this module never + * spawns a shell directly and stays unit-testable without a real subprocess. */ -import { execSync } from "child_process"; +import { basename } from "path"; import type { Logger } from "pino"; +import { getSetting } from "../settings/resolve.ts"; + +export type ProbeFn = ( + argv: [string, ...string[]], + opts: { timeoutMs: number; env?: Record }, +) => Promise; + +const BASE_TIMEOUT_MS = 5_000; +const OVERLAY_TIMEOUT_MS = 3_000; +const KILL_GRACE_MS = 500; + +/** Default probe: a detached (own process-group) Bun.spawn whose whole group is + * SIGTERM'd then SIGKILL'd at the deadline, raced so a hung shell (or a hung + * grandchild it spawned) can never block boot past the timeout. */ +const runProbe: ProbeFn = async (argv, opts) => { + let proc: ReturnType; + try { + proc = Bun.spawn(argv, { + detached: true, + env: opts.env ?? { ...process.env }, + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + }); + } catch { + return null; + } + proc.unref(); + const pid = proc.pid; + let killTimer: ReturnType | undefined; + const term = setTimeout(() => { + try { + process.kill(-pid, "SIGTERM"); + } catch { + /* group already gone */ + } + killTimer = setTimeout(() => { + try { + process.kill(-pid, "SIGKILL"); + } catch { + /* gone */ + } + }, KILL_GRACE_MS); + killTimer.unref?.(); + }, opts.timeoutMs); + const captured: Promise = (async () => { + try { + const [out] = await Promise.all([new Response(proc.stdout as ReadableStream).text(), proc.exited]); + return out; + } catch { + return null; + } + })(); + let deadlineTimer: ReturnType; + const deadline: Promise = new Promise((resolve) => { + deadlineTimer = setTimeout(() => resolve(null), opts.timeoutMs + KILL_GRACE_MS + 250); + }); + try { + return await Promise.race([captured, deadline]); + } finally { + clearTimeout(term); + if (killTimer) clearTimeout(killTimer); + clearTimeout(deadlineTimer!); + } +}; + +function validateBase( + raw: string | null, + baseline: string, +): { path: string; source: "probe" | "baseline"; reason?: string } { + if (raw === null) return { path: baseline, source: "baseline", reason: "killed-or-empty" }; + const v = raw.trim(); + if (v.length === 0) return { path: baseline, source: "baseline", reason: "empty" }; + if (/\s/.test(v)) return { path: baseline, source: "baseline", reason: "whitespace" }; + if (v.split(":").filter(Boolean).length < 2) return { path: baseline, source: "baseline", reason: "too-few-segments" }; + if (v === baseline) return { path: baseline, source: "baseline", reason: "equals-baseline" }; + return { path: v, source: "probe" }; +} + +/** Overlay contributes only well-formed absolute dirs; anything else yields []. */ +function absoluteDirsOf(raw: string | null): string[] { + if (raw === null) return []; + const v = raw.trim(); + if (v.length === 0 || /\s/.test(v)) return []; + return v.split(":").filter((d) => d.startsWith("/")); +} + +function unionAppend(base: string, extra: string[]): string { + const have = new Set(base.split(":").filter(Boolean)); + const add = extra.filter((d) => !have.has(d)); + return add.length === 0 ? base : [base, ...add].join(":"); +} /** Which of `names` is a non-empty file on `pathValue`, keyed `has`. */ export function probeTools(pathValue: string, names: string[]): Record { @@ -28,36 +124,50 @@ export function probeTools(pathValue: string, names: string[]): Record { + const baseline = process.env.PATH ?? ""; - // 1. Interactive login shell — sources both .zprofile and .zshrc. - try { - resolvedPath = execSync(`${shell} -ilc 'echo $PATH' 2>/dev/null`, { - encoding: "utf8", - timeout: 30000, - }).trim() || resolvedPath; - } catch { /* timeout or shell error — keep baseline */ } - - // 2. Explicit NVM: source nvm.sh on top of the already-resolved PATH so - // NVM prepends its bin dirs without losing Homebrew/login-shell entries. - try { - const nvmDir = process.env.NVM_DIR ?? `${process.env.HOME}/.nvm`; - const nvmScript = `${nvmDir}/nvm.sh`; - const nvmPath = execSync( - `[ -s "${nvmScript}" ] && export PATH="${resolvedPath}" && . "${nvmScript}" && echo $PATH`, - { encoding: "utf8", timeout: 5000, shell: "/bin/zsh" }, - ).trim(); - if (nvmPath) resolvedPath = nvmPath; - } catch { /* nvm not installed or failed */ } - - // Log so we can verify key tools are present after restarts - const pathEntries = resolvedPath.split(":"); - log.info( - { entries: pathEntries.length, ...probeTools(resolvedPath, ["node", "pnpm", "doppler"]) }, - "PATH resolved", - ); - - return resolvedPath; + const override = getSetting("rt.daemonPath").value; + let result: string; + let source: string; + + if (typeof override === "string" && override.trim().length > 0) { + result = override.trim(); + source = "override"; + } else { + const shell = process.env.SHELL ?? "/bin/zsh"; + const isFish = basename(shell) === "fish"; + const baseArgv: [string, ...string[]] = isFish + ? [shell, "-lc", "string join : $PATH"] + : [ + shell, + "-lc", + `{ [ -s "\${NVM_DIR:-$HOME/.nvm}/nvm.sh" ] && . "\${NVM_DIR:-$HOME/.nvm}/nvm.sh" >/dev/null 2>&1; }; printf %s "$PATH"`, + ]; + const base = validateBase(await probe(baseArgv, { timeoutMs: BASE_TIMEOUT_MS }), baseline); + result = base.path; + source = base.source; + if (base.reason) log.warn({ reason: base.reason }, "PATH base probe unusable; kept baseline"); + + const ovArgv: [string, ...string[]] = isFish ? [shell, "-ilc", "string join : $PATH"] : [shell, "-ilc", "echo $PATH"]; + const ovRaw = await probe(ovArgv, { timeoutMs: OVERLAY_TIMEOUT_MS, env: { ...process.env, TERM: "dumb" } }); + const extra = absoluteDirsOf(ovRaw); + if (extra.length === 0) { + // Warn on BOTH timeout (null) and garbage (non-null but no usable + // absolute dirs) ... the ruling says timeout OR garbage. + log.warn("PATH interactive overlay skipped (timed out or no usable dirs)"); + } else { + const before = result; + result = unionAppend(result, extra); + if (result !== before) source += "+overlay"; + } + } + + const probed = probeTools(result, ["node", "git", "bun", "pnpm"]); + const missing = Object.entries(probed) + .filter(([, v]) => !v) + .map(([k]) => k.replace(/^has/, "").toLowerCase()); + if (missing.length > 0) log.warn({ missing }, "PATH missing required tools; set rt.daemonPath to override"); + log.info({ source, entries: result.split(":").length, ...probed }, "PATH resolved"); + return result; } From 258973b76def20848f522a3eabfec4117708daba Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:20:10 -0500 Subject: [PATCH 106/142] daemon-logger: stream error listener + loggerDegraded + crash-handler raw-write fallback --- .../daemon-logger-resilience.test.ts | 14 +++++ lib/daemon-logger.ts | 51 +++++++++++++++++-- 2 files changed, 61 insertions(+), 4 deletions(-) create mode 100644 lib/__tests__/daemon-logger-resilience.test.ts diff --git a/lib/__tests__/daemon-logger-resilience.test.ts b/lib/__tests__/daemon-logger-resilience.test.ts new file mode 100644 index 00000000..1ee7aedc --- /dev/null +++ b/lib/__tests__/daemon-logger-resilience.test.ts @@ -0,0 +1,14 @@ +import { test, expect } from "bun:test"; +import { createDaemonLogger } from "../daemon-logger.ts"; +import { mkdtempSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +test("a stream write error does not throw out of log.info and flips loggerDegraded", async () => { + const dir = mkdtempSync(join(tmpdir(), "logres-")); + const handle = await createDaemonLogger({ logDir: dir, level: "info" }); + // Simulate a write failure by emitting 'error' on the underlying stream. + handle.stream.emit("error", Object.assign(new Error("no space"), { code: "ENOSPC" })); + expect(() => handle.logger.info("after enospc")).not.toThrow(); + expect(handle.loggerDegraded()).toBe(true); +}); diff --git a/lib/daemon-logger.ts b/lib/daemon-logger.ts index 634e84b0..1ba4f449 100644 --- a/lib/daemon-logger.ts +++ b/lib/daemon-logger.ts @@ -18,17 +18,30 @@ import pino, { type Logger } from "pino"; // @ts-ignore — no types shipped; the JS API is well-tested. import roll from "pino-roll"; import { dlopen, suffix, FFIType } from "bun:ffi"; -import { mkdirSync, openSync, closeSync, existsSync, statSync, renameSync } from "fs"; +import { mkdirSync, openSync, closeSync, existsSync, statSync, renameSync, writeSync } from "fs"; import { join } from "path"; import { logsDir } from "./rt-paths.ts"; +/** Last-resort write straight to fd 2, bypassing pino entirely — used only when the logger itself has failed or can't be trusted. */ +function rawStderr(text: string): void { + try { + writeSync(2, text); + } catch { + // Nothing left to do — even fd 2 is gone. + } +} + export interface DaemonLoggerHandle { /** Root logger — use when no specific module scope applies. */ logger: Logger; + /** Underlying pino-roll write stream — exposed as a test seam for simulating write errors. */ + stream: NodeJS.WritableStream; /** Returns a child logger that stamps `module: ` on every line. */ childLogger: (module: string) => Logger; /** Force a flush (best-effort; pino-roll's stream is sync but exposes flushSync). */ flush?: () => void; + /** True once the underlying stream has emitted an 'error' (e.g. ENOSPC) — writes since then were swallowed, not lost silently. */ + loggerDegraded: () => boolean; } export interface CreateOptions { @@ -56,6 +69,17 @@ export async function createDaemonLogger(opts: CreateOptions): Promise { + degraded = true; + rawStderr(`daemon-logger: stream error ${err?.code ?? ""} ${err?.message ?? err}\n`); + }); + const logger = pino( { level: opts.level ?? "info", @@ -74,11 +98,13 @@ export async function createDaemonLogger(opts: CreateOptions): Promise logger.child({ module }), flush: () => { // pino's flushSync drains any buffered writes; safe to call repeatedly. try { logger.flush(); } catch { /* */ } }, + loggerDegraded: () => degraded, }; } @@ -268,18 +294,35 @@ export function installCrashHandlers( // Because the pino-roll stream is opened with sync:true, logger.fatal() // flushes immediately to the fd — no need for pino.final() here. + // + // The logger.*() calls below are wrapped in try/catch: a logging failure + // (e.g. the stream is degraded from ENOSPC) must not itself abort a crash + // handler and skip the exit it's here to guarantee — only the logging is + // guarded, never the exit decision. process.on("uncaughtException", (err) => { - logger.fatal({ err }, "uncaughtException"); + try { + logger.fatal({ err }, "uncaughtException"); + } catch { + rawStderr(`uncaughtException (logger failed): ${err?.stack ?? err}\n`); + } process.exit(1); }); process.on("unhandledRejection", (reason) => { if (opts.booting?.()) { - logger.fatal({ err: reason }, "unhandledRejection during boot"); + try { + logger.fatal({ err: reason }, "unhandledRejection during boot"); + } catch { + rawStderr(`unhandledRejection during boot (logger failed): ${reason}\n`); + } process.exit(1); return; } - logger.error({ err: reason }, "unhandledRejection"); + try { + logger.error({ err: reason }, "unhandledRejection"); + } catch { + rawStderr(`unhandledRejection (logger failed): ${reason}\n`); + } }); // Intercept process.stderr.write so JS-side stderr writes land in the log. From 4b2e1992a6c022513cefd3fcbb73eaec48a1977b Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:20:46 -0500 Subject: [PATCH 107/142] daemon-logger: drop em dashes from new comments (style) --- lib/daemon-logger.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/lib/daemon-logger.ts b/lib/daemon-logger.ts index 1ba4f449..06ea317e 100644 --- a/lib/daemon-logger.ts +++ b/lib/daemon-logger.ts @@ -22,25 +22,25 @@ import { mkdirSync, openSync, closeSync, existsSync, statSync, renameSync, write import { join } from "path"; import { logsDir } from "./rt-paths.ts"; -/** Last-resort write straight to fd 2, bypassing pino entirely — used only when the logger itself has failed or can't be trusted. */ +/** Last-resort write straight to fd 2, bypassing pino entirely. Used only when the logger itself has failed or can't be trusted. */ function rawStderr(text: string): void { try { writeSync(2, text); } catch { - // Nothing left to do — even fd 2 is gone. + // Nothing left to do... even fd 2 is gone. } } export interface DaemonLoggerHandle { /** Root logger — use when no specific module scope applies. */ logger: Logger; - /** Underlying pino-roll write stream — exposed as a test seam for simulating write errors. */ + /** Underlying pino-roll write stream. Exposed as a test seam for simulating write errors. */ stream: NodeJS.WritableStream; /** Returns a child logger that stamps `module: ` on every line. */ childLogger: (module: string) => Logger; /** Force a flush (best-effort; pino-roll's stream is sync but exposes flushSync). */ flush?: () => void; - /** True once the underlying stream has emitted an 'error' (e.g. ENOSPC) — writes since then were swallowed, not lost silently. */ + /** True once the underlying stream has emitted an 'error' (e.g. ENOSPC); writes since then were swallowed, not lost silently. */ loggerDegraded: () => boolean; } @@ -70,10 +70,10 @@ export async function createDaemonLogger(opts: CreateOptions): Promise { degraded = true; @@ -297,7 +297,7 @@ export function installCrashHandlers( // // The logger.*() calls below are wrapped in try/catch: a logging failure // (e.g. the stream is degraded from ENOSPC) must not itself abort a crash - // handler and skip the exit it's here to guarantee — only the logging is + // handler and skip the exit it's here to guarantee. Only the logging is // guarded, never the exit decision. process.on("uncaughtException", (err) => { try { From b8a9948e2fefe35e8c179cde423f4a1cdb0688d3 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:20:57 -0500 Subject: [PATCH 108/142] user-path: reword overlay comment to drop process-artifact reference --- lib/daemon/user-path.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/daemon/user-path.ts b/lib/daemon/user-path.ts index ccafb309..eb31f729 100644 --- a/lib/daemon/user-path.ts +++ b/lib/daemon/user-path.ts @@ -153,8 +153,8 @@ export async function resolveUserPath(log: Logger, probe: ProbeFn = runProbe): P const ovRaw = await probe(ovArgv, { timeoutMs: OVERLAY_TIMEOUT_MS, env: { ...process.env, TERM: "dumb" } }); const extra = absoluteDirsOf(ovRaw); if (extra.length === 0) { - // Warn on BOTH timeout (null) and garbage (non-null but no usable - // absolute dirs) ... the ruling says timeout OR garbage. + // Warn on both timeout (null) and garbage (non-null but no usable + // absolute dirs): either way the overlay contributed nothing. log.warn("PATH interactive overlay skipped (timed out or no usable dirs)"); } else { const before = result; From 36ebab37a1797e55c92f90c22d06f72f4e353a04 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:22:51 -0500 Subject: [PATCH 109/142] daemon: await async resolveUserPath; drop user-path sync-exec allowlist (6.1) --- lib/__tests__/no-daemon-sync-exec.test.ts | 1 - lib/daemon.ts | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/__tests__/no-daemon-sync-exec.test.ts b/lib/__tests__/no-daemon-sync-exec.test.ts index f5d8c753..97baba52 100644 --- a/lib/__tests__/no-daemon-sync-exec.test.ts +++ b/lib/__tests__/no-daemon-sync-exec.test.ts @@ -7,7 +7,6 @@ import { dirname, resolve } from "path"; // A regression that reintroduces sync-exec into any OTHER daemon-reachable // module fails this gate (the rule has been re-broken twice). const ALLOWLIST = new Set([ - "lib/daemon/user-path.ts", // Phase 6 PATH rebuild (S013/S014/S062) "lib/daemon/boot-reconcile.ts", // Phase 0.6 / S044 (Bun.sleepSync) "lib/state/db.ts", // Phase 0.7 / S072-S073 busy-retry "lib/state/busy.ts", // Phase 0.7 / S072-S073 busy-retry diff --git a/lib/daemon.ts b/lib/daemon.ts index 3c3a40e9..4773a860 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -160,7 +160,7 @@ const systemProcessScanner = new SystemProcessScanner(); // runCapture forwards process.env explicitly (lib/subprocess.ts) because // Bun.spawn would otherwise ignore this assignment. { - const resolvedPath = resolveUserPath(log); + const resolvedPath = await resolveUserPath(log); if (resolvedPath) process.env.PATH = resolvedPath; } From 494bb520438e92c02fc3f8722d738700c2daa051 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:25:41 -0500 Subject: [PATCH 110/142] daemon-logger: rt.logLevel resolution, stderr->warn demotion, 50m size cap, recovered-error counter --- lib/__tests__/daemon-logger-level.test.ts | 24 ++++++++++ lib/daemon-logger.ts | 58 +++++++++++++++++++++-- 2 files changed, 79 insertions(+), 3 deletions(-) create mode 100644 lib/__tests__/daemon-logger-level.test.ts diff --git a/lib/__tests__/daemon-logger-level.test.ts b/lib/__tests__/daemon-logger-level.test.ts new file mode 100644 index 00000000..c461a639 --- /dev/null +++ b/lib/__tests__/daemon-logger-level.test.ts @@ -0,0 +1,24 @@ +import { test, expect } from "bun:test"; +import { resolveDaemonLogLevel, isPanicLine } from "../daemon-logger.ts"; + +test("RT_LOG_LEVEL env wins over the setting", () => { + expect(resolveDaemonLogLevel("debug", () => "warn")).toBe("debug"); +}); +test("setting is used when env is unset", () => { + expect(resolveDaemonLogLevel(undefined, () => "warn")).toBe("warn"); +}); +test("falls back to info when neither is set", () => { + expect(resolveDaemonLogLevel(undefined, () => undefined)).toBe("info"); +}); +test("a thrown setting read falls back to info instead of propagating", () => { + expect( + resolveDaemonLogLevel(undefined, () => { + throw new Error("unknown key: rt.logLevel"); + }), + ).toBe("info"); +}); +test("a panic-looking stderr line is escalated; ordinary noise is not", () => { + expect(isPanicLine("panic: runtime error")).toBe(true); + expect(isPanicLine("Uncaught Error: boom")).toBe(true); + expect(isPanicLine("rt: ignoring \"x\" from the team scope")).toBe(false); +}); diff --git a/lib/daemon-logger.ts b/lib/daemon-logger.ts index 06ea317e..3950de69 100644 --- a/lib/daemon-logger.ts +++ b/lib/daemon-logger.ts @@ -21,6 +21,7 @@ import { dlopen, suffix, FFIType } from "bun:ffi"; import { mkdirSync, openSync, closeSync, existsSync, statSync, renameSync, writeSync } from "fs"; import { join } from "path"; import { logsDir } from "./rt-paths.ts"; +import { getSetting } from "./settings/resolve.ts"; /** Last-resort write straight to fd 2, bypassing pino entirely. Used only when the logger itself has failed or can't be trusted. */ function rawStderr(text: string): void { @@ -42,6 +43,8 @@ export interface DaemonLoggerHandle { flush?: () => void; /** True once the underlying stream has emitted an 'error' (e.g. ENOSPC); writes since then were swallowed, not lost silently. */ loggerDegraded: () => boolean; + /** Count of errors that were observed and handled without crashing the daemon: demoted stderr noise plus steady-state recovered unhandledRejections. */ + recoveredErrorCount: () => number; } export interface CreateOptions { @@ -49,6 +52,40 @@ export interface CreateOptions { level?: pino.LevelWithSilent; } +/** + * Resolves the daemon's pino level: RT_LOG_LEVEL env, then the `rt.logLevel` + * setting, then "info". The setting read is try/catch-guarded because the + * `rt.logLevel` registry key may not exist yet (added in a later task), and + * the resolver may also run pre-boot; this must never throw. + */ +export function resolveDaemonLogLevel( + env: string | undefined, + fromSetting: () => string | undefined, +): string { + if (env) return env; + try { + const v = fromSetting(); + if (v) return v; + } catch { + // Setting unavailable (unknown key pre-registration, or resolver not + // ready yet)... fall through to the "info" default below. + } + return "info"; +} + +const PANIC_PREFIXES = ["panic:", "fatal error:", "Uncaught ", "UnhandledPromiseRejection"]; + +/** True for stderr text that looks like a native/runtime panic, not ordinary noise (warnings, CLI messages). */ +export function isPanicLine(text: string): boolean { + return PANIC_PREFIXES.some((p) => text.startsWith(p)); +} + +// Counts errors observed and handled without crashing the daemon: demoted +// stderr lines (installCrashHandlers) plus steady-state recovered +// unhandledRejections. Module-scoped (one daemon process, one counter) rather +// than per-handle, matching the process-wide handlers that increment it. +let recovered = 0; + /** * Async factory — call once at daemon startup OR in each test. * pino-roll's default export is async (it stats the dir + sets up the writer). @@ -65,6 +102,7 @@ export async function createDaemonLogger(opts: CreateOptions): Promise degraded, + recoveredErrorCount: () => recovered, }; } @@ -125,7 +164,10 @@ export async function getDaemonLogger(): Promise { if (!cachedPromise) { cachedPromise = createDaemonLogger({ logDir: logsDir(), - level: (process.env.RT_LOG_LEVEL as pino.LevelWithSilent | undefined) ?? "info", + level: resolveDaemonLogLevel( + process.env.RT_LOG_LEVEL, + () => getSetting("rt.logLevel").value, + ) as pino.LevelWithSilent, }).catch((err) => { // Clear the cache on failure — a transient cause (log dir momentarily // unwritable) may not recur, so a later call should retry rather than @@ -284,7 +326,9 @@ export function redirectNativeStderr(): void { * daemon that's already serving. No `booting` given preserves the old * always-log, never-exit behavior. * - process.stderr.write: intercept so console.error / anything writing to - * stderr lands in the JSON log instead of vanishing. + * stderr lands in the JSON log instead of vanishing, at `warn` (ordinary + * noise) or `error` (a panic-looking line per isPanicLine); demoted lines + * also count toward recoveredErrorCount(). */ export function installCrashHandlers( handle: DaemonLoggerHandle, @@ -318,6 +362,7 @@ export function installCrashHandlers( process.exit(1); return; } + recovered += 1; try { logger.error({ err: reason }, "unhandledRejection"); } catch { @@ -332,7 +377,14 @@ export function installCrashHandlers( try { const text = typeof chunk === "string" ? chunk : Buffer.from(chunk).toString(); const trimmed = text.replace(/\n+$/, ""); - if (trimmed.length > 0) logger.error({ source: "stderr" }, trimmed); + if (trimmed.length > 0) { + if (isPanicLine(trimmed)) { + logger.error({ source: "stderr" }, trimmed); + } else { + recovered += 1; + logger.warn({ source: "stderr" }, trimmed); + } + } } catch { // If anything in the logger fails, fall back to the original stderr. return origWrite(chunk, ...rest); From 65ca166548fa3145628e0208bf3d8383f0278aa1 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:25:48 -0500 Subject: [PATCH 111/142] home: stable machine-key at init, data-preserving freeze of existing stores (S071) --- commands/home.ts | 3 +- lib/home/__tests__/machine-id.test.ts | 38 ++++++++++++++ lib/home/machine-id.ts | 76 +++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 1 deletion(-) create mode 100644 lib/home/__tests__/machine-id.test.ts create mode 100644 lib/home/machine-id.ts diff --git a/commands/home.ts b/commands/home.ts index eedd39f0..2d693377 100644 --- a/commands/home.ts +++ b/commands/home.ts @@ -38,6 +38,7 @@ import { join } from "path"; import type { CommandContext } from "../lib/command-tree.ts"; import { bold, dim, green, red, reset, yellow } from "../lib/ansi.ts"; import { isSafeMachineKeySegment, machineKey, mattstackHome } from "../lib/rt-paths.ts"; +import { resolveInitialMachineKey } from "../lib/home/machine-id.ts"; import { buildInitPlan, chooseMachineProfile, @@ -549,7 +550,7 @@ export async function homeInit(args: string[], _ctx: CommandContext = {}, seams: const exec = seams.exec ?? createRealExecSeam(mattstackHome()); const ageKeySeam = seams.ageKeySeam ?? createRealAgeKeySeam(); const sopsYamlSeam = seams.sopsYamlSeam ?? defaultSopsYamlSeam(); - const key = seams.key ?? machineKey(); + const key = seams.key ?? (await resolveInitialMachineKey(mattstackHome(), probes)); const pickerSeam = seams.pickerSeam ?? createRealMachineProfilePickerSeam(); const isInteractive = seams.isInteractive ?? (() => Boolean(process.stdin.isTTY)); const materializeExec = seams.materializeExec ?? defaultMaterializeExec(); diff --git a/lib/home/__tests__/machine-id.test.ts b/lib/home/__tests__/machine-id.test.ts new file mode 100644 index 00000000..54dbedc9 --- /dev/null +++ b/lib/home/__tests__/machine-id.test.ts @@ -0,0 +1,38 @@ +import { expect, test } from "bun:test"; +import { resolveInitialMachineKey, stableMachineId } from "../machine-id.ts"; + +const IOREG_FIXTURE = ` "IOPlatformUUID" = "D9E8F7A6-1234-5678-9ABC-DEF012345678"`; + +test("stableMachineId parses IOPlatformUUID and slugs it", async () => { + const id = await stableMachineId(async () => IOREG_FIXTURE); + expect(id).toBe("d9e8f7a6-1234-5678-9abc-def012345678"); +}); + +test("stableMachineId returns null when ioreg fails", async () => { + expect(await stableMachineId(async () => null)).toBeNull(); + expect(await stableMachineId(async () => "no uuid here")).toBeNull(); +}); + +test("resolveInitialMachineKey: existing pin file is returned unchanged", async () => { + const probes = { exists: (p: string) => p.endsWith("machine-key"), listProfiles: () => [] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => "pinned-key", stableId: async () => "uuid-x" }); + expect(key).toBe("pinned-key"); +}); + +test("resolveInitialMachineKey: existing non-empty hostname-slug store freezes the slug", async () => { + const probes = { exists: () => false, listProfiles: () => ["myhost"] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => null, hostnameSlug: () => "myhost", stableId: async () => "uuid-x" }); + expect(key).toBe("myhost"); // frozen, data preserved, no move +}); + +test("resolveInitialMachineKey: fresh machine gets the stable id", async () => { + const probes = { exists: () => false, listProfiles: () => [] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => null, hostnameSlug: () => "myhost", stableId: async () => "uuid-x" }); + expect(key).toBe("uuid-x"); +}); + +test("resolveInitialMachineKey: fresh machine, ioreg fails -> hostname slug", async () => { + const probes = { exists: () => false, listProfiles: () => [] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => null, hostnameSlug: () => "myhost", stableId: async () => null }); + expect(key).toBe("myhost"); +}); diff --git a/lib/home/machine-id.ts b/lib/home/machine-id.ts new file mode 100644 index 00000000..85c93b61 --- /dev/null +++ b/lib/home/machine-id.ts @@ -0,0 +1,76 @@ +import { readFileSync } from "fs"; +import { join } from "path"; +import { isSafeMachineKeySegment, machineKey } from "../rt-paths.ts"; +import type { HomeProbes } from "../../commands/home.ts"; + +/** IOPlatformUUID via ioreg, slugged; null on any failure (non-mac, CI, no match). */ +export async function stableMachineId( + exec: (argv: string[]) => Promise = defaultIoreg, +): Promise { + const out = await exec(["ioreg", "-rd1", "-c", "IOPlatformExpertDevice"]); + if (!out) return null; + const m = out.match(/"IOPlatformUUID"\s*=\s*"([^"]+)"/); + if (!m) return null; + const slug = m[1]! + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-+|-+$/g, ""); + return isSafeMachineKeySegment(slug) ? slug : null; +} + +const defaultIoreg = async (argv: string[]): Promise => { + try { + const proc = Bun.spawn(argv, { stdin: "ignore", stdout: "pipe", stderr: "ignore" }); + const term = setTimeout(() => { + try { + proc.kill("SIGKILL"); + } catch { + // already gone + } + }, 3_000); + try { + const [out, code] = await Promise.all([new Response(proc.stdout as ReadableStream).text(), proc.exited]); + return code === 0 ? out : null; + } finally { + clearTimeout(term); + } + } catch { + return null; + } +}; + +interface InitKeyDeps { + readPin?: () => string | null; + hostnameSlug?: () => string; + stableId?: () => Promise; +} + +/** + * Establishes the machine key at `rt home init`. Data-preserving and idempotent: + * an existing pin is kept as-is; a machine whose hostname-slug store already + * carries settings freezes that slug (zero data movement); only a genuinely + * fresh machine gets the stable id. + */ +export async function resolveInitialMachineKey(home: string, probes: HomeProbes, deps: InitKeyDeps = {}): Promise { + const readPin = + deps.readPin ?? + (() => { + try { + const v = readFileSync(join(home, "machine-key"), "utf8").trim(); + return v || null; + } catch { + return null; + } + }); + const hostnameSlug = deps.hostnameSlug ?? (() => machineKey()); // machineKey() with no pin returns the hostname slug + const stableId = deps.stableId ?? (() => stableMachineId()); + + const pinned = readPin(); + if (pinned && isSafeMachineKeySegment(pinned)) return pinned; + + const slug = hostnameSlug(); + const profiles = probes.listProfiles(join(home, "user", "local")); // dirs carrying settings.local.jsonc + if (profiles.includes(slug)) return slug; // freeze existing non-empty store + + return (await stableId()) ?? slug; +} From 40aeb1351eca1d7f6f3861b693df9c2b2db06558 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:29:51 -0500 Subject: [PATCH 112/142] log-janitor: onError callback; daemon logs prune failures at warn --- lib/__tests__/log-janitor.test.ts | 7 +++++++ lib/daemon.ts | 6 ++++-- lib/log-janitor.ts | 15 +++++++++++---- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/lib/__tests__/log-janitor.test.ts b/lib/__tests__/log-janitor.test.ts index 0332a04c..fa9098e3 100644 --- a/lib/__tests__/log-janitor.test.ts +++ b/lib/__tests__/log-janitor.test.ts @@ -81,4 +81,11 @@ describe("pruneLogs", () => { const { removed } = pruneLogs(dir, 14, now); expect(removed).toEqual(["tray.2026-08-01.log"]); }); + + test("readdir failure reports via onError instead of swallowing", () => { + const calls: string[] = []; + const bogus = join("/nonexistent-xyz", "rt", "logs"); + pruneLogs(bogus, 14, Date.now(), (phase) => calls.push(phase)); + expect(calls).toContain("readdir"); + }); }); diff --git a/lib/daemon.ts b/lib/daemon.ts index 3c3a40e9..8b01036d 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -276,7 +276,8 @@ function logRetentionDays(): number { } setInterval(() => { try { - const { removed } = pruneLogs(logsDir(), logRetentionDays(), Date.now()); + const { removed } = pruneLogs(logsDir(), logRetentionDays(), Date.now(), + (phase, err, file) => log.warn({ err, phase, file }, "log prune step failed")); if (removed.length > 0) log.info({ removed: removed.length }, "pruned old surface logs"); } catch (err) { log.warn({ err }, "log prune failed"); @@ -285,7 +286,8 @@ setInterval(() => { // Boot-time sweep to handle frequent daemon restarts that would otherwise starve the daily interval. setTimeout(() => { try { - const { removed } = pruneLogs(logsDir(), logRetentionDays(), Date.now()); + const { removed } = pruneLogs(logsDir(), logRetentionDays(), Date.now(), + (phase, err, file) => log.warn({ err, phase, file }, "log prune step failed")); if (removed.length > 0) log.info({ removed: removed.length }, "pruned old surface logs"); } catch (err) { log.warn({ err }, "log prune failed"); diff --git a/lib/log-janitor.ts b/lib/log-janitor.ts index 7451de3f..618e301c 100644 --- a/lib/log-janitor.ts +++ b/lib/log-janitor.ts @@ -34,7 +34,12 @@ function assertLogsDir(dir: string): void { * directories) whose name matches the surface log pattern and whose mtime is * older than `retentionDays` back from `now`. Returns the basenames removed. */ -export function pruneLogs(dir: string, retentionDays: number, now: number): { removed: string[] } { +export function pruneLogs( + dir: string, + retentionDays: number, + now: number, + onError?: (phase: "readdir" | "unlink", err: unknown, file?: string) => void, +): { removed: string[] } { assertLogsDir(dir); const cutoff = now - retentionDays * DAY; @@ -43,7 +48,8 @@ export function pruneLogs(dir: string, retentionDays: number, now: number): { re let entries; try { entries = readdirSync(dir, { withFileTypes: true }); - } catch { + } catch (err) { + onError?.("readdir", err); return { removed }; } @@ -62,8 +68,9 @@ export function pruneLogs(dir: string, retentionDays: number, now: number): { re try { unlinkSync(full); removed.push(entry.name); - } catch { - // best-effort — a file gone or unreadable between stat and unlink is not fatal + } catch (err) { + // best-effort: a file gone or unreadable between stat and unlink is not fatal + onError?.("unlink", err, entry.name); } } From 7e83441a12dcbad30330ce60b11c261b9cc71be1 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:30:23 -0500 Subject: [PATCH 113/142] plan: Task 14 imports setSettingsWarnSink from local ./settings/resolve.ts barrel Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/superpowers/plans/2026-08-28-p2-health.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/superpowers/plans/2026-08-28-p2-health.md b/docs/superpowers/plans/2026-08-28-p2-health.md index 22834bd9..9793a565 100644 --- a/docs/superpowers/plans/2026-08-28-p2-health.md +++ b/docs/superpowers/plans/2026-08-28-p2-health.md @@ -1671,7 +1671,7 @@ import { writeHeartbeat } from "./daemon/heartbeat-file.ts"; import { computeHealth } from "./daemon/health.ts"; import { apiWsClientCount } from "./daemon/api-server.ts"; import { isCrashLooping, readSupervisionState } from "./daemon/supervision-state.ts"; -import { setSettingsWarnSink } from "@mattstack/rt-client"; +import { setSettingsWarnSink } from "./settings/resolve.ts"; // local barrel re-exports packages/rt-client resolver // Bind the resolver's warn sink to a deduped daemon log.warn (S033/R005): a // hot-path getSetting on a disallowed-scope key warns once, not every tick. From 4c725d19fb1c53153ed240b2601f89faf4160c30 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:33:29 -0500 Subject: [PATCH 114/142] rt-client: rt.logLevel registry row + injectable deduped settings warn sink (dist rebuilt, no bump) --- packages/rt-client/src/index.ts | 2 +- .../rt-client/src/settings/registry-defs.ts | 9 +++++++ packages/rt-client/src/settings/resolve.ts | 27 ++++++++++++++++--- .../rt-client/test/settings-warn-sink.test.ts | 12 +++++++++ 4 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 packages/rt-client/test/settings-warn-sink.test.ts diff --git a/packages/rt-client/src/index.ts b/packages/rt-client/src/index.ts index 54ccad6b..9c098b42 100644 --- a/packages/rt-client/src/index.ts +++ b/packages/rt-client/src/index.ts @@ -91,7 +91,7 @@ export { repoNameForPath } from "./repos.ts"; // ─── Settings (RT-50) ──────────────────────────────────────────────────────── -export { getSetting, listSettings, explainSetting, expandVariables, SCOPE_ORDER } from "./settings/resolve.ts"; +export { getSetting, listSettings, explainSetting, expandVariables, SCOPE_ORDER, setSettingsWarnSink } from "./settings/resolve.ts"; export type { Scope, Provenance, diff --git a/packages/rt-client/src/settings/registry-defs.ts b/packages/rt-client/src/settings/registry-defs.ts index 8a00016b..6ab0f683 100644 --- a/packages/rt-client/src/settings/registry-defs.ts +++ b/packages/rt-client/src/settings/registry-defs.ts @@ -196,6 +196,15 @@ export const REGISTRY: readonly SettingDef[] = [ migrated: true, description: "Age floor in days for the log janitor pruning every surface's rotated log files under ~/.mattstack/rt/logs (default 14). A fresh key, not an ownership-latch port, so a default is fine here.", }, + { + key: "rt.logLevel", + type: "string", + scopes: ["machine", "user"], + default: "info", + merge: "replace", + migrated: true, + description: "Daemon log level (trace|debug|info|warn|error). RT_LOG_LEVEL env wins, then this setting, then info (lib/daemon-logger.ts resolveDaemonLogLevel). A fresh key, not an ownership-latch port, so a default is fine here.", + }, { key: "rt.apiPort", type: "number", diff --git a/packages/rt-client/src/settings/resolve.ts b/packages/rt-client/src/settings/resolve.ts index 9c07a2a8..6e949981 100644 --- a/packages/rt-client/src/settings/resolve.ts +++ b/packages/rt-client/src/settings/resolve.ts @@ -487,8 +487,29 @@ function expandCtxFrom(opts: ResolveOpts): ExpandCtx { }; } +let warnSink: ((msg: string) => void) | null = null; +const warnedOnce = new Set(); + +/** The daemon binds a deduped log.warn here so a hot-path getSetting on a + * disallowed-scope key warns once, not every tick. Default: console.warn + * (CLI/test behavior unchanged). null restores the default. */ +export function setSettingsWarnSink(sink: ((msg: string) => void) | null): void { + warnSink = sink; + warnedOnce.clear(); +} + +export function emitSettingsWarning(msg: string): void { + if (warnSink) { + if (warnedOnce.has(msg)) return; + warnedOnce.add(msg); + warnSink(msg); + return; + } + console.warn(msg); +} + function warnInvalid(key: string, entry: InvalidScope): void { - console.warn( + emitSettingsWarning( `rt: ignoring "${key}" from the ${entry.scope} scope (${entry.file ?? "no file"}): ${entry.reason}`, ); } @@ -543,7 +564,7 @@ export function listSettings(opts: ResolveOpts = {}): ListedSetting[] { listed.value = expandVariables(resolution.value, ctx); } catch (err) { listed.expandError = (err as Error).message; - console.warn(`rt: showing "${def.key}" unexpanded — ${listed.expandError}`); + emitSettingsWarning(`rt: showing "${def.key}" unexpanded — ${listed.expandError}`); } } @@ -583,7 +604,7 @@ function listUnregistered(stores: StoreBundle, opts: ResolveOpts): ListedSetting return [...found.entries()] .sort(([a], [b]) => a.localeCompare(b)) .map(([key, hit]) => { - console.warn( + emitSettingsWarning( `rt: unregistered setting "${key}" in ${hit.file} — ignoring it (this rt may be older than the store)`, ); return { diff --git a/packages/rt-client/test/settings-warn-sink.test.ts b/packages/rt-client/test/settings-warn-sink.test.ts new file mode 100644 index 00000000..d0bc9480 --- /dev/null +++ b/packages/rt-client/test/settings-warn-sink.test.ts @@ -0,0 +1,12 @@ +import { test, expect } from "bun:test"; +import { setSettingsWarnSink } from "../src/index.ts"; +import { emitSettingsWarning } from "../src/settings/resolve.ts"; + +test("a bound sink receives warnings and dedupes on identical messages", () => { + const seen: string[] = []; + setSettingsWarnSink((m) => seen.push(m)); + emitSettingsWarning("rt: sample warning"); + emitSettingsWarning("rt: sample warning"); + expect(seen).toEqual(["rt: sample warning"]); // deduped + setSettingsWarnSink(null); // restore default +}); From 3b7daf6ced3196090d5a7b719bcacb100129d207 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:35:57 -0500 Subject: [PATCH 115/142] dev-mode: marker-based wrapper detection, bounded read, legacy fallback (S020/S067) --- commands/settings.ts | 3 +- lib/__tests__/dev-mode.test.ts | 48 +++++++++++++++++++++++++--- lib/deps/__tests__/links.test.ts | 9 ++++-- lib/deps/links.ts | 19 +++++------ lib/dev-mode.ts | 55 ++++++++++++++++++++++++-------- 5 files changed, 103 insertions(+), 31 deletions(-) diff --git a/commands/settings.ts b/commands/settings.ts index 6871dfb4..f90d4a40 100644 --- a/commands/settings.ts +++ b/commands/settings.ts @@ -16,7 +16,7 @@ import { TRAY_APP_NAME, DEV_TRAY_APP_NAME, TRAY_APP_BUNDLE, trayAppPath, devTrayAppPath, } from "../lib/rt-paths.ts"; -import { installRtBinary } from "../lib/dev-mode.ts"; +import { DEV_MODE_TAG, installRtBinary } from "../lib/dev-mode.ts"; import { describeTuple, tupleWarning, type FlavorTuple } from "./daemon.ts"; import { RT_BUNDLE_PATH } from "../lib/bundle-layout.ts"; import { spawnSync } from "child_process"; @@ -511,6 +511,7 @@ export function renderDevModeWrapper(sourcePath: string, bunPath: string): strin const bunDir = dirname(bunPath); return [ `#!/bin/zsh`, + `${DEV_MODE_TAG}`, `export PATH="${bunDir}:/opt/homebrew/bin:/usr/local/bin:$PATH"`, `export RT_LAUNCH_CWD="$PWD"`, `cd "${sourcePath}" || { echo "rt: dev-mode source checkout missing: ${sourcePath}" >&2; exit 1; }`, diff --git a/lib/__tests__/dev-mode.test.ts b/lib/__tests__/dev-mode.test.ts index e8dd0ea6..613c1c21 100644 --- a/lib/__tests__/dev-mode.test.ts +++ b/lib/__tests__/dev-mode.test.ts @@ -7,9 +7,9 @@ * activeLaunchdLabel() (which depends on it) rests on a verified foundation. */ import { afterEach, describe, expect, test } from "bun:test"; -import { existsSync, lstatSync, mkdirSync, readlinkSync, realpathSync, rmSync, writeFileSync } from "fs"; +import { existsSync, lstatSync, mkdirSync, readlinkSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "fs"; import { dirname, join } from "path"; -import { currentMode, installRtBinary } from "../dev-mode.ts"; +import { currentMode, DEV_MODE_TAG, installRtBinary, isDevModeWrapperContent } from "../dev-mode.ts"; // The dev-mode wrapper path is resolved at CALL time from process.env.HOME // (mirrors lib/rt-paths.ts's home()), so this constant only needs to match @@ -30,7 +30,7 @@ describe("currentMode", () => { test("reports dev when the wrapper exists at ~/.local/bin/rt", () => { mkdirSync(join(process.env.HOME!, ".local", "bin"), { recursive: true }); - writeFileSync(WRAPPER_PATH, "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + writeFileSync(WRAPPER_PATH, `#!/bin/sh\n${DEV_MODE_TAG}\nexit 0\n`, { mode: 0o755 }); expect(currentMode()).toBe("dev"); }); @@ -45,7 +45,7 @@ describe("currentMode", () => { expect(currentMode()).toBe("prod"); // fakeHome/.local/bin/rt doesn't exist yet mkdirSync(join(fakeHome, ".local", "bin"), { recursive: true }); - writeFileSync(join(fakeHome, ".local", "bin", "rt"), "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + writeFileSync(join(fakeHome, ".local", "bin", "rt"), `#!/bin/sh\n${DEV_MODE_TAG}\nexit 0\n`, { mode: 0o755 }); expect(currentMode()).toBe("dev"); rmSync(fakeHome, { recursive: true, force: true }); @@ -55,6 +55,44 @@ describe("currentMode", () => { }); }); +describe("isDevModeWrapperContent", () => { + test("new marked wrapper is recognized", () => { + expect(isDevModeWrapperContent(`#!/bin/zsh\n${DEV_MODE_TAG}\nexport PATH=...\n`)).toBe(true); + }); + test("legacy markerless wrapper (RT_LAUNCH_CWD tell) is recognized", () => { + expect(isDevModeWrapperContent(`#!/bin/zsh\nexport PATH="x"\nexport RT_LAUNCH_CWD="$PWD"\n`)).toBe(true); + }); + test("foreign #! script is not a dev wrapper", () => { + expect(isDevModeWrapperContent(`#!/bin/sh\necho hi\n`)).toBe(false); + }); + test("a mattstack-link file is not a dev wrapper", () => { + expect(isDevModeWrapperContent(`#!/bin/sh\n# mattstack-link: rt\nexec ...\n`)).toBe(false); + }); + test("non-shebang content is not a dev wrapper", () => { + expect(isDevModeWrapperContent(`ELF\x00binary`)).toBe(false); + }); +}); + +describe("currentMode bounded read", () => { + afterEach(() => { + try { rmSync(WRAPPER_PATH); } catch { /* already absent */ } + }); + + test("a symlink to a >4KB binary-shaped file classifies as prod without reading the whole file", () => { + mkdirSync(join(process.env.HOME!, ".local", "bin"), { recursive: true }); + const bigBinaryPath = join(process.env.HOME!, "big-binary"); + // Mach-O-ish header followed by >4KB of non-marker filler, so a + // whole-file read (rather than a bounded prefix read) would still + // correctly classify this as prod -- the real proof is that this + // doesn't throw/hang and stays fast even against a multi-MB target. + const filler = Buffer.alloc(8192, 0x41); + writeFileSync(bigBinaryPath, Buffer.concat([Buffer.from([0xcf, 0xfa, 0xed, 0xfe]), filler])); + symlinkSync(bigBinaryPath, WRAPPER_PATH); + + expect(currentMode()).toBe("prod"); + }); +}); + describe("installRtBinary", () => { const BIN = join(process.env.HOME!, ".local", "bin"); afterEach(() => { try { rmSync(join(BIN, "rt")); } catch { /* absent */ } }); @@ -84,7 +122,7 @@ describe("installRtBinary", () => { test("currentMode reads through the link: a link to a script is dev, to a Mach-O is prod", () => { const script = join(process.env.HOME!, "wrapper.sh"); - writeFileSync(script, "#!/bin/zsh\nexit 0\n", { mode: 0o755 }); + writeFileSync(script, `#!/bin/zsh\n${DEV_MODE_TAG}\nexit 0\n`, { mode: 0o755 }); installRtBinary(script); expect(currentMode()).toBe("dev"); }); diff --git a/lib/deps/__tests__/links.test.ts b/lib/deps/__tests__/links.test.ts index 63ea5048..87a20234 100644 --- a/lib/deps/__tests__/links.test.ts +++ b/lib/deps/__tests__/links.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; -import { join } from "path"; +import { dirname, join } from "path"; import { HELPERS_DIR, RT_BUNDLE_PATH, __test__ as bundleLayoutTest } from "../../bundle-layout.ts"; import { setSetting } from "../../settings/write.ts"; import { fakeProbes, type FakeProbesOpts } from "../../setup/__tests__/fakes.ts"; @@ -201,8 +201,13 @@ describe("tagged PATH links", () => { }); test("link(rt) refuses dev-mode-owns-rt when ~/.local/bin/rt is the dev-mode wrapper script", () => { + // isDevModeWrapper reads the real fs at `path` (bounded prefix, never + // through the Probes seam), so this needs a real file on disk -- `home` + // is a real mkdtempSync'd directory, not a fake one. const path = linkPath(home, "rt"); - const p = bundleProbe({ files: { [path]: "#!/bin/sh\nexec bun run cli.ts \"$@\"\n" } }); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, "#!/bin/sh\nexport RT_LAUNCH_CWD=\"$PWD\"\nexec bun run cli.ts \"$@\"\n"); + const p = bundleProbe({ files: { [path]: "#!/bin/sh\nexport RT_LAUNCH_CWD=\"$PWD\"\nexec bun run cli.ts \"$@\"\n" } }); const outcome = link(p, "rt"); expect(outcome).toEqual({ ok: false, reason: "dev-mode-owns-rt", detail: expect.any(String) }); diff --git a/lib/deps/links.ts b/lib/deps/links.ts index df570d16..e4bd3be8 100644 --- a/lib/deps/links.ts +++ b/lib/deps/links.ts @@ -8,7 +8,7 @@ */ import { dirname, join } from "path"; -import { installRtBinary } from "../dev-mode.ts"; +import { installRtBinary, isDevModeWrapperContent, readWrapperPrefix } from "../dev-mode.ts"; import type { Probes } from "../setup/probes.ts"; import { readSetupState, updateSetupState } from "../setup/state.ts"; import { bundledToolExec, isOurLink, LINK_TAG, linkPath, userCopyOnPath } from "./resolve.ts"; @@ -39,14 +39,15 @@ const REAL_SEAMS: LinkSeams = { installRtBinary: (src) => installRtBinary(src) } /** * rt in dev mode is signalled the same way lib/dev-mode.ts's currentMode() - * detects it — a "#!" wrapper script at the link path — but read through the - * Probes seam instead of raw fs, and narrowed to exclude our own tagged - * wrapper (whose second line carries LINK_TAG, not a dev-mode shebang body). + * detects it: shares isDevModeWrapperContent so the two call sites can never + * disagree. Reads the real filesystem (not the Probes seam) via the same + * bounded readWrapperPrefix currentMode() uses -- in prod this path is a + * symlink to the multi-MB compiled binary, so a whole-file read here would + * be exactly the bug this detector exists to avoid. */ -function isDevModeWrapper(p: Pick, path: string): boolean { - const content = p.readFile(path); - if (!content || !content.startsWith("#!")) return false; - return !(content.split("\n")[1] ?? "").startsWith(LINK_TAG); +function isDevModeWrapper(path: string): boolean { + const prefix = readWrapperPrefix(path); + return prefix !== null && isDevModeWrapperContent(prefix); } /** Single-quotes `s` for /bin/sh, escaping embedded single quotes via the standard '\'' trick — safe against $, `, \, " and everything else a relocated bundle path could contain. */ @@ -89,7 +90,7 @@ function clearForced(p: Probes, tool: string): void { export function link(p: Probes, tool: string, opts: { force?: boolean } = {}, seams: LinkSeams = REAL_SEAMS): LinkOutcome { const path = linkPath(p.home, tool); - if (tool === "rt" && isDevModeWrapper(p, path)) { + if (tool === "rt" && isDevModeWrapper(path)) { return { ok: false, reason: "dev-mode-owns-rt", detail: `${path} is the dev-mode wrapper script; leave dev mode before linking rt` }; } diff --git a/lib/dev-mode.ts b/lib/dev-mode.ts index 60b63732..f2c50fb9 100644 --- a/lib/dev-mode.ts +++ b/lib/dev-mode.ts @@ -64,32 +64,59 @@ export function installRtBinary(src: string): string { return dest; } +export const DEV_MODE_TAG = "# mattstack-dev-mode"; + /** - * Dev mode is signalled by the dev-mode WRAPPER SCRIPT at ~/.local/bin/rt -- - * not by any file existing there. Prod mode installs the compiled binary at - * that same path (MAT-383: leaving dev mode must leave a working rt behind), - * so presence alone can no longer tell the modes apart: the smoke showed a - * machine with the prod binary installed still reporting "dev", which made - * the flavor toggle a permanent no-op. A script starts with "#!"; a Mach-O - * binary never does. + * A recognized dev-mode wrapper: our marker on line 2, OR a legacy + * markerless wrapper (its RT_LAUNCH_CWD export line is our unique tell, + * predating the marker). A foreign #! script -- including our own tagged + * PATH-link wrapper from lib/deps/links.ts, which carries LINK_TAG instead + * -- has neither, so it correctly falls through to false. `prefix` is a + * bounded head of the file, never the whole file: in prod this path is a + * symlink to the compiled binary. */ -export function currentMode(): "dev" | "prod" { - const path = devModeWrapperPath(); - if (!existsSync(path)) return "prod"; +export function isDevModeWrapperContent(prefix: string): boolean { + if (!prefix.startsWith("#!")) return false; + const line2 = prefix.split("\n")[1] ?? ""; + return line2.startsWith(DEV_MODE_TAG) || prefix.includes("RT_LAUNCH_CWD"); +} + +/** + * A bounded head of `path` (never the whole file): in prod this path is a + * symlink to the multi-MB compiled binary, and a whole-file read there would + * be needless I/O on every mode check. Exported so lib/deps/links.ts shares + * this same real bounded read instead of re-implementing it. + */ +export function readWrapperPrefix(path: string): string | null { try { const fd = openSync(path, "r"); try { - const head = Buffer.alloc(2); - readSync(fd, head, 0, 2, 0); - return head.toString("latin1") === "#!" ? "dev" : "prod"; + const buf = Buffer.alloc(4096); + const n = readSync(fd, buf, 0, 4096, 0); + return buf.toString("latin1", 0, n); } finally { closeSync(fd); } } catch { - return "prod"; + return null; } } +/** + * Dev mode is signalled by the dev-mode WRAPPER SCRIPT at ~/.local/bin/rt -- + * not by any file existing there. Prod mode installs the compiled binary at + * that same path (MAT-383: leaving dev mode must leave a working rt behind), + * so presence alone can no longer tell the modes apart: the smoke showed a + * machine with the prod binary installed still reporting "dev", which made + * the flavor toggle a permanent no-op. + */ +export function currentMode(): "dev" | "prod" { + const path = devModeWrapperPath(); + if (!existsSync(path)) return "prod"; + const prefix = readWrapperPrefix(path); + return prefix !== null && isDevModeWrapperContent(prefix) ? "dev" : "prod"; +} + export interface IntendedMode { mode: "dev" | "prod"; provenance: "setting" | "derived-from-wrapper"; From 384b8a124264e0a44c7fd53d97ca84f845a98688 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:38:38 -0500 Subject: [PATCH 116/142] handleCommand: reqId + caller tag + per-(cmd,error) suppression + slow-command info + currentCmd --- lib/daemon.ts | 39 +++++++++++++++++-- .../__tests__/command-attribution.test.ts | 17 ++++++++ lib/daemon/command-attribution.ts | 29 ++++++++++++++ 3 files changed, 82 insertions(+), 3 deletions(-) create mode 100644 lib/daemon/__tests__/command-attribution.test.ts create mode 100644 lib/daemon/command-attribution.ts diff --git a/lib/daemon.ts b/lib/daemon.ts index 8b01036d..8602d469 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -36,6 +36,7 @@ import { SystemProcessScanner } from "./daemon/system-process-scanner.ts"; import { parkUntilIntended, probeSocketHolder, daemonFlavor } from "./daemon/park.ts"; import { evictStaleDaemon } from "./daemon/boot-reconcile.ts"; import { resolveUserPath } from "./daemon/user-path.ts"; +import { shortReqId, makeSuppressor } from "./daemon/command-attribution.ts"; // Every state.db API is reached through the lib/state barrel, never through // ./state/db.ts directly: importing the barrel is what guarantees every // store module has registered its legacy-JSON importer before the one-shot @@ -378,22 +379,54 @@ let routedHandlers: ReturnType | undefined; // policy: docs/daemon-supervision-design.md). let shuttingDownViaVerb = false; +// In-flight command name, polled by the loop monitor (wired in a later task) +// to spot a handler that never returns. +const currentCmd: { cmd: string | null } = { cmd: null }; +const rejectSuppressor = makeSuppressor(60_000); +const SLOW_COMMAND_MS = 2000; + async function handleCommand(cmd: string, payload: any, signal?: AbortSignal): Promise { const t0 = Date.now(); + const reqId = shortReqId(); + const caller = payload && typeof payload._client === "string" ? payload._client : "unknown"; + currentCmd.cmd = cmd; try { const result = await routeCommand(cmd, payload, signal); + const durationMs = Date.now() - t0; if (result && result.ok === false) { - log.warn({ cmd, error: result.error, durationMs: Date.now() - t0 }, "command rejected"); + const key = `${cmd}|${result.error ?? ""}`; + const { emit, suppressed } = rejectSuppressor.check(key, Date.now()); + if (emit) { + log.warn( + { reqId, cmd, caller, error: result.error, durationMs, digest: redactDigest(payload), ...(suppressed ? { suppressed } : {}) }, + "command rejected", + ); + } + return { ...result, reqId }; + } + if (durationMs > SLOW_COMMAND_MS) { + log.info({ reqId, cmd, caller, durationMs }, "command handled (slow)"); } else { - log.debug({ cmd, durationMs: Date.now() - t0 }, "command handled"); + log.debug({ reqId, cmd, caller, durationMs }, "command handled"); } return result; } catch (err) { - log.error({ err, cmd, durationMs: Date.now() - t0 }, "command failed"); + log.error({ err, reqId, cmd, caller, durationMs: Date.now() - t0, digest: redactDigest(payload) }, "command failed"); throw err; + } finally { + currentCmd.cmd = null; } } +/** Loggable, secret-free summary of a command payload: top-level key names + * plus a whitelist of identifying fields safe to echo into logs. */ +function redactDigest(payload: any): Record { + if (!payload || typeof payload !== "object") return {}; + const keys = Object.keys(payload); + const pick = (k: string): Record => (payload[k] !== undefined ? { [k]: payload[k] } : {}); + return { keys, ...pick("repo"), ...pick("repoName"), ...pick("branch"), ...pick("iid"), ...pick("room") }; +} + async function routeCommand(cmd: string, payload: any, signal?: AbortSignal): Promise { const routed = routedHandlers?.[cmd]; if (routed) return routed(payload, signal); diff --git a/lib/daemon/__tests__/command-attribution.test.ts b/lib/daemon/__tests__/command-attribution.test.ts new file mode 100644 index 00000000..de0c9135 --- /dev/null +++ b/lib/daemon/__tests__/command-attribution.test.ts @@ -0,0 +1,17 @@ +import { test, expect } from "bun:test"; +import { shortReqId, makeSuppressor } from "../command-attribution.ts"; + +test("shortReqId is short and unique-ish", () => { + const a = shortReqId(); const b = shortReqId(); + expect(a).toMatch(/^[a-z0-9]{6}$/); + expect(a).not.toBe(b); +}); + +test("suppressor logs first, then throttles with a running suppressed count", () => { + const s = makeSuppressor(60_000); + expect(s.check("mr:action|boom", 0)).toEqual({ emit: true, suppressed: 0 }); // first: log + expect(s.check("mr:action|boom", 1_000)).toEqual({ emit: false, suppressed: 1 }); // within window: silent + expect(s.check("mr:action|boom", 2_000)).toEqual({ emit: false, suppressed: 2 }); + expect(s.check("mr:action|boom", 61_000)).toEqual({ emit: true, suppressed: 2 }); // window elapsed: log with count + expect(s.check("mr:action|boom", 61_500)).toEqual({ emit: false, suppressed: 1 }); // count resets after an emit +}); diff --git a/lib/daemon/command-attribution.ts b/lib/daemon/command-attribution.ts new file mode 100644 index 00000000..2d2e7b85 --- /dev/null +++ b/lib/daemon/command-attribution.ts @@ -0,0 +1,29 @@ +/** Short request id for tying a daemon log line to the invocation. */ +export function shortReqId(): string { + return Math.random().toString(36).slice(2, 8).padEnd(6, "0"); +} + +interface SuppressEntry { lastEmitAt: number; suppressed: number } + +/** Per-(cmd,error) suppression: always emit the first occurrence and, once per + * window, emit again carrying the count suppressed since the last emit. */ +export function makeSuppressor(windowMs: number) { + const map = new Map(); + return { + check(key: string, now: number): { emit: boolean; suppressed: number } { + const e = map.get(key); + if (!e) { + map.set(key, { lastEmitAt: now, suppressed: 0 }); + return { emit: true, suppressed: 0 }; + } + if (now - e.lastEmitAt >= windowMs) { + const suppressed = e.suppressed; + e.lastEmitAt = now; + e.suppressed = 0; + return { emit: true, suppressed }; + } + e.suppressed += 1; + return { emit: false, suppressed: e.suppressed }; + }, + }; +} From 8ca4d9b63fd630bbd3631fb936ac78ef0095aaa2 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:41:02 -0500 Subject: [PATCH 117/142] setup: arm64/unsupported-arch row at setup (R051) --- lib/setup/__tests__/validators-mac.test.ts | 22 ++++++++++++++++++++ lib/setup/validators/mac.ts | 24 ++++++++++++++++++++-- 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/lib/setup/__tests__/validators-mac.test.ts b/lib/setup/__tests__/validators-mac.test.ts index 3198a8c0..01a699ab 100644 --- a/lib/setup/__tests__/validators-mac.test.ts +++ b/lib/setup/__tests__/validators-mac.test.ts @@ -75,6 +75,28 @@ describe("macRows — tool.clt", () => { }); }); +describe("macRows — tool.arch", () => { + test("arm64 -> ready", async () => { + const execScript: ExecScript = (argv) => (argv[0] === "uname" ? ok("arm64\n") : ok()); + const r = await pickRow(macRows(fakeProbes({ exec: execScript })), "tool.arch"); + expect(r.status).toBe("ready"); + expect(r.detail).toContain("arm64"); + expect(r.required).toBe(true); + }); + + test("x86_64 -> invalid, unsupported architecture", async () => { + const execScript: ExecScript = (argv) => (argv[0] === "uname" ? ok("x86_64\n") : ok()); + const r = await pickRow(macRows(fakeProbes({ exec: execScript })), "tool.arch"); + expect(r.status).toBe("invalid"); + }); + + test("uname unreachable -> error, never invalid (couldn't determine, not a failed determination)", async () => { + const execScript: ExecScript = (argv) => (argv[0] === "uname" ? missing("uname") : ok()); + const r = await pickRow(macRows(fakeProbes({ exec: execScript })), "tool.arch"); + expect(r.status).toBe("error"); + }); +}); + describe("macRows — tool.path", () => { test("~/.local/bin first on PATH and the precedence marker present -> ready", async () => { const p = fakeProbes({ diff --git a/lib/setup/validators/mac.ts b/lib/setup/validators/mac.ts index d7136287..9c920d97 100644 --- a/lib/setup/validators/mac.ts +++ b/lib/setup/validators/mac.ts @@ -39,6 +39,26 @@ async function cltRow(p: Probes): Promise { return row({ ...base, status: "missing", detail: "Apple command line tools not installed", action: CLT_INSTALL_ACTION }); } +async function archRow(p: Probes): Promise { + const base = { + id: "tool.arch", + kind: "tool" as const, + title: "Processor", + why: "mattstack ships an Apple-silicon (arm64) build; Intel Macs are not supported.", + required: true, + }; + const res = await p.exec(["uname", "-m"]); + const arch = res.stdout.trim(); + + // Same honesty ruling as macosVersionRow: a probe that couldn't run reports + // "error", not "invalid" — only a definite non-arm64 result is invalid. + if (res.code !== 0 || !arch) { + return row({ ...base, status: "error", detail: "Could not determine your processor" }); + } + if (arch === "arm64") return row({ ...base, status: "ready", detail: "Apple silicon (arm64)" }); + return row({ ...base, status: "invalid", detail: `${arch}: Apple silicon (arm64) required` }); +} + function pathRow(p: Probes): Row { const base = { id: "tool.path", kind: "info" as const, title: "PATH precedence", why: "Makes sure your shell finds rt's shims and team intercepts before any conflicting binary.", required: false }; const localBin = `${p.home}/.local/bin`; @@ -57,6 +77,6 @@ function pathRow(p: Probes): Row { } export async function macRows(p: Probes): Promise { - const [macos, clt] = await Promise.all([macosVersionRow(p), cltRow(p)]); - return [macos, clt, pathRow(p)]; + const [macos, clt, arch] = await Promise.all([macosVersionRow(p), cltRow(p), archRow(p)]); + return [macos, clt, arch, pathRow(p)]; } From e877114ae9d68c3c4757723cdceea86ee95f7493 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:42:54 -0500 Subject: [PATCH 118/142] unknown-command envelope (code+version); transports send X-RT-Client (dist rebuilt, no bump) --- lib/daemon-client.ts | 8 ++++++-- lib/daemon.ts | 3 ++- lib/daemon/__tests__/unknown-command.test.ts | 12 ++++++++++++ lib/daemon/unknown-command.ts | 14 ++++++++++++++ packages/rt-client/src/transport.ts | 2 +- 5 files changed, 35 insertions(+), 4 deletions(-) create mode 100644 lib/daemon/__tests__/unknown-command.test.ts create mode 100644 lib/daemon/unknown-command.ts diff --git a/lib/daemon-client.ts b/lib/daemon-client.ts index aff0ad11..4dccf799 100644 --- a/lib/daemon-client.ts +++ b/lib/daemon-client.ts @@ -60,10 +60,12 @@ async function trySocketQuery( try { const hasBody = payload && Object.keys(payload).length > 0; + const headers: Record = { "X-RT-Client": `rt-cli/${process.pid}` }; + if (hasBody) headers["Content-Type"] = "application/json"; const response = await fetch(`http://localhost/${cmd}`, { unix: DAEMON_SOCK_PATH, method: hasBody ? "POST" : "GET", - headers: hasBody ? { "Content-Type": "application/json" } : undefined, + headers, body: hasBody ? JSON.stringify(payload) : undefined, signal: AbortSignal.timeout(timeoutMs), } as any); @@ -153,10 +155,12 @@ export async function trayRequest( try { const hasBody = init.body !== undefined; + const headers: Record = { "X-RT-Client": `rt-cli/${process.pid}` }; + if (hasBody) headers["Content-Type"] = "application/json"; const response = await fetch(`http://localhost${path}`, { unix: sockPath, method: init.method, - headers: hasBody ? { "Content-Type": "application/json" } : undefined, + headers, body: hasBody ? JSON.stringify(init.body) : undefined, signal: AbortSignal.timeout(init.timeoutMs ?? REQUEST_TIMEOUT_MS), } as any); diff --git a/lib/daemon.ts b/lib/daemon.ts index 8602d469..4cc75c4e 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -37,6 +37,7 @@ import { parkUntilIntended, probeSocketHolder, daemonFlavor } from "./daemon/par import { evictStaleDaemon } from "./daemon/boot-reconcile.ts"; import { resolveUserPath } from "./daemon/user-path.ts"; import { shortReqId, makeSuppressor } from "./daemon/command-attribution.ts"; +import { unknownCommandReply } from "./daemon/unknown-command.ts"; // Every state.db API is reached through the lib/state barrel, never through // ./state/db.ts directly: importing the barrel is what guarantees every // store module has registered its legacy-JSON importer before the one-shot @@ -451,7 +452,7 @@ async function routeCommand(cmd: string, payload: any, signal?: AbortSignal): Pr return { ok: true, message: "shutting down" }; default: - return { ok: false, error: `unknown command: ${cmd}` }; + return unknownCommandReply(cmd, typeof RT_VERSION !== "undefined" ? RT_VERSION : "source"); } } diff --git a/lib/daemon/__tests__/unknown-command.test.ts b/lib/daemon/__tests__/unknown-command.test.ts new file mode 100644 index 00000000..1a6e063a --- /dev/null +++ b/lib/daemon/__tests__/unknown-command.test.ts @@ -0,0 +1,12 @@ +import { test, expect } from "bun:test"; +import { unknownCommandReply } from "../unknown-command.ts"; + +test("unknown command carries a code, version, and actionable text", () => { + const r = unknownCommandReply("chat:archive", "v0.9.0"); + expect(r.ok).toBe(false); + expect(r.code).toBe("unknown-command"); + expect(r.version).toBe("v0.9.0"); + expect(r.error).toContain("v0.9.0"); + expect(r.error).toContain("chat:archive"); + expect(r.error.toLowerCase()).toContain("restart"); +}); diff --git a/lib/daemon/unknown-command.ts b/lib/daemon/unknown-command.ts new file mode 100644 index 00000000..33306978 --- /dev/null +++ b/lib/daemon/unknown-command.ts @@ -0,0 +1,14 @@ +/** + * Reply shape for a command name routeCommand's switch doesn't recognize. + * Carries the daemon's own version so a caller can tell version skew (the + * daemon is older than the CLI/client that sent the command) from a genuine + * typo (findings R021, R008). + */ +export function unknownCommandReply(cmd: string, version: string) { + return { + ok: false as const, + code: "unknown-command" as const, + version, + error: `daemon at version ${version} does not know "${cmd}"; restart or upgrade rt (rt daemon restart)`, + }; +} diff --git a/packages/rt-client/src/transport.ts b/packages/rt-client/src/transport.ts index b8243c24..169295d0 100644 --- a/packages/rt-client/src/transport.ts +++ b/packages/rt-client/src/transport.ts @@ -57,7 +57,7 @@ export async function rtCommand( const res = await fetch(`http://localhost/${cmd}`, { unix: sockPath, method: "POST", - headers: { "Content-Type": "application/json" }, + headers: { "Content-Type": "application/json", "X-RT-Client": `rt-client/${process.pid}` }, body: JSON.stringify(payload), signal: AbortSignal.timeout(opts.timeoutMs ?? 15_000), // Bun's `unix` fetch option isn't in the standard RequestInit type. From e75d566e627ef4488c9ab4d705f177bcbb1cfbf0 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:46:25 -0500 Subject: [PATCH 119/142] servers: thread X-RT-Client into payload._client; advertise it in CORS --- lib/daemon/__tests__/caller-tag.test.ts | 7 +++++++ lib/daemon/api-server.ts | 4 +++- lib/daemon/socket-server.ts | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) create mode 100644 lib/daemon/__tests__/caller-tag.test.ts diff --git a/lib/daemon/__tests__/caller-tag.test.ts b/lib/daemon/__tests__/caller-tag.test.ts new file mode 100644 index 00000000..ce08b779 --- /dev/null +++ b/lib/daemon/__tests__/caller-tag.test.ts @@ -0,0 +1,7 @@ +import { test, expect } from "bun:test"; +import { buildCorsHeaders } from "../api-server.ts"; + +test("CORS allow-headers advertises X-RT-Client so browser preflight passes", () => { + const h = buildCorsHeaders("https://example.com", true); + expect(h["Access-Control-Allow-Headers"]).toContain("X-RT-Client"); +}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index fd4b5d66..1b2f5b2d 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -158,7 +158,7 @@ export function clearWsClients(): void { export function buildCorsHeaders(origin: string | null, trusted: boolean): Record { const headers: Record = { "Access-Control-Allow-Methods": "GET, POST, OPTIONS", - "Access-Control-Allow-Headers": "Content-Type, X-RT-Token", + "Access-Control-Allow-Headers": "Content-Type, X-RT-Token, X-RT-Client", }; if (origin && trusted) { headers["Access-Control-Allow-Origin"] = origin; @@ -436,6 +436,8 @@ export async function startApiServer(deps: ApiServerDeps): Promise> payload = coerceQueryParams(url.searchParams); } + const client = req.headers.get("x-rt-client"); + if (client) payload._client = client; const result = await handleCommand(route.cmd, payload, req.signal); return Response.json(result, { headers: corsHeaders }); } catch (err) { diff --git a/lib/daemon/socket-server.ts b/lib/daemon/socket-server.ts index fee4c27e..4dbd84d8 100644 --- a/lib/daemon/socket-server.ts +++ b/lib/daemon/socket-server.ts @@ -45,6 +45,8 @@ export function startSocketServer(opts: { try { payload = await req.json(); } catch { /* empty body is fine */ } } + const client = req.headers.get("x-rt-client"); + if (client) (payload as any)._client = client; const result = await handleCommand(cmd, payload, req.signal); return Response.json(result); } catch (err) { From 46ac5738fd7d4d2ae2a36d46d5ce4182017586a2 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:49:12 -0500 Subject: [PATCH 120/142] plan: split HandlerContext type additions into the tasks that provide their values (13/14/15) to avoid a tsc-red window Co-Authored-By: Claude Opus 4.8 (1M context) --- .../superpowers/plans/2026-08-28-p2-health.md | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/docs/superpowers/plans/2026-08-28-p2-health.md b/docs/superpowers/plans/2026-08-28-p2-health.md index 9793a565..ad04bbf7 100644 --- a/docs/superpowers/plans/2026-08-28-p2-health.md +++ b/docs/superpowers/plans/2026-08-28-p2-health.md @@ -1494,14 +1494,17 @@ git commit -m "servers: thread X-RT-Client into payload._client; advertise it in - [ ] **Step 1: Extend the type** -In `lib/daemon/handlers/types.ts`, change the `refreshStatusRef` field and add `getHealth` + `heartbeatSeq` (the ping handler in Task 14 echoes the seq; declaring it here keeps `ctx` typed): +In `lib/daemon/handlers/types.ts`, change ONLY the `refreshStatusRef` field: ```ts refreshStatusRef: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }; - getHealth: () => import("../health.ts").HealthSnapshot; - heartbeatSeq: () => number; - setLogLevel: (l: string) => void; // Task 15 uses these; declare now to avoid a second types.ts edit - getLogLevel: () => string; ``` +Do NOT add `getHealth`/`heartbeatSeq`/`setLogLevel`/`getLogLevel` here. Each of +those is a REQUIRED `HandlerContext` field, so it must be added to the type in +the same task that also provides its value in the `handlerCtx` literal (Task 14 +adds `getHealth` + `heartbeatSeq`; Task 15 adds `setLogLevel` + `getLogLevel`), +or tsc goes red across the gap. This task's `refreshStatusRef` change is fully +self-consistent: the type, the `daemon.ts` init, and `CacheRefresherDeps` all +move together below. - [ ] **Step 2: Update the init site and the refresher** @@ -1565,7 +1568,8 @@ git commit -m "ctx: extend refreshStatusRef with cycle outcome + getHealth; expo **Files:** - Modify: `lib/daemon.ts` (start the monitor + sampler; build `getHealth`; put it on `handlerCtx`) -- Modify: `lib/daemon/handlers/status.ts` (add `health`/`metrics`/`eventLoop` to `status` + `tray:status`; `health.level` + `eventLoop` to `ping`) +- Modify: `lib/daemon/handlers/types.ts` (add `getHealth` + `heartbeatSeq` to `HandlerContext` — deferred here from Task 13 so type + value land together) +- Modify: `lib/daemon/handlers/status.ts` (add `health`/`metrics`/`eventLoop` to `status` + `tray:status`; `health.level` + `eventLoop` + `heartbeatSeq` to `ping`) - Create: `lib/daemon/health-sampler.ts` (5-min metrics log + rss baseline + disk-free cache) - Test: `lib/daemon/__tests__/health-sampler.test.ts` @@ -1722,7 +1726,7 @@ Add `heartbeatSeq: loopMon.seq` to the `handlerCtx` object literal (and `heartbe getHealth: buildHealthSnapshot, heartbeatSeq: loopMon.seq, ``` -Add `getHealth: buildHealthSnapshot` to the `handlerCtx` object literal (lines ~353-364). Import `getFreshnessSnapshot` if not already in `daemon.ts` scope (it lives in `lib/daemon/freshness.ts`). Ensure `loopMon.stop()` is called in `cleanup()`. +Add `getHealth: buildHealthSnapshot` to the `handlerCtx` object literal (lines ~353-364), AND add `getHealth: () => import("./health.ts").HealthSnapshot` to `HandlerContext` in `lib/daemon/handlers/types.ts` (Task 13 deliberately left this to Task 14 so the type and its value land together). Import `getFreshnessSnapshot` if not already in `daemon.ts` scope (it lives in `lib/daemon/freshness.ts`). Ensure `loopMon.stop()` is called in `cleanup()`. - [ ] **Step 5: Surface the snapshot in the handlers** @@ -1773,7 +1777,9 @@ git commit -m "daemon: wire loop monitor + heartbeat + health sampler; surface h **Files:** - Modify: `commands/daemon.ts` (add `setLogLevel`) - Modify: `lib/command-tree-def.ts` (add the `log-level` leaf in the `daemon` subtree) -- Modify: `lib/daemon/handlers/status.ts` (add `daemon:log-level` handler) OR a small dedicated handler module registered in the router +- Modify: `lib/daemon/handlers/types.ts` (add `setLogLevel` + `getLogLevel` to `HandlerContext`) +- Modify: `lib/daemon.ts` (wire `setLogLevel`/`getLogLevel` on `handlerCtx`) +- Modify: `lib/daemon/handlers/status.ts` (add `daemon:log-level` handler) - Test: `commands/__tests__/log-level.test.ts` (the pure format/parse), and picker conformance. **Interfaces:** @@ -1837,7 +1843,7 @@ export async function setLogLevel(args: string[] = []): Promise { console.log(formatLogLevelResult(res as any, Boolean(level))); } ``` -The `daemon:log-level` handler sets the live pino level on the singleton logger. The `setLogLevel`/`getLogLevel` accessors are already declared on `HandlerContext` (Task 13); wire them in `daemon.ts`'s `handlerCtx` object literal: +The `daemon:log-level` handler sets the live pino level on the singleton logger. Add `setLogLevel: (l: string) => void` and `getLogLevel: () => string` to `HandlerContext` in `lib/daemon/handlers/types.ts` (deferred here from Task 13 so type + value land together), then wire them in `daemon.ts`'s `handlerCtx` object literal: ```ts // in daemon.ts handlerCtx: setLogLevel: (l: string) => { log.level = l; log.info({ level: l }, "log level changed"); }, From a4108185ac2403f81c2cec0d89b4572d6c18d153 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:51:40 -0500 Subject: [PATCH 121/142] ctx: extend refreshStatusRef with cycle outcome; export apiWsClientCount --- lib/daemon.ts | 6 ++--- lib/daemon/__tests__/cache-refresh-gc.test.ts | 2 +- .../__tests__/refresh-status-ref.test.ts | 12 +++++++++ lib/daemon/api-server.ts | 5 ++++ lib/daemon/cache-refresh.ts | 27 +++++++++++++++++-- lib/daemon/handlers/types.ts | 4 +-- 6 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 lib/daemon/__tests__/refresh-status-ref.test.ts diff --git a/lib/daemon.ts b/lib/daemon.ts index 4cc75c4e..4ea5414f 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -207,9 +207,9 @@ const cache: BranchCacheStore = { // Port scan cache, held as a single mutable ref so handler modules can read // fresh values without getters. The port poller mutates it in place. const portCacheRef = { ports: [] as PortEntry[], updatedAt: 0 }; -// Refresh-cycle status ref (last successful cache refresh), also mutated in -// place so status handlers read a live value. -const refreshStatusRef = { lastRefreshAt: 0 }; +// Refresh-cycle status ref (last cycle's outcome), also mutated in place so +// status handlers read a live value. +const refreshStatusRef = { lastRefreshAt: 0, lastSuccessAt: 0, failedRepos: 0, enrichErrors: 0 }; const startedAt = Date.now(); // Injected at compile time via `bun build --define RT_VERSION='"v1.x.x"'` (see cli.ts) — diff --git a/lib/daemon/__tests__/cache-refresh-gc.test.ts b/lib/daemon/__tests__/cache-refresh-gc.test.ts index a959a4d1..4ad5dbf9 100644 --- a/lib/daemon/__tests__/cache-refresh-gc.test.ts +++ b/lib/daemon/__tests__/cache-refresh-gc.test.ts @@ -133,7 +133,7 @@ function wireCycle(): Wiring { const refresh = createCacheRefresher({ log: silentLog, cache, - refreshStatusRef: { lastRefreshAt: 0 }, + refreshStatusRef: { lastRefreshAt: 0, lastSuccessAt: 0, failedRepos: 0, enrichErrors: 0 }, portCacheRef: { ports: [], updatedAt: 0 }, repoIndex: () => ({ [CLEAN]: tempDir("rt-gcwire-clean-"), [FLAKY]: tempDir("rt-gcwire-flaky-") }), broadcast: () => {}, diff --git a/lib/daemon/__tests__/refresh-status-ref.test.ts b/lib/daemon/__tests__/refresh-status-ref.test.ts new file mode 100644 index 00000000..4fa9c02d --- /dev/null +++ b/lib/daemon/__tests__/refresh-status-ref.test.ts @@ -0,0 +1,12 @@ +import { test, expect } from "bun:test"; +import { applyRefreshOutcome } from "../cache-refresh.ts"; + +test("a clean cycle advances lastSuccessAt; a failing cycle does not", () => { + const ref = { lastRefreshAt: 0, lastSuccessAt: 0, failedRepos: 0, enrichErrors: 0 }; + applyRefreshOutcome(ref, 1000, 0, 0); + expect(ref.lastSuccessAt).toBe(1000); + applyRefreshOutcome(ref, 2000, 2, 5); + expect(ref.lastRefreshAt).toBe(2000); + expect(ref.lastSuccessAt).toBe(1000); // unchanged on failure + expect(ref.failedRepos).toBe(2); +}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index 1b2f5b2d..8490309a 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -79,6 +79,11 @@ interface ApiWSData { const wsClients = new Set>(); +/** Count of currently connected WS broadcast clients, for health reporting. */ +export function apiWsClientCount(): number { + return wsClients.size; +} + let apiServerLog: { warn: (o: unknown, m: string) => void } = { warn: () => {} }; /** Consecutive Bun `ws.send()` backpressure (-1) returns tolerated before a diff --git a/lib/daemon/cache-refresh.ts b/lib/daemon/cache-refresh.ts index 93c9f7af..49450c8e 100644 --- a/lib/daemon/cache-refresh.ts +++ b/lib/daemon/cache-refresh.ts @@ -29,7 +29,7 @@ export interface CacheRefresherDeps { log: Logger; /** The process-wide branch-cache store; `cache.reload()` replaces the old read-from-disk. */ cache: BranchCacheStore; - refreshStatusRef: { lastRefreshAt: number }; + refreshStatusRef: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }; portCacheRef: PortCacheRef; repoIndex: () => RepoIndex; broadcast: (type: string, data: any) => void; @@ -82,6 +82,24 @@ export function makeCoalescer( }; } +/** + * Records one refresh cycle's outcome onto the shared ref. `lastSuccessAt` + * only advances when the cycle was clean (no failed repos, no enrich + * errors) — a health signal downstream must be able to trust as "refresh is + * actually working", not just "a cycle ran". + */ +export function applyRefreshOutcome( + ref: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }, + at: number, + failedReposCount: number, + enrichErrorsCount: number, +): void { + ref.lastRefreshAt = at; + ref.failedRepos = failedReposCount; + ref.enrichErrors = enrichErrorsCount; + if (failedReposCount === 0 && enrichErrorsCount === 0) ref.lastSuccessAt = at; +} + export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise { const { log, cache, refreshStatusRef, portCacheRef, repoIndex, broadcast } = deps; @@ -107,6 +125,10 @@ export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise(); + // Cycle-wide total; the per-repo `enrichErrors` below is scoped to one + // iteration and resets each repo, so this is the only place the total + // for the whole cycle (fed to applyRefreshOutcome below) accumulates. + let totalEnrichErrors = 0; // `repos` keys on the serialized repo identity (repo-index.ts), so every // `repoName` below — passed on into refreshAllMRs, project-sync, and the @@ -181,6 +203,7 @@ export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise Promise Promise; /** Start a directory watch over a repo's .git/config and run an initial check. */ startWatchingRepo: (repoName: string, repoPath: string) => void; - /** Holder for the last cache-refresh timestamp (0 = never). */ - refreshStatusRef: { lastRefreshAt: number }; + /** Holder for the last cache-refresh cycle's outcome (0s = never run). */ + refreshStatusRef: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }; } export type Handler = (payload: any, signal?: AbortSignal) => Promise; From af36aece9208dfa1423c9f76bcbda7d90db801c4 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:54:57 -0500 Subject: [PATCH 122/142] cache-refresh: remove em dash from applyRefreshOutcome docstring --- lib/daemon/cache-refresh.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/daemon/cache-refresh.ts b/lib/daemon/cache-refresh.ts index 49450c8e..cb3438d5 100644 --- a/lib/daemon/cache-refresh.ts +++ b/lib/daemon/cache-refresh.ts @@ -85,8 +85,8 @@ export function makeCoalescer( /** * Records one refresh cycle's outcome onto the shared ref. `lastSuccessAt` * only advances when the cycle was clean (no failed repos, no enrich - * errors) — a health signal downstream must be able to trust as "refresh is - * actually working", not just "a cycle ran". + * errors). Downstream health reporting must be able to trust it as "refresh + * is actually working", not just "a cycle ran". */ export function applyRefreshOutcome( ref: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }, From dd3353be971ee011982a01b0ee2de624a9366cb7 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 17:54:58 -0500 Subject: [PATCH 123/142] dev-mode: fix rt-health/steps-a consumer tests for the new wrapper detector (S020/S067) --- lib/deps/__tests__/links.test.ts | 11 +++++------ lib/deps/links.ts | 19 +++++++++++-------- lib/setup/__tests__/fakes.ts | 10 ++++++++++ lib/setup/__tests__/steps-a.test.ts | 12 +++++++++++- .../__tests__/validators-rt-health.test.ts | 7 ++++--- lib/setup/probes.ts | 12 ++++++++++++ 6 files changed, 53 insertions(+), 18 deletions(-) diff --git a/lib/deps/__tests__/links.test.ts b/lib/deps/__tests__/links.test.ts index 87a20234..56a1a97d 100644 --- a/lib/deps/__tests__/links.test.ts +++ b/lib/deps/__tests__/links.test.ts @@ -1,7 +1,7 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import { mkdirSync, mkdtempSync, realpathSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; -import { dirname, join } from "path"; +import { join } from "path"; import { HELPERS_DIR, RT_BUNDLE_PATH, __test__ as bundleLayoutTest } from "../../bundle-layout.ts"; import { setSetting } from "../../settings/write.ts"; import { fakeProbes, type FakeProbesOpts } from "../../setup/__tests__/fakes.ts"; @@ -201,12 +201,11 @@ describe("tagged PATH links", () => { }); test("link(rt) refuses dev-mode-owns-rt when ~/.local/bin/rt is the dev-mode wrapper script", () => { - // isDevModeWrapper reads the real fs at `path` (bounded prefix, never - // through the Probes seam), so this needs a real file on disk -- `home` - // is a real mkdtempSync'd directory, not a fake one. + // isDevModeWrapper reads through p.readPrefix (Probes-routed, bounded), + // so the fake in-memory files map is enough -- no real fs write, and the + // content must be genuinely recognized (RT_LAUNCH_CWD tell) rather than + // any bare "#!" script, matching the shared detector's real rule. const path = linkPath(home, "rt"); - mkdirSync(dirname(path), { recursive: true }); - writeFileSync(path, "#!/bin/sh\nexport RT_LAUNCH_CWD=\"$PWD\"\nexec bun run cli.ts \"$@\"\n"); const p = bundleProbe({ files: { [path]: "#!/bin/sh\nexport RT_LAUNCH_CWD=\"$PWD\"\nexec bun run cli.ts \"$@\"\n" } }); const outcome = link(p, "rt"); diff --git a/lib/deps/links.ts b/lib/deps/links.ts index e4bd3be8..87fe4675 100644 --- a/lib/deps/links.ts +++ b/lib/deps/links.ts @@ -8,7 +8,7 @@ */ import { dirname, join } from "path"; -import { installRtBinary, isDevModeWrapperContent, readWrapperPrefix } from "../dev-mode.ts"; +import { installRtBinary, isDevModeWrapperContent } from "../dev-mode.ts"; import type { Probes } from "../setup/probes.ts"; import { readSetupState, updateSetupState } from "../setup/state.ts"; import { bundledToolExec, isOurLink, LINK_TAG, linkPath, userCopyOnPath } from "./resolve.ts"; @@ -40,13 +40,16 @@ const REAL_SEAMS: LinkSeams = { installRtBinary: (src) => installRtBinary(src) } /** * rt in dev mode is signalled the same way lib/dev-mode.ts's currentMode() * detects it: shares isDevModeWrapperContent so the two call sites can never - * disagree. Reads the real filesystem (not the Probes seam) via the same - * bounded readWrapperPrefix currentMode() uses -- in prod this path is a - * symlink to the multi-MB compiled binary, so a whole-file read here would - * be exactly the bug this detector exists to avoid. + * disagree. Reads through the Probes seam's readPrefix -- a real bounded + * (4096-byte) read in production, same as currentMode()'s own standalone + * read, but routed through `p` so link()'s tests (which drive the rest of + * this function entirely via a fake bundle/home) don't have to touch the + * real machine's HOME just to simulate this one check. currentMode() itself + * keeps its own direct real-fs read -- it has no Probes seam to route + * through, and is not this function's concern. */ -function isDevModeWrapper(path: string): boolean { - const prefix = readWrapperPrefix(path); +function isDevModeWrapper(p: Pick, path: string): boolean { + const prefix = p.readPrefix(path); return prefix !== null && isDevModeWrapperContent(prefix); } @@ -90,7 +93,7 @@ function clearForced(p: Probes, tool: string): void { export function link(p: Probes, tool: string, opts: { force?: boolean } = {}, seams: LinkSeams = REAL_SEAMS): LinkOutcome { const path = linkPath(p.home, tool); - if (tool === "rt" && isDevModeWrapper(path)) { + if (tool === "rt" && isDevModeWrapper(p, path)) { return { ok: false, reason: "dev-mode-owns-rt", detail: `${path} is the dev-mode wrapper script; leave dev mode before linking rt` }; } diff --git a/lib/setup/__tests__/fakes.ts b/lib/setup/__tests__/fakes.ts index 6844ad04..70c379d7 100644 --- a/lib/setup/__tests__/fakes.ts +++ b/lib/setup/__tests__/fakes.ts @@ -88,6 +88,16 @@ export function fakeProbes(opts: FakeProbesOpts = {}): Probes & { return files[path] ?? null; }, + // Mirrors the real bounded (4096-byte) prefix read, through `links` + // exactly like fileSize does -- a symlinked fixture (e.g. `p.symlink` + // registering an rt bundle target) must resolve here too, not just for + // real files planted via `files`. + readPrefix(path) { + const resolved = resolveThroughLinks(path); + if (resolved === null || resolved in dirs) return null; + return (files[resolved] ?? "").slice(0, 4096); + }, + readDir(path) { // A copy, not the live array: real readdirSync snapshots the directory // at call time, so a caller iterating the result while also removing diff --git a/lib/setup/__tests__/steps-a.test.ts b/lib/setup/__tests__/steps-a.test.ts index 7e573612..8e0becb1 100644 --- a/lib/setup/__tests__/steps-a.test.ts +++ b/lib/setup/__tests__/steps-a.test.ts @@ -11,6 +11,7 @@ import { setSetting } from "../../settings/write.ts"; import { closeStateDb, setKvValue } from "../../state/index.ts"; import { serializeIdentity } from "../../settings/identity.ts"; import { linkPath } from "../../deps/links.ts"; +import { DEV_MODE_TAG } from "../../dev-mode.ts"; import type { ExecResult, Probes } from "../probes.ts"; import type { SecretsExecResult, SecretsExecSeam, SecretsSeams } from "../../secrets/store.ts"; import { readTeamSecret, teamSopsYamlPath } from "../../secrets/team-store.ts"; @@ -668,7 +669,16 @@ describe("path.link / settings.seed / repos.clone / intercepts.install (real HOM } test("path.link: links fast-browser/gitq/deck, skips rt when the dev-mode wrapper owns ~/.local/bin/rt, installs shell + zshenv precedence", async () => { - const p = bundleProbe({ files: { [linkPath(home, "rt")]: "#!/bin/sh\nexec bun run cli.ts \"$@\"\n" } }); + // isDevModeWrapper reads through p.readPrefix (Probes-routed, bounded), + // so the fake in-memory files map is enough -- no real fs write needed. + // (A real write here would also flip the REAL currentMode(), which this + // describe block's `home` is real HOME for: appBundleRoot() would then + // hunt for the dev-flavor bundle name and miss this fixture's prod-named + // appRoot entirely, skipping fast-browser/gitq/deck too.) Content must be + // genuinely recognized (the marker), not any bare "#!" script. + const rtLinkPath = linkPath(home, "rt"); + const wrapperContent = `#!/bin/sh\n${DEV_MODE_TAG}\nexec bun run cli.ts "$@"\n`; + const p = bundleProbe({ files: { [rtLinkPath]: wrapperContent } }); const { ctx, logs } = makeCtx(p); const outcome = await pathLinkStep.run(ctx); diff --git a/lib/setup/__tests__/validators-rt-health.test.ts b/lib/setup/__tests__/validators-rt-health.test.ts index d5f2fc12..9e3501de 100644 --- a/lib/setup/__tests__/validators-rt-health.test.ts +++ b/lib/setup/__tests__/validators-rt-health.test.ts @@ -4,6 +4,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, import { tmpdir } from "os"; import { dirname, join } from "path"; import { DAEMON_CONFIG_PATH } from "../../daemon-config.ts"; +import { DEV_MODE_TAG } from "../../dev-mode.ts"; import { LOGIN_ITEMS_SETTINGS_ACTION } from "../permissions.ts"; import { setSetting } from "../../settings/write.ts"; import { homeBackupRow, rtHealthRows } from "../validators/rt-health.ts"; @@ -139,7 +140,7 @@ describe("rtHealthRows — rows that resolve the app bundle", () => { test("dev mode wrapper at ~/.local/bin/rt -> skipped, dev mode owns it", async () => { const wrapperDir = join(home, ".local", "bin"); mkdirSync(wrapperDir, { recursive: true }); - writeFileSync(join(wrapperDir, "rt"), "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + writeFileSync(join(wrapperDir, "rt"), `#!/bin/sh\n${DEV_MODE_TAG}\nexit 0\n`, { mode: 0o755 }); const p = bundleProbe(); const r = await pickRow(rtHealthRows(p, { ci: false }, NOOP_FZF), "tool.rt-link"); expect(r.status).toBe("skipped"); @@ -529,10 +530,10 @@ describe("rtHealthRows — tool.flavor", () => { rmSync(home, { recursive: true, force: true }); }); - /** A script at ~/.local/bin/rt is the dev-mode signal currentMode() reads. */ + /** A recognized wrapper at ~/.local/bin/rt is the dev-mode signal currentMode() reads. */ function writeDevWrapper(): void { mkdirSync(join(home, ".local", "bin"), { recursive: true }); - writeFileSync(join(home, ".local", "bin", "rt"), "#!/bin/sh\necho dev\n"); + writeFileSync(join(home, ".local", "bin", "rt"), `#!/bin/sh\n${DEV_MODE_TAG}\necho dev\n`); } test("live daemon of the wrong flavor: fail, names all three legs", async () => { diff --git a/lib/setup/probes.ts b/lib/setup/probes.ts index 1c14d3b1..4f4f6505 100644 --- a/lib/setup/probes.ts +++ b/lib/setup/probes.ts @@ -8,6 +8,7 @@ import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, readlinkSync, rmSync, statSync, symlinkSync, unlinkSync, writeFileSync } from "fs"; import { homedir } from "os"; import { daemonSocketQuery, trayRequest, type DaemonResponse, type TrayClient } from "../daemon-client.ts"; +import { readWrapperPrefix } from "../dev-mode.ts"; import { UserActionableError } from "./errors.ts"; export interface ExecResult { @@ -23,6 +24,13 @@ export interface Probes { /** Byte size, following symlinks, only for a REGULAR file (a directory, a symlink to one, or anything missing/unreadable is null) — the cheap "is this actually a file worth reading" check callers need before decoding one. */ fileSize(path: string): number | null; readFile(path: string): string | null; + /** + * A bounded 4096-byte prefix of `path`, following symlinks -- never a + * whole-file read. Exists for callers that must classify a file by its + * head (e.g. dev-mode wrapper detection) where the target may be a symlink + * to a multi-MB binary; `readFile` would read the whole thing. + */ + readPrefix(path: string): string | null; readDir(path: string): string[]; readlink(path: string): string | null; writeFile(path: string, content: string, mode?: number): void; @@ -173,6 +181,10 @@ export function createRealProbes(): Probes { } }, + readPrefix(path) { + return readWrapperPrefix(path); + }, + readDir(path) { try { return readdirSync(path); From 8f3b92a627a2f65ba3911c1f056cce765e884886 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 18:06:03 -0500 Subject: [PATCH 124/142] home-snapshot: diagnose not-provisioned and missing git identity (S090/R043) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/daemon/__tests__/home-snapshot.test.ts | 85 +++++++++++++++++++--- lib/daemon/home-snapshot.ts | 27 ++++++- lib/home/__tests__/init-exec.test.ts | 23 ++++++ lib/home/init-exec.ts | 8 ++ 4 files changed, 133 insertions(+), 10 deletions(-) diff --git a/lib/daemon/__tests__/home-snapshot.test.ts b/lib/daemon/__tests__/home-snapshot.test.ts index 33cc9df4..52ffbc45 100644 --- a/lib/daemon/__tests__/home-snapshot.test.ts +++ b/lib/daemon/__tests__/home-snapshot.test.ts @@ -43,10 +43,11 @@ function defaultResponders(opts: { pushStderr?: string; sha?: string; hasRemote?: boolean; + hasIdentity?: boolean; } = {}): Responder[] { const { isRepo = true, branch = "main", branchExit = 0, statusZ = "", commitExit = 0, addExit = 0, pushExit = 0, pushStderr = "", sha = "abc123", - hasRemote = true, + hasRemote = true, hasIdentity = true, } = opts; return [ (argv) => (argv[1] === "rev-parse" && argv[2] === "--is-inside-work-tree") @@ -60,6 +61,14 @@ function defaultResponders(opts: { : undefined, (argv) => (argv[1] === "status") ? { stdout: statusZ, stderr: "", exitCode: 0 } : undefined, (argv) => (argv[1] === "add") ? { stdout: "", stderr: "", exitCode: addExit } : undefined, + // git identity probe, checked right before the first auto commit — + // defaults to "configured" so every fixture not testing R043 stays green. + (argv) => (argv[1] === "config" && argv[2] === "user.name") + ? (hasIdentity ? { stdout: "rt test\n", stderr: "", exitCode: 0 } : { stdout: "", stderr: "", exitCode: 1 }) + : undefined, + (argv) => (argv[1] === "config" && argv[2] === "user.email") + ? (hasIdentity ? { stdout: "rt@example.test\n", stderr: "", exitCode: 0 } : { stdout: "", stderr: "", exitCode: 1 }) + : undefined, (argv) => (gitVerb(argv) === "commit") ? { stdout: "", stderr: "", exitCode: commitExit } : undefined, // `hasRemote()`'s own probe — most fixtures simulate a repo that already has origin configured, matching every pre-existing push test's assumption. (argv) => (argv[1] === "remote" && argv.length === 2) ? { stdout: hasRemote ? "origin\n" : "", stderr: "", exitCode: 0 } : undefined, @@ -168,6 +177,15 @@ const DEFAULT_SETTINGS: HomeSnapshotSettings = { const NO_OWNERS: Owners = { zones: {} }; +// A real (but never touched — every git call underneath it is faked) +// directory: the S090 existsSync guard runs against the real filesystem, so +// the fixture repoDir the whole suite shares must actually exist on disk, +// not just look plausible as a string. +const FAKE_REPO_DIR = realpathSync(mkdtempSync(join(tmpdir(), "rt-home-snapshot-fakerepo-"))); +afterAll(() => { + try { rmSync(FAKE_REPO_DIR, { recursive: true, force: true }); } catch { /* best-effort cleanup */ } +}); + async function flushAsync(): Promise { // Real macrotask hop — lets fake-exec's async chain (all microtasks, no // real timers involved) fully settle before assertions run. @@ -211,7 +229,7 @@ function baseDeps(overrides: Partial = {}): { const deps: HomeSnapshotDeps = { log, broadcast: (type, data) => broadcasts.push({ type, data }), - repoDir: "/fake/repo", + repoDir: FAKE_REPO_DIR, exec: execFn, watch: watch.fn, setTimeout: timers.setTimeoutFn, @@ -261,6 +279,23 @@ describe("startHomeSnapshot — inert paths", () => { expect(execCalls.length).toBe(1); }); + test("S090: a missing repoDir is diagnosed 'not-provisioned', names `rt home init`, never spawns git", async () => { + const { fn: execFn, calls: execCalls } = makeFakeExec(defaultResponders()); + const { deps, log, watch } = baseDeps({ exec: execFn, repoDir: "/does/not/exist/rt-home-snapshot-s090" }); + const handle = startHomeSnapshot(deps); + await handle.ready; + + expect(watch.calls.length).toBe(0); + const warnCall = log.calls.find((c) => c.level === "warn"); + expect(warnCall?.args[1]).toContain("rt home init"); + // The existsSync guard runs before any git spawn at all. + expect(execCalls.length).toBe(0); + + const result = await handle.runNow("manual"); + expect(result.skipped).toBe("not-provisioned"); + expect(execCalls.length).toBe(0); + }); + test("a throwing watch seam (fs.watch EMFILE/ENOSPC/ENOENT) resolves ready promptly and makes runNow return an inert result, not hang", async () => { const throwingWatch = () => { throw new Error("EMFILE: too many open files"); }; const { deps, log } = baseDeps({ watch: throwingWatch as any }); @@ -319,7 +354,7 @@ describe("startHomeSnapshot — live enabled toggle", () => { expect(result.committed).toBe(true); expect(handle.status().watching).toBe(true); - expect(watch.calls).toEqual([{ path: "/fake/repo", options: { recursive: true } }]); + expect(watch.calls).toEqual([{ path: FAKE_REPO_DIR, options: { recursive: true } }]); const janitorEntry = [...timers.pending.values()].find((t) => t.ms === DEFAULT_SETTINGS.janitorIntervalMin * 60_000); expect(janitorEntry).toBeDefined(); }); @@ -345,7 +380,7 @@ describe("startHomeSnapshot — watcher", () => { const handle = startHomeSnapshot(deps); await handle.ready; - expect(watch.calls).toEqual([{ path: "/fake/repo", options: { recursive: true } }]); + expect(watch.calls).toEqual([{ path: FAKE_REPO_DIR, options: { recursive: true } }]); expect(handle.status().watching).toBe(true); // One pending timer: the janitor interval (debounceSec*1000 == distinguishable via ms below). const janitorEntry = [...timers.pending.values()].find((t) => t.ms === DEFAULT_SETTINGS.janitorIntervalMin * 60_000); @@ -565,7 +600,7 @@ describe("startHomeSnapshot — commit shapes", () => { }, }; const { fn: execFn, calls: execCalls, optsLog } = makeFakeExec(defaultResponders({ statusZ: "?? notes/a.md\0" })); - const { deps } = baseDeps({ exec: execFn, readOwners: () => owners, repoDir: "/fake/repo" }); + const { deps } = baseDeps({ exec: execFn, readOwners: () => owners, repoDir: FAKE_REPO_DIR }); const handle = startHomeSnapshot(deps); await handle.ready; @@ -582,9 +617,9 @@ describe("startHomeSnapshot — commit shapes", () => { "git", "-c", "commit.gpgsign=false", "commit", "-q", "-m", "snapshot (manual): notes", "--", ".", ":(exclude)prefs/", ":(exclude)secrets/", ]); - expect(optsLog[addIdx]?.cwd).toBe("/fake/repo"); + expect(optsLog[addIdx]?.cwd).toBe(FAKE_REPO_DIR); expect(optsLog[addIdx]?.timeoutMs).toBeGreaterThan(0); - expect(optsLog[commitIdx]?.cwd).toBe("/fake/repo"); + expect(optsLog[commitIdx]?.cwd).toBe(FAKE_REPO_DIR); expect(optsLog[commitIdx]?.timeoutMs).toBeGreaterThan(0); }); @@ -667,6 +702,37 @@ describe("startHomeSnapshot — commit shapes", () => { expect(commits.length).toBe(2); // the auto commit and the janitor zone commit for (const argv of commits) expect(argv.slice(0, 3)).toEqual(["git", "-c", "commit.gpgsign=false"]); }); + + test("R043: missing git identity skips with 'no-git-identity', warns once, never attempts the commit", async () => { + const { fn: execFn, calls: execCalls } = makeFakeExec(defaultResponders({ statusZ: "?? a.txt\0", hasIdentity: false })); + const { deps, log } = baseDeps({ exec: execFn }); + const handle = startHomeSnapshot(deps); + await handle.ready; + + const result = await handle.runNow("manual"); + expect(result.skipped).toBe("no-git-identity"); + expect(execCalls.some((c) => gitVerb(c) === "commit")).toBe(false); + expect(log.calls.filter((c) => c.level === "warn" && String(c.args[c.args.length - 1]).includes("git config --global user.name")).length).toBe(1); + + // A later manual call short-circuits without re-probing identity or git status. + execCalls.length = 0; + const secondResult = await handle.runNow("manual"); + expect(secondResult.skipped).toBe("no-git-identity"); + expect(execCalls.length).toBe(0); + expect(log.calls.filter((c) => c.level === "warn").length).toBe(1); // still just the one warn + }); + + test("git identity present — commits normally, exactly one identity probe pair", async () => { + const { fn: execFn, calls: execCalls } = makeFakeExec(defaultResponders({ statusZ: "?? a.txt\0" })); + const { deps } = baseDeps({ exec: execFn }); + const handle = startHomeSnapshot(deps); + await handle.ready; + + const result = await handle.runNow("manual"); + expect(result.committed).toBe(true); + expect(execCalls.filter((c) => c[1] === "config" && c[2] === "user.name").length).toBe(1); + expect(execCalls.filter((c) => c[1] === "config" && c[2] === "user.email").length).toBe(1); + }); }); // ─── concurrency guard ─────────────────────────────────────────────────────── @@ -691,7 +757,7 @@ describe("startHomeSnapshot — concurrency guard", () => { describe("startHomeSnapshot — push", () => { test("a commit schedules a trailing push after pushDelaySec; success clears pushPending", async () => { const { fn: execFn, calls: execCalls, optsLog } = makeFakeExec(defaultResponders({ statusZ: "?? a.txt\0", pushExit: 0 })); - const { deps, timers } = baseDeps({ exec: execFn, repoDir: "/fake/repo" }); + const { deps, timers } = baseDeps({ exec: execFn, repoDir: FAKE_REPO_DIR }); const handle = startHomeSnapshot(deps); await handle.ready; @@ -704,7 +770,7 @@ describe("startHomeSnapshot — push", () => { const pushIdx = execCalls.findIndex((c) => c[0] === "git" && c[1] === "push"); expect(execCalls[pushIdx]).toEqual(["git", "push", "-q", "origin", "HEAD"]); - expect(optsLog[pushIdx]?.cwd).toBe("/fake/repo"); + expect(optsLog[pushIdx]?.cwd).toBe(FAKE_REPO_DIR); expect(optsLog[pushIdx]?.timeoutMs).toBeGreaterThan(0); expect(handle.status().pushPending).toBe(false); expect(handle.status().lastPushAt).toBe(1_000_000); @@ -829,6 +895,7 @@ describe("startHomeSnapshot — push", () => { if (argv[1] === "rev-parse" && argv[2] === "HEAD") return { stdout: "sha1\n", stderr: "", exitCode: 0 }; if (argv[1] === "status") return { stdout: "?? a.txt\0", stderr: "", exitCode: 0 }; if (argv[1] === "add") return { stdout: "", stderr: "", exitCode: 0 }; + if (argv[1] === "config") return { stdout: "rt test\n", stderr: "", exitCode: 0 }; if (gitVerb(argv) === "commit") { await gate; return { stdout: "", stderr: "", exitCode: 0 }; } return { stdout: "", stderr: "", exitCode: 0 }; }; diff --git a/lib/daemon/home-snapshot.ts b/lib/daemon/home-snapshot.ts index 3c793ff9..f835b2ee 100644 --- a/lib/daemon/home-snapshot.ts +++ b/lib/daemon/home-snapshot.ts @@ -46,6 +46,8 @@ export type SnapshotReason = "manual" | "watch" | "janitor"; export type SkipReason = | "disabled" | "not-a-repo" + | "not-provisioned" + | "no-git-identity" | "init-failed" | "detached" | "merge-in-progress" @@ -278,7 +280,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle const ownersPath = ownersPathFor(deps.repoDir); - let disabledReason: "not-a-repo" | "init-failed" | null = null; + let disabledReason: SkipReason | null = null; let stopped = false; let watcher: { close(): void } | null = null; let debounceTimer: ReturnType | null = null; @@ -356,6 +358,15 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle async function init(): Promise { try { + // Checked before spawning git at all: a missing repoDir (never `rt + // home init`'d) otherwise reaches the same exitCode === -1 branch as a + // genuinely missing git binary, misdiagnosing "not provisioned" as + // "could not run git". + if (!existsSync(deps.repoDir)) { + disabledReason = "not-provisioned"; + deps.log.warn({ repoDir: deps.repoDir }, "home-snapshot: home repo not provisioned; run `rt home init`; inert"); + return; + } const check = await deps.exec(["git", "rev-parse", "--is-inside-work-tree"], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, @@ -688,6 +699,20 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle // `-c commit.gpgsign=false`: a global signing config with an unusable // key fails every snapshot commit outright (exit 128), and nothing // about an unattended backup commit needs a signature. + // Checked once, right before the first commit attempt: an unconfigured + // identity fails every commit the same way (exit 128, "empty ident + // name"), so this latches disabledReason rather than retrying the + // same doomed commit every cycle. + const name = await deps.exec(["git", "config", "user.name"], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, stderr: "pipe" }); + const email = await deps.exec(["git", "config", "user.email"], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, stderr: "pipe" }); + if (name.exitCode !== 0 || !name.stdout.trim() || email.exitCode !== 0 || !email.stdout.trim()) { + disabledReason = "no-git-identity"; + if (lastLoggedCommitError !== "no-git-identity") { + deps.log.warn("home-snapshot: no git identity; run `git config --global user.name` and `git config --global user.email`; snapshots inert"); + lastLoggedCommitError = "no-git-identity"; + } + return { committed: false, sha: null, paths: [], reason, skipped: "no-git-identity" }; + } const message = reason === "manual" ? plan.message.replace(/^snapshot:/, "snapshot (manual):") : plan.message; const commitResult = await deps.exec(["git", "-c", "commit.gpgsign=false", "commit", "-q", "-m", message, "--", ".", ...excludeArgs], { cwd: deps.repoDir, diff --git a/lib/home/__tests__/init-exec.test.ts b/lib/home/__tests__/init-exec.test.ts index 5b520d6f..b186cfd9 100644 --- a/lib/home/__tests__/init-exec.test.ts +++ b/lib/home/__tests__/init-exec.test.ts @@ -26,6 +26,8 @@ class FakeExecSeam implements ExecSeam { failRun?: (cmd: string[]) => string | undefined; exists?: (path: string) => boolean; blocksSymlink?: (path: string) => boolean; + /** git config user.name/user.email answers for commitInitialUserRepo's identity check — defaults to a fully-configured identity so every other test's commit step doesn't have to opt in. */ + identity?: { name?: string; email?: string }; } = {}, ) {} @@ -33,6 +35,12 @@ class FakeExecSeam implements ExecSeam { this.calls.push({ kind: "run", cmd, cwd: runOpts?.cwd }); const failure = this.opts.failRun?.(cmd); if (failure) return { code: 1, stdout: "", stderr: failure }; + const configKey = cmd[cmd.length - 2] === "config" ? cmd[cmd.length - 1] : undefined; + if (configKey === "user.name" || configKey === "user.email") { + const identity = this.opts.identity ?? { name: "rt test", email: "rt@example.test" }; + const value = configKey === "user.name" ? identity.name : identity.email; + return value ? { code: 0, stdout: `${value}\n`, stderr: "" } : { code: 1, stdout: "", stderr: "" }; + } return { code: 0, stdout: "", stderr: "" }; } @@ -120,6 +128,21 @@ describe("executeInitPlan", () => { expect(result.ok).toBe(false); if (!result.ok) expect(result.failedStep).toBe("commitInitialUserRepo"); }); + + test("R043: no git identity fails with an actionable message, never attempts the commit", async () => { + const seam = new FakeExecSeam({ identity: { name: "", email: "" } }); + + const result = await executeInitPlan([{ kind: "commitInitialUserRepo" }], seam, noopLog); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.failedStep).toBe("commitInitialUserRepo"); + expect(result.stderr).toContain("git config --global user.name"); + expect(result.stderr).toContain("git config --global user.email"); + expect(result.stderr).toContain("rt home init"); + } + expect(seam.calls.some((c) => c.kind === "run" && c.cmd.includes("commit"))).toBe(false); + }); }); describe("writeGitignore / writeOwners — write-if-absent, decided at exec time", () => { diff --git a/lib/home/init-exec.ts b/lib/home/init-exec.ts index 71a59a72..deb589be 100644 --- a/lib/home/init-exec.ts +++ b/lib/home/init-exec.ts @@ -82,6 +82,14 @@ async function runStep(step: InitStep, exec: ExecSeam, log: StepLog): Promise Date: Fri, 28 Aug 2026 18:06:10 -0500 Subject: [PATCH 125/142] daemon: wire loop monitor + heartbeat + health sampler; surface health/metrics/eventLoop in status/tray:status/ping status-identity.test.ts's fakeCtx() gained getHealth/heartbeatSeq stubs to match the widened HandlerContext. --- lib/daemon.ts | 77 +++++++++++++++++++- lib/daemon/__tests__/health-sampler.test.ts | 15 ++++ lib/daemon/__tests__/status-identity.test.ts | 7 ++ lib/daemon/handlers/status.ts | 12 +++ lib/daemon/handlers/types.ts | 5 ++ lib/daemon/health-sampler.ts | 66 +++++++++++++++++ 6 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 lib/daemon/__tests__/health-sampler.test.ts create mode 100644 lib/daemon/health-sampler.ts diff --git a/lib/daemon.ts b/lib/daemon.ts index 4ea5414f..6eb3f3d4 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -52,7 +52,7 @@ import { runBootIdentityMigration } from "./daemon/boot-migrate.ts"; import { runCapture } from "./subprocess.ts"; import { buildRoutedHandlers } from "./daemon/command-router.ts"; import { startSocketServer } from "./daemon/socket-server.ts"; -import { startApiServer, withApiPortParkRetry, broadcast } from "./daemon/api-server.ts"; +import { startApiServer, withApiPortParkRetry, broadcast, apiWsClientCount } from "./daemon/api-server.ts"; import { loadCronConfig, startCron } from "./daemon/cron.ts"; import { startPollers } from "./daemon/pollers.ts"; import { startHomeSnapshot } from "./daemon/home-snapshot.ts"; @@ -60,8 +60,15 @@ import { startAgentStatusPoller } from "./daemon/agent-status-poller.ts"; import { initFreshness, reconcileFreshness, + getFreshnessSnapshot, type FreshnessEnv, } from "./daemon/freshness.ts"; +import { startLoopMonitor } from "./daemon/loop-monitor.ts"; +import { createHealthSampler } from "./daemon/health-sampler.ts"; +import { writeHeartbeat } from "./daemon/heartbeat-file.ts"; +import { computeHealth } from "./daemon/health.ts"; +import { isCrashLooping, readSupervisionState } from "./daemon/supervision-state.ts"; +import { setSettingsWarnSink } from "./settings/resolve.ts"; import { startDiscussionsPoller } from "./daemon/discussions-poller.ts"; import { createCleanup, installSignalHandlers } from "./daemon/shutdown.ts"; import { createEventsBus } from "./daemon/events-bus.ts"; @@ -121,6 +128,11 @@ redirectNativeStderr(); const loggerHandle = await getDaemonLogger(); const log = loggerHandle.logger; +// Route the settings resolver's dedup'd warn sink into structured daemon +// logging, so a hot-path getSetting on a disallowed-scope key surfaces once +// in the daemon log instead of the resolver's own console fallback. +setSettingsWarnSink((m) => log.warn({ src: "settings" }, m)); + // Wire uncaughtException + unhandledRejection through pino as early as the // logger allows: every module-scope side effect below this point // (createEventsBus, cron, home-snapshot, sweep timers) can throw, and this @@ -352,6 +364,63 @@ const agentStatusPoller = startAgentStatusPoller({ log: loggerHandle.childLogger("agent-status"), }); +// In-flight command name, polled by the loop monitor to spot a handler that +// never returns. Declared before the monitor so its `currentCmd` closure +// captures this same mutable ref, not a stale one. +const currentCmd: { cmd: string | null } = { cmd: null }; + +// 5-min metrics log + the two cached signals health needs but is too costly +// to compute per call: the 1h rss-growth baseline and free disk under RT_DIR. +const healthSampler = createHealthSampler({ + log, + rtDir: RT_DIR, + wsClients: apiWsClientCount, + // Sourced exactly as handlerCtx.watchedConfigs is below... there is no bare + // `watchedConfigs` alias at this scope. + watchers: () => hooksGuard.watchedConfigs.size, + startedAt, +}); +healthSampler.sample(); // seed baseline/free immediately, don't wait 5min for the first reading +safeInterval(() => healthSampler.sample(), 5 * 60_000, "health-sample", log); + +// 250ms event-loop drift monitor; also writes the cross-process liveness +// heartbeat file every ~2s. Both timers are unref'd and db-free internally. +const loopMon = startLoopMonitor({ + log, + currentCmd: () => currentCmd.cmd, + onHeartbeat: (at, seq) => writeHeartbeat(RT_DIR, { at, seq }), +}); + +/** Not cached: computeHealth is pure/cheap, and every input it reads is + * already either a live ref or a fast getter, so recomputing per call keeps + * the snapshot honest without a staleness window to reason about. */ +function buildHealthSnapshot() { + const now = Date.now(); + const sup = readSupervisionState(); + const failuresLastHour = sup.recentFailures.filter((f) => f.at > now - 60 * 60_000).length; + return computeHealth({ + now, + uptimeMs: now - startedAt, + mem: process.memoryUsage(), + rssBaseline: healthSampler.rssBaseline(), + wsClients: apiWsClientCount(), + watchers: hooksGuard.watchedConfigs.size, + freshness: getFreshnessSnapshot(), + refresh: { + lastSuccessAt: refreshStatusRef.lastSuccessAt, + failedRepos: refreshStatusRef.failedRepos, + enrichErrors: refreshStatusRef.enrichErrors, + }, + refreshIntervalMs: 5 * 60_000, + eventLoop: { ...loopMon.stats }, + supervisionFailuresLastHour: failuresLastHour, + crashLooping: isCrashLooping(sup, now), + loggerDegraded: loggerHandle.loggerDegraded?.() ?? false, + recoveredErrorRateLastWindow: loggerHandle.recoveredErrorCount?.() ?? 0, + freeBytes: healthSampler.freeBytes(), + }); +} + // ─── Handler context + command routing ─────────────────────────────────────── const handlerCtx: HandlerContext = { @@ -365,6 +434,8 @@ const handlerCtx: HandlerContext = { checkAndRepairHooksPath: hooksGuard.checkAndRepairHooksPath, startWatchingRepo: hooksGuard.startWatchingRepo, refreshStatusRef, + getHealth: buildHealthSnapshot, + heartbeatSeq: loopMon.seq, }; /** Env bundle for the live-freshness subsystem. */ @@ -380,9 +451,6 @@ let routedHandlers: ReturnType | undefined; // policy: docs/daemon-supervision-design.md). let shuttingDownViaVerb = false; -// In-flight command name, polled by the loop monitor (wired in a later task) -// to spot a handler that never returns. -const currentCmd: { cmd: string | null } = { cmd: null }; const rejectSuppressor = makeSuppressor(60_000); const SLOW_COMMAND_MS = 2000; @@ -469,6 +537,7 @@ const cleanup = (): void => { eventsBus.close(); homeSnapshot.stop(); agentStatusPoller.stop(); + loopMon.stop(); cleanupCore(); }; diff --git a/lib/daemon/__tests__/health-sampler.test.ts b/lib/daemon/__tests__/health-sampler.test.ts new file mode 100644 index 00000000..4024d864 --- /dev/null +++ b/lib/daemon/__tests__/health-sampler.test.ts @@ -0,0 +1,15 @@ +// lib/daemon/__tests__/health-sampler.test.ts +import { test, expect } from "bun:test"; +import { rollRssBaseline } from "../health-sampler.ts"; + +test("rss baseline rolls forward only after the window elapses", () => { + // baseline null -> set on first sample + let b = rollRssBaseline(null, { rss: 100, at: 0 }, 60 * 60_000); + expect(b).toEqual({ rss: 100, at: 0 }); + // within the hour: unchanged + b = rollRssBaseline(b, { rss: 200, at: 30 * 60_000 }, 60 * 60_000); + expect(b).toEqual({ rss: 100, at: 0 }); + // after the hour: rolls to the new sample + b = rollRssBaseline(b, { rss: 250, at: 61 * 60_000 }, 60 * 60_000); + expect(b).toEqual({ rss: 250, at: 61 * 60_000 }); +}); diff --git a/lib/daemon/__tests__/status-identity.test.ts b/lib/daemon/__tests__/status-identity.test.ts index b92d66b4..c3c9d99d 100644 --- a/lib/daemon/__tests__/status-identity.test.ts +++ b/lib/daemon/__tests__/status-identity.test.ts @@ -8,6 +8,13 @@ function fakeCtx(): any { watchedConfigs: new Map(), cache: { entries: {} }, portCacheRef: { ports: [], updatedAt: null }, + getHealth: () => ({ + level: "ok", + reasons: [], + metrics: { rss: 0, heapUsed: 0, external: 0, uptimeMs: 0, wsClients: 0, watchers: 0 }, + eventLoop: { maxLagMs: 0, lastStallAt: null, lastStallCmd: null, stalls: 0 }, + }), + heartbeatSeq: () => 0, }; } diff --git a/lib/daemon/handlers/status.ts b/lib/daemon/handlers/status.ts index 19db8227..5f9abf5c 100644 --- a/lib/daemon/handlers/status.ts +++ b/lib/daemon/handlers/status.ts @@ -26,16 +26,21 @@ export function createStatusHandlers(ctx: HandlerContext): HandlerMap { // must see this run's own boot-attempt/failure counters, not whatever // they were when the daemon started. const { bootAttempts, lastReadyAt, recentFailures, lastExit } = readSupervisionState(); + const h = ctx.getHealth(); return { ok: true, uptime: Date.now() - ctx.startedAt, pid: process.pid, ...ctx.identity, + health: h.level, + eventLoop: h.eventLoop, + heartbeatSeq: ctx.heartbeatSeq(), supervision: { bootAttempts, lastReadyAt, recentFailures: recentFailures.slice(-3), lastExit }, }; }, "status": async () => { + const h = ctx.getHealth(); return { ok: true, data: { @@ -47,6 +52,9 @@ export function createStatusHandlers(ctx: HandlerContext): HandlerMap { portCacheAge: ctx.portCacheRef.updatedAt ? Date.now() - ctx.portCacheRef.updatedAt : null, freshness: getFreshnessSnapshot(), identity: ctx.identity, + health: { level: h.level, reasons: h.reasons }, + metrics: h.metrics, + eventLoop: h.eventLoop, }, }; }, @@ -58,6 +66,7 @@ export function createStatusHandlers(ctx: HandlerContext): HandlerMap { const repo = p.repo || "unknown"; portsByRepo[repo] = (portsByRepo[repo] || 0) + 1; } + const h = ctx.getHealth(); return { ok: true, @@ -72,6 +81,9 @@ export function createStatusHandlers(ctx: HandlerContext): HandlerMap { lastRefresh: ctx.refreshStatusRef.lastRefreshAt || null, portsByRepo, pendingNotifications: peekNotifications().length, + health: { level: h.level, reasons: h.reasons }, + metrics: h.metrics, + eventLoop: h.eventLoop, }, }; }, diff --git a/lib/daemon/handlers/types.ts b/lib/daemon/handlers/types.ts index 6eca7e7e..647e7608 100644 --- a/lib/daemon/handlers/types.ts +++ b/lib/daemon/handlers/types.ts @@ -10,6 +10,7 @@ import type { FSWatcher } from "fs"; import type { Logger } from "pino"; import type { PortEntry } from "../../port-scanner.ts"; import type { BranchCacheStore } from "../../state/index.ts"; +import type { HealthSnapshot } from "../health.ts"; /** * RT-48: `CacheEntry` used to be DECLARED here — a third copy of the same @@ -70,6 +71,10 @@ export interface HandlerContext { startWatchingRepo: (repoName: string, repoPath: string) => void; /** Holder for the last cache-refresh cycle's outcome (0s = never run). */ refreshStatusRef: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }; + /** Computes the current health verdict (level/reasons/metrics/eventLoop) on demand; not cached, cheap enough per call. */ + getHealth: () => HealthSnapshot; + /** Current loop-monitor heartbeat sequence number, echoed by `ping`. */ + heartbeatSeq: () => number; } export type Handler = (payload: any, signal?: AbortSignal) => Promise; diff --git a/lib/daemon/health-sampler.ts b/lib/daemon/health-sampler.ts new file mode 100644 index 00000000..2d2a7c5e --- /dev/null +++ b/lib/daemon/health-sampler.ts @@ -0,0 +1,66 @@ +// lib/daemon/health-sampler.ts +/** Periodic (5-min) metrics logging + the two cached signals health needs that + * are too costly to compute per ping: the 1h rss baseline (growth) and free + * disk under RT_DIR. Pure helpers are unit-tested; the timer just calls sample. */ +import { statfsSync } from "fs"; +import type { Logger } from "pino"; + +export function rollRssBaseline( + prev: { rss: number; at: number } | null, + now: { rss: number; at: number }, + windowMs: number, +): { rss: number; at: number } { + if (!prev) return now; + if (now.at - prev.at >= windowMs) return now; + return prev; +} + +export interface HealthSampler { + sample(): void; + freeBytes(): number | null; + rssBaseline(): { rss: number; at: number } | null; +} + +export function createHealthSampler(opts: { + log: Logger; + rtDir: string; + wsClients: () => number; + watchers: () => number; + startedAt: number; +}): HealthSampler { + let baseline: { rss: number; at: number } | null = null; + let free: number | null = null; + + function statfsFree(dir: string): number | null { + // Not every platform/runtime implements statfs; leave free=null and disk + // checks are simply skipped rather than treated as an error. + try { + const s = statfsSync(dir); + return s.bavail * s.bsize; + } catch { + return null; + } + } + + return { + freeBytes: () => free, + rssBaseline: () => baseline, + sample() { + const mem = process.memoryUsage(); + const now = Date.now(); + baseline = rollRssBaseline(baseline, { rss: mem.rss, at: now }, 60 * 60_000); + free = statfsFree(opts.rtDir); + opts.log.info( + { + rss: mem.rss, + heapUsed: mem.heapUsed, + external: mem.external, + wsClients: opts.wsClients(), + watchers: opts.watchers(), + uptimeMs: now - opts.startedAt, + }, + "daemon metrics", + ); + }, + }; +} From 38cf5612bdae8d232b552fdcbb090cf671eccb9d Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 18:11:56 -0500 Subject: [PATCH 126/142] add rt daemon log-level: live level set/show via daemon:log-level verb --- commands/__tests__/log-level.test.ts | 9 +++++++++ commands/daemon.ts | 16 ++++++++++++++++ lib/command-tree-def.ts | 14 ++++++++++++++ lib/daemon.ts | 2 ++ lib/daemon/handlers/status.ts | 10 ++++++++++ lib/daemon/handlers/types.ts | 4 ++++ 6 files changed, 55 insertions(+) create mode 100644 commands/__tests__/log-level.test.ts diff --git a/commands/__tests__/log-level.test.ts b/commands/__tests__/log-level.test.ts new file mode 100644 index 00000000..73a6f8a0 --- /dev/null +++ b/commands/__tests__/log-level.test.ts @@ -0,0 +1,9 @@ +import { test, expect } from "bun:test"; +import { formatLogLevelResult } from "../daemon.ts"; + +test("formats a set result", () => { + expect(formatLogLevelResult({ ok: true, level: "debug" }, true)).toContain("debug"); +}); +test("formats a show result", () => { + expect(formatLogLevelResult({ ok: true, level: "info" }, false)).toContain("info"); +}); diff --git a/commands/daemon.ts b/commands/daemon.ts index 2c1374ec..dad5abeb 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -1166,3 +1166,19 @@ async function waitForPort(port: number, timeoutMs: number): Promise { await new Promise(r => setTimeout(r, 100)); } } + +/** Pure formatter for `daemon:log-level` results — shared by the CLI and its tests. */ +export function formatLogLevelResult(res: { ok: boolean; level?: string; error?: string }, wasSet: boolean): string { + if (!res.ok) return ` ${red}●${reset} ${res.error ?? "failed"}`; + return ` ${green}●${reset} daemon log level ${wasSet ? "set to" : "is"} ${res.level}`; +} + +/** Show (no arg) or set (level arg) the running daemon's live pino log level. */ +export async function setLogLevel(args: string[] = []): Promise { + const json = args.includes("--json"); + const level = args.find((a) => !a.startsWith("--")); + const res = await daemonQuery("daemon:log-level", level ? { level } : {}); + if (!res) { console.log(` ${red}●${reset} daemon not reachable`); return; } + if (json) { console.log(JSON.stringify(res)); return; } + console.log(formatLogLevelResult(res as any, Boolean(level))); +} diff --git a/lib/command-tree-def.ts b/lib/command-tree-def.ts index 291614d3..7aff0159 100644 --- a/lib/command-tree-def.ts +++ b/lib/command-tree-def.ts @@ -664,6 +664,20 @@ export const TREE: Record = { { name: "Terminal", flag: "--terminal", type: "boolean", default: false, hint: "Tail logs in terminal via lnav or pino-pretty instead of opening the web viewer (alias -t)" }, ], }, + "log-level": { + description: "Show or set the daemon's live log level", + module: "./commands/daemon.ts", + fn: "setLogLevel", + omitBehavior: "list", + args: [ + { name: "Level", type: "select", hint: "Omit to show the current level", + options: [ + { value: "trace", label: "trace" }, { value: "debug", label: "debug" }, + { value: "info", label: "info" }, { value: "warn", label: "warn" }, + { value: "error", label: "error" }, + ] }, + ], + }, }, }, diff --git a/lib/daemon.ts b/lib/daemon.ts index 6eb3f3d4..d44adb14 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -436,6 +436,8 @@ const handlerCtx: HandlerContext = { refreshStatusRef, getHealth: buildHealthSnapshot, heartbeatSeq: loopMon.seq, + setLogLevel: (l) => { log.level = l; log.info({ level: l }, "log level changed"); }, + getLogLevel: () => log.level, }; /** Env bundle for the live-freshness subsystem. */ diff --git a/lib/daemon/handlers/status.ts b/lib/daemon/handlers/status.ts index 5f9abf5c..3800bd83 100644 --- a/lib/daemon/handlers/status.ts +++ b/lib/daemon/handlers/status.ts @@ -9,6 +9,7 @@ * ports — cached port-scan data, optionally filtered by repo * notifications — drain the notification queue * notifications:peek — peek at the notification queue (diagnostics) + * daemon:log-level — show or set the live pino log level */ import { existsSync, readdirSync } from "fs"; @@ -184,5 +185,14 @@ export function createStatusHandlers(ctx: HandlerContext): HandlerMap { // Peek without draining — for diagnostics return { ok: true, data: peekNotifications() }; }, + + "daemon:log-level": async (payload?: { level?: string }) => { + const VALID = ["trace", "debug", "info", "warn", "error"]; + if (payload?.level) { + if (!VALID.includes(payload.level)) return { ok: false, error: `invalid level: ${payload.level}` }; + ctx.setLogLevel(payload.level); + } + return { ok: true, level: ctx.getLogLevel() }; + }, }; } diff --git a/lib/daemon/handlers/types.ts b/lib/daemon/handlers/types.ts index 647e7608..90b12202 100644 --- a/lib/daemon/handlers/types.ts +++ b/lib/daemon/handlers/types.ts @@ -75,6 +75,10 @@ export interface HandlerContext { getHealth: () => HealthSnapshot; /** Current loop-monitor heartbeat sequence number, echoed by `ping`. */ heartbeatSeq: () => number; + /** Sets the daemon logger's live level (trace/debug/info/warn/error). */ + setLogLevel: (l: string) => void; + /** Reads the daemon logger's current live level. */ + getLogLevel: () => string; } export type Handler = (payload: any, signal?: AbortSignal) => Promise; From 22a2f525a6f1b71281bb42871d7e08c89648361b Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 18:14:44 -0500 Subject: [PATCH 127/142] fix: remove em dashes from log-level comments --- commands/daemon.ts | 2 +- lib/daemon/handlers/status.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/commands/daemon.ts b/commands/daemon.ts index dad5abeb..6f3c79a4 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -1167,7 +1167,7 @@ async function waitForPort(port: number, timeoutMs: number): Promise { } } -/** Pure formatter for `daemon:log-level` results — shared by the CLI and its tests. */ +/** Pure formatter for `daemon:log-level` results, shared by the CLI and its tests. */ export function formatLogLevelResult(res: { ok: boolean; level?: string; error?: string }, wasSet: boolean): string { if (!res.ok) return ` ${red}●${reset} ${res.error ?? "failed"}`; return ` ${green}●${reset} daemon log level ${wasSet ? "set to" : "is"} ${res.level}`; diff --git a/lib/daemon/handlers/status.ts b/lib/daemon/handlers/status.ts index 3800bd83..2767376e 100644 --- a/lib/daemon/handlers/status.ts +++ b/lib/daemon/handlers/status.ts @@ -9,7 +9,7 @@ * ports — cached port-scan data, optionally filtered by repo * notifications — drain the notification queue * notifications:peek — peek at the notification queue (diagnostics) - * daemon:log-level — show or set the live pino log level + * daemon:log-level - show or set the live pino log level */ import { existsSync, readdirSync } from "fs"; From 16de2bab28160a3bb9ffb6b8a69fb6da25d23e6b Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 18:14:59 -0500 Subject: [PATCH 128/142] home-snapshot: gate janitor-zone commit on git identity; drop em dashes (R043) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/daemon/__tests__/home-snapshot.test.ts | 34 +++++++++++++++-- lib/daemon/home-snapshot.ts | 44 +++++++++++++--------- lib/home/__tests__/init-exec.test.ts | 2 +- 3 files changed, 57 insertions(+), 23 deletions(-) diff --git a/lib/daemon/__tests__/home-snapshot.test.ts b/lib/daemon/__tests__/home-snapshot.test.ts index 52ffbc45..5e351720 100644 --- a/lib/daemon/__tests__/home-snapshot.test.ts +++ b/lib/daemon/__tests__/home-snapshot.test.ts @@ -61,7 +61,7 @@ function defaultResponders(opts: { : undefined, (argv) => (argv[1] === "status") ? { stdout: statusZ, stderr: "", exitCode: 0 } : undefined, (argv) => (argv[1] === "add") ? { stdout: "", stderr: "", exitCode: addExit } : undefined, - // git identity probe, checked right before the first auto commit — + // git identity probe, checked right before either commit site runs... // defaults to "configured" so every fixture not testing R043 stays green. (argv) => (argv[1] === "config" && argv[2] === "user.name") ? (hasIdentity ? { stdout: "rt test\n", stderr: "", exitCode: 0 } : { stdout: "", stderr: "", exitCode: 1 }) @@ -177,8 +177,8 @@ const DEFAULT_SETTINGS: HomeSnapshotSettings = { const NO_OWNERS: Owners = { zones: {} }; -// A real (but never touched — every git call underneath it is faked) -// directory: the S090 existsSync guard runs against the real filesystem, so +// A real directory (never touched; every git call underneath it is faked): +// the S090 existsSync guard runs against the real filesystem, so // the fixture repoDir the whole suite shares must actually exist on disk, // not just look plausible as a string. const FAKE_REPO_DIR = realpathSync(mkdtempSync(join(tmpdir(), "rt-home-snapshot-fakerepo-"))); @@ -722,7 +722,7 @@ describe("startHomeSnapshot — commit shapes", () => { expect(log.calls.filter((c) => c.level === "warn").length).toBe(1); // still just the one warn }); - test("git identity present — commits normally, exactly one identity probe pair", async () => { + test("git identity present: commits normally, exactly one identity probe pair", async () => { const { fn: execFn, calls: execCalls } = makeFakeExec(defaultResponders({ statusZ: "?? a.txt\0" })); const { deps } = baseDeps({ exec: execFn }); const handle = startHomeSnapshot(deps); @@ -733,6 +733,32 @@ describe("startHomeSnapshot — commit shapes", () => { expect(execCalls.filter((c) => c[1] === "config" && c[2] === "user.name").length).toBe(1); expect(execCalls.filter((c) => c[1] === "config" && c[2] === "user.email").length).toBe(1); }); + + test("R043: a janitor-only cycle (no auto paths, one dirty claimed zone past threshold) with no git identity also skips 'no-git-identity', never attempts the janitor commit", async () => { + const owners: Owners = { zones: { "prefs/": { owner: "matt", claimedAt: "2026-01-01T00:00:00.000Z" } } }; + const db = freshDb(); + db.query("INSERT INTO kv (ns, k, v, updated_at) VALUES ('home-snapshot', 'state', ?, 0);") + .run(JSON.stringify({ firstSeenDirty: { "prefs/": 0 } })); + + const { fn: execFn, calls: execCalls } = makeFakeExec(defaultResponders({ statusZ: "?? prefs/x.md\0", hasIdentity: false })); + const { deps, log } = baseDeps({ + exec: execFn, + readOwners: () => owners, + db, + now: () => 10_000_000, // far past a 1-hour threshold from firstSeenDirty=0 + }); + + const handle = startHomeSnapshot(deps); + await handle.ready; + + const result = await handle.runNow("manual"); + expect(result.skipped).toBe("no-git-identity"); + // Only the claimed zone was dirty, so this cycle has no auto commit at + // all: the identity gate must still catch the janitor-only path. + expect(execCalls.some((c) => c[1] === "add")).toBe(false); + expect(execCalls.some((c) => gitVerb(c) === "commit")).toBe(false); + expect(log.calls.filter((c) => c.level === "warn" && String(c.args[c.args.length - 1]).includes("git config --global user.name")).length).toBe(1); + }); }); // ─── concurrency guard ─────────────────────────────────────────────────────── diff --git a/lib/daemon/home-snapshot.ts b/lib/daemon/home-snapshot.ts index f835b2ee..d6a1f1d9 100644 --- a/lib/daemon/home-snapshot.ts +++ b/lib/daemon/home-snapshot.ts @@ -667,7 +667,28 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle let committed = false; let sha: string | null = null; - if (plan.autoPaths.length > 0 && plan.message !== null) { + // Two independent commit sites below (the auto commit and the + // janitor-zone loop) can each attempt a commit this cycle; both fail the + // same doomed way (exit 128, "empty ident name") against an unconfigured + // identity. Checked once, up front, whenever EITHER would run, so a + // janitor-only cycle (no auto paths, one dirty claimed zone) is covered + // too, not just the auto-commit path. + const willAutoCommit = plan.autoPaths.length > 0 && plan.message !== null; + const willJanitorCommit = (reason === "janitor" || reason === "manual") && plan.janitorZones.length > 0; + if (willAutoCommit || willJanitorCommit) { + const name = await deps.exec(["git", "config", "user.name"], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, stderr: "pipe" }); + const email = await deps.exec(["git", "config", "user.email"], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, stderr: "pipe" }); + if (name.exitCode !== 0 || !name.stdout.trim() || email.exitCode !== 0 || !email.stdout.trim()) { + disabledReason = "no-git-identity"; + if (lastLoggedCommitError !== "no-git-identity") { + deps.log.warn("home-snapshot: no git identity; run `git config --global user.name` and `git config --global user.email`; snapshots inert"); + lastLoggedCommitError = "no-git-identity"; + } + return { committed: false, sha: null, paths: [], reason, skipped: "no-git-identity" }; + } + } + + if (willAutoCommit) { // `plan.autoPaths` describes what the STATUS SNAPSHOT at the top of // this run looked like — a purely descriptive record of intent. The // exclude pathspec built from `plan.excludedZones` (identical on both @@ -698,22 +719,9 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle // // `-c commit.gpgsign=false`: a global signing config with an unusable // key fails every snapshot commit outright (exit 128), and nothing - // about an unattended backup commit needs a signature. - // Checked once, right before the first commit attempt: an unconfigured - // identity fails every commit the same way (exit 128, "empty ident - // name"), so this latches disabledReason rather than retrying the - // same doomed commit every cycle. - const name = await deps.exec(["git", "config", "user.name"], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, stderr: "pipe" }); - const email = await deps.exec(["git", "config", "user.email"], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, stderr: "pipe" }); - if (name.exitCode !== 0 || !name.stdout.trim() || email.exitCode !== 0 || !email.stdout.trim()) { - disabledReason = "no-git-identity"; - if (lastLoggedCommitError !== "no-git-identity") { - deps.log.warn("home-snapshot: no git identity; run `git config --global user.name` and `git config --global user.email`; snapshots inert"); - lastLoggedCommitError = "no-git-identity"; - } - return { committed: false, sha: null, paths: [], reason, skipped: "no-git-identity" }; - } - const message = reason === "manual" ? plan.message.replace(/^snapshot:/, "snapshot (manual):") : plan.message; + // about an unattended backup commit needs a signature. (Git identity + // is confirmed once, above, before either commit site runs.) + const message = reason === "manual" ? plan.message!.replace(/^snapshot:/, "snapshot (manual):") : plan.message!; const commitResult = await deps.exec(["git", "-c", "commit.gpgsign=false", "commit", "-q", "-m", message, "--", ".", ...excludeArgs], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, @@ -736,7 +744,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle } } - if ((reason === "janitor" || reason === "manual") && plan.janitorZones.length > 0) { + if (willJanitorCommit) { for (const jz of plan.janitorZones) { const dirtyHours = Math.floor((deps.now() - jz.dirtySinceMs) / (60 * 60 * 1000)); const message = `snapshot (janitor): ${jz.zone} dirty >${dirtyHours}h, owner ${jz.owner}`; diff --git a/lib/home/__tests__/init-exec.test.ts b/lib/home/__tests__/init-exec.test.ts index b186cfd9..8bbf40fb 100644 --- a/lib/home/__tests__/init-exec.test.ts +++ b/lib/home/__tests__/init-exec.test.ts @@ -26,7 +26,7 @@ class FakeExecSeam implements ExecSeam { failRun?: (cmd: string[]) => string | undefined; exists?: (path: string) => boolean; blocksSymlink?: (path: string) => boolean; - /** git config user.name/user.email answers for commitInitialUserRepo's identity check — defaults to a fully-configured identity so every other test's commit step doesn't have to opt in. */ + /** git config user.name/user.email answers for commitInitialUserRepo's identity check. Defaults to a fully-configured identity so every other test's commit step doesn't have to opt in. */ identity?: { name?: string; email?: string }; } = {}, ) {} From 27537820d5c33baed1f08e2e63adf0d167f96fc4 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 18:18:29 -0500 Subject: [PATCH 129/142] secrets: timeout + SecretsTimeoutError on the sops spawn (S070 sops half) --- lib/secrets/__tests__/store.test.ts | 18 ++++++++++ lib/secrets/store.ts | 52 +++++++++++++++++++++++------ 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/lib/secrets/__tests__/store.test.ts b/lib/secrets/__tests__/store.test.ts index 2a4fd182..799579bc 100644 --- a/lib/secrets/__tests__/store.test.ts +++ b/lib/secrets/__tests__/store.test.ts @@ -8,8 +8,10 @@ import { resetSecretsMemo, formatDebugLine, buildSecretsSpawnOptions, + createRealSecretsExecSeam, NoAgeKeyError, InvalidSecretsSegmentError, + SecretsTimeoutError, type SecretsExecResult, type SecretsExecSeam, type SecretsSeams, @@ -573,6 +575,22 @@ describe("real seam spawn options — cwd pin", () => { }); }); +describe("real seam spawn timeout", () => { + test("a hanging sops spawn times out with SecretsTimeoutError, does not hang", async () => { + let resolveExit: (code: number) => void = () => {}; + const fakeProc = { + pid: 1, + stdout: new Response("").body, + stderr: new Response("").body, + exited: new Promise((r) => { resolveExit = r; }), + kill: () => resolveExit(143), // killable: kill resolves exit, no real process + }; + const seam = createRealSecretsExecSeam(undefined, () => fakeProc as any); + await expect(seam.run(["sops", "-d", "x"], { timeoutMs: 50 } as any)) + .rejects.toBeInstanceOf(SecretsTimeoutError); + }, 2_000); +}); + describe("formatDebugLine (the debugLog path)", () => { test("a sensitive call's line never includes env values or stdout/stderr, whatever they'd contain", () => { const line = formatDebugLine(["sops", "-d", "/some/path"], { sensitive: true }); diff --git a/lib/secrets/store.ts b/lib/secrets/store.ts index 19283211..e261b048 100644 --- a/lib/secrets/store.ts +++ b/lib/secrets/store.ts @@ -44,7 +44,8 @@ export interface SecretsExecResult { } export interface SecretsExecSeam { - run(cmd: string[], opts?: { env?: Record; sensitive?: boolean }): Promise; + /** `timeoutMs` overrides the default kill-and-reject deadline (see SecretsTimeoutError). */ + run(cmd: string[], opts?: { env?: Record; sensitive?: boolean; timeoutMs?: number }): Promise; fileExists(path: string): boolean; /** * Direct child names of a directory (not recursive); [] when the directory @@ -138,6 +139,11 @@ export function validateSlug(slug: string): void { if (!SLUG_PATTERN.test(slug)) throw new InvalidSecretsSegmentError("slug", slug, SLUG_PATTERN); } +/** Thrown when a sops/keychain spawn does not exit in time (a locked keychain pops a GUI dialog and blocks until clicked). */ +export class SecretsTimeoutError extends Error {} + +const DEFAULT_SECRETS_TIMEOUT_MS = 30_000; + /** Validates `domain` — every path construction routes through here, so this is the one choke point. */ export function secretsFilePath(domain: string): string { validateDomain(domain); @@ -458,14 +464,23 @@ export function buildSecretsSpawnOptions(opts?: { env?: Record; }; } +type SecretsSpawn = (argv: string[], opts: any) => { + stdout: ReadableStream; + stderr: ReadableStream; + exited: Promise; + kill: (sig?: number | string) => void; +}; + /** * Real seam: Bun.spawn-based capture, real fs reads/writes. `cwd`, when * given, is pinned for every sops spawn this seam instance makes — the * personal store's default seam (`cwd` omitted) resolves `/user`; * team-store.ts constructs its own instance per team with `cwd` set to that - * team's clone root (see `buildTeamSpawnOptions`). + * team's clone root (see `buildTeamSpawnOptions`). `spawn` is injectable so + * tests can model a hanging child without a real subprocess; the default is + * the real `Bun.spawn`, so production behavior is unchanged. */ -export function createRealSecretsExecSeam(cwd?: string): SecretsExecSeam { +export function createRealSecretsExecSeam(cwd?: string, spawn: SecretsSpawn = Bun.spawn as unknown as SecretsSpawn): SecretsExecSeam { return { async run(cmd, opts) { debugLog(cmd, opts?.sensitive); @@ -475,13 +490,30 @@ export function createRealSecretsExecSeam(cwd?: string): SecretsExecSeam { // stay unaware of the bundle. const [bin, ...args] = cmd; const resolved = bin === undefined ? cmd : [resolveBundledTool(bin), ...args]; - const proc = Bun.spawn(resolved, buildSecretsSpawnOptions({ env: opts?.env, cwd })); - const [stdout, stderr, code] = await Promise.all([ - new Response(proc.stdout).text(), - new Response(proc.stderr).text(), - proc.exited, - ]); - return { code, stdout, stderr }; + const proc = spawn(resolved, buildSecretsSpawnOptions({ env: opts?.env, cwd })); + const timeoutMs = opts?.timeoutMs ?? DEFAULT_SECRETS_TIMEOUT_MS; + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + try { + proc.kill(); + } catch { + // already exited + } + }, timeoutMs); + try { + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + if (timedOut) { + throw new SecretsTimeoutError(`${cmd[0]}: did not exit within ${timeoutMs}ms (keychain prompt pending?)`); + } + return { code, stdout, stderr }; + } finally { + clearTimeout(timer); + } }, fileExists(path) { return existsSync(path); From 81928fd2158b97df5354c360a144d73a59a4e01a Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 18:26:28 -0500 Subject: [PATCH 130/142] e2e: assert additive health/metrics/eventLoop + heartbeat file --- e2e/tests/daemon.test.ts | 128 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 127 insertions(+), 1 deletion(-) diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index a7dc3cdb..dca7bfb1 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from "bun:test"; -import { existsSync, mkdirSync, writeFileSync, readdirSync } from "fs"; +import { existsSync, mkdirSync, writeFileSync, readdirSync, readFileSync } from "fs"; import { join } from "path"; import { createTestHome, rt, RT_BINARY } from "../harness.ts"; @@ -214,3 +214,129 @@ describe("daemon", () => { }); }); }); + +// Additive coverage for the health snapshot (level/reasons + metrics + +// eventLoop) that computeHealth (lib/daemon/health.ts) attaches to every +// status-shaped surface, and for the heartbeat file the loop monitor writes +// alongside it. A live foreground daemon on a per-run free RT_API_PORT, same +// pattern as e2e/tests/events.test.ts and e2e/tests/endpoint.test.ts. +describe("health surfaces", () => { + let home: string; + let cleanup: () => void; + let apiPort = 0; + let daemon: ReturnType; + + /** Grab a free TCP port by binding port 0 and releasing it. */ + function freePort(): number { + const srv = Bun.serve({ port: 0, fetch: () => new Response("") }); + const port = srv.port; + srv.stop(true); + if (!port) throw new Error("failed to allocate a free port"); + return port; + } + + beforeAll(async () => { + apiPort = freePort(); + ({ path: home, cleanup } = createTestHome()); + // `rt daemon status` short-circuits to "not installed" before it ever + // reaches a liveness classification, install first. + await rt(["daemon", "install"], { home }); + const bunDir = join(process.execPath, ".."); + daemon = Bun.spawn([RT_BINARY, "--daemon"], { + env: { + HOME: home, + PATH: `${join(RT_BINARY, "..")}:${bunDir}:/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin`, + TERM: "xterm-256color", + RT_SKIP_SETUP: "1", + CI: "true", + RT_API_PORT: String(apiPort), + }, + stdout: "pipe", + stderr: "pipe", + }); + await waitForSocket(join(home, ".mattstack", "rt", "rt.sock")); + if (daemon.exitCode !== null) { + throw new Error( + `daemon process exited (code ${daemon.exitCode}) right after creating its socket ` + + `(port ${apiPort} collision or daemon boot crash; check the daemon's stderr).`, + ); + } + }, 60_000); + + afterAll(async () => { + try { daemon?.kill(); } catch { /* already gone */ } + await daemon?.exited; + cleanup(); + }); + + function expectHealthLevel(level: unknown) { + expect(["ok", "degraded", "unhealthy"]).toContain(level as string); + } + + test("rt daemon status --json carries health, metrics, and eventLoop, additive to the existing fields", async () => { + const result = await rt(["daemon", "status", "--json"], { home }); + expect(result.exitCode).toBe(0); + const out = JSON.parse(result.stdout); + + // Pre-existing fields still present. + expect(out.ok).toBe(true); + expect(out.state).toBe("running"); + expect(typeof out.data.pid).toBe("number"); + expect(typeof out.data.watchedRepos).toBe("number"); + + // New blocks. + expectHealthLevel(out.data.health.level); + expect(Array.isArray(out.data.health.reasons)).toBe(true); + expect(typeof out.data.metrics.rss).toBe("number"); + expect(typeof out.data.eventLoop.maxLagMs).toBe("number"); + }, 30_000); + + test("GET /api/status (tray:status) carries health, metrics, and eventLoop, additive to the existing fields", async () => { + const res = await fetch(`http://127.0.0.1:${apiPort}/api/status`); + expect(res.status).toBe(200); + const out = (await res.json()) as any; + + // Pre-existing fields still present. + expect(out.ok).toBe(true); + expect(typeof out.data.pid).toBe("number"); + expect(typeof out.data.memoryUsage).toBe("number"); + + // New blocks. + expectHealthLevel(out.data.health.level); + expect(Array.isArray(out.data.health.reasons)).toBe(true); + expect(typeof out.data.metrics.rss).toBe("number"); + expect(typeof out.data.eventLoop.maxLagMs).toBe("number"); + }, 15_000); + + test("ping over rt.sock carries the health level and eventLoop, additive to the existing fields", async () => { + const sockPath = join(home, ".mattstack", "rt", "rt.sock"); + const res = await fetch("http://localhost/ping", { + unix: sockPath, + signal: AbortSignal.timeout(5_000), + } as any); + const out = (await res.json()) as any; + + // Pre-existing fields still present. + expect(out.ok).toBe(true); + expect(typeof out.uptime).toBe("number"); + expect(typeof out.pid).toBe("number"); + + // New blocks. ping's `health` field is the level string itself (not an + // object), unlike status/tray:status where health.level is nested. + expectHealthLevel(out.health); + expect(typeof out.eventLoop.maxLagMs).toBe("number"); + }, 15_000); + + test("a heartbeat file appears under the isolated HOME's RT_DIR within a few seconds", async () => { + const heartbeatPath = join(home, ".mattstack", "rt", "daemon-heartbeat.json"); + const deadline = Date.now() + 5_000; + while (!existsSync(heartbeatPath) && Date.now() < deadline) { + await Bun.sleep(200); + } + expect(existsSync(heartbeatPath)).toBe(true); + + const hb = JSON.parse(readFileSync(heartbeatPath, "utf8")); + expect(typeof hb.at).toBe("number"); + expect(typeof hb.seq).toBe("number"); + }, 10_000); +}); From 16d071de06cd4b111c2c3be0f8e4a85415d85798 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 18:28:55 -0500 Subject: [PATCH 131/142] branch-cache: add composeKey/branchOf/identityOf + get/getByBranch (S069 part 1) Also updates the two other BranchCacheStore implementers (lib/daemon.ts's delegating facade, fake-cache-store.ts's test double) and one exact-key-set test assertion so tsc stays clean; store PK/upsert/delete/gc behavior is unchanged. --- lib/daemon.ts | 2 ++ lib/daemon/__tests__/fake-cache-store.ts | 9 +++++- .../__tests__/freshness-mapping.test.ts | 2 +- lib/state/__tests__/branch-cache.test.ts | 20 +++++++++++- lib/state/branch-cache.ts | 31 ++++++++++++++++++- 5 files changed, 60 insertions(+), 4 deletions(-) diff --git a/lib/daemon.ts b/lib/daemon.ts index 4773a860..f7677c73 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -201,6 +201,8 @@ const cache: BranchCacheStore = { delete: (branch) => openBranchCacheStore().delete(branch), reload: () => openBranchCacheStore().reload(), gc: (repos, maxAgeMs) => openBranchCacheStore().gc(repos, maxAgeMs), + get: (identity, branch) => openBranchCacheStore().get(identity, branch), + getByBranch: (branch) => openBranchCacheStore().getByBranch(branch), }; // Port scan cache, held as a single mutable ref so handler modules can read // fresh values without getters. The port poller mutates it in place. diff --git a/lib/daemon/__tests__/fake-cache-store.ts b/lib/daemon/__tests__/fake-cache-store.ts index b81c6d9f..de44c012 100644 --- a/lib/daemon/__tests__/fake-cache-store.ts +++ b/lib/daemon/__tests__/fake-cache-store.ts @@ -9,6 +9,7 @@ * `openStateDb(tempPath)` instead of this. */ +import { composeKey } from "../../state/branch-cache.ts"; import type { BranchCacheStore, CacheEntry } from "../../state/index.ts"; export function fakeStore(entries: Record = {}): BranchCacheStore { @@ -16,7 +17,13 @@ export function fakeStore(entries: Record = {}): BranchCache entries, put(branch, entry) { entries[branch] = entry; }, delete(branch) { delete entries[branch]; }, - reload() { /* no db behind this fake — the map is the whole store */ }, + reload() { /* no db behind this fake, the map is the whole store */ }, gc() { /* GC is exercised against a real store, not here */ }, + get(identity, branch) { return entries[composeKey(identity, branch)]; }, + getByBranch(branch) { + const suffix = `:${branch}`; + for (const [k, v] of Object.entries(entries)) if (k === branch || k.endsWith(suffix)) return v; + return undefined; + }, }; } diff --git a/lib/daemon/__tests__/freshness-mapping.test.ts b/lib/daemon/__tests__/freshness-mapping.test.ts index 186e18e5..bb65663d 100644 --- a/lib/daemon/__tests__/freshness-mapping.test.ts +++ b/lib/daemon/__tests__/freshness-mapping.test.ts @@ -826,7 +826,7 @@ describe("write-through at updateEntry (RT-48)", () => { const dir = mkdtempSync(join(tmpdir(), "rt-writethrough-api-")); const db = openStateDb(join(dir, "state.db")); const store = getBranchCacheStore(db); - expect(Object.keys(store).sort()).toEqual(["delete", "entries", "gc", "put", "reload"]); + expect(Object.keys(store).sort()).toEqual(["delete", "entries", "gc", "get", "getByBranch", "put", "reload"]); db.close(); }); }); diff --git a/lib/state/__tests__/branch-cache.test.ts b/lib/state/__tests__/branch-cache.test.ts index f58805c1..9b44e03b 100644 --- a/lib/state/__tests__/branch-cache.test.ts +++ b/lib/state/__tests__/branch-cache.test.ts @@ -13,7 +13,25 @@ import { existsSync, mkdtempSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; import { closeStateDb, getStateDb, openStateDb } from "../db.ts"; -import { getBranchCacheStore, rekeyBranchCacheTable, type CacheEntry } from "../branch-cache.ts"; +import { branchOf, composeKey, getBranchCacheStore, identityOf, rekeyBranchCacheTable, type CacheEntry } from "../branch-cache.ts"; + +test("composeKey/branchOf/identityOf round-trip with a serialized identity", () => { + const id = "remote:gitlab.com%2Facme%2Facme-dev"; + const k = composeKey(id, "feature/x"); + expect(k).toBe(`${id}:feature/x`); + expect(branchOf(k)).toBe("feature/x"); + expect(identityOf(k)).toBe(id); +}); +test("bare key (no identity) degrades gracefully", () => { + expect(composeKey(undefined, "main")).toBe("main"); + expect(branchOf("main")).toBe("main"); + expect(identityOf("main")).toBeUndefined(); +}); +test("branch never contains a colon, so lastIndexOf split is unambiguous", () => { + const k = composeKey("path:%2FUsers%2Fdev%2Fscratch", "release"); + expect(branchOf(k)).toBe("release"); + expect(identityOf(k)).toBe("path:%2FUsers%2Fdev%2Fscratch"); +}); let dir: string; diff --git a/lib/state/branch-cache.ts b/lib/state/branch-cache.ts index 72789473..0bc5923f 100644 --- a/lib/state/branch-cache.ts +++ b/lib/state/branch-cache.ts @@ -54,6 +54,21 @@ export function rekeyBranchCacheTable(): Promise { return rekeyTableColumn("branch_cache", "repo"); } +/** state.db keys the branch cache on `${serializedIdentity}:${branch}`. Split + * on the LAST colon: git branch names contain none, serialized identities + * always carry their own (remote:/path:), so this is unambiguous. */ +export function composeKey(identity: string | undefined, branch: string): string { + return identity ? `${identity}:${branch}` : branch; +} +export function branchOf(key: string): string { + const i = key.lastIndexOf(":"); + return i < 0 ? key : key.slice(i + 1); +} +export function identityOf(key: string): string | undefined { + const i = key.lastIndexOf(":"); + return i < 0 ? undefined : key.slice(0, i); +} + export interface BranchCacheStore { /** The live map — ctx.cache-compatible. Same object identity across reload(). */ entries: Record; @@ -61,6 +76,10 @@ export interface BranchCacheStore { put(branch: string, entry: CacheEntry): void; /** Map + row delete, one call. */ delete(branch: string): void; + /** Looks up by composeKey(identity, branch). Store is still bare-keyed this task, so this degrades to a bare-branch lookup until Task 10 flips the PK. */ + get(identity: string | undefined, branch: string): CacheEntry | undefined; + /** Scans for any key ending in `:${branch}` (or the bare branch itself), for callers without an identity yet. */ + getByBranch(branch: string): CacheEntry | undefined; /** Rebuilds `entries` in place from the db (replaces loadCache-from-file). */ reload(): void; /** @@ -183,9 +202,19 @@ function createStore(db: Database): BranchCacheStore { }, { op: "gc", count: toDelete.length }); } + function get(identity: string | undefined, branch: string): CacheEntry | undefined { + return entries[composeKey(identity, branch)]; + } + + function getByBranch(branch: string): CacheEntry | undefined { + const suffix = `:${branch}`; + for (const [k, v] of Object.entries(entries)) if (k === branch || k.endsWith(suffix)) return v; + return undefined; + } + reload(); - return { entries, put, delete: del, reload, gc }; + return { entries, put, delete: del, reload, gc, get, getByBranch }; } let singletonStore: BranchCacheStore | null = null; From 1186b79d6cfd0bf694b811896386e9445757d671 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 18:29:53 -0500 Subject: [PATCH 132/142] registry.test: add rt.logLevel to migrated-key fixtures (24 keys) --- .../rt-client/src/settings/__tests__/registry.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/rt-client/src/settings/__tests__/registry.test.ts b/packages/rt-client/src/settings/__tests__/registry.test.ts index e84146b4..3d24dc81 100644 --- a/packages/rt-client/src/settings/__tests__/registry.test.ts +++ b/packages/rt-client/src/settings/__tests__/registry.test.ts @@ -51,7 +51,7 @@ describe("settings/registry", () => { } }); - test("exactly 23 keys are migrated:true", () => { + test("exactly 24 keys are migrated:true", () => { const migrated = allDefs().filter((d) => d.migrated); expect(migrated.map((d) => d.key).sort()).toEqual( @@ -59,7 +59,7 @@ describe("settings/registry", () => { "rt.intercepts", "rt.repoIdentityOverrides", "rt.repoRoots", "rt.roles", "rt.worktrees", "rt.notifications", "rt.cron", "rt.repoTracking", "rt.runsPruneDays", "rt.runaway", "rt.workspacePrefs", "rt.sync", "rt.branchNaming", "rt.variations", "rt.presets", "rt.dopplerTemplate", - "rt.homeSnapshot", "rt.worktreeApp", "rt.sdmEnrichment", "rt.logRetentionDays", "rt.integrations", "rt.hooks", + "rt.homeSnapshot", "rt.worktreeApp", "rt.sdmEnrichment", "rt.logRetentionDays", "rt.logLevel", "rt.integrations", "rt.hooks", "rt.apiPort", ].sort(), ); @@ -198,13 +198,13 @@ describe("settings/registry", () => { expect(def?.merge).toBe("replace"); }); - test("has exactly the 23 migrated:true keys and the 43 suite keys", () => { + test("has exactly the 24 migrated:true keys and the 43 suite keys", () => { const migratedFalseKeys: string[] = []; const migratedTrueKeys = [ "rt.roles", "rt.intercepts", "rt.worktrees", "rt.repoIdentityOverrides", "rt.repoRoots", "rt.notifications", "rt.cron", "rt.repoTracking", "rt.runsPruneDays", "rt.runaway", "rt.workspacePrefs", "rt.sync", "rt.branchNaming", "rt.variations", "rt.presets", "rt.dopplerTemplate", - "rt.homeSnapshot", "rt.worktreeApp", "rt.sdmEnrichment", "rt.logRetentionDays", "rt.integrations", "rt.hooks", + "rt.homeSnapshot", "rt.worktreeApp", "rt.sdmEnrichment", "rt.logRetentionDays", "rt.logLevel", "rt.integrations", "rt.hooks", "rt.apiPort", ]; const suiteKeys = [ From 23a6d3d620bba23fa5c2aee47696579c8ce33961 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 18:34:29 -0500 Subject: [PATCH 133/142] branch-cache: make get/getByBranch free functions; restore daemon.ts fence (S069 part 1) get/getByBranch on the BranchCacheStore interface forced an edit to lib/daemon.ts's cache facade, which is under a write fence owned by the p2-health lane. Drop get entirely (Task 10 consumers will use entries[composeKey(identity, branch)] directly); reshape getByBranch into a free function over an entries map instead of a store method, so the interface -- and daemon.ts's facade -- do not change. Reverts the daemon.ts and fake-cache-store.ts edits from the prior commit. --- lib/daemon.ts | 2 -- lib/daemon/__tests__/fake-cache-store.ts | 9 +----- .../__tests__/freshness-mapping.test.ts | 2 +- lib/state/__tests__/branch-cache.test.ts | 28 ++++++++++++++++++- lib/state/branch-cache.ts | 28 +++++++++---------- 5 files changed, 42 insertions(+), 27 deletions(-) diff --git a/lib/daemon.ts b/lib/daemon.ts index f7677c73..4773a860 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -201,8 +201,6 @@ const cache: BranchCacheStore = { delete: (branch) => openBranchCacheStore().delete(branch), reload: () => openBranchCacheStore().reload(), gc: (repos, maxAgeMs) => openBranchCacheStore().gc(repos, maxAgeMs), - get: (identity, branch) => openBranchCacheStore().get(identity, branch), - getByBranch: (branch) => openBranchCacheStore().getByBranch(branch), }; // Port scan cache, held as a single mutable ref so handler modules can read // fresh values without getters. The port poller mutates it in place. diff --git a/lib/daemon/__tests__/fake-cache-store.ts b/lib/daemon/__tests__/fake-cache-store.ts index de44c012..b81c6d9f 100644 --- a/lib/daemon/__tests__/fake-cache-store.ts +++ b/lib/daemon/__tests__/fake-cache-store.ts @@ -9,7 +9,6 @@ * `openStateDb(tempPath)` instead of this. */ -import { composeKey } from "../../state/branch-cache.ts"; import type { BranchCacheStore, CacheEntry } from "../../state/index.ts"; export function fakeStore(entries: Record = {}): BranchCacheStore { @@ -17,13 +16,7 @@ export function fakeStore(entries: Record = {}): BranchCache entries, put(branch, entry) { entries[branch] = entry; }, delete(branch) { delete entries[branch]; }, - reload() { /* no db behind this fake, the map is the whole store */ }, + reload() { /* no db behind this fake — the map is the whole store */ }, gc() { /* GC is exercised against a real store, not here */ }, - get(identity, branch) { return entries[composeKey(identity, branch)]; }, - getByBranch(branch) { - const suffix = `:${branch}`; - for (const [k, v] of Object.entries(entries)) if (k === branch || k.endsWith(suffix)) return v; - return undefined; - }, }; } diff --git a/lib/daemon/__tests__/freshness-mapping.test.ts b/lib/daemon/__tests__/freshness-mapping.test.ts index bb65663d..186e18e5 100644 --- a/lib/daemon/__tests__/freshness-mapping.test.ts +++ b/lib/daemon/__tests__/freshness-mapping.test.ts @@ -826,7 +826,7 @@ describe("write-through at updateEntry (RT-48)", () => { const dir = mkdtempSync(join(tmpdir(), "rt-writethrough-api-")); const db = openStateDb(join(dir, "state.db")); const store = getBranchCacheStore(db); - expect(Object.keys(store).sort()).toEqual(["delete", "entries", "gc", "get", "getByBranch", "put", "reload"]); + expect(Object.keys(store).sort()).toEqual(["delete", "entries", "gc", "put", "reload"]); db.close(); }); }); diff --git a/lib/state/__tests__/branch-cache.test.ts b/lib/state/__tests__/branch-cache.test.ts index 9b44e03b..98cfadd3 100644 --- a/lib/state/__tests__/branch-cache.test.ts +++ b/lib/state/__tests__/branch-cache.test.ts @@ -13,7 +13,7 @@ import { existsSync, mkdtempSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; import { closeStateDb, getStateDb, openStateDb } from "../db.ts"; -import { branchOf, composeKey, getBranchCacheStore, identityOf, rekeyBranchCacheTable, type CacheEntry } from "../branch-cache.ts"; +import { branchOf, composeKey, getBranchCacheStore, getByBranch, identityOf, rekeyBranchCacheTable, type CacheEntry } from "../branch-cache.ts"; test("composeKey/branchOf/identityOf round-trip with a serialized identity", () => { const id = "remote:gitlab.com%2Facme%2Facme-dev"; @@ -33,6 +33,32 @@ test("branch never contains a colon, so lastIndexOf split is unambiguous", () => expect(identityOf(k)).toBe("path:%2FUsers%2Fdev%2Fscratch"); }); +describe("getByBranch — free function over an entries map", () => { + function makeCacheEntry(linearId: string): CacheEntry { + return { ticket: null, linearId, mr: null, fetchedAt: Date.now() }; + } + + test("exact bare-key hit", () => { + const entries: Record = { main: makeCacheEntry("bare") }; + expect(getByBranch(entries, "main")?.linearId).toBe("bare"); + }); + + test("suffix hit across two repos sharing the same branch name picks a match, not a false negative", () => { + const entries: Record = { + "remote:gitlab.com%2Facme%2Frepo-a:feature/x": makeCacheEntry("repo-a"), + "remote:gitlab.com%2Facme%2Frepo-b:feature/x": makeCacheEntry("repo-b"), + }; + const hit = getByBranch(entries, "feature/x"); + expect(hit).toBeDefined(); + expect(["repo-a", "repo-b"]).toContain(hit!.linearId); + }); + + test("miss returns undefined", () => { + const entries: Record = { main: makeCacheEntry("bare") }; + expect(getByBranch(entries, "nonexistent")).toBeUndefined(); + }); +}); + let dir: string; beforeEach(() => { diff --git a/lib/state/branch-cache.ts b/lib/state/branch-cache.ts index 0bc5923f..f76801b1 100644 --- a/lib/state/branch-cache.ts +++ b/lib/state/branch-cache.ts @@ -69,6 +69,18 @@ export function identityOf(key: string): string | undefined { return i < 0 ? undefined : key.slice(0, i); } +/** + * Free function, not a store method, on purpose: `BranchCacheStore` is a + * structural interface with implementers outside this module's ownership + * (lib/daemon.ts's facade), so growing the interface forces edits there too. + * Scans for any key ending in `:${branch}` (or the bare branch itself). + */ +export function getByBranch(entries: Record, branch: string): CacheEntry | undefined { + const suffix = `:${branch}`; + for (const [k, v] of Object.entries(entries)) if (k === branch || k.endsWith(suffix)) return v; + return undefined; +} + export interface BranchCacheStore { /** The live map — ctx.cache-compatible. Same object identity across reload(). */ entries: Record; @@ -76,10 +88,6 @@ export interface BranchCacheStore { put(branch: string, entry: CacheEntry): void; /** Map + row delete, one call. */ delete(branch: string): void; - /** Looks up by composeKey(identity, branch). Store is still bare-keyed this task, so this degrades to a bare-branch lookup until Task 10 flips the PK. */ - get(identity: string | undefined, branch: string): CacheEntry | undefined; - /** Scans for any key ending in `:${branch}` (or the bare branch itself), for callers without an identity yet. */ - getByBranch(branch: string): CacheEntry | undefined; /** Rebuilds `entries` in place from the db (replaces loadCache-from-file). */ reload(): void; /** @@ -202,19 +210,9 @@ function createStore(db: Database): BranchCacheStore { }, { op: "gc", count: toDelete.length }); } - function get(identity: string | undefined, branch: string): CacheEntry | undefined { - return entries[composeKey(identity, branch)]; - } - - function getByBranch(branch: string): CacheEntry | undefined { - const suffix = `:${branch}`; - for (const [k, v] of Object.entries(entries)) if (k === branch || k.endsWith(suffix)) return v; - return undefined; - } - reload(); - return { entries, put, delete: del, reload, gc, get, getByBranch }; + return { entries, put, delete: del, reload, gc }; } let singletonStore: BranchCacheStore | null = null; From d8de94f3e834a8336fc63d95eabe0d526a2b13d6 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 18:46:57 -0500 Subject: [PATCH 134/142] fix: windowed maxLag recovery + named loop-lag threshold + review fix-now items Applies the final whole-branch review's approved fix wave for the rt daemon health feature: maxLagMs now decays as a windowed max instead of a lifetime high-water mark (and currentlyStalled drops its redundant maxLagMs OR-leg), the health degraded threshold for event-loop lag is named instead of hardcoded, one branch-introduced em dash is fixed, a process-citation is dropped from a comment, the daemon log-level resolver validates against pino's level set, and the heartbeat reader now shape-guards against a partial-but-valid JSON object. --- commands/daemon.ts | 2 +- lib/__tests__/daemon-logger-level.test.ts | 6 ++++++ lib/daemon-logger.ts | 24 +++++++++++++-------- lib/daemon/__tests__/health.test.ts | 16 ++++++++++++++ lib/daemon/__tests__/heartbeat-file.test.ts | 6 ++++++ lib/daemon/__tests__/loop-monitor.test.ts | 11 ++++++++++ lib/daemon/health.ts | 3 ++- lib/daemon/heartbeat-file.ts | 6 +++++- lib/daemon/loop-monitor.ts | 22 +++++++++++++------ lib/daemon/unknown-command.ts | 2 +- 10 files changed, 78 insertions(+), 20 deletions(-) diff --git a/commands/daemon.ts b/commands/daemon.ts index 6f3c79a4..b4f604d9 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -532,7 +532,7 @@ export function statusLines(verdict: DaemonStatusVerdict, now: number): string[] lines.push(` ${dim}status command failed: ${verdict.detail ?? "unknown error"}${reset}`); } else if (verdict.eventLoop && verdict.eventLoop.maxLagMs > 0) { const el = verdict.eventLoop; - lines.push(` ${dim}answers ping, status timed out — event loop maxLag ${el.maxLagMs}ms${el.lastStallCmd ? ` (last stall in ${el.lastStallCmd})` : ""}${reset}`); + lines.push(` ${dim}answers ping, status timed out: event loop maxLag ${el.maxLagMs}ms${el.lastStallCmd ? ` (last stall in ${el.lastStallCmd})` : ""}${reset}`); } else { lines.push(` ${dim}answers ping, but status timed out — likely mid-sync${reset}`); } diff --git a/lib/__tests__/daemon-logger-level.test.ts b/lib/__tests__/daemon-logger-level.test.ts index c461a639..feb3564d 100644 --- a/lib/__tests__/daemon-logger-level.test.ts +++ b/lib/__tests__/daemon-logger-level.test.ts @@ -17,6 +17,12 @@ test("a thrown setting read falls back to info instead of propagating", () => { }), ).toBe("info"); }); +test("an unknown level falls back to info instead of reaching pino", () => { + expect(resolveDaemonLogLevel("verbose", () => undefined)).toBe("info"); +}); +test("a valid level still passes through", () => { + expect(resolveDaemonLogLevel("debug", () => undefined)).toBe("debug"); +}); test("a panic-looking stderr line is escalated; ordinary noise is not", () => { expect(isPanicLine("panic: runtime error")).toBe(true); expect(isPanicLine("Uncaught Error: boom")).toBe(true); diff --git a/lib/daemon-logger.ts b/lib/daemon-logger.ts index 3950de69..a8700c86 100644 --- a/lib/daemon-logger.ts +++ b/lib/daemon-logger.ts @@ -52,25 +52,31 @@ export interface CreateOptions { level?: pino.LevelWithSilent; } +const VALID_LOG_LEVELS = new Set(["trace", "debug", "info", "warn", "error", "fatal", "silent"]); + /** * Resolves the daemon's pino level: RT_LOG_LEVEL env, then the `rt.logLevel` * setting, then "info". The setting read is try/catch-guarded because the * `rt.logLevel` registry key may not exist yet (added in a later task), and - * the resolver may also run pre-boot; this must never throw. + * the resolver may also run pre-boot; this must never throw. The resolved + * value is validated against pino's level set: an unrecognized value (a typo + * like "warning") must not reach pino's constructor, which throws on it. */ export function resolveDaemonLogLevel( env: string | undefined, fromSetting: () => string | undefined, ): string { - if (env) return env; - try { - const v = fromSetting(); - if (v) return v; - } catch { - // Setting unavailable (unknown key pre-registration, or resolver not - // ready yet)... fall through to the "info" default below. + let resolved = env; + if (!resolved) { + try { + resolved = fromSetting(); + } catch { + // Setting unavailable (unknown key pre-registration, or resolver not + // ready yet)... fall through to the "info" default below. + } } - return "info"; + if (!resolved || !VALID_LOG_LEVELS.has(resolved)) return "info"; + return resolved; } const PANIC_PREFIXES = ["panic:", "fatal error:", "Uncaught ", "UnhandledPromiseRejection"]; diff --git a/lib/daemon/__tests__/health.test.ts b/lib/daemon/__tests__/health.test.ts index 387ef901..b8a5f52c 100644 --- a/lib/daemon/__tests__/health.test.ts +++ b/lib/daemon/__tests__/health.test.ts @@ -73,3 +73,19 @@ test("stale refresh (older than 2 intervals) is degraded", () => { i.refresh = { lastSuccessAt: i.now - 11 * 60_000, failedRepos: 0, enrichErrors: 0 }; expect(computeHealth(i).level).toBe("degraded"); }); + +test("event-loop lag over the named threshold flips degraded", () => { + const i = base(); + i.eventLoop.maxLagMs = 600; + const h = computeHealth(i); + expect(h.level).toBe("degraded"); + expect(h.reasons.some((r) => r.startsWith("event-loop:"))).toBe(true); +}); + +test("event-loop lag under the named threshold stays ok", () => { + const i = base(); + i.eventLoop.maxLagMs = 400; + const h = computeHealth(i); + expect(h.level).toBe("ok"); + expect(h.reasons).toEqual([]); +}); diff --git a/lib/daemon/__tests__/heartbeat-file.test.ts b/lib/daemon/__tests__/heartbeat-file.test.ts index 1a4fc93d..7e3b0b67 100644 --- a/lib/daemon/__tests__/heartbeat-file.test.ts +++ b/lib/daemon/__tests__/heartbeat-file.test.ts @@ -21,6 +21,12 @@ test("corrupt file reads as null", () => { expect(readHeartbeat(dir)).toBeNull(); }); +test("a partial-but-valid-JSON object (missing `at`) reads as null", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + writeFileSync(join(dir, "daemon-heartbeat.json"), JSON.stringify({ seq: 1 })); + expect(readHeartbeat(dir)).toBeNull(); +}); + test("a second write overwrites atomically", () => { const dir = mkdtempSync(join(tmpdir(), "hb-")); writeHeartbeat(dir, { at: 1, seq: 1 }); diff --git a/lib/daemon/__tests__/loop-monitor.test.ts b/lib/daemon/__tests__/loop-monitor.test.ts index 7ca34211..b4378b14 100644 --- a/lib/daemon/__tests__/loop-monitor.test.ts +++ b/lib/daemon/__tests__/loop-monitor.test.ts @@ -39,3 +39,14 @@ test("maxLagMs is a high-water mark", () => { applyTick(s, 1550, 1600, null, OPTS, () => {}); expect(s.maxLagMs).toBe(300); }); + +test("maxLagMs decays once no bigger spike lands within the window", () => { + const s: LoopStats = newLoopStats(); + applyTick(s, 1000, 1800, null, OPTS, () => {}); // drift 800, maxLagMs -> 800 at now=1800 + expect(s.maxLagMs).toBe(800); + // OPTS has no maxLagWindowMs, so it falls back to stallRecentMs (10_000). + // now=12000 is 10200ms past maxLagAt(1800), past the window, so an + // on-time tick (drift 0) decays maxLagMs to the current lagMs. + applyTick(s, 12000, 12000, null, OPTS, () => {}); + expect(s.maxLagMs).toBe(0); +}); diff --git a/lib/daemon/health.ts b/lib/daemon/health.ts index 57c687a3..dede9f79 100644 --- a/lib/daemon/health.ts +++ b/lib/daemon/health.ts @@ -13,6 +13,7 @@ export const HEALTH_THRESHOLDS = { diskHardFloorBytes: 100 * 1024 * 1024, restartsPerHourUnhealthy: 5, recoveredErrorRate: 10, + loopLagDegradedMs: 500, } as const; export interface HealthMetrics { @@ -93,7 +94,7 @@ export function computeHealth(i: HealthInputs): HealthSnapshot { if (i.rssBaseline && i.mem.rss > i.rssBaseline.rss * (1 + T.rssGrowthPct / 100)) { degraded.push(`memory: rss grew >${T.rssGrowthPct}% in the last hour`); } - if (i.eventLoop.maxLagMs > 500) degraded.push(`event-loop: lag ${i.eventLoop.maxLagMs}ms`); + if (i.eventLoop.maxLagMs > T.loopLagDegradedMs) degraded.push(`event-loop: lag ${i.eventLoop.maxLagMs}ms`); if (i.recoveredErrorRateLastWindow > T.recoveredErrorRate) { degraded.push(`errors: ${i.recoveredErrorRateLastWindow} recovered in 5min`); } diff --git a/lib/daemon/heartbeat-file.ts b/lib/daemon/heartbeat-file.ts index 77206798..62224d0a 100644 --- a/lib/daemon/heartbeat-file.ts +++ b/lib/daemon/heartbeat-file.ts @@ -31,7 +31,11 @@ export function readHeartbeat(dir: string): Heartbeat | null { try { const p = heartbeatPath(dir); if (!existsSync(p)) return null; - return JSON.parse(readFileSync(p, "utf8")) as Heartbeat; + const parsed = JSON.parse(readFileSync(p, "utf8")); + if (typeof parsed?.at === "number" && typeof parsed?.seq === "number") { + return parsed as Heartbeat; + } + return null; } catch { return null; } diff --git a/lib/daemon/loop-monitor.ts b/lib/daemon/loop-monitor.ts index 78f03046..45e6cc1d 100644 --- a/lib/daemon/loop-monitor.ts +++ b/lib/daemon/loop-monitor.ts @@ -2,14 +2,15 @@ * Event-loop drift monitor. A ~250ms unref'd interval measures how late each * tick fires vs its scheduled time; a large drift means the loop was blocked. * The interval callback is created once and the stats object is preallocated, - * so the hot tick allocates nothing. Every ~2s it also writes the heartbeat - * file the cross-process classifier reads. + * so the hot tick allocates nothing. Every ~2s it also invokes an + * `onHeartbeat` callback (the daemon writes the heartbeat file from that). */ import type { Logger } from "pino"; export interface LoopStats { lagMs: number; maxLagMs: number; + maxLagAt: number; stalls: number; lastStallAt: number | null; lastStallCmd: string | null; @@ -17,13 +18,14 @@ export interface LoopStats { } export function newLoopStats(): LoopStats { - return { lagMs: 0, maxLagMs: 0, stalls: 0, lastStallAt: null, lastStallCmd: null, currentlyStalled: false }; + return { lagMs: 0, maxLagMs: 0, maxLagAt: 0, stalls: 0, lastStallAt: null, lastStallCmd: null, currentlyStalled: false }; } interface TickOpts { stallLogMs: number; stallUnhealthyMs: number; stallRecentMs: number; + maxLagWindowMs?: number; } /** Pure: fold one tick into `stats`. `onStall` fires once per stall (warn sink). */ @@ -37,7 +39,11 @@ export function applyTick( ): void { const drift = now - expected; stats.lagMs = drift > 0 ? drift : 0; - if (stats.lagMs > stats.maxLagMs) stats.maxLagMs = stats.lagMs; + const maxLagWindowMs = opts.maxLagWindowMs ?? opts.stallRecentMs; + if (stats.lagMs > stats.maxLagMs || now - stats.maxLagAt > maxLagWindowMs) { + stats.maxLagMs = stats.lagMs; + stats.maxLagAt = now; + } if (drift > opts.stallLogMs) { stats.stalls += 1; stats.lastStallAt = now; @@ -47,7 +53,7 @@ export function applyTick( stats.currentlyStalled = stats.lastStallAt !== null && now - stats.lastStallAt <= opts.stallRecentMs && - (drift > opts.stallUnhealthyMs || stats.maxLagMs > opts.stallUnhealthyMs); + drift > opts.stallUnhealthyMs; } export interface LoopMonitorOpts { @@ -56,6 +62,7 @@ export interface LoopMonitorOpts { stallLogMs?: number; stallUnhealthyMs?: number; stallRecentMs?: number; + maxLagWindowMs?: number; heartbeatMs?: number; currentCmd: () => string | null; onHeartbeat: (at: number, seq: number) => void; @@ -69,6 +76,7 @@ export function startLoopMonitor( stallLogMs: opts.stallLogMs ?? 1000, stallUnhealthyMs: opts.stallUnhealthyMs ?? 2000, stallRecentMs: opts.stallRecentMs ?? 10_000, + maxLagWindowMs: opts.maxLagWindowMs ?? (opts.stallRecentMs ?? 10_000), }; const heartbeatMs = opts.heartbeatMs ?? 2000; const stats = newLoopStats(); @@ -77,8 +85,8 @@ export function startLoopMonitor( let seq = 0; let warnedThisStall = false; - // Hoisted once (allocation-free ruling): the tick must not build a fresh - // closure every 250ms. onStall closes over warnedThisStall by reference. + // Hoisted once so the hot tick allocates nothing: it must not build a + // fresh closure every 250ms. onStall closes over warnedThisStall by reference. const onStall = (drift: number, cmd: string | null): void => { if (!warnedThisStall) { opts.log.warn({ driftMs: drift, cmd }, "event loop stalled"); diff --git a/lib/daemon/unknown-command.ts b/lib/daemon/unknown-command.ts index 33306978..139fc68b 100644 --- a/lib/daemon/unknown-command.ts +++ b/lib/daemon/unknown-command.ts @@ -2,7 +2,7 @@ * Reply shape for a command name routeCommand's switch doesn't recognize. * Carries the daemon's own version so a caller can tell version skew (the * daemon is older than the CLI/client that sent the command) from a genuine - * typo (findings R021, R008). + * typo. */ export function unknownCommandReply(cmd: string, version: string) { return { From 3ac78f2e8992ad6d8f90ac847792cf33c3554e30 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 19:24:34 -0500 Subject: [PATCH 135/142] fix: drop em dashes in arch-row comment and test describe (no-em-dashes) Co-Authored-By: Claude Opus 4.8 (1M context) --- lib/setup/__tests__/validators-mac.test.ts | 2 +- lib/setup/validators/mac.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/setup/__tests__/validators-mac.test.ts b/lib/setup/__tests__/validators-mac.test.ts index 01a699ab..1b81998d 100644 --- a/lib/setup/__tests__/validators-mac.test.ts +++ b/lib/setup/__tests__/validators-mac.test.ts @@ -75,7 +75,7 @@ describe("macRows — tool.clt", () => { }); }); -describe("macRows — tool.arch", () => { +describe("macRows: tool.arch", () => { test("arm64 -> ready", async () => { const execScript: ExecScript = (argv) => (argv[0] === "uname" ? ok("arm64\n") : ok()); const r = await pickRow(macRows(fakeProbes({ exec: execScript })), "tool.arch"); diff --git a/lib/setup/validators/mac.ts b/lib/setup/validators/mac.ts index 9c920d97..dad7b44e 100644 --- a/lib/setup/validators/mac.ts +++ b/lib/setup/validators/mac.ts @@ -51,7 +51,7 @@ async function archRow(p: Probes): Promise { const arch = res.stdout.trim(); // Same honesty ruling as macosVersionRow: a probe that couldn't run reports - // "error", not "invalid" — only a definite non-arm64 result is invalid. + // "error", not "invalid": only a definite non-arm64 result is invalid. if (res.code !== 0 || !arch) { return row({ ...base, status: "error", detail: "Could not determine your processor" }); } From e966d030a037e0ddebde044837bc7a0feb24506c Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 19:24:34 -0500 Subject: [PATCH 136/142] branch-cache: flip to composite ${identity}:${branch} key; scope all consumers (S069) Consumers updated: store put (keys off entry.repoName), enrich (cold-start sets identity from remoteUrl), notifier (composite fired-state), worktree-reconciler (branchOf/mrKey), freshness (composeKey lookups), handlers/cache (bare-branch read contract + optional repoIdentity), handlers/system-processes and handlers/worktree (repo-scoped lookups), status/data (branchOf display). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../status/__tests__/status-fallback.test.ts | 24 ++++ commands/status/data.ts | 3 +- lib/__tests__/enrich-cache-identity.test.ts | 82 +++++++++++++ lib/__tests__/notifier-fired-hygiene.test.ts | 29 +++++ .../__tests__/cache-read-bare-branch.test.ts | 70 +++++++++++ lib/daemon/__tests__/cache-refresh-gc.test.ts | 9 +- .../__tests__/discussions-semantics.test.ts | 3 +- lib/daemon/__tests__/fake-cache-store.ts | 6 +- .../__tests__/freshness-mapping.test.ts | 114 +++++++++++------- .../system-processes-handlers.test.ts | 9 +- .../__tests__/worktree-reconciler.test.ts | 22 ++++ lib/daemon/freshness.ts | 19 +-- lib/daemon/handlers/cache.ts | 40 ++++-- lib/daemon/handlers/system-processes.ts | 5 +- lib/daemon/handlers/worktree.ts | 20 ++- lib/daemon/worktree-reconciler.ts | 26 ++-- lib/enrich.ts | 24 +++- lib/notifier.ts | 9 +- lib/state/__tests__/branch-cache.test.ts | 107 +++++++++++----- lib/state/branch-cache.ts | 11 +- lib/worktree/__tests__/dispose.test.ts | 16 ++- 21 files changed, 524 insertions(+), 124 deletions(-) create mode 100644 lib/__tests__/enrich-cache-identity.test.ts create mode 100644 lib/daemon/__tests__/cache-read-bare-branch.test.ts diff --git a/commands/status/__tests__/status-fallback.test.ts b/commands/status/__tests__/status-fallback.test.ts index a95027b6..63135681 100644 --- a/commands/status/__tests__/status-fallback.test.ts +++ b/commands/status/__tests__/status-fallback.test.ts @@ -87,6 +87,30 @@ describe("rt status fallback (no daemon)", () => { expect(existsSync(rtDirPath) ? readdirSync(rtDirPath) : []).toEqual([]); }); + test("S069/Task 10: two repos sharing a branch name display as a bare branch, not a composite key", async () => { + const dbPath = stateDbPath(); + mkdirSync(join(home, ".mattstack", "rt"), { recursive: true }); + const db = openStateDb(dbPath); + const store = getBranchCacheStore(db); + store.put("shared-branch", { + ticket: null, linearId: "", mr: null, fetchedAt: 1, repoName: "repo-a", + }); + store.put("shared-branch", { + ticket: null, linearId: "", mr: null, fetchedAt: 2, repoName: "repo-b", + }); + db.close(); + + const data = await fetchStatusData(); + + // Both rows are real, distinct composite-keyed rows in state.db...the + // dashboard's flat bare-branch dict can only show one, never a raw + // composite key, and never crashes reconciling the two. + expect(Object.keys(data.branches)).toEqual(["shared-branch"]); + const winner = data.branches["shared-branch"]!.repoName; + expect(winner).toBeDefined(); + expect(["repo-a", "repo-b"]).toContain(winner!); + }); + test("an empty branch_cache table serves an empty dashboard", async () => { mkdirSync(join(home, ".mattstack", "rt"), { recursive: true }); openStateDb(stateDbPath()).close(); diff --git a/commands/status/data.ts b/commands/status/data.ts index a58ccad6..6106ff7c 100644 --- a/commands/status/data.ts +++ b/commands/status/data.ts @@ -10,6 +10,7 @@ import type { CacheEntry, StatusData } from "./types.ts"; import type { PortEntry } from "../../lib/port-scanner.ts"; +import { branchOf } from "../../lib/state/branch-cache.ts"; interface BranchCacheRow { branch: string; @@ -56,7 +57,7 @@ async function readBranchesFromStateDb(): Promise> { // with looser optionality on the ticket fields (`stateName?: string` // vs `string | null`). Same data that used to arrive here as parsed // JSON out of branch-cache.json. - branches[row.branch] = { + branches[branchOf(row.branch)] = { ticket: row.ticket !== null ? (JSON.parse(row.ticket) as CacheEntry["ticket"]) : null, linearId: row.linear_id, mr: row.mr !== null ? (JSON.parse(row.mr) as CacheEntry["mr"]) : null, diff --git a/lib/__tests__/enrich-cache-identity.test.ts b/lib/__tests__/enrich-cache-identity.test.ts new file mode 100644 index 00000000..9dedb7ea --- /dev/null +++ b/lib/__tests__/enrich-cache-identity.test.ts @@ -0,0 +1,82 @@ +/** + * lib/enrich.ts: cold-start writes are keyed by the composite + * `${identity}:${branch}` (S069/Task 10), not the bare branch. Without this, + * two repos enriching a same-named branch overwrite each other's cache row. + * + * `loadSecrets` is mocked to report no API keys, so `fetchAndCache` never + * reaches GitLab/Linear; it still writes a cache row per branch (mr/ticket + * null), which is all this test needs to observe the key format. + */ +import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from "bun:test"; +import { mkdtempSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import * as linearModule from "../linear.ts"; +import { enrichBranches } from "../enrich.ts"; +import { closeStateDb, getBranchCacheStore } from "../state/index.ts"; +import { composeKey } from "../state/branch-cache.ts"; + +let home: string; +let realHome: string | undefined; + +beforeEach(() => { + realHome = process.env.HOME; + home = mkdtempSync(join(tmpdir(), "rt-enrich-identity-")); + process.env.HOME = home; + spyOn(linearModule, "loadSecrets").mockResolvedValue({ linearApiKey: undefined, gitlabToken: undefined } as any); +}); + +afterEach(() => { + mock.restore(); + closeStateDb(); + process.env.HOME = realHome; + rmSync(home, { recursive: true, force: true }); +}); + +describe("enrichBranches cold-start: repoName/key is the serialized remote identity", () => { + test("writes the cache row under composeKey(identity, branch), not the bare branch", async () => { + await enrichBranches( + [{ path: "/tmp/repo-a", branch: "main" }], + "git@gitlab.com:acme/repo-a.git", + { silent: true }, + ); + + const entries = getBranchCacheStore().entries; + const identityKeys = Object.keys(entries).filter((k) => k.endsWith(":main")); + expect(identityKeys.length).toBe(1); + expect(entries["main"]).toBeUndefined(); // never the bare branch + expect(entries[identityKeys[0]!]?.repoName).toBe(identityKeys[0]!.replace(/:main$/, "")); + }); + + test("two repos enriching the same branch name coexist (no collision)", async () => { + await enrichBranches( + [{ path: "/tmp/repo-a", branch: "main" }], + "git@gitlab.com:acme/repo-a.git", + { silent: true }, + ); + await enrichBranches( + [{ path: "/tmp/repo-b", branch: "main" }], + "git@gitlab.com:acme/repo-b.git", + { silent: true }, + ); + + const entries = getBranchCacheStore().entries; + const keyA = composeKey("remote:gitlab.com%2Facme%2Frepo-a", "main"); + const keyB = composeKey("remote:gitlab.com%2Facme%2Frepo-b", "main"); + expect(entries[keyA]).toBeDefined(); + expect(entries[keyB]).toBeDefined(); + expect(entries[keyA]).not.toBe(entries[keyB]); + }); + + test("no remote (path-only repo) degrades to a bare-branch key", async () => { + await enrichBranches( + [{ path: "/tmp/repo-local", branch: "scratch" }], + undefined, + { silent: true }, + ); + + const entries = getBranchCacheStore().entries; + expect(entries["scratch"]).toBeDefined(); + expect(entries["scratch"]?.repoName).toBeUndefined(); + }); +}); diff --git a/lib/__tests__/notifier-fired-hygiene.test.ts b/lib/__tests__/notifier-fired-hygiene.test.ts index 88fb6615..6bca3089 100644 --- a/lib/__tests__/notifier-fired-hygiene.test.ts +++ b/lib/__tests__/notifier-fired-hygiene.test.ts @@ -18,6 +18,7 @@ import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"; import * as notifierModule from "../notifier.ts"; import { closeStateDb, getNotifierStateBlob } from "../state/index.ts"; +import { composeKey } from "../state/branch-cache.ts"; interface NotifierStateShape { branches: Record; @@ -102,6 +103,34 @@ describe("checkAndNotify fired-ledger hygiene", () => { }); }); +describe("checkAndNotify: composite-key repo scoping (S069/Task 10)", () => { + test("evicting one repo's branch does not prune the other repo's fired key", () => { + spyOn(notifierModule, "notify").mockImplementation(() => {}); + + const keyA = composeKey("repo-a", "main"); + const keyB = composeKey("repo-b", "main"); + + // Cycle 1: both repos' "main" baseline (same bare branch, distinct + // composite keys...the collision this task fixes). + notifierModule.checkAndNotify({ [keyA]: mrEntry("running"), [keyB]: mrEntry("running") }, undefined, 123); + // Cycle 2: repo-a's pipeline fails; repo-b's stays running. Fires and + // persists repo-a's pipeline:failed key, keyed by the FULL composite key. + notifierModule.checkAndNotify({ [keyA]: mrEntry("failed"), [keyB]: mrEntry("running") }, undefined, 123); + + const firedKeyA = notifierModule.__test__.firedKey("pipeline:failed", keyA); + expect(readState().fired).toContain(firedKeyA); + + // Cycle 3: repo-a's branch is evicted (GC, or just absent this cycle); + // repo-b's "main" is still present under its own composite key. + notifierModule.checkAndNotify({ [keyB]: mrEntry("running") }, undefined, 123); + + const state = readState(); + expect(state.fired).not.toContain(firedKeyA); + // repo-b's own baseline snapshot survives untouched by repo-a's eviction. + expect(state.branches[keyB]).toBeDefined(); + }); +}); + describe("pruneFiredForEvictedBranches (unit)", () => { test("keeps only keys reconstructable from the live branch set", () => { const fired = new Set([ diff --git a/lib/daemon/__tests__/cache-read-bare-branch.test.ts b/lib/daemon/__tests__/cache-read-bare-branch.test.ts new file mode 100644 index 00000000..3292f775 --- /dev/null +++ b/lib/daemon/__tests__/cache-read-bare-branch.test.ts @@ -0,0 +1,70 @@ +/** + * cache:read's read contract (S069/Task 10): the store keys `ctx.cache.entries` + * by the composite `${identity}:${branch}` now, but cache:read's OUTPUT must + * stay keyed by the bare branch, never a composite key, so the CLI/board/ + * tray see exactly the same shape they always have. An absent `repoIdentity` + * falls back to a suffix match across repos; a present one scopes exactly. + */ +import { describe, test, expect } from "bun:test"; +import { createCacheHandlers } from "../handlers/cache.ts"; +import { composeKey } from "../../state/branch-cache.ts"; +import { fakeStore } from "./fake-cache-store.ts"; + +function makeCtx(entries: Record) { + const ctx = { + cache: fakeStore(entries), + refreshCache: async () => {}, + } as any; + return createCacheHandlers(ctx); +} + +describe("cache:read: bare-branch output", () => { + test("an unfiltered read returns bare-branch keys, never the store's composite keys", async () => { + const entries = { + [composeKey("remote:host%2Fa", "main")]: { linearId: "A", ticket: null, mr: null, fetchedAt: 1 }, + }; + const handlers = makeCtx(entries); + + const res = await handlers["cache:read"]!({}); + + expect(Object.keys(res.data)).toEqual(["main"]); + expect(res.data.main.linearId).toBe("A"); + }); + + test("a filtered read (branches list) resolves a bare branch by suffix match when repoIdentity is absent", async () => { + const entries = { + [composeKey("remote:host%2Fa", "main")]: { linearId: "A", ticket: null, mr: null, fetchedAt: 1 }, + }; + const handlers = makeCtx(entries); + + const res = await handlers["cache:read"]!({ branches: ["main"] }); + + expect(Object.keys(res.data)).toEqual(["main"]); + expect(res.data.main.linearId).toBe("A"); + }); + + test("two repos sharing a branch name: an unscoped read picks one entry, never crashes or merges them", async () => { + const entries = { + [composeKey("remote:host%2Fa", "main")]: { linearId: "A", ticket: null, mr: null, fetchedAt: 1 }, + [composeKey("remote:host%2Fb", "main")]: { linearId: "B", ticket: null, mr: null, fetchedAt: 2 }, + }; + const handlers = makeCtx(entries); + + const res = await handlers["cache:read"]!({ branches: ["main"] }); + + expect(Object.keys(res.data)).toEqual(["main"]); + expect(["A", "B"]).toContain(res.data.main.linearId); + }); + + test("an explicit repoIdentity scopes exactly, disambiguating two repos sharing a branch name", async () => { + const entries = { + [composeKey("remote:host%2Fa", "main")]: { linearId: "A", ticket: null, mr: null, fetchedAt: 1 }, + [composeKey("remote:host%2Fb", "main")]: { linearId: "B", ticket: null, mr: null, fetchedAt: 2 }, + }; + const handlers = makeCtx(entries); + + const res = await handlers["cache:read"]!({ branches: ["main"], repoIdentity: "remote:host%2Fb" }); + + expect(res.data.main.linearId).toBe("B"); + }); +}); diff --git a/lib/daemon/__tests__/cache-refresh-gc.test.ts b/lib/daemon/__tests__/cache-refresh-gc.test.ts index a959a4d1..11902fe9 100644 --- a/lib/daemon/__tests__/cache-refresh-gc.test.ts +++ b/lib/daemon/__tests__/cache-refresh-gc.test.ts @@ -40,6 +40,7 @@ import { createCacheRefresher } from "../cache-refresh.ts"; import { createProjectMRs } from "../project-mrs-store.ts"; import { createDiscussionsFileStore } from "../discussions-file-store.ts"; import { getBranchCacheStore, openStateDb, getNotifierStateBlob, setNotifierStateBlob, type CacheEntry } from "../../state/index.ts"; +import { composeKey } from "../../state/branch-cache.ts"; const DAY_MS = 24 * 60 * 60 * 1000; const CLEAN = "gcwire-clean"; @@ -158,11 +159,11 @@ describe("cache-refresh cycle: branch-cache GC", () => { await runCycle(); // Clean repo: aged out. Fresh row of the same repo: kept. - expect(cache.entries["gcwire-clean-stale"]).toBeUndefined(); - expect(cache.entries["gcwire-clean-fresh"]).toBeDefined(); + expect(cache.entries[composeKey(CLEAN, "gcwire-clean-stale")]).toBeUndefined(); + expect(cache.entries[composeKey(CLEAN, "gcwire-clean-fresh")]).toBeDefined(); // Flaky repo: `onError` fired, so the repo never entered succeededRepos // and NOTHING of its rows may be aged out this cycle. - expect(cache.entries["gcwire-flaky-stale"]).toBeDefined(); + expect(cache.entries[composeKey(FLAKY, "gcwire-flaky-stale")]).toBeDefined(); // NULL-repo rows are unattributable: prunable by age alone. expect(cache.entries["gcwire-orphan-stale"]).toBeUndefined(); }, 20_000); @@ -205,7 +206,7 @@ describe("cache-refresh cycle: branch-cache GC", () => { await runCycle(); - expect(cache.entries["gcwire-clean-stale"]).toBeUndefined(); + expect(cache.entries[composeKey(CLEAN, "gcwire-clean-stale")]).toBeUndefined(); const after = getNotifierStateBlob<{ fired: string[] }>({ fired: [] }); expect(after.fired).not.toContain(evictedKey); }, 20_000); diff --git a/lib/daemon/__tests__/discussions-semantics.test.ts b/lib/daemon/__tests__/discussions-semantics.test.ts index da23b6b3..126058b8 100644 --- a/lib/daemon/__tests__/discussions-semantics.test.ts +++ b/lib/daemon/__tests__/discussions-semantics.test.ts @@ -6,6 +6,7 @@ import { createDiscussionsFileStore, pruneDiscussionsStore } from "../discussion import { collectSweepTargets } from "../discussions-poller.ts"; import { createProjectMRs } from "../project-mrs-store.ts"; import { openStateDb, getBranchCacheStore } from "../../state/index.ts"; +import { composeKey } from "../../state/branch-cache.ts"; const tmp = (n: string) => join(mkdtempSync(join(tmpdir(), "rt-dsem-")), n); const tmpDb = () => openStateDb(tmp("state.db"), "cli"); @@ -97,7 +98,7 @@ describe("pruneDiscussionsStore", () => { // GC runs (repo "r" refreshed cleanly this cycle — gating per spec // "New: branch-cache GC"), evicting the stale row. cache.gc(new Set(["r"]), 30 * DAY_MS); - expect(cache.entries["stale-branch"]).toBeUndefined(); + expect(cache.entries[composeKey("r", "stale-branch")]).toBeUndefined(); // The union's branch-cache leg just shrank; the discussion is now a // true orphan and prunes — intended cleanup, not a regression. diff --git a/lib/daemon/__tests__/fake-cache-store.ts b/lib/daemon/__tests__/fake-cache-store.ts index b81c6d9f..3eac54a3 100644 --- a/lib/daemon/__tests__/fake-cache-store.ts +++ b/lib/daemon/__tests__/fake-cache-store.ts @@ -10,11 +10,15 @@ */ import type { BranchCacheStore, CacheEntry } from "../../state/index.ts"; +import { composeKey } from "../../state/branch-cache.ts"; export function fakeStore(entries: Record = {}): BranchCacheStore { return { entries, - put(branch, entry) { entries[branch] = entry; }, + // Mirrors the real store's put (Task 10): keyed by composeKey(entry.repoName, + // branch), not the bare branch, so a fixture pre-seeded under a composite + // key stays addressable at the same key after a consumer writes through it. + put(branch, entry) { entries[composeKey(entry.repoName, branch)] = entry; }, delete(branch) { delete entries[branch]; }, reload() { /* no db behind this fake — the map is the whole store */ }, gc() { /* GC is exercised against a real store, not here */ }, diff --git a/lib/daemon/__tests__/freshness-mapping.test.ts b/lib/daemon/__tests__/freshness-mapping.test.ts index 186e18e5..7beb1a73 100644 --- a/lib/daemon/__tests__/freshness-mapping.test.ts +++ b/lib/daemon/__tests__/freshness-mapping.test.ts @@ -12,6 +12,11 @@ import { createProjectMRs } from "../project-mrs-store.ts"; import type { InvalidationKey } from "@mattstack/glance"; import { fakeStore } from "./fake-cache-store.ts"; import { getBranchCacheStore, openStateDb } from "../../state/index.ts"; +import { composeKey } from "../../state/branch-cache.ts"; + +/** Composite key for a repo-x-attributed entry, matching what a real + * composeKey(entry.repoName, branch) put would produce. */ +const K = (branch: string) => composeKey("repo-x", branch); function tmpStorePath(): string { return join(mkdtempSync(join(tmpdir(), "rt-freshness-mapping-")), "state.db"); @@ -115,7 +120,7 @@ function key(kind: InvalidationKey["kind"], ref: string): InvalidationKey { describe("applyInvalidationBatch", () => { test("mr key with cached iid refetches that MR and updates the entry", async () => { const entries: Record = { - "feat-a": { mr: { iid: 42 }, ticket: { id: "T-1" }, linearId: "T-1", fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 42 }, ticket: { id: "T-1" }, linearId: "T-1", fetchedAt: 1, repoName: "repo-x" }, }; const { env, broadcasts, puts } = makeEnv(entries); const calls: any[] = []; @@ -132,18 +137,18 @@ describe("applyInvalidationBatch", () => { await applyInvalidationBatch(env, target, makeRunner(), [key("mr", "42")], noNotify); expect(calls).toEqual([["single", "g/p", 42]]); - expect(entries["feat-a"].mr.iid).toBe(42); - expect(entries["feat-a"].fetchedAt).toBeGreaterThan(1); - expect(entries["feat-a"].ticket).toEqual({ id: "T-1" }); // enrichment preserved - expect(entries["feat-a"].linearId).toBe("T-1"); - expect(puts).toEqual(["feat-a"]); + expect(entries[K("feat-a")].mr.iid).toBe(42); + expect(entries[K("feat-a")].fetchedAt).toBeGreaterThan(1); + expect(entries[K("feat-a")].ticket).toEqual({ id: "T-1" }); // enrichment preserved + expect(entries[K("feat-a")].linearId).toBe("T-1"); + expect(puts).toEqual(["feat-a"]); // store.put is still called with the BARE branch expect(broadcasts.filter((b) => b.type === "mr:update").length).toBe(1); - expect(broadcasts[0]!.data).toEqual({ repoName: "repo-x", mrs: { 42: entries["feat-a"].mr } }); + expect(broadcasts[0]!.data).toEqual({ repoName: "repo-x", mrs: { 42: entries[K("feat-a")].mr } }); }); test("mr key for another repo's iid is ignored", async () => { const entries: Record = { - "feat-a": { mr: { iid: 42 }, fetchedAt: 1, repoName: "other-repo" }, + [composeKey("other-repo", "feat-a")]: { mr: { iid: 42 }, fetchedAt: 1, repoName: "other-repo" }, }; const { env, puts } = makeEnv(entries); let called = false; @@ -166,8 +171,8 @@ describe("applyInvalidationBatch", () => { test("unknown mr key gap-fills null-mr branches after debounce", async () => { const entries: Record = { - "no-mr-branch": { mr: null, fetchedAt: 1, repoName: "repo-x" }, - "has-mr": { mr: { iid: 7 }, fetchedAt: 1, repoName: "repo-x" }, + [K("no-mr-branch")]: { mr: null, fetchedAt: 1, repoName: "repo-x" }, + [K("has-mr")]: { mr: { iid: 7 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); const batchCalls: string[][] = []; @@ -186,13 +191,13 @@ describe("applyInvalidationBatch", () => { await applyInvalidationBatch(env, target, runner, [key("mr", "999")], noNotify); expect(batchCalls.length).toBe(0); // debounced, not immediate await new Promise((r) => setTimeout(r, 40)); // > gapFillDebounceMs (10) - expect(batchCalls).toEqual([["no-mr-branch"]]); // only null-mr branches - expect(entries["no-mr-branch"].mr.iid).toBe(99); + expect(batchCalls).toEqual([["no-mr-branch"]]); // only null-mr branches (bare) + expect(entries[K("no-mr-branch")].mr.iid).toBe(99); }); test("disposed runner never arms gapFillTimer for an unknown mr key", async () => { const entries: Record = { - "no-mr-branch": { mr: null, fetchedAt: 1, repoName: "repo-x" }, + [K("no-mr-branch")]: { mr: null, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); let batchCalled = false; @@ -213,7 +218,7 @@ describe("applyInvalidationBatch", () => { test("unknown mr key with no null-mr branches skips the batch fetch entirely", async () => { const entries: Record = { - "has-mr": { mr: { iid: 7 }, fetchedAt: 1, repoName: "repo-x" }, + [K("has-mr")]: { mr: { iid: 7 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); let batchCalled = false; @@ -233,7 +238,7 @@ describe("applyInvalidationBatch", () => { test("notes key routes through refreshDiscussions override for cached iids only", async () => { const entries: Record = { - "feat-a": { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); const refreshed: Array<[string, number]> = []; @@ -263,7 +268,7 @@ describe("applyInvalidationBatch", () => { test("branch key refetches by branch; unknown branch and pipelines are ignored", async () => { const entries: Record = { - "feat-a": { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); const calls: any[] = []; @@ -290,7 +295,7 @@ describe("applyInvalidationBatch", () => { test("branch refetch returning null writes mr: null (MR deleted/never existed)", async () => { const entries: Record = { - "feat-a": { mr: { iid: 42 }, ticket: null, linearId: "", fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 42 }, ticket: null, linearId: "", fetchedAt: 1, repoName: "repo-x" }, }; const { env, puts } = makeEnv(entries); const target: RepoTarget = { @@ -302,14 +307,14 @@ describe("applyInvalidationBatch", () => { } as any, }; await applyInvalidationBatch(env, target, makeRunner(), [key("branch", "feat-a")], noNotify); - expect(entries["feat-a"].mr).toBeNull(); + expect(entries[K("feat-a")].mr).toBeNull(); expect(puts).toEqual(["feat-a"]); }); test("concurrent batch merges into pending and processes after current run", async () => { const entries: Record = { - "feat-a": { mr: { iid: 1 }, fetchedAt: 1, repoName: "repo-x" }, - "feat-b": { mr: { iid: 2 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 1 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-b")]: { mr: { iid: 2 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); const fetched: number[] = []; @@ -339,7 +344,7 @@ describe("applyInvalidationBatch", () => { test("duplicate keys within a batch are processed once", async () => { const entries: Record = { - "feat-a": { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); let count = 0; @@ -357,8 +362,8 @@ describe("applyInvalidationBatch", () => { test("a throwing fetch drops that key and continues with the rest", async () => { const entries: Record = { - "feat-a": { mr: { iid: 1 }, fetchedAt: 1, repoName: "repo-x" }, - "feat-b": { mr: { iid: 2 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 1 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-b")]: { mr: { iid: 2 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); const target: RepoTarget = { @@ -373,14 +378,14 @@ describe("applyInvalidationBatch", () => { } as any, }; await applyInvalidationBatch(env, target, makeRunner(), [key("mr", "1"), key("mr", "2")], noNotify); - expect(entries["feat-a"].mr.iid).toBe(1); // untouched - expect(entries["feat-b"].mr.iid).toBe(2); // still updated + expect(entries[K("feat-a")].mr.iid).toBe(1); // untouched + expect(entries[K("feat-b")].mr.iid).toBe(2); // still updated }); test("notify fires once per mutating batch with current userId", async () => { const entries: Record = { - "feat-a": { mr: { iid: 1 }, fetchedAt: 1, repoName: "repo-x" }, - "feat-b": { mr: { iid: 2 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 1 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-b")]: { mr: { iid: 2 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); let notifyCount = 0; @@ -428,7 +433,7 @@ describe("applyInvalidationBatch", () => { test("mr event, iid on a local branch: ONE fetch feeds branch entry AND project store", async () => { const store = pmrsStore(); const entries: Record = { - feat: { mr: { iid: 7 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat")]: { mr: { iid: 7 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); const calls: number[] = []; @@ -444,14 +449,14 @@ describe("applyInvalidationBatch", () => { ...noNotify, grantsFor: projectGrants, projectStore: store, }); expect(calls).toEqual([7]); // exactly one fetch, not two - expect(entries.feat.fetchedAt).toBeGreaterThan(1); // branch entry refreshed + expect(entries[K("feat")].fetchedAt).toBeGreaterThan(1); // branch entry refreshed expect(store.read("repo-x")!.mrs[7]).toBeDefined(); // project store also fed }); test("mr event, iid NOT in branchByIid but entry keyed by PR's sourceBranch has mr: null: ONE fetch feeds branch entry AND project store", async () => { const store = pmrsStore(); const entries: Record = { - "branch-42": { mr: null, fetchedAt: 1, repoName: "repo-x" }, + [K("branch-42")]: { mr: null, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); const calls: number[] = []; @@ -468,16 +473,16 @@ describe("applyInvalidationBatch", () => { ...noNotify, grantsFor: projectGrants, projectStore: store, }); expect(calls).toEqual([42]); // exactly one fetch - expect(entries["branch-42"].mr).not.toBeNull(); // branch entry filled via sourceBranch feed - expect(entries["branch-42"].mr.iid).toBe(42); - expect(entries["branch-42"].fetchedAt).toBeGreaterThan(1); + expect(entries[K("branch-42")].mr).not.toBeNull(); // branch entry filled via sourceBranch feed + expect(entries[K("branch-42")].mr.iid).toBe(42); + expect(entries[K("branch-42")].fetchedAt).toBeGreaterThan(1); expect(store.read("repo-x")!.mrs[42]).toBeDefined(); // project store also fed }); test("branch push with local entry + project grant: fetchPullRequestByBranch result reused, NO fetchSingleMR", async () => { const store = pmrsStore(); const entries: Record = { - "feat-a": { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); let singleCalled = false; @@ -740,6 +745,33 @@ describe("applyInvalidationBatch", () => { }); }); +// ─── S069/Task 10: composite-key collision safety ──────────────────────────── + +describe("freshness composite-key scoping (S069/Task 10)", () => { + test("a branch in two repos resolves to the right repo's entry", async () => { + const entries: Record = { + [composeKey("repo-a", "main")]: { mr: { iid: 1 }, fetchedAt: 1, repoName: "repo-a" }, + [composeKey("repo-b", "main")]: { mr: { iid: 2 }, fetchedAt: 1, repoName: "repo-b" }, + }; + const { env } = makeEnv(entries); + const target: RepoTarget = { + repoName: "repo-a", projectPath: "g/p", + provider: { + fetchSingleMR: async () => null, + fetchPullRequestByBranch: async () => fakePR(1, { sourceBranch: "main" }), + fetchPullRequestsByBranches: async () => new Map(), + } as any, + }; + + await applyInvalidationBatch(env, target, makeRunner(), [key("branch", "main")], noNotify); + + expect(entries[composeKey("repo-a", "main")].mr.iid).toBe(1); + // repo-b's same-named branch is a different composite key entirely, + // untouched by a refresh scoped to repo-a. + expect(entries[composeKey("repo-b", "main")].mr.iid).toBe(2); + }); +}); + // ─── RT-48 write-through (spec test 4) ─────────────────────────────────────── /** @@ -792,13 +824,13 @@ describe("write-through at updateEntry (RT-48)", () => { db.close(); // the daemon dies here — no flush, no shutdown hook, nothing const rebuilt = rebuildFromDb(dbPath); - expect(rebuilt["feat-a"]).toBeDefined(); - expect(rebuilt["feat-a"].mr.iid).toBe(42); - expect(rebuilt["feat-a"].fetchedAt).toBeGreaterThan(1); + expect(rebuilt[K("feat-a")]).toBeDefined(); + expect(rebuilt[K("feat-a")].mr.iid).toBe(42); + expect(rebuilt[K("feat-a")].fetchedAt).toBeGreaterThan(1); // Enrichment the events path never touches is preserved through the row. - expect(rebuilt["feat-a"].ticket).toEqual({ id: "T-1" }); - expect(rebuilt["feat-a"].linearId).toBe("T-1"); - expect(rebuilt["feat-a"].repoName).toBe("repo-x"); + expect(rebuilt[K("feat-a")].ticket).toEqual({ id: "T-1" }); + expect(rebuilt[K("feat-a")].linearId).toBe("T-1"); + expect(rebuilt[K("feat-a")].repoName).toBe("repo-x"); }); test("a refresh that clears an MR persists the null, not the stale MR", async () => { @@ -819,7 +851,7 @@ describe("write-through at updateEntry (RT-48)", () => { await applyInvalidationBatch(env, target, makeRunner(), [key("branch", "feat-b")], noNotify); db.close(); - expect(rebuildFromDb(dbPath)["feat-b"].mr).toBeNull(); + expect(rebuildFromDb(dbPath)[K("feat-b")].mr).toBeNull(); }); test("the store exposes no flush of any kind — persistence is not optional", () => { diff --git a/lib/daemon/__tests__/system-processes-handlers.test.ts b/lib/daemon/__tests__/system-processes-handlers.test.ts index 2597c9c4..e40bd62b 100644 --- a/lib/daemon/__tests__/system-processes-handlers.test.ts +++ b/lib/daemon/__tests__/system-processes-handlers.test.ts @@ -2,6 +2,7 @@ import { describe, test, expect } from "bun:test"; import { createSystemProcessHandlers } from "../handlers/system-processes.ts"; import type { SystemProcess } from "../system-process-scanner.ts"; +import { composeKey } from "../../state/branch-cache.ts"; function makeProcess(overrides: Partial = {}): SystemProcess { return { @@ -53,9 +54,9 @@ describe("system-processes handler", () => { }); test("enriches processes with Linear ticket from branch cache", async () => { - const proc = makeProcess({ branch: "feature/foo" }); + const proc = makeProcess({ branch: "feature/foo", repo: "myrepo" }); const handlers = setup([proc], { - "feature/foo": { + [composeKey("myrepo", "feature/foo")]: { ticket: { identifier: "ENG-123", title: "Do the thing" }, }, }); @@ -75,8 +76,8 @@ describe("system-processes handler", () => { }); test("leaves linearTicket null when cache entry has no ticket", async () => { - const proc = makeProcess({ branch: "feature/foo" }); - const handlers = setup([proc], { "feature/foo": { ticket: null } }); + const proc = makeProcess({ branch: "feature/foo", repo: "myrepo" }); + const handlers = setup([proc], { [composeKey("myrepo", "feature/foo")]: { ticket: null } }); const res = await handlers["system-processes"]!({}); diff --git a/lib/daemon/__tests__/worktree-reconciler.test.ts b/lib/daemon/__tests__/worktree-reconciler.test.ts index 87b17260..a0ef67d5 100644 --- a/lib/daemon/__tests__/worktree-reconciler.test.ts +++ b/lib/daemon/__tests__/worktree-reconciler.test.ts @@ -6,6 +6,7 @@ import { basename, dirname, join } from "path"; import type { Logger } from "pino"; import { readJson, writeJson } from "../../json-store.ts"; import { closeStateDb, listKvValues, setKvValue } from "../../state/index.ts"; +import { composeKey } from "../../state/branch-cache.ts"; import { machineSettingsPath, rtDir, teamSettingsPath } from "../../rt-paths.ts"; import { deriveRepoIdentity, parseIdentity } from "../../settings/identity.ts"; import { findByPath, loadRegistry, saveRegistry, type TreeRecord } from "../../worktree/registry.ts"; @@ -924,6 +925,27 @@ describe("merge reactor (detectTransitions)", () => { expect(tracked(rec.path)!.state).toBe("claimed"); }); + test("S069/Task 10: mrState is built only from the reconciled repo's composite-keyed entries", async () => { + const rec = ephemeralTree("kilo", "feat-kilo"); + const sameBranchInThisRepo = (state: string) => ({ + [composeKey(repoName, "feat-kilo")]: { repoName, mr: { iid: 42, state }, fetchedAt: Date.now() }, + // Same bare branch name, a DIFFERENT repo's composite key: must never + // be read as this repo's opened->merged edge, nor advance its snapshot. + [composeKey("beta-repo", "feat-kilo")]: { repoName: "beta-repo", mr: { iid: 99, state: "merged" }, fetchedAt: Date.now() }, + }); + + await detect(sameBranchInThisRepo("opened")); + expect(reactorState().mrState[`${repoName}:feat-kilo`]).toBe("opened"); + expect(reactorState().mrState["beta-repo:feat-kilo"]).toBeUndefined(); + + await detect(sameBranchInThisRepo("merged")); + + expect(existsSync(rec.path)).toBe(false); // this repo's tree disposed + expect(reactorState().fired).toContain(`disposed:${repoName}:42:merged`); + // beta-repo's own MR (99) never fired through this repo's pass. + expect(reactorState().fired).not.toContain("disposed:beta-repo:99:merged"); + }); + test("runOnce runs the reactor after the reconcile pass", async () => { const rec = ephemeralTree("golf", "feat-golf"); await declareWorktrees(repo, repoName, {}); diff --git a/lib/daemon/freshness.ts b/lib/daemon/freshness.ts index 91ce9f6e..aade086c 100644 --- a/lib/daemon/freshness.ts +++ b/lib/daemon/freshness.ts @@ -38,6 +38,7 @@ import { redactCredentials } from "./redact-credentials.ts"; import { getProjectMRs, type ProjectMRs } from "./project-mrs-store.ts"; import { getDiscussionsFileStore } from "./discussions-file-store.ts"; import { createCursorStore, type CursorStore } from "../state/index.ts"; +import { composeKey, branchOf } from "../state/branch-cache.ts"; import { runCapture } from "../subprocess.ts"; const log = lazyChildLogger("freshness"); @@ -502,9 +503,9 @@ async function processKeys( // iid → branch for this repo, rebuilt per batch (the cache may have been // reloaded from disk by the full poll since the last tick). const branchByIid = new Map(); - for (const [branch, entry] of Object.entries(ctx.cache.entries)) { + for (const [key, entry] of Object.entries(ctx.cache.entries)) { if (entry.repoName !== repoName) continue; - if (typeof entry.mr?.iid === "number") branchByIid.set(entry.mr.iid, branch); + if (typeof entry.mr?.iid === "number") branchByIid.set(entry.mr.iid, branchOf(key)); } const g = (overrides.grantsFor ?? ((r: string) => grants(loadRepoTracking(), r)))(repoName); @@ -542,7 +543,7 @@ async function processKeys( } const pr = await provider.fetchSingleMR(projectPath, iid, getCurrentUserId()); const feedBranch = branch - ?? (pr && ctx.cache.entries[pr.sourceBranch]?.repoName === repoName ? pr.sourceBranch : undefined); + ?? (pr && ctx.cache.entries[composeKey(repoName, pr.sourceBranch)] !== undefined ? pr.sourceBranch : undefined); if (feedBranch) mutated = updateEntry(env, repoName, feedBranch, pr) || mutated; if (wantProject && pr) upsertProject(pr); // GitLab's approval action bumps no updatedAt, so this is the only @@ -576,8 +577,8 @@ async function processKeys( break; } case "branch": { - const entry = ctx.cache.entries[k.ref]; - const isOurs = entry !== undefined && entry.repoName === repoName; + const entry = ctx.cache.entries[composeKey(repoName, k.ref)]; + const isOurs = entry !== undefined; let fetchedForRef: PullRequest | null | undefined; if (isOurs) { fetchedForRef = await provider.fetchPullRequestByBranch(projectPath, k.ref, "all"); @@ -636,7 +637,7 @@ async function processKeys( */ function updateEntry(env: FreshnessEnv, repoName: string, branch: string, pr: PullRequest | null): boolean { const { ctx } = env; - const existing = ctx.cache.entries[branch]; + const existing = ctx.cache.entries[composeKey(repoName, branch)]; if (!existing) return false; // lost race with a full refresh — skip const mr = pr ? toMRInfo(pr) : null; ctx.cache.put(branch, { ...existing, mr, fetchedAt: Date.now(), repoName }); @@ -654,8 +655,8 @@ function updateEntry(env: FreshnessEnv, repoName: string, branch: string, pr: Pu */ export function applyMRWriteback(env: FreshnessEnv, repoName: string, projectPath: string, pr: PullRequest): void { let branch: string | null = null; - for (const [b, entry] of Object.entries(env.ctx.cache.entries)) { - if (entry.repoName === repoName && entry.mr?.iid === pr.iid) { branch = b; break; } + for (const [key, entry] of Object.entries(env.ctx.cache.entries)) { + if (entry.repoName === repoName && entry.mr?.iid === pr.iid) { branch = branchOf(key); break; } } if (branch) updateEntry(env, repoName, branch, pr); @@ -702,7 +703,7 @@ async function runGapFill(env: FreshnessEnv, target: RepoTarget, overrides: Mapp const nullMrBranches = Object.entries(ctx.cache.entries) .filter(([, e]) => e.repoName === repoName && e.mr == null) - .map(([branch]) => branch); + .map(([key]) => branchOf(key)); if (nullMrBranches.length === 0) return; if (!provider.fetchPullRequestsByBranches) return; diff --git a/lib/daemon/handlers/cache.ts b/lib/daemon/handlers/cache.ts index 1823b702..7e9493af 100644 --- a/lib/daemon/handlers/cache.ts +++ b/lib/daemon/handlers/cache.ts @@ -10,6 +10,7 @@ */ import type { HandlerContext, HandlerMap, CacheEntry } from "./types.ts"; +import { branchOf, composeKey, getByBranch } from "../../state/branch-cache.ts"; /** How long an entry that resolved a ticket id but never got the ticket is left alone before another lookup is spent on it. Short enough that a key @@ -36,26 +37,40 @@ export function createCacheHandlers(ctx: HandlerContext): HandlerMap { "cache:read": async (payload) => { const branches = payload?.branches as string[] | undefined; const maxAgeMs = payload?.maxAgeMs as number | undefined; + // Optional exact scoping: an absent repoIdentity falls back to a + // suffix match across repos (today's callers never pass this yet). + const repoIdentity = payload?.repoIdentity as string | undefined; + + const lookup = (b: string): CacheEntry | undefined => + repoIdentity ? ctx.cache.entries[composeKey(repoIdentity, b)] : getByBranch(ctx.cache.entries, b); // Freshness gate: when the caller sets maxAgeMs, refresh first if the // oldest requested entry is older than that. Missing entries and an // empty cache count as infinitely stale. refreshCache is coalesced, so // concurrent stale readers share one refresh. if (typeof maxAgeMs === "number") { - const pool = branches ?? Object.keys(ctx.cache.entries); + const pool = branches ?? Object.keys(ctx.cache.entries).map(branchOf); let oldestFetchedAt = 0; if (pool.length > 0) { - oldestFetchedAt = Math.min(...pool.map((b) => ctx.cache.entries[b]?.fetchedAt ?? 0)); + oldestFetchedAt = Math.min(...pool.map((b) => lookup(b)?.fetchedAt ?? 0)); } if (Date.now() - oldestFetchedAt >= maxAgeMs) { await ctx.refreshCache(); } } - if (!branches) return { ok: true, data: ctx.cache.entries }; + // The output is always bare-branch keyed, never the store's internal + // composite keys, so cache:read's contract to the CLI/board/tray + // never changes underneath them. + if (!branches) { + const out: Record = {}; + for (const [k, v] of Object.entries(ctx.cache.entries)) out[branchOf(k)] = v; + return { ok: true, data: out }; + } const filtered: Record = {}; for (const b of branches) { - if (ctx.cache.entries[b]) filtered[b] = ctx.cache.entries[b]; + const entry = lookup(b); + if (entry) filtered[b] = entry; } return { ok: true, data: filtered }; }, @@ -66,9 +81,10 @@ export function createCacheHandlers(ctx: HandlerContext): HandlerMap { }, "branch:enrich": async (payload) => { - const branch = payload?.branch as string; - const repoPath = payload?.repoPath as string; - const remoteUrl = payload?.remoteUrl as string | undefined; + const branch = payload?.branch as string; + const repoPath = payload?.repoPath as string; + const remoteUrl = payload?.remoteUrl as string | undefined; + const repoIdentity = payload?.repoIdentity as string | undefined; // Test seam: the enricher, so a test never reaches Linear or the forge. const inject = payload?.enrich as | ((b: unknown, r: unknown, o: unknown) => Promise) @@ -76,7 +92,10 @@ export function createCacheHandlers(ctx: HandlerContext): HandlerMap { if (!branch) return { ok: false, error: "missing branch" }; - const cached = ctx.cache.entries[branch]; + const lookupBranch = (): CacheEntry | undefined => + repoIdentity ? ctx.cache.entries[composeKey(repoIdentity, branch)] : getByBranch(ctx.cache.entries, branch); + + const cached = lookupBranch(); const healing = !!cached; if (cached && !isIncomplete(cached)) { return { ok: true, data: cached, source: "cache" }; @@ -102,8 +121,9 @@ export function createCacheHandlers(ctx: HandlerContext): HandlerMap { // it also picks up rows a racing CLI enrichment upserted. ctx.cache.reload(); - if (ctx.cache.entries[branch]) { - return { ok: true, data: ctx.cache.entries[branch], source: "fresh" }; + const fresh = lookupBranch(); + if (fresh) { + return { ok: true, data: fresh, source: "fresh" }; } return { ok: true, data: null, source: "empty" }; } catch (err) { diff --git a/lib/daemon/handlers/system-processes.ts b/lib/daemon/handlers/system-processes.ts index 545cf7b9..34e92d36 100644 --- a/lib/daemon/handlers/system-processes.ts +++ b/lib/daemon/handlers/system-processes.ts @@ -1,6 +1,7 @@ import type { HandlerMap, HandlerContext } from "./types.ts"; import type { SystemProcessScanner, SystemProcess } from "../system-process-scanner.ts"; import { repoLabel } from "../../repo-arg.ts"; +import { composeKey } from "../../state/branch-cache.ts"; function shortName(proc: SystemProcess): string { // Use fullCommand (complete argv) to get the real binary name, @@ -95,7 +96,9 @@ export function createSystemProcessHandlers( const processes = scanner.getProcesses().map(proc => { let linearTicket: string | null = null; if (proc.branch) { - const cacheEntry = ctx.cache.entries[proc.branch]; + // proc.repo is already the serialized identity (scanner tags rows + // with it post-rekey), so an exact composeKey lookup is safe here. + const cacheEntry = ctx.cache.entries[composeKey(proc.repo, proc.branch)]; if (cacheEntry?.ticket) { linearTicket = `${cacheEntry.ticket.identifier}: ${cacheEntry.ticket.title}`; } diff --git a/lib/daemon/handlers/worktree.ts b/lib/daemon/handlers/worktree.ts index b4c3f223..f712b840 100644 --- a/lib/daemon/handlers/worktree.ts +++ b/lib/daemon/handlers/worktree.ts @@ -44,6 +44,7 @@ import { import { disambiguate, slugifyTicketTitle } from "../../worktree/branch-name.ts"; import { createTree } from "../../worktree/create.ts"; import { classifyDirtyAsync, disposeTree, type DisposeDeps } from "../../worktree/dispose.ts"; +import { branchOf, composeKey } from "../../state/branch-cache.ts"; import { isTreeLocked, withTreeLock } from "../../worktree/locks.ts"; import { branchExistsLocalAsync, @@ -172,10 +173,18 @@ function disposeDeps( repoName: string, repoPath: string, ): DisposeDeps { + // disposeTree's joinedMr looks up by the BARE branch: hand it a + // bare-keyed, this-repo-only view of the (now composite-keyed) cache map + // so a same-named branch in another repo can never shadow the real entry. + const cacheEntries: DisposeDeps["cacheEntries"] = {}; + for (const [key, entry] of Object.entries(ctx.cache.entries)) { + if (entry.repoName && entry.repoName !== repoName) continue; + cacheEntries[branchOf(key)] = entry; + } return { repoName, repoPath, - cacheEntries: ctx.cache.entries as DisposeDeps["cacheEntries"], + cacheEntries, emit: opts.emit, log: ctx.log, killProcesses: loadWorktreeAppConfig().killProcesses, @@ -558,9 +567,12 @@ export function createWorktreeHandlers( } for (const t of trees) { - // The join key is (repoName, branch): a bare-branch join would hand - // a tree another repo's MR when both repos use the same name. - const entry = t.branch ? entries[t.branch] : undefined; + // The join key is composeKey(repoName, branch): an exact hit scopes + // to this repo so a same-named branch elsewhere can never join here. + // The bare-key fallback only ever matches an unattributed entry + // (older caches predate repoName), never another repo's, since + // every attributed write now composes under its own identity. + const entry = t.branch ? (entries[composeKey(repoName, t.branch)] ?? entries[t.branch]) : undefined; const mr = entry?.mr && (!entry.repoName || entry.repoName === repoName) ? { iid: entry.mr.iid, state: entry.mr.state, title: entry.mr.title } diff --git a/lib/daemon/worktree-reconciler.ts b/lib/daemon/worktree-reconciler.ts index 9ac37eac..bc75d864 100644 --- a/lib/daemon/worktree-reconciler.ts +++ b/lib/daemon/worktree-reconciler.ts @@ -38,6 +38,7 @@ import { import { isTreeLocked, withTreeLock } from "../worktree/locks.ts"; import { ensureWorktreeRegistryRekeyed } from "../repo-index.ts"; import { createTree, scrapTree, type CreateDeps } from "../worktree/create.ts"; +import { branchOf } from "../state/branch-cache.ts"; import { classifyDirtyAsync, disposeTree } from "../worktree/dispose.ts"; import { changedSince, stepsToRun, runReadySteps } from "../worktree/ready.ts"; import { MAX_LOGGED_OUTPUT, outputTail } from "../subprocess.ts"; @@ -513,11 +514,21 @@ async function actOnTree( return "fired"; } + // disposeTree's joinedMr looks up by the BARE branch (its own contract, + // unaware of the composite `${identity}:${branch}` keys this repo's + // cache map now carries): hand it a bare-keyed, this-repo-only view so a + // same-named branch in another repo can never shadow the real entry. + const scopedEntries: Record = {}; + for (const [key, entry] of Object.entries(deps.cacheEntries)) { + if (entry.repoName && entry.repoName !== deps.repoName) continue; + scopedEntries[branchOf(key)] = entry; + } + const outcome = await disposeTree( { repoName: deps.repoName, repoPath: deps.repoPath, - cacheEntries: deps.cacheEntries as Record, + cacheEntries: scopedEntries as Record, emit: deps.emit, log: deps.log, killProcesses: appConfig.killProcesses, @@ -591,19 +602,20 @@ export async function detectTransitions(deps: ReactorDeps): Promise { if (!key.startsWith(prefix)) nextMrState[key] = value; } - for (const [branch, entry] of Object.entries(cacheEntries)) { + for (const [mapKey, entry] of Object.entries(cacheEntries)) { // Unattributed entries (older caches predate repoName) may join any repo; // an entry attributed elsewhere never does. if (entry.repoName && entry.repoName !== repoName) continue; if (!entry.mr) continue; + const branch = branchOf(mapKey); const cur = entry.mr.state ?? null; - const key = prefix + branch; - const prev = state.mrState[key] ?? null; + const mrKey = prefix + branch; + const prev = state.mrState[mrKey] ?? null; const iid = typeof entry.mr.iid === "number" ? String(entry.mr.iid) : branch; if (cur === "opened") { - nextMrState[key] = "opened"; + nextMrState[mrKey] = "opened"; // Reopen: forget this MR's fires so a later merge acts again, and hand // any disposable tree back to its owner. for (const fireKey of [...fired]) { @@ -613,7 +625,7 @@ export async function detectTransitions(deps: ReactorDeps): Promise { continue; } - nextMrState[key] = cur; + nextMrState[mrKey] = cur; if (prev !== "opened") continue; // cold-boot safety: unknown prev never fires if (!cur || !TERMINAL_STATES.has(cur)) continue; @@ -633,7 +645,7 @@ export async function detectTransitions(deps: ReactorDeps): Promise { reaction = worse(reaction, result === "busy" ? "retry" : result); } - if (reaction === "retry") nextMrState[key] = "opened"; + if (reaction === "retry") nextMrState[mrKey] = "opened"; else if (reaction === "fired") fired.add(fireKey); } diff --git a/lib/enrich.ts b/lib/enrich.ts index 1317b6fd..d6b7b5d3 100644 --- a/lib/enrich.ts +++ b/lib/enrich.ts @@ -23,6 +23,8 @@ import { type BranchCacheStore, type CacheEntry, } from "./state/index.ts"; +import { composeKey } from "./state/branch-cache.ts"; +import { identityFromRemote, serializeIdentity } from "./settings/identity.ts"; import { GitLabProvider, type PullRequest, @@ -36,6 +38,13 @@ import { type LinearTicket, } from "./linear.ts"; +/** Best-effort serialized identity for a remote URL; undefined with no remote. */ +function identityForRemote(remoteUrl: string | undefined): string | undefined { + if (!remoteUrl) return undefined; + const parsed = identityFromRemote(remoteUrl); + return parsed ? serializeIdentity(parsed) : undefined; +} + // ─── Remote URL parser ─────────────────────────────────────────────────────── export function parseRemoteUrl(url: string): { host: string; projectPath: string } | null { @@ -263,12 +272,14 @@ export async function enrichBranches( const secrets = await loadSecrets(); const willFetch = !!(secrets.linearApiKey || secrets.gitlabToken); const store = getBranchCacheStore(); + const identity = identityForRemote(remoteUrl); - const allCached = !options?.forceRefresh && willFetch && branches.every((b) => b.branch in store.entries); + const allCached = !options?.forceRefresh && willFetch + && branches.every((b) => composeKey(identity, b.branch) in store.entries); if (allCached) { const cachedResults = branches.map((b) => { - const entry = store.entries[b.branch]!; + const entry = store.entries[composeKey(identity, b.branch)]!; return { path: b.path, dirName: b.path.split("/").pop() || b.path, @@ -297,6 +308,7 @@ async function fetchAndCache( ): Promise { const secrets = await loadSecrets(); const willFetch = !!(secrets.linearApiKey || secrets.gitlabToken); + const identity = identityForRemote(remoteUrl); let showSpinner = false; if (!silent && willFetch && process.stderr.isTTY) { @@ -362,7 +374,7 @@ async function fetchAndCache( const results: EnrichedBranch[] = branches.map((b, idx) => { const dirName = b.path.split("/").pop() || b.path; const { linearId } = branchLinearIds[idx]!; - const existing = store.entries[b.branch]; + const existing = store.entries[composeKey(identity, b.branch)]; const pr = mrMap.get(b.branch) ?? null; const mr = mrFetchSucceeded ? (pr ? toMRInfo(pr) : null) : (existing?.mr ?? null); @@ -376,7 +388,7 @@ async function fetchAndCache( linearId: linearId || existing?.linearId || "", mr, fetchedAt: mrFetchSucceeded ? Date.now() : (existing?.fetchedAt ?? Date.now()), - repoName: existing?.repoName, + repoName: identity, }]); return { path: b.path, dirName, branch: b.branch, linearId, ticket, mr }; @@ -492,7 +504,7 @@ export async function refreshAllMRs( // preserve the existing entry to avoid overwriting good enrichment data that was // previously resolved via a full enrich (e.g., from an older/renamed MR title). if (!mr && !linearId) { - const existing = store.entries[b.branch]; + const existing = store.entries[composeKey(repoName, b.branch)]; if (existing?.linearId || existing?.ticket) { // Keep existing enrichment — we have nothing better to replace it with enriched.push([b.branch, { ...existing, fetchedAt: now, repoName }]); @@ -511,7 +523,7 @@ export async function refreshAllMRs( // GitLab API failed entirely — preserve existing MR data to avoid false transitions. // If we also couldn't resolve a linearId (non-standard branch name, no MR title to fall // back on), preserve existing ticket/linearId too — we have nothing better to substitute. - const existing = store.entries[b.branch]; + const existing = store.entries[composeKey(repoName, b.branch)]; enriched.push([b.branch, { ticket: linearId ? ticket : (existing?.ticket ?? null), linearId: linearId || existing?.linearId || "", diff --git a/lib/notifier.ts b/lib/notifier.ts index 0c235c87..ff3ccf2c 100644 --- a/lib/notifier.ts +++ b/lib/notifier.ts @@ -26,6 +26,7 @@ import type { SystemProcess } from "./daemon/system-process-scanner.ts"; import { agentSessionPids } from "./daemon/worktree-process-kill.ts"; import { lazyChildLogger } from "./daemon-logger.ts"; import { repoLabel } from "./repo-arg.ts"; +import { branchOf } from "./state/branch-cache.ts"; import { getNotifierStateBlob, setNotifierStateBlob, @@ -552,6 +553,11 @@ function detectBranchTransitions( prefs: NotificationPrefs, currentUserId: number | null, ): void { + // `branch` is the branch-cache map key (composite `${identity}:${branch}` + // when attributed, bare otherwise), kept as-is here rather than unwrapped + // to the bare branch, so `firedKey` and the snapshot map come out + // repo-scoped for free. `branchOf(branch)` is used only for the + // human-readable notification text. for (const [branch, entry] of Object.entries(current)) { // If the MR slot is null we have no fresh data — skipping prevents // false "transition" detection that would clear the fired key set. @@ -566,7 +572,8 @@ function detectBranchTransitions( if (!was) continue; // First time seeing this branch — no transition const now = snapshotBranch(entry, was); - const branchShort = branch.length > 40 ? branch.slice(0, 39) + "…" : branch; + const displayBranch = branchOf(branch); + const branchShort = displayBranch.length > 40 ? displayBranch.slice(0, 39) + "…" : displayBranch; const mrUrl = entry.mr?.webUrl ?? undefined; // MR merged (opened → merged) — check BEFORE skipping merged MRs diff --git a/lib/state/__tests__/branch-cache.test.ts b/lib/state/__tests__/branch-cache.test.ts index 98cfadd3..16bfef7d 100644 --- a/lib/state/__tests__/branch-cache.test.ts +++ b/lib/state/__tests__/branch-cache.test.ts @@ -33,7 +33,7 @@ test("branch never contains a colon, so lastIndexOf split is unambiguous", () => expect(identityOf(k)).toBe("path:%2FUsers%2Fdev%2Fscratch"); }); -describe("getByBranch — free function over an entries map", () => { +describe("getByBranch: free function over an entries map", () => { function makeCacheEntry(linearId: string): CacheEntry { return { ticket: null, linearId, mr: null, fetchedAt: Date.now() }; } @@ -59,6 +59,32 @@ describe("getByBranch — free function over an entries map", () => { }); }); +describe("put: composite-key collision safety (S069/Task 10)", () => { + let collisionDir: string; + + beforeEach(() => { + collisionDir = mkdtempSync(join(tmpdir(), "rt-branch-cache-collision-")); + }); + + afterEach(() => { + rmSync(collisionDir, { recursive: true, force: true }); + }); + + test("put keys by entry.repoName so same-name branches in two repos coexist", () => { + const dbPath = join(collisionDir, "state.db"); + const db = openStateDb(dbPath, "cli"); + const store = getBranchCacheStore(db); + + store.put("main", { repoName: "remote:host%2Fa", ticket: null, linearId: "", mr: null, fetchedAt: 1 }); + store.put("main", { repoName: "remote:host%2Fb", ticket: null, linearId: "", mr: null, fetchedAt: 2 }); + + expect(store.entries[composeKey("remote:host%2Fa", "main")]?.fetchedAt).toBe(1); + expect(store.entries[composeKey("remote:host%2Fb", "main")]?.fetchedAt).toBe(2); + expect(Object.keys(store.entries).length).toBe(2); + db.close(); + }); +}); + let dir: string; beforeEach(() => { @@ -214,19 +240,34 @@ describe("two handles, per-row last-writer-wins", () => { }); describe("bare-branch upsert semantics", () => { - test("a no-repoName upsert hits the same row a repoName-bearing write created (no NULL duplicate)", () => { + test("two puts with the same repoName hit the same row (no duplicate)", () => { const dbPath = join(dir, "state.db"); const db = openStateDb(dbPath, "cli"); const store = getBranchCacheStore(db); + const key = composeKey("repo-tools", "feature/x"); store.put("feature/x", makeEntry({ repoName: "repo-tools", linearId: "first" })); - expect(rowCount(db, "feature/x")).toBe(1); + expect(rowCount(db, key)).toBe(1); + + store.put("feature/x", makeEntry({ repoName: "repo-tools", linearId: "second" })); + + expect(rowCount(db, key)).toBe(1); + expect(store.entries[key]?.linearId).toBe("second"); + db.close(); + }); + + test("a repoName-bearing write and a bare (no-repoName) write to the same branch land in different rows (Task 10: key is composeKey(entry.repoName, branch), not the bare branch)", () => { + const dbPath = join(dir, "state.db"); + const db = openStateDb(dbPath, "cli"); + const store = getBranchCacheStore(db); + store.put("feature/x", makeEntry({ repoName: "repo-tools", linearId: "attributed" })); // enrichBranches-style upsert: same bare branch, no repoName available. - store.put("feature/x", makeEntry({ linearId: "second" })); + store.put("feature/x", makeEntry({ linearId: "bare" })); - expect(rowCount(db, "feature/x")).toBe(1); - expect(store.entries["feature/x"]?.linearId).toBe("second"); + expect(store.entries[composeKey("repo-tools", "feature/x")]?.linearId).toBe("attributed"); + expect(store.entries["feature/x"]?.linearId).toBe("bare"); + expect(Object.keys(store.entries).length).toBe(2); db.close(); }); @@ -259,13 +300,15 @@ describe("gc — succeeded-repo gating and NULL-repo age rule", () => { store.gc(new Set(["repo-a"]), 30 * DAY_MS); - expect(store.entries["stale-succeeded"]).toBeUndefined(); - expect(rowCount(db, "stale-succeeded")).toBe(0); + const succeededKey = composeKey("repo-a", "stale-succeeded"); + const failedKey = composeKey("repo-b", "stale-failed"); + expect(store.entries[succeededKey]).toBeUndefined(); + expect(rowCount(db, succeededKey)).toBe(0); // repo-b had a swallowed fetch error this cycle (never made it into // succeededRepos) — its stale rows must survive (r2 finding 1). - expect(store.entries["stale-failed"]).toBeDefined(); - expect(rowCount(db, "stale-failed")).toBe(1); + expect(store.entries[failedKey]).toBeDefined(); + expect(rowCount(db, failedKey)).toBe(1); db.close(); }); @@ -292,8 +335,9 @@ describe("gc — succeeded-repo gating and NULL-repo age rule", () => { store.gc(new Set(["repo-a"]), 30 * DAY_MS); - expect(store.entries["fresh-succeeded"]).toBeDefined(); - expect(rowCount(db, "fresh-succeeded")).toBe(1); + const key = composeKey("repo-a", "fresh-succeeded"); + expect(store.entries[key]).toBeDefined(); + expect(rowCount(db, key)).toBe(1); db.close(); }); @@ -308,9 +352,10 @@ describe("gc — succeeded-repo gating and NULL-repo age rule", () => { store.gc(new Set(["repo-a"]), 30 * DAY_MS); - expect(Object.keys(store.entries).sort()).toEqual(["keep"]); + const keepKey = composeKey("repo-a", "keep"); + expect(Object.keys(store.entries).sort()).toEqual([keepKey]); const remaining = db.query("SELECT branch FROM branch_cache;").all() as { branch: string }[]; - expect(remaining.map(r => r.branch).sort()).toEqual(["keep"]); + expect(remaining.map(r => r.branch).sort()).toEqual([keepKey]); db.close(); }); @@ -319,6 +364,8 @@ describe("gc — succeeded-repo gating and NULL-repo age rule", () => { const db = openStateDb(dbPath, "cli"); const store = getBranchCacheStore(db); + const racyKey = composeKey("repo-a", "racy"); + const doomedKey = composeKey("repo-a", "doomed"); store.put("racy", makeEntry({ repoName: "repo-a", fetchedAt: oldTs })); store.put("doomed", makeEntry({ repoName: "repo-a", fetchedAt: oldTs })); @@ -340,7 +387,7 @@ describe("gc — succeeded-repo gating and NULL-repo age rule", () => { (stmt as unknown as { all: unknown }).all = (...args: never[]) => { const rows = realAll(...args); (stmt as unknown as { all: unknown }).all = realAll; // one-shot - cli.query("UPDATE branch_cache SET fetched_at = ? WHERE branch = ?;").run(freshTs, "racy"); + cli.query("UPDATE branch_cache SET fetched_at = ? WHERE branch = ?;").run(freshTs, racyKey); return rows; }; } @@ -355,13 +402,13 @@ describe("gc — succeeded-repo gating and NULL-repo age rule", () => { // The freshly enriched row survives with its new timestamp; its stale // sibling is still pruned. - expect(rowCount(db, "racy")).toBe(1); - const { fetched_at } = db.query("SELECT fetched_at FROM branch_cache WHERE branch = ?;").get("racy") as { fetched_at: number }; + expect(rowCount(db, racyKey)).toBe(1); + const { fetched_at } = db.query("SELECT fetched_at FROM branch_cache WHERE branch = ?;").get(racyKey) as { fetched_at: number }; expect(fetched_at).toBe(freshTs); - expect(rowCount(db, "doomed")).toBe(0); + expect(rowCount(db, doomedKey)).toBe(0); // Row/map parity holds on both sides of the re-guard. - expect(store.entries["racy"]).toBeDefined(); - expect(store.entries["doomed"]).toBeUndefined(); + expect(store.entries[racyKey]).toBeDefined(); + expect(store.entries[doomedKey]).toBeUndefined(); cli.close(); db.close(); @@ -393,7 +440,7 @@ describe("daemon-flavor busy handling", () => { expect(() => store.put("busy-branch", makeEntry({ repoName: "repo-a" }))).not.toThrow(); // The map is the daemon's read model (spec "In-memory ownership") — // it must carry the enrichment even when the row could not. - expect(store.entries["busy-branch"]).toBeDefined(); + expect(store.entries[composeKey("repo-a", "busy-branch")]).toBeDefined(); } finally { lock.release(); } @@ -404,6 +451,7 @@ describe("daemon-flavor busy handling", () => { const dbPath = join(dir, "state.db"); const db = openStateDb(dbPath, "daemon"); const store = getBranchCacheStore(db); + const key = composeKey("repo-a", "stale-busy"); store.put("stale-busy", makeEntry({ repoName: "repo-a", fetchedAt: Date.now() - 40 * 24 * 60 * 60 * 1000 })); const lock = holdWriteLock(dbPath); @@ -411,11 +459,11 @@ describe("daemon-flavor busy handling", () => { expect(() => store.gc(new Set(["repo-a"]), 30 * 24 * 60 * 60 * 1000)).not.toThrow(); // Rows survived, so the map must too — otherwise the next reload() // would resurrect an entry the map had already dropped. - expect(store.entries["stale-busy"]).toBeDefined(); + expect(store.entries[key]).toBeDefined(); } finally { lock.release(); } - expect(rowCount(db, "stale-busy")).toBe(1); + expect(rowCount(db, key)).toBe(1); db.close(); }, 10_000); @@ -423,16 +471,17 @@ describe("daemon-flavor busy handling", () => { const dbPath = join(dir, "state.db"); const db = openStateDb(dbPath, "daemon"); const store = getBranchCacheStore(db); + const key = composeKey("repo-a", "doomed"); store.put("doomed", makeEntry({ repoName: "repo-a" })); const lock = holdWriteLock(dbPath); try { - expect(() => store.delete("doomed")).not.toThrow(); - expect(store.entries["doomed"]).toBeDefined(); + expect(() => store.delete(key)).not.toThrow(); + expect(store.entries[key]).toBeDefined(); } finally { lock.release(); } - expect(rowCount(db, "doomed")).toBe(1); + expect(rowCount(db, key)).toBe(1); db.close(); }, 10_000); }); @@ -487,7 +536,8 @@ describe("rekeyBranchCacheTable", () => { getBranchCacheStore().put("feature/x", makeEntry({ repoName: "remote:gitlab.com%2Fg%2Fr" })); const report = await rekeyBranchCacheTable(); expect(report.migrated).toEqual([]); - const row = getStateDb().query("SELECT repo FROM branch_cache WHERE branch = ?;").get("feature/x") as { repo: string }; + const row = getStateDb().query("SELECT repo FROM branch_cache WHERE branch = ?;") + .get(composeKey("remote:gitlab.com%2Fg%2Fr", "feature/x")) as { repo: string }; expect(row.repo).toBe("remote:gitlab.com%2Fg%2Fr"); }); @@ -495,7 +545,8 @@ describe("rekeyBranchCacheTable", () => { getBranchCacheStore().put("feature/y", makeEntry({ repoName: "ghost-repo" })); const report = await rekeyBranchCacheTable(); expect(report.retained).toEqual(["ghost-repo"]); - const row = getStateDb().query("SELECT repo FROM branch_cache WHERE branch = ?;").get("feature/y") as { repo: string }; + const row = getStateDb().query("SELECT repo FROM branch_cache WHERE branch = ?;") + .get(composeKey("ghost-repo", "feature/y")) as { repo: string }; expect(row.repo).toBe("ghost-repo"); expect(warnSpy).toHaveBeenCalled(); }); diff --git a/lib/state/branch-cache.ts b/lib/state/branch-cache.ts index f76801b1..8fa12342 100644 --- a/lib/state/branch-cache.ts +++ b/lib/state/branch-cache.ts @@ -147,21 +147,26 @@ function createStore(db: Database): BranchCacheStore { } function put(branch: string, entry: CacheEntry): void { + // Keyed by composeKey(entry.repoName, branch), not the bare branch: two + // repos with a same-named branch must land in different rows/map slots, + // never overwrite each other (the collision Task 10 fixes). The row's + // `branch` column and the map key are always the same composite string. + const key = composeKey(entry.repoName, branch); // The map update sits OUTSIDE the wrapper on purpose: it is this cycle's // freshly enriched truth and the thing handlers serve, so it must land // even when the row defers. (gc/delete keep the two together instead — // see below.) persistOrWarn("branch-cache", () => { db.query(UPSERT_SQL).run( - branch, + key, entry.repoName ?? null, entry.ticket !== null ? JSON.stringify(entry.ticket) : null, entry.linearId, entry.mr !== null ? JSON.stringify(entry.mr) : null, entry.fetchedAt, ); - }, { op: "put", branch }); - entries[branch] = entry; + }, { op: "put", branch: key }); + entries[key] = entry; } function del(branch: string): void { diff --git a/lib/worktree/__tests__/dispose.test.ts b/lib/worktree/__tests__/dispose.test.ts index aed1d695..f3dc096e 100644 --- a/lib/worktree/__tests__/dispose.test.ts +++ b/lib/worktree/__tests__/dispose.test.ts @@ -6,6 +6,7 @@ import { basename, dirname, join } from "path"; import { teamSettingsPath } from "../../rt-paths.ts"; import { setSetting } from "../../settings/write.ts"; import { closeStateDb, getBranchCacheStore, type CacheEntry } from "../../state/index.ts"; +import { branchOf } from "../../state/branch-cache.ts"; import { loadRegistry, saveRegistry, type TreeRecord } from "../registry.ts"; import { branchExistsLocalAsync, listWorktreesAsync, remoteRefExists } from "../git-async.ts"; import { hasFreshAttendantLease } from "../lease.ts"; @@ -761,7 +762,10 @@ describe("disposeTree against the real branch_cache store (identity-keyed)", () })); // Seeded exactly as cache-refresh.ts writes it: repoName is the same - // identity the daemon iterates the repo index under. + // identity the daemon iterates the repo index under. The store now keys + // its own map by composeKey(repoName, branch); the daemon's caller + // (worktree-reconciler.ts actOnTree) hands disposeTree a bare-keyed, + // this-repo-only view; reproduce that same remap here. const store = getBranchCacheStore(); store.put("feature-a", { ticket: null, @@ -770,11 +774,14 @@ describe("disposeTree against the real branch_cache store (identity-keyed)", () mr: { iid: 42, sha, state: "merged" } as unknown as CacheEntry["mr"], repoName: identityRepoName, }); + const cacheEntries = Object.fromEntries( + Object.entries(store.entries).map(([key, entry]) => [branchOf(key), entry]), + ); const deps: DisposeDeps = { repoName: identityRepoName, repoPath: repo, - cacheEntries: store.entries, + cacheEntries, emit: (type, data) => events.push({ type, data }), log: { info: () => {}, warn: () => {} }, killProcesses: false, @@ -801,11 +808,14 @@ describe("disposeTree against the real branch_cache store (identity-keyed)", () mr: { iid: 42, sha, state: "merged" } as unknown as CacheEntry["mr"], repoName: "acme", // pre-rekey legacy display name }); + const cacheEntries = Object.fromEntries( + Object.entries(store.entries).map(([key, entry]) => [branchOf(key), entry]), + ); const deps: DisposeDeps = { repoName: identityRepoName, repoPath: repo, - cacheEntries: store.entries, + cacheEntries, emit: (type, data) => events.push({ type, data }), log: { info: () => {}, warn: () => {} }, killProcesses: false, From 12f89670810bd18ab572f7473864caf62fdd170d Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 19:30:56 -0500 Subject: [PATCH 137/142] test: adapt dev-mode + home-init consumer fixtures to stricter detector/identity check Two earlier-committed behavior changes broke four consumer test fixtures: the dev-mode wrapper detector now requires a real marker (# mattstack-dev-mode or RT_LAUNCH_CWD) instead of treating any file's presence as dev mode, and commitInitialUserRepo now checks git identity before committing. Production code is unchanged; fixtures now plant a recognized wrapper and answer the git config user.name/email probes, matching the pattern already used in home-snapshot.test.ts's defaultResponders. --- commands/__tests__/home.test.ts | 10 ++++++++++ lib/__tests__/daemon-config.test.ts | 3 ++- lib/__tests__/intended-mode.test.ts | 4 ++-- 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/commands/__tests__/home.test.ts b/commands/__tests__/home.test.ts index c38419f6..4cd2b56f 100644 --- a/commands/__tests__/home.test.ts +++ b/commands/__tests__/home.test.ts @@ -172,6 +172,16 @@ class FakeSeam implements ExecSeam { async run(cmd: string[]): Promise { this.calls.push({ kind: "run", arg: cmd }); if (this.opts.failRun?.(cmd)) return { code: 1, stdout: "", stderr: "boom" }; + // commitInitialUserRepo probes identity before committing; default to + // "configured" so fixtures not exercising the no-identity path stay + // green, mirroring lib/daemon/__tests__/home-snapshot.test.ts's + // defaultResponders. + if (cmd[cmd.length - 2] === "config" && cmd[cmd.length - 1] === "user.name") { + return { code: 0, stdout: "rt test\n", stderr: "" }; + } + if (cmd[cmd.length - 2] === "config" && cmd[cmd.length - 1] === "user.email") { + return { code: 0, stdout: "rt@example.test\n", stderr: "" }; + } return { code: 0, stdout: "", stderr: "" }; } async writeFile(path: string, content: string): Promise { diff --git a/lib/__tests__/daemon-config.test.ts b/lib/__tests__/daemon-config.test.ts index 32f06d15..dfc1a58a 100644 --- a/lib/__tests__/daemon-config.test.ts +++ b/lib/__tests__/daemon-config.test.ts @@ -10,6 +10,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { mkdirSync, rmSync, writeFileSync } from "fs"; import { join } from "path"; import { activeLaunchdLabel, resolveApiPort } from "../daemon-config.ts"; +import { DEV_MODE_TAG } from "../dev-mode.ts"; const WRAPPER_PATH = join(process.env.HOME!, ".local", "bin", "rt"); @@ -24,7 +25,7 @@ describe("activeLaunchdLabel", () => { test("resolves to com.mattstack.daemon.dev in dev mode (wrapper present)", () => { mkdirSync(join(process.env.HOME!, ".local", "bin"), { recursive: true }); - writeFileSync(WRAPPER_PATH, "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + writeFileSync(WRAPPER_PATH, `#!/bin/sh\n${DEV_MODE_TAG}\nexit 0\n`, { mode: 0o755 }); expect(activeLaunchdLabel()).toBe("com.mattstack.daemon.dev"); }); }); diff --git a/lib/__tests__/intended-mode.test.ts b/lib/__tests__/intended-mode.test.ts index 626ebb0b..d747ccec 100644 --- a/lib/__tests__/intended-mode.test.ts +++ b/lib/__tests__/intended-mode.test.ts @@ -2,7 +2,7 @@ import { describe, test, expect, beforeEach, afterEach } from "bun:test"; import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; -import { resolveIntendedMode } from "../dev-mode.ts"; +import { resolveIntendedMode, DEV_MODE_TAG } from "../dev-mode.ts"; import { setSetting } from "../settings/write.ts"; let home: string; @@ -27,7 +27,7 @@ describe("resolveIntendedMode", () => { test("unset: derives from wrapper — script at ~/.local/bin/rt means dev", () => { mkdirSync(join(home, ".local", "bin"), { recursive: true }); - writeFileSync(join(home, ".local", "bin", "rt"), "#!/bin/sh\necho dev\n"); + writeFileSync(join(home, ".local", "bin", "rt"), `#!/bin/sh\n${DEV_MODE_TAG}\necho dev\n`); expect(resolveIntendedMode()).toEqual({ mode: "dev", provenance: "derived-from-wrapper" }); }); From 1e79eb39ffa9c0a8bb4163e27508b92050abdfbd Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 19:50:41 -0500 Subject: [PATCH 138/142] docs: regenerate command reference for wave-1 daemon flags and verbs Co-Authored-By: Claude Fable 5 --- website/docs/reference/daemon/status.mdx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/website/docs/reference/daemon/status.mdx b/website/docs/reference/daemon/status.mdx index 0d7c2410..bfd689ac 100644 --- a/website/docs/reference/daemon/status.mdx +++ b/website/docs/reference/daemon/status.mdx @@ -12,9 +12,15 @@ Show daemon status ## Usage ```bash -rt daemon status +rt daemon status [flags] ``` +## Arguments & flags + +| Flag / Arg | Type | Default | Description | +| --- | --- | --- | --- | +| [`--json`](/guides/common-flags) | boolean | `false` | Emit the verdict as JSON instead of the formatted lines | + _See code: [commands/daemon.ts › showStatus](https://github.com/m4ttstack/rt/blob/main/commands/daemon.ts)_ {/* generated by scripts/gen-docs.ts; edit prose in _partials, not here */} \ No newline at end of file From 730d00810be4737197850125f24d94d2a092a381 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 19:53:50 -0500 Subject: [PATCH 139/142] enrich: repo-scope daemon cache:read; SecretsTimeoutError.name (final review #1/#2) enrichBranches's daemon-first path now passes repoIdentity on cache:read, so a branch name shared across two tracked repos (main/master) no longer risks a suffix-match cross-repo hit. SecretsTimeoutError now sets its name, mirroring AgeKeyTimeoutError. --- lib/__tests__/enrich-cache-identity.test.ts | 48 +++++++++++++++++++++ lib/enrich.ts | 2 + lib/secrets/store.ts | 7 ++- 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/lib/__tests__/enrich-cache-identity.test.ts b/lib/__tests__/enrich-cache-identity.test.ts index 9dedb7ea..2b88cc2a 100644 --- a/lib/__tests__/enrich-cache-identity.test.ts +++ b/lib/__tests__/enrich-cache-identity.test.ts @@ -15,6 +15,14 @@ import * as linearModule from "../linear.ts"; import { enrichBranches } from "../enrich.ts"; import { closeStateDb, getBranchCacheStore } from "../state/index.ts"; import { composeKey } from "../state/branch-cache.ts"; +import { createCacheHandlers } from "../daemon/handlers/cache.ts"; +import { fakeStore } from "../daemon/__tests__/fake-cache-store.ts"; + +// Captured before any mock.module call... mock.module mutates the live +// namespace object in place, so restoring with the ORIGINAL binding (not a +// re-import) is what undoes it for every other test file sharing this process. +const realDaemonClient = await import("../daemon-client.ts"); +const realDaemonQuery = realDaemonClient.daemonQuery; let home: string; let realHome: string | undefined; @@ -27,6 +35,10 @@ beforeEach(() => { }); afterEach(() => { + mock.module("../daemon-client.ts", () => ({ + ...realDaemonClient, + daemonQuery: realDaemonQuery, + })); mock.restore(); closeStateDb(); process.env.HOME = realHome; @@ -80,3 +92,39 @@ describe("enrichBranches cold-start: repoName/key is the serialized remote ident expect(entries["scratch"]?.repoName).toBeUndefined(); }); }); + +describe("enrichBranches daemon-first path: cache:read is repo-scoped", () => { + test("two tracked repos both have branch 'main': a repoIdentity-scoped read returns repo A's entry, never repo B's", async () => { + const identityA = "remote:gitlab.com%2Facme%2Frepo-a"; + const identityB = "remote:gitlab.com%2Facme%2Frepo-b"; + const entries: Record = { + [composeKey(identityA, "main")]: { + linearId: "A-1", ticket: null, mr: null, fetchedAt: Date.now(), repoName: identityA, + }, + [composeKey(identityB, "main")]: { + linearId: "B-1", ticket: null, mr: null, fetchedAt: Date.now(), repoName: identityB, + }, + }; + // The real cache:read handler over an in-memory store: this exercises the + // actual scoping logic, not a stand-in for it. + const handlers = createCacheHandlers({ + cache: fakeStore(entries), + refreshCache: async () => {}, + } as any); + + mock.module("../daemon-client.ts", () => ({ + ...realDaemonClient, + daemonQuery: async (cmd: string, payload: any) => { + if (cmd !== "cache:read") throw new Error(`unexpected daemon command: ${cmd}`); + return handlers["cache:read"]!(payload); + }, + })); + + const result = await enrichBranches( + [{ path: "/tmp/repo-a", branch: "main" }], + "git@gitlab.com:acme/repo-a.git", + ); + + expect(result[0]?.linearId).toBe("A-1"); + }); +}); diff --git a/lib/enrich.ts b/lib/enrich.ts index d6b7b5d3..5988a17e 100644 --- a/lib/enrich.ts +++ b/lib/enrich.ts @@ -241,8 +241,10 @@ export async function enrichBranches( if (!options?.silent) { try { const { daemonQuery } = await import("./daemon-client.ts"); + const identity = identityForRemote(remoteUrl); const response = await daemonQuery("cache:read", { branches: branches.map(b => b.branch), + repoIdentity: identity, }); if (response?.ok && response.data) { diff --git a/lib/secrets/store.ts b/lib/secrets/store.ts index e261b048..0876b194 100644 --- a/lib/secrets/store.ts +++ b/lib/secrets/store.ts @@ -140,7 +140,12 @@ export function validateSlug(slug: string): void { } /** Thrown when a sops/keychain spawn does not exit in time (a locked keychain pops a GUI dialog and blocks until clicked). */ -export class SecretsTimeoutError extends Error {} +export class SecretsTimeoutError extends Error { + constructor(message: string) { + super(message); + this.name = "SecretsTimeoutError"; + } +} const DEFAULT_SECRETS_TIMEOUT_MS = 30_000; From e8d841706d0ead7e46a61e8dead20db6418ce896 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 20:14:12 -0500 Subject: [PATCH 140/142] ci: retrigger checks after docs regeneration From afe0f903bbf27d1db2e35792a9279639e0b3fc6f Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 20:17:05 -0500 Subject: [PATCH 141/142] docs: regenerate command reference after wave-2 merge (J5) --- website/docs/reference/daemon/index.mdx | 1 + website/docs/reference/daemon/log-level.mdx | 26 +++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 website/docs/reference/daemon/log-level.mdx diff --git a/website/docs/reference/daemon/index.mdx b/website/docs/reference/daemon/index.mdx index fecbec70..2991b00a 100644 --- a/website/docs/reference/daemon/index.mdx +++ b/website/docs/reference/daemon/index.mdx @@ -27,5 +27,6 @@ rt daemon | [`status`](status) | Show daemon status | | [`track`](track) | Per-repo background tracking (live/poll/off) | | [`logs`](logs) | Show daemon logs | +| [`log-level`](log-level) | Show or set the daemon's live log level | {/* generated by scripts/gen-docs.ts; edit prose in _partials, not here */} \ No newline at end of file diff --git a/website/docs/reference/daemon/log-level.mdx b/website/docs/reference/daemon/log-level.mdx new file mode 100644 index 00000000..041d3a55 --- /dev/null +++ b/website/docs/reference/daemon/log-level.mdx @@ -0,0 +1,26 @@ +--- +title: rt daemon log-level +sidebar_label: log-level +--- + +# rt daemon log-level + +`rt › daemon › log-level` + +Show or set the daemon's live log level + +## Usage + +```bash +rt daemon log-level +``` + +## Arguments & flags + +| Flag / Arg | Type | Default | Description | +| --- | --- | --- | --- | +| `` | select | | Omit to show the current level | + +_See code: [commands/daemon.ts › setLogLevel](https://github.com/m4ttstack/rt/blob/main/commands/daemon.ts)_ + +{/* generated by scripts/gen-docs.ts; edit prose in _partials, not here */} \ No newline at end of file From 68c2618f8db90211e57ffe18566660c0c18ffb61 Mon Sep 17 00:00:00 2001 From: Matthew Goodwin Date: Fri, 28 Aug 2026 20:52:05 -0500 Subject: [PATCH 142/142] chat-handlers.test: drop test for chat:unread-waking, a verb main's delivery-v2 removed --- lib/daemon/__tests__/chat-handlers.test.ts | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/lib/daemon/__tests__/chat-handlers.test.ts b/lib/daemon/__tests__/chat-handlers.test.ts index 93f4e9b9..f1941e82 100644 --- a/lib/daemon/__tests__/chat-handlers.test.ts +++ b/lib/daemon/__tests__/chat-handlers.test.ts @@ -174,23 +174,6 @@ test("chat:dm rejects a missing or empty body", async () => { expect(empty.ok).toBe(false); }); -test("chat:unread-waking reports what would wake a handle without advancing its cursor", async () => { - const h = freshHandlers(); - await h["chat:join"]({ room: "r", handle: "a" }); - await h["chat:join"]({ room: "r", handle: "b" }); - await h["chat:post"]({ room: "r", handle: "a", body: "@b hi" }); - const res1 = await h["chat:unread-waking"]({ handle: "b" }); - if (!res1.ok) throw new Error("unreachable"); - const first = res1.data; - expect(first).toMatchObject({ rooms: [{ room: "r", count: 1, mentions: 1 }] }); - // maxId is the watermark the tail's stream loop skips at or below; without - // it the tail cannot tell which wakes the catch-up already covered. - expect(first.rooms[0]!.maxId).toBeGreaterThan(0); - const res2 = await h["chat:unread-waking"]({ handle: "b" }); - if (!res2.ok) throw new Error("unreachable"); - expect(res2.data).toEqual(first); -}); - // R034: `limit: -1` reaches `ORDER BY id ASC LIMIT ?`, where SQLite treats a // negative LIMIT as unlimited, so a viewer/agent bug returns and // JSON-serializes an entire (100k-row) room on the event loop.