diff --git a/commands/__tests__/daemon-logs-render.test.ts b/commands/__tests__/daemon-logs-render.test.ts new file mode 100644 index 000000000..eb14e8b76 --- /dev/null +++ b/commands/__tests__/daemon-logs-render.test.ts @@ -0,0 +1,38 @@ +/** + * nativeStderrDisplay (showLogs' stale-crash mtime gate). + * + * daemon-stderr.log is rotated on daemon boot (lib/daemon-logger.ts), but a + * leftover file can still predate the *currently running* daemon (e.g. it was + * never rotated because the daemon has been up for days). This pins the + * show/hide + header decision without needing a live daemon or a real file. + */ + +import { describe, test, expect } from "bun:test"; +import { nativeStderrDisplay } from "../daemon.ts"; + +const NOW = 1_785_000_000_000; + +describe("nativeStderrDisplay", () => { + test("hides the block when the file predates the daemon's startedAt", () => { + const { show, header } = nativeStderrDisplay(NOW - 10_000, NOW); + expect(show).toBe(false); + expect(header).toBe("no crash since this daemon started"); + }); + + test("hides the block when the file mtime exactly equals startedAt", () => { + const { show } = nativeStderrDisplay(NOW, NOW); + expect(show).toBe(false); + }); + + test("shows the block, with the mtime in the header, when the file postdates startedAt", () => { + const mtimeMs = NOW + 5_000; + const { show, header } = nativeStderrDisplay(mtimeMs, NOW); + expect(show).toBe(true); + expect(header).toBe(`native stderr (captured ${new Date(mtimeMs).toISOString()})`); + }); + + test("fails open (shows) when the daemon's startedAt is unknown (nothing to compare against)", () => { + const { show } = nativeStderrDisplay(NOW - 999_999, null); + expect(show).toBe(true); + }); +}); diff --git a/commands/__tests__/daemon-status-render.test.ts b/commands/__tests__/daemon-status-render.test.ts index eb4ff2dfb..98e27b516 100644 --- a/commands/__tests__/daemon-status-render.test.ts +++ b/commands/__tests__/daemon-status-render.test.ts @@ -71,4 +71,41 @@ describe("statusLines", () => { expect(out).toContain("watching: 1 repo"); expect(out).not.toContain("1 repos"); }); + + // ── Task 10 ── + + test("a parked pid points at the flavor mismatch, not 'not running'", () => { + const out = plain({ state: "parked", pid: 42, holderFlavor: "prod" }); + expect(out).toContain("parked"); + expect(out).toContain("pid 42"); + expect(out).toContain("held by: prod"); + expect(out).not.toContain("installed but not running"); + }); + + test("alive-not-serving names the pid and the stuck detail", () => { + const out = plain({ state: "alive-not-serving", pid: 42, detail: "booting" }); + expect(out).toContain("process 42 is running but not answering rt.sock"); + expect(out).toContain("still booting"); + expect(out).not.toContain("installed but not running"); + }); + + test("alive-not-serving wedged/quarantined get their own detail lines", () => { + expect(plain({ state: "alive-not-serving", pid: 1, detail: "wedged" })).toContain("deadlocked"); + expect(plain({ state: "alive-not-serving", pid: 1, detail: "quarantined" })).toContain("recovered from a corrupt db"); + }); + + test("crash-looping surfaces the failure count and the last reason", () => { + const out = plain({ state: "crash-looping", failures: 4, reason: "EADDRINUSE" }); + expect(out).toContain("crash-looping"); + expect(out).toContain("4 failures"); + expect(out).toContain("EADDRINUSE"); + }); + + test("boot-failed surfaces the phase and reason, and points at rt daemon start", () => { + const out = plain({ state: "boot-failed", reason: "EADDRINUSE", phase: "api" }); + expect(out).toContain("boot failed"); + expect(out).toContain("phase: api"); + expect(out).toContain("EADDRINUSE"); + expect(out).toContain("rt daemon start"); + }); }); diff --git a/commands/__tests__/daemon-uninstall-start.test.ts b/commands/__tests__/daemon-uninstall-start.test.ts new file mode 100644 index 000000000..926f5dcc0 --- /dev/null +++ b/commands/__tests__/daemon-uninstall-start.test.ts @@ -0,0 +1,145 @@ +/** + * `rt daemon uninstall`/`start` (the CLI-side liveness guards, Task 14, + * S027/S030/S028-CLI). Fakes the tray over a real Bun.serve on + * TRAY_SOCK_PATH (same rig as commands/__tests__/settings-dev-mode.test.ts) + * and, where a scenario needs "the daemon is live", a real Bun.serve on + * DAEMON_SOCK_PATH answering /ping (isDaemonProcessRunning's pid check and + * probeSocketHolder/isDaemonRunning's socket ping are both exercised for + * real, never mocked module internals). + */ +import { afterEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from "fs"; +import { start, uninstall } from "../daemon.ts"; +import { + DAEMON_CONFIG_PATH, + DAEMON_PID_PATH, + DAEMON_SOCK_PATH, + RT_DIR, + TRAY_SOCK_PATH, + markDaemonInstalled, +} from "../../lib/daemon-config.ts"; +import { resolveIntendedMode } from "../../lib/dev-mode.ts"; + +let servers: ReturnType[] = []; +let logs: string[] = []; +const realLog = console.log; + +function captureLogs(): void { + logs = []; + console.log = (...args: unknown[]) => { logs.push(args.map(String).join(" ")); }; +} + +function serveTray(handlers: Record Response>): void { + servers.push(Bun.serve({ + unix: TRAY_SOCK_PATH, + fetch(req) { + const url = new URL(req.url); + const handler = handlers[url.pathname]; + return handler ? handler() : new Response("not found", { status: 404 }); + }, + })); +} + +/** A real listener on rt.sock that answers /ping (what both isDaemonRunning() + * (daemon-client.ts) and probeSocketHolder() (lib/daemon/park.ts) fetch. + * Flavor defaults to the CURRENT intended mode (not a hardcoded "prod") so + * start()'s post-liveness warnIfWrongFlavor() check never fires a spurious + * mismatch when this file runs after another test flips mattstack.mode in + * the shared isolated HOME `bun test` uses for the whole process). */ +function serveDaemonPing(body?: Record): void { + const resolvedBody = body ?? { ok: true, pid: 4242, flavor: resolveIntendedMode().mode }; + servers.push(Bun.serve({ + unix: DAEMON_SOCK_PATH, + fetch(req) { + const url = new URL(req.url); + if (url.pathname === "/ping") return Response.json(resolvedBody); + return new Response("not found", { status: 404 }); + }, + })); +} + +afterEach(() => { + console.log = realLog; + for (const s of servers) { try { s.stop(true); } catch { /* already stopped */ } } + servers = []; + for (const p of [DAEMON_SOCK_PATH, DAEMON_PID_PATH, TRAY_SOCK_PATH, DAEMON_CONFIG_PATH]) { + try { rmSync(p); } catch { /* absent */ } + } +}); + +describe("uninstall (liveness guard)", () => { + test("leaves rt.pid/daemon.json when isDaemonProcessRunning() says the daemon is alive", async () => { + mkdirSync(RT_DIR, { recursive: true }); + markDaemonInstalled(); + writeFileSync(DAEMON_PID_PATH, String(process.pid)); // this test process is genuinely alive + // tray unreachable: trayQuery('/daemon/stop') resolves null (no server on TRAY_SOCK_PATH) + + captureLogs(); + await uninstall(); + + expect(existsSync(DAEMON_PID_PATH)).toBe(true); // cleanupDaemonFiles did NOT run + expect(JSON.parse(readFileSync(DAEMON_CONFIG_PATH, "utf8")).installed).toBe(true); // markDaemonUninstalled did NOT run + expect(logs.join("\n")).toContain("launchctl bootout"); + }); + + test("leaves rt.sock/daemon.json when probeSocketHolder() finds a live holder (no rt.pid at all)", async () => { + mkdirSync(RT_DIR, { recursive: true }); + markDaemonInstalled(); + serveDaemonPing(); + + captureLogs(); + await uninstall(); + + expect(JSON.parse(readFileSync(DAEMON_CONFIG_PATH, "utf8")).installed).toBe(true); + expect(logs.join("\n")).toContain("launchctl bootout"); + }); + + test("cleans up rt.sock/rt.pid/daemon.json when nothing is alive", async () => { + mkdirSync(RT_DIR, { recursive: true }); + markDaemonInstalled(); + writeFileSync(DAEMON_PID_PATH, "999999"); // no such pid + writeFileSync(DAEMON_SOCK_PATH, ""); // stale file, not a real listener (probeSocketHolder's fetch fails) + + captureLogs(); + await uninstall(); + + expect(existsSync(DAEMON_PID_PATH)).toBe(false); + expect(JSON.parse(readFileSync(DAEMON_CONFIG_PATH, "utf8")).installed).toBe(false); + expect(logs.join("\n")).not.toContain("launchctl bootout"); + expect(logs.join("\n")).toContain("daemon fully uninstalled"); + }); +}); + +describe("start (kickstart escalation)", () => { + test("falls back to /daemon/restart when the tray acks /daemon/start but the socket never comes up", async () => { + mkdirSync(RT_DIR, { recursive: true }); + markDaemonInstalled(); + let restartCalled = false; + serveTray({ + "/daemon/start": () => Response.json({ ok: true }), + "/daemon/restart": () => { restartCalled = true; return Response.json({ ok: true }); }, + }); + + captureLogs(); + await start(); + + expect(restartCalled).toBe(true); + }, 20_000); + + test("escalation succeeds once /daemon/restart actually brings the socket up", async () => { + mkdirSync(RT_DIR, { recursive: true }); + markDaemonInstalled(); + serveTray({ + "/daemon/start": () => Response.json({ ok: true }), + "/daemon/restart": () => { + serveDaemonPing(); + return Response.json({ ok: true }); + }, + }); + + captureLogs(); + await start(); + + expect(logs.join("\n")).toContain("daemon started"); + }, 20_000); +}); diff --git a/commands/__tests__/probe-pid-alive.test.ts b/commands/__tests__/probe-pid-alive.test.ts new file mode 100644 index 000000000..e699d4569 --- /dev/null +++ b/commands/__tests__/probe-pid-alive.test.ts @@ -0,0 +1,19 @@ +import { describe, test, expect } from "bun:test"; +import { probePidAlive } from "../daemon.ts"; +import { readSupervisionState } from "../../lib/daemon/supervision-state.ts"; + +describe("probePidAlive", () => { + // Regression: the lsof fallback must exclude the CALLING process itself. + // showStatus opens a bun:sqlite handle on state.db (inside RT_DIR) via + // readSupervisionState() immediately before this probe runs ... `lsof +D + // RT_DIR` then legitimately reports the calling CLI process as a live + // holder of the directory, with no daemon involved at all. Without the + // process.pid filter this self-matches and a genuinely dead daemon + // (no recorded pid, no breadcrumb pid) misclassifies as alive. + test("a state.db handle held by THIS process does not self-match as a live daemon", async () => { + readSupervisionState(); // opens (and keeps open) the isolated HOME's state.db + const result = await probePidAlive(null, undefined); + expect(result.alive).toBe(false); + expect(result.pid).toBeNull(); + }); +}); diff --git a/commands/daemon.ts b/commands/daemon.ts index ab571ad6a..8d492ab50 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -25,6 +25,8 @@ import { existsSync, mkdirSync, readdirSync, readFileSync, statSync, unlinkSync, import { bold, dim, green, yellow, red, reset } from "../lib/tui.ts"; import { isDaemonInstalled, + isDaemonProcessRunning, + activeLaunchdLabel, markDaemonInstalled, markDaemonUninstalled, cleanupDaemonFiles, readDaemonPid, RT_DIR, @@ -35,6 +37,8 @@ import { daemonQuery, isDaemonRunning, trayQuery } from "../lib/daemon-client.ts import { classifyDaemonStatus, type DaemonStatusVerdict } from "../lib/daemon-status.ts"; import { resolveIntendedMode, currentMode, type IntendedMode } from "../lib/dev-mode.ts"; import { probeSocketHolder } from "../lib/daemon/park.ts"; +import { readBreadcrumb, readSupervisionState } from "../lib/daemon/supervision-state.ts"; +import { runCapture } from "../lib/subprocess.ts"; import { isGitLabRemote } from "../lib/enrich.ts"; import type { CacheKind, RepoTrackingEntry } from "../lib/repo-tracking.ts"; import { loadRepoTracking, loadMachineRepoTracking, loadMachineRepoTrackingRaw, saveRepoTrackingRaw, grants, parseCachesArg, CACHE_KINDS, DEFAULT_PROJECT_MRS_WINDOW_DAYS, teamNamesIdentity } from "../lib/repo-tracking.ts"; @@ -222,7 +226,20 @@ export async function uninstall(): Promise { console.log(` ${green}✓${reset} removed legacy launchd plist`); } - // 3. Clear install flag + sock/pid files. + // 3. A failed/absent tray stop must never delete rt.sock/rt.pid/daemon.json + // out from under a daemon that's actually still alive (that would orphan + // it, still running, launchd-supervised, but rt's own bookkeeping says + // uninstalled). Check both liveness signals: the recorded pid, and whether + // anything still answers on rt.sock (a daemon can be alive with no + // matching rt.pid, e.g. after a crash-and-respawn under launchd). + const stillAlive = isDaemonProcessRunning() || (await probeSocketHolder()) !== null; + if (stillAlive) { + console.log(`\n ${yellow}⚠${reset} daemon is still running, leaving rt.sock/rt.pid/daemon.json in place`); + console.log(` ${dim}Fix: ${bold}launchctl bootout gui/$UID/${activeLaunchdLabel()}${reset}\n`); + return; + } + + // 4. Clear install flag + sock/pid files. markDaemonUninstalled(); cleanupDaemonFiles(); console.log(` ${green}✓${reset} cleared install flag`); @@ -256,16 +273,31 @@ export async function start(): Promise { } console.log(` ${dim}starting ${intended.mode} daemon via tray…${reset}`); + if (await pollForDaemonUp(intended)) return; + + // The tray acked /daemon/start, but SMAppService can register a job that + // never actually launches (still booting, crash-looping, etc.); kick it + // via /daemon/restart, which forces launchd to invoke it, rather than + // leaving the operator staring at "check logs" for something a retry fixes. + console.log(` ${dim}not up yet, escalating to restart (kickstart)…${reset}`); + const restartResult = await trayQuery("/daemon/restart", "POST"); + if (restartResult?.ok && (await pollForDaemonUp(intended))) return; + + console.log(`\n ${yellow}daemon starting… check logs: rt daemon logs${reset}\n`); +} + +/** Shared poll loop for start()'s initial wait and its kickstart escalation. */ +async function pollForDaemonUp(intended: IntendedMode): Promise { for (let i = 0; i < 12; i++) { await Bun.sleep(250); if (await isDaemonRunning()) { if (!(await warnIfWrongFlavor("start", intended))) { console.log(`\n ${green}✓ daemon started${reset}\n`); } - return; + return true; } } - console.log(`\n ${yellow}daemon starting… check logs: rt daemon logs${reset}\n`); + return false; } export async function stop(): Promise { @@ -316,8 +348,53 @@ export async function restart(): Promise { // ─── Status ────────────────────────────────────────────────────────────────── -export async function showStatus(): Promise { +/** + * Raw OS-level liveness, independent of rt.sock. Tries a direct pid check + * first against every pid this HOME actually recorded: rt.pid, then the + * boot breadcrumb's pid (the breadcrumb survives failures rt.pid never gets + * written for, per Ruling P1), before falling back to a last-resort scan. + * + * That scan is `lsof +D `, not the brief's suggested system-wide + * `pgrep -f 'rt --daemon|lib/daemon.ts'`: a raw pgrep matches ANY rt daemon + * on the machine regardless of which HOME started it, and on an ordinary dev + * workstation there usually IS one (the developer's own real daemon), so a + * pgrep-based check on an isolated/alternate HOME reliably misreports a dead + * boot attempt as alive-not-serving (verified live against this repo's own + * dev daemon while writing the e2e test below). `lsof +D` instead asks "does + * any process hold a file open under THIS HOME's rt dir", home-scoped by + * construction, immune to that false positive and to pid-reuse. Only worth + * calling once both `status` and a plain ping have already failed. + * + * The CALLER itself is excluded from the lsof result: `showStatus` opens its + * own `bun:sqlite` handle on `state.db` (inside RT_DIR) via + * `readSupervisionState()` just before this probe runs, so `lsof +D RT_DIR` + * legitimately reports the calling CLI process as a live holder of the + * directory, with no daemon involved at all. Left unfiltered, a genuinely + * dead daemon self-matches and misclassifies as alive-not-serving/parked; + * this only failed to show up in manual testing because incidental work + * happened to separate the state.db open from the lsof call by enough time + * for state.db's own transient lock window to close, an accident of + * timing, not a guarantee. + */ +export async function probePidAlive(recordedPid: number | null, breadcrumbPid?: number): Promise<{ alive: boolean; pid: number | null }> { + for (const candidate of [recordedPid, breadcrumbPid ?? null]) { + if (candidate === null) continue; + try { + process.kill(candidate, 0); + return { alive: true, pid: candidate }; + } catch { /* not this one, try the next candidate */ } + } + const { stdout } = await runCapture(["lsof", "-t", "+D", RT_DIR], { timeoutMs: 3000 }); + const pids = stdout.trim().split(/\s+/).filter(Boolean).map(Number) + .filter((n) => !isNaN(n) && n !== process.pid); + return pids.length > 0 ? { alive: true, pid: pids[0]! } : { alive: false, pid: recordedPid }; +} + +export async function showStatus(args: string[] = []): Promise { + const json = args.includes("--json"); + if (!isDaemonInstalled()) { + if (json) return void console.log(JSON.stringify({ ok: true, state: "not-installed" })); console.log(` ${dim}○${reset} not installed ${dim}(run rt daemon install)${reset}\n`); return; } @@ -326,14 +403,40 @@ export async function showStatus(): Promise { // A failed status query does NOT mean the daemon is down — it answers `ping` // in a fraction of the budget a loaded `status` needs. Establish liveness // before reporting, and only pay for the probe when nothing came back. - const alive = classifyDaemonStatus.needsLivenessProbe(response) ? await isDaemonRunning() : false; + const pingOk = classifyDaemonStatus.needsLivenessProbe(response) ? await isDaemonRunning() : false; + const recordedPid = readDaemonPid() ?? null; + + // Ping ALSO failed: the only remaining ground is the pid/breadcrumb/kv + // trail Task 9 left behind. Read it here, once, rather than on every status + // call, since it's the uncommon path. + let pidAlive: boolean | undefined; + let pid = recordedPid; + let breadcrumb: ReturnType | undefined; + let supervision: ReturnType | undefined; + if (classifyDaemonStatus.needsPidProbe(response, pingOk)) { + breadcrumb = readBreadcrumb(); + // The kv tier can be legitimately empty (or reflect nothing useful) when + // a failure happened before state.db ever opened (Ruling P1). The + // breadcrumb read above is what classifyDaemonStatus falls back to then. + supervision = readSupervisionState(); + const probed = await probePidAlive(recordedPid, breadcrumb?.pid); + pidAlive = probed.alive; + pid = probed.pid; + } + const verdict = classifyDaemonStatus({ installed: true, response, - alive, - pid: readDaemonPid() ?? null, + pingOk, + pid, + pidAlive, + intendedFlavor: resolveIntendedMode().mode, + breadcrumb, + supervision, }); + if (json) return void console.log(JSON.stringify({ ok: true, ...verdict })); + for (const line of statusLines(verdict, Date.now())) console.log(line); if (verdict.state === "running") { @@ -417,6 +520,46 @@ export function statusLines(verdict: DaemonStatusVerdict, now: number): string[] return lines; } + if (verdict.state === "parked") { + const lines = [` ${yellow}◐${reset} parked ${dim}(pid ${verdict.pid}, another flavor owns rt.sock)${reset}`]; + lines.push( + verdict.holderFlavor + ? ` ${dim}held by: ${verdict.holderFlavor}${reset}` + : ` ${dim}waiting for the intended flavor to take rt.sock${reset}`, + ); + lines.push(` ${dim}check: rt settings dev-mode${reset}`); + return lines; + } + + if (verdict.state === "alive-not-serving") { + const detailLine = { + booting: "still booting", + wedged: "reached ready but stopped answering (likely deadlocked)", + quarantined: "recovered from a corrupt db but still not answering", + }[verdict.detail]; + return [ + ` ${yellow}●${reset} process ${verdict.pid} is running but not answering rt.sock`, + ` ${dim}${detailLine}${reset}`, + ` ${dim}check: rt daemon logs -t${reset}`, + ]; + } + + if (verdict.state === "crash-looping") { + return [ + ` ${red}●${reset} crash-looping ${dim}(${verdict.failures} failures recently)${reset}`, + ` ${dim}last reason: ${verdict.reason}${reset}`, + ` ${dim}check: rt daemon logs -t${reset}`, + ]; + } + + if (verdict.state === "boot-failed") { + return [ + ` ${red}●${reset} boot failed ${dim}(phase: ${verdict.phase})${reset}`, + ` ${dim}reason: ${verdict.reason}${reset}`, + ` ${dim}run: rt daemon start${reset}`, + ]; + } + if (verdict.state === "not-running") { const lines = [` ${red}●${reset} installed but not running`]; if (verdict.pid) lines.push(` ${dim}last pid: ${verdict.pid}${reset}`); @@ -706,6 +849,25 @@ export async function manageTracking(args: string[] = []): Promise { // ─── Logs ──────────────────────────────────────────────────────────────────── +/** + * Decides whether showLogs' native-stderr block is worth printing, and its + * header. `daemon-stderr.log` is rotated on open (daemon-logger.ts) but the + * fresh file can still be non-empty from a crash that happened before *this* + * boot's rotation ran (e.g. a bun panic mid-startup), so staleness is judged + * by mtime vs. the live daemon's startedAt, not by rotation alone. A `null` + * startedAt (daemon unreachable, nothing to compare against) fails open: + * show it, since a down daemon is exactly when the last crash matters most. + */ +export function nativeStderrDisplay( + mtimeMs: number, + daemonStartedAt: number | null, +): { show: boolean; header: string } { + if (daemonStartedAt !== null && mtimeMs <= daemonStartedAt) { + return { show: false, header: "no crash since this daemon started" }; + } + return { show: true, header: `native stderr (captured ${new Date(mtimeMs).toISOString()})` }; +} + /** * Show daemon logs. * @@ -723,16 +885,28 @@ export async function showLogs(args: string[] = []): Promise { // Surface captured native stderr first — these are bun panics/asserts that // bypassed the JS-side interceptor and were caught by the swift-shim's - // freopen of fd 2. If non-empty, the most recent crash leads the output. + // freopen of fd 2. Only shown when it postdates the running daemon's boot, + // otherwise it's a previous life's crash, not "the most recent crash". const stderrPath = join(LOG_DIR, "daemon-stderr.log"); if (existsSync(stderrPath)) { const content = readFileSync(stderrPath, "utf8").trim(); if (content) { - console.log(`\n ${red}${bold}native stderr${reset} ${dim}(${stderrPath})${reset}`); - for (const line of content.split("\n").slice(-20)) { - console.log(` ${red}${line}${reset}`); + const mtimeMs = statSync(stderrPath).mtimeMs; + const ping = await daemonQuery("ping"); + const daemonStartedAt = + ping && (ping as any).ok && typeof (ping as any).startedAt === "number" + ? ((ping as any).startedAt as number) + : null; + const { show, header } = nativeStderrDisplay(mtimeMs, daemonStartedAt); + if (show) { + console.log(`\n ${red}${bold}${header}${reset} ${dim}(${stderrPath})${reset}`); + for (const line of content.split("\n").slice(-20)) { + console.log(` ${red}${line}${reset}`); + } + console.log(""); + } else { + console.log(`\n ${dim}${header}${reset}\n`); } - console.log(""); } } diff --git a/docs/daemon-api-auth.md b/docs/daemon-api-auth.md new file mode 100644 index 000000000..3e2b1584b --- /dev/null +++ b/docs/daemon-api-auth.md @@ -0,0 +1,62 @@ +# The :9401 trust boundary + +How the daemon's REST/WS surface decides who to trust, and the follow-up +wiring two standalone modules from this phase still need in sibling-owned +files. + +## The model + +- **No `Origin` header at all** (the CLI, the Swift tray, rt-client from a + Bun/Node process, mr-board, gitq, the VS Code extension): unaffected, no + gate applies beyond what already existed. None of today's consumers send + an `Origin` header to `:9401`. +- **A browser `Origin` header is present**: trusted only if the request + presents the local `X-RT-Token` (`?token=` query param for `/ws`, since + browsers cannot set custom headers on a WS handshake) OR the Origin is on + the `rt.trustedBrowserOrigins` settings allowlist (see + `packages/rt-client/src/settings/registry-defs.ts`; `docs/settings-architecture.md` + is the settings-system contract). Otherwise: no `Access-Control-Allow-Origin` + on REST reads (default-deny CORS), and a 403 on `/ws`. +- **Mutating routes** (every method except GET/HEAD/OPTIONS, plus + `/api/secrets` and `/api/notifications` despite being GETs) require the + local `X-RT-Token` regardless of Origin... this is the CSRF defense against + a browser form/simple-request bypassing CORS preflight entirely, and it is + orthogonal to the Origin check above. + +See `lib/daemon/api-auth.ts` (`isBrowserRequestTrusted`, `needsToken`, +`getTrustedBrowserOrigins`) and `lib/daemon/api-server.ts` +(`buildCorsHeaders`, the `/ws` gate in `fetch()`) for the implementation. + +## Follow-up wiring for sibling-owned files (not done in this job) + +**S010** (`lib/daemon/handlers/worktree.ts`): `lib/daemon/git-ref-validation.ts` +exports `validateGitRef(ref)`. Call it right after `payload.branch` is read +(around `worktree.ts:282`) and return `{ ok: false, error }` on a rejection +BEFORE any `runGit` call reaches it... that single call site also covers the +weaker secondary instance in `divergence()` (`worktree.ts:211-213`), since +both read the same `branch` value. + +**S050** (`lib/daemon/freshness.ts`): `lib/daemon/redact-credentials.ts` +exports `redactCredentials(text)`. Wrap every log/error interpolation of a +remote URL with it... the audit names `freshness.ts:142, 148, 275, 279` as the +current call sites. + +**S043 caller side** (`lib/daemon.ts`): `lib/daemon/api-server.ts` exports +`ApiPortInUseError` (a named `Error` subclass with `.name === "ApiPortInUseError"` +and `.port`). Catch it around the `startApiServer()` call and park-and-retry +with backoff instead of letting it reach the top-level crash path; any other +error out of `startApiServer()` is a genuine misconfiguration and should keep +crashing as it does today. + +## Known gap: GET routes that trigger real work stay ungated + +`GET /api/cache?maxAgeMs=` can force a full cache refresh, and +`GET /api/sdm/recents` spawns `sdm status`; neither requires the local token. +Default-deny CORS stops an untrusted browser Origin from reading the +response, but a plain cross-origin GET needs no preflight, so the request +still lands and the work still runs even when the response is unreadable. +This is deliberately not fixed here: gating these routes risks breaking +non-browser REST consumers that read them untokened today (the tray, +editor extensions), and there was no time in this pass to audit every such +consumer, which the audit's own fixer notes flag as the real risk of +tokening reads. Tracked as an open follow-up, not a silently-closed finding. diff --git a/docs/daemon-runner-health.md b/docs/daemon-runner-health.md index 95b5ae629..f9ee31b15 100644 --- a/docs/daemon-runner-health.md +++ b/docs/daemon-runner-health.md @@ -1,184 +1,9 @@ -# Daemon & Runner Code-Health Audit +# Daemon runner health (superseded) -Reference analysis for the `rt` daemon ([lib/daemon.ts](../lib/daemon.ts) and [lib/daemon/](../lib/daemon/)) and the runner TUI ([commands/runner.tsx](../commands/runner.tsx)). +This document audited subsystems (process-manager, remedy-engine, +runner.tsx, workspace-sync) that no longer exist. It is retained only as +a redirect. -Both subsystems are central — the daemon owns all process lifecycle and long-lived state; the runner is our flagship feature and the primary surface that touches that state. Defects here leak processes, corrupt UI state, or silently drop features without any obvious error. This document exists to catalog known issues so they can be fixed deliberately rather than re-discovered. - -## Legend - -- **P0** — Active correctness bug or resource leak with user-visible impact. -- **P1** — Reliability issue, silent feature drop, or unbounded growth under realistic use. -- **P2** — Edge case or latent bug that will bite under specific conditions. -- **P3** — Code quality / testability — not a bug, but the surrounding code will rot without it. - -Each item is scoped by file:line-link so a fix PR can jump straight to the site. - ---- - -## 1. Orphaned processes - -### ~~P0 — Child process groups are not reaped on kill~~ ✅ FIXED -[lib/daemon/process-manager.ts:34-54](../lib/daemon/process-manager.ts#L34-L54), [lib/daemon/process-manager.ts:140-147](../lib/daemon/process-manager.ts#L140-L147), [lib/daemon/process-manager.ts:182-190](../lib/daemon/process-manager.ts#L182-L190) - -`Bun.spawn` now passes `detached: true` so the child is a session/pgroup leader (pgid == pid). A new `killGroup(pid, signal)` helper sends signals to `-pid` with guards against pid ≤ 1 and ESRCH. Both the kill() path and the existing-process eviction path now use it. New regression test [`process-manager.test.ts`](../lib/daemon/__tests__/process-manager.test.ts) `"kill reaps grandchildren (detached pgroup)"` verifies the fix by spawning backgrounded sleepers, killing the parent, and asserting every grandchild pid is gone. - -### ~~P0 — `process:respawn` drops the remedy subscription~~ ✅ FIXED -[lib/daemon.ts:720-726](../lib/daemon.ts#L720-L726) - -Added `remedyEngine.onSpawn(id)` after `processManager.respawn(id)`. Re-uses stored `processMeta` (cwd/cmd unchanged across respawn). - -### ~~P1 — Warm (SIGSTOP) processes survive daemon death as stopped zombies~~ ✅ FIXED -[lib/daemon/state-store.ts:28-104](../lib/daemon/state-store.ts#L28-L104), [lib/daemon.ts:1398-1411](../lib/daemon.ts#L1398-L1411), [lib/daemon/process-manager.ts:168-174](../lib/daemon/process-manager.ts#L168-L174) - -`StateStore` now persists `{state, pid}` per id. `ProcessManager.spawn` records pid on spawn; terminal transitions (`stopped`/`crashed`) clear it. `reconcileAfterRestart()` returns orphans (non-stopped with a pid), which the daemon reaps via `killGroup(pid, SIGCONT)` followed by `killGroup(pid, SIGKILL)`. Legacy `{id: state}` persistence format is still accepted on load. Regression coverage in [state-store.test.ts](../lib/daemon/__tests__/state-store.test.ts#L200-L230). - -### ~~P1 — `subscribeToOutput` unsubscribers leave empty Sets behind~~ ✅ FIXED -[lib/daemon/process-manager.ts:100-109](../lib/daemon/process-manager.ts#L100-L109) - -Unsubscribe now deletes the Set from `outputHooks` once its size reaches zero. - ---- - -## 2. Memory leaks - -### ~~P0 — `:wt` git watcher is torn down every reconcile tick~~ ✅ FIXED -[commands/runner.tsx:1580-1584](../commands/runner.tsx#L1580-L1584) - -Reconcile loop now strips a trailing `:wt` from the key before checking `activeLaneIds`, so the companion watcher lives and dies with its parent lane instead of being torn down every tick. - -### ~~P1 — Global-remedy `fs.watch` has no handle and no cleanup~~ ✅ FIXED -[lib/daemon.ts:132-163](../lib/daemon.ts#L132-L163), [lib/daemon.ts:1320](../lib/daemon.ts#L1320) - -Handle retained in `globalRemedyWatcher` and closed in `cleanup()`. - -### ~~P1 — `ProcessManager.spawnConfigs` and `outputHooks` grow unbounded~~ ✅ FIXED -[lib/daemon.ts:770-779](../lib/daemon.ts#L770-L779), [commands/runner.tsx](../commands/runner.tsx) - -Added `process:remove` IPC handler that tears down all five daemon-side maps (remedy state, attach socket, log buffer, process config, state-store entry). The runner now calls it from every entry-deletion path (remove-entry, remove-lane, reset). Entries with fresh ids no longer leak daemon-side state. - -### ~~P2 — `RemedyEngine` can create orphan state for globals-only matches~~ ✅ FIXED -[lib/daemon.ts:773](../lib/daemon.ts#L773) - -`process:remove` handler (added in Session B) calls `remedyEngine.unregister(id)` before tearing down the other maps, so globals-only state is cleaned up when the runner deletes an entry. - -### ~~P2 — `refreshCache` has no in-flight guard~~ ✅ FIXED -[lib/daemon.ts:390-407](../lib/daemon.ts#L390-L407) - -Split into a coalescing `refreshCache()` wrapper and the original logic as `refreshCacheImpl()`. Concurrent callers await the same promise. - -### ~~P2 — outputHooks Map retains empty Sets~~ ✅ FIXED -Covered by the subscribeToOutput unsubscribe cleanup above. - ---- - -## 3. Bugs - -### ~~P1 — Invalid JSON during save wipes all globals~~ ✅ FIXED -[lib/runner-store.ts:154-168](../lib/runner-store.ts#L154-L168), [lib/daemon.ts:137-158](../lib/daemon.ts#L137-L158) - -`loadGlobalRemedies` now throws on parse failure / non-array shape; missing file still returns `[]`. Daemon watcher callback catches and logs without calling `reloadGlobals`, so live rules persist through transient invalid states. - -### ~~P1 — Global-remedy watcher is not debounced~~ ✅ FIXED -[lib/daemon.ts:147-161](../lib/daemon.ts#L147-L161) - -100ms settle timer collapses the rename+change burst into a single reload. - -### ~~P1 — Concurrent `cache:refresh` via REST API~~ ✅ FIXED -Resolved by the `refreshCache` in-flight guard above. - -### ~~P2 — `entry.id` basename collision~~ ✅ FIXED -[lib/runner-store.ts:410-420](../lib/runner-store.ts#L410-L420) - -`normalizeLane` now detects within-lane entry id duplicates and appends a 6-char sha1 of the worktree path to the loser (`~a1b2c3`). Collisions no longer silently alias two processes' PTY/state. - -### ~~P2 — `onSpawn` re-merge uses stale meta for callers that pass neither cwd nor cmd~~ ✅ FIXED -[lib/daemon.ts:753-761](../lib/daemon.ts#L753-L761), [lib/daemon/process-manager.ts:245-247](../lib/daemon/process-manager.ts#L245-L247) - -`process:respawn` — the only caller that used to pass no cwd/cmd — now pulls them from `processManager.getSpawnConfig(id)` so globals-only matches survive respawn. Other `onSpawn` callers already pass cwd/cmd from their handler payload. - -### ~~P2 — `reloadGlobals` didn't subscribe for processes registered with empty remedies~~ ✅ FIXED -[lib/daemon/remedy-engine.ts:108-127](../lib/daemon/remedy-engine.ts#L108-L127) - -If `register()` was called with no per-entry remedies and `onSpawn` early-returned (no subscription because merged was empty), a later `reloadGlobals` that added matching rules updated `s.remedies` but never subscribed to output — so nothing fired. Found while writing the RemedyEngine test suite. Fix: `reloadGlobals` now subscribes on empty→non-empty and unsubscribes on non-empty→empty. Regression test in [remedy-engine.test.ts "reloadGlobals mid-flight re-merges active states"](../lib/daemon/__tests__/remedy-engine.test.ts#L300-L319). - -### ~~P3 — Dead constant~~ ✅ FIXED -[lib/daemon.ts:48](../lib/daemon.ts#L48) - -`LINEAR_REFRESH_INTERVAL_MS` removed. - -### ~~P3 — Compaction ordering is interleaving-sensitive~~ ✅ FIXED -[lib/runner-store.ts:338-348](../lib/runner-store.ts#L338-L348) - -Each entry now gets its absolute input index as its position. Groups inherit the position of their first member. Output order follows input order deterministically. - -### ~~P3 — State-store permits invalid transitions silently~~ ✅ FIXED -[lib/daemon/state-store.ts:111-119](../lib/daemon/state-store.ts#L111-L119), [lib/daemon.ts:1403-1407](../lib/daemon.ts#L1403-L1407) - -Forced transitions remain permitted (needed for kill-of-warm, reconcile, etc.) but `StateStore` now exposes `onInvalidTransition(cb)` and fires it when the move isn't in `VALID_TRANSITIONS`. The daemon wires this to its log so drift surfaces in the daemon log instead of being silently swallowed. - ---- - -## 4. Code organization / testability - -### ~~P1 — `lib/daemon.ts` is 1400+ lines with no seams~~ ✅ FIXED -[lib/daemon.ts](../lib/daemon.ts), [lib/daemon/handlers/](../lib/daemon/handlers/) - -`handleCommand` split into four domain modules behind a routed-lookup-first-then-switch dispatch: - -- [lib/daemon/handlers/process.ts](../lib/daemon/handlers/process.ts) — `process:spawn|kill|respawn|remove|start|stop|restart|list|state|states|logs|attach-info|suspend|resume` -- [lib/daemon/handlers/cache.ts](../lib/daemon/handlers/cache.ts) — `cache:read|refresh` + `branch:enrich` -- [lib/daemon/handlers/remedy.ts](../lib/daemon/handlers/remedy.ts) — `remedy:set|clear|drain` -- [lib/daemon/handlers/proxy.ts](../lib/daemon/handlers/proxy.ts) — `proxy:start|stop|set-upstream|status|list` - -Each module is a factory that takes a `HandlerContext` and returns a `HandlerMap`. Daemon.ts constructs the ctx once and merges the maps into `routedHandlers`; `handleCommand` does `routedHandlers[cmd] ?? switch` so non-extracted commands (ping, hooks:*, repos, ports, status, tcc:check, notifications*, tray:status, group:*, port:*, shutdown) remain inline because they read daemon-local state (watchers, repos index, notifications, port allocator, groups) that wouldn't benefit from being pushed out. - -Live cache access goes through `ctx.cache.entries` — `loadCache()` now mutates `cache.entries` in place instead of reassigning, so handlers see disk reloads without plumbing getters. - -### ~~P1 — `lib/runner-store.ts` doubled in size with no tests~~ ✅ FIXED -[lib/runner-store/compact.ts](../lib/runner-store/compact.ts), [lib/__tests__/runner-store-compact.test.ts](../lib/__tests__/runner-store-compact.test.ts) - -Added 7 round-trip tests and physically extracted `normalizeRemedy`, `normalizeEntry`, `compactEntries` (and their helpers) to `lib/runner-store/compact.ts`. runner-store.ts now re-exports `compactEntries`/`normalizeEntry` and delegates all compact↔expand logic. The extracted module imports only types from `runner-store.ts`, so there's no runtime cycle. - -### ~~P1 — `commands/runner.tsx` is 2500+ lines~~ ✅ PARTIALLY FIXED -[lib/runner/git-watchers.ts](../lib/runner/git-watchers.ts) - -Git watcher setup, per-lane debounce, and linked-worktree `.git/worktrees//HEAD` handling extracted into `createGitWatcherPool(onChange) → { sync, dispose }`. `repoGitDir`, `readCurrentBranch`, `readCurrentBranchAsync` now live there too. runner.tsx drops from 2500 → ~2430 lines and no longer imports `fs.watch` or `FSWatcher` at all. Remaining deferrals (keymap-handler split, `reconcileWatchers` module) are cosmetic and not bugs. - -### ~~P2 — `ProcessManager.spawn` does too many things~~ ✅ PARTIALLY FIXED -[lib/daemon/process-manager.ts:31-44](../lib/daemon/process-manager.ts#L31-L44) - -Port eviction is now an async `evictPort(port)` helper — no more synchronous `Bun.spawnSync(["sh","-c","lsof..."])` stalling the event loop. The broader decomposition of `spawn()` is deferred with the other structural refactors. - -### ~~P2 — No unit tests for `RemedyEngine` lifecycle invariants~~ ✅ FIXED -[lib/daemon/__tests__/remedy-engine.test.ts](../lib/daemon/__tests__/remedy-engine.test.ts) - -Added 13 tests driving `register`/`onSpawn`/`unregister` against a `FakeProcessManager` stub. Covers all five invariants (hook accumulation, orphan on removal, concurrent triggers, cooldown gating, double-respawn guard) plus pattern handling (array OR, invalid regex tolerance, ANSI stripping) and global remedies (cwdContains match/non-match, reload mid-flight re-merge). Writing the tests uncovered the latent `reloadGlobals`-doesn't-subscribe bug fixed above. - -### ~~P3 — Repeated sync git reads on runner startup~~ ✅ FIXED -[commands/runner.tsx:483-498](../commands/runner.tsx#L483-L498), [commands/runner.tsx:2527-2545](../commands/runner.tsx#L2527-L2545) - -Added async `readCurrentBranchAsync` (uses `Bun.spawn` + `proc.exited`) and parallelized the startup fan-out with `Promise.all`. Cost is now ~max-spawn-time, not N×spawn-time. - ---- - -## Status summary (audit sweep) - -All correctness, leak, and resource-safety items from the audit have been addressed: - -- **P0 / P1 bugs & leaks:** all fixed — pgroup reaping, remedy respawn subscription, `:wt` watcher, global-remedy hardening, refreshCache coalescing, warm-process crash recovery, empty-Set cleanup, map growth. -- **P2 correctness:** all fixed — entry.id collision salting, onSpawn meta passthrough, reloadGlobals missed-subscribe, RemedyEngine lifecycle tests, state-store invalid-transition observability. -- **P3:** all fixed — dead constant removed, compaction ordering deterministic, async parallel git reads on startup. - -**Structural refactors completed:** -- `handleCommand` split into `lib/daemon/handlers/{process,cache,remedy,proxy}.ts` — routed-lookup dispatch in daemon.ts, 14 `process:*` + 3 `cache:*`/`branch:*` + 3 `remedy:*` + 5 `proxy:*` commands all flow through factory-injected `HandlerContext`. -- compact/expand physically extracted to `lib/runner-store/compact.ts`; runner-store.ts re-exports for back-compat. -- Git watcher pool extracted to `lib/runner/git-watchers.ts` — `createGitWatcherPool(onChange) → { sync, dispose }` with built-in 150ms per-lane debounce. - -**Remaining cosmetic deferrals:** -- `commands/runner.tsx` keymap handlers could be split into sibling files for testability, but that's pure reorg — no correctness signal behind it. - -## Not covered here - -- `SuspendManager`, `ExclusiveGroup`, `AttachServer`, `WorkspaceSync`, `PortAllocator` — not re-read this pass. Worth a follow-up audit on the same four axes. -- Test coverage itself — there is very little for the daemon. Any of the extraction work above should ship with the matching test file. -- Observability — `diag()` is sprinkled throughout but there's no sampling, no correlation id, no structured reader. A shared `makeChildLogger(component, id)` helper would clean up call sites and produce greppable logs. +- Current stability audit + roadmap: `docs/daemon-stability-audit-2026-08.md` + (in the daemon-stability-audit worktree). +- Supervision verdicts + exit-code semantics: `docs/daemon-supervision-design.md`. diff --git a/docs/daemon-supervision-design.md b/docs/daemon-supervision-design.md new file mode 100644 index 000000000..61b4f6c1a --- /dev/null +++ b/docs/daemon-supervision-design.md @@ -0,0 +1,71 @@ +# Daemon supervision: status verdicts and exit-code semantics + +Phase 0 design anchor for the rt daemon stability roadmap (audit +2026-08). Tasks 9-14 of the Phase 0 plan implement this. + +## launchd contract + +The prod plist sets `KeepAlive = { SuccessfulExit: false }`: launchd +respawns the daemon ONLY on a non-zero exit. A zero exit means "stay +down". Every exit-code decision below follows from that single fact. + +## Exit-code policy + +| Path | Exit | Why | +|-----------------------------------------|------|-----| +| `startDaemon()` boot throw (prod path) | 1 | Visible + launchd relaunches. Paired with crash-loop detection so it cannot loop silently forever. | +| `shutdown` IPC/REST verb | 0 | Intentional stop; launchd must not respawn. Records `last-exit.kind = "shutdown"`. | +| Bare OS signal SIGTERM/SIGINT/SIGHUP | 1 | External kill (pkill, script, memory pressure); launchd SHOULD respawn. The sanctioned stop path goes through SMAppService.unregister, where the exit code is irrelevant, so exiting non-zero here does not break intended stops. | +| Crash-loop guard trips (N in M minutes) | 0 (park) | Stop the flapping; surface `crash-looping` so a human intervenes instead of launchd hammering every ~10s. | + +Mechanism: a module-scope `shuttingDownViaVerb` flag is set true by the +`shutdown` verb before it calls cleanup; `gracefulExit(signal)` reads it: +set → exit(0), unset (bare signal) → exit(1). + +Boot-phase gate: a module-scope `bootPhase: "booting" | "ready"` flips +to `"ready"` immediately before the `daemon ready` log. The +`unhandledRejection` handler exits(1) while `bootPhase === "booting"` +and only logs (recovers) once ready, so a boot-time stray rejection is +fatal but a steady-state one is not. + +## Status verdicts + +`rt daemon status` and `/api/status` classify by first match: + +1. `not-installed`: SMAppService not registered. +2. `serving`: ping on rt.sock succeeds. +3. `parked`: ping fails, a live rt pid exists, and the boot breadcrumb + phase is a flavor standoff (park). Named distinctly so the user is + told "another flavor owns the socket", not "wedged". +4. `alive-not-serving`: ping fails but a live rt pid exists + (`process.kill(pid,0)` on rt.pid, falling back to `lsof +D RT_DIR` + scoped to this HOME's rt dir and filtered to exclude the calling + process's own pid). + Sub-detail from the breadcrumb phase: `booting` / `wedged`, or + `quarantined` when a state.db/events.db boot-failed marker is present. + Prints "process is running but not answering rt.sock ... rt daemon logs -t". +5. `crash-looping`: no live pid AND the kv failure record shows ≥ N + failures within the last M minutes (N=3, M=5). Prints the last reason. +6. `boot-failed`: no live pid AND the most recent kv exit record is a + boot throw (fewer than N failures). Prints the last reason + phase. +7. `installed-not-running`: registered, no live pid, clean/again-absent + exit record. + +## Persisted state (kv, ns `daemon-supervision`, no schema change) + +- `boot-attempts` (number): incremented at the top of `runDaemon()`. +- `last-ready-at` (number, epoch ms): stamped just before `daemon ready`. +- `recent-failures` (array of `{ at, phase, reason }`, capped to 10): + appended by the boot fatal path and by state.db/events.db boot-failed + markers. Crash-loop = ≥ N entries newer than now − M minutes. +- `last-exit` (`{ at, kind: "shutdown" | "signal" | "boot-failed", code, reason? }`): + written by the shutdown verb, the signal handlers, and the boot + fatal path. Lets status distinguish "cleanly stopped" from "died". + +## Boot breadcrumb + +`~/.mattstack/rt/daemon-boot.json` = `{ at, pid, flavor, phase }`, +rewritten at each boot phase: `start` → `events-db` → `state-db` → +`api` → `socket` → `ready`. Lets `alive-not-serving` name where a +live-but-silent daemon is stuck even when the logs are unreadable. +Removed (or stamped `ready`) on successful boot. diff --git a/docs/superpowers/plans/2026-08-28-p0-supervision.md b/docs/superpowers/plans/2026-08-28-p0-supervision.md new file mode 100644 index 000000000..72453ddae --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-p0-supervision.md @@ -0,0 +1,1088 @@ +# Phase 0 · Honest Supervision Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make rt daemon crashes visible and recoverable: every boot failure exits non-zero (or parks visibly) instead of becoming a silent zombie, crash/restart history is persisted and surfaced by `rt daemon status`, and start/stop/eviction stop deleting a live daemon's runtime files. + +**Architecture:** Retire the P0/P1 crash-and-hide class in `lib/daemon.ts` and its immediate collaborators. Boot failures become fatal (log + `process.exit(1)`) on the prod path, gated by a boot-phase flag so steady-state stray rejections still recover; crash handlers and stderr redirection move above every module-scope side effect; restart counters and last-exit reasons live in the existing `kv` table (no schema change); `rt daemon status` and `/api/status` gain `alive-not-serving` / `parked` / `boot-failed` / `crash-looping` verdicts; start/stop/eviction become ownership-aware. Two one-line correctness fixes (state.db flavor on the snapshot path, extended busy-code matching + `BEGIN IMMEDIATE`) make the contention policy real. + +**Tech Stack:** Bun, `bun:sqlite`, pino, TypeScript. Tests are `bun test` (unit) and `bun test --preload ./e2e/setup.ts` (e2e, isolated-HOME daemon spawns). + +**Spec:** `/Users/matt/Documents/GitHub/repo-tools/.claude/worktrees/daemon-stability-audit/docs/daemon-stability-audit-2026-08.md` ... "Roadmap › Phase 0" plus Appendix A/B entries S001, S003, S004, S009, S011, S012, S026, S027, S028, S029, S030, S035, S036, S037, S043, S044, S060, S072, S073, S074, R001, R002, R007, R017. Each carries a failure scenario, prescribed fix, and fixer notes; read the relevant entry before implementing its task. + +## Global Constraints + +- **No `SCHEMA_VERSION` bump and no new/edited `V*_SCHEMA` block.** Persist all supervision state (restart counters, last-exit reason, boot-failed markers) in the existing `kv` table under namespace `daemon-supervision`, via `setKvValue`/`getKvValue` from `lib/state/kv-blob.ts`. If any task appears to need a schema change, STOP and ask the user (per the job brief's question format) ... do not proceed. +- **Never start a daemon or run `rt` against the real machine.** Every daemon or `dist/rt` invocation in a test or check runs under `env -i HOME=` only (repo CLAUDE.md, "Operating on this machine"). e2e daemon spawns already do this via `e2e/setup.ts`; new e2e tests must follow the same isolation. +- **Write fence ... do NOT modify these sibling-owned files** (ask the user if a task seems to need one): `lib/daemon/api-server.ts`, `lib/daemon/api-auth.ts`, `lib/daemon/socket-server.ts`, `lib/daemon/handlers/secrets.ts`, `lib/subprocess.ts`, `lib/daemon/cache-refresh.ts`, `lib/git-worktrees.ts`, `lib/daemon/freshness.ts`, `lib/daemon/pollers.ts`, `lib/daemon/worktree-process-kill.ts`, `lib/daemon/system-process-scanner.ts`, `lib/runs/store.ts`, `lib/notifier.ts`, `lib/daemon/handlers/discussions.ts`, `lib/daemon/handlers/chat.ts`, `lib/daemon/handlers/agent.ts`, `lib/daemon/handlers/pane.ts`, `lib/daemon/handlers/project-mrs.ts`, `lib/daemon/handlers/worktree.ts`, `lib/herdr/client.ts`, `lib/port-scanner.ts`, `lib/deps/links.ts`, `lib/worktree/trash.ts`, `lib/agent-herdr.ts`, `lib/daemon/cron.ts`, `lib/daemon/hooks-guard.ts`, `lib/home/age-key.ts`, `lib/daemon/discussions-store.ts`, `lib/state/presence-store.ts`. +- **Every subagent dispatched during execution carries an explicit `model`** (`sonnet` for mechanical tasks, `haiku` for lookups). +- **`packages/rt-client` is touched** (Task 5 edits `registry-defs.ts`). After that task and before the final whole-branch review, run `bun run build` inside `packages/rt-client` (keeps `dist/` and `dist-freshness.test.ts` green). +- **Verification (must pass before the work is done):** + - `bun test lib commands packages scripts` green (from the worktree root) + - `bunx tsc --noEmit` reports zero errors + - `bun run test:e2e` green, or at minimum `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts` (say which was run) + +## Deferred / out-of-fence items (documented, not implemented here) + +- **S073 · `presence-store.ts:signIn` → `.immediate()`** ... `lib/state/presence-store.ts` is fenced. Task 7 converts the chat-store and notifier-store read-then-write transactions; the `signIn` caller is a follow-up for the presence-store owner. Not required for verification. +- **0.3 · `rt.apiPort` bind-time consumption** ... `lib/daemon/api-server.ts` (the binder) is fenced. Task 5 registers the `rt.apiPort` setting and exposes `resolveApiPort()`; the bind-time read is the api-server sibling's hop. The escape hatch is therefore wired daemon-side but consumed sibling-side; do not claim it functions end-to-end until the sibling reads it. +- **Swift tray edits (S026 dot mapping, S028 `DaemonLifecycle` kickstart fallback, S029 `tray-crash.log` rotation, S060 `AppDelegate` comment)** ... grouped as optional Task 16. They cannot be verified by the bun/tsc/e2e gate and the operating rules forbid rebuilding the blessed bundle. Scope confirmation is raised at the plan-review checkpoint. The verifiable CLI-side halves (S028 start→kickstart fallback, S060 exit-code policy) live in Tasks 14 and 12 and are done regardless. + +--- + +## Task 1: Status-verdict + exit-code design sketch + +The half-page design that Tasks 9-14 depend on. No production code; the deliverable is a committed design doc. (Retires nothing directly; anchors R001, R002, S036, S060, S026, S028.) + +**Files:** +- Create: `docs/daemon-supervision-design.md` + +**Interfaces:** +- Produces: the verdict names (`serving`, `alive-not-serving`, `parked`, `boot-failed`, `crash-looping`, `installed-not-running`, `not-installed`), the boot-phase names (`booting` → `ready`), the kv keys under ns `daemon-supervision`, the boot-breadcrumb shape, and the exit-code policy table. Tasks 9/10/12 consume these names verbatim. + +- [ ] **Step 1: Write the design doc** with exactly this content: + +```markdown +# Daemon supervision: status verdicts and exit-code semantics + +Phase 0 design anchor for the rt daemon stability roadmap (audit +2026-08). Tasks 9-14 of the Phase 0 plan implement this. + +## launchd contract + +The prod plist sets `KeepAlive = { SuccessfulExit: false }`: launchd +respawns the daemon ONLY on a non-zero exit. A zero exit means "stay +down". Every exit-code decision below follows from that single fact. + +## Exit-code policy + +| Path | Exit | Why | +|-----------------------------------------|------|-----| +| `startDaemon()` boot throw (prod path) | 1 | Visible + launchd relaunches. Paired with crash-loop detection so it cannot loop silently forever. | +| `shutdown` IPC/REST verb | 0 | Intentional stop; launchd must not respawn. Records `last-exit.kind = "shutdown"`. | +| Bare OS signal SIGTERM/SIGINT/SIGHUP | 1 | External kill (pkill, script, memory pressure); launchd SHOULD respawn. The sanctioned stop path goes through SMAppService.unregister, where the exit code is irrelevant, so exiting non-zero here does not break intended stops. | +| Crash-loop guard trips (N in M minutes) | 0 (park) | Stop the flapping; surface `crash-looping` so a human intervenes instead of launchd hammering every ~10s. | + +Mechanism: a module-scope `shuttingDownViaVerb` flag is set true by the +`shutdown` verb before it calls cleanup; `gracefulExit(signal)` reads it +... set → exit(0), unset (bare signal) → exit(1). + +Boot-phase gate: a module-scope `bootPhase: "booting" | "ready"` flips +to `"ready"` immediately before the `daemon ready` log. The +`unhandledRejection` handler exits(1) while `bootPhase === "booting"` +and only logs (recovers) once ready ... so a boot-time stray rejection is +fatal but a steady-state one is not. + +## Status verdicts + +`rt daemon status` and `/api/status` classify by first match: + +1. `not-installed` ... SMAppService not registered. +2. `serving` ... ping on rt.sock succeeds. +3. `parked` ... ping fails, a live rt pid exists, and the boot breadcrumb + phase is a flavor standoff (park). Named distinctly so the user is + told "another flavor owns the socket", not "wedged". +4. `alive-not-serving` ... ping fails but a live rt pid exists + (`process.kill(pid,0)` on rt.pid, or `pgrep -f 'rt --daemon|lib/daemon.ts'`). + Sub-detail from the breadcrumb phase: `booting` / `wedged`, or + `quarantined` when a state.db/events.db boot-failed marker is present. + Prints "process is running but not answering rt.sock ... rt daemon logs -t". +5. `crash-looping` ... no live pid AND the kv failure record shows ≥ N + failures within the last M minutes (N=3, M=5). Prints the last reason. +6. `boot-failed` ... no live pid AND the most recent kv exit record is a + boot throw (fewer than N failures). Prints the last reason + phase. +7. `installed-not-running` ... registered, no live pid, clean/again-absent + exit record. + +## Persisted state (kv, ns `daemon-supervision`, no schema change) + +- `boot-attempts` (number) ... incremented at the top of `runDaemon()`. +- `last-ready-at` (number, epoch ms) ... stamped just before `daemon ready`. +- `recent-failures` (array of `{ at, phase, reason }`, capped to 10) ... + appended by the boot fatal path and by state.db/events.db boot-failed + markers. Crash-loop = ≥ N entries newer than now − M minutes. +- `last-exit` (`{ at, kind: "shutdown" | "signal" | "boot-failed", code, reason? }`) + ... written by the shutdown verb, the signal handlers, and the boot + fatal path. Lets status distinguish "cleanly stopped" from "died". + +## Boot breadcrumb + +`~/.mattstack/rt/daemon-boot.json` = `{ at, pid, flavor, phase }`, +rewritten at each boot phase: `start` → `crash-handlers` → `events-db` +→ `state-db` → `socket` → `api` → `ready`. Lets `alive-not-serving` +name where a live-but-silent daemon is stuck even when the logs are +unreadable. Removed (or stamped `ready`) on successful boot. +``` + +- [ ] **Step 2: Commit** + +```bash +git add docs/daemon-supervision-design.md +git commit -m "docs: sketch daemon supervision verdicts + exit-code semantics" +``` + +--- + +## Task 2: Fatal boot means exit (0.1 ... S001, S037) + +Boot failures on the prod path currently become an `unhandledRejection` that only logs, leaving a live-pid zombie with no socket/API. Make a `runDaemon()` throw fatal, gate the rejection handler on a boot-phase flag, and move the rt.pid write to after both binds. + +**Files:** +- Modify: `lib/daemon-logger.ts:232-264` (`installCrashHandlers` gains a `booting` predicate) +- Modify: `lib/daemon.ts:385-514` (wrap `runDaemon()` body; boot-phase flag; move rt.pid write), `lib/daemon.ts:392` (pass predicate), `lib/daemon.ts:513` (flip flag) +- Test: `lib/__tests__/daemon-logger.test.ts` (rejection handler), `e2e/tests/daemon.test.ts` (fatal boot) + +**Interfaces:** +- Produces: `installCrashHandlers(logger, opts?: { booting?: () => boolean })` ... when `booting()` is true, `unhandledRejection` logs `fatal` and `process.exit(1)`; otherwise it logs `error` only (today's behavior). Default (no `booting`) preserves today's error-only behavior. +- Produces: module-scope `let bootPhase: "booting" | "ready" = "booting"` in `lib/daemon.ts`, flipped to `"ready"` at line 513. + +- [ ] **Step 1: Write the failing unit test** in `lib/__tests__/daemon-logger.test.ts`: + +```ts +test("unhandledRejection exits(1) while booting, only logs once ready", () => { + const exits: number[] = []; + const origExit = process.exit; + // @ts-expect-error test stub + process.exit = (code?: number) => { exits.push(code ?? 0); }; + const fatal = mock(() => {}); + const error = mock(() => {}); + const logger = makeFakeLogger({ fatal, error }); // existing test helper + let booting = true; + installCrashHandlers(logger, { booting: () => booting }); + process.emit("unhandledRejection", new Error("boot boom"), Promise.resolve()); + expect(fatal).toHaveBeenCalledTimes(1); + expect(exits).toEqual([1]); + booting = false; + process.emit("unhandledRejection", new Error("steady boom"), Promise.resolve()); + expect(error).toHaveBeenCalledTimes(1); + expect(exits).toEqual([1]); // no second exit + process.exit = origExit; +}); +``` + +(If `makeFakeLogger` does not exist, build a minimal `{ info, warn, error, fatal }` of `mock(() => {})`. Remove the listeners this test adds in `afterEach` via `process.removeAllListeners("unhandledRejection")` scoped to the test, matching the file's existing cleanup convention.) + +- [ ] **Step 2: Run it ... expect FAIL** (`booting` option not supported): + +Run: `bun test lib/__tests__/daemon-logger.test.ts -t "unhandledRejection exits"` +Expected: FAIL. + +- [ ] **Step 3: Implement in `lib/daemon-logger.ts`.** Change the signature and the `unhandledRejection` branch (currently lines 242-244): + +```ts +export function installCrashHandlers( + logger: Logger, + opts: { booting?: () => boolean } = {}, +): void { + // ... uncaughtException handler unchanged (still fatal + exit 1) ... + process.on("unhandledRejection", (reason) => { + if (opts.booting?.()) { + logger.fatal({ err: reason }, "unhandledRejection during boot"); + process.exit(1); + return; + } + logger.error({ err: reason }, "unhandledRejection"); + }); + // ... stderr interception unchanged ... +} +``` + +- [ ] **Step 4: Run the unit test ... expect PASS.** + +- [ ] **Step 5: Wire the boot-phase flag + fatal wrap + rt.pid move in `lib/daemon.ts`.** + - Add near the top of module scope (after imports, before line 78): `let bootPhase: "booting" | "ready" = "booting";` + - At the `installCrashHandlers(loggerHandle)` call (line 392): `installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" });` + - Wrap the body of `runDaemon()` (385-514) in try/catch: + +```ts +async function runDaemon() { + try { + // ... existing body 386-513 ... + } catch (err) { + loggerHandle.fatal?.({ err }, "daemon boot failed"); + // Task 9 adds recordBootFailure(currentPhase, err) here. + try { loggerHandle.flush?.(); } catch {} + process.exit(1); + } +} +``` + + - **Move the rt.pid write** (currently `writeFileSync(DAEMON_PID_PATH, String(process.pid))` at line 416) to AFTER both binds ... i.e. after the API bind at line 469 and the socket bind at line 468 (Task 5 will make API bind first; either way, rt.pid is written only once both `servers.socket` and `servers.api` are assigned). A failed boot then never leaves a live-pid file. + - Set `bootPhase = "ready";` immediately before `log.info({ pid }, "daemon ready")` at line 513. + +- [ ] **Step 6: Write the failing e2e test** in `e2e/tests/daemon.test.ts` (uses the isolated-HOME harness already in `e2e/setup.ts`): + +```ts +test("daemon boot with API port already bound exits non-zero and leaves no stale rt.pid", async () => { + // Bind the API port inside the isolated HOME so the daemon cannot. + const port = 9411; + const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + try { + const proc = Bun.spawn({ + cmd: [rtBinary, "--daemon"], + env: { ...isolatedEnv, RT_API_PORT: String(port) }, + stdout: "pipe", stderr: "pipe", + }); + const code = await proc.exited; + expect(code).not.toBe(0); + expect(existsSync(join(isolatedHome, ".mattstack/rt/rt.pid"))).toBe(false); + } finally { + squatter.stop(true); + } +}); +``` + +(Reuse the harness's existing `rtBinary`, `isolatedEnv`, `isolatedHome` fixtures ... mirror `e2e/tests/daemon.test.ts`'s existing setup. The daemon's own `startApiServer` retries the bind 6× before throwing, so allow up to the 60s timeout.) + +- [ ] **Step 7: Run the e2e test ... expect PASS.** Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "API port already bound"` + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon-logger.ts lib/daemon.ts lib/__tests__/daemon-logger.test.ts e2e/tests/daemon.test.ts +git commit -m "daemon: boot failure is fatal (exit 1), gated by boot-phase flag; rt.pid after binds" +``` + +--- + +## Task 3: Crash handlers first (0.2 ... S003, S004, R007) + +Move `redirectNativeStderr()` and `installCrashHandlers()` above every module-scope side effect so a pre-`startDaemon` failure lands in the crash log instead of a discarded stderr. + +**Files:** +- Modify: `lib/daemon.ts` (hoist two calls to module scope; remove the duplicates inside `runDaemon()` at 391-392) +- Test: `e2e/tests/daemon.test.ts` + +**Interfaces:** +- Consumes: `installCrashHandlers(logger, { booting })` from Task 2. + +- [ ] **Step 1: Hoist `redirectNativeStderr()`** to the very first executable statement of `lib/daemon.ts` module scope ... before `migrateLegacyRtDir()` at line 78. It depends only on `logsDir()` and has its own internal try/catch, so a subsequent module-scope throw's fd-2 output lands in `daemon-stderr.log`. + +- [ ] **Step 2: Hoist `installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" })`** to immediately after `getDaemonLogger()` resolves (right after line ~95, before `parkUntilIntended` at 103 and before `createEventsBus` at 191). The logger must exist first (line 85), so this is the earliest correct point. + +- [ ] **Step 3: Remove the now-duplicate `redirectNativeStderr()` and `installCrashHandlers()` calls** inside `runDaemon()` (lines 391-392). Keep `mkdirSync(RT_DIR, …)` at 386 (redirect needs the logs dir; `redirectNativeStderr` already mkdirs its own dir, and RT_DIR creation is idempotent ... verify the hoisted `redirectNativeStderr` still finds/creates `logsDir()`). + +- [ ] **Step 4: Write the failing e2e test** in `e2e/tests/daemon.test.ts`: + +```ts +test("a corrupt events.db does not crash the daemon silently ... error is captured", async () => { + // Pre-create a corrupt events.db in the isolated HOME. + const rtDir = join(isolatedHome, ".mattstack/rt"); + mkdirSync(rtDir, { recursive: true }); + writeFileSync(join(rtDir, "events.db"), "not a sqlite file at all"); + const proc = Bun.spawn({ cmd: [rtBinary, "--daemon"], env: isolatedEnv, stdout: "pipe", stderr: "pipe" }); + // Task 4 makes this self-heal; for Task 3 we only require the failure is captured, not /dev/null. + await Bun.sleep(3000); proc.kill(); + const stderrLog = join(rtDir, "logs", "daemon-stderr.log"); + const quarantined = readdirSync(rtDir).some((f) => f.startsWith("events.db.corrupt-")); + const captured = existsSync(stderrLog) || quarantined; + expect(captured).toBe(true); +}); +``` + +- [ ] **Step 5: Run ... expect PASS** (the redirect now runs before events.db construction, so a corruption throw is captured in `daemon-stderr.log`; after Task 4 it is quarantined instead). Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "corrupt events.db"` + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon.ts e2e/tests/daemon.test.ts +git commit -m "daemon: install stderr redirect + crash handlers before every module-scope side effect" +``` + +--- + +## Task 4: events.db joins the discipline (0.4 ... S009, S035) + +Give `events.db` the corruption quarantine + busy_timeout/synchronous pragmas state.db has, and guard the two sweep timers so a sync sqlite throw cannot exit the daemon. + +**Files:** +- Modify: `lib/daemon/events-bus.ts:64-224` (quarantine + pragmas in `createEventsBus`) +- Modify: `lib/daemon.ts:194,196` (wrap sweep timers; add a `safeInterval`/`safeTimeout` helper) +- Test: `lib/daemon/__tests__/events-bus.test.ts` + +**Interfaces:** +- Consumes: `isCorruptionError` and `quarantine` shape from `lib/state/db.ts` (reuse the `SQLITE_CORRUPT`/`SQLITE_NOTADB` detection; `events.db` is a bounded-retention journal, so total loss on quarantine is harmless ... no migration concern). + +- [ ] **Step 1: Write the failing test** in `lib/daemon/__tests__/events-bus.test.ts`: + +```ts +test("createEventsBus quarantines and recreates a corrupt events.db instead of throwing", () => { + const dir = mkdtempSync(join(tmpdir(), "events-corrupt-")); + const dbPath = join(dir, "events.db"); + writeFileSync(dbPath, "garbage not sqlite"); + const bus = createEventsBus({ dbPath, log: silentLog }); + expect(readdirSync(dir).some((f) => f.startsWith("events.db.corrupt-"))).toBe(true); + // fresh db works: + bus.emit("test", { hi: 1 }); + expect(bus.list({ limit: 1 }).length).toBe(1); + bus.close(); +}); + +test("createEventsBus sets busy_timeout and synchronous=NORMAL", () => { + const dir = mkdtempSync(join(tmpdir(), "events-pragma-")); + const bus = createEventsBus({ dbPath: join(dir, "events.db"), log: silentLog }); + // @ts-expect-error internal handle access for the test, or expose a debug getter + const handle = bus.__db ?? getBusDb(bus); + expect(handle.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); + expect(handle.query("PRAGMA synchronous").get()).toEqual({ synchronous: 1 }); // NORMAL + bus.close(); +}); +``` + +(If the bus does not expose its handle, add a minimal `__db` back-reference or a `busyTimeout()` debug accessor in `events-bus.ts` for the test; do not expose it beyond the module's test needs.) + +- [ ] **Step 2: Run ... expect FAIL.** Run: `bun test lib/daemon/__tests__/events-bus.test.ts -t "quarantine"` + +- [ ] **Step 3: Implement in `lib/daemon/events-bus.ts`** ... wrap the open (lines 73-86): + +```ts +import { isCorruptionError } from "../state/db"; // export it if not already exported +// ... +mkdirSync(dirname(opts.dbPath), { recursive: true }); +let db: Database; +try { + db = new Database(opts.dbPath, { create: true }); + db.exec("PRAGMA busy_timeout = 250;"); // before journal_mode, matching state/db.ts ordering + db.exec("PRAGMA journal_mode = WAL;"); + db.exec("PRAGMA synchronous = NORMAL;"); + db.query("PRAGMA user_version").get(); // touch to force a real read that trips corruption +} catch (err) { + if (!isCorruptionError(err)) throw err; + quarantineEventsDb(opts.dbPath, opts.log); // rename to events.db.corrupt-, warn + db = new Database(opts.dbPath, { create: true }); + db.exec("PRAGMA busy_timeout = 250;"); + db.exec("PRAGMA journal_mode = WAL;"); + db.exec("PRAGMA synchronous = NORMAL;"); +} +// then the existing CREATE TABLE / CREATE INDEX (78-86) +``` + +Write a local `quarantineEventsDb(path, log)` mirroring `state/db.ts`'s `quarantine` (rename the db + `-wal`/`-shm` sidecars to `path.corrupt-`, `log.warn`). If `isCorruptionError` is not exported from `state/db.ts`, add the export (it is a pure predicate, safe to export). + +- [ ] **Step 4: Run the events-bus tests ... expect PASS.** + +- [ ] **Step 5: Write the failing sweep-guard test** in `lib/daemon/__tests__/events-bus.test.ts` OR a small `lib/__tests__/daemon-sweep-guard.test.ts` for the helper: + +```ts +test("safeInterval swallows a throwing tick and logs warn", () => { + const warn = mock(() => {}); + const log = { ...silentLog, warn }; + let ticks = 0; + const handle = safeInterval(() => { ticks++; throw new Error("SQLITE_FULL"); }, 10, "test-sweep", log); + return new Promise((r) => setTimeout(() => { + clearInterval(handle); + expect(ticks).toBeGreaterThan(0); + expect(warn).toHaveBeenCalled(); + r(); + }, 45)); +}); +``` + +- [ ] **Step 6: Implement `safeInterval`/`safeTimeout`** in `lib/daemon.ts` (or a tiny `lib/daemon/safe-timers.ts` exporting both): + +```ts +export function safeInterval(fn: () => void, ms: number, label: string, log: Logger) { + return setInterval(() => { try { fn(); } catch (err) { log.warn({ err, label }, "timer tick failed"); } }, ms); +} +export function safeTimeout(fn: () => void, ms: number, label: string, log: Logger) { + return setTimeout(() => { try { fn(); } catch (err) { log.warn({ err, label }, "timer tick failed"); } }, ms); +} +``` + +Replace the two bare sweep timers at `lib/daemon.ts:194` and `:196` with `safeInterval(() => eventsBus.sweep(), 60*60*1000, "events-sweep", log)` and `safeTimeout(() => eventsBus.sweep(), 30_000, "events-sweep-boot", log)`. (The `pruneRuns`/`pruneLogs` timers at 200-248 already wrap their bodies; leaving them is fine, but converting them to `safeInterval` is a welcome DRY cleanup if trivial.) + +- [ ] **Step 7: Run ... expect PASS.** Then `bun test lib/daemon/__tests__/events-bus.test.ts`. + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon/events-bus.ts lib/daemon.ts lib/state/db.ts lib/daemon/__tests__/events-bus.test.ts +git commit -m "events.db: corruption quarantine + busy_timeout/synchronous pragmas; guard sweep timers" +``` + +--- + +## Task 5: Bind order + rt.apiPort setting (0.3, my side ... S043, S030 seam) + +Bind the API server before the unix socket so a failed API bind never strands a socket-bound zombie, and register the `rt.apiPort` escape-hatch setting (the api-server sibling consumes it at bind time). + +**Files:** +- Modify: `lib/daemon.ts:468-469` (swap: API bind first, then socket bind; rt.pid still written after both, per Task 2) +- Modify: `lib/daemon-config.ts:72` (add `resolveApiPort()`; keep `API_PORT` const non-breaking) +- Modify: `packages/rt-client/src/settings/registry-defs.ts` (register `rt.apiPort`) +- Test: `lib/__tests__/daemon-config.test.ts`, `packages/rt-client` settings test, `e2e/tests/daemon.test.ts` + +**Interfaces:** +- Produces: `resolveApiPort(): number` in `lib/daemon-config.ts` = `Number(process.env.RT_API_PORT) || getSetting("rt.apiPort").value || 9401`, resolved lazily (never at module load). `API_PORT` const stays as-is for `api-server.ts`'s existing import. + +- [ ] **Step 1: Register the setting.** Append to `packages/rt-client/src/settings/registry-defs.ts` (shaped like `rt.logRetentionDays` at 190-198): + +```ts +{ + key: "rt.apiPort", + type: "number", + scopes: ["machine", "user"], + default: 9401, + merge: "replace", + migrated: true, + description: "TCP port the daemon's local HTTP/WS API binds (escape hatch when 9401 is held).", +}, +``` + +- [ ] **Step 2: Rebuild rt-client** so the daemon and `dist-freshness` see the new key: + +Run: `cd packages/rt-client && bun run build && cd -` + +- [ ] **Step 3: Write the failing test** in `lib/__tests__/daemon-config.test.ts`: + +```ts +test("resolveApiPort: env wins, then setting, then 9401", () => { + const prev = process.env.RT_API_PORT; + process.env.RT_API_PORT = "12345"; + expect(resolveApiPort()).toBe(12345); + delete process.env.RT_API_PORT; + expect(resolveApiPort()).toBe(9401); // default setting value + if (prev !== undefined) process.env.RT_API_PORT = prev; +}); +``` + +- [ ] **Step 4: Run ... expect FAIL** (`resolveApiPort` undefined). + +- [ ] **Step 5: Implement in `lib/daemon-config.ts`** (leave `API_PORT` at line 72 untouched): + +```ts +import { getSetting } from "./settings/resolve"; +export function resolveApiPort(): number { + const env = Number(process.env.RT_API_PORT); + if (env) return env; + try { return getSetting("rt.apiPort").value || 9401; } catch { return 9401; } +} +``` + +- [ ] **Step 6: Run ... expect PASS.** + +- [ ] **Step 7: Swap the bind order in `lib/daemon.ts`.** Reorder so the API binds before the socket: + +```ts +servers.api = await startApiServer({ handleCommand, log }); // was line 469 ... now first +servers.socket = startSocketServer({ handleCommand, log }); // was line 468 ... now second +// rt.pid write (moved by Task 2) stays after BOTH assignments +``` + +- [ ] **Step 8: Write the failing e2e test** in `e2e/tests/daemon.test.ts` (extends the Task 2 squatter test): + +```ts +test("API-bind failure leaves neither rt.sock nor rt.pid", async () => { + const port = 9412; + const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + try { + const proc = Bun.spawn({ cmd: [rtBinary, "--daemon"], env: { ...isolatedEnv, RT_API_PORT: String(port) }, stdout: "pipe", stderr: "pipe" }); + await proc.exited; + expect(existsSync(join(isolatedHome, ".mattstack/rt/rt.sock"))).toBe(false); + expect(existsSync(join(isolatedHome, ".mattstack/rt/rt.pid"))).toBe(false); + } finally { squatter.stop(true); } +}); +``` + +- [ ] **Step 9: Run ... expect PASS.** Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts -t "API-bind failure"` + +- [ ] **Step 10: Commit** + +```bash +git add lib/daemon.ts lib/daemon-config.ts packages/rt-client/src/settings/registry-defs.ts packages/rt-client/dist lib/__tests__/daemon-config.test.ts e2e/tests/daemon.test.ts +git commit -m "daemon: bind API before socket; register rt.apiPort setting + resolveApiPort()" +``` + +--- + +## Task 6: home-snapshot opens state.db daemon-flavored (0.7 ... S011) + +`startHomeSnapshot` opens the state.db singleton at module scope with the default `cli` flavor (5000ms busy_timeout), so the daemon runs with the wrong contention policy forever. Make its db lazy and daemon-flavored, and harden `getStateDb` against a silent flavor mismatch. + +**Files:** +- Modify: `lib/daemon/home-snapshot.ts:272,299` (lazy daemon-flavored db resolver) +- Modify: `lib/state/db.ts:522-530` (`getStateDb` mismatch hardening) +- Test: `lib/state/__tests__/db.test.ts` + +**Interfaces:** +- Consumes: `getStateDb("daemon")` ... 250ms busy_timeout. + +- [ ] **Step 1: Write the failing test** in `lib/state/__tests__/db.test.ts` (`describe("pragma values per flavor")`): + +```ts +test("getStateDb('daemon') reports busy_timeout 250 even after a default open", () => { + const cli = getStateDb(); // opens singleton, cli flavor + expect(cli.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 5000 }); + const daemon = getStateDb("daemon"); // same singleton ... must not stay at 5000 + expect(daemon.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); +}); +``` + +(Use the file's existing isolated-HOME / `closeStateDb` setup so this does not leak into other tests.) + +- [ ] **Step 2: Run ... expect FAIL** (singleton keeps the cli 5000 timeout). + +- [ ] **Step 3: Harden `getStateDb` in `lib/state/db.ts:522-530`** ... when the singleton is already open and a caller requests a stronger (shorter) flavor timeout, re-apply the pragma: + +```ts +export function getStateDb(flavor: DbFlavor = "cli"): Database { + const path = stateDbPath(); + if (singleton && singletonPath === path) { + const want = BUSY_TIMEOUT_MS[flavor]; + const have = Number((singleton.query("PRAGMA busy_timeout").get() as any)?.timeout ?? 0); + if (want < have) singleton.exec(`PRAGMA busy_timeout = ${want};`); + return singleton; + } + // ... existing (re)open path ... +} +``` + +- [ ] **Step 4: Run ... expect PASS.** + +- [ ] **Step 5: Make home-snapshot's db lazy + daemon-flavored** in `lib/daemon/home-snapshot.ts`. Replace the eager `db: rawDeps.db ?? getStateDb()` (line 272) with a thunk defaulting to `() => getStateDb("daemon")`, and resolve it on first use inside `loadState`/`runNow`/`status` rather than at construction (line 299). Concretely: store `const resolveDb = rawDeps.db ? () => rawDeps.db! : () => getStateDb("daemon");` and call `resolveDb()` where `deps.db` was read, so no db opens until `startDaemon` has already opened it daemon-flavored via `openBranchCacheStore`. + +- [ ] **Step 6: Add a boot-order regression test** in `lib/state/__tests__/db.test.ts` (or `home-snapshot.test.ts`) asserting that constructing `startHomeSnapshot` does NOT open the state.db singleton (call it, then assert `getStateDb` was not yet invoked ... spy on the module or assert no `state.db` file exists until first use in an isolated HOME). + +- [ ] **Step 7: Run the db tests ... expect PASS.** Run: `bun test lib/state/__tests__/db.test.ts` + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon/home-snapshot.ts lib/state/db.ts lib/state/__tests__/db.test.ts +git commit -m "home-snapshot: lazy daemon-flavored state.db; getStateDb re-applies a stronger flavor timeout" +``` + +--- + +## Task 7: Extended busy codes + IMMEDIATE transactions (0.7 ... S072, S073) + +`isBusyError` misses `SQLITE_BUSY_SNAPSHOT`/`_RECOVERY`, and read-then-write daemon transactions use a deferred `BEGIN` that produces snapshot conflicts busy_timeout cannot absorb. Widen the match and take the write lock up front. + +**Files:** +- Modify: `lib/state/busy.ts:39-41` (`isBusyError`) +- Modify: `lib/state/chat-store.ts` (`readUnread`, `joinRoom`, `archiveRoom`, `dmRoomFor` → `.immediate()`) +- Modify: `lib/state/notifier-store.ts` (`drainNotificationQueue` → `.immediate()`) +- Test: `lib/state/__tests__/busy.test.ts` +- **Deferred (fenced):** `lib/state/presence-store.ts:signIn` ... documented follow-up, not done here. + +**Interfaces:** +- Produces: `isBusyError` returns true for any `code` starting `SQLITE_BUSY`. + +- [ ] **Step 1: Write the failing test** in `lib/state/__tests__/busy.test.ts` ... a real two-connection snapshot conflict: + +```ts +test("isBusyError matches SQLITE_BUSY_SNAPSHOT from a real conflict", () => { + const dir = mkdtempSync(join(tmpdir(), "busy-snap-")); + const path = join(dir, "t.db"); + const a = new Database(path); a.exec("PRAGMA journal_mode=WAL; CREATE TABLE t(id INTEGER PRIMARY KEY, v INTEGER);"); + a.exec("INSERT INTO t(id,v) VALUES(1,0);"); + const b = new Database(path); + a.exec("BEGIN;"); a.query("SELECT v FROM t WHERE id=1").get(); // pin snapshot on A + b.exec("UPDATE t SET v=1 WHERE id=1;"); // B commits (autocommit) + let caught: unknown; + try { a.exec("UPDATE t SET v=2 WHERE id=1;"); } catch (e) { caught = e; } + expect(caught).toBeDefined(); + expect((caught as any).code?.startsWith("SQLITE_BUSY")).toBe(true); + expect(isBusyError(caught)).toBe(true); + try { a.exec("ROLLBACK;"); } catch {} + a.close(); b.close(); +}); +``` + +- [ ] **Step 2: Run ... expect FAIL** (`isBusyError` returns false for `SQLITE_BUSY_SNAPSHOT`). + +- [ ] **Step 3: Widen `isBusyError`** in `lib/state/busy.ts:39-41`: + +```ts +export function isBusyError(err: unknown): boolean { + const code = (err as { code?: string })?.code; + return code === "SQLITE_BUSY" || (typeof code === "string" && code.startsWith("SQLITE_BUSY_")); +} +``` + +- [ ] **Step 4: Run ... expect PASS.** + +- [ ] **Step 5: Convert read-then-write transactions to `.immediate()`.** In `lib/state/chat-store.ts`, change the four cited transaction builders (`readUnread` ~483, `joinRoom`, `archiveRoom`, `dmRoomFor`) from `db.transaction(fn)(...)` to `db.transaction(fn).immediate(...)` so the write lock is taken at `BEGIN IMMEDIATE`. Do the same for `drainNotificationQueue` in `lib/state/notifier-store.ts:111`. Leave a one-line comment at the first site: `// BEGIN IMMEDIATE: read-then-write must lock up front or SQLITE_BUSY_SNAPSHOT bypasses busy_timeout.` + +- [ ] **Step 6: Verify no regression** ... run the chat/notifier/state suites: `bun test lib/state` and any `chat` command tests. Expect PASS (behavior identical under no contention; the change only affects lock acquisition timing). + +- [ ] **Step 7: Commit** + +```bash +git add lib/state/busy.ts lib/state/chat-store.ts lib/state/notifier-store.ts lib/state/__tests__/busy.test.ts +git commit -m "state: isBusyError matches SQLITE_BUSY_*; read-then-write daemon txns use BEGIN IMMEDIATE" +``` + +--- + +## Task 8: state.db importer isolation (0.3 ... S074) + +A throwing legacy importer inside the v0 migration rolls back the whole migration, so `user_version` stays 0 and every subsequent open repeats the failure. Wrap each importer in a SAVEPOINT; on throw, roll back that one importer, warn, and still rename the file. + +**Files:** +- Modify: `lib/state/db.ts:400-417` (`importLegacyStores` ... SAVEPOINT per importer) +- Test: `lib/state/__tests__/db.test.ts` + +**Interfaces:** +- Consumes: `LEGACY_IMPORTS` array (line 71), `runMigrations`'s `consumed` list (renames to `.migrated`). + +- [ ] **Step 1: Write the failing test** in `lib/state/__tests__/db.test.ts`: + +```ts +test("a throwing legacy importer is isolated: db reaches SCHEMA_VERSION, other stores import, file renamed", () => { + // Arrange an isolated HOME with a project-mrs.json whose keys "5" and "05" normalize to the same iid, + // plus one benign legacy file another importer consumes. + writeFileSync(join(rtDir, "project-mrs.json"), JSON.stringify({ "host/repo": { mrs: { "5": {...}, "05": {...} } } })); + writeFileSync(join(rtDir, ".json"), JSON.stringify(benignFixture)); + const db = openStateDb(join(rtDir, "state.db"), "daemon"); + expect(db.query("PRAGMA user_version").get()).toEqual({ user_version: SCHEMA_VERSION }); + // benign store's rows landed: + expect(db.query("SELECT COUNT(*) c FROM ").get()).toMatchObject({ c: benignCount }); + // offending file still renamed: + expect(existsSync(join(rtDir, "project-mrs.json"))).toBe(false); + expect(existsSync(join(rtDir, "project-mrs.json.migrated"))).toBe(true); +}); +``` + +(Fill ``/``/fixtures from an existing importer in `LEGACY_IMPORTS`; the db.test.ts legacy-import cases already build such fixtures ... reuse one.) + +- [ ] **Step 2: Run ... expect FAIL** (the throw rolls back the whole migration; `user_version` stays 0 / benign rows absent). + +- [ ] **Step 3: Implement SAVEPOINT-per-importer** in `importLegacyStores` (`lib/state/db.ts:400-417`). For each `LEGACY_IMPORTS` entry, wrap its `run(db)` in a savepoint scoped to that importer only (do NOT loosen the surrounding schema-DDL migration, which must stay loud): + +```ts +for (const imp of LEGACY_IMPORTS) { + if (!existsSync(imp.path())) continue; + db.exec("SAVEPOINT legacy_import;"); + try { + imp.run(db); + db.exec("RELEASE legacy_import;"); + consumed.push(imp.path()); + } catch (err) { + db.exec("ROLLBACK TO legacy_import; RELEASE legacy_import;"); + log?.warn?.({ err, file: imp.path() }, "legacy import failed; skipping (file will still be renamed)"); + consumed.push(imp.path()); // spec: corrupt = warn + skip, but still rename + } +} +``` + +(Match the exact `LegacyImport` shape at db.ts:64-69 ... `path()`/`run(db)` names may differ; adapt to the real fields.) + +- [ ] **Step 4: Run ... expect PASS.** + +- [ ] **Step 5: Commit** + +```bash +git add lib/state/db.ts lib/state/__tests__/db.test.ts +git commit -m "state.db: isolate each legacy importer in a SAVEPOINT so one bad file cannot wedge migration" +``` + +--- + +## Task 9: Restart counter + last-exit reason in kv (0.5 ... R002, S037) + +Persist boot attempts, last-ready stamp, recent failures, and last-exit reason in the `kv` table (ns `daemon-supervision`), plus a boot breadcrumb file. Wire the record calls into the boot path, the shutdown verb, the signal handlers, and the boot fatal path. No schema change. + +**Files:** +- Create: `lib/daemon/supervision-state.ts` +- Create: `lib/daemon/__tests__/supervision-state.test.ts` +- Modify: `lib/daemon.ts` (record boot-attempt at `runDaemon` top; breadcrumb per phase; ready stamp; `recordBootFailure` in Task 2's catch; `recordCleanExit("shutdown")` in the shutdown verb), `lib/daemon/shutdown.ts` (record exit in `gracefulExit`) + +**Interfaces:** +- Produces: + - `recordBootAttempt(): void` ... increments `boot-attempts`, appends nothing. + - `recordDaemonReady(): void` ... sets `last-ready-at = Date.now()`. + - `recordBootFailure(phase: BootPhase, reason: string): void` ... appends `{ at, phase, reason }` to `recent-failures` (cap 10), sets `last-exit = { at, kind: "boot-failed", code: 1, reason }`. + - `recordCleanExit(kind: "shutdown" | "signal", code: number): void` ... sets `last-exit = { at, kind, code }`. + - `readSupervisionState(): { bootAttempts, lastReadyAt, recentFailures, lastExit }`. + - `isCrashLooping(state, now, n = 3, windowMs = 5*60_000): boolean` ... ≥ n failures newer than `now - windowMs`. + - `writeBreadcrumb(phase: BootPhase): void` / `clearBreadcrumb(): void` ... `~/.mattstack/rt/daemon-boot.json`. + - `type BootPhase = "start" | "crash-handlers" | "events-db" | "state-db" | "socket" | "api" | "ready"`. +- Consumes: `getKvValue`/`setKvValue` from `lib/state/kv-blob.ts`; `getStateDb("daemon")`. + +- [ ] **Step 1: Write the failing test** in `lib/daemon/__tests__/supervision-state.test.ts`: + +```ts +test("boot attempts, ready stamp, failures and last-exit round-trip through kv", () => { + recordBootAttempt(); recordBootAttempt(); + recordDaemonReady(); + recordBootFailure("api", "EADDRINUSE"); + const s = readSupervisionState(); + expect(s.bootAttempts).toBe(2); + expect(s.lastReadyAt).toBeGreaterThan(0); + expect(s.recentFailures.at(-1)).toMatchObject({ phase: "api", reason: "EADDRINUSE" }); + expect(s.lastExit).toMatchObject({ kind: "boot-failed", code: 1 }); +}); + +test("isCrashLooping true at >=3 failures within the window", () => { + const now = 1_000_000; + const fails = [now-10, now-20, now-30].map((at) => ({ at, phase: "api" as const, reason: "x" })); + expect(isCrashLooping({ bootAttempts: 3, lastReadyAt: 0, recentFailures: fails, lastExit: null }, now)).toBe(true); + const old = [{ at: now-10*60_000, phase: "api" as const, reason: "x" }]; + expect(isCrashLooping({ bootAttempts: 1, lastReadyAt: 0, recentFailures: old, lastExit: null }, now)).toBe(false); +}); +``` + +(Use the file's isolated-HOME convention ... `bunfig` preload already repoints HOME for `bun test`; `recordBootAttempt` writes to the test state.db.) + +- [ ] **Step 2: Run ... expect FAIL** (module does not exist). + +- [ ] **Step 3: Implement `lib/daemon/supervision-state.ts`** with the interfaces above. All reads/writes go through `getKvValue("daemon-supervision", key, fallback, getStateDb("daemon"))` / `setKvValue("daemon-supervision", key, value, getStateDb("daemon"))`. Cap `recent-failures` at 10 on append. Breadcrumb via `writeFileSync(join(RT_DIR, "daemon-boot.json"), JSON.stringify({ at, pid: process.pid, flavor: currentMode(), phase }))` inside a try/catch (never fatal). + +- [ ] **Step 4: Run ... expect PASS.** + +- [ ] **Step 5: Wire into the boot path** (`lib/daemon.ts`): + - `recordBootAttempt(); writeBreadcrumb("start");` at the top of `runDaemon()` (after Task 3's crash handlers are already at module scope; call these first thing inside `runDaemon`). + - `writeBreadcrumb("events-db" | "state-db" | "socket" | "api")` at each corresponding phase (events.db is module-scope ... write that breadcrumb right after `createEventsBus`; state-db right after `openBranchCacheStore`; socket/api at the binds). + - In Task 2's catch: `recordBootFailure(currentPhase, String(err));` (track `currentPhase` in a module var updated alongside each `writeBreadcrumb`). + - `recordDaemonReady(); writeBreadcrumb("ready");` right where `bootPhase = "ready"` is set (line 513). + - Shutdown verb (349-364): before `process.exit(0)`, `recordCleanExit("shutdown", 0);` and set the Task 12 `shuttingDownViaVerb = true` flag (Task 12 adds the flag; here just add the record call). + +- [ ] **Step 6: Wire into signal exit** (`lib/daemon/shutdown.ts` `gracefulExit`): before exit, `recordCleanExit("signal", code)` (Task 12 sets `code` to 1 for bare signals; for now record with the code it exits with). + +- [ ] **Step 7: Add an e2e assertion** in `e2e/tests/daemon.test.ts`: after the Task 5 API-bind-failure spawn, assert `daemon-boot.json` exists with `phase: "api"` and the kv `recent-failures` has an entry (read the state.db in the isolated HOME, or assert via `rt daemon status --json` once Task 10 lands ... for Task 9, assert the breadcrumb file only). + +- [ ] **Step 8: Run the unit + e2e tests ... expect PASS.** + +- [ ] **Step 9: Commit** + +```bash +git add lib/daemon/supervision-state.ts lib/daemon/__tests__/supervision-state.test.ts lib/daemon.ts lib/daemon/shutdown.ts e2e/tests/daemon.test.ts +git commit -m "daemon: persist boot attempts, failures, last-exit in kv + boot breadcrumb" +``` + +--- + +## Task 10: Status verdicts (0.5 ... R001, S026 daemon-side) + +Extend `DaemonStatusVerdict` and `classifyDaemonStatus` with `alive-not-serving`, `parked`, `boot-failed`, `crash-looping`, read from the liveness probe + supervision state + breadcrumb. Expose the fields in `ping`/`/api/status`, and print them in `rt daemon status`. + +**Files:** +- Modify: `lib/daemon-status.ts:14-19` (verdict type), `31-…` (`classifyDaemonStatus`) +- Modify: `lib/daemon/handlers/status.ts:23` (`ping` includes supervision summary; NOT fenced) +- Modify: `commands/daemon.ts:379-…` (`statusLines` prints new verdicts) +- Test: `lib/__tests__/daemon-status.test.ts` + +**Interfaces:** +- Consumes: `readSupervisionState`, `isCrashLooping`, breadcrumb reader from Task 9. +- Produces: `DaemonStatusVerdict` extended with `{ state: "alive-not-serving"; pid: number; detail: "booting" | "wedged" | "quarantined" }`, `{ state: "parked"; pid: number; holderFlavor?: string }`, `{ state: "boot-failed"; reason: string; phase: string }`, `{ state: "crash-looping"; failures: number; reason: string }`. + +- [ ] **Step 1: Write the failing tests** in `lib/__tests__/daemon-status.test.ts`: + +```ts +test("alive pid + failed ping -> alive-not-serving with breadcrumb detail", () => { + const v = classifyDaemonStatus({ installed: true, pingOk: false, pidAlive: true, pid: 42, + breadcrumb: { phase: "socket" }, supervision: emptySupervision() }); + expect(v).toMatchObject({ state: "alive-not-serving", pid: 42, detail: "booting" }); +}); +test("no pid + >=3 recent failures -> crash-looping", () => { + const v = classifyDaemonStatus({ installed: true, pingOk: false, pidAlive: false, pid: null, + supervision: { recentFailures: threeRecentFailures(), lastExit: { kind: "boot-failed", reason: "EADDRINUSE" } } }); + expect(v).toMatchObject({ state: "crash-looping" }); +}); +test("no pid + single boot-failed -> boot-failed with reason", () => { + const v = classifyDaemonStatus({ installed: true, pingOk: false, pidAlive: false, pid: null, + supervision: { recentFailures: [oneFailure("api","EADDRINUSE")], lastExit: { kind: "boot-failed", reason: "EADDRINUSE", phase: "api" } } }); + expect(v).toMatchObject({ state: "boot-failed", reason: "EADDRINUSE", phase: "api" }); +}); +``` + +- [ ] **Step 2: Run ... expect FAIL.** + +- [ ] **Step 3: Extend the verdict type and `classifyDaemonStatus`** in `lib/daemon-status.ts` following the resolution order from the design doc (Task 1): not-installed → serving → parked → alive-not-serving → crash-looping → boot-failed → installed-not-running. `classifyDaemonStatus` takes the already-gathered inputs (`pingOk`, `pidAlive`, `pid`, `breadcrumb`, `supervision`); keep it a pure function (the liveness probe and kv/breadcrumb reads happen in `commands/daemon.ts`/the caller, matching the existing `needsLivenessProbe` split). `parked` when breadcrumb phase indicates a flavor standoff; `alive-not-serving` detail = `booting` (phase < ready), `wedged` (phase == ready), `quarantined` (a `*.boot-failed` marker present). + +- [ ] **Step 4: Run ... expect PASS.** + +- [ ] **Step 5: Surface the data.** In `lib/daemon/handlers/status.ts` `ping`, add a `supervision: { bootAttempts, lastReadyAt, recentFailures: recentFailures.slice(-3), lastExit }` field (read via Task 9). In `commands/daemon.ts` `showStatus`/liveness probe, gather `pidAlive` (`process.kill(pid,0)` on rt.pid, fallback `pgrep -f 'rt --daemon|lib/daemon.ts'` via `runCapture`) and read the breadcrumb + supervision state (when ping fails), then pass to `classifyDaemonStatus`. In `statusLines`, print a line per new verdict (see the design doc's status strings). + +- [ ] **Step 6: Add a `--json` assertion e2e** in `e2e/tests/daemon.test.ts`: after an API-bind-failure spawn, `rt daemon status --json` under the same isolated HOME reports `boot-failed` or `crash-looping` (whichever the failure count yields). Run under `env -i HOME=`. + +- [ ] **Step 7: Run the unit + e2e tests ... expect PASS.** `bun test lib/__tests__/daemon-status.test.ts` + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon-status.ts lib/daemon/handlers/status.ts commands/daemon.ts lib/__tests__/daemon-status.test.ts e2e/tests/daemon.test.ts +git commit -m "daemon status: alive-not-serving / parked / boot-failed / crash-looping verdicts" +``` + +--- + +## Task 11: stderr log rotation + stale-crash stamp (0.5 ... S029) + +`daemon-stderr.log` is never rotated and its stale contents are shown as "most recent crash". Rotate on open and gate the crash block on mtime. + +**Files:** +- Modify: `lib/daemon-logger.ts:204-219` (`redirectNativeStderr` rotate-on-open) +- Modify: `commands/daemon.ts:716-765` (`showLogs` mtime gate + label) +- Test: `lib/__tests__/daemon-logger.test.ts`, `commands/__tests__/daemon.test.ts` (or wherever `showLogs` is tested) + +**Interfaces:** +- Produces: rotated filename `daemon-stderr..log` (matches the janitor's `LOG_FILE_PATTERN`, so it prunes for free). + +- [ ] **Step 1: Write the failing test** in `lib/__tests__/daemon-logger.test.ts`: + +```ts +test("redirectNativeStderr rotates a non-empty daemon-stderr.log before reopening", () => { + // isolated logs dir with a pre-existing non-empty daemon-stderr.log + writeFileSync(join(logsDir(), "daemon-stderr.log"), "old panic\n"); + redirectNativeStderr(); + const rotated = readdirSync(logsDir()).filter((f) => /^daemon-stderr\.\d{4}-\d{2}-\d{2}\.log$/.test(f)); + expect(rotated.length).toBe(1); + expect(statSync(join(logsDir(), "daemon-stderr.log")).size).toBe(0); +}); +``` + +- [ ] **Step 2: Run ... expect FAIL.** + +- [ ] **Step 3: Implement rotate-on-open** in `redirectNativeStderr` (`lib/daemon-logger.ts:204-219`): before `openSync(path, "a")`, if the file exists and is non-empty, `renameSync` it to `daemon-stderr..log` (dedupe with a `.N` suffix if that name already exists, matching the janitor's dated-file convention). Keep the existing swallow-on-failure behavior. + +- [ ] **Step 4: Run ... expect PASS.** + +- [ ] **Step 5: Write the failing `showLogs` test** ... the native-stderr block is hidden when the file mtime predates the current daemon's start: + +```ts +test("showLogs hides the native-stderr block when the file is older than the daemon start", () => { + // stub daemon startedAt newer than the file mtime; assert the red block is not printed +}); +``` + +- [ ] **Step 6: Implement in `showLogs`** (`commands/daemon.ts:724-737`): only print the native-stderr block when the file's `mtime` is newer than the current daemon's `startedAt` (from `ping`); include the mtime in the header (`native stderr (captured )`). When older, skip it silently (or print a one-line "no crash since this daemon started"). + +- [ ] **Step 7: Run ... expect PASS.** + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon-logger.ts commands/daemon.ts lib/__tests__/daemon-logger.test.ts +git commit -m "logs: rotate daemon-stderr.log on open; hide stale crash block by mtime" +``` + +--- + +## Task 12: Exit-code semantics (0.6 ... S036, S060) + +The `shutdown` verb correctly exits 0, but bare OS signals also exit 0, so an externally-killed daemon stays down. Reserve exit 0 for the verb; exit non-zero on bare signals. + +**Files:** +- Modify: `lib/daemon.ts:349-364` (shutdown verb sets `shuttingDownViaVerb = true`) +- Modify: `lib/daemon/shutdown.ts:57-71` (`gracefulExit` reads the flag) +- Test: `lib/daemon/__tests__/shutdown.test.ts` (new) + +**Interfaces:** +- Produces: a shared boolean the shutdown verb sets before cleanup; `installSignalHandlers` gains a `wasVerbShutdown: () => boolean` option (avoids a cross-module global). `gracefulExit` exits 0 when `wasVerbShutdown()` is true, else `process.exit(1)`. +- Consumes: `recordCleanExit` (Task 9). + +- [ ] **Step 1: Write the failing test** in `lib/daemon/__tests__/shutdown.test.ts`: + +```ts +test("gracefulExit exits 0 after the shutdown verb, 1 on a bare signal", () => { + const exits: number[] = []; + const exit = (c?: number) => { exits.push(c ?? 0); }; + let viaVerb = false; + const handlers = makeGracefulExit({ cleanup: () => {}, flushLogs: () => {}, log: silentLog, + wasVerbShutdown: () => viaVerb, exit, recordCleanExit: () => {} }); + handlers("SIGTERM"); + expect(exits).toEqual([1]); + viaVerb = true; + handlers("SIGTERM"); + expect(exits).toEqual([1, 0]); +}); +``` + +(Refactor `gracefulExit` into a testable `makeGracefulExit(deps)` that returns the handler, injecting `exit`/`recordCleanExit` so no real `process.exit` fires in the test.) + +- [ ] **Step 2: Run ... expect FAIL.** + +- [ ] **Step 3: Implement.** In `lib/daemon/shutdown.ts`, refactor `installSignalHandlers`/`gracefulExit` to `makeGracefulExit(deps)` reading `deps.wasVerbShutdown()`: true → `recordCleanExit("shutdown", 0)` + `exit(0)`; false → `recordCleanExit("signal", 1)` + `exit(1)`. In `lib/daemon.ts`, add module-scope `let shuttingDownViaVerb = false;`, set it `true` in the shutdown verb before `cleanup()`, and pass `wasVerbShutdown: () => shuttingDownViaVerb` into `installSignalHandlers` at line 511. The shutdown verb keeps `process.exit(0)`. + +- [ ] **Step 4: Run ... expect PASS.** + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon.ts lib/daemon/shutdown.ts lib/daemon/__tests__/shutdown.test.ts +git commit -m "daemon: bare-signal exit is non-zero (launchd respawns); shutdown verb stays exit 0" +``` + +--- + +## Task 13: Ownership-aware cleanup + eviction death-confirmation (0.6 ... S012, S044) + +`cleanup()` unlinks rt.sock/rt.pid unconditionally, and eviction sleeps a blind 300ms. Make cleanup compare-and-delete, and make eviction wait for the old pid to actually die. + +**Files:** +- Modify: `lib/daemon/shutdown.ts:44-46` (ownership-aware unlink) +- Modify: `lib/daemon/boot-reconcile.ts:16-26` (poll-to-death, escalate to SIGKILL) +- Test: `lib/daemon/__tests__/shutdown.test.ts`, `lib/daemon/__tests__/boot-reconcile.test.ts` (new) + +**Interfaces:** +- Produces: `cleanup()` unlinks rt.pid only when its content `=== String(process.pid)`, and gates the rt.sock unlink on that same check. `evictStaleDaemon` polls `process.kill(pid, 0)` up to a bound (e.g. 3s / 30×100ms), escalates to SIGKILL, and only returns once the pid is gone. + +- [ ] **Step 1: Write the failing cleanup test** in `lib/daemon/__tests__/shutdown.test.ts`: + +```ts +test("cleanup does not unlink rt.pid/rt.sock when the pid file belongs to another process", () => { + writeFileSync(DAEMON_PID_PATH, "999999"); // not our pid + writeFileSync(DAEMON_SOCK_PATH, ""); + createCleanup({ ...deps, pid: process.pid })(); + expect(existsSync(DAEMON_PID_PATH)).toBe(true); + expect(existsSync(DAEMON_SOCK_PATH)).toBe(true); +}); +test("cleanup unlinks when the pid file is ours", () => { + writeFileSync(DAEMON_PID_PATH, String(process.pid)); + writeFileSync(DAEMON_SOCK_PATH, ""); + createCleanup({ ...deps, pid: process.pid })(); + expect(existsSync(DAEMON_PID_PATH)).toBe(false); + expect(existsSync(DAEMON_SOCK_PATH)).toBe(false); +}); +``` + +(Inject `pid` into `createCleanup` deps for testability; default to `process.pid`.) + +- [ ] **Step 2: Run ... expect FAIL.** + +- [ ] **Step 3: Implement ownership-aware unlink** in `createCleanup` (`lib/daemon/shutdown.ts:44-46`): + +```ts +try { + if (existsSync(DAEMON_PID_PATH) && readFileSync(DAEMON_PID_PATH, "utf8").trim() === String(deps.pid)) { + unlinkSync(DAEMON_PID_PATH); + if (existsSync(DAEMON_SOCK_PATH)) unlinkSync(DAEMON_SOCK_PATH); + } +} catch (err) { deps.log.warn({ err }, "cleanup unlink skipped"); } +``` + +- [ ] **Step 4: Run ... expect PASS.** + +- [ ] **Step 5: Write the failing eviction test** in `lib/daemon/__tests__/boot-reconcile.test.ts`: + +```ts +test("evictStaleDaemon waits for the old pid to die, escalating to SIGKILL", async () => { + // Spawn a child that ignores SIGTERM; write its pid to rt.pid; assert evict SIGKILLs it and returns only once gone. + const child = Bun.spawn({ cmd: ["bash", "-c", "trap '' TERM; sleep 30"] }); + writeFileSync(DAEMON_PID_PATH, String(child.pid)); + const start = Date.now(); + await evictStaleDaemon(silentLog); // now async + expect(isAlive(child.pid)).toBe(false); + expect(Date.now() - start).toBeLessThan(5000); +}); +``` + +- [ ] **Step 6: Implement in `evictStaleDaemon`** (`lib/daemon/boot-reconcile.ts`): replace `Bun.sleepSync(300)` with an async poll ... after SIGTERM, loop `process.kill(pid, 0)` every 100ms up to ~2.5s; if still alive, `process.kill(pid, "SIGKILL")` and poll another ~0.5s; return once `process.kill(pid,0)` throws (pid gone). Make the function `async` and `await` it at the call site (`lib/daemon.ts:396`). Keep the `previousPid === process.pid` self-guard. + +- [ ] **Step 7: Run ... expect PASS.** Then `bun test lib/daemon/__tests__/boot-reconcile.test.ts lib/daemon/__tests__/shutdown.test.ts` + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon/shutdown.ts lib/daemon/boot-reconcile.ts lib/daemon.ts lib/daemon/__tests__/shutdown.test.ts lib/daemon/__tests__/boot-reconcile.test.ts +git commit -m "daemon: ownership-aware socket/pid unlink; eviction waits for pid death then SIGKILL" +``` + +--- + +## Task 14: uninstall + start guards (0.6 ... S027, S030, S028 CLI-side) + +`rt daemon uninstall` deletes rt.sock/rt.pid from under a live daemon, and `rt daemon start` cannot revive a registered-but-exited-0 daemon. Guard uninstall on liveness; make start escalate to a kickstart route. + +**Files:** +- Modify: `commands/daemon.ts:210-231` (`uninstall` guard), `commands/daemon.ts:235-269` (`start` escalation) +- Modify: `lib/daemon-client.ts:171-185` (`attemptRestart` re-probes liveness) +- Test: `commands/__tests__/daemon.test.ts` (or the existing commands/daemon test location) + +**Interfaces:** +- Consumes: `isDaemonProcessRunning()` (daemon-config.ts:152), `probeSocketHolder()` (park.ts), `activeLaunchdLabel()` (daemon-config.ts:61), `isDaemonRunning()`. + +- [ ] **Step 1: Write the failing uninstall test**: + +```ts +test("uninstall leaves rt.sock/rt.pid and daemon.json when the daemon is still running", async () => { + // stub trayQuery('/daemon/stop') to fail, isDaemonProcessRunning -> true + await uninstall(); + expect(cleanupDaemonFilesSpy).not.toHaveBeenCalled(); + expect(printedRemedy).toContain("launchctl bootout"); +}); +``` + +- [ ] **Step 2: Run ... expect FAIL.** + +- [ ] **Step 3: Implement the uninstall guard** (`commands/daemon.ts:210-231`): after a failed/absent `trayQuery("/daemon/stop")`, call `isDaemonProcessRunning()` (and `probeSocketHolder()` as a second signal). Only run `markDaemonUninstalled()` + `cleanupDaemonFiles()` when no live holder; otherwise print the remedy `launchctl bootout gui/$UID/${activeLaunchdLabel()}` and leave the files. Audit other callers of `cleanupDaemonFiles`/`markDaemonUninstalled` (e.g. the dev-mode toggle in `commands/settings.ts`) for the same missing guard and note any in the commit body. + +- [ ] **Step 4: Run ... expect PASS.** + +- [ ] **Step 5: Write the failing start-escalation test**: + +```ts +test("start escalates to the restart/kickstart route when the tray acks but the socket stays absent", async () => { + // trayQuery('/daemon/start') returns ok, isDaemonRunning stays false through the poll + await start(); + expect(restartRouteSpy).toHaveBeenCalled(); +}); +``` + +- [ ] **Step 6: Implement start escalation** (`commands/daemon.ts:235-269`): when the tray acked `/daemon/start` but `isDaemonRunning()` stays false through the 12×250ms poll, fall back to the `/daemon/restart` route (kickstart). In `lib/daemon-client.ts:171-185`, make `attemptRestart` re-probe `isDaemonRunning()` after `trayQuery("/daemon/start")` and return `true` only when the daemon actually answers ... so the `daemonQuery` nag stops misdirecting. Do NOT change signal-handler exit codes here (that is Task 12). + +- [ ] **Step 7: Run ... expect PASS.** + +- [ ] **Step 8: Commit** + +```bash +git add commands/daemon.ts lib/daemon-client.ts commands/__tests__/daemon.test.ts +git commit -m "daemon CLI: uninstall guards on liveness; start escalates to kickstart; attemptRestart re-probes" +``` + +--- + +## Task 15: Retire the stale audit doc (0.8 ... R017) + +Replace `docs/daemon-runner-health.md` (which audits deleted subsystems) with a pointer to the current audit and the new supervision design doc. + +**Files:** +- Modify: `docs/daemon-runner-health.md` + +- [ ] **Step 1: Replace the file's contents** with a short pointer: + +```markdown +# Daemon runner health ... superseded + +This document audited subsystems (process-manager, remedy-engine, +runner.tsx, workspace-sync) that no longer exist. It is retained only as +a redirect. + +- Current stability audit + roadmap: `docs/daemon-stability-audit-2026-08.md` + (in the daemon-stability-audit worktree). +- Supervision verdicts + exit-code semantics: `docs/daemon-supervision-design.md`. +``` + +(If R058's comment sweep is trivially co-located, remove the one stale comment it names; otherwise leave it.) + +- [ ] **Step 2: Commit** + +```bash +git add docs/daemon-runner-health.md +git commit -m "docs: retire stale daemon-runner-health.md, point at the current audit + supervision design" +``` + +--- + +## Task 16 (OPTIONAL ... pending plan-review scope confirmation): Swift tray consumption + +**Do not start without the reviewer's go-ahead** (raised in the plan-milestone report). These edits cannot be verified by the bun/tsc/e2e gate, and the operating rules forbid rebuilding the blessed bundle, so they ship as source-only, unverified changes following the fixer notes: + +- **S026** ... `rt-tray/Sources/AppDelegate.swift`: give `.starting` an expiry (record `startingSince`; in `refreshStatus` treat `.starting` as expired after ~30s / 3 failed polls and fall through to `setHealth(.down)`), so the health dot stops sticking yellow; map the new daemon verdicts (Task 10) to dot colors. +- **S028** ... `rt-tray/Sources/DaemonLifecycle.swift`: when `register()` returns already-registered but the socket stays unreachable, fall back to `launchctl kickstart` (Kickstart.arguments already exists). +- **S029** ... `rt-tray/Sources/TrayLog.swift`: rotate `tray-crash.log` on open (rename-if-nonempty), matching Task 11's `daemon-stderr.log` treatment. +- **S060** ... `rt-tray/Sources/AppDelegate.swift:625-627`: fix the stale comment to say `KeepAlive: SuccessfulExit=false` (not `KeepAlive=true`). + +If confirmed, do the comment fix (S060) first (trivial), then S026/S028/S029, one commit each, each commit body noting "source-only, unverified: blessed bundle not rebuilt". + +--- + +## Final verification (before the whole-branch review) + +- [ ] `cd packages/rt-client && bun run build && cd -` (rt-client was touched in Task 5) +- [ ] `bunx tsc --noEmit` ... zero errors +- [ ] `bun test lib commands packages scripts` ... green +- [ ] `bun run test:e2e` (or `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts` ... record which) +- [ ] Request the whole-branch code review (superpowers:requesting-code-review); address findings; re-run the gate. diff --git a/docs/superpowers/plans/2026-08-28-p1-event-loop.md b/docs/superpowers/plans/2026-08-28-p1-event-loop.md new file mode 100644 index 000000000..c8d355275 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-p1-event-loop.md @@ -0,0 +1,1490 @@ +# Phase 1 · Event-Loop Sacred Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Excise every synchronous subprocess call from daemon-reachable code, make `runCapture`'s timeout actually enforceable, add an import-graph gate that keeps sync-exec dead, and stop the coalesced refresh and background scans from wedging or taxing the event loop. + +**Architecture:** Route all daemon-thread subprocess work through the existing async helpers (`runCapture` in `lib/subprocess.ts`, `runGit`/`listWorktreesAsync` in `lib/worktree/git-async.ts`). Harden `runCapture` so a pipe-holding grandchild can no longer hold its promise open. Add a source-graph test that walks the daemon's import tree and fails on any sync-exec import. Bound the refresh cycle and the provider cache, and gate/cheapen the periodic scans. + +**Tech Stack:** Bun 1.3.13, TypeScript (strict), `bun:test`, `bun:sqlite`, pino. `@mattstack/glance` provides `GitLabProvider`. + +**Spec:** `/Users/matt/Documents/GitHub/repo-tools/.claude/worktrees/daemon-stability-audit/docs/daemon-stability-audit-2026-08.md` ... "Roadmap > Phase 1" (items 1.1-1.5) plus Appendix A/B for each finding (S007, S008, S015, S016, S021, S023, S024, S038, S039, S045, S048, S049, S055, S058, S061, S093, S098, S101, S104, R032). + +## Global Constraints + +- **Isolated HOME for any binary run.** Never start a daemon or run `dist/rt` against the real machine. Any daemon or `dist/rt` invocation runs under `env -i HOME=` only. Tests use `bun:test` with tmp HOME/`RT_RUNS_ROOT` fixtures ... never touch the developer's `~/.mattstack` or `~/.rt`. +- **Write fence.** These sibling-owned files must NOT be modified this run: `cli.ts`, `lib/daemon.ts`, `lib/daemon/park.ts`, `lib/daemon/boot-reconcile.ts`, `lib/daemon/boot-migrate.ts`, `lib/daemon/shutdown.ts`, `lib/daemon-logger.ts`, `lib/daemon-config.ts`, `lib/daemon-status.ts`, `commands/daemon.ts`, `lib/daemon/events-bus.ts`, `lib/daemon/home-snapshot.ts`, `lib/daemon/handlers/status.ts`, `lib/state/*`, `rt-tray/**`, `lib/daemon/api-server.ts`, `lib/daemon/api-auth.ts`, `lib/daemon/socket-server.ts`, `lib/daemon/handlers/secrets.ts`, `lib/notifier.ts`, `lib/daemon/handlers/discussions.ts`, `lib/daemon/handlers/chat.ts`, `lib/daemon/handlers/agent.ts`, `lib/daemon/handlers/pane.ts`, `lib/daemon/handlers/project-mrs.ts`, `lib/daemon/handlers/worktree.ts`, `lib/herdr/client.ts`, `lib/port-scanner.ts`, `lib/deps/links.ts`, `lib/worktree/trash.ts`, `lib/agent-herdr.ts`, `lib/daemon/cron.ts`, `lib/daemon/hooks-guard.ts`, `lib/home/age-key.ts`, `lib/daemon/discussions-store.ts`. +- **S055 carve-out.** `lib/daemon/handlers/status.ts` is sibling-owned. Do NOT edit it. The async replacement it needs (`listWorktreesAsync`) already lives in a file we own (`lib/worktree/git-async.ts`). The one-line call-site swap is documented in the report Notes, not applied here. The import-graph gate (Task 7) allowlists `status.ts` with a comment naming S055. +- **rt-client mirror.** `packages/rt-client/src/settings/exec.ts` is a byte-for-byte mirror of `lib/subprocess.ts`'s `runCapture`. Change `lib/subprocess.ts` first, mirror there, then run `bun run build` inside `packages/rt-client` (its `dist/` is gitignored and copied verbatim by `file:` consumers; `packages/rt-client/test/dist-freshness.test.ts` fails otherwise). +- **Canonical fixtures.** `listWorktreesAsync` returns git's canonicalized paths (`/private/var/...` on macOS tmpdirs). Tests that build temp repos must compare against `realpathSync`'d paths. +- **Subagent models.** Every subagent dispatched during execution carries an explicit `model` (`sonnet` for mechanical tasks, `haiku` for lookups). +- **Verification (all must pass before done):** `bun test lib commands packages scripts` green; `bunx tsc --noEmit` zero errors; the Task 7 gate passes AND is proven to fail when a forbidden import is reintroduced (show the RED run in the report). +- **Comments:** clean-code rules. A comment states a constraint the code cannot show (a parity anchor, an ordering trap, a non-obvious invariant). No narration, no finding IDs in source. No em dashes. + +--- + +### Task 1: Make `runCapture`'s timeout enforceable (S023, S024) + +`runCapture` awaits `new Response(proc.stdout).text()` unconditionally after the kill timer fires; a grandchild that inherited the pipe keeps that read pending forever, so the promise never settles and every in-flight guard latches. Fix: race the reads against the deadline so `runCapture` always settles within `timeoutMs`, and escalate SIGTERM→SIGKILL on the child. (Group-kill via `process.kill(-pid)` is NOT reliable on Bun 1.3.13 ... verified ESRCH once the direct child exits ... so this uses the race as the load-critical fix, per the S023 fixer note that racing the reads is mandatory and kill escalation is belt-and-suspenders. A surviving grandchild leaks its fd until the OS reaps it, which the audit accepts.) + +**Files:** +- Modify: `lib/subprocess.ts` (the `runCapture` body and `RunResult`) +- Mirror: `packages/rt-client/src/settings/exec.ts` (byte-for-byte) +- Test: `lib/__tests__/subprocess.test.ts` (create if absent) + +**Interfaces:** +- Produces: `runCapture(argv, opts) => Promise` where `RunResult` gains `timedOut?: boolean`. Contract unchanged otherwise: never throws; on spawn failure/timeout/read error returns `exitCode: -1` (callers already branch on `!== 0`). `timedOut: true` is additive and set only on the deadline path. + +- [ ] **Step 1: Write the failing test** + +Create `lib/__tests__/subprocess.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { runCapture } from "../subprocess.ts"; + +test("resolves within the deadline even when a grandchild holds the pipe", async () => { + // zsh exits after ~0.2s, but backgrounds `sleep 20` which inherits stdout. + const t0 = Date.now(); + const r = await runCapture( + ["/bin/zsh", "-c", "sleep 20 & echo started; sleep 0.2"], + { timeoutMs: 1000 }, + ); + const elapsed = Date.now() - t0; + expect(elapsed).toBeLessThan(4000); // must NOT wait for the 20s grandchild + expect(r.timedOut).toBe(true); + expect(r.exitCode).toBe(-1); +}); + +test("a SIGTERM-ignoring child is bounded by SIGKILL escalation", async () => { + const t0 = Date.now(); + const r = await runCapture( + ["/bin/zsh", "-c", "trap '' TERM; sleep 20"], + { timeoutMs: 800 }, + ); + expect(Date.now() - t0).toBeLessThan(4000); + expect(r.timedOut).toBe(true); +}); + +test("normal fast command still returns real stdout and exitCode 0", async () => { + const r = await runCapture(["/bin/echo", "hello"], { timeoutMs: 5000 }); + expect(r.stdout.trim()).toBe("hello"); + expect(r.exitCode).toBe(0); + expect(r.timedOut).toBeUndefined(); +}); + +test("timed-out call reports exitCode -1 so callers treat it as failure", async () => { + const r = await runCapture(["/bin/sleep", "20"], { timeoutMs: 500 }); + expect(r.exitCode).toBe(-1); + expect(r.timedOut).toBe(true); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/__tests__/subprocess.test.ts` +Expected: the grandchild test FAILS (elapsed ~20s, exceeds 4000ms) or times out ... proving the current unconditional-await bug. + +- [ ] **Step 3: Rewrite `runCapture` to race the reads against the deadline** + +In `lib/subprocess.ts`, add `timedOut` to `RunResult` and replace the timer + read block: + +```ts +export interface RunResult { + stdout: string; + stderr: string; + exitCode: number; + /** Set true only when the deadline fired before the child settled. */ + timedOut?: boolean; +} +``` + +Replace the body from the `const timer = setTimeout(...)` line through the closing of the `try/catch/finally` (lines 60-76) with: + +```ts + const timeoutMs = opts.timeoutMs ?? 10_000; + // SIGTERM at the deadline, SIGKILL a short grace later. A child that ignores + // SIGTERM (or a D-state descendant) cannot be reaped in-band, so the read is + // raced against the deadline below rather than awaited unconditionally: that + // is what lets runCapture settle while a grandchild still holds the pipe. + let killTimer: ReturnType | undefined; + const term = setTimeout(() => { + try { proc.kill("SIGTERM"); } catch { /* already exited */ } + killTimer = setTimeout(() => { + try { proc.kill("SIGKILL"); } catch { /* already exited */ } + }, 2000); + }, timeoutMs); + + const captured: Promise = (async () => { + try { + const stdoutPromise = new Response(proc.stdout as ReadableStream).text(); + const stderrPromise = captureStderr + ? new Response(proc.stderr as ReadableStream).text() + : Promise.resolve(""); + const [stdout, stderr, exitCode] = await Promise.all([ + stdoutPromise, + stderrPromise, + proc.exited, + ]); + return { stdout, stderr, exitCode }; + } catch { + return { stdout: "", stderr: "", exitCode: -1 }; + } + })(); + + const deadline: Promise = new Promise((resolve) => { + setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); + }); + + try { + return await Promise.race([captured, deadline]); + } finally { + clearTimeout(term); + if (killTimer) clearTimeout(killTimer); + } +``` + +(Keep the `Bun.spawn` block above it unchanged, including the env comment.) + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/__tests__/subprocess.test.ts` +Expected: PASS (all four). + +- [ ] **Step 5: Mirror into rt-client and rebuild** + +Apply the identical `RunResult`/body change to `packages/rt-client/src/settings/exec.ts` (its `runCapture` is the same shape; keep its own env comment). Then: + +Run: `bun run build` (from `packages/rt-client/`) +Then: `bun test packages/rt-client` +Expected: `dist-freshness.test.ts` PASS. + +- [ ] **Step 6: Commit** + +```bash +git add lib/subprocess.ts packages/rt-client/src/settings/exec.ts packages/rt-client/dist lib/__tests__/subprocess.test.ts +git commit -m "runCapture: race reads against the deadline so a pipe-holding grandchild can't wedge it (S023, S024)" +``` + +--- + +### Task 2: Long timeouts for mutating git verbs (S104) + +Mutating git verbs (checkout, merge, stash push/pop) and `status` share the 60s `DEFAULT_TIMEOUT_MS`; a large-repo checkout gets SIGKILLed half-applied at 60s. Give them the same 5-minute budget that fetch/`worktree add` already have. + +**Files:** +- Modify: `lib/worktree/git-async.ts` (add constant; thread `timeoutMs` into stash helpers and `statusPorcelainAsync`) +- Modify: `lib/daemon/worktree-reconciler.ts` (checkout/merge call sites in `freshenOne` and `autoReturnMain`) +- Test: `lib/__tests__/git-async-timeouts.test.ts` (create) + +**Interfaces:** +- Produces: `export const MUTATING_TIMEOUT_MS = 5 * 60_000;` in `git-async.ts`. `stashChangesAsync(cwd, label, opts?: { timeoutMs?: number })` and `popStashAsync(cwd, stashName, opts?: { timeoutMs?: number })` now accept an optional timeout, defaulting to `MUTATING_TIMEOUT_MS`. + +- [ ] **Step 1: Write the failing test** + +Create `lib/__tests__/git-async-timeouts.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { MUTATING_TIMEOUT_MS } from "../worktree/git-async.ts"; + +test("mutating timeout is 5 minutes", () => { + expect(MUTATING_TIMEOUT_MS).toBe(5 * 60_000); +}); +``` + +Also assert the stash helper signature accepts an override (compile-time guard): + +```ts +import { stashChangesAsync, popStashAsync } from "../worktree/git-async.ts"; +test("stash helpers accept a timeout override", () => { + // Type-level: these must type-check with an opts arg. + const a: typeof stashChangesAsync = stashChangesAsync; + const b: typeof popStashAsync = popStashAsync; + expect(typeof a).toBe("function"); + expect(typeof b).toBe("function"); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/__tests__/git-async-timeouts.test.ts` +Expected: FAIL ... `MUTATING_TIMEOUT_MS` not exported. + +- [ ] **Step 3: Add the constant and thread it** + +In `lib/worktree/git-async.ts`, below `const DEFAULT_TIMEOUT_MS = 60_000;`: + +```ts +/** Checkout/merge/stash on a large tree can legitimately exceed a minute; a + * 60s SIGKILL leaves the working tree half-switched. Match fetch/worktree-add. */ +export const MUTATING_TIMEOUT_MS = 5 * 60_000; +``` + +Update the helpers: + +```ts +export async function statusPorcelainAsync(cwd: string): Promise { + const r = await runGit(cwd, ["status", "--porcelain"], { timeoutMs: MUTATING_TIMEOUT_MS }); + return r.stdout; +} + +export async function stashChangesAsync( + cwd: string, + label: string, + opts: { timeoutMs?: number } = {}, +): Promise { + const message = `!!GitHub_Desktop<${label}>`; + await runGit(cwd, ["stash", "push", "-u", "-m", message], { + timeoutMs: opts.timeoutMs ?? MUTATING_TIMEOUT_MS, + }); +} + +export async function popStashAsync( + cwd: string, + stashName: string, + opts: { timeoutMs?: number } = {}, +): Promise { + await runGit(cwd, ["stash", "pop", stashName], { + timeoutMs: opts.timeoutMs ?? MUTATING_TIMEOUT_MS, + }); +} +``` + +- [ ] **Step 4: Bump the reconciler checkout/merge call sites** + +In `lib/daemon/worktree-reconciler.ts`, add `MUTATING_TIMEOUT_MS` to the existing `../worktree/git-async.ts` import (the file already imports `runGit` from there). Add `{ timeoutMs: MUTATING_TIMEOUT_MS }` to these `runGit` calls: +- `freshenOne`: `["checkout", "--", ...classify.discard]` (~line 727); `["merge", "--ff-only", defaultRef]` (~line 756). +- `autoReturnMain`: `["checkout", defaultBranch]` (~line 469); `["merge", "--ff-only", defaultRef]` (~line 475). + +Example: + +```ts +const merge = await runGit(rec.path, ["merge", "--ff-only", defaultRef], { timeoutMs: MUTATING_TIMEOUT_MS }); +``` + +The stash push/pop calls in these functions inherit the new default automatically. (Leave `worktree prune` and `branch -D` at the 60s default; they are not in S104's named set and are fast.) + +- [ ] **Step 5: Run tests** + +Run: `bun test lib/__tests__/git-async-timeouts.test.ts && bun test lib/daemon/__tests__/worktree-reconciler` +Expected: PASS. If a reconciler test asserts an exact `runGit` argv without opts, update it to allow the opts arg. + +- [ ] **Step 6: Commit** + +```bash +git add lib/worktree/git-async.ts lib/daemon/worktree-reconciler.ts lib/__tests__/git-async-timeouts.test.ts +git commit -m "git-async: 5-min timeout for checkout/merge/stash/status so a large-repo checkout isn't killed half-applied (S104)" +``` + +--- + +### Task 3: Excise sync exec from cache-refresh and git-worktrees (S008, S045, S021) + +The 5-minute refresh runs `execSync` (`for-each-ref`, `git config`) and the sync `listWorktrees`/`listWorktreeRoots` per repo on the daemon thread. Swap to the async helpers. Add the missing `listWorktreeRootsAsync` twin. Gate the doppler loop on grants (honors the "off = zero background work" contract). + +**Files:** +- Modify: `lib/worktree/git-async.ts` (add `listWorktreeRootsAsync`) +- Modify: `lib/daemon/cache-refresh.ts` (three swaps + doppler grant gate) +- Modify: `lib/daemon/__tests__/cache-refresh-gc.test.ts` (retarget the `listWorktrees`/`listWorktreeRoots` spies) +- Test: `lib/__tests__/git-async.test.ts` or add to existing git-async coverage for `listWorktreeRootsAsync` + +**Interfaces:** +- Consumes: `listWorktreesAsync(path) => Promise`, `runGit(cwd, args) => Promise` (Task 1/2 hardened). +- Produces: `listWorktreeRootsAsync(repoPath) => Promise` (returns `[]` on git failure, matching the sync contract). + +- [ ] **Step 1: Write the failing test for `listWorktreeRootsAsync`** + +Add to a git-async test file (create `lib/__tests__/git-worktree-roots-async.test.ts`): + +```ts +import { test, expect } from "bun:test"; +import { mkdtempSync } from "fs"; +import { realpathSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { listWorktreeRootsAsync } from "../worktree/git-async.ts"; +import { runGit } from "../worktree/git-async.ts"; + +test("listWorktreeRootsAsync returns the main worktree path", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-wt-"))); + await runGit(dir, ["init", "-q"]); + await runGit(dir, ["commit", "--allow-empty", "-m", "init", "-c", "user.email=a@b.c", "-c", "user.name=t"]); + const roots = await listWorktreeRootsAsync(dir); + expect(roots).toContain(dir); +}); + +test("listWorktreeRootsAsync returns [] on a non-repo", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-nonrepo-"))); + expect(await listWorktreeRootsAsync(dir)).toEqual([]); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/__tests__/git-worktree-roots-async.test.ts` +Expected: FAIL ... `listWorktreeRootsAsync` not exported. + +- [ ] **Step 3: Add `listWorktreeRootsAsync`** + +In `lib/worktree/git-async.ts`, after `listWorktreesAsync`: + +```ts +/** Worktree root paths (main + linked), existing-on-disk only. `[]` on git + * failure ... the async twin of git-worktrees.ts listWorktreeRoots. */ +export async function listWorktreeRootsAsync(repoPath: string): Promise { + return (await listWorktreesAsync(repoPath) ?? []).map((w) => w.path); +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/__tests__/git-worktree-roots-async.test.ts` +Expected: PASS. + +- [ ] **Step 5: Swap the cache-refresh call sites** + +In `lib/daemon/cache-refresh.ts`: + +Imports ... remove `import { execSync } from "child_process";` and `import { listWorktreeRoots, listWorktrees } from "../git-worktrees.ts";`; add `import { listWorktreesAsync, listWorktreeRootsAsync, runGit } from "../worktree/git-async.ts";`. + +Branch listing (line ~112): + +```ts +const branches: Array<{ path: string; branch: string }> = ((await listWorktreesAsync(repoPath)) ?? []) + .filter((w): w is { path: string; branch: string } => !!w.branch && !w.branch.startsWith("on-deck/")); +``` + +`for-each-ref` (lines ~118-133): + +```ts +const worktreeBranchSet = new Set(branches.map((b) => b.branch)); +const localBranches = await runGit(repoPath, ["for-each-ref", "--format=%(refname:short)", "refs/heads/"]); +if (localBranches.exitCode === 0) { + for (const name of localBranches.stdout.split("\n")) { + const trimmed = name.trim(); + if (!trimmed || worktreeBranchSet.has(trimmed) || trimmed.startsWith("on-deck/")) continue; + if (extractLinearId(trimmed)) branches.push({ path: repoPath, branch: trimmed }); + } +} else { + log.warn({ repo: repoPath }, "local branch listing failed"); +} +``` + +(argv form removes the shell, so the old `replace(/^'|'$/g, "")` quote-stripping is no longer needed ... the `--format=%(refname:short)` has no surrounding quotes without a shell.) + +Remote URL (lines ~138-142): + +```ts +let remoteUrl: string | undefined; +const remote = await runGit(repoPath, ["config", "--get", "remote.origin.url"]); +if (remote.exitCode === 0) remoteUrl = remote.stdout.trim() || undefined; +``` + +Doppler loop (line ~209): replace `listWorktreeRoots(repoPath)` with `await listWorktreeRootsAsync(repoPath)`, and gate the loop body on grants. Load tracking once above the loop and skip repos with no cache grant: + +```ts +const tracking = loadRepoTracking(); +for (const [repoName, repoPath] of Object.entries(repoIndex())) { + if (!existsSync(repoPath)) continue; + if (grants(tracking, repoName).caches.size === 0) continue; // off = zero background work + try { + const worktreeRoots = await listWorktreeRootsAsync(repoPath); + // ...unchanged body... +``` + +(`loadRepoTracking` and `grants` are already imported in this file.) + +- [ ] **Step 6: Retarget the GC test spies** + +In `lib/daemon/__tests__/cache-refresh-gc.test.ts` (~lines 105, 108), the test spies on `gitWorktreesModule.listWorktrees`/`listWorktreeRoots`. Retarget to the async twins: + +```ts +import * as gitAsync from "../../worktree/git-async.ts"; +// ... +spyOn(gitAsync, "listWorktreesAsync").mockResolvedValue([]); +spyOn(gitAsync, "listWorktreeRootsAsync").mockResolvedValue([]); +``` + +(Remove the now-unused `gitWorktreesModule` import if nothing else uses it.) + +- [ ] **Step 7: Run the cache-refresh tests** + +Run: `bun test lib/daemon/__tests__/cache-refresh-gc.test.ts` +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add lib/worktree/git-async.ts lib/daemon/cache-refresh.ts lib/daemon/__tests__/cache-refresh-gc.test.ts lib/__tests__/git-worktree-roots-async.test.ts +git commit -m "cache-refresh: async git + grant-gated doppler loop; add listWorktreeRootsAsync (S008, S045, S021)" +``` + +--- + +### Task 4: Async, cached `getRemoteUrl` in freshness (R032) + +`getRemoteUrl` uses `execSync` on the daemon thread inside every forge handler (mr:action, discussions:*, project sync) and the freshness reconcile loop. Make it async via `runCapture` with a 5s timeout, cached per repoPath for the process lifetime. + +**Files:** +- Modify: `lib/daemon/freshness.ts` +- Test: `lib/daemon/__tests__/freshness-remote-url.test.ts` (create) + +**Interfaces:** +- Produces: `getRemoteUrl(repoPath) => Promise` (was sync). All four call sites become `await getRemoteUrl(...)`. + +- [ ] **Step 1: Write the failing test** + +Create `lib/daemon/__tests__/freshness-remote-url.test.ts` ... since `getRemoteUrl` is module-private, test through a real repo via a tiny exported probe is not available; instead assert no `execSync`/`child_process` import remains in freshness.ts (the observable contract for R032) plus a behavioral cache test using a real temp repo through the public `getRepoContext` is heavy. Keep it to the source guard, which the Task 7 gate will also enforce, plus a targeted unit if a seam exists: + +```ts +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("freshness.ts no longer imports execSync/child_process", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + expect(src).not.toMatch(/from\s+["']child_process["']/); + expect(src).not.toMatch(/\bexecSync\b/); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/freshness-remote-url.test.ts` +Expected: FAIL ... `child_process`/`execSync` still present. + +- [ ] **Step 3: Convert `getRemoteUrl` and add the cache** + +In `lib/daemon/freshness.ts`: remove `import { execSync } from "child_process";`; add `import { runCapture } from "../subprocess.ts";`. Add a module-level cache near the other module state (below the `providers`/`userId` block ~line 91): + +```ts +const remoteUrlCache = new Map(); +``` + +Replace `getRemoteUrl` (lines 95-103): + +```ts +/** remote.origin.url, cached per repoPath for the process lifetime (remotes + * rarely change). Async so it never blocks the event loop. */ +async function getRemoteUrl(repoPath: string): Promise { + const cached = remoteUrlCache.get(repoPath); + if (cached !== undefined) return cached; + const r = await runCapture(["git", "config", "--get", "remote.origin.url"], { + cwd: repoPath, + timeoutMs: 5000, + stderr: "ignore", + }); + const url = r.exitCode === 0 ? (r.stdout.trim() || null) : null; + remoteUrlCache.set(repoPath, url); + return url; +} +``` + +Add `await` at the four call sites: `ensureProvider` (~135), `getRepoContext` (~273 and ~292), `reconcileFreshnessImpl` (~692). All three enclosing functions are already `async`. + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/freshness-remote-url.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc zero errors (the `await` additions type-check because callers are async). + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/freshness.ts lib/daemon/__tests__/freshness-remote-url.test.ts +git commit -m "freshness: async, cached getRemoteUrl via runCapture (R032)" +``` + +--- + +### Task 5: Async `killWorktreeProcesses` (S015, S016) + +`killWorktreeProcesses` runs three `execSync` calls (system-wide `lsof`, two `ps`) on the daemon thread per disposal and on the unattended reactor path; a stuck `lsof` wedges the loop. Port to `runCapture` (the identical async `lsof` already exists in `system-process-scanner.ts`). + +**Files:** +- Modify: `lib/daemon/worktree-process-kill.ts` (make `killWorktreeProcesses` async; three `runCapture` swaps) +- Modify: `lib/worktree/dispose.ts:238` (add `await`) +- Modify: `lib/daemon/worktree-reconciler.ts:447` (add `await`; update the "sync by design" comment) +- Test: `lib/daemon/__tests__/worktree-process-kill.test.ts` (add a source guard + keep `selectKillTargets` coverage) + +**Interfaces:** +- Produces: `killWorktreeProcesses(worktreePath) => Promise` (was sync). `WorktreeKillResult` unchanged. + +- [ ] **Step 1: Write the failing test** + +Add to `lib/daemon/__tests__/worktree-process-kill.test.ts`: + +```ts +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("worktree-process-kill.ts imports no sync exec", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "worktree-process-kill.ts"), "utf8"); + expect(src).not.toMatch(/from\s+["']child_process["']/); + expect(src).not.toMatch(/\bexecSync\b/); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/worktree-process-kill.test.ts` +Expected: FAIL ... `execSync` still imported. + +- [ ] **Step 3: Port the three calls to `runCapture`** + +In `lib/daemon/worktree-process-kill.ts`: remove `import { execSync } from "child_process";`; add `import { runCapture } from "../subprocess.ts";`. Make the function async and swap each call (argv form, no shell, no `2>/dev/null`): + +```ts +export async function killWorktreeProcesses(worktreePath: string): Promise { + const lsof = await runCapture(["lsof", "-d", "cwd", "-Fpn"], { timeoutMs: 10_000 }); + if (lsof.exitCode !== 0 && !lsof.stdout) { + log.warn({ exitCode: lsof.exitCode, worktreePath }, "lsof failed; skipping worktree process kill"); + return { terminated: [] }; + } + const lsofOut = lsof.stdout; + // ...existing parse of lsofOut into candidate pids (unchanged)... +``` + +Then the `ps -p -o pid=,ppid=,comm=,args=` call: + +```ts + const ps = await runCapture(["ps", "-p", pidList, "-o", "pid=,ppid=,comm=,args="], { timeoutMs: 5000 }); + if (ps.exitCode !== 0 && !ps.stdout) { + log.warn({ exitCode: ps.exitCode, worktreePath }, "ps failed; skipping worktree process kill"); + return { terminated: [] }; + } + for (const line of ps.stdout.split("\n")) { /* unchanged parse */ } +``` + +Then the `ps eww` label call: + +```ts + const eww = await runCapture(["ps", "eww", "-o", "pid=,command=", "-p", targets.map((t) => t.pid).join(",")], { timeoutMs: 5000 }); + if (eww.exitCode === 0 || eww.stdout) scripts = parsePackageScripts(eww.stdout); +``` + +(`pidList` is the same comma/space-joined pid string the old `ps` used; pass it as one argv element ... `ps -p` accepts a comma-separated list.) + +- [ ] **Step 4: Await at the two call sites** + +`lib/worktree/dispose.ts:238`: `const { terminated } = await killWorktreeProcesses(rec.path);` +`lib/daemon/worktree-reconciler.ts:447`: `const { terminated } = await killWorktreeProcesses(rec.path);` ... and replace the "ruled execSync exception (the process killer is sync by design)" comment with one that no longer claims sync (e.g. drop it; the try/catch keeps its "a failure here never blocks the return" rationale). + +- [ ] **Step 5: Run tests** + +Run: `bun test lib/daemon/__tests__/worktree-process-kill.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean (both call sites already sit in `async` functions). + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/worktree-process-kill.ts lib/worktree/dispose.ts lib/daemon/worktree-reconciler.ts lib/daemon/__tests__/worktree-process-kill.test.ts +git commit -m "worktree-process-kill: async lsof/ps via runCapture (S015, S016)" +``` + +--- + +### Task 6: Async index write in `resolveIndexPathForIdentity` (S098) + +`endpoint:claim`/`endpoint:lookup` can reach `resolveIndexPathForIdentity`, which on a legacy-key match calls `updateRepoIndex` → `observedMainPath` → a synchronous `git worktree list` on the daemon thread. Resolve the observed main path asynchronously. + +**Files:** +- Modify: `lib/repo-index.ts` (add `observedMainPathAsync`; use it on the async path) +- Test: `lib/__tests__/repo-index-async.test.ts` (create) + +**Interfaces:** +- Consumes: `listWorktreesAsync(path) => Promise`, `setIndexPath(key, mainPath)` (existing sync KV write, no subprocess). +- Produces: `observedMainPathAsync(repoRoot) => Promise` (degrades to `repoRoot`). + +- [ ] **Step 1: Write the failing test** + +Create `lib/__tests__/repo-index-async.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("resolveIndexPathForIdentity no longer reaches a sync git via observedMainPath", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "repo-index.ts"), "utf8"); + // observedMainPath (sync execSync) must not be called from the async resolver path. + expect(src).toMatch(/observedMainPathAsync/); +}); +``` + +(The Task 7 import-graph gate is the real enforcement that no sync git reaches the daemon graph; this test just anchors the async twin's existence.) + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/__tests__/repo-index-async.test.ts` +Expected: FAIL ... `observedMainPathAsync` not present. + +- [ ] **Step 3: Add `observedMainPathAsync` and use it** + +In `lib/repo-index.ts`, add `import { listWorktreesAsync } from "./worktree/git-async.ts";` (top imports). Add next to `observedMainPath`: + +```ts +/** Async twin of observedMainPath: the repo's MAIN worktree path as git + * reports it, degrading to repoRoot. Safe on the daemon thread. */ +async function observedMainPathAsync(repoRoot: string): Promise { + const wts = await listWorktreesAsync(repoRoot); + return wts?.[0]?.path ?? repoRoot; +} +``` + +In `resolveIndexPathForIdentity` (line ~277), replace `updateRepoIndex(serialized, path);` with: + +```ts +setIndexPath(serialized, await observedMainPathAsync(path)); +``` + +(`setIndexPath` is the raw KV write already exported in this file; it does the same persistence `updateRepoIndex` does, minus the sync git probe, which `observedMainPathAsync` now supplies.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/__tests__/repo-index-async.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean (`resolveIndexPathForIdentity` is already async). + +- [ ] **Step 5: Commit** + +```bash +git add lib/repo-index.ts lib/__tests__/repo-index-async.test.ts +git commit -m "repo-index: async observed-main-path on the endpoint:claim resolve path (S098)" +``` + +--- + +### Task 7: The import-graph gate (1.3) + +A test that walks the daemon's import graph from `lib/daemon.ts` and fails if any daemon-reachable module has a sync-exec call site (`execSync(`, `spawnSync(`, `Bun.spawnSync(`, `Bun.sleepSync(`). The rule has been re-broken twice; only a gate keeps it dead. Ships GREEN with an honest allowlist of every current out-of-Phase-1 offender, each commented with the finding/phase that removes it (decision: full-closure walk with honest allowlist). + +**Verified facts (from a throwaway walker run against the live tree):** +- Closure from `lib/daemon.ts` = 151 files. `lib/daemon.ts` and other entry files carry a `#!/usr/bin/env bun` shebang that makes `Bun.Transpiler.scanImports` throw ... strip a leading shebang before scanning. +- `.tsx` files (e.g. `lib/rt-render.tsx`, in the closure) need the `tsx` loader, not `ts`. +- Relative imports are spelled with explicit `.ts`/`.tsx` extensions throughout, so `resolve(dirname(file), importPath)` needs no extension guessing. +- Matching call sites (`\bexecSync\s*\(`) rather than bare `execSync` avoids false positives: `api-server.ts:105` mentions `Bun.sleepSync` in a comment with no paren, correctly NOT flagged. Raw-source matching yields the same 13 offenders as comment-stripped matching, so the test uses raw matching. +- After Tasks 3/4/5 remove cache-refresh.ts, freshness.ts, and worktree-process-kill.ts from the offender set, exactly these 10 remain (the allowlist): + +| Allowlist entry | Removing finding / phase | +|---|---| +| `lib/daemon/user-path.ts` | Phase 6 PATH rebuild (S013/S014/S062) | +| `lib/daemon/boot-reconcile.ts` | Phase 0.6 / S044 (`Bun.sleepSync`) | +| `lib/state/db.ts` | Phase 0.7 / S072-S073 busy-retry (`Bun.sleepSync`) | +| `lib/state/busy.ts` | Phase 0.7 / S072-S073 busy-retry (`Bun.sleepSync`) | +| `lib/git-worktrees.ts` | S055 (reached only via `handlers/status.ts`; that swap to `listWorktreesAsync` removes the edge) | +| `lib/daemon/handlers/status.ts` | S055 (the edge that pulls in git-worktrees.ts; own source has no sync-exec, listed per the brief) | +| `lib/repo-index.ts` | Phase 5.3 dedup (retains `execSync` at heal/derive paths; Task 6's endpoint path is covered by `repo-index-async.test.ts`) | +| `lib/repo.ts` | R050 / Phase 5.4 (reached via `handlers/system-processes.ts` → `repo-arg.ts`) | +| `lib/git.ts` | R050 / Phase 5.4 (via `repo.ts`) | +| `lib/herdr-launch.ts` | Phase 5 herdr (reached via `handlers/pane.ts`) | +| `lib/rt-render.tsx` | R050 / Phase 5.4 (daemon carries the TUI; `no-eager-tui` extension breaks the chain) | + +**Files:** +- Create: `lib/__tests__/no-daemon-sync-exec.test.ts` + +**Interfaces:** +- Self-contained test; consumes nothing from other tasks. Must run AFTER Tasks 3/4/5 land (else the three fixed files fail it). + +- [ ] **Step 1: Write the gate test (RED against current tree, before Tasks 3/4/5, or GREEN after)** + +Create `lib/__tests__/no-daemon-sync-exec.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { dirname, resolve } from "path"; + +// Files with sync-exec that Phase 1 does NOT remove. Each entry names the +// finding/phase that will delete it, so this list shrinks as later phases land. +// A regression that reintroduces sync-exec into any OTHER daemon-reachable +// module fails this gate (the rule has been re-broken twice). +const ALLOWLIST = new Set([ + "lib/daemon/user-path.ts", // Phase 6 PATH rebuild (S013/S014/S062) + "lib/daemon/boot-reconcile.ts", // Phase 0.6 / S044 (Bun.sleepSync) + "lib/state/db.ts", // Phase 0.7 / S072-S073 busy-retry + "lib/state/busy.ts", // Phase 0.7 / S072-S073 busy-retry + "lib/git-worktrees.ts", // S055: reached only via handlers/status.ts + "lib/daemon/handlers/status.ts", // S055: the edge into git-worktrees.ts + "lib/repo-index.ts", // Phase 5.3 dedup (heal/derive execSync) + "lib/repo.ts", // R050 / Phase 5.4 (via handlers/system-processes.ts) + "lib/git.ts", // R050 / Phase 5.4 (via repo.ts) + "lib/herdr-launch.ts", // Phase 5 herdr (via handlers/pane.ts) + "lib/rt-render.tsx", // R050 / Phase 5.4 (daemon carries the TUI) +]); + +const SYNC_EXEC = [ + /\bexecSync\s*\(/, + /\bspawnSync\s*\(/, + /\bBun\.spawnSync\s*\(/, + /\bBun\.sleepSync\s*\(/, +]; + +const REPO_ROOT = resolve(import.meta.dir, "..", ".."); +const stripShebang = (s: string) => s.replace(/^#!.*\n/, ""); +const tsT = new Bun.Transpiler({ loader: "ts" }); +const tsxT = new Bun.Transpiler({ loader: "tsx" }); +const loaderFor = (f: string) => (f.endsWith(".tsx") || f.endsWith(".jsx") ? tsxT : tsT); + +/** Files reachable from lib/daemon.ts via relative imports (the daemon graph). */ +function daemonClosure(): string[] { + const entry = resolve(REPO_ROOT, "lib/daemon.ts"); + const visited = new Set(); + const stack = [entry]; + while (stack.length) { + const file = stack.pop()!; + if (visited.has(file)) continue; + visited.add(file); + let src: string; + try { src = stripShebang(readFileSync(file, "utf8")); } catch { continue; } + let imports: { path: string }[]; + try { imports = loaderFor(file).scanImports(src); } catch { continue; } + for (const imp of imports) { + if (!imp.path.startsWith(".")) continue; // external package + stack.push(resolve(dirname(file), imp.path)); + } + } + return [...visited]; +} + +function hasSyncExec(source: string): boolean { + return SYNC_EXEC.some((re) => re.test(source)); +} + +test("no daemon-reachable module calls sync exec (outside the allowlist)", () => { + const offenders: string[] = []; + for (const file of daemonClosure()) { + const rel = file.replace(REPO_ROOT + "/", ""); + if (ALLOWLIST.has(rel)) continue; + let src: string; + try { src = readFileSync(file, "utf8"); } catch { continue; } + if (hasSyncExec(src)) offenders.push(rel); + } + expect(offenders).toEqual([]); +}); + +test("the checker flags a reintroduced sync-exec call (proves the gate bites)", () => { + // Permanent RED proof: the matcher must catch a fresh offense. + expect(hasSyncExec(`import { execSync } from "child_process";\nexecSync("true");`)).toBe(true); + expect(hasSyncExec(`await Bun.sleepSync(10);`)).toBe(true); + expect(hasSyncExec(`// a comment mentioning execSync without a call`)).toBe(false); +}); + +test("the daemon closure actually resolves (guards against a walker that finds nothing)", () => { + const closure = daemonClosure(); + expect(closure.length).toBeGreaterThan(50); // ~151 today; a collapse means the walk broke +}); +``` + +- [ ] **Step 2: Run the gate** + +Run: `bun test lib/__tests__/no-daemon-sync-exec.test.ts` +Expected: GREEN (Tasks 3/4/5 already removed cache-refresh, freshness, worktree-process-kill). If it lists an offender not in the allowlist, that file is either a Phase-1 target that regressed (fix it) or a daemon-reachable sync-exec outside Phase 1 (add it to the allowlist with a finding/phase comment). The closure-size guard and the checker RED-proof tests must also pass. + +- [ ] **Step 3: Demonstrate the gate fails on reintroduction (report evidence)** + +Temporarily add to `lib/daemon/pollers.ts` (a Phase-1-clean, non-allowlisted daemon file): `import { execSync } from "child_process";` and a call `execSync("true");`. Run: + +Run: `bun test lib/__tests__/no-daemon-sync-exec.test.ts` +Expected: FAIL ... offenders `["lib/daemon/pollers.ts"]`. Capture this RED output for the report, then revert both lines. + +- [ ] **Step 4: Commit** + +```bash +git add lib/__tests__/no-daemon-sync-exec.test.ts +git commit -m "gate: fail on sync-exec anywhere in the daemon import graph (1.3)" +``` + +--- + +### Task 8: Refresh cannot wedge (S007) + +One hung GitLab call latches `refreshInFlight` forever, freezing the coalesced refresh, and merged `runner.pending` grows unbounded. Add a whole-cycle deadline that clears the latch so the next tick can start, and cap the pending queue. + +**Files:** +- Modify: `lib/daemon/cache-refresh.ts` (extract a coalescer with a deadline) +- Modify: `lib/daemon/freshness.ts` (`applyInvalidationBatch` pending cap) +- Test: `lib/daemon/__tests__/cache-refresh-coalesce.test.ts` (create), `lib/daemon/__tests__/freshness-pending-cap.test.ts` (create) + +**Interfaces:** +- Produces: `makeCoalescer(run, deadlineMs, onTimeout) => () => Promise` exported from `cache-refresh.ts`. +- Produces: `PENDING_CAP` const in `freshness.ts`; `applyInvalidationBatch` dedupes-and-caps pending. + +- [ ] **Step 1: Write the failing coalescer test** + +Create `lib/daemon/__tests__/cache-refresh-coalesce.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { makeCoalescer } from "../cache-refresh.ts"; + +test("clears the in-flight latch after the deadline even if run never settles", async () => { + let starts = 0; + let timedOut = 0; + const coalesce = makeCoalescer( + () => { starts++; return new Promise(() => {}); }, // never resolves + 50, + () => { timedOut++; }, + ); + const t0 = Date.now(); + await coalesce(); // resolves at the deadline, not never + expect(Date.now() - t0).toBeLessThan(500); + expect(timedOut).toBe(1); + await coalesce(); // latch cleared, a new run can start + expect(starts).toBe(2); +}); + +test("coalesces concurrent callers onto one run", async () => { + let starts = 0; + let resolveRun!: () => void; + const coalesce = makeCoalescer( + () => { starts++; return new Promise((r) => { resolveRun = r; }); }, + 10_000, + () => {}, + ); + const a = coalesce(); + const b = coalesce(); + expect(starts).toBe(1); + resolveRun(); + await Promise.all([a, b]); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/cache-refresh-coalesce.test.ts` +Expected: FAIL ... `makeCoalescer` not exported. + +- [ ] **Step 3: Add `makeCoalescer` and use it** + +In `lib/daemon/cache-refresh.ts`, add above `createCacheRefresher`: + +```ts +/** Below the 5-min tick, above the slowest legitimate deep sync. */ +const REFRESH_CYCLE_DEADLINE_MS = 4 * 60 * 1000; + +/** + * Coalesce concurrent callers onto one in-flight run, but clear the latch after + * `deadlineMs` even if the run never settles, so a wedged cycle (a half-open + * GitLab socket that never rejects) cannot pin the latch forever. The wedged + * run's frame still leaks until the OS reaps the socket; this only frees the + * next tick. + */ +export function makeCoalescer( + run: () => Promise, + deadlineMs: number, + onTimeout: () => void, +): () => Promise { + let inFlight: Promise | null = null; + return () => { + if (inFlight) return inFlight; + const impl = run().catch(() => {}); // a rejected cycle still clears the latch + const guarded = Promise.race([ + impl, + new Promise((resolve) => setTimeout(() => { onTimeout(); resolve(); }, deadlineMs)), + ]).finally(() => { inFlight = null; }); + inFlight = guarded; + return guarded; + }; +} +``` + +Replace the `refreshInFlight` coalescer (lines 54-60) inside `createCacheRefresher`: + +```ts + const refreshCache = makeCoalescer( + refreshCacheImpl, + REFRESH_CYCLE_DEADLINE_MS, + () => log.warn("cache refresh timed out; cleared in-flight latch for next tick"), + ); +``` + +(Delete `let refreshInFlight` and the old `function refreshCache`. `refreshCacheImpl` stays as the async body; `return refreshCache;` at the end is unchanged.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/cache-refresh-coalesce.test.ts` +Expected: PASS. + +- [ ] **Step 5: Add the pending-cap test** + +Create `lib/daemon/__tests__/freshness-pending-cap.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { applyInvalidationBatch, PENDING_CAP } from "../freshness.ts"; + +test("merged pending is deduped by kind:ref and capped", async () => { + const runner: any = { processing: true, pending: [] }; + // Push more distinct keys than the cap; plus duplicates. + const keys = Array.from({ length: PENDING_CAP + 500 }, (_, i) => ({ kind: "mr", ref: String(i) })); + const dupes = [{ kind: "mr", ref: "0" }, { kind: "mr", ref: "0" }]; + await applyInvalidationBatch({} as any, {} as any, runner, [...keys, ...dupes], {}); + expect(runner.pending.length).toBeLessThanOrEqual(PENDING_CAP); + const ids = runner.pending.map((k: any) => `${k.kind}:${k.ref}`); + expect(new Set(ids).size).toBe(ids.length); // no duplicates +}); +``` + +- [ ] **Step 6: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/freshness-pending-cap.test.ts` +Expected: FAIL ... `PENDING_CAP` not exported (and current code pushes unbounded). + +- [ ] **Step 7: Cap the pending queue** + +In `lib/daemon/freshness.ts`, add near the module state (below `const watches`): + +```ts +/** Bound merged pending so a wedged processKeys cannot grow memory unbounded. */ +export const PENDING_CAP = 1000; +``` + +Replace the `if (runner.processing)` block in `applyInvalidationBatch` (lines 392-395): + +```ts + if (runner.processing) { + const seen = new Set(runner.pending.map((k) => `${k.kind}:${k.ref}`)); + for (const k of keys) { + if (runner.pending.length >= PENDING_CAP) break; + const id = `${k.kind}:${k.ref}`; + if (seen.has(id)) continue; + seen.add(id); + runner.pending.push(k); + } + return; + } +``` + +- [ ] **Step 8: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/freshness-pending-cap.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean. + +- [ ] **Step 9: Commit** + +```bash +git add lib/daemon/cache-refresh.ts lib/daemon/freshness.ts lib/daemon/__tests__/cache-refresh-coalesce.test.ts lib/daemon/__tests__/freshness-pending-cap.test.ts +git commit -m "refresh: whole-cycle deadline clears the coalesce latch; cap RepoWatch.pending (S007)" +``` + +--- + +### Task 9: Provider cache invalidation on token rotation (S048, S049) + +`providers` caches `GitLabProvider` instances by repoName with no token check, so a rotated `gitlabToken` never reaches watchers or forge handlers until a daemon restart. Key the cache on a token fingerprint and rebuild on mismatch; reset the `userIdResolved` latch. + +**Files:** +- Modify: `lib/daemon/freshness.ts` +- Test: `lib/daemon/__tests__/freshness-provider-rotation.test.ts` (create) + +**Interfaces:** +- Consumes: `loadSecrets()` (from `../linear.ts`, returns `{ gitlabToken }`), `makeProvider(host, token)`. +- Internal: `providers` map value becomes `{ provider: GitLabProvider; token: string }`. + +- [ ] **Step 1: Write the failing test** + +Create `lib/daemon/__tests__/freshness-provider-rotation.test.ts`. Since `ensureProvider` is module-private, test the observable contract: the source no longer returns a cached provider without comparing the current token. Assert the fingerprint mechanism exists and the `if (cached) return cached;` short-circuit is gone: + +```ts +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("ensureProvider compares the current token before reusing a cached provider", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + // The unconditional cache-hit return is the S049 bug; it must be gone. + expect(src).not.toMatch(/const cached = providers\.get\(repoName\);\s*\n\s*if \(cached\) return cached;/); + // A token fingerprint must be stored alongside the provider. + expect(src).toMatch(/providers\.set\(repoName,\s*\{\s*provider/); +}); +``` + +(A behavioral test would need a real GitLabProvider + secrets seam; the freshness module has no injection point for `loadSecrets`. This source-contract test plus `bunx tsc` is the achievable guard; note the limitation in the report.) + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/freshness-provider-rotation.test.ts` +Expected: FAIL ... the unconditional cache-hit return still present. + +- [ ] **Step 3: Key the provider cache on a token fingerprint** + +In `lib/daemon/freshness.ts`, change the `providers` map type (line 88): + +```ts +const providers = new Map(); +``` + +Rewrite `ensureProvider` so `loadSecrets()` runs before the cache decision and a token mismatch rebuilds (lines 125-155): + +```ts +async function ensureProvider(repoName: string, repoPath: string): Promise { + const secrets = await loadSecrets(); + if (!secrets.gitlabToken) { + log.info(`no gitlabToken; skipping ${repoName}`); + return null; + } + const cached = providers.get(repoName); + if (cached && cached.token === secrets.gitlabToken) return cached.provider; + if (cached) { + // Token rotated: drop the stale provider and any live watch built on it so + // the next reconcile rebuilds with the new token, and re-resolve userId. + stopWatch(repoName); + userIdResolved = false; + } + + const remoteUrl = await getRemoteUrl(repoPath); + if (!remoteUrl) { log.info(`no origin remote for ${repoName}; skipping`); return null; } + if (!isGitLabRemote(remoteUrl)) { log.info(`remote "${remoteUrl}" for ${repoName} is not GitLab; skipping events watch`); return null; } + const remote = parseRemoteUrl(remoteUrl); + if (!remote) { log.info(`could not parse remote "${remoteUrl}" for ${repoName}; skipping`); return null; } + + const provider = makeProvider(remote.host, secrets.gitlabToken); + providers.set(repoName, { provider, token: secrets.gitlabToken }); + return provider; +} +``` + +Update the other cache reads to the new value shape: +- `getRepoContext` line 251: `let provider = providers.get(repoName)?.provider ?? null;` +- `getRepoContext` line 282 (after building): `providers.set(repoName, { provider, token: secrets.gitlabToken });` +- `ensureUserId` line 160: `const anyProvider = providers.values().next().value?.provider as GitLabProvider | undefined;` +- `disposeFreshness` `providers.clear()` unchanged. + +(`stopWatch` is already defined in this module; calling it for a repo with no watch is a no-op ... verify its guard, add an early return if absent.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/freshness-provider-rotation.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean (all `providers.get/set/values` sites updated to the new shape). + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/freshness.ts lib/daemon/__tests__/freshness-provider-rotation.test.ts +git commit -m "freshness: rebuild the provider cache when gitlabToken rotates (S048, S049)" +``` + +--- + +### Task 10: A single failed lsof no longer resets runaway detection (S061) + +`gather()` returns `[]` on both a real empty scan and an `lsof`/`ps` failure; `scan()` then prunes every tracked pid, resetting `firstSeen` and the runaway sample window so a machine where `lsof` fails intermittently can never fire a runaway notification. Distinguish failure (null) from empty. + +**Files:** +- Modify: `lib/daemon/system-process-scanner.ts` (`getAllRepoPids`, `gather`, `scan`, `refresh`) +- Test: `lib/daemon/__tests__/system-process-scanner-resilience.test.ts` (create) + +**Interfaces:** +- Internal: `getAllRepoPids` and `gather` return `... | null` (null = the underlying `lsof`/`ps` scan failed). `scan`/`refresh` preserve `tracked`/`lastResult`/`lastScanAt` when `gather` returns null. +- Change `gather` from `private` to `protected` so a test subclass can drive its return value. + +- [ ] **Step 1: Write the failing test** + +Create `lib/daemon/__tests__/system-process-scanner-resilience.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { SystemProcessScanner } from "../system-process-scanner.ts"; + +class FakeScanner extends SystemProcessScanner { + next: any[] | null = []; + protected async gather(): Promise { return this.next; } +} + +test("a failed gather (null) keeps tracked windows and lastResult intact", async () => { + const s = new FakeScanner(); + // Seed a tracked process across enough scans that firstSeen is established. + s.next = [{ pid: 4242, cpuPercent: 95, command: "node", args: "x", ppid: 1, port: null, memoryMB: 10, etime: "01:00", repo: "r", linearTicket: null, packageScript: null }]; + const first = await s.scan(); + expect(first.find((p) => p.pid === 4242)).toBeTruthy(); + const firstSeen = s.getTracked().get(4242)?.firstSeen; + + // Now lsof fails: gather returns null. tracked and lastResult must survive. + s.next = null; + const during = await s.scan(); + expect(s.getTracked().get(4242)?.firstSeen).toBe(firstSeen); + expect(during.find((p) => p.pid === 4242)).toBeTruthy(); // lastResult carried forward +}); +``` + +(Field names on the fake process object follow `GatheredProcess`; adjust to the exact shape when implementing. `getTracked()` is an existing accessor.) + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/system-process-scanner-resilience.test.ts` +Expected: FAIL ... `gather` is `private` (can't override) and/or `scan` wipes `tracked` on the null tick. + +- [ ] **Step 3: Thread the failure signal** + +In `lib/daemon/system-process-scanner.ts`: + +`getAllRepoPids` ... return null on lsof failure instead of an empty Map: + +```ts +async function getAllRepoPids(trackedPaths: string[]): Promise | null> { + if (trackedPaths.length === 0) return new Map(); + const { stdout, exitCode } = await runCapture(["lsof", "-d", "cwd", "-Fpn"], { timeoutMs: 10_000 }); + if (exitCode !== 0 && !stdout) { + log.warn({ exitCode }, "lsof scan failed; preserving prior process state"); + return null; + } + return parseLsofCwdMap(stdout, trackedPaths); +} +``` + +`gather` ... change signature to `protected async gather(...): Promise` and propagate null (lines 380, 389): + +```ts + const cwdMap = await getAllRepoPids(trackedPaths); + if (cwdMap === null) return null; // lsof failed + if (cwdMap.size === 0) return []; // genuinely no tracked-cwd processes + // ... + const psRes = await runCapture([...], { timeoutMs: 5000 }); + if (psRes.exitCode !== 0 && !psRes.stdout) return null; // ps failed + if (!psRes.stdout) return []; +``` + +`scan` ... early-return on null before the try/finally so `tracked`/`lastResult`/`lastScanAt` are untouched: + +```ts + async scan(portEntries: PortEntry[] = []): Promise { + const gathered = await this.gather(portEntries); + if (gathered === null) return this.lastResult; // failed tick: preserve everything + try { + // ...existing loop over `gathered`, prune, this.lastResult = results... + return results; + } finally { + this.lastScanAt = Date.now(); + } + } +``` + +`refresh` ... same null guard: + +```ts + async refresh(portEntries: PortEntry[] = []): Promise { + const gathered = await this.gather(portEntries); + if (gathered === null) return this.lastResult; + try { + // ...existing map... + return results; + } finally { + this.lastScanAt = Date.now(); + } + } +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/system-process-scanner-resilience.test.ts && bun test lib/daemon/__tests__/system-process-scanner.test.ts && bunx tsc --noEmit` +Expected: PASS (new + existing scanner tests); tsc clean. + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/system-process-scanner.ts lib/daemon/__tests__/system-process-scanner-resilience.test.ts +git commit -m "system-process-scanner: a failed lsof preserves runaway windows (S061)" +``` + +--- + +### Task 11: Cheapen run-DB opens: mtime memoization + herdr backoff (S101, S038, S039) + +The agent-status poller and `runs:list` open every retained run's SQLite db every 10s (cost scales with retained runs, not live runs), and the poller spawns herdr every 10s forever on a herdr-less machine. Memoize finished-run summaries by state.db mtime, and back off the herdr probe after repeated failures. + +**Files:** +- Modify: `lib/runs/store.ts` (mtime cache in `listRuns`) +- Modify: `lib/daemon/agent-status-poller.ts` (consecutive-failure backoff) +- Test: `lib/runs/__tests__/store-memo.test.ts` (create), add to `lib/daemon/__tests__/agent-status-poller.test.ts` + +**Interfaces:** +- Internal: `listRuns` caches finished-run `RunSummary` keyed by `${repo}/${id}` → `{ mtimeMs, summary }`; running runs are never cached (their db still mutates and their liveness overlay changes). +- Internal: poller gains `FAILURE_THRESHOLD` / `BACKOFF_TICKS` backoff. + +- [ ] **Step 1: Write the failing memoization test** + +Create `lib/runs/__tests__/store-memo.test.ts`. Use the existing `seedRun`/`root` fixtures and a spy to prove a finished run's db is not reopened when its mtime is unchanged: + +```ts +import { test, expect, afterEach, spyOn } from "bun:test"; +import { Database } from "bun:sqlite"; +import { listRuns } from "../store.ts"; +import { root, seedRun } from "./fixtures.ts"; + +afterEach(() => { delete process.env.RT_RUNS_ROOT; }); + +test("a finished run's db is opened once, then served from the mtime cache", () => { + root(); // sets RT_RUNS_ROOT to a temp dir + seedRun("repoA", "run1", { status: "done" }); + const openSpy = spyOn(Database.prototype, "query"); + listRuns(); // first call opens + reads + const afterFirst = openSpy.mock.calls.length; + listRuns(); // second call: mtime unchanged -> no reopen + expect(openSpy.mock.calls.length).toBe(afterFirst); // no additional queries for run1 + openSpy.mockRestore(); +}); +``` + +(If `seedRun`'s signature differs, match it; the point is a finished run plus a proof the second `listRuns` does not re-query it.) + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/runs/__tests__/store-memo.test.ts` +Expected: FAIL ... every `listRuns` reopens and re-queries every run. + +- [ ] **Step 3: Add the mtime cache** + +In `lib/runs/store.ts`, add `import { statSync } from "fs";` (if absent) and a module cache: + +```ts +// Finished runs never change; skip the open+PRAGMA+4-reads when the db mtime +// is unchanged. Running runs are never cached: their db still mutates and their +// liveness overlay is recomputed per call. +const summaryCache = new Map(); +``` + +Rewrite the per-run body of `listRuns` (lines 122-130): + +```ts + for (const id of dirs(join(runsRoot(), r))) { + const dbPath = join(runsRoot(), r, id, "state.db"); + let mtimeMs: number; + try { mtimeMs = statSync(dbPath).mtimeMs; } catch { continue; } + const key = `${r}/${id}`; + const hit = summaryCache.get(key); + if (hit && hit.mtimeMs === mtimeMs) { out.push(hit.summary); continue; } + + const opened = openRun(r, id); + if (!opened) continue; + try { + const row = runRow(opened.db); + if (row) { + const summary = withAttention(opened.db, row, liveness); + out.push(summary); + if (summary.status !== "running") summaryCache.set(key, { mtimeMs, summary }); + } + } finally { + opened.db.close(); + } + } +``` + +(Confirm the field is `summary.status`; the poller reads `run.status`. If `RunSummary` names it `state`, use that.) + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/runs/__tests__/store-memo.test.ts && bun test lib/runs/__tests__/store.test.ts` +Expected: PASS (new + existing store tests, including corrupt/missing-db cases). + +- [ ] **Step 5: Write the failing herdr-backoff test** + +Add to `lib/daemon/__tests__/agent-status-poller.test.ts` (using the existing `probe`/`list` DI seams and manual `tick()`): + +```ts +test("backs off the herdr probe after repeated failures", async () => { + let probeCalls = 0; + const handle = startAgentStatusPoller({ + intervalMs: 3_600_000, // real timer never fires + probe: async () => { probeCalls++; return null; }, // herdr absent + list: () => [], + }); + for (let i = 0; i < 20; i++) await handle.tick(); + handle.stop(); + // Without backoff this would be 20; with backoff (threshold 3, 1-in-6) far fewer. + expect(probeCalls).toBeLessThan(10); +}); +``` + +- [ ] **Step 6: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/agent-status-poller.test.ts` +Expected: FAIL ... `probeCalls` is 20 (probes every tick). + +- [ ] **Step 7: Add the backoff** + +In `lib/daemon/agent-status-poller.ts`, add constants and counter state in the closure around `tick`: + +```ts +const FAILURE_THRESHOLD = 3; // consecutive null probes before backing off +const BACKOFF_TICKS = 6; // then probe once every 6 ticks (~60s at 10s cadence) +``` + +```ts + let consecutiveFailures = 0; + let ticksSkipped = 0; + async function tick() { + if (consecutiveFailures >= FAILURE_THRESHOLD) { + if (++ticksSkipped < BACKOFF_TICKS) return; + ticksSkipped = 0; + } + const entries = await probe(); + if (entries === null) { consecutiveFailures++; return; } + consecutiveFailures = 0; + // ...existing tick body... + } +``` + +- [ ] **Step 8: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/agent-status-poller.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean. + +- [ ] **Step 9: Commit** + +```bash +git add lib/runs/store.ts lib/daemon/agent-status-poller.ts lib/runs/__tests__/store-memo.test.ts lib/daemon/__tests__/agent-status-poller.test.ts +git commit -m "runs: mtime-memoize finished-run summaries; back off herdr probe (S101, S038, S039)" +``` + +--- + +### Task 12: Gate background scans on demand (S058, S093) + +The 10s system-process scan and 30s port scan run at full cadence with zero consumers (idle tray, battery). Gate them on a recent consumer read (tray/CLI/REST hit of `ports`/`system-processes`/`tray:status`). Wired via a new tracker read from `pollers.ts` and set by wrapping the demand commands in `command-router.ts` (no forbidden-file edit; verified reachable). + +**Files:** +- Create: `lib/daemon/demand-tracker.ts` +- Modify: `lib/daemon/command-router.ts` (wrap the demand-command entries) +- Modify: `lib/daemon/pollers.ts` (skip scans when no recent demand) +- Test: `lib/daemon/__tests__/demand-tracker.test.ts` (create) + +**Interfaces:** +- Produces: `recordDemand()`, `demandedWithin(ms) => boolean`, `wrapWithDemand(handlers, cmds) => handlers` in `demand-tracker.ts`. + +- [ ] **Step 1: Write the failing test** + +Create `lib/daemon/__tests__/demand-tracker.test.ts`: + +```ts +import { test, expect } from "bun:test"; +import { recordDemand, demandedWithin, wrapWithDemand } from "../demand-tracker.ts"; + +test("demandedWithin reflects a recent recordDemand", () => { + recordDemand(); + expect(demandedWithin(60_000)).toBe(true); + expect(demandedWithin(0)).toBe(false); // window of 0ms is never "recent" +}); + +test("wrapWithDemand records demand and delegates to the inner handler", async () => { + let called = false; + const handlers = { "system-processes": async () => { called = true; return { ok: true }; }, other: async () => ({ ok: true }) }; + wrapWithDemand(handlers, ["system-processes"]); + const before = demandedWithin(50); + await handlers["system-processes"](undefined as any); + expect(called).toBe(true); + expect(demandedWithin(1000)).toBe(true); + void before; +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `bun test lib/daemon/__tests__/demand-tracker.test.ts` +Expected: FAIL ... module does not exist. + +- [ ] **Step 3: Create the tracker** + +Create `lib/daemon/demand-tracker.ts`: + +```ts +/** + * "A consumer is watching" signal for the background scans. The tray/CLI/console + * calling ports/system-processes/tray:status stamps demand here (via the + * command-router wrapper); pollers skip the 10s/30s scans when nothing has asked + * recently, so an idle machine stops paying the lsof/git tax (S058, S093). + */ +let lastDemandAt = 0; + +export function recordDemand(): void { + lastDemandAt = Date.now(); +} + +/** True when a consumer read a scan-backed command within `ms`. */ +export function demandedWithin(ms: number): boolean { + return lastDemandAt !== 0 && Date.now() - lastDemandAt < ms; +} + +/** Wrap the named handler entries so each call stamps demand, then delegates. */ +export function wrapWithDemand>(handlers: T, cmds: string[]): T { + for (const cmd of cmds) { + const inner = handlers[cmd]; + if (typeof inner !== "function") continue; + (handlers as any)[cmd] = (...args: any[]) => { recordDemand(); return inner(...args); }; + } + return handlers; +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `bun test lib/daemon/__tests__/demand-tracker.test.ts` +Expected: PASS. + +- [ ] **Step 5: Wrap the demand commands in the router** + +In `lib/daemon/command-router.ts`: `import { wrapWithDemand } from "./demand-tracker.ts";`. Change the trailing `return { ...handlers... }` of `buildRoutedHandlers` to assign then wrap: + +```ts + const handlers: TypedHandlers & HandlerMap = { + ...createCacheHandlers(ctx), + // ...unchanged spreads... + "freshness:reconcile": async () => { + await reconcileFreshness({ ctx, broadcast }); + return { ok: true, data: getFreshnessSnapshot() }; + }, + }; + // A tray/CLI/console read of any scan-backed command means "someone is + // watching", which un-gates the background scans (see pollers.ts, S058/S093). + return wrapWithDemand(handlers, ["ports", "system-processes", "tray:status"]); +``` + +- [ ] **Step 6: Gate the scans in pollers** + +In `lib/daemon/pollers.ts`: `import { demandedWithin } from "./demand-tracker.ts";` and add: + +```ts +/** Consider a consumer "present" for 5 min after its last scan-backed read. */ +const DEMAND_WINDOW_MS = 5 * 60 * 1000; +``` + +Add the gate as the first line inside both scan bodies (after the in-flight guard): + +```ts + async function refreshPortCache(): Promise { + if (portScanInFlight) return; + if (!demandedWithin(DEMAND_WINDOW_MS)) return; // no consumer asked recently + portScanInFlight = true; + // ...unchanged... + } + + async function refreshSystemProcesses(): Promise { + if (processScanInFlight) return; + if (!demandedWithin(DEMAND_WINDOW_MS)) return; + processScanInFlight = true; + // ...unchanged... + } +``` + +(The tray's on-demand `system-processes` handler already calls `scanner.refresh()` when its cache is stale, so a freshly-connecting consumer gets immediate data and its read stamps demand, resuming the poller on the next tick. The hooks-scan and cache-refresh intervals are unaffected.) + +- [ ] **Step 7: Run tests + typecheck** + +Run: `bun test lib/daemon/__tests__/demand-tracker.test.ts && bunx tsc --noEmit` +Expected: PASS; tsc clean. + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon/demand-tracker.ts lib/daemon/command-router.ts lib/daemon/pollers.ts lib/daemon/__tests__/demand-tracker.test.ts +git commit -m "pollers: gate the 10s/30s scans on recent consumer demand (S058, S093)" +``` + +--- + +### Final verification (run before the whole-branch review) + +- [ ] `bun run build` inside `packages/rt-client` (rt-client's `runCapture` mirror changed in Task 1). +- [ ] `bun test lib commands packages scripts` ... all green. +- [ ] `bunx tsc --noEmit` ... zero errors. +- [ ] `bun test lib/__tests__/no-daemon-sync-exec.test.ts` ... green; capture the Task 7 Step 3 RED demonstration for the report. +- [ ] Confirm no sibling-owned file in the write fence was modified (`git diff --name-only main...` reviewed against the fence). diff --git a/docs/superpowers/plans/2026-08-28-p3-trust-boundary.md b/docs/superpowers/plans/2026-08-28-p3-trust-boundary.md new file mode 100644 index 000000000..5c8ce0d90 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-p3-trust-boundary.md @@ -0,0 +1,1745 @@ +# Phase 3 — The 127.0.0.1 Trust Boundary — Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Close the daemon's `:9401` trust boundary — WS/CORS origin auth, token-gate the ungated mutating/destructive routes, one shared api-token loader, input hygiene (body-size cap, path-param decode safety, query coercion) — plus the two standalone items S043 (EADDRINUSE diagnostics + typed failure) and S083 (`pathParam()` 400s). + +**Architecture:** All logic lands as small, pure, dependency-injected functions in `lib/daemon/api-auth.ts` and `lib/daemon/api-server.ts` (mirroring the existing `bindApiServerWithRetry`/`needsToken`/`tokenOk` style already in this codebase) so every gate is unit-testable without spinning a real socket. The one exception is the request-body-size cap, which is a Bun.serve runtime option — that gets one small live `Bun.serve({port:0})` integration test. Two findings (S010, S050) have their fix location in sibling-owned files (`lib/daemon/handlers/worktree.ts`, `lib/daemon/freshness.ts`); this plan creates standalone, tested validator/utility modules for them and documents the one-line call-site wiring for the owning job, per the job brief's explicit instruction for S010 (mirrored here for S050 since its fix location is equally out of this job's write fence). + +**Tech Stack:** Bun, TypeScript, `bun:test`, existing `@mattstack/rt-client` settings resolver. + +**Spec:** `/Users/matt/Documents/GitHub/repo-tools/.claude/worktrees/daemon-stability-audit/docs/daemon-stability-audit-2026-08.md` — "Roadmap > Phase 3" (lines 72-79) plus Appendix A findings S005, S006, S010, S040, S041, S042, S043, S050, S054, S083, S084, S085, S092 (read-only input; never modify). + +## Global Constraints + +- Write fence: only `lib/daemon/api-server.ts`, `lib/daemon/api-auth.ts`, `lib/daemon/socket-server.ts`, `lib/daemon/handlers/secrets.ts`, new files under `lib/daemon/` (+ their tests under `lib/daemon/__tests__/`), `e2e/tests/`, `packages/rt-client/src/` (only if the client must send a token), the settings registry file, and `docs/`. Never touch `lib/daemon/handlers/worktree.ts`, `lib/daemon/freshness.ts`, or `lib/daemon.ts` — those are sibling-owned. +- Never start a daemon or run `rt`/`dist/rt` against the real machine; any such invocation in a test must run under `env -i HOME=`. +- `bun test lib commands packages scripts` must stay green; `bunx tsc --noEmit` must report zero errors. +- If `packages/rt-client` is touched, run `bun run build` inside it before the final review (dist-freshness test enforces this). +- Non-browser clients (no `Origin` header: the Swift tray, rt-client from Bun processes, mr-board, gitq, the VS Code extension) must keep working completely unchanged. A browser `Origin` must present the `X-RT-Token` (or, for `/ws`, a `?token=` query param) OR match the new `rt.trustedBrowserOrigins` settings allowlist. +- Never use em dashes or en dashes in code comments, commit messages, or docs (project convention — use parens or "..."). +- Follow TDD: write the failing test first, watch it fail, then implement. + +--- + +## File Map + +| File | Change | +|---|---| +| `lib/daemon/request-limits.ts` | **new** — shared `MAX_REQUEST_BODY_SIZE` constant (S092) | +| `lib/daemon/api-auth.ts` | token singleton + warn (S054); settings-backed origin allowlist + `isBrowserRequestTrusted` (S005/S006); `needsToken` invert-default (S040/S041/S084) | +| `lib/daemon/api-server.ts` | CORS default-deny + `/ws` gate (S005/S006); `broadcastToClients` backpressure/dead-client handling (S042); `pathParam()` + 400 (S083); `coerceQueryParams` (S085); `ApiPortInUseError` + lsof probe (S043) | +| `lib/daemon/socket-server.ts` | wire `MAX_REQUEST_BODY_SIZE` (S092) | +| `lib/daemon/handlers/secrets.ts` | use the shared token singleton (S054) | +| `packages/rt-client/src/settings/registry-defs.ts` | new `rt.trustedBrowserOrigins` key | +| `lib/daemon/git-ref-validation.ts` | **new** — S010 validator (standalone; sibling wires into `worktree.ts`) | +| `lib/daemon/redact-credentials.ts` | **new** — S050 utility (standalone; sibling wires into `freshness.ts`) | +| `docs/daemon-api-auth.md` | **new** — short note on the auth model + the two wiring pointers | + +All new tests live under `lib/daemon/__tests__/`. + +--- + +## Task 1: Shared request body-size cap (S092) + +**Files:** +- Create: `lib/daemon/request-limits.ts` +- Modify: `lib/daemon/api-server.ts` (add `maxRequestBodySize` to the `Bun.serve` options) +- Modify: `lib/daemon/socket-server.ts` (add `maxRequestBodySize` to the `Bun.serve` options) +- Test: `lib/daemon/__tests__/request-body-size.test.ts` + +**Interfaces:** +- Produces: `export const MAX_REQUEST_BODY_SIZE: number` (1 MiB) from `lib/daemon/request-limits.ts`, imported by both servers. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/request-body-size.test.ts +/** + * Bun enforces `maxRequestBodySize` itself (413 before the handler runs) -- + * this is a live-server test, not a pure-function one, because there is no + * pure function to unit test: the cap is a Bun.serve runtime option. + */ +import { describe, test, expect, afterEach } from "bun:test"; +import type { Server } from "bun"; +import { MAX_REQUEST_BODY_SIZE } from "../request-limits.ts"; + +let server: Server | undefined; + +afterEach(() => { + server?.stop(true); + server = undefined; +}); + +describe("MAX_REQUEST_BODY_SIZE", () => { + test("is set to 1 MiB", () => { + expect(MAX_REQUEST_BODY_SIZE).toBe(1024 * 1024); + }); + + test("Bun rejects a body over the cap with a 4xx before the handler runs", async () => { + let handlerRan = false; + server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + maxRequestBodySize: 10, // tiny cap for a fast, deterministic test + async fetch(req) { + handlerRan = true; + await req.text(); + return new Response("ok"); + }, + }); + const res = await fetch(`http://127.0.0.1:${server.port}/`, { + method: "POST", + body: "x".repeat(1000), + }); + expect(res.status).toBeGreaterThanOrEqual(400); + expect(res.status).toBeLessThan(500); + expect(handlerRan).toBe(false); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/request-body-size.test.ts` +Expected: FAIL — `request-limits.ts` does not exist yet (module not found). + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/request-limits.ts +/** + * Shared cap on request body size for both daemon servers (api-server.ts's + * :9401 HTTP/WS surface and socket-server.ts's unix-socket IPC channel). + * Neither transport authenticates reads, so an unbounded body (Bun's + * default is 128 MB) lets any same-user process or a cross-origin browser + * request stall the daemon's single event loop parsing a giant payload. + * Real payloads on both transports are kilobytes; 1 MiB costs nothing and + * turns an oversized request into an immediate 413 instead. + */ +export const MAX_REQUEST_BODY_SIZE = 1024 * 1024; +``` + +Then in `lib/daemon/api-server.ts`, add the import and option: + +```ts +import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; +``` + +...and inside the `Bun.serve({ ... })` options object passed to `bindApiServerWithRetry`, add: + +```ts + maxRequestBodySize: MAX_REQUEST_BODY_SIZE, +``` + +(alongside the existing `port`, `hostname`, `idleTimeout` keys). + +In `lib/daemon/socket-server.ts`, add the same import and, inside the `Bun.serve({ ... })` call, add: + +```ts + maxRequestBodySize: MAX_REQUEST_BODY_SIZE, +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/request-body-size.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice for a quick regression check** + +Run: `bun test lib/daemon` +Expected: PASS (no existing test asserts a specific absence of `maxRequestBodySize`) + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/request-limits.ts lib/daemon/api-server.ts lib/daemon/socket-server.ts lib/daemon/__tests__/request-body-size.test.ts +git commit -m "daemon: cap request body size at 1 MiB on both servers (S092)" +``` + +--- + +## Task 2: One shared api-token loader, with a warn on persist failure (S054) + +**Files:** +- Modify: `lib/daemon/api-auth.ts` +- Modify: `lib/daemon/api-server.ts` (use the new getter instead of the raw loader) +- Modify: `lib/daemon/handlers/secrets.ts` (use the new getter as the default override) +- Test: `lib/daemon/__tests__/api-auth.test.ts` (extend) + +**Interfaces:** +- Produces: `export function getApiToken(tokenPath?: string): string` and `export function reloadApiToken(tokenPath?: string): string` from `lib/daemon/api-auth.ts`. `loadOrCreateApiToken` keeps its existing signature and export (still the underlying file I/O primitive; `getApiToken`/`reloadApiToken` wrap it with an in-memory cache). +- Consumes: `lazyChildLogger` from `../daemon-logger.ts` (already exported; see `getDaemonLogger`/`lazyChildLogger` in `lib/daemon-logger.ts`). + +- [ ] **Step 1: Write the failing test** + +Append to `lib/daemon/__tests__/api-auth.test.ts`: + +```ts +import { getApiToken, reloadApiToken, loadOrCreateApiToken } from "../api-auth.ts"; +import { mkdtempSync, rmSync, writeFileSync, chmodSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +describe("getApiToken / reloadApiToken singleton", () => { + test("getApiToken caches: a second call does not re-read the file even if it changes underneath", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + const first = reloadApiToken(tokenPath); // seed the cache with a known path + writeFileSync(tokenPath, "a-different-token", { mode: 0o600 }); + const second = getApiToken(tokenPath); // ignores the new file content -- cached + expect(second).toBe(first); + expect(second).not.toBe("a-different-token"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("reloadApiToken re-reads and updates the cache", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + reloadApiToken(tokenPath); + writeFileSync(tokenPath, "rotated-token", { mode: 0o600 }); + const reloaded = reloadApiToken(tokenPath); + expect(reloaded).toBe("rotated-token"); + expect(getApiToken(tokenPath)).toBe("rotated-token"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("loadOrCreateApiToken still works standalone (unchanged primitive)", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + const a = loadOrCreateApiToken(tokenPath); + const b = loadOrCreateApiToken(tokenPath); + expect(a).toBe(b); + expect(a.length).toBeGreaterThan(0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-auth.test.ts` +Expected: FAIL — `getApiToken`/`reloadApiToken` are not exported yet. + +- [ ] **Step 3: Write minimal implementation** + +In `lib/daemon/api-auth.ts`, add near the top (after the existing imports) and after `loadOrCreateApiToken`: + +```ts +import { lazyChildLogger } from "../daemon-logger.ts"; + +const log = lazyChildLogger("api-auth"); +``` + +Change the existing `loadOrCreateApiToken`'s silent write-failure catch to log a warning (this is the only edit to that function's body): + +```ts +export function loadOrCreateApiToken(tokenPath: string = API_TOKEN_PATH): string { + try { + if (existsSync(tokenPath)) { + const existing = readFileSync(tokenPath, "utf8").trim(); + if (existing) return existing; + } + } catch { /* fall through to regenerate */ } + const token = randomUUID(); + try { + mkdirSync(RT_DIR, { recursive: true }); + writeFileSync(tokenPath, token, { mode: 0o600 }); + } catch (err) { + log.warn({ err, tokenPath }, "could not persist api-token; enforced in-memory only this run, so a client reading the file will disagree until the daemon restarts"); + } + return token; +} + +/** + * `getApiToken`/`reloadApiToken` share ONE in-memory value between + * api-server.ts and the secrets handler (S054): before this, api-server + * captured a token once at boot while the secrets handler called + * `loadOrCreateApiToken()` fresh on every request, so an external rotation + * (deleting api-token to force a new one) left the two permanently + * disagreeing about which token is current -- and if the token dir was + * unwritable, the secrets handler regenerated a brand new random token on + * every single call, never matching anything a client could read from disk. + * Both consumers now read the SAME cached value; a rotation only takes + * effect for both after `reloadApiToken()` runs or the daemon restarts, + * either of which was already the closest thing to a happy path before. + */ +let cachedApiToken: string | null = null; + +export function getApiToken(tokenPath: string = API_TOKEN_PATH): string { + if (cachedApiToken === null) cachedApiToken = loadOrCreateApiToken(tokenPath); + return cachedApiToken; +} + +export function reloadApiToken(tokenPath: string = API_TOKEN_PATH): string { + cachedApiToken = loadOrCreateApiToken(tokenPath); + return cachedApiToken; +} +``` + +In `lib/daemon/api-server.ts`, change the import and the one call site: + +```ts +import { needsToken, tokenOk, getApiToken } from "./api-auth.ts"; +``` + +```ts + const apiToken = getApiToken(); +``` + +(replaces `const apiToken = loadOrCreateApiToken();`) + +In `lib/daemon/handlers/secrets.ts`, change the import and the one default-override: + +```ts +import { getApiToken, tokenOk } from "../api-auth.ts"; +``` + +```ts + /** Defaults to `getApiToken` (the real ~/.mattstack/rt/api-token, shared with api-auth.ts and api-server.ts). */ + apiToken?: () => string; +``` + +(only the doc comment line changes; the field name/type is unchanged) + +```ts + const apiToken = overrides.apiToken ?? (() => getApiToken()); +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-auth.test.ts lib/daemon/__tests__/secrets-handler.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice** + +Run: `bun test lib/daemon` +Expected: PASS + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/api-auth.ts lib/daemon/api-server.ts lib/daemon/handlers/secrets.ts lib/daemon/__tests__/api-auth.test.ts +git commit -m "daemon: one shared api-token cache for api-server and secrets handler (S054)" +``` + +--- + +## Task 3: Origin allowlist settings key + trust helper + needsToken invert-default (S005, S006, S040, S041, S084) + +**Files:** +- Modify: `packages/rt-client/src/settings/registry-defs.ts` +- Modify: `lib/daemon/api-auth.ts` +- Test: `lib/daemon/__tests__/api-auth.test.ts` (extend) + +**Interfaces:** +- Produces: + - `export function getTrustedBrowserOrigins(): readonly string[]` (reads the `rt.trustedBrowserOrigins` setting, `[]` on any error or when unset) + - `export function isOriginAllowed(origin: string, allowedOrigins: readonly string[]): boolean` + - `export function isBrowserRequestTrusted(origin: string | null, token: string | null, apiToken: string, allowedOrigins: readonly string[]): boolean` (the shared decision Task 4 wires into both the CORS header and the `/ws` gate: no `Origin` header at all -> trusted (non-browser); otherwise a valid token OR an allowlisted origin) + - `needsToken(method, pathname)` changes shape: default-gated for every method except `GET`/`HEAD`/`OPTIONS`, plus two explicit GET exceptions (`/api/secrets`, `/api/notifications`) that are gated despite being reads. +- Consumes: `getSetting` from `../settings/resolve.ts` (already re-exported from `@mattstack/rt-client`; see `lib/chat-viewer-url.ts` for the exact import/usage pattern), `tokenOk` (already in this file). + +- [ ] **Step 1: Add the registry key** + +In `packages/rt-client/src/settings/registry-defs.ts`, add a new entry after the `rt.hooks` block (around line 207, right before the `// --- mattstack (installer-lane) ---` comment): + +```ts + { + key: "rt.trustedBrowserOrigins", + type: "array", + scopes: ["user", "machine"], + default: [], + merge: "replace", + description: "Browser Origins (scheme://host:port, exact string match) trusted to read the :9401 daemon API and subscribe to /ws without presenting the local api-token -- e.g. a locally-hosted console or chat-viewer dev server. Empty by default: every current mattstack consumer (the CLI, the Swift tray, rt-client from Bun/Node processes, the VS Code extension) is a non-browser client (sends no Origin header at all) and is unaffected either way.", + }, +``` + +- [ ] **Step 2: Build rt-client so the new key is live for tests that resolve it** + +Run: `cd packages/rt-client && bun run build && cd -` +Expected: build succeeds; `dist/` picks up the new registry row. + +- [ ] **Step 3: Write the failing tests** + +Append to `lib/daemon/__tests__/api-auth.test.ts`: + +```ts +import { isOriginAllowed, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "../api-auth.ts"; + +describe("isOriginAllowed", () => { + test("exact match", () => { + expect(isOriginAllowed("http://localhost:5544", ["http://localhost:5544"])).toBe(true); + }); + test("no match", () => { + expect(isOriginAllowed("http://evil.example", ["http://localhost:5544"])).toBe(false); + }); + test("empty allowlist matches nothing", () => { + expect(isOriginAllowed("http://localhost:5544", [])).toBe(false); + }); +}); + +describe("isBrowserRequestTrusted", () => { + const apiToken = "the-real-token"; + + test("no Origin header at all -- a non-browser client -- is trusted regardless of token or allowlist", () => { + expect(isBrowserRequestTrusted(null, null, apiToken, [])).toBe(true); + }); + + test("a browser Origin with the correct token is trusted even off the allowlist", () => { + expect(isBrowserRequestTrusted("http://evil.example", apiToken, apiToken, [])).toBe(true); + }); + + test("a browser Origin with a wrong token and not on the allowlist is rejected", () => { + expect(isBrowserRequestTrusted("http://evil.example", "wrong", apiToken, [])).toBe(false); + }); + + test("a browser Origin with no token but on the allowlist is trusted", () => { + expect(isBrowserRequestTrusted("http://localhost:5544", null, apiToken, ["http://localhost:5544"])).toBe(true); + }); + + test("a browser Origin with no token and not on the allowlist is rejected", () => { + expect(isBrowserRequestTrusted("http://localhost:5544", null, apiToken, [])).toBe(false); + }); +}); + +describe("getTrustedBrowserOrigins", () => { + test("returns an array (empty by default in an isolated test HOME)", () => { + const origins = getTrustedBrowserOrigins(); + expect(Array.isArray(origins)).toBe(true); + }); +}); +``` + +Also extend the existing `needsToken` describe block in the same file with the new cases (add these `test`s inside the existing `describe("needsToken", ...)`): + +```ts + test("refresh requires a token now (S040)", () => { + expect(needsToken("POST", "/api/refresh")).toBe(true); + }); + + test("hooks repair requires a token now (S040/S084)", () => { + expect(needsToken("POST", "/api/hooks/my-repo/repair")).toBe(true); + }); + + test("notifications GET (destructive drain) requires a token now (S041)", () => { + expect(needsToken("GET", "/api/notifications")).toBe(true); + }); + + test("every non-GET/HEAD/OPTIONS method defaults to requiring a token", () => { + expect(needsToken("POST", "/api/some-future-mutating-route")).toBe(true); + expect(needsToken("PUT", "/api/anything")).toBe(true); + expect(needsToken("DELETE", "/api/anything")).toBe(true); + }); + + test("plain reads still do not require a token", () => { + expect(needsToken("GET", "/api/repos")).toBe(false); + expect(needsToken("GET", "/api/cache")).toBe(false); + expect(needsToken("HEAD", "/api/repos")).toBe(false); + }); +``` + +- [ ] **Step 4: Run tests to verify they fail** + +Run: `bun test lib/daemon/__tests__/api-auth.test.ts` +Expected: FAIL — `isOriginAllowed`/`isBrowserRequestTrusted`/`getTrustedBrowserOrigins` not exported; the two new `needsToken` cases for `/api/refresh` and `/api/hooks/.../repair` and `/api/notifications` fail against the current allowlist-based implementation. + +- [ ] **Step 5: Write minimal implementation** + +In `lib/daemon/api-auth.ts`, add the import (alongside the existing ones) and the new functions: + +```ts +import { getSetting } from "../settings/resolve.ts"; +``` + +```ts +/** + * `rt.trustedBrowserOrigins` -- see registry-defs.ts. Read fresh on every + * call (the settings resolver is deliberately unmemoized), wrapped in a + * try/catch since a request-path settings read must never 500 the daemon + * over a malformed store file. + */ +export function getTrustedBrowserOrigins(): readonly string[] { + try { + const resolved = getSetting("rt.trustedBrowserOrigins"); + return Array.isArray(resolved.value) ? resolved.value : []; + } catch { + return []; + } +} + +export function isOriginAllowed(origin: string, allowedOrigins: readonly string[]): boolean { + return allowedOrigins.includes(origin); +} + +/** + * The 127.0.0.1 trust boundary (S005/S006): the daemon binds loopback-only, + * but any web page the user visits also runs on 127.0.0.1 and can send a + * request. A request with NO Origin header at all is not a browser fetch -- + * it is the CLI, the Swift tray, rt-client from a Bun/Node process, or the + * VS Code extension, none of which send one -- so it is trusted unchanged. + * A request that DOES carry an Origin header is trusted only if it presents + * the local api-token or its Origin is on the explicit allowlist. + */ +export function isBrowserRequestTrusted( + origin: string | null, + token: string | null, + apiToken: string, + allowedOrigins: readonly string[], +): boolean { + if (!origin) return true; + if (tokenOk(token, apiToken)) return true; + return isOriginAllowed(origin, allowedOrigins); +} +``` + +Replace the whole `needsToken` function with: + +```ts +/** + * True when a request mutates state, or (secrets/notifications) returns or + * drains something a GET should not silently consume, and must present the + * local token. Default-gated for every method except GET/HEAD/OPTIONS (S040: + * an allowlist-by-path guaranteed the next mutating route would ship + * unguarded) plus two explicit GET exceptions whose verb lies about being a + * read. + */ +export function needsToken(method: string, pathname: string): boolean { + if (method === "GET" || method === "HEAD" || method === "OPTIONS") { + // Gated despite being a GET: /api/secrets's response body IS a + // credential (S054); /api/notifications DRAINS the queue (S041), so its + // verb lies about being a read the way every other GET here is not. + if (pathname === "/api/secrets") return true; + if (pathname === "/api/notifications") return true; + return false; + } + return true; +} +``` + +- [ ] **Step 6: Run tests to verify they pass** + +Run: `bun test lib/daemon/__tests__/api-auth.test.ts` +Expected: PASS + +- [ ] **Step 7: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. + +- [ ] **Step 8: Commit** + +```bash +git add packages/rt-client/src/settings/registry-defs.ts packages/rt-client/dist lib/daemon/api-auth.ts lib/daemon/__tests__/api-auth.test.ts +git commit -m "daemon: rt.trustedBrowserOrigins allowlist + needsToken invert-default (S005/S006/S040/S041/S084)" +``` + +--- + +## Task 4: Wire CORS default-deny and the `/ws` origin/token gate into api-server.ts (S005, S006) + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Test: `lib/daemon/__tests__/api-server-cors-ws.test.ts` (new) + +**Interfaces:** +- Consumes: `isBrowserRequestTrusted`, `getTrustedBrowserOrigins` from `./api-auth.ts` (Task 3). +- Produces: `export function buildCorsHeaders(origin: string | null, trusted: boolean): Record` — a pure function so the header-shape logic is unit-testable without a real server. The live `/ws` gate and the live CORS-header wiring inside `fetch()` are exercised indirectly through this same function plus the `isBrowserRequestTrusted` tests from Task 3 (this codebase's existing convention -- see `bindApiServerWithRetry` -- is to keep the decision logic in pure, tested functions and keep the `Bun.serve` wiring itself thin). + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/api-server-cors-ws.test.ts +import { describe, test, expect } from "bun:test"; +import { buildCorsHeaders } from "../api-server.ts"; + +describe("buildCorsHeaders", () => { + test("no Origin header: no Access-Control-Allow-Origin is set (non-browser request, CORS is irrelevant)", () => { + const headers = buildCorsHeaders(null, true); + expect(headers["Access-Control-Allow-Origin"]).toBeUndefined(); + }); + + test("an untrusted Origin gets no Access-Control-Allow-Origin (default-deny, S006)", () => { + const headers = buildCorsHeaders("http://evil.example", false); + expect(headers["Access-Control-Allow-Origin"]).toBeUndefined(); + }); + + test("a trusted Origin is echoed back with Vary: Origin", () => { + const headers = buildCorsHeaders("http://localhost:5544", true); + expect(headers["Access-Control-Allow-Origin"]).toBe("http://localhost:5544"); + expect(headers["Vary"]).toBe("Origin"); + }); + + test("always advertises the methods/headers a preflight needs, trusted or not", () => { + const headers = buildCorsHeaders("http://evil.example", false); + expect(headers["Access-Control-Allow-Methods"]).toContain("POST"); + expect(headers["Access-Control-Allow-Headers"]).toContain("X-RT-Token"); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-server-cors-ws.test.ts` +Expected: FAIL — `buildCorsHeaders` is not exported yet. + +- [ ] **Step 3: Write minimal implementation** + +In `lib/daemon/api-server.ts`, change the import line to add the two new helpers: + +```ts +import { needsToken, tokenOk, getApiToken, isBrowserRequestTrusted, getTrustedBrowserOrigins } from "./api-auth.ts"; +``` + +Add this exported pure function near the top of the file (after the `wsClients`/`broadcast` block, before `startApiServer`): + +```ts +/** + * CORS default-deny (S006): a browser page on an untrusted Origin still gets + * its request served (127.0.0.1 loopback + the per-route token gate are the + * real defenses), but the response carries no Access-Control-Allow-Origin, + * so the page's own JavaScript cannot read the body. A request with no + * Origin at all (every non-browser consumer today) needs no CORS headers. + */ +export function buildCorsHeaders(origin: string | null, trusted: boolean): Record { + const headers: Record = { + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type, X-RT-Token", + }; + if (origin && trusted) { + headers["Access-Control-Allow-Origin"] = origin; + headers["Vary"] = "Origin"; + } + return headers; +} +``` + +Now replace the body of `fetch(req, server)` from the top through the old `corsHeaders` declaration. The current code (for reference) is: + +```ts + async fetch(req, server) { + const url = new URL(req.url); + + // WebSocket upgrade — broadcast channel + if (url.pathname === "/ws") { + if (server.upgrade(req, { data: { kind: "broadcast" } })) return undefined as any; + return new Response("WebSocket upgrade failed", { status: 400 }); + } + + // CORS headers for local dev + const corsHeaders = { + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type", + }; +``` + +Replace it with: + +```ts + async fetch(req, server) { + const url = new URL(req.url); + const origin = req.headers.get("origin"); + const allowedOrigins = getTrustedBrowserOrigins(); + + // WebSocket upgrade — broadcast channel. Browsers cannot set custom + // headers on a WS handshake, so the token (when a browser page wants + // to identify itself) travels as a ?token= query param instead of + // X-RT-Token (S005). + if (url.pathname === "/ws") { + const wsToken = url.searchParams.get("token"); + if (!isBrowserRequestTrusted(origin, wsToken, apiToken, allowedOrigins)) { + return new Response("origin not allowed", { status: 403 }); + } + if (server.upgrade(req, { data: { kind: "broadcast" } })) return undefined as any; + return new Response("WebSocket upgrade failed", { status: 400 }); + } + + // CORS: default-deny. A trusted Origin (token or allowlist) gets its + // Origin echoed back; anything else gets no Access-Control-Allow-Origin + // at all, so a malicious page's own JS cannot read the response (S006). + const trusted = isBrowserRequestTrusted(origin, req.headers.get("x-rt-token"), apiToken, allowedOrigins); + const corsHeaders = buildCorsHeaders(origin, trusted); +``` + +No other lines in `fetch()` need to change — every later reference to `corsHeaders` in the function already reads from this same local binding. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-server-cors-ws.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/__tests__/api-server-cors-ws.test.ts +git commit -m "daemon: default-deny CORS and gate /ws on origin/token (S005/S006)" +``` + +--- + +## Task 5: broadcast() drops dead/backpressured WS clients (S042) + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Test: `lib/daemon/__tests__/api-server-broadcast.test.ts` (new) + +**Interfaces:** +- Produces: `export function broadcastToClients(clients: Iterable, type: string, data: any, log: { warn: (o: unknown, m: string) => void }): void` where `BroadcastTarget = { send(data: string): number; close(): void }` (exported type). `broadcast()` becomes a thin wrapper calling `broadcastToClients(wsClients, type, data, log)`. +- Consumes: nothing new. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/api-server-broadcast.test.ts +import { describe, test, expect } from "bun:test"; +import { broadcastToClients, type BroadcastTarget } from "../api-server.ts"; + +function fakeClient(sendReturns: number[]): BroadcastTarget & { closed: boolean; sent: string[] } { + const sent: string[] = []; + let i = 0; + const client = { + closed: false, + sent, + send(data: string) { + sent.push(data); + const ret = sendReturns[Math.min(i, sendReturns.length - 1)]; + i++; + return ret; + }, + close() { client.closed = true; }, + }; + return client; +} + +function fakeLog() { + const warns: unknown[] = []; + return { warn: (o: unknown, _m: string) => { warns.push(o); }, warns }; +} + +describe("broadcastToClients", () => { + test("a healthy client (positive send return) is never closed", () => { + const client = fakeClient([42]); + broadcastToClients([client], "status", { ok: true }, fakeLog()); + expect(client.closed).toBe(false); + expect(client.sent.length).toBe(1); + }); + + test("a send() returning 0 (dropped frame) closes the client immediately and logs a warning", () => { + const client = fakeClient([0]); + const log = fakeLog(); + broadcastToClients([client], "status", { ok: true }, log); + expect(client.closed).toBe(true); + expect(log.warns.length).toBe(1); + }); + + test("a send() returning -1 (backpressure) is tolerated for a few sends before closing", () => { + const client = fakeClient([-1, -1, -1, -1]); + const log = fakeLog(); + broadcastToClients([client], "a", {}, log); + expect(client.closed).toBe(false); + broadcastToClients([client], "b", {}, log); + expect(client.closed).toBe(false); + broadcastToClients([client], "c", {}, log); + // third consecutive backpressure event closes the client + expect(client.closed).toBe(true); + }); + + test("a successful send resets the backpressure counter", () => { + const client = fakeClient([-1, -1, 99, -1, -1, -1]); + const log = fakeLog(); + broadcastToClients([client], "a", {}, log); // -1 (count=1) + broadcastToClients([client], "b", {}, log); // -1 (count=2) + broadcastToClients([client], "c", {}, log); // 99 -- resets to 0 + expect(client.closed).toBe(false); + broadcastToClients([client], "d", {}, log); // -1 (count=1) + broadcastToClients([client], "e", {}, log); // -1 (count=2) + expect(client.closed).toBe(false); + broadcastToClients([client], "f", {}, log); // -1 (count=3) -- closes + expect(client.closed).toBe(true); + }); + + test("a client whose send() throws is treated as gone: caught, not propagated", () => { + const client: BroadcastTarget = { + send() { throw new Error("ECONNRESET"); }, + close() { /* no-op */ }, + }; + expect(() => broadcastToClients([client], "a", {}, fakeLog())).not.toThrow(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-server-broadcast.test.ts` +Expected: FAIL — `broadcastToClients`/`BroadcastTarget` not exported yet. + +- [ ] **Step 3: Write minimal implementation** + +Replace the existing `broadcast()` function and the `wsClients` declaration block in `lib/daemon/api-server.ts`: + +Current code (for reference): + +```ts +const wsClients = new Set>(); + +/** Broadcast an event to all connected WebSocket clients. */ +export function broadcast(type: string, data: any): void { + if (wsClients.size === 0) return; + const msg = JSON.stringify({ type, data, timestamp: Date.now() }); + for (const ws of wsClients) { + try { ws.send(msg); } catch { /* client disconnected */ } + } +} +``` + +Replace with: + +```ts +const wsClients = new Set>(); + +/** Consecutive Bun `ws.send()` backpressure (-1) returns tolerated before a + client is dropped as chronically stalled. */ +const BACKPRESSURE_CLOSE_THRESHOLD = 3; +const backpressureCounts = new WeakMap(); + +export interface BroadcastTarget { + send(data: string): number; + close(): void; +} + +/** + * Sends one frame to every client, dropping any that Bun's own send() return + * value marks as gone (S042). `ws.send()` never throws on a dead socket -- + * it returns 0 (this send silently failed) or -1 (backpressure) -- so a + * disconnected or stalled console/chat-viewer tab used to keep receiving a + * SUBSET of frames forever with nothing logged. 0 means Bun already dropped + * this exact frame for this client: closing immediately (rather than + * counting) is correct because the client's own reconnect logic is the only + * way it recovers a consistent stream. -1 means backpressure, which can be + * transient, so a few in a row are tolerated before giving up on the client. + */ +export function broadcastToClients( + clients: Iterable, + type: string, + data: any, + log: { warn: (o: unknown, m: string) => void }, +): void { + const msg = JSON.stringify({ type, data, timestamp: Date.now() }); + for (const client of clients) { + let result: number; + try { + result = client.send(msg); + } catch (err) { + log.warn({ err }, "ws client send threw; dropping"); + try { client.close(); } catch { /* already gone */ } + continue; + } + if (result === 0) { + log.warn({ type }, "ws client dropped a frame (send()=0); closing so its reconnect resyncs"); + backpressureCounts.delete(client); + try { client.close(); } catch { /* already gone */ } + } else if (result === -1) { + const count = (backpressureCounts.get(client) ?? 0) + 1; + if (count >= BACKPRESSURE_CLOSE_THRESHOLD) { + log.warn({ type, count }, "ws client chronically backpressured; closing"); + backpressureCounts.delete(client); + try { client.close(); } catch { /* already gone */ } + } else { + backpressureCounts.set(client, count); + } + } else { + backpressureCounts.delete(client); + } + } +} + +/** Broadcast an event to all connected WebSocket clients. */ +export function broadcast(type: string, data: any): void { + if (wsClients.size === 0) return; + broadcastToClients(wsClients, type, data, apiServerLog); +} +``` + +This introduces one new module-level binding, `apiServerLog`, since `broadcast()` previously had no logger in scope at all (it is called from many places across the daemon, not just from inside `startApiServer`). Add it right after the existing `wsClients`-adjacent declarations, and set it from `startApiServer`: + +```ts +let apiServerLog: { warn: (o: unknown, m: string) => void } = { warn: () => {} }; +``` + +...and inside `startApiServer`, right after `const { handleCommand, log } = deps;`, add: + +```ts + apiServerLog = log; +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-server-broadcast.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. (`ServerWebSocket` structurally satisfies `BroadcastTarget` since it has both `send(string): number` and `close(): void`, so `broadcastToClients(wsClients, ...)` type-checks with no cast.) + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/__tests__/api-server-broadcast.test.ts +git commit -m "daemon: broadcast() drops dead/backpressured ws clients instead of silently dropping frames (S042)" +``` + +--- + +## Task 6: pathParam() helper — malformed %-encoding returns 400, not a logged 500 (S083) + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Test: `lib/daemon/__tests__/api-server-path-param.test.ts` (new) + +**Interfaces:** +- Produces: `export function pathParam(pathname: string, prefix: string, suffix?: string): string | undefined` (returns `undefined` when the pathname doesn't match the prefix/suffix shape, the captured segment is empty, or `decodeURIComponent` throws). + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/api-server-path-param.test.ts +import { describe, test, expect } from "bun:test"; +import { pathParam } from "../api-server.ts"; + +describe("pathParam", () => { + test("decodes a clean prefix-only param", () => { + expect(pathParam("/api/cache/main", "/api/cache/")).toBe("main"); + }); + + test("decodes a URL-encoded segment", () => { + expect(pathParam("/api/cache/feature%2Ffoo", "/api/cache/")).toBe("feature/foo"); + }); + + test("returns undefined for malformed %-encoding instead of throwing", () => { + expect(pathParam("/api/cache/%E0%A4%A", "/api/cache/")).toBeUndefined(); + }); + + test("returns undefined when the pathname doesn't start with the prefix", () => { + expect(pathParam("/api/other/main", "/api/cache/")).toBeUndefined(); + }); + + test("handles a prefix+suffix pair (hooks repair shape)", () => { + expect(pathParam("/api/hooks/my-repo/repair", "/api/hooks/", "/repair")).toBe("my-repo"); + }); + + test("prefix+suffix: malformed encoding still returns undefined", () => { + expect(pathParam("/api/hooks/%E0%A4%A/repair", "/api/hooks/", "/repair")).toBeUndefined(); + }); + + test("prefix+suffix: wrong suffix returns undefined", () => { + expect(pathParam("/api/hooks/my-repo/other", "/api/hooks/", "/repair")).toBeUndefined(); + }); + + test("an empty captured segment returns undefined", () => { + expect(pathParam("/api/cache/", "/api/cache/")).toBeUndefined(); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-server-path-param.test.ts` +Expected: FAIL — `pathParam` not exported yet. + +- [ ] **Step 3: Write minimal implementation** + +Add this exported function to `lib/daemon/api-server.ts` (near `buildCorsHeaders`, before `startApiServer`): + +```ts +/** + * Decodes one path segment between a fixed prefix (and optional suffix), + * returning `undefined` (never throwing) on any shape mismatch or malformed + * %-encoding (S083). Before this, each parameterized route hand-rolled its + * own decodeURIComponent inside the route's try block, so a malformed + * segment fell through to the OUTER catch and came back as a logged 500; + * every route using this helper instead gets a clean 400. + */ +export function pathParam(pathname: string, prefix: string, suffix = ""): string | undefined { + if (!pathname.startsWith(prefix)) return undefined; + if (suffix && !pathname.endsWith(suffix)) return undefined; + const end = suffix ? pathname.length - suffix.length : pathname.length; + if (end <= prefix.length) return undefined; + const raw = pathname.slice(prefix.length, end); + try { + return decodeURIComponent(raw); + } catch { + return undefined; + } +} +``` + +Now wire it into the three routes inside `fetch()`. Replace the `/api/cache/:branch` block: + +```ts + // Single branch lookup: /api/cache/:branch + if (url.pathname.startsWith("/api/cache/") && req.method === "GET") { + const branch = pathParam(url.pathname, "/api/cache/"); + if (branch === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); + } + const result = await handleCommand("cache:read", { branches: [branch] }, req.signal); + return Response.json(result, { headers: corsHeaders }); + } +``` + +Replace the `/api/hooks/:repo/repair` block: + +```ts + // Hooks repair: /api/hooks/:repo/repair + if (url.pathname.startsWith("/api/hooks/") && url.pathname.endsWith("/repair") && req.method === "POST") { + const repo = pathParam(url.pathname, "/api/hooks/", "/repair"); + if (repo === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); + } + const result = await handleCommand("hooks:repair", { repo }, req.signal); + return Response.json(result, { headers: corsHeaders }); + } +``` + +Replace the `/api/runs/:repo/:runId` block (this one already guarded against the URIError; simplify it onto the shared helper so there is one decode path in the file, not two): + +```ts + // Runs detail: /api/runs/:repo/:runId + if (url.pathname.startsWith("/api/runs/") && req.method === "GET") { + const rest = pathParam(url.pathname, "/api/runs/"); + if (rest === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); + } + const slash = rest.indexOf("/"); + if (slash > 0 && slash < rest.length - 1) { + const result = await handleCommand("runs:get", { repo: rest.slice(0, slash), runId: rest.slice(slash + 1) }, req.signal); + return Response.json(result, { headers: corsHeaders }); + } + // falls through to the 404 path below for a shape mismatch, e.g. "/api/runs/onlyonesegment" + } +``` + +Note the behavior change from before: a malformed `/api/runs/...` now returns 400 instead of falling through to the generic 404. This is intentional and matches S083's ask across all three routes uniformly (a malformed path parameter is a 400, a genuinely unknown route is a 404); update no other test, since no existing test in this repo asserts the old runs-route 500-vs-404 distinction (grep `lib/daemon/__tests__/` for `runs:get` before assuming otherwise, and if one exists, adjust it to expect 400 for the malformed case instead of removing coverage). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-server-path-param.test.ts` +Expected: PASS + +- [ ] **Step 5: Check for an existing runs-route test that might need updating** + +Run: `grep -rn "runs:get\|/api/runs/" lib/daemon/__tests__/` +If a test asserts the old 500-on-malformed or 404-on-malformed behavior for `/api/runs/`, update its expected status to 400 to match the new shared helper. + +- [ ] **Step 6: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. + +- [ ] **Step 7: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/__tests__/api-server-path-param.test.ts +git commit -m "daemon: pathParam() helper -- malformed %-encoding is a 400, not a logged 500 (S083)" +``` + +--- + +## Task 7: GET query param coercion (S085) + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Test: `lib/daemon/__tests__/api-server-query-coerce.test.ts` (new) + +**Interfaces:** +- Produces: `export function coerceQueryParams(params: URLSearchParams): Record` — converts `"true"`/`"false"` to booleans and plain-integer/decimal strings to numbers; everything else stays a string. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/api-server-query-coerce.test.ts +import { describe, test, expect } from "bun:test"; +import { coerceQueryParams } from "../api-server.ts"; + +describe("coerceQueryParams", () => { + test("coerces maxAgeMs to a number (the documented cache:read flag)", () => { + const out = coerceQueryParams(new URLSearchParams("maxAgeMs=60000")); + expect(out.maxAgeMs).toBe(60000); + expect(typeof out.maxAgeMs).toBe("number"); + }); + + test("coerces refresh=true to a boolean (the documented ports flag)", () => { + const out = coerceQueryParams(new URLSearchParams("refresh=true")); + expect(out.refresh).toBe(true); + }); + + test("coerces refresh=false to a boolean false, not a truthy string", () => { + const out = coerceQueryParams(new URLSearchParams("refresh=false")); + expect(out.refresh).toBe(false); + }); + + test("leaves a non-numeric, non-boolean string alone", () => { + const out = coerceQueryParams(new URLSearchParams("repo=my-repo-name")); + expect(out.repo).toBe("my-repo-name"); + }); + + test("leaves an empty string alone rather than coercing to 0", () => { + const out = coerceQueryParams(new URLSearchParams("q=")); + expect(out.q).toBe(""); + }); + + test("coerces a decimal number too", () => { + const out = coerceQueryParams(new URLSearchParams("ratio=1.5")); + expect(out.ratio).toBe(1.5); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-server-query-coerce.test.ts` +Expected: FAIL — `coerceQueryParams` not exported yet. + +- [ ] **Step 3: Write minimal implementation** + +Add to `lib/daemon/api-server.ts` (near `pathParam`): + +```ts +const PLAIN_NUMBER_RE = /^-?\d+(\.\d+)?$/; + +/** + * REST query strings arrive as strings no matter what the client meant + * (S085): `?maxAgeMs=60000` and `?refresh=true` reached handlers that do a + * strict `typeof x === "number"` or `x === true` check, so the documented + * flag silently no-op'd over HTTP while working over the socket (where + * payloads are real JSON). One coercion at the REST seam fixes every such + * flag at once instead of a per-handler private parser. + */ +export function coerceQueryParams(params: URLSearchParams): Record { + const out: Record = {}; + for (const [key, value] of params) { + if (value === "true") out[key] = true; + else if (value === "false") out[key] = false; + else if (value !== "" && PLAIN_NUMBER_RE.test(value)) out[key] = Number(value); + else out[key] = value; + } + return out; +} +``` + +Change the payload-building line inside `fetch()` from: + +```ts + } else { + payload = Object.fromEntries(url.searchParams); + } +``` + +to: + +```ts + } else { + payload = coerceQueryParams(url.searchParams); + } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-server-query-coerce.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/__tests__/api-server-query-coerce.test.ts +git commit -m "daemon: coerce REST GET query params to number/boolean at the seam (S085)" +``` + +--- + +## Task 8: ApiPortInUseError + lsof diagnostics when bind retries are exhausted (S043 addendum) + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Test: `lib/daemon/__tests__/api-server-bind.test.ts` (extend) + +**Interfaces:** +- Produces: `export class ApiPortInUseError extends Error { readonly code: "EADDRINUSE"; readonly port: number }`. `BindRetryDeps` grows an optional `probePortHolder?: (port: number) => Promise` field (defaults to a real `lsof -i :` via `runCapture`). `bindApiServerWithRetry` throws `ApiPortInUseError` (instead of the raw EADDRINUSE `Error`) once `BIND_RETRY_ATTEMPTS` is exhausted, after logging the probe result at `warn`. +- Consumes: `runCapture` from `../subprocess.ts`. +- **Contract for the sibling job owning `lib/daemon.ts`'s caller side:** catch `ApiPortInUseError` (check `err instanceof ApiPortInUseError`, or `err.name === "ApiPortInUseError"`) around the `startApiServer()` call and route to a park-and-retry-with-backoff loop instead of letting it propagate to the top-level crash/unhandledRejection path. Every other error out of `startApiServer()` is a real misconfiguration and should keep crashing as it does today. + +- [ ] **Step 1: Write the failing test** + +Append to `lib/daemon/__tests__/api-server-bind.test.ts`: + +```ts +import { ApiPortInUseError, BIND_RETRY_ATTEMPTS } from "../api-server.ts"; + +function depsWithProbe(overrides: Partial = {}) { + const logs: Array<{ o: unknown; m: string }> = []; + const sleeps: number[] = []; + const probeCalls: number[] = []; + return { + sleep: async (ms: number) => { sleeps.push(ms); }, + log: { warn: (o: unknown, m: string) => logs.push({ o, m }) }, + probePortHolder: async (port: number) => { probeCalls.push(port); return "COMMAND PID USER\nnode 123 matt"; }, + logs, + sleeps, + probeCalls, + ...overrides, + }; +} + +describe("bindApiServerWithRetry — exhausted retries (S043)", () => { + test("throws ApiPortInUseError (not the raw EADDRINUSE Error) once attempts are exhausted", async () => { + const d = depsWithProbe(); + let error: unknown; + try { + await bindApiServerWithRetry(() => { throw eaddrinuse(); }, d); + } catch (err) { + error = err; + } + expect(error).toBeInstanceOf(ApiPortInUseError); + expect((error as ApiPortInUseError).code).toBe("EADDRINUSE"); + expect((error as Error).message).toContain("EADDRINUSE"); + }); + + test("probes the port holder exactly once, only after the final attempt", async () => { + const d = depsWithProbe(); + let calls = 0; + await expect( + bindApiServerWithRetry(() => { calls++; throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + expect(calls).toBe(BIND_RETRY_ATTEMPTS); + expect(d.probeCalls.length).toBe(1); + }); + + test("logs the probe result at warn before throwing", async () => { + const d = depsWithProbe(); + await expect( + bindApiServerWithRetry(() => { throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + const finalWarn = d.logs.at(-1)!; + expect(finalWarn.o).toMatchObject({ holder: expect.stringContaining("node") }); + }); + + test("a probe failure does not prevent the ApiPortInUseError from being thrown", async () => { + const d = depsWithProbe({ probePortHolder: async () => { throw new Error("lsof: command not found"); } }); + await expect( + bindApiServerWithRetry(() => { throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + }); + + test("a successful bind never probes", async () => { + const d = depsWithProbe(); + await bindApiServerWithRetry(() => "server" as any, d); + expect(d.probeCalls.length).toBe(0); + }); +}); +``` + +Also update the OLD test that currently asserts the raw-error message (it now gets a differently-shaped, but still EADDRINUSE-mentioning, error): + +Find this existing test: + +```ts + test("exhausting retries rethrows the original error after exactly BIND_RETRY_ATTEMPTS calls", async () => { + const d = deps(); + let calls = 0; + await expect( + bindApiServerWithRetry(() => { calls++; throw eaddrinuse(); }, d), + ).rejects.toThrow("EADDRINUSE"); + expect(calls).toBe(BIND_RETRY_ATTEMPTS); + expect(d.sleeps.length).toBe(BIND_RETRY_ATTEMPTS - 1); + }); +``` + +Change its assertion to also cover the new type, since `deps()` (the original helper, with no `probePortHolder` override) must still work by falling back to a default real-`lsof` probe — which would actually shell out in a test. To keep this test hermetic, add `probePortHolder: async () => "n/a"` to the base `deps()` helper's return object (it is an optional field on `BindRetryDeps`, so this is a backward-compatible addition, not a signature break): + +```ts +function deps(overrides: Partial = {}): BindRetryDeps & { logs: string[]; sleeps: number[] } { + const logs: string[] = []; + const sleeps: number[] = []; + return { + sleep: async (ms: number) => { sleeps.push(ms); }, + log: { warn: (_o: unknown, m: string) => logs.push(`warn:${m}`) }, + probePortHolder: async () => "n/a", + logs, + sleeps, + ...overrides, + }; +} +``` + +Then the pre-existing test's assertion becomes: + +```ts + test("exhausting retries rethrows as ApiPortInUseError after exactly BIND_RETRY_ATTEMPTS calls", async () => { + const d = deps(); + let calls = 0; + await expect( + bindApiServerWithRetry(() => { calls++; throw eaddrinuse(); }, d), + ).rejects.toThrow("EADDRINUSE"); + expect(calls).toBe(BIND_RETRY_ATTEMPTS); + expect(d.sleeps.length).toBe(BIND_RETRY_ATTEMPTS - 1); + }); +``` + +(only the test name and this doc comment change; `.rejects.toThrow("EADDRINUSE")` still matches since `ApiPortInUseError`'s message contains that substring.) + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/api-server-bind.test.ts` +Expected: FAIL — `ApiPortInUseError` not exported; `probePortHolder` not a recognized field yet. + +- [ ] **Step 3: Write minimal implementation** + +In `lib/daemon/api-server.ts`, add the import: + +```ts +import { runCapture } from "../subprocess.ts"; +``` + +Add the error class and update `BindRetryDeps`/`bindApiServerWithRetry`. Replace the current block: + +```ts +export interface BindRetryDeps { + sleep: (ms: number) => Promise; + log: { warn: (o: unknown, m: string) => void }; +} + +// evictStaleDaemon (lib/daemon/boot-reconcile.ts) already assumes a prior +// holder is gone after a 300ms Bun.sleepSync — 6 attempts at 500ms (~3s +// worst case, exported so tests assert against these, not hardcoded copies) +// gives that same assumption room to be wrong once before this gives up too. +export const BIND_RETRY_ATTEMPTS = 6; +export const BIND_RETRY_DELAY_MS = 500; + +/** + * evictStaleDaemon() SIGTERMs the previous holder of this port before a new + * daemon binds, but the kill isn't synchronous with the exit — a fresh + * daemon can reach this bind before the old one has actually released + * 9401. Only EADDRINUSE is retried (bounded, ~3s total); anything else is a + * real misconfiguration and fails on the first attempt, same as before. + */ +export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDeps): Promise { + for (let attempt = 1; ; attempt++) { + try { + return bind(); + } catch (err) { + const isAddrInUse = err instanceof Error && (err as NodeJS.ErrnoException).code === "EADDRINUSE"; + if (!isAddrInUse || attempt >= BIND_RETRY_ATTEMPTS) throw err; + deps.log.warn({ attempt, port: API_PORT }, "api port in use, retrying — another daemon is likely still shutting down"); + await deps.sleep(BIND_RETRY_DELAY_MS); + } + } +} +``` + +with: + +```ts +/** + * Thrown when every bind retry is exhausted with EADDRINUSE still held. A + * NAMED error type (S043) rather than the raw EADDRINUSE Error, so + * lib/daemon.ts's caller can distinguish "the port is genuinely squatted" + * from any other startup failure and park-and-retry with backoff instead of + * crash-looping (that caller-side change belongs to a sibling job; this + * class is the contract it wires into). + */ +export class ApiPortInUseError extends Error { + readonly code = "EADDRINUSE" as const; + readonly port: number; + constructor(port: number) { + super(`EADDRINUSE: api server port ${port} is still in use after ${BIND_RETRY_ATTEMPTS} bind attempts`); + this.name = "ApiPortInUseError"; + this.port = port; + } +} + +export interface BindRetryDeps { + sleep: (ms: number) => Promise; + log: { warn: (o: unknown, m: string) => void }; + /** Defaults to a real `lsof -i :` via runCapture; overridable so tests never shell out. */ + probePortHolder?: (port: number) => Promise; +} + +async function defaultProbePortHolder(port: number): Promise { + const result = await runCapture(["lsof", "-i", `:${port}`], { timeoutMs: 5_000, stderr: "pipe" }); + return result.stdout.trim(); +} + +// evictStaleDaemon (lib/daemon/boot-reconcile.ts) already assumes a prior +// holder is gone after a 300ms Bun.sleepSync — 6 attempts at 500ms (~3s +// worst case, exported so tests assert against these, not hardcoded copies) +// gives that same assumption room to be wrong once before this gives up too. +export const BIND_RETRY_ATTEMPTS = 6; +export const BIND_RETRY_DELAY_MS = 500; + +/** + * evictStaleDaemon() SIGTERMs the previous holder of this port before a new + * daemon binds, but the kill isn't synchronous with the exit — a fresh + * daemon can reach this bind before the old one has actually released + * 9401. Only EADDRINUSE is retried (bounded, ~3s total); anything else is a + * real misconfiguration and fails on the first attempt, same as before. + * + * Once retries are exhausted, this logs WHO holds the port (S043's + * diagnostic ask) and throws ApiPortInUseError instead of the bare + * EADDRINUSE Error, so a caller can tell "give up cleanly" apart from "the + * bind function itself is broken". + */ +export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDeps): Promise { + const probe = deps.probePortHolder ?? defaultProbePortHolder; + for (let attempt = 1; ; attempt++) { + try { + return bind(); + } catch (err) { + const isAddrInUse = err instanceof Error && (err as NodeJS.ErrnoException).code === "EADDRINUSE"; + if (!isAddrInUse) throw err; + if (attempt >= BIND_RETRY_ATTEMPTS) { + const holder = await probe(API_PORT).catch((probeErr) => `lsof failed: ${String(probeErr)}`); + deps.log.warn({ port: API_PORT, holder }, "api port still in use after retries; giving up bind — the daemon should park and retry with backoff rather than crash-loop"); + throw new ApiPortInUseError(API_PORT); + } + deps.log.warn({ attempt, port: API_PORT }, "api port in use, retrying — another daemon is likely still shutting down"); + await deps.sleep(BIND_RETRY_DELAY_MS); + } + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/api-server-bind.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the full daemon test slice and tsc** + +Run: `bun test lib/daemon` +Run: `bunx tsc --noEmit` +Expected: both clean. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/__tests__/api-server-bind.test.ts +git commit -m "daemon: log the EADDRINUSE port holder and throw a typed ApiPortInUseError (S043)" +``` + +--- + +## Task 9: git-ref validation utility (S010 — standalone, sibling wires it into worktree.ts) + +**Files:** +- Create: `lib/daemon/git-ref-validation.ts` +- Test: `lib/daemon/__tests__/git-ref-validation.test.ts` + +**Interfaces:** +- Produces: `export function isSafeGitRef(ref: string): boolean` and `export function validateGitRef(ref: string): { ok: true } | { ok: false; error: string }`. + +This module is NOT wired into `lib/daemon/handlers/worktree.ts` by this job — that file is sibling-owned per the write fence. The wiring is documented in `docs/daemon-api-auth.md` (Task 11) and the job report's Notes. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/git-ref-validation.test.ts +import { describe, test, expect } from "bun:test"; +import { isSafeGitRef, validateGitRef } from "../git-ref-validation.ts"; + +describe("isSafeGitRef", () => { + test("a normal branch name is safe", () => { + expect(isSafeGitRef("feature/my-branch")).toBe(true); + }); + + test("a leading dash is unsafe (option injection, S010)", () => { + expect(isSafeGitRef("--upload-pack=touch /tmp/x")).toBe(false); + }); + + test("a bare dash is unsafe", () => { + expect(isSafeGitRef("-")).toBe(false); + }); + + test("an empty string is unsafe", () => { + expect(isSafeGitRef("")).toBe(false); + }); + + test("a branch containing a dash mid-name is safe", () => { + expect(isSafeGitRef("job/p3-trust-boundary")).toBe(true); + }); +}); + +describe("validateGitRef", () => { + test("returns ok:true for a safe ref", () => { + expect(validateGitRef("main")).toEqual({ ok: true }); + }); + + test("returns ok:false with the offending ref named in the error for an unsafe one", () => { + const result = validateGitRef("--upload-pack=x"); + expect(result.ok).toBe(false); + expect((result as { ok: false; error: string }).error).toContain("--upload-pack=x"); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/git-ref-validation.test.ts` +Expected: FAIL — module does not exist. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/git-ref-validation.ts +/** + * Rejects a branch/ref that could be parsed by git as an OPTION rather than + * a ref (S010): a caller-supplied `branch` like "--upload-pack=touch /tmp/x" + * reaches `git fetch origin ` and `git rev-list ......` + * unescaped in lib/daemon/handlers/worktree.ts, and git happily runs it as + * an option since nothing on that path validates the string first. This is + * the ONE guard both call sites need; a future caller (or a consumer app + * like mr-board/console/the chat viewer forwarding an untrusted string as + * `branch`) inherits the same hole without it. + * + * Deliberately narrow: reject a leading '-' rather than allowlisting a + * character set, since `git check-ref-format --branch` accepts far more + * punctuation than is worth re-deriving here, and the vulnerable shape is + * specifically "parses as an option", not "contains an unusual character". + */ +export function isSafeGitRef(ref: string): boolean { + return ref.length > 0 && !ref.startsWith("-"); +} + +export function validateGitRef(ref: string): { ok: true } | { ok: false; error: string } { + if (!isSafeGitRef(ref)) { + return { ok: false, error: `unsafe git ref (starts with '-' or empty): ${ref}` }; + } + return { ok: true }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/git-ref-validation.test.ts` +Expected: PASS + +- [ ] **Step 5: Run tsc** + +Run: `bunx tsc --noEmit` +Expected: clean. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/git-ref-validation.ts lib/daemon/__tests__/git-ref-validation.test.ts +git commit -m "daemon: standalone git-ref validator for S010 (worktree.ts wiring documented, not wired here)" +``` + +--- + +## Task 10: credential redaction utility (S050 — standalone, sibling wires it into freshness.ts) + +**Files:** +- Create: `lib/daemon/redact-credentials.ts` +- Test: `lib/daemon/__tests__/redact-credentials.test.ts` + +**Interfaces:** +- Produces: `export function redactCredentials(text: string): string`. + +Same treatment as Task 9: `lib/daemon/freshness.ts` (where the audit's fixer notes say this belongs, at lines ~142/148/275/279) is not in this job's write fence. This module is standalone and tested; the wiring is documented in `docs/daemon-api-auth.md` (Task 11) and the report's Notes. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/redact-credentials.test.ts +import { describe, test, expect } from "bun:test"; +import { redactCredentials } from "../redact-credentials.ts"; + +describe("redactCredentials", () => { + test("redacts userinfo (user:token@) out of an https remote URL", () => { + const input = "https://oauth2:glpat-XXXXXXXXXXXXXXXXXXXX@gitlab.example.com/g/p.git"; + const out = redactCredentials(input); + expect(out).not.toContain("glpat-XXXXXXXXXXXXXXXXXXXX"); + expect(out).toContain("gitlab.example.com/g/p.git"); + }); + + test("redacts a GitHub PAT embedded in the URL", () => { + const input = "https://ghp_abcdefghijklmnopqrstuvwxyz012345@github.com/o/r.git"; + const out = redactCredentials(input); + expect(out).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz012345"); + expect(out).toContain("github.com/o/r.git"); + }); + + test("leaves a URL with no embedded credentials unchanged", () => { + const input = "https://gitlab.example.com/g/p.git"; + expect(redactCredentials(input)).toBe(input); + }); + + test("leaves plain text with no URL unchanged", () => { + const input = "local branch listing failed"; + expect(redactCredentials(input)).toBe(input); + }); + + test("redacts every match when more than one credential-bearing URL appears in the same string", () => { + const input = "tried https://oauth2:tok1@a.example/x then https://oauth2:tok2@b.example/y"; + const out = redactCredentials(input); + expect(out).not.toContain("tok1"); + expect(out).not.toContain("tok2"); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/redact-credentials.test.ts` +Expected: FAIL — module does not exist. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/redact-credentials.ts +/** + * Strips userinfo (user:token@ or user@) out of any http(s) URL embedded in + * a string (S050): freshness.ts logs `remote.origin.url` verbatim at info + * on every reconcile, and echoes it into thrown errors every mr/discussions + * handler returns to callers. A repo cloned as + * `https://oauth2:glpat-XXXX@gitlab.example.com/...` (routine for + * dotfiles/CI-derived clones) puts that token into ~/.rt/logs/daemon.*.log + * and into any client-facing error message -- logs are the first thing a + * user pastes into a bug report. + */ +const CREDENTIAL_URL_RE = /(https?:\/\/)[^/@\s]+@/gi; + +export function redactCredentials(text: string): string { + return text.replace(CREDENTIAL_URL_RE, "$1[redacted]@"); +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/redact-credentials.test.ts` +Expected: PASS + +- [ ] **Step 5: Run tsc** + +Run: `bunx tsc --noEmit` +Expected: clean. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/redact-credentials.ts lib/daemon/__tests__/redact-credentials.test.ts +git commit -m "daemon: standalone credential-redaction utility for S050 (freshness.ts wiring documented, not wired here)" +``` + +--- + +## Task 11: docs note + final whole-branch verification + +**Files:** +- Create: `docs/daemon-api-auth.md` + +- [ ] **Step 1: Write the doc** + +```markdown +# The :9401 trust boundary + +How the daemon's REST/WS surface decides who to trust, and the follow-up +wiring two standalone modules from this phase still need in sibling-owned +files. + +## The model + +- **No `Origin` header at all** (the CLI, the Swift tray, rt-client from a + Bun/Node process, mr-board, gitq, the VS Code extension): unaffected, no + gate applies beyond what already existed. None of today's consumers send + an `Origin` header to `:9401`. +- **A browser `Origin` header is present**: trusted only if the request + presents the local `X-RT-Token` (`?token=` query param for `/ws`, since + browsers cannot set custom headers on a WS handshake) OR the Origin is on + the `rt.trustedBrowserOrigins` settings allowlist (see + `packages/rt-client/src/settings/registry-defs.ts`; `docs/settings-architecture.md` + is the settings-system contract). Otherwise: no `Access-Control-Allow-Origin` + on REST reads (default-deny CORS), and a 403 on `/ws`. +- **Mutating routes** (every method except GET/HEAD/OPTIONS, plus + `/api/secrets` and `/api/notifications` despite being GETs) require the + local `X-RT-Token` regardless of Origin — this is the CSRF defense against + a browser form/simple-request bypassing CORS preflight entirely, and it is + orthogonal to the Origin check above. + +See `lib/daemon/api-auth.ts` (`isBrowserRequestTrusted`, `needsToken`, +`getTrustedBrowserOrigins`) and `lib/daemon/api-server.ts` +(`buildCorsHeaders`, the `/ws` gate in `fetch()`) for the implementation. + +## Follow-up wiring for sibling-owned files (not done in this job) + +**S010** (`lib/daemon/handlers/worktree.ts`): `lib/daemon/git-ref-validation.ts` +exports `validateGitRef(ref)`. Call it right after `payload.branch` is read +(around `worktree.ts:282`) and return `{ ok: false, error }` on a rejection +BEFORE any `runGit` call reaches it — that single call site also covers the +weaker secondary instance in `divergence()` (`worktree.ts:211-213`), since +both read the same `branch` value. + +**S050** (`lib/daemon/freshness.ts`): `lib/daemon/redact-credentials.ts` +exports `redactCredentials(text)`. Wrap every log/error interpolation of a +remote URL with it — the audit names `freshness.ts:142, 148, 275, 279` as the +current call sites. + +**S043 caller side** (`lib/daemon.ts`): `lib/daemon/api-server.ts` exports +`ApiPortInUseError` (a named `Error` subclass with `.name === "ApiPortInUseError"` +and `.port`). Catch it around the `startApiServer()` call and park-and-retry +with backoff instead of letting it reach the top-level crash path; any other +error out of `startApiServer()` is a genuine misconfiguration and should keep +crashing as it does today. +``` + +- [ ] **Step 2: Run the full verification suite** + +Run: `bun test lib commands packages scripts` +Expected: all green. + +Run: `bunx tsc --noEmit` +Expected: zero errors. + +Run: `cd packages/rt-client && bun run build && cd -` +Expected: clean build (already run in Task 3, but re-run here as the final gate since later tasks may have touched files rt-client's dist-freshness test watches). + +- [ ] **Step 3: Commit** + +```bash +git add docs/daemon-api-auth.md +git commit -m "docs: the :9401 trust boundary model and the S010/S050/S043 sibling wiring notes" +``` + +--- + +## Self-Review Notes (already applied above) + +- **Spec coverage:** S092 (Task 1), S054 (Task 2), S005/S006/S040/S041/S084 (Tasks 3-4), S042 (Task 5), S083 (Task 6), S085 (Task 7), S043 (Task 8), S010 (Task 9), S050 (Task 10). All 13 cited findings have a task. +- **Write fence:** every modified/created file is inside the job's write fence; Tasks 9-10 deliberately stop short of editing `worktree.ts`/`freshness.ts` and document the wiring instead, mirroring the job brief's explicit instruction for S010 and extending the same treatment to S050 since its fix location is equally out of fence. +- **Type consistency:** `isBrowserRequestTrusted(origin, token, apiToken, allowedOrigins)` has the same parameter order and names everywhere it's used (Task 3 definition, Task 4 call sites). `BroadcastTarget`/`broadcastToClients` names match between Task 5's definition and its test. `pathParam`/`coerceQueryParams` signatures match between definition and test across Tasks 6-7. diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index 48345d0ff..a7dc3cdb5 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -1,7 +1,169 @@ import { describe, test, expect, beforeAll, afterAll } from "bun:test"; -import { existsSync } from "fs"; +import { existsSync, mkdirSync, writeFileSync, readdirSync } from "fs"; import { join } from "path"; -import { createTestHome, rt } from "../harness.ts"; +import { createTestHome, rt, RT_BINARY } from "../harness.ts"; + +async function waitForSocket(sockPath: string, timeoutMs = 15_000): Promise { + const deadline = Date.now() + timeoutMs; + while (!existsSync(sockPath)) { + if (Date.now() > deadline) throw new Error(`daemon socket never appeared at ${sockPath}`); + await Bun.sleep(100); + } +} + +/** Grab a free TCP port by binding port 0 and releasing it. */ +function freePort(): number { + const srv = Bun.serve({ port: 0, fetch: () => new Response("") }); + const port = srv.port; + srv.stop(true); + if (!port) throw new Error("failed to allocate a free port"); + return port; +} + +describe("fatal boot", () => { + // These three used to assert a crash (S043 pre-fix: EADDRINUSE on the API + // port took the daemon down the fatal boot-failed path). The integration + // job's I5(a) wiring (lib/daemon.ts's withApiPortParkRetry around + // startApiServer, docs/daemon-api-auth.md's S043 caller-side contract) + // makes this recoverable instead: the daemon parks and retries with + // backoff rather than crashing, so it now boots successfully once the + // squatted port frees. + test("daemon parks (does not crash) while the API port is squatted, and boots once it frees", async () => { + const { path: home, cleanup } = createTestHome(); + const bunDir = join(process.execPath, ".."); + const port = 9411; + const rtDir = join(home, ".mattstack", "rt"); + const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + let daemon: ReturnType | undefined; + try { + daemon = Bun.spawn([RT_BINARY, "--daemon"], { + env: { + HOME: home, + PATH: `${join(RT_BINARY, "..")}:${bunDir}:/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin`, + TERM: "xterm-256color", + RT_SKIP_SETUP: "1", + CI: "true", + RT_API_PORT: String(port), + }, + stdout: "pipe", + stderr: "pipe", + }); + + // Give bindApiServerWithRetry's own ~3s inner retry a full cycle to + // exhaust and reach the outer park-retry loop; it must still be alive + // (parked, not crashed) and must not have written rt.sock/rt.pid yet + // (neither server has bound). + await Bun.sleep(4_000); + expect(daemon.exitCode).toBeNull(); + expect(existsSync(join(rtDir, "rt.sock"))).toBe(false); + expect(existsSync(join(rtDir, "rt.pid"))).toBe(false); + + squatter.stop(true); + + await waitForSocket(join(rtDir, "rt.sock"), 40_000); + expect(daemon.exitCode).toBeNull(); + expect(existsSync(join(rtDir, "rt.pid"))).toBe(true); + } finally { + squatter.stop(true); + try { daemon?.kill(); } catch { /* already gone */ } + await daemon?.exited; + cleanup(); + } + }, 60_000); + + test("daemon status --json never claims 'running' while parked on a squatted API port, and does once it recovers", async () => { + const { path: home, cleanup } = createTestHome(); + const bunDir = join(process.execPath, ".."); + // A different port than the sibling test above, so parallel test files + // can never collide on the same bound TCP port. + const port = 9412; + const rtDir = join(home, ".mattstack", "rt"); + const squatter = Bun.serve({ port, hostname: "127.0.0.1", fetch: () => new Response("busy") }); + let daemon: ReturnType | undefined; + try { + // `rt daemon status` short-circuits to "not installed" before it ever + // reaches a liveness classification, install first. + await rt(["daemon", "install"], { home }); + + daemon = Bun.spawn([RT_BINARY, "--daemon"], { + env: { + HOME: home, + PATH: `${join(RT_BINARY, "..")}:${bunDir}:/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin`, + TERM: "xterm-256color", + RT_SKIP_SETUP: "1", + CI: "true", + RT_API_PORT: String(port), + }, + stdout: "pipe", + stderr: "pipe", + }); + + await Bun.sleep(4_000); + expect(daemon.exitCode).toBeNull(); + + const parked = await rt(["daemon", "status", "--json"], { home, env: { RT_API_PORT: String(port) } }); + expect(parked.exitCode).toBe(0); + expect(JSON.parse(parked.stdout).state).not.toBe("running"); + + squatter.stop(true); + await waitForSocket(join(rtDir, "rt.sock"), 40_000); + + const recovered = await rt(["daemon", "status", "--json"], { home, env: { RT_API_PORT: String(port) } }); + expect(recovered.exitCode).toBe(0); + expect(JSON.parse(recovered.stdout).state).toBe("running"); + } finally { + squatter.stop(true); + try { daemon?.kill(); } catch { /* already gone */ } + await daemon?.exited; + cleanup(); + } + }, 60_000); + + test("a corrupt events.db self-heals (quarantined), and the daemon boots and serves", async () => { + const { path: home, cleanup } = createTestHome(); + const bunDir = join(process.execPath, ".."); + let daemon: ReturnType | undefined; + try { + // Pre-create a corrupt events.db in the isolated HOME, before the + // daemon ever runs; createEventsBus (module scope) opens it. + const rtDir = join(home, ".mattstack", "rt"); + mkdirSync(rtDir, { recursive: true }); + writeFileSync(join(rtDir, "events.db"), "not a sqlite file at all"); + + const apiPort = freePort(); + daemon = Bun.spawn([RT_BINARY, "--daemon"], { + env: { + HOME: home, + PATH: `${join(RT_BINARY, "..")}:${bunDir}:/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin`, + TERM: "xterm-256color", + RT_SKIP_SETUP: "1", + CI: "true", + RT_API_PORT: String(apiPort), + }, + stdout: "pipe", + stderr: "pipe", + }); + + await waitForSocket(join(rtDir, "rt.sock")); + expect(daemon.exitCode).toBeNull(); + + // (a) the corrupt events.db was quarantined, not just failed on. + expect(readdirSync(rtDir).some((f) => f.startsWith("events.db.corrupt-"))).toBe(true); + + // (b) the daemon actually boots and serves: a live round trip through + // the recreated events.db proves it, not just the socket's existence. + const served = await rt(["events", "emit", "e2e/corrupt-events-recover"], { + home, + env: { RT_API_PORT: String(apiPort) }, + }); + expect(served.exitCode).toBe(0); + } finally { + try { daemon?.kill(); } catch { /* already gone */ } + await daemon?.exited; + cleanup(); + } + }, 60_000); +}); describe("daemon", () => { describe("install creates config", () => { diff --git a/lib/__tests__/agent-herdr.test.ts b/lib/__tests__/agent-herdr.test.ts index 568fdc19a..60b0f5f18 100644 --- a/lib/__tests__/agent-herdr.test.ts +++ b/lib/__tests__/agent-herdr.test.ts @@ -1,5 +1,8 @@ import { expect, test } from "bun:test"; -import { herdrAgentWait, launchInWorkspace, type HerdrRunner } from "../agent-herdr.ts"; +import { mkdtempSync, writeFileSync, chmodSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { defaultHerdrRunner, herdrAgentWait, launchInWorkspace, resolveHerdrBin, type HerdrRunner } from "../agent-herdr.ts"; function scripted(responses: Record) { const calls: string[][] = []; @@ -90,3 +93,34 @@ test("herdrAgentWait builds the current verb (agent wait --until)", async () => await herdrAgentWait("wA:p1", ["idle", "done"], 45000, runner); expect(calls[0]).toEqual(["agent", "wait", "wA:p1", "--until", "idle", "--until", "done", "--timeout", "45000"]); }); + +test("resolveHerdrBin prefers HERDR_BIN over everything else", () => { + const bin = resolveHerdrBin({ HERDR_BIN: "/custom/herdr", HOME: "/home/x" }, () => "/opt/homebrew/bin/herdr"); + expect(bin).toBe("/custom/herdr"); +}); + +test("resolveHerdrBin prefers a PATH-resolved herdr over the ~/.local/bin fallback", () => { + const bin = resolveHerdrBin({ HOME: "/home/x" }, () => "/opt/homebrew/bin/herdr"); + expect(bin).toBe("/opt/homebrew/bin/herdr"); +}); + +test("resolveHerdrBin falls back to ~/.local/bin/herdr when PATH resolution fails", () => { + const bin = resolveHerdrBin({ HOME: "/home/x" }, () => null); + expect(bin).toBe("/home/x/.local/bin/herdr"); +}); + +test("defaultHerdrRunner throws a clear error when the resolved bin does not exist", async () => { + const runner = defaultHerdrRunner({ HERDR_BIN: "/nonexistent/herdr", HOME: "/home/x" }); + await expect(runner(["workspace", "list"])).rejects.toThrow(/herdr not found/); +}); + +test("defaultHerdrRunner passes the supplied env through to the child (C9: it must not fall back to bare process.env)", async () => { + const dir = mkdtempSync(join(tmpdir(), "agent-herdr-env-")); + const fakeHerdr = join(dir, "herdr"); + writeFileSync(fakeHerdr, "#!/bin/sh\necho \"$SENTINEL_VAR\"\n"); + chmodSync(fakeHerdr, 0o755); + + const runner = defaultHerdrRunner({ HERDR_BIN: fakeHerdr, HOME: dir, SENTINEL_VAR: "from-supplied-env" }); + const result = await runner(["workspace", "list"]); + expect(result.stdout.trim()).toBe("from-supplied-env"); +}); diff --git a/lib/__tests__/daemon-client-attribution.test.ts b/lib/__tests__/daemon-client-attribution.test.ts new file mode 100644 index 000000000..7d087d1e6 --- /dev/null +++ b/lib/__tests__/daemon-client-attribution.test.ts @@ -0,0 +1,104 @@ +/** + * S081: daemon-client's timed-out/refused attribution used to live in + * module-level flags (`_lastQueryTimedOut`/`_lastQueryWasRefused`) shared + * across every concurrent query. A fast query resolving in between a slow + * query's own trySocketQuery call and the moment its caller reads + * lastQueryTimedOut() could reset those flags out from under it — the slow + * query's caller would then see "daemon unavailable" for a call that + * actually just exceeded its own window. daemonQueryAttributed returns the + * failure kind alongside the response instead, so it can never be + * clobbered by an unrelated concurrent call. + */ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { DAEMON_SOCK_PATH, TRAY_SOCK_PATH, markDaemonInstalled, markDaemonUninstalled } from "../daemon-config.ts"; +import { daemonQueryAttributed } from "../daemon-client.ts"; + +describe("daemonQueryAttributed", () => { + let server: ReturnType; + + beforeEach(() => { + markDaemonInstalled(); + server = Bun.serve({ + unix: DAEMON_SOCK_PATH, + async fetch(req) { + const url = new URL(req.url); + if (url.pathname === "/slow") { + await new Promise(() => {}); // never resolves — the client's own AbortSignal.timeout fires + } + return new Response(JSON.stringify({ ok: true, data: {} }), { headers: { "Content-Type": "application/json" } }); + }, + }); + }); + + afterEach(() => { + server.stop(true); + markDaemonUninstalled(); + }); + + test("a slow query that times out reports timedOut:true even resolved after a concurrent fast success", async () => { + const slow = daemonQueryAttributed("slow", {}, 50); + const fast = await daemonQueryAttributed("fast", {}); + expect(fast.response?.ok).toBe(true); + expect(fast.timedOut).toBe(false); + + const slowResult = await slow; + expect(slowResult.response).toBeNull(); + expect(slowResult.timedOut).toBe(true); + expect(slowResult.refused).toBe(false); + }); + + test("a fast query's own success is reported correctly even started after a slow one is already in flight", async () => { + const slow = daemonQueryAttributed("slow", {}, 200); + const fast = await daemonQueryAttributed("fast", {}); + expect(fast.response?.ok).toBe(true); + expect(fast.timedOut).toBe(false); + await slow; // drain — this one times out at 200ms, don't leave it dangling + }); +}); + +// S082: auto-start retried the real query exactly once, 300ms after asking +// the tray to start the daemon — but parkUntilIntended's own socket probe, +// state.db open, and the identity migration routinely take longer than +// that, so a start that genuinely succeeds still gets reported as +// "installed but not running". +describe("auto-start bounded poll (S082)", () => { + const origRtAppSocket = process.env.RT_APP_SOCKET; + let daemonServer: ReturnType | undefined; + let traySock: string | undefined; + let trayServer: ReturnType | undefined; + let bindTimer: ReturnType | undefined; + + beforeEach(() => { + markDaemonInstalled(); + }); + + afterEach(() => { + if (bindTimer) clearTimeout(bindTimer); + daemonServer?.stop(true); + trayServer?.stop(true); + markDaemonUninstalled(); + if (origRtAppSocket === undefined) delete process.env.RT_APP_SOCKET; + else process.env.RT_APP_SOCKET = origRtAppSocket; + }); + + test("a restart that takes longer than 300ms to bind its socket is still picked up, not reported down", async () => { + // attemptRestart() reads the fixed TRAY_SOCK_PATH (not RT_APP_SOCKET), + // so the fake tray must listen there. + trayServer = Bun.serve({ + unix: TRAY_SOCK_PATH, + fetch: () => new Response(JSON.stringify({ ok: true }), { headers: { "Content-Type": "application/json" } }), + }); + + // No real daemon socket exists yet — simulates the gap between the tray + // accepting /daemon/start and the daemon actually binding rt.sock. + bindTimer = setTimeout(() => { + daemonServer = Bun.serve({ + unix: DAEMON_SOCK_PATH, + fetch: () => new Response(JSON.stringify({ ok: true, data: {} }), { headers: { "Content-Type": "application/json" } }), + }); + }, 600); + + const result = await daemonQueryAttributed("ping", {}, 200); + expect(result.response?.ok).toBe(true); + }, 5000); +}); diff --git a/lib/__tests__/daemon-config.test.ts b/lib/__tests__/daemon-config.test.ts index 21fc45b54..32f06d15c 100644 --- a/lib/__tests__/daemon-config.test.ts +++ b/lib/__tests__/daemon-config.test.ts @@ -9,7 +9,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { mkdirSync, rmSync, writeFileSync } from "fs"; import { join } from "path"; -import { activeLaunchdLabel } from "../daemon-config.ts"; +import { activeLaunchdLabel, resolveApiPort } from "../daemon-config.ts"; const WRAPPER_PATH = join(process.env.HOME!, ".local", "bin", "rt"); @@ -28,3 +28,15 @@ describe("activeLaunchdLabel", () => { expect(activeLaunchdLabel()).toBe("com.mattstack.daemon.dev"); }); }); + +describe("resolveApiPort", () => { + test("env wins, then setting, then 9401", () => { + const prev = process.env.RT_API_PORT; + process.env.RT_API_PORT = "12345"; + expect(resolveApiPort()).toBe(12345); + delete process.env.RT_API_PORT; + expect(resolveApiPort()).toBe(9401); // default setting value + if (prev !== undefined) process.env.RT_API_PORT = prev; + else delete process.env.RT_API_PORT; + }); +}); diff --git a/lib/__tests__/daemon-logger.test.ts b/lib/__tests__/daemon-logger.test.ts index 1370be577..dd8e6b3b2 100644 --- a/lib/__tests__/daemon-logger.test.ts +++ b/lib/__tests__/daemon-logger.test.ts @@ -1,10 +1,20 @@ -import { describe, it, expect, beforeEach, afterEach } from "bun:test"; -import { mkdtempSync, rmSync, readFileSync, readdirSync, existsSync } from "node:fs"; +import { describe, it, expect, beforeEach, afterEach, mock } from "bun:test"; +import { mkdtempSync, rmSync, readFileSync, readdirSync, existsSync, writeFileSync, statSync, unlinkSync, mkdirSync, closeSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; +import { dlopen, suffix, FFIType } from "bun:ffi"; +import type { Logger } from "pino"; // We import the factory (not the singleton) so each test gets isolation. -import { createDaemonLogger, lazyChildLogger, getDaemonLogger, __test__ } from "../daemon-logger.ts"; +import { + createDaemonLogger, + lazyChildLogger, + getDaemonLogger, + installCrashHandlers, + redirectNativeStderr, + __test__, + type DaemonLoggerHandle, +} from "../daemon-logger.ts"; import { logsDir } from "../rt-paths.ts"; let logDir: string; @@ -126,6 +136,90 @@ describe("lazyChildLogger", () => { }); }); +describe("redirectNativeStderr (rotation)", () => { + // redirectNativeStderr dup2's the REAL process fd 2 to the log file (that is + // the whole point of the function); a bare call here would swallow this + // test process's own stderr for the rest of the run. Save/restore fd 2 + // around the call with the same dup/dup2 pair the implementation uses. + function withRealFd2Saved(fn: () => void): void { + const libc = dlopen(`libSystem.${suffix}`, { + dup: { args: [FFIType.i32], returns: FFIType.i32 }, + dup2: { args: [FFIType.i32, FFIType.i32], returns: FFIType.i32 }, + }); + const savedFd2 = libc.symbols.dup(2); + try { + fn(); + } finally { + libc.symbols.dup2(savedFd2, 2); + closeSync(savedFd2); + libc.close(); + } + } + + function stderrPath(): string { + return join(logsDir(), "daemon-stderr.log"); + } + + function clearRotatedFiles(): void { + const dir = logsDir(); + if (!existsSync(dir)) return; + for (const f of readdirSync(dir)) { + if (/^daemon-stderr\.\d{4}-\d{2}-\d{2}(\.\d+)?\.log$/.test(f)) unlinkSync(join(dir, f)); + } + } + + beforeEach(() => { + mkdirSync(logsDir(), { recursive: true }); + clearRotatedFiles(); + try { unlinkSync(stderrPath()); } catch { /* not present yet */ } + }); + + afterEach(() => { + clearRotatedFiles(); + try { unlinkSync(stderrPath()); } catch { /* already gone */ } + }); + + it("rotates a non-empty daemon-stderr.log before reopening, matching the janitor's dated pattern", () => { + if (process.platform !== "darwin") return; // redirectNativeStderr is a darwin-only no-op elsewhere + + writeFileSync(stderrPath(), "old panic\n"); + + withRealFd2Saved(() => redirectNativeStderr()); + + const rotated = readdirSync(logsDir()).filter((f) => /^daemon-stderr\.\d{4}-\d{2}-\d{2}\.log$/.test(f)); + expect(rotated.length).toBe(1); + expect(readFileSync(join(logsDir(), rotated[0]!), "utf8")).toBe("old panic\n"); + expect(statSync(stderrPath()).size).toBe(0); + }); + + it("dedupes with a .N suffix when the dated rotation name is already taken", () => { + if (process.platform !== "darwin") return; + + const d = new Date(); + const date = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`; + writeFileSync(join(logsDir(), `daemon-stderr.${date}.log`), "yesterday's rotation\n"); + writeFileSync(stderrPath(), "today's panic\n"); + + withRealFd2Saved(() => redirectNativeStderr()); + + const rotated = readdirSync(logsDir()).filter((f) => new RegExp(`^daemon-stderr\\.${date}(\\.\\d+)?\\.log$`).test(f)); + expect(rotated.length).toBe(2); + const suffixed = rotated.find((f) => f !== `daemon-stderr.${date}.log`); + expect(suffixed).toBeDefined(); + expect(readFileSync(join(logsDir(), suffixed!), "utf8")).toBe("today's panic\n"); + }); + + it("leaves an empty or missing daemon-stderr.log alone (nothing to rotate)", () => { + if (process.platform !== "darwin") return; + + withRealFd2Saved(() => redirectNativeStderr()); + + const rotated = readdirSync(logsDir()).filter((f) => /^daemon-stderr\.\d{4}-\d{2}-\d{2}(\.\d+)?\.log$/.test(f)); + expect(rotated.length).toBe(0); + expect(statSync(stderrPath()).size).toBe(0); + }); +}); + describe("getDaemonLogger — concurrency", () => { it("shares one in-flight initialization across concurrent callers", async () => { __test__.resetDaemonLoggerCache(); @@ -191,3 +285,33 @@ describe("lazyChildLogger — Proxy guard", () => { expect(() => (log as any).level).toThrow(); }); }); + +describe("installCrashHandlers (boot-phase gating)", () => { + it("unhandledRejection exits(1) while booting, only logs once ready", () => { + const exits: number[] = []; + const origExit = process.exit; + const origStderrWrite = process.stderr.write.bind(process.stderr); + // @ts-expect-error test stub (captures the exit code instead of terminating) + process.exit = (code?: number) => { exits.push(code ?? 0); }; + const fatal = mock(() => {}); + const error = mock(() => {}); + const logger = { fatal, error } as unknown as Logger; + let booting = true; + try { + installCrashHandlers({ logger } as DaemonLoggerHandle, { booting: () => booting }); + process.emit("unhandledRejection", new Error("boot boom"), Promise.resolve()); + expect(fatal).toHaveBeenCalledTimes(1); + expect(exits).toEqual([1]); + + booting = false; + process.emit("unhandledRejection", new Error("steady boom"), Promise.resolve()); + expect(error).toHaveBeenCalledTimes(1); + expect(exits).toEqual([1]); // no second exit + } finally { + process.exit = origExit; + process.stderr.write = origStderrWrite; + process.removeAllListeners("unhandledRejection"); + process.removeAllListeners("uncaughtException"); + } + }); +}); diff --git a/lib/__tests__/daemon-status.test.ts b/lib/__tests__/daemon-status.test.ts index 842429053..17187267f 100644 --- a/lib/__tests__/daemon-status.test.ts +++ b/lib/__tests__/daemon-status.test.ts @@ -1,9 +1,35 @@ import { describe, expect, test } from "bun:test"; import { classifyDaemonStatus } from "../daemon-status.ts"; +import type { BootFailure, LastExit, SupervisionState } from "../daemon/supervision-state.ts"; + +// `alive` (Task 1's field name for "a plain ping succeeded") is renamed +// `pingOk` here: Task 10 adds a second, distinct liveness signal (`pidAlive`, +// a raw OS-level pid check independent of rt.sock), and keeping both named +// `alive` would make call sites ambiguous about which one they mean. +function emptySupervision(): SupervisionState { + return { bootAttempts: 0, lastReadyAt: 0, recentFailures: [], lastExit: null }; +} + +function oneFailure(phase: BootFailure["phase"], reason: string): BootFailure { + return { at: Date.now(), phase, reason }; +} + +function threeRecentFailures(): BootFailure[] { + const now = Date.now(); + return [ + { at: now - 3000, phase: "api", reason: "EADDRINUSE" }, + { at: now - 2000, phase: "api", reason: "EADDRINUSE" }, + { at: now - 1000, phase: "api", reason: "EADDRINUSE" }, + ]; +} + +function bootFailedExit(reason: string): LastExit { + return { at: Date.now(), kind: "boot-failed", code: 1, reason }; +} describe("classifyDaemonStatus", () => { test("not installed short-circuits everything else", () => { - const v = classifyDaemonStatus({ installed: false, response: null, alive: false, pid: null }); + const v = classifyDaemonStatus({ installed: false, response: null, pingOk: false, pid: null }); expect(v.state).toBe("not-installed"); }); @@ -11,7 +37,7 @@ describe("classifyDaemonStatus", () => { const v = classifyDaemonStatus({ installed: true, response: { ok: true, data: { pid: 42, uptime: 1000 } }, - alive: true, + pingOk: true, pid: 42, }); expect(v.state).toBe("running"); @@ -24,7 +50,7 @@ describe("classifyDaemonStatus", () => { const v = classifyDaemonStatus({ installed: true, response: { ok: false, error: "freshness store unreadable" }, - alive: false, // never consulted: the answer itself is proof of life + pingOk: false, // never consulted: the answer itself is proof of life pid: 89290, }); expect(v.state).toBe("degraded"); @@ -37,7 +63,7 @@ describe("classifyDaemonStatus", () => { // The other half of the bug: daemonQuery returns null on a 2s timeout even // when it has already established the socket is live (daemon-client.ts:149). test("a null response with a live ping means running but unresponsive", () => { - const v = classifyDaemonStatus({ installed: true, response: null, alive: true, pid: 89290 }); + const v = classifyDaemonStatus({ installed: true, response: null, pingOk: true, pid: 89290 }); expect(v.state).toBe("degraded"); if (v.state === "degraded") { expect(v.reason).toBe("unresponsive"); @@ -46,13 +72,13 @@ describe("classifyDaemonStatus", () => { }); test("a null response and a dead ping is genuinely not running", () => { - const v = classifyDaemonStatus({ installed: true, response: null, alive: false, pid: 123 }); + const v = classifyDaemonStatus({ installed: true, response: null, pingOk: false, pid: 123 }); expect(v.state).toBe("not-running"); if (v.state === "not-running") expect(v.pid).toBe(123); }); test("not running without a recorded pid is still not running", () => { - const v = classifyDaemonStatus({ installed: true, response: null, alive: false, pid: null }); + const v = classifyDaemonStatus({ installed: true, response: null, pingOk: false, pid: null }); expect(v.state).toBe("not-running"); if (v.state === "not-running") expect(v.pid).toBeNull(); }); @@ -61,11 +87,104 @@ describe("classifyDaemonStatus", () => { const v = classifyDaemonStatus({ installed: true, response: { ok: false, error: "boom" }, - alive: false, + pingOk: false, pid: null, }); expect(v.state).toBe("degraded"); }); + + // ── Task 10: alive-not-serving / parked / crash-looping / boot-failed ── + + test("alive pid + failed ping -> alive-not-serving with breadcrumb detail", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: true, + pid: 42, + breadcrumb: { phase: "socket" }, + supervision: emptySupervision(), + }); + expect(v).toMatchObject({ state: "alive-not-serving", pid: 42, detail: "booting" }); + }); + + test("alive pid stuck after reaching ready -> wedged", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: true, + pid: 42, + breadcrumb: { phase: "ready" }, + supervision: emptySupervision(), + }); + expect(v).toMatchObject({ state: "alive-not-serving", pid: 42, detail: "wedged" }); + }); + + test("alive pid at ready with a boot-failed exit on record -> quarantined", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: true, + pid: 42, + breadcrumb: { phase: "ready" }, + supervision: { ...emptySupervision(), lastExit: bootFailedExit("events.db corrupt") }, + }); + expect(v).toMatchObject({ state: "alive-not-serving", pid: 42, detail: "quarantined" }); + }); + + test("alive pid whose breadcrumb flavor disagrees with the intended flavor -> parked", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: true, + pid: 42, + intendedFlavor: "prod", + breadcrumb: { phase: "start", flavor: "dev" }, + supervision: emptySupervision(), + }); + expect(v).toMatchObject({ state: "parked", pid: 42 }); + }); + + test("a breadcrumb with no supervision (pre-state.db failure) still classifies from the file alone", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: true, + pid: 42, + breadcrumb: { phase: "events-db" }, + }); + expect(v).toMatchObject({ state: "alive-not-serving", pid: 42, detail: "booting" }); + }); + + test("no pid + >=3 recent failures -> crash-looping", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: false, + pid: null, + supervision: { ...emptySupervision(), recentFailures: threeRecentFailures(), lastExit: bootFailedExit("EADDRINUSE") }, + }); + expect(v).toMatchObject({ state: "crash-looping" }); + if (v.state === "crash-looping") { + expect(v.failures).toBeGreaterThanOrEqual(3); + expect(v.reason).toBe("EADDRINUSE"); + } + }); + + test("no pid + single boot-failed -> boot-failed with reason and phase", () => { + const v = classifyDaemonStatus({ + installed: true, + pingOk: false, + pidAlive: false, + pid: null, + supervision: { ...emptySupervision(), recentFailures: [oneFailure("api", "EADDRINUSE")], lastExit: bootFailedExit("EADDRINUSE") }, + }); + expect(v).toMatchObject({ state: "boot-failed", reason: "EADDRINUSE", phase: "api" }); + }); + + test("no pid + no supervision at all -> plain not-running (additive: old callers unaffected)", () => { + const v = classifyDaemonStatus({ installed: true, pingOk: false, pidAlive: false, pid: null }); + expect(v.state).toBe("not-running"); + }); }); describe("needsLivenessProbe", () => { diff --git a/lib/__tests__/git-async-timeouts.test.ts b/lib/__tests__/git-async-timeouts.test.ts new file mode 100644 index 000000000..5ed9782c8 --- /dev/null +++ b/lib/__tests__/git-async-timeouts.test.ts @@ -0,0 +1,14 @@ +import { test, expect } from "bun:test"; +import { MUTATING_TIMEOUT_MS, stashChangesAsync, popStashAsync } from "../worktree/git-async.ts"; + +test("mutating timeout is 5 minutes", () => { + expect(MUTATING_TIMEOUT_MS).toBe(5 * 60_000); +}); + +test("stash helpers accept a timeout override", () => { + // Type-level: these must type-check with an opts arg. + const a: typeof stashChangesAsync = stashChangesAsync; + const b: typeof popStashAsync = popStashAsync; + expect(typeof a).toBe("function"); + expect(typeof b).toBe("function"); +}); diff --git a/lib/__tests__/git-worktree-roots-async.test.ts b/lib/__tests__/git-worktree-roots-async.test.ts new file mode 100644 index 000000000..3c933432c --- /dev/null +++ b/lib/__tests__/git-worktree-roots-async.test.ts @@ -0,0 +1,20 @@ +import { test, expect } from "bun:test"; +import { mkdtempSync } from "fs"; +import { realpathSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { listWorktreeRootsAsync } from "../worktree/git-async.ts"; +import { runGit } from "../worktree/git-async.ts"; + +test("listWorktreeRootsAsync returns the main worktree path", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-wt-"))); + await runGit(dir, ["init", "-q"]); + await runGit(dir, ["commit", "--allow-empty", "-m", "init", "-c", "user.email=a@b.c", "-c", "user.name=t"]); + const roots = await listWorktreeRootsAsync(dir); + expect(roots).toContain(dir); +}); + +test("listWorktreeRootsAsync returns [] on a non-repo", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-nonrepo-"))); + expect(await listWorktreeRootsAsync(dir)).toEqual([]); +}); diff --git a/lib/__tests__/no-daemon-sync-exec.test.ts b/lib/__tests__/no-daemon-sync-exec.test.ts new file mode 100644 index 000000000..f5d8c7532 --- /dev/null +++ b/lib/__tests__/no-daemon-sync-exec.test.ts @@ -0,0 +1,81 @@ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { dirname, resolve } from "path"; + +// Files with sync-exec that Phase 1 does NOT remove. Each entry names the +// finding/phase that will delete it, so this list shrinks as later phases land. +// A regression that reintroduces sync-exec into any OTHER daemon-reachable +// module fails this gate (the rule has been re-broken twice). +const ALLOWLIST = new Set([ + "lib/daemon/user-path.ts", // Phase 6 PATH rebuild (S013/S014/S062) + "lib/daemon/boot-reconcile.ts", // Phase 0.6 / S044 (Bun.sleepSync) + "lib/state/db.ts", // Phase 0.7 / S072-S073 busy-retry + "lib/state/busy.ts", // Phase 0.7 / S072-S073 busy-retry + "lib/repo-index.ts", // Phase 5.3 dedup (heal/derive execSync) + "lib/repo.ts", // R050 / Phase 5.4 (via handlers/system-processes.ts) + "lib/git.ts", // R050 / Phase 5.4 (via repo.ts) + "lib/herdr-launch.ts", // Phase 5 herdr (via handlers/pane.ts) + "lib/rt-render.tsx", // R050 / Phase 5.4 (daemon carries the TUI) +]); + +const SYNC_EXEC = [ + /\bexecSync\s*\(/, + /\bspawnSync\s*\(/, + /\bBun\.spawnSync\s*\(/, + /\bBun\.sleepSync\s*\(/, +]; + +const REPO_ROOT = resolve(import.meta.dir, "..", ".."); +const stripShebang = (s: string) => s.replace(/^#!.*\n/, ""); +const tsT = new Bun.Transpiler({ loader: "ts" }); +const tsxT = new Bun.Transpiler({ loader: "tsx" }); +const loaderFor = (f: string) => (f.endsWith(".tsx") || f.endsWith(".jsx") ? tsxT : tsT); + +/** Files reachable from lib/daemon.ts via relative imports (the daemon graph). */ +function daemonClosure(): string[] { + const entry = resolve(REPO_ROOT, "lib/daemon.ts"); + const visited = new Set(); + const stack = [entry]; + while (stack.length) { + const file = stack.pop()!; + if (visited.has(file)) continue; + visited.add(file); + let src: string; + try { src = stripShebang(readFileSync(file, "utf8")); } catch { continue; } + let imports: { path: string }[]; + try { imports = loaderFor(file).scanImports(src); } catch { continue; } + for (const imp of imports) { + if (!imp.path.startsWith(".")) continue; // external package + stack.push(resolve(dirname(file), imp.path)); + } + } + return [...visited]; +} + +function hasSyncExec(source: string): boolean { + return SYNC_EXEC.some((re) => re.test(source)); +} + +test("no daemon-reachable module calls sync exec (outside the allowlist)", () => { + const offenders: string[] = []; + for (const file of daemonClosure()) { + const rel = file.replace(REPO_ROOT + "/", ""); + if (ALLOWLIST.has(rel)) continue; + let src: string; + try { src = readFileSync(file, "utf8"); } catch { continue; } + if (hasSyncExec(src)) offenders.push(rel); + } + expect(offenders).toEqual([]); +}); + +test("the checker flags a reintroduced sync-exec call (proves the gate bites)", () => { + // Permanent RED proof: the matcher must catch a fresh offense. + expect(hasSyncExec(`import { execSync } from "child_process";\nexecSync("true");`)).toBe(true); + expect(hasSyncExec(`await Bun.sleepSync(10);`)).toBe(true); + expect(hasSyncExec(`// a comment mentioning execSync without a call`)).toBe(false); +}); + +test("the daemon closure actually resolves (guards against a walker that finds nothing)", () => { + const closure = daemonClosure(); + expect(closure.length).toBeGreaterThan(50); // ~151 today; a collapse means the walk broke +}); diff --git a/lib/__tests__/notifier.test.ts b/lib/__tests__/notifier.test.ts index ed044d389..f9e5e103e 100644 --- a/lib/__tests__/notifier.test.ts +++ b/lib/__tests__/notifier.test.ts @@ -280,3 +280,33 @@ describe("conflict-free streak", () => { expect(__test__.shouldRearmConflicts(conflicted, snap)).toBe(false); }); }); + +describe("removeFromQueueWithRetry (busy-swallowed post-push removal)", () => { + test("succeeds on the first attempt when the removal actually took", async () => { + let queued = true; + const removeFn = () => { queued = false; }; + const isQueuedFn = () => queued; + const ok = await __test__.removeFromQueueWithRetry("evt-1", 3, 1, removeFn, isQueuedFn); + expect(ok).toBe(true); + }); + + test("retries when the removal is busy-swallowed, then succeeds", async () => { + let queued = true; + let calls = 0; + const removeFn = () => { + calls++; + if (calls >= 3) queued = false; // the first two deletes are swallowed by SQLITE_BUSY + }; + const isQueuedFn = () => queued; + const ok = await __test__.removeFromQueueWithRetry("evt-2", 5, 1, removeFn, isQueuedFn); + expect(ok).toBe(true); + expect(calls).toBe(3); + }); + + test("gives up after the attempt budget and reports failure instead of hanging", async () => { + const removeFn = () => {}; // never actually removes it + const isQueuedFn = () => true; + const ok = await __test__.removeFromQueueWithRetry("evt-3", 3, 1, removeFn, isQueuedFn); + expect(ok).toBe(false); + }); +}); diff --git a/lib/__tests__/port-scanner.test.ts b/lib/__tests__/port-scanner.test.ts index 750c574b9..033c0adad 100644 --- a/lib/__tests__/port-scanner.test.ts +++ b/lib/__tests__/port-scanner.test.ts @@ -1,10 +1,16 @@ import { describe, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, realpathSync, rmSync, symlinkSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; import { parseListeningLsof, parsePidValueMap, parseCwdMap, parseWorktreePorcelain, parseEtimeMs, + matchCwdToRepo, + canonicalizeRepoIndex, + canonicalizeWorktreeMap, } from "../port-scanner.ts"; describe("parseListeningLsof", () => { @@ -120,3 +126,61 @@ describe("parseEtimeMs", () => { expect(parseEtimeMs("")).toBeNull(); }); }); + +// S097: lsof reports the kernel's resolved (real) cwd; the repo index and +// worktree map carry whatever path the user cd'd through when registering +// a repo. A symlinked repo root (~/code -> /Volumes/Dev/code) previously +// matched nothing at all — no ports, no runaway detection, and stale dev +// servers kept running past dispose because killWorktreeProcesses found +// nothing either. +describe("canonicalizeRepoIndex / canonicalizeWorktreeMap (S097)", () => { + let dir: string; + let real: string; + let link: string; + + const setup = () => { + dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-port-scanner-"))); + real = join(dir, "real-repo"); + link = join(dir, "linked-repo"); + mkdirSync(real, { recursive: true }); + symlinkSync(real, link); + }; + const teardown = () => { rmSync(dir, { recursive: true, force: true }); }; + + test("a symlinked repo root resolves to the real path lsof would report", () => { + setup(); + try { + const repos = canonicalizeRepoIndex({ acme: link }); + expect(repos.acme).toBe(realpathSync(real)); + } finally { + teardown(); + } + }); + + test("a nonexistent path (deleted mid-scan) falls back to the literal rather than throwing", () => { + const repos = canonicalizeRepoIndex({ acme: "/does/not/exist/anywhere" }); + expect(repos.acme).toBe("/does/not/exist/anywhere"); + }); + + test("worktree map keys are canonicalized the same way", () => { + setup(); + try { + const wt = canonicalizeWorktreeMap(new Map([[link, { repo: "acme", branch: "main" }]])); + expect([...wt.keys()]).toEqual([realpathSync(real)]); + } finally { + teardown(); + } + }); + + test("end-to-end: matchCwdToRepo finds a repo registered under a symlinked root once canonicalized", () => { + setup(); + try { + const cwd = realpathSync(real); // what lsof would report + const repos = canonicalizeRepoIndex({ acme: link }); // what the index has + const match = matchCwdToRepo(cwd, repos, new Map()); + expect(match.repo).toBe("acme"); + } finally { + teardown(); + } + }); +}); diff --git a/lib/__tests__/repo-index-async.test.ts b/lib/__tests__/repo-index-async.test.ts new file mode 100644 index 000000000..fdcc423e2 --- /dev/null +++ b/lib/__tests__/repo-index-async.test.ts @@ -0,0 +1,9 @@ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("resolveIndexPathForIdentity no longer reaches a sync git via observedMainPath", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "repo-index.ts"), "utf8"); + // observedMainPath (sync execSync) must not be called from the async resolver path. + expect(src).toMatch(/observedMainPathAsync/); +}); diff --git a/lib/__tests__/rt-paths.test.ts b/lib/__tests__/rt-paths.test.ts index 50cf38a68..665eafdbc 100644 --- a/lib/__tests__/rt-paths.test.ts +++ b/lib/__tests__/rt-paths.test.ts @@ -228,6 +228,7 @@ describe("rt-paths", () => { const home = makeHome(); process.env.HOME = home; mkdirSync(join(home, ".rt"), { recursive: true }); + writeFileSync(join(home, ".rt", "repos.json"), "{}"); // rt signature marker (S099) writeFileSync(join(home, ".rt", "old.json"), "old"); mkdirSync(join(home, ".mattstack", "rt"), { recursive: true }); writeFileSync(join(home, ".mattstack", "rt", "new.json"), "new"); @@ -241,11 +242,58 @@ describe("rt-paths", () => { const home = makeHome(); process.env.HOME = home; mkdirSync(join(home, ".rt"), { recursive: true }); + writeFileSync(join(home, ".rt", "repos.json"), "{}"); // rt signature marker (S099) expect(migrateLegacyRtDir()).toBe("migrated"); expect(migrateLegacyRtDir()).toBe("none"); rmSync(home, { recursive: true, force: true }); }); + // S099: a new user who has never run rt, but has another tool that also + // uses ~/.rt as its config dir, must not have that directory silently + // annexed as rt state (parsed, quarantined, or renamed) — only a + // directory carrying an actual rt signature is ever touched. + test("migrate: a foreign ~/.rt with no rt signature is left alone, no ~/.mattstack/rt materializes (S099)", () => { + const home = makeHome(); + process.env.HOME = home; + mkdirSync(join(home, ".rt"), { recursive: true }); + writeFileSync(join(home, ".rt", "config.toml"), "some-other-tools-config"); + const result = migrateLegacyRtDir(); + expect(result).not.toBe("migrated"); + expect(result).not.toBe("conflict"); + expect(readFileSync(join(home, ".rt", "config.toml"), "utf8")).toBe("some-other-tools-config"); + expect(existsSync(join(home, ".mattstack", "rt"))).toBe(false); + rmSync(home, { recursive: true, force: true }); + }); + + test("migrate: a foreign ~/.rt is left alone even when ~/.mattstack/rt already exists — not reported as a conflict (S099)", () => { + const home = makeHome(); + process.env.HOME = home; + mkdirSync(join(home, ".rt"), { recursive: true }); + writeFileSync(join(home, ".rt", "config.toml"), "some-other-tools-config"); + mkdirSync(join(home, ".mattstack", "rt"), { recursive: true }); + const result = migrateLegacyRtDir(); + expect(result).not.toBe("conflict"); + expect(readFileSync(join(home, ".rt", "config.toml"), "utf8")).toBe("some-other-tools-config"); + rmSync(home, { recursive: true, force: true }); + }); + + test("migrate: a bare 'logs' dir alone is a sufficient rt signature", () => { + const home = makeHome(); + process.env.HOME = home; + mkdirSync(join(home, ".rt", "logs"), { recursive: true }); + expect(migrateLegacyRtDir()).toBe("migrated"); + rmSync(home, { recursive: true, force: true }); + }); + + test("migrate: a bare 'state.db' file alone is a sufficient rt signature", () => { + const home = makeHome(); + process.env.HOME = home; + mkdirSync(join(home, ".rt"), { recursive: true }); + writeFileSync(join(home, ".rt", "state.db"), ""); + expect(migrateLegacyRtDir()).toBe("migrated"); + rmSync(home, { recursive: true, force: true }); + }); + // ── legacyDirsPresent (the canary probe) ──────────────────────────────────── test("canary: reports real legacy dirs, ignores symlinks, skips absent", () => { diff --git a/lib/__tests__/subprocess.test.ts b/lib/__tests__/subprocess.test.ts index 9390076c6..88eba03c7 100644 --- a/lib/__tests__/subprocess.test.ts +++ b/lib/__tests__/subprocess.test.ts @@ -1,4 +1,7 @@ import { describe, test, expect, afterEach } from "bun:test"; +import { unlink } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; import { runCapture, outputTail, MAX_LOGGED_OUTPUT } from "../subprocess.ts"; const SENTINEL = "RT_SPAWN_ENV_SENTINEL"; @@ -29,6 +32,66 @@ describe("runCapture env", () => { }); }); +describe("runCapture timeout enforcement", () => { + test("resolves within the deadline even when a grandchild holds the pipe", async () => { + // zsh exits after ~0.2s, but backgrounds `sleep 20` which inherits stdout. + const t0 = Date.now(); + const r = await runCapture( + ["/bin/zsh", "-c", "sleep 20 & echo started; sleep 0.2"], + { timeoutMs: 1000 }, + ); + const elapsed = Date.now() - t0; + expect(elapsed).toBeLessThan(4000); // must NOT wait for the 20s grandchild + expect(r.timedOut).toBe(true); + expect(r.exitCode).toBe(-1); + }); + + test("a SIGTERM-ignoring child is bounded by SIGKILL escalation", async () => { + // Timing alone doesn't prove the kill worked: the deadline race resolves + // runCapture on schedule regardless of whether SIGKILL ever fires. So this + // asserts the process is actually gone, via its own recorded pid. + const pidFile = path.join( + os.tmpdir(), + `rt-subprocess-test-${Date.now()}-${Math.random().toString(36).slice(2)}.pid`, + ); + try { + const t0 = Date.now(); + const r = await runCapture( + ["/bin/zsh", "-c", `echo $$ > ${pidFile}; trap '' TERM; sleep 20`], + { timeoutMs: 500 }, + ); + expect(Date.now() - t0).toBeLessThan(4000); + expect(r.timedOut).toBe(true); + + // Past the 2s SIGKILL grace, so the escalation has had time to land. + await new Promise((resolve) => setTimeout(resolve, 2500)); + const pid = Number((await Bun.file(pidFile).text()).trim()); + let alive = true; + try { + process.kill(pid, 0); + } catch { + alive = false; + } + expect(alive).toBe(false); + } finally { + await unlink(pidFile).catch(() => {}); + } + }, 8000); + + test("normal fast command still returns real stdout and exitCode 0", async () => { + const r = await runCapture(["/bin/echo", "hello"], { timeoutMs: 5000 }); + expect(r.stdout.trim()).toBe("hello"); + expect(r.exitCode).toBe(0); + expect(r.timedOut).toBeUndefined(); + }); + + test("timed-out call reports exitCode -1 so callers treat it as failure", async () => { + const r = await runCapture(["/bin/sleep", "20"], { timeoutMs: 500 }); + expect(r.exitCode).toBe(-1); + expect(r.timedOut).toBe(true); + }); +}); + describe("outputTail", () => { test("passes short output through, trimmed", () => { expect(outputTail(" env: node: No such file or directory\n", 2000)) diff --git a/lib/agent-herdr.ts b/lib/agent-herdr.ts index ec3dccee2..9cb964bad 100644 --- a/lib/agent-herdr.ts +++ b/lib/agent-herdr.ts @@ -16,6 +16,7 @@ * visible pane, rt verifies the result from real state afterward. */ +import { existsSync } from "fs"; import { homedir } from "os"; import { join } from "path"; import { runCapture } from "./subprocess.ts"; @@ -23,15 +24,36 @@ import { runCapture } from "./subprocess.ts"; export interface HerdrResult { stdout: string; exitCode: number } export type HerdrRunner = (args: string[]) => Promise; -export function defaultHerdrRunner(): HerdrRunner { - const home = process.env.HOME ?? homedir(); - const bin = process.env.HERDR_BIN ?? join(home, ".local", "bin", "herdr"); - const socket = process.env.HERDR_SOCKET_PATH ?? join(home, ".config", "herdr", "herdr.sock"); +/** + * Mirrors lib/cswap.ts's cswapBin(): Bun.which reads process.env.PATH at + * call time, and the daemon overlays the user's full login PATH onto + * process.env.PATH at boot (lib/daemon.ts resolveUserPath), so this resolves + * a brew-installed herdr even though the daemon's start-env PATH does not + * carry it. The ~/.local/bin fallback preserves the vendor-script install. + */ +export function resolveHerdrBin( + env: NodeJS.ProcessEnv = process.env, + which: (cmd: string) => string | null = (cmd) => Bun.which(cmd), +): string { + if (env.HERDR_BIN) return env.HERDR_BIN; + const onPath = which("herdr"); + if (onPath) return onPath; + const home = env.HOME ?? homedir(); + return join(home, ".local", "bin", "herdr"); +} + +export function defaultHerdrRunner(env: NodeJS.ProcessEnv = process.env): HerdrRunner { + const home = env.HOME ?? homedir(); + const bin = resolveHerdrBin(env); + const socket = env.HERDR_SOCKET_PATH ?? join(home, ".config", "herdr", "herdr.sock"); return async (args) => { + if (!existsSync(bin)) { + throw new Error(`herdr not found at ${bin} (install via \`rt setup\` / brew)`); + } const r = await runCapture([bin, ...args], { timeoutMs: 15_000, stderr: "pipe", - env: { ...process.env, HERDR_SOCKET_PATH: socket }, + env: { ...env, HERDR_SOCKET_PATH: socket }, }); return { stdout: r.stdout || r.stderr, exitCode: r.exitCode }; }; diff --git a/lib/command-tree-def.ts b/lib/command-tree-def.ts index b28c6ded6..648a4388e 100644 --- a/lib/command-tree-def.ts +++ b/lib/command-tree-def.ts @@ -640,7 +640,9 @@ export const TREE: Record = { description: "Show daemon status", module: "./commands/daemon.ts", fn: "showStatus", - args: [], + args: [ + { name: "JSON", flag: "--json", type: "boolean", default: false, hint: "Emit the verdict as JSON instead of the formatted lines" }, + ], }, track: { description: "Per-repo background tracking (live/poll/off)", diff --git a/lib/daemon-client.ts b/lib/daemon-client.ts index 2a03a629c..da89c789c 100644 --- a/lib/daemon-client.ts +++ b/lib/daemon-client.ts @@ -30,19 +30,32 @@ export interface DaemonResponse { const REQUEST_TIMEOUT_MS = 2000; +// Deprecated shim state (see lastQueryTimedOut's doc): kept in sync so an +// external caller still reading it via the module-level accessor keeps +// working, but nothing in this file reads these two vars anymore — every +// internal caller carries its own attribution on its own return value +// instead, which a concurrent query can never clobber. let _lastQueryWasRefused = false; let _lastQueryTimedOut = false; +interface SocketQueryAttempt { + response: DaemonResponse | null; + /** True on ECONNREFUSED — the socket is stale, not merely slow. */ + refused: boolean; + /** True when the request itself exceeded timeoutMs. */ + timedOut: boolean; +} + async function trySocketQuery( cmd: string, payload?: Record, timeoutMs: number = REQUEST_TIMEOUT_MS, -): Promise { - // Reset per-query flags up front so a missing-socket early return doesn't - // leave a previous query's timeout/refused state visible to callers. - _lastQueryWasRefused = false; - _lastQueryTimedOut = false; - if (!existsSync(DAEMON_SOCK_PATH)) return null; +): Promise { + if (!existsSync(DAEMON_SOCK_PATH)) { + _lastQueryWasRefused = false; + _lastQueryTimedOut = false; + return { response: null, refused: false, timedOut: false }; + } try { const hasBody = payload && Object.keys(payload).length > 0; @@ -57,14 +70,16 @@ async function trySocketQuery( _lastQueryWasRefused = false; _lastQueryTimedOut = false; - return (await response.json()) as DaemonResponse; + return { response: (await response.json()) as DaemonResponse, refused: false, timedOut: false }; } catch (err) { const code = (err as any)?.code ?? ""; const name = (err as any)?.name ?? ""; const msg = err instanceof Error ? err.message : ""; - _lastQueryWasRefused = code === "ECONNREFUSED" || msg.includes("ECONNREFUSED") || msg.includes("Connection refused"); - _lastQueryTimedOut = name === "TimeoutError" || name === "AbortError" || msg.includes("timed out"); - return null; + const refused = code === "ECONNREFUSED" || msg.includes("ECONNREFUSED") || msg.includes("Connection refused"); + const timedOut = name === "TimeoutError" || name === "AbortError" || msg.includes("timed out"); + _lastQueryWasRefused = refused; + _lastQueryTimedOut = timedOut; + return { response: null, refused, timedOut }; } } @@ -102,7 +117,7 @@ export async function daemonSocketQuery( payload?: Record, timeoutMs?: number, ): Promise { - return trySocketQuery(cmd, payload, timeoutMs); + return (await trySocketQuery(cmd, payload, timeoutMs)).response; } // ─── Tray request client (MAT-383 setup verbs) ─────────────────────────────── @@ -178,12 +193,48 @@ async function attemptRestart(): Promise { // null response (tray socket absent / request failed) means no restart // actually happened. const res = await trayQuery("/daemon/start", "POST"); - return res !== null; + if (res === null) return false; + + // The tray ack only proves the request was received, not that the + // daemon actually came up, so re-probe rt.sock before reporting success, + // so daemonQuery's caller isn't told "restarted" while the daemon is + // still mid-boot and then misdirected into warnDaemonDown() on the very + // next query instead of actually waiting for it. + for (let i = 0; i < 12; i++) { + await Bun.sleep(250); + if (await isDaemonRunning()) return true; + } + return false; } catch { return false; } } +const SOCKET_POLL_TOTAL_MS = 3_000; +const SOCKET_POLL_INTERVAL_MS = 150; + +/** + * Polls for rt.sock to exist and answer for up to ~3s after a restart + * request. parkUntilIntended's own socket probe, state.db open, and the + * identity migration routinely take longer than a single fixed delay, so a + * start that genuinely succeeds must not be reported as "installed but not + * running" just because the retry landed too early. + */ +async function waitForSocket( + totalMs: number = SOCKET_POLL_TOTAL_MS, + intervalMs: number = SOCKET_POLL_INTERVAL_MS, +): Promise { + const deadline = Date.now() + totalMs; + while (Date.now() < deadline) { + if (existsSync(DAEMON_SOCK_PATH)) { + const ping = await trySocketQuery("ping", {}, Math.min(intervalMs * 2, 1000)); + if (ping.response !== null) return true; + } + await Bun.sleep(intervalMs); + } + return existsSync(DAEMON_SOCK_PATH); +} + function warnDaemonDown(): void { if (hasWarnedThisSession || _warningSuppressed) return; hasWarnedThisSession = true; @@ -205,48 +256,71 @@ export function suppressDaemonDownWarning(): void { // ─── Public API ────────────────────────────────────────────────────────────── /** - * Send a command to the daemon and return the response. - * - * Returns null if daemon is not available (either not installed or not running - * and can't be auto-restarted). Callers should fall back to direct execution. + * Same contract as `daemonQuery`, but the failure kind travels on the + * return value instead of the module-level `_lastQuery*` flags — so a + * concurrent query (a 30s mr:action merge racing a 2s status poll, say) + * can never clobber this call's own attribution between its trySocketQuery + * resolving and its caller reading it. Prefer this over + * `daemonQuery` + `lastQueryTimedOut()` for any new caller. */ -export async function daemonQuery( +export async function daemonQueryAttributed( cmd: string, payload?: Record, timeoutMs?: number, -): Promise { +): Promise { // 1. Try HTTP request over Unix socket - const result = await trySocketQuery(cmd, payload, timeoutMs); - if (result !== null) return result; + const first = await trySocketQuery(cmd, payload, timeoutMs); + if (first.response !== null) return first; // 2. Check if user opted in - if (!isDaemonInstalled()) return null; // not installed → silent fallback + if (!isDaemonInstalled()) return { response: null, refused: false, timedOut: false }; // not installed → silent fallback // 3. If the socket file still exists AND the connection wasn't refused, // the daemon IS running — this query timed out or hit a transient error. // Return null silently. But if the connection was refused, the socket is // stale (daemon died without cleaning up) — fall through to attempt restart. - if (existsSync(DAEMON_SOCK_PATH) && !_lastQueryWasRefused) return null; + if (existsSync(DAEMON_SOCK_PATH) && !first.refused) return first; // 4. Socket is gone → daemon is genuinely not running. Attempt restart. const restarted = await attemptRestart(); + let last = first; if (restarted) { - // Retry once after short delay - await Bun.sleep(300); - const retryResult = await trySocketQuery(cmd, payload, timeoutMs); - if (retryResult !== null) return retryResult; + // Bounded poll, not one fixed-delay retry: parkUntilIntended's own + // socket probe, state.db open, and the identity migration routinely + // take longer than 300ms, and a successful start must not be reported + // as "installed but not running" just because the retry landed early. + await waitForSocket(); + const retry = await trySocketQuery(cmd, payload, timeoutMs); + if (retry.response !== null) return retry; + last = retry; } // 5. Restart failed → warn (once per session) warnDaemonDown(); - return null; + return last; +} + +/** + * Send a command to the daemon and return the response. + * + * Returns null if daemon is not available (either not installed or not running + * and can't be auto-restarted). Callers should fall back to direct execution. + */ +export async function daemonQuery( + cmd: string, + payload?: Record, + timeoutMs?: number, +): Promise { + return (await daemonQueryAttributed(cmd, payload, timeoutMs)).response; } /** * True if the last `daemonQuery` returned null because the request timed out - * (as opposed to the daemon being genuinely down). Used by action callers so - * they can surface "timed out — verify on GitLab" instead of the misleading - * "daemon unavailable". + * (as opposed to the daemon being genuinely down). Deprecated: still backed + * by shared module state, so it remains vulnerable to the exact + * cross-query attribution race `daemonQueryAttributed` closes — kept only + * for callers that predate that function. New callers should use + * `daemonQueryAttributed` and read `.timedOut` off their own result. */ export function lastQueryTimedOut(): boolean { return _lastQueryTimedOut; @@ -256,7 +330,7 @@ export function lastQueryTimedOut(): boolean { * Quick check: is the daemon reachable right now? */ export async function isDaemonRunning(): Promise { - const response = await trySocketQuery("ping"); + const { response } = await trySocketQuery("ping"); return response?.ok === true; } @@ -294,8 +368,8 @@ const MR_ACTION_TIMEOUT_MS = 30_000; export function mrActions(repoName: string, iid: number): DaemonMRActions { const fire = async (action: string, args: any[] = []): Promise => { - const res = await daemonQuery("mr:action", { repoName, iid, action, args }, MR_ACTION_TIMEOUT_MS); - if (!res) throw new Error(lastQueryTimedOut() ? `${action} timed out — verify on GitLab` : "daemon unavailable"); + const { response: res, timedOut } = await daemonQueryAttributed("mr:action", { repoName, iid, action, args }, MR_ACTION_TIMEOUT_MS); + if (!res) throw new Error(timedOut ? `${action} timed out — verify on GitLab` : "daemon unavailable"); if (!res.ok) throw new Error(res.error || `${action} failed`); }; @@ -312,14 +386,14 @@ export function mrActions(repoName: string, iid: number): DaemonMRActions { requestReReview: (uid) => fire("requestReReview", [uid]), fetchJobDetail: async (jobId, pipelineId) => { - const res = await daemonQuery("mr:fetch-job-detail", { repoName, iid, jobId, pipelineId }, MR_ACTION_TIMEOUT_MS); - if (!res) throw new Error(lastQueryTimedOut() ? "fetchJobDetail timed out" : "daemon unavailable"); + const { response: res, timedOut } = await daemonQueryAttributed("mr:fetch-job-detail", { repoName, iid, jobId, pipelineId }, MR_ACTION_TIMEOUT_MS); + if (!res) throw new Error(timedOut ? "fetchJobDetail timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "fetchJobDetail failed"); return res.data; }, fetchJobTrace: async (jobId) => { - const res = await daemonQuery("mr:fetch-job-trace", { repoName, iid, jobId }, MR_ACTION_TIMEOUT_MS); - if (!res) throw new Error(lastQueryTimedOut() ? "fetchJobTrace timed out" : "daemon unavailable"); + const { response: res, timedOut } = await daemonQueryAttributed("mr:fetch-job-trace", { repoName, iid, jobId }, MR_ACTION_TIMEOUT_MS); + if (!res) throw new Error(timedOut ? "fetchJobTrace timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "fetchJobTrace failed"); return res.data as string; }, @@ -352,12 +426,12 @@ export async function fetchDiscussions( iid: number, opts?: { force?: boolean }, ): Promise { - const res = await daemonQuery( + const { response: res, timedOut } = await daemonQueryAttributed( "discussions:read", { repoName, iid, force: opts?.force === true }, DISCUSSIONS_TIMEOUT_MS, ); - if (!res) throw new Error(lastQueryTimedOut() ? "discussions timed out" : "daemon unavailable"); + if (!res) throw new Error(timedOut ? "discussions timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "discussions:read failed"); return res.data as DiscussionsSnapshot; } @@ -369,12 +443,12 @@ export async function setDiscussionResolved( discussionId: string, resolved: boolean, ): Promise { - const res = await daemonQuery( + const { response: res, timedOut } = await daemonQueryAttributed( "discussions:resolve", { repoName, iid, discussionId, resolved }, DISCUSSIONS_TIMEOUT_MS, ); - if (!res) throw new Error(lastQueryTimedOut() ? "resolve timed out" : "daemon unavailable"); + if (!res) throw new Error(timedOut ? "resolve timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "discussions:resolve failed"); return res.data as DiscussionsSnapshot; } @@ -384,12 +458,12 @@ export async function fetchMRDiffs( repoName: string, iid: number, ): Promise> { - const res = await daemonQuery( + const { response: res, timedOut } = await daemonQueryAttributed( "discussions:diffs", { repoName, iid }, DISCUSSIONS_TIMEOUT_MS, ); - if (!res) throw new Error(lastQueryTimedOut() ? "diffs timed out" : "daemon unavailable"); + if (!res) throw new Error(timedOut ? "diffs timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "discussions:diffs failed"); return (res.data as { diffs: Array<{ newPath: string; diff: string }> }).diffs; } @@ -401,12 +475,12 @@ export async function replyToDiscussion( discussionId: string, body: string, ): Promise { - const res = await daemonQuery( + const { response: res, timedOut } = await daemonQueryAttributed( "discussions:reply", { repoName, iid, discussionId, body }, DISCUSSIONS_TIMEOUT_MS, ); - if (!res) throw new Error(lastQueryTimedOut() ? "reply timed out" : "daemon unavailable"); + if (!res) throw new Error(timedOut ? "reply timed out" : "daemon unavailable"); if (!res.ok) throw new Error(res.error || "discussions:reply failed"); return res.data as DiscussionsSnapshot; } diff --git a/lib/daemon-config.ts b/lib/daemon-config.ts index 4164f10e2..6396a3a3f 100644 --- a/lib/daemon-config.ts +++ b/lib/daemon-config.ts @@ -12,6 +12,7 @@ import { homedir } from "os"; import { dirname, join } from "path"; import { rtDir } from "./rt-paths.ts"; import { currentMode } from "./dev-mode.ts"; +import { getSetting } from "./settings/resolve.ts"; // ─── Types ─────────────────────────────────────────────────────────────────── @@ -71,6 +72,22 @@ export const TRAY_SOCK_PATH = join(RT_DIR, "tray.sock"); * daemon's hardcoded port (RT-45). */ export const API_PORT = Number(process.env.RT_API_PORT) || 9401; +/** + * Call-time API port resolution: RT_API_PORT env wins (e2e isolation, RT-45), + * then the rt.apiPort setting (escape hatch when 9401 is held), then 9401. + * A function, not a const: it must never be evaluated at module load, since + * getSetting() reads the settings stores off ambient HOME. + */ +export function resolveApiPort(): number { + const env = Number(process.env.RT_API_PORT); + if (env) return env; + try { + return getSetting("rt.apiPort").value || 9401; + } catch { + return 9401; + } +} + // ─── Read / Write ──────────────────────────────────────────────────────────── // `home`, when passed, overrides the module-load `DAEMON_CONFIG_PATH` const diff --git a/lib/daemon-logger.ts b/lib/daemon-logger.ts index ecca0df9a..634e84b05 100644 --- a/lib/daemon-logger.ts +++ b/lib/daemon-logger.ts @@ -18,7 +18,7 @@ import pino, { type Logger } from "pino"; // @ts-ignore — no types shipped; the JS API is well-tested. import roll from "pino-roll"; import { dlopen, suffix, FFIType } from "bun:ffi"; -import { mkdirSync, openSync, closeSync } from "fs"; +import { mkdirSync, openSync, closeSync, existsSync, statSync, renameSync } from "fs"; import { join } from "path"; import { logsDir } from "./rt-paths.ts"; @@ -192,6 +192,26 @@ export const __test__ = { // ─── Native stderr capture ─────────────────────────────────────────────────── +/** Local `yyyy-MM-dd`, matching the janitor's dated-file convention (lib/cli-logger.ts's `today()`). */ +function todayDate(): string { + const d = new Date(); + return `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, "0")}-${String(d.getDate()).padStart(2, "0")}`; +} + +/** + * Picks the rotation target for `daemon-stderr.log`: `daemon-stderr..log`, + * or `..log` if that name is already taken (e.g. two boots same day); both + * shapes match log-janitor's LOG_FILE_PATTERN, so pruneLogs sweeps them for free. + */ +function nextRotatedStderrPath(dir: string, date: string): string { + const base = join(dir, `daemon-stderr.${date}.log`); + if (!existsSync(base)) return base; + for (let n = 1; ; n++) { + const candidate = join(dir, `daemon-stderr.${date}.${n}.log`); + if (!existsSync(candidate)) return candidate; + } +} + /** * Point fd 2 at ~/.mattstack/rt/logs/daemon-stderr.log so native output that bypasses * JS entirely (bun panics, segfault reports, runtime asserts) is captured no @@ -206,7 +226,14 @@ export function redirectNativeStderr(): void { try { const dir = logsDir(); mkdirSync(dir, { recursive: true }); - const fd = openSync(join(dir, "daemon-stderr.log"), "a"); + const stderrPath = join(dir, "daemon-stderr.log"); + // Rotate any leftover content from a previous crash before reopening, + // otherwise `rt daemon logs` keeps showing yesterday's panic as "most + // recent". A rename here can never lose data (unlike truncation). + if (existsSync(stderrPath) && statSync(stderrPath).size > 0) { + renameSync(stderrPath, nextRotatedStderrPath(dir, todayDate())); + } + const fd = openSync(stderrPath, "a"); const libc = dlopen(`libSystem.${suffix}`, { dup2: { args: [FFIType.i32, FFIType.i32], returns: FFIType.i32 }, }); @@ -225,11 +252,18 @@ export function redirectNativeStderr(): void { * logger before exit. Call ONCE during daemon startup, AFTER logger init. * * - uncaughtException: log as fatal (sync), exit 1 - * - unhandledRejection: log as error, do NOT exit (let the daemon recover) + * - unhandledRejection: fatal + exit 1 while `opts.booting()` is true (no + * socket/API bound yet, nothing worth staying up for); log as error and + * stay alive once booted, so a stray steady-state rejection doesn't kill a + * daemon that's already serving. No `booting` given preserves the old + * always-log, never-exit behavior. * - process.stderr.write: intercept so console.error / anything writing to * stderr lands in the JSON log instead of vanishing. */ -export function installCrashHandlers(handle: DaemonLoggerHandle): void { +export function installCrashHandlers( + handle: DaemonLoggerHandle, + opts: { booting?: () => boolean } = {}, +): void { const { logger } = handle; // Because the pino-roll stream is opened with sync:true, logger.fatal() @@ -240,6 +274,11 @@ export function installCrashHandlers(handle: DaemonLoggerHandle): void { }); process.on("unhandledRejection", (reason) => { + if (opts.booting?.()) { + logger.fatal({ err: reason }, "unhandledRejection during boot"); + process.exit(1); + return; + } logger.error({ err: reason }, "unhandledRejection"); }); diff --git a/lib/daemon-status.ts b/lib/daemon-status.ts index c550985c6..2cdbbcfc2 100644 --- a/lib/daemon-status.ts +++ b/lib/daemon-status.ts @@ -7,29 +7,98 @@ * attempting a restart). A caller that reads "no usable response" as "the * daemon is down" then tells the user to start a daemon that is already * running. This maps the raw outcome onto what is actually known. + * + * Below that, a second tier answers a harder question: rt.sock isn't + * answering at all, but is the daemon actually down, or alive-and-stuck? + * `pidAlive`/`breadcrumb`/`supervision` (Task 9's supervision-state.ts) are + * the only signals that can tell, and per Ruling P1 (2026-08-28 p0-supervision + * ledger) the breadcrumb FILE is the sole record a pre-state.db boot failure + * leaves. `supervision` (the kv tier) can be absent even when `breadcrumb` + * is present, and classification must still resolve to something useful from + * the breadcrumb alone. */ import type { DaemonResponse } from "./daemon-client.ts"; +import { isCrashLooping, type BootPhase, type SupervisionState } from "./daemon/supervision-state.ts"; export type DaemonStatusVerdict = | { state: "not-installed" } | { state: "running"; data: any } /** Up — proven by an answer or a ping — but `status` itself did not deliver. */ | { state: "degraded"; reason: "error" | "unresponsive"; detail?: string; pid: number | null } + /** Ping fails, a live pid exists, and it's parked waiting for a different + * flavor to hold rt.sock (park.ts): a flavor standoff, not a stuck boot. */ + | { state: "parked"; pid: number; holderFlavor?: string } + /** Ping fails but the pid is alive: still mid-boot, stuck after reaching + * ready, or alive-but-quarantined (recovered from a corrupt db). */ + | { state: "alive-not-serving"; pid: number; detail: "booting" | "wedged" | "quarantined" } + /** No live pid, and the kv failure record shows >= N failures within the window. */ + | { state: "crash-looping"; failures: number; reason: string } + /** No live pid, and the most recent recorded exit was a boot throw (fewer than N failures). */ + | { state: "boot-failed"; reason: string; phase: string } | { state: "not-running"; pid: number | null }; +/** The boot breadcrumb (`daemon-boot.json`), as classifyDaemonStatus needs it. Not + * imported from supervision-state.ts, since that module's `Breadcrumb` interface is + * intentionally unexported, and this shape only needs to be structurally + * compatible with it. */ +export interface DaemonBreadcrumbInput { + phase: BootPhase; + flavor?: "dev" | "prod"; + pid?: number; + at?: number; +} + export interface DaemonStatusInputs { installed: boolean; - /** The `status` reply, or null if the transport gave up. */ - response: DaemonResponse | null; - /** Result of a `ping` probe. Only meaningful when `response` is null. */ - alive: boolean; + /** The `status` reply, or null/absent if the transport gave up. */ + response?: DaemonResponse | null; + /** Result of a `ping` probe. Only meaningful when `response` is absent. */ + pingOk?: boolean; /** Last recorded pid, for the operator to act on. */ pid: number | null; + /** Raw OS-level liveness of `pid` (process.kill(pid,0), or a pgrep-found + * stand-in), independent of rt.sock. Only worth gathering once `pingOk` + * has already come back false; see `classifyDaemonStatus.needsPidProbe`. */ + pidAlive?: boolean; + /** This machine's currently-intended flavor (`resolveIntendedMode().mode`). + * A live pid whose own breadcrumb flavor disagrees with this is parked + * (park.ts), not stuck: the same signal `parkUntilIntended` itself acts on. */ + intendedFlavor?: "dev" | "prod"; + /** The socket holder's flavor, when the caller managed to learn it (best + * effort: probing rt.sock again after a failed ping/status round rarely + * succeeds, since a parked pid never binds it). Display-only. */ + holderFlavor?: string | null; + breadcrumb?: DaemonBreadcrumbInput | null; + /** Task 9's kv tier. Can be absent even when `breadcrumb` is present: a + * pre-state.db failure leaves only the breadcrumb file (Ruling P1). */ + supervision?: SupervisionState; + /** Injected for deterministic crash-loop window checks under test; defaults to Date.now(). */ + now?: number; +} + +const PHASE_ORDER: BootPhase[] = ["start", "events-db", "state-db", "api", "socket", "ready"]; + +function classifyAliveNotServingDetail( + breadcrumb: DaemonBreadcrumbInput | null | undefined, + supervision: SupervisionState | undefined, +): "booting" | "wedged" | "quarantined" { + const phase = breadcrumb?.phase; + if (!phase || PHASE_ORDER.indexOf(phase) < PHASE_ORDER.indexOf("ready")) return "booting"; + // Reached ready this run, but a prior attempt is on record as boot-failed, + // most likely a corrupt-db quarantine (lib/state/db.ts, events-bus.ts) it + // recovered from and is now stuck behind for an unrelated reason. + if (supervision?.lastExit?.kind === "boot-failed") return "quarantined"; + return "wedged"; +} + +function countRecentFailures(supervision: SupervisionState, now: number, windowMs = 5 * 60_000): number { + const floor = now - windowMs; + return supervision.recentFailures.filter((f) => f.at > floor).length; } export function classifyDaemonStatus(opts: DaemonStatusInputs): DaemonStatusVerdict { - const { installed, response, alive, pid } = opts; + const { installed, response, pingOk, pid, pidAlive, intendedFlavor, holderFlavor, breadcrumb, supervision } = opts; if (!installed) return { state: "not-installed" }; @@ -42,9 +111,33 @@ export function classifyDaemonStatus(opts: DaemonStatusInputs): DaemonStatusVerd return { state: "degraded", reason: "error", detail: response.error, pid }; } - // No reply. Ping is the only ground truth left: a daemon busy enough to blow - // the status timeout still answers a trivial ping. - if (alive) return { state: "degraded", reason: "unresponsive", pid }; + // No reply. A plain ping is the next ground truth: a daemon busy enough to + // blow the status timeout still answers a trivial ping. + if (pingOk) return { state: "degraded", reason: "unresponsive", pid }; + + // Ping failed too. From here, only pidAlive/breadcrumb/supervision (new + // signals) can say more than "not running"; absent them, fall straight + // through to the pre-existing not-running verdict. + if (pidAlive && pid !== null) { + if (breadcrumb?.flavor && intendedFlavor && breadcrumb.flavor !== intendedFlavor) { + return { state: "parked", pid, ...(holderFlavor ? { holderFlavor } : {}) }; + } + return { state: "alive-not-serving", pid, detail: classifyAliveNotServingDetail(breadcrumb, supervision) }; + } + + if (supervision) { + const now = opts.now ?? Date.now(); + if (isCrashLooping(supervision, now)) { + const reason = supervision.lastExit?.kind === "boot-failed" + ? supervision.lastExit.reason + : (supervision.recentFailures.at(-1)?.reason ?? "unknown"); + return { state: "crash-looping", failures: countRecentFailures(supervision, now), reason }; + } + if (supervision.lastExit?.kind === "boot-failed") { + const phase = supervision.recentFailures.at(-1)?.phase ?? "unknown"; + return { state: "boot-failed", reason: supervision.lastExit.reason, phase }; + } + } return { state: "not-running", pid }; } @@ -55,3 +148,10 @@ export function classifyDaemonStatus(opts: DaemonStatusInputs): DaemonStatusVerd */ classifyDaemonStatus.needsLivenessProbe = (response: DaemonResponse | null): boolean => response === null; + +/** + * Whether the pid/breadcrumb/supervision probes are worth paying for: only + * once both `status` and a plain `ping` have failed to answer. + */ +classifyDaemonStatus.needsPidProbe = (response: DaemonResponse | null, pingOk: boolean): boolean => + response === null && !pingOk; diff --git a/lib/daemon.ts b/lib/daemon.ts index bffeacf08..e12db4fb2 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -50,7 +50,7 @@ import { runBootIdentityMigration } from "./daemon/boot-migrate.ts"; import { runCapture } from "./subprocess.ts"; import { buildRoutedHandlers } from "./daemon/command-router.ts"; import { startSocketServer } from "./daemon/socket-server.ts"; -import { startApiServer, broadcast } from "./daemon/api-server.ts"; +import { startApiServer, withApiPortParkRetry, broadcast } from "./daemon/api-server.ts"; import { loadCronConfig, startCron } from "./daemon/cron.ts"; import { startPollers } from "./daemon/pollers.ts"; import { startHomeSnapshot } from "./daemon/home-snapshot.ts"; @@ -63,6 +63,15 @@ import { import { startDiscussionsPoller } from "./daemon/discussions-poller.ts"; import { createCleanup, installSignalHandlers } from "./daemon/shutdown.ts"; import { createEventsBus } from "./daemon/events-bus.ts"; +import { + writeBreadcrumb, + recordBootAttempt, + recordDaemonReady, + recordBootFailure, + recordCleanExit, + type BootPhase, +} from "./daemon/supervision-state.ts"; +import { safeInterval, safeTimeout } from "./daemon/safe-timers.ts"; import { pruneRuns } from "./runs/prune.ts"; import { pruneLogs } from "./log-janitor.ts"; import { getSetting } from "./settings/resolve.ts"; @@ -75,8 +84,33 @@ import type { PortEntry } from "./port-scanner.ts"; // a clean rename of a real legacy tree into a conflict. Idempotent — the CLI // entry (cli.ts) also runs it, but `bun run lib/daemon.ts` skips cli.ts. import { migrateLegacyRtDir, LEGACY_RT_LABEL, RT_DIR_LABEL, logsDir } from "./rt-paths.ts"; + +// Gates installCrashHandlers' unhandledRejection handler: fatal during boot +// (no socket/API bound yet, nothing to recover), advisory-only once ready. +let bootPhase: "booting" | "ready" = "booting"; + +// Tracks the finer-grained boot phase for the breadcrumb file and for +// attributing a Task-2 fatal boot error to the phase it happened in. +// setPhase is db-free (writeBreadcrumb only writes a file), so it is safe +// to call from module scope, before state.db exists. +let currentPhase: BootPhase = "start"; +function setPhase(phase: BootPhase): void { + currentPhase = phase; + writeBreadcrumb(phase); +} + const rtMigration = migrateLegacyRtDir(); +// Capture native panics (bypass JS entirely) at the fd level, so a throw +// during any later module-scope construction (createEventsBus, cron, +// home-snapshot, …) lands in daemon-stderr.log instead of vanishing down +// whatever fd 2 the launcher gave us. Depends only on logsDir() and mkdirs +// its own dir (this MUST run after migrateLegacyRtDir(): mkdirSync(logsDir()) +// creates the new rt dir, and migrateLegacyRtDir() treats that dir merely +// existing as a "conflict" with a real legacy tree, so redirecting first +// would defeat the migration). +redirectNativeStderr(); + // ─── Logging ───────────────────────────────────────────────────────────────── // Pino-backed structured logger. See lib/daemon-logger.ts. Top-level await // initializes the singleton before any other startup code runs, so `log` is @@ -85,6 +119,14 @@ const rtMigration = migrateLegacyRtDir(); const loggerHandle = await getDaemonLogger(); const log = loggerHandle.logger; +// Wire uncaughtException + unhandledRejection through pino as early as the +// logger allows: every module-scope side effect below this point +// (createEventsBus, cron, home-snapshot, sweep timers) can throw, and this +// must run BEFORE any of it does. +installCrashHandlers(loggerHandle, { booting: () => bootPhase === "booting" }); + +setPhase("start"); + if (rtMigration === "migrated") { log.info(`migrated legacy ${LEGACY_RT_LABEL} state to ${RT_DIR_LABEL}`); } else if (rtMigration === "conflict") { @@ -189,11 +231,14 @@ const hooksGuard = createHooksGuard(log); // Pane-communication events bus (RT-44): SQLite journal + in-memory waiters. const eventsBus = createEventsBus({ dbPath: join(RT_DIR, "events.db"), log }); +setPhase("events-db"); // Hourly retention sweep — cheap; rides its own interval rather than pollers -// because it needs no poller deps. -setInterval(() => eventsBus.sweep(), 60 * 60 * 1000); +// because it needs no poller deps. safeInterval/safeTimeout: a sync sqlite +// throw here (e.g. SQLITE_FULL) must warn, not become an uncaughtException +// that exits the daemon. +safeInterval(() => eventsBus.sweep(), 60 * 60 * 1000, "events-sweep", log); // Boot-time sweep to handle frequent daemon restarts that would otherwise starve the hourly interval. -setTimeout(() => eventsBus.sweep(), 30_000); +safeTimeout(() => eventsBus.sweep(), 30_000, "events-sweep-boot", log); // Age-floor prune of pipeline run dirs — daily; assertPrunable in prune.ts // guards every deletion against the runs root. @@ -326,6 +371,11 @@ const freshnessEnv: FreshnessEnv = { ctx: handlerCtx, broadcast: emit }; // branch-cache facade above). let routedHandlers: ReturnType | undefined; +// Set by the `shutdown` verb before it exits, so a bare OS signal arriving +// mid-teardown is still distinguishable from the intentional stop (exit-code +// policy: docs/daemon-supervision-design.md). +let shuttingDownViaVerb = false; + async function handleCommand(cmd: string, payload: any, signal?: AbortSignal): Promise { const t0 = Date.now(); try { @@ -349,10 +399,16 @@ async function routeCommand(cmd: string, payload: any, signal?: AbortSignal): Pr switch (cmd) { case "shutdown": log.info("received shutdown command"); + // Set before the delay, not inside the setTimeout callback: a bare + // SIGTERM arriving in the 100ms window must see this flag already + // true, or the signal handler treats an intentional stop as a crash + // (exit 1, launchd respawns). + shuttingDownViaVerb = true; // Delay cleanup so this response can be written first — cleanup() // force-closes all in-flight connections, including the one that // carried the shutdown request. setTimeout(() => { + recordCleanExit("shutdown", 0); cleanup(); loggerHandle.flush?.(); process.exit(0); @@ -383,151 +439,162 @@ const cleanup = (): void => { // ─── Entry ─────────────────────────────────────────────────────────────────── async function runDaemon(): Promise { - mkdirSync(RT_DIR, { recursive: true }); - - // Capture native panics (bypass JS entirely) at the fd level, then wire - // uncaughtException + unhandledRejection through pino. Must run BEFORE - // any async work that could throw uncaught. - redirectNativeStderr(); - installCrashHandlers(loggerHandle); - - // If a previous daemon process is still alive (orphan from a failed - // restart), evict it before we bind the socket. - evictStaleDaemon(log); - - // Auto-unlink any tagged tool link whose tool now has a genuine user copy - // elsewhere on PATH (e.g. the user ran `brew install gh` after rt linked - // the bundled one). reconcile() itself is synchronous (a ~/.local/bin - // readDir plus a handful of stats) — wrapping the call in `async` alone - // would NOT defer it, since nothing inside actually awaits. setTimeout(0) - // is what actually pushes it past the rest of this function: the PID - // write, openBranchCacheStore, and both server binds below all run first, - // on this same synchronous pass, before the timer callback ever fires. - setTimeout(() => { - try { - const { removed } = reconcileLinks(createRealProbes()); - if (removed.length > 0) log.info({ removed }, "deps: auto-unlinked tools now shadowed by a user copy"); - } catch (err) { - log.warn({ err }, "deps: link reconcile failed"); - } - }, 0); - - log.info("daemon starting"); - writeFileSync(DAEMON_PID_PATH, String(process.pid)); - - // Open state.db and build the in-memory branch-cache map BEFORE serving - // (spec "Migration & contention"): the one long transaction is the - // legacy-JSON import, and it must never land inside the event loop. If a - // CLI process is mid-import right now, we block here, in startup. - openBranchCacheStore(); - log.info({ count: Object.keys(cache.entries).length }, "branch cache loaded from state.db"); - - // one-shot re-key of every legacy NAME-keyed store row onto its - // serialized repo identity. Fire-and-forget (not awaited) like the PATH - // reconcile above — the ordering guarantee this depends on (running before - // anything prunes the repo index) only needs this to be on the boot path, - // not blocking the socket bind; a prune only ever arrives as a command sent - // to an already-running daemon. - runBootIdentityMigration(log).catch((err) => { - log.warn({ err }, "boot identity migration failed"); - }); - - routedHandlers = buildRoutedHandlers({ - ctx: handlerCtx, - broadcast: emit, - systemProcessScanner, - worktree: { - emit, - kick: worktreeReconciler.kick, - creationInFlight: worktreeReconciler.creationInFlight, - }, - eventsBus, - homeSnapshot, - repos: { - withReconcilerHeld: worktreeReconciler.withReconcilerHeld, - refreshWatchedRepos: hooksGuard.refreshWatchedRepos, - }, - stateDb: getStateDb("daemon"), - }); - - // Daemon startup is one of the two moments a handle is about to be - // needed (spec "Pruning"); sign-in is the other, inside signIn itself. - // Pruning is best-effort cleanup, so a concurrent CLI writer's - // SQLITE_BUSY must not abort startup before the socket binds. - let prunedPresence = 0; - persistOrWarn("daemon", () => { prunedPresence = prunePresence(Date.now(), getStateDb("daemon"), snapshotRegistryDeps()); }, { op: "prunePresence" }); - if (prunedPresence > 0) log.info({ prunedPresence }, "chat: pruned stale presence rows at daemon startup"); - - // Socket server (Unix socket for CLI/tray) + REST/WS server (external clients) - servers.socket = startSocketServer({ handleCommand, log }); - servers.api = await startApiServer({ handleCommand, log }); - - // Wire notification broadcasts to WebSocket clients - onNotification(emit); - - // Discover and watch repos - hooksGuard.refreshWatchedRepos(); - - // Team tracking intent (mattstack.tracking) resolves through a primed - // identity→name map, not live derivation — loadRepoTracking is sync and - // runs on every freshness tick. Team intent is inert until this completes. - // The repo index moved into state.db (RT-50): there is no file to fs.watch - // for new-repo changes any more, so the 60s hooks-scan poller (pollers.ts) - // is the only re-prime mechanism now, not just the reliable one. - primeTeamTrackingIdentityMap(loadRepoIndex()).catch((err) => { - log.warn({ err }, "repo-tracking: failed to prime team-intent identity map"); - }); - - // Periodic background work: cache refresh, port scan, system-process scan, - // hooks-guard fallback rescan. - startPollers({ - log, refreshCache, portCacheRef, broadcast: emit, systemProcessScanner, - repoIndex: loadRepoIndex, - checkAndRepairHooksPath: hooksGuard.checkAndRepairHooksPath, - }); - - // Kick off the events watchers once the first refresh has populated the - // cache with repoName stamps. reconcileFreshness inside the cache refresher - // follows repo-index changes from there. - setTimeout(() => { - initFreshness(freshnessEnv).catch((err) => { - log.error({ err }, "freshness: init failed"); + try { + mkdirSync(RT_DIR, { recursive: true }); + + // If a previous daemon process is still alive (orphan from a failed + // restart), evict it before we bind the socket. + await evictStaleDaemon(log); + + // Auto-unlink any tagged tool link whose tool now has a genuine user copy + // elsewhere on PATH (e.g. the user ran `brew install gh` after rt linked + // the bundled one). reconcile() itself is synchronous (a ~/.local/bin + // readDir plus a handful of stats); wrapping the call in `async` alone + // would NOT defer it, since nothing inside actually awaits. setTimeout(0) + // is what actually pushes it past the rest of this function: the PID + // write, openBranchCacheStore, and both server binds below all run first, + // on this same synchronous pass, before the timer callback ever fires. + setTimeout(() => { + try { + const { removed } = reconcileLinks(createRealProbes()); + if (removed.length > 0) log.info({ removed }, "deps: auto-unlinked tools now shadowed by a user copy"); + } catch (err) { + log.warn({ err }, "deps: link reconcile failed"); + } + }, 0); + + log.info("daemon starting"); + + // Open state.db and build the in-memory branch-cache map BEFORE serving + // (spec "Migration & contention"): the one long transaction is the + // legacy-JSON import, and it must never land inside the event loop. If a + // CLI process is mid-import right now, we block here, in startup. + setPhase("state-db"); + openBranchCacheStore(); + recordBootAttempt(); + log.info({ count: Object.keys(cache.entries).length }, "branch cache loaded from state.db"); + + // one-shot re-key of every legacy NAME-keyed store row onto its + // serialized repo identity. Fire-and-forget (not awaited) like the PATH + // reconcile above: the ordering guarantee this depends on (running before + // anything prunes the repo index) only needs this to be on the boot path, + // not blocking the socket bind; a prune only ever arrives as a command sent + // to an already-running daemon. + runBootIdentityMigration(log).catch((err) => { + log.warn({ err }, "boot identity migration failed"); }); - }, 7000); - // Background sweep for new MR comments → `discussions:new-comments` events. - startDiscussionsPoller({ ctx: handlerCtx, broadcast: emit }); + routedHandlers = buildRoutedHandlers({ + ctx: handlerCtx, + broadcast: emit, + systemProcessScanner, + worktree: { + emit, + kick: worktreeReconciler.kick, + creationInFlight: worktreeReconciler.creationInFlight, + }, + eventsBus, + homeSnapshot, + repos: { + withReconcilerHeld: worktreeReconciler.withReconcilerHeld, + refreshWatchedRepos: hooksGuard.refreshWatchedRepos, + }, + stateDb: getStateDb("daemon"), + }); - // Sandbox ground-truth reconcile: port-forwards, dev-ports mirroring, and - // typed-event → notification fan-out (no-op while no controller answers). + // Daemon startup is one of the two moments a handle is about to be + // needed (spec "Pruning"); sign-in is the other, inside signIn itself. + // Pruning is best-effort cleanup, so a concurrent CLI writer's + // SQLITE_BUSY must not abort startup before the socket binds. + let prunedPresence = 0; + persistOrWarn("daemon", () => { prunedPresence = prunePresence(Date.now(), getStateDb("daemon"), snapshotRegistryDeps()); }, { op: "prunePresence" }); + if (prunedPresence > 0) log.info({ prunedPresence }, "chat: pruned stale presence rows at daemon startup"); + + // API server first: a failed bind exits fatally (boot-phase catch below), + // and binding API before the unix socket means that fatal exit never + // strands a socket-bound zombie behind it. ApiPortInUseError is the one + // exception to "fatal": bindApiServerWithRetry has already exhausted its + // own ~3s inner retry, so the holder is a whole other process that may + // take much longer to exit — park-and-retry with backoff instead of + // crash-looping (S043 caller-side contract, docs/daemon-api-auth.md). + setPhase("api"); + servers.api = await withApiPortParkRetry( + () => startApiServer({ handleCommand, log }), + { sleep: (ms) => Bun.sleep(ms), log }, + ); + setPhase("socket"); + servers.socket = startSocketServer({ handleCommand, log }); + + // Only write rt.pid once both servers are actually bound: a boot that + // fails before this point must never leave a live-pid file with no + // socket/API behind it. + writeFileSync(DAEMON_PID_PATH, String(process.pid)); + + // Wire notification broadcasts to WebSocket clients + onNotification(emit); + + // Discover and watch repos + hooksGuard.refreshWatchedRepos(); + + // Team tracking intent (mattstack.tracking) resolves through a primed + // identity→name map, not live derivation; loadRepoTracking is sync and + // runs on every freshness tick. Team intent is inert until this completes. + // The repo index moved into state.db (RT-50): there is no file to fs.watch + // for new-repo changes any more, so the 60s hooks-scan poller (pollers.ts) + // is the only re-prime mechanism now, not just the reliable one. + primeTeamTrackingIdentityMap(loadRepoIndex()).catch((err) => { + log.warn({ err }, "repo-tracking: failed to prime team-intent identity map"); + }); - // Graceful shutdown on all termination signals - installSignalHandlers({ cleanup, flushLogs: () => loggerHandle.flush?.(), log }); + // Periodic background work: cache refresh, port scan, system-process scan, + // hooks-guard fallback rescan. + startPollers({ + log, refreshCache, portCacheRef, broadcast: emit, systemProcessScanner, + repoIndex: loadRepoIndex, + checkAndRepairHooksPath: hooksGuard.checkAndRepairHooksPath, + }); - log.info({ pid: process.pid }, "daemon ready"); -} + // Kick off the events watchers once the first refresh has populated the + // cache with repoName stamps. reconcileFreshness inside the cache refresher + // follows repo-index changes from there. + setTimeout(() => { + initFreshness(freshnessEnv).catch((err) => { + log.error({ err }, "freshness: init failed"); + }); + }, 7000); + + // Background sweep for new MR comments → `discussions:new-comments` events. + startDiscussionsPoller({ ctx: handlerCtx, broadcast: emit }); + + // Sandbox ground-truth reconcile: port-forwards, dev-ports mirroring, and + // typed-event → notification fan-out (no-op while no controller answers). + + // Graceful shutdown on all termination signals + installSignalHandlers({ + cleanup, + flushLogs: () => loggerHandle.flush?.(), + log, + wasVerbShutdown: () => shuttingDownViaVerb, + }); -/** - * Both real callers (cli.ts's `--daemon` entry, and this file's own - * import.meta.main guard below) invoke this fire-and-forget — neither awaits - * or catches. Left as a bare async function, ANY failure inside runDaemon() - * (this bind, openBranchCacheStore, anything else awaited) becomes an - * unhandledRejection, whose handler (installCrashHandlers, above) logs and - * deliberately does NOT exit — the daemon would stay alive with rt.sock - * possibly bound but startup never having reached signal handlers, pollers, - * or "daemon ready". Catching here and exiting explicitly gives every - * startup failure the same outcome a synchronous one already had: the - * process dies and whatever restarts a crashed daemon restarts this one too. - */ -export async function startDaemon(): Promise { - try { - await runDaemon(); + bootPhase = "ready"; + recordDaemonReady(); + setPhase("ready"); + log.info({ pid: process.pid }, "daemon ready"); } catch (err) { - log.fatal({ err }, "daemon startup failed — exiting"); + log.fatal({ err }, "daemon boot failed"); + recordBootFailure(currentPhase, String(err)); + try { loggerHandle.flush?.(); } catch (flushErr) { log.warn({ err: flushErr }, "daemon boot log flush failed"); } process.exit(1); } } +// runDaemon() never rejects: it logs fatal and exit(1)s internally on any +// boot failure, so this wrapper needs no catch of its own. +export async function startDaemon(): Promise { + await runDaemon(); +} + // Auto-run when executed directly (source mode: bun run lib/daemon.ts) if (import.meta.main) { startDaemon(); diff --git a/lib/daemon/__tests__/agent-handlers.test.ts b/lib/daemon/__tests__/agent-handlers.test.ts index 608147513..30ec7cbaa 100644 --- a/lib/daemon/__tests__/agent-handlers.test.ts +++ b/lib/daemon/__tests__/agent-handlers.test.ts @@ -5,6 +5,7 @@ import { AGENT_NAMES } from "../../chat-names.ts"; import { openStateDb, signIn } from "../../state/index.ts"; import { createAgentHandlers, type HeadlessChild } from "../handlers/agent.ts"; import type { HerdrRunner } from "../../agent-herdr.ts"; +import { repoLabel } from "../../repo-arg.ts"; let n = 0; const REPO = "remote:example.com%2Fa%2Fb"; @@ -62,6 +63,34 @@ test("agent:start herdr rolls back the inserted record when launch fails", async expect(list.data.agents).toHaveLength(0); }); +// Pins S051: a tab-label dedup must never report success with a phantom +// record nothing is listening on (rt agent resume on it would run +// `claude --resume` for a session that never started). +test("agent:start herdr returns ok:false and rolls back when herdr dedups the tab label", async () => { + const label = "!7"; + const focusCalls: string[][] = []; + const runner: HerdrRunner = async (args) => { + focusCalls.push(args); + if (args[0] === "workspace" && args[1] === "list") { + return { stdout: JSON.stringify({ result: { workspaces: [{ workspace_id: "w1", label: repoLabel(REPO) }] } }), exitCode: 0 }; + } + if (args[0] === "tab" && args[1] === "list") { + return { stdout: JSON.stringify({ result: { tabs: [{ tab_id: "w1:t9", label } ] } }), exitCode: 0 }; + } + return { stdout: "{}", exitCode: 0 }; + }; + const h = fresh({ runner }); + const res = await h["agent:start"]({ repo: REPO, cwd: "/tmp/x", prompt: "hi", surface: "herdr", tab: label }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toMatch(/already open/); + expect(focusCalls.some((c) => c[0] === "tab" && c[1] === "focus")).toBe(true); + expect(focusCalls.some((c) => c[0] === "pane" && c[1] === "run")).toBe(false); + const list = await h["agent:list"]({}); + if (!list.ok) throw new Error("unreachable"); + expect(list.data.agents).toHaveLength(0); +}); + // Pins the guard: a no-op insert (standing in for runCriticalWrite giving up // after sustained SQLITE_BUSY) must block the launch, not just the record. test("agent:start refuses to launch when the insert did not persist", async () => { @@ -91,6 +120,22 @@ test("agent:start headless refuses a missing prompt", async () => { expect(res.error).toMatch(/prompt/); }); +// R033: an unchecked surface value falls through to the headless spawn path +// with headless=false, spawning an interactive claude with stdin ignored +// and recording surface "bogus" — never a caller-visible error. +test("agent:start rejects a surface outside herdr/headless, naming the allowed values", async () => { + const h = fresh(); + const res = await h["agent:start"]({ repo: REPO, cwd: "/tmp/x", prompt: "hi", surface: "bogus" as any }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("surface"); + expect(res.error).toContain("herdr"); + expect(res.error).toContain("headless"); + const list = await h["agent:list"]({}); + if (!list.ok) throw new Error("unreachable"); + expect(list.data.agents).toHaveLength(0); +}); + test("agent:start headless finishes the record and emits agent/done", async () => { const emitted: string[] = []; let resolveExit!: (c: number) => void; diff --git a/lib/daemon/__tests__/agent-status-poller.test.ts b/lib/daemon/__tests__/agent-status-poller.test.ts index 57020710c..7f9813b1d 100644 --- a/lib/daemon/__tests__/agent-status-poller.test.ts +++ b/lib/daemon/__tests__/agent-status-poller.test.ts @@ -73,3 +73,45 @@ test("finished runs are ignored and dropped from tracking", async () => { await handle!.tick(); expect(events).toHaveLength(0); }); + +test("backs off the herdr probe after repeated failures", async () => { + let probeCalls = 0; + handle = startAgentStatusPoller({ + emitEvent: () => {}, + log: quietLog, + intervalMs: 3_600_000, // real timer never fires + probe: async () => { probeCalls++; return null; }, // herdr absent + list: () => [], + }); + for (let i = 0; i < 20; i++) await handle.tick(); + // Without backoff this would be 20; with backoff (threshold 3, 1-in-6) far fewer. + expect(probeCalls).toBeLessThan(10); + // Backoff never stops probing forever: 3 initial failures engage it, then + // one probe every BACKOFF_TICKS (ticks 9 and 15 of 20) keeps checking. + expect(probeCalls).toBe(5); +}); + +test("a successful probe after backoff resets consecutiveFailures and resumes per-tick probing", async () => { + let probeCalls = 0; + handle = startAgentStatusPoller({ + emitEvent: () => {}, + log: quietLog, + intervalMs: 3_600_000, + probe: async () => { + probeCalls++; + // Invocations 1-3 cross FAILURE_THRESHOLD and engage backoff; the + // backoff window then skips ticks 4-8 without invoking probe at all, + // so invocation 4 is the tick-9 retry ... make it succeed. + return probeCalls <= 3 ? null : []; + }, + list: () => [], + }); + for (let i = 0; i < 9; i++) await handle.tick(); + expect(probeCalls).toBe(4); // probed at ticks 1, 2, 3, then again at tick 9 + const callsBeforeRecovery = probeCalls; + await handle.tick(); // tick 10, immediately after the successful tick-9 probe + // A successful probe resets consecutiveFailures to 0, so the very next + // tick is not gated by backoff ... it probes right away instead of + // waiting out another BACKOFF_TICKS window. + expect(probeCalls).toBe(callsBeforeRecovery + 1); +}); diff --git a/lib/daemon/__tests__/api-auth.test.ts b/lib/daemon/__tests__/api-auth.test.ts index d9cda0b1f..b5113660d 100644 --- a/lib/daemon/__tests__/api-auth.test.ts +++ b/lib/daemon/__tests__/api-auth.test.ts @@ -4,7 +4,11 @@ */ import { describe, test, expect } from "bun:test"; -import { needsToken, tokenOk } from "../api-auth.ts"; +import { needsToken, tokenOk, getApiToken, reloadApiToken, loadOrCreateApiToken } from "../api-auth.ts"; +import { isOriginAllowed, isBrowserRequestTrusted, getTrustedBrowserOrigins, resolveOriginTrust } from "../api-auth.ts"; +import { mkdtempSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; describe("needsToken", () => { test("shutdown requires a token", () => { @@ -39,6 +43,35 @@ describe("needsToken", () => { test("secrets requires a token even though it's a GET — the response body is a credential, not metadata", () => { expect(needsToken("GET", "/api/secrets")).toBe(true); }); + + test("refresh requires a token now (S040)", () => { + expect(needsToken("POST", "/api/refresh")).toBe(true); + }); + + test("hooks repair requires a token now (S040/S084)", () => { + expect(needsToken("POST", "/api/hooks/my-repo/repair")).toBe(true); + }); + + test("notifications GET (destructive drain) requires a token now (S041)", () => { + expect(needsToken("GET", "/api/notifications")).toBe(true); + }); + + test("every non-GET/HEAD/OPTIONS method defaults to requiring a token", () => { + expect(needsToken("POST", "/api/some-future-mutating-route")).toBe(true); + expect(needsToken("PUT", "/api/anything")).toBe(true); + expect(needsToken("DELETE", "/api/anything")).toBe(true); + }); + + test("plain reads still do not require a token", () => { + expect(needsToken("GET", "/api/repos")).toBe(false); + expect(needsToken("GET", "/api/cache")).toBe(false); + expect(needsToken("HEAD", "/api/repos")).toBe(false); + }); + + test("OPTIONS never requires a token even for secrets/notifications (preflight must never be gated)", () => { + expect(needsToken("OPTIONS", "/api/secrets")).toBe(false); + expect(needsToken("OPTIONS", "/api/notifications")).toBe(false); + }); }); describe("tokenOk", () => { @@ -58,3 +91,121 @@ describe("tokenOk", () => { expect(tokenOk("anything", "")).toBe(false); }); }); + +describe("getApiToken / reloadApiToken singleton", () => { + test("getApiToken caches: a second call does not re-read the file even if it changes underneath", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + const first = reloadApiToken(tokenPath); // seed the cache with a known path + writeFileSync(tokenPath, "a-different-token", { mode: 0o600 }); + const second = getApiToken(tokenPath); // ignores the new file content -- cached + expect(second).toBe(first); + expect(second).not.toBe("a-different-token"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("reloadApiToken re-reads and updates the cache", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + reloadApiToken(tokenPath); + writeFileSync(tokenPath, "rotated-token", { mode: 0o600 }); + const reloaded = reloadApiToken(tokenPath); + expect(reloaded).toBe("rotated-token"); + expect(getApiToken(tokenPath)).toBe("rotated-token"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("loadOrCreateApiToken still works standalone (unchanged primitive)", () => { + const dir = mkdtempSync(join(tmpdir(), "rt-api-token-")); + const tokenPath = join(dir, "api-token"); + try { + const a = loadOrCreateApiToken(tokenPath); + const b = loadOrCreateApiToken(tokenPath); + expect(a).toBe(b); + expect(a.length).toBeGreaterThan(0); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +describe("isOriginAllowed", () => { + test("exact match", () => { + expect(isOriginAllowed("http://localhost:5544", ["http://localhost:5544"])).toBe(true); + }); + test("no match", () => { + expect(isOriginAllowed("http://evil.example", ["http://localhost:5544"])).toBe(false); + }); + test("empty allowlist matches nothing", () => { + expect(isOriginAllowed("http://localhost:5544", [])).toBe(false); + }); +}); + +describe("isBrowserRequestTrusted", () => { + const apiToken = "the-real-token"; + + test("no Origin header at all -- a non-browser client -- is trusted regardless of token or allowlist", () => { + expect(isBrowserRequestTrusted(null, null, apiToken, [])).toBe(true); + }); + + test("a browser Origin with the correct token is trusted even off the allowlist", () => { + expect(isBrowserRequestTrusted("http://evil.example", apiToken, apiToken, [])).toBe(true); + }); + + test("a browser Origin with a wrong token and not on the allowlist is rejected", () => { + expect(isBrowserRequestTrusted("http://evil.example", "wrong", apiToken, [])).toBe(false); + }); + + test("a browser Origin with no token but on the allowlist is trusted", () => { + expect(isBrowserRequestTrusted("http://localhost:5544", null, apiToken, ["http://localhost:5544"])).toBe(true); + }); + + test("a browser Origin with no token and not on the allowlist is rejected", () => { + expect(isBrowserRequestTrusted("http://localhost:5544", null, apiToken, [])).toBe(false); + }); +}); + +describe("getTrustedBrowserOrigins", () => { + test("returns an array (empty by default in an isolated test HOME)", () => { + const origins = getTrustedBrowserOrigins(); + expect(Array.isArray(origins)).toBe(true); + }); +}); + +describe("resolveOriginTrust", () => { + const apiToken = "the-real-token"; + + test("no Origin header: never calls getAllowedOrigins (the settings read is disk I/O every non-browser request would otherwise pay for)", () => { + let calls = 0; + const getAllowedOrigins = () => { calls++; return []; }; + const trusted = resolveOriginTrust(null, null, apiToken, getAllowedOrigins); + expect(trusted).toBe(true); + expect(calls).toBe(0); + }); + + test("an Origin header present: does call getAllowedOrigins", () => { + let calls = 0; + const getAllowedOrigins = () => { calls++; return ["http://localhost:5544"]; }; + const trusted = resolveOriginTrust("http://localhost:5544", null, apiToken, getAllowedOrigins); + expect(trusted).toBe(true); + expect(calls).toBe(1); + }); + + test("an Origin header with the correct token is trusted without needing the allowlist call to matter", () => { + let calls = 0; + const getAllowedOrigins = () => { calls++; return []; }; + const trusted = resolveOriginTrust("http://evil.example", apiToken, apiToken, getAllowedOrigins); + expect(trusted).toBe(true); + expect(calls).toBe(1); + }); + + test("defaults to the real getTrustedBrowserOrigins when no override is passed", () => { + expect(resolveOriginTrust(null, null, apiToken)).toBe(true); + }); +}); diff --git a/lib/daemon/__tests__/api-server-bind.test.ts b/lib/daemon/__tests__/api-server-bind.test.ts index 3fc417f4d..d8ff59421 100644 --- a/lib/daemon/__tests__/api-server-bind.test.ts +++ b/lib/daemon/__tests__/api-server-bind.test.ts @@ -1,5 +1,9 @@ -import { describe, test, expect } from "bun:test"; -import { bindApiServerWithRetry, BIND_RETRY_ATTEMPTS, BIND_RETRY_DELAY_MS, type BindRetryDeps } from "../api-server.ts"; +import { describe, test, expect, afterEach } from "bun:test"; +import type { Server } from "bun"; +import { bindApiServerWithRetry, BIND_RETRY_ATTEMPTS, BIND_RETRY_DELAY_MS, startApiServer, type BindRetryDeps } from "../api-server.ts"; +import { ApiPortInUseError } from "../api-server.ts"; +import { setSetting, unsetSetting } from "../../settings/write.ts"; +import { getSetting } from "../../settings/resolve.ts"; function eaddrinuse(): Error { return Object.assign(new Error("EADDRINUSE"), { code: "EADDRINUSE" }); @@ -11,6 +15,7 @@ function deps(overrides: Partial = {}): BindRetryDeps & { logs: s return { sleep: async (ms: number) => { sleeps.push(ms); }, log: { warn: (_o: unknown, m: string) => logs.push(`warn:${m}`) }, + probePortHolder: async () => "n/a", logs, sleeps, ...overrides, @@ -41,7 +46,7 @@ describe("bindApiServerWithRetry", () => { expect(d.logs.some((l) => l.includes("retrying"))).toBe(true); }); - test("exhausting retries rethrows the original error after exactly BIND_RETRY_ATTEMPTS calls", async () => { + test("exhausting retries rethrows as ApiPortInUseError after exactly BIND_RETRY_ATTEMPTS calls", async () => { const d = deps(); let calls = 0; await expect( @@ -61,3 +66,105 @@ describe("bindApiServerWithRetry", () => { expect(d.sleeps).toEqual([]); }); }); + +function depsWithProbe(overrides: Partial = {}) { + const logs: Array<{ o: unknown; m: string }> = []; + const sleeps: number[] = []; + const probeCalls: number[] = []; + return { + sleep: async (ms: number) => { sleeps.push(ms); }, + log: { warn: (o: unknown, m: string) => logs.push({ o, m }) }, + probePortHolder: async (port: number) => { probeCalls.push(port); return "COMMAND PID USER\nnode 123 matt"; }, + logs, + sleeps, + probeCalls, + ...overrides, + }; +} + +describe("bindApiServerWithRetry — exhausted retries (S043)", () => { + test("throws ApiPortInUseError (not the raw EADDRINUSE Error) once attempts are exhausted", async () => { + const d = depsWithProbe(); + let error: unknown; + try { + await bindApiServerWithRetry(() => { throw eaddrinuse(); }, d); + } catch (err) { + error = err; + } + expect(error).toBeInstanceOf(ApiPortInUseError); + expect((error as ApiPortInUseError).code).toBe("EADDRINUSE"); + expect((error as Error).message).toContain("EADDRINUSE"); + }); + + test("probes the port holder exactly once, only after the final attempt", async () => { + const d = depsWithProbe(); + let calls = 0; + await expect( + bindApiServerWithRetry(() => { calls++; throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + expect(calls).toBe(BIND_RETRY_ATTEMPTS); + expect(d.probeCalls.length).toBe(1); + }); + + test("logs the probe result at warn before throwing", async () => { + const d = depsWithProbe(); + await expect( + bindApiServerWithRetry(() => { throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + const finalWarn = d.logs.at(-1)!; + expect(finalWarn.o).toMatchObject({ holder: expect.stringContaining("node") }); + }); + + test("a probe failure does not prevent the ApiPortInUseError from being thrown", async () => { + const d = depsWithProbe({ probePortHolder: async () => { throw new Error("lsof: command not found"); } }); + await expect( + bindApiServerWithRetry(() => { throw eaddrinuse(); }, d), + ).rejects.toBeInstanceOf(ApiPortInUseError); + }); + + test("a successful bind never probes", async () => { + const d = depsWithProbe(); + await bindApiServerWithRetry(() => "server" as any, d); + expect(d.probeCalls.length).toBe(0); + }); +}); + +describe("startApiServer — binds via resolveApiPort() (S043 caller-side wiring)", () => { + let server: Server | undefined; + let prevEnv: string | undefined; + + afterEach(() => { + server?.stop(true); + server = undefined; + // Runs even when an assertion above threw — restore both regardless of + // pass/fail, and regardless of test HOME being shared across the whole + // `bun test` run (test-setup.ts preloads it once, not per file). + if (prevEnv !== undefined) process.env.RT_API_PORT = prevEnv; + else delete process.env.RT_API_PORT; + unsetSetting("rt.apiPort", "user"); + }); + + test("binds to the rt.apiPort setting value, not the hardcoded 9401 default", async () => { + prevEnv = process.env.RT_API_PORT; + delete process.env.RT_API_PORT; + + // Measure a free port rather than hardcoding one, then release it + // immediately — startApiServer binds it back before anything else can. + const probe = Bun.serve({ port: 0, fetch: () => new Response() }); + const port = probe.port; + probe.stop(true); + + setSetting("rt.apiPort", port, "user"); + const log = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as any; + + server = await startApiServer({ handleCommand: async () => ({ ok: true }), log }); + + expect(server.port).toBe(port); + }); + + test("the rt.apiPort setting from the previous test does not leak into later tests (C8)", () => { + // 9401 is the registry default — proof the "user" scope value was + // actually removed, not just that some value happens to be present. + expect(getSetting("rt.apiPort").value).toBe(9401); + }); +}); diff --git a/lib/daemon/__tests__/api-server-broadcast.test.ts b/lib/daemon/__tests__/api-server-broadcast.test.ts new file mode 100644 index 000000000..56faf914c --- /dev/null +++ b/lib/daemon/__tests__/api-server-broadcast.test.ts @@ -0,0 +1,75 @@ +import { describe, test, expect } from "bun:test"; +import { broadcastToClients, type BroadcastTarget } from "../api-server.ts"; + +function fakeClient(sendReturns: number[]): BroadcastTarget & { closed: boolean; sent: string[] } { + const sent: string[] = []; + let i = 0; + const client = { + closed: false, + sent, + send(data: string) { + sent.push(data); + const ret = sendReturns[Math.min(i, sendReturns.length - 1)] as number; + i++; + return ret; + }, + close() { client.closed = true; }, + }; + return client; +} + +function fakeLog() { + const warns: unknown[] = []; + return { warn: (o: unknown, _m: string) => { warns.push(o); }, warns }; +} + +describe("broadcastToClients", () => { + test("a healthy client (positive send return) is never closed", () => { + const client = fakeClient([42]); + broadcastToClients([client], "status", { ok: true }, fakeLog()); + expect(client.closed).toBe(false); + expect(client.sent.length).toBe(1); + }); + + test("a send() returning 0 (dropped frame) closes the client immediately and logs a warning", () => { + const client = fakeClient([0]); + const log = fakeLog(); + broadcastToClients([client], "status", { ok: true }, log); + expect(client.closed).toBe(true); + expect(log.warns.length).toBe(1); + }); + + test("a send() returning -1 (backpressure) is tolerated for a few sends before closing", () => { + const client = fakeClient([-1, -1, -1, -1]); + const log = fakeLog(); + broadcastToClients([client], "a", {}, log); + expect(client.closed).toBe(false); + broadcastToClients([client], "b", {}, log); + expect(client.closed).toBe(false); + broadcastToClients([client], "c", {}, log); + // third consecutive backpressure event closes the client + expect(client.closed).toBe(true); + }); + + test("a successful send resets the backpressure counter", () => { + const client = fakeClient([-1, -1, 99, -1, -1, -1]); + const log = fakeLog(); + broadcastToClients([client], "a", {}, log); // -1 (count=1) + broadcastToClients([client], "b", {}, log); // -1 (count=2) + broadcastToClients([client], "c", {}, log); // 99 -- resets to 0 + expect(client.closed).toBe(false); + broadcastToClients([client], "d", {}, log); // -1 (count=1) + broadcastToClients([client], "e", {}, log); // -1 (count=2) + expect(client.closed).toBe(false); + broadcastToClients([client], "f", {}, log); // -1 (count=3) -- closes + expect(client.closed).toBe(true); + }); + + test("a client whose send() throws is treated as gone: caught, not propagated", () => { + const client: BroadcastTarget = { + send() { throw new Error("ECONNRESET"); }, + close() { /* no-op */ }, + }; + expect(() => broadcastToClients([client], "a", {}, fakeLog())).not.toThrow(); + }); +}); diff --git a/lib/daemon/__tests__/api-server-cors-ws.test.ts b/lib/daemon/__tests__/api-server-cors-ws.test.ts new file mode 100644 index 000000000..500e45468 --- /dev/null +++ b/lib/daemon/__tests__/api-server-cors-ws.test.ts @@ -0,0 +1,96 @@ +import { describe, test, expect, afterEach } from "bun:test"; +import type { Server } from "bun"; +import { buildCorsHeaders, startApiServer } from "../api-server.ts"; +import { getApiToken } from "../api-auth.ts"; +import { setSetting, unsetSetting } from "../../settings/write.ts"; + +describe("buildCorsHeaders", () => { + test("no Origin header: no Access-Control-Allow-Origin is set (non-browser request, CORS is irrelevant)", () => { + const headers = buildCorsHeaders(null, true); + expect(headers["Access-Control-Allow-Origin"]).toBeUndefined(); + }); + + test("an untrusted Origin gets no Access-Control-Allow-Origin (default-deny, S006)", () => { + const headers = buildCorsHeaders("http://evil.example", false); + expect(headers["Access-Control-Allow-Origin"]).toBeUndefined(); + }); + + test("a trusted Origin is echoed back with Vary: Origin", () => { + const headers = buildCorsHeaders("http://localhost:5544", true); + expect(headers["Access-Control-Allow-Origin"]).toBe("http://localhost:5544"); + expect(headers["Vary"]).toBe("Origin"); + }); + + test("always advertises the methods/headers a preflight needs, trusted or not", () => { + const headers = buildCorsHeaders("http://evil.example", false); + expect(headers["Access-Control-Allow-Methods"]).toContain("POST"); + expect(headers["Access-Control-Allow-Headers"]).toContain("X-RT-Token"); + }); +}); + +describe("token-authenticated browser preflight (S-C1: off-allowlist Origin, X-RT-Token preflight)", () => { + let server: Server | undefined; + + afterEach(() => { + server?.stop(true); + server = undefined; + unsetSetting("rt.apiPort", "user"); + }); + + test("an OPTIONS preflight requesting X-RT-Token from an off-allowlist Origin still gets Access-Control-Allow-Origin, so the browser proceeds to the real token-bearing request", async () => { + const probe = Bun.serve({ port: 0, fetch: () => new Response() }); + const port = probe.port; + probe.stop(true); + setSetting("rt.apiPort", port, "user"); + const log = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as any; + server = await startApiServer({ handleCommand: async () => ({ ok: true }), log }); + + const apiToken = getApiToken(); + const origin = "http://off-allowlist.example"; + + const preflight = await fetch(`http://127.0.0.1:${port}/api/refresh`, { + method: "OPTIONS", + headers: { + Origin: origin, + "Access-Control-Request-Method": "GET", + "Access-Control-Request-Headers": "x-rt-token", + }, + }); + expect(preflight.status).toBe(204); + expect(preflight.headers.get("access-control-allow-origin")).toBe(origin); + + const actual = await fetch(`http://127.0.0.1:${port}/api/refresh`, { + method: "POST", + headers: { Origin: origin, "X-RT-Token": apiToken }, + }); + expect(actual.status).not.toBe(401); + expect(actual.headers.get("access-control-allow-origin")).toBe(origin); + }); +}); + +describe("advertised URLs use the resolved bind port, not the compile-time default (S9-a)", () => { + let server: Server | undefined; + + afterEach(() => { + server?.stop(true); + server = undefined; + unsetSetting("rt.apiPort", "user"); + }); + + test("GET / and a 404 both report the actually-bound port when it differs from 9401", async () => { + const probe = Bun.serve({ port: 0, fetch: () => new Response() }); + const port = probe.port; + probe.stop(true); + expect(port).not.toBe(9401); + setSetting("rt.apiPort", port, "user"); + const log = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as any; + server = await startApiServer({ handleCommand: async () => ({ ok: true }), log }); + + const root = await (await fetch(`http://127.0.0.1:${port}/`)).json() as { docs: string; websocket: string }; + expect(root.docs).toBe(`http://localhost:${port}/`); + expect(root.websocket).toBe(`ws://localhost:${port}/ws`); + + const notFound = await (await fetch(`http://127.0.0.1:${port}/nope`)).json() as { docs: string }; + expect(notFound.docs).toBe(`http://localhost:${port}/`); + }); +}); diff --git a/lib/daemon/__tests__/api-server-park-retry.test.ts b/lib/daemon/__tests__/api-server-park-retry.test.ts new file mode 100644 index 000000000..413427b9b --- /dev/null +++ b/lib/daemon/__tests__/api-server-park-retry.test.ts @@ -0,0 +1,48 @@ +import { describe, test, expect } from "bun:test"; +import { withApiPortParkRetry, ApiPortInUseError } from "../api-server.ts"; + +describe("withApiPortParkRetry (S043 caller-side wiring)", () => { + test("retries with backoff on ApiPortInUseError and returns once start succeeds", async () => { + const sleeps: number[] = []; + const warns: unknown[] = []; + let attempts = 0; + const result = await withApiPortParkRetry( + async () => { + attempts++; + if (attempts < 3) throw new ApiPortInUseError(9401); + return "server"; + }, + { sleep: async (ms) => { sleeps.push(ms); }, log: { warn: (o) => warns.push(o) } }, + ); + expect(result).toBe("server"); + expect(attempts).toBe(3); + expect(sleeps.length).toBe(2); + expect(warns.length).toBe(2); + }); + + test("backs off with an increasing delay on each successive attempt", async () => { + const sleeps: number[] = []; + let attempts = 0; + await withApiPortParkRetry( + async () => { + attempts++; + if (attempts < 4) throw new ApiPortInUseError(9401); + return "server"; + }, + { sleep: async (ms) => { sleeps.push(ms); }, log: { warn: () => {} } }, + ); + expect(sleeps[1]).toBeGreaterThan(sleeps[0]!); + expect(sleeps[2]).toBeGreaterThan(sleeps[1]!); + }); + + test("a non-ApiPortInUseError error is never retried", async () => { + let attempts = 0; + await expect( + withApiPortParkRetry( + async () => { attempts++; throw new Error("state.db open failed"); }, + { sleep: async () => {}, log: { warn: () => {} } }, + ), + ).rejects.toThrow("state.db open failed"); + expect(attempts).toBe(1); + }); +}); diff --git a/lib/daemon/__tests__/api-server-path-param.test.ts b/lib/daemon/__tests__/api-server-path-param.test.ts new file mode 100644 index 000000000..353b9a0a5 --- /dev/null +++ b/lib/daemon/__tests__/api-server-path-param.test.ts @@ -0,0 +1,36 @@ +import { describe, test, expect } from "bun:test"; +import { pathParam } from "../api-server.ts"; + +describe("pathParam", () => { + test("decodes a clean prefix-only param", () => { + expect(pathParam("/api/cache/main", "/api/cache/")).toBe("main"); + }); + + test("decodes a URL-encoded segment", () => { + expect(pathParam("/api/cache/feature%2Ffoo", "/api/cache/")).toBe("feature/foo"); + }); + + test("returns undefined for malformed %-encoding instead of throwing", () => { + expect(pathParam("/api/cache/%E0%A4%A", "/api/cache/")).toBeUndefined(); + }); + + test("returns undefined when the pathname doesn't start with the prefix", () => { + expect(pathParam("/api/other/main", "/api/cache/")).toBeUndefined(); + }); + + test("handles a prefix+suffix pair (hooks repair shape)", () => { + expect(pathParam("/api/hooks/my-repo/repair", "/api/hooks/", "/repair")).toBe("my-repo"); + }); + + test("prefix+suffix: malformed encoding still returns undefined", () => { + expect(pathParam("/api/hooks/%E0%A4%A/repair", "/api/hooks/", "/repair")).toBeUndefined(); + }); + + test("prefix+suffix: wrong suffix returns undefined", () => { + expect(pathParam("/api/hooks/my-repo/other", "/api/hooks/", "/repair")).toBeUndefined(); + }); + + test("an empty captured segment returns undefined", () => { + expect(pathParam("/api/cache/", "/api/cache/")).toBeUndefined(); + }); +}); diff --git a/lib/daemon/__tests__/api-server-query-coerce.test.ts b/lib/daemon/__tests__/api-server-query-coerce.test.ts new file mode 100644 index 000000000..f3f28cb31 --- /dev/null +++ b/lib/daemon/__tests__/api-server-query-coerce.test.ts @@ -0,0 +1,35 @@ +import { describe, test, expect } from "bun:test"; +import { coerceQueryParams } from "../api-server.ts"; + +describe("coerceQueryParams", () => { + test("coerces maxAgeMs to a number (the documented cache:read flag)", () => { + const out = coerceQueryParams(new URLSearchParams("maxAgeMs=60000")); + expect(out.maxAgeMs).toBe(60000); + expect(typeof out.maxAgeMs).toBe("number"); + }); + + test("coerces refresh=true to a boolean (the documented ports flag)", () => { + const out = coerceQueryParams(new URLSearchParams("refresh=true")); + expect(out.refresh).toBe(true); + }); + + test("coerces refresh=false to a boolean false, not a truthy string", () => { + const out = coerceQueryParams(new URLSearchParams("refresh=false")); + expect(out.refresh).toBe(false); + }); + + test("leaves a non-numeric, non-boolean string alone", () => { + const out = coerceQueryParams(new URLSearchParams("repo=my-repo-name")); + expect(out.repo).toBe("my-repo-name"); + }); + + test("leaves an empty string alone rather than coercing to 0", () => { + const out = coerceQueryParams(new URLSearchParams("q=")); + expect(out.q).toBe(""); + }); + + test("coerces a decimal number too", () => { + const out = coerceQueryParams(new URLSearchParams("ratio=1.5")); + expect(out.ratio).toBe(1.5); + }); +}); diff --git a/lib/daemon/__tests__/boot-reconcile.test.ts b/lib/daemon/__tests__/boot-reconcile.test.ts new file mode 100644 index 000000000..30ff990ad --- /dev/null +++ b/lib/daemon/__tests__/boot-reconcile.test.ts @@ -0,0 +1,50 @@ +import { mkdirSync, writeFileSync } from "fs"; +import { expect, test } from "bun:test"; +import type { Logger } from "pino"; +import { DAEMON_PID_PATH, RT_DIR } from "../../daemon-config.ts"; +import { evictStaleDaemon } from "../boot-reconcile.ts"; + +mkdirSync(RT_DIR, { recursive: true }); + +const silentLog = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as unknown as Logger; + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +test("evictStaleDaemon waits for the old pid to die, escalating to SIGKILL", async () => { + const child = Bun.spawn({ cmd: ["bash", "-c", "trap '' TERM; sleep 30"] }); + writeFileSync(DAEMON_PID_PATH, String(child.pid)); + const start = Date.now(); + await evictStaleDaemon(silentLog); + expect(isAlive(child.pid)).toBe(false); + expect(Date.now() - start).toBeLessThan(5000); + child.kill(); +}); + +test("evictStaleDaemon does not throw when the pid exits between the liveness check and the SIGTERM send (ESRCH)", async () => { + writeFileSync(DAEMON_PID_PATH, "999999"); + const originalKill = process.kill; + let sigtermSent = false; + (process as any).kill = (pid: number, signal?: string | number) => { + if (pid !== 999999) return originalKill(pid, signal as any); + if (signal === 0) return true; // liveness check still sees it alive + if (signal === "SIGTERM") { + sigtermSent = true; + const err = Object.assign(new Error("kill ESRCH"), { code: "ESRCH" }); + throw err; + } + throw new Error(`unexpected signal in test: ${String(signal)}`); + }; + try { + await expect(evictStaleDaemon(silentLog)).resolves.toBeUndefined(); + } finally { + process.kill = originalKill; + } + expect(sigtermSent).toBe(true); +}); diff --git a/lib/daemon/__tests__/cache-refresh-coalesce.test.ts b/lib/daemon/__tests__/cache-refresh-coalesce.test.ts new file mode 100644 index 000000000..12191e6ed --- /dev/null +++ b/lib/daemon/__tests__/cache-refresh-coalesce.test.ts @@ -0,0 +1,61 @@ +import { test, expect } from "bun:test"; +import { makeCoalescer } from "../cache-refresh.ts"; + +test("clears the in-flight latch after the deadline even if run never settles", async () => { + let starts = 0; + let timedOut = 0; + const coalesce = makeCoalescer( + () => { starts++; return new Promise(() => {}); }, // never resolves + 50, + () => { timedOut++; }, + ); + const t0 = Date.now(); + await coalesce(); // resolves at the deadline, not never + expect(Date.now() - t0).toBeLessThan(500); + expect(timedOut).toBe(1); + await coalesce(); // latch cleared, a new run can start + expect(starts).toBe(2); +}); + +test("coalesces concurrent callers onto one run", async () => { + let starts = 0; + let resolveRun!: () => void; + const coalesce = makeCoalescer( + () => { starts++; return new Promise((r) => { resolveRun = r; }); }, + 10_000, + () => {}, + ); + const a = coalesce(); + const b = coalesce(); + expect(starts).toBe(1); + resolveRun(); + await Promise.all([a, b]); +}); + +test("a fast success does not fire onTimeout after the deadline elapses", async () => { + let timedOut = 0; + const coalesce = makeCoalescer( + () => Promise.resolve(), // settles well before the deadline + 50, + () => { timedOut++; }, + ); + await coalesce(); + await new Promise((r) => setTimeout(r, 150)); // past the deadline + expect(timedOut).toBe(0); // the deadline timer must have been cleared, not just outraced +}); + +test("refuses to admit a replacement cycle once maxOrphanCycles stalled runs are already stuck in the background", async () => { + let starts = 0; + let refused = 0; + const coalesce = makeCoalescer( + () => { starts++; return new Promise(() => {}); }, // never resolves — every cycle orphans + 10, + () => {}, + { maxOrphanCycles: 2, onRefused: () => { refused++; } }, + ); + await coalesce(); // orphans (1) + await coalesce(); // orphans (2) + await coalesce(); // cap hit — refused, no new socket work started + expect(starts).toBe(2); + expect(refused).toBe(1); +}); diff --git a/lib/daemon/__tests__/cache-refresh-gc.test.ts b/lib/daemon/__tests__/cache-refresh-gc.test.ts index db2953300..a959a4d10 100644 --- a/lib/daemon/__tests__/cache-refresh-gc.test.ts +++ b/lib/daemon/__tests__/cache-refresh-gc.test.ts @@ -31,7 +31,7 @@ import { join } from "path"; import type { Logger } from "pino"; import * as enrichModule from "../../enrich.ts"; -import * as gitWorktreesModule from "../../git-worktrees.ts"; +import * as gitAsync from "../../worktree/git-async.ts"; import * as notifierModule from "../../notifier.ts"; import * as repoTrackingModule from "../../repo-tracking.ts"; import * as discussionsModule from "../discussions-file-store.ts"; @@ -101,11 +101,13 @@ function wireCycle(): Wiring { }); // A real git tree is irrelevant to the GC claim; one branch per repo is - // enough to make the loop reach the enrichment call. - spyOn(gitWorktreesModule, "listWorktrees").mockImplementation((repoPath: string) => [ - { path: repoPath, branch: `wt-${repoPath}` } as any, + // enough to make the loop reach the enrichment call and let FLAKY's onError + // fire (an empty branch list would short-circuit refreshAllMRs entirely and + // make FLAKY look clean, defeating the test). + spyOn(gitAsync, "listWorktreesAsync").mockImplementation(async (repoPath: string) => [ + { path: repoPath, branch: `wt-${repoPath}` }, ]); - spyOn(gitWorktreesModule, "listWorktreeRoots").mockReturnValue([]); + spyOn(gitAsync, "listWorktreeRootsAsync").mockResolvedValue([]); spyOn(enrichModule, "refreshAllMRs").mockImplementation( async (_branches, _remoteUrl, onError, repoName) => { diff --git a/lib/daemon/__tests__/chat-handlers.test.ts b/lib/daemon/__tests__/chat-handlers.test.ts index 20bc0f369..8b0de5f91 100644 --- a/lib/daemon/__tests__/chat-handlers.test.ts +++ b/lib/daemon/__tests__/chat-handlers.test.ts @@ -4,7 +4,7 @@ import { execSync } from "child_process"; import { mkdtempSync, realpathSync, writeFileSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; -import { openStateDb } from "../../state/index.ts"; +import { openStateDb, postMessage } from "../../state/index.ts"; import { createChatHandlers, inviteText, renderWelcome, type InboxDeps } from "../handlers/chat.ts"; import { herdrRequest } from "../../herdr/client.ts"; import { fakeHerdr, HerdrFakeError, type FakeHerdrHandler } from "../../herdr/__tests__/fake-herdr.ts"; @@ -65,6 +65,21 @@ test("chat:join rejects an invalid handle with a reason rather than normalizing expect(res.error).toContain("handle"); }); +// R033: an unchecked wakeOn value is stored on chat_members and, when the +// join creates the room, stamped into chat_room_defaults for every future +// joiner too — recipientsFromMembers treats it as neither "none" nor "all" +// (falls through to mention-only) and nothing ever reports the bad value. +test("chat:join rejects a wakeOn value outside mention/all/none, naming the allowed values", async () => { + const h = freshHandlers(); + const res = await h["chat:join"]({ room: "build", handle: "a", wakeOn: "sometimes" as any }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("wakeOn"); + expect(res.error).toContain("mention"); + expect(res.error).toContain("all"); + expect(res.error).toContain("none"); +}); + test("chat:post returns the recipients and emits only the room's msg event", async () => { const emitted: string[] = []; const h = freshHandlers((topic) => { emitted.push(topic); return 0; }); @@ -77,6 +92,23 @@ test("chat:post returns the recipients and emits only the room's msg event", asy expect(emitted).toEqual(["chat/r/msg"]); }); +test("chat:post still reports success when emitEvent throws after the message is durable", async () => { + const h = freshHandlers(() => { throw new Error("events.db locked"); }); + await h["chat:join"]({ room: "r", handle: "a" }); + const res = await h["chat:post"]({ room: "r", handle: "a", body: "hi" }); + expect(res.ok).toBe(true); + if (!res.ok) throw new Error("unreachable"); + expect(res.data.id).toBeGreaterThan(0); +}); + +test("chat:dm still reports success when emitEvent throws after the message is durable", async () => { + const h = freshHandlers(() => { throw new Error("events.db locked"); }); + const res = await h["chat:dm"]({ from: "agent", to: "matt", body: "ping" }); + expect(res.ok).toBe(true); + if (!res.ok) throw new Error("unreachable"); + expect(res.data.id).toBeGreaterThan(0); +}); + test("chat:post rejects an invalid mentions element with a reason rather than storing it", async () => { const h = freshHandlers(); await h["chat:join"]({ room: "r", handle: "a" }); @@ -86,6 +118,102 @@ test("chat:post rejects an invalid mentions element with a reason rather than st expect(res.error).toContain("handle"); }); +// R010: a typo'd room silently no-op'd through postMessage's REVIVE (a +// no-op for a room with no chat_rooms row) and returned {ok:true, +// recipients:[]} — no error, no listing, unreachable except by the exact +// typo'd name. It must fail loudly instead. +test("chat:post refuses a room nobody has joined instead of silently black-holing the message", async () => { + const h = freshHandlers(); + const res = await h["chat:post"]({ room: "typo-room", handle: "a", body: "hi" }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("typo-room"); +}); + +test("chat:post's unknown-room error names a close existing room", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "deck-main", handle: "a" }); + const res = await h["chat:post"]({ room: "deck", handle: "a", body: "hi" }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("deck-main"); +}); + +test("chat:post rejects a missing or empty body rather than routing it through unenforced", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + const missing = await h["chat:post"]({ room: "r", handle: "a" } as any); + expect(missing.ok).toBe(false); + const empty = await h["chat:post"]({ room: "r", handle: "a", body: "" }); + expect(empty.ok).toBe(false); +}); + +test("chat:post rejects a body over the size cap", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + const res = await h["chat:post"]({ room: "r", handle: "a", body: "x".repeat(64 * 1024 + 1) }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error.toLowerCase()).toContain("body"); +}); + +test("chat:post rejects a non-array mentions field", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + const res = await h["chat:post"]({ room: "r", handle: "a", body: "hi", mentions: "b" as any }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("mentions"); +}); + +test("chat:dm rejects a missing or empty body", async () => { + const h = freshHandlers(); + const missing = await h["chat:dm"]({ from: "a", to: "b" } as any); + expect(missing.ok).toBe(false); + const empty = await h["chat:dm"]({ from: "a", to: "b", body: "" }); + expect(empty.ok).toBe(false); +}); + +// R034: `limit: -1` reaches `ORDER BY id ASC LIMIT ?`, where SQLite treats a +// negative LIMIT as unlimited, so a viewer/agent bug returns and +// JSON-serializes an entire (100k-row) room on the event loop. +test("chat:messages clamps a negative limit into [1,500] instead of reaching SQLite's unlimited LIMIT", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + for (let i = 0; i < 502; i++) postMessage({ room: "r", handle: "a", body: `msg ${i}` }, h.db); + const res = await h["chat:messages"]({ room: "r", limit: -1 }); + if (!res.ok) throw new Error("unreachable"); + expect(res.data.messages.length).toBeGreaterThanOrEqual(1); + expect(res.data.messages.length).toBeLessThanOrEqual(500); +}); + +test("chat:messages clamps an absurdly large limit to the cap", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + for (let i = 0; i < 502; i++) postMessage({ room: "r", handle: "a", body: `msg ${i}` }, h.db); + const res = await h["chat:messages"]({ room: "r", limit: 1_000_000 }); + if (!res.ok) throw new Error("unreachable"); + expect(res.data.messages.length).toBe(500); +}); + +test("chat:messages coerces a non-numeric limit to the default instead of a datatype 500", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "a" }); + await h["chat:post"]({ room: "r", handle: "a", body: "hi" }); + const res = await h["chat:messages"]({ room: "r", limit: "lots" as any }); + expect(res.ok).toBe(true); +}); + +test("chat:read clamps a negative limit into [1,500] rather than reaching SQLite's unlimited LIMIT", async () => { + const h = freshHandlers(); + await h["chat:join"]({ room: "r", handle: "b" }); // b's own cursor starts before every message below + for (let i = 0; i < 502; i++) postMessage({ room: "r", handle: "a", body: `msg ${i}` }, h.db); + const res = await h["chat:read"]({ handle: "b", limit: -1 } as any); + if (!res.ok) throw new Error("unreachable"); + expect(res.data.rooms[0]!.messages.length).toBeGreaterThanOrEqual(1); + expect(res.data.rooms[0]!.messages.length).toBeLessThanOrEqual(500); +}); + test("the read-only handlers mutate nothing", async () => { const h = freshHandlers(); await h["chat:join"]({ room: "r", handle: "a" }); @@ -473,6 +601,35 @@ test("sign-in rejects an invalid baseHandle with a reason rather than normalizin expect(res.error).toContain("handle"); }); +// S076: a missing/empty sessionId binds as NULL against session_id TEXT +// PRIMARY KEY, which SQLite accepts — the row then holds the UNIQUE handle +// but the reclaim-by-session_id path can never match it, wedging every +// later sign-in under that base handle with a UNIQUE constraint failure. +test("sign-in rejects a missing sessionId rather than storing a NULL-keyed row", async () => { + const h = freshHandlers(); + const res = await h["chat:sign-in"]({ baseHandle: "x" } as any); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("sessionId"); +}); + +test("sign-in rejects an empty-string sessionId the same way", async () => { + const h = freshHandlers(); + const res = await h["chat:sign-in"]({ sessionId: "", baseHandle: "x" }); + expect(res.ok).toBe(false); + if (res.ok) throw new Error("unreachable"); + expect(res.error).toContain("sessionId"); +}); + +test("a rejected sign-in never wedges the next sign-in under the same base handle", async () => { + const h = freshHandlers(); + await h["chat:sign-in"]({ baseHandle: "x" } as any); // rejected, must not persist a row + const ok = await h["chat:sign-in"]({ sessionId: "s1", baseHandle: "x" }); + expect(ok.ok).toBe(true); + if (!ok.ok) throw new Error("unreachable"); + expect(ok.data).toMatchObject({ handle: "x" }); +}); + test("chat:sign-out is a no-op success for a session that never signed in", async () => { const h = freshHandlers(); const res = await h["chat:sign-out"]({ sessionId: "never-signed-in" }); @@ -894,3 +1051,21 @@ test("chat:dm-open refuses a reclaimed sender the same way chat:dm does", async const res = await h["chat:dm-open"]({ from: "a", to: "b", sessionId: "s1" }); expect(res.ok).toBe(false); }); + +test("chat:post warns through the injected logger (ctx.log), not a module-private lazyChildLogger (C6)", async () => { + const db = openStateDb(join(tmpdir(), `chat-h-log-${process.pid}-${n++}.db`)); + const warnCalls: unknown[] = []; + const log = { + info: () => {}, debug: () => {}, error: () => {}, + warn: (...args: unknown[]) => { warnCalls.push(args); }, + } as any; + const h = createChatHandlers({ + db, + emitEvent: () => { throw new Error("emit boom"); }, // postAndNotify's own warn path + log, + }); + await h["chat:join"]({ room: "r", handle: "a" }); + const res = await h["chat:post"]({ room: "r", handle: "a", body: "hi" }); + expect(res.ok).toBe(true); + expect(warnCalls.length).toBeGreaterThan(0); +}); diff --git a/lib/daemon/__tests__/cron.test.ts b/lib/daemon/__tests__/cron.test.ts index 1c535b195..b09ab97b0 100644 --- a/lib/daemon/__tests__/cron.test.ts +++ b/lib/daemon/__tests__/cron.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { mkdtempSync, realpathSync, rmSync } from "fs"; +import { mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; import { setSetting } from "../../settings/write.ts"; @@ -121,4 +121,36 @@ describe("startCron", () => { await sleep(40); expect(runs).toHaveLength(0); }); + + // The real (non-overridden) spawn path: Bun.spawn without an explicit + // `env` key gives the child the env snapshot from when THIS bun process + // started, not process.env as mutated at runtime (verified directly: + // omitting `env` leaves a child reading a var reassigned post-startup at + // its ORIGINAL value; `env: { ...process.env }` gives it the live one). + // That is exactly what the daemon's boot-time PATH overlay does to + // process.env.PATH, so this exercises the same shape with a plain var — + // an absolute argv[0] sidesteps unrelated PATH-search mechanics. + test("cron spawns with the live process.env, not the frozen start snapshot", async () => { + const dir = realpathSync(mkdtempSync(join(tmpdir(), "rt-cron-env-"))); + const marker = join(dir, "ran.txt"); + const script = join(dir, "marker.sh"); + writeFileSync(script, `#!/bin/sh\necho "$CRON_TEST_VALUE" > "${marker}"\n`, { mode: 0o755 }); + + const varName = "CRON_TEST_VALUE"; + delete process.env[varName]; // absent at this bun process's own startup + process.env[varName] = "live-value"; // set only after startup, like resolveUserPath does to PATH + try { + const cron = startCron( + { triggers: [{ name: "t", event: "tick", run: [script], debounceMs: 5 }] }, + { log }, + ); + cron.onBroadcast("tick", null); + await sleep(500); // real process spawn + exit, not a mocked runCommand + cron.dispose(); + expect(readFileSync(marker, "utf8").trim()).toBe("live-value"); + } finally { + delete process.env[varName]; + rmSync(dir, { recursive: true, force: true }); + } + }); }); diff --git a/lib/daemon/__tests__/demand-tracker.test.ts b/lib/daemon/__tests__/demand-tracker.test.ts new file mode 100644 index 000000000..82c4ebc34 --- /dev/null +++ b/lib/daemon/__tests__/demand-tracker.test.ts @@ -0,0 +1,24 @@ +import { test, expect } from "bun:test"; +import { recordDemand, demandedWithin, wrapWithDemand } from "../demand-tracker.ts"; + +test("demandedWithin reflects a recent recordDemand", () => { + recordDemand(); + expect(demandedWithin(60_000)).toBe(true); + expect(demandedWithin(0)).toBe(false); // window of 0ms is never "recent" +}); + +test("wrapWithDemand records demand and delegates to the inner handler", async () => { + let called = false; + // Typed variadic, matching how buildRoutedHandlers' entries are actually invoked + // (some take a payload, some don't) — a fixed 0-arg signature would reject that call shape. + const handlers: Record Promise<{ ok: boolean }>> = { + "system-processes": async () => { called = true; return { ok: true }; }, + other: async () => ({ ok: true }), + }; + wrapWithDemand(handlers, ["system-processes"]); + const before = demandedWithin(50); + await handlers["system-processes"]!(undefined as any); + expect(called).toBe(true); + expect(demandedWithin(1000)).toBe(true); + void before; +}); diff --git a/lib/daemon/__tests__/discussions-diffs.test.ts b/lib/daemon/__tests__/discussions-diffs.test.ts new file mode 100644 index 000000000..bfd945c11 --- /dev/null +++ b/lib/daemon/__tests__/discussions-diffs.test.ts @@ -0,0 +1,62 @@ +/** + * discussions:diffs (S053 + S086): the outbound GitLab fetch must carry a + * bound signal (so a stalled connection doesn't orphan the promise and the + * sops-decrypted token in its closure forever), and a full page (100 rows) + * must be reported as `truncated` rather than silently dropped. + */ +import { expect, test } from "bun:test"; +import { fetchMrDiffs } from "../handlers/discussions.ts"; + +function fakeDiffs(n: number) { + return Array.from({ length: n }, (_, i) => ({ new_path: `file${i}.ts`, diff: `@@ diff ${i}` })); +} + +test("truncated is false when fewer than a full page comes back", async () => { + const fetchFn = (async () => new Response(JSON.stringify(fakeDiffs(3)), { status: 200 })) as unknown as typeof fetch; + const out = await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); + expect(out.diffs).toHaveLength(3); + expect(out.truncated).toBe(false); +}); + +test("truncated is true when exactly a full page (100) comes back", async () => { + const fetchFn = (async () => new Response(JSON.stringify(fakeDiffs(100)), { status: 200 })) as unknown as typeof fetch; + const out = await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); + expect(out.diffs).toHaveLength(100); + expect(out.truncated).toBe(true); +}); + +test("maps new_path/diff to newPath/diff", async () => { + const fetchFn = (async () => new Response(JSON.stringify([{ new_path: "a.ts", diff: "@@" }]), { status: 200 })) as unknown as typeof fetch; + const out = await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); + expect(out.diffs).toEqual([{ newPath: "a.ts", diff: "@@" }]); +}); + +test("a non-ok response throws with the status", async () => { + const fetchFn = (async () => new Response("", { status: 502 })) as unknown as typeof fetch; + await expect(fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn })).rejects.toThrow(/502/); +}); + +test("the fetch call carries an AbortSignal", async () => { + let sawSignal: AbortSignal | undefined; + const fetchFn = (async (_url: any, init: any) => { + sawSignal = init?.signal; + return new Response(JSON.stringify([]), { status: 200 }); + }) as typeof fetch; + await fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn }); + expect(sawSignal).toBeInstanceOf(AbortSignal); +}); + +test("aborting the caller's own request signal cancels the in-flight fetch", async () => { + const controller = new AbortController(); + let sawSignal: AbortSignal | undefined; + const fetchFn = (async (_url: any, init: any) => { + sawSignal = init?.signal; + return new Promise((_resolve, reject) => { + init.signal.addEventListener("abort", () => reject(new Error("aborted"))); + }); + }) as typeof fetch; + const promise = fetchMrDiffs("https://gitlab.example.com", "g/repo", 7, "tok", { fetchFn, reqSignal: controller.signal }); + controller.abort(); + await expect(promise).rejects.toThrow(); + expect(sawSignal?.aborted).toBe(true); +}); diff --git a/lib/daemon/__tests__/events-bus.test.ts b/lib/daemon/__tests__/events-bus.test.ts index a1a85c03d..c10e038f0 100644 --- a/lib/daemon/__tests__/events-bus.test.ts +++ b/lib/daemon/__tests__/events-bus.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect, beforeEach, afterEach } from "bun:test"; -import { mkdtempSync, rmSync } from "fs"; +import { mkdtempSync, rmSync, writeFileSync, readdirSync } from "fs"; import { join } from "path"; import { tmpdir } from "os"; import pino from "pino"; @@ -7,6 +7,37 @@ import { createEventsBus, matchTopic, type EventsBus } from "../events-bus.ts"; const log = pino({ level: "silent" }); +describe("events bus corruption + pragmas", () => { + test("createEventsBus quarantines and recreates a corrupt events.db instead of throwing", () => { + const dir = mkdtempSync(join(tmpdir(), "events-corrupt-")); + try { + const dbPath = join(dir, "events.db"); + writeFileSync(dbPath, "garbage not sqlite"); + const bus = createEventsBus({ dbPath, log }); + expect(readdirSync(dir).some((f) => f.startsWith("events.db.corrupt-"))).toBe(true); + // fresh db works: + bus.emit("test", { hi: 1 }); + expect(bus.list({ pattern: "**" }).events.length).toBe(1); + bus.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("createEventsBus sets busy_timeout and synchronous=NORMAL", () => { + const dir = mkdtempSync(join(tmpdir(), "events-pragma-")); + try { + const bus = createEventsBus({ dbPath: join(dir, "events.db"), log }); + const handle = bus.__db!; + expect(handle.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); + expect(handle.query("PRAGMA synchronous").get()).toEqual({ synchronous: 1 }); // NORMAL + bus.close(); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + describe("matchTopic", () => { test("bare topic matches itself only", () => { expect(matchTopic("job/x/question", "job/x/question")).toBe(true); diff --git a/lib/daemon/__tests__/freshness-pending-cap.test.ts b/lib/daemon/__tests__/freshness-pending-cap.test.ts new file mode 100644 index 000000000..f488ba9e3 --- /dev/null +++ b/lib/daemon/__tests__/freshness-pending-cap.test.ts @@ -0,0 +1,13 @@ +import { test, expect } from "bun:test"; +import { applyInvalidationBatch, PENDING_CAP } from "../freshness.ts"; + +test("merged pending is deduped by kind:ref and capped", async () => { + const runner: any = { processing: true, pending: [] }; + // Push more distinct keys than the cap; plus duplicates. + const keys = Array.from({ length: PENDING_CAP + 500 }, (_, i) => ({ kind: "mr" as const, ref: String(i), cause: "test" })); + const dupes = [{ kind: "mr" as const, ref: "0", cause: "test" }, { kind: "mr" as const, ref: "0", cause: "test" }]; + await applyInvalidationBatch({} as any, {} as any, runner, [...keys, ...dupes], {}); + expect(runner.pending.length).toBeLessThanOrEqual(PENDING_CAP); + const ids = runner.pending.map((k: any) => `${k.kind}:${k.ref}`); + expect(new Set(ids).size).toBe(ids.length); // no duplicates +}); diff --git a/lib/daemon/__tests__/freshness-poll-userid.test.ts b/lib/daemon/__tests__/freshness-poll-userid.test.ts new file mode 100644 index 000000000..5e3cbc559 --- /dev/null +++ b/lib/daemon/__tests__/freshness-poll-userid.test.ts @@ -0,0 +1,39 @@ +/** + * S022: userId used to resolve only inside reconcileFreshnessImpl's + * live-mode-only loop, so a poll-only tracked repo (mode: "poll") never + * built a provider, ensureUserId() never ran, getCurrentUserId() stayed + * null forever, and checkAndNotify silently suppressed every + * self-authored transition. Static source checks (matching this file's + * established S048/S049 test style — see freshness-provider-rotation.test.ts — + * since GitLabProvider is an external network client with no test seam). + */ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +const freshnessSrc = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); +const cacheRefreshSrc = readFileSync(resolve(import.meta.dir, "..", "cache-refresh.ts"), "utf8"); + +test("S022: a mode-independent userId resolver is exported from freshness.ts", () => { + expect(freshnessSrc).toMatch(/export async function resolveUserIdAcrossTracking\(/); +}); + +test("S022: the resolver gates on the branches/project-mrs grant, not on live mode", () => { + const fn = freshnessSrc.match(/export async function resolveUserIdAcrossTracking\([\s\S]*?\n\}\n/)?.[0]; + expect(fn).toBeTruthy(); + expect(fn).toMatch(/caches\.has\(["']branches["']\)/); + expect(fn).toMatch(/caches\.has\(["']project-mrs["']\)/); + expect(fn).not.toMatch(/mode\s*!==\s*["']live["']/); +}); + +test("S022: cache-refresh.ts resolves userId before checkAndNotify (cycle-1 fix)", () => { + const resolveIndex = cacheRefreshSrc.indexOf("resolveUserIdAcrossTracking("); + const checkAndNotifyIndex = cacheRefreshSrc.indexOf("checkAndNotify(cache.entries"); + expect(resolveIndex).toBeGreaterThan(-1); + expect(checkAndNotifyIndex).toBeGreaterThan(-1); + expect(resolveIndex).toBeLessThan(checkAndNotifyIndex); +}); + +test("S022: warns once when transitions are suppressed because userId never resolved", () => { + expect(freshnessSrc).toMatch(/userId is unresolved/); +}); diff --git a/lib/daemon/__tests__/freshness-provider-rotation.test.ts b/lib/daemon/__tests__/freshness-provider-rotation.test.ts new file mode 100644 index 000000000..bcda8181c --- /dev/null +++ b/lib/daemon/__tests__/freshness-provider-rotation.test.ts @@ -0,0 +1,44 @@ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("ensureProvider compares the current token before reusing a cached provider", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + // The unconditional cache-hit return is the S049 bug; it must be gone. + expect(src).not.toMatch(/const cached = providers\.get\(repoName\);\s*\n\s*if \(cached\) return cached;/); + // A token fingerprint must be stored alongside the provider. + expect(src).toMatch(/providers\.set\(repoName,\s*\{\s*provider/); +}); + +test("getRepoContext drops a stale-token provider before serving the cached one (S048/S049 fix 1)", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + const tokenCheckMatch = src.match(/cachedForToken\.token !== currentSecrets\.gitlabToken/); + expect(tokenCheckMatch).not.toBeNull(); + // The comparison must run before the cache is read for the fast path, so a + // rotated token can never reach a poll-mode repo or an already-cached + // forge-handler provider. + const fastPathIndex = src.indexOf("const watch = watches.get(repoName);"); + expect(fastPathIndex).toBeGreaterThan(-1); + expect(tokenCheckMatch!.index!).toBeLessThan(fastPathIndex); +}); + +test("getRepoContext invalidates the cached provider when gitlabToken is removed, not just rotated (C5)", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + // The buggy gate: invalidation only ran when a NEW token was present, so a + // caller whose secrets lost gitlabToken entirely kept getting served the + // stale authenticated provider. It must be gone. + expect(src).not.toMatch(/if \(currentSecrets\.gitlabToken && cachedForToken\.token !== currentSecrets\.gitlabToken\)/); + // The fixed condition must still invalidate on a plain mismatch, absent value included. + expect(src).toMatch(/if \(cachedForToken\.token !== currentSecrets\.gitlabToken\)/); +}); + +test("reconcileFreshnessImpl drops a stale-token watch before skipping already-watched repos (S048/S049 fix 2)", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + const staleWatchDrop = src.match(/existing\.token !== secrets\.gitlabToken\)\s*stopWatch\(repoName\);/); + expect(staleWatchDrop).not.toBeNull(); + // The drop must happen before the "already watched, skip" short-circuit, + // or a live watcher built on a rotated token never rebuilds. + const skipIndex = src.indexOf("if (watches.has(repoName)) continue;"); + expect(skipIndex).toBeGreaterThan(-1); + expect(staleWatchDrop!.index!).toBeLessThan(skipIndex); +}); diff --git a/lib/daemon/__tests__/freshness-redact-credentials.test.ts b/lib/daemon/__tests__/freshness-redact-credentials.test.ts new file mode 100644 index 000000000..c4228bb3a --- /dev/null +++ b/lib/daemon/__tests__/freshness-redact-credentials.test.ts @@ -0,0 +1,22 @@ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; +import { redactCredentials } from "../redact-credentials.ts"; + +const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + +test("S050: every log/error interpolation of a remote URL runs it through redactCredentials", () => { + expect(src).toMatch(/log\.info\(`remote "\$\{redactCredentials\(remoteUrl\)\}" for \$\{repoName\}/); + expect(src).toMatch(/log\.info\(`could not parse remote "\$\{redactCredentials\(remoteUrl\)\}"/); + expect(src).toMatch(/throw new Error\(`could not parse remote URL "\$\{redactCredentials\(remoteUrl\)\}"`\)/); +}); + +test("S050: no remaining bare ${remoteUrl} interpolation in freshness.ts", () => { + expect(src).not.toMatch(/\$\{remoteUrl\}/); +}); + +test("redactCredentials strips userinfo from a credentialed URL", () => { + expect(redactCredentials("https://oauth2:glpat-XXXX@gitlab.example.com/a/b.git")).toBe( + "https://[redacted]@gitlab.example.com/a/b.git", + ); +}); diff --git a/lib/daemon/__tests__/freshness-remote-url.test.ts b/lib/daemon/__tests__/freshness-remote-url.test.ts new file mode 100644 index 000000000..ec17a900a --- /dev/null +++ b/lib/daemon/__tests__/freshness-remote-url.test.ts @@ -0,0 +1,9 @@ +import { test, expect } from "bun:test"; +import { readFileSync } from "fs"; +import { resolve } from "path"; + +test("freshness.ts no longer imports execSync/child_process", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + expect(src).not.toMatch(/from\s+["']child_process["']/); + expect(src).not.toMatch(/\bexecSync\b/); +}); diff --git a/lib/daemon/__tests__/git-ref-validation.test.ts b/lib/daemon/__tests__/git-ref-validation.test.ts new file mode 100644 index 000000000..27c9fc3d7 --- /dev/null +++ b/lib/daemon/__tests__/git-ref-validation.test.ts @@ -0,0 +1,36 @@ +import { describe, test, expect } from "bun:test"; +import { isSafeGitRef, validateGitRef } from "../git-ref-validation.ts"; + +describe("isSafeGitRef", () => { + test("a normal branch name is safe", () => { + expect(isSafeGitRef("feature/my-branch")).toBe(true); + }); + + test("a leading dash is unsafe (option injection)", () => { + expect(isSafeGitRef("--upload-pack=touch /tmp/x")).toBe(false); + }); + + test("a bare dash is unsafe", () => { + expect(isSafeGitRef("-")).toBe(false); + }); + + test("an empty string is unsafe", () => { + expect(isSafeGitRef("")).toBe(false); + }); + + test("a branch containing a dash mid-name is safe", () => { + expect(isSafeGitRef("job/p3-trust-boundary")).toBe(true); + }); +}); + +describe("validateGitRef", () => { + test("returns ok:true for a safe ref", () => { + expect(validateGitRef("main")).toEqual({ ok: true }); + }); + + test("returns ok:false with the offending ref named in the error for an unsafe one", () => { + const result = validateGitRef("--upload-pack=x"); + expect(result.ok).toBe(false); + expect((result as { ok: false; error: string }).error).toContain("--upload-pack=x"); + }); +}); diff --git a/lib/daemon/__tests__/home-snapshot.test.ts b/lib/daemon/__tests__/home-snapshot.test.ts index 569aa4c63..33cc9df46 100644 --- a/lib/daemon/__tests__/home-snapshot.test.ts +++ b/lib/daemon/__tests__/home-snapshot.test.ts @@ -1005,6 +1005,43 @@ describe("startHomeSnapshot — state persistence", () => { }); }); +// ─── boot order: db must open daemon-flavored, never at construction ──────── + +describe("startHomeSnapshot (boot order)", () => { + test("constructing startHomeSnapshot does not open the state.db singleton before the caller's next await", async () => { + const home = mkdtempSync(join(tmpdir(), "rt-home-snapshot-bootorder-")); + const origHome = process.env.HOME; + process.env.HOME = home; + closeStateDb(); + try { + const stateDbPath = join(home, ".mattstack", "rt", "state.db"); + const { fn: execFn } = makeFakeExec(defaultResponders({ statusZ: "?? a.txt\0" })); + // No `db` override: this exercises the real getStateDb() singleton, + // matching lib/daemon.ts's module-scope `startHomeSnapshot(...)` call, + // the exact call site that used to open state.db "cli"-flavored before + // startDaemon() ever got to openBranchCacheStore(). + const { deps } = baseDeps({ exec: execFn, db: undefined }); + + const handle = startHomeSnapshot(deps); + + // Synchronously, right after construction returns, mirroring the + // module-scope call in lib/daemon.ts, which runs to completion before + // startDaemon() (and its openBranchCacheStore() daemon-flavored open) + // is ever reached, no db file may exist yet. + expect(existsSync(stateDbPath)).toBe(false); + + await handle.ready; + // First real use (init()'s loadState, past its own `await deps.exec`) + // has by now opened it. + expect(existsSync(stateDbPath)).toBe(true); + } finally { + process.env.HOME = origHome; + closeStateDb(); + rmSync(home, { recursive: true, force: true }); + } + }); +}); + // ─── stop() ─────────────────────────────────────────────────────────────── describe("startHomeSnapshot — stop", () => { diff --git a/lib/daemon/__tests__/hooks-guard.test.ts b/lib/daemon/__tests__/hooks-guard.test.ts new file mode 100644 index 000000000..ee72af08e --- /dev/null +++ b/lib/daemon/__tests__/hooks-guard.test.ts @@ -0,0 +1,57 @@ +/** + * S057: fs.watch handles in hooks-guard have no 'error' listener (an + * emitted error with no listener is an uncaught exception, which + * installCrashHandlers turns into a daemon exit(1) and a launchd relaunch + * that re-arms the same watchers and hits the same limit); and + * refreshWatchedRepos is add-only, so a relocated or removed repo's stale + * watcher on a dead .git dir is kept forever. + */ +import { afterEach, beforeEach, expect, test } from "bun:test"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import pino from "pino"; +import { createHooksGuard } from "../hooks-guard.ts"; + +const log = pino({ level: "silent" }); + +let dir: string; +beforeEach(() => { dir = mkdtempSync(join(tmpdir(), "rt-hooks-guard-")); }); +afterEach(() => { rmSync(dir, { recursive: true, force: true }); }); + +function makeRepo(name: string): string { + const repoPath = join(dir, name); + mkdirSync(join(repoPath, ".git"), { recursive: true }); + writeFileSync(join(repoPath, ".git", "config"), "[core]\n"); + return repoPath; +} + +test("an emitted 'error' event on the watcher does not throw, and self-heals the bookkeeping", async () => { + const repoPath = makeRepo("a"); + const guard = createHooksGuard(log); + guard.startWatchingRepo("a", repoPath); + expect(guard.watchedConfigs.size).toBe(1); + + const [configPath, watcher] = [...guard.watchedConfigs.entries()][0]!; + expect(() => watcher.emit("error", new Error("EMFILE"))).not.toThrow(); + // give the close/delete a tick if it's deferred + await Bun.sleep(0); + expect(guard.watchedConfigs.has(configPath)).toBe(false); + guard.closeAll(); +}); + +test("refreshWatchedRepos closes and drops a watcher whose repo left the index (relocated or removed)", () => { + const repoPathA = makeRepo("a"); + const repoPathB = makeRepo("b"); + let index: Record = { a: repoPathA, b: repoPathB }; + const guard = createHooksGuard(log, { loadRepoIndexFn: () => index }); + + guard.refreshWatchedRepos(); + expect(guard.watchedConfigs.size).toBe(2); + + // "b" is relocated/removed: the index no longer carries it. + index = { a: repoPathA }; + guard.refreshWatchedRepos(); + expect(guard.watchedConfigs.size).toBe(1); + guard.closeAll(); +}); diff --git a/lib/daemon/__tests__/pane-handlers.test.ts b/lib/daemon/__tests__/pane-handlers.test.ts index fbae6a7e2..11f89ccf7 100644 --- a/lib/daemon/__tests__/pane-handlers.test.ts +++ b/lib/daemon/__tests__/pane-handlers.test.ts @@ -329,6 +329,28 @@ test("pane:spawn stops polling for registration at the wall-clock budget, not a expect(calls.filter((c) => c === "agent.get").length).toBeLessThanOrEqual(2); }); +// S087: pane:spawn's worst-case daemon budget (register + idle wait + trust +// retry + prompt) exceeds rt-client's client-side timeout — the client +// gives up and reports failure while the daemon keeps working, so a retry +// spawns a second claude pane in the same cwd. Once the caller's own +// AbortSignal fires, the handler must stop spending further budget and +// return the pane it already created instead of continuing the full flow. +test("pane:spawn stops after tab creation once the caller's AbortSignal fires, returning the pane already created", async () => { + const controller = new AbortController(); + const { handler, calls } = spawnFake({ statuses: ["idle"] }); + const abortingHandler: typeof handler = (method, params) => { + if (method === "pane.send_input") controller.abort(); + return handler(method, params); + }; + const { pane } = harness(abortingHandler); + const res = await pane["pane:spawn"]({ cwd: "/repos/chat" }, controller.signal); + if (!res.ok) throw new Error(res.error); + expect(res.data.pane.paneId).toBe("w2:p7"); + expect(res.data.ready).toBe(false); + expect(calls).not.toContain("agent.get"); + expect(calls).not.toContain("agent.wait"); +}); + test("pane:spawn refuses an unknown cswap account before touching herdr", async () => { const { handler, calls } = spawnFake({ statuses: ["idle"] }); const { pane } = harness(handler); diff --git a/lib/daemon/__tests__/project-sync.test.ts b/lib/daemon/__tests__/project-sync.test.ts index 2437da6fa..576123849 100644 --- a/lib/daemon/__tests__/project-sync.test.ts +++ b/lib/daemon/__tests__/project-sync.test.ts @@ -494,6 +494,35 @@ describe("project-mrs:read handler", async () => { expect(forgeCalls).toBe(1); }); + // S088: mr:by-branch's forge write-back upserted unconditionally, bypassing + // the same scope/tagged gate upsertProject (lib/daemon/freshness.ts) + // enforces for every other write path — a demand-scoped repo could pick up + // a stranger's MR that the next delta sync filters right back out, then + // reappears on the next by-branch call. The caller still gets the PR (it + // asked for this exact branch); it just must not land in the store. + test("by-branch: forge write-back respects the same demand scope the sync path enforces", async () => { + const store = tmpStore(); + store.fullSync("remote:repo", "g/p", [], Date.now()); + store.setScope("remote:repo", { authors: ["alice"], windowDays: 30 }); + const h = createProjectMRsHandlers(fakeCtx, () => {}, { store, sync: async () => {}, tracking: grantedTracking, + fetchByBranch: async (_r, branch) => ({ pr: pr(9, { sourceBranch: branch, author: { id: 2, username: "stranger" } as any }), projectPath: "g/p" }) }); + const res = await h["mr:by-branch"]!({ repoName: "remote:repo", branches: ["feat-x"] }); + expect(res.ok).toBe(true); + expect((dataOf(res) as any).byBranch["feat-x"]).toMatchObject({ source: "forge", pr: { iid: 9 } }); + expect(store.read("remote:repo")!.mrs[9]).toBeUndefined(); + }); + + test("by-branch: forge write-back still stores an in-scope author's MR", async () => { + const store = tmpStore(); + store.fullSync("remote:repo", "g/p", [], Date.now()); + store.setScope("remote:repo", { authors: ["alice"], windowDays: 30 }); + const h = createProjectMRsHandlers(fakeCtx, () => {}, { store, sync: async () => {}, tracking: grantedTracking, + fetchByBranch: async (_r, branch) => ({ pr: pr(10, { sourceBranch: branch, author: { id: 1, username: "alice" } as any }), projectPath: "g/p" }) }); + const res = await h["mr:by-branch"]!({ repoName: "remote:repo", branches: ["feat-y"] }); + expect(res.ok).toBe(true); + expect(store.read("remote:repo")!.mrs[10]).toBeDefined(); + }); + test("by-branch: no MR anywhere is null; a per-branch forge failure is null with a warn, not a batch failure", async () => { const store = tmpStore(); store.fullSync("remote:repo", "g/p", [pr(1, { sourceBranch: "ok" })], Date.now()); diff --git a/lib/daemon/__tests__/redact-credentials.test.ts b/lib/daemon/__tests__/redact-credentials.test.ts new file mode 100644 index 000000000..57e5a5e25 --- /dev/null +++ b/lib/daemon/__tests__/redact-credentials.test.ts @@ -0,0 +1,35 @@ +import { describe, test, expect } from "bun:test"; +import { redactCredentials } from "../redact-credentials.ts"; + +describe("redactCredentials", () => { + test("redacts userinfo (user:token@) out of an https remote URL", () => { + const input = "https://oauth2:glpat-XXXXXXXXXXXXXXXXXXXX@gitlab.example.com/g/p.git"; + const out = redactCredentials(input); + expect(out).not.toContain("glpat-XXXXXXXXXXXXXXXXXXXX"); + expect(out).toContain("gitlab.example.com/g/p.git"); + }); + + test("redacts a GitHub PAT embedded in the URL", () => { + const input = "https://ghp_abcdefghijklmnopqrstuvwxyz012345@github.com/o/r.git"; + const out = redactCredentials(input); + expect(out).not.toContain("ghp_abcdefghijklmnopqrstuvwxyz012345"); + expect(out).toContain("github.com/o/r.git"); + }); + + test("leaves a URL with no embedded credentials unchanged", () => { + const input = "https://gitlab.example.com/g/p.git"; + expect(redactCredentials(input)).toBe(input); + }); + + test("leaves plain text with no URL unchanged", () => { + const input = "local branch listing failed"; + expect(redactCredentials(input)).toBe(input); + }); + + test("redacts every match when more than one credential-bearing URL appears in the same string", () => { + const input = "tried https://oauth2:tok1@a.example/x then https://oauth2:tok2@b.example/y"; + const out = redactCredentials(input); + expect(out).not.toContain("tok1"); + expect(out).not.toContain("tok2"); + }); +}); diff --git a/lib/daemon/__tests__/request-body-size.test.ts b/lib/daemon/__tests__/request-body-size.test.ts new file mode 100644 index 000000000..b6ee04a59 --- /dev/null +++ b/lib/daemon/__tests__/request-body-size.test.ts @@ -0,0 +1,42 @@ +/** + * Bun enforces `maxRequestBodySize` itself (413 before the handler runs) -- + * this is a live-server test, not a pure-function one, because there is no + * pure function to unit test: the cap is a Bun.serve runtime option. + */ +import { describe, test, expect, afterEach } from "bun:test"; +import type { Server } from "bun"; +import { MAX_REQUEST_BODY_SIZE } from "../request-limits.ts"; + +let server: Server | undefined; + +afterEach(() => { + server?.stop(true); + server = undefined; +}); + +describe("MAX_REQUEST_BODY_SIZE", () => { + test("is set to 1 MiB", () => { + expect(MAX_REQUEST_BODY_SIZE).toBe(1024 * 1024); + }); + + test("Bun rejects a body over the cap with a 4xx before the handler runs", async () => { + let handlerRan = false; + server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + maxRequestBodySize: 10, // tiny cap for a fast, deterministic test + async fetch(req) { + handlerRan = true; + await req.text(); + return new Response("ok"); + }, + }); + const res = await fetch(`http://127.0.0.1:${server.port}/`, { + method: "POST", + body: "x".repeat(1000), + }); + expect(res.status).toBeGreaterThanOrEqual(400); + expect(res.status).toBeLessThan(500); + expect(handlerRan).toBe(false); + }); +}); diff --git a/lib/daemon/__tests__/safe-timers.test.ts b/lib/daemon/__tests__/safe-timers.test.ts new file mode 100644 index 000000000..13ab83b89 --- /dev/null +++ b/lib/daemon/__tests__/safe-timers.test.ts @@ -0,0 +1,55 @@ +import { describe, test, expect, mock } from "bun:test"; +import pino from "pino"; +import { safeInterval, safeTimeout } from "../safe-timers.ts"; + +const silentLog = pino({ level: "silent" }); + +describe("safeInterval / safeTimeout", () => { + test("safeInterval swallows a throwing tick and logs warn", async () => { + const warn = mock(() => {}); + const log = { ...silentLog, warn } as unknown as typeof silentLog; + let ticks = 0; + const handle = safeInterval( + () => { + ticks++; + throw new Error("SQLITE_FULL"); + }, + 10, + "test-sweep", + log, + ); + await new Promise((r) => setTimeout(r, 45)); + clearInterval(handle); + expect(ticks).toBeGreaterThan(0); + expect(warn).toHaveBeenCalled(); + }); + + test("safeTimeout swallows a throwing tick and logs warn", async () => { + const warn = mock(() => {}); + const log = { ...silentLog, warn } as unknown as typeof silentLog; + let ran = false; + safeTimeout( + () => { + ran = true; + throw new Error("SQLITE_FULL"); + }, + 10, + "test-timeout", + log, + ); + await new Promise((r) => setTimeout(r, 45)); + expect(ran).toBe(true); + expect(warn).toHaveBeenCalled(); + }); + + test("safeInterval does not swallow a non-throwing tick's normal operation", async () => { + const warn = mock(() => {}); + const log = { ...silentLog, warn } as unknown as typeof silentLog; + let ticks = 0; + const handle = safeInterval(() => { ticks++; }, 10, "test-sweep-ok", log); + await new Promise((r) => setTimeout(r, 45)); + clearInterval(handle); + expect(ticks).toBeGreaterThan(0); + expect(warn).not.toHaveBeenCalled(); + }); +}); diff --git a/lib/daemon/__tests__/shutdown.test.ts b/lib/daemon/__tests__/shutdown.test.ts new file mode 100644 index 000000000..139cd69f1 --- /dev/null +++ b/lib/daemon/__tests__/shutdown.test.ts @@ -0,0 +1,60 @@ +import { existsSync, writeFileSync } from "fs"; +import { expect, test } from "bun:test"; +import type { Logger } from "pino"; +import { DAEMON_PID_PATH, DAEMON_SOCK_PATH } from "../../daemon-config.ts"; +import { createCleanup, makeGracefulExit } from "../shutdown.ts"; + +const silentLog = { info: () => {}, warn: () => {}, error: () => {}, debug: () => {} } as unknown as Logger; + +const cleanupDeps = { + servers: {}, + hooksGuard: { closeAll: () => {} } as any, + log: silentLog, +}; + +test("gracefulExit exits 0 after the shutdown verb, 1 on a bare signal", () => { + const exits: number[] = []; + const exit = (c?: number) => { exits.push(c ?? 0); }; + let viaVerb = false; + const handlers = makeGracefulExit({ cleanup: () => {}, flushLogs: () => {}, log: silentLog, + wasVerbShutdown: () => viaVerb, exit, recordCleanExit: () => {} }); + handlers("SIGTERM"); + expect(exits).toEqual([1]); + viaVerb = true; + handlers("SIGTERM"); + expect(exits).toEqual([1, 0]); +}); + +test("gracefulExit records the exit kind and code via recordCleanExit", () => { + const recorded: Array<{ kind: string; code: number }> = []; + let viaVerb = false; + const handlers = makeGracefulExit({ + cleanup: () => {}, + flushLogs: () => {}, + log: silentLog, + wasVerbShutdown: () => viaVerb, + exit: () => {}, + recordCleanExit: (kind, code) => { recorded.push({ kind, code }); }, + }); + handlers("SIGINT"); + expect(recorded).toEqual([{ kind: "signal", code: 1 }]); + viaVerb = true; + handlers("SIGHUP"); + expect(recorded).toEqual([{ kind: "signal", code: 1 }, { kind: "shutdown", code: 0 }]); +}); + +test("cleanup does not unlink rt.pid/rt.sock when the pid file belongs to another process", () => { + writeFileSync(DAEMON_PID_PATH, "999999"); + writeFileSync(DAEMON_SOCK_PATH, ""); + createCleanup({ ...cleanupDeps, pid: process.pid })(); + expect(existsSync(DAEMON_PID_PATH)).toBe(true); + expect(existsSync(DAEMON_SOCK_PATH)).toBe(true); +}); + +test("cleanup unlinks when the pid file is ours", () => { + writeFileSync(DAEMON_PID_PATH, String(process.pid)); + writeFileSync(DAEMON_SOCK_PATH, ""); + createCleanup({ ...cleanupDeps, pid: process.pid })(); + expect(existsSync(DAEMON_PID_PATH)).toBe(false); + expect(existsSync(DAEMON_SOCK_PATH)).toBe(false); +}); diff --git a/lib/daemon/__tests__/status-identity.test.ts b/lib/daemon/__tests__/status-identity.test.ts index c292aec2a..b92d66b43 100644 --- a/lib/daemon/__tests__/status-identity.test.ts +++ b/lib/daemon/__tests__/status-identity.test.ts @@ -18,6 +18,20 @@ describe("daemon identity", () => { expect(res).toMatchObject({ ok: true, flavor: "dev", version: "source", sourceRev: "abc1234" }); }); + test("ping carries a supervision summary (Task 10)", async () => { + const h = createStatusHandlers(fakeCtx()); + const res = (await h["ping"]!({}, undefined as any)) as any; + // Loose on values deliberately: daemon-supervision kv is process-wide + // (lib/daemon/supervision-state.ts, `getStateDb("daemon")`), so this test + // sharing a `bun test` process with supervision-state.test.ts can see + // whatever that suite last wrote. The shape/cap is what this test owns. + expect(typeof res.supervision.bootAttempts).toBe("number"); + expect(typeof res.supervision.lastReadyAt).toBe("number"); + expect(Array.isArray(res.supervision.recentFailures)).toBe(true); + expect(res.supervision.recentFailures.length).toBeLessThanOrEqual(3); + expect(res.supervision.lastExit === null || typeof res.supervision.lastExit === "object").toBe(true); + }); + test("status.data carries the identity object", async () => { const h = createStatusHandlers(fakeCtx()); const res = (await h["status"]!({}, undefined as any)) as any; diff --git a/lib/daemon/__tests__/status-repos-async.test.ts b/lib/daemon/__tests__/status-repos-async.test.ts new file mode 100644 index 000000000..37283af07 --- /dev/null +++ b/lib/daemon/__tests__/status-repos-async.test.ts @@ -0,0 +1,64 @@ +/** + * S055: "repos" used the sync `listWorktrees` (lib/git-worktrees.ts, + * execSync) on the daemon's event loop. Swapped for the async + * `listWorktreesAsync` (lib/worktree/git-async.ts, already used elsewhere + * in the daemon). + */ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { execSync } from "child_process"; +import { mkdtempSync, realpathSync, rmSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { createStatusHandlers } from "../handlers/status.ts"; + +let tmpRoot: string; + +beforeEach(() => { + tmpRoot = realpathSync(mkdtempSync(join(tmpdir(), "rt-status-repos-"))); +}); + +afterEach(() => { + try { rmSync(tmpRoot, { recursive: true, force: true }); } catch { /* */ } +}); + +function initRepo(path: string): void { + execSync(`git init -q "${path}"`); + writeFileSync(join(path, "README"), "x"); + execSync(`git -C "${path}" add . && git -C "${path}" -c user.email=t@t -c user.name=t commit -q -m init`); +} + +function fakeCtx(repos: Record): any { + return { + startedAt: 123, + identity: { flavor: "dev", version: "source", sourceRev: "abc1234", startedAt: 123 }, + watchedConfigs: new Map(), + cache: { entries: {} }, + portCacheRef: { ports: [], updatedAt: null }, + repoIndex: () => repos, + }; +} + +describe("status handlers — repos (S055 async worktree listing)", () => { + test("lists worktrees with a branch, omitting detached ones", async () => { + const repo = mkdtempSync(join(tmpRoot, "repo-")); + initRepo(repo); + const linked = join(tmpRoot, "linked"); + execSync(`git -C "${repo}" worktree add -q "${linked}" -b feat/x`); + + const h = createStatusHandlers(fakeCtx({ myrepo: repo })); + const res = (await h["repos"]!({}, undefined as any)) as any; + + expect(res.ok).toBe(true); + const worktrees = res.data.repos.myrepo.worktrees; + expect(worktrees.map((w: any) => w.path).sort()).toEqual([linked, repo].sort()); + expect(worktrees.every((w: any) => typeof w.branch === "string" && w.branch.length > 0)).toBe(true); + }); + + test("a repo whose git command fails (bad repoPath) yields no worktrees, not a throw", async () => { + const notARepo = mkdtempSync(join(tmpRoot, "not-a-repo-")); + const h = createStatusHandlers(fakeCtx({ broken: notARepo })); + const res = (await h["repos"]!({}, undefined as any)) as any; + expect(res.ok).toBe(true); + expect(res.data.repos.broken.worktrees).toEqual([]); + }); +}); diff --git a/lib/daemon/__tests__/supervision-state.test.ts b/lib/daemon/__tests__/supervision-state.test.ts new file mode 100644 index 000000000..828ead75a --- /dev/null +++ b/lib/daemon/__tests__/supervision-state.test.ts @@ -0,0 +1,73 @@ +import { describe, expect, test } from "bun:test"; +import { rmSync } from "fs"; +import { join } from "path"; +import { + recordBootAttempt, + recordDaemonReady, + recordBootFailure, + recordCleanExit, + readSupervisionState, + isCrashLooping, + writeBreadcrumb, + readBreadcrumb, +} from "../supervision-state.ts"; +import { RT_DIR } from "../../daemon-config.ts"; + +/** Test-only cleanup mirroring the breadcrumb file's path (production has + * no clear API; the daemon only ever writes or reads it). */ +function removeBreadcrumbFile(): void { + rmSync(join(RT_DIR, "daemon-boot.json"), { force: true }); +} + +describe("supervision-state kv round-trip", () => { + test("boot attempts, ready stamp, failures and last-exit round-trip through kv", () => { + recordBootAttempt(); + recordBootAttempt(); + recordDaemonReady(); + recordBootFailure("api", "EADDRINUSE"); + const s = readSupervisionState(); + expect(s.bootAttempts).toBe(2); + expect(s.lastReadyAt).toBeGreaterThan(0); + expect(s.recentFailures.at(-1)).toMatchObject({ phase: "api", reason: "EADDRINUSE" }); + expect(s.lastExit).toMatchObject({ kind: "boot-failed", code: 1 }); + }); + + test("recordCleanExit sets last-exit with the given kind and code", () => { + recordCleanExit("shutdown", 0); + const s = readSupervisionState(); + expect(s.lastExit).toMatchObject({ kind: "shutdown", code: 0 }); + }); + + test("recent-failures is capped at 10 entries", () => { + for (let i = 0; i < 15; i++) recordBootFailure("api", `err-${i}`); + const s = readSupervisionState(); + expect(s.recentFailures.length).toBe(10); + expect(s.recentFailures.at(-1)).toMatchObject({ reason: "err-14" }); + }); +}); + +describe("isCrashLooping", () => { + test("true at >=3 failures within the window", () => { + const now = 1_000_000; + const fails = [now - 10, now - 20, now - 30].map((at) => ({ at, phase: "api" as const, reason: "x" })); + expect(isCrashLooping({ bootAttempts: 3, lastReadyAt: 0, recentFailures: fails, lastExit: null }, now)).toBe(true); + const old = [{ at: now - 10 * 60_000, phase: "api" as const, reason: "x" }]; + expect(isCrashLooping({ bootAttempts: 1, lastReadyAt: 0, recentFailures: old, lastExit: null }, now)).toBe(false); + }); +}); + +describe("breadcrumb file", () => { + test("writeBreadcrumb then readBreadcrumb round-trips phase, pid, flavor", () => { + writeBreadcrumb("api"); + const b = readBreadcrumb(); + expect(b).not.toBeNull(); + expect(b?.phase).toBe("api"); + expect(b?.pid).toBe(process.pid); + expect(typeof b?.at).toBe("number"); + }); + + test("readBreadcrumb returns null when no breadcrumb has been written", () => { + removeBreadcrumbFile(); + expect(readBreadcrumb()).toBeNull(); + }); +}); diff --git a/lib/daemon/__tests__/system-process-scanner-resilience.test.ts b/lib/daemon/__tests__/system-process-scanner-resilience.test.ts new file mode 100644 index 000000000..c5db5939c --- /dev/null +++ b/lib/daemon/__tests__/system-process-scanner-resilience.test.ts @@ -0,0 +1,46 @@ +import { test, expect } from "bun:test"; +import { SystemProcessScanner } from "../system-process-scanner.ts"; + +/** Drives gather()'s return value directly so a null (failed lsof/ps tick) + * can be simulated without shelling out. */ +class FakeScanner extends SystemProcessScanner { + next: any[] | null = []; + protected override async gather(): Promise { + return this.next; + } +} + +function fakeProcess(pid: number, cpuPercent: number) { + return { + pid, + ppid: 1, + command: "node", + fullCommand: "node build.js", + cpuPercent, + rssKb: 10_000, + uptime: "01:00", + cwd: "/repo", + repo: "r", + worktree: null, + branch: null, + relativeDir: "", + port: null, + linearTicket: null, + packageScript: null, + }; +} + +test("a failed gather (null) keeps tracked windows and lastResult intact", async () => { + const s = new FakeScanner(); + + s.next = [fakeProcess(4242, 95)]; + const first = await s.scan(); + expect(first.find((p) => p.pid === 4242)).toBeTruthy(); + const firstSeen = s.getTracked(4242)?.firstSeen; + expect(firstSeen).toBeDefined(); + + s.next = null; + const during = await s.scan(); + expect(s.getTracked(4242)?.firstSeen).toBe(firstSeen); + expect(during.find((p) => p.pid === 4242)).toBeTruthy(); +}); diff --git a/lib/daemon/__tests__/worktree-handlers.test.ts b/lib/daemon/__tests__/worktree-handlers.test.ts index c95cfb7b0..7b68a7a97 100644 --- a/lib/daemon/__tests__/worktree-handlers.test.ts +++ b/lib/daemon/__tests__/worktree-handlers.test.ts @@ -235,6 +235,17 @@ describe("worktree:provision", () => { expect(res.error).toBe("repo-unknown"); }); + test("S010: refuses a branch that git would parse as an option, before any git call", async () => { + const repo = makeRepo(); + const { h, events } = makeHandlers({ [repoName]: repo }); + + const res: any = await h["worktree:provision"]!({ repoName, branch: "--upload-pack=touch /tmp/x" }); + + expect(res.ok).toBe(false); + expect(res.error).toContain("unsafe git ref"); + expect(events.length).toBe(0); + }); + test("Hard cutover: a bare legacy name refuses even when it IS a registered repoIndex key", async () => { const repo = makeRepo(); // Registered under a non-identity key on purpose — proves the rejection diff --git a/lib/daemon/__tests__/worktree-process-kill.test.ts b/lib/daemon/__tests__/worktree-process-kill.test.ts index faf05ed2e..c86ebb8e7 100644 --- a/lib/daemon/__tests__/worktree-process-kill.test.ts +++ b/lib/daemon/__tests__/worktree-process-kill.test.ts @@ -1,3 +1,5 @@ +import { readFileSync } from "fs"; +import { resolve } from "path"; import { describe, expect, test } from "bun:test"; import { selectKillTargets, type KillCandidate } from "../worktree-process-kill.ts"; @@ -5,6 +7,12 @@ function row(pid: number, ppid: number, command: string, fullCommand: string): K return { pid, ppid, command, fullCommand }; } +test("worktree-process-kill.ts imports no sync exec", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "worktree-process-kill.ts"), "utf8"); + expect(src).not.toMatch(/from\s+["']child_process["']/); + expect(src).not.toMatch(/\bexecSync\b/); +}); + describe("selectKillTargets", () => { test("kills package-script chains and orphaned compilers", () => { const rows = [ diff --git a/lib/daemon/__tests__/worktree-reconciler.test.ts b/lib/daemon/__tests__/worktree-reconciler.test.ts index 8f30830ae..87b17260f 100644 --- a/lib/daemon/__tests__/worktree-reconciler.test.ts +++ b/lib/daemon/__tests__/worktree-reconciler.test.ts @@ -2,7 +2,7 @@ import { describe, test, expect, beforeEach, spyOn } from "bun:test"; import { execSync } from "child_process"; import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; -import { basename, join } from "path"; +import { basename, dirname, join } from "path"; import type { Logger } from "pino"; import { readJson, writeJson } from "../../json-store.ts"; import { closeStateDb, listKvValues, setKvValue } from "../../state/index.ts"; @@ -19,7 +19,7 @@ import { import { createTree } from "../../worktree/create.ts"; import type { WorktreeAppConfig } from "../../worktree/config.ts"; import { RETENTION_MS } from "../../worktree/trash.ts"; -import { reconcileRepoRegistry, createWorktreeReconciler, __test__ } from "../worktree-reconciler.ts"; +import { reconcileRepoRegistry, createWorktreeReconciler, withCreateLock, __test__ } from "../worktree-reconciler.ts"; function makeRepo(): string { // realpathSync: git canonicalizes /var -> /private/var on macOS (Global Constraints) @@ -1348,6 +1348,40 @@ describe("detached trigger / latency", () => { expect(events.filter((e) => e.type === "worktree:created").length).toBe(1); }, 10_000); + + // S065: a kick() during an in-flight pass, once the pass has started + // working (its per-repo loop has begun — this repo's replenish may + // already have run), must not be silently dropped. Observed here via + // repoIndex() call count: it's read fresh once per runOnce() invocation, + // so a queued follow-up pass is externally visible as a second call. + test("a kick() arriving after the pass has started work triggers a follow-up pass, not a silent drop (S065)", async () => { + process.env.HOME = realpathSync(mkdtempSync(join(tmpdir(), "rtkick2-home-"))); + closeStateDb(); + __test__.createBackoff.clear(); + const repoName = "acme-kick2"; + const repo = makeRepo(); + addBareOrigin(repo); + writeFileSync(join(repo, "wip.txt"), "not idle\n"); + await declareWorktrees(repo, repoName, { onDeck: 1, root: join(repo, ".worktrees"), ready: [{ run: "sleep 2" }] }); + + let repoIndexCalls = 0; + const reconciler = createWorktreeReconciler({ + cache: { entries: {} }, + repoIndex: () => { repoIndexCalls++; return { [repoName]: repo }; }, + emit: () => {}, + log: fakeLog(), + }); + + reconciler.kick(); + await waitFor(() => reconciler.creationInFlight(repoName) !== null, 2000); + // Mid-pass: this repo's replenish step is already running, so this kick + // must queue a follow-up rather than being dropped. + reconciler.kick(); + + await waitFor(() => reconciler.creationInFlight(repoName) === null, 6000); + await waitFor(() => repoIndexCalls >= 2, 6000); // the queued follow-up pass actually ran + expect(repoIndexCalls).toBeGreaterThanOrEqual(2); + }, 15_000); }); describe("reapRepoTrash", () => { @@ -1380,4 +1414,75 @@ describe("reapRepoTrash", () => { await waitFor(() => !existsSync(leftover) && !existsSync(expired)); expect(existsSync(fresh)).toBe(true); }); + + // S079: sanitizeRoot (lib/worktree/config.ts) has no ancestor check, so a + // repo configured with e.g. `root: "${repoRoot}/.."` makes the crash sweep + // walk the parent directory of every sibling repo for `.trash-*` names. + test("refuses a configured root outside the repo and warns instead of sweeping it", async () => { + const parent = dirname(repo); + const siblingLeftover = join(parent, ".trash-should-survive-123"); + mkdirSync(siblingLeftover, { recursive: true }); + await declareWorktrees(repo, "acme", { root: parent }); + + const warns: unknown[][] = []; + const log = { info: () => {}, warn: (...a: unknown[]) => warns.push(a), error: () => {}, debug: () => {} } as unknown as Logger; + + await __test__.reapRepoTrash({ repoName: "acme", repoPath: repo, log }); + await new Promise((r) => setTimeout(r, 300)); // give a wrongly-spawned detached rm time to run + + expect(existsSync(siblingLeftover)).toBe(true); + expect(warns.some((w) => JSON.stringify(w).includes(parent))).toBe(true); + }); +}); + +// S089: a provision's cold createTree and the reconciler's own replenish +// createTree can run concurrently for the same repo, both `git fetch origin +// ` against the same repoPath — the loser fails to lock +// refs/remotes/origin/, and that failure gets charged to +// createBackoff (a 5-to-30-minute replenish hold) for what was really just +// contention, not a genuine failure. Serializing createTree per repoPath +// closes the race at its root. +describe("withCreateLock", () => { + test("serializes concurrent calls for the same repoPath — never two holders at once", async () => { + const order: string[] = []; + let active = 0; + let maxActive = 0; + const run = (id: string) => withCreateLock("/repo/a", async () => { + active++; + maxActive = Math.max(maxActive, active); + order.push(`start-${id}`); + await new Promise((r) => setTimeout(r, 10)); + order.push(`end-${id}`); + active--; + }); + await Promise.all([run("1"), run("2"), run("3")]); + expect(maxActive).toBe(1); + expect(order).toEqual(["start-1", "end-1", "start-2", "end-2", "start-3", "end-3"]); + }); + + test("different repoPaths are not serialized against each other", async () => { + let active = 0; + let maxActive = 0; + const run = (path: string) => withCreateLock(path, async () => { + active++; + maxActive = Math.max(maxActive, active); + await new Promise((r) => setTimeout(r, 10)); + active--; + }); + await Promise.all([run("/repo/b"), run("/repo/c")]); + expect(maxActive).toBe(2); + }); + + test("a holder that throws still releases the lock for the next caller", async () => { + await expect(withCreateLock("/repo/d", async () => { throw new Error("boom"); })).rejects.toThrow("boom"); + let ran = false; + await withCreateLock("/repo/d", async () => { ran = true; }); + expect(ran).toBe(true); + }); +}); + +test("both cold-create call sites in handlers/worktree.ts serialize createTree through the shared per-repo lock (S089)", () => { + const source = readFileSync(new URL("../handlers/worktree.ts", import.meta.url), "utf8"); + const matches = source.match(/withCreateLock\(/g) ?? []; + expect(matches.length).toBeGreaterThanOrEqual(2); }); diff --git a/lib/daemon/agent-status-poller.ts b/lib/daemon/agent-status-poller.ts index faf30ed2c..fb50a4108 100644 --- a/lib/daemon/agent-status-poller.ts +++ b/lib/daemon/agent-status-poller.ts @@ -16,6 +16,8 @@ import type { RunLiveness } from "../runs/attention.ts"; import { listRuns } from "../runs/store.ts"; const POLL_MS = 10_000; +const FAILURE_THRESHOLD = 3; // consecutive null probes before backing off +const BACKOFF_TICKS = 6; // then probe once every 6 ticks (~60s at 10s cadence) interface Log { info(obj: unknown, msg?: string): void; @@ -39,10 +41,17 @@ export function startAgentStatusPoller(opts: { const list = opts.list ?? ((liveness: RunLiveness) => listRuns(undefined, liveness)); const last = new Map(); let seeded = false; + let consecutiveFailures = 0; + let ticksSkipped = 0; const tick = async (): Promise => { + if (consecutiveFailures >= FAILURE_THRESHOLD) { + if (++ticksSkipped < BACKOFF_TICKS) return; + ticksSkipped = 0; + } const entries = await probe(); - if (entries === null) return; + if (entries === null) { consecutiveFailures++; return; } + consecutiveFailures = 0; primeLivenessCache(entries); let runs: RunSummary[]; try { diff --git a/lib/daemon/api-auth.ts b/lib/daemon/api-auth.ts index e41b923bc..45a058be6 100644 --- a/lib/daemon/api-auth.ts +++ b/lib/daemon/api-auth.ts @@ -1,8 +1,10 @@ /** * Local-token auth for the :9401 API server. * - * The server binds to 127.0.0.1, but CORS is `*`, so a malicious web page could - * still drive *mutating* endpoints via the browser. Requiring a custom header + * The server binds to 127.0.0.1, and CORS is default-deny (only a trusted + * Origin gets its response readable), but CORS alone does not stop a + * malicious web page from firing a mutating request in the first place -- it + * only stops the page from reading the reply. Requiring a custom header * (X-RT-Token) on those routes forces a CORS preflight the page can't satisfy * (it can't read the token), blocking cross-site control while leaving reads * open for convenience. @@ -12,6 +14,10 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs"; import { join } from "path"; import { randomUUID } from "crypto"; import { RT_DIR } from "../daemon-config.ts"; +import { lazyChildLogger } from "../daemon-logger.ts"; +import { getSetting } from "../settings/resolve.ts"; + +const log = lazyChildLogger("api-auth"); /** Where the local API token is persisted (0600) for trusted local clients. */ export const API_TOKEN_PATH = join(RT_DIR, "api-token"); @@ -32,24 +38,131 @@ export function loadOrCreateApiToken(tokenPath: string = API_TOKEN_PATH): string try { mkdirSync(RT_DIR, { recursive: true }); writeFileSync(tokenPath, token, { mode: 0o600 }); - } catch { /* best-effort; token still enforced in-memory this run */ } + } catch (err) { + log.warn({ err, tokenPath }, "could not persist api-token; enforced in-memory only this run, so a client reading the file will disagree until the daemon restarts"); + } return token; } -/** True when a request mutates state, or (secrets) returns raw credential values, and must present the local token. */ +/** + * `getApiToken`/`reloadApiToken` share ONE in-memory value between + * api-server.ts and the secrets handler: before this, api-server + * captured a token once at boot while the secrets handler called + * `loadOrCreateApiToken()` fresh on every request, so an external rotation + * (deleting api-token to force a new one) left the two permanently + * disagreeing about which token is current -- and if the token dir was + * unwritable, the secrets handler regenerated a brand new random token on + * every single call, never matching anything a client could read from disk. + * Both consumers now read the SAME cached value; a rotation only takes + * effect for both after `reloadApiToken()` runs or the daemon restarts, + * either of which was already the closest thing to a happy path before. + */ +let cachedApiToken: string | null = null; + +export function getApiToken(tokenPath: string = API_TOKEN_PATH): string { + if (cachedApiToken === null) cachedApiToken = loadOrCreateApiToken(tokenPath); + return cachedApiToken; +} + +export function reloadApiToken(tokenPath: string = API_TOKEN_PATH): string { + cachedApiToken = loadOrCreateApiToken(tokenPath); + return cachedApiToken; +} + +/** + * True when a request mutates state, or (secrets/notifications) returns or + * drains something a GET should not silently consume, and must present the + * local token. A CORS preflight (OPTIONS) can never present the custom + * X-RT-Token header, so it is never gated, on any path. Otherwise + * default-gated for every method except GET/HEAD (an allowlist-by-path + * approach guarantees the next mutating route would ship unguarded), plus two + * explicit GET exceptions whose verb lies about being a read. + */ export function needsToken(method: string, pathname: string): boolean { if (method === "OPTIONS") return false; - if (pathname === "/api/shutdown") return true; - if (pathname === "/api/sdm/reconnect") return true; - if (pathname === "/api/events/emit") return true; - // Gated despite being a GET: every other read-only route returns metadata - // (branch names, MR titles, ports) safe under the open-CORS "reads are - // free" policy above; this one's response body IS the credential. - if (pathname === "/api/secrets") return true; - return false; + if (method === "GET" || method === "HEAD") { + // Gated despite being a GET: /api/secrets's response body IS a + // credential; /api/notifications DRAINS the queue, so its + // verb lies about being a read the way every other GET here is not. + if (pathname === "/api/secrets") return true; + if (pathname === "/api/notifications") return true; + return false; + } + return true; } /** True when the presented token matches the configured one (and one exists). */ export function tokenOk(provided: string | null, expected: string): boolean { return expected.length > 0 && provided === expected; } + +/** + * `rt.trustedBrowserOrigins` -- see registry-defs.ts. Read fresh on every + * call (the settings resolver is deliberately unmemoized), wrapped in a + * try/catch since a request-path settings read must never 500 the daemon + * over a malformed store file. + */ +export function getTrustedBrowserOrigins(): readonly string[] { + try { + const resolved = getSetting("rt.trustedBrowserOrigins"); + return Array.isArray(resolved.value) ? resolved.value : []; + } catch { + return []; + } +} + +export function isOriginAllowed(origin: string, allowedOrigins: readonly string[]): boolean { + return allowedOrigins.includes(origin); +} + +/** + * The 127.0.0.1 trust boundary: the daemon binds loopback-only, + * but any web page the user visits also runs on 127.0.0.1 and can send a + * request. A request with NO Origin header at all is not a browser fetch -- + * it is the CLI, the Swift tray, rt-client from a Bun/Node process, or the + * VS Code extension, none of which send one -- so it is trusted unchanged. + * A request that DOES carry an Origin header is trusted only if it presents + * the local api-token or its Origin is on the explicit allowlist. + */ +export function isBrowserRequestTrusted( + origin: string | null, + token: string | null, + apiToken: string, + allowedOrigins: readonly string[], +): boolean { + if (!origin) return true; + if (tokenOk(token, apiToken)) return true; + return isOriginAllowed(origin, allowedOrigins); +} + +/** + * Same trust decision as isBrowserRequestTrusted, but resolves the allowlist + * lazily: getAllowedOrigins() only runs when an Origin header is present. + * getTrustedBrowserOrigins does synchronous disk I/O on the settings store, + * and the vast majority of :9401 traffic (the CLI, the tray, rt-client from a + * Bun/Node process) carries no Origin at all, so it must never pay that cost. + */ +export function resolveOriginTrust( + origin: string | null, + presentedToken: string | null, + apiToken: string, + getAllowedOrigins: () => readonly string[] = getTrustedBrowserOrigins, +): boolean { + if (!origin) return true; + return isBrowserRequestTrusted(origin, presentedToken, apiToken, getAllowedOrigins()); +} + +/** + * A browser CORS preflight (OPTIONS) cannot carry the X-RT-Token value + * itself -- only Access-Control-Request-Headers names it as a header the + * follow-up request will use -- so an off-allowlist Origin that intends to + * authenticate with the token must be granted the preflight on trust alone. + * tokenOk() still gates the actual request; this only lets the browser send it. + */ +export function isTokenPreflight(method: string, requestHeaders: string | null): boolean { + if (method !== "OPTIONS" || !requestHeaders) return false; + return requestHeaders + .split(",") + .map((h) => h.trim().toLowerCase()) + .includes("x-rt-token"); +} diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index 7dab9b6c9..073bf3429 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -9,15 +9,18 @@ import type { Server, ServerWebSocket } from "bun"; import type { Logger } from "pino"; -import { API_PORT } from "../daemon-config.ts"; -import { needsToken, tokenOk, loadOrCreateApiToken } from "./api-auth.ts"; +import { API_PORT, resolveApiPort } from "../daemon-config.ts"; +import { needsToken, tokenOk, getApiToken, resolveOriginTrust, isTokenPreflight } from "./api-auth.ts"; import { getAggregatedConnection } from "./freshness.ts"; +import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; +import { runCapture } from "../subprocess.ts"; -const API_INDEX = { +function buildApiIndex(port: number) { + return { name: "rt daemon", version: "1.0.0", - docs: `http://localhost:${API_PORT}/`, - websocket: `ws://localhost:${API_PORT}/ws`, + docs: `http://localhost:${port}/`, + websocket: `ws://localhost:${port}/ws`, endpoints: [ { method: "GET", path: "/api/status", description: "Daemon health, uptime, memory, cache stats" }, { method: "GET", path: "/api/ports", description: "Listening ports grouped by repo/worktree" }, @@ -48,7 +51,8 @@ const API_INDEX = { header: "X-RT-Token", description: "Required on mutating routes (shutdown, sdm reconnect, events emit) and /api/secrets. Token at ~/.mattstack/rt/api-token.", }, -}; + }; +} const REST_ROUTES: Record = { "/api/status": { cmd: "tray:status", method: "GET" }, @@ -77,13 +81,68 @@ interface ApiWSData { const wsClients = new Set>(); +let apiServerLog: { warn: (o: unknown, m: string) => void } = { warn: () => {} }; + +/** Consecutive Bun `ws.send()` backpressure (-1) returns tolerated before a + client is dropped as chronically stalled. */ +const BACKPRESSURE_CLOSE_THRESHOLD = 3; +const backpressureCounts = new WeakMap(); + +export interface BroadcastTarget { + send(data: string): number; + close(): void; +} + +/** + * Sends one frame to every client, dropping any that Bun's own send() return + * value marks as gone. `ws.send()` never throws on a dead socket -- + * it returns 0 (this send silently failed) or -1 (backpressure) -- so a + * disconnected or stalled console/chat-viewer tab used to keep receiving a + * SUBSET of frames forever with nothing logged. 0 means Bun already dropped + * this exact frame for this client: closing immediately (rather than + * counting) is correct because the client's own reconnect logic is the only + * way it recovers a consistent stream. -1 means backpressure, which can be + * transient, so a few in a row are tolerated before giving up on the client. + */ +export function broadcastToClients( + clients: Iterable, + type: string, + data: any, + log: { warn: (o: unknown, m: string) => void }, +): void { + const msg = JSON.stringify({ type, data, timestamp: Date.now() }); + for (const client of clients) { + let result: number; + try { + result = client.send(msg); + } catch (err) { + log.warn({ err }, "ws client send threw; dropping"); + try { client.close(); } catch { /* already gone */ } + continue; + } + if (result === 0) { + log.warn({ type }, "ws client dropped a frame (send()=0); closing so its reconnect resyncs"); + backpressureCounts.delete(client); + try { client.close(); } catch { /* already gone */ } + } else if (result === -1) { + const count = (backpressureCounts.get(client) ?? 0) + 1; + if (count >= BACKPRESSURE_CLOSE_THRESHOLD) { + log.warn({ type, count }, "ws client chronically backpressured; closing"); + backpressureCounts.delete(client); + try { client.close(); } catch { /* already gone */ } + } else { + backpressureCounts.set(client, count); + } + } else { + backpressureCounts.delete(client); + } + } +} + /** Broadcast an event to all connected WebSocket clients. */ export function broadcast(type: string, data: any): void { if (wsClients.size === 0) return; - const msg = JSON.stringify({ type, data, timestamp: Date.now() }); - for (const ws of wsClients) { - try { ws.send(msg); } catch { /* client disconnected */ } - } + broadcastToClients(wsClients, type, data, apiServerLog); } /** Drop all broadcast clients (shutdown). */ @@ -91,14 +150,100 @@ export function clearWsClients(): void { wsClients.clear(); } +/** + * CORS default-deny: a browser page on an untrusted Origin still gets + * its request served (127.0.0.1 loopback + the per-route token gate are the + * real defenses), but the response carries no Access-Control-Allow-Origin, + * so the page's own JavaScript cannot read the body. A request with no + * Origin at all (every non-browser consumer today) needs no CORS headers. + */ +export function buildCorsHeaders(origin: string | null, trusted: boolean): Record { + const headers: Record = { + "Access-Control-Allow-Methods": "GET, POST, OPTIONS", + "Access-Control-Allow-Headers": "Content-Type, X-RT-Token", + }; + if (origin && trusted) { + headers["Access-Control-Allow-Origin"] = origin; + headers["Vary"] = "Origin"; + } + return headers; +} + +/** + * Decodes one path segment between a fixed prefix (and optional suffix), + * returning `undefined` (never throwing) on any shape mismatch or malformed + * %-encoding. Before this, each parameterized route hand-rolled its + * own decodeURIComponent inside the route's try block, so a malformed + * segment fell through to the OUTER catch and came back as a logged 500; + * every route using this helper instead gets a clean 400. + */ +export function pathParam(pathname: string, prefix: string, suffix = ""): string | undefined { + if (!pathname.startsWith(prefix)) return undefined; + if (suffix && !pathname.endsWith(suffix)) return undefined; + const end = suffix ? pathname.length - suffix.length : pathname.length; + if (end <= prefix.length) return undefined; + const raw = pathname.slice(prefix.length, end); + try { + return decodeURIComponent(raw); + } catch { + return undefined; + } +} + +const PLAIN_NUMBER_RE = /^-?\d+(\.\d+)?$/; + +/** + * REST query strings arrive as strings no matter what the client meant: + * "?maxAgeMs=60000" and "?refresh=true" reached handlers that do a + * strict `typeof x === "number"` or `x === true` check, so the documented + * flag silently no-op'd over HTTP while working over the socket (where + * payloads are real JSON). One coercion at the REST seam fixes every such + * flag at once instead of a per-handler private parser. + */ +export function coerceQueryParams(params: URLSearchParams): Record { + const out: Record = {}; + for (const [key, value] of params) { + if (value === "true") out[key] = true; + else if (value === "false") out[key] = false; + else if (value !== "" && PLAIN_NUMBER_RE.test(value)) out[key] = Number(value); + else out[key] = value; + } + return out; +} + export interface ApiServerDeps { handleCommand: (cmd: string, payload: any, signal?: AbortSignal) => Promise; log: Logger; } +/** + * Thrown when every bind retry is exhausted with EADDRINUSE still held. A + * named error type rather than the raw EADDRINUSE Error, so lib/daemon.ts's + * caller can distinguish "the port is genuinely squatted" from any other + * startup failure and park-and-retry with backoff instead of crash-looping + * (that caller-side change belongs to a sibling job; this class is the + * contract it wires into). + */ +export class ApiPortInUseError extends Error { + readonly code = "EADDRINUSE" as const; + readonly port: number; + constructor(port: number) { + super(`EADDRINUSE: api server port ${port} is still in use after ${BIND_RETRY_ATTEMPTS} bind attempts`); + this.name = "ApiPortInUseError"; + this.port = port; + } +} + export interface BindRetryDeps { sleep: (ms: number) => Promise; log: { warn: (o: unknown, m: string) => void }; + /** Defaults to a real `lsof -i :` via runCapture; overridable so tests never shell out. */ + probePortHolder?: (port: number) => Promise; +} + +async function defaultProbePortHolder(port: number): Promise { + const result = await runCapture(["lsof", "-i", `:${port}`], { timeoutMs: 5_000, stderr: "pipe" }); + return result.stdout.trim(); } // evictStaleDaemon (lib/daemon/boot-reconcile.ts) already assumes a prior @@ -114,54 +259,115 @@ export const BIND_RETRY_DELAY_MS = 500; * daemon can reach this bind before the old one has actually released * 9401. Only EADDRINUSE is retried (bounded, ~3s total); anything else is a * real misconfiguration and fails on the first attempt, same as before. + * + * Once retries are exhausted, this logs who holds the port and throws + * ApiPortInUseError instead of the bare EADDRINUSE Error, so a caller can + * tell "give up cleanly" apart from "the bind function itself is broken". */ -export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDeps): Promise { +export async function bindApiServerWithRetry(bind: () => T, deps: BindRetryDeps, port: number = API_PORT): Promise { + const probe = deps.probePortHolder ?? defaultProbePortHolder; for (let attempt = 1; ; attempt++) { try { return bind(); } catch (err) { const isAddrInUse = err instanceof Error && (err as NodeJS.ErrnoException).code === "EADDRINUSE"; - if (!isAddrInUse || attempt >= BIND_RETRY_ATTEMPTS) throw err; - deps.log.warn({ attempt, port: API_PORT }, "api port in use, retrying — another daemon is likely still shutting down"); + if (!isAddrInUse) throw err; + if (attempt >= BIND_RETRY_ATTEMPTS) { + const holder = await probe(port).catch((probeErr) => `lsof failed: ${String(probeErr)}`); + deps.log.warn({ port, holder }, "api port still in use after retries; giving up bind (the daemon should park and retry with backoff rather than crash-loop)"); + throw new ApiPortInUseError(port); + } + deps.log.warn({ attempt, port }, "api port in use, retrying — another daemon is likely still shutting down"); await deps.sleep(BIND_RETRY_DELAY_MS); } } } +/** + * Backoff base/cap for {@link withApiPortParkRetry}'s outer loop. Distinct + * from BIND_RETRY_* (bindApiServerWithRetry's own ~3s inner retry, already + * exhausted before an ApiPortInUseError ever reaches here): this loop + * assumes the holder is a whole other process that may take much longer + * than 3s to exit, so it backs off further between each full re-attempt. + */ +const PARK_RETRY_BASE_MS = 3_000; +const PARK_RETRY_MAX_MS = 60_000; + +export interface ParkRetryDeps { + sleep: (ms: number) => Promise; + log: { warn: (o: unknown, m: string) => void }; +} + +/** + * Wraps a `startApiServer`-shaped call: on `ApiPortInUseError` (bind retries + * already exhausted), logs and waits with exponential backoff, then calls + * `start` again — indefinitely, never giving up — instead of letting the + * error reach the daemon's top-level crash path (S043 caller-side contract, + * docs/daemon-api-auth.md). Any other error propagates immediately: that is + * a genuine misconfiguration, not a transient port squat. + */ +export async function withApiPortParkRetry(start: () => Promise, deps: ParkRetryDeps): Promise { + for (let attempt = 1; ; attempt++) { + try { + return await start(); + } catch (err) { + if (!(err instanceof ApiPortInUseError)) throw err; + const delayMs = Math.min(PARK_RETRY_BASE_MS * 2 ** (attempt - 1), PARK_RETRY_MAX_MS); + deps.log.warn({ attempt, port: err.port, delayMs }, "api server port still in use; parked, retrying with backoff"); + await deps.sleep(delayMs); + } + } +} + export async function startApiServer(deps: ApiServerDeps): Promise> { const { handleCommand, log } = deps; - const apiToken = loadOrCreateApiToken(); + apiServerLog = log; + const apiToken = getApiToken(); + const port = resolveApiPort(); const server = await bindApiServerWithRetry(() => Bun.serve({ - port: API_PORT, + port, // Bind to loopback only — never expose the control surface on the LAN. hostname: "127.0.0.1", // Raise the request idle timeout off Bun's 10s default so long-lived // clients aren't reaped every 10s ("[Bun.serve]: request timed out"). // 255 is the server max; the websocket block below sets its own larger one. idleTimeout: 255, + maxRequestBodySize: MAX_REQUEST_BODY_SIZE, async fetch(req, server) { const url = new URL(req.url); + const origin = req.headers.get("origin"); - // WebSocket upgrade — broadcast channel + // WebSocket upgrade (broadcast channel). Browsers cannot set custom + // headers on a WS handshake, so the token (when a browser page wants + // to identify itself) travels as a ?token= query param instead of + // X-RT-Token. if (url.pathname === "/ws") { + const wsToken = url.searchParams.get("token"); + if (!resolveOriginTrust(origin, wsToken, apiToken)) { + return new Response("origin not allowed", { status: 403 }); + } if (server.upgrade(req, { data: { kind: "broadcast" } })) return undefined as any; return new Response("WebSocket upgrade failed", { status: 400 }); } - // CORS headers for local dev - const corsHeaders = { - "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Methods": "GET, POST, OPTIONS", - "Access-Control-Allow-Headers": "Content-Type", - }; + // CORS: default-deny. A trusted Origin (token or allowlist) gets its + // Origin echoed back; anything else gets no Access-Control-Allow-Origin + // at all, so a malicious page's own JS cannot read the response. + // resolveOriginTrust only resolves the allowlist when origin is set, + // since the settings read behind it is synchronous disk I/O. + const trusted = resolveOriginTrust(origin, req.headers.get("x-rt-token"), apiToken) + || isTokenPreflight(req.method, req.headers.get("access-control-request-headers")); + const corsHeaders = buildCorsHeaders(origin, trusted); if (req.method === "OPTIONS") { return new Response(null, { status: 204, headers: corsHeaders }); } - // Gate mutating routes behind the local token (CORS is *, so this is the - // CSRF defense against a malicious page driving control endpoints). + // Gate mutating routes behind the local token. CORS default-deny only + // stops a malicious page from reading the response; it can still fire + // the request itself (a classic CSRF), so the token is the actual + // defense against a malicious page driving control endpoints. if (needsToken(req.method, url.pathname) && !tokenOk(req.headers.get("x-rt-token"), apiToken)) { return Response.json({ ok: false, error: "unauthorized" }, { status: 401, headers: corsHeaders }); } @@ -169,38 +375,41 @@ export async function startApiServer(deps: ApiServerDeps): Promise> try { // Self-describing root if (url.pathname === "/" || url.pathname === "") { - return Response.json(API_INDEX, { headers: corsHeaders }); + return Response.json(buildApiIndex(port), { headers: corsHeaders }); } // Single branch lookup: /api/cache/:branch if (url.pathname.startsWith("/api/cache/") && req.method === "GET") { - const branch = decodeURIComponent(url.pathname.slice("/api/cache/".length)); + const branch = pathParam(url.pathname, "/api/cache/"); + if (branch === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); + } const result = await handleCommand("cache:read", { branches: [branch] }, req.signal); return Response.json(result, { headers: corsHeaders }); } // Hooks repair: /api/hooks/:repo/repair if (url.pathname.startsWith("/api/hooks/") && url.pathname.endsWith("/repair") && req.method === "POST") { - const repo = decodeURIComponent(url.pathname.slice("/api/hooks/".length, -"/repair".length)); + const repo = pathParam(url.pathname, "/api/hooks/", "/repair"); + if (repo === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); + } const result = await handleCommand("hooks:repair", { repo }, req.signal); return Response.json(result, { headers: corsHeaders }); } // Runs detail: /api/runs/:repo/:runId if (url.pathname.startsWith("/api/runs/") && req.method === "GET") { - let rest: string | undefined; - try { - rest = decodeURIComponent(url.pathname.slice("/api/runs/".length)); - } catch { - rest = undefined; // malformed %-encoding -> fall through to the 404 path below + const rest = pathParam(url.pathname, "/api/runs/"); + if (rest === undefined) { + return Response.json({ ok: false, error: "malformed path parameter" }, { status: 400, headers: corsHeaders }); } - if (rest !== undefined) { - const slash = rest.indexOf("/"); - if (slash > 0 && slash < rest.length - 1) { - const result = await handleCommand("runs:get", { repo: rest.slice(0, slash), runId: rest.slice(slash + 1) }, req.signal); - return Response.json(result, { headers: corsHeaders }); - } + const slash = rest.indexOf("/"); + if (slash > 0 && slash < rest.length - 1) { + const result = await handleCommand("runs:get", { repo: rest.slice(0, slash), runId: rest.slice(slash + 1) }, req.signal); + return Response.json(result, { headers: corsHeaders }); } + // falls through to the 404 path below for a shape mismatch, e.g. "/api/runs/onlyonesegment" } // Secrets: forward the X-RT-Token header (already verified above by @@ -215,7 +424,7 @@ export async function startApiServer(deps: ApiServerDeps): Promise> // Static routes const route = REST_ROUTES[url.pathname]; if (!route) { - return Response.json({ ok: false, error: "not found", docs: `http://localhost:${API_PORT}/` }, { status: 404, headers: corsHeaders }); + return Response.json({ ok: false, error: "not found", docs: `http://localhost:${port}/` }, { status: 404, headers: corsHeaders }); } if (req.method !== route.method && req.method !== "OPTIONS") { @@ -227,7 +436,7 @@ export async function startApiServer(deps: ApiServerDeps): Promise> if (req.method === "POST") { try { payload = await req.json(); } catch { /* empty body */ } } else { - payload = Object.fromEntries(url.searchParams); + payload = coerceQueryParams(url.searchParams); } const result = await handleCommand(route.cmd, payload, req.signal); @@ -260,8 +469,8 @@ export async function startApiServer(deps: ApiServerDeps): Promise> // Broadcast clients are read-only; inbound frames are ignored. }, }, - }), { sleep: (ms) => Bun.sleep(ms), log }); + }), { sleep: (ms) => Bun.sleep(ms), log }, port); - log.info({ port: API_PORT }, "api server listening"); + log.info({ port }, "api server listening"); return server; } diff --git a/lib/daemon/boot-reconcile.ts b/lib/daemon/boot-reconcile.ts index 231fdec24..23aff7738 100644 --- a/lib/daemon/boot-reconcile.ts +++ b/lib/daemon/boot-reconcile.ts @@ -8,19 +8,54 @@ import type { Logger } from "pino"; import { readDaemonPid } from "../daemon-config.ts"; +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); // signal 0 = existence check, throws if not alive + return true; + } catch { + return false; + } +} + +function sleep(ms: number): Promise { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +/** Poll every ~100ms until `pid` is gone or `maxMs` elapses. */ +async function waitForDeath(pid: number, maxMs: number): Promise { + const deadline = Date.now() + maxMs; + while (isAlive(pid)) { + if (Date.now() >= deadline) return false; + await sleep(100); + } + return true; +} + /** * Evict a still-alive previous daemon. This is the last line of defence when * the `start` command's orphan-detection doesn't fire (e.g. launchd relaunches * us automatically without going through `rt daemon start`). + * + * Waits for the old process to actually die rather than a blind sleep: a + * daemon that survives the eviction window can still race the new one for + * rt.sock/rt.pid (S044). Escalates to SIGKILL if SIGTERM alone doesn't land. */ -export function evictStaleDaemon(log: Logger): void { +export async function evictStaleDaemon(log: Logger): Promise { const previousPid = readDaemonPid(); if (!previousPid || previousPid === process.pid) return; + if (!isAlive(previousPid)) return; try { - process.kill(previousPid, 0); // throws if not alive process.kill(previousPid, "SIGTERM"); - log.warn({ pid: previousPid }, "evicted stale daemon process"); - // Brief pause so the old process can exit and release any shared resources - Bun.sleepSync(300); - } catch { /* process not found — nothing to evict */ } + } catch (err) { + // Exited between the liveness probe and this send (ESRCH), or is not + // ours (EPERM). Nothing left to evict either way. + log.warn({ err, pid: previousPid }, "stale daemon SIGTERM skipped"); + return; + } + log.warn({ pid: previousPid }, "evicted stale daemon process"); + if (await waitForDeath(previousPid, 2500)) return; + try { + process.kill(previousPid, "SIGKILL"); + } catch { /* already gone */ } + await waitForDeath(previousPid, 500); } diff --git a/lib/daemon/cache-refresh.ts b/lib/daemon/cache-refresh.ts index 70194bd93..8c1e5d035 100644 --- a/lib/daemon/cache-refresh.ts +++ b/lib/daemon/cache-refresh.ts @@ -6,25 +6,24 @@ * * Concurrent callers are coalesced: the 5-minute timer and `cache:refresh` IPC * both fire-and-forget into the returned function. Without a guard they stack - * up, each running execSync across every repo + a batch GraphQL. If a refresh + * up, each running async git across every repo + a batch GraphQL. If a refresh * is already in flight, callers await the existing run instead of starting a * second one. */ import { existsSync } from "fs"; -import { execSync } from "child_process"; import type { Logger } from "pino"; import type { PortCacheRef, RepoIndex } from "./handlers/types.ts"; import type { BranchCacheStore } from "../state/index.ts"; import { checkAndNotify } from "../notifier.ts"; -import { getCurrentUserId } from "./freshness.ts"; +import { getCurrentUserId, resolveUserIdAcrossTracking } from "./freshness.ts"; import { loadRepoTracking, grants } from "../repo-tracking.ts"; import { syncProjectMRs } from "./project-sync.ts"; import { getProjectMRs } from "./project-mrs-store.ts"; import { pruneDiscussionsStore } from "./discussions-file-store.ts"; import { reconcileForRepo } from "./doppler-sync.ts"; import { deriveRepoIdentity } from "../settings/identity.ts"; -import { listWorktreeRoots, listWorktrees } from "../git-worktrees.ts"; +import { listWorktreesAsync, listWorktreeRootsAsync, runGit } from "../worktree/git-async.ts"; export interface CacheRefresherDeps { log: Logger; @@ -48,16 +47,80 @@ export interface CacheRefresherDeps { */ const BRANCH_CACHE_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; +/** Below the 5-min tick, above the slowest legitimate deep sync. */ +const REFRESH_CYCLE_DEADLINE_MS = 4 * 60 * 1000; + +export interface CoalescerOptions { + /** + * Cap on cycles still running in the background after their own deadline + * fired ("orphans"). `run` is never cancelled at the deadline (no + * AbortSignal threads through the GitLab/git calls it makes), so without a + * cap, repeated stalls could pile up half-open sockets and pending work + * without bound. Once the cap is hit, a new cycle is refused (not + * started) until an orphan settles. + */ + maxOrphanCycles?: number; + /** Called when a cycle is refused because the orphan cap is at capacity. */ + onRefused?: () => void; +} + +/** + * Coalesce concurrent callers onto one in-flight run, but clear the latch after + * `deadlineMs` even if the run never settles, so a wedged cycle (a half-open + * GitLab socket that never rejects) cannot pin the latch forever. The wedged + * run's frame still leaks until the OS reaps the socket; this only frees the + * next tick. `maxOrphanCycles` bounds how many such wedged runs may be alive + * at once (see CoalescerOptions) — the cap is the mitigation; it does not + * cancel the wedged runs themselves. + */ +export function makeCoalescer( + run: () => Promise, + deadlineMs: number, + onTimeout: () => void, + { maxOrphanCycles = 2, onRefused = () => {} }: CoalescerOptions = {}, +): () => Promise { + let inFlight: Promise | null = null; + let orphanCycles = 0; + return () => { + if (inFlight) return inFlight; + if (orphanCycles >= maxOrphanCycles) { + onRefused(); + return Promise.resolve(); + } + let timedOut = false; + const impl = run() + .catch(() => {}) // a rejected cycle still clears the latch + .finally(() => { if (timedOut) orphanCycles--; }); + // Promise.race never cancels the losing branch, so the deadline timer must be + // captured and cleared on every settle path or a fast success still fires + // onTimeout deadlineMs later, misreported as a wedge. + let deadlineTimer: ReturnType; + const deadline = new Promise((resolve) => { + deadlineTimer = setTimeout(() => { + timedOut = true; + orphanCycles++; + onTimeout(); + resolve(); + }, deadlineMs); + }); + const guarded = Promise.race([impl, deadline]).finally(() => { + clearTimeout(deadlineTimer); + inFlight = null; + }); + inFlight = guarded; + return guarded; + }; +} + export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise { const { log, cache, refreshStatusRef, portCacheRef, repoIndex, broadcast } = deps; - let refreshInFlight: Promise | null = null; - - function refreshCache(): Promise { - if (refreshInFlight) return refreshInFlight; - refreshInFlight = refreshCacheImpl().finally(() => { refreshInFlight = null; }); - return refreshInFlight; - } + const refreshCache = makeCoalescer( + refreshCacheImpl, + REFRESH_CYCLE_DEADLINE_MS, + () => log.warn("cache refresh timed out; cleared in-flight latch for next tick"), + { onRefused: () => log.warn("cache refresh skipped; too many stalled cycles already running") }, + ); async function refreshCacheImpl(): Promise { log.debug("cache: starting background refresh"); @@ -109,37 +172,29 @@ export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise = listWorktrees(repoPath).filter( - (w) => w.branch && !w.branch.startsWith("on-deck/"), - ); + const branches: Array<{ path: string; branch: string }> = ((await listWorktreesAsync(repoPath)) ?? []) + .filter((w): w is { path: string; branch: string } => !!w.branch && !w.branch.startsWith("on-deck/")); // 2. Discover local branches (not just worktrees) const worktreeBranchSet = new Set(branches.map(b => b.branch)); - try { - const localBranchOutput = execSync( - "git for-each-ref --format='%(refname:short)' refs/heads/", - { cwd: repoPath, encoding: "utf8", stdio: "pipe" }, - ); - - for (const name of localBranchOutput.split("\n")) { - const trimmed = name.trim().replace(/^'|'$/g, ""); + const localBranches = await runGit(repoPath, ["for-each-ref", "--format=%(refname:short)", "refs/heads/"]); + if (localBranches.exitCode === 0) { + for (const name of localBranches.stdout.split("\n")) { + const trimmed = name.trim(); if (!trimmed || worktreeBranchSet.has(trimmed) || trimmed.startsWith("on-deck/")) continue; if (extractLinearId(trimmed)) { branches.push({ path: repoPath, branch: trimmed }); } } - } catch (err) { - log.warn({ err, repo: repoPath }, "local branch listing failed"); + } else { + log.warn({ repo: repoPath }, "local branch listing failed"); } if (branches.length > 0) { // Get remote URL let remoteUrl: string | undefined; - try { - remoteUrl = execSync("git config --get remote.origin.url", { - cwd: repoPath, encoding: "utf8", stdio: "pipe", - }).trim(); - } catch { /* no remote */ } + const remote = await runGit(repoPath, ["config", "--get", "remote.origin.url"]); + if (remote.exitCode === 0) remoteUrl = remote.stdout.trim() || undefined; // Optimized: 3 GraphQL calls for ALL open MRs + 1 Linear batch. // The onError callback fires on per-MR enrich failures (GitLab, @@ -197,6 +252,16 @@ export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise Promise log.info(`cron ${trigger.name}: exited ${code}`)); } catch (err) { diff --git a/lib/daemon/demand-tracker.ts b/lib/daemon/demand-tracker.ts new file mode 100644 index 000000000..1db1dd930 --- /dev/null +++ b/lib/daemon/demand-tracker.ts @@ -0,0 +1,34 @@ +/** + * "A consumer is watching" signal for the background scans. The tray/CLI/console + * calling ports/system-processes/tray:status stamps demand here (via the + * command-router wrapper); pollers skip the 10s/30s scans when nothing has asked + * recently, so an idle machine stops paying the lsof/git tax (S058, S093). + */ +let lastDemandAt = 0; + +/** + * Only the wrapped command handlers (ports / system-processes / tray:status, + * via wrapWithDemand) stamp demand. WS relay and SSE topic subscriptions do + * not: a push-only consumer that subscribes to a broadcast topic but never + * calls a command gets no demand credit and can starve past the demand + * window. A subscribe-side stamp would live in api-server.ts, out of scope + * this phase. + */ +export function recordDemand(): void { + lastDemandAt = Date.now(); +} + +/** True when a consumer read a scan-backed command within `ms`. */ +export function demandedWithin(ms: number): boolean { + return lastDemandAt !== 0 && Date.now() - lastDemandAt < ms; +} + +/** Wrap the named handler entries so each call stamps demand, then delegates. */ +export function wrapWithDemand>(handlers: T, cmds: string[]): T { + for (const cmd of cmds) { + const inner = handlers[cmd]; + if (typeof inner !== "function") continue; + (handlers as any)[cmd] = (...args: any[]) => { recordDemand(); return inner(...args); }; + } + return handlers; +} diff --git a/lib/daemon/events-bus.ts b/lib/daemon/events-bus.ts index aca1dd085..bbc914a33 100644 --- a/lib/daemon/events-bus.ts +++ b/lib/daemon/events-bus.ts @@ -8,9 +8,10 @@ */ import { Database } from "bun:sqlite"; -import { mkdirSync } from "fs"; +import { mkdirSync, renameSync } from "fs"; import { dirname } from "path"; import type { Logger } from "pino"; +import { isCorruptionError } from "../state/db.ts"; export interface BusEvent { id: number; topic: string; payload: unknown; emittedAt: number } export interface WaitResult { events: BusEvent[]; cursor: number } @@ -31,6 +32,8 @@ export interface EventsBus { sweep(): number; waiterCount(): number; close(): void; + /** Test-only debug accessor for the underlying handle (e.g. pragma checks). Not for feature code. */ + __db?: Database; } // One matcher for wait AND list. Bun.Glob: `*` does not cross `/`, `**` does. @@ -61,6 +64,30 @@ function rowToEvent(row: EventRow): BusEvent { return { id: row.id, topic: row.topic, payload, emittedAt: row.emittedAt }; } +/** + * Renames a corrupt events.db out of the way and warns loudly, mirroring + * lib/state/db.ts's `quarantine`. events.db is a bounded-retention journal + * (sweep() already discards old rows), so losing it entirely on corruption + * is harmless: recreate empty rather than attempt any repair. WAL sidecars + * are best-effort cleaned since they are meaningless without the main file. + */ +function quarantineEventsDb(path: string, log: Logger): void { + const stamp = new Date().toISOString().replace(/[:.]/g, "-"); + const quarantinedPath = `${path}.corrupt-${stamp}`; + log.warn( + { path, quarantinedPath }, + "events db could not be opened (corrupt), quarantining and recreating empty", + ); + renameSync(path, quarantinedPath); + for (const sidecar of [`${path}-wal`, `${path}-shm`]) { + try { + renameSync(sidecar, `${sidecar}.corrupt-${stamp}`); + } catch { + // sidecar absent, fine: WAL mode doesn't always leave one + } + } +} + export function createEventsBus(opts: { dbPath: string; log: Logger; @@ -73,8 +100,26 @@ export function createEventsBus(opts: { // Self-sufficient about its parent dir — daemon.ts constructs the bus at // module scope, before startDaemon()'s mkdirSync(RT_DIR) runs. mkdirSync(dirname(opts.dbPath), { recursive: true }); - const db = new Database(opts.dbPath, { create: true }); - db.exec("PRAGMA journal_mode = WAL;"); + // PRAGMA order matches lib/state/db.ts's applyPragmas: busy_timeout FIRST + // so the WAL conversion itself respects it, then journal_mode, then + // synchronous. + let db: Database; + try { + db = new Database(opts.dbPath, { create: true }); + db.exec("PRAGMA busy_timeout = 250;"); + db.exec("PRAGMA journal_mode = WAL;"); + db.exec("PRAGMA synchronous = NORMAL;"); + // Pragmas alone don't always force sqlite to validate the file header; + // touch it now so a corrupt file surfaces here, not mid-query later. + db.query("PRAGMA user_version").get(); + } catch (err) { + if (!isCorruptionError(err)) throw err; + quarantineEventsDb(opts.dbPath, log); + db = new Database(opts.dbPath, { create: true }); + db.exec("PRAGMA busy_timeout = 250;"); + db.exec("PRAGMA journal_mode = WAL;"); + db.exec("PRAGMA synchronous = NORMAL;"); + } db.exec(` CREATE TABLE IF NOT EXISTS events ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -215,6 +260,8 @@ export function createEventsBus(opts: { waiterCount() { return waiters.size; }, + __db: db, + close() { const head = maxId(); for (const w of [...waiters]) settle(w, { events: [], cursor: head }); diff --git a/lib/daemon/freshness.ts b/lib/daemon/freshness.ts index a5f7cebac..8b4d09561 100644 --- a/lib/daemon/freshness.ts +++ b/lib/daemon/freshness.ts @@ -26,17 +26,19 @@ * disposeFreshness() — daemon shutdown */ -import { execSync } from "child_process"; import { GitLabProvider, type InvalidationKey, type MRApprovalRules, type PullRequest } from "@mattstack/glance"; -import { loadRepoTracking, grants, type RepoGrants } from "../repo-tracking.ts"; +import { loadRepoTracking, grants, type RepoGrants, type RepoTracking } from "../repo-tracking.ts"; +import { existsSync } from "fs"; import { loadSecrets } from "../linear.ts"; import { parseRemoteUrl, isGitLabRemote, toMRInfo } from "../enrich.ts"; import { checkAndNotify } from "../notifier.ts"; import type { HandlerContext } from "./handlers/types.ts"; import { lazyChildLogger } from "../daemon-logger.ts"; +import { redactCredentials } from "./redact-credentials.ts"; import { getProjectMRs, type ProjectMRs } from "./project-mrs-store.ts"; import { getDiscussionsFileStore } from "./discussions-file-store.ts"; import { createCursorStore, type CursorStore } from "../state/index.ts"; +import { runCapture } from "../subprocess.ts"; const log = lazyChildLogger("freshness"); @@ -85,21 +87,32 @@ interface RepoWatch { } const watches = new Map(); -const providers = new Map(); + +/** Bound merged pending so a wedged processKeys cannot grow memory unbounded. */ +export const PENDING_CAP = 1000; + +const providers = new Map(); let userId: number | null = null; let userIdResolved = false; let selfUsername: string | null = null; +const remoteUrlCache = new Map(); + // ─── Helpers ───────────────────────────────────────────────────────────────── -function getRemoteUrl(repoPath: string): string | null { - try { - return execSync("git config --get remote.origin.url", { - cwd: repoPath, encoding: "utf8", stdio: "pipe", - }).trim(); - } catch { - return null; - } +/** remote.origin.url, cached per repoPath for the process lifetime (remotes + * rarely change). Async so it never blocks the event loop. */ +async function getRemoteUrl(repoPath: string): Promise { + const cached = remoteUrlCache.get(repoPath); + if (cached !== undefined) return cached; + const r = await runCapture(["git", "config", "--get", "remote.origin.url"], { + cwd: repoPath, + timeoutMs: 5000, + stderr: "ignore", + }); + const url = r.exitCode === 0 ? (r.stdout.trim() || null) : null; + remoteUrlCache.set(repoPath, url); + return url; } /** The same debug-accounting hook makeProvider wires, for SDK helpers rt constructs directly (NoteMutator). */ @@ -123,41 +136,47 @@ function makeProvider(host: string, token: string): GitLabProvider { } async function ensureProvider(repoName: string, repoPath: string): Promise { - const cached = providers.get(repoName); - if (cached) return cached; - const secrets = await loadSecrets(); if (!secrets.gitlabToken) { log.info(`no gitlabToken; skipping ${repoName}`); return null; } - const remoteUrl = getRemoteUrl(repoPath); + const cached = providers.get(repoName); + if (cached && cached.token === secrets.gitlabToken) return cached.provider; + if (cached) { + // Token rotated: drop the stale watch built on the old token and + // re-resolve userId against the new one on the next reconcile. + stopWatch(repoName); + userIdResolved = false; + } + + const remoteUrl = await getRemoteUrl(repoPath); if (!remoteUrl) { log.info(`no origin remote for ${repoName}; skipping`); return null; } if (!isGitLabRemote(remoteUrl)) { - log.info(`remote "${remoteUrl}" for ${repoName} is not GitLab; skipping events watch`); + log.info(`remote "${redactCredentials(remoteUrl)}" for ${repoName} is not GitLab; skipping events watch`); return null; } const remote = parseRemoteUrl(remoteUrl); if (!remote) { - log.info(`could not parse remote "${remoteUrl}" for ${repoName}; skipping`); + log.info(`could not parse remote "${redactCredentials(remoteUrl)}" for ${repoName}; skipping`); return null; } const provider = makeProvider(remote.host, secrets.gitlabToken); - providers.set(repoName, provider); + providers.set(repoName, { provider, token: secrets.gitlabToken }); return provider; } async function ensureUserId(): Promise { if (userIdResolved) return userId; // Resolve via any available provider. If none exist yet, defer until one does. - const anyProvider = providers.values().next().value as GitLabProvider | undefined; + const anyProvider = providers.values().next().value?.provider as GitLabProvider | undefined; if (!anyProvider) return null; try { @@ -173,6 +192,40 @@ async function ensureUserId(): Promise { return userId; } +let userIdSuppressedWarned = false; + +/** + * S022: reconcileFreshnessImpl only ever builds a provider (and thus only + * ever calls ensureUserId) for repos in live mode, so a poll-only tracked + * user's getCurrentUserId() stayed null forever and checkAndNotify silently + * suppressed every self-authored transition. Called from cache-refresh.ts + * BEFORE its checkAndNotify (reconcileSubscriptions/reconcileFreshnessImpl + * runs after, so relying on it alone would leave the first cache-refresh + * cycle still passing null). Gates on the `branches`/`project-mrs` grant, + * never on mode, and is a no-op once userIdResolved (ensureUserId's own + * guard) or once every candidate repo has been tried this cycle. + */ +export async function resolveUserIdAcrossTracking( + repoIndex: Record, + tracking: RepoTracking, +): Promise { + if (userIdResolved) return; + for (const [repoName, repoPath] of Object.entries(repoIndex)) { + const g = grants(tracking, repoName); + if (g.mode === "off") continue; + if (!g.caches.has("branches") && !g.caches.has("project-mrs")) continue; + if (!existsSync(repoPath)) continue; + const provider = await ensureProvider(repoName, repoPath); + if (!provider) continue; + await ensureUserId(); + if (userIdResolved) return; + } + if (!userIdResolved && !userIdSuppressedWarned) { + userIdSuppressedWarned = true; + log.warn("userId is unresolved; self-authored MR transitions are being suppressed (no gitlabToken, or token validation failed)"); + } +} + export function getSelfUsername(): string | null { return selfUsername; } export async function resolveSelfUsername(repoName: string, repoPath: string): Promise { @@ -247,8 +300,22 @@ export async function getRepoContext( repoPath?: string, projectPathOverride?: string, ): Promise<{ provider: GitLabProvider; projectPath: string; projectId: number }> { + // A cached provider's token may have rotated since it was built; poll-mode + // repos and already-cached forge-handler providers never pass back through + // ensureProvider, so this is the only place that catches a stale token for + // them (S048/S049). + const cachedForToken = providers.get(repoName); + if (cachedForToken) { + const currentSecrets = await loadSecrets(); + if (cachedForToken.token !== currentSecrets.gitlabToken) { + stopWatch(repoName); + userIdResolved = false; + providers.delete(repoName); + } + } + const watch = watches.get(repoName); - let provider = providers.get(repoName) ?? null; + let provider = providers.get(repoName)?.provider ?? null; // Live-watch fast path — but only when the caller didn't override projectPath. // If they did, fall through to the ephemeral path so we use the canonical path. @@ -270,16 +337,16 @@ export async function getRepoContext( if (!secrets.gitlabToken) { throw new Error("missing gitlabToken (run: rt secrets set rt gitlabToken)"); } - const remoteUrl = getRemoteUrl(repoPath); + const remoteUrl = await getRemoteUrl(repoPath); if (!remoteUrl) { throw new Error(`could not read git remote.origin.url in ${repoPath}`); } const remote = parseRemoteUrl(remoteUrl); if (!remote) { - throw new Error(`could not parse remote URL "${remoteUrl}"`); + throw new Error(`could not parse remote URL "${redactCredentials(remoteUrl)}"`); } provider = makeProvider(remote.host, secrets.gitlabToken); - providers.set(repoName, provider); + providers.set(repoName, { provider, token: secrets.gitlabToken }); } // Pick projectPath: explicit override > previously-cached ephemeral > git remote. @@ -289,7 +356,7 @@ export async function getRepoContext( if (cached) projectPath = cached.projectPath; } if (!projectPath && repoPath) { - const remoteUrl = getRemoteUrl(repoPath); + const remoteUrl = await getRemoteUrl(repoPath); const remote = remoteUrl ? parseRemoteUrl(remoteUrl) : null; if (remote) projectPath = remote.projectPath; } @@ -390,7 +457,14 @@ export async function applyInvalidationBatch( overrides: MappingOverrides = {}, ): Promise { if (runner.processing) { - runner.pending.push(...keys); + const seen = new Set(runner.pending.map((k) => `${k.kind}:${k.ref}`)); + for (const k of keys) { + if (runner.pending.length >= PENDING_CAP) break; + const id = `${k.kind}:${k.ref}`; + if (seen.has(id)) continue; + seen.add(id); + runner.pending.push(k); + } return; } runner.processing = true; @@ -678,6 +752,15 @@ async function reconcileFreshnessImpl(env: FreshnessEnv): Promise { for (const [repoName, repoPath] of Object.entries(repoIndex)) { if (grants(tracking, repoName).mode !== "live") continue; + + // A live watch whose provider token has since rotated must be dropped so the + // ensureProvider/startWatch below rebuilds it with the current token (S048/S049). + const existing = providers.get(repoName); + if (existing && watches.has(repoName)) { + const secrets = await loadSecrets(); + if (secrets.gitlabToken && existing.token !== secrets.gitlabToken) stopWatch(repoName); + } + if (watches.has(repoName)) continue; const provider = await ensureProvider(repoName, repoPath); @@ -689,7 +772,7 @@ async function reconcileFreshnessImpl(env: FreshnessEnv): Promise { // startWatch rather than trust a check made before it. if (watches.has(repoName)) continue; - const remoteUrl = getRemoteUrl(repoPath); + const remoteUrl = await getRemoteUrl(repoPath); const remote = remoteUrl ? parseRemoteUrl(remoteUrl) : null; if (!remote) continue; diff --git a/lib/daemon/git-ref-validation.ts b/lib/daemon/git-ref-validation.ts new file mode 100644 index 000000000..809a733fd --- /dev/null +++ b/lib/daemon/git-ref-validation.ts @@ -0,0 +1,24 @@ +/** + * Rejects a branch/ref string that git would parse as an OPTION rather than + * a ref. A caller-supplied `branch` like "--upload-pack=touch /tmp/x" reaches + * `git fetch origin ` and `git rev-list ......` unescaped, + * and git happily executes it as an option since nothing on that path + * validates the string first. This is the one guard both call sites need; + * any future caller (including a consumer app forwarding an untrusted string + * as `branch`) inherits the same hole without it. + * + * Deliberately narrow: reject a leading '-' rather than allowlisting a + * character set. `git check-ref-format --branch` accepts far more + * punctuation than is worth re-deriving here, and the vulnerable shape is + * specifically "parses as an option", not "contains an unusual character". + */ +export function isSafeGitRef(ref: string): boolean { + return ref.length > 0 && !ref.startsWith("-"); +} + +export function validateGitRef(ref: string): { ok: true } | { ok: false; error: string } { + if (!isSafeGitRef(ref)) { + return { ok: false, error: `unsafe git ref (starts with '-' or empty): ${ref}` }; + } + return { ok: true }; +} diff --git a/lib/daemon/handlers/agent.ts b/lib/daemon/handlers/agent.ts index f79628b8e..268791b5e 100644 --- a/lib/daemon/handlers/agent.ts +++ b/lib/daemon/handlers/agent.ts @@ -96,11 +96,17 @@ export function createAgentHandlers(opts: { { workspaceLabel, tabLabel, paneCommand: buildPaneCommand(rec.cwd, inv) }, runner, ); - if (!out.focusedExisting) { - rec.paneId = out.paneId; - rec.tabId = out.tabId; - rec.workspaceId = out.workspaceId; + if (out.focusedExisting) { + // A live tab already answers to this label: launchInWorkspace + // focused it and ran nothing. Reporting ok:true here would insert a + // record with a freshly minted sessionId nothing is listening on — + // rt agent resume against it would run `claude --resume` for a + // session that was never started. + return { ok: false, error: `tab "${tabLabel}" already open; focused it` }; } + rec.paneId = out.paneId; + rec.tabId = out.tabId; + rec.workspaceId = out.workspaceId; return { ok: true, data: rec }; } @@ -130,6 +136,9 @@ export function createAgentHandlers(opts: { "agent:start": async (payload: Commands["agent:start"]["payload"]): Promise> => { const { repo, cwd } = payload; if (!repo || !cwd) return { ok: false, error: "agent:start requires repo (serialized identity) and cwd" }; + if (payload.surface !== undefined && payload.surface !== "herdr" && payload.surface !== "headless") { + return { ok: false, error: `invalid surface "${payload.surface}"; must be one of herdr, headless` }; + } const surface: AgentSurface = payload.surface ?? "herdr"; const prompt = payload.prompt; if (surface === "headless" && !prompt) { @@ -193,6 +202,9 @@ export function createAgentHandlers(opts: { "agent:resume": async (payload: Commands["agent:resume"]["payload"]): Promise> => { const rec = getAgent(payload.id, db); if (!rec) return { ok: false, error: `no agent record for "${payload.id}"` }; + if (payload.surface !== undefined && payload.surface !== "herdr" && payload.surface !== "headless") { + return { ok: false, error: `invalid surface "${payload.surface}"; must be one of herdr, headless` }; + } const surface: AgentSurface = payload.surface ?? rec.surface; if (surface === "headless" && !payload.prompt) { return { ok: false, error: "headless resume requires a prompt (claude -p with no prompt blocks on stdin)" }; diff --git a/lib/daemon/handlers/chat.ts b/lib/daemon/handlers/chat.ts index 0852d46dd..d0a0f7025 100644 --- a/lib/daemon/handlers/chat.ts +++ b/lib/daemon/handlers/chat.ts @@ -5,6 +5,7 @@ */ import type { Database } from "bun:sqlite"; +import type { Logger } from "pino"; import { isValidChatName, joinRoom, @@ -19,6 +20,7 @@ import { pendingMessages, listRooms, archiveRoom, + roomArchivedAt, roomDefaultWake, listMembers, dmRoomFor, @@ -55,7 +57,8 @@ import type { CommandResult, TypedHandlers } from "./types.ts"; export type InboxDeps = { resolve: typeof resolveInbox; deliver: typeof deliverToInbox }; const defaultInboxDeps: InboxDeps = { resolve: resolveInbox, deliver: deliverToInbox }; -const log = lazyChildLogger("chat"); +// Fallback only: real wiring threads ctx.log in from command-router.ts. +const defaultLog = lazyChildLogger("chat"); const CHAT_COMMANDS = [ "chat:join", @@ -87,6 +90,42 @@ function assertionError(fn: () => void): string | null { } } +/** Generous, not tight: bounds a single message body without constraining any real conversation. */ +const MAX_BODY_BYTES = 64 * 1024; + +function isValidBody(body: unknown): body is string { + return typeof body === "string" && body.length > 0 && Buffer.byteLength(body, "utf8") <= MAX_BODY_BYTES; +} + +/** Rooms `handle` already belongs to whose name is a prefix/suffix of the typo'd one — the common shape of a "deck" vs "deck-main" miss. */ +function closestRoomNames(typo: string, handle: string, db: Database): string[] { + const known = listRooms(handle, db, { includeArchived: true }).map((r) => r.room); + return known.filter((r) => r.startsWith(typo) || typo.startsWith(r)).slice(0, 3); +} + +/** + * `limit: -1` reaches `ORDER BY id ASC LIMIT ?`, where SQLite treats a + * negative LIMIT as unlimited, so a viewer/agent bug returns and + * JSON-serializes an entire (100k-row) room on the event loop; a + * non-numeric limit hits a datatype-mismatch SQLite error instead. + * Mirrors the events handler's `num()` coercion pattern. + */ +const MAX_CHAT_LIMIT = 500; + +function clampLimit(v: unknown, fallback: number): number { + if (v == null || v === "") return fallback; + const n = Number(v); + if (!Number.isFinite(n)) return fallback; + return Math.min(Math.max(Math.trunc(n), 1), MAX_CHAT_LIMIT); +} + +/** An unchecked value here lands on chat_members (and, on a join-creates, chat_room_defaults for every future joiner) and is silently treated as mention-only — never "none", never "all", never reported. */ +const VALID_WAKE_ON = ["mention", "all", "none"] as const; + +function isValidWakeOn(v: unknown): v is (typeof VALID_WAKE_ON)[number] { + return typeof v === "string" && (VALID_WAKE_ON as readonly string[]).includes(v); +} + // herdr's report_metadata schema types `seq` as a uint64 INTEGER: a // nanosecond-bigint string is rejected (invalid_request), and Date.now() // alone repeats within the same millisecond -- either loses a badge, the @@ -342,11 +381,20 @@ function postAndNotify( inboxDeps: InboxDeps, herdr: typeof herdrRequest, deliveryChains: Map>, + log: Logger, ): { id: number; recipients: string[] } | undefined { const { room, handle, body, mentions } = args; const posted = postMessage({ room, handle, body, mentions }, db); if (!posted) return undefined; - emitEvent(`chat/${room}/msg`, { id: posted.id }); + // The row is durable at this point. The msg emit is best-effort: a throw + // here (a full disk, an orphan daemon holding an events.db lock) must + // never surface as a failed post — the caller would retry and post the + // message twice. + try { + emitEvent(`chat/${room}/msg`, { id: posted.id }); + } catch (err) { + log.warn({ err, id: posted.id, room }, "chat: emit for the posted message threw; message is durable, this emit was not"); + } const dm = dmParticipants(room, db); for (const recipient of posted.recipients) { queueMicrotask(() => { @@ -361,17 +409,21 @@ function postAndNotify( const humanHandle = getSetting("chat.humanHandle").value; const allMentions = mergeMentions(body, mentions); if (humanHandle && allMentions.includes(humanHandle)) { - const title = dm ? `DM from ${handle}` : `#${room}`; - // The click target: the viewer at this exact message, when the viewer is - // configured. The tray opens `url` on a default click for any category. - notifyEnabled( - CHAT_NOTIFICATION_CATEGORY, - title, - `${handle}: ${body}`, - chatViewerUrl(readChatViewerUrlSetting(), room, posted.id), - undefined, - `chat:${posted.id}`, - ); + try { + const title = dm ? `DM from ${handle}` : `#${room}`; + // The click target: the viewer at this exact message, when the viewer is + // configured. The tray opens `url` on a default click for any category. + notifyEnabled( + CHAT_NOTIFICATION_CATEGORY, + title, + `${handle}: ${body}`, + chatViewerUrl(readChatViewerUrlSetting(), room, posted.id), + undefined, + `chat:${posted.id}`, + ); + } catch (err) { + log.warn({ err, id: posted.id, room }, "chat: desk notify threw after a successful post"); + } } return posted; } @@ -396,8 +448,11 @@ export function createChatHandlers(opts: { /** `findPaneSessionRetrying`'s wall-clock budget/poll; overridable so a test whose fake herdr never resolves does not have to wait out the real production budget. */ paneSessionBudgetMs?: number; paneSessionPollMs?: number; + /** Request logger; wired from ctx.log by command-router.ts. */ + log?: Logger; }): Pick & { db: Database } { const { db, emitEvent } = opts; + const log = opts.log ?? defaultLog; const herdr = opts.herdr ?? herdrRequest; const inboxDeps = opts.inboxDeps ?? defaultInboxDeps; const registryDeps = opts.registryDeps; @@ -416,6 +471,9 @@ export function createChatHandlers(opts: { const { room, handle, wakeOn, cwd, pane } = payload; if (!isValidChatName(handle)) return { ok: false, error: `invalid handle "${handle}"` }; if (!isValidChatName(room)) return { ok: false, error: `invalid room "${room}"` }; + if (wakeOn !== undefined && !isValidWakeOn(wakeOn)) { + return { ok: false, error: `invalid wakeOn "${wakeOn}"; must be one of ${VALID_WAKE_ON.join(", ")}` }; + } try { const data = joinRoom({ room, handle, wakeOn, cwd, pane }, db); return { ok: true, data }; @@ -431,16 +489,27 @@ export function createChatHandlers(opts: { "chat:post": async (payload: Commands["chat:post"]["payload"]): Promise> => { const { room, handle, body, mentions } = payload; + if (!isValidChatName(room)) return { ok: false, error: `invalid room "${room}"` }; + if (!isValidChatName(handle)) return { ok: false, error: `invalid handle "${handle}"` }; + if (!isValidBody(body)) return { ok: false, error: `body must be a non-empty string under ${MAX_BODY_BYTES} bytes` }; + if (mentions !== undefined && !Array.isArray(mentions)) return { ok: false, error: "mentions must be an array of handles" }; const invalidMention = mentions?.find((m) => !isValidChatName(m)); if (invalidMention !== undefined) return { ok: false, error: `invalid handle "${invalidMention}"` }; - const posted = postAndNotify(db, emitEvent, { room, handle, body, mentions }, inboxDeps, herdr, deliveryChains); + // A typo'd room previously no-op'd through postMessage's REVIVE (a + // no-op for a room with no chat_rooms row) and returned ok with no + // recipients — unreachable except by the exact typo'd name. + if (roomArchivedAt(room, db) === undefined) { + const nearby = closestRoomNames(room, handle, db); + return { ok: false, error: `unknown room "${room}"${nearby.length ? ` — did you mean: ${nearby.join(", ")}` : ""}` }; + } + const posted = postAndNotify(db, emitEvent, { room, handle, body, mentions }, inboxDeps, herdr, deliveryChains, log); if (!posted) return { ok: false, error: "chat: post failed (retry budget exhausted)" }; return { ok: true, data: posted }; }, "chat:read": async (payload: Commands["chat:read"]["payload"]): Promise> => { const { handle, room, limit, sinceMs } = payload; - const rooms = readUnread({ handle, room, limit: limit ?? 20, sinceMs }, db); + const rooms = readUnread({ handle, room, limit: clampLimit(limit, 20), sinceMs }, db); return { ok: true, data: { rooms } }; }, @@ -489,7 +558,7 @@ export function createChatHandlers(opts: { "chat:messages": async (payload: Commands["chat:messages"]["payload"]): Promise> => { const { room, before, limit } = payload; - const messages = listMessages({ room, before, limit: limit ?? 50 }, db); + const messages = listMessages({ room, before, limit: clampLimit(limit, 50) }, db); return { ok: true, data: { messages } }; }, @@ -632,6 +701,7 @@ export function createChatHandlers(opts: { const { from, to, body, sessionId } = payload; if (!isValidChatName(from)) return { ok: false, error: `invalid handle "${from}"` }; if (!isValidChatName(to)) return { ok: false, error: `invalid handle "${to}"` }; + if (!isValidBody(body)) return { ok: false, error: `body must be a non-empty string under ${MAX_BODY_BYTES} bytes` }; const err = assertionError(() => assertSessionOwnsHandle(from, sessionId, db)); if (err) return { ok: false, error: err }; const humanHandle = getSetting("chat.humanHandle").value; @@ -647,7 +717,7 @@ export function createChatHandlers(opts: { // Recipient travels in `mentions`, not the body, so the transcript // shows the text as typed and the desk still notifies when `to` is // the human. - const posted = postAndNotify(db, emitEvent, { room, handle: from, body, mentions: [to] }, inboxDeps, herdr, deliveryChains); + const posted = postAndNotify(db, emitEvent, { room, handle: from, body, mentions: [to] }, inboxDeps, herdr, deliveryChains, log); if (!posted) return { ok: false, error: "chat: dm failed (retry budget exhausted)" }; return { ok: true, data: { room, id: posted.id, recipients: posted.recipients } }; }, diff --git a/lib/daemon/handlers/discussions.ts b/lib/daemon/handlers/discussions.ts index 86f5adabc..5cbc3b3fb 100644 --- a/lib/daemon/handlers/discussions.ts +++ b/lib/daemon/handlers/discussions.ts @@ -31,6 +31,40 @@ import type { Commands } from "../../../packages/rt-client/src/commands.ts"; /** Discussions are stable per push; 2min TTL keeps reads fast without going stale. */ const DISCUSSIONS_TTL_MS = 2 * 60 * 1000; +/** GitLab's page size for this endpoint; a full page means there may be more. */ +const DIFFS_PAGE_SIZE = 100; +const DIFFS_FETCH_TIMEOUT_MS = 30_000; + +/** + * Every other outbound fetch in the daemon carries a bound (linear.ts, + * notifier.ts, park.ts); this one previously had none, so a stalled GitLab + * connection left the promise (and the sops-decrypted token in its closure) + * pending indefinitely. `reqSignal` is the client's own request signal + * (handlers/types.ts's Handler(payload, signal?)) so a client giving up + * actually cancels the in-flight GitLab request instead of orphaning it. + * `truncated` reports a full page rather than silently dropping files past it. + */ +export async function fetchMrDiffs( + baseURL: string, + projectPath: string, + iid: number, + token: string, + opts: { reqSignal?: AbortSignal; fetchFn?: typeof fetch } = {}, +): Promise<{ diffs: Array<{ newPath: string; diff: string }>; truncated: boolean }> { + const fetchFn = opts.fetchFn ?? fetch; + const encoded = encodeURIComponent(projectPath); + const url = `${baseURL}/api/v4/projects/${encoded}/merge_requests/${iid}/diffs?per_page=${DIFFS_PAGE_SIZE}`; + const timeout = AbortSignal.timeout(DIFFS_FETCH_TIMEOUT_MS); + const signal = opts.reqSignal ? AbortSignal.any([timeout, opts.reqSignal]) : timeout; + const res = await fetchFn(url, { headers: { "PRIVATE-TOKEN": token }, signal }); + if (!res.ok) throw new Error(`GitLab diffs API: ${res.status}`); + const raw = (await res.json()) as Array<{ new_path: string; diff: string }>; + return { + diffs: raw.map((d) => ({ newPath: d.new_path, diff: d.diff })), + truncated: raw.length >= DIFFS_PAGE_SIZE, + }; +} + export function createDiscussionHandlers( ctx: HandlerContext, broadcast: BroadcastFn, @@ -122,7 +156,7 @@ export function createDiscussionHandlers( } }, - "discussions:diffs": async (payload) => { + "discussions:diffs": async (payload, signal) => { const repoName = payload?.repoName as string | undefined; const iid = payload?.iid as number | undefined; if (!repoName || typeof iid !== "number") { @@ -138,14 +172,14 @@ export function createDiscussionHandlers( const secrets = await loadSecrets(); if (!secrets.gitlabToken) return { ok: false, error: "no gitlabToken in secrets" }; - const encoded = encodeURIComponent(repoCtx.projectPath); - const url = `${repoCtx.provider.baseURL}/api/v4/projects/${encoded}/merge_requests/${iid}/diffs?per_page=100`; - const res = await fetch(url, { headers: { "PRIVATE-TOKEN": secrets.gitlabToken } }); - if (!res.ok) return { ok: false, error: `GitLab diffs API: ${res.status}` }; - - const raw = await res.json() as Array<{ new_path: string; diff: string }>; - const diffs = raw.map((d) => ({ newPath: d.new_path, diff: d.diff })); - return { ok: true, data: { diffs } }; + const { diffs, truncated } = await fetchMrDiffs( + repoCtx.provider.baseURL, + repoCtx.projectPath, + iid, + secrets.gitlabToken, + { reqSignal: signal }, + ); + return { ok: true, data: { diffs, truncated } }; } catch (err) { return { ok: false, error: String(err) }; } diff --git a/lib/daemon/handlers/pane.ts b/lib/daemon/handlers/pane.ts index 0930e10db..e3a2dc1b2 100644 --- a/lib/daemon/handlers/pane.ts +++ b/lib/daemon/handlers/pane.ts @@ -126,7 +126,9 @@ export function createPaneHandlers(opts: { registry?: (repoName: string) => Array<{ path: string; branch: string | null | undefined }>; /** The registry probe behind buddyStatus, fakeable the same way lib/daemon/handlers/chat.ts's registryDeps is. */ registryDeps?: RegistryDeps; -}): Pick & { db: Database } { +}): Pick + & { "pane:spawn": (payload: Commands["pane:spawn"]["payload"], signal?: AbortSignal) => Promise> } + & { db: Database } { const { db, repoIndex } = opts; const herdr = opts.herdr ?? herdrRequest; const tray = opts.tray ?? trayRequest; @@ -194,7 +196,7 @@ export function createPaneHandlers(opts: { return { ok: true, data: { directories: out } }; }, - "pane:spawn": async (payload: Commands["pane:spawn"]["payload"]): Promise> => { + "pane:spawn": async (payload: Commands["pane:spawn"]["payload"], signal?: AbortSignal): Promise> => { const { cwd, account, model, effort, prompt } = payload; if (!cwd || !cwd.startsWith("/")) return { ok: false, error: "cwd must be an absolute path" }; if (account) { @@ -217,8 +219,23 @@ export function createPaneHandlers(opts: { if (!tab.ok) return herdrError(tab); const paneId = tab.result.root_pane.pane_id; + // pane:spawn's summed worst-case budget (register + idle wait + a + // blocked-trust retry + the opening prompt) can run longer than + // rt-client's own client-side timeout for this call. Once the caller + // has given up, continuing to spend the daemon's budget only risks a + // retry racing a second claude pane into the same cwd — so every step + // past tab creation checks the signal first and returns the pane + // already created (not-ready) rather than pressing on for a client + // that is no longer listening. + const earlyReturn = async (status: AgentStatus): Promise> => { + const ctx: PaneRowContext = { db, repoIndex, exec, now, workspaces: new Map([[workspaceId!, label]]), ...presenceMaps(db, now(), registryDeps) }; + const pane = await paneRow({ ...tab.result.root_pane, agent: "claude", agent_status: status }, ctx); + return { ok: true, data: { pane, ready: false } }; + }; + const sent = await herdr("pane.send_input", { pane_id: paneId, text: launchCommand({ cwd, account, model, effort }), keys: ["enter"] }); if (!sent.ok) return herdrError(sent); + if (signal?.aborted) return earlyReturn("unknown"); // herdr registers the agent a few hundred ms after the shell starts claude. // Bound the wait by wall-clock, not a fixed attempt count: a slow-but-alive @@ -227,7 +244,7 @@ export function createPaneHandlers(opts: { // holds its caller. let registered = false; const registerDeadline = now() + REGISTER_BUDGET_MS; - while (now() < registerDeadline) { + while (now() < registerDeadline && !signal?.aborted) { const got = await herdr("agent.get", { target: paneId }); if (got.ok) { registered = true; @@ -235,22 +252,26 @@ export function createPaneHandlers(opts: { } await Bun.sleep(REGISTER_POLL_MS); } + if (signal?.aborted) return earlyReturn("unknown"); let status: AgentStatus = "unknown"; let ready = false; if (registered) { const settled = await herdr<{ agent: HerdrAgent }>("agent.wait", { target: paneId, until: SETTLED, timeout_ms: IDLE_BUDGET_MS }, { timeoutMs: waitTimeout(IDLE_BUDGET_MS) }); if (settled.ok) status = settled.result.agent.agent_status; + if (signal?.aborted) return earlyReturn(status); if (status === "blocked") { const screen = await herdr<{ read: { text: string } }>("pane.read", { pane_id: paneId, source: "visible" }); if (screen.ok && /trust/i.test(screen.result.read.text)) { await herdr("pane.send_keys", { pane_id: paneId, keys: ["enter"] }); + if (signal?.aborted) return earlyReturn(status); const again = await herdr<{ agent: HerdrAgent }>("agent.wait", { target: paneId, until: SETTLED, timeout_ms: TRUST_BUDGET_MS }, { timeoutMs: waitTimeout(TRUST_BUDGET_MS) }); if (again.ok) status = again.result.agent.agent_status; } } ready = status === "idle" || status === "done"; } + if (signal?.aborted) return earlyReturn(status); if (ready && prompt) { await herdr("agent.prompt", { target: paneId, text: prompt, wait: { until: ["working"], timeout_ms: PROMPT_BUDGET_MS } }, { timeoutMs: waitTimeout(PROMPT_BUDGET_MS) }); diff --git a/lib/daemon/handlers/project-mrs.ts b/lib/daemon/handlers/project-mrs.ts index c9441a784..0395ecdca 100644 --- a/lib/daemon/handlers/project-mrs.ts +++ b/lib/daemon/handlers/project-mrs.ts @@ -245,7 +245,18 @@ export function createProjectMRsHandlers( try { const { pr, projectPath } = await fetchByBranch(repoName, branch); if (pr) { - store().upsert(repoName, projectPath, pr, "events"); + // Same scope/tagged gate upsertProject (lib/daemon/freshness.ts) + // enforces for every other write path: a demand-scoped repo + // must not pick up a stranger's MR here just because a client + // happened to ask for their branch by name — the next delta + // sync would filter it right back out, and it would reappear + // on the next by-branch call. The caller still gets the PR + // (it asked for this exact branch); it just isn't stored. + const rec = store().read(repoName); + const scope = rec?.scope; + const tagged = (rec?.mrs[pr.iid]?.codeownerSections?.length ?? 0) > 0; + const inScope = !scope || !pr.author?.username || scope.authors.includes(pr.author.username) || tagged; + if (inScope) store().upsert(repoName, projectPath, pr, "events"); byBranch[branch] = { pr, source: "forge" }; } else { byBranch[branch] = null; diff --git a/lib/daemon/handlers/secrets.ts b/lib/daemon/handlers/secrets.ts index a0e33021b..2608e603b 100644 --- a/lib/daemon/handlers/secrets.ts +++ b/lib/daemon/handlers/secrets.ts @@ -46,7 +46,7 @@ import { loadSecrets } from "../../linear.ts"; import { parseIdentity } from "../../settings/identity.ts"; import { loadMachineRepoTracking, grants, type RepoTracking } from "../../repo-tracking.ts"; -import { loadOrCreateApiToken, tokenOk } from "../api-auth.ts"; +import { getApiToken, tokenOk } from "../api-auth.ts"; import { readSecret, createRealSecretsExecSeam, type SecretsSeams } from "../../secrets/store.ts"; import { createRealAgeKeySeam } from "../../home/age-key.ts"; import type { Commands, ForgeSlug } from "../../../packages/rt-client/src/commands.ts"; @@ -125,7 +125,7 @@ export interface SecretsHandlerOverrides { deckSecrets?: () => Promise<{ cfApiToken?: string; cfZoneId?: string }>; /** Defaults to `loadBoardSecrets` (cross-domain: `board` + `rt`) for secrets:read's "board" scope. */ boardSecrets?: () => Promise; - /** Defaults to `loadOrCreateApiToken` (the real ~/.mattstack/rt/api-token, shared with api-auth.ts). */ + /** Defaults to `getApiToken` (the real ~/.mattstack/rt/api-token, shared with api-auth.ts and api-server.ts). */ apiToken?: () => string; } @@ -142,7 +142,7 @@ export function createSecretsHandlers( const extensionSecrets = overrides.extensionSecrets ?? loadSecrets; const deckSecrets = overrides.deckSecrets ?? loadDeckSecrets; const boardSecrets = overrides.boardSecrets ?? loadBoardSecrets; - const apiToken = overrides.apiToken ?? (() => loadOrCreateApiToken()); + const apiToken = overrides.apiToken ?? (() => getApiToken()); return { "secrets:forge-token": async (payload: Commands["secrets:forge-token"]["payload"]) => { diff --git a/lib/daemon/handlers/status.ts b/lib/daemon/handlers/status.ts index 96e0c9b56..19db8227e 100644 --- a/lib/daemon/handlers/status.ts +++ b/lib/daemon/handlers/status.ts @@ -14,14 +14,25 @@ import { existsSync, readdirSync } from "fs"; import type { HandlerContext, HandlerMap } from "./types.ts"; import type { PortEntry } from "../../port-scanner.ts"; -import { listWorktrees } from "../../git-worktrees.ts"; +import { listWorktreesAsync } from "../../worktree/git-async.ts"; import { drainNotifications, peekNotifications } from "../../notifier.ts"; import { getFreshnessSnapshot } from "../freshness.ts"; +import { readSupervisionState } from "../supervision-state.ts"; export function createStatusHandlers(ctx: HandlerContext): HandlerMap { return { "ping": async () => { - return { ok: true, uptime: Date.now() - ctx.startedAt, pid: process.pid, ...ctx.identity }; + // Read here (not once at ctx build time): a status/status.ts request + // must see this run's own boot-attempt/failure counters, not whatever + // they were when the daemon started. + const { bootAttempts, lastReadyAt, recentFailures, lastExit } = readSupervisionState(); + return { + ok: true, + uptime: Date.now() - ctx.startedAt, + pid: process.pid, + ...ctx.identity, + supervision: { bootAttempts, lastReadyAt, recentFailures: recentFailures.slice(-3), lastExit }, + }; }, "status": async () => { @@ -104,7 +115,9 @@ export function createStatusHandlers(ctx: HandlerContext): HandlerMap { for (const [repoName, repoPath] of Object.entries(repos)) { if (!existsSync(repoPath)) continue; // Detached worktrees have no branch — omit them from the listing. - const worktrees = listWorktrees(repoPath).filter((w) => w.branch); + const worktrees = ((await listWorktreesAsync(repoPath)) ?? []).filter( + (w): w is { path: string; branch: string } => Boolean(w.branch), + ); detailed[repoName] = { path: repoPath, worktrees }; } diff --git a/lib/daemon/handlers/worktree.ts b/lib/daemon/handlers/worktree.ts index b65f672aa..b4c3f2232 100644 --- a/lib/daemon/handlers/worktree.ts +++ b/lib/daemon/handlers/worktree.ts @@ -32,6 +32,7 @@ import { realpathSync, rmSync } from "fs"; import { join } from "path"; import { parseIdentity } from "../../settings/identity.ts"; +import { validateGitRef } from "../git-ref-validation.ts"; import type { HandlerContext, HandlerMap } from "./types.ts"; import { findByBranch, @@ -57,7 +58,7 @@ import { resolveReadySteps, } from "../../worktree/config.ts"; import { changedSince, runReadySteps, stepsToRun } from "../../worktree/ready.ts"; -import { freshenRepo, reconcileRepoRegistry } from "../worktree-reconciler.ts"; +import { freshenRepo, reconcileRepoRegistry, withCreateLock } from "../worktree-reconciler.ts"; import { repoDataDir, rtDir } from "../../rt-paths.ts"; const PROVISION_FETCH_TIMEOUT_MS = 5 * 60_000; @@ -295,6 +296,12 @@ export function createWorktreeHandlers( return { ok: false, error: "branch-unresolved" }; } + // S010: a branch that git would parse as an option (e.g. + // "--upload-pack=...") must never reach a runGit call, including + // divergence()'s below — both read this same `branch`. + const refCheck = validateGitRef(branch); + if (!refCheck.ok) return { ok: false, error: refCheck.error }; + const attached = findByBranch(trees, branch); if (attached.length > 1) return { ok: false, error: "branch-duplicated" }; if (attached.length === 1) return { ok: false, error: `branch-attached:${attached[0]!.name}` }; @@ -313,9 +320,13 @@ export function createWorktreeHandlers( } } if (!rec) { - const created = await createTree({ + // Serialized against the reconciler's own replenish createTree for + // this repo (S089): both `git fetch origin ` against the + // same repoPath, and an unserialized race charges the loser's + // ref-lock failure to createBackoff for what was just contention. + const created = await withCreateLock(repoPath, () => createTree({ repoName, repoPath, emit: opts.emit, log: ctx.log, - }); + })); if (!created.ok) { if (created.error === "busy") return { ok: false, error: "busy" }; return { @@ -467,7 +478,7 @@ export function createWorktreeHandlers( const repoPath = repoName ? ctx.repoIndex()[repoName] : undefined; if (!repoName || !repoPath || parseIdentity(repoName) === null) return { ok: false, error: "repo-unknown" }; - const created = await createTree({ repoName, repoPath, emit: opts.emit, log: ctx.log }); + const created = await withCreateLock(repoPath, () => createTree({ repoName, repoPath, emit: opts.emit, log: ctx.log })); if (!created.ok) { if (created.error === "busy") return { ok: false, error: "busy" }; return { ok: false, error: createFailedError(created) }; diff --git a/lib/daemon/home-snapshot.ts b/lib/daemon/home-snapshot.ts index 0d0fb56c9..3c793ff9e 100644 --- a/lib/daemon/home-snapshot.ts +++ b/lib/daemon/home-snapshot.ts @@ -258,6 +258,11 @@ function persistPushRecord(db: Database, record: HomePushRecord, log: Logger): v export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle { const repoDir = rawDeps.repoDir ?? join(mattstackHome(), "user"); const rawReadSettings = rawDeps.readSettings ?? (() => getSetting("rt.homeSnapshot").value); + // Thunk, not a resolved value: module-scope construction (lib/daemon.ts) + // must not open state.db before startDaemon() has opened it daemon-flavored + // via openBranchCacheStore; see loadState's call site inside init() below, + // which is the first place this ever actually gets invoked. + const resolveDb = rawDeps.db ? (() => rawDeps.db!) : (() => getStateDb("daemon")); const deps = { log: rawDeps.log, broadcast: rawDeps.broadcast, @@ -269,7 +274,6 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle now: rawDeps.now ?? (() => Date.now()), readSettings: () => clampSettings(rawReadSettings()), readOwners: rawDeps.readOwners ?? readOwnersReal, - db: rawDeps.db ?? getStateDb(), }; const ownersPath = ownersPathFor(deps.repoDir); @@ -296,7 +300,8 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle let lastPushError: string | null = null; /** True once `home:push-failed` has been broadcast for the CURRENT unbroken run of push failures — reset to false the moment a push succeeds, so a retry storm broadcasts once, not on every attempt. */ let pushFailureBroadcast = false; - let firstSeenDirty: Record = loadState(deps.db, deps.log); + /** Populated in init(), after the is-inside-work-tree check; see resolveDb's comment for why this can't happen at construction time. */ + let firstSeenDirty: Record = {}; let lastLoggedOwnersError: string | null = null; /** Shared dedup key for every "deps.readSettings() itself threw" warn (armWatcher's debounce read, status()) — a settings store that broke after boot and stays broken must warn once, not on every fs event or every `rt home snapshot --status` poll. */ let lastLoggedSettingsError: string | null = null; @@ -370,6 +375,10 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle deps.log.warn({ repoDir: deps.repoDir }, "home-snapshot: repoDir is not a git repository; inert"); return; } + // First real db touch: this await already put us past the daemon's + // synchronous boot pass, so by now startDaemon() has opened state.db + // daemon-flavored via openBranchCacheStore (see resolveDb above). + firstSeenDirty = loadState(resolveDb(), deps.log); if (deps.readSettings().enabled !== false) { tryArm(); } @@ -525,7 +534,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle pushFailureBroadcast = false; lastPushAt = deps.now(); lastPushError = null; - persistPushRecord(deps.db, { at: lastPushAt, ok: true }, deps.log); + persistPushRecord(resolveDb(), { at: lastPushAt, ok: true }, deps.log); if (pushRetryTimer) { deps.clearTimeout(pushRetryTimer); pushRetryTimer = null; @@ -538,7 +547,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle const redactedStderr = redactCredentials(result.stderr); pushPending = true; lastPushError = redactedStderr; - persistPushRecord(deps.db, { at: deps.now(), ok: false, error: redactedStderr }, deps.log); + persistPushRecord(resolveDb(), { at: deps.now(), ok: false, error: redactedStderr }, deps.log); deps.log.warn({ stderr: redactedStderr }, "home-snapshot: push failed"); // Only the FIRST failure of an unbroken streak broadcasts — a retry // storm (schedulePushRetry firing every pushDelaySec*5) would @@ -642,7 +651,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle }); firstSeenDirty = plan.nextFirstSeenDirty; - persistState(deps.db, firstSeenDirty, deps.log); + persistState(resolveDb(), firstSeenDirty, deps.log); let committed = false; let sha: string | null = null; diff --git a/lib/daemon/hooks-guard.ts b/lib/daemon/hooks-guard.ts index f0b30d535..758b15690 100644 --- a/lib/daemon/hooks-guard.ts +++ b/lib/daemon/hooks-guard.ts @@ -12,6 +12,7 @@ import type { Logger } from "pino"; import { repoDataDir } from "../rt-paths.ts"; import { runCapture } from "../subprocess.ts"; import { loadRepoIndex, resolveGitConfigPath } from "./repo-index.ts"; +import type { RepoIndex } from "./handlers/types.ts"; export interface HooksGuard { /** Live map of repo git-config watchers (configPath → FSWatcher). */ @@ -26,7 +27,11 @@ export interface HooksGuard { closeAll(): void; } -export function createHooksGuard(log: Logger): HooksGuard { +export function createHooksGuard( + log: Logger, + deps: { loadRepoIndexFn?: () => RepoIndex } = {}, +): HooksGuard { + const loadRepoIndexFn = deps.loadRepoIndexFn ?? loadRepoIndex; const watchedConfigs = new Map(); async function checkAndRepairHooksPath(repoName: string, repoPath: string): Promise { @@ -90,6 +95,16 @@ export function createHooksGuard(log: Logger): HooksGuard { }, 100); // slightly longer debounce: rename events can cluster }); + // FSWatcher is an EventEmitter: an 'error' with no listener is an + // uncaught exception, which installCrashHandlers turns into a daemon + // exit(1) and a launchd relaunch that re-arms the same watchers and + // hits the same limit (EMFILE, a watched dir unlinked, ...). + watcher.on("error", (err) => { + log.warn({ err, repo: repoName, configPath }, "hooks-guard watcher error; dropping this watch"); + watchedConfigs.delete(configPath); + try { watcher.close(); } catch { /* already gone */ } + }); + watchedConfigs.set(configPath, watcher); log.debug({ repo: repoName, file: `${gitDir}/${configFile}` }, "watching repo"); @@ -98,11 +113,22 @@ export function createHooksGuard(log: Logger): HooksGuard { } function refreshWatchedRepos(): void { - const repos = loadRepoIndex(); + const repos = loadRepoIndexFn(); + const liveConfigPaths = new Set(); for (const [repoName, repoPath] of Object.entries(repos)) { if (!existsSync(repoPath)) continue; + const configPath = resolveGitConfigPath(repoPath); + if (configPath) liveConfigPaths.add(configPath); startWatchingRepo(repoName, repoPath); } + // Reconcile, not just add: a repo relocated (rt repos locate) or removed + // from the index leaves its old watcher pointed at a dead .git dir, and + // status().watchedRepos over-reports forever without this. + for (const [configPath, watcher] of watchedConfigs) { + if (liveConfigPaths.has(configPath)) continue; + try { watcher.close(); } catch { /* already gone */ } + watchedConfigs.delete(configPath); + } } function closeAll(): void { diff --git a/lib/daemon/pollers.ts b/lib/daemon/pollers.ts index 91c82c53f..c8a38475b 100644 --- a/lib/daemon/pollers.ts +++ b/lib/daemon/pollers.ts @@ -12,11 +12,14 @@ import { checkRunawayProcesses } from "../notifier.ts"; import { primeTeamTrackingIdentityMap } from "../repo-tracking.ts"; import type { SystemProcessScanner } from "./system-process-scanner.ts"; import type { PortCacheRef, RepoIndex } from "./handlers/types.ts"; +import { demandedWithin } from "./demand-tracker.ts"; const MR_REFRESH_INTERVAL_MS = 5 * 60 * 1000; // 5 minutes const PORT_SCAN_INTERVAL_MS = 30 * 1000; // 30 seconds const HOOKS_SCAN_INTERVAL_MS = 60 * 1000; // 60 seconds (fallback for stale watchers) const SYSTEM_PROCESS_SCAN_INTERVAL_MS = 10 * 1000; // 10 seconds +/** Consider a consumer "present" for 5 min after its last scan-backed read. */ +const DEMAND_WINDOW_MS = 5 * 60 * 1000; export interface PollerDeps { log: Logger; @@ -39,6 +42,7 @@ export function startPollers(deps: PollerDeps): void { async function refreshPortCache(): Promise { if (portScanInFlight) return; + if (!demandedWithin(DEMAND_WINDOW_MS)) return; // no consumer asked recently portScanInFlight = true; try { portCacheRef.ports = await scanListeningPorts(); @@ -55,6 +59,7 @@ export function startPollers(deps: PollerDeps): void { async function refreshSystemProcesses(): Promise { if (processScanInFlight) return; + if (!demandedWithin(DEMAND_WINDOW_MS)) return; processScanInFlight = true; try { const processes = await systemProcessScanner.scan(portCacheRef.ports); diff --git a/lib/daemon/redact-credentials.ts b/lib/daemon/redact-credentials.ts new file mode 100644 index 000000000..b46d6f55e --- /dev/null +++ b/lib/daemon/redact-credentials.ts @@ -0,0 +1,14 @@ +/** + * Strips userinfo (user:token@ or user@) out of any http(s) URL embedded in + * a string. freshness.ts logs `remote.origin.url` verbatim on every + * reconcile and echoes it into thrown errors returned to callers; a repo + * cloned as `https://oauth2:glpat-XXXX@gitlab.example.com/...` (routine for + * dotfiles/CI-derived clones) puts that token into the daemon's rotated log + * files and into any client-facing error message. Logs are the first thing + * a user pastes into a bug report. + */ +const CREDENTIAL_URL_RE = /(https?:\/\/)[^/@\s]+@/gi; + +export function redactCredentials(text: string): string { + return text.replace(CREDENTIAL_URL_RE, "$1[redacted]@"); +} diff --git a/lib/daemon/request-limits.ts b/lib/daemon/request-limits.ts new file mode 100644 index 000000000..d8448818c --- /dev/null +++ b/lib/daemon/request-limits.ts @@ -0,0 +1,10 @@ +/** + * Shared cap on request body size for both daemon servers (api-server.ts's + * :9401 HTTP/WS surface and socket-server.ts's unix-socket IPC channel). + * Neither transport authenticates reads, so an unbounded body (Bun's + * default is 128 MB) lets any same-user process or a cross-origin browser + * request stall the daemon's single event loop parsing a giant payload. + * Real payloads on both transports are kilobytes; 1 MiB costs nothing and + * turns an oversized request into an immediate 413 instead. + */ +export const MAX_REQUEST_BODY_SIZE = 1024 * 1024; diff --git a/lib/daemon/safe-timers.ts b/lib/daemon/safe-timers.ts new file mode 100644 index 000000000..b28a5de75 --- /dev/null +++ b/lib/daemon/safe-timers.ts @@ -0,0 +1,42 @@ +/** + * lib/daemon/safe-timers.ts: try/catch-wrapped setInterval/setTimeout. + * + * A bare `setInterval`/`setTimeout` callback that throws synchronously + * (e.g. a sqlite SQLITE_FULL on a WAL write) becomes an uncaughtException + * with no stack frame back to the timer that scheduled it; Node/Bun's + * event loop has nothing to attribute the throw to but the process itself, + * so installCrashHandlers treats it as fatal and exits the daemon. Wrapping + * the tick converts that crash into a logged warning. + */ + +import type { Logger } from "pino"; + +export function safeInterval( + fn: () => void, + ms: number, + label: string, + log: Logger, +): ReturnType { + return setInterval(() => { + try { + fn(); + } catch (err) { + log.warn({ err, label }, "timer tick failed"); + } + }, ms); +} + +export function safeTimeout( + fn: () => void, + ms: number, + label: string, + log: Logger, +): ReturnType { + return setTimeout(() => { + try { + fn(); + } catch (err) { + log.warn({ err, label }, "timer tick failed"); + } + }, ms); +} diff --git a/lib/daemon/shutdown.ts b/lib/daemon/shutdown.ts index 4b4274333..29015fcbf 100644 --- a/lib/daemon/shutdown.ts +++ b/lib/daemon/shutdown.ts @@ -3,7 +3,7 @@ * runtime files in an order that beats launchd's 5s ExitTimeOut. */ -import { existsSync, unlinkSync } from "fs"; +import { existsSync, readFileSync, unlinkSync } from "fs"; import type { Server } from "bun"; import type { Logger } from "pino"; import { DAEMON_SOCK_PATH, DAEMON_PID_PATH } from "../daemon-config.ts"; @@ -11,16 +11,19 @@ import { clearWsClients } from "./api-server.ts"; import { disposeFreshness } from "./freshness.ts"; import { stopDiscussionsPoller } from "./discussions-poller.ts"; import type { HooksGuard } from "./hooks-guard.ts"; +import { recordCleanExit } from "./supervision-state.ts"; export interface ShutdownDeps { /** Mutable holder — daemon.ts assigns the servers after boot. */ servers: { socket?: Server; api?: Server }; hooksGuard: HooksGuard; log: Logger; + /** This process's pid. Injected so cleanup's ownership check is testable. */ + pid?: number; } export function createCleanup(deps: ShutdownDeps): () => void { - const { servers, hooksGuard, log } = deps; + const { servers, hooksGuard, log, pid = process.pid } = deps; return function cleanup(): void { // Stop accepting new traffic first, and force-close all in-flight @@ -40,31 +43,64 @@ export function createCleanup(deps: ShutdownDeps): () => void { // through at every mutation site (lib/state/branch-cache.ts), so there // is nothing dirty in memory to race launchd's 5s ExitTimeOut. - // Remove runtime files - for (const path of [DAEMON_SOCK_PATH, DAEMON_PID_PATH]) { - try { if (existsSync(path)) unlinkSync(path); } catch { /* */ } - } + // Remove runtime files, but only if rt.pid still names THIS process. + // A shutting-down old daemon that unlinks unconditionally can delete a + // new daemon's rt.sock/rt.pid out from under it (S012/S044). + try { + if (existsSync(DAEMON_PID_PATH) && readFileSync(DAEMON_PID_PATH, "utf8").trim() === String(pid)) { + unlinkSync(DAEMON_PID_PATH); + if (existsSync(DAEMON_SOCK_PATH)) unlinkSync(DAEMON_SOCK_PATH); + } + } catch (err) { log.warn({ err }, "cleanup unlink skipped"); } log.info("daemon stopped"); }; } +export interface GracefulExitDeps { + cleanup: () => void; + flushLogs: () => void; + log: Logger; + /** True once the `shutdown` verb has claimed this exit as intentional. */ + wasVerbShutdown: () => boolean; + exit: (code?: number) => void; + recordCleanExit: (kind: "shutdown" | "signal", code: number) => void; +} + /** * Graceful shutdown on all termination signals. SIGHUP is sent when the * parent process exits (e.g. launchd session ends, or a tray-spawned daemon's - * parent tray is killed) — treat it as a clean stop. + * parent tray is killed). + * + * launchd's KeepAlive.SuccessfulExit=false only respawns on a non-zero exit, + * so the code here must distinguish the intentional `shutdown` verb (exit 0, + * stay down) from a bare external signal (pkill, memory pressure, a stray + * script, exit 1, launchd respawns). The sanctioned stop path + * (SMAppService.unregister) doesn't go through this signal path at all, so + * exiting non-zero on a bare signal never fights an intended stop. */ +export function makeGracefulExit(deps: GracefulExitDeps): (signal: NodeJS.Signals) => void { + return (signal: NodeJS.Signals) => { + deps.log.info({ signal }, "received signal; shutting down"); + deps.cleanup(); + deps.flushLogs(); + if (deps.wasVerbShutdown()) { + deps.recordCleanExit("shutdown", 0); + deps.exit(0); + } else { + deps.recordCleanExit("signal", 1); + deps.exit(1); + } + }; +} + export function installSignalHandlers(opts: { cleanup: () => void; flushLogs: () => void; log: Logger; + wasVerbShutdown: () => boolean; }): void { - const gracefulExit = (signal: NodeJS.Signals) => { - opts.log.info({ signal }, "received signal; shutting down"); - opts.cleanup(); - opts.flushLogs(); - process.exit(0); - }; + const gracefulExit = makeGracefulExit({ ...opts, exit: process.exit, recordCleanExit }); process.on("SIGTERM", () => gracefulExit("SIGTERM")); process.on("SIGINT", () => gracefulExit("SIGINT")); process.on("SIGHUP", () => gracefulExit("SIGHUP")); diff --git a/lib/daemon/socket-server.ts b/lib/daemon/socket-server.ts index 7ab37fb8b..fee4c27e4 100644 --- a/lib/daemon/socket-server.ts +++ b/lib/daemon/socket-server.ts @@ -9,6 +9,7 @@ import { existsSync, unlinkSync } from "fs"; import type { Server } from "bun"; import type { Logger } from "pino"; import { DAEMON_SOCK_PATH } from "../daemon-config.ts"; +import { MAX_REQUEST_BODY_SIZE } from "./request-limits.ts"; export function startSocketServer(opts: { handleCommand: (cmd: string, payload: any, signal?: AbortSignal) => Promise; @@ -33,6 +34,7 @@ export function startSocketServer(opts: { // on how long an idle connection may sit before Bun kills it — it never // holds connections open on its own. idleTimeout: 255, + maxRequestBodySize: MAX_REQUEST_BODY_SIZE, async fetch(req) { try { const url = new URL(req.url); diff --git a/lib/daemon/supervision-state.ts b/lib/daemon/supervision-state.ts new file mode 100644 index 000000000..7e364cbfb --- /dev/null +++ b/lib/daemon/supervision-state.ts @@ -0,0 +1,142 @@ +/** + * lib/daemon/supervision-state.ts: daemon boot/crash history, so + * `rt daemon status` can report boot-failed/crash-looping and a stuck-phase + * breadcrumb for a live-but-silent daemon. + * + * Two tiers, deliberately not one: + * - The breadcrumb FILE (`writeBreadcrumb`/`readBreadcrumb`) + * opens no database, so it is safe to call at module scope, before + * state.db exists. It is the only tier a pre-db boot failure can reach. + * - The kv tier (`recordBootAttempt`, `recordDaemonReady`, + * `recordCleanExit`, and the kv half of `recordBootFailure`) goes through + * `getStateDb("daemon")`, so callers must not reach it until the daemon + * has opened its state.db (lib/daemon.ts's `openBranchCacheStore()`). + * `recordBootFailure` is safe to call at any point regardless: its kv + * write is try/catch'd and silently no-ops if the db isn't open yet. + */ + +import { existsSync, readFileSync, writeFileSync } from "fs"; +import { join } from "path"; +import { RT_DIR } from "../daemon-config.ts"; +import { daemonFlavor } from "./park.ts"; +import { getKvValue, setKvValue } from "../state/kv-blob.ts"; +import { getStateDb } from "../state/db.ts"; + +export type BootPhase = "start" | "events-db" | "state-db" | "api" | "socket" | "ready"; + +const NS = "daemon-supervision"; +const KEY_BOOT_ATTEMPTS = "boot-attempts"; +const KEY_LAST_READY_AT = "last-ready-at"; +const KEY_RECENT_FAILURES = "recent-failures"; +const KEY_LAST_EXIT = "last-exit"; + +const RECENT_FAILURES_CAP = 10; + +export interface BootFailure { + at: number; + phase: BootPhase; + reason: string; +} + +export type LastExit = + | { at: number; kind: "boot-failed"; code: number; reason: string } + | { at: number; kind: "shutdown" | "signal"; code: number }; + +export interface SupervisionState { + bootAttempts: number; + lastReadyAt: number; + recentFailures: BootFailure[]; + lastExit: LastExit | null; +} + +function db() { + return getStateDb("daemon"); +} + +export function recordBootAttempt(): void { + const n = getKvValue(NS, KEY_BOOT_ATTEMPTS, 0, db()); + setKvValue(NS, KEY_BOOT_ATTEMPTS, n + 1, db()); +} + +export function recordDaemonReady(): void { + setKvValue(NS, KEY_LAST_READY_AT, Date.now(), db()); +} + +/** + * Always writes the breadcrumb file, which never needs state.db. The kv + * append is best-effort: a failure this early in boot may predate state.db + * being open at all, and that must not throw back into the caller's catch. + */ +export function recordBootFailure(phase: BootPhase, reason: string): void { + writeBreadcrumb(phase); + try { + const at = Date.now(); + const existing = getKvValue(NS, KEY_RECENT_FAILURES, [], db()); + const next = [...existing, { at, phase, reason }].slice(-RECENT_FAILURES_CAP); + setKvValue(NS, KEY_RECENT_FAILURES, next, db()); + setKvValue(NS, KEY_LAST_EXIT, { at, kind: "boot-failed", code: 1, reason }, db()); + } catch { + // Pre-db failure (or a busy/corrupt state.db): the breadcrumb file above + // is the only record this failure gets, and that's fine. + } +} + +export function recordCleanExit(kind: "shutdown" | "signal", code: number): void { + try { + setKvValue(NS, KEY_LAST_EXIT, { at: Date.now(), kind, code }, db()); + } catch { + // Best-effort, same as recordBootFailure's kv half. + } +} + +export function readSupervisionState(): SupervisionState { + const store = db(); + return { + bootAttempts: getKvValue(NS, KEY_BOOT_ATTEMPTS, 0, store), + lastReadyAt: getKvValue(NS, KEY_LAST_READY_AT, 0, store), + recentFailures: getKvValue(NS, KEY_RECENT_FAILURES, [], store), + lastExit: getKvValue(NS, KEY_LAST_EXIT, null, store), + }; +} + +export function isCrashLooping( + state: SupervisionState, + now: number, + n = 3, + windowMs = 5 * 60_000, +): boolean { + const floor = now - windowMs; + return state.recentFailures.filter((f) => f.at > floor).length >= n; +} + +// ─── Breadcrumb file (db-free) ──────────────────────────────────────────── + +interface Breadcrumb { + at: number; + pid: number; + flavor: "dev" | "prod"; + phase: BootPhase; +} + +function breadcrumbPath(): string { + return join(RT_DIR, "daemon-boot.json"); +} + +/** Never fatal: a breadcrumb is a diagnostic aid, not something boot may fail over. */ +export function writeBreadcrumb(phase: BootPhase): void { + try { + const breadcrumb: Breadcrumb = { at: Date.now(), pid: process.pid, flavor: daemonFlavor(), phase }; + writeFileSync(breadcrumbPath(), JSON.stringify(breadcrumb)); + } catch { + // Best-effort. + } +} + +export function readBreadcrumb(): Breadcrumb | null { + try { + if (!existsSync(breadcrumbPath())) return null; + return JSON.parse(readFileSync(breadcrumbPath(), "utf8")) as Breadcrumb; + } catch { + return null; + } +} diff --git a/lib/daemon/system-process-scanner.ts b/lib/daemon/system-process-scanner.ts index dfb121c26..2ac189417 100644 --- a/lib/daemon/system-process-scanner.ts +++ b/lib/daemon/system-process-scanner.ts @@ -222,7 +222,7 @@ async function getPackageScripts(pidList: string): Promise> return parsePackageScripts(stdout); } -async function getAllRepoPids(trackedPaths: string[]): Promise> { +async function getAllRepoPids(trackedPaths: string[]): Promise | null> { if (trackedPaths.length === 0) return new Map(); // Single lsof call to get cwds for ALL processes at once. @@ -232,8 +232,11 @@ async function getAllRepoPids(trackedPaths: string[]): Promise { + const gathered = await this.gather(portEntries); + if (gathered === null) return this.lastResult; // failed tick: preserve tracked/lastResult/lastScanAt try { - const gathered = await this.gather(portEntries); const now = Date.now(); const currentPids = new Set(); @@ -344,8 +348,9 @@ export class SystemProcessScanner { * alert. Also advances `lastScanAt` so freshness checks see the update. */ async refresh(portEntries: PortEntry[] = []): Promise { + const gathered = await this.gather(portEntries); + if (gathered === null) return this.lastResult; // failed tick: preserve tracked/lastResult/lastScanAt try { - const gathered = await this.gather(portEntries); const prev = new Map(this.lastResult.map(p => [p.pid, p])); const results = gathered.map(proc => { const before = prev.get(proc.pid); @@ -368,7 +373,7 @@ export class SystemProcessScanner { * decorate it with port + package-script, but no runaway/tracking state. * Shared by `scan` (adds tracking) and `refresh` (carries flags forward). */ - private async gather(portEntries: PortEntry[]): Promise { + protected async gather(portEntries: PortEntry[]): Promise { const repos = loadRepoIndex(); if (Object.keys(repos).length === 0) return []; @@ -378,7 +383,8 @@ export class SystemProcessScanner { const worktreeMap = await buildWorktreeMap(repos); const trackedPaths = [...new Set([...Object.values(repos), ...worktreeMap.keys()])]; const cwdMap = await getAllRepoPids(trackedPaths); - if (cwdMap.size === 0) return []; + if (cwdMap === null) return null; // lsof failed + if (cwdMap.size === 0) return []; // genuinely no tracked-cwd processes // Get CPU/memory for discovered PIDs const pidList = [...cwdMap.keys()].join(","); @@ -386,6 +392,7 @@ export class SystemProcessScanner { ["ps", "-p", pidList, "-o", "pid=,ppid=,pcpu=,rss=,etime=,comm=,args="], { timeoutMs: 5000 }, ); + if (psRes.exitCode !== 0 && !psRes.stdout) return null; // ps failed if (!psRes.stdout) return []; const parsed = parseProcessList(psRes.stdout, repos, cwdMap, worktreeMap); diff --git a/lib/daemon/worktree-process-kill.ts b/lib/daemon/worktree-process-kill.ts index ca499bc26..e41ee707f 100644 --- a/lib/daemon/worktree-process-kill.ts +++ b/lib/daemon/worktree-process-kill.ts @@ -12,9 +12,8 @@ * the caller — parking proceeds while stragglers wind down. */ -import { execSync } from "child_process"; - import { lazyChildLogger } from "../daemon-logger.ts"; +import { runCapture } from "../subprocess.ts"; import { parseLsofCwdMap, parsePackageScripts } from "./system-process-scanner.ts"; const log = lazyChildLogger("worktree-kill"); @@ -126,18 +125,14 @@ export interface WorktreeKillResult { * Discovery is done fresh (not from the scanner's 10s-old snapshot) so we act * on ground truth at park time. */ -export function killWorktreeProcesses(worktreePath: string): WorktreeKillResult { - let lsofOut: string; - try { - lsofOut = execSync("lsof -d cwd -Fpn 2>/dev/null", { - encoding: "utf8", stdio: "pipe", timeout: 10000, - }); - } catch (err) { - log.warn({ err, worktreePath }, "lsof failed; skipping worktree process kill"); +export async function killWorktreeProcesses(worktreePath: string): Promise { + const lsof = await runCapture(["lsof", "-d", "cwd", "-Fpn"], { timeoutMs: 10_000 }); + if (lsof.exitCode !== 0 && !lsof.stdout) { + log.warn({ exitCode: lsof.exitCode, worktreePath }, "lsof failed; skipping worktree process kill"); return { terminated: [] }; } - const cwdMap = parseLsofCwdMap(lsofOut, [worktreePath]); + const cwdMap = parseLsofCwdMap(lsof.stdout, [worktreePath]); // Drop close-parent matches (parseLsofCwdMap keeps them for scanner display; // a process merely *above* the worktree must not be killed). for (const [pid, cwd] of cwdMap) { @@ -147,24 +142,21 @@ export function killWorktreeProcesses(worktreePath: string): WorktreeKillResult const pidList = [...cwdMap.keys()].join(","); const rows: KillCandidate[] = []; - try { - const psOut = execSync(`ps -p ${pidList} -o pid=,ppid=,comm=,args= 2>/dev/null`, { - encoding: "utf8", stdio: "pipe", timeout: 5000, - }); - for (const line of psOut.split("\n")) { - const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\S+)\s+(.*)/); - if (!match) continue; - rows.push({ - pid: parseInt(match[1]!, 10), - ppid: parseInt(match[2]!, 10), - command: basename(match[3]!), - fullCommand: match[4]!, - }); - } - } catch (err) { - log.warn({ err, worktreePath }, "ps failed; skipping worktree process kill"); + const ps = await runCapture(["ps", "-p", pidList, "-o", "pid=,ppid=,comm=,args="], { timeoutMs: 5000 }); + if (ps.exitCode !== 0 && !ps.stdout) { + log.warn({ exitCode: ps.exitCode, worktreePath }, "ps failed; skipping worktree process kill"); return { terminated: [] }; } + for (const line of ps.stdout.split("\n")) { + const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\S+)\s+(.*)/); + if (!match) continue; + rows.push({ + pid: parseInt(match[1]!, 10), + ppid: parseInt(match[2]!, 10), + command: basename(match[3]!), + fullCommand: match[4]!, + }); + } const targets = selectKillTargets(rows, { protectedPids: [process.pid, process.ppid], @@ -174,12 +166,11 @@ export function killWorktreeProcesses(worktreePath: string): WorktreeKillResult // Label with the package-script invocation when known — that's the name the // developer recognizes ("pnpm start:lite:watch", not "node"). let scripts = new Map(); - try { - const ewwOut = execSync(`ps eww -o pid=,command= -p ${targets.map(t => t.pid).join(",")} 2>/dev/null`, { - encoding: "utf8", stdio: "pipe", timeout: 5000, maxBuffer: 32 * 1024 * 1024, - }); - scripts = parsePackageScripts(ewwOut); - } catch { /* labels fall back to comm */ } + const eww = await runCapture( + ["ps", "eww", "-o", "pid=,command=", "-p", targets.map(t => t.pid).join(",")], + { timeoutMs: 5000 }, + ); + if (eww.exitCode === 0 || eww.stdout) scripts = parsePackageScripts(eww.stdout); const terminated: { pid: number; label: string }[] = []; for (const target of targets) { diff --git a/lib/daemon/worktree-reconciler.ts b/lib/daemon/worktree-reconciler.ts index d427c9426..9ac37eac1 100644 --- a/lib/daemon/worktree-reconciler.ts +++ b/lib/daemon/worktree-reconciler.ts @@ -8,7 +8,7 @@ * `creationInFlight`). */ -import { basename, join } from "path"; +import { basename, isAbsolute, join, relative, resolve } from "path"; import { realpathSync } from "fs"; import type { Logger } from "pino"; import { rtDir } from "../rt-paths.ts"; @@ -29,6 +29,7 @@ import { gitOk, headSha, listWorktreesAsync, + MUTATING_TIMEOUT_MS, remoteDefaultRef, runGit, stashChangesAsync, @@ -441,24 +442,23 @@ async function autoReturnMain( } if (appConfig.killProcesses) { - // The ruled execSync exception (the process killer is sync by design); - // a failure here never blocks the return. + // A failure here never blocks the return. try { - const { terminated } = killWorktreeProcesses(rec.path); + const { terminated } = await killWorktreeProcesses(rec.path); if (terminated.length > 0) log.info({ ...fields, count: terminated.length }, "worktree processes terminated"); } catch (err) { log.warn({ err, ...fields }, "auto-return: process kill failed; returning anyway"); } } - const status = await runGit(rec.path, ["status", "--porcelain"]); + const status = await runGit(rec.path, ["status", "--porcelain"], { timeoutMs: MUTATING_TIMEOUT_MS }); if (status.exitCode !== 0) { log.warn({ ...fields, output: status.stderr.trim() }, "auto-return: git status failed"); return "retry"; } if (status.stdout.trim().length > 0) { await stashChangesAsync(rec.path, mergedBranch); - const after = await runGit(rec.path, ["status", "--porcelain"]); + const after = await runGit(rec.path, ["status", "--porcelain"], { timeoutMs: MUTATING_TIMEOUT_MS }); if (after.exitCode !== 0 || after.stdout.trim().length > 0) { log.warn({ ...fields }, "auto-return: stash did not clear the worktree"); return "retry"; @@ -466,13 +466,13 @@ async function autoReturnMain( log.info({ ...fields }, `stashed uncommitted changes on "${mergedBranch}"`); } - const checkout = await runGit(rec.path, ["checkout", defaultBranch]); + const checkout = await runGit(rec.path, ["checkout", defaultBranch], { timeoutMs: MUTATING_TIMEOUT_MS }); if (checkout.exitCode !== 0) { log.warn({ ...fields, defaultBranch, output: checkout.stderr.trim() }, "auto-return: checkout failed"); return "retry"; } - const ff = await runGit(rec.path, ["merge", "--ff-only", defaultRef]); + const ff = await runGit(rec.path, ["merge", "--ff-only", defaultRef], { timeoutMs: MUTATING_TIMEOUT_MS }); if (ff.exitCode !== 0) { log.warn({ ...fields, defaultRef, output: ff.stderr.trim() }, "auto-return: fast-forward failed"); return "retry"; @@ -724,7 +724,7 @@ async function freshenOne(deps: FreshenDeps, rec: TreeRecord): Promise const classify = await classifyDirtyAsync(rec.path); if (classify.discard.length > 0) { - await runGit(rec.path, ["checkout", "--", ...classify.discard]); + await runGit(rec.path, ["checkout", "--", ...classify.discard], { timeoutMs: MUTATING_TIMEOUT_MS }); } // Blockers stashed under the tree's own branch name (Desktop-compatible @@ -744,7 +744,7 @@ async function freshenOne(deps: FreshenDeps, rec: TreeRecord): Promise const popStash = async (): Promise => { if (!stashName) return; - const pop = await runGit(rec.path, ["stash", "pop", stashName]); + const pop = await runGit(rec.path, ["stash", "pop", stashName], { timeoutMs: MUTATING_TIMEOUT_MS }); if (pop.exitCode !== 0) { log.warn( { ...fields, stashName }, @@ -753,7 +753,7 @@ async function freshenOne(deps: FreshenDeps, rec: TreeRecord): Promise } }; - const ff = await runGit(rec.path, ["merge", "--ff-only", defaultRef]); + const ff = await runGit(rec.path, ["merge", "--ff-only", defaultRef], { timeoutMs: MUTATING_TIMEOUT_MS }); if (ff.exitCode !== 0) { log.warn({ ...fields, defaultRef, output: ff.stderr.trim() }, "freshen: fast-forward failed"); await popStash(); @@ -851,6 +851,30 @@ export async function freshenRepo( */ const createBackoff = new Map(); +/** + * S089: a provision's cold `createTree` (handlers/worktree.ts) and this + * reconciler's own replenish `createTree` can run concurrently for the same + * repo, both `git fetch origin ` against the same repoPath — the + * loser fails to lock refs/remotes/origin/, and that failure gets + * charged to createBackoff (a 5-to-30-minute replenish hold) for what was + * really just contention, not a genuine failure. Chained per repoPath so + * concurrent callers queue instead of racing; a rejected holder still + * releases the lock for the next one. + */ +const createLocks = new Map>(); + +export function withCreateLock(repoPath: string, fn: () => Promise): Promise { + const prior = createLocks.get(repoPath) ?? Promise.resolve(); + const ready = prior.catch(() => {}); // a previous holder's rejection must not block the next one + const result = ready.then(fn); + const tracked: Promise = result.then(() => undefined, () => undefined); + createLocks.set(repoPath, tracked); + void tracked.finally(() => { + if (createLocks.get(repoPath) === tracked) createLocks.delete(repoPath); + }); + return result; +} + /** The active backoff deadline for a repo, or null when creates may run now. */ function createBlockedUntil(repoName: string): string | null { const entry = createBackoff.get(repoName); @@ -920,7 +944,7 @@ async function replenishAndShrink( break; } budget--; - const p: Promise = createTree({ repoName, repoPath, emit, log }) + const p: Promise = withCreateLock(repoPath, () => createTree({ repoName, repoPath, emit, log })) .then((result) => { if (result.ok) { createBackoff.delete(repoName); @@ -1002,10 +1026,30 @@ async function replenishAndShrink( * parks trees stripped-but-recoverable (RT-51) — are reaped only past the * retention window. */ +/** + * Whether `root` is repoPath itself or a strict ancestor of it — + * sanitizeRoot (lib/worktree/config.ts) has no such check, so a value like + * `${repoRoot}/..` sweeps the parent directory shared by every sibling repo + * for `.trash-*` names. An unrelated, dedicated external root (the + * documented `root: "~/wt"` case) is fine to sweep — it's a repo-specific + * destination nothing else shares — so this only refuses the ancestor + * shape, not "root lies outside repoPath" in general. + */ +function isRootAnAncestorOfRepo(repoPath: string, root: string): boolean { + const rel = relative(resolve(root), resolve(repoPath)); + return rel === "" || (!rel.startsWith("..") && !isAbsolute(rel)); +} + async function reapRepoTrash(deps: { repoName: string; repoPath: string; log: Logger }): Promise { const { repoName, repoPath, log } = deps; const cfg = await loadWorktreeRepoConfig(repoName, repoPath); - const reaped = await reapTrashInRoots([join(repoPath, ".worktrees"), cfg.root], log); + const roots = [join(repoPath, ".worktrees")]; + if (isRootAnAncestorOfRepo(repoPath, cfg.root)) { + log.warn({ repo: repoName, root: cfg.root, repoPath }, "worktree trash sweep refused a configured root that is an ancestor of the repo"); + } else { + roots.push(cfg.root); + } + const reaped = await reapTrashInRoots(roots, log); if (reaped > 0) log.info({ repo: repoName, count: reaped }, "worktree trash reaped"); const expired = await reapExpiredTrash(repoPath, log); if (expired > 0) log.info({ repo: repoName, count: expired }, "worktree retention trash reaped"); @@ -1057,6 +1101,16 @@ export function createWorktreeReconciler(deps: ReconcilerDeps): { /** Non-null while a holder owns the reconciler. */ let hold: Promise | null = null; let kickQueued = false; + /** + * True once the current pass's per-repo loop has begun processing at + * least one repo. Two kicks that both land before this flips (the common + * "two synchronous kicks" case) still collapse to one pass — the + * upcoming loop reads fresh state regardless. A kick landing after it + * flips might be about a repo this pass has already stepped past (e.g. a + * provision claiming the last on-deck tree right after replenish ran for + * it), so it queues a follow-up instead of being silently dropped. + */ + let passStartedWork = false; const creationPromises = new Map>(); async function runOnce(): Promise { @@ -1074,6 +1128,7 @@ export function createWorktreeReconciler(deps: ReconcilerDeps): { const appConfig = loadWorktreeAppConfig(); for (const [repoName, repoPath] of Object.entries(repos)) { + passStartedWork = true; if (!(await repoHasWorktreeActivity(repoName, repoPath))) continue; try { await reconcileRepoRegistry({ repoName, repoPath, emit: deps.emit, log: deps.log }); @@ -1123,13 +1178,25 @@ export function createWorktreeReconciler(deps: ReconcilerDeps): { kickQueued = true; return; } - if (inFlight) return; + if (inFlight) { + // Two kicks landing before this pass has stepped into its per-repo + // loop still collapse to one pass; once it has, a kick might be about + // a repo already stepped past (its replenish already ran this pass), + // so queue a follow-up rather than dropping it silently. + if (passStartedWork) kickQueued = true; + return; + } + passStartedWork = false; const p = runOnce() .catch((err) => { deps.log.warn({ err }, "worktree reconciler: kick failed"); }) .finally(() => { if (inFlight === p) inFlight = null; + if (kickQueued) { + kickQueued = false; + kick(); + } }); inFlight = p; } diff --git a/lib/deps/__tests__/links.test.ts b/lib/deps/__tests__/links.test.ts index 5b9fef2e9..63ea50488 100644 --- a/lib/deps/__tests__/links.test.ts +++ b/lib/deps/__tests__/links.test.ts @@ -289,6 +289,26 @@ describe("tagged PATH links", () => { expect(result).toEqual({ removed: ["gh"], kept: [] }); }); + // S066: a DEFAULT_EXPOSED tool (rt's own product surface, not a vendored + // third-party tool like "gh") is never auto-unlinked by a same-named PATH + // collision — an unrelated foreign tool of the same name (e.g. Kong's own + // "deck") must never shadow-remove mattstack's. + test("reconcile never auto-unlinks deck/gitq/rt even when a same-named binary appears elsewhere on PATH (S066)", () => { + const p = bundleProbe({ env: { PATH: "/opt/homebrew/bin" } }); + p.symlink(join(appRoot, HELPERS_DIR, "deck"), linkPath(home, "deck")); + p.symlink(join(appRoot, HELPERS_DIR, "gitq"), linkPath(home, "gitq")); + link(p, "rt", {}, { installRtBinary: (src, dest) => { p.symlink(src, dest); return dest; } }); + + // An unrelated foreign binary happens to share each name. + p.writeFile("/opt/homebrew/bin/deck", "kongs-deck-binary"); + p.writeFile("/opt/homebrew/bin/gitq", "some-other-gitq-binary"); + p.writeFile("/opt/homebrew/bin/rt", "some-other-rt-binary"); + + const result = reconcile(p); + expect(result.removed).toEqual([]); + expect(result.kept.sort()).toEqual(["deck", "gitq", "rt"]); + }); + test("real-fs repro (F1): after the daemon's boot-time PATH prepend, reconcile removes nothing", () => { // Real fs (existsSync/statSync/symlinkSync/readdirSync via createRealProbes) // is what actually matters for this proof — the file/directory distinction diff --git a/lib/deps/links.ts b/lib/deps/links.ts index e70d56a95..df570d162 100644 --- a/lib/deps/links.ts +++ b/lib/deps/links.ts @@ -147,11 +147,20 @@ export function unlink(p: Probes, tool: string): { removed: boolean } { export function reconcile(p: Probes): { removed: string[]; kept: string[] } { const dir = join(p.home, ".local", "bin"); const forced = new Set(readSetupState(p).forcedLinks); + const defaultExposed = new Set(DEFAULT_EXPOSED); const removed: string[] = []; const kept: string[] = []; for (const tool of p.readDir(dir)) { if (!isOurLink(p, tool)) continue; // not one of ours — nothing to reconcile - if (forced.has(tool) || !userCopyOnPath(p, tool)) { + // DEFAULT_EXPOSED (rt, fast-browser, gitq, deck) is mattstack's own + // product surface, not a vendored third-party tool like the bundled + // "gh" — userCopyOnPath matches by name only, so a same-named PATH + // entry is at least as likely to be an unrelated namesake (Kong's own + // "deck") as a genuine second copy of ours, and shadow-removing our own + // "rt" this way breaks every command. Only a vendored tool's step-aside + // (deferring to a real system copy the user installed) is safe to + // decide by name alone. + if (defaultExposed.has(tool) || forced.has(tool) || !userCopyOnPath(p, tool)) { kept.push(tool); } else { p.removeFile(linkPath(p.home, tool)); diff --git a/lib/herdr/__tests__/client.test.ts b/lib/herdr/__tests__/client.test.ts index 431b71328..952601ab6 100644 --- a/lib/herdr/__tests__/client.test.ts +++ b/lib/herdr/__tests__/client.test.ts @@ -66,3 +66,33 @@ test("herdrAvailable probes session.snapshot", async () => { test("waitTimeout adds the 5s margin herdr needs to answer at its own budget", () => { expect(waitTimeout(60_000)).toBe(65_000); }); + +// S095: herdrRequest decoded each socket chunk independently — a multibyte +// character (box-drawing, emoji, ...) split across a chunk boundary became +// two partial byte sequences, each decoding to U+FFFD, corrupting the reply +// even though JSON.parse still succeeded on the well-formed-but-wrong text. +test("a multibyte character split across two socket chunks decodes correctly", async () => { + const sock = join(tmpdir(), `herdr-split-${process.pid}.sock`); + const text = "a─b"; // U+2500 BOX DRAWINGS LIGHT HORIZONTAL: E2 94 80, 3 bytes + const payload = JSON.stringify({ id: "x", result: { text } }) + "\n"; + const bytes = Buffer.from(payload, "utf8"); + const charStart = bytes.indexOf(Buffer.from("─", "utf8")); + const splitAt = charStart + 1; // split inside the multibyte sequence, after its first byte + + const server = Bun.listen({ + unix: sock, + socket: { + open(socket) { + socket.write(bytes.subarray(0, splitAt)); + setTimeout(() => socket.write(bytes.subarray(splitAt)), 20); + }, + data() {}, + close() {}, + error() {}, + }, + }); + stops.push(() => server.stop(true)); + + const res = await herdrRequest<{ text: string }>("whatever", {}, { sockPath: sock }); + expect(res).toEqual({ ok: true, result: { text: "a─b" } }); +}); diff --git a/lib/herdr/client.ts b/lib/herdr/client.ts index eb1dc3737..c252b406c 100644 --- a/lib/herdr/client.ts +++ b/lib/herdr/client.ts @@ -37,7 +37,12 @@ export function herdrRequest( return new Promise((resolve) => { let settled = false; - let buf = ""; + // A multibyte UTF-8 character (box-drawing, emoji, ...) can straddle a + // socket chunk boundary; decoding each chunk independently turns each + // half into its own U+FFFD replacement character, corrupting the reply + // (JSON.parse still succeeds — the text is just wrong). Accumulating raw + // bytes and decoding the whole buffer together avoids that entirely. + const chunks: Buffer[] = []; let conn: { end(): void } | undefined; const finish = (r: HerdrResult) => { if (settled) return; @@ -55,6 +60,7 @@ export function herdrRequest( } const tryParseBuffered = (): HerdrResult | undefined => { + const buf = Buffer.concat(chunks).toString("utf8"); const nl = buf.indexOf("\n"); if (nl < 0) return undefined; const text = buf.slice(0, nl); @@ -81,7 +87,7 @@ export function herdrRequest( socket.write(line); }, data(_socket, chunk) { - buf += chunk.toString(); + chunks.push(chunk); const parsed = tryParseBuffered(); if (parsed) finish(parsed); }, diff --git a/lib/home/__tests__/age-key.test.ts b/lib/home/__tests__/age-key.test.ts index 4aad547d7..7615dab42 100644 --- a/lib/home/__tests__/age-key.test.ts +++ b/lib/home/__tests__/age-key.test.ts @@ -9,6 +9,8 @@ import { keyExport, withArgvRedaction, AgeKeyAbsentError, + AgeKeyTimeoutError, + createRealAgeKeySeam, type AgeExecResult, type AgeKeySeam, } from "../age-key.ts"; @@ -378,3 +380,47 @@ describe("keyExport", () => { expect('import { statfs } from "./statfs-helpers.ts";').not.toMatch(FS_IMPORT_RE); }); }); + +// S070: a locked keychain (screen-lock, or an ACL mismatch between the dev +// shim and mattstack.app) pops a GUI dialog that blocks the real spawn +// forever; every seam.run must have a bounded deadline instead. +describe("createRealAgeKeySeam timeout (S070)", () => { + test("a spawn that outlives its timeout rejects with a distinguished AgeKeyTimeoutError instead of hanging", async () => { + const seam = createRealAgeKeySeam(); + await expect(seam.run(["sleep", "5"], { timeoutMs: 50 })).rejects.toThrow(AgeKeyTimeoutError); + }); + + test("the timeout error names the command without leaking a sensitive -w value", async () => { + const seam = createRealAgeKeySeam(); + // sh -c's trailing args become unused positional params ($0, $1, ...) — + // this still sleeps for the script text alone, while still carrying a + // "-w " pair later in argv for redactArgv to find. + const cmd = ["sh", "-c", "sleep 5", "argv0", "-w", "AGE-SECRET-KEY-should-not-appear"]; + try { + await seam.run(cmd, { timeoutMs: 50, sensitive: true }); + throw new Error("expected a timeout"); + } catch (err) { + expect(err).toBeInstanceOf(AgeKeyTimeoutError); + expect((err as Error).message).not.toContain("AGE-SECRET-KEY-should-not-appear"); + expect((err as Error).message).toContain(""); + } + }); + + test("a spawn that finishes well within its timeout resolves normally", async () => { + const seam = createRealAgeKeySeam(); + const res = await seam.run(["echo", "hi"], { timeoutMs: 5000 }); + expect(res.code).toBe(0); + expect(res.stdout.trim()).toBe("hi"); + }); + + test("a child that ignores SIGTERM still rejects promptly with AgeKeyTimeoutError (C7: the deadline settles independently of proc.exited)", async () => { + const seam = createRealAgeKeySeam(); + const start = Date.now(); + await expect( + seam.run(["bash", "-c", "trap '' TERM; sleep 30"], { timeoutMs: 50 }), + ).rejects.toThrow(AgeKeyTimeoutError); + // Must settle on the timeout deadline (plus the SIGKILL escalation grace), + // not wait out proc.exited for a child that never dies from SIGTERM alone. + expect(Date.now() - start).toBeLessThan(4000); + }); +}); diff --git a/lib/home/age-key.ts b/lib/home/age-key.ts index 1df556ab6..fd1aa0fa4 100644 --- a/lib/home/age-key.ts +++ b/lib/home/age-key.ts @@ -29,8 +29,26 @@ export interface AgeKeySeam { * the private key this way, keeping it out of argv). * `sensitive` marks a call whose argv or stdin carries key material, so a * logging wrapper (withArgvRedaction) knows to redact it. + * `timeoutMs` overrides the default kill-and-reject deadline (see + * AgeKeyTimeoutError) — a locked keychain or an unexpected access-control + * dialog otherwise blocks this call, and every caller behind it, forever. */ - run(cmd: string[], opts?: { input?: string; sensitive?: boolean }): Promise; + run(cmd: string[], opts?: { input?: string; sensitive?: boolean; timeoutMs?: number }): Promise; +} + +/** + * Thrown instead of resolving when the spawn outlives its timeout — a + * distinct class (not a generic Error, and never a resolved AgeExecResult) + * so a caller like loadSecrets's domainMemo can recognize "the keychain is + * probably showing a dialog right now" and avoid caching it as a permanent + * decrypt failure that would poison every later retry, including the one + * after the user dismisses the dialog. + */ +export class AgeKeyTimeoutError extends Error { + constructor(message: string) { + super(message); + this.name = "AgeKeyTimeoutError"; + } } const KEYCHAIN_ACCOUNT = "mattstack"; @@ -277,6 +295,9 @@ function debugLog(cmd: string[]): void { if (CLI_DEBUG) console.error(`[age-key] ${cmd.join(" ")}`); } +/** A locked keychain (screen-lock, or a keychain item whose ACL belongs to a different signed binary) pops a GUI dialog and blocks until clicked; this bounds every spawn against that. */ +const DEFAULT_AGE_KEY_TIMEOUT_MS = 30_000; + /** Bun.spawn-based capture, env passed live (PATH-snapshot gotcha). Unexported: only reachable wrapped, via createRealAgeKeySeam. */ function createRawAgeKeySeam(): AgeKeySeam { return { @@ -298,12 +319,47 @@ function createRawAgeKeySeam(): AgeKeySeam { // read it ignore the close, but age-keygen -y blocks on EOF otherwise. if (opts?.input !== undefined) proc.stdin.write(opts.input); proc.stdin.end(); - const [stdout, stderr, code] = await Promise.all([ - new Response(proc.stdout).text(), - new Response(proc.stderr).text(), - proc.exited, - ]); - return { code, stdout, stderr }; + + const timeoutMs = opts?.timeoutMs ?? DEFAULT_AGE_KEY_TIMEOUT_MS; + + // A child that ignores SIGTERM (or a locked-keychain dialog that never + // closes) would otherwise keep `proc.exited` pending forever, so the + // deadline is raced against the read independently rather than + // discovered only after Promise.all settles — mirrors lib/subprocess.ts. + const captured: Promise = (async () => { + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + return { code, stdout, stderr }; + })(); + + let killTimer: ReturnType | undefined; + let deadlineTimer!: ReturnType; + const deadline: Promise = new Promise((_, reject) => { + deadlineTimer = setTimeout(() => { + try { proc.kill("SIGTERM"); } catch { /* already exited */ } + // SIGTERM alone is not guaranteed (a trapped or hung child can + // ignore it); escalate to SIGKILL after a short grace, unref'd so + // it never holds this process open past the caller's own use of it. + killTimer = setTimeout(() => { + try { proc.kill("SIGKILL"); } catch { /* already exited */ } + }, 2000); + killTimer.unref?.(); + reject(new AgeKeyTimeoutError( + `${redactArgv(cmd).join(" ")}: did not exit within ${timeoutMs}ms (keychain prompt pending?)`, + )); + }, timeoutMs); + }); + + try { + return await Promise.race([captured, deadline]); + } finally { + clearTimeout(deadlineTimer); + // killTimer intentionally NOT cleared here: on the timeout path it + // must survive to fire SIGKILL against a child that ignored SIGTERM. + } }, }; } diff --git a/lib/notifier.ts b/lib/notifier.ts index 55db3c325..0c235c875 100644 --- a/lib/notifier.ts +++ b/lib/notifier.ts @@ -214,6 +214,29 @@ export function peekNotifications(): NotificationEvent[] { * Attempt to push a notification event to the tray app via its Unix socket. * Returns true if the push succeeded, false if tray is unavailable. */ +/** + * removeQueuedNotification's own write already retries on SQLITE_BUSY (3 x + * 20ms, lib/state/notifier-store.ts) and gives up silently. When that + * happens after a successful tray push, the row stays queued and the next + * drainNotifications()/peekNotifications() redelivers it as a duplicate. + * Retry the removal at this layer, bounded, before reporting success. + */ +async function removeFromQueueWithRetry( + eventId: string, + attempts = 3, + delayMs = 50, + removeFn: (id: string) => void = removeQueuedNotification, + isQueuedFn: (id: string) => boolean = isNotificationQueued, +): Promise { + for (let i = 0; i < attempts; i++) { + removeFn(eventId); + if (!isQueuedFn(eventId)) return true; + if (i < attempts - 1) await new Promise((r) => setTimeout(r, delayMs)); + } + log.warn({ eventId }, "notification queue removal kept failing after a successful push; it may redeliver on the next drain"); + return false; +} + async function pushToTray(event: NotificationEvent): Promise { if (!existsSync(TRAY_SOCK_PATH)) return false; @@ -227,8 +250,7 @@ async function pushToTray(event: NotificationEvent): Promise { } as any); if (response.ok) { - // Push succeeded — remove from queue - removeQueuedNotification(event.id); + await removeFromQueueWithRetry(event.id); return true; } return false; @@ -878,6 +900,7 @@ export const __test__ = { notifyFallback, firedKey, pruneFiredForEvictedBranches, + removeFromQueueWithRetry, setFallbackNotifier(path: string | null): void { fallbackNotifier = path ?? "osascript"; }, diff --git a/lib/port-scanner.ts b/lib/port-scanner.ts index c437b620e..8ff5b4d99 100644 --- a/lib/port-scanner.ts +++ b/lib/port-scanner.ts @@ -10,7 +10,7 @@ * would freeze the event loop long enough to time out status polls. */ -import { existsSync } from "fs"; +import { existsSync, realpathSync } from "fs"; import { homedir } from "os"; import { loadRepoIndex as loadRepoIndexFromStore } from "./repo-index.ts"; import { runCapture } from "./subprocess.ts"; @@ -167,6 +167,39 @@ export async function buildWorktreeMap( return new Map(perRepo.flat()); } +/** + * lsof reports the kernel's resolved (real, canonically-cased) path for a + * process's cwd. The repo index and worktree map instead carry whatever + * path the user cd'd through when the repo was registered (behind a + * symlink, under a case-insensitive APFS volume's non-canonical spelling, + * ...), so a raw string comparison against lsof's cwd can miss even a repo + * that is genuinely running. Falls back to the literal path when it does + * not (yet) exist on disk — mirrors lib/runs/prune.ts's canon(). + */ +function canon(path: string): string { + try { + return realpathSync(path); + } catch { + return path; + } +} + +/** Canonicalizes every path in the repo index once per scan (not once per matched port). */ +export function canonicalizeRepoIndex(repos: Record): Record { + const out: Record = {}; + for (const [name, path] of Object.entries(repos)) out[name] = canon(path); + return out; +} + +/** Canonicalizes every worktree path's key once per scan (not once per matched port). */ +export function canonicalizeWorktreeMap( + worktreeMap: Map, +): Map { + const out = new Map(); + for (const [path, info] of worktreeMap) out.set(canon(path), info); + return out; +} + export function matchCwdToRepo( cwd: string, repos: Record, @@ -215,13 +248,17 @@ export function matchCwdToRepo( * ports whose process CWD matches a known repo. */ export async function scanListeningPorts(): Promise { - const repos = loadRepoIndex(); - if (Object.keys(repos).length === 0) return []; + const rawRepos = loadRepoIndex(); + if (Object.keys(rawRepos).length === 0) return []; - const [worktreeMap, listenersRes] = await Promise.all([ - buildWorktreeMap(repos), + const [rawWorktreeMap, listenersRes] = await Promise.all([ + buildWorktreeMap(rawRepos), runCapture(["lsof", "-iTCP", "-sTCP:LISTEN", "-P", "-n"], { timeoutMs: 10_000 }), ]); + // Once per scan, not once per matched port: lsof's cwd is already real, + // so the index/worktree side is what needs canonicalizing to match it. + const repos = canonicalizeRepoIndex(rawRepos); + const worktreeMap = canonicalizeWorktreeMap(rawWorktreeMap); const listeners = parseListeningLsof(listenersRes.stdout); if (listeners.length === 0) return []; diff --git a/lib/repo-index.ts b/lib/repo-index.ts index e5ddf614d..6a8a4f0f2 100644 --- a/lib/repo-index.ts +++ b/lib/repo-index.ts @@ -32,6 +32,7 @@ import { repoLabel, repoLabelFull, repoLabelQualified } from "./repo-label.ts"; import { dim } from "./ansi.ts"; import { getSetting } from "./settings/resolve.ts"; import { mergeRegistries, type TreeRecord } from "./worktree/registry.ts"; +import { listWorktreesAsync } from "./worktree/git-async.ts"; // ─── Types ─────────────────────────────────────────────────────────────────── @@ -151,6 +152,13 @@ function observedMainPath(repoRoot: string): string { } } +/** Async twin of observedMainPath: the repo's MAIN worktree path as git + * reports it, degrading to repoRoot. Safe on the daemon thread. */ +async function observedMainPathAsync(repoRoot: string): Promise { + const wts = await listWorktreesAsync(repoRoot); + return wts?.[0]?.path ?? repoRoot; +} + /** * The row's current path, read straight from the namespace rather than through * `loadRepoIndex()`: that function's legacy-repos.json import is a migration @@ -274,7 +282,7 @@ export async function resolveIndexPathForIdentity(serialized: string): Promise existsSync(join(dir, name))); +} + /** * One-shot migration of a real legacy ~/.rt directory to ~/.mattstack/rt. * Called early from the CLI entry and daemon boot — BEFORE anything (loggers @@ -287,9 +301,12 @@ export const RT_DIR_LABEL = "~/.mattstack/rt"; * ~/.rt would land in "conflict" instead of migrating. * * - ~/.rt absent, or a symlink (the RT-33 compat shim): "none", untouched. - * - real ~/.rt, no ~/.mattstack/rt: rename it into place → "migrated". - * - real ~/.rt AND ~/.mattstack/rt both exist: "conflict" — state is split - * and a human must merge; nothing is touched. + * - ~/.rt real but carries no rt signature (another tool's config dir of + * the same name): "none", untouched — never renamed, parsed, or reported + * as a conflict against ~/.mattstack/rt. + * - real ~/.rt (rt's), no ~/.mattstack/rt: rename it into place → "migrated". + * - real ~/.rt (rt's) AND ~/.mattstack/rt both exist: "conflict" — state is + * split and a human must merge; nothing is touched. */ export function migrateLegacyRtDir(): LegacyMigrationResult { const legacy = legacyRtDir(); @@ -300,6 +317,7 @@ export function migrateLegacyRtDir(): LegacyMigrationResult { return "none"; // no ~/.rt at all } if (legacyStat.isSymbolicLink() || !legacyStat.isDirectory()) return "none"; + if (!hasRtSignature(legacy)) return "none"; // real directory, but not rt's — never touch it const target = rtDir(); try { diff --git a/lib/runs/__tests__/prune.test.ts b/lib/runs/__tests__/prune.test.ts index 75adbbb26..86a7a815f 100644 --- a/lib/runs/__tests__/prune.test.ts +++ b/lib/runs/__tests__/prune.test.ts @@ -35,6 +35,15 @@ function seedRun(dir: string, repo: string, id: string, startedAt: number, userV db.close(); } +/** Polls until the detached rm -rf a prune spawns has actually removed `path` (S100: the delete is no longer synchronous). */ +async function waitGone(path: string, timeoutMs = 5000): Promise { + const deadline = Date.now() + timeoutMs; + while (existsSync(path)) { + if (Date.now() > deadline) throw new Error(`timed out waiting for ${path} to be reaped`); + await new Promise((r) => setTimeout(r, 20)); + } +} + function seedRunEnded(dir: string, repo: string, id: string, startedAt: number, endedAt: number, status = "done"): void { const runDir = join(dir, repo, id); mkdirSync(runDir, { recursive: true }); @@ -55,7 +64,7 @@ function seedRunEnded(dir: string, repo: string, id: string, startedAt: number, } describe("pruneRuns", () => { - test("removes runs ended past the floor, keeps recent and running ones", () => { + test("removes runs ended past the floor, keeps recent and running ones", async () => { const dir = root(); const now = Date.now(); seedRunEnded(dir, "alpha", "old-done", now - 40 * DAY, now - 40 * DAY); // ended 40d ago -> pruned @@ -64,12 +73,12 @@ describe("pruneRuns", () => { // never-finished: age by state.db mtime (fresh in this test) -> kept const { removed } = pruneRuns(now); expect(removed).toEqual([join(dir, "alpha", "old-done")]); - expect(existsSync(join(dir, "alpha", "old-done"))).toBe(false); + await waitGone(join(dir, "alpha", "old-done")); // reaped by a detached rm -rf, not synchronously expect(existsSync(join(dir, "alpha", "new-done"))).toBe(true); expect(existsSync(join(dir, "alpha", "still-running"))).toBe(true); }); - test("a running run ages out once its state.db mtime crosses the floor", () => { + test("a running run ages out once its state.db mtime crosses the floor", async () => { const dir = root(); const now = Date.now(); seedRun(dir, "alpha", "stale-running", now - 40 * DAY); @@ -78,7 +87,7 @@ describe("pruneRuns", () => { utimesSync(dbPath, oldTime, oldTime); const { removed } = pruneRuns(now); expect(removed).toEqual([join(dir, "alpha", "stale-running")]); - expect(existsSync(join(dir, "alpha", "stale-running"))).toBe(false); + await waitGone(join(dir, "alpha", "stale-running")); }); test("a stale regular file inside a repo dir is not mistaken for a run and survives pruning", () => { @@ -110,4 +119,27 @@ describe("pruneRuns", () => { expect(() => assertPrunable(join(dir, "alpha", "victim-run"), dir)).toThrow(); expect(existsSync(join(outside, "victim-run"))).toBe(true); }); + + // S100: the boot-time prune (60s after start) previously unlinked every + // expired run tree synchronously, blocking the daemon's single thread for + // the full duration of each recursive rm — a tray poll or chat post + // arriving mid-sweep would time out. The delete must be off-thread. + test("prune spawns the delete asynchronously — the run dir is not synchronously unlinked (S100)", () => { + const dir = root(); + const now = Date.now(); + seedRunEnded(dir, "alpha", "old-done", now - 40 * DAY, now - 40 * DAY); + const { removed } = pruneRuns(now); + expect(removed).toEqual([join(dir, "alpha", "old-done")]); + // Not gone yet: the unlink runs in a detached child process, off this + // thread, so pruneRuns returning never means the disk is clean yet. + expect(existsSync(join(dir, "alpha", "old-done"))).toBe(true); + }); + + test("prune's detached delete eventually removes the run dir", async () => { + const dir = root(); + const now = Date.now(); + seedRunEnded(dir, "alpha", "old-done", now - 40 * DAY, now - 40 * DAY); + pruneRuns(now); + await waitGone(join(dir, "alpha", "old-done")); + }); }); diff --git a/lib/runs/__tests__/store-memo.test.ts b/lib/runs/__tests__/store-memo.test.ts new file mode 100644 index 000000000..da1b9a463 --- /dev/null +++ b/lib/runs/__tests__/store-memo.test.ts @@ -0,0 +1,28 @@ +import { Database } from "bun:sqlite"; +import { afterEach, expect, spyOn, test } from "bun:test"; +import { listRuns } from "../store.ts"; +import { root, seedRun } from "./fixtures.ts"; + +afterEach(() => { delete process.env.RT_RUNS_ROOT; }); + +test("a finished run's db is opened once, then served from the mtime cache", () => { + const dir = root(); + seedRun(dir, "repoA", "run1", 1000, 1, { status: "done" }); + const openSpy = spyOn(Database.prototype, "query"); + listRuns(); // first call opens + reads + const afterFirst = openSpy.mock.calls.length; + listRuns(); // second call: mtime unchanged -> no reopen + expect(openSpy.mock.calls.length).toBe(afterFirst); // no additional queries for run1 + openSpy.mockRestore(); +}); + +test("a running run is never cached: it is reopened even when its mtime is unchanged", () => { + const dir = root(); + seedRun(dir, "repoA", "run2", 1000, 1, { status: "running" }); + const openSpy = spyOn(Database.prototype, "query"); + listRuns(); + const afterFirst = openSpy.mock.calls.length; + listRuns(); + expect(openSpy.mock.calls.length).toBeGreaterThan(afterFirst); + openSpy.mockRestore(); +}); diff --git a/lib/runs/prune.ts b/lib/runs/prune.ts index b42b05a28..e2b336388 100644 --- a/lib/runs/prune.ts +++ b/lib/runs/prune.ts @@ -5,7 +5,7 @@ * and must stay in front of every rmSync. */ import { Database } from "bun:sqlite"; -import { existsSync, readdirSync, realpathSync, rmSync, statSync, type Dirent } from "fs"; +import { existsSync, readdirSync, realpathSync, statSync, type Dirent } from "fs"; import { basename, dirname, join, sep } from "path"; import { getSetting } from "../settings/resolve.ts"; import { runsRoot } from "./store.ts"; @@ -56,6 +56,26 @@ function realDirNames(path: string): string[] { } } +/** + * Detached, unawaited `rm -rf` — mirrors lib/worktree/trash.ts's reap + * pattern. A recursive unlink of a large run tree must never block the + * daemon's single event-loop thread: a sync rmSync here froze every + * concurrent tray poll and chat post for the sweep's full duration. + */ +function reapAsync(path: string): void { + try { + const proc = Bun.spawn(["rm", "-rf", "--", path], { + stdin: "ignore", + stdout: "ignore", + stderr: "ignore", + detached: true, + }); + proc.unref(); + } catch { + // Best-effort: a run tree that survives a failed spawn costs disk, never correctness. + } +} + function floorDays(): number { try { const v = getSetting("rt.runsPruneDays").value; @@ -98,7 +118,7 @@ export function pruneRuns(now: number = Date.now()): { removed: string[] } { } if (stamp < cutoff) { assertPrunable(runDir, root); - rmSync(runDir, { recursive: true, force: true }); + reapAsync(runDir); removed.push(runDir); } } diff --git a/lib/runs/store.ts b/lib/runs/store.ts index 3e290ac28..b9719ea62 100644 --- a/lib/runs/store.ts +++ b/lib/runs/store.ts @@ -4,7 +4,7 @@ * and per-call — no held connections, so a run dir can be pruned under us. */ import { Database } from "bun:sqlite"; -import { existsSync, readdirSync, type Dirent } from "fs"; +import { existsSync, readdirSync, statSync, type Dirent } from "fs"; import { homedir } from "os"; import { join } from "path"; import type { Attention, RunDetail, RunFieldRow, RunStageRow, RunSummary } from "../../packages/rt-client/src/commands.ts"; @@ -114,17 +114,33 @@ function withAttention(db: Database, row: RunSummary, liveness?: RunLiveness): R } } +// Finished runs never change; skip the open+PRAGMA+4-reads when the db mtime +// is unchanged. Running runs are never cached: their db still mutates and +// their liveness overlay is recomputed per call. +const summaryCache = new Map(); + export function listRuns(repo?: string, liveness?: RunLiveness): RunSummary[] { if (repo != null && !isPathComponent(repo)) return []; const repos = repo ? [repo] : dirs(runsRoot()); const out: RunSummary[] = []; for (const r of repos) { for (const id of dirs(join(runsRoot(), r))) { + const dbPath = join(runsRoot(), r, id, "state.db"); + let mtimeMs: number; + try { mtimeMs = statSync(dbPath).mtimeMs; } catch { continue; } + const key = `${runsRoot()}/${r}/${id}`; + const hit = summaryCache.get(key); + if (hit && hit.mtimeMs === mtimeMs) { out.push(hit.summary); continue; } + const opened = openRun(r, id); if (!opened) continue; try { const row = runRow(opened.db); - if (row) out.push(withAttention(opened.db, row, liveness)); + if (row) { + const summary = withAttention(opened.db, row, liveness); + out.push(summary); + if (summary.status !== "running") summaryCache.set(key, { mtimeMs, summary }); + } } finally { opened.db.close(); } diff --git a/lib/state/__tests__/busy.test.ts b/lib/state/__tests__/busy.test.ts index 1f8eb70f1..01791698b 100644 --- a/lib/state/__tests__/busy.test.ts +++ b/lib/state/__tests__/busy.test.ts @@ -1,5 +1,9 @@ import { expect, test } from "bun:test"; -import { runCriticalWrite } from "../busy.ts"; +import { Database } from "bun:sqlite"; +import { mkdtempSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { isBusyError, runCriticalWrite } from "../busy.ts"; test("returns the value when fn succeeds", () => { expect(runCriticalWrite("t", () => 42, {})).toBe(42); @@ -26,3 +30,20 @@ test("returns undefined after exhausting attempts on a busy error", () => { test("rethrows a non-busy error rather than retrying", () => { expect(() => runCriticalWrite("t", () => { throw new Error("syntax error"); }, {})).toThrow("syntax error"); }); + +test("isBusyError matches SQLITE_BUSY_SNAPSHOT from a real conflict", () => { + const dir = mkdtempSync(join(tmpdir(), "busy-snap-")); + const path = join(dir, "t.db"); + const a = new Database(path); a.exec("PRAGMA journal_mode=WAL; CREATE TABLE t(id INTEGER PRIMARY KEY, v INTEGER);"); + a.exec("INSERT INTO t(id,v) VALUES(1,0);"); + const b = new Database(path); + a.exec("BEGIN;"); a.query("SELECT v FROM t WHERE id=1").get(); // pin snapshot on A + b.exec("UPDATE t SET v=1 WHERE id=1;"); // B commits (autocommit) + let caught: unknown; + try { a.exec("UPDATE t SET v=2 WHERE id=1;"); } catch (e) { caught = e; } + expect(caught).toBeDefined(); + expect((caught as any).code?.startsWith("SQLITE_BUSY")).toBe(true); + expect(isBusyError(caught)).toBe(true); + try { a.exec("ROLLBACK;"); } catch {} + a.close(); b.close(); +}); diff --git a/lib/state/__tests__/db.test.ts b/lib/state/__tests__/db.test.ts index 39e4f7374..86be77397 100644 --- a/lib/state/__tests__/db.test.ts +++ b/lib/state/__tests__/db.test.ts @@ -6,7 +6,7 @@ * HOME isolation is handled by the repo-wide bun test preload * (test-setup.ts) — never removed here. */ -import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test"; import { Database } from "bun:sqlite"; import { chmodSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync, writeFileSync } from "fs"; import { tmpdir } from "os"; @@ -18,6 +18,12 @@ import { openStateDb, SCHEMA_VERSION, } from "../db.ts"; +// Side-effect imports: registering the REAL project-mrs and discussions +// importers (module-load LEGACY_IMPORTS.push), not fakes, so the isolation +// test below exercises a genuine throw (duplicate-iid UNIQUE violation) +// and a genuine benign import, not a hand-rolled stand-in for either. +import "../../daemon/project-mrs-store.ts"; +import "../../daemon/discussions-file-store.ts"; const DB_TS_PATH = join(import.meta.dir, "..", "db.ts"); @@ -342,6 +348,62 @@ describe("legacy import seam", () => { }); expect(() => openStateDb(dbPath, "cli").close()).not.toThrow(); }); + + test("a throwing legacy importer is isolated: db reaches SCHEMA_VERSION, the other importer's rows land, and the offending file is still renamed", () => { + const warnSpy = spyOn(console, "warn").mockImplementation(() => {}); + try { + const dbPath = join(dir, "state.db"); + + // project-mrs-store's real importer: keys "5" and "05" both bind + // Number(iidStr) === 5, so the second INSERT hits project_mrs's + // (repo, iid) PRIMARY KEY and throws mid-transaction. + const projectMrsPath = join(dir, "project-mrs.json"); + writeFileSync( + projectMrsPath, + JSON.stringify({ + "host/repo": { + mrs: { + "5": { pr: { iid: 5, title: "first" }, fetchedAt: 111 }, + "05": { pr: { iid: 5, title: "duplicate" }, fetchedAt: 222 }, + }, + }, + }), + ); + + // discussions-file-store's real importer: a benign, unrelated file + // that must still land even though the importer above throws. + const discussionsPath = join(dir, "discussions.json"); + writeFileSync( + discussionsPath, + JSON.stringify({ + "host/repo:7": { discussions: [{ id: "d1" }], fetchedAt: 333 }, + }), + ); + + const db = openStateDb(dbPath, "cli"); + + expect(db.query("PRAGMA user_version;").get()).toEqual({ user_version: SCHEMA_VERSION }); + + const discussionsRow = db + .query("SELECT repo, iid, fetched_at FROM discussions WHERE repo = ? AND iid = ?;") + .get("host/repo", 7); + expect(discussionsRow).toEqual({ repo: "host/repo", iid: 7, fetched_at: 333 }); + + expect(existsSync(projectMrsPath)).toBe(false); + expect(existsSync(`${projectMrsPath}.migrated`)).toBe(true); + expect(existsSync(discussionsPath)).toBe(false); + expect(existsSync(`${discussionsPath}.migrated`)).toBe(true); + + const warnedAboutOffender = warnSpy.mock.calls.some((call) => + call.some((arg) => typeof arg === "string" && arg.includes("project-mrs.json")), + ); + expect(warnedAboutOffender).toBe(true); + + db.close(); + } finally { + warnSpy.mockRestore(); + } + }); }); describe("pragma values per flavor", () => { @@ -375,6 +437,13 @@ describe("pragma values per flavor", () => { expect(timeout).toBe(5000); db.close(); }); + + test("getStateDb('daemon') reports busy_timeout 250 even after a default open", () => { + const cli = getStateDb(); // opens singleton, cli flavor + expect(cli.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 5000 }); + const daemon = getStateDb("daemon"); // same singleton, must not stay at 5000 + expect(daemon.query("PRAGMA busy_timeout").get()).toEqual({ timeout: 250 }); + }); }); describe("startup busy budget — open+migrate blocks, it does not throw", () => { diff --git a/lib/state/__tests__/presence-store.test.ts b/lib/state/__tests__/presence-store.test.ts index 8cc4f029b..01146a47e 100644 --- a/lib/state/__tests__/presence-store.test.ts +++ b/lib/state/__tests__/presence-store.test.ts @@ -6,8 +6,9 @@ * wiring those tests cover. */ import { expect, test } from "bun:test"; +import { readFileSync } from "fs"; import { tmpdir } from "os"; -import { join } from "path"; +import { join, resolve } from "path"; import { openStateDb } from "../db.ts"; import { joinRoom, listMembers } from "../chat-store.ts"; import { @@ -180,6 +181,30 @@ test("assertSessionOwnsHandle throws only on a mismatched signed handle", () => expect(() => assertSessionOwnsHandle("x", undefined, db)).not.toThrow(); // no session id offered, no enforcement }); +test("S073: signIn's read-then-write transaction uses .immediate() (BEGIN IMMEDIATE), not a deferred BEGIN", () => { + // A plain db.transaction()'s deferred BEGIN lets signIn's own reads + // (prunePresence, SELECT_PRESENCE_BY_SESSION_SQL) open a snapshot before + // any write; a commit by another connection in that window turns the + // eventual write into an unretryable SQLITE_BUSY_SNAPSHOT that the + // flavor's busy_timeout cannot absorb. .immediate() takes the write lock + // at BEGIN, so contention surfaces as an ordinary, retryable SQLITE_BUSY + // instead (matching the chat-store.ts/dm-store.ts/notifier-store.ts + // siblings already converted for the same reason). + const src = readFileSync(resolve(import.meta.dir, "..", "presence-store.ts"), "utf8"); + const runIndex = src.indexOf("const run = db.transaction("); + expect(runIndex).toBeGreaterThan(-1); + expect(src.indexOf("return run.immediate();", runIndex)).toBeGreaterThan(runIndex); +}); + +test("C9: reserveAgentHandle's read-then-write transaction also uses .immediate(), same reason as signIn's S073 fix", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "presence-store.ts"), "utf8"); + const fnIndex = src.indexOf("export function reserveAgentHandle("); + expect(fnIndex).toBeGreaterThan(-1); + const runIndex = src.indexOf("const run = db.transaction(", fnIndex); + expect(runIndex).toBeGreaterThan(fnIndex); + expect(src.indexOf("return run.immediate();", runIndex)).toBeGreaterThan(runIndex); +}); + test("assertSessionSignedIn throws when the session's row is gone", () => { const db = fresh(); expect(() => assertSessionSignedIn("ghost", db)).toThrow(/handle reclaimed/); @@ -222,6 +247,16 @@ test("prune: a never-signed-out row past 24h with no live binding is deleted", ( expect(db.query("SELECT COUNT(*) c FROM chat_presence").get()).toMatchObject({ c: 0 }); }); +test("prune: a signed-out row within its 24h offline window survives even when last_seen_at is stale (C9: PRUNABLE_SQL must not let a signed-out row's last_seen_at leg bypass its own signed_out_at age bound)", () => { + const db = fresh(); + signIn({ sessionId: "s1", baseHandle: "x", now }, db); // last_seen_at pinned at `now`, no touches + signOut("s1", now + 30 * HOUR, db); // signed out well after last_seen_at went stale + // 1h after signing out: signed_out_at leg is nowhere near its 24h bound, + // but last_seen_at (still `now`, 31h stale) trips the OTHER leg. + expect(prunePresence(now + 31 * HOUR, db)).toBe(0); + expect(db.query("SELECT COUNT(*) c FROM chat_presence").get()).toMatchObject({ c: 1 }); +}); + test("touchLastSeen refreshes only last_seen_at -- the sole remaining route to it now that chat:pulse is gone", () => { const db = fresh(); signIn({ sessionId: "s1", baseHandle: "x", now }, db); diff --git a/lib/state/__tests__/source-guards.test.ts b/lib/state/__tests__/source-guards.test.ts index 40dac2fdd..806e0313f 100644 --- a/lib/state/__tests__/source-guards.test.ts +++ b/lib/state/__tests__/source-guards.test.ts @@ -92,9 +92,10 @@ describe("legacy JSON state files are retired", () => { describe("daemon startup opens state.db before serving", () => { test("openBranchCacheStore() precedes both server binds in runDaemon", () => { const src = readFileSync(join(REPO_ROOT, "lib", "daemon.ts"), "utf8"); - // startDaemon() itself is now a thin catch-and-exit wrapper (see - // "startDaemon exits on any runDaemon failure" below); the real ordered - // startup sequence this test asserts on lives in runDaemon(). + // startDaemon() itself is now just `await runDaemon()` (see "boot + // failure is fatal" below; the catch-and-exit lives in runDaemon + // itself); the real ordered startup sequence this test asserts on + // lives in runDaemon() too. const start = src.indexOf("async function runDaemon("); expect(start).toBeGreaterThan(-1); @@ -122,26 +123,33 @@ describe("daemon startup opens state.db before serving", () => { }); }); -describe("startDaemon exits on any runDaemon failure", () => { +describe("boot failure is fatal for both fire-and-forget callers", () => { test("runDaemon is not exported — startDaemon is the only safe entry point", () => { const src = readFileSync(join(REPO_ROOT, "lib", "daemon.ts"), "utf8"); expect(src).not.toContain("export async function runDaemon("); expect(src).not.toContain("export function runDaemon("); }); - test("startDaemon wraps runDaemon in try/catch and exits nonzero on failure", () => { + test("runDaemon wraps its own body in try/catch and exits nonzero on failure; startDaemon just awaits it", () => { const src = readFileSync(join(REPO_ROOT, "lib", "daemon.ts"), "utf8"); - const start = src.indexOf("export async function startDaemon("); - expect(start).toBeGreaterThan(-1); + const runStart = src.indexOf("async function runDaemon("); + const startDaemonStart = src.indexOf("export async function startDaemon("); + expect(runStart).toBeGreaterThan(-1); + expect(startDaemonStart).toBeGreaterThan(runStart); // Both real callers (cli.ts's --daemon entry, this file's own - // import.meta.main guard) invoke startDaemon() fire-and-forget — the - // catch-and-exit MUST live inside startDaemon itself, not at either - // call site, or an unhandledRejection silently leaves the daemon - // half-up (rt.sock possibly bound, nothing past the failure ever wired). - const body = src.slice(start, start + 400); - expect(body).toContain("await runDaemon()"); - expect(body).toContain("catch"); - expect(body).toContain("process.exit(1)"); + // import.meta.main guard) invoke startDaemon() fire-and-forget, and + // startDaemon() is now just `await runDaemon()`, so the catch-and-exit + // MUST live inside runDaemon() itself, or an unhandledRejection could + // silently leave the daemon half-up (rt.sock possibly bound, nothing + // past the failure ever wired). The booting-gated unhandledRejection + // handler (installCrashHandlers) is only the backstop for whatever + // still manages to slip past this try/catch. + const runBody = src.slice(runStart, startDaemonStart); + expect(runBody).toContain("catch"); + expect(runBody).toContain("process.exit(1)"); + + const startDaemonBody = src.slice(startDaemonStart, startDaemonStart + 200); + expect(startDaemonBody).toContain("await runDaemon()"); }); }); diff --git a/lib/state/busy.ts b/lib/state/busy.ts index 1124378c4..fbcb2258b 100644 --- a/lib/state/busy.ts +++ b/lib/state/busy.ts @@ -35,9 +35,15 @@ import type { DaemonLoggerHandle } from "../daemon-logger.ts"; let logHandle: Promise | null = null; -/** True for the bun:sqlite error thrown when a write can't get the lock inside busy_timeout. */ +/** + * True for the bun:sqlite error thrown when a write can't get the lock + * inside busy_timeout, including the SNAPSHOT/RECOVERY variants a + * deferred-BEGIN read-then-write transaction can throw, which busy_timeout + * does not retry the way it retries a plain SQLITE_BUSY. + */ export function isBusyError(err: unknown): boolean { - return (err as { code?: string } | undefined)?.code === "SQLITE_BUSY"; + const code = (err as { code?: string } | undefined)?.code; + return code === "SQLITE_BUSY" || (typeof code === "string" && code.startsWith("SQLITE_BUSY_")); } function warnBusy(module: string, context: Record): void { diff --git a/lib/state/chat-store.ts b/lib/state/chat-store.ts index fad13dbd2..d2127600c 100644 --- a/lib/state/chat-store.ts +++ b/lib/state/chat-store.ts @@ -221,7 +221,8 @@ export function joinRoom( return { handle, memberCount, unread: Math.max(0, maxId - lastReadId) }; }); - return run(); + // BEGIN IMMEDIATE: read-then-write must lock up front or SQLITE_BUSY_SNAPSHOT bypasses busy_timeout. + return run.immediate(); } export function leaveRoom(room: string, handle: string, db: Database = getStateDb()): void { @@ -280,7 +281,7 @@ export function archiveRoom( if (current === null) db.query(UPDATE_ROOM_ARCHIVED_SQL).run(archivedAt, room); return { room, archivedAt }; }); - return run(); + return run.immediate(); } /** The wake mode stamped by whichever join created `room`; undefined for a room never stamped (including every DM room — dmRoomFor never stamps one). */ @@ -420,7 +421,7 @@ export function readUnread( return results; }); - return run(); + return run.immediate(); } /** diff --git a/lib/state/db.ts b/lib/state/db.ts index e292bf78d..008ad6474 100644 --- a/lib/state/db.ts +++ b/lib/state/db.ts @@ -312,7 +312,7 @@ function addHandleColumnIfMissing(db: Database): void { } /** bun:sqlite error codes that mean "the file on disk is not a usable db". */ -function isCorruptionError(err: unknown): boolean { +export function isCorruptionError(err: unknown): boolean { const code = (err as { code?: string } | undefined)?.code; return code === "SQLITE_CORRUPT" || code === "SQLITE_NOTADB"; } @@ -399,11 +399,22 @@ function quarantine(path: string): void { /** * Runs each registered legacy importer whose source file exists, inside the * caller's transaction. Returns the list of source paths that were consumed - * (successfully imported OR corrupt-and-skipped) — both cases still rename - * per spec "Migration & contention" ("corrupt = warn + skip"; brief: "warn + - * skip + still rename"). Renaming itself happens AFTER COMMIT (the caller - * does it), since a filesystem rename cannot participate in the sqlite - * transaction. + * (successfully imported OR corrupt/throwing-and-skipped): all three cases + * still rename per spec "Migration & contention" ("corrupt = warn + skip"; + * brief: "warn + skip + still rename"). Renaming itself happens AFTER COMMIT + * (the caller does it), since a filesystem rename cannot participate in the + * sqlite transaction. + * + * Each importer's `import(db, json)` runs inside its own SAVEPOINT, nested + * inside the caller's outer BEGIN IMMEDIATE. A throwing importer (e.g. two + * legacy keys that normalize to the same primary key, tripping a UNIQUE + * constraint) previously rolled back the WHOLE v0->v1 migration: user_version + * stayed 0, so every later openStateDb call replayed the identical throw + * forever with no self-heal. The savepoint confines that rollback to the one + * importer's own writes, so the schema DDL and every OTHER importer still + * commit and the db reaches SCHEMA_VERSION. This is deliberately narrower + * than the outer transaction's own error handling: schema/DDL failures are + * not wrapped here and still abort the whole migration loudly. */ function importLegacyStores(db: Database, dir: string): string[] { const consumed: string[] = []; @@ -418,7 +429,14 @@ function importLegacyStores(db: Database, dir: string): string[] { consumed.push(path); continue; } - entry.import(db, json); + db.exec("SAVEPOINT legacy_import;"); + try { + entry.import(db, json); + db.exec("RELEASE legacy_import;"); + } catch (err) { + db.exec("ROLLBACK TO legacy_import; RELEASE legacy_import;"); + console.warn(`rt: legacy import failed for ${path}, skipping (file will still be renamed): ${(err as Error).message}`); + } consumed.push(path); } return consumed; @@ -550,11 +568,19 @@ export function stateDbPath(): string { */ export function getStateDb(flavor: DbFlavor = "cli"): Database { const path = stateDbPath(); - if (!singleton || singletonPath !== path) { - singleton?.close(); - singleton = openStateDb(path, flavor); - singletonPath = path; + if (singleton && singletonPath === path) { + // A caller asking for a stronger (shorter) contention policy than the + // singleton currently holds must not silently inherit whatever flavor + // opened it first (e.g. a "cli" 5000ms opener beating the daemon's own + // "daemon" 250ms open); re-tighten in place rather than reopening. + const want = BUSY_TIMEOUT_MS[flavor]; + const have = Number((singleton.query("PRAGMA busy_timeout").get() as { timeout?: number } | null)?.timeout ?? 0); + if (want < have) singleton.exec(`PRAGMA busy_timeout = ${want};`); + return singleton; } + singleton?.close(); + singleton = openStateDb(path, flavor); + singletonPath = path; return singleton; } diff --git a/lib/state/dm-store.ts b/lib/state/dm-store.ts index 39c66b9f4..f7d712f42 100644 --- a/lib/state/dm-store.ts +++ b/lib/state/dm-store.ts @@ -72,7 +72,7 @@ export function dmRoomFor( return { room, created: true }; }); - return run(); + return run.immediate(); } export function dmParticipants(room: string, db: Database = getStateDb()): { a: string; b: string } | null { diff --git a/lib/state/notifier-store.ts b/lib/state/notifier-store.ts index 682a822a3..0b08238c4 100644 --- a/lib/state/notifier-store.ts +++ b/lib/state/notifier-store.ts @@ -108,7 +108,7 @@ export function drainNotificationQueue(db: Database = getStateDb()): Notificatio db.exec(`DELETE FROM notify_queue;`); return rows.map(rowToEvent); }); - return runCriticalWrite("drain", () => run(), {}) ?? []; + return runCriticalWrite("drain", () => run.immediate(), {}) ?? []; } /** Peek reads without deleting — diagnostics, no mutation, no retry needed. */ diff --git a/lib/state/presence-store.ts b/lib/state/presence-store.ts index c8236933c..773844d4f 100644 --- a/lib/state/presence-store.ts +++ b/lib/state/presence-store.ts @@ -111,7 +111,7 @@ function isReclaimable(row: PresenceRawRow, sessionStaleCutoff: number, deps: Re * SQL only narrows to "old enough to be worth a registry check", never the * final delete decision. Bind params in order: dayAgo, dayAgo (both legs). */ -const PRUNABLE_SQL = `(signed_out_at IS NOT NULL AND signed_out_at < ?) OR last_seen_at < ?`; +const PRUNABLE_SQL = `(signed_out_at IS NOT NULL AND signed_out_at < ?) OR (signed_out_at IS NULL AND last_seen_at < ?)`; const SELECT_PRESENCE_BY_HANDLE_SQL = `SELECT ${PRESENCE_COLUMNS} FROM chat_presence WHERE handle = ?;`; const SELECT_PRESENCE_BY_SESSION_SQL = `SELECT ${PRESENCE_COLUMNS} FROM chat_presence WHERE session_id = ?;`; @@ -240,6 +240,12 @@ export function signIn( deps: RegistryDeps = defaultRegistryDeps, ): { handle: string; baseHandle: string; reclaimed: boolean } { const { sessionId, statusText } = args; + // Defense in depth: the handler (lib/daemon/handlers/chat.ts) is the + // root-cause guard, but session_id is a bare TEXT PRIMARY KEY with no + // NOT NULL/CHECK constraint (bun:sqlite binds undefined as NULL, which + // SQLite accepts), so any future caller of this store function directly + // must not be able to wedge the same NULL-keyed-row failure mode. + if (!sessionId) throw new Error("signIn: sessionId is required"); const cwd = args.cwd ?? null; const repo = args.repo ?? null; const branch = args.branch ?? null; @@ -320,7 +326,7 @@ export function signIn( return { handle, baseHandle, reclaimed: winnerRow !== null }; }); - return run(); + return run.immediate(); } const NAMES_KV_NS = "chat"; @@ -353,7 +359,9 @@ export function reserveAgentHandle(db: Database = getStateDb(), now: number = Da recordPoolNameUse(name, now, db); return name; }); - return run(); + // BEGIN IMMEDIATE: read-then-write must lock up front or SQLITE_BUSY_SNAPSHOT + // bypasses busy_timeout (same reason as signIn's S073 fix above). + return run.immediate(); } export function signOut(sessionId: string, now: number = Date.now(), db: Database = getStateDb()): void { diff --git a/lib/subprocess.ts b/lib/subprocess.ts index afc62be90..addc7eab7 100644 --- a/lib/subprocess.ts +++ b/lib/subprocess.ts @@ -10,6 +10,8 @@ export interface RunResult { stdout: string; stderr: string; exitCode: number; + /** Set true only when the deadline fired before the child settled. */ + timedOut?: boolean; } /** Longest slice of a failed step's output carried into a log line. */ @@ -57,21 +59,53 @@ export async function runCapture( return { stdout: "", stderr: "", exitCode: -1 }; } - const timer = setTimeout(() => { - try { proc.kill(); } catch { /* already exited */ } - }, opts.timeoutMs ?? 10_000); + const timeoutMs = opts.timeoutMs ?? 10_000; + // SIGTERM at the deadline, SIGKILL a short grace later. A child that ignores + // SIGTERM (or a D-state descendant) cannot be reaped in-band, so the read is + // raced against the deadline below rather than awaited unconditionally: that + // is what lets runCapture settle while a grandchild still holds the pipe. + let killTimer: ReturnType | undefined; + const term = setTimeout(() => { + try { proc.kill("SIGTERM"); } catch { /* already exited */ } + killTimer = setTimeout(() => { + try { proc.kill("SIGKILL"); } catch { /* already exited */ } + }, 2000); + // unref: a short-lived CLI process must not be held open by a timed-out + // call waiting on this timer; the daemon stays alive regardless, so its + // SIGKILL still fires. + killTimer.unref?.(); + }, timeoutMs); + + const captured: Promise = (async () => { + try { + const stdoutPromise = new Response(proc.stdout as ReadableStream).text(); + const stderrPromise = captureStderr + ? new Response(proc.stderr as ReadableStream).text() + : Promise.resolve(""); + const [stdout, stderr, exitCode] = await Promise.all([ + stdoutPromise, + stderrPromise, + proc.exited, + ]); + return { stdout, stderr, exitCode }; + } catch { + return { stdout: "", stderr: "", exitCode: -1 }; + } + })(); + + let deadlineTimer: ReturnType; + const deadline: Promise = new Promise((resolve) => { + deadlineTimer = setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); + }); try { - const stdoutPromise = new Response(proc.stdout as ReadableStream).text(); - const stderrPromise = captureStderr - ? new Response(proc.stderr as ReadableStream).text() - : Promise.resolve(""); - const [stdout, stderr] = await Promise.all([stdoutPromise, stderrPromise]); - const exitCode = await proc.exited; - return { stdout, stderr, exitCode }; - } catch { - return { stdout: "", stderr: "", exitCode: -1 }; + return await Promise.race([captured, deadline]); } finally { - clearTimeout(timer); + clearTimeout(term); + clearTimeout(deadlineTimer!); + // killTimer intentionally NOT cleared here: on the timeout path it must + // survive this finally to fire SIGKILL against a child that ignored + // SIGTERM. proc.kill is already try/catch guarded, so it is a harmless + // no-op if the child exited before the 2s grace elapses. } } diff --git a/lib/worktree/__tests__/trash.test.ts b/lib/worktree/__tests__/trash.test.ts index c4ed23482..f732e550c 100644 --- a/lib/worktree/__tests__/trash.test.ts +++ b/lib/worktree/__tests__/trash.test.ts @@ -164,6 +164,31 @@ describe("worktree trash", () => { expect(result.ok).toBe(false); expect(existsSync(tree)).toBe(true); }); + + // S078: `.worktrees/` was never added to info/exclude unless the tree + // went through createTree first (e.g. `rt worktree adopt`'s disposals, + // or any repo whose worktrees root differs from the default, skip that + // call entirely) — the retention store then shows up as `?? .worktrees/` + // in the user's own `git status`, and `git add -A` stages a whole second + // copy of the source tree into it. + test("retireTree excludes .worktrees/ from the repo's own git status, even without going through createTree first", async () => { + Bun.spawnSync(["git", "init", "-q", repo]); + Bun.spawnSync(["git", "-C", repo, "config", "user.email", "test@example.com"]); + Bun.spawnSync(["git", "-C", repo, "config", "user.name", "Test"]); + writeFileSync(join(repo, "README.md"), "hi\n"); + Bun.spawnSync(["git", "-C", repo, "add", "README.md"]); + Bun.spawnSync(["git", "-C", repo, "commit", "-q", "-m", "init"]); + + const tree = makeTree(root, "hotel"); + const result = await retireTree(tree, "hotel", repo); + expect(result.ok).toBe(true); + + const exclude = readFileSync(join(repo, ".git", "info", "exclude"), "utf8"); + expect(exclude).toContain(".worktrees/"); + + const status = Bun.spawnSync(["git", "-C", repo, "status", "--porcelain"]).stdout.toString(); + expect(status).not.toContain(".worktrees"); + }); }); describe("stripTrashDir", () => { @@ -242,6 +267,19 @@ describe("worktree trash", () => { expect(existsSync(stray)).toBe(true); expect(warns.length).toBe(1); }); + + // S079: a trailing small integer (a manual "backup-3", "notes-42" dropped + // "with the other trash") parses as a number just fine and, taken at face + // value as a ms-epoch, is always ancient — rm -rf'd on the very next pass + // despite the doc comment promising non-rt entries are kept. This is a + // distinct failure shape from "not-rt-made" above (no digits at all). + test("an entry whose trailing digits are not a plausible rt epoch is kept and warned about", async () => { + const stray = makeTree(retainedTrashRoot(repo), "backup-3"); + const { log, warns } = capturingLog(); + expect(await reapExpiredTrash(repo, log, Date.now())).toBe(0); + expect(existsSync(stray)).toBe(true); + expect(warns.length).toBe(1); + }); }); describe("reapTrashDir on retained entries", () => { diff --git a/lib/worktree/dispose.ts b/lib/worktree/dispose.ts index 52e2bb1d1..3f3efeb0d 100644 --- a/lib/worktree/dispose.ts +++ b/lib/worktree/dispose.ts @@ -235,7 +235,7 @@ export async function disposeTree( } if (deps.killProcesses) { - const { terminated } = killWorktreeProcesses(rec.path); + const { terminated } = await killWorktreeProcesses(rec.path); if (terminated.length > 0) { log.info( { repo: repoName, tree: rec.name, count: terminated.length }, diff --git a/lib/worktree/git-async.ts b/lib/worktree/git-async.ts index 1d994b668..1563e1964 100644 --- a/lib/worktree/git-async.ts +++ b/lib/worktree/git-async.ts @@ -32,6 +32,10 @@ const NO_HOOKS = ["-c", "core.hooksPath=/dev/null"]; const DEFAULT_TIMEOUT_MS = 60_000; +/** Checkout/merge/stash on a large tree can legitimately exceed a minute; a + * 60s SIGKILL leaves the working tree half-switched. Match fetch/worktree-add. */ +export const MUTATING_TIMEOUT_MS = 5 * 60_000; + const DESKTOP_STASH_RE = /!!GitHub_Desktop<(.+)>$/; /** Run a git command with hooks suppressed, capturing stdout+stderr. Never throws. */ @@ -64,7 +68,7 @@ export async function currentBranchAsync(cwd: string): Promise { } export async function statusPorcelainAsync(cwd: string): Promise { - const r = await runGit(cwd, ["status", "--porcelain"]); + const r = await runGit(cwd, ["status", "--porcelain"], { timeoutMs: MUTATING_TIMEOUT_MS }); return r.stdout; } @@ -144,6 +148,12 @@ export async function listWorktreesAsync(repoPath: string): Promise { + return (await listWorktreesAsync(repoPath) ?? []).map((w) => w.path); +} + /** * Idempotently append `pattern` to the common git dir's info/exclude, with a * "# rt worktree" marker comment written on first use of this file. Returns @@ -174,13 +184,25 @@ export async function ensureInfoExclude(repoPath: string, pattern: string): Prom * Async port of git-ops.ts stashChanges — interoperable with GitHub Desktop * and worktree-context. */ -export async function stashChangesAsync(cwd: string, label: string): Promise { +export async function stashChangesAsync( + cwd: string, + label: string, + opts: { timeoutMs?: number } = {}, +): Promise { const message = `!!GitHub_Desktop<${label}>`; - await runGit(cwd, ["stash", "push", "-u", "-m", message]); + await runGit(cwd, ["stash", "push", "-u", "-m", message], { + timeoutMs: opts.timeoutMs ?? MUTATING_TIMEOUT_MS, + }); } -export async function popStashAsync(cwd: string, stashName: string): Promise { - await runGit(cwd, ["stash", "pop", stashName]); +export async function popStashAsync( + cwd: string, + stashName: string, + opts: { timeoutMs?: number } = {}, +): Promise { + await runGit(cwd, ["stash", "pop", stashName], { + timeoutMs: opts.timeoutMs ?? MUTATING_TIMEOUT_MS, + }); } /** diff --git a/lib/worktree/trash.ts b/lib/worktree/trash.ts index 9b39f72a1..709217345 100644 --- a/lib/worktree/trash.ts +++ b/lib/worktree/trash.ts @@ -22,6 +22,7 @@ import { mkdir, readdir, rename } from "fs/promises"; import { basename, dirname, join } from "path"; +import { ensureInfoExclude } from "./git-async.ts"; /** Marks a directory as rt's to delete. Nothing without this prefix is ever reaped. */ export const TRASH_PREFIX = ".trash-"; @@ -38,6 +39,21 @@ const RETAIN_DIR = ".trash"; /** How long a retained tree survives before the reconciler reaps it. */ export const RETENTION_MS = 14 * 24 * 60 * 60 * 1000; +/** + * A ms-epoch rt actually wrote (`${name}-${Date.now()}`) is always after + * this. A trailing small integer — a manual "backup-3" or "notes-42" + * dropped "with the other trash" — parses as a number just fine and, taken + * at face value as an epoch, is always ancient; without this floor it gets + * reaped on the very next pass despite not being rt's to delete. + */ +const EPOCH_FLOOR_MS = Date.UTC(2020, 0, 1); + +/** Whether `raw` looks like an epoch rt itself would have written, not merely any integer. */ +function looksLikeRtEpoch(raw: string): boolean { + const n = Number(raw); + return Number.isInteger(n) && n >= EPOCH_FLOOR_MS; +} + /** * Top-level dirs inside a retained tree that are reinstallable and deleted at * dispose time (exact names, plus the `dist-*` family). Mispredicting here @@ -110,6 +126,13 @@ export async function retireTree( if (!name || name.includes("/") || name.includes("\\")) { throw new Error(`worktree trash name must be a single path component: ${JSON.stringify(name)}`); } + // retainedTrashRoot is always /.worktrees/.trash regardless of + // cfg.root, so this exclude pattern is correct in every case — including + // a tree disposed without ever going through createTree first (e.g. `rt + // worktree adopt`'s disposals), whose repo may never have had + // ".worktrees/" excluded at all. Without it the retention store shows up + // as `?? .worktrees/` in the user's own `git status`. + await ensureInfoExclude(repoPath, ".worktrees/"); const root = retainedTrashRoot(repoPath); await mkdir(root, { recursive: true }); trashPath = join(root, `${name}-${Date.now()}`); @@ -194,7 +217,7 @@ export async function reapExpiredTrash( let reaped = 0; for (const entry of entries) { const epoch = /-(\d+)$/.exec(entry)?.[1]; - if (!epoch) { + if (!epoch || !looksLikeRtEpoch(epoch)) { log.warn({ root, entry }, "worktree retention sweep skipped an entry it did not write"); continue; } diff --git a/packages/rt-client/src/settings/__tests__/registry.test.ts b/packages/rt-client/src/settings/__tests__/registry.test.ts index 1b534ea0d..e84146b4f 100644 --- a/packages/rt-client/src/settings/__tests__/registry.test.ts +++ b/packages/rt-client/src/settings/__tests__/registry.test.ts @@ -51,7 +51,7 @@ describe("settings/registry", () => { } }); - test("exactly 22 keys are migrated:true", () => { + test("exactly 23 keys are migrated:true", () => { const migrated = allDefs().filter((d) => d.migrated); expect(migrated.map((d) => d.key).sort()).toEqual( @@ -60,6 +60,7 @@ describe("settings/registry", () => { "rt.notifications", "rt.cron", "rt.repoTracking", "rt.runsPruneDays", "rt.runaway", "rt.workspacePrefs", "rt.sync", "rt.branchNaming", "rt.variations", "rt.presets", "rt.dopplerTemplate", "rt.homeSnapshot", "rt.worktreeApp", "rt.sdmEnrichment", "rt.logRetentionDays", "rt.integrations", "rt.hooks", + "rt.apiPort", ].sort(), ); }); @@ -197,13 +198,14 @@ describe("settings/registry", () => { expect(def?.merge).toBe("replace"); }); - test("has exactly the 22 migrated:true keys and the 42 suite keys", () => { + test("has exactly the 23 migrated:true keys and the 43 suite keys", () => { const migratedFalseKeys: string[] = []; const migratedTrueKeys = [ "rt.roles", "rt.intercepts", "rt.worktrees", "rt.repoIdentityOverrides", "rt.repoRoots", "rt.notifications", "rt.cron", "rt.repoTracking", "rt.runsPruneDays", "rt.runaway", "rt.workspacePrefs", "rt.sync", "rt.branchNaming", "rt.variations", "rt.presets", "rt.dopplerTemplate", "rt.homeSnapshot", "rt.worktreeApp", "rt.sdmEnrichment", "rt.logRetentionDays", "rt.integrations", "rt.hooks", + "rt.apiPort", ]; const suiteKeys = [ "mattstack.integrations", @@ -248,8 +250,9 @@ describe("settings/registry", () => { "agent.effort", "agent.account", "agent.extraArgs", + "rt.trustedBrowserOrigins", ]; - expect(suiteKeys).toHaveLength(42); + expect(suiteKeys).toHaveLength(43); expect(allDefs().map((d) => d.key).sort()).toEqual( [...migratedFalseKeys, ...migratedTrueKeys, ...suiteKeys].sort(), diff --git a/packages/rt-client/src/settings/exec.ts b/packages/rt-client/src/settings/exec.ts index b5c68b075..5ecb3cb8d 100644 --- a/packages/rt-client/src/settings/exec.ts +++ b/packages/rt-client/src/settings/exec.ts @@ -12,6 +12,8 @@ export interface RunResult { stdout: string; stderr: string; exitCode: number; + /** Set true only when the deadline fired before the child settled. */ + timedOut?: boolean; } /** @@ -47,21 +49,53 @@ export async function runCapture( return { stdout: "", stderr: "", exitCode: -1 }; } - const timer = setTimeout(() => { - try { proc.kill(); } catch { /* already exited */ } - }, opts.timeoutMs ?? 10_000); + const timeoutMs = opts.timeoutMs ?? 10_000; + // SIGTERM at the deadline, SIGKILL a short grace later. A child that ignores + // SIGTERM (or a D-state descendant) cannot be reaped in-band, so the read is + // raced against the deadline below rather than awaited unconditionally: that + // is what lets runCapture settle while a grandchild still holds the pipe. + let killTimer: ReturnType | undefined; + const term = setTimeout(() => { + try { proc.kill("SIGTERM"); } catch { /* already exited */ } + killTimer = setTimeout(() => { + try { proc.kill("SIGKILL"); } catch { /* already exited */ } + }, 2000); + // unref: a short-lived CLI process must not be held open by a timed-out + // call waiting on this timer; the daemon stays alive regardless, so its + // SIGKILL still fires. + killTimer.unref?.(); + }, timeoutMs); + + const captured: Promise = (async () => { + try { + const stdoutPromise = new Response(proc.stdout as ReadableStream).text(); + const stderrPromise = captureStderr + ? new Response(proc.stderr as ReadableStream).text() + : Promise.resolve(""); + const [stdout, stderr, exitCode] = await Promise.all([ + stdoutPromise, + stderrPromise, + proc.exited, + ]); + return { stdout, stderr, exitCode }; + } catch { + return { stdout: "", stderr: "", exitCode: -1 }; + } + })(); + + let deadlineTimer: ReturnType; + const deadline: Promise = new Promise((resolve) => { + deadlineTimer = setTimeout(() => resolve({ stdout: "", stderr: "", exitCode: -1, timedOut: true }), timeoutMs); + }); try { - const stdoutPromise = new Response(proc.stdout as ReadableStream).text(); - const stderrPromise = captureStderr - ? new Response(proc.stderr as ReadableStream).text() - : Promise.resolve(""); - const [stdout, stderr] = await Promise.all([stdoutPromise, stderrPromise]); - const exitCode = await proc.exited; - return { stdout, stderr, exitCode }; - } catch { - return { stdout: "", stderr: "", exitCode: -1 }; + return await Promise.race([captured, deadline]); } finally { - clearTimeout(timer); + clearTimeout(term); + clearTimeout(deadlineTimer!); + // killTimer intentionally NOT cleared here: on the timeout path it must + // survive this finally to fire SIGKILL against a child that ignored + // SIGTERM. proc.kill is already try/catch guarded, so it is a harmless + // no-op if the child exited before the 2s grace elapses. } } diff --git a/packages/rt-client/src/settings/registry-defs.ts b/packages/rt-client/src/settings/registry-defs.ts index f0109ddd3..8a00016b7 100644 --- a/packages/rt-client/src/settings/registry-defs.ts +++ b/packages/rt-client/src/settings/registry-defs.ts @@ -196,6 +196,15 @@ export const REGISTRY: readonly SettingDef[] = [ migrated: true, description: "Age floor in days for the log janitor pruning every surface's rotated log files under ~/.mattstack/rt/logs (default 14). A fresh key, not an ownership-latch port, so a default is fine here.", }, + { + key: "rt.apiPort", + type: "number", + scopes: ["machine", "user"], + default: 9401, + merge: "replace", + migrated: true, + description: "TCP port for the daemon's local HTTP/WS API. Escape hatch when 9401 is held: RT_API_PORT env wins, then this setting, then 9401 (lib/daemon-config.ts resolveApiPort(), read at bind time by lib/daemon/api-server.ts).", + }, { key: "rt.hooks", type: "object", @@ -205,6 +214,14 @@ export const REGISTRY: readonly SettingDef[] = [ migrated: true, description: "Per-repo git hook enable/disable state ({enabled, hooks: {: boolean}}); ownership-latch port of repos//hooks.json, store wins per field once it owns the key — including per-hook-name entries inside the nested hooks map, each defaulting to enabled when absent. The installed git-hook shim still greps repos//hooks.json with zero process spawns (a hook fires on every git operation); that file is now a DERIVED CACHE this key writes through, kept current by commands/hooks.ts's regenerateHooksCache at every write seam.", }, + { + key: "rt.trustedBrowserOrigins", + type: "array", + scopes: ["user", "machine"], + default: [], + merge: "replace", + description: "Browser Origins (scheme://host:port, exact string match) trusted to read the :9401 daemon API and subscribe to /ws without presenting the local api-token -- e.g. a locally-hosted console or chat-viewer dev server. Empty by default: every current mattstack consumer (the CLI, the Swift tray, rt-client from Bun/Node processes, the VS Code extension) is a non-browser client (sends no Origin header at all) and is unaffected either way.", + }, // --- mattstack (installer-lane) ----------------------------------------- { diff --git a/website/docs/reference/daemon/status.mdx b/website/docs/reference/daemon/status.mdx index 0d7c24101..bfd689ac3 100644 --- a/website/docs/reference/daemon/status.mdx +++ b/website/docs/reference/daemon/status.mdx @@ -12,9 +12,15 @@ Show daemon status ## Usage ```bash -rt daemon status +rt daemon status [flags] ``` +## Arguments & flags + +| Flag / Arg | Type | Default | Description | +| --- | --- | --- | --- | +| [`--json`](/guides/common-flags) | boolean | `false` | Emit the verdict as JSON instead of the formatted lines | + _See code: [commands/daemon.ts › showStatus](https://github.com/m4ttstack/rt/blob/main/commands/daemon.ts)_ {/* generated by scripts/gen-docs.ts; edit prose in _partials, not here */} \ No newline at end of file diff --git a/website/docs/reference/pane/focus.mdx b/website/docs/reference/pane/focus.mdx new file mode 100644 index 000000000..9f66c8128 --- /dev/null +++ b/website/docs/reference/pane/focus.mdx @@ -0,0 +1,27 @@ +--- +title: rt pane focus +sidebar_label: focus +--- + +# rt pane focus + +`rt › pane › focus` + +Bring a herdr pane to the front (via the tray: workspace + tab focus and terminal window raise) + +## Usage + +```bash +rt pane focus [flags] +``` + +## Arguments & flags + +| Flag / Arg | Type | Default | Description | +| --- | --- | --- | --- | +| `` | text | | herdr pane id to focus | +| [`--json`](/guides/common-flags) | boolean | `false` | Emit the focus result as JSON instead of the plain line | + +_See code: [commands/pane.ts › paneFocus](https://github.com/m4ttstack/rt/blob/main/commands/pane.ts)_ + +{/* generated by scripts/gen-docs.ts; edit prose in _partials, not here */} \ No newline at end of file diff --git a/website/docs/reference/pane/index.mdx b/website/docs/reference/pane/index.mdx index 920ebecda..65abc8ca5 100644 --- a/website/docs/reference/pane/index.mdx +++ b/website/docs/reference/pane/index.mdx @@ -23,6 +23,7 @@ rt pane | [`peek`](peek) | The last lines of a pane's visible screen | | [`spawn`](spawn) | Open a herdr tab in a directory and start claude in it, optionally under a cswap account | | [`send`](send) | Inject text into a pane as if typed and submitted (--text - reads stdin) | +| [`focus`](focus) | Bring a herdr pane to the front (via the tray: workspace + tab focus and terminal window raise) | | [`accounts`](accounts) | cswap accounts with rate-limit headroom, for spawn --account | | [`directories`](directories) | Repos and worktrees rt knows, as suggestions for spawn --cwd |