diff --git a/commands/__tests__/home.test.ts b/commands/__tests__/home.test.ts index c38419f6..4cd2b56f 100644 --- a/commands/__tests__/home.test.ts +++ b/commands/__tests__/home.test.ts @@ -172,6 +172,16 @@ class FakeSeam implements ExecSeam { async run(cmd: string[]): Promise { this.calls.push({ kind: "run", arg: cmd }); if (this.opts.failRun?.(cmd)) return { code: 1, stdout: "", stderr: "boom" }; + // commitInitialUserRepo probes identity before committing; default to + // "configured" so fixtures not exercising the no-identity path stay + // green, mirroring lib/daemon/__tests__/home-snapshot.test.ts's + // defaultResponders. + if (cmd[cmd.length - 2] === "config" && cmd[cmd.length - 1] === "user.name") { + return { code: 0, stdout: "rt test\n", stderr: "" }; + } + if (cmd[cmd.length - 2] === "config" && cmd[cmd.length - 1] === "user.email") { + return { code: 0, stdout: "rt@example.test\n", stderr: "" }; + } return { code: 0, stdout: "", stderr: "" }; } async writeFile(path: string, content: string): Promise { diff --git a/commands/__tests__/log-level.test.ts b/commands/__tests__/log-level.test.ts new file mode 100644 index 00000000..73a6f8a0 --- /dev/null +++ b/commands/__tests__/log-level.test.ts @@ -0,0 +1,9 @@ +import { test, expect } from "bun:test"; +import { formatLogLevelResult } from "../daemon.ts"; + +test("formats a set result", () => { + expect(formatLogLevelResult({ ok: true, level: "debug" }, true)).toContain("debug"); +}); +test("formats a show result", () => { + expect(formatLogLevelResult({ ok: true, level: "info" }, false)).toContain("info"); +}); diff --git a/commands/__tests__/status-lines.test.ts b/commands/__tests__/status-lines.test.ts new file mode 100644 index 00000000..85c81df8 --- /dev/null +++ b/commands/__tests__/status-lines.test.ts @@ -0,0 +1,33 @@ +import { test, expect } from "bun:test"; +import { statusLines } from "../daemon.ts"; + +const strip = (s: string) => s.replace(/\[[0-9;]*m/g, ""); + +test("degraded/unresponsive prints ping-carried maxLag, not 'likely mid-sync'", () => { + const lines = statusLines( + { state: "degraded", reason: "unresponsive", pid: 42, eventLoop: { maxLagMs: 1400, lastStallAt: 1, lastStallCmd: "mr:action", stalls: 2 } } as any, + 2000, + ).map(strip).join("\n"); + expect(lines).not.toContain("likely mid-sync"); + expect(lines).toContain("1400ms"); + expect(lines).toContain("mr:action"); +}); + +test("alive-not-serving 'stalled' prints stalled Ns ago", () => { + const lines = statusLines( + { state: "alive-not-serving", pid: 42, detail: "stalled", stalledForMs: 8000 } as any, + 0, + ).map(strip).join("\n"); + expect(lines).toContain("event loop stalled"); + expect(lines).toContain("8s"); +}); + +test("running prints the health level and reasons when present", () => { + const lines = statusLines( + { state: "running", data: { pid: 42, uptime: 60000, watchedRepos: 3, cacheEntries: 10, + health: { level: "degraded", reasons: ["refresh: 3 repos failing (auth?)"] } } } as any, + 0, + ).map(strip).join("\n"); + expect(lines).toContain("degraded"); + expect(lines).toContain("refresh: 3 repos failing"); +}); diff --git a/commands/daemon.ts b/commands/daemon.ts index 8d492ab5..b4f604d9 100644 --- a/commands/daemon.ts +++ b/commands/daemon.ts @@ -33,11 +33,12 @@ import { LOG_DIR, LAUNCHD_PLIST_PATH, } from "../lib/daemon-config.ts"; -import { daemonQuery, isDaemonRunning, trayQuery } from "../lib/daemon-client.ts"; +import { daemonQuery, isDaemonRunning, pingDaemon, trayQuery } from "../lib/daemon-client.ts"; import { classifyDaemonStatus, type DaemonStatusVerdict } from "../lib/daemon-status.ts"; import { resolveIntendedMode, currentMode, type IntendedMode } from "../lib/dev-mode.ts"; import { probeSocketHolder } from "../lib/daemon/park.ts"; import { readBreadcrumb, readSupervisionState } from "../lib/daemon/supervision-state.ts"; +import { readHeartbeat } from "../lib/daemon/heartbeat-file.ts"; import { runCapture } from "../lib/subprocess.ts"; import { isGitLabRemote } from "../lib/enrich.ts"; import type { CacheKind, RepoTrackingEntry } from "../lib/repo-tracking.ts"; @@ -403,22 +404,27 @@ export async function showStatus(args: string[] = []): Promise { // A failed status query does NOT mean the daemon is down — it answers `ping` // in a fraction of the budget a loaded `status` needs. Establish liveness // before reporting, and only pay for the probe when nothing came back. - const pingOk = classifyDaemonStatus.needsLivenessProbe(response) ? await isDaemonRunning() : false; + // pingDaemon (not isDaemonRunning) so the raw reply's eventLoop summary is + // still on hand to render, and so this probe never risks a restart. + const pingResp = classifyDaemonStatus.needsLivenessProbe(response) ? await pingDaemon() : null; + const pingOk = pingResp?.ok === true; const recordedPid = readDaemonPid() ?? null; - // Ping ALSO failed: the only remaining ground is the pid/breadcrumb/kv - // trail Task 9 left behind. Read it here, once, rather than on every status + // Ping ALSO failed: the only remaining ground is the pid/breadcrumb/kv/heartbeat + // trail Task 9/2 left behind. Read it here, once, rather than on every status // call, since it's the uncommon path. let pidAlive: boolean | undefined; let pid = recordedPid; let breadcrumb: ReturnType | undefined; let supervision: ReturnType | undefined; + let heartbeat: ReturnType | undefined; if (classifyDaemonStatus.needsPidProbe(response, pingOk)) { breadcrumb = readBreadcrumb(); // The kv tier can be legitimately empty (or reflect nothing useful) when // a failure happened before state.db ever opened (Ruling P1). The // breadcrumb read above is what classifyDaemonStatus falls back to then. supervision = readSupervisionState(); + heartbeat = readHeartbeat(RT_DIR); const probed = await probePidAlive(recordedPid, breadcrumb?.pid); pidAlive = probed.alive; pid = probed.pid; @@ -433,6 +439,8 @@ export async function showStatus(args: string[] = []): Promise { intendedFlavor: resolveIntendedMode().mode, breadcrumb, supervision, + heartbeat, + pingEventLoop: (pingResp as any)?.eventLoop, }); if (json) return void console.log(JSON.stringify({ ok: true, ...verdict })); @@ -503,6 +511,15 @@ export function statusLines(verdict: DaemonStatusVerdict, now: number): string[] }); lines.push(` ${dim}events: ${parts.join(" · ")}${reset}`); } + + const health = verdict.data.health as { level: string; reasons: string[] } | undefined; + if (health && health.level !== "ok") { + const dot = health.level === "unhealthy" ? red : yellow; + lines.push(` ${dot}health: ${health.level}${reset}`); + for (const r of health.reasons) lines.push(` ${dim}- ${r}${reset}`); + } + const el = verdict.data.eventLoop as { maxLagMs: number } | undefined; + if (el && el.maxLagMs >= 500) lines.push(` ${dim}event loop: maxLag ${el.maxLagMs}ms${reset}`); return lines; } @@ -511,11 +528,14 @@ export function statusLines(verdict: DaemonStatusVerdict, now: number): string[] // the operator to `rt daemon start` against a daemon that is already up. const lines = [` ${yellow}●${reset} running, but not reporting status`]; if (verdict.pid) lines.push(` ${dim}pid: ${verdict.pid}${reset}`); - lines.push( - verdict.reason === "error" - ? ` ${dim}status command failed: ${verdict.detail ?? "unknown error"}${reset}` - : ` ${dim}answers ping, but status timed out — likely mid-sync${reset}`, - ); + if (verdict.reason === "error") { + lines.push(` ${dim}status command failed: ${verdict.detail ?? "unknown error"}${reset}`); + } else if (verdict.eventLoop && verdict.eventLoop.maxLagMs > 0) { + const el = verdict.eventLoop; + lines.push(` ${dim}answers ping, status timed out: event loop maxLag ${el.maxLagMs}ms${el.lastStallCmd ? ` (last stall in ${el.lastStallCmd})` : ""}${reset}`); + } else { + lines.push(` ${dim}answers ping, but status timed out — likely mid-sync${reset}`); + } lines.push(` ${dim}check: rt daemon logs${reset}`); return lines; } @@ -536,6 +556,7 @@ export function statusLines(verdict: DaemonStatusVerdict, now: number): string[] booting: "still booting", wedged: "reached ready but stopped answering (likely deadlocked)", quarantined: "recovered from a corrupt db but still not answering", + stalled: `event loop stalled ${Math.round((verdict.stalledForMs ?? 0) / 1000)}s ago (no heartbeat)`, }[verdict.detail]; return [ ` ${yellow}●${reset} process ${verdict.pid} is running but not answering rt.sock`, @@ -1145,3 +1166,19 @@ async function waitForPort(port: number, timeoutMs: number): Promise { await new Promise(r => setTimeout(r, 100)); } } + +/** Pure formatter for `daemon:log-level` results, shared by the CLI and its tests. */ +export function formatLogLevelResult(res: { ok: boolean; level?: string; error?: string }, wasSet: boolean): string { + if (!res.ok) return ` ${red}●${reset} ${res.error ?? "failed"}`; + return ` ${green}●${reset} daemon log level ${wasSet ? "set to" : "is"} ${res.level}`; +} + +/** Show (no arg) or set (level arg) the running daemon's live pino log level. */ +export async function setLogLevel(args: string[] = []): Promise { + const json = args.includes("--json"); + const level = args.find((a) => !a.startsWith("--")); + const res = await daemonQuery("daemon:log-level", level ? { level } : {}); + if (!res) { console.log(` ${red}●${reset} daemon not reachable`); return; } + if (json) { console.log(JSON.stringify(res)); return; } + console.log(formatLogLevelResult(res as any, Boolean(level))); +} diff --git a/commands/home.ts b/commands/home.ts index eedd39f0..2d693377 100644 --- a/commands/home.ts +++ b/commands/home.ts @@ -38,6 +38,7 @@ import { join } from "path"; import type { CommandContext } from "../lib/command-tree.ts"; import { bold, dim, green, red, reset, yellow } from "../lib/ansi.ts"; import { isSafeMachineKeySegment, machineKey, mattstackHome } from "../lib/rt-paths.ts"; +import { resolveInitialMachineKey } from "../lib/home/machine-id.ts"; import { buildInitPlan, chooseMachineProfile, @@ -549,7 +550,7 @@ export async function homeInit(args: string[], _ctx: CommandContext = {}, seams: const exec = seams.exec ?? createRealExecSeam(mattstackHome()); const ageKeySeam = seams.ageKeySeam ?? createRealAgeKeySeam(); const sopsYamlSeam = seams.sopsYamlSeam ?? defaultSopsYamlSeam(); - const key = seams.key ?? machineKey(); + const key = seams.key ?? (await resolveInitialMachineKey(mattstackHome(), probes)); const pickerSeam = seams.pickerSeam ?? createRealMachineProfilePickerSeam(); const isInteractive = seams.isInteractive ?? (() => Boolean(process.stdin.isTTY)); const materializeExec = seams.materializeExec ?? defaultMaterializeExec(); diff --git a/commands/settings.ts b/commands/settings.ts index 6871dfb4..f90d4a40 100644 --- a/commands/settings.ts +++ b/commands/settings.ts @@ -16,7 +16,7 @@ import { TRAY_APP_NAME, DEV_TRAY_APP_NAME, TRAY_APP_BUNDLE, trayAppPath, devTrayAppPath, } from "../lib/rt-paths.ts"; -import { installRtBinary } from "../lib/dev-mode.ts"; +import { DEV_MODE_TAG, installRtBinary } from "../lib/dev-mode.ts"; import { describeTuple, tupleWarning, type FlavorTuple } from "./daemon.ts"; import { RT_BUNDLE_PATH } from "../lib/bundle-layout.ts"; import { spawnSync } from "child_process"; @@ -511,6 +511,7 @@ export function renderDevModeWrapper(sourcePath: string, bunPath: string): strin const bunDir = dirname(bunPath); return [ `#!/bin/zsh`, + `${DEV_MODE_TAG}`, `export PATH="${bunDir}:/opt/homebrew/bin:/usr/local/bin:$PATH"`, `export RT_LAUNCH_CWD="$PWD"`, `cd "${sourcePath}" || { echo "rt: dev-mode source checkout missing: ${sourcePath}" >&2; exit 1; }`, diff --git a/commands/status/__tests__/status-fallback.test.ts b/commands/status/__tests__/status-fallback.test.ts index a95027b6..63135681 100644 --- a/commands/status/__tests__/status-fallback.test.ts +++ b/commands/status/__tests__/status-fallback.test.ts @@ -87,6 +87,30 @@ describe("rt status fallback (no daemon)", () => { expect(existsSync(rtDirPath) ? readdirSync(rtDirPath) : []).toEqual([]); }); + test("S069/Task 10: two repos sharing a branch name display as a bare branch, not a composite key", async () => { + const dbPath = stateDbPath(); + mkdirSync(join(home, ".mattstack", "rt"), { recursive: true }); + const db = openStateDb(dbPath); + const store = getBranchCacheStore(db); + store.put("shared-branch", { + ticket: null, linearId: "", mr: null, fetchedAt: 1, repoName: "repo-a", + }); + store.put("shared-branch", { + ticket: null, linearId: "", mr: null, fetchedAt: 2, repoName: "repo-b", + }); + db.close(); + + const data = await fetchStatusData(); + + // Both rows are real, distinct composite-keyed rows in state.db...the + // dashboard's flat bare-branch dict can only show one, never a raw + // composite key, and never crashes reconciling the two. + expect(Object.keys(data.branches)).toEqual(["shared-branch"]); + const winner = data.branches["shared-branch"]!.repoName; + expect(winner).toBeDefined(); + expect(["repo-a", "repo-b"]).toContain(winner!); + }); + test("an empty branch_cache table serves an empty dashboard", async () => { mkdirSync(join(home, ".mattstack", "rt"), { recursive: true }); openStateDb(stateDbPath()).close(); diff --git a/commands/status/data.ts b/commands/status/data.ts index a58ccad6..6106ff7c 100644 --- a/commands/status/data.ts +++ b/commands/status/data.ts @@ -10,6 +10,7 @@ import type { CacheEntry, StatusData } from "./types.ts"; import type { PortEntry } from "../../lib/port-scanner.ts"; +import { branchOf } from "../../lib/state/branch-cache.ts"; interface BranchCacheRow { branch: string; @@ -56,7 +57,7 @@ async function readBranchesFromStateDb(): Promise> { // with looser optionality on the ticket fields (`stateName?: string` // vs `string | null`). Same data that used to arrive here as parsed // JSON out of branch-cache.json. - branches[row.branch] = { + branches[branchOf(row.branch)] = { ticket: row.ticket !== null ? (JSON.parse(row.ticket) as CacheEntry["ticket"]) : null, linearId: row.linear_id, mr: row.mr !== null ? (JSON.parse(row.mr) as CacheEntry["mr"]) : null, diff --git a/docs/superpowers/plans/2026-08-28-p2-health.md b/docs/superpowers/plans/2026-08-28-p2-health.md new file mode 100644 index 00000000..ad04bbf7 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-p2-health.md @@ -0,0 +1,1929 @@ +# Daemon Health You Can See (Phase 2 / RT-79) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Give the rt daemon a single server-computed health verdict (ok / degraded / unhealthy) that names the failing subsystem, add event-loop stall detection, and fix log ergonomics and request attribution, so a non-author can tell from `rt daemon status`, `/api/status`, and the tray whether the daemon is serving, degraded, stalled, or dead, and why. + +**Architecture:** A pure `computeHealth` plus a daemon-side adapter feed a `health`/`metrics`/`eventLoop` block into the `status`, `tray:status`, and `ping` verbs. A 250ms unref'd loop monitor measures event-loop drift and writes a monotonic heartbeat file (atomic rename, db-free); the cross-process status classifier reads that file to report a stall when the pid is alive but ping fails. Logger gains a level-from-setting, a stream error listener, stderr demotion, and a size cap; handleCommand gains a request id, caller tag, and per-(cmd,error) suppression; unknown commands gain a distinct envelope. + +**Tech Stack:** Bun, TypeScript, pino + pino-roll, bun:sqlite (kv only, no schema change), `@mattstack/rt-client` settings registry. + +**Spec:** `docs/superpowers/specs/2026-08-28-p2-health-design.md` (read it alongside this plan; the plan argues from it). + +## Global Constraints + +- **No `SCHEMA_VERSION` bump.** The only new persisted state is the heartbeat file `RT_DIR/daemon-heartbeat.json`; everything else is computed live or reuses the Phase 0 `daemon-supervision` kv namespace. +- **`rt.logLevel` goes through the settings registry** per `docs/settings-architecture.md`: add the row in `registry-defs.ts`, then `cd packages/rt-client && bun run build` so the dist-freshness test stays green. **Do not bump the rt-client version and do not publish** (publishing is release-class, from `main` only). Its estate rollout rides the next release. +- **No `rt-tray/` edits.** Document the tray read contract only. +- **Never start a daemon or run `dist/rt` except under `env -i HOME=`.** Tests use isolated HOME via the bunfig preload; never touch the real `~/.mattstack`. +- **Do not modify these p6-portability-owned files:** `lib/daemon/user-path.ts`, `lib/rt-paths.ts`, `lib/deps/links.ts`, `lib/agent-herdr.ts`, `lib/dev-mode.ts`, `lib/setup/**`, `lib/daemon/agent-status-poller.ts`, `lib/enrich.ts`, `lib/daemon/home-snapshot.ts`, `lib/home/**`, `lib/daemon/cron.ts`, `lib/daemon/handlers/agent.ts`. Also do not touch the module-scope `resolveUserPath()` call in `lib/daemon.ts` (p6 makes it awaited-async). Everything else in `lib/daemon.ts` is in scope. +- **Default thresholds** (spec's table): `loopTickMs`=250, `loopLagDegradedMs`=500, `loopStallLogMs`=1000, `loopStallUnhealthyMs`=2000, `stallRecentMs`=10_000, `heartbeatIntervalMs`=2000, `heartbeatStaleMs`=6000, `refreshStaleMultiplier`=2, `rssSoftThresholdMB`=1024, `rssGrowthPct`=50 over 1h, `diskSoftFloorMB`=500, `diskHardFloorMB`=100, `restartsPerHourUnhealthy`=5 (or Phase 0 `isCrashLooping`), `recoveredErrorRate`=10 per 5min, `slowCommandMs`=2000. +- **Commit after every task.** Run `bunx tsc --noEmit` (0 errors) before each commit that touches TS. + +--- + +## File Structure + +**New files:** +- `lib/daemon/health.ts` — pure `computeHealth(inputs): HealthSnapshot`, the `HealthInputs`/`HealthSnapshot`/threshold types and constants. +- `lib/daemon/heartbeat-file.ts` — `writeHeartbeat`/`readHeartbeat` (atomic rename, db-free). +- `lib/daemon/loop-monitor.ts` — `startLoopMonitor` + the pure `applyTick` drift function. +- Tests colocated under `lib/daemon/__tests__/` and `commands/__tests__/` following the repo pattern. + +**Modified files:** `lib/daemon-status.ts`, `commands/daemon.ts`, `lib/daemon-client.ts`, `lib/daemon-logger.ts`, `lib/log-janitor.ts`, `lib/daemon.ts`, `lib/daemon/handlers/status.ts`, `lib/daemon/handlers/types.ts`, `lib/daemon/cache-refresh.ts`, `lib/daemon/api-server.ts`, `lib/daemon/socket-server.ts`, `lib/command-tree-def.ts`, `packages/rt-client/src/settings/registry-defs.ts`, `packages/rt-client/src/settings/resolve.ts`, `packages/rt-client/src/settings/registry-machinery.ts` (ResolveOpts only), `packages/rt-client/src/transport.ts`, `packages/rt-client/src/index.ts`. + +--- + +## Task 1: `lib/daemon/health.ts` — pure health computation + +**Files:** +- Create: `lib/daemon/health.ts` +- Test: `lib/daemon/__tests__/health.test.ts` + +**Interfaces:** +- Produces: `computeHealth(inputs: HealthInputs): HealthSnapshot`; `HealthInputs`, `HealthSnapshot`, `HealthMetrics`, `HealthEventLoop`, `HEALTH_THRESHOLDS`. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/health.test.ts +import { test, expect } from "bun:test"; +import { computeHealth, type HealthInputs } from "../health.ts"; + +function base(): HealthInputs { + return { + now: 1_000_000, + uptimeMs: 60_000, + mem: { rss: 200 * 1024 * 1024, heapUsed: 50 * 1024 * 1024, external: 1 * 1024 * 1024 }, + rssBaseline: null, + wsClients: 0, + watchers: 3, + freshness: { "remote:gitlab/acme": { state: "live" } }, + refresh: { lastSuccessAt: 1_000_000 - 60_000, failedRepos: 0, enrichErrors: 0 }, + refreshIntervalMs: 5 * 60_000, + eventLoop: { maxLagMs: 20, lastStallAt: null, lastStallCmd: null, stalls: 0, currentlyStalled: false }, + supervisionFailuresLastHour: 0, + crashLooping: false, + loggerDegraded: false, + recoveredErrorRateLastWindow: 0, + freeBytes: 50 * 1024 * 1024 * 1024, + }; +} + +test("all-nominal inputs are ok with no reasons", () => { + const h = computeHealth(base()); + expect(h.level).toBe("ok"); + expect(h.reasons).toEqual([]); + expect(h.metrics.watchers).toBe(3); + expect(h.eventLoop.maxLagMs).toBe(20); +}); + +test("a degraded freshness watcher flips degraded and names refresh", () => { + const i = base(); + i.freshness = { "remote:gitlab/acme": { state: "degraded" } }; + const h = computeHealth(i); + expect(h.level).toBe("degraded"); + expect(h.reasons.some((r) => r.startsWith("refresh:"))).toBe(true); +}); + +test("failed repos in the last cycle flip degraded", () => { + const i = base(); + i.refresh = { lastSuccessAt: i.now - 60_000, failedRepos: 3, enrichErrors: 5 }; + expect(computeHealth(i).level).toBe("degraded"); +}); + +test("logger degraded flips unhealthy and names logging", () => { + const i = base(); + i.loggerDegraded = true; + const h = computeHealth(i); + expect(h.level).toBe("unhealthy"); + expect(h.reasons.some((r) => r.startsWith("logging:"))).toBe(true); +}); + +test("currently stalled event loop is unhealthy; unhealthy wins over a degraded signal", () => { + const i = base(); + i.eventLoop.currentlyStalled = true; + i.freshness = { r: { state: "degraded" } }; // also degraded + const h = computeHealth(i); + expect(h.level).toBe("unhealthy"); + expect(h.reasons[0].startsWith("event-loop:")).toBe(true); // unhealthy reasons first +}); + +test("critical disk is unhealthy; low disk is degraded", () => { + const crit = base(); crit.freeBytes = 50 * 1024 * 1024; + expect(computeHealth(crit).level).toBe("unhealthy"); + const low = base(); low.freeBytes = 300 * 1024 * 1024; + expect(computeHealth(low).level).toBe("degraded"); +}); + +test("stale refresh (older than 2 intervals) is degraded", () => { + const i = base(); + i.refresh = { lastSuccessAt: i.now - 11 * 60_000, failedRepos: 0, enrichErrors: 0 }; + expect(computeHealth(i).level).toBe("degraded"); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/health.test.ts` +Expected: FAIL, `Cannot find module '../health.ts'`. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/health.ts +/** + * Pure daemon health verdict. computeHealth takes a fully-gathered input + * struct (the daemon-side adapter does all I/O) and returns the level, the + * named reasons, and the metrics/eventLoop blocks the surfaces echo. + */ + +export const HEALTH_THRESHOLDS = { + refreshStaleMultiplier: 2, + rssSoftThresholdBytes: 1024 * 1024 * 1024, + rssGrowthPct: 50, + diskSoftFloorBytes: 500 * 1024 * 1024, + diskHardFloorBytes: 100 * 1024 * 1024, + restartsPerHourUnhealthy: 5, + recoveredErrorRate: 10, +} as const; + +export interface HealthMetrics { + rss: number; + heapUsed: number; + external: number; + uptimeMs: number; + wsClients: number; + watchers: number; +} + +export interface HealthEventLoop { + maxLagMs: number; + lastStallAt: number | null; + lastStallCmd: string | null; + stalls: number; +} + +export interface HealthInputs { + now: number; + uptimeMs: number; + mem: { rss: number; heapUsed: number; external: number }; + /** rss + timestamp from ~1h ago, for growth detection; null if not yet sampled. */ + rssBaseline: { rss: number; at: number } | null; + wsClients: number; + watchers: number; + freshness: Record; + refresh: { lastSuccessAt: number; failedRepos: number; enrichErrors: number }; + refreshIntervalMs: number; + eventLoop: HealthEventLoop & { currentlyStalled: boolean }; + supervisionFailuresLastHour: number; + crashLooping: boolean; + loggerDegraded: boolean; + recoveredErrorRateLastWindow: number; + freeBytes: number | null; + /** Deferred inputs (spec): wired in a later phase, ignored today. */ + busySkips?: number; + criticalWriteFailures?: number; +} + +export interface HealthSnapshot { + level: "ok" | "degraded" | "unhealthy"; + reasons: string[]; + metrics: HealthMetrics; + eventLoop: HealthEventLoop; +} + +function mb(bytes: number): number { + return Math.round(bytes / (1024 * 1024)); +} + +export function computeHealth(i: HealthInputs): HealthSnapshot { + const T = HEALTH_THRESHOLDS; + const unhealthy: string[] = []; + const degraded: string[] = []; + + // --- unhealthy --- + if (i.loggerDegraded) unhealthy.push("logging: disabled (ENOSPC)"); + if (i.eventLoop.currentlyStalled) unhealthy.push("event-loop: currently stalled"); + if (i.crashLooping || i.supervisionFailuresLastHour >= T.restartsPerHourUnhealthy) { + unhealthy.push(`restarts: ${i.supervisionFailuresLastHour} in the last hour`); + } + if (i.freeBytes !== null && i.freeBytes < T.diskHardFloorBytes) { + unhealthy.push(`disk: ${mb(i.freeBytes)}MB free (critical)`); + } + + // --- degraded --- + const degradedRepos = Object.values(i.freshness).filter((f) => f.state === "degraded").length; + if (degradedRepos > 0) degraded.push(`refresh: ${degradedRepos} watcher${degradedRepos !== 1 ? "s" : ""} degraded`); + if (i.refresh.failedRepos > 0 || i.refresh.enrichErrors > 0) { + degraded.push(`refresh: ${i.refresh.failedRepos} repos failing (auth?)`); + } + const refreshAge = i.now - i.refresh.lastSuccessAt; + if (i.refresh.lastSuccessAt > 0 && refreshAge > T.refreshStaleMultiplier * i.refreshIntervalMs) { + degraded.push(`refresh: last success ${Math.round(refreshAge / 1000)}s ago`); + } + if (i.mem.rss > T.rssSoftThresholdBytes) degraded.push(`memory: rss ${mb(i.mem.rss)}MB`); + if (i.rssBaseline && i.mem.rss > i.rssBaseline.rss * (1 + T.rssGrowthPct / 100)) { + degraded.push(`memory: rss grew >${T.rssGrowthPct}% in the last hour`); + } + if (i.eventLoop.maxLagMs > 500) degraded.push(`event-loop: lag ${i.eventLoop.maxLagMs}ms`); + if (i.recoveredErrorRateLastWindow > T.recoveredErrorRate) { + degraded.push(`errors: ${i.recoveredErrorRateLastWindow} recovered in 5min`); + } + if (i.freeBytes !== null && i.freeBytes >= T.diskHardFloorBytes && i.freeBytes < T.diskSoftFloorBytes) { + degraded.push(`disk: ${mb(i.freeBytes)}MB free`); + } + + const level = unhealthy.length > 0 ? "unhealthy" : degraded.length > 0 ? "degraded" : "ok"; + return { + level, + reasons: level === "ok" ? [] : [...unhealthy, ...degraded], + metrics: { + rss: i.mem.rss, + heapUsed: i.mem.heapUsed, + external: i.mem.external, + uptimeMs: i.uptimeMs, + wsClients: i.wsClients, + watchers: i.watchers, + }, + eventLoop: { + maxLagMs: i.eventLoop.maxLagMs, + lastStallAt: i.eventLoop.lastStallAt, + lastStallCmd: i.eventLoop.lastStallCmd, + stalls: i.eventLoop.stalls, + }, + }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/health.test.ts` +Expected: PASS (all 7). + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/health.ts lib/daemon/__tests__/health.test.ts +git commit -m "add lib/daemon/health.ts: pure computeHealth + thresholds" +``` + +--- + +## Task 2: `lib/daemon/heartbeat-file.ts` — atomic-rename heartbeat + +**Files:** +- Create: `lib/daemon/heartbeat-file.ts` +- Test: `lib/daemon/__tests__/heartbeat-file.test.ts` + +**Interfaces:** +- Produces: `writeHeartbeat(dir: string, hb: Heartbeat): void`, `readHeartbeat(dir: string): Heartbeat | null`, `interface Heartbeat { at: number; seq: number }`. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/heartbeat-file.test.ts +import { test, expect } from "bun:test"; +import { mkdtempSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { writeHeartbeat, readHeartbeat } from "../heartbeat-file.ts"; + +test("write then read round-trips", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + writeHeartbeat(dir, { at: 123, seq: 7 }); + expect(readHeartbeat(dir)).toEqual({ at: 123, seq: 7 }); +}); + +test("missing file reads as null", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + expect(readHeartbeat(dir)).toBeNull(); +}); + +test("corrupt file reads as null", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + writeFileSync(join(dir, "daemon-heartbeat.json"), "{not json"); + expect(readHeartbeat(dir)).toBeNull(); +}); + +test("a second write overwrites atomically", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + writeHeartbeat(dir, { at: 1, seq: 1 }); + writeHeartbeat(dir, { at: 2, seq: 2 }); + expect(readHeartbeat(dir)).toEqual({ at: 2, seq: 2 }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/heartbeat-file.test.ts` +Expected: FAIL, module not found. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/heartbeat-file.ts +/** + * Monotonic liveness heartbeat, written to a small file via atomic rename so + * it never opens state.db. A stalled/lock-wedged daemon is exactly when the + * WAL is least readable, so the cross-process classifier reads THIS, not kv. + * Same db-free pattern as the Phase 0 breadcrumb. + */ +import { existsSync, readFileSync, renameSync, writeFileSync } from "fs"; +import { join } from "path"; + +export interface Heartbeat { + at: number; + seq: number; +} + +function heartbeatPath(dir: string): string { + return join(dir, "daemon-heartbeat.json"); +} + +/** Never fatal: a heartbeat is a diagnostic aid, not something a tick may fail over. */ +export function writeHeartbeat(dir: string, hb: Heartbeat): void { + try { + const tmp = `${heartbeatPath(dir)}.${process.pid}.tmp`; + writeFileSync(tmp, JSON.stringify(hb)); + renameSync(tmp, heartbeatPath(dir)); + } catch { + // best-effort + } +} + +export function readHeartbeat(dir: string): Heartbeat | null { + try { + const p = heartbeatPath(dir); + if (!existsSync(p)) return null; + return JSON.parse(readFileSync(p, "utf8")) as Heartbeat; + } catch { + return null; + } +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/heartbeat-file.test.ts` +Expected: PASS (4). + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/heartbeat-file.ts lib/daemon/__tests__/heartbeat-file.test.ts +git commit -m "add lib/daemon/heartbeat-file.ts: atomic-rename heartbeat" +``` + +--- + +## Task 3: `lib/daemon/loop-monitor.ts` — event-loop drift + heartbeat + +**Files:** +- Create: `lib/daemon/loop-monitor.ts` +- Test: `lib/daemon/__tests__/loop-monitor.test.ts` + +**Interfaces:** +- Consumes: nothing (pure `applyTick` plus a thin timer wrapper). +- Produces: `applyTick(stats, expected, now, cmd, opts, onStall): void` (pure), `startLoopMonitor(opts): { stats: LoopStats; seq: () => number; stop: () => void }`, `interface LoopStats { lagMs; maxLagMs; stalls; lastStallAt; lastStallCmd; currentlyStalled }`. + +- [ ] **Step 1: Write the failing test** (test the pure tick math; the timer wrapper is thin) + +```ts +// lib/daemon/__tests__/loop-monitor.test.ts +import { test, expect } from "bun:test"; +import { applyTick, newLoopStats, type LoopStats } from "../loop-monitor.ts"; + +const OPTS = { stallLogMs: 1000, stallUnhealthyMs: 2000, stallRecentMs: 10_000 }; + +test("an on-time tick records small lag and no stall", () => { + const s = newLoopStats(); + applyTick(s, /*expected*/ 1000, /*now*/ 1010, "cache:refresh", OPTS, () => {}); + expect(s.lagMs).toBe(10); + expect(s.maxLagMs).toBe(10); + expect(s.stalls).toBe(0); + expect(s.currentlyStalled).toBe(false); +}); + +test("a >1s drift counts a stall, records the in-flight cmd, and warns", () => { + const s = newLoopStats(); + let warned = 0; + applyTick(s, 1000, 2500, "mr:action", OPTS, () => { warned++; }); + expect(s.stalls).toBe(1); + expect(s.lastStallCmd).toBe("mr:action"); + expect(s.lastStallAt).toBe(2500); + expect(s.maxLagMs).toBe(1500); + expect(warned).toBe(1); +}); + +test("currentlyStalled is true when the last big drift is within stallRecentMs", () => { + const s = newLoopStats(); + applyTick(s, 1000, 3500, "x", OPTS, () => {}); // 2500ms drift >= 2000 unhealthy, lastStallAt=3500 + expect(s.currentlyStalled).toBe(true); + // a small-drift tick whose `now` is past lastStallAt + stallRecentMs clears it + // (10ms drift, so no new stall; now-lastStallAt = 10500 > 10000 recent window) + applyTick(s, 13990, 14000, null, OPTS, () => {}); + expect(s.currentlyStalled).toBe(false); +}); + +test("maxLagMs is a high-water mark", () => { + const s: LoopStats = newLoopStats(); + applyTick(s, 1000, 1300, null, OPTS, () => {}); + applyTick(s, 1550, 1600, null, OPTS, () => {}); + expect(s.maxLagMs).toBe(300); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/loop-monitor.test.ts` +Expected: FAIL, module not found. + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/loop-monitor.ts +/** + * Event-loop drift monitor. A ~250ms unref'd interval measures how late each + * tick fires vs its scheduled time; a large drift means the loop was blocked. + * The interval callback is created once and the stats object is preallocated, + * so the hot tick allocates nothing. Every ~2s it also writes the heartbeat + * file the cross-process classifier reads. + */ +import type { Logger } from "pino"; + +export interface LoopStats { + lagMs: number; + maxLagMs: number; + stalls: number; + lastStallAt: number | null; + lastStallCmd: string | null; + currentlyStalled: boolean; +} + +export function newLoopStats(): LoopStats { + return { lagMs: 0, maxLagMs: 0, stalls: 0, lastStallAt: null, lastStallCmd: null, currentlyStalled: false }; +} + +interface TickOpts { + stallLogMs: number; + stallUnhealthyMs: number; + stallRecentMs: number; +} + +/** Pure: fold one tick into `stats`. `onStall` fires once per stall (warn sink). */ +export function applyTick( + stats: LoopStats, + expected: number, + now: number, + currentCmd: string | null, + opts: TickOpts, + onStall: (drift: number, cmd: string | null) => void, +): void { + const drift = now - expected; + stats.lagMs = drift > 0 ? drift : 0; + if (stats.lagMs > stats.maxLagMs) stats.maxLagMs = stats.lagMs; + if (drift > opts.stallLogMs) { + stats.stalls += 1; + stats.lastStallAt = now; + stats.lastStallCmd = currentCmd; + onStall(drift, currentCmd); + } + stats.currentlyStalled = + stats.lastStallAt !== null && + now - stats.lastStallAt <= opts.stallRecentMs && + (drift > opts.stallUnhealthyMs || stats.maxLagMs > opts.stallUnhealthyMs); +} + +export interface LoopMonitorOpts { + log: Logger; + tickMs?: number; + stallLogMs?: number; + stallUnhealthyMs?: number; + stallRecentMs?: number; + heartbeatMs?: number; + currentCmd: () => string | null; + onHeartbeat: (at: number, seq: number) => void; +} + +export function startLoopMonitor(opts: LoopMonitorOpts): { stats: LoopStats; stop: () => void } { + const tickMs = opts.tickMs ?? 250; + const tickOpts: TickOpts = { + stallLogMs: opts.stallLogMs ?? 1000, + stallUnhealthyMs: opts.stallUnhealthyMs ?? 2000, + stallRecentMs: opts.stallRecentMs ?? 10_000, + }; + const heartbeatMs = opts.heartbeatMs ?? 2000; + const stats = newLoopStats(); + let expected = Date.now() + tickMs; + let lastHeartbeat = 0; + let seq = 0; + let warnedThisStall = false; + + // Hoisted once (allocation-free ruling): the tick must not build a fresh + // closure every 250ms. onStall closes over warnedThisStall by reference. + const onStall = (drift: number, cmd: string | null): void => { + if (!warnedThisStall) { + opts.log.warn({ driftMs: drift, cmd }, "event loop stalled"); + warnedThisStall = true; + } + }; + + const timer = setInterval(() => { + const now = Date.now(); + applyTick(stats, expected, now, opts.currentCmd(), tickOpts, onStall); + if (stats.lagMs <= tickOpts.stallLogMs) warnedThisStall = false; + expected = now + tickMs; + if (now - lastHeartbeat >= heartbeatMs) { + lastHeartbeat = now; + opts.onHeartbeat(now, ++seq); + } + }, tickMs); + timer.unref(); + + return { stats, seq: () => seq, stop: () => clearInterval(timer) }; +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/daemon/__tests__/loop-monitor.test.ts` +Expected: PASS (4). + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/loop-monitor.ts lib/daemon/__tests__/loop-monitor.test.ts +git commit -m "add lib/daemon/loop-monitor.ts: drift monitor + heartbeat cadence" +``` + +--- + +## Task 4: `lib/daemon-status.ts` — heartbeat input + `stalled` detail + +**Files:** +- Modify: `lib/daemon-status.ts` +- Test: extend `lib/__tests__/daemon-status.test.ts` (add cases; do not rewrite existing ones). + +**Interfaces:** +- Consumes: `Heartbeat` shape `{ at, seq }` (structural; do not import to avoid a cycle), `HealthEventLoop` shape for the degraded passthrough. +- Produces: `DaemonStatusInputs` gains `heartbeat?`, `heartbeatStaleMs?`, `pingEventLoop?`; the `alive-not-serving` verdict gains `detail: "...|stalled"` and optional `stalledForMs`; the `degraded` verdict gains optional `eventLoop`. + +- [ ] **Step 1: Write the failing test** + +```ts +// add to lib/__tests__/daemon-status.test.ts +import { classifyDaemonStatus } from "../daemon-status.ts"; + +test("alive + ping-fail + ready + stale heartbeat => alive-not-serving 'stalled'", () => { + const now = 1_000_000; + const v = classifyDaemonStatus({ + installed: true, response: null, pingOk: false, pid: 42, pidAlive: true, + breadcrumb: { phase: "ready" }, + heartbeat: { at: now - 8000, seq: 3 }, heartbeatStaleMs: 6000, + now, + }); + expect(v.state).toBe("alive-not-serving"); + if (v.state === "alive-not-serving") { + expect(v.detail).toBe("stalled"); + expect(v.stalledForMs).toBe(8000); + } +}); + +test("alive + ready + FRESH heartbeat => 'wedged', not 'stalled'", () => { + const now = 1_000_000; + const v = classifyDaemonStatus({ + installed: true, response: null, pingOk: false, pid: 42, pidAlive: true, + breadcrumb: { phase: "ready" }, + heartbeat: { at: now - 500, seq: 9 }, heartbeatStaleMs: 6000, + now, + }); + expect(v.state === "alive-not-serving" && v.detail).toBe("wedged"); +}); + +test("degraded/unresponsive carries the ping-supplied eventLoop", () => { + const v = classifyDaemonStatus({ + installed: true, response: null, pingOk: true, pid: 42, + pingEventLoop: { maxLagMs: 1400, lastStallAt: 123, lastStallCmd: "mr:action", stalls: 2 }, + }); + expect(v.state).toBe("degraded"); + if (v.state === "degraded") expect(v.eventLoop?.maxLagMs).toBe(1400); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/__tests__/daemon-status.test.ts` +Expected: FAIL (unknown properties `heartbeat`/`pingEventLoop`; `stalled` not in union). + +- [ ] **Step 3: Write minimal implementation** + +In `lib/daemon-status.ts`: + +Add a structural type near the top: +```ts +export interface HeartbeatInput { at: number; seq: number } +export interface StatusEventLoop { maxLagMs: number; lastStallAt: number | null; lastStallCmd: string | null; stalls: number } +``` + +Extend the verdict union (the two affected members): +```ts + | { state: "degraded"; reason: "error" | "unresponsive"; detail?: string; pid: number | null; eventLoop?: StatusEventLoop } + | { state: "alive-not-serving"; pid: number; detail: "booting" | "wedged" | "quarantined" | "stalled"; stalledForMs?: number } +``` + +Extend `DaemonStatusInputs`: +```ts + heartbeat?: HeartbeatInput | null; + heartbeatStaleMs?: number; + pingEventLoop?: StatusEventLoop; +``` + +Change `classifyAliveNotServingDetail` to also detect stall, and return the age: +```ts +function classifyAliveNotServingDetail( + breadcrumb: DaemonBreadcrumbInput | null | undefined, + supervision: SupervisionState | undefined, + heartbeat: HeartbeatInput | null | undefined, + heartbeatStaleMs: number, + now: number, +): { detail: "booting" | "wedged" | "quarantined" | "stalled"; stalledForMs?: number } { + const phase = breadcrumb?.phase; + if (!phase || PHASE_ORDER.indexOf(phase) < PHASE_ORDER.indexOf("ready")) return { detail: "booting" }; + if (heartbeat && now - heartbeat.at > heartbeatStaleMs) { + return { detail: "stalled", stalledForMs: now - heartbeat.at }; + } + if (supervision?.lastExit?.kind === "boot-failed") return { detail: "quarantined" }; + return { detail: "wedged" }; +} +``` + +In `classifyDaemonStatus`, thread `now` into the alive-not-serving branch and pass eventLoop into degraded/unresponsive: +```ts + if (response) { + return { state: "degraded", reason: "error", detail: response.error, pid }; + } + if (pingOk) return { state: "degraded", reason: "unresponsive", pid, eventLoop: opts.pingEventLoop }; + ... + if (pidAlive && pid !== null) { + if (breadcrumb?.flavor && intendedFlavor && breadcrumb.flavor !== intendedFlavor) { + return { state: "parked", pid, ...(holderFlavor ? { holderFlavor } : {}) }; + } + const now = opts.now ?? Date.now(); + const d = classifyAliveNotServingDetail(breadcrumb, supervision, opts.heartbeat, opts.heartbeatStaleMs ?? 6000, now); + return { state: "alive-not-serving", pid, detail: d.detail, ...(d.stalledForMs ? { stalledForMs: d.stalledForMs } : {}) }; + } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/__tests__/daemon-status.test.ts` +Expected: PASS (new + existing). + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon-status.ts lib/__tests__/daemon-status.test.ts +git commit -m "daemon-status: heartbeat-stale 'stalled' detail + degraded eventLoop" +``` + +--- + +## Task 5: `commands/daemon.ts` + `lib/daemon-client.ts` — render health, read heartbeat, non-restarting ping + +**Files:** +- Modify: `commands/daemon.ts` (`statusLines`, `showStatus`) +- Modify: `lib/daemon-client.ts` (add `pingDaemon`) +- Test: extend `commands/__tests__/daemon-status-lines.test.ts` (or the existing statusLines test file; if none, create `commands/__tests__/status-lines.test.ts`). + +**Interfaces:** +- Consumes: `classifyDaemonStatus` (Task 4 shape), `readHeartbeat` (Task 2), `RT_DIR`. +- Produces: `pingDaemon(timeoutMs?): Promise` (non-restarting); `statusLines` renders the new degraded/stalled/health lines. + +- [ ] **Step 1: Write the failing test** + +```ts +// commands/__tests__/status-lines.test.ts +import { test, expect } from "bun:test"; +import { statusLines } from "../daemon.ts"; + +const strip = (s: string) => s.replace(/\[[0-9;]*m/g, ""); + +test("degraded/unresponsive prints ping-carried maxLag, not 'likely mid-sync'", () => { + const lines = statusLines( + { state: "degraded", reason: "unresponsive", pid: 42, eventLoop: { maxLagMs: 1400, lastStallAt: 1, lastStallCmd: "mr:action", stalls: 2 } } as any, + 2000, + ).map(strip).join("\n"); + expect(lines).not.toContain("likely mid-sync"); + expect(lines).toContain("1400ms"); + expect(lines).toContain("mr:action"); +}); + +test("alive-not-serving 'stalled' prints stalled Ns ago", () => { + const lines = statusLines( + { state: "alive-not-serving", pid: 42, detail: "stalled", stalledForMs: 8000 } as any, + 0, + ).map(strip).join("\n"); + expect(lines).toContain("event loop stalled"); + expect(lines).toContain("8s"); +}); + +test("running prints the health level and reasons when present", () => { + const lines = statusLines( + { state: "running", data: { pid: 42, uptime: 60000, watchedRepos: 3, cacheEntries: 10, + health: { level: "degraded", reasons: ["refresh: 3 repos failing (auth?)"] } } } as any, + 0, + ).map(strip).join("\n"); + expect(lines).toContain("degraded"); + expect(lines).toContain("refresh: 3 repos failing"); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test commands/__tests__/status-lines.test.ts` +Expected: FAIL (old text / missing health line). + +- [ ] **Step 3: Write minimal implementation** + +In `commands/daemon.ts` `statusLines`: + +Replace the degraded ternary (currently line ~514-518) so the `unresponsive` branch shows lag when present: +```ts + if (verdict.reason === "error") { + lines.push(` ${dim}status command failed: ${verdict.detail ?? "unknown error"}${reset}`); + } else if (verdict.eventLoop && verdict.eventLoop.maxLagMs > 0) { + const el = verdict.eventLoop; + lines.push(` ${dim}answers ping, status timed out — event loop maxLag ${el.maxLagMs}ms${el.lastStallCmd ? ` (last stall in ${el.lastStallCmd})` : ""}${reset}`); + } else { + lines.push(` ${dim}answers ping, but status timed out — likely mid-sync${reset}`); + } +``` + +In the `alive-not-serving` branch, add the `stalled` case and render the age: +```ts + const detailLine = { + booting: "still booting", + wedged: "reached ready but stopped answering (likely deadlocked)", + quarantined: "recovered from a corrupt db but still not answering", + stalled: `event loop stalled ${Math.round((verdict.stalledForMs ?? 0) / 1000)}s ago (no heartbeat)`, + }[verdict.detail]; +``` + +In the `running` branch, after the cache line, append health + metrics/eventLoop when present: +```ts + const health = verdict.data.health as { level: string; reasons: string[] } | undefined; + if (health && health.level !== "ok") { + const dot = health.level === "unhealthy" ? red : yellow; + lines.push(` ${dot}health: ${health.level}${reset}`); + for (const r of health.reasons) lines.push(` ${dim}- ${r}${reset}`); + } + const el = verdict.data.eventLoop as { maxLagMs: number } | undefined; + if (el && el.maxLagMs >= 500) lines.push(` ${dim}event loop: maxLag ${el.maxLagMs}ms${reset}`); +``` + +In `lib/daemon-client.ts`, add a non-restarting ping (uses the existing single-attempt `trySocketQuery`): +```ts +/** Single-attempt ping that never triggers the restart machinery, so + * `rt daemon status` can probe liveness and read the daemon's eventLoop + * summary without spawning a daemon as a side effect. */ +export async function pingDaemon(timeoutMs?: number): Promise { + return (await trySocketQuery("ping", undefined, timeoutMs)).response; +} +``` + +In `commands/daemon.ts` `showStatus`, capture the ping payload and heartbeat, and pass them in. Replace the `pingOk` line and the `needsPidProbe` block: +```ts + const pingResp = classifyDaemonStatus.needsLivenessProbe(response) ? await pingDaemon() : null; + const pingOk = pingResp?.ok === true; + ... + let heartbeat: ReturnType | undefined; + if (classifyDaemonStatus.needsPidProbe(response, pingOk)) { + breadcrumb = readBreadcrumb(); + supervision = readSupervisionState(); + heartbeat = readHeartbeat(RT_DIR); + const probed = await probePidAlive(recordedPid, breadcrumb?.pid); + pidAlive = probed.alive; + pid = probed.pid; + } + + const verdict = classifyDaemonStatus({ + installed: true, response, pingOk, pid, pidAlive, + intendedFlavor: resolveIntendedMode().mode, + breadcrumb, supervision, + heartbeat, pingEventLoop: (pingResp as any)?.eventLoop, + }); +``` +Add imports: `import { pingDaemon } from "../lib/daemon-client.ts"` (or extend the existing daemon-client import), `import { readHeartbeat } from "../lib/daemon/heartbeat-file.ts"`, `import { RT_DIR } from "../lib/daemon-config.ts"`. (Confirm exact relative paths against the file's existing imports.) + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test commands/__tests__/status-lines.test.ts` +Expected: PASS. Then `bunx tsc --noEmit` (0 errors). + +- [ ] **Step 5: Commit** + +```bash +git add commands/daemon.ts lib/daemon-client.ts commands/__tests__/status-lines.test.ts +git commit -m "daemon status: render health/stall lines; add non-restarting pingDaemon" +``` + +--- + +## Task 6: `lib/daemon-logger.ts` — stream error listener + crash-handler resilience + +**Files:** +- Modify: `lib/daemon-logger.ts` +- Test: `lib/__tests__/daemon-logger-resilience.test.ts` + +**Interfaces:** +- Produces: `DaemonLoggerHandle` gains `loggerDegraded(): boolean`; `createDaemonLogger` installs a stream `error` listener; the crash handlers fall back to a raw write and still `process.exit(1)`. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/__tests__/daemon-logger-resilience.test.ts +import { test, expect } from "bun:test"; +import { createDaemonLogger } from "../daemon-logger.ts"; +import { mkdtempSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +test("a stream write error does not throw out of log.info and flips loggerDegraded", async () => { + const dir = mkdtempSync(join(tmpdir(), "logres-")); + const handle = await createDaemonLogger({ logDir: dir, level: "info" }); + // Simulate a write failure by emitting 'error' on the underlying stream. + handle.stream.emit("error", Object.assign(new Error("no space"), { code: "ENOSPC" })); + expect(() => handle.logger.info("after enospc")).not.toThrow(); + expect(handle.loggerDegraded()).toBe(true); +}); +``` + +Note: this requires `createDaemonLogger` to expose the `stream` on the returned handle (add it to the handle type). If exposing `stream` is undesirable, the test may instead construct the logger against an injected stream; adjust `createDaemonLogger`'s options to accept an optional `stream` for testing. Prefer exposing `stream` on the handle (smallest change). + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/__tests__/daemon-logger-resilience.test.ts` +Expected: FAIL (`loggerDegraded` undefined / `stream` undefined). + +- [ ] **Step 3: Write minimal implementation** + +In `createDaemonLogger` (after the `roll(...)` stream is created, before building the pino logger): +```ts + let degraded = false; + stream.on("error", (err: any) => { + degraded = true; + try { + require("fs").writeSync(2, `daemon-logger: ${err?.code ?? ""} ${err?.message ?? err}\n`); + } catch { /* nothing left to do */ } + }); +``` +Return the handle with the new members: +```ts + return { + logger, + stream, + loggerDegraded: () => degraded, + childLogger: (module: string) => logger.child({ module }), + flush: () => { try { logger.flush(); } catch {} }, + }; +``` +Update the `DaemonLoggerHandle` interface to include `stream: NodeJS.WritableStream` and `loggerDegraded(): boolean`. + +In `installCrashHandlers`, wrap each handler body so a throwing logger cannot abort the handler (keep the existing exit semantics — boot-vs-steady per Phase 0): +```ts + process.on("uncaughtException", (err) => { + try { + handle.logger.fatal({ err }, "uncaughtException"); + } catch { + try { require("fs").writeSync(2, `uncaughtException (logger failed): ${err?.stack ?? err}\n`); } catch {} + } + handle.flush?.(); + process.exit(1); + }); +``` +Apply the identical try/catch + raw-write fallback to the `unhandledRejection` handler, preserving its current boot-phase-aware exit decision (do not change whether it exits; only guard the logging). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/__tests__/daemon-logger-resilience.test.ts` +Expected: PASS. `bunx tsc --noEmit` clean. + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon-logger.ts lib/__tests__/daemon-logger-resilience.test.ts +git commit -m "daemon-logger: stream error listener + loggerDegraded + crash-handler raw-write fallback" +``` + +--- + +## Task 7: `lib/daemon-logger.ts` — level from setting, stderr demotion, size cap, recovered-error counter + +**Files:** +- Modify: `lib/daemon-logger.ts` +- Test: `lib/__tests__/daemon-logger-level.test.ts` + +**Interfaces:** +- Consumes: `getSetting` from `@mattstack/rt-client`. +- Produces: `getDaemonLogger` resolves level `RT_LOG_LEVEL env ?? getSetting("rt.logLevel") ?? "info"`; `createDaemonLogger` adds `size: "50m"` to pino-roll; the stderr interceptor logs at `warn` unless a panic prefix; a `recoveredErrorCount()` getter on the handle. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/__tests__/daemon-logger-level.test.ts +import { test, expect } from "bun:test"; +import { resolveDaemonLogLevel, isPanicLine } from "../daemon-logger.ts"; + +test("RT_LOG_LEVEL env wins over the setting", () => { + expect(resolveDaemonLogLevel("debug", () => "warn")).toBe("debug"); +}); +test("setting is used when env is unset", () => { + expect(resolveDaemonLogLevel(undefined, () => "warn")).toBe("warn"); +}); +test("falls back to info when neither is set", () => { + expect(resolveDaemonLogLevel(undefined, () => undefined)).toBe("info"); +}); +test("a panic-looking stderr line is escalated; ordinary noise is not", () => { + expect(isPanicLine("panic: runtime error")).toBe(true); + expect(isPanicLine("Uncaught Error: boom")).toBe(true); + expect(isPanicLine("rt: ignoring \"x\" from the team scope")).toBe(false); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/__tests__/daemon-logger-level.test.ts` +Expected: FAIL (functions not exported). + +- [ ] **Step 3: Write minimal implementation** + +Add pure helpers: +```ts +export function resolveDaemonLogLevel( + env: string | undefined, + fromSetting: () => string | undefined, +): string { + if (env) return env; + try { + const v = fromSetting(); + if (v) return v; + } catch { /* resolver may be unavailable pre-boot */ } + return "info"; +} + +const PANIC_PREFIXES = ["panic:", "fatal error:", "Uncaught ", "UnhandledPromiseRejection"]; +export function isPanicLine(text: string): boolean { + return PANIC_PREFIXES.some((p) => text.startsWith(p)); +} +``` +In `getDaemonLogger`, use it: +```ts + cachedPromise = createDaemonLogger({ + logDir: logsDir(), + level: resolveDaemonLogLevel(process.env.RT_LOG_LEVEL, () => { + return getSetting("rt.logLevel").value; + }) as pino.LevelWithSilent, + })... +``` +Add `import { getSetting } from "@mattstack/rt-client"` (match the existing import style in the repo; if daemon-logger must stay dependency-light, inject the getter from `lib/daemon.ts` instead — but the settings resolver is in-process and cheap, so a direct import is fine). + +In `createDaemonLogger`, add the size cap to the `roll(...)` options: +```ts + const stream = await roll({ + file: `${opts.logDir}/daemon`, + extension: ".log", + frequency: "daily", + dateFormat: "yyyy-MM-dd", + mkdir: true, + size: "50m", + limit: { count: 14 }, + sync: true, + }); +``` + +In the stderr interceptor (`redirectNativeStderr`/the `process.stderr.write` override that routes into pino), route at `warn` with `source: "stderr"` unless `isPanicLine`, and increment a recovered-error counter that the handle exposes: +```ts + // inside the intercept, `text` is the stderr chunk: + if (isPanicLine(text)) { + handleLogger.error({ source: "stderr" }, text.trimEnd()); + } else { + recovered += 1; + handleLogger.warn({ source: "stderr" }, text.trimEnd()); + } +``` +Expose `recoveredErrorCount: () => recovered` on the handle (module-scope `let recovered = 0`). Also increment `recovered` in the `unhandledRejection` recovered path (steady-state). Add `recoveredErrorCount(): number` to `DaemonLoggerHandle`. + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/__tests__/daemon-logger-level.test.ts` +Expected: PASS. `bunx tsc --noEmit` clean. + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon-logger.ts lib/__tests__/daemon-logger-level.test.ts +git commit -m "daemon-logger: rt.logLevel resolution, stderr->warn demotion, 50m size cap, recovered-error counter" +``` + +--- + +## Task 8: `lib/log-janitor.ts` — `onError` callback + +**Files:** +- Modify: `lib/log-janitor.ts` +- Test: extend `lib/__tests__/log-janitor.test.ts` (or create if absent). + +**Interfaces:** +- Produces: `pruneLogs(dir, retentionDays, now, onError?): { removed }` where `onError?: (phase: "readdir" | "unlink", err: unknown, file?: string) => void`. + +- [ ] **Step 1: Write the failing test** + +```ts +// add to lib/__tests__/log-janitor.test.ts +import { test, expect } from "bun:test"; +import { pruneLogs } from "../log-janitor.ts"; +import { join } from "path"; + +test("readdir failure reports via onError instead of swallowing", () => { + const calls: string[] = []; + const bogus = join("/nonexistent-xyz", "rt", "logs"); + pruneLogs(bogus, 14, Date.now(), (phase) => calls.push(phase)); + expect(calls).toContain("readdir"); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/__tests__/log-janitor.test.ts` +Expected: FAIL (`onError` not a parameter). + +- [ ] **Step 3: Write minimal implementation** + +Add the optional param and call it in both catches: +```ts +export function pruneLogs( + dir: string, + retentionDays: number, + now: number, + onError?: (phase: "readdir" | "unlink", err: unknown, file?: string) => void, +): { removed: string[] } { + ... + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch (err) { + onError?.("readdir", err); + return { removed }; + } + ... + try { + unlinkSync(full); + removed.push(entry.name); + } catch (err) { + onError?.("unlink", err, entry.name); + } +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/__tests__/log-janitor.test.ts` +Expected: PASS. + +- [ ] **Step 5: Wire the daemon's callers to log at warn, then commit** + +In `lib/daemon.ts`, both `pruneLogs(logsDir(), logRetentionDays(), Date.now())` call sites (the daily interval and the boot timeout) pass an onError that warns: +```ts + const { removed } = pruneLogs(logsDir(), logRetentionDays(), Date.now(), + (phase, err, file) => log.warn({ err, phase, file }, "log prune step failed")); +``` + +```bash +git add lib/log-janitor.ts lib/daemon.ts lib/__tests__/log-janitor.test.ts +git commit -m "log-janitor: onError callback; daemon logs prune failures at warn" +``` + +--- + +## Task 9: Settings — `rt.logLevel` row, injectable resolver warn sink, rebuild dist + +**Files:** +- Modify: `packages/rt-client/src/settings/registry-defs.ts` +- Modify: `packages/rt-client/src/settings/resolve.ts` +- Modify: `packages/rt-client/src/settings/registry-machinery.ts` (ResolveOpts) OR add a module-level sink (chosen below) +- Modify: `packages/rt-client/src/index.ts` (export the sink setter) +- Test: `packages/rt-client/test/settings-warn-sink.test.ts`; existing `settings-paths-parity` and `dist-freshness` tests must stay green. + +**Interfaces:** +- Produces: registry key `rt.logLevel`; `setSettingsWarnSink(sink: ((msg: string) => void) | null): void` exported from the package (default `console.warn`). + +- [ ] **Step 1: Add the registry row** + +In `registry-defs.ts`, insert directly after the `rt.logRetentionDays` row (before `rt.apiPort`): +```ts + { + key: "rt.logLevel", + type: "string", + scopes: ["machine", "user"], + default: "info", + merge: "replace", + migrated: true, + description: "Daemon log level (trace|debug|info|warn|error). RT_LOG_LEVEL env wins, then this setting, then info (lib/daemon-logger.ts resolveDaemonLogLevel). A fresh key, not an ownership-latch port, so a default is fine here.", + }, +``` + +- [ ] **Step 2: Write the failing warn-sink test** + +```ts +// packages/rt-client/test/settings-warn-sink.test.ts +import { test, expect } from "bun:test"; +import { setSettingsWarnSink } from "../src/index.ts"; +import { emitSettingsWarning } from "../src/settings/resolve.ts"; + +test("a bound sink receives warnings and dedupes on identical messages", () => { + const seen: string[] = []; + setSettingsWarnSink((m) => seen.push(m)); + emitSettingsWarning("rt: sample warning"); + emitSettingsWarning("rt: sample warning"); + expect(seen).toEqual(["rt: sample warning"]); // deduped + setSettingsWarnSink(null); // restore default +}); +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `cd packages/rt-client && bun test test/settings-warn-sink.test.ts` +Expected: FAIL (exports missing). + +- [ ] **Step 4: Implement the sink in `resolve.ts`** + +Add a module-level, deduping sink and a public emit used by the 3 warn sites: +```ts +let warnSink: ((msg: string) => void) | null = null; +const warnedOnce = new Set(); + +/** The daemon binds a deduped log.warn here so a hot-path getSetting on a + * disallowed-scope key warns once, not every tick. Default: console.warn + * (CLI/test behavior unchanged). null restores the default. */ +export function setSettingsWarnSink(sink: ((msg: string) => void) | null): void { + warnSink = sink; + warnedOnce.clear(); +} + +export function emitSettingsWarning(msg: string): void { + if (warnSink) { + if (warnedOnce.has(msg)) return; + warnedOnce.add(msg); + warnSink(msg); + return; + } + console.warn(msg); +} +``` +Replace the three `console.warn(...)` calls (`warnInvalid` line ~491, `listSettings` line ~546, `listUnregistered` line ~586) with `emitSettingsWarning(...)` passing the same message string. Note: dedupe-by-message collapses to the spec's per-`(key, scope, reason)` tuple only because each message string is deterministic in exactly those fields (`warnInvalid` interpolates key + scope + file + reason). That holds for all three sites today; keep it true if you edit the message text. + +Export `setSettingsWarnSink` from `packages/rt-client/src/index.ts` alongside the other settings exports: +```ts +export { getSetting, listSettings, explainSetting, expandVariables, SCOPE_ORDER, setSettingsWarnSink } from "./settings/resolve.ts"; +``` + +- [ ] **Step 5: Run tests, rebuild dist** + +Run: `cd packages/rt-client && bun test test/settings-warn-sink.test.ts && bun run build` +Expected: PASS; `bun run build` regenerates `dist/` so `test/dist-freshness.test.ts` stays green. **Do not bump the version; do not publish.** + +- [ ] **Step 6: Commit** + +```bash +git add packages/rt-client/src/settings/registry-defs.ts packages/rt-client/src/settings/resolve.ts packages/rt-client/src/index.ts packages/rt-client/test/settings-warn-sink.test.ts packages/rt-client/dist +git commit -m "rt-client: rt.logLevel registry row + injectable deduped settings warn sink (dist rebuilt, no bump)" +``` + +--- + +## Task 10: `lib/daemon.ts` `handleCommand` — reqId, caller, suppression, slow-command, currentCmd + +**Files:** +- Modify: `lib/daemon.ts` +- Create: `lib/daemon/command-attribution.ts` (pure helpers: reqId, suppression bookkeeping) +- Test: `lib/daemon/__tests__/command-attribution.test.ts` + +**Interfaces:** +- Produces: `shortReqId(): string`; `shouldLogSuppressed(map, key, now, windowMs): { emit: boolean; suppressed: number }`; module-scope `currentCmd` ref that `handleCommand` sets; `handleCommand` logs `{ reqId, cmd, caller, durationMs }` and echoes `reqId` in `ok:false` envelopes. + +- [ ] **Step 1: Write the failing test (pure helpers)** + +```ts +// lib/daemon/__tests__/command-attribution.test.ts +import { test, expect } from "bun:test"; +import { shortReqId, makeSuppressor } from "../command-attribution.ts"; + +test("shortReqId is short and unique-ish", () => { + const a = shortReqId(); const b = shortReqId(); + expect(a).toMatch(/^[a-z0-9]{6}$/); + expect(a).not.toBe(b); +}); + +test("suppressor logs first, then throttles with a running suppressed count", () => { + const s = makeSuppressor(60_000); + expect(s.check("mr:action|boom", 0)).toEqual({ emit: true, suppressed: 0 }); // first: log + expect(s.check("mr:action|boom", 1_000)).toEqual({ emit: false, suppressed: 1 }); // within window: silent + expect(s.check("mr:action|boom", 2_000)).toEqual({ emit: false, suppressed: 2 }); + expect(s.check("mr:action|boom", 61_000)).toEqual({ emit: true, suppressed: 2 }); // window elapsed: log with count + expect(s.check("mr:action|boom", 61_500)).toEqual({ emit: false, suppressed: 1 }); // count resets after an emit +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/command-attribution.test.ts` +Expected: FAIL (module not found). + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/command-attribution.ts +/** Short request id for tying a daemon log line to the invocation. */ +export function shortReqId(): string { + return Math.random().toString(36).slice(2, 8).padEnd(6, "0"); +} + +interface SuppressEntry { lastEmitAt: number; suppressed: number } + +/** Per-(cmd,error) suppression: always emit the first occurrence and, once per + * window, emit again carrying the count suppressed since the last emit. */ +export function makeSuppressor(windowMs: number) { + const map = new Map(); + return { + check(key: string, now: number): { emit: boolean; suppressed: number } { + const e = map.get(key); + if (!e) { + map.set(key, { lastEmitAt: now, suppressed: 0 }); + return { emit: true, suppressed: 0 }; + } + if (now - e.lastEmitAt >= windowMs) { + const suppressed = e.suppressed; + e.lastEmitAt = now; + e.suppressed = 0; + return { emit: true, suppressed }; + } + e.suppressed += 1; + return { emit: false, suppressed: e.suppressed }; + }, + }; +} +``` + +In `lib/daemon.ts`, add module-scope state near `handleCommand`: +```ts +import { shortReqId, makeSuppressor } from "./daemon/command-attribution.ts"; +const currentCmd: { cmd: string | null } = { cmd: null }; +const rejectSuppressor = makeSuppressor(60_000); +const SLOW_COMMAND_MS = 2000; +``` +Rewrite `handleCommand` (keep the throw-on-exception contract): +```ts +async function handleCommand(cmd: string, payload: any, signal?: AbortSignal): Promise { + const t0 = Date.now(); + const reqId = shortReqId(); + const caller = (payload && typeof payload._client === "string" ? payload._client : "unknown"); + currentCmd.cmd = cmd; + try { + const result = await routeCommand(cmd, payload, signal); + const durationMs = Date.now() - t0; + if (result && result.ok === false) { + const key = `${cmd}|${result.error ?? ""}`; + const { emit, suppressed } = rejectSuppressor.check(key, Date.now()); + if (emit) log.warn({ reqId, cmd, caller, error: result.error, durationMs, digest: redactDigest(payload), ...(suppressed ? { suppressed } : {}) }, "command rejected"); + return { ...result, reqId }; + } + if (durationMs > SLOW_COMMAND_MS) log.info({ reqId, cmd, caller, durationMs }, "command handled (slow)"); + else log.debug({ reqId, cmd, caller, durationMs }, "command handled"); + return result; + } catch (err) { + log.error({ err, reqId, cmd, caller, durationMs: Date.now() - t0, digest: redactDigest(payload) }, "command failed"); + throw err; + } finally { + currentCmd.cmd = null; + } +} + +function redactDigest(payload: any): Record { + if (!payload || typeof payload !== "object") return {}; + const keys = Object.keys(payload); + const pick = (k: string) => (payload[k] !== undefined ? { [k]: payload[k] } : {}); + return { keys, ...pick("repo"), ...pick("repoName"), ...pick("branch"), ...pick("iid"), ...pick("room") }; +} +``` +Wire `currentCmd.cmd` into the loop monitor in Task 13 (the monitor's `currentCmd: () => currentCmd.cmd`). + +- [ ] **Step 4: Run tests** + +Run: `bun test lib/daemon/__tests__/command-attribution.test.ts` then `bunx tsc --noEmit`. +Expected: PASS; 0 type errors. + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/command-attribution.ts lib/daemon.ts lib/daemon/__tests__/command-attribution.test.ts +git commit -m "handleCommand: reqId + caller tag + per-(cmd,error) suppression + slow-command info + currentCmd" +``` + +--- + +## Task 11: Unknown-command envelope + `X-RT-Client` in both transports + +**Files:** +- Modify: `lib/daemon.ts` (`routeCommand` default) +- Modify: `lib/daemon-client.ts` (send `X-RT-Client` on GET+POST) +- Modify: `packages/rt-client/src/transport.ts` (send `X-RT-Client`) +- Test: `lib/daemon/__tests__/unknown-command.test.ts` + +**Interfaces:** +- Produces: unknown-command returns `{ ok: false, code: "unknown-command", error, version }`. Both transports set `X-RT-Client: /`. + +- [ ] **Step 1: Write the failing test** + +Extract the default-branch shape into a tiny pure helper so it's testable: +```ts +// lib/daemon/__tests__/unknown-command.test.ts +import { test, expect } from "bun:test"; +import { unknownCommandReply } from "../unknown-command.ts"; + +test("unknown command carries a code, version, and actionable text", () => { + const r = unknownCommandReply("chat:archive", "v0.9.0"); + expect(r.ok).toBe(false); + expect(r.code).toBe("unknown-command"); + expect(r.version).toBe("v0.9.0"); + expect(r.error).toContain("v0.9.0"); + expect(r.error).toContain("chat:archive"); + expect(r.error.toLowerCase()).toContain("restart"); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/unknown-command.test.ts` +Expected: FAIL (module not found). + +- [ ] **Step 3: Write minimal implementation** + +```ts +// lib/daemon/unknown-command.ts +export function unknownCommandReply(cmd: string, version: string) { + return { + ok: false as const, + code: "unknown-command" as const, + version, + error: `daemon at version ${version} does not know "${cmd}"; restart or upgrade rt (rt daemon restart)`, + }; +} +``` +In `lib/daemon.ts` `routeCommand`, replace the default branch: +```ts + default: + return unknownCommandReply(cmd, typeof RT_VERSION !== "undefined" ? RT_VERSION : "source"); +``` +(add `import { unknownCommandReply } from "./daemon/unknown-command.ts"`). + +In `lib/daemon-client.ts` `trySocketQuery`, always send the client header (restructure the `hasBody` ternary so headers fire on GET too): +```ts + const headers: Record = { "X-RT-Client": `rt-cli/${process.pid}` }; + if (hasBody) headers["Content-Type"] = "application/json"; + const response = await fetch(`http://localhost/${cmd}`, { + unix: DAEMON_SOCK_PATH, + method: hasBody ? "POST" : "GET", + headers, + body: hasBody ? JSON.stringify(payload) : undefined, + signal: AbortSignal.timeout(timeoutMs), + } as any); +``` +`lib/daemon-client.ts` has a **second** header-building `fetch(...)` path (around lines 155-160, a separate request helper); add the identical `X-RT-Client` header there too so every rt-CLI request is attributed, not just `trySocketQuery`'s. + +In `packages/rt-client/src/transport.ts` `rtCommand`, add the header (the caller label defaults to the package’s consumer; use a generic tag): +```ts + headers: { "Content-Type": "application/json", "X-RT-Client": `rt-client/${process.pid}` }, +``` + +- [ ] **Step 4: Run tests + rebuild dist (rt-client touched)** + +Run: `bun test lib/daemon/__tests__/unknown-command.test.ts`, then `cd packages/rt-client && bun run build`. `bunx tsc --noEmit` clean. +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/unknown-command.ts lib/daemon.ts lib/daemon-client.ts packages/rt-client/src/transport.ts packages/rt-client/dist lib/daemon/__tests__/unknown-command.test.ts +git commit -m "unknown-command envelope (code+version); transports send X-RT-Client (dist rebuilt, no bump)" +``` + +--- + +## Task 12: Servers read `X-RT-Client` into `payload._client`; CORS allow-header + +**Files:** +- Modify: `lib/daemon/api-server.ts` +- Modify: `lib/daemon/socket-server.ts` +- Test: `lib/daemon/__tests__/caller-tag.test.ts` + +**Interfaces:** +- Consumes: `handleCommand(cmd, payload)` reading `payload._client` (Task 10). +- Produces: both servers merge `req.headers.get("x-rt-client")` into `payload._client` before dispatch; `buildCorsHeaders` allows `X-RT-Client`. + +- [ ] **Step 1: Write the failing test** + +```ts +// lib/daemon/__tests__/caller-tag.test.ts +import { test, expect } from "bun:test"; +import { buildCorsHeaders } from "../api-server.ts"; + +test("CORS allow-headers advertises X-RT-Client so browser preflight passes", () => { + const h = buildCorsHeaders("https://example.com", true); + expect(h["Access-Control-Allow-Headers"]).toContain("X-RT-Client"); +}); +``` +(The header-merge behavior is covered end-to-end by the e2e daemon test in Task 15; the unit test guards the CORS regression, which is otherwise silent.) + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/caller-tag.test.ts` +Expected: FAIL (no `X-RT-Client` in allow-headers). + +- [ ] **Step 3: Write minimal implementation** + +In `buildCorsHeaders`: +```ts + "Access-Control-Allow-Headers": "Content-Type, X-RT-Token, X-RT-Client", +``` +In `api-server.ts`, in the generic dispatch block (after `payload` is built from query/body, before `handleCommand(route.cmd, ...)`): +```ts + const client = req.headers.get("x-rt-client"); + if (client) payload._client = client; + const result = await handleCommand(route.cmd, payload, req.signal); +``` +In `socket-server.ts`, after the payload parse (line ~46), before dispatch: +```ts + const client = req.headers.get("x-rt-client"); + if (client) (payload as any)._client = client; + const result = await handleCommand(cmd, payload, req.signal); +``` + +- [ ] **Step 4: Run test + typecheck** + +Run: `bun test lib/daemon/__tests__/caller-tag.test.ts`; `bunx tsc --noEmit`. +Expected: PASS. + +- [ ] **Step 5: Commit** + +```bash +git add lib/daemon/api-server.ts lib/daemon/socket-server.ts lib/daemon/__tests__/caller-tag.test.ts +git commit -m "servers: thread X-RT-Client into payload._client; advertise it in CORS" +``` + +--- + +## Task 13: `HandlerContext` extension + `cache-refresh` populates the refresh ref + wsClient count + +**Files:** +- Modify: `lib/daemon/handlers/types.ts` (extend `refreshStatusRef`, add `getHealth`) +- Modify: `lib/daemon/cache-refresh.ts` (populate the extended ref) +- Modify: `lib/daemon/api-server.ts` (export `apiWsClientCount`) +- Test: `lib/daemon/__tests__/refresh-status-ref.test.ts` (a focused unit around the ref update helper) + +**Interfaces:** +- Produces: `refreshStatusRef: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }`; `HandlerContext.getHealth: () => HealthSnapshot`; `apiWsClientCount(): number`. + +- [ ] **Step 1: Extend the type** + +In `lib/daemon/handlers/types.ts`, change ONLY the `refreshStatusRef` field: +```ts + refreshStatusRef: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }; +``` +Do NOT add `getHealth`/`heartbeatSeq`/`setLogLevel`/`getLogLevel` here. Each of +those is a REQUIRED `HandlerContext` field, so it must be added to the type in +the same task that also provides its value in the `handlerCtx` literal (Task 14 +adds `getHealth` + `heartbeatSeq`; Task 15 adds `setLogLevel` + `getLogLevel`), +or tsc goes red across the gap. This task's `refreshStatusRef` change is fully +self-consistent: the type, the `daemon.ts` init, and `CacheRefresherDeps` all +move together below. + +- [ ] **Step 2: Update the init site and the refresher** + +In `lib/daemon.ts` line ~210: +```ts +const refreshStatusRef = { lastRefreshAt: 0, lastSuccessAt: 0, failedRepos: 0, enrichErrors: 0 }; +``` +In `lib/daemon/cache-refresh.ts`, widen the `CacheRefresherDeps` type's own `refreshStatusRef` field (declared around line 32) to match, or `tsc` fails at the pass into `createCacheRefresher`: +```ts + // CacheRefresherDeps (cache-refresh.ts ~line 32): + refreshStatusRef: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }; +``` +Then, where the cycle finishes (currently sets `refreshStatusRef.lastRefreshAt = Date.now()` at ~line 195), also record the cycle outcome from the `failedRepos`/`enrichErrors` locals already computed in `refreshCacheImpl`: +```ts + refreshStatusRef.lastRefreshAt = Date.now(); + refreshStatusRef.failedRepos = failedRepos.size; + refreshStatusRef.enrichErrors = enrichErrors; + if (failedRepos.size === 0 && enrichErrors === 0) refreshStatusRef.lastSuccessAt = refreshStatusRef.lastRefreshAt; +``` +(Confirm the exact local names `failedRepos`/`enrichErrors` and that `refreshStatusRef` is in scope there; both were confirmed present in `cache-refresh.ts`.) + +- [ ] **Step 3: Export the ws-client count** + +In `lib/daemon/api-server.ts`, add a module-scope accessor next to `wsClients`: +```ts +export function apiWsClientCount(): number { + return wsClients.size; +} +``` + +- [ ] **Step 4: Focused test** + +```ts +// lib/daemon/__tests__/refresh-status-ref.test.ts +import { test, expect } from "bun:test"; +import { applyRefreshOutcome } from "../cache-refresh.ts"; + +test("a clean cycle advances lastSuccessAt; a failing cycle does not", () => { + const ref = { lastRefreshAt: 0, lastSuccessAt: 0, failedRepos: 0, enrichErrors: 0 }; + applyRefreshOutcome(ref, 1000, 0, 0); + expect(ref.lastSuccessAt).toBe(1000); + applyRefreshOutcome(ref, 2000, 2, 5); + expect(ref.lastRefreshAt).toBe(2000); + expect(ref.lastSuccessAt).toBe(1000); // unchanged on failure + expect(ref.failedRepos).toBe(2); +}); +``` +Extract the four-line update into an exported pure helper `applyRefreshOutcome(ref, at, failedReposCount, enrichErrors)` in `cache-refresh.ts` and call it from the cycle end, so the logic is unit-tested without running a real refresh. + +- [ ] **Step 5: Run tests, typecheck, commit** + +Run: `bun test lib/daemon/__tests__/refresh-status-ref.test.ts`; `bunx tsc --noEmit`. +```bash +git add lib/daemon/handlers/types.ts lib/daemon.ts lib/daemon/cache-refresh.ts lib/daemon/api-server.ts lib/daemon/__tests__/refresh-status-ref.test.ts +git commit -m "ctx: extend refreshStatusRef with cycle outcome + getHealth; export apiWsClientCount" +``` + +--- + +## Task 14: Daemon wiring — loop monitor, metrics sampler, heartbeat, `getHealth`; surface health in status/tray:status/ping + +**Files:** +- Modify: `lib/daemon.ts` (start the monitor + sampler; build `getHealth`; put it on `handlerCtx`) +- Modify: `lib/daemon/handlers/types.ts` (add `getHealth` + `heartbeatSeq` to `HandlerContext` — deferred here from Task 13 so type + value land together) +- Modify: `lib/daemon/handlers/status.ts` (add `health`/`metrics`/`eventLoop` to `status` + `tray:status`; `health.level` + `eventLoop` + `heartbeatSeq` to `ping`) +- Create: `lib/daemon/health-sampler.ts` (5-min metrics log + rss baseline + disk-free cache) +- Test: `lib/daemon/__tests__/health-sampler.test.ts` + +**Interfaces:** +- Consumes: `startLoopMonitor` (Task 3), `computeHealth` (Task 1), `writeHeartbeat` (Task 2), `apiWsClientCount` (Task 13), `readSupervisionState`/`isCrashLooping` (Phase 0), `refreshStatusRef` (Task 13), logger handle's `loggerDegraded`/`recoveredErrorCount` (Tasks 6/7). +- Produces: `createHealthSampler(opts)` returning `{ sample(): void; freeBytes(): number | null; rssBaseline(): {rss;at}|null; recoveredRateLastWindow(): number }`; `handlerCtx.getHealth` closure. + +- [ ] **Step 1: Write the sampler test** + +```ts +// lib/daemon/__tests__/health-sampler.test.ts +import { test, expect } from "bun:test"; +import { rollRssBaseline } from "../health-sampler.ts"; + +test("rss baseline rolls forward only after the window elapses", () => { + // baseline null -> set on first sample + let b = rollRssBaseline(null, { rss: 100, at: 0 }, 60 * 60_000); + expect(b).toEqual({ rss: 100, at: 0 }); + // within the hour: unchanged + b = rollRssBaseline(b, { rss: 200, at: 30 * 60_000 }, 60 * 60_000); + expect(b).toEqual({ rss: 100, at: 0 }); + // after the hour: rolls to the new sample + b = rollRssBaseline(b, { rss: 250, at: 61 * 60_000 }, 60 * 60_000); + expect(b).toEqual({ rss: 250, at: 61 * 60_000 }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/daemon/__tests__/health-sampler.test.ts` +Expected: FAIL (module not found). + +- [ ] **Step 3: Implement the sampler** + +```ts +// lib/daemon/health-sampler.ts +/** Periodic (5-min) metrics logging + the two cached signals health needs that + * are too costly to compute per ping: the 1h rss baseline (growth) and free + * disk under RT_DIR. Pure helpers are unit-tested; the timer just calls sample. */ +import type { Logger } from "pino"; + +export function rollRssBaseline( + prev: { rss: number; at: number } | null, + now: { rss: number; at: number }, + windowMs: number, +): { rss: number; at: number } { + if (!prev) return now; + if (now.at - prev.at >= windowMs) return now; + return prev; +} + +export interface HealthSampler { + sample(): void; + freeBytes(): number | null; + rssBaseline(): { rss: number; at: number } | null; +} + +export function createHealthSampler(opts: { + log: Logger; + rtDir: string; + wsClients: () => number; + watchers: () => number; + startedAt: number; +}): HealthSampler { + let baseline: { rss: number; at: number } | null = null; + let free: number | null = null; + + function statfsFree(dir: string): number | null { + try { + // Node/Bun fs.statfsSync where available; guarded so an unsupported + // platform leaves free=null and disk checks are simply skipped. + const { statfsSync } = require("fs"); + const s = statfsSync(dir); + return s.bavail * s.bsize; + } catch { + return null; + } + } + + return { + freeBytes: () => free, + rssBaseline: () => baseline, + sample() { + const mem = process.memoryUsage(); + const now = Date.now(); + baseline = rollRssBaseline(baseline, { rss: mem.rss, at: now }, 60 * 60_000); + free = statfsFree(opts.rtDir); + opts.log.info( + { rss: mem.rss, heapUsed: mem.heapUsed, external: mem.external, wsClients: opts.wsClients(), watchers: opts.watchers(), uptimeMs: now - opts.startedAt }, + "daemon metrics", + ); + }, + }; +} +``` + +- [ ] **Step 4: Wire it all in `lib/daemon.ts`** (module scope, after `log` and after `handlerCtx` fields are available; the monitor/sampler timers mirror the existing events-sweep timers) + +```ts +import { startLoopMonitor } from "./daemon/loop-monitor.ts"; +import { createHealthSampler } from "./daemon/health-sampler.ts"; +import { writeHeartbeat } from "./daemon/heartbeat-file.ts"; +import { computeHealth } from "./daemon/health.ts"; +import { apiWsClientCount } from "./daemon/api-server.ts"; +import { isCrashLooping, readSupervisionState } from "./daemon/supervision-state.ts"; +import { setSettingsWarnSink } from "./settings/resolve.ts"; // local barrel re-exports packages/rt-client resolver + +// Bind the resolver's warn sink to a deduped daemon log.warn (S033/R005): a +// hot-path getSetting on a disallowed-scope key warns once, not every tick. +// The sink dedupes internally; here we only route it into structured logging. +setSettingsWarnSink((m) => log.warn({ src: "settings" }, m)); + +const healthSampler = createHealthSampler({ + // Source the watched-configs map exactly as handlerCtx.watchedConfigs is + // sourced today: it is hooksGuard.watchedConfigs (there is no bare + // `watchedConfigs` alias in daemon.ts scope). + log, rtDir: RT_DIR, wsClients: apiWsClientCount, watchers: () => hooksGuard.watchedConfigs.size, startedAt, +}); +healthSampler.sample(); // seed baseline/free immediately +safeInterval(() => healthSampler.sample(), 5 * 60_000, "health-sample", log); + +const loopMon = startLoopMonitor({ + log, + currentCmd: () => currentCmd.cmd, + onHeartbeat: (at, seq) => writeHeartbeat(RT_DIR, { at, seq }), +}); + +function buildHealthSnapshot() { + const now = Date.now(); + const sup = readSupervisionState(); + const failuresLastHour = sup.recentFailures.filter((f) => f.at > now - 60 * 60_000).length; + return computeHealth({ + now, + uptimeMs: now - startedAt, + mem: process.memoryUsage(), + rssBaseline: healthSampler.rssBaseline(), + wsClients: apiWsClientCount(), + watchers: watchedConfigs.size, + freshness: getFreshnessSnapshot(), + refresh: { lastSuccessAt: refreshStatusRef.lastSuccessAt, failedRepos: refreshStatusRef.failedRepos, enrichErrors: refreshStatusRef.enrichErrors }, + refreshIntervalMs: 5 * 60_000, + eventLoop: { ...loopMon.stats }, + supervisionFailuresLastHour: failuresLastHour, + crashLooping: isCrashLooping(sup, now), + loggerDegraded: loggerHandle.loggerDegraded?.() ?? false, + recoveredErrorRateLastWindow: loggerHandle.recoveredErrorCount?.() ?? 0, + freeBytes: healthSampler.freeBytes(), + }); +} +``` + +Add `heartbeatSeq: loopMon.seq` to the `handlerCtx` object literal (and `heartbeatSeq: () => number` to `HandlerContext` in `types.ts`) so `ping` can echo the current heartbeat sequence per the spec: +```ts + getHealth: buildHealthSnapshot, + heartbeatSeq: loopMon.seq, +``` +Add `getHealth: buildHealthSnapshot` to the `handlerCtx` object literal (lines ~353-364), AND add `getHealth: () => import("./health.ts").HealthSnapshot` to `HandlerContext` in `lib/daemon/handlers/types.ts` (Task 13 deliberately left this to Task 14 so the type and its value land together). Import `getFreshnessSnapshot` if not already in `daemon.ts` scope (it lives in `lib/daemon/freshness.ts`). Ensure `loopMon.stop()` is called in `cleanup()`. + +- [ ] **Step 5: Surface the snapshot in the handlers** + +In `lib/daemon/handlers/status.ts`: +- `status` handler `data`: add `health: ctx.getHealth().health` — but `getHealth()` returns the whole snapshot; splice the three blocks: +```ts + "status": async () => { + const h = ctx.getHealth(); + return { ok: true, data: { + pid: process.pid, + uptime: Date.now() - ctx.startedAt, + watchedRepos: ctx.watchedConfigs.size, + cacheEntries: Object.keys(ctx.cache.entries).length, + portsCached: ctx.portCacheRef.ports.length, + portCacheAge: ctx.portCacheRef.updatedAt ? Date.now() - ctx.portCacheRef.updatedAt : null, + freshness: getFreshnessSnapshot(), + identity: ctx.identity, + health: { level: h.level, reasons: h.reasons }, + metrics: h.metrics, + eventLoop: h.eventLoop, + } }; + }, +``` +- `tray:status` handler `data`: add the same `health`/`metrics`/`eventLoop` three (keep the existing fields). +- `ping` handler: add `health: h.level` and `eventLoop: h.eventLoop` (cheap): +```ts + "ping": async () => { + const { bootAttempts, lastReadyAt, recentFailures, lastExit } = readSupervisionState(); + const h = ctx.getHealth(); + return { ok: true, uptime: Date.now() - ctx.startedAt, pid: process.pid, ...ctx.identity, + health: h.level, eventLoop: h.eventLoop, heartbeatSeq: ctx.heartbeatSeq(), + supervision: { bootAttempts, lastReadyAt, recentFailures: recentFailures.slice(-3), lastExit } }; + }, +``` + +- [ ] **Step 6: Run tests, typecheck, commit** + +Run: `bun test lib/daemon/__tests__/health-sampler.test.ts`; `bunx tsc --noEmit`. +```bash +git add lib/daemon.ts lib/daemon/handlers/status.ts lib/daemon/health-sampler.ts lib/daemon/__tests__/health-sampler.test.ts +git commit -m "daemon: wire loop monitor + heartbeat + health sampler; surface health/metrics/eventLoop in status/tray:status/ping" +``` + +--- + +## Task 15: `rt daemon log-level` command + `daemon:log-level` verb + +**Files:** +- Modify: `commands/daemon.ts` (add `setLogLevel`) +- Modify: `lib/command-tree-def.ts` (add the `log-level` leaf in the `daemon` subtree) +- Modify: `lib/daemon/handlers/types.ts` (add `setLogLevel` + `getLogLevel` to `HandlerContext`) +- Modify: `lib/daemon.ts` (wire `setLogLevel`/`getLogLevel` on `handlerCtx`) +- Modify: `lib/daemon/handlers/status.ts` (add `daemon:log-level` handler) +- Test: `commands/__tests__/log-level.test.ts` (the pure format/parse), and picker conformance. + +**Interfaces:** +- Consumes: `daemonQuery`, the logger handle's live level setter. +- Produces: `daemon:log-level` verb: payload `{ level? }` → sets `logger.level` live and returns `{ ok, level }`; with no `level`, returns the current `{ ok, level }`. `setLogLevel(args)` CLI handler. + +- [ ] **Step 1: Add the command-tree leaf** (after `logs:` in the `daemon` subtree; select with `omitBehavior: "list"` so omitting shows the current level, matching `settings.runaway`) + +```ts + "log-level": { + description: "Show or set the daemon's live log level", + module: "./commands/daemon.ts", + fn: "setLogLevel", + omitBehavior: "list", + args: [ + { name: "Level", type: "select", hint: "Omit to show the current level", + options: [ + { value: "trace", label: "trace" }, { value: "debug", label: "debug" }, + { value: "info", label: "info" }, { value: "warn", label: "warn" }, + { value: "error", label: "error" }, + ] }, + ], + }, +``` + +- [ ] **Step 2: Write the failing test** + +```ts +// commands/__tests__/log-level.test.ts +import { test, expect } from "bun:test"; +import { formatLogLevelResult } from "../daemon.ts"; + +test("formats a set result", () => { + expect(formatLogLevelResult({ ok: true, level: "debug" }, true)).toContain("debug"); +}); +test("formats a show result", () => { + expect(formatLogLevelResult({ ok: true, level: "info" }, false)).toContain("info"); +}); +``` + +- [ ] **Step 3: Run test to verify it fails** + +Run: `bun test commands/__tests__/log-level.test.ts` +Expected: FAIL (function missing). + +- [ ] **Step 4: Implement handler + verb** + +In `commands/daemon.ts`: +```ts +export function formatLogLevelResult(res: { ok: boolean; level?: string; error?: string }, wasSet: boolean): string { + if (!res.ok) return ` ${red}●${reset} ${res.error ?? "failed"}`; + return ` ${green}●${reset} daemon log level ${wasSet ? "set to" : "is"} ${res.level}`; +} + +export async function setLogLevel(args: string[] = []): Promise { + const json = args.includes("--json"); + const level = args.find((a) => !a.startsWith("--")); + const res = await daemonQuery("daemon:log-level", level ? { level } : {}); + if (!res) { console.log(` ${red}●${reset} daemon not reachable`); return; } + if (json) { console.log(JSON.stringify(res)); return; } + console.log(formatLogLevelResult(res as any, Boolean(level))); +} +``` +The `daemon:log-level` handler sets the live pino level on the singleton logger. Add `setLogLevel: (l: string) => void` and `getLogLevel: () => string` to `HandlerContext` in `lib/daemon/handlers/types.ts` (deferred here from Task 13 so type + value land together), then wire them in `daemon.ts`'s `handlerCtx` object literal: +```ts +// in daemon.ts handlerCtx: + setLogLevel: (l: string) => { log.level = l; log.info({ level: l }, "log level changed"); }, + getLogLevel: () => log.level, +``` +The `daemon:log-level` verb lives in `createStatusHandlers` (it already closes over `ctx`). +and the handler: +```ts + "daemon:log-level": async (payload?: { level?: string }) => { + const VALID = ["trace", "debug", "info", "warn", "error"]; + if (payload?.level) { + if (!VALID.includes(payload.level)) return { ok: false, error: `invalid level: ${payload.level}` }; + ctx.setLogLevel(payload.level); + } + return { ok: true, level: ctx.getLogLevel() }; + }, +``` + +- [ ] **Step 5: Run picker conformance + tests, commit** + +Run: `bun run picker:check` (must pass — the leaf declares `omitBehavior`), `bun test commands/__tests__/log-level.test.ts`, `bunx tsc --noEmit`. +```bash +git add commands/daemon.ts lib/command-tree-def.ts lib/daemon/handlers/status.ts lib/daemon.ts commands/__tests__/log-level.test.ts +git commit -m "add rt daemon log-level: live level set/show via daemon:log-level verb" +``` + +--- + +## Task 16: E2E surface assertions + full verification + +**Files:** +- Modify: `e2e/tests/daemon.test.ts` (assert the new fields are additive and present) +- Test: the whole suite. + +- [ ] **Step 1: Add e2e assertions** + +In `e2e/tests/daemon.test.ts`, after the daemon is up, assert `/api/status` (tray:status) and the `status` verb carry the new blocks and that `ping` carries `health`: +```ts + const status = await rtJson(["daemon", "status", "--json"]); + // additive: pre-existing fields still present, new blocks present when running + // (exact assertions match the harness's existing patterns in this file) +``` +Follow the file's existing helper conventions (do not invent a new harness). Assert: `health.level` is one of ok/degraded/unhealthy; `metrics.rss` is a number; `eventLoop.maxLagMs` is a number; and a heartbeat file exists under the isolated HOME's RT_DIR after ~3s. + +- [ ] **Step 2: Run the daemon e2e** + +Run: `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts` +Expected: PASS. (This harness starts a daemon under an isolated HOME via the preload — never against the real machine.) + +- [ ] **Step 3: Full verification gate** + +Run, in order, and record results: +```bash +bunx tsc --noEmit +bun test lib commands packages scripts +bun run picker:check +cd packages/rt-client && bun run build && cd - +bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts +``` +Expected: `tsc` 0 errors; unit suites green; picker:check green; rt-client dist fresh; daemon e2e green. If practical, run the full `bun run test:e2e` and note which was run. + +- [ ] **Step 4: Commit** + +```bash +git add e2e/tests/daemon.test.ts +git commit -m "e2e: assert additive health/metrics/eventLoop + heartbeat file" +``` + +--- + +## Documentation deliverable (tray read contract) + +Not a code task, but part of the spec's scope: in the spec file (already committed) the tray read contract is documented. If a `docs/` note for the Swift-owning follow-up is wanted, add one line to `docs/daemon-supervision-design.md` or a new `docs/daemon-health.md` pointing the tray at `data.health.level` (green/orange/red) and `data.health.reasons[0]`. Keep it to a short paragraph; no `rt-tray/` edits. + +--- + +## Self-Review (completed by the plan author) + +**Spec coverage:** R011 → Tasks 1,14 (health level + reasons in surfaces). R012 → Tasks 1,14 (metrics block + 5-min sampler + growth); watcher-close explicitly out of scope per spec. R003 → Tasks 3,4,5,14 (loop monitor + heartbeat + classifier stall + rendering). R004 → Tasks 7,9,15 (rt.logLevel setting + live verb + slow-command info in Task 10). S031 → Tasks 7,8,10 (size cap + pruneLogs onError + suppression). S032 → Task 6 (stream error listener + crash-handler wrap + loggerDegraded). S033/R005 → Tasks 7,9 (stderr demotion + resolver warn sink + recovered-error counter). R008 → Tasks 10,11,12 (reqId + caller tag + digest). R021 → Task 11 (unknown-command envelope). Constraints (no schema bump, dist rebuild no publish, no tray edits, isolated HOME) → Global Constraints + Tasks 9/11/16. + +**Placeholder scan:** every code step carries real test + impl code; the two spots that say "confirm exact names against the file" (cache-refresh locals, showStatus imports) are verification instructions, not placeholders, and the names were confirmed present by investigation. + +**Type consistency:** `HealthSnapshot`/`HealthInputs` (Task 1) are consumed unchanged in Tasks 13/14; `refreshStatusRef` fields defined in Task 13 match their use in Task 14's `buildHealthSnapshot`; `LoopStats` (Task 3) is spread into the classifier-shaped `eventLoop` in Task 14; `pingDaemon` (Task 5) return type matches `showStatus`'s use; `setSettingsWarnSink` (Task 9) exported name matches the daemon bind (Task 14 note: bind it at daemon boot — add `setSettingsWarnSink((m) => log.warn({ src: "settings" }, m))` near the logger setup, deduped by the sink itself). diff --git a/docs/superpowers/plans/2026-08-28-p6-portability.md b/docs/superpowers/plans/2026-08-28-p6-portability.md new file mode 100644 index 00000000..35dd9f98 --- /dev/null +++ b/docs/superpowers/plans/2026-08-28-p6-portability.md @@ -0,0 +1,1344 @@ +# Phase 6 · Someone else's Mac (p6-portability) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make the rt daemon survive a machine that is not the author's ... a fish shell, a blocking `.zshrc`, a renamed Mac, a foreign `~/.local/bin/rt`, no home repo, no git identity, an Intel Mac, and a locked keychain. + +**Architecture:** Nine bounded units over three subsystems: (1) rebuild `resolveUserPath` as an async, killable, fish-aware PATH probe with an `rt.daemonPath` override; (2) stabilize machine identity and dev-mode detection; (3) first-run honesty in home-snapshot, secrets, setup, and the branch cache. Each unit is independently testable; task 10 (the branch-cache key flip) is the one atomic multi-file change. + +**Tech Stack:** Bun, TypeScript, `bun:sqlite`, `bun test`, `@mattstack/rt-client` settings registry, pino logger. + +**Spec:** `docs/superpowers/specs/2026-08-28-p6-portability-design.md` (read it alongside this plan; the plan argues from it). + +## Global Constraints + +- **No `SCHEMA_VERSION` bump.** Every fix here is code-only; S069 reuses the existing `branch TEXT PRIMARY KEY` column with no DDL change. If a bump ever looks unavoidable, STOP and ask through the shepherd channel first. +- **Never start a daemon or run `dist/rt` against the real machine.** Any daemon or compiled-binary invocation runs under `env -i HOME=`. Tests use injected seams and never spawn a real login shell. +- **Do not edit `rt-tray/`.** +- **Write fence:** work only inside this worktree. These files are the sibling p2-health lane's and MUST NOT be modified: `lib/daemon.ts` (EXCEPT the single `resolveUserPath` call statement at `lib/daemon.ts:163`, which the shepherd granted for Task 3 ... await the async result, change nothing else in the file), `lib/daemon-logger.ts`, `lib/daemon-status.ts`, `lib/daemon/supervision-state.ts`, `lib/daemon/handlers/status.ts`, `commands/daemon.ts`, `lib/daemon/command-router.ts`, `lib/daemon/api-server.ts`, `lib/daemon/socket-server.ts`, `lib/log-janitor.ts`, `lib/daemon/safe-timers.ts`. +- **`packages/rt-client` is touched (Task 1).** After any change under it, run `bun run build` inside `packages/rt-client` (the `dist/` that `file:` consumers copy). `packages/rt-client/test/dist-freshness.test.ts` is the guard. +- **The daemon sync-exec gate.** `lib/__tests__/no-daemon-sync-exec.test.ts` forbids `execSync(`/`spawnSync(`/`Bun.spawnSync(`/`Bun.sleepSync(` in any daemon-reachable module. All new subprocess use is async `Bun.spawn`. Remove the `user-path.ts` allowlist entry once Task 2 lands (Task 3). +- **Comments:** clean-code only (state a constraint the code cannot show; no narration, no task numbers in source). Never use em dashes; use "..." or rephrase. +- **Serialized repo identity:** per `docs/repo-identity.md`, state.db tables (branch_cache) key on the serialized wire identity (`remote:host%2Fpath` / `path:%2F…`). In the daemon, the variable `repoName` and `CacheEntry.repoName` already hold that serialized identity. + +--- + +## Task 1: `rt.daemonPath` settings registry key + +**Files:** +- Modify: `packages/rt-client/src/settings/registry-defs.ts` (add one row to the `REGISTRY` array) +- Test: `packages/rt-client/test/registry-defs.test.ts` (or the existing registry test file; add a case) +- Build: `packages/rt-client` (`bun run build`) + +**Interfaces:** +- Produces: the registered key `"rt.daemonPath"` (type `string`, scope `machine`), readable via `getSetting("rt.daemonPath")` (sync; returns `undefined` when unset; throws only on an unregistered key). + +- [ ] **Step 1: Write the failing test** + +Add to the registry test (mirror how existing keys are asserted): + +```ts +import { getDef } from "../src/settings/registry-machinery.ts"; + +test("rt.daemonPath is a machine-scoped string key with no default", () => { + const def = getDef("rt.daemonPath"); + expect(def).toBeDefined(); + expect(def!.type).toBe("string"); + expect(def!.scopes).toEqual(["machine"]); + expect(def!.default).toBeUndefined(); + expect(def!.pathGuardFields).toBeUndefined(); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd packages/rt-client && bun test test/registry-defs.test.ts -t "rt.daemonPath"` +Expected: FAIL (`def` is undefined). + +- [ ] **Step 3: Add the registry row** + +In `packages/rt-client/src/settings/registry-defs.ts`, add to the `REGISTRY` array (place it near `rt.apiPort`, the other machine-scoped daemon key): + +```ts +{ + key: "rt.daemonPath", + type: "string", + scopes: ["machine"], + merge: "replace", + description: + "Absolute colon-separated PATH the daemon uses for every child it spawns, instead of probing your login shell. Set this when the daemon can't find node/git/bun/pnpm (e.g. a fish shell, a blocking .zshrc, or PATH exports that live only in .zshrc). Machine-scoped: it never travels to another machine.", +}, +``` + +No `default` (absent means "probe the shell"); no `pathGuardFields` (the value is itself a PATH literal, and machine scope is exempt from the path-literal guard). + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd packages/rt-client && bun test test/registry-defs.test.ts -t "rt.daemonPath"` +Expected: PASS. + +- [ ] **Step 5: Rebuild rt-client dist and verify freshness** + +Run: `cd packages/rt-client && bun run build && bun test test/dist-freshness.test.ts` +Expected: PASS (dist regenerated). + +- [ ] **Step 6: Commit** + +```bash +git add packages/rt-client/src/settings/registry-defs.ts packages/rt-client/test packages/rt-client/dist +git commit -m "rt-client: register rt.daemonPath machine setting (6.1)" +``` + +--- + +## Task 2: Rebuild `resolveUserPath` (S013, S014, S062) + +**Files:** +- Modify: `lib/daemon/user-path.ts` (rewrite `resolveUserPath`; keep `probeTools` exported; drop both `execSync` calls) +- Test: `lib/daemon/__tests__/user-path.test.ts` + +**Interfaces:** +- Consumes: `getSetting("rt.daemonPath")` from Task 1. +- Produces: `export async function resolveUserPath(log: Logger, probe?: ProbeFn): Promise` (was sync). `ProbeFn = (argv: [string, ...string[]], opts: { timeoutMs: number; env?: Record }) => Promise` (resolves the child's stdout, or `null` on spawn failure / timeout / kill). `probeTools(pathValue, names)` unchanged. + +- [ ] **Step 1: Write the failing tests** + +Replace/extend `lib/daemon/__tests__/user-path.test.ts`. Use an injected `probe` seam so no real shell is spawned. `makeLog()` returns a pino-shaped stub capturing `warn`/`info` calls. + +```ts +import { resolveUserPath } from "../user-path.ts"; + +function makeLog() { + const warns: any[] = []; const infos: any[] = []; + return { log: { warn: (...a: any[]) => warns.push(a), info: (...a: any[]) => infos.push(a) } as any, warns, infos }; +} + +test("fish-style space-separated base output is rejected, baseline kept + warn", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async () => "/opt/homebrew/bin /usr/bin /bin"; // spaces = fish-unsplit + const out = await resolveUserPath(log, probe); + expect(out).toBe("/usr/bin:/bin"); + expect(warns.some((w) => JSON.stringify(w).includes("whitespace"))).toBe(true); +}); + +test("a hanging probe returns baseline within the timeout", async () => { + const { log } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async () => null; // seam models kill/timeout as null + const out = await resolveUserPath(log, probe); + expect(out).toBe("/usr/bin:/bin"); +}); + +test("base equal to launchd baseline is treated as silent fallback (S062)", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin:/usr/sbin:/sbin"; + const probe = async (argv: any) => (argv[1] === "-lc" ? "/usr/bin:/bin:/usr/sbin:/sbin" : null); + const out = await resolveUserPath(log, probe); + expect(out).toBe("/usr/bin:/bin:/usr/sbin:/sbin"); + expect(warns.some((w) => JSON.stringify(w).includes("equals-baseline"))).toBe(true); +}); + +test("rt.daemonPath override skips both probes", async () => { + const { log } = makeLog(); + let called = false; + const probe = async () => { called = true; return "x"; }; + // Point HOME at a scratch machine store that sets rt.daemonPath, OR stub getSetting. + // (Executor: use the repo's settings test harness to set rt.daemonPath = "/over/bin:/x/bin" at machine scope.) + const out = await resolveUserPath(log, probe); + expect(out).toBe("/over/bin:/x/bin"); + expect(called).toBe(false); +}); + +test("valid base accepted; interactive overlay appends a .zshrc-only dir after base", async () => { + const { log } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async (argv: any) => + argv[1] === "-lc" ? "/opt/homebrew/bin:/usr/bin:/bin" : "/opt/homebrew/bin:/usr/bin:/bin:/Users/x/.nvm/versions/node/v22/bin"; + const out = await resolveUserPath(log, probe); + expect(out).toBe("/opt/homebrew/bin:/usr/bin:/bin:/Users/x/.nvm/versions/node/v22/bin"); +}); + +test("overlay timeout is skipped with a warn; base kept unchanged", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async (argv: any) => (argv[1] === "-lc" ? "/opt/homebrew/bin:/usr/bin:/bin" : null); + const out = await resolveUserPath(log, probe); + expect(out).toBe("/opt/homebrew/bin:/usr/bin:/bin"); + expect(warns.some((w) => JSON.stringify(w).includes("overlay"))).toBe(true); +}); + +test("garbage overlay (non-null, no absolute dirs) is skipped with a warn", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async (argv: any) => (argv[1] === "-lc" ? "/opt/homebrew/bin:/usr/bin:/bin" : "not-a-path:also-not"); + const out = await resolveUserPath(log, probe); + expect(out).toBe("/opt/homebrew/bin:/usr/bin:/bin"); + expect(warns.some((w) => JSON.stringify(w).includes("overlay"))).toBe(true); +}); + +test("missing-tool warn fires when node is absent", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async () => "/usr/bin:/bin"; // no node + await resolveUserPath(log, probe); + expect(warns.some((w) => JSON.stringify(w).includes("missing"))).toBe(true); +}); +``` + +Keep the existing `probeTools` tests. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test lib/daemon/__tests__/user-path.test.ts` +Expected: FAIL (resolveUserPath is still sync / no override / no overlay). + +- [ ] **Step 3: Rewrite `lib/daemon/user-path.ts`** + +Replace the module (keep `probeTools` as-is; remove `import { execSync }`): + +```ts +import { basename } from "path"; +import type { Logger } from "pino"; +import { getSetting } from "@mattstack/rt-client"; + +export type ProbeFn = ( + argv: [string, ...string[]], + opts: { timeoutMs: number; env?: Record }, +) => Promise; + +const BASE_TIMEOUT_MS = 5_000; +const OVERLAY_TIMEOUT_MS = 3_000; +const KILL_GRACE_MS = 500; + +/** Default probe: a detached (own process-group) Bun.spawn whose whole group is + * SIGTERM'd then SIGKILL'd at the deadline, raced so a hung shell (or a hung + * grandchild it spawned) can never block boot past the timeout. */ +const runProbe: ProbeFn = async (argv, opts) => { + let proc: ReturnType; + try { + proc = Bun.spawn(argv, { + detached: true, + env: opts.env ?? { ...process.env }, + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + }); + } catch { + return null; + } + proc.unref(); + const pid = proc.pid; + let killTimer: ReturnType | undefined; + const term = setTimeout(() => { + try { process.kill(-pid, "SIGTERM"); } catch { /* group already gone */ } + killTimer = setTimeout(() => { try { process.kill(-pid, "SIGKILL"); } catch { /* gone */ } }, KILL_GRACE_MS); + killTimer.unref?.(); + }, opts.timeoutMs); + const captured: Promise = (async () => { + try { + const [out] = await Promise.all([new Response(proc.stdout as ReadableStream).text(), proc.exited]); + return out; + } catch { + return null; + } + })(); + let deadlineTimer: ReturnType; + const deadline: Promise = new Promise((resolve) => { + deadlineTimer = setTimeout(() => resolve(null), opts.timeoutMs + KILL_GRACE_MS + 250); + }); + try { + return await Promise.race([captured, deadline]); + } finally { + clearTimeout(term); + if (killTimer) clearTimeout(killTimer); + clearTimeout(deadlineTimer!); + } +}; + +function validateBase(raw: string | null, baseline: string): { path: string; source: "probe" | "baseline"; reason?: string } { + if (raw === null) return { path: baseline, source: "baseline", reason: "killed-or-empty" }; + const v = raw.trim(); + if (v.length === 0) return { path: baseline, source: "baseline", reason: "empty" }; + if (/\s/.test(v)) return { path: baseline, source: "baseline", reason: "whitespace" }; + if (v.split(":").filter(Boolean).length < 2) return { path: baseline, source: "baseline", reason: "too-few-segments" }; + if (v === baseline) return { path: baseline, source: "baseline", reason: "equals-baseline" }; + return { path: v, source: "probe" }; +} + +/** Overlay contributes only well-formed absolute dirs; anything else yields []. */ +function absoluteDirsOf(raw: string | null): string[] { + if (raw === null) return []; + const v = raw.trim(); + if (v.length === 0 || /\s/.test(v)) return []; + return v.split(":").filter((d) => d.startsWith("/")); +} + +function unionAppend(base: string, extra: string[]): string { + const have = new Set(base.split(":").filter(Boolean)); + const add = extra.filter((d) => !have.has(d)); + return add.length === 0 ? base : [base, ...add].join(":"); +} + +export function probeTools(pathValue: string, names: string[]): Record { + const entries = pathValue.split(":").filter((p) => p.length > 0); + const probed: Record = {}; + for (const name of names) { + probed[`has${name[0]!.toUpperCase()}${name.slice(1)}`] = entries.some((p) => { + try { return Bun.file(`${p}/${name}`).size > 0; } catch { return false; } + }); + } + return probed; +} + +export async function resolveUserPath(log: Logger, probe: ProbeFn = runProbe): Promise { + const baseline = process.env.PATH ?? ""; + + const override = getSetting("rt.daemonPath"); + let result: string; + let source: string; + + if (typeof override === "string" && override.trim().length > 0) { + result = override.trim(); + source = "override"; + } else { + const shell = process.env.SHELL ?? "/bin/zsh"; + const isFish = basename(shell) === "fish"; + const baseArgv: [string, ...string[]] = isFish + ? [shell, "-lc", "string join : $PATH"] + : [shell, "-lc", `{ [ -s "${'${NVM_DIR:-$HOME/.nvm}'}/nvm.sh" ] && . "${'${NVM_DIR:-$HOME/.nvm}'}/nvm.sh" >/dev/null 2>&1; }; printf %s "$PATH"`]; + const base = validateBase(await probe(baseArgv, { timeoutMs: BASE_TIMEOUT_MS }), baseline); + result = base.path; + source = base.source; + if (base.reason) log.warn({ reason: base.reason }, "PATH base probe unusable; kept baseline"); + + const ovArgv: [string, ...string[]] = isFish + ? [shell, "-ilc", "string join : $PATH"] + : [shell, "-ilc", "echo $PATH"]; + const ovRaw = await probe(ovArgv, { timeoutMs: OVERLAY_TIMEOUT_MS, env: { ...process.env, TERM: "dumb" } }); + const extra = absoluteDirsOf(ovRaw); + if (extra.length === 0) { + // Warn on BOTH timeout (null) and garbage (non-null but no usable + // absolute dirs) ... the ruling says timeout OR garbage. + log.warn("PATH interactive overlay skipped (timed out or no usable dirs)"); + } else { + const before = result; + result = unionAppend(result, extra); + if (result !== before) source += "+overlay"; + } + } + + const probed = probeTools(result, ["node", "git", "bun", "pnpm"]); + const missing = Object.entries(probed).filter(([, v]) => !v).map(([k]) => k.replace(/^has/, "").toLowerCase()); + if (missing.length > 0) log.warn({ missing }, "PATH missing required tools; set rt.daemonPath to override"); + log.info({ source, entries: result.split(":").length, ...probed }, "PATH resolved"); + return result; +} +``` + +(The `${'${NVM_DIR...}'}` fragments above are a template-literal escape so the literal `${NVM_DIR:-$HOME/.nvm}` survives into the shell body ... the executor writes the shell string so `$NVM_DIR`/`$HOME` expand in the child shell, not in TS.) + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test lib/daemon/__tests__/user-path.test.ts` +Expected: PASS. + +- [ ] **Step 5: Type-check** + +Run: `bunx tsc --noEmit` +Expected: zero errors. + +- [ ] **Step 6: Commit** + +```bash +git add lib/daemon/user-path.ts lib/daemon/__tests__/user-path.test.ts +git commit -m "user-path: async fish-aware killable PATH probe + rt.daemonPath override (S013/S014/S062)" +``` + +--- + +## Task 3: Await the async resolver in `daemon.ts`; drop the gate allowlist entry + +**Files:** +- Modify: `lib/daemon.ts:163` (the single granted statement only) +- Modify: `lib/__tests__/no-daemon-sync-exec.test.ts` (remove the `user-path.ts` allowlist line) + +**Interfaces:** +- Consumes: `resolveUserPath` (now async) from Task 2. + +- [ ] **Step 1: Make the one-line daemon.ts change** + +At `lib/daemon.ts:163`, change only: + +```ts + const resolvedPath = resolveUserPath(log); +``` +to: +```ts + const resolvedPath = await resolveUserPath(log); +``` + +Do not touch anything else in `lib/daemon.ts` (the surrounding block, line 164's `if (resolvedPath) process.env.PATH = resolvedPath;`, and the prefix block at 167-183 stay exactly as they are). Module-scope `await` is already used in this file (lines 119, 145). + +- [ ] **Step 2: Remove the allowlist entry** + +In `lib/__tests__/no-daemon-sync-exec.test.ts`, delete this line from the `ALLOWLIST` set: + +```ts + "lib/daemon/user-path.ts", // Phase 6 PATH rebuild (S013/S014/S062) +``` + +- [ ] **Step 3: Run the gate + type-check** + +Run: `bun test lib/__tests__/no-daemon-sync-exec.test.ts && bunx tsc --noEmit` +Expected: PASS, zero errors. (If the gate fails naming `user-path.ts`, a stray sync-exec remains in Task 2 ... fix there.) + +- [ ] **Step 4: Commit** + +```bash +git add lib/daemon.ts lib/__tests__/no-daemon-sync-exec.test.ts +git commit -m "daemon: await async resolveUserPath; drop user-path sync-exec allowlist (6.1)" +``` + +--- + +## Task 4: Stable machine-key at setup (S071) + +**Files:** +- Create: `lib/home/machine-id.ts` (`stableMachineId`, `resolveInitialMachineKey`) +- Modify: `commands/home.ts:552` (use `resolveInitialMachineKey`) +- Test: `lib/home/__tests__/machine-id.test.ts` + +**Interfaces:** +- Consumes: `machineKey()`, `isSafeMachineKeySegment` from `lib/rt-paths.ts`; `HomeProbes` (has `listProfiles(userLocalDir)`, `exists`). +- Produces: `export async function stableMachineId(exec?: (argv: string[]) => Promise): Promise`; `export async function resolveInitialMachineKey(home: string, probes: HomeProbes, deps?: {...}): Promise`. + +- [ ] **Step 1: Write the failing tests** + +```ts +import { stableMachineId, resolveInitialMachineKey } from "../machine-id.ts"; + +const IOREG_FIXTURE = ` "IOPlatformUUID" = "D9E8F7A6-1234-5678-9ABC-DEF012345678"`; + +test("stableMachineId parses IOPlatformUUID and slugs it", async () => { + const id = await stableMachineId(async () => IOREG_FIXTURE); + expect(id).toBe("d9e8f7a6-1234-5678-9abc-def012345678"); +}); + +test("stableMachineId returns null when ioreg fails", async () => { + expect(await stableMachineId(async () => null)).toBeNull(); + expect(await stableMachineId(async () => "no uuid here")).toBeNull(); +}); + +test("resolveInitialMachineKey: existing pin file is returned unchanged", async () => { + const probes = { exists: (p: string) => p.endsWith("machine-key"), listProfiles: () => [] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => "pinned-key", stableId: async () => "uuid-x" }); + expect(key).toBe("pinned-key"); +}); + +test("resolveInitialMachineKey: existing non-empty hostname-slug store freezes the slug", async () => { + const probes = { exists: () => false, listProfiles: () => ["myhost"] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => null, hostnameSlug: () => "myhost", stableId: async () => "uuid-x" }); + expect(key).toBe("myhost"); // frozen, data preserved, no move +}); + +test("resolveInitialMachineKey: fresh machine gets the stable id", async () => { + const probes = { exists: () => false, listProfiles: () => [] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => null, hostnameSlug: () => "myhost", stableId: async () => "uuid-x" }); + expect(key).toBe("uuid-x"); +}); + +test("resolveInitialMachineKey: fresh machine, ioreg fails -> hostname slug", async () => { + const probes = { exists: () => false, listProfiles: () => [] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => null, hostnameSlug: () => "myhost", stableId: async () => null }); + expect(key).toBe("myhost"); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test lib/home/__tests__/machine-id.test.ts` +Expected: FAIL (module missing). + +- [ ] **Step 3: Implement `lib/home/machine-id.ts`** + +```ts +import { hostname } from "os"; +import { join } from "path"; +import { readFileSync } from "fs"; +import { isSafeMachineKeySegment, machineKey } from "../rt-paths.ts"; +import type { HomeProbes } from "../../commands/home.ts"; + +/** IOPlatformUUID via ioreg, slugged; null on any failure (non-mac, CI, no match). */ +export async function stableMachineId( + exec: (argv: string[]) => Promise = defaultIoreg, +): Promise { + const out = await exec(["ioreg", "-rd1", "-c", "IOPlatformExpertDevice"]); + if (!out) return null; + const m = out.match(/"IOPlatformUUID"\s*=\s*"([^"]+)"/); + if (!m) return null; + const slug = m[1]!.toLowerCase().replace(/[^a-z0-9-]+/g, "-").replace(/^-+|-+$/g, ""); + return isSafeMachineKeySegment(slug) ? slug : null; +} + +const defaultIoreg = async (argv: [string, ...string[]] | string[]): Promise => { + try { + const proc = Bun.spawn(argv as string[], { stdin: "ignore", stdout: "pipe", stderr: "ignore" }); + const term = setTimeout(() => { try { proc.kill("SIGKILL"); } catch { /* gone */ } }, 3_000); + try { + const [out, code] = await Promise.all([new Response(proc.stdout as ReadableStream).text(), proc.exited]); + return code === 0 ? out : null; + } finally { clearTimeout(term); } + } catch { return null; } +}; + +interface InitKeyDeps { + readPin?: () => string | null; + hostnameSlug?: () => string; + stableId?: () => Promise; +} + +/** Establishes the machine key at `rt home init`. Data-preserving + idempotent: + * an existing pin is kept; a machine with existing data freezes its current + * slug (zero move); only a genuinely fresh machine gets the stable id. */ +export async function resolveInitialMachineKey(home: string, probes: HomeProbes, deps: InitKeyDeps = {}): Promise { + const readPin = deps.readPin ?? (() => { try { const v = readFileSync(join(home, "machine-key"), "utf8").trim(); return v || null; } catch { return null; } }); + const hostnameSlug = deps.hostnameSlug ?? (() => machineKey()); // machineKey() with no pin returns the hostname slug + const stableId = deps.stableId ?? (() => stableMachineId()); + + const pinned = readPin(); + if (pinned && isSafeMachineKeySegment(pinned)) return pinned; + + const slug = hostnameSlug(); + const profiles = probes.listProfiles(join(home, "user", "local")); // dirs carrying settings.local.jsonc + if (profiles.includes(slug)) return slug; // freeze existing non-empty store + + return (await stableId()) ?? slug; +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test lib/home/__tests__/machine-id.test.ts` +Expected: PASS. + +- [ ] **Step 5: Wire it into `rt home init`** + +In `commands/home.ts:552`, change: + +```ts + const key = seams.key ?? machineKey(); +``` +to: +```ts + const key = seams.key ?? (await resolveInitialMachineKey(mattstackHome(), probes)); +``` + +Add `import { resolveInitialMachineKey } from "../lib/home/machine-id.ts";` at the top. `homeInit` is already `async`. + +- [ ] **Step 6: Run the home command tests + type-check** + +Run: `bun test commands/__tests__/home.test.ts && bunx tsc --noEmit` +Expected: PASS (existing tests pass `seams.key`, so they bypass the new path; zero type errors). + +- [ ] **Step 7: Commit** + +```bash +git add lib/home/machine-id.ts lib/home/__tests__/machine-id.test.ts commands/home.ts +git commit -m "home: stable machine-key at init, data-preserving freeze of existing stores (S071)" +``` + +--- + +## Task 5: Dev-mode wrapper marker (S020, S067) + +**Files:** +- Modify: `commands/settings.ts` (`renderDevModeWrapper`: add marker line 2) +- Modify: `lib/dev-mode.ts` (`currentMode`: bounded-prefix read + delegate; export `isDevModeWrapperContent`, `DEV_MODE_TAG`) +- Modify: `lib/deps/links.ts` (`isDevModeWrapper`: bounded-prefix read + delegate) +- Test: `lib/__tests__/dev-mode.test.ts` (or the existing dev-mode test file) + +**Interfaces:** +- Produces: `export const DEV_MODE_TAG = "# mattstack-dev-mode";` and `export function isDevModeWrapperContent(prefix: string): boolean` in `lib/dev-mode.ts`. + +- [ ] **Step 1: Write the failing tests** + +```ts +import { isDevModeWrapperContent, DEV_MODE_TAG } from "../dev-mode.ts"; + +test("new marked wrapper is recognized", () => { + expect(isDevModeWrapperContent(`#!/bin/zsh\n${DEV_MODE_TAG}\nexport PATH=...\n`)).toBe(true); +}); +test("legacy markerless wrapper (RT_LAUNCH_CWD tell) is recognized", () => { + expect(isDevModeWrapperContent(`#!/bin/zsh\nexport PATH="x"\nexport RT_LAUNCH_CWD="$PWD"\n`)).toBe(true); +}); +test("foreign #! script is not a dev wrapper", () => { + expect(isDevModeWrapperContent(`#!/bin/sh\necho hi\n`)).toBe(false); +}); +test("a mattstack-link file is not a dev wrapper", () => { + expect(isDevModeWrapperContent(`#!/bin/sh\n# mattstack-link: rt\nexec ...\n`)).toBe(false); +}); +test("non-shebang content is not a dev wrapper", () => { + expect(isDevModeWrapperContent(`ELF\x00binary`)).toBe(false); +}); +``` + +Plus a `currentMode()` test: write a symlink at `devModeWrapperPath()` to a >4KB binary-shaped file and assert `currentMode() === "prod"` (and that it does not throw / read the whole file). Use the existing dev-mode test's HOME-scratch pattern. + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test lib/__tests__/dev-mode.test.ts` +Expected: FAIL (`isDevModeWrapperContent` missing). + +- [ ] **Step 3: Add the marker to the emitted wrapper** + +In `commands/settings.ts renderDevModeWrapper`, insert the marker as line 2: + +```ts + return [ + `#!/bin/zsh`, + `# mattstack-dev-mode`, + `export PATH="${bunDir}:/opt/homebrew/bin:/usr/local/bin:$PATH"`, + `export RT_LAUNCH_CWD="$PWD"`, + `cd "${sourcePath}" || { echo "rt: dev-mode source checkout missing: ${sourcePath}" >&2; exit 1; }`, + `exec "${bunPath}" run --preload="${DEV_MODE_PRELOAD}" "${sourcePath}/cli.ts" "$@"`, + ].join("\n") + "\n"; +``` + +- [ ] **Step 4: Add the shared detector + bounded read in `lib/dev-mode.ts`** + +Add: + +```ts +export const DEV_MODE_TAG = "# mattstack-dev-mode"; + +/** A recognized dev-mode wrapper: our new marker on line 2, OR a legacy + * markerless wrapper (its RT_LAUNCH_CWD line is our unique tell). A foreign + * #! script has neither. `prefix` is a bounded head of the file, never the + * whole file: in prod this path is a symlink to the compiled binary. */ +export function isDevModeWrapperContent(prefix: string): boolean { + if (!prefix.startsWith("#!")) return false; + const line2 = prefix.split("\n")[1] ?? ""; + return line2.startsWith(DEV_MODE_TAG) || prefix.includes("RT_LAUNCH_CWD"); +} + +/** A bounded head of the file (never the whole file: in prod the wrapper path + * is a symlink to the multi-MB compiled binary). Exported so links.ts shares + * the same real bounded read. */ +export function readWrapperPrefix(path: string): string | null { + try { + const fd = openSync(path, "r"); + try { + const buf = Buffer.alloc(4096); + const n = readSync(fd, buf, 0, 4096, 0); + return buf.toString("latin1", 0, n); + } finally { closeSync(fd); } + } catch { return null; } +} +``` + +Rewrite `currentMode()` to use them: + +```ts +export function currentMode(): "dev" | "prod" { + const path = devModeWrapperPath(); + if (!existsSync(path)) return "prod"; + const prefix = readWrapperPrefix(path); + return prefix !== null && isDevModeWrapperContent(prefix) ? "dev" : "prod"; +} +``` + +(Keep the existing `openSync`/`readSync`/`closeSync` imports; add `Buffer` if not already available via global.) + +- [ ] **Step 5: Delegate from `lib/deps/links.ts` via a REAL bounded read** + +`p.readFile` is `readFileSync`, which in prod follows the `~/.local/bin/rt` +symlink into the multi-MB compiled binary ... a whole-file read. Slicing its +result does NOT deliver the bounded-read ruling (the whole file is already in +memory). Use the exported bounded `readWrapperPrefix` (openSync + readSync +4096) instead, so no whole-file read ever happens: + +```ts +import { isDevModeWrapperContent, readWrapperPrefix } from "../dev-mode.ts"; + +function isDevModeWrapper(path: string): boolean { + const prefix = readWrapperPrefix(path); + return prefix !== null && isDevModeWrapperContent(prefix); +} +``` + +Drop the now-unused `p: Pick` parameter and update +`isDevModeWrapper`'s single call site in `links.ts` to pass just the path. +Any `links.ts` test that drove this through an injected `readFile` switches to +writing a real temp file at `path` (the detector reads the actual symlink +target's head on the real fs, by design). + +- [ ] **Step 6: Run tests + type-check** + +Run: `bun test lib/__tests__/dev-mode.test.ts lib/deps/__tests__/links.test.ts && bunx tsc --noEmit` +Expected: PASS. + +- [ ] **Step 7: Commit** + +```bash +git add commands/settings.ts lib/dev-mode.ts lib/deps/links.ts lib/__tests__/dev-mode.test.ts +git commit -m "dev-mode: marker-based wrapper detection, bounded read, legacy fallback (S020/S067)" +``` + +--- + +## Task 6: Unsupported-platform row at setup (R051) + +**Files:** +- Modify: `lib/setup/validators/mac.ts` (add `archRow`, include in `macRows`) +- Test: `lib/setup/validators/__tests__/mac.test.ts` (or the existing mac validator test) + +**Interfaces:** +- Consumes: `Probes` (`p.exec(argv)` → `{ stdout, code }`), `row()` from `lib/setup/contract.ts`. + +- [ ] **Step 1: Write the failing tests** + +```ts +import { macRows } from "../mac.ts"; + +function probes(uname: { stdout: string; code: number }) { + return { exec: async (argv: string[]) => (argv[0] === "uname" ? uname : { stdout: "", code: 0 }), exists: () => false, readFile: () => null, env: {}, home: "/h" } as any; +} + +test("arm64 -> ready", async () => { + const rows = await macRows(probes({ stdout: "arm64", code: 0 })); + const arch = rows.find((r) => r.id === "tool.arch")!; + expect(arch.status).toBe("ready"); +}); +test("x86_64 -> invalid", async () => { + const rows = await macRows(probes({ stdout: "x86_64", code: 0 })); + expect(rows.find((r) => r.id === "tool.arch")!.status).toBe("invalid"); +}); +test("probe failure -> error, not invalid", async () => { + const rows = await macRows(probes({ stdout: "", code: 127 })); + expect(rows.find((r) => r.id === "tool.arch")!.status).toBe("error"); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test lib/setup/validators/__tests__/mac.test.ts -t arch` +Expected: FAIL (no `tool.arch` row). + +- [ ] **Step 3: Add `archRow` and include it** + +In `lib/setup/validators/mac.ts`: + +```ts +async function archRow(p: Probes): Promise { + const base = { id: "tool.arch", kind: "tool" as const, title: "Processor", + why: "mattstack ships an Apple-silicon (arm64) build; Intel Macs are not supported.", required: true }; + const res = await p.exec(["uname", "-m"]); + const arch = res.stdout.trim(); + if (res.code !== 0 || !arch) return row({ ...base, status: "error", detail: "Could not determine your processor" }); + if (arch === "arm64") return row({ ...base, status: "ready", detail: "Apple silicon (arm64)" }); + return row({ ...base, status: "invalid", detail: `${arch}: Apple silicon (arm64) required` }); +} +``` + +And change `macRows`: + +```ts +export async function macRows(p: Probes): Promise { + const [macos, clt, arch] = await Promise.all([macosVersionRow(p), cltRow(p), archRow(p)]); + return [macos, clt, arch, pathRow(p)]; +} +``` + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test lib/setup/validators/__tests__/mac.test.ts` +Expected: PASS. (Note: an existing "macRows returns N rows" count assertion may need +1 ... update it.) + +- [ ] **Step 5: Commit** + +```bash +git add lib/setup/validators/mac.ts lib/setup/validators/__tests__/mac.test.ts +git commit -m "setup: arm64/unsupported-arch row at setup (R051)" +``` + +--- + +## Task 7: Home-repo first-run honesty (S090, R043) + +**Files:** +- Modify: `lib/daemon/home-snapshot.ts` (init `existsSync` check; identity check before commit; new SkipReason values) +- Modify: `lib/home/init-exec.ts` (identity check before the initial commit) +- Test: `lib/daemon/__tests__/home-snapshot.test.ts` (or the existing home-snapshot test) + +**Interfaces:** +- Consumes: `deps.exec` (async runCapture-shaped: `{ exitCode, stdout, stderr }`), `deps.repoDir`, `deps.log`. + +- [ ] **Step 1: Write the failing tests** + +For S090 (init) and R043 (identity), use the home-snapshot test's fake `exec`/`deps`: + +```ts +test("S090: missing repoDir is diagnosed not-provisioned, names rt home init", async () => { + // deps.repoDir points at a path that does not exist; exec is never reached for rev-parse. + const snap = makeSnapshot({ repoDir: "/does/not/exist" }); + await snap.init(); + expect(snap.__disabledReason()).toBe("not-provisioned"); + expect(warnsInclude(snap, "rt home init")).toBe(true); +}); + +test("R043: missing git identity blocks the commit with an actionable reason", async () => { + const exec = fakeExec({ + "git rev-parse --is-inside-work-tree": { exitCode: 0, stdout: "true" }, + "git config user.name": { exitCode: 1, stdout: "" }, + "git config user.email": { exitCode: 1, stdout: "" }, + }); + const snap = makeSnapshot({ repoDir: existingRepoDir, exec }); + await snap.init(); + const r = await snap.snapshot("watch"); + expect(r.skipped ?? snap.__disabledReason()).toBe("no-git-identity"); + expect(execCalled(exec, "git ... commit")).toBe(false); // never attempted +}); +``` + +(Executor: adapt to the test file's real seam names; the assertions ... `not-provisioned`, the `rt home init` string, `no-git-identity`, and "commit never attempted" ... are the contract.) + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test lib/daemon/__tests__/home-snapshot.test.ts -t "S090\|R043"` +Expected: FAIL. + +- [ ] **Step 3: Add the new SkipReason values** + +In `lib/daemon/home-snapshot.ts`, extend the `SkipReason` union (lines 46-55): + +```ts +export type SkipReason = + | "disabled" + | "not-a-repo" + | "not-provisioned" + | "no-git-identity" + | "init-failed" + | "detached" + | "merge-in-progress" + | "owners-read-error" + | "index-locked" + | "add-failed" + | "no-changes"; +``` + +Also widen the local `disabledReason` declaration ... it is currently narrowed +(`let disabledReason: "not-a-repo" | "init-failed" | null;` around +`lib/daemon/home-snapshot.ts:281`), so assigning `"not-provisioned"` / +`"no-git-identity"` fails `tsc`. Change it to: + +```ts + let disabledReason: SkipReason | null = null; +``` + +- [ ] **Step 4: S090 ... existsSync guard in `init()`** + +In `init()` (around line 357), before the `git rev-parse` spawn: + +```ts + async function init(): Promise { + try { + if (!existsSync(deps.repoDir)) { + disabledReason = "not-provisioned"; + deps.log.warn({ repoDir: deps.repoDir }, "home-snapshot: home repo not provisioned; run `rt home init`; inert"); + return; + } + const check = await deps.exec(["git", "rev-parse", "--is-inside-work-tree"], { /* unchanged */ }); + // ... existing exitCode === -1 / not-a-repo branches unchanged ... +``` + +(`existsSync` is already imported at line 24.) + +- [ ] **Step 5: R043 ... identity check before the snapshot commit** + +In the snapshot path, immediately before the commit spawn (line 692), gate once: + +```ts + const name = await deps.exec(["git", "config", "user.name"], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, stderr: "pipe" }); + const email = await deps.exec(["git", "config", "user.email"], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, stderr: "pipe" }); + if (name.exitCode !== 0 || !name.stdout.trim() || email.exitCode !== 0 || !email.stdout.trim()) { + disabledReason = "no-git-identity"; + if (lastLoggedCommitError !== "no-git-identity") { + deps.log.warn("home-snapshot: no git identity; run `git config --global user.name` and `git config --global user.email`; snapshots inert"); + lastLoggedCommitError = "no-git-identity"; + } + return { committed: false, sha: null, paths: [], reason, skipped: "no-git-identity" }; + } + const message = /* unchanged */; + const commitResult = await deps.exec(["git", "-c", "commit.gpgsign=false", "commit", ...]); +``` + +(Return shape mirrors the existing skipped-return objects in this function ... executor matches the actual local return type; the contract is: identity missing → skip with `no-git-identity`, commit never attempted, logged once.) + +- [ ] **Step 6: R043 companion ... initial commit in `init-exec.ts`** + +In `lib/home/init-exec.ts commitInitialUserRepo` (line 82), before the `commit`: + +```ts + case "commitInitialUserRepo": { + log("committing the initial user/ tree"); + await run(exec, ["git", "-C", "user", "add", "-A"]); + const name = await exec.run(["git", "-C", "user", "config", "user.name"]); + const email = await exec.run(["git", "-C", "user", "config", "user.email"]); + if (name.code !== 0 || !name.stdout.trim() || email.code !== 0 || !email.stdout.trim()) { + throw new StepFailed("no git identity: run `git config --global user.name` and `git config --global user.email`, then re-run `rt home init`"); + } + const result = await exec.run(["git", "-c", "commit.gpgsign=false", "-C", "user", "commit", "-m", "initial home repo"]); + // ... existing nothing-to-commit tolerance unchanged ... +``` + +- [ ] **Step 7: Run tests + type-check** + +Run: `bun test lib/daemon/__tests__/home-snapshot.test.ts lib/home/__tests__/init-exec.test.ts && bunx tsc --noEmit` +Expected: PASS. + +- [ ] **Step 8: Commit** + +```bash +git add lib/daemon/home-snapshot.ts lib/home/init-exec.ts lib/daemon/__tests__/home-snapshot.test.ts lib/home/__tests__ +git commit -m "home-snapshot: diagnose not-provisioned and missing git identity (S090/R043)" +``` + +--- + +## Task 8: Timeout on the sops secrets spawn (S070, sops half) + +**Files:** +- Modify: `lib/secrets/store.ts` (`createRealSecretsExecSeam`: add timeout/kill + `SecretsTimeoutError`) +- Test: `lib/secrets/__tests__/store.test.ts` + +**Interfaces:** +- Produces: `export class SecretsTimeoutError extends Error`. + +- [ ] **Step 1: Write the failing test** + +Do NOT spawn a real `trap '' TERM; sleep 60` process: the seam's kill is +SIGTERM-only (mirroring age-key), so a SIGTERM-immune child would hang the +test (`proc.exited` never resolves). Instead inject a fake spawn whose child +resolves `exited` only when `kill()` is called (a killable process), so the +timeout timer fires, kills it, and the seam throws: + +```ts +import { createRealSecretsExecSeam, SecretsTimeoutError } from "../store.ts"; + +test("a hanging sops spawn times out with SecretsTimeoutError, does not hang", async () => { + let resolveExit: (code: number) => void = () => {}; + const fakeProc = { + pid: 1, + stdout: new Response("").body, + stderr: new Response("").body, + exited: new Promise((r) => { resolveExit = r; }), + kill: () => resolveExit(143), // killable: kill resolves exit, no real process + }; + const seam = createRealSecretsExecSeam(undefined, () => fakeProc as any); + await expect(seam.run(["sops", "-d", "x"], { timeoutMs: 50 } as any)) + .rejects.toBeInstanceOf(SecretsTimeoutError); +}, 2_000); +``` + +(The contract: a child that does not exit on its own rejects with +`SecretsTimeoutError` within the timeout, and the timer's `kill()` terminates +it. The fake models a real killable process without one.) + +- [ ] **Step 2: Run test to verify it fails** + +Run: `bun test lib/secrets/__tests__/store.test.ts -t "times out"` +Expected: FAIL (`SecretsTimeoutError` undefined; the call hangs). + +- [ ] **Step 3: Add the error + timeout, mirroring `age-key.ts`** + +In `lib/secrets/store.ts`, add near the other error classes (after `InvalidSecretsSegmentError`): + +```ts +/** Thrown when a sops/keychain spawn does not exit in time (a locked keychain pops a GUI dialog and blocks until clicked). */ +export class SecretsTimeoutError extends Error {} + +const DEFAULT_SECRETS_TIMEOUT_MS = 30_000; +``` + +Make the spawn injectable (mirroring how age-key isolates its raw seam for +testability) and wrap the await with the same timer pattern `age-key.ts` uses. +Change the factory signature to accept an optional spawn seam: + +```ts +type SecretsSpawn = (argv: string[], opts: any) => { + stdout: ReadableStream; stderr: ReadableStream; exited: Promise; kill: (sig?: number | string) => void; +}; + +export function createRealSecretsExecSeam(cwd?: string, spawn: SecretsSpawn = Bun.spawn as unknown as SecretsSpawn): SecretsExecSeam { + return { + async run(cmd, opts) { + debugLog(cmd, opts?.sensitive); + const [bin, ...args] = cmd; + const resolved = bin === undefined ? cmd : [resolveBundledTool(bin), ...args]; + const proc = spawn(resolved, buildSecretsSpawnOptions({ env: opts?.env, cwd })); + const timeoutMs = (opts as { timeoutMs?: number } | undefined)?.timeoutMs ?? DEFAULT_SECRETS_TIMEOUT_MS; + let timedOut = false; + const timer = setTimeout(() => { timedOut = true; try { proc.kill(); } catch { /* already exited */ } }, timeoutMs); + try { + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + if (timedOut) throw new SecretsTimeoutError(`${cmd[0]}: did not exit within ${timeoutMs}ms (keychain prompt pending?)`); + return { code, stdout, stderr }; + } finally { clearTimeout(timer); } + }, + // ... fileExists / listDir / the rest of the seam unchanged ... + }; +} +``` + +(If `SecretsExecSeam.run`'s opts type has no `timeoutMs`, add it to the interface as optional. The default `spawn` is the real `Bun.spawn`, so production behavior is unchanged; only tests inject a fake.) + +- [ ] **Step 4: Run test to verify it passes** + +Run: `bun test lib/secrets/__tests__/store.test.ts -t "times out"` +Expected: PASS. + +- [ ] **Step 5: Type-check + commit** + +Run: `bunx tsc --noEmit` + +```bash +git add lib/secrets/store.ts lib/secrets/__tests__/store.test.ts +git commit -m "secrets: timeout + SecretsTimeoutError on the sops spawn (S070 sops half)" +``` + +--- + +## Task 9: Branch-cache key helpers (S069, part 1 of 2) + +**Files:** +- Modify: `lib/state/branch-cache.ts` (add pure helpers + `get`/`getByBranch`; store still bare-keyed here) +- Test: `lib/state/__tests__/branch-cache.test.ts` + +**Interfaces:** +- Produces: `export function composeKey(identity: string | undefined, branch: string): string`; `export function branchOf(key: string): string`; `export function identityOf(key: string): string | undefined`; store methods `get(identity: string | undefined, branch: string): CacheEntry | undefined` and `getByBranch(branch: string): CacheEntry | undefined`. + +- [ ] **Step 1: Write the failing tests** + +```ts +import { composeKey, branchOf, identityOf } from "../branch-cache.ts"; + +test("composeKey/branchOf/identityOf round-trip with a serialized identity", () => { + const id = "remote:gitlab.com%2Facme%2Facme-dev"; + const k = composeKey(id, "feature/x"); + expect(k).toBe(`${id}:feature/x`); + expect(branchOf(k)).toBe("feature/x"); + expect(identityOf(k)).toBe(id); +}); +test("bare key (no identity) degrades gracefully", () => { + expect(composeKey(undefined, "main")).toBe("main"); + expect(branchOf("main")).toBe("main"); + expect(identityOf("main")).toBeUndefined(); +}); +test("branch never contains a colon, so lastIndexOf split is unambiguous", () => { + const k = composeKey("path:%2FUsers%2Fdev%2Fscratch", "release"); + expect(branchOf(k)).toBe("release"); + expect(identityOf(k)).toBe("path:%2FUsers%2Fdev%2Fscratch"); +}); +``` + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test lib/state/__tests__/branch-cache.test.ts -t "composeKey"` +Expected: FAIL (helpers missing). + +- [ ] **Step 3: Add the helpers + accessors** + +In `lib/state/branch-cache.ts`, add module-level: + +```ts +/** state.db keys the branch cache on `${serializedIdentity}:${branch}`. Split + * on the LAST colon: git branch names contain none, serialized identities + * always carry their own (remote:/path:), so this is unambiguous. */ +export function composeKey(identity: string | undefined, branch: string): string { + return identity ? `${identity}:${branch}` : branch; +} +export function branchOf(key: string): string { + const i = key.lastIndexOf(":"); + return i < 0 ? key : key.slice(i + 1); +} +export function identityOf(key: string): string | undefined { + const i = key.lastIndexOf(":"); + return i < 0 ? undefined : key.slice(0, i); +} +``` + +In `createStore`, add to the returned object: + +```ts + function get(identity: string | undefined, branch: string): CacheEntry | undefined { + return entries[composeKey(identity, branch)]; + } + function getByBranch(branch: string): CacheEntry | undefined { + const suffix = `:${branch}`; + for (const [k, v] of Object.entries(entries)) if (k === branch || k.endsWith(suffix)) return v; + return undefined; + } +``` + +and include `get, getByBranch` in the returned store object and in the `BranchCacheStore` interface. + +- [ ] **Step 4: Run tests to verify they pass** + +Run: `bun test lib/state/__tests__/branch-cache.test.ts && bunx tsc --noEmit` +Expected: PASS (helpers are additive; store behavior unchanged this task). + +- [ ] **Step 5: Commit** + +```bash +git add lib/state/branch-cache.ts lib/state/__tests__/branch-cache.test.ts +git commit -m "branch-cache: add composeKey/branchOf/identityOf + get/getByBranch (S069 part 1)" +``` + +--- + +## Task 10: Flip branch-cache to the composite key (S069, part 2 of 2) ... ATOMIC + +This is the one multi-file atomic change: the store key becomes composite and every direct-lookup consumer switches in the same commit. Intermediate states are not green, so land it as one commit after the whole suite passes. Read contract: every by-branch lookup still resolves a bare branch, scoped to the caller's repo (exact key) or suffix-matched across repos when the repo is unknown; `cache:read`, CLI, board, and tray see bare branch names exactly as before. + +**Files:** +- Modify: `lib/state/branch-cache.ts` (`put` keys off `entry.repoName`) +- Modify: `lib/enrich.ts` (cold-start sets `repoName` from identity; `allCached`/lookup use composeKey) +- Modify: `lib/notifier.ts` (loop var is the composite key; `branchOf` only for display) +- Modify: `lib/daemon/worktree-reconciler.ts` (`branchOf(key)` for the bare branch) +- Modify: `lib/daemon/freshness.ts` (direct lookups compose; iterations use `branchOf`) +- Modify: `lib/daemon/handlers/cache.ts` (`cache:read` returns bare-branch keys via suffix-match; optional `repoIdentity`) +- Modify: `commands/status/data.ts` (display `branchOf(row.branch)`) +- Test: add cases to `branch-cache.test.ts`, `enrich` test, `notifier` test, `worktree-reconciler` test, `freshness` test, `cache` handler test. + +**Interfaces:** +- Consumes: `composeKey/branchOf/identityOf/getByBranch` (Task 9); `serializeIdentity`, `identityFromRemote` from `lib/settings/identity.ts`. + +- [ ] **Step 1: Write the failing tests (collision-safety across all sites)** + +```ts +// branch-cache: two repos, same branch, coexist +test("put keys by entry.repoName so same-name branches in two repos coexist", () => { + const s = makeStore(); // over a temp db + s.put("main", { repoName: "remote:host%2Fa", ticket: null, linearId: "", mr: null, fetchedAt: 1 }); + s.put("main", { repoName: "remote:host%2Fb", ticket: null, linearId: "", mr: null, fetchedAt: 2 }); + expect(s.get("remote:host%2Fa", "main")!.fetchedAt).toBe(1); + expect(s.get("remote:host%2Fb", "main")!.fetchedAt).toBe(2); + expect(Object.keys(s.entries).length).toBe(2); +}); + +// cache:read returns bare-branch keys +test("cache:read returns bare branch names (suffix match), never composite keys", async () => { + // seed ctx.cache with a composite-keyed entry, call the handler with branches:["main"] + const res = await handler["cache:read"]({ branches: ["main"] }); + expect(Object.keys(res.data)).toEqual(["main"]); +}); + +// notifier: same branch, two repos, independent fired-state +test("evicting one repo's branch does not prune the other repo's fired key", () => { + // build cacheEntries with composeKey("remote:host%2Fa","main") and ...b/main + // fire on a's main, then run checkAndNotify with only b/main present + // assert a's fired key survives (it is keyed by the composite branch var) +}); + +// reconciler: mrState only from the reconciled repo +test("reactor builds mrState only from the reconciled repo's entries", async () => { + // cacheEntries has ${A}:main (opened->merged) and ${B}:main (opened) + // run for repo A; assert only A:main transitions, B untouched +}); + +// freshness: a branch in two repos resolves to the right repo's entry +test("updateEntry composes the repo-scoped key", () => { + // seed ${A}:main and ${B}:main; updateEntry(env, A, "main", pr) + // assert only ${A}:main.mr changed +}); +``` + +(These are the acceptance contracts; the executor fills fixtures using each test file's existing helpers.) + +- [ ] **Step 2: Run tests to verify they fail** + +Run: `bun test lib/state/__tests__/branch-cache.test.ts lib/daemon/__tests__ lib/__tests__/notifier.test.ts` +Expected: FAIL (collisions overwrite; composite keys not yet used). + +- [ ] **Step 3: `branch-cache.ts` ... `put` keys off `entry.repoName`** + +In `createStore.put`, derive the key and use it for BOTH the row PK and the map (the `branch` column now stores the composite key; `repo` still stores the identity, so `gc`/`reload`/`delete` keep working transparently since they operate on the `branch` column value): + +```ts + function put(branch: string, entry: CacheEntry): void { + const key = composeKey(entry.repoName, branch); + persistOrWarn("branch-cache", () => { + db.query(UPSERT_SQL).run( + key, + entry.repoName ?? null, + entry.ticket !== null ? JSON.stringify(entry.ticket) : null, + entry.linearId, + entry.mr !== null ? JSON.stringify(entry.mr) : null, + entry.fetchedAt, + ); + }, { op: "put", branch: key }); + entries[key] = entry; + } +``` + +`delete(branch)` callers pass a key that is already the map key; if any caller passes a bare branch, route it through `getByBranch`/`composeKey` at that call site. `gc` is unchanged (it deletes by the `branch` column value, which is now the composite key, and gates on `row.repo` = identity). + +- [ ] **Step 4: `enrich.ts` ... cold-start sets identity; lookups compose** + +In `fetchAndCache`, compute the identity once and set it on every entry it writes, and in `enrichBranches`'s cached path compose the key: + +```ts +import { composeKey } from "./state/branch-cache.ts"; +import { serializeIdentity, identityFromRemote } from "./settings/identity.ts"; + +// inside fetchAndCache: derive identity from remoteUrl (best-effort; undefined if no remote) +const identity = remoteUrl ? serializeIdentity(identityFromRemote(remoteUrl)) : undefined; +// ...when building each CacheEntry, set repoName: identity +// ...store.put(branch, { ...entry, repoName: identity }) + +// inside enrichBranches cached path: +const allCached = !options?.forceRefresh && willFetch + && branches.every((b) => composeKey(identity, b.branch) in store.entries); +// ... +const entry = store.entries[composeKey(identity, b.branch)]!; +``` + +(`enrichBranches` must compute `identity` from its `remoteUrl` the same way, before the cached check.) + +- [ ] **Step 5: `notifier.ts` ... composite key through, `branchOf` for display** + +`detectBranchTransitions` and the `checkAndNotify` snapshot loop already key `state.branches`, `newBranches`, `firedKey`, and `pruneFiredForEvictedBranches` off the `cacheEntries` map keys. With composite keys those become repo-scoped automatically. The only change: wherever a human-readable branch name is put into a notification message, use `branchOf(key)`. Add `import { branchOf } from "../state/branch-cache.ts";` (adjust path) and apply it at the message-construction sites inside `detectBranchTransitions`. + +- [ ] **Step 6: `worktree-reconciler.ts` ... `branchOf(key)` for the bare branch** + +In the reactor loop (line 594 onward), the loop key is now the composite key. Derive the bare branch for registry lookups; keep the repo filter: + +```ts + for (const [key, entry] of Object.entries(cacheEntries)) { + if (entry.repoName && entry.repoName !== repoName) continue; + if (!entry.mr) continue; + const branch = branchOf(key); + // ... `const mrKey = prefix + branch;` (rename the local `key` used for mrState to `mrKey` + // to avoid colliding with the composite map key) ... + // findByBranch(loadRegistry(repoName), branch) and resumeTrees(deps, branch) use the bare branch. + } +``` + +Add `import { branchOf } from "../state/branch-cache.ts";`. Rename the existing local `const key = prefix + branch;` to `mrKey` and update its uses (`nextMrState[mrKey]`, `state.mrState[mrKey]`). + +- [ ] **Step 7: `freshness.ts` ... compose direct lookups, `branchOf` iterations** + +`repoName` here is the serialized identity, so: + +```ts +import { composeKey, branchOf } from "../state/branch-cache.ts"; + +// line 505 branchByIid: iterate, filter entry.repoName !== repoName, store bare branch: +for (const [key, entry] of Object.entries(ctx.cache.entries)) { + if (entry.repoName !== repoName) continue; + if (typeof entry.mr?.iid === "number") branchByIid.set(entry.mr.iid, branchOf(key)); +} + +// line 545: `ctx.cache.entries[pr.sourceBranch]?.repoName === repoName` +// -> `ctx.cache.entries[composeKey(repoName, pr.sourceBranch)] !== undefined` + +// line 579: `const entry = ctx.cache.entries[k.ref];` then `entry.repoName === repoName` +// -> `const entry = ctx.cache.entries[composeKey(repoName, k.ref)];` (the repoName check is then redundant) + +// updateEntry (line 637): compose for both read and write +function updateEntry(env, repoName, branch, pr) { + const key = composeKey(repoName, branch); + const existing = env.ctx.cache.entries[key]; + if (!existing) return false; + env.ctx.cache.put(branch, { ...existing, mr: pr ? toMRInfo(pr) : null, fetchedAt: Date.now(), repoName }); + // ... +} + +// applyMRWriteback (657) + runGapFill (703): iterate, filter by entry.repoName, map keys via branchOf, +// pass bare branch to updateEntry (which recomposes). +``` + +- [ ] **Step 8: `handlers/cache.ts` ... bare-branch output via suffix-match** + +`cache:read` must return bare-branch keys. Accept an optional `repoIdentity` for exact scoping; otherwise suffix-match. Import `branchOf`/`getByBranch` semantics: + +```ts +"cache:read": async (payload) => { + const branches = payload?.branches as string[] | undefined; + const repoIdentity = payload?.repoIdentity as string | undefined; + const maxAgeMs = payload?.maxAgeMs as number | undefined; + + const lookup = (b: string): CacheEntry | undefined => + repoIdentity ? ctx.cache.entries[`${repoIdentity}:${b}`] + : Object.entries(ctx.cache.entries).find(([k]) => k === b || k.endsWith(`:${b}`))?.[1]; + + if (typeof maxAgeMs === "number") { + const pool = branches ?? Object.keys(ctx.cache.entries).map(branchOf); + let oldest = 0; + if (pool.length > 0) oldest = Math.min(...pool.map((b) => lookup(b)?.fetchedAt ?? 0)); + if (Date.now() - oldest >= maxAgeMs) await ctx.refreshCache(); + } + + if (!branches) { + const out: Record = {}; + for (const [k, v] of Object.entries(ctx.cache.entries)) out[branchOf(k)] = v; // bare-branch keyed + return { ok: true, data: out }; + } + const filtered: Record = {}; + for (const b of branches) { const e = lookup(b); if (e) filtered[b] = e; } + return { ok: true, data: filtered }; +}, +``` + +Add `import { branchOf } from "../../state/branch-cache.ts";` (adjust path). `branch:enrich`'s `ctx.cache.entries[branch]` lookups: compose with the payload's repo identity when present, else `getByBranch`-style suffix match. + +- [ ] **Step 9: `commands/status/data.ts` ... display bare branch** + +In `readBranchesFromStateDb`, key the returned dict by the bare branch: + +```ts +import { branchOf } from "../../lib/state/branch-cache.ts"; +// ... +for (const row of rows) { + branches[branchOf(row.branch)] = { + ticket: row.ticket !== null ? JSON.parse(row.ticket) : null, + linearId: row.linear_id, + mr: row.mr !== null ? JSON.parse(row.mr) : null, + fetchedAt: row.fetched_at, + repoName: row.repo ?? undefined, + }; +} +``` + +- [ ] **Step 10: Run the whole affected suite + type-check** + +Run: `bun test lib commands packages scripts && bunx tsc --noEmit` +Expected: PASS, zero errors. Confirm `lib/daemon/discussions-poller.ts` needs no change (it iterates `Object.values`, self-healing). + +- [ ] **Step 11: Commit (single atomic commit)** + +```bash +git add lib/state/branch-cache.ts lib/enrich.ts lib/notifier.ts lib/daemon/worktree-reconciler.ts lib/daemon/freshness.ts lib/daemon/handlers/cache.ts commands/status/data.ts lib/state/__tests__ lib/daemon/__tests__ lib/__tests__/notifier.test.ts commands/__tests__ +git commit -m "branch-cache: flip to composite ${identity}:${branch} key; scope all consumers (S069 part 2)" +``` + +--- + +## Final verification (run before the whole-branch review) + +- [ ] `bun test lib commands packages scripts` green (worktree root). +- [ ] `bunx tsc --noEmit` reports zero errors. +- [ ] `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts e2e/tests/setup.test.ts e2e/tests/first-run.test.ts` green (run full `bun run test:e2e` if practical; record which was run). +- [ ] `lib/__tests__/no-daemon-sync-exec.test.ts` green with the `user-path.ts` allowlist entry removed. +- [ ] `cd packages/rt-client && bun run build && bun test test/dist-freshness.test.ts` green. + +## Self-review (author checklist ... completed before saving) + +- **Spec coverage:** every spec item maps to a task ... 6.1 → Tasks 1-3; S071 → Task 4; S020/S067 → Task 5; R051 → Task 6; S090/R043 → Task 7; S070 sops → Task 8; S069 → Tasks 9-10. +- **Placeholder scan:** no TBD/TODO; new code is inlined; edit sites carry before/after snippets and exact anchors. +- **Type consistency:** `composeKey/branchOf/identityOf` used identically in Tasks 9-10; `ProbeFn` signature consistent in Task 2; `SkipReason` additions consistent in Task 7. diff --git a/docs/superpowers/specs/2026-08-28-p2-health-design.md b/docs/superpowers/specs/2026-08-28-p2-health-design.md new file mode 100644 index 00000000..1f296251 --- /dev/null +++ b/docs/superpowers/specs/2026-08-28-p2-health-design.md @@ -0,0 +1,250 @@ +# Daemon health you can see (Phase 2 / RT-79) + +Design record for Phase 2 of the rt daemon stability roadmap. Makes a +non-author user able to tell, from `rt daemon status`, the tray dot, and +`/api/status`, whether the daemon is serving, degraded, stalled, or dead, and +why. Builds on Phase 0 (the status classifier in `lib/daemon-status.ts`, the +`daemon-supervision` kv namespace, the pre-db breadcrumb file). Covers audit +findings R011, R012, R003, R004, S031, S032, S033, R005, R008, R021. + +## Health model + +A single server-computed verdict every surface reads, replacing three +independent client-side classifications (CLI, Swift tray, none in `/api/status`). + +`lib/daemon/health.ts` exports a pure `computeHealth(inputs): HealthSnapshot`: + +``` +HealthSnapshot = { + level: "ok" | "degraded" | "unhealthy", + reasons: string[], // one subsystem-prefixed line per trigger + metrics: { rss, heapUsed, external, uptimeMs, wsClients, watchers }, + // watchers = fs.watch handle count (watchedConfigs.size) + eventLoop: { maxLagMs, lastStallAt, lastStallCmd, stalls }, +} +``` + +Level is severity-ordered; unhealthy wins over degraded. + +- **unhealthy** if any: logger degraded (ENOSPC, from S032); event loop currently + stalled (heartbeat/monitor); restart storm (Phase 0 `isCrashLooping`, or >= N + supervision failures in the last hour); disk free under a hard floor. +- **degraded** if any: a freshness watcher is `degraded` + (`getFreshnessSnapshot()`); the last refresh cycle had `failedRepos > 0` or + `enrichErrors > 0`; last successful refresh age > 2x the refresh interval; rss + over a soft threshold or grown > 50% in the last hour; event-loop `maxLagMs` + over the lag threshold within the window; recovered-error rate over a small + threshold in the window; disk free under a soft floor. +- **ok** otherwise. + +`reasons` name the failing subsystem so the operator knows where to look, e.g. +`"refresh: 3 repos failing (auth?)"`, `"event-loop: stalled 8s"`, +`"logging: disabled (ENOSPC)"`, `"disk: 180MB free"` (R011). `metrics` gives the +memory/handle/uptime numbers a leak hunt needs (R012). R012's watcher-close +remediation (closing fs.watch handles for repos no longer indexed) is **out of +Phase 2 scope**: Phase 2 ships the metrics + growth-alarm half only, and the +leak-close rides the later watcher-lifecycle work. + +A daemon-side adapter (`buildHealthSnapshot(ctx)`) gathers the inputs from `ctx`, +the loop monitor, the logger handle, Phase 0 supervision, and an optional +`fs.statfs` probe, then calls the pure function. Inputs that need new tracking: +`refreshStatusRef` grows from `{ lastRefreshAt }` to also carry the last cycle's +`{ lastSuccessAt, failedRepos, enrichErrors }` (populated in `cache-refresh.ts`); +`wsClients.size` is exposed from `api-server.ts` to the adapter. **Deferred behind +a typed hook** (V1 does not wire them): SQLITE_BUSY-skip and critical-write-failure +counters. `HealthInputs` declares the fields so they slot in later without a shape +change. + +### Where each surface reads it + +- `status` and `tray:status` verbs (`lib/daemon/handlers/status.ts`) gain + additive `health`, `metrics`, `eventLoop` blocks. `/api/status` aliases + `tray:status`, so it carries the verdict. +- `ping` gains `health.level`, `version` (`ctx.identity.version`), the heartbeat + `seq`, and the `eventLoop` summary (`maxLagMs`, `lastStallAt`, `lastStallCmd`) so + a caller that only got a ping through can still show lag (all cheap). +- `rt daemon status` (`commands/daemon.ts` `statusLines`) renders `level` + + `reasons` and the metrics/eventLoop lines on the running branch. Two corrections + to today's guesswork, on two different verdicts (R003): + - **degraded / unresponsive** (ping answered but the `status` verb timed out): + print the ping-carried `maxLagMs` / last stall instead of "likely mid-sync". + - **alive-not-serving** (ping failed, pid alive): print the new "stalled Ns ago" + detail from `now - heartbeat.at`. The heartbeat file is the only signal + reachable here and carries `{ at, seq }`, no `maxLag`. +- **Swift tray is deferred** (brief constraint): no `rt-tray/` edits. Documented + contract for the follow-up: read `data.health.level` -> green/orange/red and + `data.health.reasons[0]` as `statusText`; the current client derivation + (pendingNotifications + two-miss) becomes the fallback when `health` is absent. + +## Heartbeat and stall detection (R003) + +`lib/daemon/loop-monitor.ts`: a ~250ms `setInterval` measuring drift +(`actualElapsed - expected`) into a preallocated `LoopStats` object. The tick is +allocation-free (no per-tick closures/objects) and the timer is `unref()`'d so it +never keeps the process alive. It maintains `{ lagMs, maxLagMs, stalls, +lastStallAt, lastStallCmd }`; on drift > 1s it increments `stalls`, records +`lastStallCmd` (a module `currentCmd` set by `handleCommand`), and logs one warn. + +**Heartbeat is a file, not kv** (ratified): every ~2s the monitor writes +`{ at, seq }` (seq monotonic) to `RT_DIR/daemon-heartbeat.json` via atomic rename +(write temp + `renameSync`), the same db-free pattern as Phase 0's breadcrumb. +Rationale: state.db is the WAL every CLI contends on, and a stalled or +lock-wedged daemon is exactly when it is least readable. `lib/daemon/heartbeat-file.ts` +owns `writeHeartbeat`/`readHeartbeat` (missing/corrupt -> null). + +Cross-process detection extends `lib/daemon-status.ts`: `classifyDaemonStatus` +takes an optional `heartbeat: { at, seq } | null` plus a stale threshold. In the +alive-not-serving branch, when `breadcrumb.phase === "ready"` and +`now - heartbeat.at` exceeds the threshold, the detail becomes a new `"stalled"` +(with age) instead of `"wedged"`. The `alive-not-serving` detail union grows to +`"booting" | "wedged" | "quarantined" | "stalled"`. `commands/daemon.ts` reads the +heartbeat file only when the pid probe is already needed (`needsPidProbe`), and +`statusLines` prints "event loop stalled Ns ago". + +## Default thresholds + +`computeHealth` stays pure; these are the defaults the daemon-side adapter feeds +it (and the classifier's heartbeat threshold). Named constants, tunable later. + +| Constant | Default | Drives | +|---|---|---| +| `loopTickMs` | 250 ms | loop-monitor tick cadence | +| `loopLagDegradedMs` | 500 ms | degraded: `maxLagMs` in the window exceeds this | +| `loopStallLogMs` | 1000 ms | warn + `stalls++` when a single tick's drift exceeds this (R003's "> 1s") | +| `loopStallUnhealthyMs` | 2000 ms | "currently stalled" -> unhealthy: the most recent tick's drift exceeded this within the last `stallRecentMs` | +| `stallRecentMs` | 10 s | window in which a large recent drift still counts as "currently stalled" | +| `heartbeatIntervalMs` | 2000 ms | heartbeat file write cadence | +| `heartbeatStaleMs` | 6000 ms | classifier: alive-not-serving + heartbeat age over this -> "stalled Ns ago" (3 missed writes) | +| `refreshStaleMultiplier` | 2x the refresh interval | degraded: last successful refresh older than this | +| `rssSoftThresholdMB` | 1024 MB | degraded: rss over this | +| `rssGrowthPct` / `rssGrowthWindow` | 50% over 1 h | degraded: rss grew this much in the window | +| `diskSoftFloorMB` | 500 MB | degraded: free space under RT_DIR below this | +| `diskHardFloorMB` | 100 MB | unhealthy: free space below this | +| `restartsPerHourUnhealthy` | 5 (or Phase 0 `isCrashLooping`, >= 3 / 5 min) | unhealthy: restart storm | +| `recoveredErrorRate` / window | > 10 in 5 min | degraded: stderr/rejection error churn | +| `slowCommandMs` | 2000 ms | log a successful command at `info` above this | + +"Currently stalled" is necessarily an in-process near-miss (a daemon answering +`status` is not stalled at that instant): the adapter sets it when the last tick's +drift exceeded `loopStallUnhealthyMs` within `stallRecentMs`, catching a daemon +that just unstuck. An ongoing stall is caught cross-process instead, by the +classifier's stale-heartbeat path above. + +## Log level and growth policy + +- **rt.logLevel** (R004): new registry row, `type: "string"`, + `scopes: ["machine", "user"]`, `default: "info"`, `merge: "replace"`, following + `docs/settings-architecture.md`'s checklist exactly as `rt.apiPort` did (add the + row, `cd packages/rt-client && bun run build` so the dist-freshness test stays + green). `getDaemonLogger` resolves `level = RT_LOG_LEVEL env ?? getSetting("rt.logLevel") ?? "info"` + (env wins, mirroring `resolveApiPort`). +- **Live control** (R004): a `daemon:log-level` IPC verb sets `logger.level` + at runtime and logs the change; `rt daemon log-level ` dispatches it. The + new command registers in `command-tree-def.ts` and `lib/module-registry.ts`, and + its required positional `level` (a select over pino levels) declares + `omitBehavior: "picker"` so `bun run picker:check` stays green. +- **Slow-command visibility** (R004): `handleCommand` logs successful commands at + `info` when `durationMs > 2s` (else `debug`, as today), so latency outliers are + visible at the default level. +- **Growth cap** (S031): pino-roll gets `size: "50m"` beside `limit: { count: 14 }` + (bounds within-day growth independent of the daily/age sweep). Per-(cmd,error) + suppression in `handleCommand`: a Map keyed `${cmd}|${errorKey}` tracking + `{ count, lastLoggedAt }`. **Guardrail:** always log the first occurrence + immediately; within 60s of the last logged line, increment silently; at >= 60s + emit one line carrying `suppressed: ` and reset. `pruneLogs` takes an + `onError` callback so the janitor's readdir/unlink failures log at warn instead + of being swallowed. + +## Logger resilience and stderr noise + +- **Stream error listener** (S032): `createDaemonLogger` adds + `stream.on("error", ...)` that sets a `loggerDegraded` flag and does a raw + `fs.writeSync(2, ...)`, so a full-disk write never throws out of a log call. The + handle exposes `loggerDegraded` (feeds health -> unhealthy). The + `uncaughtException` / `unhandledRejection` handler bodies are wrapped in + try/catch with a raw-write fallback that still calls `process.exit(1)` (Phase 0's + boot-vs-steady-state semantics preserved). +- **stderr demotion** (S033, R005): the stderr interceptor logs at `warn` with + `source: "stderr"`, escalating to `error` only for known panic/exception + prefixes. `unhandledRejection` and recovered errors increment a process-wide + counter exposed in `health`/`metrics`. +- **Resolver warn sink** (S033): `packages/rt-client`'s resolver + (`resolve.ts` `warnInvalid` and siblings) takes an injectable warn sink + defaulting to `console.warn` (CLI/test behavior unchanged). The daemon binds a + sink that dedupes per `(key, scope, reason)` to `log.warn`, so a hot-path + `getSetting` on a disallowed-scope key warns once, not every tick. + +## Request attribution + +- **reqId + caller** (R008): `handleCommand` mints a short request id per request + and logs `{ reqId, cmd, caller, durationMs }` on every seam line; `ok:false` + envelopes echo `reqId`. Caller comes from an `X-RT-Client` header (REST) or a + `_client` field on the socket frame, formatted `/` (default + `unknown`). On reject/fail, log a redacted payload digest: top-level keys plus + the whitelisted `repo`/`branch`/`iid`/`room` when present. rt's own transport + (`lib/daemon-client.ts`) and `packages/rt-client`'s transport both send the tag. +- **Unknown-command envelope** (R021): the `routeCommand` default returns + `{ ok: false, code: "unknown-command", error, version }`. Both transports map + `code === "unknown-command"` to distinct text ("daemon at version X does not know + ; restart or upgrade rt"). `ping` optionally exposes the command-name list + for pre-checks. + +**rt-client blast radius:** the `packages/rt-client` edits (registry row, warn +sink, caller tag, unknown-command text) ship as source + a `dist` rebuild; the +version is **not** bumped and the package is **not** published (publishing is +release-class, from `main` only). The estate-wide rollout to board/gitq/console +rides the next release from `main`. + +## Constraints and invariants + +- No `SCHEMA_VERSION` bump. The only new persisted state is the heartbeat file + (`RT_DIR/daemon-heartbeat.json`); everything else is computed live or reuses the + Phase 0 `daemon-supervision` kv namespace. +- `rt.logLevel` goes through the settings registry per the checklist; rebuild + rt-client `dist`; no version bump, no publish. +- No `rt-tray/` edits; the tray read contract above is a documented follow-up. +- Never start a daemon or run `dist/rt` except under `env -i HOME=`. +- Do not touch the p6-portability-owned files, nor the module-scope + `resolveUserPath()` call in `lib/daemon.ts` (p6 makes it awaited-async). + +## Components + +**New:** `lib/daemon/health.ts` (pure `computeHealth` + `HealthInputs`), +`lib/daemon/loop-monitor.ts`, `lib/daemon/heartbeat-file.ts`, the +`buildHealthSnapshot` adapter, the `rt daemon log-level` command handler. + +**Changed:** `lib/daemon.ts` (handleCommand reqId/caller/suppression/currentCmd, +loop-monitor + metrics-logger wiring), `lib/daemon/handlers/status.ts` +(health/metrics/eventLoop in status + tray:status + ping; unknown-command code), +`lib/daemon/handlers/types.ts` (extend `refreshStatusRef`, expose health inputs), +`lib/daemon/cache-refresh.ts` (populate the extended ref), +`lib/daemon/api-server.ts` (read `X-RT-Client`, expose `wsClients`), +`lib/daemon-logger.ts` (level from setting, stream error listener, stderr +demotion, crash-handler wrap, size cap, `loggerDegraded`), `lib/daemon-status.ts` +(heartbeat input + `stalled` detail), `commands/daemon.ts` (render health + read +heartbeat), `lib/log-janitor.ts` (`onError`), `lib/daemon-client.ts` (send caller +tag, surface unknown-command), `packages/rt-client` (registry row, warn sink, +transport caller tag + unknown-command text), `lib/command-tree-def.ts` + +`lib/module-registry.ts` (log-level command). + +## Testing + +- `health.ts`: each level transition and reason string (pure, table-driven). +- `loop-monitor.ts`: drift math with injected clock; unref'd; no per-tick + allocation. +- `heartbeat-file.ts`: write/read round trip via atomic rename; missing/corrupt + -> null. +- `daemon-status.ts`: `stalled` detail when heartbeat is stale + pid alive + boot + reached ready; every existing verdict unchanged. +- `daemon-logger.ts`: stream error -> `info()` does not throw and `loggerDegraded` + set; crash handler still exits under a throwing logger; a non-panic stderr line + logs at warn not error; the size cap option is present. +- `handleCommand`: reqId minted and echoed in `ok:false`; caller logged; a burst + of identical `ok:false` produces a bounded number of lines with a suppressed + count. +- unknown-command envelope carries `code` + `version`; transport surfaces the + distinct text. +- resolver warn sink dedupes once per `(key, scope, reason)`. +- settings: `rt.logLevel` row + settings-paths parity + dist freshness. +- E2E `e2e/tests/daemon.test.ts`: status/tray:status/ping additive fields; + `/api/status` shape stays additive. diff --git a/docs/superpowers/specs/2026-08-28-p6-portability-design.md b/docs/superpowers/specs/2026-08-28-p6-portability-design.md new file mode 100644 index 00000000..393bc2fd --- /dev/null +++ b/docs/superpowers/specs/2026-08-28-p6-portability-design.md @@ -0,0 +1,520 @@ +# Phase 6 · Someone else's Mac (p6-portability) ... design + +**Status:** proposed +**Date:** 2026-08-28 +**Branch:** `job/p6-portability` (stacked on `job/integration`, wave 1) +**Roadmap:** daemon-stability-audit-2026-08 §"Phase 6 · Someone else's Mac" (RT-83) + +Phase 6 makes the daemon survive a machine that is not the author's: a +teammate whose login shell is fish, whose `.zshrc` blocks or execs into tmux, +whose Mac has a different hostname, who has no home repo yet, whose git has no +identity, or who is on an Intel Mac. The audit lists ~8 items across four +sub-themes. The `superpowers` chain runs a full spec for 6.1 and treats the +rest as bounded, plan-sized units. + +## Verification pass ... what survived, what wave 1 already closed + +Every Phase 6 finding was re-verified against the merged wave-1 code before +scoping this spec. Result: + +**Open, in scope:** + +| Item | Finding(s) | One-line defect | +|---|---|---| +| 6.1 PATH rebuild | S013, S014, S062 | fish emits space-separated PATH; boot hangs forever on a blocking `.zshrc`; silent fallback to launchd's bare PATH when `.zshrc` execs fish/tmux | +| 6.2 machine key | S071 | machine settings key derives from the mutable hostname | +| 6.2 dev wrapper | S020, S067 | a foreign `~/.local/bin/rt` `#!` script is misread as our dev-mode wrapper, parking prod | +| 6.3 platform | R051 | no Intel / unsupported-arch warning at setup | +| 6.4 first-run | S090 | missing `~/.mattstack/user` diagnosed as "git missing" not "not provisioned" | +| 6.4 first-run | R043 | no git `user.name`/`user.email` check with an actionable message | +| 6.4 first-run | S070 (sops half) | age-key spawn got a timeout in wave 1; the sops spawn in `lib/secrets/store.ts` still has none, so a locked keychain hangs `loadSecrets()` | +| 6.4 first-run | S069 | `branch_cache` keys on the bare branch name; a same-name branch in a second repo overwrites the first | + +**Already closed by wave 1, dropped from scope** (verified in code): + +- **S046** ... `lib/daemon/cron.ts:84` now passes `env: { ...process.env }`. +- **S099** ... `lib/rt-paths.ts` gates the `~/.rt` rename behind `hasRtSignature()` (`RT_SIGNATURE_ENTRIES`). +- **S066** ... `lib/deps/links.ts` keeps every `DEFAULT_EXPOSED` tool; reconcile never unlinks our own product surface by name. +- **S002** ... `lib/agent-herdr.ts` resolves herdr via `resolveHerdrBin()` (`HERDR_BIN` ?? `Bun.which("herdr")` ?? `~/.local/bin/herdr`) with a clear error. +- **S039** ... `agent-status-poller.ts` backs the herdr probe off after 3 null probes; `lib/runs/store.ts` memoizes run summaries by db mtime. +- **S051** ... `handlers/agent.ts` returns `ok:false` and rolls back the record when herdr focuses an existing tab. +- **S022** ... `lib/daemon/freshness.ts resolveUserIdAcrossTracking()` gates on grant, not live-vs-poll, so poll-only users get notifications. +- **S070 (age-key half)** ... `lib/home/age-key.ts` has the 30s timeout + `AgeKeyTimeoutError` already. + +The one correction to the brief: the brief listed **S070 as done**. Only the +age-key half is; the sops spawn in `lib/secrets/store.ts` still has no timeout. +That half is kept in scope (6.4). + +No change here requires a `SCHEMA_VERSION` bump (S069 reuses the existing +`branch TEXT PRIMARY KEY` column ... see 6.4). `packages/rt-client` is touched +(one new registry key), so `bun run build` runs in it before the final review. + +--- + +## 6.1 · PATH resolution rebuilt (S013, S014, S062) + +### The problem + +`lib/daemon/user-path.ts resolveUserPath()` scrapes the user's PATH with +`execSync($SHELL -ilc 'echo $PATH')` at daemon boot (called synchronously at +`lib/daemon.ts:163`). Three failure classes on someone else's Mac: + +- **S013 (fish):** `fish -ilc 'echo $PATH'` prints a *space-separated* list. + The daemon splits on `:`, so every real dir lands inside one bogus entry; + `git`/`node`/`pnpm` vanish from every child's PATH. `entries: 1` is logged; + nothing warns. +- **S014 (hang):** `-i` sources `.zshrc`. Under launchd (no TTY, no network + yet) a plugin, `gpg-agent`/pinentry, `direnv`, or a `read` can block + forever. `execSync`'s timeout only SIGTERMs; an interactive shell ignores + SIGTERM, so the daemon hangs before it binds anything ... the exact "starts, + binds nothing, logs nothing" symptom CLAUDE.md warns gets misdiagnosed. +- **S062 (silent fallback):** `.zshrc` ending in `exec fish` / `exec tmux` + replaces zsh before `-c` runs; stdout is empty, `|| resolvedPath` silently + keeps launchd's `/usr/bin:/bin:/usr/sbin:/sbin`. The pool then wedges with + `env: node: No such file or directory` (the 2026-08-21 spawn-env incident), + now for any common `.zshrc` idiom. + +### Decisions + +1. **Non-interactive login base probe (`-lc`), interactive overlay unioned on + top.** The base probe uses a *non-interactive login* shell (`-lc`), which + sources `.zprofile`/`.zshenv` (zsh) or `.bash_profile` (bash) but never the + interactive rc files (`.zshrc`, `.bashrc`). This is the safe floor: it + cannot hang on an interactive plugin (S014) or exec into tmux/fish (S062), + because those idioms live in `.zshrc`. `.zshenv`'s absolute-path fnm + bootstrap and `.zprofile`'s `brew shellenv` (both fixed by the 2026-08-21 + spawn-env contract) are still sourced, so a standard Homebrew+fnm Mac + resolves fully from the base alone. + + **Interactive overlay (best-effort), per the shepherd ruling.** After the + base resolves, run a best-effort `$SHELL -ilc 'echo $PATH'` (fish: + `-ilc 'string join : $PATH'`) with **stdin from `/dev/null`**, **`TERM=dumb`** + in the child env, and a **3s hard timeout in the same killable process + group** (decision 2). Validate its output as a colon-separated list of + *absolute* dirs; **union its unique dirs after the base entries** (append, + never prepend, so the base and the daemon's own prefix keep priority). On + timeout or garbage, skip it with one `warn` line and keep the base result. + This recovers the common `.zshrc`-only PATH exports (`nvm`, `pyenv`, + `cargo`) without reintroducing the hang: the interactive shell can block or + exec, but the base has already resolved and the overlay is bounded and + killable, so a bad `.zshrc` only costs the overlay, never the daemon. The + `rt.daemonPath` override (decision 4) and the missing-tools warning + (decision 6) remain the backstops when both probes fall short. + +2. **Hard timeout in a killable process group.** The probe spawns via + `Bun.spawn([...], { detached: true })` (a new session/process group; the + same option `lib/worktree/trash.ts:183` already uses) and `proc.unref()`s + it. A `setTimeout` pair escalates `process.kill(-proc.pid, "SIGTERM")` then, + after a short grace, `process.kill(-proc.pid, "SIGKILL")` ... the negative + pid targets the whole group, so a hung grandchild (pinentry, a stuck + `direnv`) is reaped too, not just the shell. The result is a + `Promise.race([captured, deadline])` so `resolveUserPath` always resolves + within the timeout regardless of what the child does. `detached: true` is + what makes `-pid` safe: without it, `-pid` would signal the daemon's own + group. Default timeout 5000ms (a login shell resolves in well under 1s), + overridable via `RT_PATH_PROBE_TIMEOUT_MS` and via an injected seam for + tests. + +3. **fish-aware, colon-joined output, nvm overlay.** + - `shellName = basename($SHELL || "/bin/zsh")`. + - fish: `[$SHELL, "-lc", "string join : $PATH"]` ... emits a colon-joined + list (fixes S013). + - everything else: `[$SHELL, "-lc", '{ [ -s "${NVM_DIR:-$HOME/.nvm}/nvm.sh" ] && . "${NVM_DIR:-$HOME/.nvm}/nvm.sh" >/dev/null 2>&1; }; printf %s "$PATH"']` + ... `printf %s "$PATH"` is already colon-joined; the nvm overlay replaces + the current file's second `execSync`. It is the fast, safe recovery of an + nvm node in the base probe (so nvm resolves even when the interactive + overlay in decision 1 is skipped on timeout); the interactive overlay is + the broader net for pyenv/cargo/hand-rolled `.zshrc` exports. + +4. **Explicit `rt.daemonPath` override (settings registry).** A new + machine-scoped key. When set to a non-empty value, `resolveUserPath` uses it + verbatim and skips the shell probe entirely ... instant, deterministic, and + the honest replacement for scraping an exotic shell. Registry row (in + `packages/rt-client/src/settings/registry-defs.ts`, mirrored nowhere else): + + ```ts + { + key: "rt.daemonPath", + type: "string", + scopes: ["machine"], + merge: "replace", + // no `default`: absent means "resolve via the shell probe below". + // no `pathGuardFields`: the value IS a PATH literal, and machine scope + // is exempt from the path-literal guard anyway (write.ts). + description: "Absolute colon-separated PATH the daemon uses for every child it spawns, instead of probing your login shell. Set this when the daemon can't find node/git/bun/pnpm (e.g. a fish shell, a blocking .zshrc, or PATH exports that live only in .zshrc). Machine-scoped: it never travels to another machine.", + } + ``` + + Read synchronously via `getSetting("rt.daemonPath")` (getSetting is + sync and throws only on an *unregistered* key; an unset registered key + resolves to `undefined`). + +5. **Validate before trusting the probe.** Trim the output; reject it (keep the + baseline `process.env.PATH`, warn) when it is empty, contains whitespace + (a space/tab means a fish-unsplit or corrupt value), splits into fewer than + two colon segments, or equals the launchd baseline verbatim (the S062 + silent-fallback signature). Acceptance is the only path that overwrites the + baseline. + +6. **Observability.** The probe's tool set becomes `node`, `git`, `bun`, + `pnpm`. `doppler` is intentionally dropped from the logged tool set: it is + an optional integration, not a toolchain prerequisite, and its absence is + not a portability failure worth a boot-time signal. After resolution + (override, base, base+overlay, or baseline), if any of node/git/bun/pnpm are + missing from the resolved PATH, log one `warn` naming the remedy ("set + `rt.daemonPath`"). Distinguish, in the log, the outcomes: override used / + base accepted / overlay unioned / fell back to baseline (with the reason: + killed, empty, invalid). + +7. **Async integration (fence-granted).** `resolveUserPath` becomes + `async (log) => Promise`. `lib/daemon.ts:163` changes the one + statement to `const resolvedPath = await resolveUserPath(log);` (shepherd + granted this single-statement exception to the p2-health lane's ownership of + `daemon.ts`; the surrounding block and everything else in the file stay + p2's). `daemon.ts` already uses top-level `await` (lines 119, 145, ...), so + this is a literal one-line change. Boot waits at most the hard timeout and + can never hang. The bundle-Helpers + `~/.local/bin` prefix block that + follows (`daemon.ts:167-183`) is unchanged and still runs after the await. + +8. **Remove the sync-exec allowlist entry.** With both `execSync` calls gone + (the code now uses async `Bun.spawn` only), delete + `"lib/daemon/user-path.ts", // Phase 6 PATH rebuild (S013/S014/S062)` from + the `ALLOWLIST` in `lib/__tests__/no-daemon-sync-exec.test.ts`. The gate's + static regex scans for `execSync(`/`spawnSync(`/`Bun.spawnSync(`/ + `Bun.sleepSync(`; `Bun.spawn(` + `setTimeout` + `process.kill` match none. + +### Shape + +``` +resolveUserPath(log): // async, Promise + override = getSetting("rt.daemonPath") // sync read + if override non-empty: + result = override; source = "override" + else: + raw = await probe(shell, "-lc", 5000) // base: detached pgroup, hard timeout, race + base = validate(raw) ? raw : baseline // reject fish-space / empty / launchd-baseline + ov = await probe(shell, "-ilc", 3000, // overlay: stdin=/dev/null, TERM=dumb, + { stdin: "/dev/null", env: { TERM: "dumb" } }) // same detached pgroup kill + result = union(base, absoluteDirsOf(ov)) // append overlay's unique absolute dirs + source = base-rejected ? "baseline" : ("base" + overlay-unioned? "+overlay" : "") + warnIfMissing(result, [node, git, bun, pnpm]) // one warn line, names rt.daemonPath + log.info({ source, entries, hasNode, hasGit, hasBun, hasPnpm }, "PATH resolved") + return result +``` + +`probe` is a single injectable seam (default: the real detached `Bun.spawn` + +pgroup-kill + deadline-race), used for both the base and the overlay, so tests +never spawn a real shell. `absoluteDirsOf` rejects a non-colon / whitespace / +non-absolute overlay value (returns `[]`, logs the skip). + +### Tests (`lib/daemon/__tests__/user-path.test.ts`) + +- fish-style space-separated base output is rejected → baseline kept + warn. +- a hanging base probe (injected seam that never settles) → resolves within the + timeout, returns baseline, logs the killed/fallback reason. +- an `exec`-into-empty `.zshrc` (base returns "") → baseline kept, distinguishable in the log. +- `rt.daemonPath` set → neither probe called, value used verbatim. +- a valid colon base PATH → accepted; `hasNode` etc. reflected; no warn. +- interactive overlay contributes a `.zshrc`-only dir (e.g. an nvm/pyenv dir) → + it is appended after the base entries, unique-only, order preserved. +- a hanging or garbage overlay → skipped with one warn; the base result is kept + unchanged (overlay never regresses the base). +- missing-tool warn fires once when node/git absent. +- `probeTools` existing coverage retained. + +--- + +## 6.2 · Machine key from a stable identifier (S071) + +### The problem + +`machineKey()` (`lib/rt-paths.ts:140`, mirrored byte-for-byte in +`packages/rt-client/src/settings/paths.ts`) reads the `~/.mattstack/machine-key` +pin file, and *falls back to a slug of `os.hostname()`* when no pin exists. The +machine settings store lives at `user/local//settings.local.jsonc`. +Rename the Mac and the key changes; the machine's settings silently vanish. +`machineKey()` is on a hot synchronous path (every `getSetting` calls +`readStores()` → `machineSettingsPath()` → `machineKey()`), so it must stay +sync and subprocess-free. + +Today the only pin writer is `rt home init` (`lib/home/init-exec.ts:103` +`writeMachineKey`), and it writes `config.machineKey`, which *defaults to +`machineKey()` itself* (`commands/home.ts:552`) ... i.e. the hostname slug. So +even a set-up machine self-pins its hostname slug rather than a stable id. + +### Decision + +Keep `machineKey()` exactly as-is (sync, pin-first, hostname fallback ... no +subprocess, so rt-paths ↔ rt-client parity is preserved). Establish a *stable* +pin at setup time, data-preservingly: + +- New `async stableMachineId(): Promise` (rt-side, e.g. + `lib/home/machine-id.ts`): `Bun.spawn(["ioreg", "-rd1", "-c", + "IOPlatformExpertDevice"])` (async, hard timeout, detached), parse + `"IOPlatformUUID" = ""`, slug it through `isSafeMachineKeySegment`. + Returns `null` on any failure (non-mac, CI, parse miss). +- `rt home init`'s default key becomes `seams.key ?? (await resolveInitialMachineKey())`: + 1. pin file already exists → return `machineKey()` (its current value; **no change**). + 2. else the hostname-slug machine store already has data on disk → return the + hostname slug (freeze the current key; this is the "migrate the + hostname-keyed section" step, done with zero data movement). The predicate + is the one `gatherHomeState` already uses ... `profileDirPresent` + (`commands/home.ts:152`, `probes.exists(join(home, "user", "local", + ))`) ... except the freeze guard requires the dir to be **non-empty** + (an empty stub is a fresh machine, not data to preserve), so it checks + existence AND at least one entry (e.g. `settings.local.jsonc`), not bare + existence. + 3. else (truly fresh) → `(await stableMachineId()) ?? machineKey()` (hardware + UUID for new installs; hostname slug as the last resort). + The interactive picker's explicit key still wins (`seams.key`). + +**Data-preserving + idempotent:** an existing pin is never rewritten; a machine +with existing data keeps its current key (frozen); only a genuinely fresh +machine gets the hardware UUID. `machineKey()` reads the same value before and +after, so there is no within-boot key drift on any machine that has data. + +**Deliberately out of this item:** no daemon-boot pin write (the write fence +grants only the one `resolveUserPath` statement in `daemon.ts`; adding a call +there is out of bounds, and setup is the correct owner of the pin anyway). No +hot-path warn (it would diverge the rt-paths ↔ rt-client mirror). A machine run +without `rt home init` therefore keeps the live hostname slug; this is a +dev-only residual, and the spec-review gate can add a setup-row surface for it +if wanted. + +### Tests + +- fresh (no pin, no data) + injected `stableMachineId` → pin written with the + stable id. +- existing pin → `resolveInitialMachineKey` returns it unchanged. +- existing hostname-slug data, no pin → pin written with the hostname slug (frozen). +- `stableMachineId` parses a real `ioreg` fixture; returns null on a failing/empty probe. + +--- + +## 6.2 · Dev-mode wrapper marker (S020, S067) + +### The problem + +`currentMode()` (`lib/dev-mode.ts:76`) classifies `~/.local/bin/rt` as "dev" +whenever its first two bytes are `#!`. Any foreign `#!` script parked there +reads as dev and the prod daemon parks forever. Its companion +`isDevModeWrapper()` (`lib/deps/links.ts:46`) treats any `#!` file whose line 2 +does not start with `LINK_TAG` as our dev wrapper, so `rt deps link rt --force` +refuses to replace a foreign script (`dev-mode-owns-rt`). The current wrapper +(`renderDevModeWrapper()`, `commands/settings.ts:510`) carries no marker ... its +line 2 is a real `export PATH=...`. + +### Decision + +Mirror the `LINK_TAG` pattern (`lib/deps/resolve.ts:113`, +`# mattstack-link:`). Add a marker line 2 to new wrappers and centralize +detection so the two call sites cannot diverge (the audit's "fix both +together"). + +- `renderDevModeWrapper()` emits `# mattstack-dev-mode` as line 2 (after the + shebang, before the `export PATH`). +- New shared `isDevModeWrapperContent(prefix): boolean` (in `lib/dev-mode.ts`, + imported by `lib/deps/links.ts`): true iff `prefix` starts with `#!` **and** + either line 2 starts with `# mattstack-dev-mode` (new wrappers) **or** + `prefix.includes("RT_LAUNCH_CWD")` (the legacy markerless body's unique tell, + which is line 3). A foreign `#!` script has neither → false → classified + prod / eligible for replacement. +- **Read a bounded prefix, never the whole file.** In prod, `~/.local/bin/rt` + is a symlink to the compiled binary inside the app bundle, and `readFileSync` + follows the symlink ... reading the whole file would slurp a multi-MB binary. + Both detectors read only the first few KB (e.g. an `openSync` + + `readSync(4096)`, extending the current `currentMode()` 2-byte read), which + is more than enough for the marker on line 2 and the `RT_LAUNCH_CWD` tell on + line 3. `currentMode()` reads that prefix and delegates to + `isDevModeWrapperContent`; `isDevModeWrapper()` in `links.ts` reads a bounded + prefix (not `p.readFile`'s full read) and delegates too. + +**Backward-compatible:** existing dev machines whose wrapper predates the +marker still classify as dev (via the `RT_LAUNCH_CWD` tell), so no re-link is +needed and no dev machine flips to prod. This matters: a wrong flip is the +dev/prod standoff that `rt` daemon verbs cannot themselves repair. + +### Tests + +- a foreign `#!/bin/sh\necho hi` at the wrapper path → `currentMode()` prod, + `isDevModeWrapper()` false. +- a legacy markerless wrapper (`RT_LAUNCH_CWD` body) → dev / true. +- a new marked wrapper → dev / true. +- a `LINK_TAG` link → not a dev wrapper. +- the wrapper path is a symlink to a large (>4KB) binary-shaped file → prod, + and only the bounded prefix is read (no whole-file slurp). + +--- + +## 6.3 · Unsupported platform at setup (R051) + +### Decision + +Add an architecture row to `lib/setup/validators/mac.ts`, mirroring +`macosVersionRow`'s honesty ruling: + +```ts +async function archRow(p: Probes): Promise { + const base = { id: "tool.arch", kind: "tool" as const, title: "Processor", + why: "mattstack ships an Apple-silicon (arm64) build; Intel Macs are not supported.", required: true }; + const res = await p.exec(["uname", "-m"]); + const arch = res.stdout.trim(); + if (res.code !== 0 || !arch) return row({ ...base, status: "error", detail: "Could not determine your processor" }); + if (arch === "arm64") return row({ ...base, status: "ready", detail: "Apple silicon (arm64)" }); + return row({ ...base, status: "invalid", detail: `${arch}: Apple silicon (arm64) required` }); +} +``` + +`macRows()` returns `[macos, clt, archRow, pathRow]` (arch and macos/clt probe +in the same `Promise.all`). A failed probe reports `error` ("couldn't +determine"), never `invalid` ... same ruling as the macOS-version row. + +### Tests + +- `uname -m` = `arm64` → ready. +- `= x86_64` → invalid with an arm64 message. +- probe fails (code !== 0) → error, not invalid. + +--- + +## 6.4 · First-run honesty + +### S090 · "not provisioned" vs "git missing" + +In `lib/daemon/home-snapshot.ts init()` (line 357), before the +`git rev-parse --is-inside-work-tree` spawn, `existsSync(deps.repoDir)`. When +the dir is absent set a distinct `disabledReason` (`"not-provisioned"`) and a +`warn` naming `rt home init`. The existing `exitCode === -1` branch stays for a +genuine spawn failure (git truly missing from PATH). Pure code change. + +### R043 · git identity checked once + +Before the first `home-snapshot` commit (and in +`lib/home/init-exec.ts commitInitialUserRepo`, which has the same gap), check +identity once: `git config user.name` and `git config user.email` via +`deps.exec`. If either is empty, set a distinct `disabledReason` +(`"no-git-identity"`), log one actionable `warn` (`git config --global +user.name/…user.email`), and skip committing (a commit would fail anyway). +Checking config directly is cleaner than parsing "Author identity unknown" out +of stderr and gives an actionable message once, not per cycle. + +### S070 (sops half) · timeout on the secrets spawn + +`createRealSecretsExecSeam` in `lib/secrets/store.ts` (the sops `Bun.spawn`, +awaited via `Promise.all([..., proc.exited])`) gains the exact pattern +`lib/home/age-key.ts` already uses: a `DEFAULT_SECRETS_TIMEOUT_MS` (30s), a +`setTimeout` → `proc.kill()` (SIGTERM then SIGKILL grace), and a distinguished +error (`SecretsTimeoutError`) so a locked-keychain hang surfaces as a timeout +rather than poisoning any cache with a generic failure. Async already; just add +the timer + distinguished error. + +### S069 · branch_cache keyed by repo + branch + +`branch_cache` has `branch TEXT PRIMARY KEY` with `repo` as a nullable +attribute column (already holding the serialized repo identity post the wave-1 +`rekeyBranchCacheTable` migration). A same-name branch in a second repo +overwrites the first. Per `docs/repo-identity.md`, a state.db table keys on the +**serialized wire identity** (`remote:host%2Fpath` / `path:%2F…`). + +**Fix without a schema bump:** make the primary-key *value* the composite +`${serializedIdentity}:${branch}`, reusing the existing `branch TEXT PRIMARY +KEY` column (no DDL change). This is safe to parse because a git branch name +cannot contain `:` (git `check-ref-format`), so the bare branch is always +`key.slice(key.lastIndexOf(":") + 1)` and the identity is everything before it. + +**Read contract (unchanged externally).** `cache:read` and every by-branch +lookup keep resolving a **bare branch name**: scoped to the caller's repo when +the repo is known (compose the exact `${identity}:${branch}` key), falling back +to a **suffix match** across repos (`key.endsWith(":" + branch)`) when it is +not. The CLI, board, and tray therefore see bare branch names exactly as today +and never regress; the composite key is an internal storage detail. + +**Store API** (`lib/state/branch-cache.ts`): shared `composeKey(identity, +branch)`, `branchOf(key)`, `identityOf(key)` helpers (split on the LAST `:`, +safe because branches contain none). `put(identity, branch, entry)` / +`get(identity, branch)` compose the exact key; `getByBranch(branch)` does the +suffix-match fallback for callers without an identity. `entries` stays a map, +now keyed by the composite; iterating consumers use `branchOf`/`identityOf`. + +**Consumer sites** (each gets a test): + +- **Writers** ... `lib/enrich.ts` (`writeEnriched`/`fetchAndCache`/ + `refreshAllMRs` and the standalone `import.meta.main` entry) already hold the + identity (`serializeIdentity(await deriveRepoIdentity(...))` via the local + `lib/settings/identity.ts` barrel); they call `put(identity, branch, …)` and + compose keys for their own `branch in store.entries` / lookup checks. +- **`lib/notifier.ts`** ... `state.branches` and the `fired` set key off the + same map keys as `cacheEntries` (`ctx.cache.entries`), and + `pruneFiredForEvictedBranches(fired, Object.keys(cacheEntries))` (line 887) + compares them directly. Carrying the composite key through + `state.branches`/`fired`/`detectBranchTransitions` makes the fired-state + correctly repo-scoped for free (two repos' same-named branch no longer share + one fired entry); `branchOf(key)` is used only where a human-readable branch + name is shown in the notification. **Test:** two repos, same branch name → + independent fired-state; evicting one repo's branch does not prune the + other's. +- **`lib/daemon/worktree-reconciler.ts`** ... `for (const [branch, entry] of + Object.entries(cacheEntries))` (line 594) treats the map key as a bare branch + and builds `mrState` keyed `:` (comment line 262). It switches + to `branchOf(key)` for the branch and scopes to the repo being reconciled via + `identityOf(key)` (the reconciler always knows its repo). **Test:** the + reactor builds `mrState` only from the reconciled repo's entries; a same-named + branch in another repo does not leak in. +- **`lib/daemon/freshness.ts`** ... direct lookups + `ctx.cache.entries[pr.sourceBranch]` (545), `[k.ref]` (579), `[branch]` (639) + and the `Object.entries(ctx.cache.entries)` iterations (505, 657, 703) run + per repo (the enclosing loop carries `repoName`/`repoPath` → identity). Each + direct lookup composes the exact key; each iteration filters by + `identityOf(key)` and uses `branchOf(key)`. **Test:** a branch present in two + repos resolves to the correct repo's entry. +- **`lib/daemon/handlers/cache.ts` (`cache:read`)** ... returns bare-branch- + keyed data per the read contract: exact-key when the request names a repo, + suffix-match otherwise. **Test:** `cache:read` returns bare branch names, and + a repo-scoped read never returns another repo's same-named branch. +- **`commands/status/data.ts`** (raw `SELECT branch,…` for display) ... shows + `branchOf(row.branch)`, taking identity from the row's `repo` column. +- **`lib/daemon/discussions-poller.ts`** ... iterates `Object.values(entries)` + (line 73), never keying by branch, so it **self-heals** and needs no change. +- `boot-migrate.ts`'s existing `repo`-column rekey is untouched and coexists. + +**Migration:** none. Old bare-branch rows become unused and age out via the +existing GC; the cache self-heals. Idempotent, no backfill, no schema bump. + +### Tests + +- S090: missing `repoDir` → `disabledReason "not-provisioned"`, message names `rt home init`; present-but-not-a-repo and git-missing branches unchanged. +- R043: empty `user.name`/`user.email` → `no-git-identity`, one warn, no commit; identity present → commits normally. +- S070: an injected hanging sops seam → `SecretsTimeoutError` within the timeout; the daemon/caller never blocks. +- S069: two repos, same branch name → two distinct rows; lookups resolve per repo; `branchOf` recovers the display name; a branch containing no `:` round-trips. + +--- + +## Task decomposition (preview for the plan) + +Independent enough to parallelize; 6.1 is the spine. + +1. **6.1a** ... `rt.daemonPath` registry key + `bun run build` in rt-client (unblocks 6.1b's override read). +2. **6.1b** ... rewrite `resolveUserPath` (async probe, detached pgroup kill, fish-aware, nvm overlay, validation, override, missing-tools warn) + its tests. +3. **6.1c** ... `daemon.ts:163` one-line `await`; remove the `user-path.ts` allowlist entry; gate stays green. +4. **6.2a** ... `stableMachineId` + `resolveInitialMachineKey` at `rt home init` + tests. +5. **6.2b** ... dev-mode marker: `renderDevModeWrapper` + shared `isDevModeWrapperContent` + both call sites + tests. +6. **6.3** ... `archRow` in `mac.ts` + tests. +7. **6.4a** ... S090 `existsSync` + R043 identity check in home-snapshot / init-exec + tests. +8. **6.4b** ... S070 sops timeout in `lib/secrets/store.ts` + test. +9. **6.4c** ... S069 composite branch-cache key. Wider blast radius (each with + a test): store API (`composeKey`/`branchOf`/`identityOf`/`getByBranch`) → + writers (`enrich.ts`) → `notifier.ts` (fired-state + prune) → + `worktree-reconciler.ts` (mrState) → `freshness.ts` (direct lookups + + iterations) → `handlers/cache.ts` (`cache:read` read contract) → + `status/data.ts` (display). `discussions-poller.ts` self-heals (no change). + Land the store + helpers first, then the consumers; keep the read contract + (bare branch out) intact at each step. + +## Verification (must pass) + +- `bun test lib commands packages scripts` green (worktree root). +- `bunx tsc --noEmit` zero errors. +- `bun test --preload ./e2e/setup.ts --timeout 60000 e2e/tests/daemon.test.ts e2e/tests/setup.test.ts e2e/tests/first-run.test.ts` green. +- `lib/__tests__/no-daemon-sync-exec.test.ts` green with the `user-path.ts` allowlist entry removed. +- `packages/rt-client`: `bun run build` before the final review (registry touched). +- Never start a daemon or run `dist/rt` against the real machine; any such run uses `env -i HOME=`. diff --git a/e2e/tests/daemon.test.ts b/e2e/tests/daemon.test.ts index a7dc3cdb..dca7bfb1 100644 --- a/e2e/tests/daemon.test.ts +++ b/e2e/tests/daemon.test.ts @@ -1,5 +1,5 @@ import { describe, test, expect, beforeAll, afterAll } from "bun:test"; -import { existsSync, mkdirSync, writeFileSync, readdirSync } from "fs"; +import { existsSync, mkdirSync, writeFileSync, readdirSync, readFileSync } from "fs"; import { join } from "path"; import { createTestHome, rt, RT_BINARY } from "../harness.ts"; @@ -214,3 +214,129 @@ describe("daemon", () => { }); }); }); + +// Additive coverage for the health snapshot (level/reasons + metrics + +// eventLoop) that computeHealth (lib/daemon/health.ts) attaches to every +// status-shaped surface, and for the heartbeat file the loop monitor writes +// alongside it. A live foreground daemon on a per-run free RT_API_PORT, same +// pattern as e2e/tests/events.test.ts and e2e/tests/endpoint.test.ts. +describe("health surfaces", () => { + let home: string; + let cleanup: () => void; + let apiPort = 0; + let daemon: ReturnType; + + /** Grab a free TCP port by binding port 0 and releasing it. */ + function freePort(): number { + const srv = Bun.serve({ port: 0, fetch: () => new Response("") }); + const port = srv.port; + srv.stop(true); + if (!port) throw new Error("failed to allocate a free port"); + return port; + } + + beforeAll(async () => { + apiPort = freePort(); + ({ path: home, cleanup } = createTestHome()); + // `rt daemon status` short-circuits to "not installed" before it ever + // reaches a liveness classification, install first. + await rt(["daemon", "install"], { home }); + const bunDir = join(process.execPath, ".."); + daemon = Bun.spawn([RT_BINARY, "--daemon"], { + env: { + HOME: home, + PATH: `${join(RT_BINARY, "..")}:${bunDir}:/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin`, + TERM: "xterm-256color", + RT_SKIP_SETUP: "1", + CI: "true", + RT_API_PORT: String(apiPort), + }, + stdout: "pipe", + stderr: "pipe", + }); + await waitForSocket(join(home, ".mattstack", "rt", "rt.sock")); + if (daemon.exitCode !== null) { + throw new Error( + `daemon process exited (code ${daemon.exitCode}) right after creating its socket ` + + `(port ${apiPort} collision or daemon boot crash; check the daemon's stderr).`, + ); + } + }, 60_000); + + afterAll(async () => { + try { daemon?.kill(); } catch { /* already gone */ } + await daemon?.exited; + cleanup(); + }); + + function expectHealthLevel(level: unknown) { + expect(["ok", "degraded", "unhealthy"]).toContain(level as string); + } + + test("rt daemon status --json carries health, metrics, and eventLoop, additive to the existing fields", async () => { + const result = await rt(["daemon", "status", "--json"], { home }); + expect(result.exitCode).toBe(0); + const out = JSON.parse(result.stdout); + + // Pre-existing fields still present. + expect(out.ok).toBe(true); + expect(out.state).toBe("running"); + expect(typeof out.data.pid).toBe("number"); + expect(typeof out.data.watchedRepos).toBe("number"); + + // New blocks. + expectHealthLevel(out.data.health.level); + expect(Array.isArray(out.data.health.reasons)).toBe(true); + expect(typeof out.data.metrics.rss).toBe("number"); + expect(typeof out.data.eventLoop.maxLagMs).toBe("number"); + }, 30_000); + + test("GET /api/status (tray:status) carries health, metrics, and eventLoop, additive to the existing fields", async () => { + const res = await fetch(`http://127.0.0.1:${apiPort}/api/status`); + expect(res.status).toBe(200); + const out = (await res.json()) as any; + + // Pre-existing fields still present. + expect(out.ok).toBe(true); + expect(typeof out.data.pid).toBe("number"); + expect(typeof out.data.memoryUsage).toBe("number"); + + // New blocks. + expectHealthLevel(out.data.health.level); + expect(Array.isArray(out.data.health.reasons)).toBe(true); + expect(typeof out.data.metrics.rss).toBe("number"); + expect(typeof out.data.eventLoop.maxLagMs).toBe("number"); + }, 15_000); + + test("ping over rt.sock carries the health level and eventLoop, additive to the existing fields", async () => { + const sockPath = join(home, ".mattstack", "rt", "rt.sock"); + const res = await fetch("http://localhost/ping", { + unix: sockPath, + signal: AbortSignal.timeout(5_000), + } as any); + const out = (await res.json()) as any; + + // Pre-existing fields still present. + expect(out.ok).toBe(true); + expect(typeof out.uptime).toBe("number"); + expect(typeof out.pid).toBe("number"); + + // New blocks. ping's `health` field is the level string itself (not an + // object), unlike status/tray:status where health.level is nested. + expectHealthLevel(out.health); + expect(typeof out.eventLoop.maxLagMs).toBe("number"); + }, 15_000); + + test("a heartbeat file appears under the isolated HOME's RT_DIR within a few seconds", async () => { + const heartbeatPath = join(home, ".mattstack", "rt", "daemon-heartbeat.json"); + const deadline = Date.now() + 5_000; + while (!existsSync(heartbeatPath) && Date.now() < deadline) { + await Bun.sleep(200); + } + expect(existsSync(heartbeatPath)).toBe(true); + + const hb = JSON.parse(readFileSync(heartbeatPath, "utf8")); + expect(typeof hb.at).toBe("number"); + expect(typeof hb.seq).toBe("number"); + }, 10_000); +}); diff --git a/lib/__tests__/daemon-config.test.ts b/lib/__tests__/daemon-config.test.ts index 32f06d15..dfc1a58a 100644 --- a/lib/__tests__/daemon-config.test.ts +++ b/lib/__tests__/daemon-config.test.ts @@ -10,6 +10,7 @@ import { afterEach, describe, expect, test } from "bun:test"; import { mkdirSync, rmSync, writeFileSync } from "fs"; import { join } from "path"; import { activeLaunchdLabel, resolveApiPort } from "../daemon-config.ts"; +import { DEV_MODE_TAG } from "../dev-mode.ts"; const WRAPPER_PATH = join(process.env.HOME!, ".local", "bin", "rt"); @@ -24,7 +25,7 @@ describe("activeLaunchdLabel", () => { test("resolves to com.mattstack.daemon.dev in dev mode (wrapper present)", () => { mkdirSync(join(process.env.HOME!, ".local", "bin"), { recursive: true }); - writeFileSync(WRAPPER_PATH, "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + writeFileSync(WRAPPER_PATH, `#!/bin/sh\n${DEV_MODE_TAG}\nexit 0\n`, { mode: 0o755 }); expect(activeLaunchdLabel()).toBe("com.mattstack.daemon.dev"); }); }); diff --git a/lib/__tests__/daemon-logger-level.test.ts b/lib/__tests__/daemon-logger-level.test.ts new file mode 100644 index 00000000..feb3564d --- /dev/null +++ b/lib/__tests__/daemon-logger-level.test.ts @@ -0,0 +1,30 @@ +import { test, expect } from "bun:test"; +import { resolveDaemonLogLevel, isPanicLine } from "../daemon-logger.ts"; + +test("RT_LOG_LEVEL env wins over the setting", () => { + expect(resolveDaemonLogLevel("debug", () => "warn")).toBe("debug"); +}); +test("setting is used when env is unset", () => { + expect(resolveDaemonLogLevel(undefined, () => "warn")).toBe("warn"); +}); +test("falls back to info when neither is set", () => { + expect(resolveDaemonLogLevel(undefined, () => undefined)).toBe("info"); +}); +test("a thrown setting read falls back to info instead of propagating", () => { + expect( + resolveDaemonLogLevel(undefined, () => { + throw new Error("unknown key: rt.logLevel"); + }), + ).toBe("info"); +}); +test("an unknown level falls back to info instead of reaching pino", () => { + expect(resolveDaemonLogLevel("verbose", () => undefined)).toBe("info"); +}); +test("a valid level still passes through", () => { + expect(resolveDaemonLogLevel("debug", () => undefined)).toBe("debug"); +}); +test("a panic-looking stderr line is escalated; ordinary noise is not", () => { + expect(isPanicLine("panic: runtime error")).toBe(true); + expect(isPanicLine("Uncaught Error: boom")).toBe(true); + expect(isPanicLine("rt: ignoring \"x\" from the team scope")).toBe(false); +}); diff --git a/lib/__tests__/daemon-logger-resilience.test.ts b/lib/__tests__/daemon-logger-resilience.test.ts new file mode 100644 index 00000000..1ee7aedc --- /dev/null +++ b/lib/__tests__/daemon-logger-resilience.test.ts @@ -0,0 +1,14 @@ +import { test, expect } from "bun:test"; +import { createDaemonLogger } from "../daemon-logger.ts"; +import { mkdtempSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; + +test("a stream write error does not throw out of log.info and flips loggerDegraded", async () => { + const dir = mkdtempSync(join(tmpdir(), "logres-")); + const handle = await createDaemonLogger({ logDir: dir, level: "info" }); + // Simulate a write failure by emitting 'error' on the underlying stream. + handle.stream.emit("error", Object.assign(new Error("no space"), { code: "ENOSPC" })); + expect(() => handle.logger.info("after enospc")).not.toThrow(); + expect(handle.loggerDegraded()).toBe(true); +}); diff --git a/lib/__tests__/daemon-status.test.ts b/lib/__tests__/daemon-status.test.ts index 17187267..9a9a83bb 100644 --- a/lib/__tests__/daemon-status.test.ts +++ b/lib/__tests__/daemon-status.test.ts @@ -185,6 +185,44 @@ describe("classifyDaemonStatus", () => { const v = classifyDaemonStatus({ installed: true, pingOk: false, pidAlive: false, pid: null }); expect(v.state).toBe("not-running"); }); + + + // ── Task 4: heartbeat-stale "stalled" detail + degraded eventLoop ── + + test("alive + ping-fail + ready + stale heartbeat => alive-not-serving 'stalled'", () => { + const now = 1_000_000; + const v = classifyDaemonStatus({ + installed: true, response: null, pingOk: false, pid: 42, pidAlive: true, + breadcrumb: { phase: "ready" }, + heartbeat: { at: now - 8000, seq: 3 }, heartbeatStaleMs: 6000, + now, + }); + expect(v.state).toBe("alive-not-serving"); + if (v.state === "alive-not-serving") { + expect(v.detail).toBe("stalled"); + expect(v.stalledForMs).toBe(8000); + } + }); + + test("alive + ready + FRESH heartbeat => 'wedged', not 'stalled'", () => { + const now = 1_000_000; + const v = classifyDaemonStatus({ + installed: true, response: null, pingOk: false, pid: 42, pidAlive: true, + breadcrumb: { phase: "ready" }, + heartbeat: { at: now - 500, seq: 9 }, heartbeatStaleMs: 6000, + now, + }); + expect(v.state === "alive-not-serving" && v.detail).toBe("wedged"); + }); + + test("degraded/unresponsive carries the ping-supplied eventLoop", () => { + const v = classifyDaemonStatus({ + installed: true, response: null, pingOk: true, pid: 42, + pingEventLoop: { maxLagMs: 1400, lastStallAt: 123, lastStallCmd: "mr:action", stalls: 2 }, + }); + expect(v.state).toBe("degraded"); + if (v.state === "degraded") expect(v.eventLoop?.maxLagMs).toBe(1400); + }); }); describe("needsLivenessProbe", () => { diff --git a/lib/__tests__/dev-mode.test.ts b/lib/__tests__/dev-mode.test.ts index e8dd0ea6..613c1c21 100644 --- a/lib/__tests__/dev-mode.test.ts +++ b/lib/__tests__/dev-mode.test.ts @@ -7,9 +7,9 @@ * activeLaunchdLabel() (which depends on it) rests on a verified foundation. */ import { afterEach, describe, expect, test } from "bun:test"; -import { existsSync, lstatSync, mkdirSync, readlinkSync, realpathSync, rmSync, writeFileSync } from "fs"; +import { existsSync, lstatSync, mkdirSync, readlinkSync, realpathSync, rmSync, symlinkSync, writeFileSync } from "fs"; import { dirname, join } from "path"; -import { currentMode, installRtBinary } from "../dev-mode.ts"; +import { currentMode, DEV_MODE_TAG, installRtBinary, isDevModeWrapperContent } from "../dev-mode.ts"; // The dev-mode wrapper path is resolved at CALL time from process.env.HOME // (mirrors lib/rt-paths.ts's home()), so this constant only needs to match @@ -30,7 +30,7 @@ describe("currentMode", () => { test("reports dev when the wrapper exists at ~/.local/bin/rt", () => { mkdirSync(join(process.env.HOME!, ".local", "bin"), { recursive: true }); - writeFileSync(WRAPPER_PATH, "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + writeFileSync(WRAPPER_PATH, `#!/bin/sh\n${DEV_MODE_TAG}\nexit 0\n`, { mode: 0o755 }); expect(currentMode()).toBe("dev"); }); @@ -45,7 +45,7 @@ describe("currentMode", () => { expect(currentMode()).toBe("prod"); // fakeHome/.local/bin/rt doesn't exist yet mkdirSync(join(fakeHome, ".local", "bin"), { recursive: true }); - writeFileSync(join(fakeHome, ".local", "bin", "rt"), "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + writeFileSync(join(fakeHome, ".local", "bin", "rt"), `#!/bin/sh\n${DEV_MODE_TAG}\nexit 0\n`, { mode: 0o755 }); expect(currentMode()).toBe("dev"); rmSync(fakeHome, { recursive: true, force: true }); @@ -55,6 +55,44 @@ describe("currentMode", () => { }); }); +describe("isDevModeWrapperContent", () => { + test("new marked wrapper is recognized", () => { + expect(isDevModeWrapperContent(`#!/bin/zsh\n${DEV_MODE_TAG}\nexport PATH=...\n`)).toBe(true); + }); + test("legacy markerless wrapper (RT_LAUNCH_CWD tell) is recognized", () => { + expect(isDevModeWrapperContent(`#!/bin/zsh\nexport PATH="x"\nexport RT_LAUNCH_CWD="$PWD"\n`)).toBe(true); + }); + test("foreign #! script is not a dev wrapper", () => { + expect(isDevModeWrapperContent(`#!/bin/sh\necho hi\n`)).toBe(false); + }); + test("a mattstack-link file is not a dev wrapper", () => { + expect(isDevModeWrapperContent(`#!/bin/sh\n# mattstack-link: rt\nexec ...\n`)).toBe(false); + }); + test("non-shebang content is not a dev wrapper", () => { + expect(isDevModeWrapperContent(`ELF\x00binary`)).toBe(false); + }); +}); + +describe("currentMode bounded read", () => { + afterEach(() => { + try { rmSync(WRAPPER_PATH); } catch { /* already absent */ } + }); + + test("a symlink to a >4KB binary-shaped file classifies as prod without reading the whole file", () => { + mkdirSync(join(process.env.HOME!, ".local", "bin"), { recursive: true }); + const bigBinaryPath = join(process.env.HOME!, "big-binary"); + // Mach-O-ish header followed by >4KB of non-marker filler, so a + // whole-file read (rather than a bounded prefix read) would still + // correctly classify this as prod -- the real proof is that this + // doesn't throw/hang and stays fast even against a multi-MB target. + const filler = Buffer.alloc(8192, 0x41); + writeFileSync(bigBinaryPath, Buffer.concat([Buffer.from([0xcf, 0xfa, 0xed, 0xfe]), filler])); + symlinkSync(bigBinaryPath, WRAPPER_PATH); + + expect(currentMode()).toBe("prod"); + }); +}); + describe("installRtBinary", () => { const BIN = join(process.env.HOME!, ".local", "bin"); afterEach(() => { try { rmSync(join(BIN, "rt")); } catch { /* absent */ } }); @@ -84,7 +122,7 @@ describe("installRtBinary", () => { test("currentMode reads through the link: a link to a script is dev, to a Mach-O is prod", () => { const script = join(process.env.HOME!, "wrapper.sh"); - writeFileSync(script, "#!/bin/zsh\nexit 0\n", { mode: 0o755 }); + writeFileSync(script, `#!/bin/zsh\n${DEV_MODE_TAG}\nexit 0\n`, { mode: 0o755 }); installRtBinary(script); expect(currentMode()).toBe("dev"); }); diff --git a/lib/__tests__/enrich-cache-identity.test.ts b/lib/__tests__/enrich-cache-identity.test.ts new file mode 100644 index 00000000..2b88cc2a --- /dev/null +++ b/lib/__tests__/enrich-cache-identity.test.ts @@ -0,0 +1,130 @@ +/** + * lib/enrich.ts: cold-start writes are keyed by the composite + * `${identity}:${branch}` (S069/Task 10), not the bare branch. Without this, + * two repos enriching a same-named branch overwrite each other's cache row. + * + * `loadSecrets` is mocked to report no API keys, so `fetchAndCache` never + * reaches GitLab/Linear; it still writes a cache row per branch (mr/ticket + * null), which is all this test needs to observe the key format. + */ +import { afterEach, beforeEach, describe, expect, mock, spyOn, test } from "bun:test"; +import { mkdtempSync, rmSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import * as linearModule from "../linear.ts"; +import { enrichBranches } from "../enrich.ts"; +import { closeStateDb, getBranchCacheStore } from "../state/index.ts"; +import { composeKey } from "../state/branch-cache.ts"; +import { createCacheHandlers } from "../daemon/handlers/cache.ts"; +import { fakeStore } from "../daemon/__tests__/fake-cache-store.ts"; + +// Captured before any mock.module call... mock.module mutates the live +// namespace object in place, so restoring with the ORIGINAL binding (not a +// re-import) is what undoes it for every other test file sharing this process. +const realDaemonClient = await import("../daemon-client.ts"); +const realDaemonQuery = realDaemonClient.daemonQuery; + +let home: string; +let realHome: string | undefined; + +beforeEach(() => { + realHome = process.env.HOME; + home = mkdtempSync(join(tmpdir(), "rt-enrich-identity-")); + process.env.HOME = home; + spyOn(linearModule, "loadSecrets").mockResolvedValue({ linearApiKey: undefined, gitlabToken: undefined } as any); +}); + +afterEach(() => { + mock.module("../daemon-client.ts", () => ({ + ...realDaemonClient, + daemonQuery: realDaemonQuery, + })); + mock.restore(); + closeStateDb(); + process.env.HOME = realHome; + rmSync(home, { recursive: true, force: true }); +}); + +describe("enrichBranches cold-start: repoName/key is the serialized remote identity", () => { + test("writes the cache row under composeKey(identity, branch), not the bare branch", async () => { + await enrichBranches( + [{ path: "/tmp/repo-a", branch: "main" }], + "git@gitlab.com:acme/repo-a.git", + { silent: true }, + ); + + const entries = getBranchCacheStore().entries; + const identityKeys = Object.keys(entries).filter((k) => k.endsWith(":main")); + expect(identityKeys.length).toBe(1); + expect(entries["main"]).toBeUndefined(); // never the bare branch + expect(entries[identityKeys[0]!]?.repoName).toBe(identityKeys[0]!.replace(/:main$/, "")); + }); + + test("two repos enriching the same branch name coexist (no collision)", async () => { + await enrichBranches( + [{ path: "/tmp/repo-a", branch: "main" }], + "git@gitlab.com:acme/repo-a.git", + { silent: true }, + ); + await enrichBranches( + [{ path: "/tmp/repo-b", branch: "main" }], + "git@gitlab.com:acme/repo-b.git", + { silent: true }, + ); + + const entries = getBranchCacheStore().entries; + const keyA = composeKey("remote:gitlab.com%2Facme%2Frepo-a", "main"); + const keyB = composeKey("remote:gitlab.com%2Facme%2Frepo-b", "main"); + expect(entries[keyA]).toBeDefined(); + expect(entries[keyB]).toBeDefined(); + expect(entries[keyA]).not.toBe(entries[keyB]); + }); + + test("no remote (path-only repo) degrades to a bare-branch key", async () => { + await enrichBranches( + [{ path: "/tmp/repo-local", branch: "scratch" }], + undefined, + { silent: true }, + ); + + const entries = getBranchCacheStore().entries; + expect(entries["scratch"]).toBeDefined(); + expect(entries["scratch"]?.repoName).toBeUndefined(); + }); +}); + +describe("enrichBranches daemon-first path: cache:read is repo-scoped", () => { + test("two tracked repos both have branch 'main': a repoIdentity-scoped read returns repo A's entry, never repo B's", async () => { + const identityA = "remote:gitlab.com%2Facme%2Frepo-a"; + const identityB = "remote:gitlab.com%2Facme%2Frepo-b"; + const entries: Record = { + [composeKey(identityA, "main")]: { + linearId: "A-1", ticket: null, mr: null, fetchedAt: Date.now(), repoName: identityA, + }, + [composeKey(identityB, "main")]: { + linearId: "B-1", ticket: null, mr: null, fetchedAt: Date.now(), repoName: identityB, + }, + }; + // The real cache:read handler over an in-memory store: this exercises the + // actual scoping logic, not a stand-in for it. + const handlers = createCacheHandlers({ + cache: fakeStore(entries), + refreshCache: async () => {}, + } as any); + + mock.module("../daemon-client.ts", () => ({ + ...realDaemonClient, + daemonQuery: async (cmd: string, payload: any) => { + if (cmd !== "cache:read") throw new Error(`unexpected daemon command: ${cmd}`); + return handlers["cache:read"]!(payload); + }, + })); + + const result = await enrichBranches( + [{ path: "/tmp/repo-a", branch: "main" }], + "git@gitlab.com:acme/repo-a.git", + ); + + expect(result[0]?.linearId).toBe("A-1"); + }); +}); diff --git a/lib/__tests__/intended-mode.test.ts b/lib/__tests__/intended-mode.test.ts index 626ebb0b..d747ccec 100644 --- a/lib/__tests__/intended-mode.test.ts +++ b/lib/__tests__/intended-mode.test.ts @@ -2,7 +2,7 @@ import { describe, test, expect, beforeEach, afterEach } from "bun:test"; import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; -import { resolveIntendedMode } from "../dev-mode.ts"; +import { resolveIntendedMode, DEV_MODE_TAG } from "../dev-mode.ts"; import { setSetting } from "../settings/write.ts"; let home: string; @@ -27,7 +27,7 @@ describe("resolveIntendedMode", () => { test("unset: derives from wrapper — script at ~/.local/bin/rt means dev", () => { mkdirSync(join(home, ".local", "bin"), { recursive: true }); - writeFileSync(join(home, ".local", "bin", "rt"), "#!/bin/sh\necho dev\n"); + writeFileSync(join(home, ".local", "bin", "rt"), `#!/bin/sh\n${DEV_MODE_TAG}\necho dev\n`); expect(resolveIntendedMode()).toEqual({ mode: "dev", provenance: "derived-from-wrapper" }); }); diff --git a/lib/__tests__/log-janitor.test.ts b/lib/__tests__/log-janitor.test.ts index 0332a04c..fa9098e3 100644 --- a/lib/__tests__/log-janitor.test.ts +++ b/lib/__tests__/log-janitor.test.ts @@ -81,4 +81,11 @@ describe("pruneLogs", () => { const { removed } = pruneLogs(dir, 14, now); expect(removed).toEqual(["tray.2026-08-01.log"]); }); + + test("readdir failure reports via onError instead of swallowing", () => { + const calls: string[] = []; + const bogus = join("/nonexistent-xyz", "rt", "logs"); + pruneLogs(bogus, 14, Date.now(), (phase) => calls.push(phase)); + expect(calls).toContain("readdir"); + }); }); diff --git a/lib/__tests__/notifier-fired-hygiene.test.ts b/lib/__tests__/notifier-fired-hygiene.test.ts index 88fb6615..6bca3089 100644 --- a/lib/__tests__/notifier-fired-hygiene.test.ts +++ b/lib/__tests__/notifier-fired-hygiene.test.ts @@ -18,6 +18,7 @@ import { afterEach, describe, expect, mock, spyOn, test } from "bun:test"; import * as notifierModule from "../notifier.ts"; import { closeStateDb, getNotifierStateBlob } from "../state/index.ts"; +import { composeKey } from "../state/branch-cache.ts"; interface NotifierStateShape { branches: Record; @@ -102,6 +103,34 @@ describe("checkAndNotify fired-ledger hygiene", () => { }); }); +describe("checkAndNotify: composite-key repo scoping (S069/Task 10)", () => { + test("evicting one repo's branch does not prune the other repo's fired key", () => { + spyOn(notifierModule, "notify").mockImplementation(() => {}); + + const keyA = composeKey("repo-a", "main"); + const keyB = composeKey("repo-b", "main"); + + // Cycle 1: both repos' "main" baseline (same bare branch, distinct + // composite keys...the collision this task fixes). + notifierModule.checkAndNotify({ [keyA]: mrEntry("running"), [keyB]: mrEntry("running") }, undefined, 123); + // Cycle 2: repo-a's pipeline fails; repo-b's stays running. Fires and + // persists repo-a's pipeline:failed key, keyed by the FULL composite key. + notifierModule.checkAndNotify({ [keyA]: mrEntry("failed"), [keyB]: mrEntry("running") }, undefined, 123); + + const firedKeyA = notifierModule.__test__.firedKey("pipeline:failed", keyA); + expect(readState().fired).toContain(firedKeyA); + + // Cycle 3: repo-a's branch is evicted (GC, or just absent this cycle); + // repo-b's "main" is still present under its own composite key. + notifierModule.checkAndNotify({ [keyB]: mrEntry("running") }, undefined, 123); + + const state = readState(); + expect(state.fired).not.toContain(firedKeyA); + // repo-b's own baseline snapshot survives untouched by repo-a's eviction. + expect(state.branches[keyB]).toBeDefined(); + }); +}); + describe("pruneFiredForEvictedBranches (unit)", () => { test("keeps only keys reconstructable from the live branch set", () => { const fired = new Set([ diff --git a/lib/command-tree-def.ts b/lib/command-tree-def.ts index 648a4388..c25b0a1b 100644 --- a/lib/command-tree-def.ts +++ b/lib/command-tree-def.ts @@ -664,6 +664,20 @@ export const TREE: Record = { { name: "Terminal", flag: "--terminal", type: "boolean", default: false, hint: "Tail logs in terminal via lnav or pino-pretty instead of opening the web viewer (alias -t)" }, ], }, + "log-level": { + description: "Show or set the daemon's live log level", + module: "./commands/daemon.ts", + fn: "setLogLevel", + omitBehavior: "list", + args: [ + { name: "Level", type: "select", hint: "Omit to show the current level", + options: [ + { value: "trace", label: "trace" }, { value: "debug", label: "debug" }, + { value: "info", label: "info" }, { value: "warn", label: "warn" }, + { value: "error", label: "error" }, + ] }, + ], + }, }, }, diff --git a/lib/daemon-client.ts b/lib/daemon-client.ts index da89c789..4dccf799 100644 --- a/lib/daemon-client.ts +++ b/lib/daemon-client.ts @@ -60,10 +60,12 @@ async function trySocketQuery( try { const hasBody = payload && Object.keys(payload).length > 0; + const headers: Record = { "X-RT-Client": `rt-cli/${process.pid}` }; + if (hasBody) headers["Content-Type"] = "application/json"; const response = await fetch(`http://localhost/${cmd}`, { unix: DAEMON_SOCK_PATH, method: hasBody ? "POST" : "GET", - headers: hasBody ? { "Content-Type": "application/json" } : undefined, + headers, body: hasBody ? JSON.stringify(payload) : undefined, signal: AbortSignal.timeout(timeoutMs), } as any); @@ -153,10 +155,12 @@ export async function trayRequest( try { const hasBody = init.body !== undefined; + const headers: Record = { "X-RT-Client": `rt-cli/${process.pid}` }; + if (hasBody) headers["Content-Type"] = "application/json"; const response = await fetch(`http://localhost${path}`, { unix: sockPath, method: init.method, - headers: hasBody ? { "Content-Type": "application/json" } : undefined, + headers, body: hasBody ? JSON.stringify(init.body) : undefined, signal: AbortSignal.timeout(init.timeoutMs ?? REQUEST_TIMEOUT_MS), } as any); @@ -334,6 +338,13 @@ export async function isDaemonRunning(): Promise { return response?.ok === true; } +/** Single-attempt ping that never triggers the restart machinery, so + * `rt daemon status` can probe liveness and read the daemon's eventLoop + * summary without spawning a daemon as a side effect. */ +export async function pingDaemon(timeoutMs?: number): Promise { + return (await trySocketQuery("ping", undefined, timeoutMs)).response; +} + // ─── MR action facade ──────────────────────────────────────────────────────── /** diff --git a/lib/daemon-logger.ts b/lib/daemon-logger.ts index 634e84b0..a8700c86 100644 --- a/lib/daemon-logger.ts +++ b/lib/daemon-logger.ts @@ -18,17 +18,33 @@ import pino, { type Logger } from "pino"; // @ts-ignore — no types shipped; the JS API is well-tested. import roll from "pino-roll"; import { dlopen, suffix, FFIType } from "bun:ffi"; -import { mkdirSync, openSync, closeSync, existsSync, statSync, renameSync } from "fs"; +import { mkdirSync, openSync, closeSync, existsSync, statSync, renameSync, writeSync } from "fs"; import { join } from "path"; import { logsDir } from "./rt-paths.ts"; +import { getSetting } from "./settings/resolve.ts"; + +/** Last-resort write straight to fd 2, bypassing pino entirely. Used only when the logger itself has failed or can't be trusted. */ +function rawStderr(text: string): void { + try { + writeSync(2, text); + } catch { + // Nothing left to do... even fd 2 is gone. + } +} export interface DaemonLoggerHandle { /** Root logger — use when no specific module scope applies. */ logger: Logger; + /** Underlying pino-roll write stream. Exposed as a test seam for simulating write errors. */ + stream: NodeJS.WritableStream; /** Returns a child logger that stamps `module: ` on every line. */ childLogger: (module: string) => Logger; /** Force a flush (best-effort; pino-roll's stream is sync but exposes flushSync). */ flush?: () => void; + /** True once the underlying stream has emitted an 'error' (e.g. ENOSPC); writes since then were swallowed, not lost silently. */ + loggerDegraded: () => boolean; + /** Count of errors that were observed and handled without crashing the daemon: demoted stderr noise plus steady-state recovered unhandledRejections. */ + recoveredErrorCount: () => number; } export interface CreateOptions { @@ -36,6 +52,46 @@ export interface CreateOptions { level?: pino.LevelWithSilent; } +const VALID_LOG_LEVELS = new Set(["trace", "debug", "info", "warn", "error", "fatal", "silent"]); + +/** + * Resolves the daemon's pino level: RT_LOG_LEVEL env, then the `rt.logLevel` + * setting, then "info". The setting read is try/catch-guarded because the + * `rt.logLevel` registry key may not exist yet (added in a later task), and + * the resolver may also run pre-boot; this must never throw. The resolved + * value is validated against pino's level set: an unrecognized value (a typo + * like "warning") must not reach pino's constructor, which throws on it. + */ +export function resolveDaemonLogLevel( + env: string | undefined, + fromSetting: () => string | undefined, +): string { + let resolved = env; + if (!resolved) { + try { + resolved = fromSetting(); + } catch { + // Setting unavailable (unknown key pre-registration, or resolver not + // ready yet)... fall through to the "info" default below. + } + } + if (!resolved || !VALID_LOG_LEVELS.has(resolved)) return "info"; + return resolved; +} + +const PANIC_PREFIXES = ["panic:", "fatal error:", "Uncaught ", "UnhandledPromiseRejection"]; + +/** True for stderr text that looks like a native/runtime panic, not ordinary noise (warnings, CLI messages). */ +export function isPanicLine(text: string): boolean { + return PANIC_PREFIXES.some((p) => text.startsWith(p)); +} + +// Counts errors observed and handled without crashing the daemon: demoted +// stderr lines (installCrashHandlers) plus steady-state recovered +// unhandledRejections. Module-scoped (one daemon process, one counter) rather +// than per-handle, matching the process-wide handlers that increment it. +let recovered = 0; + /** * Async factory — call once at daemon startup OR in each test. * pino-roll's default export is async (it stats the dir + sets up the writer). @@ -52,10 +108,22 @@ export async function createDaemonLogger(opts: CreateOptions): Promise { + degraded = true; + rawStderr(`daemon-logger: stream error ${err?.code ?? ""} ${err?.message ?? err}\n`); + }); + const logger = pino( { level: opts.level ?? "info", @@ -74,11 +142,14 @@ export async function createDaemonLogger(opts: CreateOptions): Promise logger.child({ module }), flush: () => { // pino's flushSync drains any buffered writes; safe to call repeatedly. try { logger.flush(); } catch { /* */ } }, + loggerDegraded: () => degraded, + recoveredErrorCount: () => recovered, }; } @@ -99,7 +170,10 @@ export async function getDaemonLogger(): Promise { if (!cachedPromise) { cachedPromise = createDaemonLogger({ logDir: logsDir(), - level: (process.env.RT_LOG_LEVEL as pino.LevelWithSilent | undefined) ?? "info", + level: resolveDaemonLogLevel( + process.env.RT_LOG_LEVEL, + () => getSetting("rt.logLevel").value, + ) as pino.LevelWithSilent, }).catch((err) => { // Clear the cache on failure — a transient cause (log dir momentarily // unwritable) may not recur, so a later call should retry rather than @@ -258,7 +332,9 @@ export function redirectNativeStderr(): void { * daemon that's already serving. No `booting` given preserves the old * always-log, never-exit behavior. * - process.stderr.write: intercept so console.error / anything writing to - * stderr lands in the JSON log instead of vanishing. + * stderr lands in the JSON log instead of vanishing, at `warn` (ordinary + * noise) or `error` (a panic-looking line per isPanicLine); demoted lines + * also count toward recoveredErrorCount(). */ export function installCrashHandlers( handle: DaemonLoggerHandle, @@ -268,18 +344,36 @@ export function installCrashHandlers( // Because the pino-roll stream is opened with sync:true, logger.fatal() // flushes immediately to the fd — no need for pino.final() here. + // + // The logger.*() calls below are wrapped in try/catch: a logging failure + // (e.g. the stream is degraded from ENOSPC) must not itself abort a crash + // handler and skip the exit it's here to guarantee. Only the logging is + // guarded, never the exit decision. process.on("uncaughtException", (err) => { - logger.fatal({ err }, "uncaughtException"); + try { + logger.fatal({ err }, "uncaughtException"); + } catch { + rawStderr(`uncaughtException (logger failed): ${err?.stack ?? err}\n`); + } process.exit(1); }); process.on("unhandledRejection", (reason) => { if (opts.booting?.()) { - logger.fatal({ err: reason }, "unhandledRejection during boot"); + try { + logger.fatal({ err: reason }, "unhandledRejection during boot"); + } catch { + rawStderr(`unhandledRejection during boot (logger failed): ${reason}\n`); + } process.exit(1); return; } - logger.error({ err: reason }, "unhandledRejection"); + recovered += 1; + try { + logger.error({ err: reason }, "unhandledRejection"); + } catch { + rawStderr(`unhandledRejection (logger failed): ${reason}\n`); + } }); // Intercept process.stderr.write so JS-side stderr writes land in the log. @@ -289,7 +383,14 @@ export function installCrashHandlers( try { const text = typeof chunk === "string" ? chunk : Buffer.from(chunk).toString(); const trimmed = text.replace(/\n+$/, ""); - if (trimmed.length > 0) logger.error({ source: "stderr" }, trimmed); + if (trimmed.length > 0) { + if (isPanicLine(trimmed)) { + logger.error({ source: "stderr" }, trimmed); + } else { + recovered += 1; + logger.warn({ source: "stderr" }, trimmed); + } + } } catch { // If anything in the logger fails, fall back to the original stderr. return origWrite(chunk, ...rest); diff --git a/lib/daemon-status.ts b/lib/daemon-status.ts index 2cdbbcfc..2f469515 100644 --- a/lib/daemon-status.ts +++ b/lib/daemon-status.ts @@ -25,19 +25,37 @@ export type DaemonStatusVerdict = | { state: "not-installed" } | { state: "running"; data: any } /** Up — proven by an answer or a ping — but `status` itself did not deliver. */ - | { state: "degraded"; reason: "error" | "unresponsive"; detail?: string; pid: number | null } + | { state: "degraded"; reason: "error" | "unresponsive"; detail?: string; pid: number | null; eventLoop?: StatusEventLoop } /** Ping fails, a live pid exists, and it's parked waiting for a different * flavor to hold rt.sock (park.ts): a flavor standoff, not a stuck boot. */ | { state: "parked"; pid: number; holderFlavor?: string } /** Ping fails but the pid is alive: still mid-boot, stuck after reaching - * ready, or alive-but-quarantined (recovered from a corrupt db). */ - | { state: "alive-not-serving"; pid: number; detail: "booting" | "wedged" | "quarantined" } + * ready, alive-but-quarantined (recovered from a corrupt db), or stalled + * (reached ready but the heartbeat file has gone stale). */ + | { state: "alive-not-serving"; pid: number; detail: "booting" | "wedged" | "quarantined" | "stalled"; stalledForMs?: number } /** No live pid, and the kv failure record shows >= N failures within the window. */ | { state: "crash-looping"; failures: number; reason: string } /** No live pid, and the most recent recorded exit was a boot throw (fewer than N failures). */ | { state: "boot-failed"; reason: string; phase: string } | { state: "not-running"; pid: number | null }; +/** Structural match for the daemon's heartbeat-file record; not imported from + * its owning module to avoid a cycle. */ +export interface HeartbeatInput { + at: number; + seq: number; +} + +/** Structural match for the ping-supplied event-loop summary, passed through + * on the degraded verdict for display. */ +export interface StatusEventLoop { + maxLagMs: number; + lastStallAt: number | null; + lastStallCmd: string | null; + stalls: number; +} + + /** The boot breadcrumb (`daemon-boot.json`), as classifyDaemonStatus needs it. Not * imported from supervision-state.ts, since that module's `Breadcrumb` interface is * intentionally unexported, and this shape only needs to be structurally @@ -75,6 +93,12 @@ export interface DaemonStatusInputs { supervision?: SupervisionState; /** Injected for deterministic crash-loop window checks under test; defaults to Date.now(). */ now?: number; + /** The daemon's heartbeat-file record, when the caller read one. */ + heartbeat?: HeartbeatInput | null; + /** How old `heartbeat` must be to count as stale. Defaults to 6000ms. */ + heartbeatStaleMs?: number; + /** Ping's event-loop summary, passed through onto a `degraded` verdict. */ + pingEventLoop?: StatusEventLoop; } const PHASE_ORDER: BootPhase[] = ["start", "events-db", "state-db", "api", "socket", "ready"]; @@ -82,14 +106,23 @@ const PHASE_ORDER: BootPhase[] = ["start", "events-db", "state-db", "api", "sock function classifyAliveNotServingDetail( breadcrumb: DaemonBreadcrumbInput | null | undefined, supervision: SupervisionState | undefined, -): "booting" | "wedged" | "quarantined" { + heartbeat: HeartbeatInput | null | undefined, + heartbeatStaleMs: number, + now: number, +): { detail: "booting" | "wedged" | "quarantined" | "stalled"; stalledForMs?: number } { const phase = breadcrumb?.phase; - if (!phase || PHASE_ORDER.indexOf(phase) < PHASE_ORDER.indexOf("ready")) return "booting"; + if (!phase || PHASE_ORDER.indexOf(phase) < PHASE_ORDER.indexOf("ready")) return { detail: "booting" }; + // A live heartbeat gone stale outranks the boot-failed check below: it is + // ground truth that the process stopped ticking, not a record of a past + // recovery it may be running fine behind. + if (heartbeat && now - heartbeat.at > heartbeatStaleMs) { + return { detail: "stalled", stalledForMs: now - heartbeat.at }; + } // Reached ready this run, but a prior attempt is on record as boot-failed, // most likely a corrupt-db quarantine (lib/state/db.ts, events-bus.ts) it // recovered from and is now stuck behind for an unrelated reason. - if (supervision?.lastExit?.kind === "boot-failed") return "quarantined"; - return "wedged"; + if (supervision?.lastExit?.kind === "boot-failed") return { detail: "quarantined" }; + return { detail: "wedged" }; } function countRecentFailures(supervision: SupervisionState, now: number, windowMs = 5 * 60_000): number { @@ -113,7 +146,7 @@ export function classifyDaemonStatus(opts: DaemonStatusInputs): DaemonStatusVerd // No reply. A plain ping is the next ground truth: a daemon busy enough to // blow the status timeout still answers a trivial ping. - if (pingOk) return { state: "degraded", reason: "unresponsive", pid }; + if (pingOk) return { state: "degraded", reason: "unresponsive", pid, eventLoop: opts.pingEventLoop }; // Ping failed too. From here, only pidAlive/breadcrumb/supervision (new // signals) can say more than "not running"; absent them, fall straight @@ -122,7 +155,9 @@ export function classifyDaemonStatus(opts: DaemonStatusInputs): DaemonStatusVerd if (breadcrumb?.flavor && intendedFlavor && breadcrumb.flavor !== intendedFlavor) { return { state: "parked", pid, ...(holderFlavor ? { holderFlavor } : {}) }; } - return { state: "alive-not-serving", pid, detail: classifyAliveNotServingDetail(breadcrumb, supervision) }; + const now = opts.now ?? Date.now(); + const d = classifyAliveNotServingDetail(breadcrumb, supervision, opts.heartbeat, opts.heartbeatStaleMs ?? 6000, now); + return { state: "alive-not-serving", pid, detail: d.detail, ...(d.stalledForMs !== undefined ? { stalledForMs: d.stalledForMs } : {}) }; } if (supervision) { diff --git a/lib/daemon.ts b/lib/daemon.ts index e12db4fb..acec94ad 100644 --- a/lib/daemon.ts +++ b/lib/daemon.ts @@ -36,6 +36,8 @@ import { SystemProcessScanner } from "./daemon/system-process-scanner.ts"; import { parkUntilIntended, probeSocketHolder, daemonFlavor } from "./daemon/park.ts"; import { evictStaleDaemon } from "./daemon/boot-reconcile.ts"; import { resolveUserPath } from "./daemon/user-path.ts"; +import { shortReqId, makeSuppressor } from "./daemon/command-attribution.ts"; +import { unknownCommandReply } from "./daemon/unknown-command.ts"; // Every state.db API is reached through the lib/state barrel, never through // ./state/db.ts directly: importing the barrel is what guarantees every // store module has registered its legacy-JSON importer before the one-shot @@ -50,7 +52,7 @@ import { runBootIdentityMigration } from "./daemon/boot-migrate.ts"; import { runCapture } from "./subprocess.ts"; import { buildRoutedHandlers } from "./daemon/command-router.ts"; import { startSocketServer } from "./daemon/socket-server.ts"; -import { startApiServer, withApiPortParkRetry, broadcast } from "./daemon/api-server.ts"; +import { startApiServer, withApiPortParkRetry, broadcast, apiWsClientCount } from "./daemon/api-server.ts"; import { loadCronConfig, startCron } from "./daemon/cron.ts"; import { startPollers } from "./daemon/pollers.ts"; import { startHomeSnapshot } from "./daemon/home-snapshot.ts"; @@ -58,8 +60,15 @@ import { startAgentStatusPoller } from "./daemon/agent-status-poller.ts"; import { initFreshness, reconcileFreshness, + getFreshnessSnapshot, type FreshnessEnv, } from "./daemon/freshness.ts"; +import { startLoopMonitor } from "./daemon/loop-monitor.ts"; +import { createHealthSampler } from "./daemon/health-sampler.ts"; +import { writeHeartbeat } from "./daemon/heartbeat-file.ts"; +import { computeHealth } from "./daemon/health.ts"; +import { isCrashLooping, readSupervisionState } from "./daemon/supervision-state.ts"; +import { setSettingsWarnSink } from "./settings/resolve.ts"; import { startDiscussionsPoller } from "./daemon/discussions-poller.ts"; import { createCleanup, installSignalHandlers } from "./daemon/shutdown.ts"; import { createEventsBus } from "./daemon/events-bus.ts"; @@ -119,6 +128,11 @@ redirectNativeStderr(); const loggerHandle = await getDaemonLogger(); const log = loggerHandle.logger; +// Route the settings resolver's dedup'd warn sink into structured daemon +// logging, so a hot-path getSetting on a disallowed-scope key surfaces once +// in the daemon log instead of the resolver's own console fallback. +setSettingsWarnSink((m) => log.warn({ src: "settings" }, m)); + // Wire uncaughtException + unhandledRejection through pino as early as the // logger allows: every module-scope side effect below this point // (createEventsBus, cron, home-snapshot, sweep timers) can throw, and this @@ -160,7 +174,7 @@ const systemProcessScanner = new SystemProcessScanner(); // runCapture forwards process.env explicitly (lib/subprocess.ts) because // Bun.spawn would otherwise ignore this assignment. { - const resolvedPath = resolveUserPath(log); + const resolvedPath = await resolveUserPath(log); if (resolvedPath) process.env.PATH = resolvedPath; } @@ -205,9 +219,9 @@ const cache: BranchCacheStore = { // Port scan cache, held as a single mutable ref so handler modules can read // fresh values without getters. The port poller mutates it in place. const portCacheRef = { ports: [] as PortEntry[], updatedAt: 0 }; -// Refresh-cycle status ref (last successful cache refresh), also mutated in -// place so status handlers read a live value. -const refreshStatusRef = { lastRefreshAt: 0 }; +// Refresh-cycle status ref (last cycle's outcome), also mutated in place so +// status handlers read a live value. +const refreshStatusRef = { lastRefreshAt: 0, lastSuccessAt: 0, failedRepos: 0, enrichErrors: 0 }; const startedAt = Date.now(); // Injected at compile time via `bun build --define RT_VERSION='"v1.x.x"'` (see cli.ts) — @@ -276,7 +290,8 @@ function logRetentionDays(): number { } setInterval(() => { try { - const { removed } = pruneLogs(logsDir(), logRetentionDays(), Date.now()); + const { removed } = pruneLogs(logsDir(), logRetentionDays(), Date.now(), + (phase, err, file) => log.warn({ err, phase, file }, "log prune step failed")); if (removed.length > 0) log.info({ removed: removed.length }, "pruned old surface logs"); } catch (err) { log.warn({ err }, "log prune failed"); @@ -285,7 +300,8 @@ setInterval(() => { // Boot-time sweep to handle frequent daemon restarts that would otherwise starve the daily interval. setTimeout(() => { try { - const { removed } = pruneLogs(logsDir(), logRetentionDays(), Date.now()); + const { removed } = pruneLogs(logsDir(), logRetentionDays(), Date.now(), + (phase, err, file) => log.warn({ err, phase, file }, "log prune step failed")); if (removed.length > 0) log.info({ removed: removed.length }, "pruned old surface logs"); } catch (err) { log.warn({ err }, "log prune failed"); @@ -348,6 +364,63 @@ const agentStatusPoller = startAgentStatusPoller({ log: loggerHandle.childLogger("agent-status"), }); +// In-flight command name, polled by the loop monitor to spot a handler that +// never returns. Declared before the monitor so its `currentCmd` closure +// captures this same mutable ref, not a stale one. +const currentCmd: { cmd: string | null } = { cmd: null }; + +// 5-min metrics log + the two cached signals health needs but is too costly +// to compute per call: the 1h rss-growth baseline and free disk under RT_DIR. +const healthSampler = createHealthSampler({ + log, + rtDir: RT_DIR, + wsClients: apiWsClientCount, + // Sourced exactly as handlerCtx.watchedConfigs is below... there is no bare + // `watchedConfigs` alias at this scope. + watchers: () => hooksGuard.watchedConfigs.size, + startedAt, +}); +healthSampler.sample(); // seed baseline/free immediately, don't wait 5min for the first reading +safeInterval(() => healthSampler.sample(), 5 * 60_000, "health-sample", log); + +// 250ms event-loop drift monitor; also writes the cross-process liveness +// heartbeat file every ~2s. Both timers are unref'd and db-free internally. +const loopMon = startLoopMonitor({ + log, + currentCmd: () => currentCmd.cmd, + onHeartbeat: (at, seq) => writeHeartbeat(RT_DIR, { at, seq }), +}); + +/** Not cached: computeHealth is pure/cheap, and every input it reads is + * already either a live ref or a fast getter, so recomputing per call keeps + * the snapshot honest without a staleness window to reason about. */ +function buildHealthSnapshot() { + const now = Date.now(); + const sup = readSupervisionState(); + const failuresLastHour = sup.recentFailures.filter((f) => f.at > now - 60 * 60_000).length; + return computeHealth({ + now, + uptimeMs: now - startedAt, + mem: process.memoryUsage(), + rssBaseline: healthSampler.rssBaseline(), + wsClients: apiWsClientCount(), + watchers: hooksGuard.watchedConfigs.size, + freshness: getFreshnessSnapshot(), + refresh: { + lastSuccessAt: refreshStatusRef.lastSuccessAt, + failedRepos: refreshStatusRef.failedRepos, + enrichErrors: refreshStatusRef.enrichErrors, + }, + refreshIntervalMs: 5 * 60_000, + eventLoop: { ...loopMon.stats }, + supervisionFailuresLastHour: failuresLastHour, + crashLooping: isCrashLooping(sup, now), + loggerDegraded: loggerHandle.loggerDegraded?.() ?? false, + recoveredErrorRateLastWindow: loggerHandle.recoveredErrorCount?.() ?? 0, + freeBytes: healthSampler.freeBytes(), + }); +} + // ─── Handler context + command routing ─────────────────────────────────────── const handlerCtx: HandlerContext = { @@ -361,6 +434,10 @@ const handlerCtx: HandlerContext = { checkAndRepairHooksPath: hooksGuard.checkAndRepairHooksPath, startWatchingRepo: hooksGuard.startWatchingRepo, refreshStatusRef, + getHealth: buildHealthSnapshot, + heartbeatSeq: loopMon.seq, + setLogLevel: (l) => { log.level = l; log.info({ level: l }, "log level changed"); }, + getLogLevel: () => log.level, }; /** Env bundle for the live-freshness subsystem. */ @@ -376,22 +453,51 @@ let routedHandlers: ReturnType | undefined; // policy: docs/daemon-supervision-design.md). let shuttingDownViaVerb = false; +const rejectSuppressor = makeSuppressor(60_000); +const SLOW_COMMAND_MS = 2000; + async function handleCommand(cmd: string, payload: any, signal?: AbortSignal): Promise { const t0 = Date.now(); + const reqId = shortReqId(); + const caller = payload && typeof payload._client === "string" ? payload._client : "unknown"; + currentCmd.cmd = cmd; try { const result = await routeCommand(cmd, payload, signal); + const durationMs = Date.now() - t0; if (result && result.ok === false) { - log.warn({ cmd, error: result.error, durationMs: Date.now() - t0 }, "command rejected"); + const key = `${cmd}|${result.error ?? ""}`; + const { emit, suppressed } = rejectSuppressor.check(key, Date.now()); + if (emit) { + log.warn( + { reqId, cmd, caller, error: result.error, durationMs, digest: redactDigest(payload), ...(suppressed ? { suppressed } : {}) }, + "command rejected", + ); + } + return { ...result, reqId }; + } + if (durationMs > SLOW_COMMAND_MS) { + log.info({ reqId, cmd, caller, durationMs }, "command handled (slow)"); } else { - log.debug({ cmd, durationMs: Date.now() - t0 }, "command handled"); + log.debug({ reqId, cmd, caller, durationMs }, "command handled"); } return result; } catch (err) { - log.error({ err, cmd, durationMs: Date.now() - t0 }, "command failed"); + log.error({ err, reqId, cmd, caller, durationMs: Date.now() - t0, digest: redactDigest(payload) }, "command failed"); throw err; + } finally { + currentCmd.cmd = null; } } +/** Loggable, secret-free summary of a command payload: top-level key names + * plus a whitelist of identifying fields safe to echo into logs. */ +function redactDigest(payload: any): Record { + if (!payload || typeof payload !== "object") return {}; + const keys = Object.keys(payload); + const pick = (k: string): Record => (payload[k] !== undefined ? { [k]: payload[k] } : {}); + return { keys, ...pick("repo"), ...pick("repoName"), ...pick("branch"), ...pick("iid"), ...pick("room") }; +} + async function routeCommand(cmd: string, payload: any, signal?: AbortSignal): Promise { const routed = routedHandlers?.[cmd]; if (routed) return routed(payload, signal); @@ -416,7 +522,7 @@ async function routeCommand(cmd: string, payload: any, signal?: AbortSignal): Pr return { ok: true, message: "shutting down" }; default: - return { ok: false, error: `unknown command: ${cmd}` }; + return unknownCommandReply(cmd, typeof RT_VERSION !== "undefined" ? RT_VERSION : "source"); } } @@ -433,6 +539,7 @@ const cleanup = (): void => { eventsBus.close(); homeSnapshot.stop(); agentStatusPoller.stop(); + loopMon.stop(); cleanupCore(); }; diff --git a/lib/daemon/__tests__/cache-read-bare-branch.test.ts b/lib/daemon/__tests__/cache-read-bare-branch.test.ts new file mode 100644 index 00000000..3292f775 --- /dev/null +++ b/lib/daemon/__tests__/cache-read-bare-branch.test.ts @@ -0,0 +1,70 @@ +/** + * cache:read's read contract (S069/Task 10): the store keys `ctx.cache.entries` + * by the composite `${identity}:${branch}` now, but cache:read's OUTPUT must + * stay keyed by the bare branch, never a composite key, so the CLI/board/ + * tray see exactly the same shape they always have. An absent `repoIdentity` + * falls back to a suffix match across repos; a present one scopes exactly. + */ +import { describe, test, expect } from "bun:test"; +import { createCacheHandlers } from "../handlers/cache.ts"; +import { composeKey } from "../../state/branch-cache.ts"; +import { fakeStore } from "./fake-cache-store.ts"; + +function makeCtx(entries: Record) { + const ctx = { + cache: fakeStore(entries), + refreshCache: async () => {}, + } as any; + return createCacheHandlers(ctx); +} + +describe("cache:read: bare-branch output", () => { + test("an unfiltered read returns bare-branch keys, never the store's composite keys", async () => { + const entries = { + [composeKey("remote:host%2Fa", "main")]: { linearId: "A", ticket: null, mr: null, fetchedAt: 1 }, + }; + const handlers = makeCtx(entries); + + const res = await handlers["cache:read"]!({}); + + expect(Object.keys(res.data)).toEqual(["main"]); + expect(res.data.main.linearId).toBe("A"); + }); + + test("a filtered read (branches list) resolves a bare branch by suffix match when repoIdentity is absent", async () => { + const entries = { + [composeKey("remote:host%2Fa", "main")]: { linearId: "A", ticket: null, mr: null, fetchedAt: 1 }, + }; + const handlers = makeCtx(entries); + + const res = await handlers["cache:read"]!({ branches: ["main"] }); + + expect(Object.keys(res.data)).toEqual(["main"]); + expect(res.data.main.linearId).toBe("A"); + }); + + test("two repos sharing a branch name: an unscoped read picks one entry, never crashes or merges them", async () => { + const entries = { + [composeKey("remote:host%2Fa", "main")]: { linearId: "A", ticket: null, mr: null, fetchedAt: 1 }, + [composeKey("remote:host%2Fb", "main")]: { linearId: "B", ticket: null, mr: null, fetchedAt: 2 }, + }; + const handlers = makeCtx(entries); + + const res = await handlers["cache:read"]!({ branches: ["main"] }); + + expect(Object.keys(res.data)).toEqual(["main"]); + expect(["A", "B"]).toContain(res.data.main.linearId); + }); + + test("an explicit repoIdentity scopes exactly, disambiguating two repos sharing a branch name", async () => { + const entries = { + [composeKey("remote:host%2Fa", "main")]: { linearId: "A", ticket: null, mr: null, fetchedAt: 1 }, + [composeKey("remote:host%2Fb", "main")]: { linearId: "B", ticket: null, mr: null, fetchedAt: 2 }, + }; + const handlers = makeCtx(entries); + + const res = await handlers["cache:read"]!({ branches: ["main"], repoIdentity: "remote:host%2Fb" }); + + expect(res.data.main.linearId).toBe("B"); + }); +}); diff --git a/lib/daemon/__tests__/cache-refresh-gc.test.ts b/lib/daemon/__tests__/cache-refresh-gc.test.ts index a959a4d1..d25af08f 100644 --- a/lib/daemon/__tests__/cache-refresh-gc.test.ts +++ b/lib/daemon/__tests__/cache-refresh-gc.test.ts @@ -40,6 +40,7 @@ import { createCacheRefresher } from "../cache-refresh.ts"; import { createProjectMRs } from "../project-mrs-store.ts"; import { createDiscussionsFileStore } from "../discussions-file-store.ts"; import { getBranchCacheStore, openStateDb, getNotifierStateBlob, setNotifierStateBlob, type CacheEntry } from "../../state/index.ts"; +import { composeKey } from "../../state/branch-cache.ts"; const DAY_MS = 24 * 60 * 60 * 1000; const CLEAN = "gcwire-clean"; @@ -133,7 +134,7 @@ function wireCycle(): Wiring { const refresh = createCacheRefresher({ log: silentLog, cache, - refreshStatusRef: { lastRefreshAt: 0 }, + refreshStatusRef: { lastRefreshAt: 0, lastSuccessAt: 0, failedRepos: 0, enrichErrors: 0 }, portCacheRef: { ports: [], updatedAt: 0 }, repoIndex: () => ({ [CLEAN]: tempDir("rt-gcwire-clean-"), [FLAKY]: tempDir("rt-gcwire-flaky-") }), broadcast: () => {}, @@ -158,11 +159,11 @@ describe("cache-refresh cycle: branch-cache GC", () => { await runCycle(); // Clean repo: aged out. Fresh row of the same repo: kept. - expect(cache.entries["gcwire-clean-stale"]).toBeUndefined(); - expect(cache.entries["gcwire-clean-fresh"]).toBeDefined(); + expect(cache.entries[composeKey(CLEAN, "gcwire-clean-stale")]).toBeUndefined(); + expect(cache.entries[composeKey(CLEAN, "gcwire-clean-fresh")]).toBeDefined(); // Flaky repo: `onError` fired, so the repo never entered succeededRepos // and NOTHING of its rows may be aged out this cycle. - expect(cache.entries["gcwire-flaky-stale"]).toBeDefined(); + expect(cache.entries[composeKey(FLAKY, "gcwire-flaky-stale")]).toBeDefined(); // NULL-repo rows are unattributable: prunable by age alone. expect(cache.entries["gcwire-orphan-stale"]).toBeUndefined(); }, 20_000); @@ -205,7 +206,7 @@ describe("cache-refresh cycle: branch-cache GC", () => { await runCycle(); - expect(cache.entries["gcwire-clean-stale"]).toBeUndefined(); + expect(cache.entries[composeKey(CLEAN, "gcwire-clean-stale")]).toBeUndefined(); const after = getNotifierStateBlob<{ fired: string[] }>({ fired: [] }); expect(after.fired).not.toContain(evictedKey); }, 20_000); diff --git a/lib/daemon/__tests__/caller-tag.test.ts b/lib/daemon/__tests__/caller-tag.test.ts new file mode 100644 index 00000000..ce08b779 --- /dev/null +++ b/lib/daemon/__tests__/caller-tag.test.ts @@ -0,0 +1,7 @@ +import { test, expect } from "bun:test"; +import { buildCorsHeaders } from "../api-server.ts"; + +test("CORS allow-headers advertises X-RT-Client so browser preflight passes", () => { + const h = buildCorsHeaders("https://example.com", true); + expect(h["Access-Control-Allow-Headers"]).toContain("X-RT-Client"); +}); diff --git a/lib/daemon/__tests__/command-attribution.test.ts b/lib/daemon/__tests__/command-attribution.test.ts new file mode 100644 index 00000000..de0c9135 --- /dev/null +++ b/lib/daemon/__tests__/command-attribution.test.ts @@ -0,0 +1,17 @@ +import { test, expect } from "bun:test"; +import { shortReqId, makeSuppressor } from "../command-attribution.ts"; + +test("shortReqId is short and unique-ish", () => { + const a = shortReqId(); const b = shortReqId(); + expect(a).toMatch(/^[a-z0-9]{6}$/); + expect(a).not.toBe(b); +}); + +test("suppressor logs first, then throttles with a running suppressed count", () => { + const s = makeSuppressor(60_000); + expect(s.check("mr:action|boom", 0)).toEqual({ emit: true, suppressed: 0 }); // first: log + expect(s.check("mr:action|boom", 1_000)).toEqual({ emit: false, suppressed: 1 }); // within window: silent + expect(s.check("mr:action|boom", 2_000)).toEqual({ emit: false, suppressed: 2 }); + expect(s.check("mr:action|boom", 61_000)).toEqual({ emit: true, suppressed: 2 }); // window elapsed: log with count + expect(s.check("mr:action|boom", 61_500)).toEqual({ emit: false, suppressed: 1 }); // count resets after an emit +}); diff --git a/lib/daemon/__tests__/discussions-semantics.test.ts b/lib/daemon/__tests__/discussions-semantics.test.ts index da23b6b3..126058b8 100644 --- a/lib/daemon/__tests__/discussions-semantics.test.ts +++ b/lib/daemon/__tests__/discussions-semantics.test.ts @@ -6,6 +6,7 @@ import { createDiscussionsFileStore, pruneDiscussionsStore } from "../discussion import { collectSweepTargets } from "../discussions-poller.ts"; import { createProjectMRs } from "../project-mrs-store.ts"; import { openStateDb, getBranchCacheStore } from "../../state/index.ts"; +import { composeKey } from "../../state/branch-cache.ts"; const tmp = (n: string) => join(mkdtempSync(join(tmpdir(), "rt-dsem-")), n); const tmpDb = () => openStateDb(tmp("state.db"), "cli"); @@ -97,7 +98,7 @@ describe("pruneDiscussionsStore", () => { // GC runs (repo "r" refreshed cleanly this cycle — gating per spec // "New: branch-cache GC"), evicting the stale row. cache.gc(new Set(["r"]), 30 * DAY_MS); - expect(cache.entries["stale-branch"]).toBeUndefined(); + expect(cache.entries[composeKey("r", "stale-branch")]).toBeUndefined(); // The union's branch-cache leg just shrank; the discussion is now a // true orphan and prunes — intended cleanup, not a regression. diff --git a/lib/daemon/__tests__/fake-cache-store.ts b/lib/daemon/__tests__/fake-cache-store.ts index b81c6d9f..3eac54a3 100644 --- a/lib/daemon/__tests__/fake-cache-store.ts +++ b/lib/daemon/__tests__/fake-cache-store.ts @@ -10,11 +10,15 @@ */ import type { BranchCacheStore, CacheEntry } from "../../state/index.ts"; +import { composeKey } from "../../state/branch-cache.ts"; export function fakeStore(entries: Record = {}): BranchCacheStore { return { entries, - put(branch, entry) { entries[branch] = entry; }, + // Mirrors the real store's put (Task 10): keyed by composeKey(entry.repoName, + // branch), not the bare branch, so a fixture pre-seeded under a composite + // key stays addressable at the same key after a consumer writes through it. + put(branch, entry) { entries[composeKey(entry.repoName, branch)] = entry; }, delete(branch) { delete entries[branch]; }, reload() { /* no db behind this fake — the map is the whole store */ }, gc() { /* GC is exercised against a real store, not here */ }, diff --git a/lib/daemon/__tests__/freshness-mapping.test.ts b/lib/daemon/__tests__/freshness-mapping.test.ts index 186e18e5..7beb1a73 100644 --- a/lib/daemon/__tests__/freshness-mapping.test.ts +++ b/lib/daemon/__tests__/freshness-mapping.test.ts @@ -12,6 +12,11 @@ import { createProjectMRs } from "../project-mrs-store.ts"; import type { InvalidationKey } from "@mattstack/glance"; import { fakeStore } from "./fake-cache-store.ts"; import { getBranchCacheStore, openStateDb } from "../../state/index.ts"; +import { composeKey } from "../../state/branch-cache.ts"; + +/** Composite key for a repo-x-attributed entry, matching what a real + * composeKey(entry.repoName, branch) put would produce. */ +const K = (branch: string) => composeKey("repo-x", branch); function tmpStorePath(): string { return join(mkdtempSync(join(tmpdir(), "rt-freshness-mapping-")), "state.db"); @@ -115,7 +120,7 @@ function key(kind: InvalidationKey["kind"], ref: string): InvalidationKey { describe("applyInvalidationBatch", () => { test("mr key with cached iid refetches that MR and updates the entry", async () => { const entries: Record = { - "feat-a": { mr: { iid: 42 }, ticket: { id: "T-1" }, linearId: "T-1", fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 42 }, ticket: { id: "T-1" }, linearId: "T-1", fetchedAt: 1, repoName: "repo-x" }, }; const { env, broadcasts, puts } = makeEnv(entries); const calls: any[] = []; @@ -132,18 +137,18 @@ describe("applyInvalidationBatch", () => { await applyInvalidationBatch(env, target, makeRunner(), [key("mr", "42")], noNotify); expect(calls).toEqual([["single", "g/p", 42]]); - expect(entries["feat-a"].mr.iid).toBe(42); - expect(entries["feat-a"].fetchedAt).toBeGreaterThan(1); - expect(entries["feat-a"].ticket).toEqual({ id: "T-1" }); // enrichment preserved - expect(entries["feat-a"].linearId).toBe("T-1"); - expect(puts).toEqual(["feat-a"]); + expect(entries[K("feat-a")].mr.iid).toBe(42); + expect(entries[K("feat-a")].fetchedAt).toBeGreaterThan(1); + expect(entries[K("feat-a")].ticket).toEqual({ id: "T-1" }); // enrichment preserved + expect(entries[K("feat-a")].linearId).toBe("T-1"); + expect(puts).toEqual(["feat-a"]); // store.put is still called with the BARE branch expect(broadcasts.filter((b) => b.type === "mr:update").length).toBe(1); - expect(broadcasts[0]!.data).toEqual({ repoName: "repo-x", mrs: { 42: entries["feat-a"].mr } }); + expect(broadcasts[0]!.data).toEqual({ repoName: "repo-x", mrs: { 42: entries[K("feat-a")].mr } }); }); test("mr key for another repo's iid is ignored", async () => { const entries: Record = { - "feat-a": { mr: { iid: 42 }, fetchedAt: 1, repoName: "other-repo" }, + [composeKey("other-repo", "feat-a")]: { mr: { iid: 42 }, fetchedAt: 1, repoName: "other-repo" }, }; const { env, puts } = makeEnv(entries); let called = false; @@ -166,8 +171,8 @@ describe("applyInvalidationBatch", () => { test("unknown mr key gap-fills null-mr branches after debounce", async () => { const entries: Record = { - "no-mr-branch": { mr: null, fetchedAt: 1, repoName: "repo-x" }, - "has-mr": { mr: { iid: 7 }, fetchedAt: 1, repoName: "repo-x" }, + [K("no-mr-branch")]: { mr: null, fetchedAt: 1, repoName: "repo-x" }, + [K("has-mr")]: { mr: { iid: 7 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); const batchCalls: string[][] = []; @@ -186,13 +191,13 @@ describe("applyInvalidationBatch", () => { await applyInvalidationBatch(env, target, runner, [key("mr", "999")], noNotify); expect(batchCalls.length).toBe(0); // debounced, not immediate await new Promise((r) => setTimeout(r, 40)); // > gapFillDebounceMs (10) - expect(batchCalls).toEqual([["no-mr-branch"]]); // only null-mr branches - expect(entries["no-mr-branch"].mr.iid).toBe(99); + expect(batchCalls).toEqual([["no-mr-branch"]]); // only null-mr branches (bare) + expect(entries[K("no-mr-branch")].mr.iid).toBe(99); }); test("disposed runner never arms gapFillTimer for an unknown mr key", async () => { const entries: Record = { - "no-mr-branch": { mr: null, fetchedAt: 1, repoName: "repo-x" }, + [K("no-mr-branch")]: { mr: null, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); let batchCalled = false; @@ -213,7 +218,7 @@ describe("applyInvalidationBatch", () => { test("unknown mr key with no null-mr branches skips the batch fetch entirely", async () => { const entries: Record = { - "has-mr": { mr: { iid: 7 }, fetchedAt: 1, repoName: "repo-x" }, + [K("has-mr")]: { mr: { iid: 7 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); let batchCalled = false; @@ -233,7 +238,7 @@ describe("applyInvalidationBatch", () => { test("notes key routes through refreshDiscussions override for cached iids only", async () => { const entries: Record = { - "feat-a": { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); const refreshed: Array<[string, number]> = []; @@ -263,7 +268,7 @@ describe("applyInvalidationBatch", () => { test("branch key refetches by branch; unknown branch and pipelines are ignored", async () => { const entries: Record = { - "feat-a": { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); const calls: any[] = []; @@ -290,7 +295,7 @@ describe("applyInvalidationBatch", () => { test("branch refetch returning null writes mr: null (MR deleted/never existed)", async () => { const entries: Record = { - "feat-a": { mr: { iid: 42 }, ticket: null, linearId: "", fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 42 }, ticket: null, linearId: "", fetchedAt: 1, repoName: "repo-x" }, }; const { env, puts } = makeEnv(entries); const target: RepoTarget = { @@ -302,14 +307,14 @@ describe("applyInvalidationBatch", () => { } as any, }; await applyInvalidationBatch(env, target, makeRunner(), [key("branch", "feat-a")], noNotify); - expect(entries["feat-a"].mr).toBeNull(); + expect(entries[K("feat-a")].mr).toBeNull(); expect(puts).toEqual(["feat-a"]); }); test("concurrent batch merges into pending and processes after current run", async () => { const entries: Record = { - "feat-a": { mr: { iid: 1 }, fetchedAt: 1, repoName: "repo-x" }, - "feat-b": { mr: { iid: 2 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 1 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-b")]: { mr: { iid: 2 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); const fetched: number[] = []; @@ -339,7 +344,7 @@ describe("applyInvalidationBatch", () => { test("duplicate keys within a batch are processed once", async () => { const entries: Record = { - "feat-a": { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); let count = 0; @@ -357,8 +362,8 @@ describe("applyInvalidationBatch", () => { test("a throwing fetch drops that key and continues with the rest", async () => { const entries: Record = { - "feat-a": { mr: { iid: 1 }, fetchedAt: 1, repoName: "repo-x" }, - "feat-b": { mr: { iid: 2 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 1 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-b")]: { mr: { iid: 2 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); const target: RepoTarget = { @@ -373,14 +378,14 @@ describe("applyInvalidationBatch", () => { } as any, }; await applyInvalidationBatch(env, target, makeRunner(), [key("mr", "1"), key("mr", "2")], noNotify); - expect(entries["feat-a"].mr.iid).toBe(1); // untouched - expect(entries["feat-b"].mr.iid).toBe(2); // still updated + expect(entries[K("feat-a")].mr.iid).toBe(1); // untouched + expect(entries[K("feat-b")].mr.iid).toBe(2); // still updated }); test("notify fires once per mutating batch with current userId", async () => { const entries: Record = { - "feat-a": { mr: { iid: 1 }, fetchedAt: 1, repoName: "repo-x" }, - "feat-b": { mr: { iid: 2 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 1 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-b")]: { mr: { iid: 2 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); let notifyCount = 0; @@ -428,7 +433,7 @@ describe("applyInvalidationBatch", () => { test("mr event, iid on a local branch: ONE fetch feeds branch entry AND project store", async () => { const store = pmrsStore(); const entries: Record = { - feat: { mr: { iid: 7 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat")]: { mr: { iid: 7 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); const calls: number[] = []; @@ -444,14 +449,14 @@ describe("applyInvalidationBatch", () => { ...noNotify, grantsFor: projectGrants, projectStore: store, }); expect(calls).toEqual([7]); // exactly one fetch, not two - expect(entries.feat.fetchedAt).toBeGreaterThan(1); // branch entry refreshed + expect(entries[K("feat")].fetchedAt).toBeGreaterThan(1); // branch entry refreshed expect(store.read("repo-x")!.mrs[7]).toBeDefined(); // project store also fed }); test("mr event, iid NOT in branchByIid but entry keyed by PR's sourceBranch has mr: null: ONE fetch feeds branch entry AND project store", async () => { const store = pmrsStore(); const entries: Record = { - "branch-42": { mr: null, fetchedAt: 1, repoName: "repo-x" }, + [K("branch-42")]: { mr: null, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); const calls: number[] = []; @@ -468,16 +473,16 @@ describe("applyInvalidationBatch", () => { ...noNotify, grantsFor: projectGrants, projectStore: store, }); expect(calls).toEqual([42]); // exactly one fetch - expect(entries["branch-42"].mr).not.toBeNull(); // branch entry filled via sourceBranch feed - expect(entries["branch-42"].mr.iid).toBe(42); - expect(entries["branch-42"].fetchedAt).toBeGreaterThan(1); + expect(entries[K("branch-42")].mr).not.toBeNull(); // branch entry filled via sourceBranch feed + expect(entries[K("branch-42")].mr.iid).toBe(42); + expect(entries[K("branch-42")].fetchedAt).toBeGreaterThan(1); expect(store.read("repo-x")!.mrs[42]).toBeDefined(); // project store also fed }); test("branch push with local entry + project grant: fetchPullRequestByBranch result reused, NO fetchSingleMR", async () => { const store = pmrsStore(); const entries: Record = { - "feat-a": { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, + [K("feat-a")]: { mr: { iid: 42 }, fetchedAt: 1, repoName: "repo-x" }, }; const { env } = makeEnv(entries); let singleCalled = false; @@ -740,6 +745,33 @@ describe("applyInvalidationBatch", () => { }); }); +// ─── S069/Task 10: composite-key collision safety ──────────────────────────── + +describe("freshness composite-key scoping (S069/Task 10)", () => { + test("a branch in two repos resolves to the right repo's entry", async () => { + const entries: Record = { + [composeKey("repo-a", "main")]: { mr: { iid: 1 }, fetchedAt: 1, repoName: "repo-a" }, + [composeKey("repo-b", "main")]: { mr: { iid: 2 }, fetchedAt: 1, repoName: "repo-b" }, + }; + const { env } = makeEnv(entries); + const target: RepoTarget = { + repoName: "repo-a", projectPath: "g/p", + provider: { + fetchSingleMR: async () => null, + fetchPullRequestByBranch: async () => fakePR(1, { sourceBranch: "main" }), + fetchPullRequestsByBranches: async () => new Map(), + } as any, + }; + + await applyInvalidationBatch(env, target, makeRunner(), [key("branch", "main")], noNotify); + + expect(entries[composeKey("repo-a", "main")].mr.iid).toBe(1); + // repo-b's same-named branch is a different composite key entirely, + // untouched by a refresh scoped to repo-a. + expect(entries[composeKey("repo-b", "main")].mr.iid).toBe(2); + }); +}); + // ─── RT-48 write-through (spec test 4) ─────────────────────────────────────── /** @@ -792,13 +824,13 @@ describe("write-through at updateEntry (RT-48)", () => { db.close(); // the daemon dies here — no flush, no shutdown hook, nothing const rebuilt = rebuildFromDb(dbPath); - expect(rebuilt["feat-a"]).toBeDefined(); - expect(rebuilt["feat-a"].mr.iid).toBe(42); - expect(rebuilt["feat-a"].fetchedAt).toBeGreaterThan(1); + expect(rebuilt[K("feat-a")]).toBeDefined(); + expect(rebuilt[K("feat-a")].mr.iid).toBe(42); + expect(rebuilt[K("feat-a")].fetchedAt).toBeGreaterThan(1); // Enrichment the events path never touches is preserved through the row. - expect(rebuilt["feat-a"].ticket).toEqual({ id: "T-1" }); - expect(rebuilt["feat-a"].linearId).toBe("T-1"); - expect(rebuilt["feat-a"].repoName).toBe("repo-x"); + expect(rebuilt[K("feat-a")].ticket).toEqual({ id: "T-1" }); + expect(rebuilt[K("feat-a")].linearId).toBe("T-1"); + expect(rebuilt[K("feat-a")].repoName).toBe("repo-x"); }); test("a refresh that clears an MR persists the null, not the stale MR", async () => { @@ -819,7 +851,7 @@ describe("write-through at updateEntry (RT-48)", () => { await applyInvalidationBatch(env, target, makeRunner(), [key("branch", "feat-b")], noNotify); db.close(); - expect(rebuildFromDb(dbPath)["feat-b"].mr).toBeNull(); + expect(rebuildFromDb(dbPath)[K("feat-b")].mr).toBeNull(); }); test("the store exposes no flush of any kind — persistence is not optional", () => { diff --git a/lib/daemon/__tests__/freshness-provider-rotation.test.ts b/lib/daemon/__tests__/freshness-provider-rotation.test.ts index bcda8181..093741ee 100644 --- a/lib/daemon/__tests__/freshness-provider-rotation.test.ts +++ b/lib/daemon/__tests__/freshness-provider-rotation.test.ts @@ -32,6 +32,19 @@ test("getRepoContext invalidates the cached provider when gitlabToken is removed expect(src).toMatch(/if \(cachedForToken\.token !== currentSecrets\.gitlabToken\)/); }); +test("getRepoContext resets the cached selfUsername when the token changes, not just userIdResolved", () => { + const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); + const tokenChangeBlock = src.match( + /if \(cachedForToken\.token !== currentSecrets\.gitlabToken\) \{\s*\n([\s\S]*?)\n\s*\}/, + ); + expect(tokenChangeBlock).not.toBeNull(); + // resolveSelfUsername() short-circuits on a truthy cached selfUsername + // (`if (selfUsername) return selfUsername;`), bypassing userIdResolved + // entirely, so the previous token's identity keeps being served unless + // selfUsername is cleared alongside it here. + expect(tokenChangeBlock![1]).toMatch(/selfUsername = null;/); +}); + test("reconcileFreshnessImpl drops a stale-token watch before skipping already-watched repos (S048/S049 fix 2)", () => { const src = readFileSync(resolve(import.meta.dir, "..", "freshness.ts"), "utf8"); const staleWatchDrop = src.match(/existing\.token !== secrets\.gitlabToken\)\s*stopWatch\(repoName\);/); diff --git a/lib/daemon/__tests__/health-sampler.test.ts b/lib/daemon/__tests__/health-sampler.test.ts new file mode 100644 index 00000000..4024d864 --- /dev/null +++ b/lib/daemon/__tests__/health-sampler.test.ts @@ -0,0 +1,15 @@ +// lib/daemon/__tests__/health-sampler.test.ts +import { test, expect } from "bun:test"; +import { rollRssBaseline } from "../health-sampler.ts"; + +test("rss baseline rolls forward only after the window elapses", () => { + // baseline null -> set on first sample + let b = rollRssBaseline(null, { rss: 100, at: 0 }, 60 * 60_000); + expect(b).toEqual({ rss: 100, at: 0 }); + // within the hour: unchanged + b = rollRssBaseline(b, { rss: 200, at: 30 * 60_000 }, 60 * 60_000); + expect(b).toEqual({ rss: 100, at: 0 }); + // after the hour: rolls to the new sample + b = rollRssBaseline(b, { rss: 250, at: 61 * 60_000 }, 60 * 60_000); + expect(b).toEqual({ rss: 250, at: 61 * 60_000 }); +}); diff --git a/lib/daemon/__tests__/health.test.ts b/lib/daemon/__tests__/health.test.ts new file mode 100644 index 00000000..b8a5f52c --- /dev/null +++ b/lib/daemon/__tests__/health.test.ts @@ -0,0 +1,91 @@ +// lib/daemon/__tests__/health.test.ts +import { test, expect } from "bun:test"; +import { computeHealth, type HealthInputs } from "../health.ts"; + +function base(): HealthInputs { + return { + now: 1_000_000, + uptimeMs: 60_000, + mem: { rss: 200 * 1024 * 1024, heapUsed: 50 * 1024 * 1024, external: 1 * 1024 * 1024 }, + rssBaseline: null, + wsClients: 0, + watchers: 3, + freshness: { "remote:gitlab/acme": { state: "live" } }, + refresh: { lastSuccessAt: 1_000_000 - 60_000, failedRepos: 0, enrichErrors: 0 }, + refreshIntervalMs: 5 * 60_000, + eventLoop: { maxLagMs: 20, lastStallAt: null, lastStallCmd: null, stalls: 0, currentlyStalled: false }, + supervisionFailuresLastHour: 0, + crashLooping: false, + loggerDegraded: false, + recoveredErrorRateLastWindow: 0, + freeBytes: 50 * 1024 * 1024 * 1024, + }; +} + +test("all-nominal inputs are ok with no reasons", () => { + const h = computeHealth(base()); + expect(h.level).toBe("ok"); + expect(h.reasons).toEqual([]); + expect(h.metrics.watchers).toBe(3); + expect(h.eventLoop.maxLagMs).toBe(20); +}); + +test("a degraded freshness watcher flips degraded and names refresh", () => { + const i = base(); + i.freshness = { "remote:gitlab/acme": { state: "degraded" } }; + const h = computeHealth(i); + expect(h.level).toBe("degraded"); + expect(h.reasons.some((r) => r.startsWith("refresh:"))).toBe(true); +}); + +test("failed repos in the last cycle flip degraded", () => { + const i = base(); + i.refresh = { lastSuccessAt: i.now - 60_000, failedRepos: 3, enrichErrors: 5 }; + expect(computeHealth(i).level).toBe("degraded"); +}); + +test("logger degraded flips unhealthy and names logging", () => { + const i = base(); + i.loggerDegraded = true; + const h = computeHealth(i); + expect(h.level).toBe("unhealthy"); + expect(h.reasons.some((r) => r.startsWith("logging:"))).toBe(true); +}); + +test("currently stalled event loop is unhealthy; unhealthy wins over a degraded signal", () => { + const i = base(); + i.eventLoop.currentlyStalled = true; + i.freshness = { r: { state: "degraded" } }; // also degraded + const h = computeHealth(i); + expect(h.level).toBe("unhealthy"); + expect(h.reasons[0]?.startsWith("event-loop:")).toBe(true); // unhealthy reasons first +}); + +test("critical disk is unhealthy; low disk is degraded", () => { + const crit = base(); crit.freeBytes = 50 * 1024 * 1024; + expect(computeHealth(crit).level).toBe("unhealthy"); + const low = base(); low.freeBytes = 300 * 1024 * 1024; + expect(computeHealth(low).level).toBe("degraded"); +}); + +test("stale refresh (older than 2 intervals) is degraded", () => { + const i = base(); + i.refresh = { lastSuccessAt: i.now - 11 * 60_000, failedRepos: 0, enrichErrors: 0 }; + expect(computeHealth(i).level).toBe("degraded"); +}); + +test("event-loop lag over the named threshold flips degraded", () => { + const i = base(); + i.eventLoop.maxLagMs = 600; + const h = computeHealth(i); + expect(h.level).toBe("degraded"); + expect(h.reasons.some((r) => r.startsWith("event-loop:"))).toBe(true); +}); + +test("event-loop lag under the named threshold stays ok", () => { + const i = base(); + i.eventLoop.maxLagMs = 400; + const h = computeHealth(i); + expect(h.level).toBe("ok"); + expect(h.reasons).toEqual([]); +}); diff --git a/lib/daemon/__tests__/heartbeat-file.test.ts b/lib/daemon/__tests__/heartbeat-file.test.ts new file mode 100644 index 00000000..7e3b0b67 --- /dev/null +++ b/lib/daemon/__tests__/heartbeat-file.test.ts @@ -0,0 +1,35 @@ +import { test, expect } from "bun:test"; +import { mkdtempSync, writeFileSync } from "fs"; +import { tmpdir } from "os"; +import { join } from "path"; +import { writeHeartbeat, readHeartbeat } from "../heartbeat-file.ts"; + +test("write then read round-trips", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + writeHeartbeat(dir, { at: 123, seq: 7 }); + expect(readHeartbeat(dir)).toEqual({ at: 123, seq: 7 }); +}); + +test("missing file reads as null", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + expect(readHeartbeat(dir)).toBeNull(); +}); + +test("corrupt file reads as null", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + writeFileSync(join(dir, "daemon-heartbeat.json"), "{not json"); + expect(readHeartbeat(dir)).toBeNull(); +}); + +test("a partial-but-valid-JSON object (missing `at`) reads as null", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + writeFileSync(join(dir, "daemon-heartbeat.json"), JSON.stringify({ seq: 1 })); + expect(readHeartbeat(dir)).toBeNull(); +}); + +test("a second write overwrites atomically", () => { + const dir = mkdtempSync(join(tmpdir(), "hb-")); + writeHeartbeat(dir, { at: 1, seq: 1 }); + writeHeartbeat(dir, { at: 2, seq: 2 }); + expect(readHeartbeat(dir)).toEqual({ at: 2, seq: 2 }); +}); diff --git a/lib/daemon/__tests__/home-snapshot.test.ts b/lib/daemon/__tests__/home-snapshot.test.ts index 33cc9df4..5e351720 100644 --- a/lib/daemon/__tests__/home-snapshot.test.ts +++ b/lib/daemon/__tests__/home-snapshot.test.ts @@ -43,10 +43,11 @@ function defaultResponders(opts: { pushStderr?: string; sha?: string; hasRemote?: boolean; + hasIdentity?: boolean; } = {}): Responder[] { const { isRepo = true, branch = "main", branchExit = 0, statusZ = "", commitExit = 0, addExit = 0, pushExit = 0, pushStderr = "", sha = "abc123", - hasRemote = true, + hasRemote = true, hasIdentity = true, } = opts; return [ (argv) => (argv[1] === "rev-parse" && argv[2] === "--is-inside-work-tree") @@ -60,6 +61,14 @@ function defaultResponders(opts: { : undefined, (argv) => (argv[1] === "status") ? { stdout: statusZ, stderr: "", exitCode: 0 } : undefined, (argv) => (argv[1] === "add") ? { stdout: "", stderr: "", exitCode: addExit } : undefined, + // git identity probe, checked right before either commit site runs... + // defaults to "configured" so every fixture not testing R043 stays green. + (argv) => (argv[1] === "config" && argv[2] === "user.name") + ? (hasIdentity ? { stdout: "rt test\n", stderr: "", exitCode: 0 } : { stdout: "", stderr: "", exitCode: 1 }) + : undefined, + (argv) => (argv[1] === "config" && argv[2] === "user.email") + ? (hasIdentity ? { stdout: "rt@example.test\n", stderr: "", exitCode: 0 } : { stdout: "", stderr: "", exitCode: 1 }) + : undefined, (argv) => (gitVerb(argv) === "commit") ? { stdout: "", stderr: "", exitCode: commitExit } : undefined, // `hasRemote()`'s own probe — most fixtures simulate a repo that already has origin configured, matching every pre-existing push test's assumption. (argv) => (argv[1] === "remote" && argv.length === 2) ? { stdout: hasRemote ? "origin\n" : "", stderr: "", exitCode: 0 } : undefined, @@ -168,6 +177,15 @@ const DEFAULT_SETTINGS: HomeSnapshotSettings = { const NO_OWNERS: Owners = { zones: {} }; +// A real directory (never touched; every git call underneath it is faked): +// the S090 existsSync guard runs against the real filesystem, so +// the fixture repoDir the whole suite shares must actually exist on disk, +// not just look plausible as a string. +const FAKE_REPO_DIR = realpathSync(mkdtempSync(join(tmpdir(), "rt-home-snapshot-fakerepo-"))); +afterAll(() => { + try { rmSync(FAKE_REPO_DIR, { recursive: true, force: true }); } catch { /* best-effort cleanup */ } +}); + async function flushAsync(): Promise { // Real macrotask hop — lets fake-exec's async chain (all microtasks, no // real timers involved) fully settle before assertions run. @@ -211,7 +229,7 @@ function baseDeps(overrides: Partial = {}): { const deps: HomeSnapshotDeps = { log, broadcast: (type, data) => broadcasts.push({ type, data }), - repoDir: "/fake/repo", + repoDir: FAKE_REPO_DIR, exec: execFn, watch: watch.fn, setTimeout: timers.setTimeoutFn, @@ -261,6 +279,23 @@ describe("startHomeSnapshot — inert paths", () => { expect(execCalls.length).toBe(1); }); + test("S090: a missing repoDir is diagnosed 'not-provisioned', names `rt home init`, never spawns git", async () => { + const { fn: execFn, calls: execCalls } = makeFakeExec(defaultResponders()); + const { deps, log, watch } = baseDeps({ exec: execFn, repoDir: "/does/not/exist/rt-home-snapshot-s090" }); + const handle = startHomeSnapshot(deps); + await handle.ready; + + expect(watch.calls.length).toBe(0); + const warnCall = log.calls.find((c) => c.level === "warn"); + expect(warnCall?.args[1]).toContain("rt home init"); + // The existsSync guard runs before any git spawn at all. + expect(execCalls.length).toBe(0); + + const result = await handle.runNow("manual"); + expect(result.skipped).toBe("not-provisioned"); + expect(execCalls.length).toBe(0); + }); + test("a throwing watch seam (fs.watch EMFILE/ENOSPC/ENOENT) resolves ready promptly and makes runNow return an inert result, not hang", async () => { const throwingWatch = () => { throw new Error("EMFILE: too many open files"); }; const { deps, log } = baseDeps({ watch: throwingWatch as any }); @@ -319,7 +354,7 @@ describe("startHomeSnapshot — live enabled toggle", () => { expect(result.committed).toBe(true); expect(handle.status().watching).toBe(true); - expect(watch.calls).toEqual([{ path: "/fake/repo", options: { recursive: true } }]); + expect(watch.calls).toEqual([{ path: FAKE_REPO_DIR, options: { recursive: true } }]); const janitorEntry = [...timers.pending.values()].find((t) => t.ms === DEFAULT_SETTINGS.janitorIntervalMin * 60_000); expect(janitorEntry).toBeDefined(); }); @@ -345,7 +380,7 @@ describe("startHomeSnapshot — watcher", () => { const handle = startHomeSnapshot(deps); await handle.ready; - expect(watch.calls).toEqual([{ path: "/fake/repo", options: { recursive: true } }]); + expect(watch.calls).toEqual([{ path: FAKE_REPO_DIR, options: { recursive: true } }]); expect(handle.status().watching).toBe(true); // One pending timer: the janitor interval (debounceSec*1000 == distinguishable via ms below). const janitorEntry = [...timers.pending.values()].find((t) => t.ms === DEFAULT_SETTINGS.janitorIntervalMin * 60_000); @@ -565,7 +600,7 @@ describe("startHomeSnapshot — commit shapes", () => { }, }; const { fn: execFn, calls: execCalls, optsLog } = makeFakeExec(defaultResponders({ statusZ: "?? notes/a.md\0" })); - const { deps } = baseDeps({ exec: execFn, readOwners: () => owners, repoDir: "/fake/repo" }); + const { deps } = baseDeps({ exec: execFn, readOwners: () => owners, repoDir: FAKE_REPO_DIR }); const handle = startHomeSnapshot(deps); await handle.ready; @@ -582,9 +617,9 @@ describe("startHomeSnapshot — commit shapes", () => { "git", "-c", "commit.gpgsign=false", "commit", "-q", "-m", "snapshot (manual): notes", "--", ".", ":(exclude)prefs/", ":(exclude)secrets/", ]); - expect(optsLog[addIdx]?.cwd).toBe("/fake/repo"); + expect(optsLog[addIdx]?.cwd).toBe(FAKE_REPO_DIR); expect(optsLog[addIdx]?.timeoutMs).toBeGreaterThan(0); - expect(optsLog[commitIdx]?.cwd).toBe("/fake/repo"); + expect(optsLog[commitIdx]?.cwd).toBe(FAKE_REPO_DIR); expect(optsLog[commitIdx]?.timeoutMs).toBeGreaterThan(0); }); @@ -667,6 +702,63 @@ describe("startHomeSnapshot — commit shapes", () => { expect(commits.length).toBe(2); // the auto commit and the janitor zone commit for (const argv of commits) expect(argv.slice(0, 3)).toEqual(["git", "-c", "commit.gpgsign=false"]); }); + + test("R043: missing git identity skips with 'no-git-identity', warns once, never attempts the commit", async () => { + const { fn: execFn, calls: execCalls } = makeFakeExec(defaultResponders({ statusZ: "?? a.txt\0", hasIdentity: false })); + const { deps, log } = baseDeps({ exec: execFn }); + const handle = startHomeSnapshot(deps); + await handle.ready; + + const result = await handle.runNow("manual"); + expect(result.skipped).toBe("no-git-identity"); + expect(execCalls.some((c) => gitVerb(c) === "commit")).toBe(false); + expect(log.calls.filter((c) => c.level === "warn" && String(c.args[c.args.length - 1]).includes("git config --global user.name")).length).toBe(1); + + // A later manual call short-circuits without re-probing identity or git status. + execCalls.length = 0; + const secondResult = await handle.runNow("manual"); + expect(secondResult.skipped).toBe("no-git-identity"); + expect(execCalls.length).toBe(0); + expect(log.calls.filter((c) => c.level === "warn").length).toBe(1); // still just the one warn + }); + + test("git identity present: commits normally, exactly one identity probe pair", async () => { + const { fn: execFn, calls: execCalls } = makeFakeExec(defaultResponders({ statusZ: "?? a.txt\0" })); + const { deps } = baseDeps({ exec: execFn }); + const handle = startHomeSnapshot(deps); + await handle.ready; + + const result = await handle.runNow("manual"); + expect(result.committed).toBe(true); + expect(execCalls.filter((c) => c[1] === "config" && c[2] === "user.name").length).toBe(1); + expect(execCalls.filter((c) => c[1] === "config" && c[2] === "user.email").length).toBe(1); + }); + + test("R043: a janitor-only cycle (no auto paths, one dirty claimed zone past threshold) with no git identity also skips 'no-git-identity', never attempts the janitor commit", async () => { + const owners: Owners = { zones: { "prefs/": { owner: "matt", claimedAt: "2026-01-01T00:00:00.000Z" } } }; + const db = freshDb(); + db.query("INSERT INTO kv (ns, k, v, updated_at) VALUES ('home-snapshot', 'state', ?, 0);") + .run(JSON.stringify({ firstSeenDirty: { "prefs/": 0 } })); + + const { fn: execFn, calls: execCalls } = makeFakeExec(defaultResponders({ statusZ: "?? prefs/x.md\0", hasIdentity: false })); + const { deps, log } = baseDeps({ + exec: execFn, + readOwners: () => owners, + db, + now: () => 10_000_000, // far past a 1-hour threshold from firstSeenDirty=0 + }); + + const handle = startHomeSnapshot(deps); + await handle.ready; + + const result = await handle.runNow("manual"); + expect(result.skipped).toBe("no-git-identity"); + // Only the claimed zone was dirty, so this cycle has no auto commit at + // all: the identity gate must still catch the janitor-only path. + expect(execCalls.some((c) => c[1] === "add")).toBe(false); + expect(execCalls.some((c) => gitVerb(c) === "commit")).toBe(false); + expect(log.calls.filter((c) => c.level === "warn" && String(c.args[c.args.length - 1]).includes("git config --global user.name")).length).toBe(1); + }); }); // ─── concurrency guard ─────────────────────────────────────────────────────── @@ -691,7 +783,7 @@ describe("startHomeSnapshot — concurrency guard", () => { describe("startHomeSnapshot — push", () => { test("a commit schedules a trailing push after pushDelaySec; success clears pushPending", async () => { const { fn: execFn, calls: execCalls, optsLog } = makeFakeExec(defaultResponders({ statusZ: "?? a.txt\0", pushExit: 0 })); - const { deps, timers } = baseDeps({ exec: execFn, repoDir: "/fake/repo" }); + const { deps, timers } = baseDeps({ exec: execFn, repoDir: FAKE_REPO_DIR }); const handle = startHomeSnapshot(deps); await handle.ready; @@ -704,7 +796,7 @@ describe("startHomeSnapshot — push", () => { const pushIdx = execCalls.findIndex((c) => c[0] === "git" && c[1] === "push"); expect(execCalls[pushIdx]).toEqual(["git", "push", "-q", "origin", "HEAD"]); - expect(optsLog[pushIdx]?.cwd).toBe("/fake/repo"); + expect(optsLog[pushIdx]?.cwd).toBe(FAKE_REPO_DIR); expect(optsLog[pushIdx]?.timeoutMs).toBeGreaterThan(0); expect(handle.status().pushPending).toBe(false); expect(handle.status().lastPushAt).toBe(1_000_000); @@ -829,6 +921,7 @@ describe("startHomeSnapshot — push", () => { if (argv[1] === "rev-parse" && argv[2] === "HEAD") return { stdout: "sha1\n", stderr: "", exitCode: 0 }; if (argv[1] === "status") return { stdout: "?? a.txt\0", stderr: "", exitCode: 0 }; if (argv[1] === "add") return { stdout: "", stderr: "", exitCode: 0 }; + if (argv[1] === "config") return { stdout: "rt test\n", stderr: "", exitCode: 0 }; if (gitVerb(argv) === "commit") { await gate; return { stdout: "", stderr: "", exitCode: 0 }; } return { stdout: "", stderr: "", exitCode: 0 }; }; diff --git a/lib/daemon/__tests__/loop-monitor.test.ts b/lib/daemon/__tests__/loop-monitor.test.ts new file mode 100644 index 00000000..b4378b14 --- /dev/null +++ b/lib/daemon/__tests__/loop-monitor.test.ts @@ -0,0 +1,52 @@ +import { test, expect } from "bun:test"; +import { applyTick, newLoopStats, type LoopStats } from "../loop-monitor.ts"; + +const OPTS = { stallLogMs: 1000, stallUnhealthyMs: 2000, stallRecentMs: 10_000 }; + +test("an on-time tick records small lag and no stall", () => { + const s = newLoopStats(); + applyTick(s, /*expected*/ 1000, /*now*/ 1010, "cache:refresh", OPTS, () => {}); + expect(s.lagMs).toBe(10); + expect(s.maxLagMs).toBe(10); + expect(s.stalls).toBe(0); + expect(s.currentlyStalled).toBe(false); +}); + +test("a >1s drift counts a stall, records the in-flight cmd, and warns", () => { + const s = newLoopStats(); + let warned = 0; + applyTick(s, 1000, 2500, "mr:action", OPTS, () => { warned++; }); + expect(s.stalls).toBe(1); + expect(s.lastStallCmd).toBe("mr:action"); + expect(s.lastStallAt).toBe(2500); + expect(s.maxLagMs).toBe(1500); + expect(warned).toBe(1); +}); + +test("currentlyStalled is true when the last big drift is within stallRecentMs", () => { + const s = newLoopStats(); + applyTick(s, 1000, 3500, "x", OPTS, () => {}); // 2500ms drift >= 2000 unhealthy, lastStallAt=3500 + expect(s.currentlyStalled).toBe(true); + // a small-drift tick whose `now` is past lastStallAt + stallRecentMs clears it + // (10ms drift, so no new stall; now-lastStallAt = 10500 > 10000 recent window) + applyTick(s, 13990, 14000, null, OPTS, () => {}); + expect(s.currentlyStalled).toBe(false); +}); + +test("maxLagMs is a high-water mark", () => { + const s: LoopStats = newLoopStats(); + applyTick(s, 1000, 1300, null, OPTS, () => {}); + applyTick(s, 1550, 1600, null, OPTS, () => {}); + expect(s.maxLagMs).toBe(300); +}); + +test("maxLagMs decays once no bigger spike lands within the window", () => { + const s: LoopStats = newLoopStats(); + applyTick(s, 1000, 1800, null, OPTS, () => {}); // drift 800, maxLagMs -> 800 at now=1800 + expect(s.maxLagMs).toBe(800); + // OPTS has no maxLagWindowMs, so it falls back to stallRecentMs (10_000). + // now=12000 is 10200ms past maxLagAt(1800), past the window, so an + // on-time tick (drift 0) decays maxLagMs to the current lagMs. + applyTick(s, 12000, 12000, null, OPTS, () => {}); + expect(s.maxLagMs).toBe(0); +}); diff --git a/lib/daemon/__tests__/refresh-status-ref.test.ts b/lib/daemon/__tests__/refresh-status-ref.test.ts new file mode 100644 index 00000000..4fa9c02d --- /dev/null +++ b/lib/daemon/__tests__/refresh-status-ref.test.ts @@ -0,0 +1,12 @@ +import { test, expect } from "bun:test"; +import { applyRefreshOutcome } from "../cache-refresh.ts"; + +test("a clean cycle advances lastSuccessAt; a failing cycle does not", () => { + const ref = { lastRefreshAt: 0, lastSuccessAt: 0, failedRepos: 0, enrichErrors: 0 }; + applyRefreshOutcome(ref, 1000, 0, 0); + expect(ref.lastSuccessAt).toBe(1000); + applyRefreshOutcome(ref, 2000, 2, 5); + expect(ref.lastRefreshAt).toBe(2000); + expect(ref.lastSuccessAt).toBe(1000); // unchanged on failure + expect(ref.failedRepos).toBe(2); +}); diff --git a/lib/daemon/__tests__/status-identity.test.ts b/lib/daemon/__tests__/status-identity.test.ts index b92d66b4..c3c9d99d 100644 --- a/lib/daemon/__tests__/status-identity.test.ts +++ b/lib/daemon/__tests__/status-identity.test.ts @@ -8,6 +8,13 @@ function fakeCtx(): any { watchedConfigs: new Map(), cache: { entries: {} }, portCacheRef: { ports: [], updatedAt: null }, + getHealth: () => ({ + level: "ok", + reasons: [], + metrics: { rss: 0, heapUsed: 0, external: 0, uptimeMs: 0, wsClients: 0, watchers: 0 }, + eventLoop: { maxLagMs: 0, lastStallAt: null, lastStallCmd: null, stalls: 0 }, + }), + heartbeatSeq: () => 0, }; } diff --git a/lib/daemon/__tests__/system-processes-handlers.test.ts b/lib/daemon/__tests__/system-processes-handlers.test.ts index 2597c9c4..e40bd62b 100644 --- a/lib/daemon/__tests__/system-processes-handlers.test.ts +++ b/lib/daemon/__tests__/system-processes-handlers.test.ts @@ -2,6 +2,7 @@ import { describe, test, expect } from "bun:test"; import { createSystemProcessHandlers } from "../handlers/system-processes.ts"; import type { SystemProcess } from "../system-process-scanner.ts"; +import { composeKey } from "../../state/branch-cache.ts"; function makeProcess(overrides: Partial = {}): SystemProcess { return { @@ -53,9 +54,9 @@ describe("system-processes handler", () => { }); test("enriches processes with Linear ticket from branch cache", async () => { - const proc = makeProcess({ branch: "feature/foo" }); + const proc = makeProcess({ branch: "feature/foo", repo: "myrepo" }); const handlers = setup([proc], { - "feature/foo": { + [composeKey("myrepo", "feature/foo")]: { ticket: { identifier: "ENG-123", title: "Do the thing" }, }, }); @@ -75,8 +76,8 @@ describe("system-processes handler", () => { }); test("leaves linearTicket null when cache entry has no ticket", async () => { - const proc = makeProcess({ branch: "feature/foo" }); - const handlers = setup([proc], { "feature/foo": { ticket: null } }); + const proc = makeProcess({ branch: "feature/foo", repo: "myrepo" }); + const handlers = setup([proc], { [composeKey("myrepo", "feature/foo")]: { ticket: null } }); const res = await handlers["system-processes"]!({}); diff --git a/lib/daemon/__tests__/unknown-command.test.ts b/lib/daemon/__tests__/unknown-command.test.ts new file mode 100644 index 00000000..1a6e063a --- /dev/null +++ b/lib/daemon/__tests__/unknown-command.test.ts @@ -0,0 +1,12 @@ +import { test, expect } from "bun:test"; +import { unknownCommandReply } from "../unknown-command.ts"; + +test("unknown command carries a code, version, and actionable text", () => { + const r = unknownCommandReply("chat:archive", "v0.9.0"); + expect(r.ok).toBe(false); + expect(r.code).toBe("unknown-command"); + expect(r.version).toBe("v0.9.0"); + expect(r.error).toContain("v0.9.0"); + expect(r.error).toContain("chat:archive"); + expect(r.error.toLowerCase()).toContain("restart"); +}); diff --git a/lib/daemon/__tests__/user-path.test.ts b/lib/daemon/__tests__/user-path.test.ts index ceb38719..984fc7af 100644 --- a/lib/daemon/__tests__/user-path.test.ts +++ b/lib/daemon/__tests__/user-path.test.ts @@ -2,7 +2,8 @@ import { describe, test, expect, beforeEach } from "bun:test"; import { mkdtempSync, writeFileSync, chmodSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; -import { probeTools } from "../user-path.ts"; +import { setSetting } from "../../settings/write.ts"; +import { resolveUserPath, probeTools } from "../user-path.ts"; describe("probeTools", () => { let binDir: string; @@ -32,3 +33,92 @@ describe("probeTools", () => { expect(probeTools("", ["node"])).toEqual({ hasNode: false }); }); }); + +function makeLog() { + const warns: any[] = []; + const infos: any[] = []; + return { log: { warn: (...a: any[]) => warns.push(a), info: (...a: any[]) => infos.push(a) } as any, warns, infos }; +} + +describe("resolveUserPath", () => { + test("fish-style space-separated base output is rejected, baseline kept + warn", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async () => "/opt/homebrew/bin /usr/bin /bin"; // spaces = fish-unsplit + const out = await resolveUserPath(log, probe); + expect(out).toBe("/usr/bin:/bin"); + expect(warns.some((w) => JSON.stringify(w).includes("whitespace"))).toBe(true); + }); + + test("a hanging probe returns baseline within the timeout", async () => { + const { log } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async () => null; // seam models kill/timeout as null + const out = await resolveUserPath(log, probe); + expect(out).toBe("/usr/bin:/bin"); + }); + + test("base equal to launchd baseline is treated as silent fallback (S062)", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin:/usr/sbin:/sbin"; + const probe = async (argv: any) => (argv[1] === "-lc" ? "/usr/bin:/bin:/usr/sbin:/sbin" : null); + const out = await resolveUserPath(log, probe); + expect(out).toBe("/usr/bin:/bin:/usr/sbin:/sbin"); + expect(warns.some((w) => JSON.stringify(w).includes("equals-baseline"))).toBe(true); + }); + + test("rt.daemonPath override skips both probes", async () => { + const { log } = makeLog(); + let called = false; + const probe = async () => { + called = true; + return "x"; + }; + const scratchHome = mkdtempSync(join(tmpdir(), "rt-daemonpath-override-")); + const originalHome = process.env.HOME; + process.env.HOME = scratchHome; + try { + setSetting("rt.daemonPath", "/over/bin:/x/bin", "machine"); + const out = await resolveUserPath(log, probe); + expect(out).toBe("/over/bin:/x/bin"); + expect(called).toBe(false); + } finally { + process.env.HOME = originalHome; + } + }); + + test("valid base accepted; interactive overlay appends a .zshrc-only dir after base", async () => { + const { log } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async (argv: any) => + argv[1] === "-lc" ? "/opt/homebrew/bin:/usr/bin:/bin" : "/opt/homebrew/bin:/usr/bin:/bin:/Users/x/.nvm/versions/node/v22/bin"; + const out = await resolveUserPath(log, probe); + expect(out).toBe("/opt/homebrew/bin:/usr/bin:/bin:/Users/x/.nvm/versions/node/v22/bin"); + }); + + test("overlay timeout is skipped with a warn; base kept unchanged", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async (argv: any) => (argv[1] === "-lc" ? "/opt/homebrew/bin:/usr/bin:/bin" : null); + const out = await resolveUserPath(log, probe); + expect(out).toBe("/opt/homebrew/bin:/usr/bin:/bin"); + expect(warns.some((w) => JSON.stringify(w).includes("overlay"))).toBe(true); + }); + + test("garbage overlay (non-null, no absolute dirs) is skipped with a warn", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async (argv: any) => (argv[1] === "-lc" ? "/opt/homebrew/bin:/usr/bin:/bin" : "not-a-path:also-not"); + const out = await resolveUserPath(log, probe); + expect(out).toBe("/opt/homebrew/bin:/usr/bin:/bin"); + expect(warns.some((w) => JSON.stringify(w).includes("overlay"))).toBe(true); + }); + + test("missing-tool warn fires when node is absent", async () => { + const { log, warns } = makeLog(); + process.env.PATH = "/usr/bin:/bin"; + const probe = async () => "/usr/bin:/bin"; // no node + await resolveUserPath(log, probe); + expect(warns.some((w) => JSON.stringify(w).includes("missing"))).toBe(true); + }); +}); diff --git a/lib/daemon/__tests__/worktree-reconciler.test.ts b/lib/daemon/__tests__/worktree-reconciler.test.ts index 87b17260..a0ef67d5 100644 --- a/lib/daemon/__tests__/worktree-reconciler.test.ts +++ b/lib/daemon/__tests__/worktree-reconciler.test.ts @@ -6,6 +6,7 @@ import { basename, dirname, join } from "path"; import type { Logger } from "pino"; import { readJson, writeJson } from "../../json-store.ts"; import { closeStateDb, listKvValues, setKvValue } from "../../state/index.ts"; +import { composeKey } from "../../state/branch-cache.ts"; import { machineSettingsPath, rtDir, teamSettingsPath } from "../../rt-paths.ts"; import { deriveRepoIdentity, parseIdentity } from "../../settings/identity.ts"; import { findByPath, loadRegistry, saveRegistry, type TreeRecord } from "../../worktree/registry.ts"; @@ -924,6 +925,27 @@ describe("merge reactor (detectTransitions)", () => { expect(tracked(rec.path)!.state).toBe("claimed"); }); + test("S069/Task 10: mrState is built only from the reconciled repo's composite-keyed entries", async () => { + const rec = ephemeralTree("kilo", "feat-kilo"); + const sameBranchInThisRepo = (state: string) => ({ + [composeKey(repoName, "feat-kilo")]: { repoName, mr: { iid: 42, state }, fetchedAt: Date.now() }, + // Same bare branch name, a DIFFERENT repo's composite key: must never + // be read as this repo's opened->merged edge, nor advance its snapshot. + [composeKey("beta-repo", "feat-kilo")]: { repoName: "beta-repo", mr: { iid: 99, state: "merged" }, fetchedAt: Date.now() }, + }); + + await detect(sameBranchInThisRepo("opened")); + expect(reactorState().mrState[`${repoName}:feat-kilo`]).toBe("opened"); + expect(reactorState().mrState["beta-repo:feat-kilo"]).toBeUndefined(); + + await detect(sameBranchInThisRepo("merged")); + + expect(existsSync(rec.path)).toBe(false); // this repo's tree disposed + expect(reactorState().fired).toContain(`disposed:${repoName}:42:merged`); + // beta-repo's own MR (99) never fired through this repo's pass. + expect(reactorState().fired).not.toContain("disposed:beta-repo:99:merged"); + }); + test("runOnce runs the reactor after the reconcile pass", async () => { const rec = ephemeralTree("golf", "feat-golf"); await declareWorktrees(repo, repoName, {}); diff --git a/lib/daemon/api-server.ts b/lib/daemon/api-server.ts index 073bf342..94ae7f03 100644 --- a/lib/daemon/api-server.ts +++ b/lib/daemon/api-server.ts @@ -81,6 +81,12 @@ interface ApiWSData { const wsClients = new Set>(); +/** Count of currently connected WS broadcast clients, for health reporting. */ +export function apiWsClientCount(): number { + return wsClients.size; +} + + let apiServerLog: { warn: (o: unknown, m: string) => void } = { warn: () => {} }; /** Consecutive Bun `ws.send()` backpressure (-1) returns tolerated before a @@ -160,7 +166,7 @@ export function clearWsClients(): void { export function buildCorsHeaders(origin: string | null, trusted: boolean): Record { const headers: Record = { "Access-Control-Allow-Methods": "GET, POST, OPTIONS", - "Access-Control-Allow-Headers": "Content-Type, X-RT-Token", + "Access-Control-Allow-Headers": "Content-Type, X-RT-Token, X-RT-Client", }; if (origin && trusted) { headers["Access-Control-Allow-Origin"] = origin; @@ -439,6 +445,8 @@ export async function startApiServer(deps: ApiServerDeps): Promise> payload = coerceQueryParams(url.searchParams); } + const client = req.headers.get("x-rt-client"); + if (client) payload._client = client; const result = await handleCommand(route.cmd, payload, req.signal); return Response.json(result, { headers: corsHeaders }); } catch (err) { diff --git a/lib/daemon/cache-refresh.ts b/lib/daemon/cache-refresh.ts index 8c1e5d03..8ab06180 100644 --- a/lib/daemon/cache-refresh.ts +++ b/lib/daemon/cache-refresh.ts @@ -29,7 +29,7 @@ export interface CacheRefresherDeps { log: Logger; /** The process-wide branch-cache store; `cache.reload()` replaces the old read-from-disk. */ cache: BranchCacheStore; - refreshStatusRef: { lastRefreshAt: number }; + refreshStatusRef: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }; portCacheRef: PortCacheRef; repoIndex: () => RepoIndex; broadcast: (type: string, data: any) => void; @@ -112,6 +112,24 @@ export function makeCoalescer( }; } +/** + * Records one refresh cycle's outcome onto the shared ref. `lastSuccessAt` + * only advances when the cycle was clean (no failed repos, no enrich + * errors). Downstream health reporting must be able to trust it as "refresh + * is actually working", not just "a cycle ran". + */ +export function applyRefreshOutcome( + ref: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }, + at: number, + failedReposCount: number, + enrichErrorsCount: number, +): void { + ref.lastRefreshAt = at; + ref.failedRepos = failedReposCount; + ref.enrichErrors = enrichErrorsCount; + if (failedReposCount === 0 && enrichErrorsCount === 0) ref.lastSuccessAt = at; +} + export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise { const { log, cache, refreshStatusRef, portCacheRef, repoIndex, broadcast } = deps; @@ -138,6 +156,10 @@ export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise(); + // Cycle-wide total; the per-repo `enrichErrors` below is scoped to one + // iteration and resets each repo, so this is the only place the total + // for the whole cycle (fed to applyRefreshOutcome below) accumulates. + let totalEnrichErrors = 0; // `repos` keys on the serialized repo identity (repo-index.ts), so every // `repoName` below — passed on into refreshAllMRs, project-sync, and the @@ -212,6 +234,7 @@ export function createCacheRefresher(deps: CacheRefresherDeps): () => Promise Promise(); + return { + check(key: string, now: number): { emit: boolean; suppressed: number } { + const e = map.get(key); + if (!e) { + map.set(key, { lastEmitAt: now, suppressed: 0 }); + return { emit: true, suppressed: 0 }; + } + if (now - e.lastEmitAt >= windowMs) { + const suppressed = e.suppressed; + e.lastEmitAt = now; + e.suppressed = 0; + return { emit: true, suppressed }; + } + e.suppressed += 1; + return { emit: false, suppressed: e.suppressed }; + }, + }; +} diff --git a/lib/daemon/freshness.ts b/lib/daemon/freshness.ts index 8b4d0956..504d11dc 100644 --- a/lib/daemon/freshness.ts +++ b/lib/daemon/freshness.ts @@ -38,6 +38,7 @@ import { redactCredentials } from "./redact-credentials.ts"; import { getProjectMRs, type ProjectMRs } from "./project-mrs-store.ts"; import { getDiscussionsFileStore } from "./discussions-file-store.ts"; import { createCursorStore, type CursorStore } from "../state/index.ts"; +import { composeKey, branchOf } from "../state/branch-cache.ts"; import { runCapture } from "../subprocess.ts"; const log = lazyChildLogger("freshness"); @@ -310,6 +311,7 @@ export async function getRepoContext( if (cachedForToken.token !== currentSecrets.gitlabToken) { stopWatch(repoName); userIdResolved = false; + selfUsername = null; providers.delete(repoName); } } @@ -502,9 +504,9 @@ async function processKeys( // iid → branch for this repo, rebuilt per batch (the cache may have been // reloaded from disk by the full poll since the last tick). const branchByIid = new Map(); - for (const [branch, entry] of Object.entries(ctx.cache.entries)) { + for (const [key, entry] of Object.entries(ctx.cache.entries)) { if (entry.repoName !== repoName) continue; - if (typeof entry.mr?.iid === "number") branchByIid.set(entry.mr.iid, branch); + if (typeof entry.mr?.iid === "number") branchByIid.set(entry.mr.iid, branchOf(key)); } const g = (overrides.grantsFor ?? ((r: string) => grants(loadRepoTracking(), r)))(repoName); @@ -542,7 +544,7 @@ async function processKeys( } const pr = await provider.fetchSingleMR(projectPath, iid, getCurrentUserId()); const feedBranch = branch - ?? (pr && ctx.cache.entries[pr.sourceBranch]?.repoName === repoName ? pr.sourceBranch : undefined); + ?? (pr && ctx.cache.entries[composeKey(repoName, pr.sourceBranch)] !== undefined ? pr.sourceBranch : undefined); if (feedBranch) mutated = updateEntry(env, repoName, feedBranch, pr) || mutated; if (wantProject && pr) upsertProject(pr); // GitLab's approval action bumps no updatedAt, so this is the only @@ -576,8 +578,8 @@ async function processKeys( break; } case "branch": { - const entry = ctx.cache.entries[k.ref]; - const isOurs = entry !== undefined && entry.repoName === repoName; + const entry = ctx.cache.entries[composeKey(repoName, k.ref)]; + const isOurs = entry !== undefined; let fetchedForRef: PullRequest | null | undefined; if (isOurs) { fetchedForRef = await provider.fetchPullRequestByBranch(projectPath, k.ref, "all"); @@ -636,7 +638,7 @@ async function processKeys( */ function updateEntry(env: FreshnessEnv, repoName: string, branch: string, pr: PullRequest | null): boolean { const { ctx } = env; - const existing = ctx.cache.entries[branch]; + const existing = ctx.cache.entries[composeKey(repoName, branch)]; if (!existing) return false; // lost race with a full refresh — skip const mr = pr ? toMRInfo(pr) : null; ctx.cache.put(branch, { ...existing, mr, fetchedAt: Date.now(), repoName }); @@ -654,8 +656,8 @@ function updateEntry(env: FreshnessEnv, repoName: string, branch: string, pr: Pu */ export function applyMRWriteback(env: FreshnessEnv, repoName: string, projectPath: string, pr: PullRequest): void { let branch: string | null = null; - for (const [b, entry] of Object.entries(env.ctx.cache.entries)) { - if (entry.repoName === repoName && entry.mr?.iid === pr.iid) { branch = b; break; } + for (const [key, entry] of Object.entries(env.ctx.cache.entries)) { + if (entry.repoName === repoName && entry.mr?.iid === pr.iid) { branch = branchOf(key); break; } } if (branch) updateEntry(env, repoName, branch, pr); @@ -702,7 +704,7 @@ async function runGapFill(env: FreshnessEnv, target: RepoTarget, overrides: Mapp const nullMrBranches = Object.entries(ctx.cache.entries) .filter(([, e]) => e.repoName === repoName && e.mr == null) - .map(([branch]) => branch); + .map(([key]) => branchOf(key)); if (nullMrBranches.length === 0) return; if (!provider.fetchPullRequestsByBranches) return; diff --git a/lib/daemon/handlers/cache.ts b/lib/daemon/handlers/cache.ts index 1823b702..7e9493af 100644 --- a/lib/daemon/handlers/cache.ts +++ b/lib/daemon/handlers/cache.ts @@ -10,6 +10,7 @@ */ import type { HandlerContext, HandlerMap, CacheEntry } from "./types.ts"; +import { branchOf, composeKey, getByBranch } from "../../state/branch-cache.ts"; /** How long an entry that resolved a ticket id but never got the ticket is left alone before another lookup is spent on it. Short enough that a key @@ -36,26 +37,40 @@ export function createCacheHandlers(ctx: HandlerContext): HandlerMap { "cache:read": async (payload) => { const branches = payload?.branches as string[] | undefined; const maxAgeMs = payload?.maxAgeMs as number | undefined; + // Optional exact scoping: an absent repoIdentity falls back to a + // suffix match across repos (today's callers never pass this yet). + const repoIdentity = payload?.repoIdentity as string | undefined; + + const lookup = (b: string): CacheEntry | undefined => + repoIdentity ? ctx.cache.entries[composeKey(repoIdentity, b)] : getByBranch(ctx.cache.entries, b); // Freshness gate: when the caller sets maxAgeMs, refresh first if the // oldest requested entry is older than that. Missing entries and an // empty cache count as infinitely stale. refreshCache is coalesced, so // concurrent stale readers share one refresh. if (typeof maxAgeMs === "number") { - const pool = branches ?? Object.keys(ctx.cache.entries); + const pool = branches ?? Object.keys(ctx.cache.entries).map(branchOf); let oldestFetchedAt = 0; if (pool.length > 0) { - oldestFetchedAt = Math.min(...pool.map((b) => ctx.cache.entries[b]?.fetchedAt ?? 0)); + oldestFetchedAt = Math.min(...pool.map((b) => lookup(b)?.fetchedAt ?? 0)); } if (Date.now() - oldestFetchedAt >= maxAgeMs) { await ctx.refreshCache(); } } - if (!branches) return { ok: true, data: ctx.cache.entries }; + // The output is always bare-branch keyed, never the store's internal + // composite keys, so cache:read's contract to the CLI/board/tray + // never changes underneath them. + if (!branches) { + const out: Record = {}; + for (const [k, v] of Object.entries(ctx.cache.entries)) out[branchOf(k)] = v; + return { ok: true, data: out }; + } const filtered: Record = {}; for (const b of branches) { - if (ctx.cache.entries[b]) filtered[b] = ctx.cache.entries[b]; + const entry = lookup(b); + if (entry) filtered[b] = entry; } return { ok: true, data: filtered }; }, @@ -66,9 +81,10 @@ export function createCacheHandlers(ctx: HandlerContext): HandlerMap { }, "branch:enrich": async (payload) => { - const branch = payload?.branch as string; - const repoPath = payload?.repoPath as string; - const remoteUrl = payload?.remoteUrl as string | undefined; + const branch = payload?.branch as string; + const repoPath = payload?.repoPath as string; + const remoteUrl = payload?.remoteUrl as string | undefined; + const repoIdentity = payload?.repoIdentity as string | undefined; // Test seam: the enricher, so a test never reaches Linear or the forge. const inject = payload?.enrich as | ((b: unknown, r: unknown, o: unknown) => Promise) @@ -76,7 +92,10 @@ export function createCacheHandlers(ctx: HandlerContext): HandlerMap { if (!branch) return { ok: false, error: "missing branch" }; - const cached = ctx.cache.entries[branch]; + const lookupBranch = (): CacheEntry | undefined => + repoIdentity ? ctx.cache.entries[composeKey(repoIdentity, branch)] : getByBranch(ctx.cache.entries, branch); + + const cached = lookupBranch(); const healing = !!cached; if (cached && !isIncomplete(cached)) { return { ok: true, data: cached, source: "cache" }; @@ -102,8 +121,9 @@ export function createCacheHandlers(ctx: HandlerContext): HandlerMap { // it also picks up rows a racing CLI enrichment upserted. ctx.cache.reload(); - if (ctx.cache.entries[branch]) { - return { ok: true, data: ctx.cache.entries[branch], source: "fresh" }; + const fresh = lookupBranch(); + if (fresh) { + return { ok: true, data: fresh, source: "fresh" }; } return { ok: true, data: null, source: "empty" }; } catch (err) { diff --git a/lib/daemon/handlers/status.ts b/lib/daemon/handlers/status.ts index 19db8227..2767376e 100644 --- a/lib/daemon/handlers/status.ts +++ b/lib/daemon/handlers/status.ts @@ -9,6 +9,7 @@ * ports — cached port-scan data, optionally filtered by repo * notifications — drain the notification queue * notifications:peek — peek at the notification queue (diagnostics) + * daemon:log-level - show or set the live pino log level */ import { existsSync, readdirSync } from "fs"; @@ -26,16 +27,21 @@ export function createStatusHandlers(ctx: HandlerContext): HandlerMap { // must see this run's own boot-attempt/failure counters, not whatever // they were when the daemon started. const { bootAttempts, lastReadyAt, recentFailures, lastExit } = readSupervisionState(); + const h = ctx.getHealth(); return { ok: true, uptime: Date.now() - ctx.startedAt, pid: process.pid, ...ctx.identity, + health: h.level, + eventLoop: h.eventLoop, + heartbeatSeq: ctx.heartbeatSeq(), supervision: { bootAttempts, lastReadyAt, recentFailures: recentFailures.slice(-3), lastExit }, }; }, "status": async () => { + const h = ctx.getHealth(); return { ok: true, data: { @@ -47,6 +53,9 @@ export function createStatusHandlers(ctx: HandlerContext): HandlerMap { portCacheAge: ctx.portCacheRef.updatedAt ? Date.now() - ctx.portCacheRef.updatedAt : null, freshness: getFreshnessSnapshot(), identity: ctx.identity, + health: { level: h.level, reasons: h.reasons }, + metrics: h.metrics, + eventLoop: h.eventLoop, }, }; }, @@ -58,6 +67,7 @@ export function createStatusHandlers(ctx: HandlerContext): HandlerMap { const repo = p.repo || "unknown"; portsByRepo[repo] = (portsByRepo[repo] || 0) + 1; } + const h = ctx.getHealth(); return { ok: true, @@ -72,6 +82,9 @@ export function createStatusHandlers(ctx: HandlerContext): HandlerMap { lastRefresh: ctx.refreshStatusRef.lastRefreshAt || null, portsByRepo, pendingNotifications: peekNotifications().length, + health: { level: h.level, reasons: h.reasons }, + metrics: h.metrics, + eventLoop: h.eventLoop, }, }; }, @@ -172,5 +185,14 @@ export function createStatusHandlers(ctx: HandlerContext): HandlerMap { // Peek without draining — for diagnostics return { ok: true, data: peekNotifications() }; }, + + "daemon:log-level": async (payload?: { level?: string }) => { + const VALID = ["trace", "debug", "info", "warn", "error"]; + if (payload?.level) { + if (!VALID.includes(payload.level)) return { ok: false, error: `invalid level: ${payload.level}` }; + ctx.setLogLevel(payload.level); + } + return { ok: true, level: ctx.getLogLevel() }; + }, }; } diff --git a/lib/daemon/handlers/system-processes.ts b/lib/daemon/handlers/system-processes.ts index 545cf7b9..34e92d36 100644 --- a/lib/daemon/handlers/system-processes.ts +++ b/lib/daemon/handlers/system-processes.ts @@ -1,6 +1,7 @@ import type { HandlerMap, HandlerContext } from "./types.ts"; import type { SystemProcessScanner, SystemProcess } from "../system-process-scanner.ts"; import { repoLabel } from "../../repo-arg.ts"; +import { composeKey } from "../../state/branch-cache.ts"; function shortName(proc: SystemProcess): string { // Use fullCommand (complete argv) to get the real binary name, @@ -95,7 +96,9 @@ export function createSystemProcessHandlers( const processes = scanner.getProcesses().map(proc => { let linearTicket: string | null = null; if (proc.branch) { - const cacheEntry = ctx.cache.entries[proc.branch]; + // proc.repo is already the serialized identity (scanner tags rows + // with it post-rekey), so an exact composeKey lookup is safe here. + const cacheEntry = ctx.cache.entries[composeKey(proc.repo, proc.branch)]; if (cacheEntry?.ticket) { linearTicket = `${cacheEntry.ticket.identifier}: ${cacheEntry.ticket.title}`; } diff --git a/lib/daemon/handlers/types.ts b/lib/daemon/handlers/types.ts index 38e99821..90b12202 100644 --- a/lib/daemon/handlers/types.ts +++ b/lib/daemon/handlers/types.ts @@ -10,6 +10,7 @@ import type { FSWatcher } from "fs"; import type { Logger } from "pino"; import type { PortEntry } from "../../port-scanner.ts"; import type { BranchCacheStore } from "../../state/index.ts"; +import type { HealthSnapshot } from "../health.ts"; /** * RT-48: `CacheEntry` used to be DECLARED here — a third copy of the same @@ -68,8 +69,16 @@ export interface HandlerContext { checkAndRepairHooksPath: (repoName: string, repoPath: string) => Promise; /** Start a directory watch over a repo's .git/config and run an initial check. */ startWatchingRepo: (repoName: string, repoPath: string) => void; - /** Holder for the last cache-refresh timestamp (0 = never). */ - refreshStatusRef: { lastRefreshAt: number }; + /** Holder for the last cache-refresh cycle's outcome (0s = never run). */ + refreshStatusRef: { lastRefreshAt: number; lastSuccessAt: number; failedRepos: number; enrichErrors: number }; + /** Computes the current health verdict (level/reasons/metrics/eventLoop) on demand; not cached, cheap enough per call. */ + getHealth: () => HealthSnapshot; + /** Current loop-monitor heartbeat sequence number, echoed by `ping`. */ + heartbeatSeq: () => number; + /** Sets the daemon logger's live level (trace/debug/info/warn/error). */ + setLogLevel: (l: string) => void; + /** Reads the daemon logger's current live level. */ + getLogLevel: () => string; } export type Handler = (payload: any, signal?: AbortSignal) => Promise; diff --git a/lib/daemon/handlers/worktree.ts b/lib/daemon/handlers/worktree.ts index b4c3f223..f712b840 100644 --- a/lib/daemon/handlers/worktree.ts +++ b/lib/daemon/handlers/worktree.ts @@ -44,6 +44,7 @@ import { import { disambiguate, slugifyTicketTitle } from "../../worktree/branch-name.ts"; import { createTree } from "../../worktree/create.ts"; import { classifyDirtyAsync, disposeTree, type DisposeDeps } from "../../worktree/dispose.ts"; +import { branchOf, composeKey } from "../../state/branch-cache.ts"; import { isTreeLocked, withTreeLock } from "../../worktree/locks.ts"; import { branchExistsLocalAsync, @@ -172,10 +173,18 @@ function disposeDeps( repoName: string, repoPath: string, ): DisposeDeps { + // disposeTree's joinedMr looks up by the BARE branch: hand it a + // bare-keyed, this-repo-only view of the (now composite-keyed) cache map + // so a same-named branch in another repo can never shadow the real entry. + const cacheEntries: DisposeDeps["cacheEntries"] = {}; + for (const [key, entry] of Object.entries(ctx.cache.entries)) { + if (entry.repoName && entry.repoName !== repoName) continue; + cacheEntries[branchOf(key)] = entry; + } return { repoName, repoPath, - cacheEntries: ctx.cache.entries as DisposeDeps["cacheEntries"], + cacheEntries, emit: opts.emit, log: ctx.log, killProcesses: loadWorktreeAppConfig().killProcesses, @@ -558,9 +567,12 @@ export function createWorktreeHandlers( } for (const t of trees) { - // The join key is (repoName, branch): a bare-branch join would hand - // a tree another repo's MR when both repos use the same name. - const entry = t.branch ? entries[t.branch] : undefined; + // The join key is composeKey(repoName, branch): an exact hit scopes + // to this repo so a same-named branch elsewhere can never join here. + // The bare-key fallback only ever matches an unattributed entry + // (older caches predate repoName), never another repo's, since + // every attributed write now composes under its own identity. + const entry = t.branch ? (entries[composeKey(repoName, t.branch)] ?? entries[t.branch]) : undefined; const mr = entry?.mr && (!entry.repoName || entry.repoName === repoName) ? { iid: entry.mr.iid, state: entry.mr.state, title: entry.mr.title } diff --git a/lib/daemon/health-sampler.ts b/lib/daemon/health-sampler.ts new file mode 100644 index 00000000..2d2a7c5e --- /dev/null +++ b/lib/daemon/health-sampler.ts @@ -0,0 +1,66 @@ +// lib/daemon/health-sampler.ts +/** Periodic (5-min) metrics logging + the two cached signals health needs that + * are too costly to compute per ping: the 1h rss baseline (growth) and free + * disk under RT_DIR. Pure helpers are unit-tested; the timer just calls sample. */ +import { statfsSync } from "fs"; +import type { Logger } from "pino"; + +export function rollRssBaseline( + prev: { rss: number; at: number } | null, + now: { rss: number; at: number }, + windowMs: number, +): { rss: number; at: number } { + if (!prev) return now; + if (now.at - prev.at >= windowMs) return now; + return prev; +} + +export interface HealthSampler { + sample(): void; + freeBytes(): number | null; + rssBaseline(): { rss: number; at: number } | null; +} + +export function createHealthSampler(opts: { + log: Logger; + rtDir: string; + wsClients: () => number; + watchers: () => number; + startedAt: number; +}): HealthSampler { + let baseline: { rss: number; at: number } | null = null; + let free: number | null = null; + + function statfsFree(dir: string): number | null { + // Not every platform/runtime implements statfs; leave free=null and disk + // checks are simply skipped rather than treated as an error. + try { + const s = statfsSync(dir); + return s.bavail * s.bsize; + } catch { + return null; + } + } + + return { + freeBytes: () => free, + rssBaseline: () => baseline, + sample() { + const mem = process.memoryUsage(); + const now = Date.now(); + baseline = rollRssBaseline(baseline, { rss: mem.rss, at: now }, 60 * 60_000); + free = statfsFree(opts.rtDir); + opts.log.info( + { + rss: mem.rss, + heapUsed: mem.heapUsed, + external: mem.external, + wsClients: opts.wsClients(), + watchers: opts.watchers(), + uptimeMs: now - opts.startedAt, + }, + "daemon metrics", + ); + }, + }; +} diff --git a/lib/daemon/health.ts b/lib/daemon/health.ts new file mode 100644 index 00000000..dede9f79 --- /dev/null +++ b/lib/daemon/health.ts @@ -0,0 +1,124 @@ +// lib/daemon/health.ts +/** + * Pure daemon health verdict. computeHealth takes a fully-gathered input + * struct (the daemon-side adapter does all I/O) and returns the level, the + * named reasons, and the metrics/eventLoop blocks the surfaces echo. + */ + +export const HEALTH_THRESHOLDS = { + refreshStaleMultiplier: 2, + rssSoftThresholdBytes: 1024 * 1024 * 1024, + rssGrowthPct: 50, + diskSoftFloorBytes: 500 * 1024 * 1024, + diskHardFloorBytes: 100 * 1024 * 1024, + restartsPerHourUnhealthy: 5, + recoveredErrorRate: 10, + loopLagDegradedMs: 500, +} as const; + +export interface HealthMetrics { + rss: number; + heapUsed: number; + external: number; + uptimeMs: number; + wsClients: number; + watchers: number; +} + +export interface HealthEventLoop { + maxLagMs: number; + lastStallAt: number | null; + lastStallCmd: string | null; + stalls: number; +} + +export interface HealthInputs { + now: number; + uptimeMs: number; + mem: { rss: number; heapUsed: number; external: number }; + /** rss + timestamp from ~1h ago, for growth detection; null if not yet sampled. */ + rssBaseline: { rss: number; at: number } | null; + wsClients: number; + watchers: number; + freshness: Record; + refresh: { lastSuccessAt: number; failedRepos: number; enrichErrors: number }; + refreshIntervalMs: number; + eventLoop: HealthEventLoop & { currentlyStalled: boolean }; + supervisionFailuresLastHour: number; + crashLooping: boolean; + loggerDegraded: boolean; + recoveredErrorRateLastWindow: number; + freeBytes: number | null; + /** Deferred inputs (spec): wired in a later phase, ignored today. */ + busySkips?: number; + criticalWriteFailures?: number; +} + +export interface HealthSnapshot { + level: "ok" | "degraded" | "unhealthy"; + reasons: string[]; + metrics: HealthMetrics; + eventLoop: HealthEventLoop; +} + +function mb(bytes: number): number { + return Math.round(bytes / (1024 * 1024)); +} + +export function computeHealth(i: HealthInputs): HealthSnapshot { + const T = HEALTH_THRESHOLDS; + const unhealthy: string[] = []; + const degraded: string[] = []; + + // --- unhealthy --- + if (i.loggerDegraded) unhealthy.push("logging: disabled (ENOSPC)"); + if (i.eventLoop.currentlyStalled) unhealthy.push("event-loop: currently stalled"); + if (i.crashLooping || i.supervisionFailuresLastHour >= T.restartsPerHourUnhealthy) { + unhealthy.push(`restarts: ${i.supervisionFailuresLastHour} in the last hour`); + } + if (i.freeBytes !== null && i.freeBytes < T.diskHardFloorBytes) { + unhealthy.push(`disk: ${mb(i.freeBytes)}MB free (critical)`); + } + + // --- degraded --- + const degradedRepos = Object.values(i.freshness).filter((f) => f.state === "degraded").length; + if (degradedRepos > 0) degraded.push(`refresh: ${degradedRepos} watcher${degradedRepos !== 1 ? "s" : ""} degraded`); + if (i.refresh.failedRepos > 0 || i.refresh.enrichErrors > 0) { + degraded.push(`refresh: ${i.refresh.failedRepos} repos failing (auth?)`); + } + const refreshAge = i.now - i.refresh.lastSuccessAt; + if (i.refresh.lastSuccessAt > 0 && refreshAge > T.refreshStaleMultiplier * i.refreshIntervalMs) { + degraded.push(`refresh: last success ${Math.round(refreshAge / 1000)}s ago`); + } + if (i.mem.rss > T.rssSoftThresholdBytes) degraded.push(`memory: rss ${mb(i.mem.rss)}MB`); + if (i.rssBaseline && i.mem.rss > i.rssBaseline.rss * (1 + T.rssGrowthPct / 100)) { + degraded.push(`memory: rss grew >${T.rssGrowthPct}% in the last hour`); + } + if (i.eventLoop.maxLagMs > T.loopLagDegradedMs) degraded.push(`event-loop: lag ${i.eventLoop.maxLagMs}ms`); + if (i.recoveredErrorRateLastWindow > T.recoveredErrorRate) { + degraded.push(`errors: ${i.recoveredErrorRateLastWindow} recovered in 5min`); + } + if (i.freeBytes !== null && i.freeBytes >= T.diskHardFloorBytes && i.freeBytes < T.diskSoftFloorBytes) { + degraded.push(`disk: ${mb(i.freeBytes)}MB free`); + } + + const level = unhealthy.length > 0 ? "unhealthy" : degraded.length > 0 ? "degraded" : "ok"; + return { + level, + reasons: level === "ok" ? [] : [...unhealthy, ...degraded], + metrics: { + rss: i.mem.rss, + heapUsed: i.mem.heapUsed, + external: i.mem.external, + uptimeMs: i.uptimeMs, + wsClients: i.wsClients, + watchers: i.watchers, + }, + eventLoop: { + maxLagMs: i.eventLoop.maxLagMs, + lastStallAt: i.eventLoop.lastStallAt, + lastStallCmd: i.eventLoop.lastStallCmd, + stalls: i.eventLoop.stalls, + }, + }; +} diff --git a/lib/daemon/heartbeat-file.ts b/lib/daemon/heartbeat-file.ts new file mode 100644 index 00000000..62224d0a --- /dev/null +++ b/lib/daemon/heartbeat-file.ts @@ -0,0 +1,42 @@ +/** + * Monotonic liveness heartbeat, written to a small file via atomic rename so + * it never opens state.db. A stalled/lock-wedged daemon is exactly when the + * WAL is least readable, so the cross-process classifier reads THIS, not kv. + * Same db-free pattern as the Phase 0 breadcrumb. + */ +import { existsSync, readFileSync, renameSync, writeFileSync } from "fs"; +import { join } from "path"; + +export interface Heartbeat { + at: number; + seq: number; +} + +function heartbeatPath(dir: string): string { + return join(dir, "daemon-heartbeat.json"); +} + +/** Never fatal: a heartbeat is a diagnostic aid, not something a tick may fail over. */ +export function writeHeartbeat(dir: string, hb: Heartbeat): void { + try { + const tmp = `${heartbeatPath(dir)}.${process.pid}.tmp`; + writeFileSync(tmp, JSON.stringify(hb)); + renameSync(tmp, heartbeatPath(dir)); + } catch { + // best-effort + } +} + +export function readHeartbeat(dir: string): Heartbeat | null { + try { + const p = heartbeatPath(dir); + if (!existsSync(p)) return null; + const parsed = JSON.parse(readFileSync(p, "utf8")); + if (typeof parsed?.at === "number" && typeof parsed?.seq === "number") { + return parsed as Heartbeat; + } + return null; + } catch { + return null; + } +} diff --git a/lib/daemon/home-snapshot.ts b/lib/daemon/home-snapshot.ts index 3c793ff9..d6a1f1d9 100644 --- a/lib/daemon/home-snapshot.ts +++ b/lib/daemon/home-snapshot.ts @@ -46,6 +46,8 @@ export type SnapshotReason = "manual" | "watch" | "janitor"; export type SkipReason = | "disabled" | "not-a-repo" + | "not-provisioned" + | "no-git-identity" | "init-failed" | "detached" | "merge-in-progress" @@ -278,7 +280,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle const ownersPath = ownersPathFor(deps.repoDir); - let disabledReason: "not-a-repo" | "init-failed" | null = null; + let disabledReason: SkipReason | null = null; let stopped = false; let watcher: { close(): void } | null = null; let debounceTimer: ReturnType | null = null; @@ -356,6 +358,15 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle async function init(): Promise { try { + // Checked before spawning git at all: a missing repoDir (never `rt + // home init`'d) otherwise reaches the same exitCode === -1 branch as a + // genuinely missing git binary, misdiagnosing "not provisioned" as + // "could not run git". + if (!existsSync(deps.repoDir)) { + disabledReason = "not-provisioned"; + deps.log.warn({ repoDir: deps.repoDir }, "home-snapshot: home repo not provisioned; run `rt home init`; inert"); + return; + } const check = await deps.exec(["git", "rev-parse", "--is-inside-work-tree"], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, @@ -656,7 +667,28 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle let committed = false; let sha: string | null = null; - if (plan.autoPaths.length > 0 && plan.message !== null) { + // Two independent commit sites below (the auto commit and the + // janitor-zone loop) can each attempt a commit this cycle; both fail the + // same doomed way (exit 128, "empty ident name") against an unconfigured + // identity. Checked once, up front, whenever EITHER would run, so a + // janitor-only cycle (no auto paths, one dirty claimed zone) is covered + // too, not just the auto-commit path. + const willAutoCommit = plan.autoPaths.length > 0 && plan.message !== null; + const willJanitorCommit = (reason === "janitor" || reason === "manual") && plan.janitorZones.length > 0; + if (willAutoCommit || willJanitorCommit) { + const name = await deps.exec(["git", "config", "user.name"], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, stderr: "pipe" }); + const email = await deps.exec(["git", "config", "user.email"], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, stderr: "pipe" }); + if (name.exitCode !== 0 || !name.stdout.trim() || email.exitCode !== 0 || !email.stdout.trim()) { + disabledReason = "no-git-identity"; + if (lastLoggedCommitError !== "no-git-identity") { + deps.log.warn("home-snapshot: no git identity; run `git config --global user.name` and `git config --global user.email`; snapshots inert"); + lastLoggedCommitError = "no-git-identity"; + } + return { committed: false, sha: null, paths: [], reason, skipped: "no-git-identity" }; + } + } + + if (willAutoCommit) { // `plan.autoPaths` describes what the STATUS SNAPSHOT at the top of // this run looked like — a purely descriptive record of intent. The // exclude pathspec built from `plan.excludedZones` (identical on both @@ -687,8 +719,9 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle // // `-c commit.gpgsign=false`: a global signing config with an unusable // key fails every snapshot commit outright (exit 128), and nothing - // about an unattended backup commit needs a signature. - const message = reason === "manual" ? plan.message.replace(/^snapshot:/, "snapshot (manual):") : plan.message; + // about an unattended backup commit needs a signature. (Git identity + // is confirmed once, above, before either commit site runs.) + const message = reason === "manual" ? plan.message!.replace(/^snapshot:/, "snapshot (manual):") : plan.message!; const commitResult = await deps.exec(["git", "-c", "commit.gpgsign=false", "commit", "-q", "-m", message, "--", ".", ...excludeArgs], { cwd: deps.repoDir, timeoutMs: GIT_TIMEOUT_MS, @@ -711,7 +744,7 @@ export function startHomeSnapshot(rawDeps: HomeSnapshotDeps): HomeSnapshotHandle } } - if ((reason === "janitor" || reason === "manual") && plan.janitorZones.length > 0) { + if (willJanitorCommit) { for (const jz of plan.janitorZones) { const dirtyHours = Math.floor((deps.now() - jz.dirtySinceMs) / (60 * 60 * 1000)); const message = `snapshot (janitor): ${jz.zone} dirty >${dirtyHours}h, owner ${jz.owner}`; diff --git a/lib/daemon/loop-monitor.ts b/lib/daemon/loop-monitor.ts new file mode 100644 index 00000000..45e6cc1d --- /dev/null +++ b/lib/daemon/loop-monitor.ts @@ -0,0 +1,110 @@ +/** + * Event-loop drift monitor. A ~250ms unref'd interval measures how late each + * tick fires vs its scheduled time; a large drift means the loop was blocked. + * The interval callback is created once and the stats object is preallocated, + * so the hot tick allocates nothing. Every ~2s it also invokes an + * `onHeartbeat` callback (the daemon writes the heartbeat file from that). + */ +import type { Logger } from "pino"; + +export interface LoopStats { + lagMs: number; + maxLagMs: number; + maxLagAt: number; + stalls: number; + lastStallAt: number | null; + lastStallCmd: string | null; + currentlyStalled: boolean; +} + +export function newLoopStats(): LoopStats { + return { lagMs: 0, maxLagMs: 0, maxLagAt: 0, stalls: 0, lastStallAt: null, lastStallCmd: null, currentlyStalled: false }; +} + +interface TickOpts { + stallLogMs: number; + stallUnhealthyMs: number; + stallRecentMs: number; + maxLagWindowMs?: number; +} + +/** Pure: fold one tick into `stats`. `onStall` fires once per stall (warn sink). */ +export function applyTick( + stats: LoopStats, + expected: number, + now: number, + currentCmd: string | null, + opts: TickOpts, + onStall: (drift: number, cmd: string | null) => void, +): void { + const drift = now - expected; + stats.lagMs = drift > 0 ? drift : 0; + const maxLagWindowMs = opts.maxLagWindowMs ?? opts.stallRecentMs; + if (stats.lagMs > stats.maxLagMs || now - stats.maxLagAt > maxLagWindowMs) { + stats.maxLagMs = stats.lagMs; + stats.maxLagAt = now; + } + if (drift > opts.stallLogMs) { + stats.stalls += 1; + stats.lastStallAt = now; + stats.lastStallCmd = currentCmd; + onStall(drift, currentCmd); + } + stats.currentlyStalled = + stats.lastStallAt !== null && + now - stats.lastStallAt <= opts.stallRecentMs && + drift > opts.stallUnhealthyMs; +} + +export interface LoopMonitorOpts { + log: Logger; + tickMs?: number; + stallLogMs?: number; + stallUnhealthyMs?: number; + stallRecentMs?: number; + maxLagWindowMs?: number; + heartbeatMs?: number; + currentCmd: () => string | null; + onHeartbeat: (at: number, seq: number) => void; +} + +export function startLoopMonitor( + opts: LoopMonitorOpts, +): { stats: LoopStats; seq: () => number; stop: () => void } { + const tickMs = opts.tickMs ?? 250; + const tickOpts: TickOpts = { + stallLogMs: opts.stallLogMs ?? 1000, + stallUnhealthyMs: opts.stallUnhealthyMs ?? 2000, + stallRecentMs: opts.stallRecentMs ?? 10_000, + maxLagWindowMs: opts.maxLagWindowMs ?? (opts.stallRecentMs ?? 10_000), + }; + const heartbeatMs = opts.heartbeatMs ?? 2000; + const stats = newLoopStats(); + let expected = Date.now() + tickMs; + let lastHeartbeat = 0; + let seq = 0; + let warnedThisStall = false; + + // Hoisted once so the hot tick allocates nothing: it must not build a + // fresh closure every 250ms. onStall closes over warnedThisStall by reference. + const onStall = (drift: number, cmd: string | null): void => { + if (!warnedThisStall) { + opts.log.warn({ driftMs: drift, cmd }, "event loop stalled"); + warnedThisStall = true; + } + }; + + const timer = setInterval(() => { + const now = Date.now(); + applyTick(stats, expected, now, opts.currentCmd(), tickOpts, onStall); + if (stats.lagMs <= tickOpts.stallLogMs) warnedThisStall = false; + expected = now + tickMs; + if (now - lastHeartbeat >= heartbeatMs) { + lastHeartbeat = now; + opts.onHeartbeat(now, ++seq); + } + }, tickMs); + timer.unref(); + + return { stats, seq: () => seq, stop: () => clearInterval(timer) }; +} diff --git a/lib/daemon/socket-server.ts b/lib/daemon/socket-server.ts index fee4c27e..4dbd84d8 100644 --- a/lib/daemon/socket-server.ts +++ b/lib/daemon/socket-server.ts @@ -45,6 +45,8 @@ export function startSocketServer(opts: { try { payload = await req.json(); } catch { /* empty body is fine */ } } + const client = req.headers.get("x-rt-client"); + if (client) (payload as any)._client = client; const result = await handleCommand(cmd, payload, req.signal); return Response.json(result); } catch (err) { diff --git a/lib/daemon/unknown-command.ts b/lib/daemon/unknown-command.ts new file mode 100644 index 00000000..139fc68b --- /dev/null +++ b/lib/daemon/unknown-command.ts @@ -0,0 +1,14 @@ +/** + * Reply shape for a command name routeCommand's switch doesn't recognize. + * Carries the daemon's own version so a caller can tell version skew (the + * daemon is older than the CLI/client that sent the command) from a genuine + * typo. + */ +export function unknownCommandReply(cmd: string, version: string) { + return { + ok: false as const, + code: "unknown-command" as const, + version, + error: `daemon at version ${version} does not know "${cmd}"; restart or upgrade rt (rt daemon restart)`, + }; +} diff --git a/lib/daemon/user-path.ts b/lib/daemon/user-path.ts index a28a66d4..eb31f729 100644 --- a/lib/daemon/user-path.ts +++ b/lib/daemon/user-path.ts @@ -1,16 +1,112 @@ /** * Resolve the user's full PATH once at daemon startup. * - * Strategy: use `$SHELL -ilc` (interactive login). Sources .zprofile AND - * .zshrc, which is where most users actually put their PATH exports - * (bun, ~/.local/bin, etc.). Slower than `-lc` due to compinit/OMZ, but - * the daemon is long-running so the one-time cost is irrelevant. - * Then layer in an explicit NVM resolution so nvm-managed tools (node, pnpm, - * etc.) are included regardless of how the daemon was launched. + * Strategy: a fast non-interactive `-lc` login shell (sources .zprofile, plus + * an explicit NVM fallback) establishes the base PATH quickly and can't hang + * on interactive-shell setup (compinit, OMZ, etc). A separate `-ilc` + * interactive probe then layers in whatever only .zshrc/.bashrc export + * (nvm's own PATH lines, ~/.local/bin, etc), unioned onto the base rather + * than replacing it, so a slow or misbehaving interactive probe can never + * regress the base PATH ... it can only fail to add to it. + * + * Both probes run through the injected `ProbeFn` seam so this module never + * spawns a shell directly and stays unit-testable without a real subprocess. */ -import { execSync } from "child_process"; +import { basename } from "path"; import type { Logger } from "pino"; +import { getSetting } from "../settings/resolve.ts"; + +export type ProbeFn = ( + argv: [string, ...string[]], + opts: { timeoutMs: number; env?: Record }, +) => Promise; + +const BASE_TIMEOUT_MS = 5_000; +const OVERLAY_TIMEOUT_MS = 3_000; +const KILL_GRACE_MS = 500; + +/** Default probe: a detached (own process-group) Bun.spawn whose whole group is + * SIGTERM'd then SIGKILL'd at the deadline, raced so a hung shell (or a hung + * grandchild it spawned) can never block boot past the timeout. */ +const runProbe: ProbeFn = async (argv, opts) => { + let proc: ReturnType; + try { + proc = Bun.spawn(argv, { + detached: true, + env: opts.env ?? { ...process.env }, + stdin: "ignore", + stdout: "pipe", + stderr: "ignore", + }); + } catch { + return null; + } + proc.unref(); + const pid = proc.pid; + let killTimer: ReturnType | undefined; + const term = setTimeout(() => { + try { + process.kill(-pid, "SIGTERM"); + } catch { + /* group already gone */ + } + killTimer = setTimeout(() => { + try { + process.kill(-pid, "SIGKILL"); + } catch { + /* gone */ + } + }, KILL_GRACE_MS); + killTimer.unref?.(); + }, opts.timeoutMs); + const captured: Promise = (async () => { + try { + const [out] = await Promise.all([new Response(proc.stdout as ReadableStream).text(), proc.exited]); + return out; + } catch { + return null; + } + })(); + let deadlineTimer: ReturnType; + const deadline: Promise = new Promise((resolve) => { + deadlineTimer = setTimeout(() => resolve(null), opts.timeoutMs + KILL_GRACE_MS + 250); + }); + try { + return await Promise.race([captured, deadline]); + } finally { + clearTimeout(term); + if (killTimer) clearTimeout(killTimer); + clearTimeout(deadlineTimer!); + } +}; + +function validateBase( + raw: string | null, + baseline: string, +): { path: string; source: "probe" | "baseline"; reason?: string } { + if (raw === null) return { path: baseline, source: "baseline", reason: "killed-or-empty" }; + const v = raw.trim(); + if (v.length === 0) return { path: baseline, source: "baseline", reason: "empty" }; + if (/\s/.test(v)) return { path: baseline, source: "baseline", reason: "whitespace" }; + if (v.split(":").filter(Boolean).length < 2) return { path: baseline, source: "baseline", reason: "too-few-segments" }; + if (v === baseline) return { path: baseline, source: "baseline", reason: "equals-baseline" }; + return { path: v, source: "probe" }; +} + +/** Overlay contributes only well-formed absolute dirs; anything else yields []. */ +function absoluteDirsOf(raw: string | null): string[] { + if (raw === null) return []; + const v = raw.trim(); + if (v.length === 0 || /\s/.test(v)) return []; + return v.split(":").filter((d) => d.startsWith("/")); +} + +function unionAppend(base: string, extra: string[]): string { + const have = new Set(base.split(":").filter(Boolean)); + const add = extra.filter((d) => !have.has(d)); + return add.length === 0 ? base : [base, ...add].join(":"); +} /** Which of `names` is a non-empty file on `pathValue`, keyed `has`. */ export function probeTools(pathValue: string, names: string[]): Record { @@ -28,36 +124,50 @@ export function probeTools(pathValue: string, names: string[]): Record { + const baseline = process.env.PATH ?? ""; - // 1. Interactive login shell — sources both .zprofile and .zshrc. - try { - resolvedPath = execSync(`${shell} -ilc 'echo $PATH' 2>/dev/null`, { - encoding: "utf8", - timeout: 30000, - }).trim() || resolvedPath; - } catch { /* timeout or shell error — keep baseline */ } - - // 2. Explicit NVM: source nvm.sh on top of the already-resolved PATH so - // NVM prepends its bin dirs without losing Homebrew/login-shell entries. - try { - const nvmDir = process.env.NVM_DIR ?? `${process.env.HOME}/.nvm`; - const nvmScript = `${nvmDir}/nvm.sh`; - const nvmPath = execSync( - `[ -s "${nvmScript}" ] && export PATH="${resolvedPath}" && . "${nvmScript}" && echo $PATH`, - { encoding: "utf8", timeout: 5000, shell: "/bin/zsh" }, - ).trim(); - if (nvmPath) resolvedPath = nvmPath; - } catch { /* nvm not installed or failed */ } - - // Log so we can verify key tools are present after restarts - const pathEntries = resolvedPath.split(":"); - log.info( - { entries: pathEntries.length, ...probeTools(resolvedPath, ["node", "pnpm", "doppler"]) }, - "PATH resolved", - ); - - return resolvedPath; + const override = getSetting("rt.daemonPath").value; + let result: string; + let source: string; + + if (typeof override === "string" && override.trim().length > 0) { + result = override.trim(); + source = "override"; + } else { + const shell = process.env.SHELL ?? "/bin/zsh"; + const isFish = basename(shell) === "fish"; + const baseArgv: [string, ...string[]] = isFish + ? [shell, "-lc", "string join : $PATH"] + : [ + shell, + "-lc", + `{ [ -s "\${NVM_DIR:-$HOME/.nvm}/nvm.sh" ] && . "\${NVM_DIR:-$HOME/.nvm}/nvm.sh" >/dev/null 2>&1; }; printf %s "$PATH"`, + ]; + const base = validateBase(await probe(baseArgv, { timeoutMs: BASE_TIMEOUT_MS }), baseline); + result = base.path; + source = base.source; + if (base.reason) log.warn({ reason: base.reason }, "PATH base probe unusable; kept baseline"); + + const ovArgv: [string, ...string[]] = isFish ? [shell, "-ilc", "string join : $PATH"] : [shell, "-ilc", "echo $PATH"]; + const ovRaw = await probe(ovArgv, { timeoutMs: OVERLAY_TIMEOUT_MS, env: { ...process.env, TERM: "dumb" } }); + const extra = absoluteDirsOf(ovRaw); + if (extra.length === 0) { + // Warn on both timeout (null) and garbage (non-null but no usable + // absolute dirs): either way the overlay contributed nothing. + log.warn("PATH interactive overlay skipped (timed out or no usable dirs)"); + } else { + const before = result; + result = unionAppend(result, extra); + if (result !== before) source += "+overlay"; + } + } + + const probed = probeTools(result, ["node", "git", "bun", "pnpm"]); + const missing = Object.entries(probed) + .filter(([, v]) => !v) + .map(([k]) => k.replace(/^has/, "").toLowerCase()); + if (missing.length > 0) log.warn({ missing }, "PATH missing required tools; set rt.daemonPath to override"); + log.info({ source, entries: result.split(":").length, ...probed }, "PATH resolved"); + return result; } diff --git a/lib/daemon/worktree-reconciler.ts b/lib/daemon/worktree-reconciler.ts index 9ac37eac..bc75d864 100644 --- a/lib/daemon/worktree-reconciler.ts +++ b/lib/daemon/worktree-reconciler.ts @@ -38,6 +38,7 @@ import { import { isTreeLocked, withTreeLock } from "../worktree/locks.ts"; import { ensureWorktreeRegistryRekeyed } from "../repo-index.ts"; import { createTree, scrapTree, type CreateDeps } from "../worktree/create.ts"; +import { branchOf } from "../state/branch-cache.ts"; import { classifyDirtyAsync, disposeTree } from "../worktree/dispose.ts"; import { changedSince, stepsToRun, runReadySteps } from "../worktree/ready.ts"; import { MAX_LOGGED_OUTPUT, outputTail } from "../subprocess.ts"; @@ -513,11 +514,21 @@ async function actOnTree( return "fired"; } + // disposeTree's joinedMr looks up by the BARE branch (its own contract, + // unaware of the composite `${identity}:${branch}` keys this repo's + // cache map now carries): hand it a bare-keyed, this-repo-only view so a + // same-named branch in another repo can never shadow the real entry. + const scopedEntries: Record = {}; + for (const [key, entry] of Object.entries(deps.cacheEntries)) { + if (entry.repoName && entry.repoName !== deps.repoName) continue; + scopedEntries[branchOf(key)] = entry; + } + const outcome = await disposeTree( { repoName: deps.repoName, repoPath: deps.repoPath, - cacheEntries: deps.cacheEntries as Record, + cacheEntries: scopedEntries as Record, emit: deps.emit, log: deps.log, killProcesses: appConfig.killProcesses, @@ -591,19 +602,20 @@ export async function detectTransitions(deps: ReactorDeps): Promise { if (!key.startsWith(prefix)) nextMrState[key] = value; } - for (const [branch, entry] of Object.entries(cacheEntries)) { + for (const [mapKey, entry] of Object.entries(cacheEntries)) { // Unattributed entries (older caches predate repoName) may join any repo; // an entry attributed elsewhere never does. if (entry.repoName && entry.repoName !== repoName) continue; if (!entry.mr) continue; + const branch = branchOf(mapKey); const cur = entry.mr.state ?? null; - const key = prefix + branch; - const prev = state.mrState[key] ?? null; + const mrKey = prefix + branch; + const prev = state.mrState[mrKey] ?? null; const iid = typeof entry.mr.iid === "number" ? String(entry.mr.iid) : branch; if (cur === "opened") { - nextMrState[key] = "opened"; + nextMrState[mrKey] = "opened"; // Reopen: forget this MR's fires so a later merge acts again, and hand // any disposable tree back to its owner. for (const fireKey of [...fired]) { @@ -613,7 +625,7 @@ export async function detectTransitions(deps: ReactorDeps): Promise { continue; } - nextMrState[key] = cur; + nextMrState[mrKey] = cur; if (prev !== "opened") continue; // cold-boot safety: unknown prev never fires if (!cur || !TERMINAL_STATES.has(cur)) continue; @@ -633,7 +645,7 @@ export async function detectTransitions(deps: ReactorDeps): Promise { reaction = worse(reaction, result === "busy" ? "retry" : result); } - if (reaction === "retry") nextMrState[key] = "opened"; + if (reaction === "retry") nextMrState[mrKey] = "opened"; else if (reaction === "fired") fired.add(fireKey); } diff --git a/lib/deps/__tests__/links.test.ts b/lib/deps/__tests__/links.test.ts index 63ea5048..56a1a97d 100644 --- a/lib/deps/__tests__/links.test.ts +++ b/lib/deps/__tests__/links.test.ts @@ -201,8 +201,12 @@ describe("tagged PATH links", () => { }); test("link(rt) refuses dev-mode-owns-rt when ~/.local/bin/rt is the dev-mode wrapper script", () => { + // isDevModeWrapper reads through p.readPrefix (Probes-routed, bounded), + // so the fake in-memory files map is enough -- no real fs write, and the + // content must be genuinely recognized (RT_LAUNCH_CWD tell) rather than + // any bare "#!" script, matching the shared detector's real rule. const path = linkPath(home, "rt"); - const p = bundleProbe({ files: { [path]: "#!/bin/sh\nexec bun run cli.ts \"$@\"\n" } }); + const p = bundleProbe({ files: { [path]: "#!/bin/sh\nexport RT_LAUNCH_CWD=\"$PWD\"\nexec bun run cli.ts \"$@\"\n" } }); const outcome = link(p, "rt"); expect(outcome).toEqual({ ok: false, reason: "dev-mode-owns-rt", detail: expect.any(String) }); diff --git a/lib/deps/links.ts b/lib/deps/links.ts index df570d16..87fe4675 100644 --- a/lib/deps/links.ts +++ b/lib/deps/links.ts @@ -8,7 +8,7 @@ */ import { dirname, join } from "path"; -import { installRtBinary } from "../dev-mode.ts"; +import { installRtBinary, isDevModeWrapperContent } from "../dev-mode.ts"; import type { Probes } from "../setup/probes.ts"; import { readSetupState, updateSetupState } from "../setup/state.ts"; import { bundledToolExec, isOurLink, LINK_TAG, linkPath, userCopyOnPath } from "./resolve.ts"; @@ -39,14 +39,18 @@ const REAL_SEAMS: LinkSeams = { installRtBinary: (src) => installRtBinary(src) } /** * rt in dev mode is signalled the same way lib/dev-mode.ts's currentMode() - * detects it — a "#!" wrapper script at the link path — but read through the - * Probes seam instead of raw fs, and narrowed to exclude our own tagged - * wrapper (whose second line carries LINK_TAG, not a dev-mode shebang body). + * detects it: shares isDevModeWrapperContent so the two call sites can never + * disagree. Reads through the Probes seam's readPrefix -- a real bounded + * (4096-byte) read in production, same as currentMode()'s own standalone + * read, but routed through `p` so link()'s tests (which drive the rest of + * this function entirely via a fake bundle/home) don't have to touch the + * real machine's HOME just to simulate this one check. currentMode() itself + * keeps its own direct real-fs read -- it has no Probes seam to route + * through, and is not this function's concern. */ -function isDevModeWrapper(p: Pick, path: string): boolean { - const content = p.readFile(path); - if (!content || !content.startsWith("#!")) return false; - return !(content.split("\n")[1] ?? "").startsWith(LINK_TAG); +function isDevModeWrapper(p: Pick, path: string): boolean { + const prefix = p.readPrefix(path); + return prefix !== null && isDevModeWrapperContent(prefix); } /** Single-quotes `s` for /bin/sh, escaping embedded single quotes via the standard '\'' trick — safe against $, `, \, " and everything else a relocated bundle path could contain. */ diff --git a/lib/dev-mode.ts b/lib/dev-mode.ts index 60b63732..f2c50fb9 100644 --- a/lib/dev-mode.ts +++ b/lib/dev-mode.ts @@ -64,32 +64,59 @@ export function installRtBinary(src: string): string { return dest; } +export const DEV_MODE_TAG = "# mattstack-dev-mode"; + /** - * Dev mode is signalled by the dev-mode WRAPPER SCRIPT at ~/.local/bin/rt -- - * not by any file existing there. Prod mode installs the compiled binary at - * that same path (MAT-383: leaving dev mode must leave a working rt behind), - * so presence alone can no longer tell the modes apart: the smoke showed a - * machine with the prod binary installed still reporting "dev", which made - * the flavor toggle a permanent no-op. A script starts with "#!"; a Mach-O - * binary never does. + * A recognized dev-mode wrapper: our marker on line 2, OR a legacy + * markerless wrapper (its RT_LAUNCH_CWD export line is our unique tell, + * predating the marker). A foreign #! script -- including our own tagged + * PATH-link wrapper from lib/deps/links.ts, which carries LINK_TAG instead + * -- has neither, so it correctly falls through to false. `prefix` is a + * bounded head of the file, never the whole file: in prod this path is a + * symlink to the compiled binary. */ -export function currentMode(): "dev" | "prod" { - const path = devModeWrapperPath(); - if (!existsSync(path)) return "prod"; +export function isDevModeWrapperContent(prefix: string): boolean { + if (!prefix.startsWith("#!")) return false; + const line2 = prefix.split("\n")[1] ?? ""; + return line2.startsWith(DEV_MODE_TAG) || prefix.includes("RT_LAUNCH_CWD"); +} + +/** + * A bounded head of `path` (never the whole file): in prod this path is a + * symlink to the multi-MB compiled binary, and a whole-file read there would + * be needless I/O on every mode check. Exported so lib/deps/links.ts shares + * this same real bounded read instead of re-implementing it. + */ +export function readWrapperPrefix(path: string): string | null { try { const fd = openSync(path, "r"); try { - const head = Buffer.alloc(2); - readSync(fd, head, 0, 2, 0); - return head.toString("latin1") === "#!" ? "dev" : "prod"; + const buf = Buffer.alloc(4096); + const n = readSync(fd, buf, 0, 4096, 0); + return buf.toString("latin1", 0, n); } finally { closeSync(fd); } } catch { - return "prod"; + return null; } } +/** + * Dev mode is signalled by the dev-mode WRAPPER SCRIPT at ~/.local/bin/rt -- + * not by any file existing there. Prod mode installs the compiled binary at + * that same path (MAT-383: leaving dev mode must leave a working rt behind), + * so presence alone can no longer tell the modes apart: the smoke showed a + * machine with the prod binary installed still reporting "dev", which made + * the flavor toggle a permanent no-op. + */ +export function currentMode(): "dev" | "prod" { + const path = devModeWrapperPath(); + if (!existsSync(path)) return "prod"; + const prefix = readWrapperPrefix(path); + return prefix !== null && isDevModeWrapperContent(prefix) ? "dev" : "prod"; +} + export interface IntendedMode { mode: "dev" | "prod"; provenance: "setting" | "derived-from-wrapper"; diff --git a/lib/enrich.ts b/lib/enrich.ts index 1317b6fd..5988a17e 100644 --- a/lib/enrich.ts +++ b/lib/enrich.ts @@ -23,6 +23,8 @@ import { type BranchCacheStore, type CacheEntry, } from "./state/index.ts"; +import { composeKey } from "./state/branch-cache.ts"; +import { identityFromRemote, serializeIdentity } from "./settings/identity.ts"; import { GitLabProvider, type PullRequest, @@ -36,6 +38,13 @@ import { type LinearTicket, } from "./linear.ts"; +/** Best-effort serialized identity for a remote URL; undefined with no remote. */ +function identityForRemote(remoteUrl: string | undefined): string | undefined { + if (!remoteUrl) return undefined; + const parsed = identityFromRemote(remoteUrl); + return parsed ? serializeIdentity(parsed) : undefined; +} + // ─── Remote URL parser ─────────────────────────────────────────────────────── export function parseRemoteUrl(url: string): { host: string; projectPath: string } | null { @@ -232,8 +241,10 @@ export async function enrichBranches( if (!options?.silent) { try { const { daemonQuery } = await import("./daemon-client.ts"); + const identity = identityForRemote(remoteUrl); const response = await daemonQuery("cache:read", { branches: branches.map(b => b.branch), + repoIdentity: identity, }); if (response?.ok && response.data) { @@ -263,12 +274,14 @@ export async function enrichBranches( const secrets = await loadSecrets(); const willFetch = !!(secrets.linearApiKey || secrets.gitlabToken); const store = getBranchCacheStore(); + const identity = identityForRemote(remoteUrl); - const allCached = !options?.forceRefresh && willFetch && branches.every((b) => b.branch in store.entries); + const allCached = !options?.forceRefresh && willFetch + && branches.every((b) => composeKey(identity, b.branch) in store.entries); if (allCached) { const cachedResults = branches.map((b) => { - const entry = store.entries[b.branch]!; + const entry = store.entries[composeKey(identity, b.branch)]!; return { path: b.path, dirName: b.path.split("/").pop() || b.path, @@ -297,6 +310,7 @@ async function fetchAndCache( ): Promise { const secrets = await loadSecrets(); const willFetch = !!(secrets.linearApiKey || secrets.gitlabToken); + const identity = identityForRemote(remoteUrl); let showSpinner = false; if (!silent && willFetch && process.stderr.isTTY) { @@ -362,7 +376,7 @@ async function fetchAndCache( const results: EnrichedBranch[] = branches.map((b, idx) => { const dirName = b.path.split("/").pop() || b.path; const { linearId } = branchLinearIds[idx]!; - const existing = store.entries[b.branch]; + const existing = store.entries[composeKey(identity, b.branch)]; const pr = mrMap.get(b.branch) ?? null; const mr = mrFetchSucceeded ? (pr ? toMRInfo(pr) : null) : (existing?.mr ?? null); @@ -376,7 +390,7 @@ async function fetchAndCache( linearId: linearId || existing?.linearId || "", mr, fetchedAt: mrFetchSucceeded ? Date.now() : (existing?.fetchedAt ?? Date.now()), - repoName: existing?.repoName, + repoName: identity, }]); return { path: b.path, dirName, branch: b.branch, linearId, ticket, mr }; @@ -492,7 +506,7 @@ export async function refreshAllMRs( // preserve the existing entry to avoid overwriting good enrichment data that was // previously resolved via a full enrich (e.g., from an older/renamed MR title). if (!mr && !linearId) { - const existing = store.entries[b.branch]; + const existing = store.entries[composeKey(repoName, b.branch)]; if (existing?.linearId || existing?.ticket) { // Keep existing enrichment — we have nothing better to replace it with enriched.push([b.branch, { ...existing, fetchedAt: now, repoName }]); @@ -511,7 +525,7 @@ export async function refreshAllMRs( // GitLab API failed entirely — preserve existing MR data to avoid false transitions. // If we also couldn't resolve a linearId (non-standard branch name, no MR title to fall // back on), preserve existing ticket/linearId too — we have nothing better to substitute. - const existing = store.entries[b.branch]; + const existing = store.entries[composeKey(repoName, b.branch)]; enriched.push([b.branch, { ticket: linearId ? ticket : (existing?.ticket ?? null), linearId: linearId || existing?.linearId || "", diff --git a/lib/home/__tests__/init-exec.test.ts b/lib/home/__tests__/init-exec.test.ts index 5b520d6f..8bbf40fb 100644 --- a/lib/home/__tests__/init-exec.test.ts +++ b/lib/home/__tests__/init-exec.test.ts @@ -26,6 +26,8 @@ class FakeExecSeam implements ExecSeam { failRun?: (cmd: string[]) => string | undefined; exists?: (path: string) => boolean; blocksSymlink?: (path: string) => boolean; + /** git config user.name/user.email answers for commitInitialUserRepo's identity check. Defaults to a fully-configured identity so every other test's commit step doesn't have to opt in. */ + identity?: { name?: string; email?: string }; } = {}, ) {} @@ -33,6 +35,12 @@ class FakeExecSeam implements ExecSeam { this.calls.push({ kind: "run", cmd, cwd: runOpts?.cwd }); const failure = this.opts.failRun?.(cmd); if (failure) return { code: 1, stdout: "", stderr: failure }; + const configKey = cmd[cmd.length - 2] === "config" ? cmd[cmd.length - 1] : undefined; + if (configKey === "user.name" || configKey === "user.email") { + const identity = this.opts.identity ?? { name: "rt test", email: "rt@example.test" }; + const value = configKey === "user.name" ? identity.name : identity.email; + return value ? { code: 0, stdout: `${value}\n`, stderr: "" } : { code: 1, stdout: "", stderr: "" }; + } return { code: 0, stdout: "", stderr: "" }; } @@ -120,6 +128,21 @@ describe("executeInitPlan", () => { expect(result.ok).toBe(false); if (!result.ok) expect(result.failedStep).toBe("commitInitialUserRepo"); }); + + test("R043: no git identity fails with an actionable message, never attempts the commit", async () => { + const seam = new FakeExecSeam({ identity: { name: "", email: "" } }); + + const result = await executeInitPlan([{ kind: "commitInitialUserRepo" }], seam, noopLog); + + expect(result.ok).toBe(false); + if (!result.ok) { + expect(result.failedStep).toBe("commitInitialUserRepo"); + expect(result.stderr).toContain("git config --global user.name"); + expect(result.stderr).toContain("git config --global user.email"); + expect(result.stderr).toContain("rt home init"); + } + expect(seam.calls.some((c) => c.kind === "run" && c.cmd.includes("commit"))).toBe(false); + }); }); describe("writeGitignore / writeOwners — write-if-absent, decided at exec time", () => { diff --git a/lib/home/__tests__/machine-id.test.ts b/lib/home/__tests__/machine-id.test.ts new file mode 100644 index 00000000..54dbedc9 --- /dev/null +++ b/lib/home/__tests__/machine-id.test.ts @@ -0,0 +1,38 @@ +import { expect, test } from "bun:test"; +import { resolveInitialMachineKey, stableMachineId } from "../machine-id.ts"; + +const IOREG_FIXTURE = ` "IOPlatformUUID" = "D9E8F7A6-1234-5678-9ABC-DEF012345678"`; + +test("stableMachineId parses IOPlatformUUID and slugs it", async () => { + const id = await stableMachineId(async () => IOREG_FIXTURE); + expect(id).toBe("d9e8f7a6-1234-5678-9abc-def012345678"); +}); + +test("stableMachineId returns null when ioreg fails", async () => { + expect(await stableMachineId(async () => null)).toBeNull(); + expect(await stableMachineId(async () => "no uuid here")).toBeNull(); +}); + +test("resolveInitialMachineKey: existing pin file is returned unchanged", async () => { + const probes = { exists: (p: string) => p.endsWith("machine-key"), listProfiles: () => [] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => "pinned-key", stableId: async () => "uuid-x" }); + expect(key).toBe("pinned-key"); +}); + +test("resolveInitialMachineKey: existing non-empty hostname-slug store freezes the slug", async () => { + const probes = { exists: () => false, listProfiles: () => ["myhost"] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => null, hostnameSlug: () => "myhost", stableId: async () => "uuid-x" }); + expect(key).toBe("myhost"); // frozen, data preserved, no move +}); + +test("resolveInitialMachineKey: fresh machine gets the stable id", async () => { + const probes = { exists: () => false, listProfiles: () => [] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => null, hostnameSlug: () => "myhost", stableId: async () => "uuid-x" }); + expect(key).toBe("uuid-x"); +}); + +test("resolveInitialMachineKey: fresh machine, ioreg fails -> hostname slug", async () => { + const probes = { exists: () => false, listProfiles: () => [] } as any; + const key = await resolveInitialMachineKey("/home", probes, { readPin: () => null, hostnameSlug: () => "myhost", stableId: async () => null }); + expect(key).toBe("myhost"); +}); diff --git a/lib/home/init-exec.ts b/lib/home/init-exec.ts index 71a59a72..deb589be 100644 --- a/lib/home/init-exec.ts +++ b/lib/home/init-exec.ts @@ -82,6 +82,14 @@ async function runStep(step: InitStep, exec: ExecSeam, log: StepLog): Promise Promise = defaultIoreg, +): Promise { + const out = await exec(["ioreg", "-rd1", "-c", "IOPlatformExpertDevice"]); + if (!out) return null; + const m = out.match(/"IOPlatformUUID"\s*=\s*"([^"]+)"/); + if (!m) return null; + const slug = m[1]! + .toLowerCase() + .replace(/[^a-z0-9-]+/g, "-") + .replace(/^-+|-+$/g, ""); + return isSafeMachineKeySegment(slug) ? slug : null; +} + +const defaultIoreg = async (argv: string[]): Promise => { + try { + const proc = Bun.spawn(argv, { stdin: "ignore", stdout: "pipe", stderr: "ignore" }); + const term = setTimeout(() => { + try { + proc.kill("SIGKILL"); + } catch { + // already gone + } + }, 3_000); + try { + const [out, code] = await Promise.all([new Response(proc.stdout as ReadableStream).text(), proc.exited]); + return code === 0 ? out : null; + } finally { + clearTimeout(term); + } + } catch { + return null; + } +}; + +interface InitKeyDeps { + readPin?: () => string | null; + hostnameSlug?: () => string; + stableId?: () => Promise; +} + +/** + * Establishes the machine key at `rt home init`. Data-preserving and idempotent: + * an existing pin is kept as-is; a machine whose hostname-slug store already + * carries settings freezes that slug (zero data movement); only a genuinely + * fresh machine gets the stable id. + */ +export async function resolveInitialMachineKey(home: string, probes: HomeProbes, deps: InitKeyDeps = {}): Promise { + const readPin = + deps.readPin ?? + (() => { + try { + const v = readFileSync(join(home, "machine-key"), "utf8").trim(); + return v || null; + } catch { + return null; + } + }); + const hostnameSlug = deps.hostnameSlug ?? (() => machineKey()); // machineKey() with no pin returns the hostname slug + const stableId = deps.stableId ?? (() => stableMachineId()); + + const pinned = readPin(); + if (pinned && isSafeMachineKeySegment(pinned)) return pinned; + + const slug = hostnameSlug(); + const profiles = probes.listProfiles(join(home, "user", "local")); // dirs carrying settings.local.jsonc + if (profiles.includes(slug)) return slug; // freeze existing non-empty store + + return (await stableId()) ?? slug; +} diff --git a/lib/log-janitor.ts b/lib/log-janitor.ts index 7451de3f..618e301c 100644 --- a/lib/log-janitor.ts +++ b/lib/log-janitor.ts @@ -34,7 +34,12 @@ function assertLogsDir(dir: string): void { * directories) whose name matches the surface log pattern and whose mtime is * older than `retentionDays` back from `now`. Returns the basenames removed. */ -export function pruneLogs(dir: string, retentionDays: number, now: number): { removed: string[] } { +export function pruneLogs( + dir: string, + retentionDays: number, + now: number, + onError?: (phase: "readdir" | "unlink", err: unknown, file?: string) => void, +): { removed: string[] } { assertLogsDir(dir); const cutoff = now - retentionDays * DAY; @@ -43,7 +48,8 @@ export function pruneLogs(dir: string, retentionDays: number, now: number): { re let entries; try { entries = readdirSync(dir, { withFileTypes: true }); - } catch { + } catch (err) { + onError?.("readdir", err); return { removed }; } @@ -62,8 +68,9 @@ export function pruneLogs(dir: string, retentionDays: number, now: number): { re try { unlinkSync(full); removed.push(entry.name); - } catch { - // best-effort — a file gone or unreadable between stat and unlink is not fatal + } catch (err) { + // best-effort: a file gone or unreadable between stat and unlink is not fatal + onError?.("unlink", err, entry.name); } } diff --git a/lib/notifier.ts b/lib/notifier.ts index 0c235c87..ff3ccf2c 100644 --- a/lib/notifier.ts +++ b/lib/notifier.ts @@ -26,6 +26,7 @@ import type { SystemProcess } from "./daemon/system-process-scanner.ts"; import { agentSessionPids } from "./daemon/worktree-process-kill.ts"; import { lazyChildLogger } from "./daemon-logger.ts"; import { repoLabel } from "./repo-arg.ts"; +import { branchOf } from "./state/branch-cache.ts"; import { getNotifierStateBlob, setNotifierStateBlob, @@ -552,6 +553,11 @@ function detectBranchTransitions( prefs: NotificationPrefs, currentUserId: number | null, ): void { + // `branch` is the branch-cache map key (composite `${identity}:${branch}` + // when attributed, bare otherwise), kept as-is here rather than unwrapped + // to the bare branch, so `firedKey` and the snapshot map come out + // repo-scoped for free. `branchOf(branch)` is used only for the + // human-readable notification text. for (const [branch, entry] of Object.entries(current)) { // If the MR slot is null we have no fresh data — skipping prevents // false "transition" detection that would clear the fired key set. @@ -566,7 +572,8 @@ function detectBranchTransitions( if (!was) continue; // First time seeing this branch — no transition const now = snapshotBranch(entry, was); - const branchShort = branch.length > 40 ? branch.slice(0, 39) + "…" : branch; + const displayBranch = branchOf(branch); + const branchShort = displayBranch.length > 40 ? displayBranch.slice(0, 39) + "…" : displayBranch; const mrUrl = entry.mr?.webUrl ?? undefined; // MR merged (opened → merged) — check BEFORE skipping merged MRs diff --git a/lib/secrets/__tests__/store.test.ts b/lib/secrets/__tests__/store.test.ts index 2a4fd182..799579bc 100644 --- a/lib/secrets/__tests__/store.test.ts +++ b/lib/secrets/__tests__/store.test.ts @@ -8,8 +8,10 @@ import { resetSecretsMemo, formatDebugLine, buildSecretsSpawnOptions, + createRealSecretsExecSeam, NoAgeKeyError, InvalidSecretsSegmentError, + SecretsTimeoutError, type SecretsExecResult, type SecretsExecSeam, type SecretsSeams, @@ -573,6 +575,22 @@ describe("real seam spawn options — cwd pin", () => { }); }); +describe("real seam spawn timeout", () => { + test("a hanging sops spawn times out with SecretsTimeoutError, does not hang", async () => { + let resolveExit: (code: number) => void = () => {}; + const fakeProc = { + pid: 1, + stdout: new Response("").body, + stderr: new Response("").body, + exited: new Promise((r) => { resolveExit = r; }), + kill: () => resolveExit(143), // killable: kill resolves exit, no real process + }; + const seam = createRealSecretsExecSeam(undefined, () => fakeProc as any); + await expect(seam.run(["sops", "-d", "x"], { timeoutMs: 50 } as any)) + .rejects.toBeInstanceOf(SecretsTimeoutError); + }, 2_000); +}); + describe("formatDebugLine (the debugLog path)", () => { test("a sensitive call's line never includes env values or stdout/stderr, whatever they'd contain", () => { const line = formatDebugLine(["sops", "-d", "/some/path"], { sensitive: true }); diff --git a/lib/secrets/store.ts b/lib/secrets/store.ts index 19283211..0876b194 100644 --- a/lib/secrets/store.ts +++ b/lib/secrets/store.ts @@ -44,7 +44,8 @@ export interface SecretsExecResult { } export interface SecretsExecSeam { - run(cmd: string[], opts?: { env?: Record; sensitive?: boolean }): Promise; + /** `timeoutMs` overrides the default kill-and-reject deadline (see SecretsTimeoutError). */ + run(cmd: string[], opts?: { env?: Record; sensitive?: boolean; timeoutMs?: number }): Promise; fileExists(path: string): boolean; /** * Direct child names of a directory (not recursive); [] when the directory @@ -138,6 +139,16 @@ export function validateSlug(slug: string): void { if (!SLUG_PATTERN.test(slug)) throw new InvalidSecretsSegmentError("slug", slug, SLUG_PATTERN); } +/** Thrown when a sops/keychain spawn does not exit in time (a locked keychain pops a GUI dialog and blocks until clicked). */ +export class SecretsTimeoutError extends Error { + constructor(message: string) { + super(message); + this.name = "SecretsTimeoutError"; + } +} + +const DEFAULT_SECRETS_TIMEOUT_MS = 30_000; + /** Validates `domain` — every path construction routes through here, so this is the one choke point. */ export function secretsFilePath(domain: string): string { validateDomain(domain); @@ -458,14 +469,23 @@ export function buildSecretsSpawnOptions(opts?: { env?: Record; }; } +type SecretsSpawn = (argv: string[], opts: any) => { + stdout: ReadableStream; + stderr: ReadableStream; + exited: Promise; + kill: (sig?: number | string) => void; +}; + /** * Real seam: Bun.spawn-based capture, real fs reads/writes. `cwd`, when * given, is pinned for every sops spawn this seam instance makes — the * personal store's default seam (`cwd` omitted) resolves `/user`; * team-store.ts constructs its own instance per team with `cwd` set to that - * team's clone root (see `buildTeamSpawnOptions`). + * team's clone root (see `buildTeamSpawnOptions`). `spawn` is injectable so + * tests can model a hanging child without a real subprocess; the default is + * the real `Bun.spawn`, so production behavior is unchanged. */ -export function createRealSecretsExecSeam(cwd?: string): SecretsExecSeam { +export function createRealSecretsExecSeam(cwd?: string, spawn: SecretsSpawn = Bun.spawn as unknown as SecretsSpawn): SecretsExecSeam { return { async run(cmd, opts) { debugLog(cmd, opts?.sensitive); @@ -475,13 +495,30 @@ export function createRealSecretsExecSeam(cwd?: string): SecretsExecSeam { // stay unaware of the bundle. const [bin, ...args] = cmd; const resolved = bin === undefined ? cmd : [resolveBundledTool(bin), ...args]; - const proc = Bun.spawn(resolved, buildSecretsSpawnOptions({ env: opts?.env, cwd })); - const [stdout, stderr, code] = await Promise.all([ - new Response(proc.stdout).text(), - new Response(proc.stderr).text(), - proc.exited, - ]); - return { code, stdout, stderr }; + const proc = spawn(resolved, buildSecretsSpawnOptions({ env: opts?.env, cwd })); + const timeoutMs = opts?.timeoutMs ?? DEFAULT_SECRETS_TIMEOUT_MS; + let timedOut = false; + const timer = setTimeout(() => { + timedOut = true; + try { + proc.kill(); + } catch { + // already exited + } + }, timeoutMs); + try { + const [stdout, stderr, code] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + if (timedOut) { + throw new SecretsTimeoutError(`${cmd[0]}: did not exit within ${timeoutMs}ms (keychain prompt pending?)`); + } + return { code, stdout, stderr }; + } finally { + clearTimeout(timer); + } }, fileExists(path) { return existsSync(path); diff --git a/lib/setup/__tests__/fakes.ts b/lib/setup/__tests__/fakes.ts index 6844ad04..70c379d7 100644 --- a/lib/setup/__tests__/fakes.ts +++ b/lib/setup/__tests__/fakes.ts @@ -88,6 +88,16 @@ export function fakeProbes(opts: FakeProbesOpts = {}): Probes & { return files[path] ?? null; }, + // Mirrors the real bounded (4096-byte) prefix read, through `links` + // exactly like fileSize does -- a symlinked fixture (e.g. `p.symlink` + // registering an rt bundle target) must resolve here too, not just for + // real files planted via `files`. + readPrefix(path) { + const resolved = resolveThroughLinks(path); + if (resolved === null || resolved in dirs) return null; + return (files[resolved] ?? "").slice(0, 4096); + }, + readDir(path) { // A copy, not the live array: real readdirSync snapshots the directory // at call time, so a caller iterating the result while also removing diff --git a/lib/setup/__tests__/steps-a.test.ts b/lib/setup/__tests__/steps-a.test.ts index 7e573612..8e0becb1 100644 --- a/lib/setup/__tests__/steps-a.test.ts +++ b/lib/setup/__tests__/steps-a.test.ts @@ -11,6 +11,7 @@ import { setSetting } from "../../settings/write.ts"; import { closeStateDb, setKvValue } from "../../state/index.ts"; import { serializeIdentity } from "../../settings/identity.ts"; import { linkPath } from "../../deps/links.ts"; +import { DEV_MODE_TAG } from "../../dev-mode.ts"; import type { ExecResult, Probes } from "../probes.ts"; import type { SecretsExecResult, SecretsExecSeam, SecretsSeams } from "../../secrets/store.ts"; import { readTeamSecret, teamSopsYamlPath } from "../../secrets/team-store.ts"; @@ -668,7 +669,16 @@ describe("path.link / settings.seed / repos.clone / intercepts.install (real HOM } test("path.link: links fast-browser/gitq/deck, skips rt when the dev-mode wrapper owns ~/.local/bin/rt, installs shell + zshenv precedence", async () => { - const p = bundleProbe({ files: { [linkPath(home, "rt")]: "#!/bin/sh\nexec bun run cli.ts \"$@\"\n" } }); + // isDevModeWrapper reads through p.readPrefix (Probes-routed, bounded), + // so the fake in-memory files map is enough -- no real fs write needed. + // (A real write here would also flip the REAL currentMode(), which this + // describe block's `home` is real HOME for: appBundleRoot() would then + // hunt for the dev-flavor bundle name and miss this fixture's prod-named + // appRoot entirely, skipping fast-browser/gitq/deck too.) Content must be + // genuinely recognized (the marker), not any bare "#!" script. + const rtLinkPath = linkPath(home, "rt"); + const wrapperContent = `#!/bin/sh\n${DEV_MODE_TAG}\nexec bun run cli.ts "$@"\n`; + const p = bundleProbe({ files: { [rtLinkPath]: wrapperContent } }); const { ctx, logs } = makeCtx(p); const outcome = await pathLinkStep.run(ctx); diff --git a/lib/setup/__tests__/validators-mac.test.ts b/lib/setup/__tests__/validators-mac.test.ts index 3198a8c0..1b81998d 100644 --- a/lib/setup/__tests__/validators-mac.test.ts +++ b/lib/setup/__tests__/validators-mac.test.ts @@ -75,6 +75,28 @@ describe("macRows — tool.clt", () => { }); }); +describe("macRows: tool.arch", () => { + test("arm64 -> ready", async () => { + const execScript: ExecScript = (argv) => (argv[0] === "uname" ? ok("arm64\n") : ok()); + const r = await pickRow(macRows(fakeProbes({ exec: execScript })), "tool.arch"); + expect(r.status).toBe("ready"); + expect(r.detail).toContain("arm64"); + expect(r.required).toBe(true); + }); + + test("x86_64 -> invalid, unsupported architecture", async () => { + const execScript: ExecScript = (argv) => (argv[0] === "uname" ? ok("x86_64\n") : ok()); + const r = await pickRow(macRows(fakeProbes({ exec: execScript })), "tool.arch"); + expect(r.status).toBe("invalid"); + }); + + test("uname unreachable -> error, never invalid (couldn't determine, not a failed determination)", async () => { + const execScript: ExecScript = (argv) => (argv[0] === "uname" ? missing("uname") : ok()); + const r = await pickRow(macRows(fakeProbes({ exec: execScript })), "tool.arch"); + expect(r.status).toBe("error"); + }); +}); + describe("macRows — tool.path", () => { test("~/.local/bin first on PATH and the precedence marker present -> ready", async () => { const p = fakeProbes({ diff --git a/lib/setup/__tests__/validators-rt-health.test.ts b/lib/setup/__tests__/validators-rt-health.test.ts index d5f2fc12..9e3501de 100644 --- a/lib/setup/__tests__/validators-rt-health.test.ts +++ b/lib/setup/__tests__/validators-rt-health.test.ts @@ -4,6 +4,7 @@ import { existsSync, mkdirSync, mkdtempSync, readFileSync, realpathSync, rmSync, import { tmpdir } from "os"; import { dirname, join } from "path"; import { DAEMON_CONFIG_PATH } from "../../daemon-config.ts"; +import { DEV_MODE_TAG } from "../../dev-mode.ts"; import { LOGIN_ITEMS_SETTINGS_ACTION } from "../permissions.ts"; import { setSetting } from "../../settings/write.ts"; import { homeBackupRow, rtHealthRows } from "../validators/rt-health.ts"; @@ -139,7 +140,7 @@ describe("rtHealthRows — rows that resolve the app bundle", () => { test("dev mode wrapper at ~/.local/bin/rt -> skipped, dev mode owns it", async () => { const wrapperDir = join(home, ".local", "bin"); mkdirSync(wrapperDir, { recursive: true }); - writeFileSync(join(wrapperDir, "rt"), "#!/bin/sh\nexit 0\n", { mode: 0o755 }); + writeFileSync(join(wrapperDir, "rt"), `#!/bin/sh\n${DEV_MODE_TAG}\nexit 0\n`, { mode: 0o755 }); const p = bundleProbe(); const r = await pickRow(rtHealthRows(p, { ci: false }, NOOP_FZF), "tool.rt-link"); expect(r.status).toBe("skipped"); @@ -529,10 +530,10 @@ describe("rtHealthRows — tool.flavor", () => { rmSync(home, { recursive: true, force: true }); }); - /** A script at ~/.local/bin/rt is the dev-mode signal currentMode() reads. */ + /** A recognized wrapper at ~/.local/bin/rt is the dev-mode signal currentMode() reads. */ function writeDevWrapper(): void { mkdirSync(join(home, ".local", "bin"), { recursive: true }); - writeFileSync(join(home, ".local", "bin", "rt"), "#!/bin/sh\necho dev\n"); + writeFileSync(join(home, ".local", "bin", "rt"), `#!/bin/sh\n${DEV_MODE_TAG}\necho dev\n`); } test("live daemon of the wrong flavor: fail, names all three legs", async () => { diff --git a/lib/setup/probes.ts b/lib/setup/probes.ts index 1c14d3b1..4f4f6505 100644 --- a/lib/setup/probes.ts +++ b/lib/setup/probes.ts @@ -8,6 +8,7 @@ import { chmodSync, existsSync, mkdirSync, readdirSync, readFileSync, readlinkSync, rmSync, statSync, symlinkSync, unlinkSync, writeFileSync } from "fs"; import { homedir } from "os"; import { daemonSocketQuery, trayRequest, type DaemonResponse, type TrayClient } from "../daemon-client.ts"; +import { readWrapperPrefix } from "../dev-mode.ts"; import { UserActionableError } from "./errors.ts"; export interface ExecResult { @@ -23,6 +24,13 @@ export interface Probes { /** Byte size, following symlinks, only for a REGULAR file (a directory, a symlink to one, or anything missing/unreadable is null) — the cheap "is this actually a file worth reading" check callers need before decoding one. */ fileSize(path: string): number | null; readFile(path: string): string | null; + /** + * A bounded 4096-byte prefix of `path`, following symlinks -- never a + * whole-file read. Exists for callers that must classify a file by its + * head (e.g. dev-mode wrapper detection) where the target may be a symlink + * to a multi-MB binary; `readFile` would read the whole thing. + */ + readPrefix(path: string): string | null; readDir(path: string): string[]; readlink(path: string): string | null; writeFile(path: string, content: string, mode?: number): void; @@ -173,6 +181,10 @@ export function createRealProbes(): Probes { } }, + readPrefix(path) { + return readWrapperPrefix(path); + }, + readDir(path) { try { return readdirSync(path); diff --git a/lib/setup/validators/mac.ts b/lib/setup/validators/mac.ts index d7136287..dad7b44e 100644 --- a/lib/setup/validators/mac.ts +++ b/lib/setup/validators/mac.ts @@ -39,6 +39,26 @@ async function cltRow(p: Probes): Promise { return row({ ...base, status: "missing", detail: "Apple command line tools not installed", action: CLT_INSTALL_ACTION }); } +async function archRow(p: Probes): Promise { + const base = { + id: "tool.arch", + kind: "tool" as const, + title: "Processor", + why: "mattstack ships an Apple-silicon (arm64) build; Intel Macs are not supported.", + required: true, + }; + const res = await p.exec(["uname", "-m"]); + const arch = res.stdout.trim(); + + // Same honesty ruling as macosVersionRow: a probe that couldn't run reports + // "error", not "invalid": only a definite non-arm64 result is invalid. + if (res.code !== 0 || !arch) { + return row({ ...base, status: "error", detail: "Could not determine your processor" }); + } + if (arch === "arm64") return row({ ...base, status: "ready", detail: "Apple silicon (arm64)" }); + return row({ ...base, status: "invalid", detail: `${arch}: Apple silicon (arm64) required` }); +} + function pathRow(p: Probes): Row { const base = { id: "tool.path", kind: "info" as const, title: "PATH precedence", why: "Makes sure your shell finds rt's shims and team intercepts before any conflicting binary.", required: false }; const localBin = `${p.home}/.local/bin`; @@ -57,6 +77,6 @@ function pathRow(p: Probes): Row { } export async function macRows(p: Probes): Promise { - const [macos, clt] = await Promise.all([macosVersionRow(p), cltRow(p)]); - return [macos, clt, pathRow(p)]; + const [macos, clt, arch] = await Promise.all([macosVersionRow(p), cltRow(p), archRow(p)]); + return [macos, clt, arch, pathRow(p)]; } diff --git a/lib/state/__tests__/branch-cache.test.ts b/lib/state/__tests__/branch-cache.test.ts index f58805c1..16bfef7d 100644 --- a/lib/state/__tests__/branch-cache.test.ts +++ b/lib/state/__tests__/branch-cache.test.ts @@ -13,7 +13,77 @@ import { existsSync, mkdtempSync, rmSync, writeFileSync } from "fs"; import { tmpdir } from "os"; import { join } from "path"; import { closeStateDb, getStateDb, openStateDb } from "../db.ts"; -import { getBranchCacheStore, rekeyBranchCacheTable, type CacheEntry } from "../branch-cache.ts"; +import { branchOf, composeKey, getBranchCacheStore, getByBranch, identityOf, rekeyBranchCacheTable, type CacheEntry } from "../branch-cache.ts"; + +test("composeKey/branchOf/identityOf round-trip with a serialized identity", () => { + const id = "remote:gitlab.com%2Facme%2Facme-dev"; + const k = composeKey(id, "feature/x"); + expect(k).toBe(`${id}:feature/x`); + expect(branchOf(k)).toBe("feature/x"); + expect(identityOf(k)).toBe(id); +}); +test("bare key (no identity) degrades gracefully", () => { + expect(composeKey(undefined, "main")).toBe("main"); + expect(branchOf("main")).toBe("main"); + expect(identityOf("main")).toBeUndefined(); +}); +test("branch never contains a colon, so lastIndexOf split is unambiguous", () => { + const k = composeKey("path:%2FUsers%2Fdev%2Fscratch", "release"); + expect(branchOf(k)).toBe("release"); + expect(identityOf(k)).toBe("path:%2FUsers%2Fdev%2Fscratch"); +}); + +describe("getByBranch: free function over an entries map", () => { + function makeCacheEntry(linearId: string): CacheEntry { + return { ticket: null, linearId, mr: null, fetchedAt: Date.now() }; + } + + test("exact bare-key hit", () => { + const entries: Record = { main: makeCacheEntry("bare") }; + expect(getByBranch(entries, "main")?.linearId).toBe("bare"); + }); + + test("suffix hit across two repos sharing the same branch name picks a match, not a false negative", () => { + const entries: Record = { + "remote:gitlab.com%2Facme%2Frepo-a:feature/x": makeCacheEntry("repo-a"), + "remote:gitlab.com%2Facme%2Frepo-b:feature/x": makeCacheEntry("repo-b"), + }; + const hit = getByBranch(entries, "feature/x"); + expect(hit).toBeDefined(); + expect(["repo-a", "repo-b"]).toContain(hit!.linearId); + }); + + test("miss returns undefined", () => { + const entries: Record = { main: makeCacheEntry("bare") }; + expect(getByBranch(entries, "nonexistent")).toBeUndefined(); + }); +}); + +describe("put: composite-key collision safety (S069/Task 10)", () => { + let collisionDir: string; + + beforeEach(() => { + collisionDir = mkdtempSync(join(tmpdir(), "rt-branch-cache-collision-")); + }); + + afterEach(() => { + rmSync(collisionDir, { recursive: true, force: true }); + }); + + test("put keys by entry.repoName so same-name branches in two repos coexist", () => { + const dbPath = join(collisionDir, "state.db"); + const db = openStateDb(dbPath, "cli"); + const store = getBranchCacheStore(db); + + store.put("main", { repoName: "remote:host%2Fa", ticket: null, linearId: "", mr: null, fetchedAt: 1 }); + store.put("main", { repoName: "remote:host%2Fb", ticket: null, linearId: "", mr: null, fetchedAt: 2 }); + + expect(store.entries[composeKey("remote:host%2Fa", "main")]?.fetchedAt).toBe(1); + expect(store.entries[composeKey("remote:host%2Fb", "main")]?.fetchedAt).toBe(2); + expect(Object.keys(store.entries).length).toBe(2); + db.close(); + }); +}); let dir: string; @@ -170,19 +240,34 @@ describe("two handles, per-row last-writer-wins", () => { }); describe("bare-branch upsert semantics", () => { - test("a no-repoName upsert hits the same row a repoName-bearing write created (no NULL duplicate)", () => { + test("two puts with the same repoName hit the same row (no duplicate)", () => { const dbPath = join(dir, "state.db"); const db = openStateDb(dbPath, "cli"); const store = getBranchCacheStore(db); + const key = composeKey("repo-tools", "feature/x"); store.put("feature/x", makeEntry({ repoName: "repo-tools", linearId: "first" })); - expect(rowCount(db, "feature/x")).toBe(1); + expect(rowCount(db, key)).toBe(1); + + store.put("feature/x", makeEntry({ repoName: "repo-tools", linearId: "second" })); + + expect(rowCount(db, key)).toBe(1); + expect(store.entries[key]?.linearId).toBe("second"); + db.close(); + }); + + test("a repoName-bearing write and a bare (no-repoName) write to the same branch land in different rows (Task 10: key is composeKey(entry.repoName, branch), not the bare branch)", () => { + const dbPath = join(dir, "state.db"); + const db = openStateDb(dbPath, "cli"); + const store = getBranchCacheStore(db); + store.put("feature/x", makeEntry({ repoName: "repo-tools", linearId: "attributed" })); // enrichBranches-style upsert: same bare branch, no repoName available. - store.put("feature/x", makeEntry({ linearId: "second" })); + store.put("feature/x", makeEntry({ linearId: "bare" })); - expect(rowCount(db, "feature/x")).toBe(1); - expect(store.entries["feature/x"]?.linearId).toBe("second"); + expect(store.entries[composeKey("repo-tools", "feature/x")]?.linearId).toBe("attributed"); + expect(store.entries["feature/x"]?.linearId).toBe("bare"); + expect(Object.keys(store.entries).length).toBe(2); db.close(); }); @@ -215,13 +300,15 @@ describe("gc — succeeded-repo gating and NULL-repo age rule", () => { store.gc(new Set(["repo-a"]), 30 * DAY_MS); - expect(store.entries["stale-succeeded"]).toBeUndefined(); - expect(rowCount(db, "stale-succeeded")).toBe(0); + const succeededKey = composeKey("repo-a", "stale-succeeded"); + const failedKey = composeKey("repo-b", "stale-failed"); + expect(store.entries[succeededKey]).toBeUndefined(); + expect(rowCount(db, succeededKey)).toBe(0); // repo-b had a swallowed fetch error this cycle (never made it into // succeededRepos) — its stale rows must survive (r2 finding 1). - expect(store.entries["stale-failed"]).toBeDefined(); - expect(rowCount(db, "stale-failed")).toBe(1); + expect(store.entries[failedKey]).toBeDefined(); + expect(rowCount(db, failedKey)).toBe(1); db.close(); }); @@ -248,8 +335,9 @@ describe("gc — succeeded-repo gating and NULL-repo age rule", () => { store.gc(new Set(["repo-a"]), 30 * DAY_MS); - expect(store.entries["fresh-succeeded"]).toBeDefined(); - expect(rowCount(db, "fresh-succeeded")).toBe(1); + const key = composeKey("repo-a", "fresh-succeeded"); + expect(store.entries[key]).toBeDefined(); + expect(rowCount(db, key)).toBe(1); db.close(); }); @@ -264,9 +352,10 @@ describe("gc — succeeded-repo gating and NULL-repo age rule", () => { store.gc(new Set(["repo-a"]), 30 * DAY_MS); - expect(Object.keys(store.entries).sort()).toEqual(["keep"]); + const keepKey = composeKey("repo-a", "keep"); + expect(Object.keys(store.entries).sort()).toEqual([keepKey]); const remaining = db.query("SELECT branch FROM branch_cache;").all() as { branch: string }[]; - expect(remaining.map(r => r.branch).sort()).toEqual(["keep"]); + expect(remaining.map(r => r.branch).sort()).toEqual([keepKey]); db.close(); }); @@ -275,6 +364,8 @@ describe("gc — succeeded-repo gating and NULL-repo age rule", () => { const db = openStateDb(dbPath, "cli"); const store = getBranchCacheStore(db); + const racyKey = composeKey("repo-a", "racy"); + const doomedKey = composeKey("repo-a", "doomed"); store.put("racy", makeEntry({ repoName: "repo-a", fetchedAt: oldTs })); store.put("doomed", makeEntry({ repoName: "repo-a", fetchedAt: oldTs })); @@ -296,7 +387,7 @@ describe("gc — succeeded-repo gating and NULL-repo age rule", () => { (stmt as unknown as { all: unknown }).all = (...args: never[]) => { const rows = realAll(...args); (stmt as unknown as { all: unknown }).all = realAll; // one-shot - cli.query("UPDATE branch_cache SET fetched_at = ? WHERE branch = ?;").run(freshTs, "racy"); + cli.query("UPDATE branch_cache SET fetched_at = ? WHERE branch = ?;").run(freshTs, racyKey); return rows; }; } @@ -311,13 +402,13 @@ describe("gc — succeeded-repo gating and NULL-repo age rule", () => { // The freshly enriched row survives with its new timestamp; its stale // sibling is still pruned. - expect(rowCount(db, "racy")).toBe(1); - const { fetched_at } = db.query("SELECT fetched_at FROM branch_cache WHERE branch = ?;").get("racy") as { fetched_at: number }; + expect(rowCount(db, racyKey)).toBe(1); + const { fetched_at } = db.query("SELECT fetched_at FROM branch_cache WHERE branch = ?;").get(racyKey) as { fetched_at: number }; expect(fetched_at).toBe(freshTs); - expect(rowCount(db, "doomed")).toBe(0); + expect(rowCount(db, doomedKey)).toBe(0); // Row/map parity holds on both sides of the re-guard. - expect(store.entries["racy"]).toBeDefined(); - expect(store.entries["doomed"]).toBeUndefined(); + expect(store.entries[racyKey]).toBeDefined(); + expect(store.entries[doomedKey]).toBeUndefined(); cli.close(); db.close(); @@ -349,7 +440,7 @@ describe("daemon-flavor busy handling", () => { expect(() => store.put("busy-branch", makeEntry({ repoName: "repo-a" }))).not.toThrow(); // The map is the daemon's read model (spec "In-memory ownership") — // it must carry the enrichment even when the row could not. - expect(store.entries["busy-branch"]).toBeDefined(); + expect(store.entries[composeKey("repo-a", "busy-branch")]).toBeDefined(); } finally { lock.release(); } @@ -360,6 +451,7 @@ describe("daemon-flavor busy handling", () => { const dbPath = join(dir, "state.db"); const db = openStateDb(dbPath, "daemon"); const store = getBranchCacheStore(db); + const key = composeKey("repo-a", "stale-busy"); store.put("stale-busy", makeEntry({ repoName: "repo-a", fetchedAt: Date.now() - 40 * 24 * 60 * 60 * 1000 })); const lock = holdWriteLock(dbPath); @@ -367,11 +459,11 @@ describe("daemon-flavor busy handling", () => { expect(() => store.gc(new Set(["repo-a"]), 30 * 24 * 60 * 60 * 1000)).not.toThrow(); // Rows survived, so the map must too — otherwise the next reload() // would resurrect an entry the map had already dropped. - expect(store.entries["stale-busy"]).toBeDefined(); + expect(store.entries[key]).toBeDefined(); } finally { lock.release(); } - expect(rowCount(db, "stale-busy")).toBe(1); + expect(rowCount(db, key)).toBe(1); db.close(); }, 10_000); @@ -379,16 +471,17 @@ describe("daemon-flavor busy handling", () => { const dbPath = join(dir, "state.db"); const db = openStateDb(dbPath, "daemon"); const store = getBranchCacheStore(db); + const key = composeKey("repo-a", "doomed"); store.put("doomed", makeEntry({ repoName: "repo-a" })); const lock = holdWriteLock(dbPath); try { - expect(() => store.delete("doomed")).not.toThrow(); - expect(store.entries["doomed"]).toBeDefined(); + expect(() => store.delete(key)).not.toThrow(); + expect(store.entries[key]).toBeDefined(); } finally { lock.release(); } - expect(rowCount(db, "doomed")).toBe(1); + expect(rowCount(db, key)).toBe(1); db.close(); }, 10_000); }); @@ -443,7 +536,8 @@ describe("rekeyBranchCacheTable", () => { getBranchCacheStore().put("feature/x", makeEntry({ repoName: "remote:gitlab.com%2Fg%2Fr" })); const report = await rekeyBranchCacheTable(); expect(report.migrated).toEqual([]); - const row = getStateDb().query("SELECT repo FROM branch_cache WHERE branch = ?;").get("feature/x") as { repo: string }; + const row = getStateDb().query("SELECT repo FROM branch_cache WHERE branch = ?;") + .get(composeKey("remote:gitlab.com%2Fg%2Fr", "feature/x")) as { repo: string }; expect(row.repo).toBe("remote:gitlab.com%2Fg%2Fr"); }); @@ -451,7 +545,8 @@ describe("rekeyBranchCacheTable", () => { getBranchCacheStore().put("feature/y", makeEntry({ repoName: "ghost-repo" })); const report = await rekeyBranchCacheTable(); expect(report.retained).toEqual(["ghost-repo"]); - const row = getStateDb().query("SELECT repo FROM branch_cache WHERE branch = ?;").get("feature/y") as { repo: string }; + const row = getStateDb().query("SELECT repo FROM branch_cache WHERE branch = ?;") + .get(composeKey("ghost-repo", "feature/y")) as { repo: string }; expect(row.repo).toBe("ghost-repo"); expect(warnSpy).toHaveBeenCalled(); }); diff --git a/lib/state/branch-cache.ts b/lib/state/branch-cache.ts index 72789473..8fa12342 100644 --- a/lib/state/branch-cache.ts +++ b/lib/state/branch-cache.ts @@ -54,6 +54,33 @@ export function rekeyBranchCacheTable(): Promise { return rekeyTableColumn("branch_cache", "repo"); } +/** state.db keys the branch cache on `${serializedIdentity}:${branch}`. Split + * on the LAST colon: git branch names contain none, serialized identities + * always carry their own (remote:/path:), so this is unambiguous. */ +export function composeKey(identity: string | undefined, branch: string): string { + return identity ? `${identity}:${branch}` : branch; +} +export function branchOf(key: string): string { + const i = key.lastIndexOf(":"); + return i < 0 ? key : key.slice(i + 1); +} +export function identityOf(key: string): string | undefined { + const i = key.lastIndexOf(":"); + return i < 0 ? undefined : key.slice(0, i); +} + +/** + * Free function, not a store method, on purpose: `BranchCacheStore` is a + * structural interface with implementers outside this module's ownership + * (lib/daemon.ts's facade), so growing the interface forces edits there too. + * Scans for any key ending in `:${branch}` (or the bare branch itself). + */ +export function getByBranch(entries: Record, branch: string): CacheEntry | undefined { + const suffix = `:${branch}`; + for (const [k, v] of Object.entries(entries)) if (k === branch || k.endsWith(suffix)) return v; + return undefined; +} + export interface BranchCacheStore { /** The live map — ctx.cache-compatible. Same object identity across reload(). */ entries: Record; @@ -120,21 +147,26 @@ function createStore(db: Database): BranchCacheStore { } function put(branch: string, entry: CacheEntry): void { + // Keyed by composeKey(entry.repoName, branch), not the bare branch: two + // repos with a same-named branch must land in different rows/map slots, + // never overwrite each other (the collision Task 10 fixes). The row's + // `branch` column and the map key are always the same composite string. + const key = composeKey(entry.repoName, branch); // The map update sits OUTSIDE the wrapper on purpose: it is this cycle's // freshly enriched truth and the thing handlers serve, so it must land // even when the row defers. (gc/delete keep the two together instead — // see below.) persistOrWarn("branch-cache", () => { db.query(UPSERT_SQL).run( - branch, + key, entry.repoName ?? null, entry.ticket !== null ? JSON.stringify(entry.ticket) : null, entry.linearId, entry.mr !== null ? JSON.stringify(entry.mr) : null, entry.fetchedAt, ); - }, { op: "put", branch }); - entries[branch] = entry; + }, { op: "put", branch: key }); + entries[key] = entry; } function del(branch: string): void { diff --git a/lib/worktree/__tests__/dispose.test.ts b/lib/worktree/__tests__/dispose.test.ts index aed1d695..f3dc096e 100644 --- a/lib/worktree/__tests__/dispose.test.ts +++ b/lib/worktree/__tests__/dispose.test.ts @@ -6,6 +6,7 @@ import { basename, dirname, join } from "path"; import { teamSettingsPath } from "../../rt-paths.ts"; import { setSetting } from "../../settings/write.ts"; import { closeStateDb, getBranchCacheStore, type CacheEntry } from "../../state/index.ts"; +import { branchOf } from "../../state/branch-cache.ts"; import { loadRegistry, saveRegistry, type TreeRecord } from "../registry.ts"; import { branchExistsLocalAsync, listWorktreesAsync, remoteRefExists } from "../git-async.ts"; import { hasFreshAttendantLease } from "../lease.ts"; @@ -761,7 +762,10 @@ describe("disposeTree against the real branch_cache store (identity-keyed)", () })); // Seeded exactly as cache-refresh.ts writes it: repoName is the same - // identity the daemon iterates the repo index under. + // identity the daemon iterates the repo index under. The store now keys + // its own map by composeKey(repoName, branch); the daemon's caller + // (worktree-reconciler.ts actOnTree) hands disposeTree a bare-keyed, + // this-repo-only view; reproduce that same remap here. const store = getBranchCacheStore(); store.put("feature-a", { ticket: null, @@ -770,11 +774,14 @@ describe("disposeTree against the real branch_cache store (identity-keyed)", () mr: { iid: 42, sha, state: "merged" } as unknown as CacheEntry["mr"], repoName: identityRepoName, }); + const cacheEntries = Object.fromEntries( + Object.entries(store.entries).map(([key, entry]) => [branchOf(key), entry]), + ); const deps: DisposeDeps = { repoName: identityRepoName, repoPath: repo, - cacheEntries: store.entries, + cacheEntries, emit: (type, data) => events.push({ type, data }), log: { info: () => {}, warn: () => {} }, killProcesses: false, @@ -801,11 +808,14 @@ describe("disposeTree against the real branch_cache store (identity-keyed)", () mr: { iid: 42, sha, state: "merged" } as unknown as CacheEntry["mr"], repoName: "acme", // pre-rekey legacy display name }); + const cacheEntries = Object.fromEntries( + Object.entries(store.entries).map(([key, entry]) => [branchOf(key), entry]), + ); const deps: DisposeDeps = { repoName: identityRepoName, repoPath: repo, - cacheEntries: store.entries, + cacheEntries, emit: (type, data) => events.push({ type, data }), log: { info: () => {}, warn: () => {} }, killProcesses: false, diff --git a/packages/rt-client/src/index.ts b/packages/rt-client/src/index.ts index 20740937..d95d2ae0 100644 --- a/packages/rt-client/src/index.ts +++ b/packages/rt-client/src/index.ts @@ -88,7 +88,7 @@ export { repoNameForPath } from "./repos.ts"; // ─── Settings (RT-50) ──────────────────────────────────────────────────────── -export { getSetting, listSettings, explainSetting, expandVariables, SCOPE_ORDER } from "./settings/resolve.ts"; +export { getSetting, listSettings, explainSetting, expandVariables, SCOPE_ORDER, setSettingsWarnSink } from "./settings/resolve.ts"; export type { Scope, Provenance, diff --git a/packages/rt-client/src/settings/__tests__/registry.test.ts b/packages/rt-client/src/settings/__tests__/registry.test.ts index e84146b4..50966037 100644 --- a/packages/rt-client/src/settings/__tests__/registry.test.ts +++ b/packages/rt-client/src/settings/__tests__/registry.test.ts @@ -51,7 +51,7 @@ describe("settings/registry", () => { } }); - test("exactly 23 keys are migrated:true", () => { + test("exactly 24 keys are migrated:true", () => { const migrated = allDefs().filter((d) => d.migrated); expect(migrated.map((d) => d.key).sort()).toEqual( @@ -59,7 +59,7 @@ describe("settings/registry", () => { "rt.intercepts", "rt.repoIdentityOverrides", "rt.repoRoots", "rt.roles", "rt.worktrees", "rt.notifications", "rt.cron", "rt.repoTracking", "rt.runsPruneDays", "rt.runaway", "rt.workspacePrefs", "rt.sync", "rt.branchNaming", "rt.variations", "rt.presets", "rt.dopplerTemplate", - "rt.homeSnapshot", "rt.worktreeApp", "rt.sdmEnrichment", "rt.logRetentionDays", "rt.integrations", "rt.hooks", + "rt.homeSnapshot", "rt.worktreeApp", "rt.sdmEnrichment", "rt.logRetentionDays", "rt.logLevel", "rt.integrations", "rt.hooks", "rt.apiPort", ].sort(), ); @@ -74,6 +74,15 @@ describe("settings/registry", () => { expect(def?.default).toBe(14); }); + test("rt.daemonPath is a machine-scoped string key with no default", () => { + const def = getDef("rt.daemonPath"); + expect(def).toBeDefined(); + expect(def!.type).toBe("string"); + expect(def!.scopes).toEqual(["machine"]); + expect(def!.default).toBeUndefined(); + expect(def!.pathGuardFields).toBeUndefined(); + }); + test("rt.worktreeApp is a machine-only field-bag object with no default (ownership latch)", () => { const def = getDef("rt.worktreeApp"); @@ -198,13 +207,13 @@ describe("settings/registry", () => { expect(def?.merge).toBe("replace"); }); - test("has exactly the 23 migrated:true keys and the 43 suite keys", () => { + test("has exactly the 24 migrated:true keys and the 44 suite keys", () => { const migratedFalseKeys: string[] = []; const migratedTrueKeys = [ "rt.roles", "rt.intercepts", "rt.worktrees", "rt.repoIdentityOverrides", "rt.repoRoots", "rt.notifications", "rt.cron", "rt.repoTracking", "rt.runsPruneDays", "rt.runaway", "rt.workspacePrefs", "rt.sync", "rt.branchNaming", "rt.variations", "rt.presets", "rt.dopplerTemplate", - "rt.homeSnapshot", "rt.worktreeApp", "rt.sdmEnrichment", "rt.logRetentionDays", "rt.integrations", "rt.hooks", + "rt.homeSnapshot", "rt.worktreeApp", "rt.sdmEnrichment", "rt.logRetentionDays", "rt.logLevel", "rt.integrations", "rt.hooks", "rt.apiPort", ]; const suiteKeys = [ @@ -251,8 +260,9 @@ describe("settings/registry", () => { "agent.account", "agent.extraArgs", "rt.trustedBrowserOrigins", + "rt.daemonPath", ]; - expect(suiteKeys).toHaveLength(43); + expect(suiteKeys).toHaveLength(44); expect(allDefs().map((d) => d.key).sort()).toEqual( [...migratedFalseKeys, ...migratedTrueKeys, ...suiteKeys].sort(), diff --git a/packages/rt-client/src/settings/registry-defs.ts b/packages/rt-client/src/settings/registry-defs.ts index 8a00016b..d33a1821 100644 --- a/packages/rt-client/src/settings/registry-defs.ts +++ b/packages/rt-client/src/settings/registry-defs.ts @@ -196,6 +196,15 @@ export const REGISTRY: readonly SettingDef[] = [ migrated: true, description: "Age floor in days for the log janitor pruning every surface's rotated log files under ~/.mattstack/rt/logs (default 14). A fresh key, not an ownership-latch port, so a default is fine here.", }, + { + key: "rt.logLevel", + type: "string", + scopes: ["machine", "user"], + default: "info", + merge: "replace", + migrated: true, + description: "Daemon log level (trace|debug|info|warn|error). RT_LOG_LEVEL env wins, then this setting, then info (lib/daemon-logger.ts resolveDaemonLogLevel). A fresh key, not an ownership-latch port, so a default is fine here.", + }, { key: "rt.apiPort", type: "number", @@ -214,6 +223,14 @@ export const REGISTRY: readonly SettingDef[] = [ migrated: true, description: "Per-repo git hook enable/disable state ({enabled, hooks: {: boolean}}); ownership-latch port of repos//hooks.json, store wins per field once it owns the key — including per-hook-name entries inside the nested hooks map, each defaulting to enabled when absent. The installed git-hook shim still greps repos//hooks.json with zero process spawns (a hook fires on every git operation); that file is now a DERIVED CACHE this key writes through, kept current by commands/hooks.ts's regenerateHooksCache at every write seam.", }, + { + key: "rt.daemonPath", + type: "string", + scopes: ["machine"], + merge: "replace", + description: + "Absolute colon-separated PATH the daemon uses for every child it spawns, instead of probing your login shell. Set this when the daemon can't find node/git/bun/pnpm (e.g. a fish shell, a blocking .zshrc, or PATH exports that live only in .zshrc). Machine-scoped: it never travels to another machine.", + }, { key: "rt.trustedBrowserOrigins", type: "array", diff --git a/packages/rt-client/src/settings/resolve.ts b/packages/rt-client/src/settings/resolve.ts index 9c07a2a8..6e949981 100644 --- a/packages/rt-client/src/settings/resolve.ts +++ b/packages/rt-client/src/settings/resolve.ts @@ -487,8 +487,29 @@ function expandCtxFrom(opts: ResolveOpts): ExpandCtx { }; } +let warnSink: ((msg: string) => void) | null = null; +const warnedOnce = new Set(); + +/** The daemon binds a deduped log.warn here so a hot-path getSetting on a + * disallowed-scope key warns once, not every tick. Default: console.warn + * (CLI/test behavior unchanged). null restores the default. */ +export function setSettingsWarnSink(sink: ((msg: string) => void) | null): void { + warnSink = sink; + warnedOnce.clear(); +} + +export function emitSettingsWarning(msg: string): void { + if (warnSink) { + if (warnedOnce.has(msg)) return; + warnedOnce.add(msg); + warnSink(msg); + return; + } + console.warn(msg); +} + function warnInvalid(key: string, entry: InvalidScope): void { - console.warn( + emitSettingsWarning( `rt: ignoring "${key}" from the ${entry.scope} scope (${entry.file ?? "no file"}): ${entry.reason}`, ); } @@ -543,7 +564,7 @@ export function listSettings(opts: ResolveOpts = {}): ListedSetting[] { listed.value = expandVariables(resolution.value, ctx); } catch (err) { listed.expandError = (err as Error).message; - console.warn(`rt: showing "${def.key}" unexpanded — ${listed.expandError}`); + emitSettingsWarning(`rt: showing "${def.key}" unexpanded — ${listed.expandError}`); } } @@ -583,7 +604,7 @@ function listUnregistered(stores: StoreBundle, opts: ResolveOpts): ListedSetting return [...found.entries()] .sort(([a], [b]) => a.localeCompare(b)) .map(([key, hit]) => { - console.warn( + emitSettingsWarning( `rt: unregistered setting "${key}" in ${hit.file} — ignoring it (this rt may be older than the store)`, ); return { diff --git a/packages/rt-client/src/transport.ts b/packages/rt-client/src/transport.ts index b8243c24..169295d0 100644 --- a/packages/rt-client/src/transport.ts +++ b/packages/rt-client/src/transport.ts @@ -57,7 +57,7 @@ export async function rtCommand( const res = await fetch(`http://localhost/${cmd}`, { unix: sockPath, method: "POST", - headers: { "Content-Type": "application/json" }, + headers: { "Content-Type": "application/json", "X-RT-Client": `rt-client/${process.pid}` }, body: JSON.stringify(payload), signal: AbortSignal.timeout(opts.timeoutMs ?? 15_000), // Bun's `unix` fetch option isn't in the standard RequestInit type. diff --git a/packages/rt-client/test/settings-warn-sink.test.ts b/packages/rt-client/test/settings-warn-sink.test.ts new file mode 100644 index 00000000..d0bc9480 --- /dev/null +++ b/packages/rt-client/test/settings-warn-sink.test.ts @@ -0,0 +1,12 @@ +import { test, expect } from "bun:test"; +import { setSettingsWarnSink } from "../src/index.ts"; +import { emitSettingsWarning } from "../src/settings/resolve.ts"; + +test("a bound sink receives warnings and dedupes on identical messages", () => { + const seen: string[] = []; + setSettingsWarnSink((m) => seen.push(m)); + emitSettingsWarning("rt: sample warning"); + emitSettingsWarning("rt: sample warning"); + expect(seen).toEqual(["rt: sample warning"]); // deduped + setSettingsWarnSink(null); // restore default +}); diff --git a/website/docs/reference/daemon/index.mdx b/website/docs/reference/daemon/index.mdx index fecbec70..2991b00a 100644 --- a/website/docs/reference/daemon/index.mdx +++ b/website/docs/reference/daemon/index.mdx @@ -27,5 +27,6 @@ rt daemon | [`status`](status) | Show daemon status | | [`track`](track) | Per-repo background tracking (live/poll/off) | | [`logs`](logs) | Show daemon logs | +| [`log-level`](log-level) | Show or set the daemon's live log level | {/* generated by scripts/gen-docs.ts; edit prose in _partials, not here */} \ No newline at end of file diff --git a/website/docs/reference/daemon/log-level.mdx b/website/docs/reference/daemon/log-level.mdx new file mode 100644 index 00000000..041d3a55 --- /dev/null +++ b/website/docs/reference/daemon/log-level.mdx @@ -0,0 +1,26 @@ +--- +title: rt daemon log-level +sidebar_label: log-level +--- + +# rt daemon log-level + +`rt › daemon › log-level` + +Show or set the daemon's live log level + +## Usage + +```bash +rt daemon log-level +``` + +## Arguments & flags + +| Flag / Arg | Type | Default | Description | +| --- | --- | --- | --- | +| `` | select | | Omit to show the current level | + +_See code: [commands/daemon.ts › setLogLevel](https://github.com/m4ttstack/rt/blob/main/commands/daemon.ts)_ + +{/* generated by scripts/gen-docs.ts; edit prose in _partials, not here */} \ No newline at end of file