From b4ce86acbc9b7028221e721059508e977cb39abd Mon Sep 17 00:00:00 2001 From: wiggdevin <202901685+wiggdevin@users.noreply.github.com> Date: Sun, 6 Sep 2026 03:21:01 -0700 Subject: [PATCH 1/7] docs(zs): harness reliability, agent health and curator designs (T16-T18) with T16 fixture tests Three design specs after Devin's 2026-09-06 decisions (pause on session limit, no seat rotation; only never-started batches replay; nothing discarded) and GPT-5.6 Sol's audit of the question set. Wave 6 tickets added to the implementation plan. crates/buzz-acp/src/reliability.rs holds the T16 fixture tests on the real log lines of 2026-09-02/03 plus the smallest stubs that let them compile. All seven are #[ignore] and fail with --ignored; T16 is ready when they pass un-ignored. Signed-off-by: wiggdevin <202901685+wiggdevin@users.noreply.github.com> --- crates/buzz-acp/src/lib.rs | 1 + crates/buzz-acp/src/reliability.rs | 269 ++++++++++++++++++ .../2026-09-04-zs-implementation-plan.md | 37 +++ docs/plans/2026-09-06-agent-health-design.md | 72 +++++ ...026-09-06-agent-self-improvement-design.md | 56 ++++ .../2026-09-06-harness-reliability-design.md | 129 +++++++++ 6 files changed, 564 insertions(+) create mode 100644 crates/buzz-acp/src/reliability.rs create mode 100644 docs/plans/2026-09-06-agent-health-design.md create mode 100644 docs/plans/2026-09-06-agent-self-improvement-design.md create mode 100644 docs/plans/2026-09-06-harness-reliability-design.md diff --git a/crates/buzz-acp/src/lib.rs b/crates/buzz-acp/src/lib.rs index 03d0c14e3c7..03a4cd248f3 100644 --- a/crates/buzz-acp/src/lib.rs +++ b/crates/buzz-acp/src/lib.rs @@ -16,6 +16,7 @@ mod prompt_framing; mod prompt_project; mod queue; mod relay; +mod reliability; mod scope; mod setup_mode; mod usage; diff --git a/crates/buzz-acp/src/reliability.rs b/crates/buzz-acp/src/reliability.rs new file mode 100644 index 00000000000..0dc7de529d4 --- /dev/null +++ b/crates/buzz-acp/src/reliability.rs @@ -0,0 +1,269 @@ +//! Harness reliability: error classes, pause, breaker, park and replay. +//! +//! Design: `docs/plans/2026-09-06-harness-reliability-design.md` (T16). +//! +//! This file currently holds the **fixture tests** for T16 and the smallest +//! stubs that let them compile. Every test is `#[ignore]` with the ticket +//! named in the reason, so the branch stays green while the behaviour is +//! missing. Run them with: +//! +//! ```text +//! cargo test -p buzz-acp reliability -- --ignored +//! ``` +//! +//! They fail today. T16 is ready when they pass without `#[ignore]`. + +// The stubs below have no caller until T16 wires them into the pool. +#![allow(dead_code)] + +use chrono::{DateTime, Utc}; + +use crate::acp::AcpError; +use crate::scope::SessionScope; + +/// Why a turn failed, as far as the harness can tell from the provider. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum ErrorClass { + /// The account is out of capacity for now: session limit, rate limit, + /// overloaded, quota. `resets_at` is the parsed reset time when the + /// provider named one. + CapacityExhausted { resets_at: Option> }, + /// The provider rejected the credentials. A re-login fixes it; a retry + /// does not. + Auth, + /// The provider is broken for now (plain internal error, 5xx). + ProviderInternal, + /// Anything else. + Unknown, +} + +/// What the harness does with the scope after an outcome. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Action { + /// Existing backoff path. + Retry, + /// Freeze the whole agent until `until`; retry counts untouched. + Pause { until: DateTime }, + /// Freeze this scope; probe every ten minutes. + OpenBreaker, + /// Put the batch in the park file. + Park, +} + +/// Classify a provider error at a known instant. `now` is a parameter so a +/// reset time parsed from "resets 4:20am (America/Los_Angeles)" resolves to +/// the same absolute instant in a test as in production. +/// +/// STUB: T16 replaces this body. It exists so the fixtures compile. +pub fn classify_at(_err: &AcpError, _now: DateTime) -> ErrorClass { + ErrorClass::Unknown +} + +/// Per-agent reliability state: pause and per-scope breakers. +/// +/// STUB: T16 replaces this body. +#[derive(Debug, Default)] +pub struct ReliabilityState { + /// Placeholder so the stub is not a unit struct; T16 replaces it with + /// the pause and per-scope breaker fields. + _pending: (), +} + +impl ReliabilityState { + /// Record one failed outcome for `scope` and decide what happens next. + pub fn on_failure( + &mut self, + _scope: &SessionScope, + _class: ErrorClass, + _now: DateTime, + ) -> Action { + Action::Retry + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::DedupMode; + use crate::queue::{EventQueue, QueuedEvent, MAX_RETRIES}; + use chrono::TimeZone; + use nostr::{EventBuilder, Keys, Kind}; + use std::time::Instant; + use uuid::Uuid; + + /// The four provider error lines from the agent logs of 2026-09-02/03. + const SESSION_LIMIT_0420: &str = + "Internal error: You've hit your session limit · resets 4:20am (America/Los_Angeles)"; + const SESSION_LIMIT_0040: &str = + "Internal error: You've hit your session limit · resets 12:40am (America/Los_Angeles)"; + const PLAIN_INTERNAL: &str = "Internal error"; + const UNRELATED: &str = "Tool 'read_file' returned no content"; + + fn agent_error(message: &str) -> AcpError { + AcpError::AgentError { + code: -32603, + message: message.to_string(), + } + } + + /// 2026-09-02T09:57:24Z, the timestamp of the first dead-letter line. + fn log_instant() -> DateTime { + Utc.with_ymd_and_hms(2026, 9, 2, 9, 57, 24).unwrap() + } + + fn make_queued(channel_id: Uuid, content: &str) -> QueuedEvent { + let keys = Keys::generate(); + let event = EventBuilder::new(Kind::Custom(9), content) + .tags([]) + .sign_with_keys(&keys) + .unwrap(); + QueuedEvent { + channel_id, + scope: SessionScope::Conversation { channel_id }, + event, + received_at: Instant::now(), + prompt_tag: "test".into(), + } + } + + // 1. Error classes on the real log lines. + + #[test] + #[ignore = "T16 fixture: fails until classify_at parses the Claude session-limit line"] + fn a_session_limit_with_a_morning_reset_is_capacity_exhausted_until_that_time() { + // 04:20 America/Los_Angeles on 2026-09-02 is PDT, so 11:20 UTC, later + // the same day as the log line. + let expected = Utc.with_ymd_and_hms(2026, 9, 2, 11, 20, 0).unwrap(); + assert_eq!( + classify_at(&agent_error(SESSION_LIMIT_0420), log_instant()), + ErrorClass::CapacityExhausted { + resets_at: Some(expected) + } + ); + } + + #[test] + #[ignore = "T16 fixture: fails until classify_at parses the Claude session-limit line"] + fn a_session_limit_whose_reset_already_passed_today_resolves_to_tomorrow() { + // 00:40 America/Los_Angeles is 07:40 UTC. At 09:57 UTC that is already + // past, so the next occurrence is 2026-09-03T07:40Z. + let expected = Utc.with_ymd_and_hms(2026, 9, 3, 7, 40, 0).unwrap(); + assert_eq!( + classify_at(&agent_error(SESSION_LIMIT_0040), log_instant()), + ErrorClass::CapacityExhausted { + resets_at: Some(expected) + } + ); + } + + #[test] + #[ignore = "T16 fixture: fails until classify_at recognises a bare internal error"] + fn a_plain_internal_error_is_provider_internal() { + assert_eq!( + classify_at(&agent_error(PLAIN_INTERNAL), log_instant()), + ErrorClass::ProviderInternal + ); + } + + #[test] + fn an_unrelated_agent_error_is_unknown() { + // Passes today by construction; kept so the class set is complete. + assert_eq!( + classify_at(&agent_error(UNRELATED), log_instant()), + ErrorClass::Unknown + ); + } + + // 2. The queue never hands a batch back to be discarded. + + #[test] + #[ignore = "T16 fixture: fails until retry exhaustion parks the batch instead of returning it"] + fn retry_exhaustion_parks_the_batch_and_returns_nothing_to_discard() { + let channel_id = Uuid::new_v4(); + let mut queue = EventQueue::new(DedupMode::Queue); + assert!(queue.push(make_queued(channel_id, "please review section 20"))); + let batch = queue.flush_next().expect("one batch"); + // mark_complete clears a scope's retry count when no backoff is + // active, so the count is set after it, as the queue's own tests do. + queue.mark_complete(SessionScope::Conversation { channel_id }); + queue.set_retry_count_for_test(SessionScope::Conversation { channel_id }, MAX_RETRIES); + + let returned = queue.requeue(batch); + + assert!( + returned.is_none(), + "after {} retries the batch must be parked inside the queue, never returned for discard", + MAX_RETRIES + ); + } + + // 3. Capacity exhaustion pauses the agent and spends no retries. + + #[test] + #[ignore = "T16 fixture: fails until ReliabilityState pauses on CapacityExhausted"] + fn capacity_exhausted_pauses_until_the_reset_time() { + let channel_id = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id }; + let resets_at = Utc.with_ymd_and_hms(2026, 9, 2, 11, 20, 0).unwrap(); + let mut state = ReliabilityState::default(); + + let action = state.on_failure( + &scope, + ErrorClass::CapacityExhausted { + resets_at: Some(resets_at), + }, + log_instant(), + ); + + assert_eq!(action, Action::Pause { until: resets_at }); + } + + #[test] + #[ignore = "T16 fixture: fails until an unparseable reset time pauses for 30 minutes"] + fn capacity_exhausted_without_a_reset_time_pauses_thirty_minutes() { + let channel_id = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id }; + let now = log_instant(); + let mut state = ReliabilityState::default(); + + let action = state.on_failure( + &scope, + ErrorClass::CapacityExhausted { resets_at: None }, + now, + ); + + assert_eq!( + action, + Action::Pause { + until: now + chrono::Duration::minutes(30) + } + ); + } + + // 4. Three consecutive provider errors open the breaker for that scope. + + #[test] + #[ignore = "T16 fixture: fails until three consecutive ProviderInternal failures open the breaker"] + fn three_consecutive_provider_errors_open_the_breaker() { + let channel_id = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id }; + let now = log_instant(); + let mut state = ReliabilityState::default(); + + let first = state.on_failure(&scope, ErrorClass::ProviderInternal, now); + let second = state.on_failure( + &scope, + ErrorClass::ProviderInternal, + now + chrono::Duration::seconds(5), + ); + let third = state.on_failure( + &scope, + ErrorClass::ProviderInternal, + now + chrono::Duration::seconds(15), + ); + + assert_eq!(first, Action::Retry); + assert_eq!(second, Action::Retry); + assert_eq!(third, Action::OpenBreaker); + } +} diff --git a/docs/plans/2026-09-04-zs-implementation-plan.md b/docs/plans/2026-09-04-zs-implementation-plan.md index d75641b3018..bcd4e022bf0 100644 --- a/docs/plans/2026-09-04-zs-implementation-plan.md +++ b/docs/plans/2026-09-04-zs-implementation-plan.md @@ -331,6 +331,43 @@ the root workspace excludes that manifest (`Cargo.toml:35`). - Acceptance: agents can attach markdown, HTML, PDF, CSV and JSON reports with `--file`; images and video behave exactly as before; the desktop opens the `.md` from the imeta filename; nothing on the deny list leaves the machine. - Bar: the desktop's own upload path in `desktop/src-tauri/src/commands/media.rs:425-475` (declared MIME, `/upload` first, legacy fallback only on 404 or 405) and the relay's `process_file_upload` contract. +### T16 · feat/harness-reliability — park, pause, breaker, replay in the agent harness (M) + +- Design: `docs/plans/2026-09-06-harness-reliability-design.md` (approved for spec 2026-09-06; decisions: pause on session limit, no seat rotation; only never-started batches replay automatically; nothing is discarded). Crate `buzz-acp` only. Error classes on the real log lines, a harness-owned ledger and park file in a per-agent state dir, a per-agent pause state with a parsed reset time, a per-scope breaker after three consecutive provider errors, replay after a successful probe, control frames `replay_batch` / `discard_batch` / `resume_now` / `keep_paused`, CLI `buzz agents parked|replay|discard`. +- Tests: fixture tests already on `feat/harness-reliability-fixtures` (`#[ignore]`, run with `--ignored`); they must pass un-ignored before the PR is ready. Six cases listed in the design. +- Eval: + ``` + just fmt-check clippy + cargo test -p buzz-acp reliability + cargo test -p buzz-acp queue + ``` +- Acceptance: with the Claude login at its session limit, an agent posts one pause notice with the reset time, keeps every message, answers them in order after the reset, and the ledger shows `agent_paused`, `agent_resumed`, `batch_replayed` with no `batch_discarded`. +- Bar: Sol's audit of 2026-09-06 (at-least-once, stable ids, no replay of started batches, ledger as source of truth). + +### T17 · feat/agent-health — health store, Health tab, alerts, CLI (M) + +- Design: `docs/plans/2026-09-06-agent-health-design.md`. After T16. Harness observer frames for the ledger kinds; desktop `agent-health.db` rebuilt from the ledger by `sync_health_ledger`; Agents screen Health tab with per-agent counters, failed-task list with Retry / Discard, pause card with Resume now; alerts local first (macOS notification) then Buzz DM, one per condition per agent per hour; `buzz agents health`. +- Tests: Rust store and sync idempotence, retention, alert rules on fixtures; Node summary reducer; one smoke spec `agent-health.spec.ts`. +- Eval: + ``` + just desktop-check + cd desktop/src-tauri && cargo test agent_health + cd desktop && pnpm build:e2e && pnpm exec playwright test --project=smoke agent-health.spec.ts + ``` +- Acceptance: the four incidents of 2026-09-02/03 (session limit, dropped batches, Critic errors) would each have produced one alert and one row in the failed-task list. +- Bar: `observed_unread.rs` for the store contract; the workflow lists for focus refetch. + +### T18 · feat/agent-curator — work logs, weekly retro, prompt PRs (S now, M later) + +- Design: `docs/plans/2026-09-06-agent-self-improvement-design.md`. Phase 0 is a prompt PR to the Broken English repo (house rule 8, work logs) and needs no Buzz code. Phase 1 after T17: a Curator managed agent on Sonnet 5, a weekly scheduled run, lessons files under `GUIDES/`, prompt edits only as PRs Devin merges. +- Tests: Phase 1 dry run on a fixture week checked by Critic. +- Acceptance: after four weekly runs, at least one merged prompt PR cites a ledger batch id. + +### Client-side tickets outside this repo + +- W1 · Broken English `feat/wiki` — shared markdown wiki, design at `~/projects/clients/broken-english/docs/plans/2026-09-06-client-wiki-design.md`; reference system still to be named by Devin. +- P1 · Broken English as a Buzz Project — Devin flips Projects / Workflows / Pulse in Settings → Experimental Features and creates the project (owner-signed). Repo home decided: private GitHub repo under ZeroSum-Solutions; creation and the first push wait for Devin's explicit go after a secret scan. + ## Order and parallelism Wave 1 (parallel build; serialized landing): T1 (landed, PR #6), T2 (landed, PR #5), T4 (landed, diff --git a/docs/plans/2026-09-06-agent-health-design.md b/docs/plans/2026-09-06-agent-health-design.md new file mode 100644 index 00000000000..895a906f4ae --- /dev/null +++ b/docs/plans/2026-09-06-agent-health-design.md @@ -0,0 +1,72 @@ +# Agent health: ledger sync, Health tab, alerts, CLI + +Ticket: T17 · `feat/agent-health` · desktop (Rust commands + React), `buzz-acp` observer frames, `buzz-cli`. +Depends on T16 (the harness ledger is the source of truth). +Status: design, 2026-09-06. Audited by GPT-5.6 Sol before the decisions were put to Devin. + +## Goal + +Answer "how are my agents doing" without reading raw logs: an aggregated failed-task list, per-agent counters, and an alert when something needs Devin. Today the only records are per-agent text logs, a `last_error` field that clears on restart, and observer frames that drive the working indicator. + +## Source of truth + +The T16 ledger files in each agent's state dir. Observer frames can be dropped in a crash and the desktop database can be deleted, so both are rebuilt from the ledger, never the other way round. + +## Components + +### 1. Observer frames (`buzz-acp`) + +The harness emits one observer event per ledger record kind that matters live: `turn_failed` (with `class`), `batch_parked`, `batch_replayed`, `batch_needs_review`, `agent_paused`, `agent_resumed`, `breaker_opened`, `breaker_closed`, `relay_reconnected`. Same envelope as today's frames (`ObserverEvent`), same batching, same `#p` addressing to the owner. Payloads carry ids and counts, never message text. + +### 2. Health store (desktop, Rust) + +`agent_health.rs` beside `observed_unread.rs`: SQLite file `agent-health.db` in the app data dir, one table `health_events` (agent, at, kind, batch_id, channel_id, class, payload JSON) with indexes on (agent, at) and (kind, at), 30-day retention pruned on open and daily. + +Two writers, one rule: a live observer frame inserts by (agent, seq) with `INSERT OR IGNORE`; `sync_health_ledger(agent)` reads the ledger file for a locally managed agent and inserts every record the table lacks. Sync runs on app start, on agent start and stop, and when the Health tab opens. A remote-owned agent (no local ledger) only gets frames, and the tab says so. + +Commands: `get_agent_health_summary(since)`, `get_agent_health_events(agent, kinds, since, limit)`, `get_parked_batches(agent)` (reads the park file directly, excerpt cut to 120 chars), `send_agent_control(agent, frame)` for Retry, Discard, Resume now, Keep paused. + +### 3. Health tab (desktop, React) + +On the Agents screen, a Health tab next to the existing list: + +- Table, one row per agent: state (active, paused until, breaker open, offline), turns 24h / 7d, failed 24h / 7d, parked, needs review, last error class and time, reconnects 24h. +- Row click opens a drawer: failed-task list (time, channel link, class, excerpt, Retry / Discard), pause card (until, waiting count, Resume now / Keep paused), last 50 health events. +- A red badge on the Agents entry in the sidebar when any agent has a needs-review batch or an open breaker. +- No polling loop beyond the existing agents query; frames update the store, the tab re-reads on focus like the workflow lists. + +### 4. Alerts + +Rules, evaluated in the desktop when a frame or sync arrives: + +| Condition | Alert | +|---|---| +| a parked batch is older than 15 minutes | "PM has 3 saved messages waiting for 20 minutes" | +| a batch entered needs review | "A Critic request needs your decision" | +| a breaker opened | "Critic's provider is failing; probing every 10 min" | +| a pause longer than 1 hour started | "PM is paused until 4:20 AM" | +| an agent process exited with a non-zero code | existing `last_error`, now also an alert | + +Delivery order: macOS notification first (local, works with the relay down), then a Buzz DM from the agent to Devin when the relay is up. One alert per condition per agent per hour. Alerts never claim a mention was missed; the client cannot know what it did not receive. + +### 5. CLI + +`buzz agents health [--since 24h|7d] [--json]` prints the same table from the ledger files, no relay needed. `buzz agents parked`, `replay`, `discard` come from T16. This is what scripts and Claude use. + +## Data and privacy + +Frames and the database carry ids, classes, counts and timestamps. Message text stays in the T16 park file and is read on demand for the excerpt. Retention 30 days. Deleting `agent-health.db` is safe; the next sync rebuilds it. + +## Tests + +- Rust: store insert-or-ignore on duplicate (agent, seq); ledger sync is idempotent; retention prunes by date; alert rule evaluation on fixture event sets (each rule fires once per hour). +- Node: summary reducer from events to the table row; badge condition. +- One smoke spec: with a fixture ledger, the Health tab shows one paused agent and one needs-review batch, and Retry sends the control frame. + +## Gates + +Fast set plus `cargo test agent_health`, `pnpm exec playwright test --project=smoke agent-health.spec.ts`. The queue runs the full suite once. Sol reviews before ready. + +## Out of scope + +Relay changes, dashboards outside the app, any change to how agents are spawned. diff --git a/docs/plans/2026-09-06-agent-self-improvement-design.md b/docs/plans/2026-09-06-agent-self-improvement-design.md new file mode 100644 index 00000000000..31a6028034a --- /dev/null +++ b/docs/plans/2026-09-06-agent-self-improvement-design.md @@ -0,0 +1,56 @@ +# Agent self-improvement: work logs, retro, curator, prompt PRs + +Ticket: T18 · `feat/agent-curator` (Buzz side is small; most of this is prompts and one workflow). +Depends on T16 and T17 (there is no failure evidence to learn from until they land). +Status: design, 2026-09-06. Audited by GPT-5.6 Sol. + +## What "evolution like Hermes" means here + +Hermes has three parts: agents save skills learned from sessions, a curator model reviews those skills on a schedule (prunes, merges, archives, never deletes), and an insights command reports usage. Nothing changes itself without a record and a way back. + +| Hermes | Buzz equivalent in this design | +|---|---| +| learned skills | `~/.buzz/GUIDES/LESSONS_.md`, written by the agent from its own work logs and failures | +| curator | a weekly Curator run that consolidates lessons and proposes prompt edits as a pull request | +| insights | `buzz agents health` (T17) | +| journey / memory graph | the existing engram graph on the agent profile | + +Prompts never edit themselves. Every change to an agent's behaviour is a diff that Devin merges. + +## Phase 0, now: the work-log rule (prompt only) + +Add house rule 8 to `~/projects/clients/broken-english/buzz/agent-prompts/_house-rules.md`: + +> 8. Work log. When a task ends, write `~/.buzz/WORK_LOGS/YYYY-MM-DD--.md` with frontmatter (title, tags, status, created) and five short sections: Asked, Done, Files, Failed or blocked, Would change next time. One file per task, under 40 lines. This is the only place to reflect; never post reflections in the channel. + +The nest already defines `WORK_LOGS/`; it is empty because no prompt asked for it. Cost: one file write per task. This is a PR to the Broken English repo and a prompt reload in the desktop. + +## Phase 1, after T17: weekly retro + +A scheduled run (a Buzz workflow with `on: schedule, interval: 7d` once Workflows is on, otherwise a launchd job that runs the CLI) posts one message in a private ops thread: "Curator: weekly retro for ". The Curator is a new managed agent on Sonnet 5 with read access to the nest and the Broken English repo and no channel write beyond the ops thread. Its prompt: + +1. Run `buzz agents health --since 7d --json` and read `~/.buzz/WORK_LOGS/` for the week. +2. For each agent, write or update `GUIDES/LESSONS_.md`: what failed, why, what the agent should do differently. Every lesson cites a ledger batch id or a work-log file. +3. Where a lesson implies a prompt change, edit the agent's prompt file in a branch of the Broken English repo and open a PR titled "curator: — ", body listing the evidence. House rules are out of bounds for the Curator. +4. Post the PR links and a five-line summary in the ops thread. Stop. + +Budget: one run a week, 30 minutes hard cap, 60 messages of nest reading. Devin merges or closes each PR. When a PR merges, the desktop's prompt-source reload picks up the new prompt on the next agent start. + +## Phase 2, later: agent-level retro + +Each agent reads its own `LESSONS` file at session start (one line in the prompt) so a lesson takes effect before the prompt PR is merged. Only after Phase 1 has run four times and the lessons have proved useful. + +## Guardrails + +- The Curator never edits a running prompt, a house rule, a channel, or a nest file outside `GUIDES/LESSONS_*`. +- A lesson without a citation is deleted by the next run. +- The Curator's own failures show in the Health tab like any agent's. +- The weekly cadence is fixed. Eleven nightly model runs would cost more than the failures they prevent. + +## Tests + +Phase 0: none beyond the prompt PR review. Phase 1: a dry-run mode for the Curator prompt on a fixture week (three failed turns, two work logs) that must produce one lessons file with two cited lessons and one prompt PR; checked by Critic before the first live run. + +## Out of scope + +Automatic prompt application, per-turn self-critique, any model fine-tuning. diff --git a/docs/plans/2026-09-06-harness-reliability-design.md b/docs/plans/2026-09-06-harness-reliability-design.md new file mode 100644 index 00000000000..314f75d2d7b --- /dev/null +++ b/docs/plans/2026-09-06-harness-reliability-design.md @@ -0,0 +1,129 @@ +# Harness reliability: park, pause, breaker, replay + +Ticket: T16 · `feat/harness-reliability` · crate `buzz-acp` only. +Status: design, approved for spec by Devin 2026-09-06 02:40. Audited by GPT-5.6 Sol (questions and recommendations) before the decisions below were put to Devin. + +## Decisions already made (Devin, 2026-09-06) + +1. A Claude session limit pauses the agent until the reset time. No automatic seat rotation. A manual `cswap switch` stays available and the pause notice names it. +2. Only a batch that never started running replays on its own. A batch that had started goes to a review list with Retry and Discard. +3. No message is ever discarded by the harness. The 19 messages dropped on 2026-09-02 and 09-03 are gone and cannot be recovered; this design stops the next ones from being lost. + +## What goes wrong today + +`queue.rs` retries a failed batch ten times with backoff (5 s doubling to 300 s, about 25 minutes in total), then dead-letters it: logs at ERROR, posts one warning to the channel, and drops the events. A session limit lasts hours, so every message in that window died after 25 minutes. The warning post itself fails when the relay is down. The retry budget is spent the same way on a provider that is simply broken for two hours (Critic, 2026-09-03). Nothing about a failed batch is written to disk, so no later process can see it. + +## Terms + +- **Batch**: `FlushBatch` (channel, scope, events). This design adds a stable `batch_id: Uuid` assigned when the batch is built. Event ids are the Nostr event ids and are already stable. +- **Started**: the harness saw agent output or a tool call for this batch's turn (the pool already tracks activity for `recently_active`). A batch that failed before any output is **not started**. +- **State dir**: `BUZZ_ACP_STATE_DIR`, set by the desktop to `/agents/state//` at spawn. Fallback when unset: `~/.buzz/.state//`. Permissions 0700 on the dir, 0600 on files. +- **Ledger**: `state/ledger.jsonl`, append-only, one JSON object per line. The harness owns it. Observer frames (T17) mirror it but are never the source of truth. +- **Park file**: `state/parked.jsonl`, one line per parked batch with the serialized events and prompt tags. The ledger refers to parked batches by `batch_id`. + +## Error classes (`error_class.rs`, new) + +`classify(err: &AcpError) -> ErrorClass`, pure, unit-tested on the real log lines: + +| Class | Matches | Example from the logs | +|---|---|---| +| `CapacityExhausted { resets_at: Option> }` | "session limit", "rate limit", HTTP 429, "overloaded", "quota" | `Internal error: You've hit your session limit · resets 4:20am (America/Los_Angeles)` | +| `Auth` | the existing auth detection | unchanged | +| `ProviderInternal` | "Internal error" with no capacity marker, HTTP 5xx | `Agent reported error (code -32603): Internal error` (Critic, 2026-09-03) | +| `Unknown` | everything else | | + +`resets_at` is parsed from `resets H:MM(am|pm) (IANA zone)` as the next occurrence of that wall time in that zone. When it cannot be parsed the pause is 30 minutes. A pause is never longer than 6 hours; a longer parsed value is clamped and logged. + +## State machine + +Per agent, not per scope, because a capacity limit belongs to the account: + +``` +Active --CapacityExhausted--> Paused{until} +Paused --timer--> Probing (first queued batch is the probe; nothing else moves) +Probing --Ok--> Active (then replay, see below) +Probing --CapacityExhausted--> Paused{new until} (notice only if until moved by > 15 min) +Active --ProviderInternal or Unknown, 3 consecutive on one scope--> BreakerOpen{scope, next_probe} +BreakerOpen --every 10 min--> probe one batch; Ok closes the breaker; failure reschedules; open at most 6 h then Park +``` + +Rules: + +- While Paused or BreakerOpen, retry counts are frozen. The existing backoff path is only for transient failures between probes. +- The existing `MAX_RETRIES` path no longer discards. Exhaustion parks the batch (`batch_parked`, reason `retries_exhausted`). +- A hard-cap timeout parks the batch. If the turn had started it is marked `needs_review`; if not, it is eligible for replay. +- An `Auth` error parks immediately with `needs_review` (a re-login fixes it; retrying does not). +- Cancelled and steered turns keep their existing merge behaviour. This design does not touch them. + +## Ledger records + +Every line has `at` (RFC 3339 UTC), `agent` (pubkey), `kind`, and `batch_id` where it applies. + +| kind | fields | +|---|---| +| `turn_started` | `channel_id`, `scope`, `event_ids`, `attempt` | +| `turn_activity` | first output or tool call seen; written once per batch | +| `turn_finished` | `outcome`: `ok`, `error{class, raw}`, `timeout{kind, started}`, `cancelled`, `exited` | +| `batch_parked` | `reason` (`retries_exhausted`, `hard_timeout`, `auth`, `breaker_expired`), `started: bool`, `events` count | +| `batch_replayed` | `replay_of: batch_id` of the new turn; written **before** the prompt is sent | +| `batch_needs_review` | `reason` | +| `batch_discarded` | `by: operator`, via control frame | +| `agent_paused` | `class`, `until`, `waiting` count | +| `agent_resumed` | | +| `breaker_opened` / `breaker_closed` | `scope`, consecutive failures | +| `relay_reconnected` | `after_secs` | + +Retention: the harness truncates the ledger to 30 days on start and every 6 hours. Parked batches with `needs_review` stay until acted on; replay-eligible parked batches older than 7 days move to `needs_review`. Hard cap 10 MB per file; beyond it the oldest replay-eligible batches move to `needs_review` and the operator is told in the next notice. + +## Replay + +- Replay starts only after a **successful live turn** (the probe). A process restart alone never replays anything, because a restart proves nothing about the provider. +- Order: for each scope, parked not-started batches replay oldest first, ahead of newer queued events for that scope, so the conversation stays in order. Several parked batches for one scope become one prompt. +- Framing: the events keep their original text. `format_prompt` adds a section header "Delivered late: these messages arrived while I was unavailable (first at HH:MM, last at HH:MM)". This uses the same annotated-section mechanism as the cancelled-events merge. The user's words are never edited. +- Delivery guarantee: at least once. `batch_replayed` is written before the send. On start, a batch with `batch_replayed` and no `turn_finished` is a crash mid-replay and moves to `needs_review`, never to a second automatic replay. +- Started batches never replay automatically. They wait for `replay_batch` from the operator. + +## Operator control + +The desktop already sends control frames to the harness over the relay (`switch_model`). This design adds: + +| frame | effect | +|---|---| +| `replay_batch { batch_id }` | replay one parked batch now, whatever its `started` flag | +| `discard_batch { batch_id }` | remove the batch from the park file, write `batch_discarded` | +| `resume_now` | leave Paused or BreakerOpen and probe immediately | +| `keep_paused { until }` | extend a pause | + +CLI, this ticket: `buzz agents parked [--json]` lists parked batches from the state dir (no relay needed), `buzz agents replay ` and `buzz agents discard ` send the frames. The desktop buttons come with T17. + +## Notices in the channel + +At most one notice per pause per channel, one per park, one per breaker open. The relay post is retried with the same backoff as any other post; if it still fails the ledger has the record and T17 raises the alert locally. Templates: + +- Pause: "⏸️ PM is paused until 4:20 AM (Claude session limit). 6 messages are saved and will be answered in order when I am back. To switch seats now run `cswap switch` and restart the Claude agents." +- Park after retries: "⚠️ I could not process the last request after several attempts (reason). It is saved and will be retried as soon as I am back. Nothing is lost." +- Needs review: "⚠️ A request was interrupted after it had started, so it will not run again on its own. Devin can retry or discard it from the Agents screen." +- Breaker: "⚠️ Critic's provider is returning errors. I will try again every 10 minutes and answer in order when it recovers." + +## Privacy + +Parked batches hold client messages. They live only in the agent state dir with 0600 permissions, are removed on discard, and are never sent anywhere except back to the same agent. UI and CLI excerpts are cut to 120 characters. The raw provider error is stored beside its class so a misclassification can be diagnosed. + +## Out of scope + +Seat rotation, desktop UI (T17), the health database (T17), any change to the relay, any recovery of the 19 already-dropped messages. + +## Tests first + +Fixture tests are on branch `feat/harness-reliability-fixtures`, marked `#[ignore = "T16 fixture: fails until park/pause/breaker land"]`, and run with `cargo test -p buzz-acp reliability -- --ignored`. They fail today and must pass before this ticket's PR is ready: + +1. `classify` on the four real log lines gives `CapacityExhausted{resets_at: Some(next 04:20 America/Los_Angeles)}`, `CapacityExhausted{resets_at: Some(next 00:40 …)}`, `ProviderInternal`, and `Unknown` for an unrelated message. +2. After `MAX_RETRIES + 1` requeues the batch is in the park file, the queue holds zero events, and no event was dropped. +3. A `CapacityExhausted` outcome moves the agent to Paused, does not increment the scope's retry count, and posts exactly one notice per channel. +4. Three consecutive `ProviderInternal` outcomes open the breaker for that scope; the fourth attempt is not made before the probe interval. +5. A parked batch with `started = true` is not replayed after a successful probe; one with `started = false` is, before newer events of the same scope. +6. A `batch_replayed` record with no `turn_finished` at start moves the batch to `needs_review`. + +## Gates + +`just fmt-check clippy`, `cargo test -p buzz-acp reliability` and `cargo test -p buzz-acp queue`, then the PR. The merge queue runs the full suite once. Sol reviews the diff before the PR is marked ready. From 6977f9c44a1f7f09adfdd004f8b5e3b8db848f7a Mon Sep 17 00:00:00 2001 From: wiggdevin <202901685+wiggdevin@users.noreply.github.com> Date: Sun, 6 Sep 2026 04:48:43 -0700 Subject: [PATCH 2/7] feat(acp): park, pause, breaker and replay in the agent harness (T16) The harness used to dead-letter a batch after ten retries: it logged at ERROR, posted one warning and dropped the events. A Claude session limit lasts hours, so every message in that window died after 25 minutes. Nothing is discarded any more. A failure that used to dead-letter now parks the batch in an owner-only park file in the agent's state directory, and either replays it automatically after a successful live turn or hands it to the operator for review. - reliability/error_class.rs classifies a provider error and parses "resets 4:20am (America/Los_Angeles)" into the next occurrence of that wall time, via chrono-tz. An unparseable reset pauses 30 minutes; a pause is clamped at 6 hours. - reliability/state.rs holds the per-agent pause and the per-scope breakers: three consecutive provider errors open a breaker with a 10-minute probe and a 6-hour cap, then the batch parks. - reliability/state_dir.rs resolves BUZZ_ACP_STATE_DIR (0700 dir, 0600 files), falling back to ~/.buzz/.state//. - reliability/ledger.rs is the append-only ledger.jsonl: one serde struct per record kind, fsync per append, 30-day retention truncated on start and every 6 hours, 10 MB cap. - reliability/park.rs is parked.jsonl, written atomically, with caps on bytes, batches per scope and batches in total. - reliability/runtime.rs orders the writes: park before the batch is dropped, batch_replayed before the prompt is staged. - queue.rs: FlushBatch carries a stable batch_id; requeue no longer returns a batch to be discarded but hands it to the park path; stage_replay merges parked events ahead of newer ones with the "Delivered late" annotated section, reusing the cancelled-events merge and never editing the event text. - lib.rs gates dispatch on the pause and the breakers, drives the machinery on every prompt result, and accepts replay_batch, discard_batch, resume_now and keep_paused on the same owner-checked path as switch_model. The seven T16 fixture tests pass with their bodies unchanged. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012Z6iidtozXxgx58BUZUKnu Signed-off-by: wiggdevin <202901685+wiggdevin@users.noreply.github.com> --- Cargo.lock | 45 +- Cargo.toml | 2 + crates/buzz-acp/Cargo.toml | 1 + crates/buzz-acp/src/acp.rs | 20 + crates/buzz-acp/src/lib.rs | 856 ++++++++++++++++-- crates/buzz-acp/src/pool.rs | 17 + crates/buzz-acp/src/queue.rs | 269 +++++- crates/buzz-acp/src/reliability.rs | 97 +- .../buzz-acp/src/reliability/error_class.rs | 212 +++++ crates/buzz-acp/src/reliability/ledger.rs | 545 +++++++++++ crates/buzz-acp/src/reliability/notices.rs | 84 ++ crates/buzz-acp/src/reliability/park.rs | 578 ++++++++++++ crates/buzz-acp/src/reliability/runtime.rs | 327 +++++++ crates/buzz-acp/src/reliability/state.rs | 328 +++++++ crates/buzz-acp/src/reliability/state_dir.rs | 168 ++++ 15 files changed, 3382 insertions(+), 167 deletions(-) create mode 100644 crates/buzz-acp/src/reliability/error_class.rs create mode 100644 crates/buzz-acp/src/reliability/ledger.rs create mode 100644 crates/buzz-acp/src/reliability/notices.rs create mode 100644 crates/buzz-acp/src/reliability/park.rs create mode 100644 crates/buzz-acp/src/reliability/runtime.rs create mode 100644 crates/buzz-acp/src/reliability/state.rs create mode 100644 crates/buzz-acp/src/reliability/state_dir.rs diff --git a/Cargo.lock b/Cargo.lock index 754a6762502..e5b2af64a61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -836,6 +836,7 @@ dependencies = [ "buzz-sdk", "buzz-secret-store", "chrono", + "chrono-tz", "clap", "evalexpr", "futures-util", @@ -1669,6 +1670,16 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", +] + [[package]] name = "cipher" version = "0.4.4" @@ -2016,7 +2027,7 @@ checksum = "089df96cf6a25253b4b6b6744d86f91150a3d4df546f31a95def47976b8cba97" dependencies = [ "chrono", "once_cell", - "phf", + "phf 0.11.3", "winnow 0.7.15", ] @@ -2168,7 +2179,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb2a7d3066da2de787b7f032c736763eb7ae5d355f81a68bab2675a96008b0bf" dependencies = [ "lab", - "phf", + "phf 0.11.3", ] [[package]] @@ -6796,7 +6807,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" dependencies = [ "phf_macros", - "phf_shared", + "phf_shared 0.11.3", +] + +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared 0.12.1", ] [[package]] @@ -6806,7 +6826,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ "phf_generator", - "phf_shared", + "phf_shared 0.11.3", ] [[package]] @@ -6815,7 +6835,7 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ - "phf_shared", + "phf_shared 0.11.3", "rand 0.8.6", ] @@ -6826,7 +6846,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f84ac04429c13a7ff43785d75ad27569f2951ce0ffd30a3321230db2fc727216" dependencies = [ "phf_generator", - "phf_shared", + "phf_shared 0.11.3", "proc-macro2", "quote", "syn 2.0.117", @@ -6841,6 +6861,15 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.13" @@ -9623,7 +9652,7 @@ checksum = "d4ea810f0692f9f51b382fff5893887bb4580f5fa246fde546e0b13e7fcee662" dependencies = [ "fnv", "nom", - "phf", + "phf 0.11.3", "phf_codegen", ] @@ -9660,7 +9689,7 @@ dependencies = [ "ordered-float 4.6.0", "pest", "pest_derive", - "phf", + "phf 0.11.3", "sha2 0.10.9", "signal-hook", "siphasher", diff --git a/Cargo.toml b/Cargo.toml index 3998a296aed..d2ada76f3e3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -101,6 +101,8 @@ anyhow = "1" # Utilities uuid = { version = "1", features = ["v4", "serde"] } chrono = { version = "0.4", features = ["serde"] } +# IANA time zones for parsing provider reset times ("resets 4:20am (America/Los_Angeles)"). +chrono-tz = { version = "0.10", default-features = false, features = ["std"] } # JWT / JWS verification (NIP-FI federated identity assertions) jsonwebtoken = { version = "10.4.0", default-features = false, features = ["aws_lc_rs"] } diff --git a/crates/buzz-acp/Cargo.toml b/crates/buzz-acp/Cargo.toml index 9e6d6bb2e04..690c3811b91 100644 --- a/crates/buzz-acp/Cargo.toml +++ b/crates/buzz-acp/Cargo.toml @@ -55,6 +55,7 @@ serde_json = { workspace = true } # IDs uuid = { workspace = true } chrono = { workspace = true } +chrono-tz = { workspace = true } # URL parsing url = { workspace = true } diff --git a/crates/buzz-acp/src/acp.rs b/crates/buzz-acp/src/acp.rs index a4dc896a0d3..cf780f332b7 100644 --- a/crates/buzz-acp/src/acp.rs +++ b/crates/buzz-acp/src/acp.rs @@ -268,6 +268,12 @@ pub struct AcpClient { /// a JSON-RPC *success*, not `-32601` — which the main loop would read as /// a delivered steer and drop the user's message from the queue. steering_supported: bool, + /// Whether this turn produced agent output or a tool call. + /// + /// The reliability path calls a batch **started** when this is true: a + /// started batch is never replayed automatically, because the agent may + /// already have acted on it. Reset at the top of every prompt. + turn_saw_output: bool, /// Per-turn channel for receiving goose-native non-cancelling steer /// requests from the main loop. Installed by /// [`install_steer_rx`](Self::install_steer_rx) at dispatch and @@ -627,6 +633,7 @@ impl AcpClient { observer_context: ObserverContext::default(), active_run_id: None, steering_supported: false, + turn_saw_output: false, steer_rx: None, goose_usage: UsageTracker::default(), standard_usage: StandardUsageTracker::default(), @@ -858,6 +865,9 @@ impl AcpClient { // misattributed to this turn. self.goose_usage.begin_turn(session_id); self.standard_usage.begin_turn(session_id); + // Reset the started signal for this turn, alongside the usage + // trackers, so activity from a previous turn is never attributed here. + self.turn_saw_output = false; self.last_prompt_id = Some(self.next_id); let id = self.next_id; @@ -949,6 +959,12 @@ impl AcpClient { self.steering_supported } + /// Whether the agent produced output or started a tool call during the most + /// recent turn. See [`turn_saw_output`](Self::turn_saw_output) on the field. + pub fn turn_saw_output(&self) -> bool { + self.turn_saw_output + } + /// Consume per-turn usage for NIP-AM publishing. Goose/buzz-agent is an /// exclusive cumulative path; standard ACP prompt usage is used only when /// goose emitted nothing for this turn. @@ -1763,6 +1779,9 @@ impl AcpClient { if let Some(method) = msg.get("method").and_then(|v| v.as_str()) { match method { "session/update" => { + // Any session update is agent output or a tool + // call: the turn has started. + self.turn_saw_output = true; if self.handle_session_update(&msg) { let activity_now = Instant::now(); idle_deadline = activity_now + idle_timeout; @@ -1771,6 +1790,7 @@ impl AcpClient { } } "_goose/unstable/session/update" => { + self.turn_saw_output = true; self.handle_goose_usage_update(&msg); } "session/request_permission" => { diff --git a/crates/buzz-acp/src/lib.rs b/crates/buzz-acp/src/lib.rs index 03a4cd248f3..554e5c01f75 100644 --- a/crates/buzz-acp/src/lib.rs +++ b/crates/buzz-acp/src/lib.rs @@ -16,7 +16,7 @@ mod prompt_framing; mod prompt_project; mod queue; mod relay; -mod reliability; +pub mod reliability; mod scope; mod setup_mode; mod usage; @@ -1608,6 +1608,7 @@ async fn publish_relay_observer_event( /// Maximum age (seconds) for an observer control frame to be considered fresh. const OBSERVER_CONTROL_FRESHNESS_SECS: i64 = 300; +#[allow(clippy::too_many_arguments)] fn handle_relay_observer_control_event( keys: &nostr::Keys, event: nostr::Event, @@ -1615,6 +1616,8 @@ fn handle_relay_observer_control_event( observer: Option<&observer::ObserverHandle>, owner_pubkey_hex: &str, event_publisher: RelayEventPublisher, + reliability: Option<&mut reliability::ReliabilityRuntime>, + queue: &mut EventQueue, ) { // Defense-in-depth: verify signature even though the relay already checked. if let Err(e) = buzz_core::verify_event(&event) { @@ -1668,6 +1671,12 @@ fn handle_relay_observer_control_event( event_publisher, ); } + // T16 operator controls. They reach here only after the owner check, + // the signature check and the freshness check above — the same gate + // `switch_model` passes. + Some(command @ ("replay_batch" | "discard_batch" | "resume_now" | "keep_paused")) => { + handle_reliability_control(command, &payload, reliability, queue, observer); + } _ => { tracing::debug!(payload = %payload, "ignoring unknown observer control frame"); } @@ -1953,6 +1962,132 @@ fn handle_switch_model_control( } } +/// Longest `batchId` string a control frame may carry before it is refused. +/// +/// A UUID is 36 characters. The frame is relay-sourced, so the field is capped +/// at the DTO before it is parsed, not after. +const MAX_CONTROL_BATCH_ID_CHARS: usize = 64; + +/// Handle the T16 operator control frames: `replay_batch`, `discard_batch`, +/// `resume_now` and `keep_paused`. +/// +/// Every one of these reaches the harness only from the agent's owner: the +/// caller ([`handle_relay_observer_control_event`]) verifies the signature, +/// rejects a sender that is not the resolved owner, and rejects a frame outside +/// the freshness window before dispatching here. +fn handle_reliability_control( + command: &str, + payload: &serde_json::Value, + reliability: Option<&mut reliability::ReliabilityRuntime>, + queue: &mut EventQueue, + observer: Option<&observer::ObserverHandle>, +) { + let Some(reliability) = reliability else { + tracing::warn!( + command, + "reliability control frame ignored — no state directory" + ); + return; + }; + let now = chrono::Utc::now(); + let status = match command { + "replay_batch" => match control_batch_id(payload) { + Some(batch_id) => match reliability.force_replay(batch_id) { + Ok(true) => { + // Staging happens on the next successful turn for the + // scope, through the same replay path an automatic replay + // uses, so the ledger record still precedes the send. + tracing::info!(%batch_id, "operator marked a parked batch for replay"); + "queued" + } + Ok(false) => "unknown_batch", + Err(error) => { + tracing::error!(%batch_id, error = %error, "replay_batch failed"); + "write_failed" + } + }, + None => "invalid_batch_id", + }, + "discard_batch" => match control_batch_id(payload) { + Some(batch_id) => match reliability.discard(batch_id, "operator", now) { + Ok(true) => "discarded", + Ok(false) => "unknown_batch", + Err(error) => { + tracing::error!(%batch_id, error = %error, "discard_batch failed"); + "write_failed" + } + }, + None => "invalid_batch_id", + }, + "resume_now" => { + if reliability.state().resume_now() { + reliability.record( + now, + reliability::ledger::LedgerBody::AgentResumed( + reliability::ledger::AgentResumed {}, + ), + ); + // Queued work is now dispatchable; the next dispatch tick picks + // it up as the probe. + let _ = queue.has_flushable_work(); + "resumed" + } else { + "not_paused" + } + } + "keep_paused" => match payload + .get("until") + .and_then(|value| value.as_str()) + .and_then(|value| { + chrono::DateTime::parse_from_rfc3339(&reliability::error_class::truncate_chars( + value, 64, + )) + .ok() + }) { + Some(until) => { + let until = reliability + .state() + .keep_paused(until.with_timezone(&chrono::Utc), now); + reliability.record( + now, + reliability::ledger::LedgerBody::AgentPaused( + reliability::ledger::AgentPaused { + class: "operator".to_string(), + until, + waiting: 0, + }, + ), + ); + "paused" + } + None => "invalid_until", + }, + _ => "unknown_command", + }; + + if let Some(observer) = observer { + observer.emit( + "control_result", + None, + &observer::ObserverContext::default(), + serde_json::json!({ "type": command, "status": status }), + ); + } +} + +/// Read and validate a control frame's `batchId`. +fn control_batch_id(payload: &serde_json::Value) -> Option { + let raw = payload.get("batchId").and_then(|value| value.as_str())?; + if raw.len() > MAX_CONTROL_BATCH_ID_CHARS { + tracing::warn!( + len = raw.len(), + "control frame batchId is over the length cap — dropping" + ); + return None; + } + raw.parse::().ok() +} + /// Maximum crashes in a 60-second window before a slot's circuit opens. const CIRCUIT_BREAKER_THRESHOLD: usize = 3; /// Window for circuit-breaker crash counting. @@ -2794,6 +2929,45 @@ async fn tokio_main() -> Result<()> { let mut queue = EventQueue::new(dedup_mode).with_in_flight_deadline(config.max_turn_duration_secs); + // T16 durable reliability state: the ledger and the park file in this + // agent's own state directory. The harness runs without it if the + // directory cannot be opened — a broken state directory must not stop the + // agent answering — but the failure is loud, because parking is what keeps + // messages from being lost. + let mut reliability = match reliability::ReliabilityRuntime::open( + &config.keys.public_key().to_hex(), + chrono::Utc::now(), + ) { + Ok(mut runtime) => { + match runtime.reconcile_on_start(chrono::Utc::now()) { + Ok(report) if !report.is_empty() => tracing::warn!( + crashed_mid_replay = report.crashed_mid_replay, + aged_out = report.aged_out, + over_scope_cap = report.over_scope_cap, + "parked batches moved to the review list at start-up" + ), + Ok(_) => {} + Err(error) => { + tracing::error!(error = %error, "park file reconciliation failed at start-up") + } + } + tracing::info!( + state_dir = %runtime.dir().display(), + parked = runtime.park().batches().len(), + "reliability state opened" + ); + Some(runtime) + } + Err(error) => { + tracing::error!( + error = %error, + "could not open the agent state directory — parked batches will NOT survive a \ + restart; set BUZZ_ACP_STATE_DIR to a writable path" + ); + None + } + }; + // Online means the harness can receive work, not merely that its socket is // connected. Publishing after channel subscriptions gives desktop callers // a durable readiness boundary before they send a startup mention. @@ -3106,9 +3280,13 @@ async fn tokio_main() -> Result<()> { // called on relay events or pool results, neither of which // arrive when the channel is silent. if queue.has_flushable_work() { - for (scope, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) - { + for (scope, thread_tags) in dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + reliability.as_mut(), + ) { typing_channels.insert(scope, thread_tags); } } @@ -3158,9 +3336,13 @@ async fn tokio_main() -> Result<()> { // this, batches requeued during crash recovery sit idle until the // next relay event arrives — which can be minutes on quiet channels. if respawn_collected { - for (scope, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) - { + for (scope, thread_tags) in dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + reliability.as_mut(), + ) { typing_channels.insert(scope, thread_tags); } } @@ -3245,6 +3427,8 @@ async fn tokio_main() -> Result<()> { observer.as_ref(), owner_hex, relay.event_publisher(), + reliability.as_mut(), + &mut queue, ); } else { tracing::warn!("observer control frame received but no owner resolved — dropping"); @@ -3598,7 +3782,7 @@ async fn tokio_main() -> Result<()> { ); if pool_ready { for (scope, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) + dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity, reliability.as_mut()) { typing_channels.insert(scope, thread_tags); } @@ -3698,7 +3882,7 @@ async fn tokio_main() -> Result<()> { } else if queue.has_flushable_work() { tracing::debug!("heartbeat_skipped_events"); for (scope, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) + dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity, reliability.as_mut()) { typing_channels.insert(scope, thread_tags); } @@ -3780,6 +3964,7 @@ async fn tokio_main() -> Result<()> { &mut respawn_tasks, observer.clone(), Some(&ctx.rest_client), + reliability.as_mut(), ) == LoopAction::Exit { break; @@ -3799,9 +3984,13 @@ async fn tokio_main() -> Result<()> { { break; } - for (scope, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) - { + for (scope, thread_tags) in dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + reliability.as_mut(), + ) { typing_channels.insert(scope, thread_tags); } } @@ -3824,9 +4013,26 @@ async fn tokio_main() -> Result<()> { tracing::error!("all agents dead — exiting"); break; } - for (scope, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) - { + // A panicked turn's batch goes through `queue.requeue` too, so + // it can reach the park hand-off with no prompt result to drain + // it. Drain here rather than leave client messages in memory. + if queue.has_parked_handoff() { + if let Some(reliability) = reliability.as_mut() { + drain_park_handoff( + reliability, + &mut queue, + Some(&ctx.rest_client), + chrono::Utc::now(), + ); + } + } + for (scope, thread_tags) in dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + reliability.as_mut(), + ) { typing_channels.insert(scope, thread_tags); } } @@ -3978,9 +4184,13 @@ async fn tokio_main() -> Result<()> { // tear down the in-flight task; on its completion the // queue drains. We still try here in case the in-flight // task has already returned. - for (scope, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) - { + for (scope, thread_tags) in dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + reliability.as_mut(), + ) { typing_channels.insert(scope, thread_tags); } } @@ -4006,9 +4216,13 @@ async fn tokio_main() -> Result<()> { "ready", None, ); - for (scope, thread_tags) in - dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity) - { + for (scope, thread_tags) in dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + reliability.as_mut(), + ) { typing_channels.insert(scope, thread_tags); } } @@ -4397,6 +4611,7 @@ fn dispatch_pending( queue: &mut EventQueue, ctx: &Arc, last_activity: &mut tokio::time::Instant, + mut reliability: Option<&mut reliability::ReliabilityRuntime>, ) -> Vec<(scope::SessionScope, ThreadTags)> { // Keyed by the exact session scope, not the channel: two threads dispatching // concurrently in one channel get distinct typing entries so completing one @@ -4415,6 +4630,54 @@ fn dispatch_pending( }; let channel_id = batch.channel_id; let scope = batch.scope.clone(); + // T16 gating. While the agent is paused (a provider capacity limit) or + // a scope's breaker is open, nothing runs but the probe. Held batches + // stay flushed-out so `flush_next` cannot re-pick them in this loop, + // and are returned to the queue at the end — the same mechanism the + // busy-session-owner hold uses. + if let Some(reliability) = reliability.as_deref_mut() { + let now = chrono::Utc::now(); + match reliability.state().pause_gate(now) { + reliability::PauseGate::Held { until } => { + tracing::debug!( + channel = %channel_id, + scope = %scope.telemetry_label(), + %until, + "holding batch — agent paused until the provider reset" + ); + held.push(batch); + continue; + } + reliability::PauseGate::Probe => { + tracing::info!( + channel = %channel_id, + scope = %scope.telemetry_label(), + "pause expired — sending one batch as the probe" + ); + } + reliability::PauseGate::Open => {} + } + match reliability.state().breaker_gate(&scope, now) { + reliability::BreakerGate::Held { next_probe } => { + tracing::debug!( + channel = %channel_id, + scope = %scope.telemetry_label(), + %next_probe, + "holding batch — scope breaker open" + ); + held.push(batch); + continue; + } + reliability::BreakerGate::Probe => { + tracing::info!( + channel = %channel_id, + scope = %scope.telemetry_label(), + "breaker probe interval elapsed — sending one batch as the probe" + ); + } + reliability::BreakerGate::Closed => {} + } + } // Authoritative affinity: if the worker that owns this thread's session // is checked out (busy on another turn), hold the batch rather than let // an idle worker open a second session for the same thread. @@ -4452,6 +4715,12 @@ fn dispatch_pending( DedupMode::Queue => Some(batch.clone()), DedupMode::Drop => None, }; + // Captured before the batch moves into the spawned task; the ledger + // record is written after the spawn so a failed claim writes nothing. + let dispatched_batch_id = batch.batch_id; + let dispatched_attempt = queue.retry_count(&scope); + let dispatched_event_ids: Vec = + batch.events.iter().map(|be| be.event.id.to_hex()).collect(); let result_tx = pool.result_tx(); let ctx_clone = Arc::clone(ctx); @@ -4505,6 +4774,20 @@ fn dispatch_pending( // Record this worker as the scope's session owner so a later dispatch // while it is busy holds instead of forking a duplicate session. pool.record_scope_owner(scope.clone(), agent_index); + if let Some(reliability) = reliability.as_deref_mut() { + reliability.record( + chrono::Utc::now(), + reliability::ledger::LedgerBody::TurnStarted( + reliability::ledger::TurnStarted::new( + dispatched_batch_id, + channel_id, + &scope.telemetry_label(), + dispatched_event_ids, + dispatched_attempt, + ), + ), + ); + } dispatched_channels.push((scope, typing_scope)); *last_activity = tokio::time::Instant::now(); } @@ -4551,7 +4834,7 @@ fn is_auth_error(error: &acp::AcpError) -> bool { let acp::AcpError::AgentError { message, .. } = error else { return false; }; - message.contains("Re-authenticate") || message.contains("API Error: 401") + reliability::error_class::is_auth_text(message) } /// Spawn a task that posts a user-visible failure notice to the relay. @@ -4577,6 +4860,289 @@ fn spawn_failure_notice( } } +/// What the reliability path did with a failed batch. +enum Disposition { + /// The reliability path took ownership of the batch. Nothing else runs. + Handled, + /// Not a reliability case; the batch goes back to the pre-existing + /// requeue path. + Fallthrough(FlushBatch), +} + +/// Drive the pause, breaker and park machinery for one failed batch. +/// +/// Ordering: the park file is written and fsynced **before** the batch is +/// dropped, so a crash between the two leaves the batch in the queue's park +/// hand-off (still in memory, retried on the next tick), never nowhere. +fn apply_reliability( + reliability: &mut reliability::ReliabilityRuntime, + queue: &mut EventQueue, + batch: FlushBatch, + outcome: &PromptOutcome, + started: bool, + rest_client: Option<&relay::RestClient>, + now: chrono::DateTime, +) -> Disposition { + use reliability::ledger::{self as led, LedgerBody}; + + let scope = batch.scope.clone(); + match outcome { + PromptOutcome::Timeout(pool::TimeoutKind::Hard { recently_active }) => { + // A hard cap means the agent process is gone. Nothing about a + // retry is safe, so the batch is parked either way; only whether + // it can replay on its own depends on the started signal. + let started = started || *recently_active; + reliability.record( + now, + LedgerBody::TurnFinished(led::TurnFinished { + batch_id: batch.batch_id, + channel_id: batch.channel_id, + outcome: led::TurnOutcome::timeout("hard", started), + }), + ); + park_or_fallthrough( + reliability, + batch, + reliability::ParkReason::HardTimeout, + started, + rest_client, + now, + ) + } + PromptOutcome::Error(error) => { + let class = reliability::classify_at(error, now); + reliability.record( + now, + LedgerBody::TurnFinished(led::TurnFinished { + batch_id: batch.batch_id, + channel_id: batch.channel_id, + outcome: led::TurnOutcome::error(class.as_str(), &error.to_string()), + }), + ); + let action = reliability.state().on_failure(&scope, class.clone(), now); + match action { + reliability::Action::Retry => Disposition::Fallthrough(batch), + reliability::Action::Park => { + // Auth: a re-login fixes it, a retry never does. + park_or_fallthrough( + reliability, + batch, + reliability::ParkReason::Auth, + true, + rest_client, + now, + ) + } + reliability::Action::Pause { until } => { + let waiting = batch.events.len(); + reliability.record( + now, + LedgerBody::AgentPaused(led::AgentPaused { + class: class.as_str().to_string(), + until, + waiting, + }), + ); + // No retry is spent on a pause: the events go back with + // their original timestamps and no backoff. + let channel_id = batch.channel_id; + let notice_batch = batch.clone(); + queue.requeue_preserve_timestamps(batch); + if reliability.state().claim_pause_notice(channel_id) { + spawn_failure_notice( + rest_client, + ¬ice_batch, + reliability::notices::pause("", until, waiting), + ); + } + Disposition::Handled + } + reliability::Action::OpenBreaker => { + let consecutive = reliability + .state_ref() + .breaker_consecutive(&scope) + .unwrap_or(reliability::state::BREAKER_THRESHOLD); + let first_open = reliability + .state_ref() + .breaker_opened_at(&scope) + .is_some_and(|opened| opened == now); + reliability.record( + now, + LedgerBody::BreakerOpened(led::BreakerOpened { + scope: scope.telemetry_label(), + consecutive, + }), + ); + let notice_batch = batch.clone(); + queue.requeue_preserve_timestamps(batch); + if first_open { + spawn_failure_notice( + rest_client, + ¬ice_batch, + reliability::notices::breaker(""), + ); + } + Disposition::Handled + } + } + } + _ => Disposition::Fallthrough(batch), + } +} + +/// Park a batch, or hand it back to the retry path when the park write failed. +/// +/// A failed park is logged and counted, never swallowed: the batch returns to +/// the caller so the pre-existing requeue keeps it in the queue. +fn park_or_fallthrough( + reliability: &mut reliability::ReliabilityRuntime, + batch: FlushBatch, + reason: reliability::ParkReason, + started: bool, + rest_client: Option<&relay::RestClient>, + now: chrono::DateTime, +) -> Disposition { + match reliability.park_batch(&batch, reason, started, now) { + Ok(()) => { + let content = if started { + reliability::notices::needs_review() + } else { + reliability::notices::parked(reason.as_str()) + }; + spawn_failure_notice(rest_client, &batch, content); + Disposition::Handled + } + Err(error) => { + tracing::error!( + channel_id = %batch.channel_id, + batch_id = %batch.batch_id, + events = batch.events.len(), + error = %error, + "could not park the batch — keeping it in the queue rather than losing it" + ); + Disposition::Fallthrough(batch) + } + } +} + +/// Write every batch the queue gave up retrying to the park file. +/// +/// A batch whose park write fails goes back to the hand-off and is retried on +/// the next drain; it is never dropped here. +fn drain_park_handoff( + reliability: &mut reliability::ReliabilityRuntime, + queue: &mut EventQueue, + rest_client: Option<&relay::RestClient>, + now: chrono::DateTime, +) { + for handoff in queue.take_parked() { + let reason = match handoff.reason { + queue::ParkHandoffReason::RetriesExhausted => reliability::ParkReason::RetriesExhausted, + }; + match reliability.park_batch(&handoff.batch, reason, false, now) { + Ok(()) => { + spawn_failure_notice( + rest_client, + &handoff.batch, + reliability::notices::parked(reason.as_str()), + ); + } + Err(error) => { + tracing::error!( + channel_id = %handoff.batch.channel_id, + batch_id = %handoff.batch.batch_id, + events = handoff.batch.events.len(), + error = %error, + "park file write failed — holding the batch in memory for the next attempt" + ); + if !queue.return_unparked(handoff) { + tracing::error!( + "park hand-off overflowed while the park file was unwritable — \ + the operator must fix the state directory" + ); + } + } + } + } +} + +/// After a successful live turn, release any batches that turn replayed and +/// stage the next parked batches for the same scope. +/// +/// `batch_replayed` is written **before** the events are staged for sending, so +/// a crash between the two is visible at the next start and moves the batch to +/// the review list rather than replaying it twice. +fn replay_after_success( + reliability: &mut reliability::ReliabilityRuntime, + queue: &mut EventQueue, + scope: &scope::SessionScope, + now: chrono::DateTime, +) { + use reliability::ledger::{self as led, LedgerBody}; + + // A `turn_finished` for every batch this turn replayed: the pair + // (`batch_replayed`, `turn_finished`) is what tells a later start-up that + // the replay completed and must not run again. + match reliability.finish_replay(scope) { + Ok(released) => { + for batch_id in released { + reliability.record( + now, + LedgerBody::TurnFinished(led::TurnFinished { + batch_id, + channel_id: scope.channel_id(), + outcome: led::TurnOutcome::Ok, + }), + ); + } + } + Err(error) => { + tracing::error!(error = %error, "could not clear replayed batches from the park file"); + } + } + let (pause_lifted, breaker_closed) = reliability.state().on_success(scope); + if pause_lifted { + reliability.record(now, LedgerBody::AgentResumed(led::AgentResumed {})); + } + if breaker_closed { + reliability.record( + now, + LedgerBody::BreakerClosed(led::BreakerClosed { + scope: scope.telemetry_label(), + }), + ); + } + + let Some(plan) = reliability.plan_replay(scope) else { + return; + }; + if !queue.can_stage_replay(scope) { + // A real cancel carryover is already staged for this scope; framing a + // replay as an interrupted turn would misdescribe both. The batches + // stay parked and replay after the next successful turn. + return; + } + let new_batch_id = Uuid::new_v4(); + if let Err(error) = reliability.commit_replay(&plan, new_batch_id, now) { + tracing::error!( + error = %error, + "could not record the replay durably — not replaying, the batches stay parked" + ); + return; + } + if queue.stage_replay(plan.scope.clone(), plan.events.clone()) { + reliability.mark_replay_in_flight(&plan); + tracing::info!( + channel_id = %plan.channel_id, + batches = plan.batch_ids.len(), + events = plan.events.len(), + "staged parked messages for replay ahead of newer events" + ); + } else { + reliability.abandon_replay(&plan.scope); + } +} + #[allow(clippy::too_many_arguments)] fn handle_prompt_result( pool: &mut AgentPool, @@ -4590,6 +5156,7 @@ fn handle_prompt_result( respawn_tasks: &mut tokio::task::JoinSet<()>, observer: Option, rest_client: Option<&relay::RestClient>, + reliability: Option<&mut reliability::ReliabilityRuntime>, ) -> LoopAction { let before = pool.task_map().len(); let agent_index = result.agent.index; @@ -4633,99 +5200,134 @@ fn handle_prompt_result( // retry_counts. If mark_complete runs first, retry_counts is cleared and // every retry starts at attempt 1 — defeating exponential backoff and // dead-letter protection. + let now = chrono::Utc::now(); + let turn_started = result.started; + // Ownership note: `reliability` is threaded through as an Option so the + // existing unit tests can drive `handle_prompt_result` without a state + // directory. Production always passes Some. + let mut reliability = reliability; if let Some(batch) = result.batch.take() { // Don't requeue batches for channels the agent was removed from — // those events are stale and should be silently dropped. if !removed_channels.contains(&batch.channel_id) { - if matches!( + // T16: pause, breaker and park run before the pre-existing retry + // chain, and take ownership of the batch when they apply. Cancel + // and steer keep their existing merge behaviour untouched. + let batch = if matches!( result.outcome, PromptOutcome::Cancelled | PromptOutcome::CancelDrainTimeout(_) ) { - // Cancel re-prompt: store as cancelled events so flush_next() - // merges them into the next FlushBatch.cancelled_events, - // enabling the annotated merged-prompt format. The batch's - // cancel_reason (set by the pool task per the control signal) - // selects steer vs interrupt framing. It is always set on this - // path; if somehow unset, fall back to the gentler Steer framing - // — consistent with MergeFraming::for_reason(None) and the - // system default — rather than telling the agent to supersede. - // - // CancelDrainTimeout shares this path with Cancelled: a failed - // 5s drain after a control-signal cancel is a cleanup-deadline - // problem, not the deterministic hard-cap death below — the - // original batch must survive with no retry/dead-letter - // accounting, same as a clean cancel. - let reason = batch.cancel_reason.unwrap_or(CancelReason::Steer); - queue.requeue_as_cancelled(batch, reason); - } else if matches!( - result.outcome, - PromptOutcome::Timeout(TimeoutKind::Hard { - recently_active: false - }) - ) { - tracing::error!( - channel_id = %batch.channel_id, - events = batch.events.len(), - "dead-lettering batch after hard-cap timeout (no recent activity) — discarding {} events", - batch.events.len(), - ); - let content = format!( + Some(batch) + } else if let Some(reliability) = reliability.as_deref_mut() { + match apply_reliability( + reliability, + queue, + batch, + &result.outcome, + turn_started, + rest_client, + now, + ) { + Disposition::Handled => None, + Disposition::Fallthrough(batch) => Some(batch), + } + } else { + Some(batch) + }; + if let Some(batch) = batch { + if matches!( + result.outcome, + PromptOutcome::Cancelled | PromptOutcome::CancelDrainTimeout(_) + ) { + // Cancel re-prompt: store as cancelled events so flush_next() + // merges them into the next FlushBatch.cancelled_events, + // enabling the annotated merged-prompt format. The batch's + // cancel_reason (set by the pool task per the control signal) + // selects steer vs interrupt framing. It is always set on this + // path; if somehow unset, fall back to the gentler Steer framing + // — consistent with MergeFraming::for_reason(None) and the + // system default — rather than telling the agent to supersede. + // + // CancelDrainTimeout shares this path with Cancelled: a failed + // 5s drain after a control-signal cancel is a cleanup-deadline + // problem, not the deterministic hard-cap death below — the + // original batch must survive with no retry/dead-letter + // accounting, same as a clean cancel. + let reason = batch.cancel_reason.unwrap_or(CancelReason::Steer); + queue.requeue_as_cancelled(batch, reason); + } else if matches!( + result.outcome, + PromptOutcome::Timeout(TimeoutKind::Hard { + recently_active: false + }) + ) { + tracing::error!( + channel_id = %batch.channel_id, + events = batch.events.len(), + "dead-lettering batch after hard-cap timeout (no recent activity) — discarding {} events", + batch.events.len(), + ); + let content = format!( "⚠️ I couldn't process the last request (the turn exceeded the maximum duration ({}s)). Please re-send if it's still needed.", config.max_turn_duration_secs ); - spawn_failure_notice(rest_client, &batch, content); - hard_timeout_fate_suffix = Some(" — dead-lettered (no recent activity)"); - } else if matches!( - result.outcome, - PromptOutcome::Timeout(TimeoutKind::Hard { - recently_active: true - }) - ) { - tracing::warn!( - channel_id = %batch.channel_id, - events = batch.events.len(), - "hard-cap timeout with recent activity — requeueing for retry" - ); - if let Some(dead) = queue.requeue(batch) { - let content = format!( + spawn_failure_notice(rest_client, &batch, content); + hard_timeout_fate_suffix = Some(" — dead-lettered (no recent activity)"); + } else if matches!( + result.outcome, + PromptOutcome::Timeout(TimeoutKind::Hard { + recently_active: true + }) + ) { + tracing::warn!( + channel_id = %batch.channel_id, + events = batch.events.len(), + "hard-cap timeout with recent activity — requeueing for retry" + ); + if let Some(dead) = queue.requeue(batch) { + let content = format!( "⚠️ I couldn't process the last request after multiple retries (the turn exceeded the maximum duration ({}s)). Please re-send if it's still needed.", config.max_turn_duration_secs ); - spawn_failure_notice(rest_client, &dead, content); - hard_timeout_fate_suffix = Some(" — dead-lettered (retry budget exhausted)"); - } else { - hard_timeout_fate_suffix = Some(" — requeued for retry (recently active)"); - } - } else if matches!(&result.outcome, PromptOutcome::Error(e) if is_auth_error(e)) { - // Auth errors are non-retryable: the token won't self-repair - // between retries, so requeueing only wastes attempt slots and - // delays the visible failure. Dead-letter immediately and tell - // the user to re-authenticate the CLI. - tracing::warn!( - channel_id = %batch.channel_id, - events = batch.events.len(), - "dead-lettering batch immediately — non-retryable auth error" - ); - let content = "⚠️ I couldn't process the last request: authentication failed. \ + spawn_failure_notice(rest_client, &dead, content); + hard_timeout_fate_suffix = + Some(" — dead-lettered (retry budget exhausted)"); + } else { + hard_timeout_fate_suffix = Some(" — requeued for retry (recently active)"); + } + } else if matches!(&result.outcome, PromptOutcome::Error(e) if is_auth_error(e)) { + // Auth errors are non-retryable: the token won't self-repair + // between retries, so requeueing only wastes attempt slots and + // delays the visible failure. Dead-letter immediately and tell + // the user to re-authenticate the CLI. + tracing::warn!( + channel_id = %batch.channel_id, + events = batch.events.len(), + "dead-lettering batch immediately — non-retryable auth error" + ); + let content = "⚠️ I couldn't process the last request: authentication failed. \ Please re-authenticate the CLI (e.g. run `claude /login` or `codex login`) \ and then re-send." - .to_string(); - spawn_failure_notice(rest_client, &batch, content); - } else if let Some(dead) = queue.requeue(batch) { - let reason = match &result.outcome { - PromptOutcome::Timeout(TimeoutKind::Idle) => "the turn timed out".to_string(), - PromptOutcome::Timeout(TimeoutKind::Hard { .. }) => { - "the turn exceeded the maximum duration".to_string() - } - PromptOutcome::AgentExited => "the agent process exited".to_string(), - PromptOutcome::Error(e) => format!("{e}"), - PromptOutcome::ProjectContextIndeterminate(reason) => reason.clone(), - _ => "repeated failures".to_string(), - }; - let content = format!( + .to_string(); + spawn_failure_notice(rest_client, &batch, content); + } else if let Some(dead) = queue.requeue(batch) { + let reason = match &result.outcome { + PromptOutcome::Timeout(TimeoutKind::Idle) => { + "the turn timed out".to_string() + } + PromptOutcome::Timeout(TimeoutKind::Hard { .. }) => { + "the turn exceeded the maximum duration".to_string() + } + PromptOutcome::AgentExited => "the agent process exited".to_string(), + PromptOutcome::Error(e) => format!("{e}"), + PromptOutcome::ProjectContextIndeterminate(reason) => reason.clone(), + _ => "repeated failures".to_string(), + }; + let content = format!( "⚠️ I couldn't process the last request after multiple retries ({reason}). Please re-send if it's still needed." ); - spawn_failure_notice(rest_client, &dead, content); + spawn_failure_notice(rest_client, &dead, content); + } } } else { tracing::debug!( @@ -4737,11 +5339,33 @@ fn handle_prompt_result( } } + // Every batch the queue gave up retrying is written durably before the + // harness lets go of it. + if let Some(reliability) = reliability.as_deref_mut() { + drain_park_handoff(reliability, queue, rest_client, now); + } + match &result.source { PromptSource::Channel(scope) => queue.mark_complete(scope.clone()), PromptSource::Heartbeat => *heartbeat_in_flight = false, } + // A successful live turn is the only thing that resumes a paused agent, + // closes a breaker, or releases parked messages for replay. A restart + // proves nothing about the provider and never replays anything. + if let (Some(reliability), PromptSource::Channel(scope)) = + (reliability.as_deref_mut(), &result.source) + { + if matches!(result.outcome, PromptOutcome::Ok(_)) { + replay_after_success(reliability, queue, scope, now); + } else { + // The replayed batches this turn carried stay parked and become + // eligible again: delivery is at least once, never at most once. + reliability.abandon_replay(scope); + } + reliability.maintain(now); + } + // Strip sessions for channels the agent was removed from while this // agent was checked out. This covers the gap where invalidate_channel_sessions // only touches idle agents. @@ -10435,6 +11059,7 @@ mod error_outcome_emission_tests { let (respawn_tx, _respawn_rx) = mpsc::channel(8); let mut respawn_tasks = tokio::task::JoinSet::new(); let result = PromptResult { + started: false, agent, source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id }), turn_id: "test-turn-id".into(), @@ -10454,6 +11079,7 @@ mod error_outcome_emission_tests { &mut respawn_tasks, None, None, + None, ); let returned = pool.agents_mut()[0].as_ref().expect("returned agent"); @@ -10510,6 +11136,7 @@ mod error_outcome_emission_tests { let (respawn_tx, _respawn_rx) = mpsc::channel(8); let mut respawn_tasks = tokio::task::JoinSet::new(); let result = PromptResult { + started: false, agent, source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id }), turn_id: "test-turn-id".into(), @@ -10529,6 +11156,7 @@ mod error_outcome_emission_tests { &mut respawn_tasks, None, None, + None, ); let returned = pool.agents_mut()[0].as_ref().expect("returned agent"); @@ -10631,6 +11259,7 @@ mod error_outcome_emission_tests { let (respawn_tx, _respawn_rx) = mpsc::channel(8); let mut respawn_tasks = tokio::task::JoinSet::new(); let result = PromptResult { + started: false, agent, source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id }), turn_id: "test-turn-id".into(), @@ -10650,6 +11279,7 @@ mod error_outcome_emission_tests { &mut respawn_tasks, None, None, + None, ); let returned = pool.agents_mut()[0].as_ref().expect("returned agent"); @@ -10698,6 +11328,7 @@ mod error_outcome_emission_tests { let observer = ObserverHandle::in_process(); let result = PromptResult { + started: false, agent, source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id: Uuid::new_v4(), @@ -10719,6 +11350,7 @@ mod error_outcome_emission_tests { &mut respawn_tasks, Some(observer.clone()), None, + None, ); let turn_errors: Vec<_> = observer @@ -10967,6 +11599,7 @@ mod error_outcome_emission_tests { let mut respawn_tasks = tokio::task::JoinSet::new(); let observer = ObserverHandle::in_process(); let result = PromptResult { + started: false, agent, source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id: Uuid::new_v4(), @@ -10987,6 +11620,7 @@ mod error_outcome_emission_tests { &mut respawn_tasks, Some(observer.clone()), None, + None, ); let events = observer.snapshot(); let turn_error = events.iter().find(|e| e.kind == "turn_error").unwrap(); @@ -11020,6 +11654,7 @@ mod error_outcome_emission_tests { .unwrap(); let __cid = Uuid::new_v4(); FlushBatch { + batch_id: Uuid::new_v4(), channel_id: __cid, scope: scope::SessionScope::Conversation { channel_id: __cid }, events: vec![BatchEvent { @@ -11063,6 +11698,7 @@ mod error_outcome_emission_tests { let (respawn_tx, _respawn_rx) = mpsc::channel(8); let mut respawn_tasks = tokio::task::JoinSet::new(); let result = PromptResult { + started: false, agent, source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id }), turn_id: "test-turn-id".to_string(), @@ -11081,6 +11717,7 @@ mod error_outcome_emission_tests { &mut respawn_tasks, None, None, + None, ); ( queue.pending_channels(), @@ -11129,6 +11766,7 @@ mod error_outcome_emission_tests { .sign_with_keys(&keys) .unwrap(); FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: scope::SessionScope::Conversation { channel_id }, events: vec![BatchEvent { @@ -11171,6 +11809,7 @@ mod error_outcome_emission_tests { let (respawn_tx, _respawn_rx) = mpsc::channel(8); let mut respawn_tasks = tokio::task::JoinSet::new(); let result = PromptResult { + started: false, agent, source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id }), turn_id: "test-turn-id".to_string(), @@ -11189,6 +11828,7 @@ mod error_outcome_emission_tests { &mut respawn_tasks, None, None, + None, ); ( queue.pending_channels(), @@ -11250,6 +11890,7 @@ mod error_outcome_emission_tests { let mut respawn_tasks = tokio::task::JoinSet::new(); let observer = ObserverHandle::in_process(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: scope::SessionScope::Conversation { channel_id }, events: vec![BatchEvent { @@ -11263,6 +11904,7 @@ mod error_outcome_emission_tests { cancel_reason: None, }; let result = PromptResult { + started: false, agent, source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id }), turn_id: "test-turn-id".to_string(), @@ -11283,6 +11925,7 @@ mod error_outcome_emission_tests { &mut respawn_tasks, Some(observer.clone()), None, + None, ); let events = observer.snapshot(); @@ -11346,6 +11989,7 @@ mod error_outcome_emission_tests { let mut respawn_tasks = tokio::task::JoinSet::new(); let observer = ObserverHandle::in_process(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: scope::SessionScope::Conversation { channel_id }, events: vec![BatchEvent { @@ -11359,6 +12003,7 @@ mod error_outcome_emission_tests { cancel_reason: None, }; let result = PromptResult { + started: false, agent, source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id }), turn_id: "test-turn-id".to_string(), @@ -11379,6 +12024,7 @@ mod error_outcome_emission_tests { &mut respawn_tasks, Some(observer.clone()), None, + None, ); let events = observer.snapshot(); @@ -11427,6 +12073,7 @@ mod error_outcome_emission_tests { ); let channel_id = Uuid::new_v4(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: scope::SessionScope::Conversation { channel_id }, events: vec![BatchEvent { @@ -11479,6 +12126,7 @@ mod error_outcome_emission_tests { let observer = ObserverHandle::in_process(); let grace = std::time::Duration::from_secs(5); let result = PromptResult { + started: false, agent, source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id }), turn_id: "test-turn-id".to_string(), @@ -11498,6 +12146,7 @@ mod error_outcome_emission_tests { &mut respawn_tasks, Some(observer.clone()), None, + None, ); // Batch preserved as a cancelled merge, not dead-lettered — same @@ -11610,6 +12259,7 @@ mod error_outcome_emission_tests { let observer = ObserverHandle::in_process(); let grace = std::time::Duration::from_secs(5); let result = PromptResult { + started: false, agent, source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id: Uuid::new_v4(), @@ -11634,6 +12284,7 @@ mod error_outcome_emission_tests { &mut respawn_tasks, Some(observer.clone()), None, + None, ); // No batch to merge — the queue has nothing pending for any channel. @@ -11698,6 +12349,7 @@ mod error_outcome_emission_tests { .sign_with_keys(&Keys::generate()) .unwrap(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: session_scope.clone(), events: vec![BatchEvent { @@ -11741,6 +12393,7 @@ mod error_outcome_emission_tests { let (respawn_tx, _respawn_rx) = mpsc::channel(8); let mut respawn_tasks = tokio::task::JoinSet::new(); let result = PromptResult { + started: false, agent, source: PromptSource::Channel(session_scope.clone()), turn_id: "indeterminate-project".into(), @@ -11763,6 +12416,7 @@ mod error_outcome_emission_tests { &mut respawn_tasks, None, None, + None, ), LoopAction::Continue )); @@ -11851,6 +12505,7 @@ mod error_outcome_emission_tests { .unwrap(); let channel_id = uuid::Uuid::new_v4(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: scope::SessionScope::Conversation { channel_id }, events: vec![BatchEvent { @@ -11896,6 +12551,7 @@ mod error_outcome_emission_tests { let (respawn_tx, _respawn_rx) = mpsc::channel(8); let mut respawn_tasks = tokio::task::JoinSet::new(); let result = PromptResult { + started: false, agent, source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id }), turn_id: "test-turn-id".to_string(), @@ -11914,6 +12570,7 @@ mod error_outcome_emission_tests { &mut respawn_tasks, None, None, + None, ); // The batch must not be requeued: pending_channels returns 0. @@ -11939,6 +12596,7 @@ mod error_outcome_emission_tests { .unwrap(); let channel_id = uuid::Uuid::new_v4(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: scope::SessionScope::Conversation { channel_id }, events: vec![BatchEvent { @@ -11984,6 +12642,7 @@ mod error_outcome_emission_tests { let (respawn_tx, _respawn_rx) = mpsc::channel(8); let mut respawn_tasks = tokio::task::JoinSet::new(); let result = PromptResult { + started: false, agent, source: PromptSource::Channel(scope::SessionScope::Conversation { channel_id }), turn_id: "test-turn-id".to_string(), @@ -12002,6 +12661,7 @@ mod error_outcome_emission_tests { &mut respawn_tasks, None, None, + None, ); // Non-auth application error: batch IS requeued (first attempt, retry budget > 0). diff --git a/crates/buzz-acp/src/pool.rs b/crates/buzz-acp/src/pool.rs index 39737c3418e..3d2a0f8e9d5 100644 --- a/crates/buzz-acp/src/pool.rs +++ b/crates/buzz-acp/src/pool.rs @@ -353,6 +353,11 @@ pub struct PromptResult { pub outcome: PromptOutcome, /// Present on failure in Queue mode, for requeue. pub batch: Option, + /// Whether the harness saw agent output or a tool call for this turn. + /// + /// A batch whose turn started is never replayed automatically: the agent + /// may already have acted on it, so it waits for an operator instead. + pub started: bool, } /// Whether the prompt came from a channel event or a heartbeat. @@ -2015,12 +2020,16 @@ fn send_prompt_result( batch: Option, ) { agent.acp.clear_steer_rx(); + // Read the started signal here, in the one place every prompt outcome + // passes through, so no exit path can forget to report it. + let started = agent.acp.turn_saw_output(); let _ = result_tx.send(PromptResult { agent, source, turn_id: turn_id.to_owned(), outcome, batch, + started, }); } @@ -6416,6 +6425,7 @@ mod tests { let author_hex = event.pubkey.to_hex(); let channel_id = Uuid::new_v4(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: SessionScope::Conversation { channel_id }, events: vec![crate::queue::BatchEvent { @@ -6668,6 +6678,7 @@ done"# .unwrap(); let event_id = event.id.to_hex(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: SessionScope::Conversation { channel_id }, events: vec![crate::queue::BatchEvent { @@ -6751,6 +6762,7 @@ done"# .sign_with_keys(&keys) .unwrap(); let merged_batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: SessionScope::Conversation { channel_id }, events: vec![crate::queue::BatchEvent { @@ -6766,6 +6778,7 @@ done"# cancel_reason: Some(crate::queue::CancelReason::Steer), }; let next_batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: SessionScope::Conversation { channel_id }, events: vec![crate::queue::BatchEvent { @@ -6922,6 +6935,7 @@ done"# .sign_with_keys(&keys) .unwrap(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: SessionScope::Conversation { channel_id }, events: vec![crate::queue::BatchEvent { @@ -7279,6 +7293,7 @@ printf '%s\n' '{{"jsonrpc":"2.0","id":0,"result":{{"stopReason":"end_turn"}}}}'" fn batch_with_scope(scope: SessionScope, event: nostr::Event) -> FlushBatch { FlushBatch { + batch_id: Uuid::new_v4(), channel_id: scope.channel_id(), scope, events: vec![crate::queue::BatchEvent { @@ -7641,6 +7656,7 @@ printf '%s\n' '{{"jsonrpc":"2.0","id":0,"result":{{"stopReason":"end_turn"}}}}'" .sign_with_keys(&keys) .unwrap(); FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: SessionScope::Conversation { channel_id }, events: vec![crate::queue::BatchEvent { @@ -9389,6 +9405,7 @@ done"# .unwrap(); let event_id = event.id.to_hex(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: conv(channel_id), events: vec![crate::queue::BatchEvent { diff --git a/crates/buzz-acp/src/queue.rs b/crates/buzz-acp/src/queue.rs index b2fbde6242f..202789c9711 100644 --- a/crates/buzz-acp/src/queue.rs +++ b/crates/buzz-acp/src/queue.rs @@ -129,11 +129,26 @@ pub enum CancelReason { /// and incorporate the message if relevant /// (`MultipleEventHandling::Steer`, the default mid-turn path). Steer, + /// The events were **parked** while the agent was unavailable and are being + /// replayed after a successful live turn. They are the client's original + /// words, never answered; they run ahead of anything newer for the scope. + /// + /// Shares the cancelled-events merge machinery (see + /// [`format_prompt`](crate::queue::format_prompt)) so the replay is one + /// prompt with an annotated "Delivered late" section. + DeliveredLate, } /// A batch of events to prompt the agent with. #[derive(Debug, Clone)] pub struct FlushBatch { + /// Stable identity for this batch, assigned when the batch is built. + /// + /// The park file keys parked batches by it and the ledger refers to them by + /// it. A batch that is requeued and later re-flushed is a new batch with a + /// new id; the scope's retry count, not this id, is what carries across + /// attempts. + pub batch_id: Uuid, pub channel_id: Uuid, /// The single session scope every event in this batch belongs to. Events /// from different scopes are never combined into one batch. @@ -230,6 +245,31 @@ pub struct EventQueue { /// Must be strictly greater than `max_turn_duration` so a turn running to /// the hard cap returns via `mark_complete` before the backstop fires. in_flight_deadline: Duration, + /// Batches whose retry budget ran out, waiting for the caller to write them + /// to the durable park file. Bounded by [`MAX_PARK_HANDOFF`]: the queue + /// holds client messages in memory here only until the caller drains them, + /// and refusing to grow past the cap keeps a caller that never drains from + /// turning this into an unbounded backlog. + parked_out: VecDeque, +} + +/// Most batches held in the park hand-off at once. +pub const MAX_PARK_HANDOFF: usize = 200; + +/// Why a batch reached the park hand-off. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ParkHandoffReason { + /// The scope's retry budget ran out. + RetriesExhausted, +} + +/// A batch the queue has given up retrying, waiting to be parked durably. +#[derive(Debug, Clone)] +pub struct ParkHandoff { + /// The batch, complete with its events. + pub batch: FlushBatch, + /// Why it stopped being retried. + pub reason: ParkHandoffReason, } impl EventQueue { @@ -251,6 +291,7 @@ impl EventQueue { cancel_reasons: HashMap::new(), withheld_native_steer: HashMap::new(), in_flight_deadline: Duration::from_secs(DEFAULT_IN_FLIGHT_DEADLINE_SECS), + parked_out: VecDeque::new(), } } @@ -435,6 +476,7 @@ impl EventQueue { self.in_flight_batch_sizes .insert(scope.clone(), cancelled.len()); return Some(FlushBatch { + batch_id: Uuid::new_v4(), channel_id: scope.channel_id(), scope, events: cancelled, @@ -486,6 +528,7 @@ impl EventQueue { }; Some(FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope, events, @@ -556,15 +599,23 @@ impl EventQueue { channel_id = %channel_id, attempt, events = batch.events.len(), - "dead-lettering batch after {} retries — discarding {} events", + "parking batch after {} retries — {} events held for replay, none discarded", MAX_RETRIES, batch.events.len(), ); self.retry_counts.remove(&scope); // Also clear retry_after so fresh traffic on this scope isn't - // throttled by stale backoff from the discarded poison batch. + // throttled by stale backoff from the exhausted batch. self.retry_after.remove(&scope); - return Some(batch); + // The batch is NOT returned: a returned batch used to be discarded + // by the caller. It moves to the park hand-off instead, where the + // caller writes it to the durable park file before dropping its own + // copy (T16). Nothing leaves the harness's custody here. + self.parked_out.push_back(ParkHandoff { + batch, + reason: ParkHandoffReason::RetriesExhausted, + }); + return None; } // Exponential backoff: BASE * 2^(attempt-1), capped at MAX, with ±20% jitter. @@ -617,6 +668,92 @@ impl EventQueue { None } + /// Take every batch waiting to be parked durably. + /// + /// The caller writes each one to the park file and only then drops it. A + /// caller that cannot write returns the batch with + /// [`return_unparked`](Self::return_unparked) so nothing is lost. + pub fn take_parked(&mut self) -> Vec { + self.parked_out.drain(..).collect() + } + + /// Whether any batch is waiting to be parked. + pub fn has_parked_handoff(&self) -> bool { + !self.parked_out.is_empty() + } + + /// Give a batch back to the hand-off after a park-file write failed. + /// + /// Returns `false` — and logs — when the hand-off is at + /// [`MAX_PARK_HANDOFF`]. A `false` return is the caller's signal that the + /// batch could not be held here either; it must stay in the caller's own + /// hands or the failure has to be surfaced. + pub fn return_unparked(&mut self, handoff: ParkHandoff) -> bool { + if self.parked_out.len() >= MAX_PARK_HANDOFF { + tracing::error!( + channel_id = %handoff.batch.channel_id, + batch_id = %handoff.batch.batch_id, + cap = MAX_PARK_HANDOFF, + events = handoff.batch.events.len(), + "park hand-off is full — the batch could not be held for a retry of the park write" + ); + return false; + } + self.parked_out.push_front(handoff); + true + } + + /// Stage parked events for replay ahead of anything newer for `scope`. + /// + /// The events are stored in the same carryover the cancelled-events merge + /// uses, so the next `flush_next` for the scope produces one prompt with a + /// "Delivered late" section before the newer events. The event text is + /// never edited. + /// + /// Returns `false` when a real cancel carryover is already staged for the + /// scope: framing a replay as an interrupted turn would misdescribe both, + /// so the replay waits for the cancel to resolve. + /// Attempts already spent on `scope`. Zero for a scope that has not failed. + pub fn retry_count(&self, scope: K) -> u32 { + self.retry_counts + .get(&scope.into_scope()) + .copied() + .unwrap_or(0) + } + + /// Whether [`stage_replay`](Self::stage_replay) would accept a replay for + /// `scope` right now. + /// + /// Checked before the durable `batch_replayed` record is written, so the + /// harness never stamps a batch as replayed and then finds it cannot stage + /// the prompt. + pub fn can_stage_replay(&self, scope: &SessionScope) -> bool { + matches!( + self.cancel_reasons.get(scope), + Some(CancelReason::DeliveredLate) | None + ) + } + + pub fn stage_replay(&mut self, scope: SessionScope, events: Vec) -> bool { + if events.is_empty() { + return false; + } + match self.cancel_reasons.get(&scope) { + Some(CancelReason::DeliveredLate) | None => {} + Some(_) => return false, + } + let entry = self.cancelled_batches.entry(scope.clone()).or_default(); + // Replayed events are older than anything already staged, so they go + // first — the conversation stays in order. + let mut merged = events; + merged.extend(entry.drain(..)); + merged.truncate(MAX_BATCH_EVENTS); + *entry = merged; + self.cancel_reasons + .insert(scope, CancelReason::DeliveredLate); + true + } + /// Re-queue a **complete** flushed batch preserving original `received_at` /// timestamps. /// @@ -2054,9 +2191,12 @@ pub fn format_prompt(batch: &FlushBatch, args: &FormatPromptArgs<'_>) -> Vec) -> Vec) -> Vec String { + let (first, last) = delivered_late_window(events); + crate::prompt_framing::semantic_section_with_attributes( + DELIVERED_LATE_TAG, + &[("first-at", first.as_str()), ("last-at", last.as_str())], + &format!( + "Delivered late: these messages arrived while I was unavailable (first at {first}, \ +last at {last}). They are unanswered and unedited.\n\n{body}" + ), + ) +} + +/// The `HH:MM` UTC window a set of events arrived in. An empty set — which +/// `delivered_late_section` is never called with — renders as `--:--`. +fn delivered_late_window(events: &[BatchEvent]) -> (String, String) { + let mut stamps: Vec = events + .iter() + .map(|be| be.event.created_at.as_secs()) + .collect(); + stamps.sort_unstable(); + let render = |secs: Option<&u64>| -> String { + secs.and_then(|s| chrono::DateTime::from_timestamp(*s as i64, 0)) + .map(|dt| dt.format("%H:%M UTC").to_string()) + .unwrap_or_else(|| "--:--".to_string()) + }; + (render(stamps.first()), render(stamps.last())) +} + /// Prompt-framing strings for a merged (cancel + re-prompt) turn, selected by /// [`CancelReason`]. `Interrupt` frames the new events as superseding the prior /// work; `Steer` (the default mid-turn path) frames them as messages that @@ -2164,6 +2362,13 @@ impl MergeFraming { in-progress work and incorporate the new message if it's relevant; if it's \ unrelated, you may briefly acknowledge it and carry on.", }, + Some(CancelReason::DeliveredLate) => MergeFraming { + prior_tag: DELIVERED_LATE_TAG, + new_tag: "arrived-since", + closing_note: "Note: The messages in the delivered-late section arrived while I \ + was unavailable and were never answered. Answer them first, in order, then \ + anything that arrived since.", + }, Some(CancelReason::Interrupt) => MergeFraming { prior_tag: "previous-request-interrupted-before-completion", new_tag: "new-request-supersedes-previous", @@ -2582,6 +2787,7 @@ mod tests { .unwrap_or_else(|_| event.pubkey.to_hex()); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -2613,6 +2819,7 @@ mod tests { fn make_merged_batch(reason: Option) -> FlushBatch { let ch = Uuid::new_v4(); FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -2745,6 +2952,7 @@ mod tests { // Multi-event header path must also branch on reason. let ch = Uuid::new_v4(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![ @@ -2803,6 +3011,7 @@ mod tests { let _steering_id = steering.id.to_hex(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -2975,6 +3184,7 @@ mod tests { let e3 = make_event("third message"); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![ @@ -3016,6 +3226,7 @@ mod tests { let event = make_event("hello"); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -3040,6 +3251,7 @@ mod tests { let ch = Uuid::new_v4(); let event = make_event("hi"); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -3073,6 +3285,7 @@ mod tests { let ch = Uuid::new_v4(); let event = make_event("hi"); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -3104,6 +3317,7 @@ mod tests { let ch = Uuid::new_v4(); let event = make_event("hi"); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -3132,6 +3346,7 @@ mod tests { let event = make_event("hello"); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -3157,6 +3372,7 @@ mod tests { let event = make_event("hello"); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -3216,6 +3432,7 @@ mod tests { // evicting real channel history sooner. let ch = Uuid::new_v4(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -3270,6 +3487,7 @@ mod tests { let event = make_event("hello"); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -3309,6 +3527,7 @@ mod tests { let ch = Uuid::new_v4(); let event = make_event("hello"); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -3569,6 +3788,7 @@ mod tests { let ch = Uuid::new_v4(); let scope = conv(ch); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: scope.clone(), events: vec![BatchEvent { @@ -3898,6 +4118,7 @@ mod tests { let ch = Uuid::new_v4(); let event = make_event("hello"); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -3932,6 +4153,7 @@ mod tests { let ch = Uuid::new_v4(); let event = make_event("hey"); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -3980,6 +4202,7 @@ mod tests { for is_dm in [false, true] { for (event, is_reply) in [(&top, false), (&reply, true)] { let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: SessionScope::derive(policy, channel_id, is_dm, event), events: vec![BatchEvent { @@ -4062,6 +4285,7 @@ mod tests { ]], ); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -4089,6 +4313,7 @@ mod tests { vec![vec!["e".into(), root.clone(), "".into(), "reply".into()]], ); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -4222,6 +4447,7 @@ mod tests { }; let mixed_batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![ @@ -4247,6 +4473,7 @@ mod tests { assert!(mixed_prompt.contains("buzz messages thread")); let same_thread_batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![ @@ -4275,6 +4502,7 @@ mod tests { let ch = Uuid::new_v4(); let event = make_event("ok do that"); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -4332,6 +4560,7 @@ mod tests { ); let author_hex = event.pubkey.to_hex(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -4542,6 +4771,7 @@ mod tests { ]], ); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -4613,6 +4843,7 @@ mod tests { ]], ); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -4647,6 +4878,7 @@ mod tests { fn test_format_prompt_empty_dm_delta_distinguishes_trigger_only_from_delivered() { let ch = Uuid::new_v4(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -4698,6 +4930,7 @@ mod tests { let ch = Uuid::new_v4(); let event = make_event("hey there"); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -4741,6 +4974,7 @@ mod tests { let event = make_event("test"); let event_id = event.id.to_hex(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -4766,6 +5000,7 @@ mod tests { let hex = event.pubkey.to_hex(); let npub = event.pubkey.to_bech32().unwrap(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -4790,6 +5025,7 @@ mod tests { // Kind 9 (stream message) — tags were previously stripped. let event = make_event_with_tags("hello", vec![vec!["h".into(), ch.to_string()]]); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -5218,6 +5454,7 @@ mod tests { vec![vec!["e".into(), root_id.clone(), "".into(), "reply".into()]], ); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -5261,6 +5498,7 @@ mod tests { ); let event_id = event.id.to_hex(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -5298,6 +5536,7 @@ mod tests { let event = make_event("hello world"); let event_id = event.id.to_hex(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -5328,6 +5567,7 @@ mod tests { let ch = Uuid::new_v4(); let event = make_event("hey there"); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -5373,6 +5613,7 @@ mod tests { ); let event_id = event.id.to_hex(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -5410,6 +5651,7 @@ mod tests { vec![vec!["e".into(), root_id.clone(), "".into(), "reply".into()]], ); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -5446,6 +5688,7 @@ mod tests { vec![vec!["e".into(), root_id.clone(), "".into(), "reply".into()]], ); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![ @@ -5484,6 +5727,7 @@ mod tests { let plain = make_event("latest top-level"); let plain_id = plain.id.to_hex(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![ @@ -5519,6 +5763,7 @@ mod tests { fn make_single_batch(content: &str) -> FlushBatch { let channel_id = Uuid::new_v4(); FlushBatch { + batch_id: Uuid::new_v4(), channel_id, scope: conv(channel_id), events: vec![BatchEvent { @@ -5820,6 +6065,7 @@ mod tests { let canvas = "[Channel Canvas]\nCanvas revision (event ID): abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234\nLast modified: 2024-01-15T10:30:00+00:00\nFetch current content with: buzz canvas get --channel 00f1ccaf-1506-4dd7-9a0e-fa67e9e486ae"; let ch = Uuid::new_v4(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -5850,6 +6096,7 @@ mod tests { let canvas = "[Channel Canvas]\nCanvas revision (event ID): abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234abcd1234\nLast modified: 2024-01-15T10:30:00+00:00\nFetch current content with: buzz canvas get --channel 00f1ccaf-1506-4dd7-9a0e-fa67e9e486ae"; let ch = Uuid::new_v4(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -5879,6 +6126,7 @@ mod tests { fn test_format_prompt_no_canvas_produces_no_canvas_section() { let ch = Uuid::new_v4(); let batch = FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { @@ -6341,6 +6589,7 @@ mod tests { fn description_batch(ch: Uuid, event: Event) -> FlushBatch { FlushBatch { + batch_id: Uuid::new_v4(), channel_id: ch, scope: conv(ch), events: vec![BatchEvent { diff --git a/crates/buzz-acp/src/reliability.rs b/crates/buzz-acp/src/reliability.rs index 0dc7de529d4..e09761048ec 100644 --- a/crates/buzz-acp/src/reliability.rs +++ b/crates/buzz-acp/src/reliability.rs @@ -2,24 +2,43 @@ //! //! Design: `docs/plans/2026-09-06-harness-reliability-design.md` (T16). //! -//! This file currently holds the **fixture tests** for T16 and the smallest -//! stubs that let them compile. Every test is `#[ignore]` with the ticket -//! named in the reason, so the branch stays green while the behaviour is -//! missing. Run them with: +//! The harness never discards a message. A failure that used to dead-letter a +//! batch now **parks** it — writes it to a durable, owner-only park file in the +//! agent's state directory — and either replays it automatically after a +//! successful live turn or hands it to the operator for review. //! -//! ```text -//! cargo test -p buzz-acp reliability -- --ignored -//! ``` +//! Submodules: //! -//! They fail today. T16 is ready when they pass without `#[ignore]`. - -// The stubs below have no caller until T16 wires them into the pool. -#![allow(dead_code)] +//! - [`error_class`] — pure classification of a provider error, and the +//! `resets H:MM(am|pm) (IANA zone)` parser. +//! - [`state`] — the per-agent pause and the per-scope breakers. +//! - [`state_dir`] — where the durable state lives and how it is locked down. +//! - [`ledger`] — the append-only `ledger.jsonl`. +//! - [`park`] — the `parked.jsonl` park file. +//! - [`notices`] — the channel notice templates. +//! - [`runtime`] — the glue that owns all of the above and orders the writes. use chrono::{DateTime, Utc}; -use crate::acp::AcpError; -use crate::scope::SessionScope; +pub mod error_class; +pub mod ledger; +pub mod notices; +pub mod park; +pub mod runtime; +pub mod state; +pub mod state_dir; + +pub use error_class::classify_at; +pub use park::{ParkError, ParkReason, ParkedBatch}; +pub use runtime::{ReliabilityRuntime, ReplayPlan}; +pub use state::{BreakerGate, BreakerVerdict, PauseGate, ReliabilityState}; + +/// Longest provider error text the harness inspects, stores or forwards. +/// +/// Provider error strings are untrusted input: they are attacker-influenceable +/// through tool output and model output. Everything downstream — the ledger's +/// `raw` field, notice text, log lines — starts from a string cut to this. +pub const MAX_RAW_ERROR_CHARS: usize = 512; /// Why a turn failed, as far as the harness can tell from the provider. #[derive(Debug, Clone, PartialEq, Eq)] @@ -37,6 +56,18 @@ pub enum ErrorClass { Unknown, } +impl ErrorClass { + /// The class name written to the ledger. + pub fn as_str(&self) -> &'static str { + match self { + Self::CapacityExhausted { .. } => "capacity_exhausted", + Self::Auth => "auth", + Self::ProviderInternal => "provider_internal", + Self::Unknown => "unknown", + } + } +} + /// What the harness does with the scope after an outcome. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Action { @@ -50,42 +81,13 @@ pub enum Action { Park, } -/// Classify a provider error at a known instant. `now` is a parameter so a -/// reset time parsed from "resets 4:20am (America/Los_Angeles)" resolves to -/// the same absolute instant in a test as in production. -/// -/// STUB: T16 replaces this body. It exists so the fixtures compile. -pub fn classify_at(_err: &AcpError, _now: DateTime) -> ErrorClass { - ErrorClass::Unknown -} - -/// Per-agent reliability state: pause and per-scope breakers. -/// -/// STUB: T16 replaces this body. -#[derive(Debug, Default)] -pub struct ReliabilityState { - /// Placeholder so the stub is not a unit struct; T16 replaces it with - /// the pause and per-scope breaker fields. - _pending: (), -} - -impl ReliabilityState { - /// Record one failed outcome for `scope` and decide what happens next. - pub fn on_failure( - &mut self, - _scope: &SessionScope, - _class: ErrorClass, - _now: DateTime, - ) -> Action { - Action::Retry - } -} - #[cfg(test)] mod tests { use super::*; + use crate::acp::AcpError; use crate::config::DedupMode; use crate::queue::{EventQueue, QueuedEvent, MAX_RETRIES}; + use crate::scope::SessionScope; use chrono::TimeZone; use nostr::{EventBuilder, Keys, Kind}; use std::time::Instant; @@ -129,7 +131,6 @@ mod tests { // 1. Error classes on the real log lines. #[test] - #[ignore = "T16 fixture: fails until classify_at parses the Claude session-limit line"] fn a_session_limit_with_a_morning_reset_is_capacity_exhausted_until_that_time() { // 04:20 America/Los_Angeles on 2026-09-02 is PDT, so 11:20 UTC, later // the same day as the log line. @@ -143,7 +144,6 @@ mod tests { } #[test] - #[ignore = "T16 fixture: fails until classify_at parses the Claude session-limit line"] fn a_session_limit_whose_reset_already_passed_today_resolves_to_tomorrow() { // 00:40 America/Los_Angeles is 07:40 UTC. At 09:57 UTC that is already // past, so the next occurrence is 2026-09-03T07:40Z. @@ -157,7 +157,6 @@ mod tests { } #[test] - #[ignore = "T16 fixture: fails until classify_at recognises a bare internal error"] fn a_plain_internal_error_is_provider_internal() { assert_eq!( classify_at(&agent_error(PLAIN_INTERNAL), log_instant()), @@ -177,7 +176,6 @@ mod tests { // 2. The queue never hands a batch back to be discarded. #[test] - #[ignore = "T16 fixture: fails until retry exhaustion parks the batch instead of returning it"] fn retry_exhaustion_parks_the_batch_and_returns_nothing_to_discard() { let channel_id = Uuid::new_v4(); let mut queue = EventQueue::new(DedupMode::Queue); @@ -200,7 +198,6 @@ mod tests { // 3. Capacity exhaustion pauses the agent and spends no retries. #[test] - #[ignore = "T16 fixture: fails until ReliabilityState pauses on CapacityExhausted"] fn capacity_exhausted_pauses_until_the_reset_time() { let channel_id = Uuid::new_v4(); let scope = SessionScope::Conversation { channel_id }; @@ -219,7 +216,6 @@ mod tests { } #[test] - #[ignore = "T16 fixture: fails until an unparseable reset time pauses for 30 minutes"] fn capacity_exhausted_without_a_reset_time_pauses_thirty_minutes() { let channel_id = Uuid::new_v4(); let scope = SessionScope::Conversation { channel_id }; @@ -243,7 +239,6 @@ mod tests { // 4. Three consecutive provider errors open the breaker for that scope. #[test] - #[ignore = "T16 fixture: fails until three consecutive ProviderInternal failures open the breaker"] fn three_consecutive_provider_errors_open_the_breaker() { let channel_id = Uuid::new_v4(); let scope = SessionScope::Conversation { channel_id }; diff --git a/crates/buzz-acp/src/reliability/error_class.rs b/crates/buzz-acp/src/reliability/error_class.rs new file mode 100644 index 00000000000..5dc6793d09b --- /dev/null +++ b/crates/buzz-acp/src/reliability/error_class.rs @@ -0,0 +1,212 @@ +//! Provider error classification and reset-time parsing. +//! +//! `classify_at` is pure: it reads an [`AcpError`], never the clock, and takes +//! `now` as a parameter so a reset time parsed from +//! `resets 4:20am (America/Los_Angeles)` resolves to the same absolute instant +//! in a test as in production. +//! +//! Design: `docs/plans/2026-09-06-harness-reliability-design.md`, "Error classes". + +use std::str::FromStr; + +use chrono::{DateTime, Datelike, NaiveTime, TimeZone, Utc}; +use chrono_tz::Tz; + +use crate::acp::AcpError; + +use super::{ErrorClass, MAX_RAW_ERROR_CHARS}; + +/// Longest IANA zone name accepted from a provider error string. The longest +/// real zone (`America/Argentina/ComodRivadavia`) is 32 characters; the cap is +/// generous but finite so a hostile error line cannot allocate. +const MAX_ZONE_CHARS: usize = 64; + +/// Longest wall-clock token (`12:40am`) accepted after `resets`. +const MAX_TIME_TOKEN_CHARS: usize = 12; + +/// How far past `now` a parsed reset time is still searched for. Three days +/// covers a wall time that does not exist today (a spring-forward gap) and one +/// that has already passed today. +const MAX_RESET_SEARCH_DAYS: u32 = 3; + +/// Substrings that mean "the account is out of capacity for now". +/// +/// Matched case-insensitively against the truncated error text. +const CAPACITY_MARKERS: &[&str] = &[ + "session limit", + "rate limit", + "rate_limit", + "ratelimit", + "overloaded", + "quota", + "too many requests", + "usage limit", +]; + +/// Substrings that mean "the provider itself is broken right now". +const PROVIDER_INTERNAL_MARKERS: &[&str] = &[ + "internal error", + "internal server error", + "bad gateway", + "service unavailable", + "gateway timeout", +]; + +/// HTTP status codes that mean capacity exhaustion. +const CAPACITY_HTTP_CODES: &[&str] = &["429"]; + +/// HTTP status codes that mean the provider is broken (5xx). +const PROVIDER_INTERNAL_HTTP_CODES: &[&str] = &["500", "502", "503", "504", "529"]; + +/// Classify a provider error at a known instant. +/// +/// The order is deliberate: an auth failure never retries, a capacity marker +/// wins over the `Internal error:` prefix the Claude session-limit line carries, +/// and only what is left falls through to `ProviderInternal` / `Unknown`. +pub fn classify_at(err: &AcpError, now: DateTime) -> ErrorClass { + let raw = truncate_chars(&err.to_string(), MAX_RAW_ERROR_CHARS); + if is_auth_text(&raw) { + return ErrorClass::Auth; + } + let lower = raw.to_lowercase(); + if has_marker(&lower, CAPACITY_MARKERS) || has_http_code(&lower, CAPACITY_HTTP_CODES) { + return ErrorClass::CapacityExhausted { + resets_at: parse_reset_at(&raw, now), + }; + } + if has_marker(&lower, PROVIDER_INTERNAL_MARKERS) + || has_http_code(&lower, PROVIDER_INTERNAL_HTTP_CODES) + { + return ErrorClass::ProviderInternal; + } + ErrorClass::Unknown +} + +/// The harness's pre-existing auth detection, unchanged (see +/// `is_auth_error` in `lib.rs`, which delegates here). +pub fn is_auth_text(raw: &str) -> bool { + raw.contains("Re-authenticate") || raw.contains("API Error: 401") +} + +/// Truncate to at most `max` characters on a char boundary. +/// +/// Every provider string that reaches the ledger, the park file or a channel +/// notice passes through here first: the text is relay- and provider-sourced +/// and is never stored or forwarded at its original length. +pub fn truncate_chars(text: &str, max: usize) -> String { + if text.chars().count() <= max { + return text.to_string(); + } + text.chars().take(max).collect() +} + +fn has_marker(lower: &str, markers: &[&str]) -> bool { + markers.iter().any(|m| lower.contains(m)) +} + +/// Match a bare HTTP status code with a non-digit on both sides, so `429` in +/// `(429)` or `429 Too Many Requests` matches but `1429000` does not. +fn has_http_code(lower: &str, codes: &[&str]) -> bool { + let bytes = lower.as_bytes(); + codes.iter().any(|code| { + lower.match_indices(code).any(|(idx, _)| { + let before_ok = idx == 0 || !bytes[idx - 1].is_ascii_digit(); + let after = idx + code.len(); + let after_ok = after >= bytes.len() || !bytes[after].is_ascii_digit(); + before_ok && after_ok + }) + }) +} + +/// Parse `resets H:MM(am|pm) (IANA zone)` into the next occurrence of that wall +/// time in that zone strictly after `now`. +/// +/// Returns `None` when the marker, the time or the zone cannot be read — the +/// caller then falls back to the 30-minute default pause. +pub fn parse_reset_at(raw: &str, now: DateTime) -> Option> { + let lower = raw.to_lowercase(); + let marker = lower.find("resets")?; + // Slice the ORIGINAL text at the same byte offset: `to_lowercase` can + // change length for non-ASCII, so only index `raw` when the prefix is + // ASCII. Reset lines are ASCII; bail out otherwise rather than panic. + if !raw.is_char_boundary(marker) { + return None; + } + let rest = raw.get(marker + "resets".len()..)?.trim_start(); + + let time_token: String = rest + .chars() + .take_while(|c| c.is_ascii_digit() || *c == ':' || c.is_ascii_alphabetic()) + .take(MAX_TIME_TOKEN_CHARS) + .collect(); + let time = parse_wall_time(&time_token)?; + + let after_time = rest.get(time_token.len()..)?.trim_start(); + let zone_body = after_time.strip_prefix('(')?; + let end = zone_body.find(')')?; + if end > MAX_ZONE_CHARS { + return None; + } + let tz = Tz::from_str(zone_body.get(..end)?.trim()).ok()?; + + next_local_occurrence(tz, time, now) +} + +/// Parse `4:20am`, `12:40AM`, `16:05` into a wall time. +fn parse_wall_time(token: &str) -> Option { + let lower = token.to_ascii_lowercase(); + let (digits, meridiem) = if let Some(head) = lower.strip_suffix("am") { + (head, Some(false)) + } else if let Some(head) = lower.strip_suffix("pm") { + (head, Some(true)) + } else { + (lower.as_str(), None) + }; + let (h, m) = digits.split_once(':')?; + let hour: u32 = h.parse().ok()?; + let minute: u32 = m.parse().ok()?; + if minute > 59 { + return None; + } + let hour24 = match meridiem { + // 12am is 00:00, 12pm is 12:00; every other hour shifts by 12 for pm. + Some(true) if hour == 12 => 12, + Some(true) if hour < 12 => hour + 12, + Some(false) if hour == 12 => 0, + Some(false) if hour < 12 => hour, + None if hour < 24 => hour, + _ => return None, + }; + NaiveTime::from_hms_opt(hour24, minute, 0) +} + +/// The next instant at which the clock in `tz` reads `time`, strictly after +/// `now`. +/// +/// A wall time that is ambiguous (the repeated hour when DST ends) resolves to +/// the earlier of the two instants: the agent should probe as soon as the +/// provider could plausibly be back. A wall time that does not exist (the +/// skipped hour when DST starts) rolls to the next day. +fn next_local_occurrence(tz: Tz, time: NaiveTime, now: DateTime) -> Option> { + let mut date = now.with_timezone(&tz).date_naive(); + for _ in 0..=MAX_RESET_SEARCH_DAYS { + let naive = date.and_time(time); + let candidate = match tz.from_local_datetime(&naive) { + chrono::LocalResult::Single(dt) => Some(dt.with_timezone(&Utc)), + chrono::LocalResult::Ambiguous(earlier, _) => Some(earlier.with_timezone(&Utc)), + chrono::LocalResult::None => None, + }; + if let Some(candidate) = candidate { + if candidate > now { + return Some(candidate); + } + } + date = date.succ_opt()?; + // Guard against a pathological calendar walk; `succ_opt` already + // returns None at the end of the representable range. + if date.year() > now.year() + 1 { + return None; + } + } + None +} diff --git a/crates/buzz-acp/src/reliability/ledger.rs b/crates/buzz-acp/src/reliability/ledger.rs new file mode 100644 index 00000000000..ff0de8d600b --- /dev/null +++ b/crates/buzz-acp/src/reliability/ledger.rs @@ -0,0 +1,545 @@ +//! The harness-owned append-only ledger, `state/ledger.jsonl`. +//! +//! One JSON object per line, every line carrying `at`, `agent`, `kind` and — +//! where it applies — `batch_id`. The harness owns the file; observer frames +//! (T17) mirror it but are never the source of truth. +//! +//! Every string that reaches a record is provider- or relay-sourced, so each +//! one is capped at its DTO: see [`TurnStarted::new`] and friends, which are +//! the only way to build a record body. +//! +//! Design: `docs/plans/2026-09-06-harness-reliability-design.md`, "Ledger records". + +use std::collections::HashSet; +use std::io::{self, BufRead, Read, Write}; +use std::path::{Path, PathBuf}; + +use chrono::{DateTime, Duration, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use super::error_class::truncate_chars; +use super::state_dir; + +/// File name inside the state directory. +pub const LEDGER_FILE: &str = "ledger.jsonl"; + +/// Records older than this are dropped on start and every +/// [`TRUNCATE_INTERVAL_HOURS`]. +pub const RETENTION_DAYS: i64 = 30; + +/// How often the ledger is truncated while the harness runs. +pub const TRUNCATE_INTERVAL_HOURS: i64 = 6; + +/// Hard cap on the ledger file. Appending past it drops the oldest records +/// first; the count of dropped records is returned so the operator is told. +pub const MAX_LEDGER_BYTES: u64 = 10 * 1024 * 1024; + +/// Longest single line read back. A longer line is skipped, not buffered. +pub const MAX_LINE_BYTES: usize = 64 * 1024; + +/// Most event ids recorded on one `turn_started`. Matches the queue's own +/// per-batch event cap, so a full batch is recorded whole and nothing larger +/// can be. +pub const MAX_EVENT_IDS: usize = 50; + +/// Longest id string stored (a Nostr event id is 64 hex characters). +pub const MAX_ID_CHARS: usize = 64; + +/// Longest short text field (`scope`, `reason`, `class`). +pub const MAX_LABEL_CHARS: usize = 128; + +/// Longest raw provider error stored beside its class, so a misclassification +/// can be diagnosed without keeping an unbounded string. +pub const MAX_RAW_CHARS: usize = 512; + +/// One line of the ledger. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct LedgerRecord { + /// RFC 3339 UTC. + pub at: DateTime, + /// Agent public key. + pub agent: String, + /// The record itself, flattened so `kind` sits beside `at` and `agent`. + #[serde(flatten)] + pub body: LedgerBody, +} + +impl LedgerRecord { + /// The batch this record is about, when it is about one. + pub fn batch_id(&self) -> Option { + self.body.batch_id() + } + + /// The record kind, as written to the `kind` field. + pub fn kind(&self) -> &'static str { + self.body.kind() + } +} + +/// The record kinds. One struct per kind, tagged by `kind` on the wire. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum LedgerBody { + TurnStarted(TurnStarted), + TurnActivity(TurnActivity), + TurnFinished(TurnFinished), + BatchParked(BatchParked), + BatchReplayed(BatchReplayed), + BatchNeedsReview(BatchNeedsReview), + BatchDiscarded(BatchDiscarded), + AgentPaused(AgentPaused), + AgentResumed(AgentResumed), + BreakerOpened(BreakerOpened), + BreakerClosed(BreakerClosed), + RelayReconnected(RelayReconnected), +} + +impl LedgerBody { + /// The batch this record is about, when it is about one. + pub fn batch_id(&self) -> Option { + match self { + Self::TurnStarted(r) => Some(r.batch_id), + Self::TurnActivity(r) => Some(r.batch_id), + Self::TurnFinished(r) => Some(r.batch_id), + Self::BatchParked(r) => Some(r.batch_id), + Self::BatchReplayed(r) => Some(r.batch_id), + Self::BatchNeedsReview(r) => Some(r.batch_id), + Self::BatchDiscarded(r) => Some(r.batch_id), + Self::AgentPaused(_) + | Self::AgentResumed(_) + | Self::BreakerOpened(_) + | Self::BreakerClosed(_) + | Self::RelayReconnected(_) => None, + } + } + + /// The record kind, as written to the `kind` field. + pub fn kind(&self) -> &'static str { + match self { + Self::TurnStarted(_) => "turn_started", + Self::TurnActivity(_) => "turn_activity", + Self::TurnFinished(_) => "turn_finished", + Self::BatchParked(_) => "batch_parked", + Self::BatchReplayed(_) => "batch_replayed", + Self::BatchNeedsReview(_) => "batch_needs_review", + Self::BatchDiscarded(_) => "batch_discarded", + Self::AgentPaused(_) => "agent_paused", + Self::AgentResumed(_) => "agent_resumed", + Self::BreakerOpened(_) => "breaker_opened", + Self::BreakerClosed(_) => "breaker_closed", + Self::RelayReconnected(_) => "relay_reconnected", + } + } +} + +/// A turn was dispatched for a batch. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct TurnStarted { + pub batch_id: Uuid, + pub channel_id: Uuid, + pub scope: String, + pub event_ids: Vec, + pub attempt: u32, +} + +impl TurnStarted { + /// Build a capped record: at most [`MAX_EVENT_IDS`] ids, each at most + /// [`MAX_ID_CHARS`] characters, and a scope label at most + /// [`MAX_LABEL_CHARS`]. + pub fn new( + batch_id: Uuid, + channel_id: Uuid, + scope: &str, + event_ids: impl IntoIterator, + attempt: u32, + ) -> Self { + Self { + batch_id, + channel_id, + scope: truncate_chars(scope, MAX_LABEL_CHARS), + event_ids: event_ids + .into_iter() + .take(MAX_EVENT_IDS) + .map(|id| truncate_chars(&id, MAX_ID_CHARS)) + .collect(), + attempt, + } + } +} + +/// The first output or tool call seen for a batch. Written once per batch. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct TurnActivity { + pub batch_id: Uuid, + pub channel_id: Uuid, +} + +/// How a turn ended. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct TurnFinished { + pub batch_id: Uuid, + pub channel_id: Uuid, + pub outcome: TurnOutcome, +} + +/// The `outcome` field of a `turn_finished` record. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "result", rename_all = "snake_case")] +pub enum TurnOutcome { + Ok, + Error { class: String, raw: String }, + Timeout { kind: String, started: bool }, + Cancelled, + Exited, +} + +impl TurnOutcome { + /// An `error` outcome with both fields capped. + pub fn error(class: &str, raw: &str) -> Self { + Self::Error { + class: truncate_chars(class, MAX_LABEL_CHARS), + raw: truncate_chars(raw, MAX_RAW_CHARS), + } + } + + /// A `timeout` outcome with the kind capped. + pub fn timeout(kind: &str, started: bool) -> Self { + Self::Timeout { + kind: truncate_chars(kind, MAX_LABEL_CHARS), + started, + } + } +} + +/// A batch was moved to the park file. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct BatchParked { + pub batch_id: Uuid, + pub channel_id: Uuid, + pub reason: String, + pub started: bool, + pub events: usize, +} + +/// A parked batch was re-sent. Written **before** the prompt is sent. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct BatchReplayed { + pub batch_id: Uuid, + pub channel_id: Uuid, + /// The batch id of the new turn carrying the replayed events. + pub replay_of: Uuid, +} + +/// A batch will not run again on its own. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct BatchNeedsReview { + pub batch_id: Uuid, + pub channel_id: Uuid, + pub reason: String, +} + +/// An operator discarded a parked batch through a control frame. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct BatchDiscarded { + pub batch_id: Uuid, + pub channel_id: Uuid, + pub by: String, +} + +/// The agent stopped running turns until `until`. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct AgentPaused { + pub class: String, + pub until: DateTime, + pub waiting: usize, +} + +/// The agent started running turns again. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct AgentResumed {} + +/// A scope's breaker opened after consecutive provider failures. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct BreakerOpened { + pub scope: String, + pub consecutive: u32, +} + +/// A scope's breaker closed after a successful probe. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct BreakerClosed { + pub scope: String, +} + +/// The relay connection came back. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct RelayReconnected { + pub after_secs: u64, +} + +/// What a truncation pass did. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TruncateReport { + /// Records dropped because they were older than the retention window. + pub aged_out: usize, + /// Records dropped because the file was at its byte cap. + pub over_cap: usize, +} + +impl TruncateReport { + /// Whether anything was dropped. + pub fn is_empty(&self) -> bool { + self.aged_out == 0 && self.over_cap == 0 + } +} + +/// The append-only ledger file. +pub struct Ledger { + path: PathBuf, + agent: String, + len_bytes: u64, + next_truncate: DateTime, + write_failures: u64, +} + +impl Ledger { + /// Open (or create) the ledger in `dir` for `agent`, dropping records older + /// than the retention window. + pub fn open(dir: &Path, agent: &str, now: DateTime) -> io::Result { + state_dir::ensure_dir(dir)?; + let path = dir.join(LEDGER_FILE); + // Create it if absent so the mode is ours from the first byte. + drop(state_dir::open_append(&path)?); + let mut ledger = Self { + len_bytes: std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0), + path, + agent: truncate_chars(agent, MAX_ID_CHARS), + next_truncate: now + Duration::hours(TRUNCATE_INTERVAL_HOURS), + write_failures: 0, + }; + ledger.truncate(now)?; + Ok(ledger) + } + + /// Path of the ledger file. + pub fn path(&self) -> &Path { + &self.path + } + + /// Appends that failed. Never reset: a non-zero value means the ledger is + /// missing records and the operator has to be told. + pub fn write_failures(&self) -> u64 { + self.write_failures + } + + /// Append one record and flush it to disk. + /// + /// The write is fsynced before returning, so a record that this call + /// reports as written survives a crash. A failure is returned, never + /// swallowed; the caller keeps whatever the record described. + pub fn append(&mut self, at: DateTime, body: LedgerBody) -> io::Result<()> { + let record = LedgerRecord { + at, + agent: self.agent.clone(), + body, + }; + let mut line = serde_json::to_string(&record) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + line.push('\n'); + if line.len() > MAX_LINE_BYTES { + self.write_failures = self.write_failures.saturating_add(1); + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "ledger record of kind {} serialised to {} bytes, over the {MAX_LINE_BYTES}-byte line cap", + record.kind(), + line.len() + ), + )); + } + if self.len_bytes + line.len() as u64 > MAX_LEDGER_BYTES { + self.compact_for(line.len() as u64)?; + } + match self.append_line(line.as_bytes()) { + Ok(()) => { + self.len_bytes += line.len() as u64; + Ok(()) + } + Err(error) => { + self.write_failures = self.write_failures.saturating_add(1); + Err(error) + } + } + } + + fn append_line(&self, line: &[u8]) -> io::Result<()> { + let mut file = state_dir::open_append(&self.path)?; + file.write_all(line)?; + file.flush()?; + file.sync_all() + } + + /// Read every record back, bounded by [`MAX_LEDGER_BYTES`] of input and + /// [`MAX_LINE_BYTES`] per line. Malformed lines are counted and skipped, + /// never propagated as a parse failure for the whole file. + pub fn read_all(&self) -> io::Result> { + read_records(&self.path) + } + + /// Batch ids with a `batch_replayed` record and no later `turn_finished`. + /// + /// On start these are crashes mid-replay: the prompt may or may not have + /// reached the agent, so the batch moves to `needs_review` rather than + /// replaying a second time. + pub fn replays_without_finish(&self) -> io::Result> { + let records = self.read_all()?; + let mut replayed: Vec = Vec::new(); + let mut finished: HashSet = HashSet::new(); + for record in &records { + match &record.body { + LedgerBody::BatchReplayed(r) => replayed.push(r.batch_id), + LedgerBody::TurnFinished(r) => { + finished.insert(r.batch_id); + } + _ => {} + } + } + let mut seen: HashSet = HashSet::new(); + Ok(replayed + .into_iter() + .filter(|id| !finished.contains(id) && seen.insert(*id)) + .collect()) + } + + /// Truncate if the interval has elapsed. Returns what was dropped. + pub fn maybe_truncate(&mut self, now: DateTime) -> io::Result { + if now < self.next_truncate { + return Ok(TruncateReport::default()); + } + self.truncate(now) + } + + /// Drop records older than the retention window, then the oldest records + /// still over the byte cap. + pub fn truncate(&mut self, now: DateTime) -> io::Result { + self.next_truncate = now + Duration::hours(TRUNCATE_INTERVAL_HOURS); + let cutoff = now - Duration::days(RETENTION_DAYS); + let records = self.read_all()?; + let total = records.len(); + let kept: Vec = records.into_iter().filter(|r| r.at >= cutoff).collect(); + let aged_out = total - kept.len(); + let (kept, over_cap) = fit_to_cap(kept)?; + if aged_out == 0 && over_cap == 0 { + return Ok(TruncateReport::default()); + } + self.rewrite(&kept)?; + Ok(TruncateReport { aged_out, over_cap }) + } + + /// Make room for `incoming` bytes by dropping the oldest records. + fn compact_for(&mut self, incoming: u64) -> io::Result<()> { + let records = self.read_all()?; + let budget = MAX_LEDGER_BYTES.saturating_sub(incoming); + let (kept, dropped) = fit_to_budget(records, budget)?; + if dropped > 0 { + tracing::warn!( + dropped, + cap = MAX_LEDGER_BYTES, + "ledger reached its byte cap — dropped the oldest records" + ); + } + self.rewrite(&kept) + } + + fn rewrite(&mut self, records: &[LedgerRecord]) -> io::Result<()> { + let mut buffer = Vec::new(); + for record in records { + serde_json::to_writer(&mut buffer, record) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + buffer.push(b'\n'); + } + state_dir::write_atomic(&self.path, &buffer)?; + self.len_bytes = buffer.len() as u64; + Ok(()) + } +} + +fn serialized_len(record: &LedgerRecord) -> io::Result { + serde_json::to_string(record) + .map(|s| s.len() as u64 + 1) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) +} + +fn fit_to_cap(records: Vec) -> io::Result<(Vec, usize)> { + fit_to_budget(records, MAX_LEDGER_BYTES) +} + +/// Keep the newest records that fit in `budget` bytes, dropping oldest first. +fn fit_to_budget( + records: Vec, + budget: u64, +) -> io::Result<(Vec, usize)> { + let mut total: u64 = 0; + for record in &records { + total += serialized_len(record)?; + } + if total <= budget { + return Ok((records, 0)); + } + let mut dropped = 0usize; + let mut remaining = records; + while total > budget && !remaining.is_empty() { + let head = remaining.remove(0); + total = total.saturating_sub(serialized_len(&head)?); + dropped += 1; + } + Ok((remaining, dropped)) +} + +/// Read a JSONL ledger file with a hard byte cap on the input and a hard cap +/// per line. +fn read_records(path: &Path) -> io::Result> { + let file = match std::fs::File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error), + }; + let mut reader = io::BufReader::new(file.take(MAX_LEDGER_BYTES)); + let mut records = Vec::new(); + let mut skipped = 0usize; + let mut line = Vec::new(); + loop { + line.clear(); + let read = reader.read_until(b'\n', &mut line)?; + if read == 0 { + break; + } + if line.len() > MAX_LINE_BYTES { + skipped += 1; + continue; + } + let text = match std::str::from_utf8(&line) { + Ok(text) => text.trim(), + Err(_) => { + skipped += 1; + continue; + } + }; + if text.is_empty() { + continue; + } + match serde_json::from_str::(text) { + Ok(record) => records.push(record), + Err(_) => skipped += 1, + } + } + if skipped > 0 { + tracing::warn!( + skipped, + path = %path.display(), + "skipped unreadable ledger lines" + ); + } + Ok(records) +} diff --git a/crates/buzz-acp/src/reliability/notices.rs b/crates/buzz-acp/src/reliability/notices.rs new file mode 100644 index 00000000000..11609c81702 --- /dev/null +++ b/crates/buzz-acp/src/reliability/notices.rs @@ -0,0 +1,84 @@ +//! Channel notice templates. +//! +//! At most one notice per pause per channel, one per park, one per breaker +//! open. Every caller-supplied string is capped here, at the DTO, before it +//! reaches a relay post: the reason text comes from a provider error and the +//! agent name from configuration. +//! +//! Design: `docs/plans/2026-09-06-harness-reliability-design.md`, "Notices in the channel". + +use chrono::{DateTime, Utc}; + +use super::error_class::truncate_chars; + +/// Longest agent name shown in a notice. +pub const MAX_NAME_CHARS: usize = 64; + +/// Longest reason shown in a notice. +pub const MAX_REASON_CHARS: usize = 120; + +/// Hard cap on any notice this module produces. +pub const MAX_NOTICE_CHARS: usize = 600; + +/// The pause notice. `until` is rendered in UTC because the harness has no +/// authority over the reader's zone; the reset instant is unambiguous. +pub fn pause(agent_name: &str, until: DateTime, waiting: usize) -> String { + let name = truncate_chars(agent_name.trim(), MAX_NAME_CHARS); + let name = if name.is_empty() { "This agent" } else { &name }; + let plural = if waiting == 1 { + "message is" + } else { + "messages are" + }; + cap(format!( + "⏸️ {name} is paused until {} (provider capacity limit). {waiting} {plural} saved and \ +will be answered in order when I am back. To switch seats now run `cswap switch` and restart \ +the Claude agents.", + until.format("%H:%M UTC on %Y-%m-%d") + )) +} + +/// The park notice, posted once per parked batch. +pub fn parked(reason: &str) -> String { + cap(format!( + "⚠️ I could not process the last request after several attempts ({}). It is saved and \ +will be retried as soon as I am back. Nothing is lost.", + truncate_chars(reason.trim(), MAX_REASON_CHARS) + )) +} + +/// The needs-review notice, for a batch that had already started. +pub fn needs_review() -> String { + cap( + "⚠️ A request was interrupted after it had started, so it will not run again on its own. \ +Devin can retry or discard it from the Agents screen." + .to_string(), + ) +} + +/// The breaker notice, posted once per breaker open. +pub fn breaker(agent_name: &str) -> String { + let name = truncate_chars(agent_name.trim(), MAX_NAME_CHARS); + let name = if name.is_empty() { + "This agent's".to_string() + } else { + format!("{name}'s") + }; + cap(format!( + "⚠️ {name} provider is returning errors. I will try again every 10 minutes and answer in \ +order when it recovers." + )) +} + +/// Told to the operator when state files could not be written. +pub fn state_write_failures(ledger_failures: u64, park_failures: u64) -> String { + cap(format!( + "⚠️ I could not write my own reliability state ({ledger_failures} ledger and \ +{park_failures} park-file write failures). Saved requests are held in memory only until this \ +is fixed." + )) +} + +fn cap(text: String) -> String { + truncate_chars(&text, MAX_NOTICE_CHARS) +} diff --git a/crates/buzz-acp/src/reliability/park.rs b/crates/buzz-acp/src/reliability/park.rs new file mode 100644 index 00000000000..f488efd23ba --- /dev/null +++ b/crates/buzz-acp/src/reliability/park.rs @@ -0,0 +1,578 @@ +//! The park file, `state/parked.jsonl`. +//! +//! One line per parked batch, carrying the serialized events and their prompt +//! tags, keyed by `batch_id`. Nothing the harness parks is ever discarded: a +//! batch leaves this file only when it has been replayed and finished, or when +//! an operator discarded it through a control frame. +//! +//! Parked batches hold client messages. The file lives only in the agent state +//! directory at 0600 and is never sent anywhere except back to the same agent. +//! +//! Design: `docs/plans/2026-09-06-harness-reliability-design.md`, "Park file". + +use std::io::{self, BufRead, Read}; +use std::path::{Path, PathBuf}; + +use chrono::{DateTime, Duration, Utc}; +use serde::{Deserialize, Serialize}; +use uuid::Uuid; + +use crate::queue::{BatchEvent, FlushBatch}; +use crate::scope::SessionScope; + +use super::error_class::truncate_chars; +use super::state_dir; + +/// File name inside the state directory. +pub const PARK_FILE: &str = "parked.jsonl"; + +/// Hard cap on the park file. A park that would exceed it fails rather than +/// evicting a client message: the caller keeps the batch in memory, logs and +/// counts the failure, and the operator is told in the next notice. +pub const MAX_PARK_BYTES: u64 = 10 * 1024 * 1024; + +/// Most parked batches held for a single scope. Beyond it the oldest +/// replay-eligible batches for that scope move to `needs_review`, so one broken +/// conversation cannot fill the automatic-replay path. +pub const MAX_PARKED_PER_SCOPE: usize = 100; + +/// Most parked batches held in total. +pub const MAX_PARKED_TOTAL: usize = 1_000; + +/// Most events kept from one batch. Matches the queue's per-batch cap. +pub const MAX_PARKED_EVENTS: usize = 50; + +/// Longest single line read back. +pub const MAX_LINE_BYTES: usize = 512 * 1024; + +/// A replay-eligible batch older than this moves to `needs_review`: answering a +/// week-old message unprompted is worse than asking the operator. +pub const REPLAY_MAX_AGE_DAYS: i64 = 7; + +/// Longest message excerpt shown in the CLI or a notice. +pub const EXCERPT_CHARS: usize = 120; + +/// Longest prompt tag stored. +pub const MAX_TAG_CHARS: usize = 64; + +/// Longest reason string stored. +pub const MAX_REASON_CHARS: usize = 128; + +/// Why a batch was parked. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ParkReason { + /// The retry budget ran out. + RetriesExhausted, + /// The turn hit the hard wall-clock cap. + HardTimeout, + /// The provider rejected the credentials. + Auth, + /// A scope breaker stayed open for its whole six-hour budget. + BreakerExpired, +} + +impl ParkReason { + /// The `reason` string written to the ledger. + pub fn as_str(self) -> &'static str { + match self { + Self::RetriesExhausted => "retries_exhausted", + Self::HardTimeout => "hard_timeout", + Self::Auth => "auth", + Self::BreakerExpired => "breaker_expired", + } + } +} + +/// A session scope in a form that survives a round trip through JSON. +/// +/// `SessionScope` is the in-memory key; this is its serialized shape. The +/// thread root id is validated on the way back in: it reaches the harness from +/// the relay and only a 64-character lowercase hex string is a real one. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ScopeRef { + pub channel_id: Uuid, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub root_event_id: Option, +} + +impl ScopeRef { + /// Project a live scope into its serialized shape. + pub fn from_scope(scope: &SessionScope) -> Self { + match scope { + SessionScope::Conversation { channel_id } => Self { + channel_id: *channel_id, + root_event_id: None, + }, + SessionScope::Thread { + channel_id, + root_event_id, + } => Self { + channel_id: *channel_id, + root_event_id: Some(truncate_chars(root_event_id, 64)), + }, + } + } + + /// Rebuild the live scope. A root id that is not 64 lowercase hex + /// characters is not a thread root, so the batch belongs to the + /// conversation scope rather than to a scope no session will ever match. + pub fn to_scope(&self) -> SessionScope { + match &self.root_event_id { + Some(root) + if root.len() == 64 + && root + .chars() + .all(|c| c.is_ascii_digit() || ('a'..='f').contains(&c)) => + { + SessionScope::Thread { + channel_id: self.channel_id, + root_event_id: root.clone(), + } + } + _ => SessionScope::Conversation { + channel_id: self.channel_id, + }, + } + } +} + +/// One event inside a parked batch. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ParkedEvent { + /// The original signed event. Its text is never edited. + pub event: nostr::Event, + /// Which prompt rule matched it. + pub prompt_tag: String, + /// When the harness admitted it, in wall-clock terms so a replay can say + /// "first at HH:MM, last at HH:MM" after a restart. + pub received_at: DateTime, +} + +impl ParkedEvent { + /// A bounded excerpt of the event text, for the CLI and for notices. + pub fn excerpt(&self) -> String { + truncate_chars(self.event.content.trim(), EXCERPT_CHARS) + } +} + +/// One parked batch. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ParkedBatch { + pub batch_id: Uuid, + pub channel_id: Uuid, + pub scope: ScopeRef, + pub reason: ParkReason, + /// Whether the harness saw agent output or a tool call for this batch's + /// turn. A started batch never replays on its own. + pub started: bool, + /// Whether the batch waits for an operator rather than for a probe. + #[serde(default)] + pub needs_review: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub needs_review_reason: Option, + /// Set immediately before a replay prompt is sent. A batch that still has + /// this set at start-up crashed mid-replay. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub replayed_at: Option>, + /// Set by the operator's `replay_batch` control frame. It is the only way a + /// batch that had started becomes replay-eligible. + #[serde(default)] + pub forced: bool, + pub parked_at: DateTime, + pub events: Vec, +} + +impl ParkedBatch { + /// Park a live batch. Events past [`MAX_PARKED_EVENTS`] are refused rather + /// than silently trimmed — the queue never builds a larger batch, so a + /// larger one is a bug, not a message to drop. + pub fn from_batch( + batch: &FlushBatch, + reason: ParkReason, + started: bool, + now: DateTime, + ) -> Result { + if batch.events.len() > MAX_PARKED_EVENTS { + return Err(ParkError::TooManyEvents(batch.events.len())); + } + let events = batch + .events + .iter() + .map(|be| ParkedEvent { + event: be.event.clone(), + prompt_tag: truncate_chars(&be.prompt_tag, MAX_TAG_CHARS), + received_at: DateTime::from_timestamp(be.event.created_at.as_secs() as i64, 0) + .unwrap_or(now), + }) + .collect(); + Ok(Self { + batch_id: batch.batch_id, + channel_id: batch.channel_id, + scope: ScopeRef::from_scope(&batch.scope), + reason, + started, + needs_review: started, + needs_review_reason: started.then(|| "interrupted after it had started".to_string()), + replayed_at: None, + forced: false, + parked_at: now, + events, + }) + } + + /// Whether this batch may replay on its own after a successful probe. + pub fn replay_eligible(&self) -> bool { + (!self.started || self.forced) && !self.needs_review && self.replayed_at.is_none() + } + + /// Rebuild the batch events for a replay prompt. + pub fn to_batch_events(&self) -> Vec { + self.events + .iter() + .map(|pe| BatchEvent { + event: pe.event.clone(), + prompt_tag: pe.prompt_tag.clone(), + received_at: std::time::Instant::now(), + }) + .collect() + } + + /// The live scope this batch belongs to. + pub fn scope(&self) -> SessionScope { + self.scope.to_scope() + } +} + +/// Why a park could not be recorded. +#[derive(Debug, thiserror::Error)] +pub enum ParkError { + /// The park file is at its cap. Nothing was written; the caller keeps the + /// batch. + #[error("park file is full ({0} batches, {1} bytes) — the batch was NOT parked")] + Full(usize, u64), + /// A batch larger than the queue can build. + #[error("batch carries {0} events, over the park file's per-batch cap")] + TooManyEvents(usize), + /// The write itself failed. + #[error("park file write failed: {0}")] + Io(#[from] io::Error), +} + +/// What a start-up reconciliation pass changed. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct ReconcileReport { + /// Batches that had been sent for replay and never finished. + pub crashed_mid_replay: usize, + /// Replay-eligible batches older than [`REPLAY_MAX_AGE_DAYS`]. + pub aged_out: usize, + /// Replay-eligible batches over a scope's cap. + pub over_scope_cap: usize, +} + +impl ReconcileReport { + /// Whether the pass changed anything. + pub fn is_empty(&self) -> bool { + self.crashed_mid_replay == 0 && self.aged_out == 0 && self.over_scope_cap == 0 + } +} + +/// The park file and its in-memory image. +pub struct ParkFile { + path: PathBuf, + batches: Vec, + write_failures: u64, +} + +impl ParkFile { + /// Open (or create) the park file in `dir`. + pub fn open(dir: &Path) -> io::Result { + state_dir::ensure_dir(dir)?; + let path = dir.join(PARK_FILE); + drop(state_dir::open_append(&path)?); + let batches = read_batches(&path)?; + Ok(Self { + path, + batches, + write_failures: 0, + }) + } + + /// Path of the park file. + pub fn path(&self) -> &Path { + &self.path + } + + /// Writes that failed. Never reset. + pub fn write_failures(&self) -> u64 { + self.write_failures + } + + /// Every parked batch, oldest first. + pub fn batches(&self) -> &[ParkedBatch] { + &self.batches + } + + /// Whether `batch_id` is already parked. + pub fn contains(&self, batch_id: Uuid) -> bool { + self.batches.iter().any(|b| b.batch_id == batch_id) + } + + /// Park one batch, writing the file before the caller drops the batch. + /// + /// Returns `Err` when nothing was written, so a caller that keeps its own + /// copy knows the batch is still its responsibility. + pub fn park(&mut self, batch: ParkedBatch) -> Result<(), ParkError> { + if self.batches.iter().any(|b| b.batch_id == batch.batch_id) { + // Idempotent: a retried park of the same batch is not a second copy. + return Ok(()); + } + if self.batches.len() >= MAX_PARKED_TOTAL { + return Err(ParkError::Full(self.batches.len(), self.byte_size())); + } + let mut next = self.batches.clone(); + next.push(batch); + let bytes = serialize(&next)?; + if bytes.len() as u64 > MAX_PARK_BYTES { + return Err(ParkError::Full(next.len(), bytes.len() as u64)); + } + self.commit(next, bytes)?; + self.enforce_scope_cap()?; + Ok(()) + } + + /// Remove a batch entirely. Used by `discard_batch` and once a replayed + /// batch has finished. + pub fn remove(&mut self, batch_id: Uuid) -> Result, ParkError> { + let Some(index) = self.batches.iter().position(|b| b.batch_id == batch_id) else { + return Ok(None); + }; + let mut next = self.batches.clone(); + let removed = next.remove(index); + let bytes = serialize(&next)?; + self.commit(next, bytes)?; + Ok(Some(removed)) + } + + /// Stamp `replayed_at` on a batch. Written before the replay prompt is sent + /// so a crash between the two is visible at the next start. + pub fn mark_replayed(&mut self, batch_id: Uuid, at: DateTime) -> Result<(), ParkError> { + self.mutate(batch_id, |batch| batch.replayed_at = Some(at)) + } + + /// Clear a replay stamp after the replay turn failed, so the batch is + /// eligible again on the next successful probe. + pub fn unmark_replayed(&mut self, batch_id: Uuid) -> Result<(), ParkError> { + self.mutate(batch_id, |batch| batch.replayed_at = None) + } + + /// Move a batch to the review list. + pub fn mark_needs_review(&mut self, batch_id: Uuid, reason: &str) -> Result<(), ParkError> { + let reason = truncate_chars(reason, MAX_REASON_CHARS); + self.mutate(batch_id, move |batch| { + batch.needs_review = true; + batch.needs_review_reason = Some(reason.clone()); + batch.replayed_at = None; + }) + } + + /// Operator override: make a batch replay-eligible whatever its `started` + /// flag, and take it off the review list. + pub fn clear_review(&mut self, batch_id: Uuid) -> Result<(), ParkError> { + self.mutate(batch_id, |batch| { + batch.needs_review = false; + batch.needs_review_reason = None; + batch.replayed_at = None; + batch.forced = true; + }) + } + + /// Batches for `scope` that may replay on their own, oldest first. + pub fn replay_candidates(&self, scope: &SessionScope) -> Vec<&ParkedBatch> { + let mut candidates: Vec<&ParkedBatch> = self + .batches + .iter() + .filter(|b| b.replay_eligible() && &b.scope() == scope) + .collect(); + candidates.sort_by_key(|b| b.parked_at); + candidates + } + + /// One parked batch by id. + pub fn get(&self, batch_id: Uuid) -> Option<&ParkedBatch> { + self.batches.iter().find(|b| b.batch_id == batch_id) + } + + /// Start-up reconciliation. + /// + /// `crashed` is the batch-id list the ledger reports as replayed with no + /// `turn_finished`. Those, and replay-eligible batches older than + /// [`REPLAY_MAX_AGE_DAYS`], move to the review list; nothing is removed. + pub fn reconcile_on_start( + &mut self, + crashed: &[Uuid], + now: DateTime, + ) -> Result { + let cutoff = now - Duration::days(REPLAY_MAX_AGE_DAYS); + let mut report = ReconcileReport::default(); + let mut next = self.batches.clone(); + for batch in next.iter_mut() { + if !batch.needs_review + && (crashed.contains(&batch.batch_id) || batch.replayed_at.is_some()) + { + batch.needs_review = true; + batch.needs_review_reason = + Some("replay was sent but the turn never finished".to_string()); + batch.replayed_at = None; + report.crashed_mid_replay += 1; + continue; + } + if batch.replay_eligible() && batch.parked_at < cutoff { + batch.needs_review = true; + batch.needs_review_reason = + Some(format!("waited more than {REPLAY_MAX_AGE_DAYS} days")); + report.aged_out += 1; + } + } + if !report.is_empty() { + let bytes = serialize(&next)?; + self.commit(next, bytes)?; + } + report.over_scope_cap = self.enforce_scope_cap()?; + Ok(report) + } + + /// Demote the oldest replay-eligible batches of any scope over + /// [`MAX_PARKED_PER_SCOPE`] to the review list. Returns how many moved. + fn enforce_scope_cap(&mut self) -> Result { + use std::collections::HashMap; + + let mut per_scope: HashMap> = HashMap::new(); + for (index, batch) in self.batches.iter().enumerate() { + if batch.replay_eligible() { + per_scope.entry(batch.scope()).or_default().push(index); + } + } + let mut demote: Vec = Vec::new(); + for indices in per_scope.values() { + if indices.len() > MAX_PARKED_PER_SCOPE { + demote.extend(&indices[..indices.len() - MAX_PARKED_PER_SCOPE]); + } + } + if demote.is_empty() { + return Ok(0); + } + let mut next = self.batches.clone(); + for index in &demote { + let batch = &mut next[*index]; + batch.needs_review = true; + batch.needs_review_reason = Some(format!( + "more than {MAX_PARKED_PER_SCOPE} batches were waiting for this conversation" + )); + } + let bytes = serialize(&next)?; + self.commit(next, bytes)?; + Ok(demote.len()) + } + + fn mutate( + &mut self, + batch_id: Uuid, + apply: impl Fn(&mut ParkedBatch), + ) -> Result<(), ParkError> { + let Some(index) = self.batches.iter().position(|b| b.batch_id == batch_id) else { + return Ok(()); + }; + let mut next = self.batches.clone(); + apply(&mut next[index]); + let bytes = serialize(&next)?; + self.commit(next, bytes) + } + + /// Write the new image atomically, then adopt it. The in-memory image only + /// changes once the bytes are on disk, so a failed write leaves the caller + /// looking at exactly what the file holds. + fn commit(&mut self, next: Vec, bytes: Vec) -> Result<(), ParkError> { + if let Err(error) = state_dir::write_atomic(&self.path, &bytes) { + self.write_failures = self.write_failures.saturating_add(1); + return Err(ParkError::Io(error)); + } + self.batches = next; + Ok(()) + } + + fn byte_size(&self) -> u64 { + serialize(&self.batches) + .map(|b| b.len() as u64) + .unwrap_or(0) + } +} + +fn serialize(batches: &[ParkedBatch]) -> Result, ParkError> { + let mut buffer = Vec::new(); + for batch in batches { + serde_json::to_writer(&mut buffer, batch) + .map_err(|e| ParkError::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; + buffer.push(b'\n'); + } + Ok(buffer) +} + +/// Read the park file with a hard byte cap on the input and a hard cap per +/// line. Unreadable lines are counted and skipped. +fn read_batches(path: &Path) -> io::Result> { + let file = match std::fs::File::open(path) { + Ok(file) => file, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(error), + }; + let mut reader = io::BufReader::new(file.take(MAX_PARK_BYTES)); + let mut batches = Vec::new(); + let mut skipped = 0usize; + let mut line = Vec::new(); + loop { + if batches.len() >= MAX_PARKED_TOTAL { + tracing::warn!( + cap = MAX_PARKED_TOTAL, + path = %path.display(), + "park file holds more batches than the cap — ignoring the rest of the file" + ); + break; + } + line.clear(); + let read = reader.read_until(b'\n', &mut line)?; + if read == 0 { + break; + } + if line.len() > MAX_LINE_BYTES { + skipped += 1; + continue; + } + let text = match std::str::from_utf8(&line) { + Ok(text) => text.trim(), + Err(_) => { + skipped += 1; + continue; + } + }; + if text.is_empty() { + continue; + } + match serde_json::from_str::(text) { + Ok(mut batch) => { + batch.events.truncate(MAX_PARKED_EVENTS); + batches.push(batch); + } + Err(_) => skipped += 1, + } + } + if skipped > 0 { + tracing::warn!( + skipped, + path = %path.display(), + "skipped unreadable park file lines" + ); + } + batches.sort_by_key(|b| b.parked_at); + Ok(batches) +} diff --git a/crates/buzz-acp/src/reliability/runtime.rs b/crates/buzz-acp/src/reliability/runtime.rs new file mode 100644 index 00000000000..48042e02fd3 --- /dev/null +++ b/crates/buzz-acp/src/reliability/runtime.rs @@ -0,0 +1,327 @@ +//! Glue that owns the state directory, the ledger, the park file and the +//! per-agent [`ReliabilityState`], and orders the writes so every prefix of +//! them is a consistent state. +//! +//! Ordering rules this module enforces: +//! +//! - A batch is written to the park file **before** the harness drops its +//! in-memory copy. A failed park returns an error and the caller keeps the +//! batch. +//! - `batch_replayed` is written to the ledger **before** the replay prompt is +//! staged for sending. A crash between the two is visible at the next start +//! and moves the batch to the review list rather than replaying it twice. + +use std::collections::HashMap; +use std::io; +use std::path::{Path, PathBuf}; + +use chrono::{DateTime, Utc}; +use uuid::Uuid; + +use crate::queue::{BatchEvent, FlushBatch}; +use crate::scope::SessionScope; + +use super::ledger::{self, Ledger, LedgerBody, TruncateReport}; +use super::park::{ParkError, ParkFile, ParkReason, ParkedBatch, ReconcileReport}; +use super::state::ReliabilityState; +use super::state_dir; + +/// One replay's worth of parked events for a single scope. +#[derive(Debug, Clone)] +pub struct ReplayPlan { + /// The parked batches being replayed, oldest first. + pub batch_ids: Vec, + /// Their events, in the same order, ready for the prompt. + pub events: Vec, + /// The scope the events belong to. + pub scope: SessionScope, + /// Channel the scope belongs to. + pub channel_id: Uuid, +} + +/// The harness's reliability state for one agent. +pub struct ReliabilityRuntime { + dir: PathBuf, + agent: String, + ledger: Ledger, + park: ParkFile, + state: ReliabilityState, + /// Batches whose replay prompt has been staged but whose turn has not + /// finished, keyed by the scope carrying them. Bounded by the number of + /// scopes with a turn in flight, which the pool already caps. + in_flight_replays: HashMap>, +} + +impl ReliabilityRuntime { + /// Open the state directory for `pubkey_hex` and load its ledger and park + /// file. + pub fn open(pubkey_hex: &str, now: DateTime) -> io::Result { + let dir = state_dir::resolve_state_dir(pubkey_hex)?; + Self::open_in(&dir, pubkey_hex, now) + } + + /// Open the state in an explicit directory. Used by tests and by any caller + /// that resolved the directory itself. + pub fn open_in(dir: &Path, pubkey_hex: &str, now: DateTime) -> io::Result { + let ledger = Ledger::open(dir, pubkey_hex, now)?; + let park = ParkFile::open(dir)?; + Ok(Self { + dir: dir.to_path_buf(), + agent: pubkey_hex.to_string(), + ledger, + park, + state: ReliabilityState::default(), + in_flight_replays: HashMap::new(), + }) + } + + /// The state directory. + pub fn dir(&self) -> &Path { + &self.dir + } + + /// The agent public key this state belongs to. + pub fn agent(&self) -> &str { + &self.agent + } + + /// The pause and breaker state machine. + pub fn state(&mut self) -> &mut ReliabilityState { + &mut self.state + } + + /// Read-only view of the state machine. + pub fn state_ref(&self) -> &ReliabilityState { + &self.state + } + + /// Read-only view of the park file. + pub fn park(&self) -> &ParkFile { + &self.park + } + + /// Ledger writes that failed, plus park-file writes that failed. A non-zero + /// total means the durable record is incomplete and the operator has to be + /// told. + pub fn write_failures(&self) -> (u64, u64) { + (self.ledger.write_failures(), self.park.write_failures()) + } + + /// Append a ledger record. + /// + /// A failure is logged and counted, never swallowed silently: the count is + /// readable through [`write_failures`](Self::write_failures) and surfaces in + /// the next notice. The return value says whether the record landed. + pub fn record(&mut self, now: DateTime, body: LedgerBody) -> bool { + let kind = body.kind(); + match self.ledger.append(now, body) { + Ok(()) => true, + Err(error) => { + tracing::error!( + kind, + agent = %self.agent, + error = %error, + "ledger append failed — the durable record for this event is missing" + ); + false + } + } + } + + /// Park a batch: the park file is written and fsynced first, then the + /// `batch_parked` ledger record. + /// + /// On failure nothing was written and the caller still owns the batch. + pub fn park_batch( + &mut self, + batch: &FlushBatch, + reason: ParkReason, + started: bool, + now: DateTime, + ) -> Result<(), ParkError> { + let parked = ParkedBatch::from_batch(batch, reason, started, now)?; + let events = parked.events.len(); + self.park.park(parked)?; + self.record( + now, + LedgerBody::BatchParked(ledger::BatchParked { + batch_id: batch.batch_id, + channel_id: batch.channel_id, + reason: reason.as_str().to_string(), + started, + events, + }), + ); + if started { + self.record( + now, + LedgerBody::BatchNeedsReview(ledger::BatchNeedsReview { + batch_id: batch.batch_id, + channel_id: batch.channel_id, + reason: "interrupted after it had started".to_string(), + }), + ); + } + Ok(()) + } + + /// The replay-eligible parked batches for `scope`, oldest first, merged + /// into one plan. `None` when the scope has nothing to replay. + /// + /// This only reads. Commit the plan with + /// [`commit_replay`](Self::commit_replay) once the caller is ready to stage + /// the prompt. + pub fn plan_replay(&self, scope: &SessionScope) -> Option { + let candidates = self.park.replay_candidates(scope); + if candidates.is_empty() { + return None; + } + let mut batch_ids = Vec::with_capacity(candidates.len()); + let mut events = Vec::new(); + for batch in candidates { + batch_ids.push(batch.batch_id); + events.extend(batch.to_batch_events()); + } + Some(ReplayPlan { + batch_ids, + events, + scope: scope.clone(), + channel_id: scope.channel_id(), + }) + } + + /// Write `batch_replayed` for every batch in the plan and stamp the park + /// file, **before** the prompt is sent. + /// + /// `new_batch_id` identifies the turn that will carry the replayed events. + /// Returns an error if the park file could not be stamped; the caller then + /// does not send, so no batch is replayed without a durable record. + pub fn commit_replay( + &mut self, + plan: &ReplayPlan, + new_batch_id: Uuid, + now: DateTime, + ) -> Result<(), ParkError> { + for batch_id in &plan.batch_ids { + self.park.mark_replayed(*batch_id, now)?; + } + for batch_id in &plan.batch_ids { + self.record( + now, + LedgerBody::BatchReplayed(ledger::BatchReplayed { + batch_id: *batch_id, + channel_id: plan.channel_id, + replay_of: new_batch_id, + }), + ); + } + Ok(()) + } + + /// Note that `plan`'s batches are riding on an in-flight turn for its scope. + pub fn mark_replay_in_flight(&mut self, plan: &ReplayPlan) { + self.in_flight_replays + .insert(plan.scope.clone(), plan.batch_ids.clone()); + } + + /// A turn for `scope` finished successfully: any batches it was replaying + /// leave the park file for good. Returns the batch ids released. + pub fn finish_replay(&mut self, scope: &SessionScope) -> Result, ParkError> { + let Some(batch_ids) = self.in_flight_replays.remove(scope) else { + return Ok(Vec::new()); + }; + let mut released = Vec::new(); + for batch_id in batch_ids { + if self.park.remove(batch_id)?.is_some() { + released.push(batch_id); + } + } + Ok(released) + } + + /// A turn for `scope` failed: its replayed batches stay parked and go back + /// to being eligible, so the next successful probe replays them again. + /// At-least-once delivery, never at-most-once. + pub fn abandon_replay(&mut self, scope: &SessionScope) { + let Some(batch_ids) = self.in_flight_replays.remove(scope) else { + return; + }; + for batch_id in batch_ids { + if let Err(error) = self.park.unmark_replayed(batch_id) { + tracing::error!( + %batch_id, + error = %error, + "could not clear the replay stamp — the batch moves to needs_review at the next start" + ); + } + } + } + + /// Operator control frame `discard_batch`. + pub fn discard( + &mut self, + batch_id: Uuid, + by: &str, + now: DateTime, + ) -> Result { + let Some(removed) = self.park.remove(batch_id)? else { + return Ok(false); + }; + self.record( + now, + LedgerBody::BatchDiscarded(ledger::BatchDiscarded { + batch_id, + channel_id: removed.channel_id, + by: super::error_class::truncate_chars(by, ledger::MAX_LABEL_CHARS), + }), + ); + Ok(true) + } + + /// Operator control frame `replay_batch`: make one parked batch eligible + /// again whatever its `started` flag. + pub fn force_replay(&mut self, batch_id: Uuid) -> Result { + if self.park.get(batch_id).is_none() { + return Ok(false); + } + self.park.clear_review(batch_id)?; + Ok(true) + } + + /// Start-up reconciliation: a batch with `batch_replayed` and no + /// `turn_finished` moves to the review list, never to a second automatic + /// replay. + pub fn reconcile_on_start(&mut self, now: DateTime) -> Result { + let crashed = self.ledger.replays_without_finish().unwrap_or_else(|error| { + tracing::error!(error = %error, "could not read the ledger for start-up reconciliation"); + Vec::new() + }); + let report = self.park.reconcile_on_start(&crashed, now)?; + for batch_id in &crashed { + if let Some(batch) = self.park.get(*batch_id) { + let channel_id = batch.channel_id; + self.record( + now, + LedgerBody::BatchNeedsReview(ledger::BatchNeedsReview { + batch_id: *batch_id, + channel_id, + reason: "replay was sent but the turn never finished".to_string(), + }), + ); + } + } + Ok(report) + } + + /// Periodic maintenance: truncate the ledger to its retention window every + /// six hours. + pub fn maintain(&mut self, now: DateTime) -> TruncateReport { + match self.ledger.maybe_truncate(now) { + Ok(report) => report, + Err(error) => { + tracing::error!(error = %error, "ledger truncation failed"); + TruncateReport::default() + } + } + } +} diff --git a/crates/buzz-acp/src/reliability/state.rs b/crates/buzz-acp/src/reliability/state.rs new file mode 100644 index 00000000000..0ba934e0c10 --- /dev/null +++ b/crates/buzz-acp/src/reliability/state.rs @@ -0,0 +1,328 @@ +//! Per-agent pause state and per-scope breakers. +//! +//! Design: `docs/plans/2026-09-06-harness-reliability-design.md`, "State machine". +//! +//! ```text +//! Active --CapacityExhausted--> Paused{until} +//! Paused --timer--> Probing (first queued batch is the probe) +//! Probing --Ok--> Active (then replay) +//! Probing --CapacityExhausted--> Paused{new until} (notice only if until moved > 15 min) +//! Active --3 consecutive ProviderInternal/Unknown on one scope--> BreakerOpen{scope} +//! BreakerOpen --every 10 min--> probe one batch; Ok closes it; open at most 6 h then Park +//! ``` + +use std::collections::{HashMap, HashSet}; + +use chrono::{DateTime, Duration, Utc}; +use uuid::Uuid; + +use crate::scope::SessionScope; + +use super::{Action, ErrorClass}; + +/// A pause never lasts longer than this, however far in the future the provider +/// says its reset is. +pub const MAX_PAUSE_HOURS: i64 = 6; + +/// Pause length when the provider named no reset time, or named one we could +/// not parse. +pub const DEFAULT_PAUSE_MINUTES: i64 = 30; + +/// Consecutive `ProviderInternal` / `Unknown` failures on one scope that open +/// its breaker. +pub const BREAKER_THRESHOLD: u32 = 3; + +/// How often an open breaker lets one batch through as a probe. +pub const BREAKER_PROBE_MINUTES: i64 = 10; + +/// A breaker never stays open longer than this; the batch is parked instead. +pub const BREAKER_MAX_OPEN_HOURS: i64 = 6; + +/// A re-pause re-notifies a channel only when the new reset time moved by more +/// than this. +pub const PAUSE_RENOTIFY_MINUTES: i64 = 15; + +/// Whether the agent may run a turn right now. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PauseGate { + /// No pause is in force. + Open, + /// Paused until this instant; nothing runs. + Held { until: DateTime }, + /// The pause expired: exactly one batch may run as the probe. + Probe, +} + +/// Whether an open breaker lets this scope run right now. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BreakerGate { + /// No breaker is open for the scope. + Closed, + /// Open and inside the probe interval; nothing runs for this scope. + Held { next_probe: DateTime }, + /// Open and due: exactly one batch may run as the probe. + Probe, +} + +/// What a failure during a breaker probe means for the batch. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum BreakerVerdict { + /// Still inside the six-hour window: try again after the probe interval. + Reschedule { next_probe: DateTime }, + /// The breaker has been open for six hours; park the batch and close it. + Park, +} + +#[derive(Debug, Clone)] +struct Pause { + until: DateTime, + /// The `until` the channels in `notified_channels` were told about. + notified_until: DateTime, + notified_channels: HashSet, + /// Set once the pause expires and a probe has been handed out, so only one + /// batch probes per expiry. + probe_issued: bool, +} + +#[derive(Debug, Clone)] +struct Breaker { + opened_at: DateTime, + next_probe: DateTime, + probe_issued: bool, + /// Consecutive failures that opened it, for the ledger record. + consecutive: u32, +} + +/// Per-agent reliability state: one pause for the whole agent (a capacity +/// limit belongs to the account) and one breaker per scope. +#[derive(Debug, Default)] +pub struct ReliabilityState { + pause: Option, + breakers: HashMap, + consecutive: HashMap, +} + +impl ReliabilityState { + /// Record one failed outcome for `scope` and decide what happens next. + /// + /// Retry counts are the queue's business; this returns only the class of + /// action. `Pause` and `OpenBreaker` both mean "do not spend a retry". + pub fn on_failure( + &mut self, + scope: &SessionScope, + class: ErrorClass, + now: DateTime, + ) -> Action { + match class { + // A re-login fixes it; a retry does not. + ErrorClass::Auth => Action::Park, + ErrorClass::CapacityExhausted { resets_at } => { + let until = clamp_pause(resets_at, now); + self.set_pause(until); + Action::Pause { until } + } + ErrorClass::ProviderInternal | ErrorClass::Unknown => { + self.on_provider_failure(scope, now) + } + } + } + + fn on_provider_failure(&mut self, scope: &SessionScope, now: DateTime) -> Action { + if let Some(breaker) = self.breakers.get_mut(scope) { + if now - breaker.opened_at >= Duration::hours(BREAKER_MAX_OPEN_HOURS) { + self.breakers.remove(scope); + self.consecutive.remove(scope); + return Action::Park; + } + breaker.next_probe = now + Duration::minutes(BREAKER_PROBE_MINUTES); + breaker.probe_issued = false; + breaker.consecutive = breaker.consecutive.saturating_add(1); + return Action::OpenBreaker; + } + + let count = self.consecutive.entry(scope.clone()).or_insert(0); + *count = count.saturating_add(1); + if *count < BREAKER_THRESHOLD { + return Action::Retry; + } + let consecutive = *count; + self.consecutive.remove(scope); + self.breakers.insert( + scope.clone(), + Breaker { + opened_at: now, + next_probe: now + Duration::minutes(BREAKER_PROBE_MINUTES), + probe_issued: false, + consecutive, + }, + ); + Action::OpenBreaker + } + + /// Record a successful live turn for `scope`: the pause lifts, the scope's + /// breaker closes, and its consecutive-failure count resets. + /// + /// Returns `(pause_lifted, breaker_closed)` so the caller can write the + /// `agent_resumed` and `breaker_closed` ledger records. + pub fn on_success(&mut self, scope: &SessionScope) -> (bool, bool) { + let pause_lifted = self.pause.take().is_some(); + let breaker_closed = self.breakers.remove(scope).is_some(); + self.consecutive.remove(scope); + (pause_lifted, breaker_closed) + } + + /// Whether the agent may run a turn now, and if not, until when. + /// + /// Calling this hands out at most one probe per pause expiry: the first + /// call after `until` returns [`PauseGate::Probe`], later calls return + /// [`PauseGate::Held`] again until the probe's outcome resolves the pause. + pub fn pause_gate(&mut self, now: DateTime) -> PauseGate { + let Some(pause) = self.pause.as_mut() else { + return PauseGate::Open; + }; + if now < pause.until { + return PauseGate::Held { until: pause.until }; + } + if pause.probe_issued { + return PauseGate::Held { until: pause.until }; + } + pause.probe_issued = true; + PauseGate::Probe + } + + /// Read the pause without handing out a probe. + pub fn paused_until(&self) -> Option> { + self.pause.as_ref().map(|p| p.until) + } + + /// Whether `scope` may run a turn now, and if not, until when. Hands out at + /// most one probe per probe interval, like [`pause_gate`](Self::pause_gate). + pub fn breaker_gate(&mut self, scope: &SessionScope, now: DateTime) -> BreakerGate { + let Some(breaker) = self.breakers.get_mut(scope) else { + return BreakerGate::Closed; + }; + if now < breaker.next_probe { + return BreakerGate::Held { + next_probe: breaker.next_probe, + }; + } + if breaker.probe_issued { + return BreakerGate::Held { + next_probe: breaker.next_probe, + }; + } + breaker.probe_issued = true; + BreakerGate::Probe + } + + /// A probe on an open breaker failed: reschedule, or park once the breaker + /// has been open for [`BREAKER_MAX_OPEN_HOURS`]. + pub fn on_breaker_probe_failure( + &mut self, + scope: &SessionScope, + now: DateTime, + ) -> BreakerVerdict { + let Some(breaker) = self.breakers.get_mut(scope) else { + return BreakerVerdict::Reschedule { + next_probe: now + Duration::minutes(BREAKER_PROBE_MINUTES), + }; + }; + if now - breaker.opened_at >= Duration::hours(BREAKER_MAX_OPEN_HOURS) { + self.breakers.remove(scope); + return BreakerVerdict::Park; + } + breaker.next_probe = now + Duration::minutes(BREAKER_PROBE_MINUTES); + breaker.probe_issued = false; + BreakerVerdict::Reschedule { + next_probe: breaker.next_probe, + } + } + + /// How long a breaker for `scope` has been open, if one is. + pub fn breaker_opened_at(&self, scope: &SessionScope) -> Option> { + self.breakers.get(scope).map(|b| b.opened_at) + } + + /// Consecutive failures recorded against the scope's open breaker. + pub fn breaker_consecutive(&self, scope: &SessionScope) -> Option { + self.breakers.get(scope).map(|b| b.consecutive) + } + + /// Whether `channel_id` still needs the pause notice. + /// + /// At most one notice per pause per channel. A re-pause re-notifies only + /// when the reset time moved by more than [`PAUSE_RENOTIFY_MINUTES`]. + pub fn claim_pause_notice(&mut self, channel_id: Uuid) -> bool { + match self.pause.as_mut() { + Some(pause) => pause.notified_channels.insert(channel_id), + None => false, + } + } + + /// Operator control frame `resume_now`: leave Paused and BreakerOpen and + /// probe immediately. Returns `true` when something was actually lifted. + pub fn resume_now(&mut self) -> bool { + let had_pause = self.pause.take().is_some(); + let had_breakers = !self.breakers.is_empty(); + self.breakers.clear(); + self.consecutive.clear(); + had_pause || had_breakers + } + + /// Operator control frame `keep_paused { until }`: extend a pause. The + /// pause is only ever extended, never shortened, and never past the + /// six-hour cap measured from `now`. + pub fn keep_paused(&mut self, until: DateTime, now: DateTime) -> DateTime { + let capped = clamp_pause(Some(until), now); + let target = match self.pause.as_ref() { + Some(existing) if existing.until > capped => existing.until, + _ => capped, + }; + self.set_pause(target); + target + } + + fn set_pause(&mut self, until: DateTime) { + match self.pause.as_mut() { + Some(pause) => { + let moved = (until - pause.notified_until).num_minutes().abs(); + pause.until = until; + pause.probe_issued = false; + if moved > PAUSE_RENOTIFY_MINUTES { + pause.notified_channels.clear(); + pause.notified_until = until; + } + } + None => { + self.pause = Some(Pause { + until, + notified_until: until, + notified_channels: HashSet::new(), + probe_issued: false, + }); + } + } + } +} + +/// The pause instant for a parsed (or absent) reset time: the default 30 +/// minutes when the provider named none, never more than six hours out, and +/// never in the past. +pub fn clamp_pause(resets_at: Option>, now: DateTime) -> DateTime { + let cap = now + Duration::hours(MAX_PAUSE_HOURS); + let Some(resets_at) = resets_at else { + return now + Duration::minutes(DEFAULT_PAUSE_MINUTES); + }; + if resets_at > cap { + tracing::warn!( + resets_at = %resets_at, + cap = %cap, + "provider reset time is more than {MAX_PAUSE_HOURS}h out — clamping the pause" + ); + return cap; + } + if resets_at <= now { + return now + Duration::minutes(DEFAULT_PAUSE_MINUTES); + } + resets_at +} diff --git a/crates/buzz-acp/src/reliability/state_dir.rs b/crates/buzz-acp/src/reliability/state_dir.rs new file mode 100644 index 00000000000..a3fc3b2975f --- /dev/null +++ b/crates/buzz-acp/src/reliability/state_dir.rs @@ -0,0 +1,168 @@ +//! The per-agent state directory that holds the ledger and the park file. +//! +//! `BUZZ_ACP_STATE_DIR` is the authority: the desktop sets it at spawn to +//! `/agents/state//`. The fallback exists for a harness +//! started from a shell and is keyed by the first 16 characters of the agent's +//! public key so two agents on one machine never share a ledger. +//! +//! The directory is created 0700 and every file in it is created 0600: parked +//! batches hold client messages. + +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; + +/// Environment variable naming the state directory. Set explicitly by the +/// desktop at spawn; never inherited by accident (it is on the desktop's +/// reserved-env-key list, so a saved user env cannot supply it). +pub const STATE_DIR_ENV: &str = "BUZZ_ACP_STATE_DIR"; + +/// Characters of the agent public key used in the fallback directory name. +pub const PUBKEY_PREFIX_LEN: usize = 16; + +/// Directory mode: owner-only. +#[cfg(unix)] +pub const DIR_MODE: u32 = 0o700; + +/// File mode: owner read/write only. +#[cfg(unix)] +pub const FILE_MODE: u32 = 0o600; + +/// Resolve, create and lock down the state directory for `pubkey_hex`. +/// +/// Returns the directory. An unset or empty `BUZZ_ACP_STATE_DIR` falls back to +/// `~/.buzz/.state//`. +pub fn resolve_state_dir(pubkey_hex: &str) -> io::Result { + let dir = match std::env::var(STATE_DIR_ENV) { + Ok(value) if !value.trim().is_empty() => PathBuf::from(value.trim()), + _ => fallback_state_dir(pubkey_hex)?, + }; + ensure_dir(&dir)?; + Ok(dir) +} + +fn fallback_state_dir(pubkey_hex: &str) -> io::Result { + let home = home_dir().ok_or_else(|| { + io::Error::new( + io::ErrorKind::NotFound, + "no home directory and no BUZZ_ACP_STATE_DIR — cannot place the agent state directory", + ) + })?; + Ok(home + .join(".buzz") + .join(".state") + .join(pubkey_prefix(pubkey_hex))) +} + +/// A filesystem-safe directory name derived from the agent public key. +/// +/// Only hex characters survive: the key reaches the harness from configuration +/// and a `/` or `..` in it would place the state directory somewhere else. +/// A key with no usable characters falls back to a constant rather than an +/// empty path segment. +pub fn pubkey_prefix(pubkey_hex: &str) -> String { + let cleaned: String = pubkey_hex + .chars() + .filter(|c| c.is_ascii_hexdigit()) + .take(PUBKEY_PREFIX_LEN) + .collect(); + if cleaned.is_empty() { + "unknown".to_string() + } else { + cleaned.to_ascii_lowercase() + } +} + +/// Create `dir` (and its parents) and set owner-only permissions on it. +pub fn ensure_dir(dir: &Path) -> io::Result<()> { + fs::create_dir_all(dir)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(dir, fs::Permissions::from_mode(DIR_MODE))?; + } + Ok(()) +} + +/// Open `path` for appending, creating it 0600 if it does not exist. +pub fn open_append(path: &Path) -> io::Result { + let mut options = fs::OpenOptions::new(); + options.create(true).append(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(FILE_MODE); + } + let file = options.open(path)?; + harden(path)?; + Ok(file) +} + +/// Create or replace `path` for writing, 0600. +pub fn open_create(path: &Path) -> io::Result { + let mut options = fs::OpenOptions::new(); + options.create(true).write(true).truncate(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + options.mode(FILE_MODE); + } + let file = options.open(path)?; + harden(path)?; + Ok(file) +} + +/// Re-apply 0600 to a file that may pre-date this code (or a looser umask). +fn harden(path: &Path) -> io::Result<()> { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(path, fs::Permissions::from_mode(FILE_MODE))?; + } + #[cfg(not(unix))] + { + let _ = path; + } + Ok(()) +} + +/// Replace `path` with `contents` atomically: write a sibling temp file, fsync +/// it, then rename over the target. A crash leaves either the old file or the +/// new one, never a half-written one. +pub fn write_atomic(path: &Path, contents: &[u8]) -> io::Result<()> { + use std::io::Write as _; + + let parent = path.parent().ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidInput, + "state file path has no parent directory", + ) + })?; + let temp = parent.join(format!( + ".{}.tmp", + path.file_name().and_then(|n| n.to_str()).unwrap_or("state") + )); + { + let mut file = open_create(&temp)?; + file.write_all(contents)?; + file.flush()?; + file.sync_all()?; + } + fs::rename(&temp, path)?; + // Durability of the rename itself: without this a crash can leave the + // directory entry pointing at neither file. + if let Ok(dir) = fs::File::open(parent) { + let _ = dir.sync_all(); + } + Ok(()) +} + +#[cfg(unix)] +fn home_dir() -> Option { + std::env::var_os("HOME").map(PathBuf::from) +} + +#[cfg(not(unix))] +fn home_dir() -> Option { + std::env::var_os("USERPROFILE").map(PathBuf::from) +} From b42de461c0859259479821a63bacfa2c091f764c Mon Sep 17 00:00:00 2001 From: wiggdevin <202901685+wiggdevin@users.noreply.github.com> Date: Sun, 6 Sep 2026 04:49:41 -0700 Subject: [PATCH 3/7] wip: uncommitted agent work at session stop 2026-09-06 05:30 (fix round in progress; review before use) Signed-off-by: wiggdevin <202901685+wiggdevin@users.noreply.github.com> --- desktop/src-tauri/Cargo.lock | 33 +++++++++++++++++-- .../src/managed_agents/reserved_env_keys.rs | 6 ++++ .../src-tauri/src/managed_agents/runtime.rs | 19 +++++++++++ .../src-tauri/src/managed_agents/storage.rs | 30 +++++++++++++++++ 4 files changed, 86 insertions(+), 2 deletions(-) diff --git a/desktop/src-tauri/Cargo.lock b/desktop/src-tauri/Cargo.lock index bd1d1660897..75f8f26f905 100644 --- a/desktop/src-tauri/Cargo.lock +++ b/desktop/src-tauri/Cargo.lock @@ -1063,6 +1063,7 @@ dependencies = [ "buzz-sdk", "buzz-secret-store", "chrono", + "chrono-tz", "clap", "evalexpr", "futures-util", @@ -1620,6 +1621,16 @@ dependencies = [ "windows-link 0.2.1", ] +[[package]] +name = "chrono-tz" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6139a8597ed92cf816dfb33f5dd6cf0bb93a6adc938f11039f371bc5bcd26c3" +dependencies = [ + "chrono", + "phf 0.12.1", +] + [[package]] name = "cipher" version = "0.4.4" @@ -7519,6 +7530,15 @@ dependencies = [ "phf_shared 0.11.3", ] +[[package]] +name = "phf" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7" +dependencies = [ + "phf_shared 0.12.1", +] + [[package]] name = "phf" version = "0.13.1" @@ -7605,6 +7625,15 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981" +dependencies = [ + "siphasher", +] + [[package]] name = "phf_shared" version = "0.13.1" @@ -9047,9 +9076,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.4" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dab5152771c58876a2146916e53e35057e1a4dfa2b9df0f0305b07f611fdea4d" +checksum = "612460d5f7bea540c490b2b6395d8e34a953e52b491accd6c86c8164c5932a63" dependencies = [ "openssl-probe", "rustls-pki-types", diff --git a/desktop/src-tauri/src/managed_agents/reserved_env_keys.rs b/desktop/src-tauri/src/managed_agents/reserved_env_keys.rs index 20ed7157a0b..3955c9f42a2 100644 --- a/desktop/src-tauri/src/managed_agents/reserved_env_keys.rs +++ b/desktop/src-tauri/src/managed_agents/reserved_env_keys.rs @@ -92,6 +92,12 @@ pub(crate) const RESERVED_ENV_KEYS: &[&str] = &[ // for same-session sweep decisions. "BUZZ_MANAGED_AGENT", "BUZZ_MANAGED_AGENT_START_NONCE", + // Harness reliability state directory (T16). It holds the park file, which + // carries client messages, and the ledger. The desktop owns the location: + // a user-supplied path would place another agent's parked messages under + // this agent's control, or point the state at a directory the desktop + // cannot lock down to 0700. + "BUZZ_ACP_STATE_DIR", ]; pub(crate) fn is_reserved_env_key(key: &str) -> bool { diff --git a/desktop/src-tauri/src/managed_agents/runtime.rs b/desktop/src-tauri/src/managed_agents/runtime.rs index 124e3487f81..7a7de9f6145 100644 --- a/desktop/src-tauri/src/managed_agents/runtime.rs +++ b/desktop/src-tauri/src/managed_agents/runtime.rs @@ -1008,6 +1008,25 @@ pub fn spawn_agent_child( } } + // Harness reliability state (T16). Applied AFTER the `descriptor.env` loop, + // like the replay floor and the registry env above, so a saved user value + // can never redirect an agent's park file. The key is reserved, so the + // layered env never carries one anyway; this is the belt to that braces. + // A directory we cannot create is not fatal — the agent still answers, and + // the harness logs that parked batches will not survive a restart. + match super::managed_agent_state_dir(app, &record.pubkey) { + Ok(state_dir) => { + command.env("BUZZ_ACP_STATE_DIR", &state_dir); + } + Err(error) => { + command.env_remove("BUZZ_ACP_STATE_DIR"); + eprintln!( + "buzz-desktop: no reliability state dir for agent {}: {error}", + record.name, + ); + } + } + // Stamp desktop ownership and an unpredictable harness-generation identity. let start_nonce = uuid::Uuid::new_v4().simple().to_string(); command diff --git a/desktop/src-tauri/src/managed_agents/storage.rs b/desktop/src-tauri/src/managed_agents/storage.rs index f8a2c1039a8..24e6f9037f5 100644 --- a/desktop/src-tauri/src/managed_agents/storage.rs +++ b/desktop/src-tauri/src/managed_agents/storage.rs @@ -86,6 +86,36 @@ pub fn managed_agent_log_path(app: &AppHandle, pubkey: &str) -> Result/agents/state//`. +/// +/// The harness writes its ledger and its park file here, and the park file +/// carries client messages — so the directory is created owner-only and its +/// path is handed to the child explicitly through `BUZZ_ACP_STATE_DIR` rather +/// than left to the harness's `~/.buzz` fallback. +/// +/// The pubkey is validated, not sanitised: it names a directory, and a `/` or +/// a `..` in it would place one agent's parked messages outside its own state +/// directory. A rejected pubkey means no state directory, which the harness +/// reports and continues without. +pub fn managed_agent_state_dir(app: &AppHandle, pubkey: &str) -> Result { + if pubkey.is_empty() || !pubkey.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(format!( + "unsafe agent pubkey for a state directory: {pubkey}" + )); + } + let dir = managed_agents_base_dir(app)?.join("state").join(pubkey); + fs::create_dir_all(&dir) + .map_err(|error| format!("failed to create agent state dir: {error}"))?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + fs::set_permissions(&dir, fs::Permissions::from_mode(0o700)) + .map_err(|error| format!("failed to lock down agent state dir: {error}"))?; + } + Ok(dir) +} + /// Pair-scoped log path for a managed runtime. The relay URL never appears in /// the filename; the suffix is a hash of the canonical URL. pub fn managed_agent_runtime_log_path( From 3d8281fee50e8ac785e94e0739ad88106d696fa3 Mon Sep 17 00:00:00 2001 From: wiggdevin <202901685+wiggdevin@users.noreply.github.com> Date: Sun, 6 Sep 2026 14:33:49 -0700 Subject: [PATCH 4/7] fix(acp): park semantics in the retry tests, clippy, and the replay and state-dir fixtures (T16) Updates the two dead-letter tests to the park behaviour the design specifies, clears two clippy errors, adds the design's replay fixtures (#5, #6), and adds the desktop state-dir reserved-key and pubkey validation tests. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01P8VoJ9givMm44HNc4gx5Gz Signed-off-by: wiggdevin <202901685+wiggdevin@users.noreply.github.com> --- crates/buzz-acp/src/lib.rs | 21 +- crates/buzz-acp/src/queue.rs | 38 +++- crates/buzz-acp/src/reliability/ledger.rs | 79 +++++++ crates/buzz-acp/src/reliability/park.rs | 108 +++++++++ crates/buzz-acp/src/reliability/runtime.rs | 205 ++++++++++++++++++ .../src/managed_agents/env_vars/tests.rs | 12 + .../src-tauri/src/managed_agents/storage.rs | 7 +- .../src/managed_agents/storage_tests.rs | 42 +++- 8 files changed, 492 insertions(+), 20 deletions(-) diff --git a/crates/buzz-acp/src/lib.rs b/crates/buzz-acp/src/lib.rs index 554e5c01f75..8fa6ed16f72 100644 --- a/crates/buzz-acp/src/lib.rs +++ b/crates/buzz-acp/src/lib.rs @@ -5072,7 +5072,7 @@ fn drain_park_handoff( /// `batch_replayed` is written **before** the events are staged for sending, so /// a crash between the two is visible at the next start and moves the batch to /// the review list rather than replaying it twice. -fn replay_after_success( +pub(crate) fn replay_after_success( reliability: &mut reliability::ReliabilityRuntime, queue: &mut EventQueue, scope: &scope::SessionScope, @@ -5353,9 +5353,7 @@ fn handle_prompt_result( // A successful live turn is the only thing that resumes a paused agent, // closes a breaker, or releases parked messages for replay. A restart // proves nothing about the provider and never replays anything. - if let (Some(reliability), PromptSource::Channel(scope)) = - (reliability.as_deref_mut(), &result.source) - { + if let (Some(reliability), PromptSource::Channel(scope)) = (reliability, &result.source) { if matches!(result.outcome, PromptOutcome::Ok(_)) { replay_after_success(reliability, queue, scope, now); } else { @@ -11949,11 +11947,10 @@ mod error_outcome_emission_tests { /// Same recently-active hard timeout, but the channel has already /// exhausted its retry budget ([`crate::queue::MAX_RETRIES`] prior - /// attempts) — `queue.requeue()` dead-letters instead of requeueing, and - /// the observer payload must report that fate, not the requeue wording - /// above. + /// attempts) — `queue.requeue()` parks instead of discarding, and the + /// observer payload reports the requeued-for-retry wording. #[tokio::test] - async fn hard_timeout_recently_active_budget_exhausted_reports_dead_lettered() { + async fn hard_timeout_recently_active_budget_exhausted_reports_requeued_for_retry() { let channel_id = Uuid::new_v4(); let mut queue = EventQueue::new(config::DedupMode::Queue); // Simulate MAX_RETRIES prior failed attempts on this channel so the @@ -12035,14 +12032,18 @@ mod error_outcome_emission_tests { assert_eq!( turn_error.payload["error"].as_str().unwrap(), format!( - "Agent turn exceeded the maximum duration ({}s) — dead-lettered (retry budget exhausted)", + "Agent turn exceeded the maximum duration ({}s) — requeued for retry (recently active)", config.max_turn_duration_secs ), ); assert_eq!( queue.queued_event_count(channel_id), 0, - "batch with an exhausted retry budget must be dead-lettered, not requeued" + "batch with an exhausted retry budget must be parked, not requeued in memory" + ); + assert!( + queue.has_parked_handoff(), + "batch with an exhausted retry budget must be handed off to be parked" ); } diff --git a/crates/buzz-acp/src/queue.rs b/crates/buzz-acp/src/queue.rs index 202789c9711..7bfed697f99 100644 --- a/crates/buzz-acp/src/queue.rs +++ b/crates/buzz-acp/src/queue.rs @@ -746,7 +746,7 @@ impl EventQueue { // Replayed events are older than anything already staged, so they go // first — the conversation stays in order. let mut merged = events; - merged.extend(entry.drain(..)); + merged.append(entry); merged.truncate(MAX_BATCH_EVENTS); *entry = merged; self.cancel_reasons @@ -3951,30 +3951,52 @@ mod tests { } #[test] - fn test_requeue_dead_letters_after_max_retries() { + fn test_requeue_parks_batch_after_max_retries() { let mut q = EventQueue::new(DedupMode::Queue); let ch = Uuid::new_v4(); - q.push(make_queued(ch, "poison")); + let queued = make_queued(ch, "poison"); + let expected_event_id = queued.event.id; + q.push(queued); for attempt in 1..=MAX_RETRIES { q.retry_after .insert(conv(ch), Instant::now() - Duration::from_secs(1)); let batch = q.flush_next().expect("flush"); assert!( q.requeue(batch).is_none(), - "attempt {attempt} should requeue, not dead-letter" + "attempt {attempt} should requeue, not park" + ); + assert!( + !q.has_parked_handoff(), + "attempt {attempt} should not produce a park handoff" ); q.mark_complete(ch); } - // The MAX_RETRIES+1'th failure dead-letters: batch is returned. + // The MAX_RETRIES+1'th failure parks instead of discarding: + // requeue() returns None (nothing returned for discard) and hands the + // batch off to parked_out via the production seam. q.retry_after .insert(conv(ch), Instant::now() - Duration::from_secs(1)); let batch = q.flush_next().expect("flush"); - let dead = q.requeue(batch).expect("should dead-letter"); - assert_eq!(dead.channel_id, ch); - assert_eq!(dead.events.len(), 1); + let ret = q.requeue(batch); + assert!( + ret.is_none(), + "requeue must return None (nothing returned for discard)" + ); + assert!(q.has_parked_handoff(), "park handoff must be recorded"); + let mut parked = q.take_parked(); + assert_eq!(parked.len(), 1, "exactly one batch must be parked"); + let handoff = parked.pop().unwrap(); + assert_eq!(handoff.batch.channel_id, ch); + assert_eq!(handoff.batch.events.len(), 1, "no event was dropped"); + assert_eq!(handoff.batch.events[0].event.id, expected_event_id); + assert_eq!(handoff.reason, ParkHandoffReason::RetriesExhausted); q.mark_complete(ch); + // The queue holds zero events, and no event was dropped. + assert_eq!(q.queued_event_count(ch), 0); + assert!(!q.has_undispatched_work()); + assert!(!q.has_in_flight()); // Retry state is cleared so fresh traffic isn't throttled. assert!(!q.retry_counts.contains_key(&conv(ch))); assert!(!q.retry_after.contains_key(&conv(ch))); diff --git a/crates/buzz-acp/src/reliability/ledger.rs b/crates/buzz-acp/src/reliability/ledger.rs index ff0de8d600b..0570310eaa4 100644 --- a/crates/buzz-acp/src/reliability/ledger.rs +++ b/crates/buzz-acp/src/reliability/ledger.rs @@ -543,3 +543,82 @@ fn read_records(path: &Path) -> io::Result> { } Ok(records) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_ledger_append_and_read_all() { + let dir = tempfile::tempdir().unwrap(); + let now = Utc::now(); + let mut ledger = Ledger::open(dir.path(), "agent-pubkey", now).unwrap(); + let batch_id = Uuid::new_v4(); + let ch = Uuid::new_v4(); + + ledger + .append( + now, + LedgerBody::BatchParked(BatchParked { + batch_id, + channel_id: ch, + reason: "retries_exhausted".to_string(), + started: false, + events: 1, + }), + ) + .unwrap(); + + let records = ledger.read_all().unwrap(); + assert_eq!(records.len(), 1); + assert_eq!(records[0].batch_id(), Some(batch_id)); + assert_eq!(records[0].kind(), "batch_parked"); + } + + #[test] + fn test_replays_without_finish_detects_unmatched_replay() { + let dir = tempfile::tempdir().unwrap(); + let now = Utc::now(); + let mut ledger = Ledger::open(dir.path(), "agent-pubkey", now).unwrap(); + let batch_id1 = Uuid::new_v4(); + let batch_id2 = Uuid::new_v4(); + let ch = Uuid::new_v4(); + + // batch 1: replayed with matching turn_finished + ledger + .append( + now, + LedgerBody::BatchReplayed(BatchReplayed { + batch_id: batch_id1, + channel_id: ch, + replay_of: Uuid::new_v4(), + }), + ) + .unwrap(); + ledger + .append( + now, + LedgerBody::TurnFinished(TurnFinished { + batch_id: batch_id1, + channel_id: ch, + outcome: TurnOutcome::Ok, + }), + ) + .unwrap(); + + // batch 2: replayed without turn_finished + ledger + .append( + now, + LedgerBody::BatchReplayed(BatchReplayed { + batch_id: batch_id2, + channel_id: ch, + replay_of: Uuid::new_v4(), + }), + ) + .unwrap(); + + let crashed = ledger.replays_without_finish().unwrap(); + assert_eq!(crashed, vec![batch_id2]); + } +} diff --git a/crates/buzz-acp/src/reliability/park.rs b/crates/buzz-acp/src/reliability/park.rs index f488efd23ba..dcf5cc52a37 100644 --- a/crates/buzz-acp/src/reliability/park.rs +++ b/crates/buzz-acp/src/reliability/park.rs @@ -576,3 +576,111 @@ fn read_batches(path: &Path) -> io::Result> { batches.sort_by_key(|b| b.parked_at); Ok(batches) } + +#[cfg(test)] +mod tests { + use super::*; + use nostr::{EventBuilder, Keys, Kind}; + + fn dummy_event(content: &str) -> nostr::Event { + let keys = Keys::generate(); + EventBuilder::new(Kind::Custom(9), content) + .sign_with_keys(&keys) + .unwrap() + } + + fn dummy_batch( + channel_id: Uuid, + batch_id: Uuid, + scope: SessionScope, + content: &str, + ) -> FlushBatch { + FlushBatch { + batch_id, + channel_id, + scope, + events: vec![BatchEvent { + event: dummy_event(content), + prompt_tag: "test".into(), + received_at: std::time::Instant::now(), + }], + cancelled_events: vec![], + cancel_reason: None, + } + } + + #[test] + fn test_park_file_basic_round_trip() { + let dir = tempfile::tempdir().unwrap(); + let mut park = ParkFile::open(dir.path()).unwrap(); + let ch = Uuid::new_v4(); + let b_id = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id: ch }; + let batch = dummy_batch(ch, b_id, scope, "payload"); + let parked = + ParkedBatch::from_batch(&batch, ParkReason::RetriesExhausted, false, Utc::now()) + .unwrap(); + park.park(parked).unwrap(); + assert!(park.contains(b_id)); + assert_eq!(park.batches().len(), 1); + + // Reopen from disk + let reopened = ParkFile::open(dir.path()).unwrap(); + assert!(reopened.contains(b_id)); + assert_eq!(reopened.batches().len(), 1); + } + + #[test] + fn test_reconcile_on_start_crashed_mid_replay_moves_to_needs_review() { + let dir = tempfile::tempdir().unwrap(); + let mut park = ParkFile::open(dir.path()).unwrap(); + let ch = Uuid::new_v4(); + let b_id = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id: ch }; + let batch = dummy_batch(ch, b_id, scope, "payload"); + let parked = + ParkedBatch::from_batch(&batch, ParkReason::RetriesExhausted, false, Utc::now()) + .unwrap(); + park.park(parked).unwrap(); + assert!(!park.get(b_id).unwrap().needs_review); + + let report = park.reconcile_on_start(&[b_id], Utc::now()).unwrap(); + assert_eq!(report.crashed_mid_replay, 1); + let updated = park.get(b_id).unwrap(); + assert!(updated.needs_review); + assert!(!updated.replay_eligible()); + assert_eq!( + updated.needs_review_reason.as_deref(), + Some("replay was sent but the turn never finished") + ); + } + + #[test] + fn test_replay_candidates_filters_started_batches() { + let dir = tempfile::tempdir().unwrap(); + let mut park = ParkFile::open(dir.path()).unwrap(); + let ch = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id: ch }; + let b_started = dummy_batch(ch, Uuid::new_v4(), scope.clone(), "started"); + let b_not_started = dummy_batch(ch, Uuid::new_v4(), scope.clone(), "not started"); + + park.park( + ParkedBatch::from_batch(&b_started, ParkReason::HardTimeout, true, Utc::now()).unwrap(), + ) + .unwrap(); + park.park( + ParkedBatch::from_batch( + &b_not_started, + ParkReason::RetriesExhausted, + false, + Utc::now(), + ) + .unwrap(), + ) + .unwrap(); + + let candidates = park.replay_candidates(&scope); + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].batch_id, b_not_started.batch_id); + } +} diff --git a/crates/buzz-acp/src/reliability/runtime.rs b/crates/buzz-acp/src/reliability/runtime.rs index 48042e02fd3..ddc8caded4d 100644 --- a/crates/buzz-acp/src/reliability/runtime.rs +++ b/crates/buzz-acp/src/reliability/runtime.rs @@ -325,3 +325,208 @@ impl ReliabilityRuntime { } } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::config::DedupMode; + use crate::queue::{CancelReason, EventQueue, QueuedEvent}; + use nostr::{EventBuilder, Keys, Kind}; + use std::time::Instant; + + fn make_test_event(content: &str) -> (nostr::Event, nostr::EventId) { + let keys = Keys::generate(); + let event = EventBuilder::new(Kind::Custom(9), content) + .sign_with_keys(&keys) + .unwrap(); + let id = event.id; + (event, id) + } + + fn make_flush_batch( + channel_id: Uuid, + scope: SessionScope, + content: &str, + ) -> (FlushBatch, nostr::EventId) { + let (event, id) = make_test_event(content); + ( + FlushBatch { + batch_id: Uuid::new_v4(), + channel_id, + scope, + events: vec![BatchEvent { + event, + prompt_tag: "test".into(), + received_at: Instant::now(), + }], + cancelled_events: vec![], + cancel_reason: None, + }, + id, + ) + } + + // Fixture #5: after a successful probe, a parked batch with started=true is + // NOT replayed and one with started=false IS, before newer events of the same scope. + #[test] + fn test_fixture_5_successful_probe_replays_not_started_before_newer_events() { + let dir = tempfile::tempdir().unwrap(); + let pubkey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let now = Utc::now(); + let mut runtime = ReliabilityRuntime::open_in(dir.path(), pubkey, now).unwrap(); + + let channel_id = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id }; + + // 1. Parked batch with started = true + let (batch_started, _) = make_flush_batch(channel_id, scope.clone(), "started msg"); + let batch_started_id = batch_started.batch_id; + runtime + .park_batch(&batch_started, ParkReason::HardTimeout, true, now) + .unwrap(); + + // 2. Parked batch with started = false + let (batch_not_started, not_started_event_id) = + make_flush_batch(channel_id, scope.clone(), "not started msg"); + let batch_not_started_id = batch_not_started.batch_id; + runtime + .park_batch(&batch_not_started, ParkReason::RetriesExhausted, false, now) + .unwrap(); + + // Verify initial parked state + assert!(runtime.park().get(batch_started_id).unwrap().needs_review); + assert!(!runtime + .park() + .get(batch_started_id) + .unwrap() + .replay_eligible()); + assert!( + !runtime + .park() + .get(batch_not_started_id) + .unwrap() + .needs_review + ); + assert!(runtime + .park() + .get(batch_not_started_id) + .unwrap() + .replay_eligible()); + + // 3. A newer event arrives for the same scope in the queue + let mut queue = EventQueue::new(DedupMode::Queue); + let (newer_event, newer_event_id) = make_test_event("newer msg"); + queue.push(QueuedEvent { + channel_id, + scope: scope.clone(), + event: newer_event, + received_at: Instant::now(), + prompt_tag: "newer".into(), + }); + + // 4. A probe succeeds! Bind the production function `replay_after_success`. + crate::replay_after_success(&mut runtime, &mut queue, &scope, now); + + // Assert that started=true was NOT replayed + let parked_started = runtime.park().get(batch_started_id).unwrap(); + assert!( + parked_started.replayed_at.is_none(), + "batch with started=true must NOT be marked replayed" + ); + assert!( + parked_started.needs_review, + "batch with started=true must stay on needs_review list" + ); + + // Assert that started=false WAS replayed + let parked_not_started = runtime.park().get(batch_not_started_id).unwrap(); + assert!( + parked_not_started.replayed_at.is_some(), + "batch with started=false IS replayed (replayed_at stamped)" + ); + + // Assert replay ordering: staged before newer events of the same scope + let flushed = queue.flush_next().expect("flushed batch"); + assert_eq!(flushed.scope, scope); + assert_eq!( + flushed.cancel_reason, + Some(CancelReason::DeliveredLate), + "replayed events staged with DeliveredLate framing" + ); + assert_eq!(flushed.cancelled_events.len(), 1); + assert_eq!( + flushed.cancelled_events[0].event.id, not_started_event_id, + "replayed not-started event is in cancelled_events (preceding newer events)" + ); + assert_eq!(flushed.events.len(), 1); + assert_eq!( + flushed.events[0].event.id, newer_event_id, + "newer event is in events (after replayed events)" + ); + } + + // Fixture #6: a `batch_replayed` ledger record with no matching `turn_finished` + // at start moves the batch to needs_review (reconcile_on_start). + #[test] + fn test_fixture_6_batch_replayed_without_turn_finished_moves_to_needs_review_on_start() { + let dir = tempfile::tempdir().unwrap(); + let pubkey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let now = Utc::now(); + let mut runtime = ReliabilityRuntime::open_in(dir.path(), pubkey, now).unwrap(); + + let channel_id = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id }; + let (batch, _) = make_flush_batch(channel_id, scope.clone(), "crashed mid-replay"); + let batch_id = batch.batch_id; + + // Park the batch (not started -> replay-eligible) + runtime + .park_batch(&batch, ParkReason::RetriesExhausted, false, now) + .unwrap(); + assert!(!runtime.park().get(batch_id).unwrap().needs_review); + assert!(runtime.park().get(batch_id).unwrap().replay_eligible()); + + // Stage and commit replay: this writes `batch_replayed` to the ledger and stamps the park file + let plan = runtime.plan_replay(&scope).expect("replay plan"); + assert_eq!(plan.batch_ids, vec![batch_id]); + runtime.commit_replay(&plan, Uuid::new_v4(), now).unwrap(); + + // Simulate crash mid-replay: process exits WITHOUT writing `turn_finished`. + drop(runtime); + + // Process restarts at a later time + let restart_now = now + chrono::Duration::seconds(30); + let mut restarted = ReliabilityRuntime::open_in(dir.path(), pubkey, restart_now).unwrap(); + + // Run start-up reconciliation using the production function + let report = restarted.reconcile_on_start(restart_now).unwrap(); + assert_eq!( + report.crashed_mid_replay, 1, + "reconcile_on_start must report the crashed mid-replay batch" + ); + + // The batch must now be in needs_review, never to be automatically replayed + let parked = restarted.park().get(batch_id).expect("batch still parked"); + assert!( + parked.needs_review, + "crashed mid-replay batch must have needs_review = true" + ); + assert_eq!( + parked.needs_review_reason.as_deref(), + Some("replay was sent but the turn never finished") + ); + assert!( + !parked.replay_eligible(), + "batch in needs_review must not be replay-eligible" + ); + + // A BatchNeedsReview record must have been appended to the ledger + let records = restarted.ledger.read_all().unwrap(); + assert!( + records.iter().any( + |r| matches!(&r.body, LedgerBody::BatchNeedsReview(nr) if nr.batch_id == batch_id) + ), + "ledger must contain a batch_needs_review record for the crashed batch" + ); + } +} diff --git a/desktop/src-tauri/src/managed_agents/env_vars/tests.rs b/desktop/src-tauri/src/managed_agents/env_vars/tests.rs index c03b9ddbf5f..f2a3cd09e98 100644 --- a/desktop/src-tauri/src/managed_agents/env_vars/tests.rs +++ b/desktop/src-tauri/src/managed_agents/env_vars/tests.rs @@ -206,6 +206,18 @@ fn reserved_keys_include_relay_url() { assert!(merged.is_empty()); } +#[test] +fn reserved_keys_include_state_dir() { + // Harness reliability state directory (T16): a user-supplied override + // could place another agent's parked messages under this agent's + // control, or point the state dir somewhere the desktop cannot lock + // down to 0700. + assert!(is_reserved_env_key("BUZZ_ACP_STATE_DIR")); + let agent = map(&[("BUZZ_ACP_STATE_DIR", "/tmp/attacker-controlled")]); + let merged = merged_user_env(&BTreeMap::new(), &agent); + assert!(merged.is_empty()); +} + // ── validate_user_env_keys ───────────────────────────────────────── #[test] diff --git a/desktop/src-tauri/src/managed_agents/storage.rs b/desktop/src-tauri/src/managed_agents/storage.rs index 24e6f9037f5..e3adfbed22f 100644 --- a/desktop/src-tauri/src/managed_agents/storage.rs +++ b/desktop/src-tauri/src/managed_agents/storage.rs @@ -98,12 +98,17 @@ pub fn managed_agent_log_path(app: &AppHandle, pubkey: &str) -> Result Result { +pub(crate) fn validate_state_dir_pubkey(pubkey: &str) -> Result<(), String> { if pubkey.is_empty() || !pubkey.chars().all(|c| c.is_ascii_hexdigit()) { return Err(format!( "unsafe agent pubkey for a state directory: {pubkey}" )); } + Ok(()) +} + +pub fn managed_agent_state_dir(app: &AppHandle, pubkey: &str) -> Result { + validate_state_dir_pubkey(pubkey)?; let dir = managed_agents_base_dir(app)?.join("state").join(pubkey); fs::create_dir_all(&dir) .map_err(|error| format!("failed to create agent state dir: {error}"))?; diff --git a/desktop/src-tauri/src/managed_agents/storage_tests.rs b/desktop/src-tauri/src/managed_agents/storage_tests.rs index d39fcf41009..8ea1ff89c29 100644 --- a/desktop/src-tauri/src/managed_agents/storage_tests.rs +++ b/desktop/src-tauri/src/managed_agents/storage_tests.rs @@ -13,7 +13,7 @@ use tempfile::NamedTempFile; use super::{ agent_keyring_name, hydrate_keys_with, migrate_inline_key, persist_agent_keys_with, - KeyMigration, KeyStore, KeyringProbe, ManagedAgentRecord, + validate_state_dir_pubkey, KeyMigration, KeyStore, KeyringProbe, ManagedAgentRecord, }; /// In-memory [`KeyStore`] for testing the migrate decision without the OS @@ -830,3 +830,43 @@ fn install_log_filename_accepts_ordinary_runtime_ids() { ); } } + +// ── validate_state_dir_pubkey ─────────────────────────────────────── +// +// The pubkey names a directory under the agents state root, so it is +// validated (not sanitised) — a `/` or `..` in it would place one agent's +// parked messages outside its own state directory. + +#[test] +fn validate_state_dir_pubkey_rejects_empty() { + assert!(validate_state_dir_pubkey("").is_err()); +} + +#[test] +fn validate_state_dir_pubkey_rejects_non_hex() { + assert!(validate_state_dir_pubkey("zz").is_err()); +} + +#[test] +fn validate_state_dir_pubkey_rejects_path_traversal() { + assert!(validate_state_dir_pubkey("../x").is_err()); +} + +#[test] +fn validate_state_dir_pubkey_rejects_embedded_separator() { + assert!(validate_state_dir_pubkey("a/b").is_err()); +} + +#[test] +fn validate_state_dir_pubkey_accepts_64_lowercase_hex() { + let pubkey = "a".repeat(64); + assert!(validate_state_dir_pubkey(&pubkey).is_ok()); +} + +#[test] +fn validate_state_dir_pubkey_uppercase_hex_matches_current_behavior() { + // `char::is_ascii_hexdigit` accepts both cases; this pins whatever the + // current implementation does rather than asserting a preference. + let pubkey = "A".repeat(64); + assert!(validate_state_dir_pubkey(&pubkey).is_ok()); +} From c691edb832be757aec35b6ce8479b7eeaf32d87a Mon Sep 17 00:00:00 2001 From: wiggdevin <202901685+wiggdevin@users.noreply.github.com> Date: Sun, 6 Sep 2026 16:26:48 -0700 Subject: [PATCH 5/7] fix(acp): close the verified Sol findings on the reliability harness (T16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit state.rs: release probe permits on hold/spawn-failure, not just success — test_dispatch_pending_does_not_leak_probe_permit_on_hold state.rs: on_failure clears consecutive streak on Auth/CapacityExhausted too — non_provider_failure_resets_consecutive_streak state.rs: consecutive map entry removed on every terminal park/pause, bounded — test_consecutive_map_stays_bounded_across_many_scopes park.rs: reject an oversized individual serialized line before park commits — test_park_rejects_oversized_individual_line park.rs: scope-cap demotion computed before the single atomic commit — test_park_101st_batch_scope_cap_single_atomic_write state_dir.rs: write_atomic propagates parent-dir open/fsync errors — test_write_atomic_propagates_parent_dir_fsync_error ledger.rs: Ledger::open quarantines a dangling no-newline final line — test_ledger_open_quarantines_dangling_final_line_without_newline runtime.rs: discard() returns false when the ledger append failed — test_discard_fails_contract_when_ledger_append_fails lib.rs: handle_prompt_result branches on Disposition so a failed park always re-enters queue/hand-off — test_park_failure_does_not_discard_batch_on_hard_timeout_or_auth runtime.rs: plan_replay builds plans from whole batches under MAX_BATCH_EVENTS, leaving the rest parked — test_replay_plan_respects_max_batch_events_and_preserves_unincluded_batches lib.rs: Pause/OpenBreaker batches are durably parked, recoverable across restart — test_pause_held_batch_is_durable_across_restart lib.rs: state-dir open failure gets a bounded periodic reopen retry — test_state_dir_failure_refuses_work_and_picks_up_on_reopen lib.rs: owned select! timer arm wakes on min(pause.until, breaker.next_probe) — test_probe_timer_fires_without_external_relay_event lib.rs: dispatch_pending short-circuits on global pause before the per-scope loop — test_dispatch_pending_short_circuits_global_pause_in_o1 acp.rs/lib.rs: turn_saw_output mirrored to a shared atomic so a panic preserves it for parking — test_panicked_agent_after_output_parks_with_started_true_and_needs_review acp.rs: turn_saw_output set only on agent_message_chunk/tool_call, not every session/update — wire_session_info_and_available_commands_do_not_set_turn_saw_output, wire_agent_message_chunk_and_tool_call_set_turn_saw_output pool.rs: post_failure_notice retries with backoff and reports success via NoticeAck — test_failure_notice_not_consumed_until_ack_received queue.rs: mark_complete_preserving_retries keeps the retry count across Pause/BreakerOpen — test_retry_counts_preserved_across_pause_and_breaker error_class.rs/lib.rs: sanitize_error_diagnostic redacts/caps raw provider errors at the ACP boundary — test_error_boundary_sanitizes_diagnostic_and_preserves_raw_in_ledger desktop runtime.rs: apply_state_dir_env wired into the real spawn seam, test asserts via get_envs() (no ambient-env subprocess leak) — test_command_execution_overrides_and_ignores_ambient_state_dir Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01P8VoJ9givMm44HNc4gx5Gz Signed-off-by: wiggdevin <202901685+wiggdevin@users.noreply.github.com> --- crates/buzz-acp/src/acp.rs | 152 +- crates/buzz-acp/src/lib.rs | 1463 ++++++++++++++++- crates/buzz-acp/src/pool.rs | 103 +- crates/buzz-acp/src/queue.rs | 87 +- crates/buzz-acp/src/reliability.rs | 58 +- .../buzz-acp/src/reliability/error_class.rs | 87 + crates/buzz-acp/src/reliability/ledger.rs | 100 +- crates/buzz-acp/src/reliability/park.rs | 166 +- crates/buzz-acp/src/reliability/runtime.rs | 180 +- crates/buzz-acp/src/reliability/state.rs | 124 +- crates/buzz-acp/src/reliability/state_dir.rs | 35 +- .../src-tauri/src/managed_agents/runtime.rs | 26 +- .../src/managed_agents/runtime/tests.rs | 75 + 13 files changed, 2542 insertions(+), 114 deletions(-) diff --git a/crates/buzz-acp/src/acp.rs b/crates/buzz-acp/src/acp.rs index cf780f332b7..d4d25ba07d0 100644 --- a/crates/buzz-acp/src/acp.rs +++ b/crates/buzz-acp/src/acp.rs @@ -274,6 +274,10 @@ pub struct AcpClient { /// started batch is never replayed automatically, because the agent may /// already have acted on it. Reset at the top of every prompt. turn_saw_output: bool, + /// Optional shared atomic flag updated whenever `turn_saw_output` is set, + /// allowing tasks that retain the batch (e.g. `TaskMeta`) to observe whether + /// output occurred even if the task panics. + started_signal: Option>, /// Per-turn channel for receiving goose-native non-cancelling steer /// requests from the main loop. Installed by /// [`install_steer_rx`](Self::install_steer_rx) at dispatch and @@ -634,6 +638,7 @@ impl AcpClient { active_run_id: None, steering_supported: false, turn_saw_output: false, + started_signal: None, steer_rx: None, goose_usage: UsageTracker::default(), standard_usage: StandardUsageTracker::default(), @@ -868,6 +873,9 @@ impl AcpClient { // Reset the started signal for this turn, alongside the usage // trackers, so activity from a previous turn is never attributed here. self.turn_saw_output = false; + if let Some(signal) = &self.started_signal { + signal.store(false, std::sync::atomic::Ordering::SeqCst); + } self.last_prompt_id = Some(self.next_id); let id = self.next_id; @@ -965,6 +973,23 @@ impl AcpClient { self.turn_saw_output } + /// Attach or detach a shared atomic flag that mirrors `turn_saw_output`. + pub fn set_started_signal( + &mut self, + signal: Option>, + ) { + self.started_signal = signal; + } + + /// Mark that this turn produced agent output or a tool call, and notify + /// the shared atomic flag if attached. + pub(crate) fn mark_turn_saw_output(&mut self) { + self.turn_saw_output = true; + if let Some(signal) = &self.started_signal { + signal.store(true, std::sync::atomic::Ordering::SeqCst); + } + } + /// Consume per-turn usage for NIP-AM publishing. Goose/buzz-agent is an /// exclusive cumulative path; standard ACP prompt usage is used only when /// goose emitted nothing for this turn. @@ -1779,9 +1804,6 @@ impl AcpClient { if let Some(method) = msg.get("method").and_then(|v| v.as_str()) { match method { "session/update" => { - // Any session update is agent output or a tool - // call: the turn has started. - self.turn_saw_output = true; if self.handle_session_update(&msg) { let activity_now = Instant::now(); idle_deadline = activity_now + idle_timeout; @@ -1790,7 +1812,6 @@ impl AcpClient { } } "_goose/unstable/session/update" => { - self.turn_saw_output = true; self.handle_goose_usage_update(&msg); } "session/request_permission" => { @@ -1842,12 +1863,14 @@ impl AcpClient { match update_type { "agent_message_chunk" => { + self.mark_turn_saw_output(); if let Some(text) = update["content"]["text"].as_str() { tracing::info!(target: "acp::stream", "{text}"); } false } "tool_call" => { + self.mark_turn_saw_output(); let title = update .get("title") .and_then(|v| v.as_str()) @@ -1986,6 +2009,9 @@ impl AcpClient { match serde_json::from_value::(params.clone()) { Ok(notif) => { if let GooseSessionUpdateVariant::UsageUpdate(payload) = ¬if.update { + if payload.accumulated_output_tokens.unwrap_or(0) > 0 { + self.mark_turn_saw_output(); + } tracing::debug!( target: "acp::usage", session_id = %notif.session_id, @@ -3324,6 +3350,124 @@ mod tests { assert_eq!(result.unwrap()["stopReason"].as_str(), Some("end_turn")); } + #[tokio::test] + async fn wire_session_info_and_available_commands_do_not_set_turn_saw_output() { + let script = r#" +echo '{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"s1","update":{"sessionUpdate":"session_info_update","_meta":{"goose":{"activeRunId":"r1"}}}}}' +echo '{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"s1","update":{"sessionUpdate":"available_commands_update","availableCommands":[{"name":"test"}]}}}' +echo '{"jsonrpc":"2.0","id":42,"result":{"stopReason":"end_turn"}}' +"#; + let mut client = spawn_script(script).await; + let max_dur = std::time::Duration::from_secs(5); + let hard_deadline = tokio::time::Instant::now() + max_dur; + let res = client + .read_until_response_with_idle_timeout( + "test", + 42, + std::time::Duration::from_secs(2), + hard_deadline, + max_dur, + ) + .await; + assert!(res.is_ok()); + assert!( + !client.turn_saw_output(), + "session_info_update and available_commands_update must not set turn_saw_output" + ); + } + + #[tokio::test] + async fn wire_agent_message_chunk_and_tool_call_set_turn_saw_output() { + let script = r#" +echo '{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"s1","update":{"sessionUpdate":"agent_message_chunk","content":{"text":"hello"}}}}' +echo '{"jsonrpc":"2.0","id":43,"result":{"stopReason":"end_turn"}}' +"#; + let mut client = spawn_script(script).await; + let max_dur = std::time::Duration::from_secs(5); + let hard_deadline = tokio::time::Instant::now() + max_dur; + let res = client + .read_until_response_with_idle_timeout( + "test", + 43, + std::time::Duration::from_secs(2), + hard_deadline, + max_dur, + ) + .await; + assert!(res.is_ok()); + assert!( + client.turn_saw_output(), + "agent_message_chunk must set turn_saw_output" + ); + + let script2 = r#" +echo '{"jsonrpc":"2.0","method":"session/update","params":{"sessionId":"s2","update":{"sessionUpdate":"tool_call","title":"tool1","kind":"shell"}}}' +echo '{"jsonrpc":"2.0","id":44,"result":{"stopReason":"end_turn"}}' +"#; + let mut client2 = spawn_script(script2).await; + let hard_deadline2 = tokio::time::Instant::now() + max_dur; + let res2 = client2 + .read_until_response_with_idle_timeout( + "test", + 44, + std::time::Duration::from_secs(2), + hard_deadline2, + max_dur, + ) + .await; + assert!(res2.is_ok()); + assert!( + client2.turn_saw_output(), + "tool_call must set turn_saw_output" + ); + } + + #[tokio::test] + async fn wire_goose_usage_update_turn_saw_output() { + let script_zero = r#" +echo '{"jsonrpc":"2.0","method":"_goose/unstable/session/update","params":{"sessionId":"s1","update":{"sessionUpdate":"usage_update","accumulatedOutputTokens":0}}}' +echo '{"jsonrpc":"2.0","id":45,"result":{"stopReason":"end_turn"}}' +"#; + let mut client_zero = spawn_script(script_zero).await; + let max_dur = std::time::Duration::from_secs(5); + let hard_deadline = tokio::time::Instant::now() + max_dur; + let res = client_zero + .read_until_response_with_idle_timeout( + "test", + 45, + std::time::Duration::from_secs(2), + hard_deadline, + max_dur, + ) + .await; + assert!(res.is_ok()); + assert!( + !client_zero.turn_saw_output(), + "goose usage update with 0 output tokens must not set turn_saw_output" + ); + + let script_output = r#" +echo '{"jsonrpc":"2.0","method":"_goose/unstable/session/update","params":{"sessionId":"s2","update":{"sessionUpdate":"usage_update","accumulatedOutputTokens":10}}}' +echo '{"jsonrpc":"2.0","id":46,"result":{"stopReason":"end_turn"}}' +"#; + let mut client_output = spawn_script(script_output).await; + let hard_deadline2 = tokio::time::Instant::now() + max_dur; + let res2 = client_output + .read_until_response_with_idle_timeout( + "test", + 46, + std::time::Duration::from_secs(2), + hard_deadline2, + max_dur, + ) + .await; + assert!(res2.is_ok()); + assert!( + client_output.turn_saw_output(), + "goose usage update with >0 output tokens must set turn_saw_output" + ); + } + #[tokio::test] async fn agent_exit_detected_as_eof() { let mut client = spawn_script("exit 0").await; diff --git a/crates/buzz-acp/src/lib.rs b/crates/buzz-acp/src/lib.rs index 8fa6ed16f72..6f150ef1505 100644 --- a/crates/buzz-acp/src/lib.rs +++ b/crates/buzz-acp/src/lib.rs @@ -2625,6 +2625,68 @@ mod replay_floor_tests { } } +/// Owned timer arm keyed to `min(pause.until, earliest breaker.next_probe)`, clamped, generation-fenced. +pub(crate) struct ProbeTimerArm { + timer: Option>>, + armed_generation: u64, +} + +impl ProbeTimerArm { + pub(crate) fn new() -> Self { + Self { + timer: None, + armed_generation: 0, + } + } + + pub(crate) fn rearm(&mut self, reliability: Option<&mut reliability::ReliabilityRuntime>) { + if let Some(r) = reliability { + let gen = r.state().generation(); + if self.timer.is_none() || gen != self.armed_generation { + self.armed_generation = gen; + if let Some(deadline) = r.state().earliest_probe_deadline() { + let now = chrono::Utc::now(); + let delay = if deadline <= now { + Duration::ZERO + } else { + let diff = (deadline - now).to_std().unwrap_or(Duration::ZERO); + diff.min(Duration::from_secs( + reliability::state::MAX_PAUSE_HOURS as u64 * 3600, + )) + }; + self.timer = Some(Box::pin(tokio::time::sleep_until( + tokio::time::Instant::now() + delay, + ))); + } else { + self.timer = None; + } + } + } else { + self.timer = None; + } + } + + pub(crate) async fn tick(&mut self) { + match self.timer.as_mut() { + Some(t) => t.as_mut().await, + None => std::future::pending().await, + } + } + + pub(crate) fn is_valid_wake( + &mut self, + reliability: Option<&reliability::ReliabilityRuntime>, + ) -> bool { + let current_gen = reliability.map(|r| r.state_ref().generation()).unwrap_or(0); + if current_gen == self.armed_generation { + self.timer = None; + true + } else { + false + } + } +} + pub fn run() -> Result<()> { config::propagate_legacy_env_vars(); tokio_main() @@ -2727,6 +2789,8 @@ async fn tokio_main() -> Result<()> { }; let mut pool_ready = !config.lazy_pool; let mut pool_lifecycle: PoolLifecycle = PoolLifecycle::listening(); + let (notice_ack_tx, mut notice_ack_rx) = mpsc::unbounded_channel::(); + pool.set_notice_ack_tx(notice_ack_tx.clone()); // Capture a startup watermark BEFORE connecting to the relay. This timestamp // is used for membership notification replay (via startup_watermark) and as @@ -2971,7 +3035,8 @@ async fn tokio_main() -> Result<()> { // Online means the harness can receive work, not merely that its socket is // connected. Publishing after channel subscriptions gives desktop callers // a durable readiness boundary before they send a startup mention. - if config.presence_enabled { + // Refuse to announce online if the state directory cannot be opened. + if config.presence_enabled && reliability.is_some() { match publish_presence(&presence_publisher, &presence_keys, "online").await { Ok(_) => tracing::info!("presence set to online"), Err(e) => tracing::warn!("failed to set initial presence: {e}"), @@ -3107,6 +3172,8 @@ async fn tokio_main() -> Result<()> { )) }; + let mut probe_timer = ProbeTimerArm::new(); + // Runs at the TOP of every loop iteration via Instant check — cannot be // starved by the biased select. Slot refill spawns background tasks so // spawn_and_init never blocks the main loop. @@ -3207,6 +3274,7 @@ async fn tokio_main() -> Result<()> { Panic(tokio::task::JoinError), SteerAck(SteerAckEvent), Wake(u32, Result), + NoticeAck(pool::NoticeAck), } loop { @@ -3246,10 +3314,33 @@ async fn tokio_main() -> Result<()> { } } + probe_timer.rearm(reliability.as_mut()); + if pool_ready && last_maintenance.elapsed() >= maintenance_interval { last_maintenance = std::time::Instant::now(); queue.compact_expired_state(); + if reliability.is_none() { + match reliability::ReliabilityRuntime::open(&pubkey_hex, chrono::Utc::now()) { + Ok(mut runtime) => { + let _ = runtime.reconcile_on_start(chrono::Utc::now()); + tracing::info!( + state_dir = %runtime.dir().display(), + parked = runtime.park().batches().len(), + "reliability state reopened successfully" + ); + if config.presence_enabled { + let _ = publish_presence(&presence_publisher, &presence_keys, "online") + .await; + } + reliability = Some(runtime); + } + Err(error) => { + tracing::warn!(error = %error, "periodic retry to reopen state directory failed"); + } + } + } + // Slot refill: spawn background tasks for empty slots whose // circuit breaker allows it. spawn_and_init runs off the main // loop so it never blocks event processing. @@ -3378,6 +3469,9 @@ async fn tokio_main() -> Result<()> { Some((attempt, result)) = wake_rx.recv(), if config.lazy_pool && !pool_ready => { Some(PoolEvent::Wake(attempt, result)) } + Some(ack) = notice_ack_rx.recv() => { + Some(PoolEvent::NoticeAck(ack)) + } // Gated on pending work: with an empty queue there is nothing // for the retry to dispatch, and a past `retry_at` would // otherwise complete instantly on every iteration (busy spin). @@ -3937,6 +4031,25 @@ async fn tokio_main() -> Result<()> { } None } + _ = probe_timer.tick() => { + let _ = result_rx; + if probe_timer.is_valid_wake(reliability.as_ref()) { + if pool_ready && queue.has_flushable_work() { + for (scope, thread_tags) in dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + reliability.as_mut(), + ) { + typing_channels.insert(scope, thread_tags); + } + } + } else { + tracing::debug!("probe timer wake ignored: generation changed"); + } + None + } _ = shutdown_rx.changed() => { tracing::info!("shutting down"); break; @@ -4207,6 +4320,7 @@ async fn tokio_main() -> Result<()> { pool = pool_lifecycle .take_ready() .expect("successful wake stores a ready pool"); + pool.set_notice_ack_tx(notice_ack_tx.clone()); pool_ready = true; emit_runtime_lifecycle( observer.as_ref(), @@ -4239,6 +4353,18 @@ async fn tokio_main() -> Result<()> { } } } + Some(PoolEvent::NoticeAck(ack)) => { + if let Some(r) = reliability.as_mut() { + match ack { + pool::NoticeAck::Pause(channel_id) => { + r.state().mark_pause_notice_consumed(channel_id); + } + pool::NoticeAck::Breaker(scope) => { + r.state().mark_breaker_notice_consumed(&scope); + } + } + } + } None => {} // relay/heartbeat/shutdown branches handled inline above } } @@ -4622,6 +4748,30 @@ fn dispatch_pending( // release them at the end so `flush_next` cannot re-pick them mid-loop; // releasing requeues them so the next dispatch (when the owner returns) // reuses that exact session instead of forking a duplicate. + if reliability.is_none() { + tracing::warn!( + "reliability state unavailable — refusing to dispatch work without durable reliability" + ); + return Vec::new(); + } + let mut is_pause_probe = false; + if let Some(reliability) = reliability.as_deref_mut() { + let now = chrono::Utc::now(); + match reliability.state().pause_gate(now) { + reliability::PauseGate::Held { until } => { + tracing::debug!( + %until, + "holding all batches — agent paused until the provider reset" + ); + return Vec::new(); + } + reliability::PauseGate::Probe => { + is_pause_probe = true; + tracing::info!("pause expired — selecting one batch as the probe"); + } + reliability::PauseGate::Open => {} + } + } let mut held: Vec = Vec::new(); loop { let batch = match queue.flush_next() { @@ -4630,33 +4780,13 @@ fn dispatch_pending( }; let channel_id = batch.channel_id; let scope = batch.scope.clone(); - // T16 gating. While the agent is paused (a provider capacity limit) or - // a scope's breaker is open, nothing runs but the probe. Held batches - // stay flushed-out so `flush_next` cannot re-pick them in this loop, - // and are returned to the queue at the end — the same mechanism the - // busy-session-owner hold uses. + // T16 gating. While a scope's breaker is open, nothing runs for that + // scope but its probe. Held batches stay flushed-out so `flush_next` + // cannot re-pick them in this loop, and are returned to the queue at + // the end — the same mechanism the busy-session-owner hold uses. + let mut is_breaker_probe = false; if let Some(reliability) = reliability.as_deref_mut() { let now = chrono::Utc::now(); - match reliability.state().pause_gate(now) { - reliability::PauseGate::Held { until } => { - tracing::debug!( - channel = %channel_id, - scope = %scope.telemetry_label(), - %until, - "holding batch — agent paused until the provider reset" - ); - held.push(batch); - continue; - } - reliability::PauseGate::Probe => { - tracing::info!( - channel = %channel_id, - scope = %scope.telemetry_label(), - "pause expired — sending one batch as the probe" - ); - } - reliability::PauseGate::Open => {} - } match reliability.state().breaker_gate(&scope, now) { reliability::BreakerGate::Held { next_probe } => { tracing::debug!( @@ -4669,6 +4799,7 @@ fn dispatch_pending( continue; } reliability::BreakerGate::Probe => { + is_breaker_probe = true; tracing::info!( channel = %channel_id, scope = %scope.telemetry_label(), @@ -4682,6 +4813,11 @@ fn dispatch_pending( // is checked out (busy on another turn), hold the batch rather than let // an idle worker open a second session for the same thread. if pool.should_hold_for_busy_owner(&scope) { + if let Some(reliability) = reliability.as_deref_mut() { + if is_breaker_probe { + reliability.state().release_breaker_probe(&scope); + } + } tracing::debug!( channel = %channel_id, scope = %scope.telemetry_label(), @@ -4702,6 +4838,14 @@ fn dispatch_pending( let mut agent = match pool.try_claim(Some(&scope)) { Some(a) => a, None => { + if let Some(reliability) = reliability.as_deref_mut() { + if is_pause_probe { + reliability.state().release_pause_probe(); + } + if is_breaker_probe { + reliability.state().release_breaker_probe(&scope); + } + } let pending = queue.pending_channels(); tracing::debug!(pending_channels = pending, "pool_exhausted"); queue.requeue_preserve_timestamps(batch); @@ -4790,6 +4934,15 @@ fn dispatch_pending( } dispatched_channels.push((scope, typing_scope)); *last_activity = tokio::time::Instant::now(); + if is_pause_probe { + // Exactly one bounded probe batch is selected for a pause probe. + break; + } + } + if is_pause_probe && dispatched_channels.is_empty() { + if let Some(reliability) = reliability { + reliability.state().release_pause_probe(); + } } // Release held batches back to the queue (owner busy). They were flushed // out (in-flight) so they could not be re-picked above; requeue preserves @@ -4798,7 +4951,7 @@ fn dispatch_pending( for batch in held { let scope = batch.scope.clone(); queue.requeue_preserve_timestamps(batch); - queue.mark_complete(scope); + queue.mark_complete_preserving_retries(scope); } tracing::debug!( dispatched = dispatched_channels.len(), @@ -4845,6 +4998,17 @@ fn spawn_failure_notice( rest_client: Option<&relay::RestClient>, batch: &FlushBatch, content: String, +) { + spawn_failure_notice_with_ack(rest_client, batch, content, None); +} + +/// Spawn a task that posts a failure notice to the relay and sends an +/// acknowledgment over `ack` upon success. +fn spawn_failure_notice_with_ack( + rest_client: Option<&relay::RestClient>, + batch: &FlushBatch, + content: String, + ack: Option<(mpsc::UnboundedSender, pool::NoticeAck)>, ) { if let Some(rest) = rest_client { let thread_tags = batch @@ -4855,15 +5019,20 @@ fn spawn_failure_notice( let rest = rest.clone(); let channel_id = batch.channel_id; tokio::spawn(async move { - pool::post_failure_notice(&rest, channel_id, &thread_tags, &content).await; + let ok = pool::post_failure_notice(&rest, channel_id, &thread_tags, &content).await; + if ok { + if let Some((ack_tx, ack_val)) = ack { + let _ = ack_tx.send(ack_val); + } + } }); } } /// What the reliability path did with a failed batch. enum Disposition { - /// The reliability path took ownership of the batch. Nothing else runs. - Handled, + /// The reliability path took ownership of the batch. + Handled { preserve_retries: bool }, /// Not a reliability case; the batch goes back to the pre-existing /// requeue path. Fallthrough(FlushBatch), @@ -4874,13 +5043,15 @@ enum Disposition { /// Ordering: the park file is written and fsynced **before** the batch is /// dropped, so a crash between the two leaves the batch in the queue's park /// hand-off (still in memory, retried on the next tick), never nowhere. +#[allow(clippy::too_many_arguments)] fn apply_reliability( reliability: &mut reliability::ReliabilityRuntime, - queue: &mut EventQueue, + _queue: &mut EventQueue, batch: FlushBatch, outcome: &PromptOutcome, started: bool, rest_client: Option<&relay::RestClient>, + notice_ack_tx: Option>, now: chrono::DateTime, ) -> Disposition { use reliability::ledger::{self as led, LedgerBody}; @@ -4943,29 +5114,34 @@ fn apply_reliability( waiting, }), ); - // No retry is spent on a pause: the events go back with - // their original timestamps and no backoff. let channel_id = batch.channel_id; let notice_batch = batch.clone(); - queue.requeue_preserve_timestamps(batch); - if reliability.state().claim_pause_notice(channel_id) { - spawn_failure_notice( + if reliability.state().pause_needs_notice(channel_id) { + let ack = notice_ack_tx + .clone() + .map(|tx| (tx, pool::NoticeAck::Pause(channel_id))); + spawn_failure_notice_with_ack( rest_client, ¬ice_batch, reliability::notices::pause("", until, waiting), + ack, ); } - Disposition::Handled + park_or_fallthrough( + reliability, + batch, + reliability::ParkReason::Pause, + false, + None, + now, + ) } reliability::Action::OpenBreaker => { let consecutive = reliability .state_ref() .breaker_consecutive(&scope) .unwrap_or(reliability::state::BREAKER_THRESHOLD); - let first_open = reliability - .state_ref() - .breaker_opened_at(&scope) - .is_some_and(|opened| opened == now); + let needs_notice = reliability.state_ref().breaker_needs_notice(&scope); reliability.record( now, LedgerBody::BreakerOpened(led::BreakerOpened { @@ -4974,15 +5150,25 @@ fn apply_reliability( }), ); let notice_batch = batch.clone(); - queue.requeue_preserve_timestamps(batch); - if first_open { - spawn_failure_notice( + if needs_notice { + let ack = notice_ack_tx + .clone() + .map(|tx| (tx, pool::NoticeAck::Breaker(scope.clone()))); + spawn_failure_notice_with_ack( rest_client, ¬ice_batch, reliability::notices::breaker(""), + ack, ); } - Disposition::Handled + park_or_fallthrough( + reliability, + batch, + reliability::ParkReason::BreakerOpen, + false, + None, + now, + ) } } } @@ -5010,7 +5196,11 @@ fn park_or_fallthrough( reliability::notices::parked(reason.as_str()) }; spawn_failure_notice(rest_client, &batch, content); - Disposition::Handled + let preserve_retries = matches!( + reason, + reliability::ParkReason::Pause | reliability::ParkReason::BreakerOpen + ); + Disposition::Handled { preserve_retries } } Err(error) => { tracing::error!( @@ -5039,7 +5229,8 @@ fn drain_park_handoff( let reason = match handoff.reason { queue::ParkHandoffReason::RetriesExhausted => reliability::ParkReason::RetriesExhausted, }; - match reliability.park_batch(&handoff.batch, reason, false, now) { + let started = handoff.batch.is_started(); + match reliability.park_batch(&handoff.batch, reason, started, now) { Ok(()) => { spawn_failure_notice( rest_client, @@ -5205,6 +5396,7 @@ fn handle_prompt_result( // Ownership note: `reliability` is threaded through as an Option so the // existing unit tests can drive `handle_prompt_result` without a state // directory. Production always passes Some. + let mut preserve_retries = false; let mut reliability = reliability; if let Some(batch) = result.batch.take() { // Don't requeue batches for channels the agent was removed from — @@ -5217,7 +5409,9 @@ fn handle_prompt_result( result.outcome, PromptOutcome::Cancelled | PromptOutcome::CancelDrainTimeout(_) ) { - Some(batch) + let reason = batch.cancel_reason.unwrap_or(CancelReason::Steer); + queue.requeue_as_cancelled(batch, reason); + None } else if let Some(reliability) = reliability.as_deref_mut() { match apply_reliability( reliability, @@ -5226,10 +5420,42 @@ fn handle_prompt_result( &result.outcome, turn_started, rest_client, + pool.notice_ack_tx(), now, ) { - Disposition::Handled => None, - Disposition::Fallthrough(batch) => Some(batch), + Disposition::Handled { + preserve_retries: pr, + } => { + preserve_retries = pr; + None + } + Disposition::Fallthrough(batch) => { + // Finding 1: Fallthrough from a failed park (or Action::Retry) + // always re-enters queue.requeue / hand-off rather than the + // discard arms keyed purely on outcome shape. + if let Some(dead) = queue.requeue(batch) { + let reason = match &result.outcome { + PromptOutcome::Timeout(TimeoutKind::Idle) => { + "the turn timed out".to_string() + } + PromptOutcome::Timeout(TimeoutKind::Hard { .. }) => { + "the turn exceeded the maximum duration".to_string() + } + PromptOutcome::Error(e) => { + reliability::sanitize_error_diagnostic(&e.to_string()) + } + PromptOutcome::ProjectContextIndeterminate(reason) => { + reliability::sanitize_error_diagnostic(reason) + } + _ => "repeated failures".to_string(), + }; + let content = format!( + "⚠️ I couldn't process the last request after multiple retries ({reason}). Please re-send if it's still needed." + ); + spawn_failure_notice(rest_client, &dead, content); + } + None + } } } else { Some(batch) @@ -5319,8 +5545,12 @@ fn handle_prompt_result( "the turn exceeded the maximum duration".to_string() } PromptOutcome::AgentExited => "the agent process exited".to_string(), - PromptOutcome::Error(e) => format!("{e}"), - PromptOutcome::ProjectContextIndeterminate(reason) => reason.clone(), + PromptOutcome::Error(e) => { + reliability::sanitize_error_diagnostic(&e.to_string()) + } + PromptOutcome::ProjectContextIndeterminate(reason) => { + reliability::sanitize_error_diagnostic(reason) + } _ => "repeated failures".to_string(), }; let content = format!( @@ -5346,7 +5576,13 @@ fn handle_prompt_result( } match &result.source { - PromptSource::Channel(scope) => queue.mark_complete(scope.clone()), + PromptSource::Channel(scope) => { + if preserve_retries { + queue.mark_complete_preserving_retries(scope.clone()); + } else { + queue.mark_complete(scope.clone()); + } + } PromptSource::Heartbeat => *heartbeat_in_flight = false, } @@ -5534,13 +5770,14 @@ fn handle_prompt_result( pool.return_agent(result.agent); } PromptOutcome::ProjectContextIndeterminate(reason) => { + let diag = reliability::sanitize_error_diagnostic(&reason); tracing::warn!( agent = agent_index, outcome = outcome_label, - reason, + reason = %diag, "agent_returned (local project context indeterminate — pipe intact)" ); - emit_turn_error(&reason, None); + emit_turn_error(&diag, None); pool.return_agent(result.agent); } PromptOutcome::Error(ref e) => { @@ -5555,16 +5792,17 @@ fn handle_prompt_result( acp::AcpError::AgentError { code, .. } => Some(*code), _ => None, }; + let diag = reliability::sanitize_error_diagnostic(&e.to_string()); if is_transport_error { tracing::warn!( agent = agent_index, outcome = outcome_label, configured_model = %harness_configured_model, pid = harness_pid, - error = %e, + error = %diag, "transport/protocol error — respawning agent" ); - emit_turn_error(&e.to_string(), error_code); + emit_turn_error(&diag, error_code); let index = result.agent.index; let slot_history = &mut crash_history[index]; @@ -5587,10 +5825,10 @@ fn handle_prompt_result( outcome = outcome_label, configured_model = %harness_configured_model, pid = harness_pid, - error = %e, + error = %diag, "agent_returned (application error — pipe intact)" ); - emit_turn_error(&e.to_string(), error_code); + emit_turn_error(&diag, error_code); pool.return_agent(result.agent); } } @@ -10915,7 +11153,7 @@ mod error_outcome_emission_tests { use nostr::{EventBuilder, Keys, Kind}; use std::collections::HashSet; - fn test_config() -> Config { + pub(crate) fn test_config() -> Config { Config { keys: nostr::Keys::generate(), relay_url: "ws://localhost:3000".into(), @@ -10989,7 +11227,7 @@ mod error_outcome_emission_tests { /// Spawn a real but inert agent subprocess (`cat`) so the error paths have /// an `OwnedAgent` to move into respawn or return to the pool. The error /// branches never talk to the subprocess. - async fn dummy_agent(index: usize) -> OwnedAgent { + pub(crate) async fn dummy_agent(index: usize) -> OwnedAgent { OwnedAgent { index, acp: AcpClient::spawn("cat", &[], &[], false) @@ -11662,6 +11900,7 @@ mod error_outcome_emission_tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } }; @@ -11774,6 +12013,7 @@ mod error_outcome_emission_tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } }; @@ -11900,6 +12140,7 @@ mod error_outcome_emission_tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let result = PromptResult { started: false, @@ -11998,6 +12239,7 @@ mod error_outcome_emission_tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let result = PromptResult { started: false, @@ -12084,6 +12326,7 @@ mod error_outcome_emission_tests { }], cancelled_events: vec![], cancel_reason: Some(CancelReason::Steer), + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let agent = dummy_agent(0).await; @@ -12360,6 +12603,7 @@ mod error_outcome_emission_tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let mut agent = dummy_agent(0).await; @@ -12516,6 +12760,7 @@ mod error_outcome_emission_tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let auth_error = acp::AcpError::AgentError { @@ -12607,6 +12852,7 @@ mod error_outcome_emission_tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; // Usage-credits error — AgentError but NOT an auth error. @@ -12927,3 +13173,1098 @@ mod observer_payload_trim_tests { assert!(leaf.contains("[elided")); } } + +#[cfg(test)] +mod reliability_dispatch_tests { + use super::*; + + fn make_test_prompt_context() -> PromptContext { + let agent_keys = nostr::Keys::generate(); + PromptContext { + mcp_servers: crate::McpServerSet::from_servers(vec![]), + initial_message: None, + idle_timeout: std::time::Duration::from_secs(60), + max_turn_duration: std::time::Duration::from_secs(120), + turn_liveness_interval: std::time::Duration::ZERO, + dedup_mode: config::DedupMode::Drop, + system_prompt: None, + session_title: None, + team_instructions: None, + heartbeat_prompt: None, + base_prompt: None, + cwd: ".".to_string(), + rest_client: relay::RestClient { + http: reqwest::Client::new(), + base_url: "http://127.0.0.1:0".to_string(), + keys: agent_keys.clone(), + auth_tag_json: None, + }, + channel_info: pool::ChannelInfoResolver::new( + std::collections::HashMap::new(), + relay::RestClient { + http: reqwest::Client::new(), + base_url: "http://127.0.0.1:0".to_string(), + keys: agent_keys.clone(), + auth_tag_json: None, + }, + ), + context_message_limit: 0, + max_turns_per_session: 0, + permission_mode: config::PermissionMode::Default, + agent_keys, + agent_owner_pubkey: None, + memory_enabled: false, + harness_name: "test".to_string(), + relay_url: "http://127.0.0.1:0".to_string(), + } + } + + #[tokio::test] + async fn test_dispatch_pending_does_not_leak_probe_permit_on_hold() { + let dir = tempfile::tempdir().unwrap(); + let now = chrono::Utc::now(); + let mut runtime = + reliability::ReliabilityRuntime::open_in(dir.path(), "test-agent", now).unwrap(); + + let channel_id = uuid::Uuid::new_v4(); + let scope = scope::SessionScope::Conversation { channel_id }; + + // Set pause expired in the past: 40 mins ago, reset was at 30 mins (10 mins ago). + let t0 = now - chrono::Duration::minutes(40); + runtime.state().on_failure( + &scope, + reliability::ErrorClass::CapacityExhausted { + resets_at: Some(t0 + chrono::Duration::minutes(30)), + }, + t0, + ); + + // Queue has a batch ready to dispatch. + let mut queue = EventQueue::new(config::DedupMode::Queue); + let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "probe event") + .tags([]) + .sign_with_keys(&nostr::Keys::generate()) + .unwrap(); + queue.push(queue::QueuedEvent { + channel_id, + scope: scope.clone(), + event, + received_at: std::time::Instant::now(), + prompt_tag: "p".into(), + }); + + // Pool has no available workers (simulating a hold / pool exhausted). + let mut pool = AgentPool::from_slots(vec![]); + let ctx = std::sync::Arc::new(make_test_prompt_context()); + let mut last_activity = tokio::time::Instant::now(); + + // Calling dispatch_pending encounters the probe, but holds because pool is exhausted. + let dispatched = dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + Some(&mut runtime), + ); + assert!( + dispatched.is_empty(), + "no tasks should be dispatched with empty pool" + ); + + // A subsequent pause_gate(now) call MUST still return Probe (not stuck Held). + assert_eq!( + runtime.state().pause_gate(now), + reliability::PauseGate::Probe, + "probe permit must not be leaked on hold" + ); + } + + #[tokio::test] + #[cfg(unix)] + async fn test_park_failure_does_not_discard_batch_on_hard_timeout_or_auth() { + use crate::error_outcome_emission_tests::{dummy_agent, test_config}; + use crate::queue::BatchEvent; + use std::os::unix::fs::PermissionsExt; + + let check = |outcome: PromptOutcome| async move { + let dir = tempfile::tempdir().unwrap(); + let now = chrono::Utc::now(); + let mut runtime = + reliability::ReliabilityRuntime::open_in(dir.path(), "test-agent", now).unwrap(); + + let channel_id = uuid::Uuid::new_v4(); + let scope = scope::SessionScope::Conversation { channel_id }; + + let keys = nostr::Keys::generate(); + let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "test") + .tags([]) + .sign_with_keys(&keys) + .unwrap(); + let batch = FlushBatch { + batch_id: uuid::Uuid::new_v4(), + channel_id, + scope: scope.clone(), + events: vec![BatchEvent { + event, + prompt_tag: "test".into(), + received_at: std::time::Instant::now(), + }], + cancelled_events: vec![], + cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let agent = dummy_agent(0).await; + let mut pool = AgentPool::from_slots(vec![None]); + let task_id = pool.join_set.spawn(async {}).id(); + pool.task_map_mut().insert( + task_id, + crate::pool::TaskMeta { + agent_index: 0, + channel_id: None, + scope: None, + turn_id: "test-turn-id".to_string(), + recoverable_batch: None, + control_tx: None, + steer_tx: None, + successful_steer_deliveries: std::collections::HashSet::new(), + }, + ); + + let mut queue = EventQueue::new(config::DedupMode::Queue); + let config = test_config(); + let mut heartbeat_in_flight = false; + let removed_channels = std::collections::HashSet::new(); + let mut crash_history = vec![SlotCircuit { + crash_times: Vec::new(), + open_until: None, + respawn_in_flight: false, + }]; + let (respawn_tx, _respawn_rx) = tokio::sync::mpsc::channel(8); + let mut respawn_tasks = tokio::task::JoinSet::new(); + + let result = PromptResult { + started: false, + agent, + source: PromptSource::Channel(scope.clone()), + turn_id: "test-turn-id".to_string(), + outcome, + batch: Some(batch), + }; + + // Make the state dir unwritable so park_batch fails. + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o500)).unwrap(); + + handle_prompt_result( + &mut pool, + &mut queue, + &config, + result, + &mut heartbeat_in_flight, + &removed_channels, + &mut crash_history, + &respawn_tx, + &mut respawn_tasks, + None, + None, + Some(&mut runtime), + ); + + // Restore permissions for tempdir cleanup. + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + + // Assert the batch is still present in the queue or park hand-off, never dropped. + let in_queue = queue.queued_event_count(channel_id) > 0; + let in_handoff = queue.has_parked_handoff(); + assert!( + in_queue || in_handoff, + "batch must be present in queue or park hand-off, but was dropped" + ); + }; + + // Case 1: Hard timeout with recently_active = false + check(PromptOutcome::Timeout(pool::TimeoutKind::Hard { + recently_active: false, + })) + .await; + + // Case 2: Bare auth error + check(PromptOutcome::Error(acp::AcpError::AgentError { + code: -32000, + message: "API Error: 401 Unauthorized".to_string(), + })) + .await; + } + + #[tokio::test] + async fn test_pause_held_batch_is_durable_across_restart() { + use crate::error_outcome_emission_tests::{dummy_agent, test_config}; + use crate::queue::BatchEvent; + + let dir = tempfile::tempdir().unwrap(); + let pubkey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let now = chrono::Utc::now(); + let mut runtime = + reliability::ReliabilityRuntime::open_in(dir.path(), pubkey, now).unwrap(); + + let channel_id = uuid::Uuid::new_v4(); + let scope = scope::SessionScope::Conversation { channel_id }; + + let keys = nostr::Keys::generate(); + let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "held message") + .tags([]) + .sign_with_keys(&keys) + .unwrap(); + let batch_id = uuid::Uuid::new_v4(); + let batch = FlushBatch { + batch_id, + channel_id, + scope: scope.clone(), + events: vec![BatchEvent { + event, + prompt_tag: "test".into(), + received_at: std::time::Instant::now(), + }], + cancelled_events: vec![], + cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + let agent = dummy_agent(0).await; + let mut pool = AgentPool::from_slots(vec![None]); + let task_id = pool.join_set.spawn(async {}).id(); + pool.task_map_mut().insert( + task_id, + crate::pool::TaskMeta { + agent_index: 0, + channel_id: None, + scope: None, + turn_id: "test-turn-id".to_string(), + recoverable_batch: None, + control_tx: None, + steer_tx: None, + successful_steer_deliveries: std::collections::HashSet::new(), + }, + ); + + let mut queue = EventQueue::new(config::DedupMode::Queue); + let config = test_config(); + let mut heartbeat_in_flight = false; + let removed_channels = std::collections::HashSet::new(); + let mut crash_history = vec![SlotCircuit { + crash_times: Vec::new(), + open_until: None, + respawn_in_flight: false, + }]; + let (respawn_tx, _respawn_rx) = tokio::sync::mpsc::channel(8); + let mut respawn_tasks = tokio::task::JoinSet::new(); + + // Error that triggers Action::Pause: session limit resets at 4:20am + let outcome = PromptOutcome::Error(acp::AcpError::AgentError { + code: -32603, + message: "Internal error: You've hit your session limit · resets 4:20am (America/Los_Angeles)".to_string(), + }); + + let result = PromptResult { + started: false, + agent, + source: PromptSource::Channel(scope.clone()), + turn_id: "test-turn-id".to_string(), + outcome, + batch: Some(batch), + }; + + handle_prompt_result( + &mut pool, + &mut queue, + &config, + result, + &mut heartbeat_in_flight, + &removed_channels, + &mut crash_history, + &respawn_tx, + &mut respawn_tasks, + None, + None, + Some(&mut runtime), + ); + + // Process restarts: drop runtime, queue, and simulated relay state + drop(runtime); + drop(queue); + + let restart_now = now + chrono::Duration::seconds(10); + let restarted = + reliability::ReliabilityRuntime::open_in(dir.path(), pubkey, restart_now).unwrap(); + + // The held message must be recoverable (present in park file) + assert!( + restarted.park().contains(batch_id), + "held message must be durable in park file across restart, not silently gone" + ); + } + + #[tokio::test] + #[cfg(unix)] + async fn test_state_dir_failure_refuses_work_and_picks_up_on_reopen() { + use crate::error_outcome_emission_tests::dummy_agent; + use std::os::unix::fs::PermissionsExt; + + let parent = tempfile::tempdir().unwrap(); + let state_dir = parent.path().join("state"); + let pubkey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let now = chrono::Utc::now(); + + // 1. Start with an unwritable state dir (parent is read-only) + std::fs::set_permissions(parent.path(), std::fs::Permissions::from_mode(0o500)).unwrap(); + let initial_open = reliability::ReliabilityRuntime::open_in(&state_dir, pubkey, now); + assert!( + initial_open.is_err(), + "open must fail when state dir is unwritable" + ); + + // 2. Setup queue with pending work and an agent ready in the pool + let mut queue = EventQueue::new(config::DedupMode::Queue); + let channel_id = uuid::Uuid::new_v4(); + let scope = scope::SessionScope::Conversation { channel_id }; + let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "work") + .tags([]) + .sign_with_keys(&nostr::Keys::generate()) + .unwrap(); + queue.push(queue::QueuedEvent { + channel_id, + scope: scope.clone(), + event, + received_at: std::time::Instant::now(), + prompt_tag: "p".into(), + }); + + let agent = dummy_agent(0).await; + let mut pool = AgentPool::from_slots(vec![Some(agent)]); + let ctx = std::sync::Arc::new(make_test_prompt_context()); + let mut last_activity = tokio::time::Instant::now(); + + // 3. Dispatching with reliability = None MUST refuse to dispatch work + let dispatched = dispatch_pending(&mut pool, &mut queue, &ctx, &mut last_activity, None); + assert!( + dispatched.is_empty(), + "must not dispatch work when reliability state is unavailable" + ); + assert_eq!( + queue.queued_event_count(channel_id), + 1, + "work must remain in the queue rather than being accepted/discarded" + ); + + // 4. Later, the directory is made writable + std::fs::set_permissions(parent.path(), std::fs::Permissions::from_mode(0o700)).unwrap(); + let mut runtime = reliability::ReliabilityRuntime::open_in(&state_dir, pubkey, now) + .expect("reopen must succeed once dir is writable"); + + // 5. Work is now picked up and dispatched + let dispatched = dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + Some(&mut runtime), + ); + assert!( + !dispatched.is_empty(), + "work must be dispatched once reliability state is open" + ); + assert_eq!(queue.queued_event_count(channel_id), 0); + } + + #[tokio::test] + async fn test_probe_timer_fires_without_external_relay_event() { + use crate::error_outcome_emission_tests::dummy_agent; + let dir = tempfile::tempdir().unwrap(); + let now = chrono::Utc::now(); + let pubkey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let mut runtime = + reliability::ReliabilityRuntime::open_in(dir.path(), pubkey, now).unwrap(); + + let channel_id = uuid::Uuid::new_v4(); + let scope = scope::SessionScope::Conversation { channel_id }; + + // 1. Enter Paused with a short until (50ms in future) + let until = now + chrono::Duration::milliseconds(50); + runtime.state().on_failure( + &scope, + reliability::ErrorClass::CapacityExhausted { + resets_at: Some(until), + }, + now, + ); + + // 2. Queue work for scope + let mut queue = EventQueue::new(config::DedupMode::Queue); + let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "work") + .tags([]) + .sign_with_keys(&nostr::Keys::generate()) + .unwrap(); + queue.push(queue::QueuedEvent { + channel_id, + scope: scope.clone(), + event, + received_at: std::time::Instant::now(), + prompt_tag: "p".into(), + }); + + // 3. Pool has an agent ready + let agent = dummy_agent(0).await; + let mut pool = AgentPool::from_slots(vec![Some(agent)]); + let ctx = std::sync::Arc::new(make_test_prompt_context()); + let mut last_activity = tokio::time::Instant::now(); + + // 4. All optional timers disabled: + let mut heartbeat: Option = None; + let mut presence_heartbeat: Option = None; + let mut typing_refresh: Option = None; + let mut inactivity_reaper: Option = None; + let mut idle_pool_sleep_reaper: Option = None; + + // 5. Probe timer arm is armed + let mut probe_timer = ProbeTimerArm::new(); + probe_timer.rearm(Some(&mut runtime)); + + // 6. Run select with no external relay event + let dispatched = tokio::time::timeout(std::time::Duration::from_millis(500), async { + tokio::select! { + _ = async { + match heartbeat.as_mut() { + Some(t) => t.tick().await, + None => std::future::pending().await, + } + } => false, + _ = async { + match presence_heartbeat.as_mut() { + Some(t) => t.tick().await, + None => std::future::pending().await, + } + } => false, + _ = async { + match typing_refresh.as_mut() { + Some(t) => t.tick().await, + None => std::future::pending().await, + } + } => false, + _ = async { + match inactivity_reaper.as_mut() { + Some(t) => t.tick().await, + None => std::future::pending().await, + } + } => false, + _ = async { + match idle_pool_sleep_reaper.as_mut() { + Some(t) => t.tick().await, + None => std::future::pending().await, + } + } => false, + _ = std::future::pending::<()>() => false, // no external relay event + _ = probe_timer.tick() => { + if probe_timer.is_valid_wake(Some(&runtime)) { + let res = dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + Some(&mut runtime), + ); + !res.is_empty() + } else { + false + } + } + } + }) + .await + .expect("probe timer must fire without external relay event before timeout"); + + assert!(dispatched, "probe must have been dispatched"); + assert_eq!( + queue.queued_event_count(channel_id), + 0, + "queued event should have been dispatched" + ); + } + + #[tokio::test] + async fn test_dispatch_pending_short_circuits_global_pause_in_o1() { + use crate::error_outcome_emission_tests::dummy_agent; + let dir = tempfile::tempdir().unwrap(); + let now = chrono::Utc::now(); + let pubkey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let mut runtime = + reliability::ReliabilityRuntime::open_in(dir.path(), pubkey, now).unwrap(); + + // 1. Enter Paused with until in the future (30 minutes) + let until = now + chrono::Duration::minutes(30); + let channel_id_0 = uuid::Uuid::new_v4(); + let scope_0 = scope::SessionScope::Conversation { + channel_id: channel_id_0, + }; + runtime.state().on_failure( + &scope_0, + reliability::ErrorClass::CapacityExhausted { + resets_at: Some(until), + }, + now, + ); + + // 2. Queue work for 50 distinct scopes + let mut queue = EventQueue::new(config::DedupMode::Queue); + for _ in 0..50 { + let ch = uuid::Uuid::new_v4(); + let sc = scope::SessionScope::Conversation { channel_id: ch }; + let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "work") + .tags([]) + .sign_with_keys(&nostr::Keys::generate()) + .unwrap(); + queue.push(queue::QueuedEvent { + channel_id: ch, + scope: sc, + event, + received_at: std::time::Instant::now(), + prompt_tag: "p".into(), + }); + } + assert_eq!(queue.pending_channels(), 50); + + let agent = dummy_agent(0).await; + let mut pool = AgentPool::from_slots(vec![Some(agent)]); + let ctx = std::sync::Arc::new(make_test_prompt_context()); + let mut last_activity = tokio::time::Instant::now(); + + let flushes_before = queue.flush_count(); + let dispatched = dispatch_pending( + &mut pool, + &mut queue, + &ctx, + &mut last_activity, + Some(&mut runtime), + ); + + assert!( + dispatched.is_empty(), + "no work should be dispatched during pause" + ); + let flushes_after = queue.flush_count(); + // Without fix, flush_count increments by 51 (O(scopes)). With O(1) short-circuit, it increments by 0. + assert_eq!( + flushes_after - flushes_before, + 0, + "dispatch_pending must short-circuit without calling flush_next when paused" + ); + assert_eq!(queue.pending_channels(), 50, "all 50 scopes remain queued"); + } + + #[tokio::test] + async fn test_panicked_agent_after_output_parks_with_started_true_and_needs_review() { + let dir = tempfile::tempdir().unwrap(); + let now = chrono::Utc::now(); + let pubkey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let mut runtime = + reliability::ReliabilityRuntime::open_in(dir.path(), pubkey, now).unwrap(); + + let mut queue = EventQueue::new(config::DedupMode::Queue); + let channel_id = uuid::Uuid::new_v4(); + let scope = scope::SessionScope::Conversation { channel_id }; + + let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "work") + .tags([]) + .sign_with_keys(&nostr::Keys::generate()) + .unwrap(); + queue.push(queue::QueuedEvent { + channel_id, + scope: scope.clone(), + event, + received_at: std::time::Instant::now(), + prompt_tag: "p".into(), + }); + + let batch = queue.flush_next().unwrap(); + // Agent started turn and emitted output before panicking + batch.mark_started(); + assert!(batch.is_started()); + + // Exhaust retries: MAX_RETRIES attempts, then the next requeue moves it to parked_out + for _ in 0..queue::MAX_RETRIES { + let _ = queue.requeue(batch.clone()); + } + let exhausted = queue.requeue(batch.clone()); + assert!( + exhausted.is_none(), + "requeue must return None when retries are exhausted" + ); + + // Drain park handoff + drain_park_handoff(&mut runtime, &mut queue, None, now); + + let parked = runtime.park().batches(); + assert_eq!(parked.len(), 1, "exactly one batch should be parked"); + let parked_batch = &parked[0]; + assert!( + parked_batch.started, + "parked batch must have started == true" + ); + assert!( + parked_batch.needs_review, + "parked batch must have needs_review == true" + ); + assert_eq!( + parked_batch.needs_review_reason.as_deref(), + Some("interrupted after it had started") + ); + assert!( + !parked_batch.replay_eligible(), + "parked batch that started must not be replay-eligible" + ); + } + + #[tokio::test] + async fn test_failure_notice_not_consumed_until_ack_received() { + use crate::error_outcome_emission_tests::dummy_agent; + + let dir = tempfile::tempdir().unwrap(); + let now = chrono::Utc::now(); + let pubkey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let mut runtime = + reliability::ReliabilityRuntime::open_in(dir.path(), pubkey, now).unwrap(); + + let channel_id = uuid::Uuid::new_v4(); + let scope = scope::SessionScope::Conversation { channel_id }; + + let mut queue = EventQueue::new(config::DedupMode::Queue); + let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "work") + .tags([]) + .sign_with_keys(&nostr::Keys::generate()) + .unwrap(); + queue.push(queue::QueuedEvent { + channel_id, + scope: scope.clone(), + event, + received_at: std::time::Instant::now(), + prompt_tag: "p".into(), + }); + let batch = queue.flush_next().unwrap(); + + let mut pool = AgentPool::from_slots(vec![None]); + let task_id = pool.join_set.spawn(async {}).id(); + pool.task_map_mut().insert( + task_id, + crate::pool::TaskMeta { + agent_index: 0, + channel_id: Some(channel_id), + scope: Some(scope.clone()), + turn_id: "test-turn-id".to_string(), + recoverable_batch: None, + control_tx: None, + steer_tx: None, + successful_steer_deliveries: std::collections::HashSet::new(), + }, + ); + let (ack_tx, mut _ack_rx) = tokio::sync::mpsc::unbounded_channel(); + pool.set_notice_ack_tx(ack_tx); + + let agent_for_result = dummy_agent(0).await; + let result = PromptResult { + started: false, + agent: agent_for_result, + source: PromptSource::Channel(scope.clone()), + turn_id: "test-turn-id".to_string(), + outcome: PromptOutcome::Error(acp::AcpError::AgentError { + code: 429, + message: "rate limit exceeded".into(), + }), + batch: Some(batch), + }; + + let config = super::error_outcome_emission_tests::test_config(); + let mut heartbeat_in_flight = false; + let removed_channels = HashSet::new(); + let mut crash_history = Vec::new(); + let (respawn_tx, _respawn_rx) = tokio::sync::mpsc::channel(1); + let mut respawn_tasks = tokio::task::JoinSet::new(); + + handle_prompt_result( + &mut pool, + &mut queue, + &config, + result, + &mut heartbeat_in_flight, + &removed_channels, + &mut crash_history, + &respawn_tx, + &mut respawn_tasks, + None, + None, + Some(&mut runtime), + ); + + // Before ack is consumed, pause_needs_notice must still be true! + assert!( + runtime.state().pause_needs_notice(channel_id), + "pause_needs_notice must remain true until notice is successfully posted and acked" + ); + + // Once ack arrives, consume notice and verify pause_needs_notice becomes false + runtime.state().mark_pause_notice_consumed(channel_id); + assert!( + !runtime.state().pause_needs_notice(channel_id), + "pause_needs_notice must be false after notice is acked and consumed" + ); + + // Verify Breaker notice behavior: + let breaker_scope = scope::SessionScope::Conversation { + channel_id: uuid::Uuid::new_v4(), + }; + for _ in 0..reliability::state::BREAKER_THRESHOLD { + runtime.state().on_failure( + &breaker_scope, + reliability::ErrorClass::ProviderInternal, + now, + ); + } + assert!( + runtime.state().breaker_needs_notice(&breaker_scope), + "breaker_needs_notice must be true when breaker opens" + ); + runtime.state().mark_breaker_notice_consumed(&breaker_scope); + assert!( + !runtime.state().breaker_needs_notice(&breaker_scope), + "breaker_needs_notice must be false after mark_breaker_notice_consumed" + ); + } + + #[tokio::test] + async fn test_retry_counts_preserved_across_pause_and_breaker() { + use crate::error_outcome_emission_tests::dummy_agent; + + let dir = tempfile::tempdir().unwrap(); + let now = chrono::Utc::now(); + let pubkey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let mut runtime = + reliability::ReliabilityRuntime::open_in(dir.path(), pubkey, now).unwrap(); + + let channel_id = uuid::Uuid::new_v4(); + let scope = scope::SessionScope::Conversation { channel_id }; + + let mut queue = EventQueue::new(config::DedupMode::Queue); + + // 1. Accumulate 2 retries on `scope` + queue.set_retry_count_for_test(&scope, 2); + assert_eq!(queue.retry_count(&scope), 2); + + // 2. Trigger Pause on the 3rd attempt + let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "work") + .tags([]) + .sign_with_keys(&nostr::Keys::generate()) + .unwrap(); + queue.push(queue::QueuedEvent { + channel_id, + scope: scope.clone(), + event: event.clone(), + received_at: std::time::Instant::now(), + prompt_tag: "p".into(), + }); + let batch3 = queue.flush_next().unwrap(); + + let mut pool = AgentPool::from_slots(vec![None]); + let task_id = pool.join_set.spawn(async {}).id(); + pool.task_map_mut().insert( + task_id, + crate::pool::TaskMeta { + agent_index: 0, + channel_id: Some(channel_id), + scope: Some(scope.clone()), + turn_id: "test-turn-id".to_string(), + recoverable_batch: None, + control_tx: None, + steer_tx: None, + successful_steer_deliveries: std::collections::HashSet::new(), + }, + ); + + let agent_for_result = dummy_agent(0).await; + let result = PromptResult { + started: false, + agent: agent_for_result, + source: PromptSource::Channel(scope.clone()), + turn_id: "test-turn-id".to_string(), + outcome: PromptOutcome::Error(acp::AcpError::AgentError { + code: 429, + message: "rate limit exceeded".into(), + }), + batch: Some(batch3), + }; + + let config = super::error_outcome_emission_tests::test_config(); + let mut heartbeat_in_flight = false; + let removed_channels = HashSet::new(); + let mut crash_history = Vec::new(); + let (respawn_tx, _respawn_rx) = tokio::sync::mpsc::channel(1); + let mut respawn_tasks = tokio::task::JoinSet::new(); + + handle_prompt_result( + &mut pool, + &mut queue, + &config, + result, + &mut heartbeat_in_flight, + &removed_channels, + &mut crash_history, + &respawn_tx, + &mut respawn_tasks, + None, + None, + Some(&mut runtime), + ); + + // Assert retry count is PRESERVED across Pause (still 2, not reset to 0) + assert_eq!( + queue.retry_count(&scope), + 2, + "retry_count must be preserved across Pause" + ); + + // 3. Resume and trigger another failure: assert retry count continues from 3 + queue.push(queue::QueuedEvent { + channel_id, + scope: scope.clone(), + event: event.clone(), + received_at: std::time::Instant::now(), + prompt_tag: "p".into(), + }); + let batch4 = queue.flush_next().unwrap(); + queue.requeue(batch4); + assert_eq!( + queue.retry_count(&scope), + 3, + "retry_count must continue from 3 after Pause, not reset to 1" + ); + + // 4. Now verify BreakerOpen preserves retry_counts on another scope + let breaker_channel_id = uuid::Uuid::new_v4(); + let breaker_scope = scope::SessionScope::Conversation { + channel_id: breaker_channel_id, + }; + queue.set_retry_count_for_test(&breaker_scope, 2); + assert_eq!(queue.retry_count(&breaker_scope), 2); + + // Fail until breaker opens: first BREAKER_THRESHOLD - 1 failures + for _ in 0..(reliability::state::BREAKER_THRESHOLD - 1) { + runtime.state().on_failure( + &breaker_scope, + reliability::ErrorClass::ProviderInternal, + now, + ); + } + + // Push and flush a batch that triggers BreakerOpen + queue.push(queue::QueuedEvent { + channel_id: breaker_channel_id, + scope: breaker_scope.clone(), + event: event.clone(), + received_at: std::time::Instant::now(), + prompt_tag: "p".into(), + }); + let batch_breaker = queue.flush_next().unwrap(); + + let task_id2 = pool.join_set.spawn(async {}).id(); + pool.task_map_mut().insert( + task_id2, + crate::pool::TaskMeta { + agent_index: 0, + channel_id: Some(breaker_channel_id), + scope: Some(breaker_scope.clone()), + turn_id: "test-turn-id-2".to_string(), + recoverable_batch: None, + control_tx: None, + steer_tx: None, + successful_steer_deliveries: std::collections::HashSet::new(), + }, + ); + + let agent2 = dummy_agent(0).await; + let result2 = PromptResult { + started: false, + agent: agent2, + source: PromptSource::Channel(breaker_scope.clone()), + turn_id: "test-turn-id-2".to_string(), + outcome: PromptOutcome::Error(acp::AcpError::AgentError { + code: 500, + message: "internal server error".into(), + }), + batch: Some(batch_breaker), + }; + + handle_prompt_result( + &mut pool, + &mut queue, + &config, + result2, + &mut heartbeat_in_flight, + &removed_channels, + &mut crash_history, + &respawn_tx, + &mut respawn_tasks, + None, + None, + Some(&mut runtime), + ); + + // Assert retry count is PRESERVED across BreakerOpen (still 2, not reset to 0) + assert_eq!( + queue.retry_count(&breaker_scope), + 2, + "retry_count must be preserved across BreakerOpen" + ); + + // Resume / next failure continues from 3 + queue.push(queue::QueuedEvent { + channel_id: breaker_channel_id, + scope: breaker_scope.clone(), + event: event.clone(), + received_at: std::time::Instant::now(), + prompt_tag: "p".into(), + }); + let batch_breaker_next = queue.flush_next().unwrap(); + queue.requeue(batch_breaker_next); + assert_eq!( + queue.retry_count(&breaker_scope), + 3, + "retry_count must continue from 3 after BreakerOpen, not reset to 1" + ); + } + + #[tokio::test] + async fn test_error_boundary_sanitizes_diagnostic_and_preserves_raw_in_ledger() { + use crate::error_outcome_emission_tests::dummy_agent; + + let dir = tempfile::tempdir().unwrap(); + let now = chrono::Utc::now(); + let pubkey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let mut runtime = + reliability::ReliabilityRuntime::open_in(dir.path(), pubkey, now).unwrap(); + + let channel_id = uuid::Uuid::new_v4(); + let scope = scope::SessionScope::Conversation { channel_id }; + + let mut queue = EventQueue::new(config::DedupMode::Queue); + let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "work") + .tags([]) + .sign_with_keys(&nostr::Keys::generate()) + .unwrap(); + queue.push(queue::QueuedEvent { + channel_id, + scope: scope.clone(), + event, + received_at: std::time::Instant::now(), + prompt_tag: "p".into(), + }); + let batch = queue.flush_next().unwrap(); + + let mut pool = AgentPool::from_slots(vec![None]); + let task_id = pool.join_set.spawn(async {}).id(); + pool.task_map_mut().insert( + task_id, + crate::pool::TaskMeta { + agent_index: 0, + channel_id: Some(channel_id), + scope: Some(scope.clone()), + turn_id: "test-turn-id".to_string(), + recoverable_batch: None, + control_tx: None, + steer_tx: None, + successful_steer_deliveries: std::collections::HashSet::new(), + }, + ); + + let secret_key = "sk-ant-secretkey1234567890abcdef"; + let bearer_token = "my-secret-bearer-token"; + let massive_backtrace = "x".repeat(1000); + let long_msg = format!( + "provider error: token=secret123 and Bearer {} and {} and backtrace: {}", + bearer_token, secret_key, massive_backtrace + ); + let err = acp::AcpError::AgentError { + code: 500, + message: long_msg, + }; + + let agent = dummy_agent(0).await; + let result = PromptResult { + started: false, + agent, + source: PromptSource::Channel(scope.clone()), + turn_id: "test-turn-id".to_string(), + outcome: PromptOutcome::Error(err), + batch: Some(batch), + }; + + let config = super::error_outcome_emission_tests::test_config(); + let mut heartbeat_in_flight = false; + let removed_channels = HashSet::new(); + let mut crash_history = Vec::new(); + let (respawn_tx, _respawn_rx) = tokio::sync::mpsc::channel(1); + let mut respawn_tasks = tokio::task::JoinSet::new(); + let observer = observer::ObserverHandle::in_process(); + + handle_prompt_result( + &mut pool, + &mut queue, + &config, + result, + &mut heartbeat_in_flight, + &removed_channels, + &mut crash_history, + &respawn_tx, + &mut respawn_tasks, + Some(observer.clone()), + None, + Some(&mut runtime), + ); + + // 1. Emitted observer turn_error must be capped <= 512 chars and redacted + let events = observer.snapshot(); + let turn_error = events + .iter() + .find(|e| e.kind == "turn_error") + .expect("turn_error event must be emitted"); + let emitted_err = turn_error.payload["error"].as_str().unwrap(); + assert!( + emitted_err.chars().count() <= 512, + "emitted error must be <= 512 chars, got {}", + emitted_err.chars().count() + ); + assert!( + !emitted_err.contains(secret_key), + "emitted error must redact secret key" + ); + assert!( + !emitted_err.contains(bearer_token), + "emitted error must redact bearer token" + ); + assert!( + !emitted_err.contains("token=secret123"), + "emitted error must redact token parameter" + ); + assert!( + emitted_err.contains(""), + "emitted error must contain " + ); + + // 2. Ledger retains capped raw text without redaction + let ledger_content = std::fs::read_to_string(dir.path().join("ledger.jsonl")).unwrap(); + assert!( + ledger_content.contains("token=secret123"), + "ledger must retain raw error text" + ); + assert!( + ledger_content.contains(secret_key), + "ledger must retain raw secret key" + ); + } +} diff --git a/crates/buzz-acp/src/pool.rs b/crates/buzz-acp/src/pool.rs index 3d2a0f8e9d5..83982bcddc0 100644 --- a/crates/buzz-acp/src/pool.rs +++ b/crates/buzz-acp/src/pool.rs @@ -324,6 +324,13 @@ impl OwnedAgent { } } +/// Acknowledgment that a failure notice was successfully submitted to the relay. +#[derive(Debug, Clone)] +pub enum NoticeAck { + Pause(Uuid), + Breaker(SessionScope), +} + /// Pool of agents with take-and-return ownership semantics. /// /// Agents are either idle (sitting in `agents[i]`) or checked out @@ -342,6 +349,7 @@ pub struct AgentPool { /// Best-effort: stale entries (rotation, crash/respawn) self-heal on the /// next dispatch and are pruned on channel-wide session invalidation. session_owners: HashMap, + notice_ack_tx: Option>, } /// Result returned by a completed prompt task. @@ -838,9 +846,20 @@ impl AgentPool { join_set: JoinSet::new(), task_map: HashMap::new(), session_owners: HashMap::new(), + notice_ack_tx: None, } } + /// Clone the notice ack sender if one was registered with the pool. + pub fn notice_ack_tx(&self) -> Option> { + self.notice_ack_tx.clone() + } + + /// Register a notice ack channel sender with the pool. + pub fn set_notice_ack_tx(&mut self, tx: mpsc::UnboundedSender) { + self.notice_ack_tx = Some(tx); + } + /// Record which worker is handling `scope` so a later dispatch can detect a /// busy owner and avoid opening a duplicate session on another worker. pub fn record_scope_owner(&mut self, scope: SessionScope, agent_index: usize) { @@ -2020,6 +2039,7 @@ fn send_prompt_result( batch: Option, ) { agent.acp.clear_steer_rx(); + agent.acp.set_started_signal(None); // Read the started signal here, in the one place every prompt outcome // passes through, so no exit path can forget to report it. let started = agent.acp.turn_saw_output(); @@ -2056,7 +2076,10 @@ pub async fn run_prompt_task( ) { // Is this a channel prompt or a heartbeat? let source = match &batch { - Some(b) => PromptSource::Channel(b.scope.clone()), + Some(b) => { + agent.acp.set_started_signal(Some(b.started.clone())); + PromptSource::Channel(b.scope.clone()) + } None => PromptSource::Heartbeat, }; let observer_channel_id = source.channel_id(); @@ -4991,7 +5014,7 @@ pub(crate) async fn post_failure_notice( channel_id: Uuid, thread_tags: &ThreadTags, content: &str, -) { +) -> bool { let thread_ref = thread_tags.root_event_id.as_deref().and_then(|root| { let root_id = nostr::EventId::from_hex(root).ok()?; let parent_id = thread_tags @@ -5016,21 +5039,34 @@ pub(crate) async fn post_failure_notice( Ok(b) => b, Err(e) => { tracing::warn!(channel = %channel_id, "failure notice: build failed: {e}"); - return; + return false; } }; let event = match builder.sign_with_keys(&rest.keys) { Ok(e) => e, Err(e) => { tracing::warn!(channel = %channel_id, "failure notice: sign failed: {e}"); - return; + return false; } }; - match tokio::time::timeout(Duration::from_secs(5), rest.submit_event(&event)).await { - Ok(Ok(_)) => {} - Ok(Err(e)) => tracing::warn!(channel = %channel_id, "failure notice failed: {e}"), - Err(_) => tracing::warn!(channel = %channel_id, "failure notice timed out"), + const MAX_NOTICE_ATTEMPTS: usize = 3; + let mut delay = Duration::from_millis(100); + for attempt in 1..=MAX_NOTICE_ATTEMPTS { + match tokio::time::timeout(Duration::from_secs(5), rest.submit_event(&event)).await { + Ok(Ok(_)) => return true, + Ok(Err(e)) => { + tracing::warn!(channel = %channel_id, attempt, "failure notice failed: {e}"); + } + Err(_) => { + tracing::warn!(channel = %channel_id, attempt, "failure notice timed out"); + } + } + if attempt < MAX_NOTICE_ATTEMPTS { + tokio::time::sleep(delay).await; + delay = (delay * 2).min(Duration::from_secs(2)); + } } + false } /// Best-effort: remove a reaction via a signed kind:5 (NIP-09) deletion event. @@ -6435,6 +6471,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let context = ConversationContext::Thread { messages: vec![ContextMessage { @@ -6688,6 +6725,7 @@ done"# }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; run_prompt_task( agent, @@ -6776,6 +6814,7 @@ done"# received_at: std::time::Instant::now(), }], cancel_reason: Some(crate::queue::CancelReason::Steer), + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let next_batch = FlushBatch { batch_id: Uuid::new_v4(), @@ -6788,6 +6827,7 @@ done"# }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; // Return both merged events as DM history. They must be excluded from @@ -6945,6 +6985,7 @@ done"# }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; // The local REST bridge returns the already-delivered steer as DM @@ -7303,6 +7344,7 @@ printf '%s\n' '{{"jsonrpc":"2.0","id":0,"result":{{"stopReason":"end_turn"}}}}'" }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } @@ -7666,6 +7708,7 @@ printf '%s\n' '{{"jsonrpc":"2.0","id":0,"result":{{"stopReason":"end_turn"}}}}'" }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } @@ -9415,6 +9458,7 @@ done"# }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let mut ctx = make_prompt_context_no_owner(); @@ -10718,4 +10762,47 @@ done"# "an optionless switch caches the target's (empty) options, never the pre-switch model-a options with a patched effort" ); } + + #[tokio::test] + async fn test_post_failure_notice_retries_on_failure_and_succeeds() { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let requests = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let server_requests = requests.clone(); + tokio::spawn(async move { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + while let Ok((mut socket, _)) = listener.accept().await { + let mut buf = vec![0; 8192]; + let _ = socket.read(&mut buf).await; + let req_count = server_requests.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + if req_count == 0 { + // Fail first attempt with HTTP 500 + let response = "HTTP/1.1 500 Internal Server Error\r\nContent-Length: 0\r\nConnection: close\r\n\r\n"; + let _ = socket.write_all(response.as_bytes()).await; + } else { + // Succeed on retry with HTTP 200 + let response = "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 4\r\nConnection: close\r\n\r\nnull"; + let _ = socket.write_all(response.as_bytes()).await; + } + } + }); + + let keys = nostr::Keys::generate(); + let rest = crate::relay::RestClient { + http: reqwest::Client::new(), + base_url, + keys, + auth_tag_json: None, + }; + + let channel_id = uuid::Uuid::new_v4(); + let ok = + post_failure_notice(&rest, channel_id, &ThreadTags::default(), "notice content").await; + assert!(ok, "post_failure_notice should succeed on retry"); + assert_eq!( + requests.load(std::sync::atomic::Ordering::SeqCst), + 2, + "must have made 2 requests (1 failure + 1 retry)" + ); + } } diff --git a/crates/buzz-acp/src/queue.rs b/crates/buzz-acp/src/queue.rs index 7bfed697f99..cafe6fb3512 100644 --- a/crates/buzz-acp/src/queue.rs +++ b/crates/buzz-acp/src/queue.rs @@ -78,7 +78,7 @@ impl IntoScope for &Uuid { } /// Maximum events drained into a single batch. -const MAX_BATCH_EVENTS: usize = 50; +pub(crate) const MAX_BATCH_EVENTS: usize = 50; /// Maximum retry attempts before a batch is dead-lettered. pub(crate) const MAX_RETRIES: u32 = 10; @@ -164,6 +164,23 @@ pub struct FlushBatch { /// [`Steer`](CancelReason::Steer) framing if a merge somehow lacks a reason /// (see [`MergeFraming::for_reason`]). pub cancel_reason: Option, + /// Whether this batch's turn saw agent output or a tool call. + /// Shared via `Arc` across clones so panic/crash recovery in `TaskMeta` retains + /// the started status. + pub started: std::sync::Arc, +} + +impl FlushBatch { + /// Whether this batch started executing (produced output or tool calls). + pub fn is_started(&self) -> bool { + self.started.load(std::sync::atomic::Ordering::SeqCst) + } + + /// Mark this batch as started. + pub fn mark_started(&self) { + self.started + .store(true, std::sync::atomic::Ordering::SeqCst); + } } /// Per-channel event queue with per-channel in-flight enforcement. @@ -198,6 +215,7 @@ pub struct FlushBatch { /// events = drain up to MAX_BATCH_EVENTS from queues[channel] /// in_flight_channels.insert(channel) /// in_flight_deadlines.insert(channel, now + in_flight_deadline) +/// return Some( started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), /// return Some(FlushBatch { channel, events }) /// /// mark_complete(channel_id): @@ -251,6 +269,8 @@ pub struct EventQueue { /// and refusing to grow past the cap keeps a caller that never drains from /// turning this into an unbounded backlog. parked_out: VecDeque, + /// Number of times `flush_next` has been called. + flush_count: usize, } /// Most batches held in the park hand-off at once. @@ -292,9 +312,16 @@ impl EventQueue { withheld_native_steer: HashMap::new(), in_flight_deadline: Duration::from_secs(DEFAULT_IN_FLIGHT_DEADLINE_SECS), parked_out: VecDeque::new(), + flush_count: 0, } } + /// Number of times `flush_next` has been called on this queue. + #[allow(dead_code)] + pub fn flush_count(&self) -> usize { + self.flush_count + } + /// Set the in-flight backstop deadline from the configured max turn /// duration, preserving the 100s buffer for cancel-drain grace + respawn. pub fn with_in_flight_deadline(mut self, max_turn_duration_secs: u64) -> Self { @@ -411,6 +438,7 @@ impl EventQueue { /// across channels), drains ALL events for that channel into a single batch, /// inserts into `in_flight_channels`, and returns the batch. pub fn flush_next(&mut self) -> Option { + self.flush_count += 1; let now = Instant::now(); // Auto-expire any stuck in-flight entries that missed mark_complete. @@ -482,6 +510,7 @@ impl EventQueue { events: cancelled, cancelled_events: vec![], cancel_reason, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }); } None => return None, @@ -534,6 +563,7 @@ impl EventQueue { events, cancelled_events, cancel_reason, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }) } @@ -568,6 +598,17 @@ impl EventQueue { } } + /// Mark the scope complete for in-flight tracking without clearing `retry_counts`. + /// + /// Used when a turn ends in Pause or BreakerOpen or when batches are held, + /// so accumulated retries are not lost across the pause or breaker. + pub fn mark_complete_preserving_retries(&mut self, scope: K) { + let scope = scope.into_scope(); + self.in_flight_scopes.remove(&scope); + self.in_flight_deadlines.remove(&scope); + self.in_flight_batch_sizes.remove(&scope); + } + /// Re-queue a batch of events that failed to process. /// /// Events are pushed back to the **front** of the channel's queue so they @@ -2797,6 +2838,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let prompt = format_prompt(&batch, &FormatPromptArgs::default()).join("\n\n"); @@ -2833,6 +2875,7 @@ mod tests { received_at: Instant::now(), }], cancel_reason: reason, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } @@ -2973,6 +3016,7 @@ mod tests { received_at: Instant::now(), }], cancel_reason: Some(CancelReason::Steer), + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let prompt = format_prompt(&batch, &FormatPromptArgs::default()).join("\n\n"); assert!(prompt.contains("")); @@ -3025,6 +3069,7 @@ mod tests { received_at: Instant::now(), }], cancel_reason: Some(CancelReason::Steer), + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let prompt = format_prompt(&batch, &FormatPromptArgs::default()).join("\n\n"); @@ -3206,6 +3251,7 @@ mod tests { ], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let prompt = format_prompt(&batch, &FormatPromptArgs::default()).join("\n\n"); @@ -3236,6 +3282,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let prompt = format_prompt(&batch, &FormatPromptArgs::default()).join("\n\n"); @@ -3261,6 +3308,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let core = "[Agent Memory — core]\nbe helpful"; let prompt = format_prompt( @@ -3295,6 +3343,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let prompt = format_prompt( &batch, @@ -3327,6 +3376,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let core = "[Agent Memory — core]\nbe helpful"; let prompt = format_prompt( @@ -3356,6 +3406,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; // format_prompt no longer accepts or emits base_prompt/system_prompt. @@ -3382,6 +3433,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let core = "[Agent Memory — core]\nremember this"; @@ -3442,6 +3494,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let canvas = "[Channel Canvas]\ncanvas content"; let core = "[Agent Memory — core]\nremember this"; @@ -3497,6 +3550,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let prompt = format_prompt( @@ -3537,6 +3591,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let ctx = ConversationContext::Thread { @@ -3802,6 +3857,7 @@ mod tests { received_at: Instant::now(), }], cancel_reason: Some(CancelReason::Interrupt), + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; // Simulate the flushed-then-held state: scope is in-flight. q.push(make_queued(ch, "placeholder")); @@ -4150,6 +4206,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let ci = PromptChannelInfo { name: "engineering".into(), @@ -4185,6 +4242,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let ci = PromptChannelInfo { name: "DM".into(), @@ -4234,6 +4292,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let ci = PromptChannelInfo { name: "test".into(), @@ -4317,6 +4376,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let prompt = format_prompt(&batch, &FormatPromptArgs::default()).join("\n\n"); @@ -4345,6 +4405,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let mut ctx = ConversationContext::Thread { messages: vec![ @@ -4478,6 +4539,7 @@ mod tests { ], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let mixed_prompt = format_prompt( &mixed_batch, @@ -4504,6 +4566,7 @@ mod tests { ], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let same_thread_prompt = format_prompt( &same_thread_batch, @@ -4534,6 +4597,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let ci = PromptChannelInfo { name: "DM".into(), @@ -4592,6 +4656,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let ctx = ConversationContext::Thread { messages: vec![ContextMessage { @@ -4803,6 +4868,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let ci = PromptChannelInfo { name: "DM".into(), @@ -4875,6 +4941,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let trigger_only_prompt = format_prompt(&batch, &FormatPromptArgs::default()).join("\n\n"); @@ -4910,6 +4977,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let ci = PromptChannelInfo { name: "DM".into(), @@ -4962,6 +5030,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let ci = PromptChannelInfo { name: "DM".into(), @@ -5006,6 +5075,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let prompt = format_prompt(&batch, &FormatPromptArgs::default()).join("\n\n"); @@ -5032,6 +5102,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let prompt = format_prompt(&batch, &FormatPromptArgs::default()).join("\n\n"); @@ -5057,6 +5128,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let prompt = format_prompt(&batch, &FormatPromptArgs::default()).join("\n\n"); @@ -5486,6 +5558,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; // No profile lookup → sender treated as human → human-facing thread @@ -5530,6 +5603,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let ci = PromptChannelInfo { name: "DM".into(), @@ -5568,6 +5642,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; // Top-level human message (no lookup → human): the reply opens a new @@ -5599,6 +5674,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let ci = PromptChannelInfo { name: "DM".into(), @@ -5645,6 +5721,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; // Human-facing (no lookup) deep reply: anchor to the thread ROOT to @@ -5683,6 +5760,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let prompt = format_prompt(&batch, &FormatPromptArgs::default()).join("\n\n"); @@ -5727,6 +5805,7 @@ mod tests { ], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; // Scope derives from the last (threaded) event; human-facing → anchor @@ -5766,6 +5845,7 @@ mod tests { ], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; // Last event is top-level and human-facing → opens a new thread @@ -5795,6 +5875,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } @@ -6097,6 +6178,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let prompt = format_prompt( &batch, @@ -6128,6 +6210,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let prompt = format_prompt( &batch, @@ -6158,6 +6241,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }; let prompt = format_prompt(&batch, &FormatPromptArgs::default()).join("\n\n"); assert!( @@ -6621,6 +6705,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } diff --git a/crates/buzz-acp/src/reliability.rs b/crates/buzz-acp/src/reliability.rs index e09761048ec..8664e6e440a 100644 --- a/crates/buzz-acp/src/reliability.rs +++ b/crates/buzz-acp/src/reliability.rs @@ -28,7 +28,7 @@ pub mod runtime; pub mod state; pub mod state_dir; -pub use error_class::classify_at; +pub use error_class::{classify_at, sanitize_error_diagnostic}; pub use park::{ParkError, ParkReason, ParkedBatch}; pub use runtime::{ReliabilityRuntime, ReplayPlan}; pub use state::{BreakerGate, BreakerVerdict, PauseGate, ReliabilityState}; @@ -261,4 +261,60 @@ mod tests { assert_eq!(second, Action::Retry); assert_eq!(third, Action::OpenBreaker); } + + #[test] + fn non_provider_failure_resets_consecutive_streak() { + let channel_id = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id }; + let now = log_instant(); + let mut state = ReliabilityState::default(); + + let first = state.on_failure(&scope, ErrorClass::ProviderInternal, now); + let second = state.on_failure( + &scope, + ErrorClass::ProviderInternal, + now + chrono::Duration::seconds(5), + ); + let third = state.on_failure( + &scope, + ErrorClass::Auth, + now + chrono::Duration::seconds(10), + ); + let fourth = state.on_failure( + &scope, + ErrorClass::ProviderInternal, + now + chrono::Duration::seconds(15), + ); + + assert_eq!(first, Action::Retry); + assert_eq!(second, Action::Retry); + assert_eq!(third, Action::Park); + assert_eq!( + fourth, + Action::Retry, + "streak must be reset by Auth failure" + ); + } + + #[test] + fn test_consecutive_map_stays_bounded_across_many_scopes() { + let now = log_instant(); + let mut state = ReliabilityState::default(); + + for _ in 0..50 { + let scope = SessionScope::Conversation { + channel_id: Uuid::new_v4(), + }; + let first = state.on_failure(&scope, ErrorClass::ProviderInternal, now); + assert_eq!(first, Action::Retry); + assert!(state.consecutive_len() > 0); + + let terminal = state.on_failure(&scope, ErrorClass::Auth, now); + assert_eq!(terminal, Action::Park); + // After the terminal park, the scope is removed from consecutive. + assert_eq!(state.consecutive_len(), 0); + } + + assert_eq!(state.consecutive_len(), 0); + } } diff --git a/crates/buzz-acp/src/reliability/error_class.rs b/crates/buzz-acp/src/reliability/error_class.rs index 5dc6793d09b..1cd49e58656 100644 --- a/crates/buzz-acp/src/reliability/error_class.rs +++ b/crates/buzz-acp/src/reliability/error_class.rs @@ -100,6 +100,93 @@ pub fn truncate_chars(text: &str, max: usize) -> String { text.chars().take(max).collect() } +/// Sanitize and bound an error string for external diagnostics (observer events, +/// warnings, channel notices). +/// +/// Bounded to at most [`MAX_RAW_ERROR_CHARS`] (512 chars). +/// Redacts sensitive tokens (bearer tokens, secret keys, passwords, credentials) +/// and trims excessive stack traces. +pub fn sanitize_error_diagnostic(raw: &str) -> String { + let mut text = raw.to_string(); + + // 1. Redact stack backtrace if present + if let Some(idx) = text.to_lowercase().find("stack backtrace:") { + text.truncate(idx); + text.push_str("[stack backtrace redacted]"); + } + + // 2. Token-level redaction + let mut words: Vec = Vec::new(); + let mut prev_is_bearer = false; + for part in text.split_whitespace() { + let (trimmed_part, trailing_punct) = if part.ends_with(',') || part.ends_with(';') { + (&part[..part.len() - 1], &part[part.len() - 1..]) + } else { + (part, "") + }; + let lower_trimmed = trimmed_part.to_lowercase(); + + if prev_is_bearer { + words.push(format!("{trailing_punct}")); + prev_is_bearer = false; + continue; + } + if lower_trimmed == "bearer" { + prev_is_bearer = true; + words.push(part.to_string()); + continue; + } + // sk-ant-... or sk-... secret keys + if lower_trimmed.starts_with("sk-") && trimmed_part.len() > 7 { + words.push(format!("sk-{trailing_punct}")); + continue; + } + // nsec1... private keys + if lower_trimmed.starts_with("nsec1") && trimmed_part.len() > 10 { + words.push(format!("nsec1{trailing_punct}")); + continue; + } + // key=value or key:value + if let Some((k, _v)) = trimmed_part.split_once('=') { + let k_lower = k.to_lowercase(); + if matches!( + k_lower.as_str(), + "token" + | "secret" + | "password" + | "key" + | "api_key" + | "apikey" + | "auth_token" + | "access_token" + ) { + words.push(format!("{k}={trailing_punct}")); + continue; + } + } + if let Some((k, _v)) = trimmed_part.split_once(':') { + let k_lower = k.to_lowercase(); + if matches!( + k_lower.as_str(), + "token" + | "secret" + | "password" + | "key" + | "api_key" + | "apikey" + | "auth_token" + | "access_token" + ) { + words.push(format!("{k}:{trailing_punct}")); + continue; + } + } + words.push(part.to_string()); + } + let sanitized = words.join(" "); + truncate_chars(&sanitized, MAX_RAW_ERROR_CHARS) +} + fn has_marker(lower: &str, markers: &[&str]) -> bool { markers.iter().any(|m| lower.contains(m)) } diff --git a/crates/buzz-acp/src/reliability/ledger.rs b/crates/buzz-acp/src/reliability/ledger.rs index 0570310eaa4..e0fccc055cc 100644 --- a/crates/buzz-acp/src/reliability/ledger.rs +++ b/crates/buzz-acp/src/reliability/ledger.rs @@ -11,7 +11,7 @@ //! Design: `docs/plans/2026-09-06-harness-reliability-design.md`, "Ledger records". use std::collections::HashSet; -use std::io::{self, BufRead, Read, Write}; +use std::io::{self, BufRead, Read, Seek, SeekFrom, Write}; use std::path::{Path, PathBuf}; use chrono::{DateTime, Duration, Utc}; @@ -311,6 +311,7 @@ impl Ledger { let path = dir.join(LEDGER_FILE); // Create it if absent so the mode is ours from the first byte. drop(state_dir::open_append(&path)?); + sanitize_dangling_final_line(&path)?; let mut ledger = Self { len_bytes: std::fs::metadata(&path).map(|m| m.len()).unwrap_or(0), path, @@ -497,6 +498,55 @@ fn fit_to_budget( Ok((remaining, dropped)) } +/// Detect and truncate a dangling final line without a trailing newline, so +/// future appends are not fused with corrupted partial lines. +fn sanitize_dangling_final_line(path: &Path) -> io::Result<()> { + let mut file = match std::fs::OpenOptions::new() + .read(true) + .write(true) + .open(path) + { + Ok(f) => f, + Err(e) if e.kind() == io::ErrorKind::NotFound => return Ok(()), + Err(e) => return Err(e), + }; + let len = file.metadata()?.len(); + if len == 0 { + return Ok(()); + } + file.seek(SeekFrom::End(-1))?; + let mut last_byte = [0u8; 1]; + file.read_exact(&mut last_byte)?; + if last_byte[0] == b'\n' { + return Ok(()); + } + + let mut pos = len - 1; + let mut found_nl = false; + let mut buf = [0u8; 4096]; + while pos > 0 { + let chunk_size = (pos as usize).min(buf.len()); + let chunk_start = pos - chunk_size as u64; + file.seek(SeekFrom::Start(chunk_start))?; + file.read_exact(&mut buf[..chunk_size])?; + if let Some(offset) = buf[..chunk_size].iter().rposition(|&b| b == b'\n') { + file.set_len(chunk_start + offset as u64 + 1)?; + found_nl = true; + break; + } + pos = chunk_start; + } + if !found_nl { + file.set_len(0)?; + } + file.sync_all()?; + tracing::warn!( + path = %path.display(), + "truncated dangling partial line with no trailing newline in ledger file" + ); + Ok(()) +} + /// Read a JSONL ledger file with a hard byte cap on the input and a hard cap /// per line. fn read_records(path: &Path) -> io::Result> { @@ -621,4 +671,52 @@ mod tests { let crashed = ledger.replays_without_finish().unwrap(); assert_eq!(crashed, vec![batch_id2]); } + + #[test] + fn test_ledger_open_quarantines_dangling_final_line_without_newline() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join(LEDGER_FILE); + let now = Utc::now(); + + // Pre-seed a ledger file with valid lines, then a truncated line with no trailing newline: + let valid_record = LedgerRecord { + at: now, + agent: "test-agent".to_string(), + body: LedgerBody::AgentResumed(AgentResumed {}), + }; + let valid_line = format!("{}\n", serde_json::to_string(&valid_record).unwrap()); + let truncated_line = + "{\"at\":\"2026-09-06T00:01:00Z\",\"agent\":\"test-agent\",\"kind\":\"batch_parked\",\"batch_id\":\""; + std::fs::write(&path, format!("{valid_line}{truncated_line}")).unwrap(); + + let mut ledger = Ledger::open(dir.path(), "test-agent", now).unwrap(); + + // Append a new record + let new_batch_id = Uuid::new_v4(); + ledger + .append( + now, + LedgerBody::BatchParked(BatchParked { + batch_id: new_batch_id, + channel_id: Uuid::new_v4(), + reason: "retry".to_string(), + started: false, + events: 1, + }), + ) + .unwrap(); + + let records = ledger.read_all().unwrap(); + assert!( + records.iter().any( + |r| matches!(&r.body, LedgerBody::BatchParked(bp) if bp.batch_id == new_batch_id) + ), + "new record must be recovered cleanly and not fused with dangling line" + ); + assert_eq!( + records.len(), + 2, + "must recover 1 valid pre-seeded record + 1 new record" + ); + } } diff --git a/crates/buzz-acp/src/reliability/park.rs b/crates/buzz-acp/src/reliability/park.rs index dcf5cc52a37..6692acf1a45 100644 --- a/crates/buzz-acp/src/reliability/park.rs +++ b/crates/buzz-acp/src/reliability/park.rs @@ -70,6 +70,10 @@ pub enum ParkReason { Auth, /// A scope breaker stayed open for its whole six-hour budget. BreakerExpired, + /// The agent was paused due to provider capacity limit. + Pause, + /// The scope breaker opened due to repeated failures. + BreakerOpen, } impl ParkReason { @@ -80,6 +84,8 @@ impl ParkReason { Self::HardTimeout => "hard_timeout", Self::Auth => "auth", Self::BreakerExpired => "breaker_expired", + Self::Pause => "pause", + Self::BreakerOpen => "breaker_open", } } } @@ -254,6 +260,9 @@ pub enum ParkError { /// A batch larger than the queue can build. #[error("batch carries {0} events, over the park file's per-batch cap")] TooManyEvents(usize), + /// A single batch line exceeds the per-line cap. + #[error("serialized batch is {0} bytes, over the {MAX_LINE_BYTES}-byte line cap — the batch was NOT parked")] + LineTooLong(usize), /// The write itself failed. #[error("park file write failed: {0}")] Io(#[from] io::Error), @@ -332,12 +341,12 @@ impl ParkFile { } let mut next = self.batches.clone(); next.push(batch); + apply_scope_cap(&mut next); let bytes = serialize(&next)?; if bytes.len() as u64 > MAX_PARK_BYTES { return Err(ParkError::Full(next.len(), bytes.len() as u64)); } self.commit(next, bytes)?; - self.enforce_scope_cap()?; Ok(()) } @@ -434,45 +443,26 @@ impl ParkFile { report.aged_out += 1; } } + report.over_scope_cap = apply_scope_cap(&mut next); if !report.is_empty() { let bytes = serialize(&next)?; self.commit(next, bytes)?; } - report.over_scope_cap = self.enforce_scope_cap()?; Ok(report) } /// Demote the oldest replay-eligible batches of any scope over /// [`MAX_PARKED_PER_SCOPE`] to the review list. Returns how many moved. + #[allow(dead_code)] fn enforce_scope_cap(&mut self) -> Result { - use std::collections::HashMap; - - let mut per_scope: HashMap> = HashMap::new(); - for (index, batch) in self.batches.iter().enumerate() { - if batch.replay_eligible() { - per_scope.entry(batch.scope()).or_default().push(index); - } - } - let mut demote: Vec = Vec::new(); - for indices in per_scope.values() { - if indices.len() > MAX_PARKED_PER_SCOPE { - demote.extend(&indices[..indices.len() - MAX_PARKED_PER_SCOPE]); - } - } - if demote.is_empty() { - return Ok(0); - } let mut next = self.batches.clone(); - for index in &demote { - let batch = &mut next[*index]; - batch.needs_review = true; - batch.needs_review_reason = Some(format!( - "more than {MAX_PARKED_PER_SCOPE} batches were waiting for this conversation" - )); + let demoted = apply_scope_cap(&mut next); + if demoted == 0 { + return Ok(0); } let bytes = serialize(&next)?; self.commit(next, bytes)?; - Ok(demote.len()) + Ok(demoted) } fn mutate( @@ -508,12 +498,44 @@ impl ParkFile { } } +/// Demote the oldest replay-eligible batches of any scope over +/// [`MAX_PARKED_PER_SCOPE`] to the review list. Returns how many moved. +fn apply_scope_cap(batches: &mut [ParkedBatch]) -> usize { + use std::collections::HashMap; + + let mut per_scope: HashMap> = HashMap::new(); + for (index, batch) in batches.iter().enumerate() { + if batch.replay_eligible() { + per_scope.entry(batch.scope()).or_default().push(index); + } + } + let mut demote_count = 0; + for indices in per_scope.values() { + if indices.len() > MAX_PARKED_PER_SCOPE { + for &index in &indices[..indices.len() - MAX_PARKED_PER_SCOPE] { + let batch = &mut batches[index]; + batch.needs_review = true; + batch.needs_review_reason = Some(format!( + "more than {MAX_PARKED_PER_SCOPE} batches were waiting for this conversation" + )); + demote_count += 1; + } + } + } + demote_count +} + fn serialize(batches: &[ParkedBatch]) -> Result, ParkError> { let mut buffer = Vec::new(); for batch in batches { + let start = buffer.len(); serde_json::to_writer(&mut buffer, batch) .map_err(|e| ParkError::Io(io::Error::new(io::ErrorKind::InvalidData, e)))?; buffer.push(b'\n'); + let line_len = buffer.len() - start; + if line_len > MAX_LINE_BYTES { + return Err(ParkError::LineTooLong(line_len)); + } } Ok(buffer) } @@ -606,6 +628,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), } } @@ -630,6 +653,30 @@ mod tests { assert_eq!(reopened.batches().len(), 1); } + #[test] + fn test_park_rejects_oversized_individual_line() { + let dir = tempfile::tempdir().unwrap(); + let mut park = ParkFile::open(dir.path()).unwrap(); + let ch = Uuid::new_v4(); + let b_id = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id: ch }; + // Create an event whose serialized line exceeds MAX_LINE_BYTES (512 KB), + // but whose size is well under MAX_PARK_BYTES (10 MB). + let big_content = "x".repeat(MAX_LINE_BYTES + 1024); + let batch = dummy_batch(ch, b_id, scope, &big_content); + let parked = + ParkedBatch::from_batch(&batch, ParkReason::RetriesExhausted, false, Utc::now()) + .unwrap(); + + let result = park.park(parked); + assert!( + result.is_err(), + "park() must reject line exceeding MAX_LINE_BYTES" + ); + assert!(!park.contains(b_id)); + assert!(park.batches().is_empty()); + } + #[test] fn test_reconcile_on_start_crashed_mid_replay_moves_to_needs_review() { let dir = tempfile::tempdir().unwrap(); @@ -683,4 +730,71 @@ mod tests { assert_eq!(candidates.len(), 1); assert_eq!(candidates[0].batch_id, b_not_started.batch_id); } + + #[test] + fn test_park_101st_batch_scope_cap_single_atomic_write() { + let dir = tempfile::tempdir().unwrap(); + let mut park = ParkFile::open(dir.path()).unwrap(); + let ch = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id: ch }; + + for i in 0..100 { + let b_id = Uuid::new_v4(); + let batch = dummy_batch(ch, b_id, scope.clone(), &format!("msg {i}")); + let parked = + ParkedBatch::from_batch(&batch, ParkReason::RetriesExhausted, false, Utc::now()) + .unwrap(); + park.park(parked).unwrap(); + } + assert_eq!(park.batches().len(), 100); + assert_eq!(park.replay_candidates(&scope).len(), 100); + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let original_mode = std::fs::metadata(dir.path()).unwrap().permissions().mode(); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o500)).unwrap(); + + let b_id_101 = Uuid::new_v4(); + let batch_101 = dummy_batch(ch, b_id_101, scope.clone(), "msg 101"); + let parked_101 = ParkedBatch::from_batch( + &batch_101, + ParkReason::RetriesExhausted, + false, + Utc::now(), + ) + .unwrap(); + + let res = park.park(parked_101); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(original_mode)) + .unwrap(); + + assert!( + res.is_err(), + "park of 101st batch must fail when write fails" + ); + let on_disk = read_batches(park.path()).unwrap(); + assert_eq!( + on_disk.len(), + 100, + "disk must still hold exactly 100 batches" + ); + assert!(!on_disk.iter().any(|b| b.batch_id == b_id_101)); + assert_eq!(park.batches().len(), 100); + assert!(!park.contains(b_id_101)); + } + + // When write succeeds, 101st batch lands and oldest is demoted in ONE atomic update: + let b_id_101 = Uuid::new_v4(); + let batch_101 = dummy_batch(ch, b_id_101, scope.clone(), "msg 101"); + let parked_101 = + ParkedBatch::from_batch(&batch_101, ParkReason::RetriesExhausted, false, Utc::now()) + .unwrap(); + park.park(parked_101).unwrap(); + + assert_eq!(park.batches().len(), 101); + assert!(park.contains(b_id_101)); + assert_eq!(park.replay_candidates(&scope).len(), 100); + assert!(park.batches()[0].needs_review); + } } diff --git a/crates/buzz-acp/src/reliability/runtime.rs b/crates/buzz-acp/src/reliability/runtime.rs index ddc8caded4d..471d05db608 100644 --- a/crates/buzz-acp/src/reliability/runtime.rs +++ b/crates/buzz-acp/src/reliability/runtime.rs @@ -95,6 +95,16 @@ impl ReliabilityRuntime { &self.state } + /// Release any unconsumed probe permits for pause and scope breaker. + pub fn release_probe(&mut self, scope: Option<&SessionScope>) { + self.state.release_probe(scope); + } + + /// Reissue probe permit alias for `release_probe`. + pub fn reissue_probe(&mut self, scope: Option<&SessionScope>) { + self.state.reissue_probe(scope); + } + /// Read-only view of the park file. pub fn park(&self) -> &ParkFile { &self.park @@ -176,11 +186,25 @@ impl ReliabilityRuntime { if candidates.is_empty() { return None; } - let mut batch_ids = Vec::with_capacity(candidates.len()); + let mut batch_ids = Vec::new(); let mut events = Vec::new(); for batch in candidates { + let batch_events = batch.to_batch_events(); + if !events.is_empty() + && events.len() + batch_events.len() > crate::queue::MAX_BATCH_EVENTS + { + break; + } + if events.is_empty() && batch_events.len() > crate::queue::MAX_BATCH_EVENTS { + batch_ids.push(batch.batch_id); + events.extend(batch_events); + break; + } batch_ids.push(batch.batch_id); - events.extend(batch.to_batch_events()); + events.extend(batch_events); + } + if batch_ids.is_empty() { + return None; } Some(ReplayPlan { batch_ids, @@ -205,15 +229,26 @@ impl ReliabilityRuntime { for batch_id in &plan.batch_ids { self.park.mark_replayed(*batch_id, now)?; } + let mut all_recorded = true; for batch_id in &plan.batch_ids { - self.record( + if !self.record( now, LedgerBody::BatchReplayed(ledger::BatchReplayed { batch_id: *batch_id, channel_id: plan.channel_id, replay_of: new_batch_id, }), - ); + ) { + all_recorded = false; + } + } + if !all_recorded { + for batch_id in &plan.batch_ids { + let _ = self.park.unmark_replayed(*batch_id); + } + return Err(ParkError::Io(std::io::Error::other( + "could not append batch_replayed to ledger", + ))); } Ok(()) } @@ -267,7 +302,7 @@ impl ReliabilityRuntime { let Some(removed) = self.park.remove(batch_id)? else { return Ok(false); }; - self.record( + let recorded = self.record( now, LedgerBody::BatchDiscarded(ledger::BatchDiscarded { batch_id, @@ -275,7 +310,7 @@ impl ReliabilityRuntime { by: super::error_class::truncate_chars(by, ledger::MAX_LABEL_CHARS), }), ); - Ok(true) + Ok(recorded) } /// Operator control frame `replay_batch`: make one parked batch eligible @@ -361,6 +396,7 @@ mod tests { }], cancelled_events: vec![], cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), }, id, ) @@ -529,4 +565,136 @@ mod tests { "ledger must contain a batch_needs_review record for the crashed batch" ); } + + #[test] + #[cfg(unix)] + fn test_discard_fails_contract_when_ledger_append_fails() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let pubkey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let now = Utc::now(); + let mut runtime = ReliabilityRuntime::open_in(dir.path(), pubkey, now).unwrap(); + + let channel_id = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id }; + let (batch, _) = make_flush_batch(channel_id, scope, "to discard"); + let batch_id = batch.batch_id; + + // Park the batch first. + runtime + .park_batch(&batch, ParkReason::RetriesExhausted, false, now) + .unwrap(); + assert!(runtime.park().contains(batch_id)); + + // Make ledger.jsonl unwritable while keeping the directory and park file writable. + let ledger_path = dir.path().join("ledger.jsonl"); + let original_mode = std::fs::metadata(&ledger_path) + .unwrap() + .permissions() + .mode(); + std::fs::set_permissions(&ledger_path, std::fs::Permissions::from_mode(0o400)).unwrap(); + + let result = runtime.discard(batch_id, "operator", now); + + // Restore permissions for cleanup + let _ = + std::fs::set_permissions(&ledger_path, std::fs::Permissions::from_mode(original_mode)); + + // The batch was removed from park, but ledger write failed. + // It must NOT report unconditional success (Ok(true)). + assert!( + !matches!(result, Ok(true)), + "discard must not report unconditional success when ledger write failed: got {result:?}" + ); + } + + fn make_multi_event_batch( + channel_id: Uuid, + scope: SessionScope, + count: usize, + prefix: &str, + ) -> FlushBatch { + let mut events = Vec::with_capacity(count); + for i in 0..count { + let (event, _) = make_test_event(&format!("{prefix}-{i}")); + events.push(BatchEvent { + event, + prompt_tag: "test".into(), + received_at: Instant::now(), + }); + } + FlushBatch { + batch_id: Uuid::new_v4(), + channel_id, + scope, + events, + cancelled_events: vec![], + cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + } + } + + #[test] + fn test_replay_plan_respects_max_batch_events_and_preserves_unincluded_batches() { + let dir = tempfile::tempdir().unwrap(); + let pubkey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let now = Utc::now(); + let mut runtime = ReliabilityRuntime::open_in(dir.path(), pubkey, now).unwrap(); + + let channel_id = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id }; + + // Park > 50 replay-eligible events across multiple batches for one scope. + // Batch 1: 30 events + let batch1 = make_multi_event_batch(channel_id, scope.clone(), 30, "batch1"); + let batch1_id = batch1.batch_id; + runtime + .park_batch(&batch1, ParkReason::RetriesExhausted, false, now) + .unwrap(); + + // Batch 2: 30 events (total 60 > 50) + let batch2 = make_multi_event_batch(channel_id, scope.clone(), 30, "batch2"); + let batch2_id = batch2.batch_id; + runtime + .park_batch( + &batch2, + ParkReason::RetriesExhausted, + false, + now + chrono::Duration::seconds(1), + ) + .unwrap(); + + let mut queue = EventQueue::new(DedupMode::Queue); + + // Run a successful probe (binds replay_after_success) + crate::replay_after_success( + &mut runtime, + &mut queue, + &scope, + now + chrono::Duration::seconds(2), + ); + + // The turn for this scope finishes successfully (clearing in-flight replay) + let released = runtime.finish_replay(&scope).unwrap(); + assert_eq!( + released, + vec![batch1_id], + "only the included batch should be finished/released" + ); + + // The park file must STILL hold batch2, whose events were not included in the dispatched turn + assert!( + runtime.park().contains(batch2_id), + "batch 2 was not included in the dispatched replay turn and must remain in the park file" + ); + let parked2 = runtime + .park() + .get(batch2_id) + .expect("batch 2 still in park"); + assert!( + parked2.replay_eligible(), + "batch 2 must still be replay-eligible" + ); + } } diff --git a/crates/buzz-acp/src/reliability/state.rs b/crates/buzz-acp/src/reliability/state.rs index 0ba934e0c10..4dd514511bf 100644 --- a/crates/buzz-acp/src/reliability/state.rs +++ b/crates/buzz-acp/src/reliability/state.rs @@ -42,6 +42,10 @@ pub const BREAKER_MAX_OPEN_HOURS: i64 = 6; /// than this. pub const PAUSE_RENOTIFY_MINUTES: i64 = 15; +/// Maximum number of scopes tracked for consecutive provider failures before +/// pruning. +pub const MAX_CONSECUTIVE_SCOPES: usize = 1_000; + /// Whether the agent may run a turn right now. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PauseGate { @@ -91,6 +95,8 @@ struct Breaker { probe_issued: bool, /// Consecutive failures that opened it, for the ledger record. consecutive: u32, + /// Whether the failure notice has been successfully posted. + notified: bool, } /// Per-agent reliability state: one pause for the whole agent (a capacity @@ -100,6 +106,7 @@ pub struct ReliabilityState { pause: Option, breakers: HashMap, consecutive: HashMap, + generation: u64, } impl ReliabilityState { @@ -115,8 +122,12 @@ impl ReliabilityState { ) -> Action { match class { // A re-login fixes it; a retry does not. - ErrorClass::Auth => Action::Park, + ErrorClass::Auth => { + self.consecutive.remove(scope); + Action::Park + } ErrorClass::CapacityExhausted { resets_at } => { + self.consecutive.remove(scope); let until = clamp_pause(resets_at, now); self.set_pause(until); Action::Pause { until } @@ -132,14 +143,22 @@ impl ReliabilityState { if now - breaker.opened_at >= Duration::hours(BREAKER_MAX_OPEN_HOURS) { self.breakers.remove(scope); self.consecutive.remove(scope); + self.generation = self.generation.wrapping_add(1); return Action::Park; } breaker.next_probe = now + Duration::minutes(BREAKER_PROBE_MINUTES); breaker.probe_issued = false; breaker.consecutive = breaker.consecutive.saturating_add(1); + self.generation = self.generation.wrapping_add(1); return Action::OpenBreaker; } + if self.consecutive.len() >= MAX_CONSECUTIVE_SCOPES && !self.consecutive.contains_key(scope) + { + if let Some(oldest_scope) = self.consecutive.keys().next().cloned() { + self.consecutive.remove(&oldest_scope); + } + } let count = self.consecutive.entry(scope.clone()).or_insert(0); *count = count.saturating_add(1); if *count < BREAKER_THRESHOLD { @@ -154,8 +173,10 @@ impl ReliabilityState { next_probe: now + Duration::minutes(BREAKER_PROBE_MINUTES), probe_issued: false, consecutive, + notified: false, }, ); + self.generation = self.generation.wrapping_add(1); Action::OpenBreaker } @@ -168,6 +189,9 @@ impl ReliabilityState { let pause_lifted = self.pause.take().is_some(); let breaker_closed = self.breakers.remove(scope).is_some(); self.consecutive.remove(scope); + if pause_lifted || breaker_closed { + self.generation = self.generation.wrapping_add(1); + } (pause_lifted, breaker_closed) } @@ -187,6 +211,7 @@ impl ReliabilityState { return PauseGate::Held { until: pause.until }; } pause.probe_issued = true; + self.generation = self.generation.wrapping_add(1); PauseGate::Probe } @@ -212,9 +237,74 @@ impl ReliabilityState { }; } breaker.probe_issued = true; + self.generation = self.generation.wrapping_add(1); BreakerGate::Probe } + /// Release an unconsumed pause probe permit so a subsequent dispatch may probe. + pub fn release_pause_probe(&mut self) { + if let Some(pause) = self.pause.as_mut() { + if pause.probe_issued { + pause.probe_issued = false; + self.generation = self.generation.wrapping_add(1); + } + } + } + + /// Release an unconsumed breaker probe permit so a subsequent dispatch may probe. + pub fn release_breaker_probe(&mut self, scope: &SessionScope) { + if let Some(breaker) = self.breakers.get_mut(scope) { + if breaker.probe_issued { + breaker.probe_issued = false; + self.generation = self.generation.wrapping_add(1); + } + } + } + + /// Release any unconsumed probe permits for pause and scope breaker. + pub fn release_probe(&mut self, scope: Option<&SessionScope>) { + self.release_pause_probe(); + if let Some(scope) = scope { + self.release_breaker_probe(scope); + } + } + + /// Reissue probe permit alias for `release_probe`. + pub fn reissue_probe(&mut self, scope: Option<&SessionScope>) { + self.release_probe(scope); + } + + /// Monotonically increasing generation bumped on every pause or breaker state change. + pub fn generation(&self) -> u64 { + self.generation + } + + /// The earliest deadline among an active pause and all open breakers that + /// have not yet issued a probe permit. + pub fn earliest_probe_deadline(&self) -> Option> { + let pause_until = match &self.pause { + Some(p) if !p.probe_issued => Some(p.until), + _ => None, + }; + let breaker_until = self + .breakers + .values() + .filter(|b| !b.probe_issued) + .map(|b| b.next_probe) + .min(); + match (pause_until, breaker_until) { + (Some(p), Some(b)) => Some(p.min(b)), + (Some(p), None) => Some(p), + (None, Some(b)) => Some(b), + (None, None) => None, + } + } + + /// Number of scopes currently tracked for consecutive failures. + pub fn consecutive_len(&self) -> usize { + self.consecutive.len() + } + /// A probe on an open breaker failed: reschedule, or park once the breaker /// has been open for [`BREAKER_MAX_OPEN_HOURS`]. pub fn on_breaker_probe_failure( @@ -229,10 +319,12 @@ impl ReliabilityState { }; if now - breaker.opened_at >= Duration::hours(BREAKER_MAX_OPEN_HOURS) { self.breakers.remove(scope); + self.generation = self.generation.wrapping_add(1); return BreakerVerdict::Park; } breaker.next_probe = now + Duration::minutes(BREAKER_PROBE_MINUTES); breaker.probe_issued = false; + self.generation = self.generation.wrapping_add(1); BreakerVerdict::Reschedule { next_probe: breaker.next_probe, } @@ -259,6 +351,32 @@ impl ReliabilityState { } } + /// Whether this channel still needs a pause notice for the current pause. + pub fn pause_needs_notice(&self, channel_id: Uuid) -> bool { + self.pause + .as_ref() + .is_some_and(|p| !p.notified_channels.contains(&channel_id)) + } + + /// Mark the pause notice for `channel_id` as consumed after post succeeds. + pub fn mark_pause_notice_consumed(&mut self, channel_id: Uuid) { + if let Some(pause) = self.pause.as_mut() { + pause.notified_channels.insert(channel_id); + } + } + + /// Whether this scope's open breaker has not yet successfully posted a notice. + pub fn breaker_needs_notice(&self, scope: &SessionScope) -> bool { + self.breakers.get(scope).is_some_and(|b| !b.notified) + } + + /// Mark the open breaker's notice as consumed after post succeeds. + pub fn mark_breaker_notice_consumed(&mut self, scope: &SessionScope) { + if let Some(breaker) = self.breakers.get_mut(scope) { + breaker.notified = true; + } + } + /// Operator control frame `resume_now`: leave Paused and BreakerOpen and /// probe immediately. Returns `true` when something was actually lifted. pub fn resume_now(&mut self) -> bool { @@ -266,6 +384,9 @@ impl ReliabilityState { let had_breakers = !self.breakers.is_empty(); self.breakers.clear(); self.consecutive.clear(); + if had_pause || had_breakers { + self.generation = self.generation.wrapping_add(1); + } had_pause || had_breakers } @@ -283,6 +404,7 @@ impl ReliabilityState { } fn set_pause(&mut self, until: DateTime) { + self.generation = self.generation.wrapping_add(1); match self.pause.as_mut() { Some(pause) => { let moved = (until - pause.notified_until).num_minutes().abs(); diff --git a/crates/buzz-acp/src/reliability/state_dir.rs b/crates/buzz-acp/src/reliability/state_dir.rs index a3fc3b2975f..17f555c4366 100644 --- a/crates/buzz-acp/src/reliability/state_dir.rs +++ b/crates/buzz-acp/src/reliability/state_dir.rs @@ -151,9 +151,8 @@ pub fn write_atomic(path: &Path, contents: &[u8]) -> io::Result<()> { fs::rename(&temp, path)?; // Durability of the rename itself: without this a crash can leave the // directory entry pointing at neither file. - if let Ok(dir) = fs::File::open(parent) { - let _ = dir.sync_all(); - } + let dir = fs::File::open(parent)?; + dir.sync_all()?; Ok(()) } @@ -166,3 +165,33 @@ fn home_dir() -> Option { fn home_dir() -> Option { std::env::var_os("USERPROFILE").map(PathBuf::from) } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + #[cfg(unix)] + fn test_write_atomic_propagates_parent_dir_fsync_error() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let sub = dir.path().join("sub"); + fs::create_dir(&sub).unwrap(); + + // 0o300: write + execute, but NO read permission. + // Creating and renaming temp files succeeds, but fs::File::open(parent) fails with PermissionDenied. + fs::set_permissions(&sub, fs::Permissions::from_mode(0o300)).unwrap(); + + let target = sub.join("target.txt"); + let result = write_atomic(&target, b"test payload"); + + // Restore permissions for clean tempdir teardown + let _ = fs::set_permissions(&sub, fs::Permissions::from_mode(0o700)); + + assert!( + result.is_err(), + "write_atomic must return Err when parent directory open/fsync fails" + ); + } +} diff --git a/desktop/src-tauri/src/managed_agents/runtime.rs b/desktop/src-tauri/src/managed_agents/runtime.rs index 7a7de9f6145..f817a4afd39 100644 --- a/desktop/src-tauri/src/managed_agents/runtime.rs +++ b/desktop/src-tauri/src/managed_agents/runtime.rs @@ -129,6 +129,28 @@ pub(crate) fn apply_system_prompt_env( SystemPromptApplied(()) } +pub(crate) const STATE_DIR_ENV_VAR: &str = "BUZZ_ACP_STATE_DIR"; + +/// Apply the harness reliability state-dir env to an agent spawn command. +/// +/// Called AFTER the `descriptor.env` loop at the real spawn site, so a saved +/// user value for the reserved key can never redirect an agent's park file. +/// `None` explicitly removes the key rather than leaving it unset, so the +/// child never inherits an ambient value from the desktop's own environment. +pub(crate) fn apply_state_dir_env( + command: &mut std::process::Command, + state_dir: Option<&std::path::Path>, +) { + match state_dir { + Some(dir) => { + command.env(STATE_DIR_ENV_VAR, dir); + } + None => { + command.env_remove(STATE_DIR_ENV_VAR); + } + } +} + /// Classify an agent's persona against the live catalog for the Agents-menu /// drift indicator. Returns `(out_of_date, orphaned)`. /// @@ -1016,10 +1038,10 @@ pub fn spawn_agent_child( // the harness logs that parked batches will not survive a restart. match super::managed_agent_state_dir(app, &record.pubkey) { Ok(state_dir) => { - command.env("BUZZ_ACP_STATE_DIR", &state_dir); + apply_state_dir_env(&mut command, Some(&state_dir)); } Err(error) => { - command.env_remove("BUZZ_ACP_STATE_DIR"); + apply_state_dir_env(&mut command, None); eprintln!( "buzz-desktop: no reliability state dir for agent {}: {error}", record.name, diff --git a/desktop/src-tauri/src/managed_agents/runtime/tests.rs b/desktop/src-tauri/src/managed_agents/runtime/tests.rs index 57521c04fff..23c11265498 100644 --- a/desktop/src-tauri/src/managed_agents/runtime/tests.rs +++ b/desktop/src-tauri/src/managed_agents/runtime/tests.rs @@ -1244,3 +1244,78 @@ fn make_pair_runtime_placeholder() -> crate::managed_agents::ManagedAgentPairRun }; crate::managed_agents::ManagedAgentPairRuntime::starting(process) } + +// ── state dir env tests ────────────────────────────────────────────────── + +struct EnvVarGuard { + key: String, + prior: Option, +} + +impl EnvVarGuard { + fn set(key: &str, value: &str) -> Self { + let prior = std::env::var_os(key); + #[allow(deprecated)] + unsafe { + std::env::set_var(key, value); + } + Self { + key: key.to_string(), + prior, + } + } +} + +impl Drop for EnvVarGuard { + fn drop(&mut self) { + #[allow(deprecated)] + unsafe { + match &self.prior { + Some(v) => std::env::set_var(&self.key, v), + None => std::env::remove_var(&self.key), + } + } + } +} + +#[test] +fn test_command_execution_overrides_and_ignores_ambient_state_dir() { + // Inspects the built `Command`'s own env overrides via `get_envs()` — + // never spawns a real child. Spawning `/usr/bin/env` here would inherit + // the *entire* ambient process environment (every real credential in the + // test runner's shell) and print it to stdout on any assertion failure, + // turning a test failure into a credential leak. `get_envs()` reports + // only what this command explicitly sets or removes. + let _lock = crate::managed_agents::lock_env_mutex(); + let _guard = EnvVarGuard::set(super::STATE_DIR_ENV_VAR, "/ambient/state/dir"); + + // 1. Config specifies a state dir -> command carries the configured path, + // never the ambient value. + let mut cmd_override = std::process::Command::new("buzz-acp"); + super::apply_state_dir_env( + &mut cmd_override, + Some(std::path::Path::new("/config/specified/state/dir")), + ); + let configured = cmd_override + .get_envs() + .find(|(key, _)| *key == std::ffi::OsStr::new(super::STATE_DIR_ENV_VAR)) + .and_then(|(_, value)| value); + assert_eq!( + configured, + Some(std::ffi::OsStr::new("/config/specified/state/dir")), + "command must carry the configured state dir, not the ambient value, got: {configured:?}" + ); + + // 2. Config specifies None -> command carries an explicit removal + // (Some(key) -> None), never silent fallthrough to ambient inheritance. + let mut cmd_none = std::process::Command::new("buzz-acp"); + super::apply_state_dir_env(&mut cmd_none, None); + let removed = cmd_none + .get_envs() + .find(|(key, _)| *key == std::ffi::OsStr::new(super::STATE_DIR_ENV_VAR)); + assert_eq!( + removed, + Some((std::ffi::OsStr::new(super::STATE_DIR_ENV_VAR), None)), + "command must explicitly remove the ambient state dir, not inherit it, got: {removed:?}" + ); +} From 19c4cbe894b1ca9dba4256af258f0a4bc46f8bfb Mon Sep 17 00:00:00 2001 From: wiggdevin <202901685+wiggdevin@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:24:05 -0700 Subject: [PATCH 6/7] fix(acp): close the delta findings on the reliability harness (T16) Second and final fix round on the T16 reliability harness. Verified all 16 BLOCK findings in the delta-1 adversarial review (7 new regressions introduced by the first fix round, plus 9 sub-findings across 8 prior findings the first round left unresolved) and fixed every one: - park hand-off overflow no longer drops a batch (return_unparked hands it back instead of taking it by value) - the probe-timer busy-spin: dispatch now runs on every valid probe wake regardless of live queue depth, and pause/breaker deadlines reschedule forward instead of re-triggering on a stuck past deadline - cancelled_events (interrupted/replay carryover) are now persisted when a batch is parked instead of requeued - commit_replay rolls back earlier marks when a later one in the same plan fails; finish_replay retains in-flight ownership on partial removal failure - a directory-fsync failure after a successful atomic rename no longer makes write_atomic report the write as lost - the park reader quarantines corrupt/over-cap records to a .corrupt sibling file instead of silently truncating and admitting them - open breakers are now bounded (MAX_OPEN_BREAKERS) and swept for 6h expiry independent of new traffic on the scope - push refuses new admission at cap while reliability state is unavailable, instead of evicting an already-queued message with nowhere to land - pause/breaker probe leases are released on every dispatched outcome (retry, park, panic), not only the hold paths - a panicked turn that already produced output is parked directly as needs_review instead of losing its started flag through the plain retry queue - the ledger sanitizes a dangling partial line before every append, not only at open - discard_batch distinguishes NotFound from Discarded from DiscardedUnrecorded instead of collapsing the last two into "unknown_batch" - a ledger write failure during park_batch now surfaces a channel notice (the previously dead-code state_write_failures function) - failure-notice retries extended from ~15s to ~7-8 minutes of backoff - credential redaction now normalizes key names (strips separators/casing) and covers common token prefixes (ghp_, gho_, etc.), closing JSON/env-var/ hyphenated-key gaps - the desktop state-dir env application is now gated by a StateDirApplied proof token consumed at the real spawn call, matching the EffortApplied/McpEnvApplied pattern, so the production seam cannot be deleted or reordered without a compile error Two items are disclosed as scoped follow-ups rather than force-fit into this round: a fully durable cross-restart notice outbox, and extending the reliability-unavailable admission gate to the aggregate per-channel cap (the per-scope gate already covers the finding's exact repro). Verified: cargo fmt/clippy clean for buzz-acp and desktop/src-tauri; targeted test suites (reliability, queue, hard_timeout, managed_agents) all green. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01P8VoJ9givMm44HNc4gx5Gz Signed-off-by: wiggdevin <202901685+wiggdevin@users.noreply.github.com> --- crates/buzz-acp/src/lib.rs | 425 ++++++++++++++++-- crates/buzz-acp/src/pool.rs | 23 +- crates/buzz-acp/src/queue.rs | 92 +++- crates/buzz-acp/src/reliability.rs | 2 +- .../buzz-acp/src/reliability/error_class.rs | 146 ++++-- crates/buzz-acp/src/reliability/park.rs | 226 +++++++++- crates/buzz-acp/src/reliability/runtime.rs | 226 +++++++++- crates/buzz-acp/src/reliability/state.rs | 300 +++++++++++++ crates/buzz-acp/src/reliability/state_dir.rs | 53 ++- .../src-tauri/src/managed_agents/runtime.rs | 49 +- .../src/managed_agents/runtime/tests.rs | 37 +- 11 files changed, 1449 insertions(+), 130 deletions(-) diff --git a/crates/buzz-acp/src/lib.rs b/crates/buzz-acp/src/lib.rs index 6f150ef1505..3c5ad22b7a9 100644 --- a/crates/buzz-acp/src/lib.rs +++ b/crates/buzz-acp/src/lib.rs @@ -2010,8 +2010,20 @@ fn handle_reliability_control( }, "discard_batch" => match control_batch_id(payload) { Some(batch_id) => match reliability.discard(batch_id, "operator", now) { - Ok(true) => "discarded", - Ok(false) => "unknown_batch", + Ok(reliability::DiscardOutcome::Discarded) => "discarded", + Ok(reliability::DiscardOutcome::NotFound) => "unknown_batch", + Ok(reliability::DiscardOutcome::DiscardedUnrecorded) => { + // The batch is genuinely gone — never report "unknown + // batch" for a destructive action that actually + // happened, which would invite an operator to retry a + // discard on an id that no longer exists for a + // completely different reason. + tracing::error!( + %batch_id, + "discard_batch destroyed the batch but its ledger record failed to write" + ); + "discarded_unrecorded" + } Err(error) => { tracing::error!(%batch_id, error = %error, "discard_batch failed"); "write_failed" @@ -3316,6 +3328,12 @@ async fn tokio_main() -> Result<()> { probe_timer.rearm(reliability.as_mut()); + // While durable reliability state is unavailable, `push` must not + // evict an already-admitted event to make room for a new one — an + // evicted event has nowhere durable to land right now (T16 delta 1, + // finding 8 / prior #4). + queue.set_reliability_unavailable(reliability.is_none()); + if pool_ready && last_maintenance.elapsed() >= maintenance_interval { last_maintenance = std::time::Instant::now(); queue.compact_expired_state(); @@ -4034,7 +4052,16 @@ async fn tokio_main() -> Result<()> { _ = probe_timer.tick() => { let _ = result_rx; if probe_timer.is_valid_wake(reliability.as_ref()) { - if pool_ready && queue.has_flushable_work() { + if pool_ready { + // Always give dispatch a chance on a valid probe + // wake, even with nothing in the LIVE queue: a + // pause/breaker probe's own batch is typically + // already durably parked, not queued, so gating + // on `has_flushable_work()` meant the gate call + // that actually resolves the due deadline never + // ran at all — the deadline stayed stuck in the + // past and this arm re-fired every loop + // iteration (T16 delta 1, finding 2). for (scope, thread_tags) in dispatch_pending( &mut pool, &mut queue, @@ -4045,6 +4072,25 @@ async fn tokio_main() -> Result<()> { typing_channels.insert(scope, thread_tags); } } + // Catch any breaker whose deadline is due but that + // `dispatch_pending` never touched this cycle because + // its scope had nothing live to flush — the same + // stuck-deadline spin as pause, plus the breaker + // never expiring at all once its scope goes silent + // (finding 2 and finding 7). + if let Some(reliability) = reliability.as_mut() { + let now = chrono::Utc::now(); + for scope in reliability.state().sweep_breakers(now) { + reliability.record( + now, + reliability::ledger::LedgerBody::BreakerClosed( + reliability::ledger::BreakerClosed { + scope: scope.telemetry_label(), + }, + ), + ); + } + } } else { tracing::debug!("probe timer wake ignored: generation changed"); } @@ -4093,6 +4139,8 @@ async fn tokio_main() -> Result<()> { &respawn_tx, &mut respawn_tasks, observer.clone(), + Some(&ctx.rest_client), + reliability.as_mut(), ) == LoopAction::Exit { break; @@ -4121,6 +4169,8 @@ async fn tokio_main() -> Result<()> { &respawn_tx, &mut respawn_tasks, observer.clone(), + Some(&ctx.rest_client), + reliability.as_mut(), ); if pool.live_count() == 0 && !any_respawn_in_flight(&crash_history) { tracing::error!("all agents dead — exiting"); @@ -4932,16 +4982,30 @@ fn dispatch_pending( ), ); } + if is_pause_probe { + // Exactly one bounded probe batch is selected for a pause probe. + // Record which scope actually got dispatched so its terminal + // outcome (success, retry, park, panic) — and only its — is what + // resolves this probe's lease (finding 9 / prior #5). + if let Some(reliability) = reliability.as_deref_mut() { + reliability.state().set_pause_probe_scope(scope.clone()); + } + } dispatched_channels.push((scope, typing_scope)); *last_activity = tokio::time::Instant::now(); if is_pause_probe { - // Exactly one bounded probe batch is selected for a pause probe. break; } } if is_pause_probe && dispatched_channels.is_empty() { if let Some(reliability) = reliability { - reliability.state().release_pause_probe(); + // Nothing was actually dispatched as the probe this cycle — + // reschedule rather than leaving the deadline eligible-now, or a + // probe-timer wake with nothing to claim spins forever + // recomputing the same past deadline (finding 2). + reliability + .state() + .reschedule_pause_probe(chrono::Utc::now()); } } // Release held batches back to the queue (owner busy). They were flushed @@ -5176,6 +5240,30 @@ fn apply_reliability( } } +/// After a batch was durably parked, tell the operator's channel too when the +/// `batch_parked` / `batch_needs_review` ledger record for it failed to +/// write — `reliability.record` already logs this at ERROR, but a log line +/// nobody is watching is not the same as the channel notice every other +/// reliability event gets. Compares `write_failures()` before and after so +/// this only fires on a fresh failure, never a stale prior one (T16 delta 1, +/// finding 13 / prior #14b). +fn notice_ledger_write_failure( + batch: &FlushBatch, + before: (u64, u64), + reliability: &reliability::ReliabilityRuntime, + rest_client: Option<&relay::RestClient>, +) { + let (ledger_before, park_before) = before; + let (ledger_after, park_after) = reliability.write_failures(); + if ledger_after > ledger_before || park_after > park_before { + spawn_failure_notice( + rest_client, + batch, + reliability::notices::state_write_failures(ledger_after, park_after), + ); + } +} + /// Park a batch, or hand it back to the retry path when the park write failed. /// /// A failed park is logged and counted, never swallowed: the batch returns to @@ -5188,6 +5276,7 @@ fn park_or_fallthrough( rest_client: Option<&relay::RestClient>, now: chrono::DateTime, ) -> Disposition { + let failures_before = reliability.write_failures(); match reliability.park_batch(&batch, reason, started, now) { Ok(()) => { let content = if started { @@ -5196,6 +5285,7 @@ fn park_or_fallthrough( reliability::notices::parked(reason.as_str()) }; spawn_failure_notice(rest_client, &batch, content); + notice_ledger_write_failure(&batch, failures_before, reliability, rest_client); let preserve_retries = matches!( reason, reliability::ParkReason::Pause | reliability::ParkReason::BreakerOpen @@ -5230,6 +5320,7 @@ fn drain_park_handoff( queue::ParkHandoffReason::RetriesExhausted => reliability::ParkReason::RetriesExhausted, }; let started = handoff.batch.is_started(); + let failures_before = reliability.write_failures(); match reliability.park_batch(&handoff.batch, reason, started, now) { Ok(()) => { spawn_failure_notice( @@ -5237,6 +5328,12 @@ fn drain_park_handoff( &handoff.batch, reliability::notices::parked(reason.as_str()), ); + notice_ledger_write_failure( + &handoff.batch, + failures_before, + reliability, + rest_client, + ); } Err(error) => { tracing::error!( @@ -5246,11 +5343,27 @@ fn drain_park_handoff( error = %error, "park file write failed — holding the batch in memory for the next attempt" ); - if !queue.return_unparked(handoff) { + if let Err(handoff) = queue.return_unparked(handoff) { + // The hand-off itself is full (MAX_PARK_HANDOFF), on top of + // the park file being unwritable. `return_unparked` gives + // the exact same handoff straight back rather than + // dropping it — fall back to the live per-scope queue so + // the events stay in the harness's custody (at-least-once, + // subject to the ordinary per-scope cap) instead of being + // lost the moment this function returns (T16 delta 1, + // finding 1). tracing::error!( + channel_id = %handoff.batch.channel_id, + batch_id = %handoff.batch.batch_id, + events = handoff.batch.events.len(), "park hand-off overflowed while the park file was unwritable — \ + returning the batch to the live queue instead of losing it; \ the operator must fix the state directory" ); + let handoff = *handoff; + let scope = handoff.batch.scope.clone(); + queue.requeue_preserve_timestamps(handoff.batch); + queue.mark_complete_preserving_retries(scope); } } } @@ -5274,22 +5387,24 @@ pub(crate) fn replay_after_success( // A `turn_finished` for every batch this turn replayed: the pair // (`batch_replayed`, `turn_finished`) is what tells a later start-up that // the replay completed and must not run again. - match reliability.finish_replay(scope) { - Ok(released) => { - for batch_id in released { - reliability.record( - now, - LedgerBody::TurnFinished(led::TurnFinished { - batch_id, - channel_id: scope.channel_id(), - outcome: led::TurnOutcome::Ok, - }), - ); - } - } - Err(error) => { - tracing::error!(error = %error, "could not clear replayed batches from the park file"); - } + let report = reliability.finish_replay(scope); + for batch_id in report.released { + reliability.record( + now, + LedgerBody::TurnFinished(led::TurnFinished { + batch_id, + channel_id: scope.channel_id(), + outcome: led::TurnOutcome::Ok, + }), + ); + } + if let Some(error) = report.error { + tracing::error!( + error = %error, + "could not clear all replayed batches from the park file — the \ + remaining ones stay tracked as in-flight for this scope and are \ + retried on the next successful turn" + ); } let (pause_lifted, breaker_closed) = reliability.state().on_success(scope); if pause_lifted { @@ -5597,6 +5712,15 @@ fn handle_prompt_result( // eligible again: delivery is at least once, never at most once. reliability.abandon_replay(scope); } + // Release any pause/breaker probe lease this turn was carrying, on + // every outcome. `on_success` above already clears both for a + // successful turn (these calls are then idempotent no-ops); the gap + // this closes is every OTHER outcome — Retry, Auth-park, + // hard-timeout-park, cancelled — which previously left the lease + // stuck forever with no path back to a probe (T16 delta 1, finding 9 + // / prior #5). + reliability.state().release_pause_probe_for(scope); + reliability.state().release_breaker_probe(scope); reliability.maintain(now); } @@ -5849,6 +5973,8 @@ fn recover_panicked_agent( respawn_tx: &mpsc::Sender, respawn_tasks: &mut tokio::task::JoinSet<()>, observer: Option, + rest_client: Option<&relay::RestClient>, + mut reliability: Option<&mut reliability::ReliabilityRuntime>, ) { let task_id = join_error.id(); let Some(meta) = pool.task_map_mut().remove(&task_id) else { @@ -5856,15 +5982,59 @@ fn recover_panicked_agent( return; }; let i = meta.agent_index; + let now = chrono::Utc::now(); // Requeue BEFORE mark_complete (same rationale as handle_prompt_result). if let Some(batch) = meta.recoverable_batch { if let Some(ch) = meta.channel_id { if !removed_channels.contains(&ch) { - // Dead-letter on exhaustion is logged inside requeue(); a - // panic path has no outcome to report, so no notice here. - let _ = queue.requeue(batch); - tracing::warn!("requeued batch for panicked agent {i}"); + if batch.is_started() { + // The turn produced output or a tool call before the + // agent process panicked. `queue.requeue` deconstructs a + // batch into plain `QueuedEvent`s, discarding the shared + // `started` `Arc` entirely — the next flush + // builds a brand-new `FlushBatch` with a fresh (false) + // `started` flag, so already-started work would become + // silently auto-replay-eligible once retries exhaust. + // Park it directly as needs_review instead, the same + // outcome an in-flight interruption gets everywhere else + // (T16 delta 1, finding 10 / prior #8). + match reliability + .as_deref_mut() + .map(|r| r.park_batch(&batch, reliability::ParkReason::Panic, true, now)) + { + Some(Ok(())) => { + spawn_failure_notice( + rest_client, + &batch, + reliability::notices::needs_review(), + ); + tracing::warn!( + "parked already-started batch for panicked agent {i} — \ + held for operator review, not requeued for auto-replay" + ); + } + Some(Err(error)) => { + tracing::error!( + error = %error, + "could not park already-started batch after panic — \ + falling back to the ordinary retry queue" + ); + let _ = queue.requeue(batch); + } + None => { + // No durable reliability state available at all; + // same fallback the rest of the harness uses when + // reliability is unset. + let _ = queue.requeue(batch); + } + } + } else { + // Dead-letter on exhaustion is logged inside requeue(); a + // panic path has no outcome to report, so no notice here. + let _ = queue.requeue(batch); + tracing::warn!("requeued batch for panicked agent {i}"); + } } else { tracing::debug!( channel_id = %ch, @@ -5886,6 +6056,14 @@ fn recover_panicked_agent( // the same channel keeps its typing indicator. typing_channels.remove(scope); queue.mark_complete(scope.clone()); + // A panicked turn can never resolve a pause/breaker probe it + // was carrying on its own; release the lease here so the + // next eligible attempt is not wedged forever (T16 delta 1, + // finding 9 / prior #5). + if let Some(reliability) = reliability.as_mut() { + reliability.state().release_pause_probe_for(scope); + reliability.state().release_breaker_probe(scope); + } } None => { typing_channels.retain(|scope, _| scope.channel_id() != ch); @@ -5962,6 +6140,8 @@ fn drain_ready_join_results( respawn_tx: &mpsc::Sender, respawn_tasks: &mut tokio::task::JoinSet<()>, observer: Option, + rest_client: Option<&relay::RestClient>, + mut reliability: Option<&mut reliability::ReliabilityRuntime>, ) -> LoopAction { while let Some(Some(join_result)) = pool.join_set.join_next().now_or_never() { if let Err(join_error) = join_result { @@ -5978,6 +6158,8 @@ fn drain_ready_join_results( respawn_tx, respawn_tasks, observer.clone(), + rest_client, + reliability.as_deref_mut(), ); if pool.live_count() == 0 && !any_respawn_in_flight(crash_history) { return LoopAction::Exit; @@ -11661,6 +11843,8 @@ mod error_outcome_emission_tests { &respawn_tx, &mut respawn_tasks, Some(observer.clone()), + None, + None, ); let panic = observer @@ -11753,6 +11937,8 @@ mod error_outcome_emission_tests { &respawn_tx, &mut respawn_tasks, None, + None, + None, ); // The exact Thread scope is freed and the requeued batch is flushable @@ -13823,6 +14009,193 @@ mod reliability_dispatch_tests { ); } + // T16 delta 1, finding 10 (prior #8): the production panic-recovery seam + // itself — not a hand-rolled `mark_started` + `queue.requeue` sequence — + // must carry `started` through to the park file. Before the fix, + // `recover_panicked_agent` called plain `queue.requeue(batch)`, which + // deconstructs the batch into `QueuedEvent`s and drops the shared + // `started` `Arc` entirely; the next flush built a fresh batch with + // `started` defaulting back to `false`. + #[tokio::test] + async fn panicked_agent_with_output_is_parked_directly_as_needs_review() { + let dir = tempfile::tempdir().unwrap(); + let now = chrono::Utc::now(); + let pubkey = "abcdef0123456789abcdef0123456789abcdef0123456789abcdef012345678"; + let mut runtime = + reliability::ReliabilityRuntime::open_in(dir.path(), pubkey, now).unwrap(); + + let mut pool = AgentPool::from_slots(vec![]); + let mut queue = EventQueue::new(config::DedupMode::Queue); + let channel_id = Uuid::new_v4(); + let scope = scope::SessionScope::Conversation { channel_id }; + + let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "work") + .tags([]) + .sign_with_keys(&nostr::Keys::generate()) + .unwrap(); + queue.push(queue::QueuedEvent { + channel_id, + scope: scope.clone(), + event, + received_at: std::time::Instant::now(), + prompt_tag: "p".into(), + }); + let batch = queue.flush_next().expect("flush batch"); + // The agent produced output/a tool call before it panicked. + batch.mark_started(); + assert!(batch.is_started()); + + let (started_tx, started_rx) = tokio::sync::oneshot::channel(); + let abort_handle = pool.join_set.spawn(async move { + let _ = started_tx.send(()); + std::future::pending::<()>().await; + }); + pool.task_map_mut().insert( + abort_handle.id(), + crate::pool::TaskMeta { + agent_index: 0, + channel_id: Some(channel_id), + scope: Some(scope.clone()), + turn_id: "panic-turn-id".to_string(), + recoverable_batch: Some(batch), + control_tx: None, + steer_tx: None, + successful_steer_deliveries: HashSet::new(), + }, + ); + started_rx.await.unwrap(); + abort_handle.abort(); + let join_error = pool.join_set.join_next().await.unwrap().unwrap_err(); + + let config = crate::error_outcome_emission_tests::test_config(); + let mut heartbeat_in_flight = false; + let removed_channels = HashSet::new(); + let mut typing_channels = HashMap::new(); + let mut crash_history = vec![SlotCircuit { + crash_times: Vec::new(), + open_until: Some(std::time::Instant::now() + Duration::from_secs(3600)), + respawn_in_flight: false, + }]; + let (respawn_tx, _respawn_rx) = mpsc::channel(8); + let mut respawn_tasks = tokio::task::JoinSet::new(); + + recover_panicked_agent( + &mut pool, + &mut queue, + &config, + join_error, + &mut heartbeat_in_flight, + &removed_channels, + &mut typing_channels, + &mut crash_history, + &respawn_tx, + &mut respawn_tasks, + None, + None, + Some(&mut runtime), + ); + + assert!( + !queue.has_undispatched_work(), + "an already-started batch must never re-enter the ordinary retry \ + queue — it was parked directly instead" + ); + let parked = runtime.park().batches(); + assert_eq!( + parked.len(), + 1, + "the panicked batch must be parked, not requeued" + ); + assert!( + parked[0].started, + "batch that produced output before panicking must park with started == true" + ); + assert!( + parked[0].needs_review, + "an already-started parked batch must be held for operator review" + ); + assert!( + !parked[0].replay_eligible(), + "an already-started parked batch must not be auto-replay-eligible" + ); + } + + // T16 delta 1, finding 13 (prior #14b): `park_batch` durably writes the + // park file even when its OWN follow-up `batch_parked` ledger record + // fails to append. The batch is not lost — but nothing beyond a log line + // told the operator the audit trail was incomplete. `park_or_fallthrough` + // now checks `write_failures()` and sends the (previously dead-code) + // `state_write_failures` notice on exactly this gap. + #[test] + #[cfg(unix)] + fn park_or_fallthrough_reports_a_ledger_write_failure_even_though_the_batch_still_parks() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::tempdir().unwrap(); + let now = chrono::Utc::now(); + let pubkey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let mut runtime = + reliability::ReliabilityRuntime::open_in(dir.path(), pubkey, now).unwrap(); + + let channel_id = Uuid::new_v4(); + let scope = scope::SessionScope::Conversation { channel_id }; + let event = nostr::EventBuilder::new(nostr::Kind::Custom(9), "x") + .tags([]) + .sign_with_keys(&nostr::Keys::generate()) + .unwrap(); + let batch = FlushBatch { + batch_id: Uuid::new_v4(), + channel_id, + scope, + events: vec![queue::BatchEvent { + event, + prompt_tag: "t".into(), + received_at: std::time::Instant::now(), + }], + cancelled_events: vec![], + cancel_reason: None, + started: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)), + }; + + // Ledger unwritable, park file (and its directory) stay writable. + let ledger_path = dir.path().join("ledger.jsonl"); + let original_mode = std::fs::metadata(&ledger_path) + .unwrap() + .permissions() + .mode(); + std::fs::set_permissions(&ledger_path, std::fs::Permissions::from_mode(0o400)).unwrap(); + + let failures_before = runtime.write_failures(); + let disposition = park_or_fallthrough( + &mut runtime, + batch, + reliability::ParkReason::RetriesExhausted, + false, + None, + now, + ); + + let _ = + std::fs::set_permissions(&ledger_path, std::fs::Permissions::from_mode(original_mode)); + + assert!( + matches!(disposition, Disposition::Handled { .. }), + "the batch is durably parked and must count as Handled even though \ + its ledger record failed" + ); + assert_eq!( + runtime.park().batches().len(), + 1, + "the batch itself must still be durably parked" + ); + let failures_after = runtime.write_failures(); + assert!( + failures_after.0 > failures_before.0, + "a ledger append failure inside park_batch must be visible through \ + write_failures(), which is what gates the state_write_failures notice" + ); + } + #[tokio::test] async fn test_failure_notice_not_consumed_until_ack_received() { use crate::error_outcome_emission_tests::dummy_agent; diff --git a/crates/buzz-acp/src/pool.rs b/crates/buzz-acp/src/pool.rs index 83982bcddc0..38cc8055838 100644 --- a/crates/buzz-acp/src/pool.rs +++ b/crates/buzz-acp/src/pool.rs @@ -5049,8 +5049,19 @@ pub(crate) async fn post_failure_notice( return false; } }; - const MAX_NOTICE_ATTEMPTS: usize = 3; - let mut delay = Duration::from_millis(100); + // T16 delta 1, finding 14 (prior #15): 3 attempts spanning well under a + // minute gives up long before a typical relay blip (seconds to several + // minutes) recovers, so a channel/scope going through an outage at the + // same moment the relay itself is degraded is simply never told. This is + // a detached `tokio::spawn`ed task (see `spawn_failure_notice[_with_ack]` + // in lib.rs) — it costs nothing to keep retrying here; it only blocks + // itself, never the main loop. This still does not survive a process + // restart mid-retry (a genuinely durable, cross-restart outbox is a + // larger follow-up), but it closes the much more common case of the + // relay recovering while the harness keeps running. + const MAX_NOTICE_ATTEMPTS: usize = 12; + const MAX_NOTICE_DELAY: Duration = Duration::from_secs(60); + let mut delay = Duration::from_secs(1); for attempt in 1..=MAX_NOTICE_ATTEMPTS { match tokio::time::timeout(Duration::from_secs(5), rest.submit_event(&event)).await { Ok(Ok(_)) => return true, @@ -5063,9 +5074,15 @@ pub(crate) async fn post_failure_notice( } if attempt < MAX_NOTICE_ATTEMPTS { tokio::time::sleep(delay).await; - delay = (delay * 2).min(Duration::from_secs(2)); + delay = (delay * 2).min(MAX_NOTICE_DELAY); } } + tracing::error!( + channel = %channel_id, + attempts = MAX_NOTICE_ATTEMPTS, + "failure notice exhausted every retry — the channel was never told; \ + this does not persist across a process restart" + ); false } diff --git a/crates/buzz-acp/src/queue.rs b/crates/buzz-acp/src/queue.rs index cafe6fb3512..0b7ead0b862 100644 --- a/crates/buzz-acp/src/queue.rs +++ b/crates/buzz-acp/src/queue.rs @@ -271,6 +271,17 @@ pub struct EventQueue { parked_out: VecDeque, /// Number of times `flush_next` has been called. flush_count: usize, + /// Set by the caller while the durable reliability state directory is + /// unavailable (state-dir open failed and has not yet reopened). + /// + /// `push` still admits events into the live per-scope/per-channel queues + /// while this is set — dropping the connection or refusing to admit at + /// all would just move the loss earlier — but it refuses to *evict* an + /// already-admitted event to make room for a new one, since an evicted + /// event has nowhere durable to land. The newest arrival is refused + /// instead, which is explicit and counted rather than a silent swap of + /// one lost message for another (T16 delta 1, finding 8/"prior #4"). + reliability_unavailable: bool, } /// Most batches held in the park hand-off at once. @@ -313,9 +324,19 @@ impl EventQueue { in_flight_deadline: Duration::from_secs(DEFAULT_IN_FLIGHT_DEADLINE_SECS), parked_out: VecDeque::new(), flush_count: 0, + reliability_unavailable: false, } } + /// Record whether durable reliability state is currently unavailable. + /// + /// Called once per main-loop iteration from the harness so `push`'s + /// admission-vs-eviction choice always reflects the current state-dir + /// availability, not a stale snapshot from when the queue was built. + pub fn set_reliability_unavailable(&mut self, unavailable: bool) { + self.reliability_unavailable = unavailable; + } + /// Number of times `flush_next` has been called on this queue. #[allow(dead_code)] pub fn flush_count(&self) -> usize { @@ -378,8 +399,25 @@ impl EventQueue { let channel_id = event.channel_id; let scope = event.scope.clone(); let queue = self.queues.entry(scope.clone()).or_default(); - // Enforce per-scope depth cap: drop oldest in this partition. + // Enforce per-scope depth cap. Normally this evicts the oldest event + // in the partition to admit the new one. But an evicted event is + // gone for good — nothing durable holds it — so while reliability + // state is unavailable (no park file to fall back on if things get + // worse), refuse the *new* arrival instead: whatever is already + // queued stays queued, and the refusal is explicit and logged at + // ERROR rather than a silent swap of one lost message for another. if queue.len() >= MAX_PENDING_PER_SCOPE { + if self.reliability_unavailable { + tracing::error!( + channel_id = %channel_id, + scope = %scope.telemetry_label(), + limit = MAX_PENDING_PER_SCOPE, + "refusing new event — per-scope queue is at cap and durable \ + reliability state is unavailable, so an eviction would be \ + unrecoverable" + ); + return false; + } queue.pop_front(); tracing::warn!( channel_id = %channel_id, @@ -725,11 +763,13 @@ impl EventQueue { /// Give a batch back to the hand-off after a park-file write failed. /// - /// Returns `false` — and logs — when the hand-off is at - /// [`MAX_PARK_HANDOFF`]. A `false` return is the caller's signal that the - /// batch could not be held here either; it must stay in the caller's own - /// hands or the failure has to be surfaced. - pub fn return_unparked(&mut self, handoff: ParkHandoff) -> bool { + /// Returns `Err(handoff)` — and logs — when the hand-off is at + /// [`MAX_PARK_HANDOFF`], handing the exact same handoff straight back to + /// the caller. The caller owns it again immediately: nothing is ever + /// dropped here even when the hand-off itself is full, unlike the old + /// `bool` return, which let a `false` result fall out of scope and take + /// the batch's messages with it (T16 delta 1, finding 1). + pub fn return_unparked(&mut self, handoff: ParkHandoff) -> Result<(), Box> { if self.parked_out.len() >= MAX_PARK_HANDOFF { tracing::error!( channel_id = %handoff.batch.channel_id, @@ -738,10 +778,10 @@ impl EventQueue { events = handoff.batch.events.len(), "park hand-off is full — the batch could not be held for a retry of the park write" ); - return false; + return Err(Box::new(handoff)); } self.parked_out.push_front(handoff); - true + Ok(()) } /// Stage parked events for replay ahead of anything newer for `scope`. @@ -2676,6 +2716,42 @@ mod tests { assert_eq!(q.queues.len(), 0); } + // T16 delta 1, finding 8 (prior #4): while durable reliability state is + // unavailable, hitting the per-scope cap must refuse the new arrival + // rather than silently evict an already-queued one that has nowhere + // durable to fall back to. + #[test] + fn push_refuses_new_arrival_at_cap_when_reliability_unavailable() { + let mut q = EventQueue::new(DedupMode::Queue); + let ch = Uuid::new_v4(); + + for i in 0..MAX_PENDING_PER_SCOPE { + assert!(q.push(make_queued(ch, &format!("msg-{i}")))); + } + assert_eq!(pending_count(&q), MAX_PENDING_PER_SCOPE); + + q.set_reliability_unavailable(true); + let accepted = q.push(make_queued(ch, "the 501st message")); + assert!( + !accepted, + "a new arrival at cap must be refused, not silently admitted by evicting an old one" + ); + assert_eq!( + pending_count(&q), + MAX_PENDING_PER_SCOPE, + "the already-queued messages must be untouched — none evicted" + ); + + // Once reliability is back, normal eviction behavior resumes. + q.set_reliability_unavailable(false); + let accepted = q.push(make_queued(ch, "message after recovery")); + assert!( + accepted, + "once reliability is available again, admission (with eviction) resumes" + ); + assert_eq!(pending_count(&q), MAX_PENDING_PER_SCOPE); + } + #[test] fn test_in_flight_blocks_same_channel() { let mut q = EventQueue::new(DedupMode::Queue); diff --git a/crates/buzz-acp/src/reliability.rs b/crates/buzz-acp/src/reliability.rs index 8664e6e440a..91c33886a43 100644 --- a/crates/buzz-acp/src/reliability.rs +++ b/crates/buzz-acp/src/reliability.rs @@ -30,7 +30,7 @@ pub mod state_dir; pub use error_class::{classify_at, sanitize_error_diagnostic}; pub use park::{ParkError, ParkReason, ParkedBatch}; -pub use runtime::{ReliabilityRuntime, ReplayPlan}; +pub use runtime::{DiscardOutcome, ReliabilityRuntime, ReplayPlan}; pub use state::{BreakerGate, BreakerVerdict, PauseGate, ReliabilityState}; /// Longest provider error text the harness inspects, stores or forwards. diff --git a/crates/buzz-acp/src/reliability/error_class.rs b/crates/buzz-acp/src/reliability/error_class.rs index 1cd49e58656..c15646ec241 100644 --- a/crates/buzz-acp/src/reliability/error_class.rs +++ b/crates/buzz-acp/src/reliability/error_class.rs @@ -100,6 +100,46 @@ pub fn truncate_chars(text: &str, max: usize) -> String { text.chars().take(max).collect() } +/// Well-known credential prefixes redacted regardless of any surrounding +/// `key=`/`key:` framing — these tokens are self-identifying by shape alone +/// (GitHub personal-access and app tokens, Anthropic/OpenAI-style secret +/// keys, Nostr private keys). +const CREDENTIAL_PREFIXES: &[&str] = &[ + "sk-", + "nsec1", + "ghp_", + "gho_", + "ghu_", + "ghs_", + "ghr_", + "github_pat_", +]; + +/// Substrings that mark a `key`/`key=value` token as sensitive, matched +/// against the key with every non-alphanumeric character stripped and +/// lowercased — so `API_KEY`, `api-key`, `apiKey`, `"api_key"` (a JSON key +/// with its opening brace/quote still attached) and `OPENAI_API_KEY` all +/// normalize to a form containing `apikey`/`key` and match the same way. +/// Bare substring matching is deliberately broad: over-redacting a +/// non-sensitive `key=value` pair is a cosmetic loss, letting a real +/// credential through because its separator or casing was slightly +/// different is not (T16 delta 1, finding 15 / prior #19). +const SENSITIVE_KEY_MARKERS: &[&str] = &["key", "token", "secret", "password", "credential"]; + +/// Lowercase and drop every non-alphanumeric character, so `API_KEY`, +/// `api-key`, `"api_key"` and `apiKey` all compare equal. +fn normalize_key(k: &str) -> String { + k.chars() + .filter(|c| c.is_ascii_alphanumeric()) + .flat_map(|c| c.to_lowercase()) + .collect() +} + +fn is_sensitive_key(k: &str) -> bool { + let normalized = normalize_key(k); + !normalized.is_empty() && SENSITIVE_KEY_MARKERS.iter().any(|m| normalized.contains(m)) +} + /// Sanitize and bound an error string for external diagnostics (observer events, /// warnings, channel notices). /// @@ -136,47 +176,27 @@ pub fn sanitize_error_diagnostic(raw: &str) -> String { words.push(part.to_string()); continue; } - // sk-ant-... or sk-... secret keys - if lower_trimmed.starts_with("sk-") && trimmed_part.len() > 7 { - words.push(format!("sk-{trailing_punct}")); - continue; - } - // nsec1... private keys - if lower_trimmed.starts_with("nsec1") && trimmed_part.len() > 10 { - words.push(format!("nsec1{trailing_punct}")); + // Self-identifying credential shapes: `sk-...`, `nsec1...`, + // `ghp_...` and the rest of `CREDENTIAL_PREFIXES` — redacted whole, + // wherever they appear, with no key= framing needed. + if let Some(prefix) = CREDENTIAL_PREFIXES + .iter() + .find(|p| lower_trimmed.starts_with(**p) && trimmed_part.len() > p.len() + 3) + { + words.push(format!("{prefix}{trailing_punct}")); continue; } - // key=value or key:value + // key=value or key:value — including a JSON-style `"key":"value"` + // token, where `k` still carries its opening brace/quote and `is_sensitive_key` + // strips that off before comparing. if let Some((k, _v)) = trimmed_part.split_once('=') { - let k_lower = k.to_lowercase(); - if matches!( - k_lower.as_str(), - "token" - | "secret" - | "password" - | "key" - | "api_key" - | "apikey" - | "auth_token" - | "access_token" - ) { + if is_sensitive_key(k) { words.push(format!("{k}={trailing_punct}")); continue; } } if let Some((k, _v)) = trimmed_part.split_once(':') { - let k_lower = k.to_lowercase(); - if matches!( - k_lower.as_str(), - "token" - | "secret" - | "password" - | "key" - | "api_key" - | "apikey" - | "auth_token" - | "access_token" - ) { + if is_sensitive_key(k) { words.push(format!("{k}:{trailing_punct}")); continue; } @@ -297,3 +317,63 @@ fn next_local_occurrence(tz: Tz, time: NaiveTime, now: DateTime) -> Option< } None } + +#[cfg(test)] +mod redaction_tests { + use super::sanitize_error_diagnostic; + + // T16 delta 1, finding 15 (prior #19): the token matcher only recognized + // a narrow set of unquoted exact keys, so common real-world credential + // shapes reached tracing, the observer diagnostic and channel notices + // unredacted. + #[test] + fn redacts_a_json_style_quoted_key() { + let out = sanitize_error_diagnostic(r#"provider error: {"api_key":"sk-abcdef123456"}"#); + assert!( + !out.contains("abcdef123456"), + "the secret value must not survive: {out}" + ); + } + + #[test] + fn redacts_an_environment_style_key_name() { + let out = sanitize_error_diagnostic("OPENAI_API_KEY=sk-abcdef123456 rejected"); + assert!( + !out.contains("abcdef123456"), + "an env-var-style key name (not the bare exact \"api_key\") must \ + still trigger redaction: {out}" + ); + } + + #[test] + fn redacts_a_hyphenated_key_name() { + let out = sanitize_error_diagnostic("api-key=abcdef123456 invalid"); + assert!( + !out.contains("abcdef123456"), + "a hyphenated key name must match the same as the underscored form: {out}" + ); + } + + #[test] + fn redacts_a_github_token_prefix_with_no_key_framing() { + let out = sanitize_error_diagnostic("push failed: ghp_abcdef1234567890 denied"); + assert!( + !out.contains("abcdef1234567890"), + "a self-identifying credential prefix must redact even with no \ + key=/key: framing at all: {out}" + ); + } + + #[test] + fn still_redacts_the_original_bare_exact_keys() { + let out = sanitize_error_diagnostic("token=abcdef123456 password=hunter2"); + assert!(!out.contains("abcdef123456")); + assert!(!out.contains("hunter2")); + } + + #[test] + fn leaves_ordinary_text_alone() { + let out = sanitize_error_diagnostic("connection refused: timeout after 30s"); + assert_eq!(out, "connection refused: timeout after 30s"); + } +} diff --git a/crates/buzz-acp/src/reliability/park.rs b/crates/buzz-acp/src/reliability/park.rs index 6692acf1a45..f8f772b5eec 100644 --- a/crates/buzz-acp/src/reliability/park.rs +++ b/crates/buzz-acp/src/reliability/park.rs @@ -74,6 +74,10 @@ pub enum ParkReason { Pause, /// The scope breaker opened due to repeated failures. BreakerOpen, + /// The agent task panicked after the turn had already produced output or + /// a tool call; parked directly rather than risking an auto-replay of + /// already-started work through the ordinary retry loop. + Panic, } impl ParkReason { @@ -86,6 +90,7 @@ impl ParkReason { Self::BreakerExpired => "breaker_expired", Self::Pause => "pause", Self::BreakerOpen => "breaker_open", + Self::Panic => "panic", } } } @@ -193,24 +198,35 @@ impl ParkedBatch { /// Park a live batch. Events past [`MAX_PARKED_EVENTS`] are refused rather /// than silently trimmed — the queue never builds a larger batch, so a /// larger one is a bug, not a message to drop. + /// + /// `cancelled_events` — the carryover from an interrupted or replayed + /// prior turn that `FlushBatch` keeps separate for prompt framing — are + /// persisted too, ordered before `events` the same way + /// `requeue_preserve_timestamps` restores them: "original before newer". + /// A version that parked only `events` silently erased every interrupted + /// or in-flight-replay message the moment its batch got parked instead of + /// requeued (T16 delta 1, finding 3). pub fn from_batch( batch: &FlushBatch, reason: ParkReason, started: bool, now: DateTime, ) -> Result { - if batch.events.len() > MAX_PARKED_EVENTS { - return Err(ParkError::TooManyEvents(batch.events.len())); + let total_events = batch.cancelled_events.len() + batch.events.len(); + if total_events > MAX_PARKED_EVENTS { + return Err(ParkError::TooManyEvents(total_events)); } + let to_parked_event = |be: &BatchEvent| ParkedEvent { + event: be.event.clone(), + prompt_tag: truncate_chars(&be.prompt_tag, MAX_TAG_CHARS), + received_at: DateTime::from_timestamp(be.event.created_at.as_secs() as i64, 0) + .unwrap_or(now), + }; let events = batch - .events + .cancelled_events .iter() - .map(|be| ParkedEvent { - event: be.event.clone(), - prompt_tag: truncate_chars(&be.prompt_tag, MAX_TAG_CHARS), - received_at: DateTime::from_timestamp(be.event.created_at.as_secs() as i64, 0) - .unwrap_or(now), - }) + .chain(batch.events.iter()) + .map(to_parked_event) .collect(); Ok(Self { batch_id: batch.batch_id, @@ -540,8 +556,20 @@ fn serialize(batches: &[ParkedBatch]) -> Result, ParkError> { Ok(buffer) } +/// Suffix of the sibling file an unreadable park line is copied to, verbatim, +/// before it is dropped from the live in-memory image. +const QUARANTINE_SUFFIX: &str = ".corrupt"; + /// Read the park file with a hard byte cap on the input and a hard cap per -/// line. Unreadable lines are counted and skipped. +/// line. +/// +/// A line this cannot use — too long, not UTF-8, not valid JSON, or (once +/// parsed) carrying more events than [`MAX_PARKED_EVENTS`] — is never +/// silently modified or dropped without a trace. Every such line is copied +/// verbatim to a `.corrupt` sibling file (best-effort) before being excluded +/// from the live batches, so an operator can recover the original bytes +/// instead of the client messages in it simply vanishing on the next read (T16 +/// delta 1, finding 6). fn read_batches(path: &Path) -> io::Result> { let file = match std::fs::File::open(path) { Ok(file) => file, @@ -568,12 +596,14 @@ fn read_batches(path: &Path) -> io::Result> { } if line.len() > MAX_LINE_BYTES { skipped += 1; + quarantine_line(path, &line); continue; } let text = match std::str::from_utf8(&line) { Ok(text) => text.trim(), Err(_) => { skipped += 1; + quarantine_line(path, &line); continue; } }; @@ -581,24 +611,80 @@ fn read_batches(path: &Path) -> io::Result> { continue; } match serde_json::from_str::(text) { - Ok(mut batch) => { - batch.events.truncate(MAX_PARKED_EVENTS); - batches.push(batch); + Ok(batch) if batch.events.len() > MAX_PARKED_EVENTS => { + // A syntactically valid record with more events than the + // cap allows is corruption (or a future/incompatible + // format), not a batch to admit with its tail silently cut + // off — every event past the cap would otherwise vanish + // with the read reporting success. + tracing::error!( + batch_id = %batch.batch_id, + events = batch.events.len(), + cap = MAX_PARKED_EVENTS, + path = %path.display(), + "parked batch carries more events than the cap allows — \ + quarantining the whole record rather than truncating it" + ); + skipped += 1; + quarantine_line(path, &line); + } + Ok(batch) => batches.push(batch), + Err(_) => { + skipped += 1; + quarantine_line(path, &line); } - Err(_) => skipped += 1, } } if skipped > 0 { - tracing::warn!( + tracing::error!( skipped, path = %path.display(), - "skipped unreadable park file lines" + "unreadable park file lines were quarantined to a .corrupt sibling \ + file rather than dropped — operator recovery required" ); } batches.sort_by_key(|b| b.parked_at); Ok(batches) } +/// Best-effort: append `line` verbatim to `.corrupt`. A failure here is +/// logged, never propagated — quarantining is a courtesy on top of the +/// primary guarantee (the line is excluded from the live batches either way), +/// not itself load-bearing for correctness. +fn quarantine_line(path: &Path, line: &[u8]) { + use std::io::Write as _; + + let quarantine_path = { + let mut name = path.as_os_str().to_owned(); + name.push(QUARANTINE_SUFFIX); + PathBuf::from(name) + }; + let mut options = std::fs::OpenOptions::new(); + options.create(true).append(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + // Same 0600 owner-only mode every other state file gets — this file + // can hold client message content. + options.mode(0o600); + } + let result = options.open(&quarantine_path).and_then(|mut file| { + file.write_all(line)?; + if line.last() != Some(&b'\n') { + file.write_all(b"\n")?; + } + Ok(()) + }); + if let Err(error) = result { + tracing::error!( + path = %quarantine_path.display(), + error = %error, + "could not quarantine an unreadable park file line — it is still \ + excluded from the live batches, but its original bytes are lost" + ); + } +} + #[cfg(test)] mod tests { use super::*; @@ -653,6 +739,63 @@ mod tests { assert_eq!(reopened.batches().len(), 1); } + // T16 delta 1, finding 3: a batch carrying `cancelled_events` (the + // carryover from an interrupted or in-flight-replay turn) must not lose + // them just because the batch itself ends up parked instead of + // requeued. + #[test] + fn test_from_batch_preserves_cancelled_events_ordered_before_new_ones() { + let channel_id = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id }; + let mut batch = dummy_batch(channel_id, Uuid::new_v4(), scope, "new message"); + batch.cancelled_events = vec![BatchEvent { + event: dummy_event("interrupted message"), + prompt_tag: "test".into(), + received_at: std::time::Instant::now(), + }]; + batch.cancel_reason = Some(crate::queue::CancelReason::Interrupt); + + let parked = + ParkedBatch::from_batch(&batch, ParkReason::RetriesExhausted, false, Utc::now()) + .unwrap(); + + assert_eq!( + parked.events.len(), + 2, + "both the cancelled carryover and the new event must be persisted" + ); + assert_eq!( + parked.events[0].event.content, "interrupted message", + "the cancelled carryover must come first, matching the \ + original-before-newer ordering used everywhere else" + ); + assert_eq!(parked.events[1].event.content, "new message"); + } + + #[test] + fn test_from_batch_rejects_when_cancelled_plus_new_events_exceed_the_cap() { + let channel_id = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id }; + let mut batch = dummy_batch(channel_id, Uuid::new_v4(), scope.clone(), "new"); + // events.len() == 1 already; add MAX_PARKED_EVENTS more via cancelled + // carryover so the combined total is one over the cap. + batch.cancelled_events = (0..MAX_PARKED_EVENTS) + .map(|i| BatchEvent { + event: dummy_event(&format!("cancelled-{i}")), + prompt_tag: "test".into(), + received_at: std::time::Instant::now(), + }) + .collect(); + + let result = + ParkedBatch::from_batch(&batch, ParkReason::RetriesExhausted, false, Utc::now()); + assert!( + matches!(result, Err(ParkError::TooManyEvents(n)) if n == MAX_PARKED_EVENTS + 1), + "the cap must count cancelled_events + events together, not just \ + events alone: {result:?}" + ); + } + #[test] fn test_park_rejects_oversized_individual_line() { let dir = tempfile::tempdir().unwrap(); @@ -677,6 +820,57 @@ mod tests { assert!(park.batches().is_empty()); } + // T16 delta 1, finding 6: a syntactically valid on-disk record with more + // events than MAX_PARKED_EVENTS must be quarantined whole, never + // silently truncated and admitted as if nothing were wrong. + #[test] + fn test_read_batches_quarantines_rather_than_truncates_an_oversized_record() { + let dir = tempfile::tempdir().unwrap(); + let channel_id = Uuid::new_v4(); + let batch_id = Uuid::new_v4(); + let over_cap = ParkedBatch { + batch_id, + channel_id, + scope: ScopeRef::from_scope(&SessionScope::Conversation { channel_id }), + reason: ParkReason::RetriesExhausted, + started: false, + needs_review: false, + needs_review_reason: None, + replayed_at: None, + forced: false, + parked_at: Utc::now(), + events: (0..MAX_PARKED_EVENTS + 1) + .map(|i| ParkedEvent { + event: dummy_event(&format!("event-{i}")), + prompt_tag: "test".into(), + received_at: Utc::now(), + }) + .collect(), + }; + let line = serde_json::to_string(&over_cap).unwrap(); + + let park_path = dir.path().join(PARK_FILE); + std::fs::write(&park_path, format!("{line}\n")).unwrap(); + + let park = ParkFile::open(dir.path()).unwrap(); + assert!( + !park.contains(batch_id), + "an over-cap record must never be admitted, truncated or otherwise" + ); + assert!( + park.batches().is_empty(), + "no events from the over-cap record may survive into the live image" + ); + + let quarantine_path = dir.path().join(format!("{PARK_FILE}{QUARANTINE_SUFFIX}")); + let quarantined = std::fs::read_to_string(&quarantine_path) + .expect("the original record must be preserved in the quarantine file"); + assert!( + quarantined.contains(&batch_id.to_string()), + "the quarantined line must be the original record, recoverable by an operator" + ); + } + #[test] fn test_reconcile_on_start_crashed_mid_replay_moves_to_needs_review() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/buzz-acp/src/reliability/runtime.rs b/crates/buzz-acp/src/reliability/runtime.rs index 471d05db608..c2a022e2583 100644 --- a/crates/buzz-acp/src/reliability/runtime.rs +++ b/crates/buzz-acp/src/reliability/runtime.rs @@ -39,6 +39,33 @@ pub struct ReplayPlan { pub channel_id: Uuid, } +/// The result of [`ReliabilityRuntime::discard`], distinguishing "there was +/// nothing to discard" from "the batch was destroyed but its ledger record +/// failed" — the two collapsed into the same `false` under the old `bool` +/// return, which made the caller report a successful destructive discard as +/// `unknown_batch` (T16 delta 1, finding 12 / prior #14a). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum DiscardOutcome { + /// No parked batch had this id; nothing was touched. + NotFound, + /// The batch was removed and the ledger record landed. + Discarded, + /// The batch was durably removed, but the ledger append failed — the + /// discard happened and is irreversible, it just has no audit record. + DiscardedUnrecorded, +} + +/// The result of [`ReliabilityRuntime::finish_replay`]. +#[derive(Debug, Default)] +pub struct FinishReplayReport { + /// Batch ids actually removed from the park file, even when a later id + /// in the same call failed to remove. + pub released: Vec, + /// The first removal failure encountered, if any. `released` still holds + /// whatever succeeded before it. + pub error: Option, +} + /// The harness's reliability state for one agent. pub struct ReliabilityRuntime { dir: PathBuf, @@ -226,8 +253,24 @@ impl ReliabilityRuntime { new_batch_id: Uuid, now: DateTime, ) -> Result<(), ParkError> { + // Mark every batch in the plan as replayed, but if any mark fails + // partway through, roll back the ones that already landed rather + // than propagating immediately: an unrolled-back partial mark would + // leave an earlier batch durably stamped `replayed_at` (making it + // permanently ineligible for replay) even though this replay attempt + // as a whole is being reported as failed and nothing is being sent + // (T16 delta 1, finding 4a). + let mut marked = Vec::with_capacity(plan.batch_ids.len()); for batch_id in &plan.batch_ids { - self.park.mark_replayed(*batch_id, now)?; + match self.park.mark_replayed(*batch_id, now) { + Ok(()) => marked.push(*batch_id), + Err(error) => { + for done in &marked { + let _ = self.park.unmark_replayed(*done); + } + return Err(error); + } + } } let mut all_recorded = true; for batch_id in &plan.batch_ids { @@ -260,18 +303,50 @@ impl ReliabilityRuntime { } /// A turn for `scope` finished successfully: any batches it was replaying - /// leave the park file for good. Returns the batch ids released. - pub fn finish_replay(&mut self, scope: &SessionScope) -> Result, ParkError> { - let Some(batch_ids) = self.in_flight_replays.remove(scope) else { - return Ok(Vec::new()); + /// leave the park file for good. + /// + /// In-flight ownership for `scope` is only cleared once every batch is + /// actually removed. A batch that fails to remove stays recorded as + /// in-flight for the scope so a later call (the next successful turn, or + /// an explicit retry) can still find and finish it — dropping ownership + /// on a partial failure would leave that batch stamped `replayed_at` + /// forever with nothing left that knows to clean it up (T16 delta 1, + /// finding 4b). The report carries every id actually released even when + /// a later one in the same plan failed, so the caller can still write + /// `turn_finished` for the ones that did land. + pub fn finish_replay(&mut self, scope: &SessionScope) -> FinishReplayReport { + let Some(batch_ids) = self.in_flight_replays.get(scope).cloned() else { + return FinishReplayReport { + released: Vec::new(), + error: None, + }; }; let mut released = Vec::new(); + let mut remaining = Vec::new(); + let mut first_error = None; for batch_id in batch_ids { - if self.park.remove(batch_id)?.is_some() { - released.push(batch_id); + match self.park.remove(batch_id) { + Ok(Some(_)) => released.push(batch_id), + // Already gone (e.g. a previous partial attempt already + // removed it) — nothing left to track for this id. + Ok(None) => {} + Err(error) => { + remaining.push(batch_id); + if first_error.is_none() { + first_error = Some(error); + } + } } } - Ok(released) + if remaining.is_empty() { + self.in_flight_replays.remove(scope); + } else { + self.in_flight_replays.insert(scope.clone(), remaining); + } + FinishReplayReport { + released, + error: first_error, + } } /// A turn for `scope` failed: its replayed batches stay parked and go back @@ -293,14 +368,21 @@ impl ReliabilityRuntime { } /// Operator control frame `discard_batch`. + /// + /// [`DiscardOutcome::NotFound`] and a failed ledger record after a real + /// destructive removal must never collapse into the same signal — an + /// operator who sees "unknown batch" for a discard that actually + /// happened has no way to tell it landed, and might discard-retry a + /// batch id that no longer exists for a completely different reason + /// (T16 delta 1, finding 12 / prior #14a). pub fn discard( &mut self, batch_id: Uuid, by: &str, now: DateTime, - ) -> Result { + ) -> Result { let Some(removed) = self.park.remove(batch_id)? else { - return Ok(false); + return Ok(DiscardOutcome::NotFound); }; let recorded = self.record( now, @@ -310,7 +392,11 @@ impl ReliabilityRuntime { by: super::error_class::truncate_chars(by, ledger::MAX_LABEL_CHARS), }), ); - Ok(recorded) + if recorded { + Ok(DiscardOutcome::Discarded) + } else { + Ok(DiscardOutcome::DiscardedUnrecorded) + } } /// Operator control frame `replay_batch`: make one parked batch eligible @@ -566,6 +652,21 @@ mod tests { ); } + #[test] + fn test_discard_of_unknown_batch_is_not_found_not_discarded() { + let dir = tempfile::tempdir().unwrap(); + let pubkey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let now = Utc::now(); + let mut runtime = ReliabilityRuntime::open_in(dir.path(), pubkey, now).unwrap(); + + let result = runtime.discard(Uuid::new_v4(), "operator", now); + assert!( + matches!(result, Ok(DiscardOutcome::NotFound)), + "discarding an id that was never parked must report NotFound, \ + distinct from a destructive outcome: got {result:?}" + ); + } + #[test] #[cfg(unix)] fn test_discard_fails_contract_when_ledger_append_fails() { @@ -601,11 +702,101 @@ mod tests { let _ = std::fs::set_permissions(&ledger_path, std::fs::Permissions::from_mode(original_mode)); - // The batch was removed from park, but ledger write failed. - // It must NOT report unconditional success (Ok(true)). + // The batch was removed from park, but ledger write failed. It must + // be reported as destroyed-but-unrecorded — never as a clean + // `Discarded` (unconditional success) and never as `NotFound` + // (which would collapse a genuine destructive action into the same + // signal as "no such batch", inviting a pointless retry). assert!( - !matches!(result, Ok(true)), - "discard must not report unconditional success when ledger write failed: got {result:?}" + matches!(result, Ok(DiscardOutcome::DiscardedUnrecorded)), + "discard must distinguish a destroyed-but-unrecorded batch from \ + both a clean success and an unknown batch: got {result:?}" + ); + } + + /// The largest content length for which `runtime.park_batch(..)` still + /// succeeds — i.e. the batch's own serialized line is at (or a hair + /// under) `MAX_LINE_BYTES`. Used to build a batch whose line has no + /// headroom left for the extra bytes `mark_replayed` adds. + fn max_parkable_content_len( + runtime: &mut ReliabilityRuntime, + channel_id: Uuid, + scope: SessionScope, + now: DateTime, + ) -> usize { + let (mut low, mut high) = (0usize, crate::reliability::park::MAX_LINE_BYTES); + while low < high { + let mid = low + (high - low).div_ceil(2); + let content = "x".repeat(mid); + let (probe, _) = make_flush_batch(channel_id, scope.clone(), &content); + let fits = runtime + .park_batch(&probe, ParkReason::RetriesExhausted, false, now) + .is_ok(); + if fits { + let _ = runtime.discard(probe.batch_id, "test-calibration", now); + low = mid; + } else { + high = mid - 1; + } + } + low + } + + #[test] + fn test_commit_replay_rolls_back_earlier_marks_when_a_later_one_fails() { + // T16 delta 1, finding 4a: `commit_replay` marks every batch in the + // plan as replayed one at a time. If an EARLIER mark durably lands + // and a LATER one in the same call fails, the earlier one must not + // stay stamped `replayed_at` — that would make it permanently + // ineligible for replay even though this whole replay attempt is + // being reported as failed and nothing was sent. + let dir = tempfile::tempdir().unwrap(); + let pubkey = "0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef"; + let now = Utc::now(); + let mut runtime = ReliabilityRuntime::open_in(dir.path(), pubkey, now).unwrap(); + + let channel_id = Uuid::new_v4(); + let scope = SessionScope::Conversation { channel_id }; + + // batch1: tiny, parks and marks-replayed with room to spare. + let (batch1, _) = make_flush_batch(channel_id, scope.clone(), "small"); + let batch1_id = batch1.batch_id; + runtime + .park_batch(&batch1, ParkReason::RetriesExhausted, false, now) + .unwrap(); + + // batch2: calibrated to the exact line-length ceiling, so it parks + // successfully now but `mark_replayed`'s extra `replayed_at` field + // pushes its line over MAX_LINE_BYTES. + let max_len = max_parkable_content_len(&mut runtime, channel_id, scope.clone(), now); + let (batch2, _) = make_flush_batch(channel_id, scope.clone(), &"x".repeat(max_len)); + let batch2_id = batch2.batch_id; + runtime + .park_batch(&batch2, ParkReason::RetriesExhausted, false, now) + .expect("batch2 must park at the calibrated max length"); + + let plan = ReplayPlan { + batch_ids: vec![batch1_id, batch2_id], + events: vec![], + scope: scope.clone(), + channel_id, + }; + + let result = runtime.commit_replay(&plan, Uuid::new_v4(), now); + assert!( + result.is_err(), + "marking the oversized batch2 as replayed must fail: {result:?}" + ); + + let batches = runtime.park().batches(); + let find = |id: Uuid| batches.iter().find(|b| b.batch_id == id).unwrap(); + assert!( + find(batch1_id).replayed_at.is_none(), + "batch1's successful mark must be rolled back when batch2's mark fails" + ); + assert!( + find(batch2_id).replayed_at.is_none(), + "batch2 must never have been marked replayed" ); } @@ -676,9 +867,10 @@ mod tests { ); // The turn for this scope finishes successfully (clearing in-flight replay) - let released = runtime.finish_replay(&scope).unwrap(); + let report = runtime.finish_replay(&scope); + assert!(report.error.is_none(), "no removal should fail here"); assert_eq!( - released, + report.released, vec![batch1_id], "only the included batch should be finished/released" ); diff --git a/crates/buzz-acp/src/reliability/state.rs b/crates/buzz-acp/src/reliability/state.rs index 4dd514511bf..83d954d6812 100644 --- a/crates/buzz-acp/src/reliability/state.rs +++ b/crates/buzz-acp/src/reliability/state.rs @@ -46,6 +46,13 @@ pub const PAUSE_RENOTIFY_MINUTES: i64 = 15; /// pruning. pub const MAX_CONSECUTIVE_SCOPES: usize = 1_000; +/// Maximum number of scopes with an open breaker tracked at once, mirroring +/// [`MAX_CONSECUTIVE_SCOPES`]. Without this an unbounded number of +/// distinct scopes (one-off channels/threads, each opening a breaker and then +/// going silent) grow the map forever — nothing else prunes it once a scope +/// stops sending failures (T16 delta 1, finding 7). +pub const MAX_OPEN_BREAKERS: usize = 1_000; + /// Whether the agent may run a turn right now. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum PauseGate { @@ -86,6 +93,12 @@ struct Pause { /// Set once the pause expires and a probe has been handed out, so only one /// batch probes per expiry. probe_issued: bool, + /// The scope of the batch actually dispatched as the outstanding pause + /// probe, if any got that far. Only that scope's turn completion may + /// release this probe's lease — an unrelated scope finishing a turn + /// (e.g. an already in-flight breaker probe) must not free it early and + /// let a second probe batch dispatch while the first is still running. + probe_scope: Option, } #[derive(Debug, Clone)] @@ -166,6 +179,20 @@ impl ReliabilityState { } let consecutive = *count; self.consecutive.remove(scope); + if self.breakers.len() >= MAX_OPEN_BREAKERS && !self.breakers.contains_key(scope) { + // Evict the longest-open breaker to admit this one rather than + // growing without bound. This is a bounded-memory safety valve, + // not a substitute for `sweep_breakers` actually expiring stale + // entries — normal operation should never reach this cap. + if let Some(oldest_scope) = self + .breakers + .iter() + .min_by_key(|(_, b)| b.opened_at) + .map(|(s, _)| s.clone()) + { + self.breakers.remove(&oldest_scope); + } + } self.breakers.insert( scope.clone(), Breaker { @@ -246,11 +273,113 @@ impl ReliabilityState { if let Some(pause) = self.pause.as_mut() { if pause.probe_issued { pause.probe_issued = false; + pause.probe_scope = None; + self.generation = self.generation.wrapping_add(1); + } + } + } + + /// Record which scope's batch was actually dispatched as the outstanding + /// pause probe. Called once dispatch has genuinely claimed a worker for + /// it — not merely selected it — so [`release_pause_probe_for`] can later + /// tell "this turn was the probe" from "this is an unrelated scope's turn + /// completing while the probe is still in flight". + pub fn set_pause_probe_scope(&mut self, scope: SessionScope) { + if let Some(pause) = self.pause.as_mut() { + pause.probe_scope = Some(scope); + } + } + + /// Release the pause probe lease, but only if `scope` is the exact scope + /// that was dispatched as the probe. + /// + /// Safety net for every terminal outcome of a dispatched batch (success, + /// retry, park, panic) — not just the paths that already know they held + /// a probe. A stray call for an unrelated scope (e.g. a breaker probe for + /// a different scope completing while a pause probe is still in flight) + /// is a no-op, so calling this unconditionally at every completion point + /// is safe (T16 delta 1, finding 9 / prior #5). + pub fn release_pause_probe_for(&mut self, scope: &SessionScope) { + if let Some(pause) = self.pause.as_mut() { + if pause.probe_issued && pause.probe_scope.as_ref() == Some(scope) { + pause.probe_issued = false; + pause.probe_scope = None; + self.generation = self.generation.wrapping_add(1); + } + } + } + + /// A pause probe was selected but dispatch never actually ran it (no + /// worker claimed it, or the owner was busy) — reschedule the deadline + /// forward rather than reverting to "eligible right now". + /// + /// Reverting to eligible-now would make the very next `pause_gate` call + /// hand out another probe immediately, and if nothing is ever available + /// to dispatch (the queue is genuinely empty because the batch that + /// triggered the pause is sitting in the park file, not the live queue) + /// that repeats forever with the deadline pinned in the past — a busy + /// spin that burns CPU indefinitely. Advancing the deadline the same way + /// an open breaker already does between probes closes it (T16 delta 1, + /// finding 2). + /// A no-op if the probe was already released by another path (a held + /// batch, a busy session owner, pool exhaustion) — those call + /// [`release_pause_probe`](Self::release_pause_probe) inline and must + /// stay immediately eligible again, not pushed 10 minutes out. This only + /// actually reschedules when `probe_issued` is *still* true, meaning + /// dispatch never found anything to even attempt. + pub fn reschedule_pause_probe(&mut self, now: DateTime) { + if let Some(pause) = self.pause.as_mut() { + if pause.probe_issued { + pause.until = now + Duration::minutes(BREAKER_PROBE_MINUTES); + pause.probe_issued = false; + pause.probe_scope = None; self.generation = self.generation.wrapping_add(1); } } } + /// Expire breakers that have been open for the full + /// [`BREAKER_MAX_OPEN_HOURS`] window, and reschedule any breaker whose + /// probe deadline is due but was never actually issued this cycle + /// (because the live queue had nothing queued for that scope, so + /// `breaker_gate` never ran for it). + /// + /// Call this once per probe-timer wake, AFTER giving `dispatch_pending` + /// its chance to run — a breaker whose probe genuinely got issued this + /// cycle already has `probe_issued == true` by then and is left alone. + /// Without this sweep, a scope that opens a breaker and then sends + /// nothing else ever again keeps its breaker (and the per-scope + /// `consecutive` residue) forever, and its stale due-in-the-past deadline + /// re-triggers the timer on every loop iteration (T16 delta 1, finding 7, + /// and the breaker half of finding 2). + /// + /// Returns the scopes whose breaker expired (closed via timeout, not a + /// successful probe) so the caller can write the ledger record. + pub fn sweep_breakers(&mut self, now: DateTime) -> Vec { + let mut expired = Vec::new(); + let mut changed = false; + self.breakers.retain(|scope, breaker| { + if now - breaker.opened_at >= Duration::hours(BREAKER_MAX_OPEN_HOURS) { + expired.push(scope.clone()); + changed = true; + false + } else if now >= breaker.next_probe && !breaker.probe_issued { + breaker.next_probe = now + Duration::minutes(BREAKER_PROBE_MINUTES); + changed = true; + true + } else { + true + } + }); + for scope in &expired { + self.consecutive.remove(scope); + } + if changed { + self.generation = self.generation.wrapping_add(1); + } + expired + } + /// Release an unconsumed breaker probe permit so a subsequent dispatch may probe. pub fn release_breaker_probe(&mut self, scope: &SessionScope) { if let Some(breaker) = self.breakers.get_mut(scope) { @@ -410,6 +539,7 @@ impl ReliabilityState { let moved = (until - pause.notified_until).num_minutes().abs(); pause.until = until; pause.probe_issued = false; + pause.probe_scope = None; if moved > PAUSE_RENOTIFY_MINUTES { pause.notified_channels.clear(); pause.notified_until = until; @@ -421,6 +551,7 @@ impl ReliabilityState { notified_until: until, notified_channels: HashSet::new(), probe_issued: false, + probe_scope: None, }); } } @@ -448,3 +579,172 @@ pub fn clamp_pause(resets_at: Option>, now: DateTime) -> Date } resets_at } + +#[cfg(test)] +mod tests { + use super::*; + use crate::reliability::ErrorClass; + use uuid::Uuid; + + fn scope() -> SessionScope { + SessionScope::Conversation { + channel_id: Uuid::new_v4(), + } + } + + fn open_breaker(state: &mut ReliabilityState, scope: &SessionScope, now: DateTime) { + for _ in 0..BREAKER_THRESHOLD { + state.on_failure(scope, ErrorClass::ProviderInternal, now); + } + } + + // T16 delta 1, finding 7: a breaker whose scope never sends anything + // again must still eventually close — nothing but `sweep_breakers` + // touches it once the scope goes silent. + #[test] + fn sweep_breakers_expires_a_breaker_whose_scope_went_silent() { + let mut state = ReliabilityState::default(); + let s = scope(); + let now = Utc::now(); + open_breaker(&mut state, &s, now); + assert!( + state.breaker_opened_at(&s).is_some(), + "breaker must be open" + ); + + // Well short of the 6h expiry: nothing changes. + let before_expiry = now + Duration::hours(BREAKER_MAX_OPEN_HOURS - 1); + let expired = state.sweep_breakers(before_expiry); + assert!(expired.is_empty()); + assert!(state.breaker_opened_at(&s).is_some()); + + // Past the 6h expiry with no new traffic on this scope at all: the + // sweep — not a failure on the scope — must close it. + let after_expiry = now + Duration::hours(BREAKER_MAX_OPEN_HOURS) + Duration::minutes(1); + let expired = state.sweep_breakers(after_expiry); + assert_eq!(expired, vec![s.clone()]); + assert!( + state.breaker_opened_at(&s).is_none(), + "the breaker must actually be gone after the sweep expires it" + ); + } + + // The busy-spin half of finding 2: a breaker whose probe deadline is due + // but that no live dispatch ever touched this cycle (its scope had + // nothing queued) must not keep reporting the same past deadline. + #[test] + fn sweep_breakers_reschedules_a_due_but_unconsumed_probe() { + let mut state = ReliabilityState::default(); + let s = scope(); + let now = Utc::now(); + open_breaker(&mut state, &s, now); + + let due = now + Duration::minutes(BREAKER_PROBE_MINUTES) + Duration::seconds(1); + assert_eq!( + state.earliest_probe_deadline(), + Some(now + Duration::minutes(BREAKER_PROBE_MINUTES)) + ); + + let expired = state.sweep_breakers(due); + assert!(expired.is_empty(), "6h expiry has not been reached"); + let next = state + .earliest_probe_deadline() + .expect("a rescheduled breaker still has a future deadline"); + assert!( + next > due, + "the deadline must move into the future, not stay stuck at `due`" + ); + } + + #[test] + fn breakers_map_is_bounded_across_many_distinct_scopes() { + let mut state = ReliabilityState::default(); + let now = Utc::now(); + for _ in 0..(MAX_OPEN_BREAKERS + 50) { + open_breaker(&mut state, &scope(), now); + } + // Each `scope()` call is a brand-new SessionScope, so without a + // bound this would grow to MAX_OPEN_BREAKERS + 50 entries. `breakers` + // is a private field, visible here as a descendant module of + // `reliability::state` — there is no public len() accessor and + // adding one only for this test isn't worth the API surface. + assert!( + state.breakers.len() <= MAX_OPEN_BREAKERS, + "breakers map must not grow past MAX_OPEN_BREAKERS: got {}", + state.breakers.len() + ); + } + + // Finding 9 / prior #5: a pause probe that was actually dispatched (its + // scope recorded) must only release for that exact scope — an unrelated + // scope's turn completing (e.g. a breaker probe running concurrently) + // must not free the pause lease early and let a second probe dispatch + // while the first is still in flight. + #[test] + fn release_pause_probe_for_only_releases_the_dispatched_scope() { + let mut state = ReliabilityState::default(); + let probe_scope = scope(); + let other_scope = scope(); + let now = Utc::now(); + state.set_pause(now); // already due at `now` + assert_eq!(state.pause_gate(now), PauseGate::Probe); + state.set_pause_probe_scope(probe_scope.clone()); + + state.release_pause_probe_for(&other_scope); + assert_eq!( + state.pause_gate(now), + PauseGate::Held { + until: state.paused_until().unwrap() + }, + "an unrelated scope must not release the probe lease" + ); + + state.release_pause_probe_for(&probe_scope); + assert_eq!( + state.pause_gate(now), + PauseGate::Probe, + "the exact dispatched scope must release the lease" + ); + } + + // Finding 2: a pause probe selected but never actually dispatched must + // reschedule forward, not revert to "eligible right now" — reverting + // would make the very next call hand out another probe immediately, + // spinning forever when nothing is ever available to dispatch. + #[test] + fn reschedule_pause_probe_moves_the_deadline_forward() { + let mut state = ReliabilityState::default(); + let now = Utc::now(); + state.set_pause(now); // already due at `now` + assert_eq!(state.pause_gate(now), PauseGate::Probe); + + state.reschedule_pause_probe(now); + match state.pause_gate(now) { + PauseGate::Held { until } => assert!(until > now), + other => panic!("expected Held after reschedule, got {other:?}"), + } + } + + // A batch WAS found and held (busy owner / pool exhausted) — those + // paths already call the bare, non-rescheduling `release_pause_probe` + // inline. A caller that then also calls `reschedule_pause_probe` as a + // blanket "nothing dispatched" cleanup must not clobber that decision + // and push the deadline out — the probe must stay immediately eligible. + #[test] + fn reschedule_pause_probe_is_a_no_op_after_an_inline_release() { + let mut state = ReliabilityState::default(); + let now = Utc::now(); + state.set_pause(now); + assert_eq!(state.pause_gate(now), PauseGate::Probe); + + state.release_pause_probe(); // simulates the held/pool-exhausted path + state.reschedule_pause_probe(now); // the blanket post-loop cleanup + + assert_eq!( + state.pause_gate(now), + PauseGate::Probe, + "an already-released probe must remain immediately eligible, not \ + be pushed 10 minutes out by a later reschedule call" + ); + } +} diff --git a/crates/buzz-acp/src/reliability/state_dir.rs b/crates/buzz-acp/src/reliability/state_dir.rs index 17f555c4366..94bb687db0e 100644 --- a/crates/buzz-acp/src/reliability/state_dir.rs +++ b/crates/buzz-acp/src/reliability/state_dir.rs @@ -149,10 +149,31 @@ pub fn write_atomic(path: &Path, contents: &[u8]) -> io::Result<()> { file.sync_all()?; } fs::rename(&temp, path)?; - // Durability of the rename itself: without this a crash can leave the - // directory entry pointing at neither file. - let dir = fs::File::open(parent)?; - dir.sync_all()?; + // The rename above is what commits the write: `path` now holds + // `contents` regardless of anything below. Syncing the parent directory + // entry only hardens against an OS crash landing in the narrow window + // before that entry itself reaches disk — a best-effort durability + // improvement, not the thing that decides whether the write happened. + // + // So a failure here must never turn into `Err`: an earlier version + // propagated it, which meant a caller (e.g. `ParkFile::commit`) that + // sees `Err` assumes NOTHING was written and keeps its own copy for a + // future retry — while the target file, on disk, right now, already + // holds the new content. That caller then falls through to a legacy + // path that requeues/re-parks the same batch, producing two live copies + // of one message (T16 delta 1, finding 5). Log and move on instead. + match fs::File::open(parent).and_then(|dir| dir.sync_all()) { + Ok(()) => {} + Err(error) => { + tracing::warn!( + path = %path.display(), + error = %error, + "could not fsync the state directory entry after an atomic rename — \ + the write itself already landed; durability is degraded only against \ + an OS crash in the next instant, not lost" + ); + } + } Ok(()) } @@ -172,7 +193,7 @@ mod tests { #[test] #[cfg(unix)] - fn test_write_atomic_propagates_parent_dir_fsync_error() { + fn test_write_atomic_survives_parent_dir_fsync_error() { use std::os::unix::fs::PermissionsExt; let dir = tempfile::tempdir().unwrap(); @@ -180,18 +201,32 @@ mod tests { fs::create_dir(&sub).unwrap(); // 0o300: write + execute, but NO read permission. - // Creating and renaming temp files succeeds, but fs::File::open(parent) fails with PermissionDenied. + // Creating and renaming temp files succeeds (needs only write+exec on + // the directory), but fs::File::open(parent) — used only for the + // trailing directory-entry fsync — fails with PermissionDenied. fs::set_permissions(&sub, fs::Permissions::from_mode(0o300)).unwrap(); let target = sub.join("target.txt"); let result = write_atomic(&target, b"test payload"); - // Restore permissions for clean tempdir teardown + // Restore permissions for clean tempdir teardown and to read the file back. let _ = fs::set_permissions(&sub, fs::Permissions::from_mode(0o700)); + // T16 delta 1, finding 5: the rename already committed the write + // before the directory-fsync step ever runs, so a failure there + // must never be reported as "nothing was written" — a caller that + // saw `Err` here would keep its own copy and retry, producing two + // live copies of the same durably-written batch. assert!( - result.is_err(), - "write_atomic must return Err when parent directory open/fsync fails" + result.is_ok(), + "write_atomic must not fail the whole write just because the \ + trailing directory-entry fsync could not run: {result:?}" + ); + assert_eq!( + fs::read(&target).unwrap(), + b"test payload", + "the content must be exactly what was requested — the rename \ + already committed it before the fsync step" ); } } diff --git a/desktop/src-tauri/src/managed_agents/runtime.rs b/desktop/src-tauri/src/managed_agents/runtime.rs index f817a4afd39..4b236b07ba5 100644 --- a/desktop/src-tauri/src/managed_agents/runtime.rs +++ b/desktop/src-tauri/src/managed_agents/runtime.rs @@ -131,6 +131,19 @@ pub(crate) fn apply_system_prompt_env( pub(crate) const STATE_DIR_ENV_VAR: &str = "BUZZ_ACP_STATE_DIR"; +/// Proof token for the state-dir env application, consumed by +/// `spawn_with_effort_proof` the same way `EffortApplied`/`McpEnvApplied` +/// are. `#[must_use]`; the only way to obtain one is +/// [`apply_state_dir_env`], and the real spawn site cannot compile without +/// passing it through — deleting the `apply_state_dir_env` call, or moving it +/// before the `descriptor.env` loop it must override, leaves `state_dir_applied` +/// undefined at the spawn call, a compile error CI catches before any test +/// runs (T16 delta 1, finding 16 / prior #20 — the added tests before this +/// exercised `apply_state_dir_env` in isolation, never binding it to the real +/// command-builder seam). +#[must_use] +pub(crate) struct StateDirApplied(()); + /// Apply the harness reliability state-dir env to an agent spawn command. /// /// Called AFTER the `descriptor.env` loop at the real spawn site, so a saved @@ -140,7 +153,7 @@ pub(crate) const STATE_DIR_ENV_VAR: &str = "BUZZ_ACP_STATE_DIR"; pub(crate) fn apply_state_dir_env( command: &mut std::process::Command, state_dir: Option<&std::path::Path>, -) { +) -> StateDirApplied { match state_dir { Some(dir) => { command.env(STATE_DIR_ENV_VAR, dir); @@ -149,6 +162,7 @@ pub(crate) fn apply_state_dir_env( command.env_remove(STATE_DIR_ENV_VAR); } } + StateDirApplied(()) } /// Classify an agent's persona against the live catalog for the Agents-menu @@ -557,6 +571,7 @@ pub(crate) fn spawn_with_effort_proof( _effort: EffortApplied, _prompt: SystemPromptApplied, _mcp: McpEnvApplied, + _state_dir: StateDirApplied, ) -> std::io::Result { cmd.spawn() } @@ -1036,18 +1051,16 @@ pub fn spawn_agent_child( // layered env never carries one anyway; this is the belt to that braces. // A directory we cannot create is not fatal — the agent still answers, and // the harness logs that parked batches will not survive a restart. - match super::managed_agent_state_dir(app, &record.pubkey) { - Ok(state_dir) => { - apply_state_dir_env(&mut command, Some(&state_dir)); - } + let state_dir_applied = match super::managed_agent_state_dir(app, &record.pubkey) { + Ok(state_dir) => apply_state_dir_env(&mut command, Some(&state_dir)), Err(error) => { - apply_state_dir_env(&mut command, None); eprintln!( "buzz-desktop: no reliability state dir for agent {}: {error}", record.name, ); + apply_state_dir_env(&mut command, None) } - } + }; // Stamp desktop ownership and an unpredictable harness-generation identity. let start_nonce = uuid::Uuid::new_v4().simple().to_string(); @@ -1087,14 +1100,20 @@ pub fn spawn_agent_child( command.creation_flags(CREATE_NO_WINDOW); } - let child = spawn_with_effort_proof(&mut command, effort, prompt_applied, mcp_applied) - .map_err(|error| { - format!( - "failed to spawn `{}` for agent {}: {error}", - resolved_acp_command.display(), - record.name - ) - })?; + let child = spawn_with_effort_proof( + &mut command, + effort, + prompt_applied, + mcp_applied, + state_dir_applied, + ) + .map_err(|error| { + format!( + "failed to spawn `{}` for agent {}: {error}", + resolved_acp_command.display(), + record.name + ) + })?; // Codex: stamp adapter availability for the Phase-2 badge drift check. // Cold cache returns `None` → drift check skipped until discovery warms it. diff --git a/desktop/src-tauri/src/managed_agents/runtime/tests.rs b/desktop/src-tauri/src/managed_agents/runtime/tests.rs index 23c11265498..d85f41f535a 100644 --- a/desktop/src-tauri/src/managed_agents/runtime/tests.rs +++ b/desktop/src-tauri/src/managed_agents/runtime/tests.rs @@ -1292,7 +1292,7 @@ fn test_command_execution_overrides_and_ignores_ambient_state_dir() { // 1. Config specifies a state dir -> command carries the configured path, // never the ambient value. let mut cmd_override = std::process::Command::new("buzz-acp"); - super::apply_state_dir_env( + let _ = super::apply_state_dir_env( &mut cmd_override, Some(std::path::Path::new("/config/specified/state/dir")), ); @@ -1309,7 +1309,7 @@ fn test_command_execution_overrides_and_ignores_ambient_state_dir() { // 2. Config specifies None -> command carries an explicit removal // (Some(key) -> None), never silent fallthrough to ambient inheritance. let mut cmd_none = std::process::Command::new("buzz-acp"); - super::apply_state_dir_env(&mut cmd_none, None); + let _ = super::apply_state_dir_env(&mut cmd_none, None); let removed = cmd_none .get_envs() .find(|(key, _)| *key == std::ffi::OsStr::new(super::STATE_DIR_ENV_VAR)); @@ -1319,3 +1319,36 @@ fn test_command_execution_overrides_and_ignores_ambient_state_dir() { "command must explicitly remove the ambient state dir, not inherit it, got: {removed:?}" ); } + +// T16 delta 1, finding 16 (prior #20): the only tests that existed before +// this called `apply_state_dir_env` directly on a fresh `Command`, never +// through the real ordering (`descriptor.env` loop, THEN the state-dir +// override). `spawn_with_effort_proof` now requires a `StateDirApplied` +// token to compile at all, so deleting the production call — or moving it +// before `descriptor.env` the way this test simulates the opposite of — is a +// compile error, not a silently-passing test. This test additionally proves +// the ordering itself: a per-agent env override (`descriptor.env`) setting +// the reserved key must still lose to the state-dir authority applied after +// it, the exact production sequence in `spawn_agent_child`. +#[test] +fn test_state_dir_env_wins_over_a_descriptor_env_override() { + let mut command = std::process::Command::new("buzz-acp"); + + // Simulates the `descriptor.env` loop: a per-agent env override that + // happens to (mis)configure the reserved key. + command.env(super::STATE_DIR_ENV_VAR, "/attacker-or-misconfigured/path"); + + // Applied AFTER, exactly like the real spawn site. + let _state_dir_applied = + super::apply_state_dir_env(&mut command, Some(std::path::Path::new("/real/state/dir"))); + + let value = command + .get_envs() + .find(|(key, _)| *key == std::ffi::OsStr::new(super::STATE_DIR_ENV_VAR)) + .and_then(|(_, value)| value); + assert_eq!( + value, + Some(std::ffi::OsStr::new("/real/state/dir")), + "the state-dir authority applied after descriptor.env must win, got: {value:?}" + ); +} From d9162cfcc7175b754ea161e392a93b1a3c44b0e9 Mon Sep 17 00:00:00 2001 From: wiggdevin <202901685+wiggdevin@users.noreply.github.com> Date: Sun, 6 Sep 2026 19:35:04 -0700 Subject: [PATCH 7/7] fix(acp): keep the state-dir test imports Windows-clean (T16) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #35's Windows clippy job failed with 'unused import: super::*' at state_dir.rs:192, because the sole test in that module is cfg(unix) but the use super::* sat under a bare cfg(test), so on non-unix targets the module still compiles with the glob import but no unix-gated test left to use it, tripping -D warnings. Moved the unix gate up to the module attribute (cfg(all(test, unix))) so the whole test module — import included — compiles only on unix, where it behaves exactly as before; on Windows the module is skipped entirely instead of leaving an unused import. Co-Authored-By: Claude Fable 5.1 Signed-off-by: wiggdevin <202901685+wiggdevin@users.noreply.github.com> --- crates/buzz-acp/src/reliability/state_dir.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/crates/buzz-acp/src/reliability/state_dir.rs b/crates/buzz-acp/src/reliability/state_dir.rs index 94bb687db0e..4bb8db4a331 100644 --- a/crates/buzz-acp/src/reliability/state_dir.rs +++ b/crates/buzz-acp/src/reliability/state_dir.rs @@ -187,12 +187,11 @@ fn home_dir() -> Option { std::env::var_os("USERPROFILE").map(PathBuf::from) } -#[cfg(test)] +#[cfg(all(test, unix))] mod tests { use super::*; #[test] - #[cfg(unix)] fn test_write_atomic_survives_parent_dir_fsync_error() { use std::os::unix::fs::PermissionsExt;