From 1b1ff92e466ee2f32c3ba281dd11d46e355eb724 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 12:50:08 +0300 Subject: [PATCH 01/68] docs(roadmap): mark 2.5.F Done (PR #65 merged) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PR #65 (ADR-0062 context commands) merged to main on 2026-07-05. Reconcile the roadmap + agent guide to reflect the completed state: - CLAUDE.md: replace the "next pickup is 2.5.F / G" status tail with 2.5.F ✅ Done — /clear (host-level fresh-session swap, TTY-interactive only), the session:compacting "Summarizing…" moment (amends ADR-0036), and the footer context-fullness indicator (contextWindowForModel); next is 2.5.G. - docs/roadmap/current.md: same reconciliation on the experience-arm pointer. - docs/roadmap/phases/phase-2.5-cli-consolidation.md: Status blockquote, the 2.5.F section header, and the closing callout now read Done/merged (incl. the Opus + Sonnet review round the merged PR carried). Spine 2.5.A/B/C/E + experience arm 2.5.D/F complete; next 2.5.G, additive lanes 2.5.H / I / J in parallel. Docs-only; no code or ADR change. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 10 +++++++++- docs/roadmap/current.md | 7 ++++++- .../roadmap/phases/phase-2.5-cli-consolidation.md | 15 ++++++++++----- 3 files changed, 25 insertions(+), 7 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 713202e2..eee6d419 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -101,7 +101,15 @@ behind [ADR-0061](docs/decisions/0061-cli-input-layer-file-injection-and-shell-e two-round maintainer security review): the accepted file / command output is queued as a compact chip and expanded into the shared UNTRUSTED nonce-fenced frame only at submit (byte-identical model context, a clean prompt); the `[chat]` command allowlist resolves as a **coupled unit** (a project setting either the exact or glob array owns -the whole allowlist). **The next pickup is the rest of the 2.5 experience arm (2.5.F / G).** +the whole allowlist); and **2.5.F** (the ADR-0062 context-history commands) is ✅ **Done (PR #65, merged 2026-07-05)**, +behind [ADR-0062](docs/decisions/0062-context-compaction-and-cli-history-commands.md): **`/clear`** — a host-level +fresh-session lifecycle swap across `relavium chat`, `chat-resume`, and the in-Home chat, rebinding the same agent +under a new `sessionId` (TTY-interactive only, rejected under `--json`/plain per ADR-0049) — plus the two +compaction-moment UX polishes (a `session:compacting` "Summarizing…" event amending ADR-0036, and the footer +context-fullness indicator via a pure `@relavium/llm` `contextWindowForModel` helper), completing the ADR-0062 +compaction story alongside the earlier-landed model-summarised `/compact` + deterministic `/trim` + automatic +compaction. **The next pickup is 2.5.G** (onboarding wizard + the Home `/models` catalog); the additive lanes +2.5.H / I / J run in parallel. For live status, per-PR history, milestone dates, and open obligations, see the canonical home [docs/roadmap/current.md](docs/roadmap/current.md); [README.md](README.md) is the public overview. diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 4982219e..e2e95b30 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -182,7 +182,12 @@ flight left no host-visible busy signal, so a message typed mid-command could cr preview, expanded into the SAME UNTRUSTED nonce-fenced frame only at submit — byte-identical model context, a clean prompt), and two follow-up review passes hardened the `[chat]` allowlist resolution (exact + glob arrays are now a **coupled unit**) and fixed a Backspace regression (ink reports the Unix physical Backspace as `key.delete`); -all recorded in the ADR-0061 "Refined at implementation" append. **Next in the experience arm: 2.5.F / G.** See the +all recorded in the ADR-0061 "Refined at implementation" append. **2.5.F** (the ADR-0062 context commands) is ✅ +**Done (PR #65, merged 2026-07-05)** behind [ADR-0062](../decisions/0062-context-compaction-and-cli-history-commands.md): +`/clear` (a host-level fresh-session swap across `chat`/`chat-resume`/Home, TTY-interactive only), the +`session:compacting` "Summarizing…" moment event (amending ADR-0036), and the footer context-fullness indicator — +completing compaction alongside the earlier `/compact` + `/trim` + automatic compaction. **Next in the experience +arm: 2.5.G** (onboarding wizard + Home `/models`); the additive lanes 2.5.H / I / J run in parallel. See the [Phase 2.5 workstreams](phases/phase-2.5-cli-consolidation.md). Carry-over hardening is tracked in [deferred-tasks.md](deferred-tasks.md) — Phase 2 picks diff --git a/docs/roadmap/phases/phase-2.5-cli-consolidation.md b/docs/roadmap/phases/phase-2.5-cli-consolidation.md index da42af77..25ef5fb8 100644 --- a/docs/roadmap/phases/phase-2.5-cli-consolidation.md +++ b/docs/roadmap/phases/phase-2.5-cli-consolidation.md @@ -7,7 +7,9 @@ > → **2.5.E** (modes + per-tool approval + mid-turn abort) ✅ **Done (PR #63, 2026-07-03)** (ADR-0057 Accepted) > — **the spine is complete**. Experience arm: **2.5.D** (chat input ergonomics + `@`/`!` chip model) ✅ **Done > (PR #64, 2026-07-03)** behind [ADR-0061](../../decisions/0061-cli-input-layer-file-injection-and-shell-escape.md). -> **Next: 2.5.F / G** (off the spine, depends on B/C). Additive lanes (no dependency chain): 2.5.H / I / J. +> **2.5.F** (`/clear` + the `session:compacting` "Summarizing…" moment + the context-fullness footer, completing +> the ADR-0062 compaction story) ✅ **Done (PR #65, merged 2026-07-05)**. **Next: 2.5.G** (onboarding wizard + +> Home `/models`). Additive lanes (no dependency chain): 2.5.H / I / J. - **Related**: [../README.md](../README.md), [phase-2-cli.md](phase-2-cli.md), [phase-2.6-conversational-authoring.md](phase-2.6-conversational-authoring.md), [phase-3-desktop.md](phase-3-desktop.md), [../../reference/cli/commands.md](../../reference/cli/commands.md), [../../reference/cli/chat-session.md](../../reference/cli/chat-session.md), [../../reference/cli/regression-harness.md](../../reference/cli/regression-harness.md), [../../decisions/README.md](../../decisions/README.md) (ADR-0054–0057) @@ -363,7 +365,7 @@ is sandbox-bounded with protected paths honoured. A security review of the resea (defense-in-depth trade-off) passes. **Required ADR: per-tool approval + reseat-less chat mode system (incl. mid-turn abort).** -### 2.5.F — `/clear`, `/trim`, and `/compact` (context compaction) +### 2.5.F — `/clear`, `/trim`, and `/compact` (context compaction) — ✅ **Done (PR #65, merged 2026-07-05)** > **Scope expanded (2026-07-04, [ADR-0062](../../decisions/0062-context-compaction-and-cli-history-commands.md)).** > The maintainer removed the Phase-3 deferral of `/compact`: we build the **full** context-compaction @@ -387,16 +389,19 @@ host + docs), each with an Opus + Sonnet review round. context (append-only, resume-preserving, cost-accounted); auto-compaction bounds a long chat before it overflows the context window; the summary is inspectable and the moment is a designed state. -> **Landed pending merge (PR for ADR-0062, three reviewed steps — shared/seam/db, engine primitive, CLI host — +> **Landed & merged (PR #65 for ADR-0062, three reviewed steps — shared/seam/db, engine primitive, CLI host — > each with an Opus + Sonnet review round):** the compaction engine primitive, automatic compaction, append-only > resume/reseat-preserving persistence, `/compact`, and `/trim [n]` are **complete**. The final 2.5.F items then -> landed (pending merge): **`/clear`** — the fresh-session lifecycle swap (ADR-0062 §7) across `relavium chat`, +> landed: **`/clear`** — the fresh-session lifecycle swap (ADR-0062 §7) across `relavium chat`, > `chat-resume`, and the in-Home chat: a host-level re-drive (standalone) / build-first `clearChat` (Home) that ends > the current session (persisted + resumable) and rebinds the same agent under a new `sessionId`, TTY-interactive-only > (rejected under `--json`/plain), zero engine change; and the two compaction-moment UX polishes — the **labeled** > "Summarizing…" spinner off a new additive `session:compacting` engine event (amends ADR-0036's event substrate) and > the footer **context-fullness** indicator (last input ÷ the model's context window, via the new pure -> `@relavium/llm` `contextWindowForModel` helper). **With these 2.5.F is feature-complete (roadmap-done-after-merge).** +> `@relavium/llm` `contextWindowForModel` helper). **With these 2.5.F is ✅ Done (PR #65, merged 2026-07-05); the +> merged PR also carried a parallel Opus + Sonnet review round — the compacting-latch spinner fix, the 0%-ctx +> footer guard, the Home double-clear MCP-leak guard, best-effort auto-compaction, and the shared adapter +> `CONTEXT_SEAM_DEFAULTS`.** ### 2.5.G — Onboarding wizard and `/models` (Home model catalog) From 9e0fbfb188572e9d95aedf6135a9680cf6980abe Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 18:16:58 +0300 Subject: [PATCH 02/68] =?UTF-8?q?docs(decisions):=202.5.G=20S1=20=E2=80=94?= =?UTF-8?q?=20ADR-0063/0064/0065=20design-lock=20(Option=20A=20live=20mode?= =?UTF-8?q?l=20catalog)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The design-lock for the Option-A expansion of 2.5.G (onboarding + /models): a LIVE model catalog, a complete model-pricing story, and provider extensibility. New ADRs (Accepted): - ADR-0063 — CLI config-write contract: the first on-disk config writer, /models -> global [preferences].default_model + a resolveChat global fallback, atomic 0600 temp+rename, secret-free, smol-toml.stringify (no new dep). - ADR-0064 — Live model catalog: an optional listModels? seam capability + a `kind` protocol abstraction + the model_catalog live cache (repurpose + 0007 migration) + refresh (first-run / models refresh / 24h-TTL non-blocking background, per-provider isolation) + a pure static/live merge; enum stays CLOSED; amends ADR-0011/0030/0031 additively. - ADR-0065 — Provider economics & extensibility: user-supplied pricing + the cost-path pricing-injection overlay (closes the silent cost-cap gap), resolveProvider host-rewiring (fixes the dead --base-url config), custom OpenAI-compatible endpoints over the SSRF floor, pricing-reference capture; extends ADR-0064's merge with a USER tier, amends ADR-0011. Append-only amendment notes added to ADR-0011/0030/0031 (seam grows a listModels? method + `kind`) and ADR-0048 (smol-toml use extends to the writer). README index updated. Roadmap reconciled: phase-2.5 §2.5.G -> Option A + a 12-step ledger (six security-flagged), and the live /v1/models fetch pulled forward from Phase 3 in both phase-2.5 and phase-2.6; current.md + CLAUDE.md note 2.5.G underway. Folds in the maintainer's adversarial multi-agent review: fixed the reversed `static ?? user` precedence note, "six tables" -> "five", the ~/.relavium/tmp 0700 claim, added Considered/rejected forks to 0064/0065, "pending reference-doc update" notes, stripped undefined S/K labels, TLS-cert DNS-rebinding backstop, and more. Verified: prettier --check clean; all three ADR relative links resolve. Refs: ADR-0063, ADR-0064, ADR-0065 Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 7 +- .../0011-internal-llm-abstraction.md | 2 + ...oning-response-format-provider-executed.md | 6 + ...-llm-seam-shape-amendment-multimodal-io.md | 2 + docs/decisions/0048-toml-config-parser.md | 4 +- .../0063-cli-config-write-contract.md | 126 ++++++++++ docs/decisions/0064-live-model-catalog.md | 234 ++++++++++++++++++ ...65-provider-economics-and-extensibility.md | 177 +++++++++++++ docs/decisions/README.md | 3 + docs/roadmap/current.md | 9 +- .../phases/phase-2.5-cli-consolidation.md | 60 +++-- .../phase-2.6-conversational-authoring.md | 10 +- 12 files changed, 616 insertions(+), 24 deletions(-) create mode 100644 docs/decisions/0063-cli-config-write-contract.md create mode 100644 docs/decisions/0064-live-model-catalog.md create mode 100644 docs/decisions/0065-provider-economics-and-extensibility.md diff --git a/CLAUDE.md b/CLAUDE.md index eee6d419..5ce5a702 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -108,8 +108,11 @@ under a new `sessionId` (TTY-interactive only, rejected under `--json`/plain per compaction-moment UX polishes (a `session:compacting` "Summarizing…" event amending ADR-0036, and the footer context-fullness indicator via a pure `@relavium/llm` `contextWindowForModel` helper), completing the ADR-0062 compaction story alongside the earlier-landed model-summarised `/compact` + deterministic `/trim` + automatic -compaction. **The next pickup is 2.5.G** (onboarding wizard + the Home `/models` catalog); the additive lanes -2.5.H / I / J run in parallel. +compaction. **2.5.G is now underway** — its scope expanded to **Option A** (a **live** model catalog + a complete +model-pricing story that governs cost) behind three new ADRs ([ADR-0063](docs/decisions/0063-cli-config-write-contract.md) +config-write, [ADR-0064](docs/decisions/0064-live-model-catalog.md) live catalog, +[ADR-0065](docs/decisions/0065-provider-economics-and-extensibility.md) provider economics), across 12 reviewed +steps; the additive lanes 2.5.H / I / J run in parallel. For live status, per-PR history, milestone dates, and open obligations, see the canonical home [docs/roadmap/current.md](docs/roadmap/current.md); [README.md](README.md) is the public overview. diff --git a/docs/decisions/0011-internal-llm-abstraction.md b/docs/decisions/0011-internal-llm-abstraction.md index b80b6679..5316ea34 100644 --- a/docs/decisions/0011-internal-llm-abstraction.md +++ b/docs/decisions/0011-internal-llm-abstraction.md @@ -28,6 +28,8 @@ The seam is the immovable contract; the adapter implementation behind it is deli > *reference* on the desktop, where Rust performs the egress via `llm_stream`. The seam's > **types and contract are unchanged**; only the per-host transport wiring is refined. +> **Amended 2026-07-05 by [ADR-0064](0064-live-model-catalog.md) and [ADR-0065](0065-provider-economics-and-extensibility.md)** (append-only — this body is unchanged). The seam's method **set** grows again (its **shape** stays frozen): ADR-0064 adds an **optional `listModels?`** capability (returning a Relavium-typed `ModelListing[]` — the vendor `models.list()` is mapped inside the adapter, no vendor type crosses) plus a provider **`kind`** protocol abstraction (`anthropic` | `openai-compatible` | `gemini`) that derives the adapter, list-models endpoint, auth, and response-mapper **per protocol** — formalizing the DeepSeek-via-OpenAI-compatible precedent noted in the Decision above. ADR-0065 makes the host `resolveProvider` build the adapter from the **stored provider row** (`kind` + a custom `base_url`) rather than only the static default registry, and injects a **pricing overlay** into the cost path so a user-priced model prices instead of throwing. The provider-**id** `z.enum` stays **closed** — a truly-open custom-provider registry is named as a future supersede of this ADR, not taken here. + > Amended 2026-06-05: the same `LLMProvider` seam is reused **unchanged** by the agent-first > `AgentSession` entry point ([ADR-0024](0024-agent-first-entry-point-agentsession.md)) — chat-mode > agents call providers through the identical contract, so no vendor SDK type crosses the seam for diff --git a/docs/decisions/0030-llm-seam-shape-amendment-reasoning-response-format-provider-executed.md b/docs/decisions/0030-llm-seam-shape-amendment-reasoning-response-format-provider-executed.md index 45c59ed8..dbe3390d 100644 --- a/docs/decisions/0030-llm-seam-shape-amendment-reasoning-response-format-provider-executed.md +++ b/docs/decisions/0030-llm-seam-shape-amendment-reasoning-response-format-provider-executed.md @@ -15,6 +15,12 @@ > **Gemini part-level `thoughtSignature`** (incl. on a `functionCall`) need a canonical opaque-continuation > carrier and are recorded follow-ups against this ADR's shape. +> **Amended 2026-07-05 by [ADR-0064](0064-live-model-catalog.md)** (append-only — this body is unchanged): the +> same seam-shape-amendment move adds one more **optional capability method** — `listModels?` (a Relavium-typed +> `ModelListing[]`; the vendor `models.list()` is mapped inside the adapter, no vendor type crosses) — alongside +> a provider `kind` discriminant. Additive and capability-varying, in the same spirit as the reasoning channel +> here; the seam's shape is otherwise unchanged. + ## Context The `@relavium/llm` seam — the request/result/stream/usage/content shapes in diff --git a/docs/decisions/0031-llm-seam-shape-amendment-multimodal-io.md b/docs/decisions/0031-llm-seam-shape-amendment-multimodal-io.md index 42b45b8a..7bd25dfe 100644 --- a/docs/decisions/0031-llm-seam-shape-amendment-multimodal-io.md +++ b/docs/decisions/0031-llm-seam-shape-amendment-multimodal-io.md @@ -23,6 +23,8 @@ > **Amended 2026-06-20 by [ADR-0046](0046-inline-media-out-via-generate-streaming-triad-deferred.md).** A refinement, not a reversal: it pins which of §5.1's two inline-media-out paths Phase-1 (1.AG) uses — the non-streaming `generate()` path ("the more common"), de-inlined at the `#emitDurable` choke point — and defers the streaming `media_start`/`media_delta`/`media_end` triad to the host (1.AH). The seam shape (incl. the handle-only `media_end`) is unchanged. +> **Amended 2026-07-05 by [ADR-0064](0064-live-model-catalog.md).** A refinement, not a reversal: the seam gains one more **optional capability method** — `listModels?` (live model discovery, returning a Relavium-typed `ModelListing[]` mapped inside each adapter — no vendor type crosses) — in the same additive spirit as the reserved media methods here. The multimodal seam shape is unchanged. + ## Context The `@relavium/llm` seam — the request/result/stream/usage/content shapes in diff --git a/docs/decisions/0048-toml-config-parser.md b/docs/decisions/0048-toml-config-parser.md index b6ef4faa..25c4df14 100644 --- a/docs/decisions/0048-toml-config-parser.md +++ b/docs/decisions/0048-toml-config-parser.md @@ -2,7 +2,9 @@ - **Status**: Accepted - **Date**: 2026-06-22 -- **Related**: [ADR-0020](0020-zod-runtime-schema-library.md), [ADR-0023](0023-strict-authored-yaml-validation.md), [ADR-0033](0033-strict-config-files-amends-0023.md), [ADR-0035](0035-yaml-parser-dependency.md), [config-spec.md](../reference/contracts/config-spec.md), [phase-2-cli.md](../roadmap/phases/phase-2-cli.md), [architectural-principles.md](../standards/architectural-principles.md), [tech-stack.md](../tech-stack.md) +- **Related**: [ADR-0020](0020-zod-runtime-schema-library.md), [ADR-0023](0023-strict-authored-yaml-validation.md), [ADR-0033](0033-strict-config-files-amends-0023.md), [ADR-0035](0035-yaml-parser-dependency.md), [config-spec.md](../reference/contracts/config-spec.md), [phase-2-cli.md](../roadmap/phases/phase-2-cli.md), [architectural-principles.md](../standards/architectural-principles.md), [tech-stack.md](../tech-stack.md), [ADR-0063](0063-cli-config-write-contract.md) (**extends this parser to the config *writer***) + +> **Amended 2026-07-05 by [ADR-0063](0063-cli-config-write-contract.md)** (append-only — this body is unchanged): `smol-toml`'s use extends from the config **loader** to the config **writer**. ADR-0063 reuses `smol-toml.stringify` (already present — **no new dependency**) for a minimal, atomic, secret-incapable write of the global `~/.relavium/config.toml`, confined to the same `apps/cli/src/config` boundary this ADR drew. The parser choice and its confinement are unchanged. ## Context diff --git a/docs/decisions/0063-cli-config-write-contract.md b/docs/decisions/0063-cli-config-write-contract.md new file mode 100644 index 00000000..b79778e6 --- /dev/null +++ b/docs/decisions/0063-cli-config-write-contract.md @@ -0,0 +1,126 @@ +# ADR-0063: CLI config-write contract — the first on-disk config writer, the global `[preferences].default_model` target, and the `resolveChat` global fallback + +- **Status**: Accepted +- **Date**: 2026-07-05 +- **Related**: [ADR-0048](0048-toml-config-parser.md) (`smol-toml`, confined to the CLI config loader — this ADR extends its use to the *writer*, same package boundary, no new dependency; **append-only top-note added there**) · [ADR-0033](0033-strict-config-files-amends-0023.md) (strict config — the written file must re-parse under `.strict()`) · [ADR-0023](0023-strict-authored-yaml-validation.md) · [ADR-0024](0024-agent-first-entry-point-agentsession.md) (the `[chat]` block + one-model-per-session) · [ADR-0006](0006-os-keychain-for-api-keys.md) (secrets live only in the keychain — never config) · [ADR-0049](0049-cli-machine-output-contract.md) (the write is a TTY-interactive action) · [ADR-0064](0064-live-model-catalog.md) + [ADR-0065](0065-provider-economics-and-extensibility.md) (the `/models` picker + onboarding wizard that drive this write). The config keys + resolution order are the canonical [config-spec.md](../reference/contracts/config-spec.md); the schemas live in [config.ts](../../packages/shared/src/config.ts). + +## Context + +The CLI config layer is **read-only**. [load.ts](../../apps/cli/src/config/load.ts) imports `smol-toml` +for `parse` only ([ADR-0048](0048-toml-config-parser.md) scoped it to "string → plain data"); no code +anywhere writes `config.toml` / `project.toml` / `workspace.toml`. Phase 2.5.G introduces the first need +to **persist** a user choice: the `/models` picker (Home) and the onboarding wizard +([ADR-0064](0064-live-model-catalog.md)) must set the model the next chat session binds, and the roadmap +names that target as `[chat].default_model`. + +Two facts make the naive reading wrong, and are the reason this needs a recorded decision: + +1. **`[chat]` is project/workspace-scoped only.** `resolveChat` ([resolve.ts](../../apps/cli/src/config/resolve.ts) L113) + reads `[chat].default_model` from `project.toml`/`workspace.toml` and has **no global layer**; the bare + Home ([ADR-0054](0054-cli-bare-invocation-interactive-home.md)) is frequently run **outside any project**, + so there is no `[chat]` file to write. Meanwhile `GlobalConfigSchema.preferences.default_model` + ([config.ts](../../packages/shared/src/config.ts) L138) already exists but today feeds only + `resolveConfig.defaultModel` (the **workflow** default, resolve.ts L83) — chat ignores it. + +2. **A config write is a read-modify-write of a hand-authored, git-committed, `.strict()`-validated TOML** + ([ADR-0033](0033-strict-config-files-amends-0023.md)) that may hold cost caps, MCP registrations, and the + `!`-shell allowlist. The failure modes are load-bearing: a **torn file** on a crash loses the user's whole + config; a **secret** written into config violates the rule-6 non-negotiable ([ADR-0006](0006-os-keychain-for-api-keys.md)); + an **invalid** emission fails the next load (`ConfigError`, exit 2), locking the user out of their own config. + +The stakes of getting this wrong are high precisely because it is the *first* writer — every later writer +(cost caps, provider settings) will inherit whatever primitive lands here. + +## Decision + +**We add a single, minimal, atomic, secret-incapable config-write primitive confined to +[apps/cli/src/config](../../apps/cli/src/config) (the sibling of `load.ts`), used to write the global +`~/.relavium/config.toml` only; `/models` and the wizard write `[preferences].default_model`, and +`resolveChat` gains a lowest-precedence global fallback layer to it.** No new dependency: `smol-toml` +([ADR-0048](0048-toml-config-parser.md)) already exports `stringify`. + +### 1. Write target — global `[preferences].default_model`, plus a `resolveChat` global fallback + +`/models` (and the wizard's default-model step) write **`[preferences].default_model`** in the always-present +global `~/.relavium/config.toml`, and `resolveChat` ([resolve.ts](../../apps/cli/src/config/resolve.ts) L113) +gains a **third, lowest-precedence layer**: `project.chat.default_model ?? workspace.chat.default_model ?? +global.preferences.default_model`. This mirrors the global fallback the workflow path **already** has +(`resolveConfig.defaultModel`, resolve.ts L83), unifying the meaning of `[preferences].default_model` to +"my preferred model everywhere" while a project's `[chat]`/`[defaults]` still override per-context. + +- **Considered — a new global `[chat]` block** (add `chat.default_model` to `GlobalConfigSchema`): rejected + for 2.5.G as a larger `@relavium/shared` schema change + drift-pin, for the same end effect; the + `[preferences]` fallback reuses an existing field. +- **Considered — write project `[chat]`, creating `.relavium/` in cwd when absent**: rejected — it drops a + surprising, possibly git-committed `.relavium/` artifact into an unrelated directory and makes a + project-less *global* preference impossible. +- **Behaviour change, disclosed:** an existing `[preferences].default_model` (today workflow-only) now also + becomes the chat default when no project/workspace `[chat].default_model` overrides it. Chat previously + had no global default at all (it fell to `DEFAULT_CHAT_MODEL`), so this is an additive, arguably-correcting + change — a user's stated preferred model now applies to chat too — recorded here, not silent. + +### 2. Mechanism — parse → set → re-validate → atomic `stringify` write + +The writer reads the existing `~/.relavium/config.toml` (or an empty object when absent), sets the single +target key on the parsed object, **re-validates the whole object against `GlobalConfigSchema`** +([ADR-0033](0033-strict-config-files-amends-0023.md)) so the emission is provably schema-valid, `stringify`s +it, and writes **atomically**: a temp file created **`0600`** (owner-only) in the owner-only (`0700`) +`~/.relavium/` directory → `fsync` → `rename` over the target — the same at-rest posture the `history.db` +write uses ([ADR-0050](0050-cli-history-db-at-rest-posture.md); the `0700` dir mode is set by +`ensureGlobalConfigDir`, [paths.ts](../../apps/cli/src/config/paths.ts) L28). Note that +`ensureGlobalConfigDir` sets `0700` on `~/.relavium/` itself, **not** on its `tmp/` subdir, so the writer sets +the temp file's `0600` mode **explicitly** rather than relying on an inherited directory mode — an +implementation prerequisite. An interrupted write leaves the original file intact. + +> **Comment/formatting loss is accepted for v1, and bounded.** `smol-toml.stringify` re-serializes the parsed +> object and **drops comments and key ordering**. We accept this for the **global** `config.toml` (the only +> file this writer touches): it is a small, largely tool-managed settings file, not the heavily-annotated +> *project* workflow config, and correctness (a guaranteed-valid, atomic, never-torn re-emission) outranks +> comment fidelity for the first writer. A **surgical, comment-preserving single-key editor** is named as a +> future refinement — it would be required before this primitive is ever pointed at a hand-curated +> `project.toml`. The write is confined to the **global** file precisely so that boundary is explicit. + +### 3. Secret-free by construction + +The writer only ever sets **schema-valid, non-secret** keys (here `default_model`) on a `GlobalConfigSchema` +object, then serializes that validated object. The writer only ever sets `default_model` (a non-secret) — there is **no +API-key field** in the config schema to write to, and API keys live only in the OS keychain +([ADR-0006](0006-os-keychain-for-api-keys.md)). The loader already rejects `api_key`-like keys +([load.ts](../../apps/cli/src/config/load.ts)); the writer's secret-free property is asserted directly in tests +(the config-write security review). A caller can never hand the +writer a free-form key/value — the surface is a typed setter (`setDefaultModel`), not a generic `writeKey`. + +### 4. Interactive-only, like every other Home mutation + +The `/models`-driven write is reached only through a TTY-interactive picker/wizard; it is never triggered +under `--json`/plain ([ADR-0049](0049-cli-machine-output-contract.md)), consistent with `/clear` +([ADR-0062](0062-context-compaction-and-cli-history-commands.md) §7). The primitive itself is surface-agnostic +and unit-tested with an injected home dir + fs, no TTY. + +## Consequences + +### Positive + +- **A user's model choice persists** — `/models` and the wizard set a durable default the next session binds; + `[preferences].default_model` becomes a coherent "preferred model everywhere" with per-project override intact. +- **No new dependency** — reuses `smol-toml.stringify` already present under [ADR-0048](0048-toml-config-parser.md); + this is a pure contract decision, confined to the same `apps/cli` config boundary. +- **Safe by construction** — atomic `0600` temp+rename (never a torn config), re-validated against the strict + schema before emission (never an unloadable file), and structurally secret-free (no key can reach disk). +- **A clean foundation** — the typed single-key setter is the primitive every later settings-writer reuses, + rather than each command hand-rolling a TOML write. + +### Negative + +- **Comment/ordering loss on the global `config.toml`** — accepted and documented (§2); mitigated by atomicity + and confinement to the global file; a comment-preserving surgical editor is named future work and is a + prerequisite before writing any hand-curated `project.toml`. +- **A disclosed behaviour change** — an existing `[preferences].default_model` now also drives the chat default + (§1); additive and arguably-correcting, but a change to document in `config-spec.md`. +- **First-writer surface area** — a new on-disk mutation path that must be security-reviewed (a dedicated + security round: secret-free + atomicity + path handling) before it ships; its blast radius is deliberately + minimized to one global file and one typed key. +- **Roadmap reconciliation** — the phase-2.5 §2.5.G text named `[chat].default_model` as the write target; this + ADR refines that to global `[preferences].default_model` + a chat fallback layer. The phase-2.5 §2.5.G text is + **already reconciled in this doc round**; the remaining `config-spec.md` update (the config-write contract + + the `[preferences].default_model` chat fallback) lands in the implementing PR. diff --git a/docs/decisions/0064-live-model-catalog.md b/docs/decisions/0064-live-model-catalog.md new file mode 100644 index 00000000..874cf545 --- /dev/null +++ b/docs/decisions/0064-live-model-catalog.md @@ -0,0 +1,234 @@ +# ADR-0064: Live model catalog — the `listModels?` seam capability, the `kind` protocol abstraction, the `model_catalog` live cache, the refresh lifecycle, and the static/live merge + +- **Status**: Accepted +- **Date**: 2026-07-05 +- **Related**: [ADR-0011](0011-internal-llm-abstraction.md) + [ADR-0030](0030-llm-seam-shape-amendment-reasoning-response-format-provider-executed.md) + [ADR-0031](0031-llm-seam-shape-amendment-multimodal-io.md) (**this ADR amends the `LLMProvider` seam shape — additively**; append-only top-notes added there) · [ADR-0038](0038-agentrunner-llm-call-boundary.md) (host-injected provider resolution — the refresh service reuses it) · [ADR-0050](0050-cli-history-db-at-rest-posture.md) (the cache shares `history.db`; a model list is non-secret) · [ADR-0044](0044-media-access-governance-read-media-save-to-cost.md) + [ADR-0045](0045-async-media-job-loop-poll-checkpoint-resume-cancel.md) (the `model_catalog` **media-routing** consumer this must not regress) · [ADR-0056](0056-cli-in-app-slash-command-system-and-manifest.md) (the `/models` REPL command + `models refresh` shell command) · [ADR-0049](0049-cli-machine-output-contract.md) (`--json`) · [ADR-0059](0059-cli-mid-session-model-reseat.md) (the **other** `/models` — mid-chat reseat, Phase 2.6 — disambiguated below) · [ADR-0063](0063-cli-config-write-contract.md) (the `/models` selection persists the next session's default via its config-write primitive) · [ADR-0065](0065-provider-economics-and-extensibility.md) (**extends** this ADR's merge with a user-pricing tier + custom providers). Canonical homes: the seam signature → [llm-provider-seam.md](../reference/shared-core/llm-provider-seam.md); the `model_catalog` DDL → [database-schema.md](../reference/desktop/database-schema.md); the commands → [commands.md](../reference/cli/commands.md); the static registry → [pricing.ts](../../packages/llm/src/pricing.ts). + +> **Amended 2026-07-05 by [ADR-0065](0065-provider-economics-and-extensibility.md)** (append-only — this body is unchanged): §6's merge precedence gains its **USER tier** — user-supplied per-model pricing fills **unknown** ids (`price = static ?? user`; static always wins for a known id), populating the merge helper's day-one optional user slot from the `model_catalog` `source='user'` rows. ADR-0065 also **injects** that merged pricing into the cost path (closing the unpriced-model cap gap) and wires custom OpenAI-compatible `base_url` endpoints over the shared SSRF floor. This ADR's static/live contract is unchanged; ADR-0065 extends it additively. + +## Context + +The model catalog is **static in-code**: `MODEL_PRICING` ([pricing.ts](../../packages/llm/src/pricing.ts)) +is the single source of truth for both **cost** and **display** — canonical id → `{provider, displayName, +contextWindowTokens, prices, …}` — and `contextWindowForModel` returns `undefined` for an unknown id. The +`model_catalog` DB table + `createModelCatalogStore` **already exist** ([schema.ts](../../packages/db/src/schema.ts) +L95, [model-catalog-store.ts](../../packages/db/src/model-catalog-store.ts)) but were built for the 2.S +**media-routing** projection, ship **empty**, and are documented as "a display projection *seeded from* +`MODEL_PRICING`" ([pricing.ts](../../packages/llm/src/pricing.ts) L8). + +Phase 2.5.G was originally scoped to a **static** `/models` catalog, and a live provider `/v1/models` fetch +was listed **out of scope → Phase 3** ([phase-2.5-cli-consolidation.md](../roadmap/phases/phase-2.5-cli-consolidation.md) +§ "Explicitly out of scope"). The maintainer has **removed that deferral** (**Option A** — the live catalog, +chosen over the originally-scoped static-registry catalog): we build a **live** catalog now, because the +static registry inevitably lags a provider shipping a new model, and a live list surfaces the models a given +**key** can actually reach (tier/allowlist-gated). But the two concerns are not symmetric: **no provider +returns pricing**, and only Anthropic and Gemini return a context window — so the catalog is *discovery* +(live) reconciled with *economics* (static). Model **selection** and model **pricing** are separable, and this +ADR keeps them cleanly separated: the live list decides **availability**, the static registry stays the +**pricing** authority. + +The stakes: this adds a method to the **frozen `LLMProvider` seam** ([ADR-0011](0011-internal-llm-abstraction.md) — +the seam is "the immovable contract"), it is an **egress/SSRF surface**, and a naive refresh could **regress +media routing** (the table's existing consumer) or **overwrite a known price**. Getting the merge precedence or +the refresh isolation wrong silently corrupts cost governance or empties the picker. + +## Decision + +**We add an optional `listModels?` capability to the `LLMProvider` seam, a `kind` protocol abstraction, widen +the existing `model_catalog` table into a live-discovery cache, add a refresh lifecycle (first-run / explicit / +TTL background), and reconcile live discovery with the static registry in one pure `@relavium/llm` merge +helper — keeping the provider-id enum CLOSED and pricing authority with the static registry.** The live fetch +for the four known providers rides each adapter's vendor SDK; no new runtime dependency. + +### 1. Seam — `listModels?` optional capability method + +We add **`listModels?(key, signal?): Promise`** to `LlmProvider`, following the existing +optional capability-varying pattern (`contextLimit?` / `generateMedia?` / `pollMediaJob?`, +[types.ts](../../packages/llm/src/types.ts) L494–521). `ModelListing` is a **Relavium/Zod** type +(`id`, `displayName?`, `contextWindowTokens?`, `maxOutputTokens?`, `capabilities?`, `deprecatedAt?`); each +adapter Zod-parses its vendor `models.list()` response and maps it to `ModelListing` **inside `src/adapters/*`**, +so **no vendor SDK type crosses the seam** ([ADR-0011](0011-internal-llm-abstraction.md), CLAUDE.md #4). The +method is **optional**: an adapter (or a future kind) without a list endpoint omits it and the host degrades to +static-only for that provider. The exact signature + `ModelListing` shape are the seam's one canonical home +([llm-provider-seam.md](../reference/shared-core/llm-provider-seam.md)), not restated here. + +### 2. The `kind` protocol abstraction + +We introduce a provider **`kind` ∈ `{anthropic, openai-compatible, gemini}`** — a closed vocabulary +`PROVIDER_KINDS` in `@relavium/shared` (mirroring how `LLM_PROVIDERS` lives there and `@relavium/db` / +`@relavium/llm` derive from it). `kind` derives, **once per protocol rather than per provider**: the adapter +factory, the list-models endpoint path (`modelsPath`), the auth style, and the response mapper. Each of the +four known providers declares a `kind` in the single-home `KNOWN_PROVIDERS` metadata +([providers.ts](../../apps/cli/src/engine/providers.ts)) — `anthropic → anthropic`, +`openai`/`deepseek → openai-compatible`, `gemini → gemini`. DeepSeek already proves the pattern +(`createOpenAiAdapter({providerId:'deepseek'})`, [providers.ts](../../packages/llm/src/providers.ts) L18). This +is the seam mechanism [ADR-0065](0065-provider-economics-and-extensibility.md) reuses for OpenAI-compatible +custom endpoints; the enum itself stays **closed** (§6). + +### 3. Live fetch via the vendor SDK path; lenient inbound parsing + +For the four known providers, `listModels` rides each adapter's **vendor SDK `models.list()`** over the +adapter's injected network seam (the injectable `fetch` on Anthropic/OpenAI; the `GeminiTransport` on Gemini) — so `@relavium/llm` gains **no `node:` import** — and inherits the +construction-time `assertHttpsBaseUrl` SSRF gate (the OpenAI adapter) or the provider's fixed public host +(Anthropic/Gemini). The call is **bounded + abortable + secret-free**, mirroring `validateProviderKey`'s +`AbortController` + hard-timeout + key-redaction discipline ([providers.ts](../../apps/cli/src/engine/providers.ts) L107). +Parsing is **lenient inbound / strict outbound**: the mapper ignores unknown vendor fields and requires only +`id`, so **additive provider drift is absorbed silently**; a per-provider filter keeps only chat-capable text +models (Gemini by `supportedGenerationMethods` including `generateContent`; OpenAI/DeepSeek by an id-family +allowlist, intersected with `MODEL_PRICING` for cost eligibility; Anthropic/Gemini lists are clean). The +per-provider endpoint contracts (Anthropic's rich `/v1/models` with `max_input_tokens`/`capabilities`; Gemini's +`/v1beta/models`; the id-only OpenAI/DeepSeek shapes) are documented in +[llm-provider-seam.md](../reference/shared-core/llm-provider-seam.md), derived once, not restated here. + +### 4. The `model_catalog` live cache — repurpose, widen, migrate + +We **invert** the table's documented role: from "static projection seeded from `MODEL_PRICING`" to the +**live-discovery cache** ("which ids exist for this key"), with `MODEL_PRICING` enriching at **read** time. +Registry pricing is **never** seeded into the DB (that would create a second, drift-prone home — CLAUDE.md #8); +the DB cost columns are reserved for **user-supplied** pricing ([ADR-0065](0065-provider-economics-and-extensibility.md)). +(Considered a **new dedicated** live-catalog table — rejected: `model_catalog` already carries the +pricing/context/deprecation columns and the FK graph, so a second table would duplicate the schema and split +the media-routing and discovery homes into two.) A drizzle migration (`0007`, ALTER-ADD only) adds a **`source`** discriminant (`static | live | user`) and a +**`last_refreshed_at`** freshness column; because SQLite `ALTER TABLE ADD` cannot carry a `CHECK`, the closed +set is validated at the **store read boundary** (mirroring `coerceMediaSurface`, +[model-catalog-store.ts](../../packages/db/src/model-catalog-store.ts) L91). We widen the store with +`listByProvider`/`listAll` readers and a **transactional bulk live-upsert** that **soft-deactivates** models +absent from the new list — **never hard-deletes** (`model_catalog.id` is an FK target from five tables: +agents, step_executions, run_costs, agent_sessions, session_messages). The existing **media-routing reader +path stays regression-clean** — the widening is additive and the narrow media projection is untouched. The +new columns' one canonical home is [database-schema.md](../reference/desktop/database-schema.md). + +### 5. Refresh lifecycle — first-run / explicit / TTL background, per-provider isolation + +Three triggers: **(a) first-run-if-empty** — a minimal blocking fetch when the cache is empty (Home open); +**(b) explicit `relavium models refresh`** — blocking, reports per-provider *added/updated/deactivated*, with a +`--json` machine form ([ADR-0049](0049-cli-machine-output-contract.md)); **(c) a 24h TTL** — opening the picker +over a stale cache renders the cache **immediately** and kicks a **non-blocking background** refresh that +updates the view as results arrive. The background refresh is **fire-and-forget and `unref`'d** so it can +**never keep a short-lived CLI process alive** past command exit, and it **swallows** per-provider failures. +Isolation is **per-provider** (`Promise.allSettled`): one provider's failure (bad key, network, drift) **never +fails the whole refresh** — it keeps its last-good cached rows and is surfaced as "couldn't refresh — showing +last-known", never an empty picker. Offline / all-fail degrades **cache → static registry**, so the picker is +**never empty**. (Considered an **event-driven** refresh keyed off a provider-key change instead of a TTL — +rejected: the connected-key set rarely changes, and a TTL + explicit `models refresh` is simpler and +offline-predictable, with no hidden trigger.) The orchestration is a **host service with injected deps** (`{resolveProvider, keyFor, +catalogStore, now}`, [ADR-0038](0038-agentrunner-llm-call-boundary.md)), so desktop/VS Code reuse it and +`@relavium/llm`/`@relavium/core` stay platform-free. CI/replay never hits the network — the refresh is +injectable/skippable behind the same recorded-fetch seam the conformance harness uses. + +### 6. The enum stays CLOSED; the merge is a pure `@relavium/llm` function + +`ProviderId` stays the **closed `z.enum(LLM_PROVIDERS)`** — it flows through the seam, the **persisted** +run-event `provider` field ([run-event.ts](../../packages/shared/src/run-event.ts) L338), authored agent YAML, +and an **exhaustive** `Record` ([providers.ts](../../packages/llm/src/providers.ts)). +Opening it to arbitrary ids is a persisted-contract + exhaustiveness change **out of 2.5.G scope**; +[ADR-0065](0065-provider-economics-and-extensibility.md) names it as future work. The pure **static/live merge** +lives in `@relavium/llm` beside `pricing.ts` (references `ModelPricing`, I/O-free) so every surface reuses it. +(Considered placing the merge in the **host** (`apps/cli`) — rejected: desktop and VS Code would each +re-implement it; a pure `@relavium/llm` function is written once and reused by all surfaces.) +Per-field **precedence**: + +- **AVAILABILITY** ← the live list. A static model **absent** from the current key's live list renders **dimmed + "not available on your key"** and is non-selectable (the maintainer's decision: dim, do not hide); a provider with **no live list** + (endpoint down or `listModels` absent) falls back to **static presence** — never "everything unavailable". +- **PRICE** ← **static** (`MODEL_PRICING`). The live tier is **never** a pricing authority (providers rarely + return price); a refresh must **never** overwrite or zero a known price. +- **CONTEXT / CAPABILITIES** ← live `??` static (live is fresher when present; e.g. Anthropic's + `max_input_tokens`). +- **DEPRECATION** ← **union** of static `deprecatedAt` and the live `deprecationDate` (§7). +- **PRICE-KNOWN** ← whether a static (or, per [ADR-0065](0065-provider-economics-and-extensibility.md), user) + price exists; **false** ⇒ the "cost cap will not apply" marker the picker surfaces. + +The helper accepts an **optional USER tier from day one** (unused here); [ADR-0065](0065-provider-economics-and-extensibility.md) +fills it with user-supplied pricing **additively**, with no re-open of this signature — the "no follow-up debt" +guarantee. The merged `ModelCatalogEntry` shape (`pricingSource`, `priceKnown`, `available`, `deprecated`, …) +is defined once beside the helper. + +### 7. Deprecation representation + +We add an optional **`deprecatedAt?: string` (ISO)** to the static `ModelPricing` +([pricing.ts](../../packages/llm/src/pricing.ts)), formalizing the DeepSeek legacy-alias prose (deprecating +2026-07-24) into a machine-readable, deterministic (`now ≥ deprecatedAt`) field. The DB +`model_catalog.deprecation_date` (`epochMs`, already present) carries the live half; the merge **unions** them; +the host projection converts ISO ↔ epochMs. The picker **flags** a deprecated model but never **forbids** it +(legacy aliases still cost correctly until their date). + +### 8. Drift resilience — endpoint/shape change behaviour + +Beyond §3's lenient inbound: a **breaking** provider change (a removed `id` field, a moved endpoint, a `4xx`) +makes the adapter's `listModels` **throw**, which §5's per-provider isolation catches → **cache + static +registry** keep the picker fully functional → a **visible, non-fatal notice** ("Provider X model list +unexpected — showing last-known") makes the drift **visible, not silent**. The **SDK path insulates** endpoint +and version moves (a vendor SDK bump handles them). A single **malformed row** is dropped at the mapper +boundary (a typed domain error, mirroring `parseCapabilities`), degrading **one** model, never the whole +provider. A drift fixture in the conformance suite proves this. + +### 9. Egress posture + +For the four known providers `listModels` hits **fixed public HTTPS hosts** via the SDK +(`assertHttpsBaseUrl` → `isPrivateOrLocalHost` on the OpenAI adapter; fixed hosts on Anthropic/Gemini), so the +residual SSRF risk is nil — the operative backstop is **SNI-pinned TLS certificate validation** (never +disabled): a private/loopback/metadata address cannot present a valid certificate for the provider's fixed +hostname, so even a DNS-rebinding answer fails the TLS handshake — and this ADR adds **no new egress surface**. A **user custom `base_url`** is a +distinct egress/SSRF surface handled entirely by [ADR-0065](0065-provider-economics-and-extensibility.md) +(host-side `connectValidated`). Gemini's key is sent in the **`x-goog-api-key` header**, never the `?key=` +query param (a URL-log leak — [ADR-0006](0006-os-keychain-for-api-keys.md)). + +### 10. Surfaces — `/models` (Home) and `relavium models refresh` + +`/models` is a new REPL command ([ADR-0056](0056-cli-in-app-slash-command-system-and-manifest.md)), +`availableIn: ['home']`: it opens an **in-tree ink picker** over the merged catalog and, on selection, **writes +the next session's default** via [ADR-0063](0063-cli-config-write-contract.md) — it does **not** rebind the live +session. This deliberately **disambiguates** the *other* `/models` +([ADR-0059](0059-cli-mid-session-model-reseat.md), Phase 2.6 — mid-chat **live reseat**): 2.5.G's Home `/models` +is a next-session **config** action, not a reseat. `relavium models` / `relavium models refresh` is a new +`COMMAND_MANIFEST` shell pair (list-cache vs force-refresh) with a `--json` contract. The picker UX is +first-class: pricing display, dimmed-unavailable, deprecated flag, an unpriced "cost cap will not apply" +hint, a loading spinner, a per-provider partial-failure banner, and a "last updated" freshness badge. + +## Consequences + +### Positive + +- **New models surface per key** — a provider's newly-shipped or tier-gated model appears without waiting for a + `pricing.ts` edit; the catalog is authoritative for **availability** while the registry stays authoritative + for **economics** — a clean separation of selection from pricing. +- **Drift-resilient and never-empty** — lenient inbound absorbs additive drift; a breaking change or an offline + provider degrades to last-known → static, visibly, never a crash or an empty picker. +- **Reusable across every surface** — the `listModels?` seam method, the pure merge helper, and the injected-deps + refresh service work identically for CLI, desktop, and VS Code (one engine, all surfaces). +- **Media routing unaffected** — the widening is additive; the existing narrow media-routing projection and its + D15 load-check are untouched; soft-deactivation preserves every FK reference. +- **No new dependency, seam frozen in shape** — reuses the vendor SDKs already fenced under `adapters/*`; the seam + gains one **optional** method (its method set is meant to grow; its shape stays frozen), amending + [ADR-0011](0011-internal-llm-abstraction.md)/[0030](0030-llm-seam-shape-amendment-reasoning-response-format-provider-executed.md)/[0031](0031-llm-seam-shape-amendment-multimodal-io.md) + additively; the provider-id enum stays closed so no persisted-contract churn. + +### Negative + +- **A wider seam surface** (one optional method + a `kind` vocabulary) and a **`model_catalog` role inversion** + with a migration. Mitigation: additive and forward-compatible ([ADR-0050](0050-cli-history-db-at-rest-posture.md) + single-user local posture); the media projection is untouched; the inversion is recorded here so the two homes + (static registry vs live cache) cannot be confused. +- **Live-cache staleness after key rotation** — a cached list can name models a newly-rotated key cannot call; + mitigated by the "last updated" badge, the TTL refresh, and a graceful run-time "model not available for this + key" failure. Serving stale-but-usable data is the deliberate offline-first trade. +- **Background-refresh lifecycle care** — a fire-and-forget refresh in a short-lived CLI must be `unref`'d, must + not surface a stack, and must tolerate two concurrent `relavium` processes racing the DB write (WAL + + `busy_timeout` already exist); getting this wrong hangs or corrupts the cache. Explicitly designed in (§5), + not bolted on, and covered by the refresh orchestration's security/robustness review. +- **A residual DNS-rebinding gap on the vendor-SDK path** — the known-provider path uses only the + construction-time string gate, not `connectValidated`; for the four **fixed public hosts** this is nil risk and + documented. The custom-`base_url` hardening is [ADR-0065](0065-provider-economics-and-extensibility.md)'s. +- **A mandatory security review** — the seam egress and the refresh orchestration (which reads provider keys) + each carry a dedicated security round before shipping (the [ADR-0057](0057-cli-chat-modes-and-per-tool-approval.md) + precedent for a security-touching regime ADR). +- **Canonical reference docs pending update** — this ADR points to `llm-provider-seam.md` (the `listModels?` + + `ModelListing` shape), `database-schema.md` (the `source` / `last_refreshed_at` columns), and `commands.md` + (the `models` / `models refresh` family) as canonical homes that are **not yet updated**; those edits land in + the implementing steps that add each artifact, matching ADR-0063's honest `config-spec.md` deferral. +- **Roadmap pull-forward** — this reverses the Phase-3 deferral of the live `/v1/models` fetch and supersedes + 2.5.G's static-only scope. The phase-2.5 § "Explicitly out of scope" and §2.5.G are **already reconciled in + this doc round**, together with the sibling + [phase-2.6-conversational-authoring.md](../roadmap/phases/phase-2.6-conversational-authoring.md) out-of-scope + list. diff --git a/docs/decisions/0065-provider-economics-and-extensibility.md b/docs/decisions/0065-provider-economics-and-extensibility.md new file mode 100644 index 00000000..9f8b9681 --- /dev/null +++ b/docs/decisions/0065-provider-economics-and-extensibility.md @@ -0,0 +1,177 @@ +# ADR-0065: Provider economics and extensibility — user-supplied pricing, the cost-path pricing-injection seam, pricing-reference capture, and custom OpenAI-compatible endpoints + +- **Status**: Accepted +- **Date**: 2026-07-05 +- **Related**: [ADR-0064](0064-live-model-catalog.md) (**this ADR extends its static/live merge with a USER tier**; append-only top-note added there) · [ADR-0011](0011-internal-llm-abstraction.md) (**this ADR amends the provider model — a `kind` protocol abstraction + building the adapter from the stored row; the id enum stays CLOSED**; append-only top-note added there) · [ADR-0028](0028-workflow-resource-governance.md) (the pre-egress budget governor whose "cost cap will not apply" gap this closes) · [ADR-0038](0038-agentrunner-llm-call-boundary.md) (host-injected resolution — the pricing overlay is injected exactly like `keyFor`) · [ADR-0006](0006-os-keychain-for-api-keys.md) + [ADR-0019](0019-cli-node-keychain-library.md) (keys stay in the keychain — user pricing is a **non-secret** storage class) · [ADR-0053](0053-mcp-network-transport-egress-security.md) + [ADR-0029](0029-tool-policy-hardening.md) (the one shared SSRF primitive a custom `base_url` reuses) · [ADR-0050](0050-cli-history-db-at-rest-posture.md) · [ADR-0056](0056-cli-in-app-slash-command-system-and-manifest.md) (the `models pricing` / `provider list --verify` commands). Canonical homes: the cost path → [cost-tracker.ts](../../packages/llm/src/cost-tracker.ts) + [budget-governor.ts](../../packages/core/src/engine/budget-governor.ts); the static registry → [pricing.ts](../../packages/llm/src/pricing.ts); the DB columns → [database-schema.md](../reference/desktop/database-schema.md); the commands → [commands.md](../reference/cli/commands.md). + +## Context + +Two latent defects, surfaced while scoping 2.5.G, frame this decision: + +1. **`provider add --base-url` is dead config.** It validates HTTPS and stores `base_url` in `llm_providers`, + and `provider list` echoes it — but **no adapter ever reads it**. `createProviderResolver` + ([providers.ts](../../apps/cli/src/engine/providers.ts) L186) is never handed the `ProviderStore`; it always + builds the keyless `defaultProviders()` ([providers.ts](../../packages/llm/src/providers.ts) L14) with the + **default** endpoints. A user who sets a custom endpoint today **silently gets the default** — and only the + OpenAI adapter even accepts a `baseURL` at all (Anthropic/Gemini factories have none). + +2. **An unpriced model silently disables the cost cap.** `priceModel` ([cost-tracker.ts](../../packages/llm/src/cost-tracker.ts) L19) + **throws `UnknownModelError`** for any id absent from the static `MODEL_PRICING`; `BudgetGovernor.evaluatePreEgress` + ([budget-governor.ts](../../packages/core/src/engine/budget-governor.ts) L141–149) **catches it and returns + `{kind:'allow'}`**. So a model with no static price runs **uncapped** — the `max_cost_microcents` governor + ([ADR-0028](0028-workflow-resource-governance.md)) silently no-ops. This is the exact "cost cap will not + apply" gap. + +The maintainer requires the model-**selection** and model-**pricing** story to be **clean and complete with no +follow-up debt** — including capturing, at provider-add time, a **pricing reference** and **user-supplied +per-model pricing** so cost governance works for models absent from the static registry. This is a distinct +decision from [ADR-0064](0064-live-model-catalog.md)'s "fetch a live list for the known providers": it centers +on `apps/cli` (the provider-add UX), `@relavium/db` (user-pricing storage), and — the sharp edge — +`@relavium/core`/`@relavium/llm` **cost-path signatures**; it has its own security surface (custom-`base_url` +SSRF + user-input validation); and it opens the **data** layer while keeping the **id enum** closed. It earns +its own ADR rather than swelling ADR-0064. + +## Decision + +**We make user-supplied per-model pricing a first-class non-secret storage class, inject the merged +static/user pricing into the cost path so a user-priced model is actually capped, rewire `resolveProvider` to +build the adapter from the stored provider row (fixing the dead-`base_url` bug and enabling custom +OpenAI-compatible endpoints over the SSRF floor), and capture a pricing-reference URL + user pricing at +provider-add — keeping the `ProviderId` enum closed.** + +### 1. User-supplied pricing — a non-secret storage class in `model_catalog` + +The `model_catalog` cost columns (`input`/`output`/`cached` microcents) **already exist** +([schema.ts](../../packages/db/src/schema.ts) L106) but are unsettable. We widen `ModelCatalogUpsert` +([model-catalog-store.ts](../../packages/db/src/model-catalog-store.ts) L51) to write them under +**`source = 'user'`** (the [ADR-0064](0064-live-model-catalog.md) §4 discriminant), so a background refresh +**never clobbers** a hand-entered row and the merge can rank precedence. Money is **integer microcents** +(`usd()` = `round(usd × 1e8)`, [pricing.ts](../../packages/llm/src/pricing.ts) L60) — no float persists; the +capture surface takes **USD/MTok** and converts at the boundary, echoing the resolved rate back. A new +`relavium models pricing` subcommand (under the [ADR-0064](0064-live-model-catalog.md) §10 `models` family; its +flags are the canonical [commands.md](../reference/cli/commands.md)'s, not restated here) writes it; +the onboarding wizard also captures it when the chosen model lacks a static price. Numeric input is +**bounds-validated** at the CLI boundary (reject negative / `NaN` / magnitudes that would break microcent math). +User pricing is **non-secret** config/data — it lives in the DB in plaintext, **never** the keychain. + +### 2. The cost-path pricing-injection seam — closing the "cap will not apply" gap + +This is the load-bearing change. We add an **optional pricing overlay** — a Relavium-typed +`ReadonlyMap` (a `resolvePrice` resolver) — and thread it through **both** cost paths: +`CostTracker` (its constructor + `cost`/`priceModel`) **and** the pre-egress estimators +(`estimateMaxNextCost`/`estimateMediaCost`) **and** the `BudgetGovernor`. Precedence: **static `MODEL_PRICING` +wins for known canonical ids; the overlay fills unknown ids only** — a user cannot silently misprice a shipped +model (matching [ADR-0064](0064-live-model-catalog.md)'s static-wins), and the same slot can also carry a +live-catalog entry for a newly-released official model. (Considered a **separate** user-pricing lookup distinct +from the static path — rejected: it would double the lookup and let realized vs pre-egress cost diverge; one +overlay through `priceModel` keeps them in lockstep. Considered mutating `MODEL_PRICING` or injecting the overlay +**unconditionally** — rejected: an **optional** overlay leaves the default path and every existing cost test +unchanged, and keeps static authoritative for known ids.) `priceModel` consults **static → overlay → then throws**, +preserving the deliberate **never-silent-zero** invariant (1.B): a *truly* unknown id (neither static nor user) +still throws, and cost governance still degrades to `allow` **with a loud, visible "cost cap will not apply" +notice** rather than a silent no-op. + +The **host** builds the overlay from **only** the `model_catalog` `source='user'` rows (a live/static cache +row's `NOT NULL DEFAULT 0` cost column is **never** read as a price — pricing authority is static per +[ADR-0064](0064-live-model-catalog.md) §6) and **injects** it — `@relavium/llm` and +`@relavium/core` never import `@relavium/db`; the overlay arrives as plain Relavium data, **byte-identical to +how `keyFor` injects the key** ([ADR-0038](0038-agentrunner-llm-call-boundary.md)). It threads +`SessionDeps → AgentSession → AgentTurnParams → new CostTracker(overlay)` **and** directly into the +`BudgetGovernor`, covering **both** the realized-cost and pre-egress paths — wiring only one would make the cap +and the ledger disagree (a governor that blocks on a price the ledger never charges, or the reverse). This is a +deliberate `core`+`llm` **seam-signature change**; under-scoping it (storing the row but not injecting it) would +leave the gap open — the exact follow-up debt the maintainer forbids. + +### 3. `resolveProvider` host-rewiring — make the stored `base_url` live (fixes bug #1) + +We upgrade `createProviderResolver` to accept the `ProviderStore` and, for a provider whose stored row carries +a **custom `base_url`**, build a **per-provider** adapter from `{kind, base_url}` rather than the static +`defaultProviders()` map — initially the **`openai-compatible`** kind +(`createOpenAiAdapter({providerId, baseURL})` + its construction-time `assertHttpsBaseUrl` gate). A stored +`base_url` is no longer dead config. The **Anthropic/Gemini** factories gain a validated `baseURL` option +**only** if a custom endpoint for those kinds is in this round's scope; otherwise custom endpoints are honestly +**`openai-compatible`-only**, documented — a `base_url` under `kind = anthropic|gemini` is refused with a clear +message rather than silently ignored (the current bug). + +### 4. Custom OpenAI-compatible endpoints + the SSRF floor + +A user registers a custom endpoint by pointing an **existing** provider id (`openai`/`deepseek`) at a custom +**HTTPS** `base_url` with `kind = openai-compatible`. `listModels`/`generate` over that custom endpoint is an +egress/SSRF surface and **must** reuse the shared HTTPS + private-range gate: `assertHttpsBaseUrl` + +`isPrivateOrLocalHost` at construction, and — for full **DNS-rebinding** protection — the host routes the +custom-endpoint hop through **`connectValidated`** ([safe-egress.ts](../../packages/db/src/safe-egress.ts), the +one shared connect-by-validated-IP primitive — [ADR-0053](0053-mcp-network-transport-egress-security.md), +[ADR-0029](0029-tool-policy-hardening.md)(d)), **never a second URL parser**. A custom `base_url` resolving to a +private/loopback/metadata address is **refused**. The known-provider fixed-host path +([ADR-0064](0064-live-model-catalog.md) §9) is unchanged — this hardening is scoped to the custom endpoint. + +### 5. Pricing-reference capture + the `kind` column + +`provider add` gains an optional **`--pricing-url`** (a non-secret, **display-only** URL where prices are looked +up), validated HTTPS at capture via `requireHttpsUrl` and stored in a **new `llm_providers.pricing_reference_url` +column** — **not** `default_headers` (that JSON is destined to be sent as wire headers once §3 wires the stored +row to the adapter — today it is dead config like `base_url`; stuffing a URL there would leak onto the wire once +live, a category error). It is **never auto-fetched** (no egress). The four known providers' pricing pages +**pre-populate** from a new `pricingUrl` field added to `KNOWN_PROVIDERS` (the pages `pricing.ts` already cites), +so the picker and `provider list` can **show where to look up prices without asking**. A **`kind` column** is added to +`llm_providers` too (nullable, populated for uniformity, load-bearing only for custom providers). These +`llm_providers` columns (`pricing_reference_url`, `kind`) ride their **own** additive migration (`0008`), +separate from [ADR-0064](0064-live-model-catalog.md)'s `model_catalog` `0007` — they land in the +provider-extensibility step, after the catalog cache. Both columns' one canonical home is +[database-schema.md](../reference/desktop/database-schema.md). + +### 6. The id enum stays CLOSED; `provider list --verify` + +`ProviderId` stays the closed `z.enum(LLM_PROVIDERS)` ([ADR-0064](0064-live-model-catalog.md) §6). A user-added +provider is **`(existing id + kind + custom base_url + keychain key + user pricing)`**; its models are unknown +ids priced solely from user rows (§2). A **truly-custom** provider id (a new, arbitrary id) would open the +closed enum — touching the **persisted** run-event `provider` field, authored agent YAML, and the exhaustive +`Record` — a deliberate future **supersede** of +[ADR-0011](0011-internal-llm-abstraction.md)'s closed-set posture. (Considered opening the enum **this round** — +rejected: that cross-package + persisted-contract churn is disproportionate to 2.5.G, and the `kind` data-layer +delivers custom endpoints without it.) It is **honestly named as future work, not this round**. `provider list` gains an **opt-in `--verify`** (reusing `validateProviderKey`'s bounded + redacted +probe) that reports per-provider verification state (the maintainer's decision) without hanging or leaking a key. One +honest limitation is documented: a custom OpenAI-compatible endpoint **reuses** the `openai`/`deepseek` id and +therefore cannot coexist with the real provider under that id — a genuinely-separate custom id awaits the +enum-opening ADR. + +## Consequences + +### Positive + +- **The cost-cap gap is closed** — a model absent from the static registry, once user-priced, is enforced by + `max_cost_microcents` on **both** the pre-egress and realized paths; a *truly* unknown model degrades to + uncapped **loudly and visibly**, never silently. +- **The dead-`base_url` bug is fixed** — a stored custom endpoint is now actually used; custom OpenAI-compatible + endpoints work end-to-end, SSRF-validated through the one shared primitive. +- **The pricing story is complete, with no follow-up debt** — capture (reference + per-model), storage + (non-secret, `source`-tagged so a refresh never clobbers it), and enforcement (the injection) all land + together; selection and pricing are cleanly separated yet both first-class. +- **A clean non-secret storage class** — user pricing + the pricing-reference URL live in the DB, distinct from + the keychain path; the two are never conflated. +- **The id enum stays closed** — no churn to the persisted run-event contract or authored YAML; extensibility is + delivered via the `kind` **data** layer, with a truly-open registry honestly deferred. + +### Negative + +- **The cost-path injection touches `core`+`llm` signatures and every cost test** — the sharpest risk. Mitigation: + it is the *only* change that closes the gap, the overlay is optional (absent ⇒ today's behaviour), precedence + is static-wins so it cannot misprice a known model, and it is covered by its security round. Wiring only + one of the two cost paths is called out as the specific hazard to avoid. +- **A user can misprice an unknown model** — bounded to *unknown* ids (static always wins for known ids), + numeric-validated at the boundary, with the USD→microcents conversion echoed back; a typo mis-governs only the + user's own custom model. +- **Custom-endpoint support is asymmetric** — `openai-compatible` works this round; `anthropic`/`gemini` custom + endpoints need those factories to gain a validated `baseURL` first, so a non-openai custom `base_url` is + **refused with a clear message** rather than over-promised. +- **A DB migration** — the `source`, `pricing_reference_url`, and `kind` columns (additive ALTER-ADD, validated + at the store boundary since SQLite `ALTER ADD` carries no `CHECK`); forward-compatible under the single-user + local posture ([ADR-0050](0050-cli-history-db-at-rest-posture.md)). +- **The reused-id limitation** — a custom OpenAI-compatible endpoint shadows the real provider under the same + id; a separate custom id awaits the future enum-opening supersede-ADR (named, not silently missing). +- **Canonical reference docs pending update** — this ADR points to `database-schema.md` (the + `pricing_reference_url` / `kind` columns) and `commands.md` (`models pricing`, `provider list --verify`, and + the now-honoured `--base-url`) as canonical homes **not yet updated**; those edits land in the implementing + steps that add each artifact, matching ADR-0063's honest `config-spec.md` deferral. +- **A mandatory security review** — the custom-`base_url` SSRF and the user-pricing/reference input validation + + the cost-path injection each carry a dedicated security round before shipping. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index 732c8908..bdd99f49 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -106,6 +106,9 @@ flowchart TD | 0060 | [Session `{{ctx.*}}` prompt interpolation](0060-session-ctx-prompt-interpolation.md) | Proposed | 2026-06-28 | | 0061 | [CLI chat input-layer file-injection (`@`-mention) and shell-escape (`!`-shell) security model](0061-cli-input-layer-file-injection-and-shell-escape.md) | Accepted | 2026-07-03 | | 0062 | [Context compaction — append-only conversation summarization and the CLI history commands (`/clear` · `/trim` · `/compact`)](0062-context-compaction-and-cli-history-commands.md) | Accepted | 2026-07-04 | +| 0063 | [CLI config-write contract — the first on-disk config writer, the global `[preferences].default_model` target, and the `resolveChat` global fallback](0063-cli-config-write-contract.md) | Accepted | 2026-07-05 | +| 0064 | [Live model catalog — the `listModels?` seam capability, the `kind` protocol abstraction, the `model_catalog` live cache, the refresh lifecycle, and the static/live merge](0064-live-model-catalog.md) | Accepted | 2026-07-05 | +| 0065 | [Provider economics and extensibility — user-supplied pricing, the cost-path pricing-injection seam, pricing-reference capture, and custom OpenAI-compatible endpoints](0065-provider-economics-and-extensibility.md) | Accepted | 2026-07-05 | ## Creating a new ADR diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index e2e95b30..63d85caa 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -186,8 +186,13 @@ all recorded in the ADR-0061 "Refined at implementation" append. **2.5.F** (the **Done (PR #65, merged 2026-07-05)** behind [ADR-0062](../decisions/0062-context-compaction-and-cli-history-commands.md): `/clear` (a host-level fresh-session swap across `chat`/`chat-resume`/Home, TTY-interactive only), the `session:compacting` "Summarizing…" moment event (amending ADR-0036), and the footer context-fullness indicator — -completing compaction alongside the earlier `/compact` + `/trim` + automatic compaction. **Next in the experience -arm: 2.5.G** (onboarding wizard + Home `/models`); the additive lanes 2.5.H / I / J run in parallel. See the +completing compaction alongside the earlier `/compact` + `/trim` + automatic compaction. **2.5.G is now +underway**, its scope **expanded to Option A** — a **live** model catalog (per-key provider discovery + a DB +cache + refresh + a static/live merge) and a complete model-pricing story (user-supplied pricing that governs +cost) — behind three new ADRs ([ADR-0063](../decisions/0063-cli-config-write-contract.md) config-write · +[ADR-0064](../decisions/0064-live-model-catalog.md) live catalog · +[ADR-0065](../decisions/0065-provider-economics-and-extensibility.md) provider economics), across 12 reviewed +steps; the additive lanes 2.5.H / I / J run in parallel. See the [Phase 2.5 workstreams](phases/phase-2.5-cli-consolidation.md). Carry-over hardening is tracked in [deferred-tasks.md](deferred-tasks.md) — Phase 2 picks diff --git a/docs/roadmap/phases/phase-2.5-cli-consolidation.md b/docs/roadmap/phases/phase-2.5-cli-consolidation.md index 25ef5fb8..98e21387 100644 --- a/docs/roadmap/phases/phase-2.5-cli-consolidation.md +++ b/docs/roadmap/phases/phase-2.5-cli-consolidation.md @@ -64,9 +64,9 @@ Along the way, close the bounded engine amendments and docs-debt that Phase 2 de [phase-2.6-conversational-authoring.md](phase-2.6-conversational-authoring.md). - `read_media` input (D12), full-fidelity reseat tool-context, in-app scrollback/pager, a **type-ahead message queue while a turn runs** (the in-flight key-swallow is handled for approval input in 2.5.E, but - queuing the *next* message is deferred), live provider `/v1/models` fetch, and a multi-pane dashboard — + queuing the *next* message is deferred), and a multi-pane dashboard — Phase 3 / later (tracked in [../deferred-tasks.md](../deferred-tasks.md)). (`/compact` model-summarised - compaction was originally listed here as Phase 3; it is now **built in 2.5.F** per [ADR-0062](../../decisions/0062-context-compaction-and-cli-history-commands.md).) + compaction was originally listed here as Phase 3; it is now **built in 2.5.F** per [ADR-0062](../../decisions/0062-context-compaction-and-cli-history-commands.md). The **live provider `/v1/models` fetch** was likewise listed here as Phase 3; it is now **built in 2.5.G** per [ADR-0064](../../decisions/0064-live-model-catalog.md) — the Option-A live model catalog.) ## Work breakdown @@ -403,19 +403,49 @@ overflows the context window; the summary is inspectable and the moment is a des > footer guard, the Home double-clear MCP-leak guard, best-effort auto-compaction, and the shared adapter > `CONTEXT_SEAM_DEFAULTS`.** -### 2.5.G — Onboarding wizard and `/models` (Home model catalog) - -**Tasks:** a `@clack` first-run wizard from the key-less Home (provider → **hidden** stdin key → -keychain, with a write-failure fallback and an env-key import offer) — reusing the two existing -ink↔clack custody patterns (the gate prompter and the 2.J create wizard); a shared `modelCatalog` -helper deriving available models from the `@relavium/llm` pricing registry filtered to -connected providers, with a staleness/deprecation guard; `/models` in Home sets the next session's -model (writing `[chat].default_model`); `provider list` shows verification state; the `[chat].max_turns` -surface wiring deferred from Phase 2 lands here. - -**Acceptance:** a key-less first run reaches a working chat via the wizard (key in the keychain, never -on disk); `/models` lists connected-provider models and sets the default; a deprecated catalog entry is -flagged. (Mid-chat model switch via reseat is Phase 2.6.) +### 2.5.G — Onboarding wizard, `/models`, and the live model catalog + +> **Scope expanded to Option A (2026-07-05).** The maintainer chose to build a **live** model catalog +> (per-key provider discovery + a DB cache + refresh + a static/live merge) and a **complete model-pricing +> story** (user-supplied pricing that actually governs cost) now, rather than the static-registry catalog +> originally scoped. It lands behind three ADRs — [ADR-0063](../../decisions/0063-cli-config-write-contract.md) +> (config-write contract), [ADR-0064](../../decisions/0064-live-model-catalog.md) (live catalog), and +> [ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) (provider economics & extensibility) +> — across **12 dependency-ordered, individually-reviewed steps** (six security-flagged). It pulls the live +> `/v1/models` fetch forward from Phase 3 (the § "Explicitly out of scope" note above is reconciled to match). + +**Tasks:** a `@clack` first-run wizard from the key-less Home (provider → **hidden** stdin key → keychain, with +a write-failure fallback + env-key import) reusing the two ink↔clack custody patterns; the **`listModels?` seam +capability** + a **`kind`** protocol abstraction + the four adapters ([ADR-0064](../../decisions/0064-live-model-catalog.md) +§1–3); the **`model_catalog` live cache** widening + migration (`source`/freshness columns) + store methods; the +pure **static/live/user merge** helper; the **refresh** lifecycle (`models refresh` + first-run + 24h-TTL +non-blocking background + per-provider partial-failure isolation); the **config-write** primitive + global +`[preferences].default_model` write + `resolveChat` global fallback ([ADR-0063](../../decisions/0063-cli-config-write-contract.md)); +the **`/models` Home picker** (first-class UX: pricing display, dimmed "not available on your key", deprecated +flag, unpriced "cost cap will not apply" hint); the **resolveProvider host-rewiring** so a stored custom +`base_url` is actually used + **custom OpenAI-compatible endpoints** over the SSRF floor +([ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) §3–4, fixing the dead-`base_url` bug); +**user-supplied pricing** capture (`models pricing`, a `--pricing-url` reference) + the **cost-path pricing +overlay** that closes the "cost cap will not apply" gap (ADR-0065 §1–2); `provider list --verify`; the +`[chat].max_turns` docs/test reconcile (already wired); and a new **adding-a-provider runbook**. + +**Acceptance:** a key-less first run reaches a working chat via the wizard (key in the keychain, never on disk); +`/models` lists the **live, connected-provider** catalog and sets the next session's default; an unavailable +static model is dimmed and a deprecated entry is flagged; `models refresh` reports per-provider outcomes and one +provider's failure never fails the whole refresh; a custom OpenAI-compatible `base_url` lists its models over an +SSRF-validated hop; a model with **no static price**, once **user-priced**, is enforced by `max_cost_microcents` +(the cost-cap gap is closed). (Mid-chat model switch via reseat stays Phase 2.6, +[ADR-0059](../../decisions/0059-cli-mid-session-model-reseat.md).) + +**Implementation steps** (each: implement + commit → Opus review → Sonnet review; **🔒** = an added security +round): **S1** ADRs 0063/0064/0065 + roadmap reconciliation (design-lock) · **S2 🔒** seam `listModels?` + `kind` ++ 4 adapters + drift + conformance · **S3** the pure static/live/user merge helper · **S4** `model_catalog` +widening + `0007` migration + store methods · **S5 🔒** refresh + `models refresh` + non-blocking background +auto-refresh + partial-failure · **S6 🔒** config-write primitive + global `[preferences]` fallback · **S7** +`/models` Home picker (first-class UX) · **S8 🔒** onboarding wizard · **S9 🔒** custom `base_url` + `kind` wiring +(dead-config fix) + `0008` migration · **S10 🔒** user-pricing capture + cost-path overlay (cap-gap fix) · +**S11** `provider list --verify` + `[chat].max_turns` reconcile · **S12** docs + adding-a-provider runbook. **Six** +are security-flagged: S2, S5, S6, S8, S9, S10. ### 2.5.H — Reasoning rendering and live-turn feedback diff --git a/docs/roadmap/phases/phase-2.6-conversational-authoring.md b/docs/roadmap/phases/phase-2.6-conversational-authoring.md index cdee0026..3e8fec1e 100644 --- a/docs/roadmap/phases/phase-2.6-conversational-authoring.md +++ b/docs/roadmap/phases/phase-2.6-conversational-authoring.md @@ -43,10 +43,12 @@ on top of the Phase 2.5 tool-environment and approval system. ### Explicitly out of scope (→ Phase 3 / later) -- `/compact` model-summarised compaction (no engine summarisation primitive); full-fidelity reseat - tool-context (the persister/schema extension, 1.X/1.Z); `read_media` input (D12); in-app - scrollback/pager; live provider `/v1/models` fetch; a multi-pane dashboard. Tracked in - [../deferred-tasks.md](../deferred-tasks.md). +- Full-fidelity reseat tool-context (the persister/schema extension, 1.X/1.Z); `read_media` input (D12); in-app + scrollback/pager; a multi-pane dashboard. Tracked in [../deferred-tasks.md](../deferred-tasks.md). (`/compact` + model-summarised compaction was originally listed here as Phase 3; it **shipped in 2.5.F** per + [ADR-0062](../../decisions/0062-context-compaction-and-cli-history-commands.md) (PR #65). The **live provider + `/v1/models` fetch** was likewise listed here; it is now **built in 2.5.G** per + [ADR-0064](../../decisions/0064-live-model-catalog.md) — the Option-A live model catalog.) ## Work breakdown From 862dc1d0dac6af9679a8a149415b6c3bbbf5d98f Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 18:47:42 +0300 Subject: [PATCH 03/68] =?UTF-8?q?feat(llm,shared):=202.5.G=20S2=20?= =?UTF-8?q?=E2=80=94=20listModels=3F=20seam=20capability=20+=20kind=20abst?= =?UTF-8?q?raction=20+=204=20adapters?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The live model-discovery half of the catalog (ADR-0064 §1-3, §7, §8). - shared: PROVIDER_KINDS closed vocabulary (anthropic | openai-compatible | gemini) + ProviderKind — the protocol axis, separate from the closed provider-id enum. - llm seam: an OPTIONAL, capability-varying listModels?(key, signal?) on LlmProvider returning a Relavium/Zod ModelListing[] (id required; displayName / context / maxOutput / deprecatedAt optional, positive-int limits so a vendor 0 == unknown is omitted). No vendor SDK type crosses the seam. providerKind(id) exhaustive helper. - adapters (shared substrate): boundedListModels — bounded (15s) + abortable + secret- free (re-scrubs + redacts the key, no cause, so neither the key nor the raw vendor payload leaks); toModelListing strict-outbound drop-on-malformed (lenient inbound, ADR-0064 §8); positiveModelInt (0/absent -> unknown). Per adapter: * anthropic: paginates models.list(), maps display_name/max_input_tokens/max_tokens. * openai (serves openai + deepseek): id-only, filters chat families + DENY embeddings/tts/whisper/image/moderation/realtime/audio/search/ft:, unions in MODEL_PRICING ids for the provider. * gemini: filters supportedActions incl. generateContent, maps name/displayName/ input+outputTokenLimit; behind the existing GeminiTransport seam. - conformance: listModels + listModelsDrift scenarios across all four providers (recorded fixtures, offline) + list-models.test.ts unit tests (mappers/filters/ redaction/pagination/bounded-timeout). Drift fixture proves an unexpected field is ignored and an id-less row dropped, never thrown. - docs/reference/shared-core/llm-provider-seam.md: the listModels? signature, the ModelListing shape, the kind families, and the per-provider endpoint contracts (the ADR-0064 canonical-home update). Verified: pnpm turbo run lint typecheck test build — 24/24 green; seam fence clean (no vendor SDK import outside adapters/, no node:* in packages/llm/src); prettier clean. Refs: ADR-0064 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../shared-core/llm-provider-seam.md | 52 ++++ packages/llm/src/adapters/anthropic.ts | 70 ++++- packages/llm/src/adapters/gemini.test.ts | 6 +- packages/llm/src/adapters/gemini.ts | 101 ++++++++ packages/llm/src/adapters/list-models.test.ts | 244 ++++++++++++++++++ packages/llm/src/adapters/openai.ts | 99 +++++++ packages/llm/src/adapters/shared.ts | 120 ++++++++- .../llm/src/conformance/fixtures/anthropic.ts | 70 +++++ .../llm/src/conformance/fixtures/deepseek.ts | 24 ++ .../llm/src/conformance/fixtures/gemini.ts | 60 +++++ .../llm/src/conformance/fixtures/openai.ts | 41 +++ .../conformance/gemini.conformance.test.ts | 15 ++ packages/llm/src/conformance/spec.ts | 94 ++++++- packages/llm/src/index.ts | 10 +- packages/llm/src/providers.ts | 22 ++ packages/llm/src/types.ts | 33 +++ packages/shared/src/constants.ts | 13 + 17 files changed, 1067 insertions(+), 7 deletions(-) create mode 100644 packages/llm/src/adapters/list-models.test.ts diff --git a/docs/reference/shared-core/llm-provider-seam.md b/docs/reference/shared-core/llm-provider-seam.md index 624a25ca..1dc63c3f 100644 --- a/docs/reference/shared-core/llm-provider-seam.md +++ b/docs/reference/shared-core/llm-provider-seam.md @@ -170,6 +170,11 @@ interface LlmProvider { contextLimit?(model: string): number | undefined; // the model's context window in tokens; undefined for an unrated/custom model (engine then skips auto-compaction) managesOwnContext?(): boolean; // provider bounds context itself ⇒ engine skips compaction; false for all current providers estimateTokens?(input: EstimateTokensInput): number; // { system, messages, tools? } → a per-provider estimate; a pre-first-turn FALLBACK only (real usage is authoritative) + // ADR-0064 live model catalog: return the models this `key` can reach, each mapped INSIDE the adapter to a + // Relavium ModelListing (no vendor models.list() type crosses). OPTIONAL (a provider without a list endpoint + // omits it → host degrades to static-only). Bounded + abortable + secret-free; one bad row is dropped, a + // breaking endpoint change throws a classified, key-redacted LlmProviderError. See "Model discovery" below. + listModels?(key: string, signal?: AbortSignalLike): Promise; } // The per-modality capability matrix (ADR-0031 decision #3). Input composability is unconstrained @@ -449,6 +454,53 @@ The platform-free **`MediaStore`** contract (`put`/`get`/`resolveForEgress` — signature are landed as reserved shape; implementations and the choke-point wiring are 1.AF. +### Model discovery — the `listModels?` capability ([ADR-0064](../../decisions/0064-live-model-catalog.md)) + +`listModels?(key, signal?): Promise` is an **optional, capability-varying** method (the same +pattern as `generateMedia?` / `contextLimit?`): a provider without a live list endpoint omits it and the host +degrades to the static registry ([pricing.ts](../../../packages/llm/src/pricing.ts)) for that provider. It +returns **live discovery** — which model ids a given `key` can actually reach (tier/allowlist-gated). The live +tier decides **availability**; the static registry stays the **pricing** authority (ADR-0064 §6), so +`ModelListing` deliberately carries **no price**. Each adapter maps its vendor `models.list()` row to +`ModelListing` **inside `src/adapters/*`** — **no vendor SDK type crosses this seam** (ADR-0011). + +```ts +// The Relavium/Zod projection of a live model-list row (ADR-0064 §1). MINIMAL + lenient-inbound / +// strict-outbound: only `id` is required; the rest are provider-varying and OMITTED when unknown. +interface ModelListing { + id: string; // provider-native id (Gemini: the `models/` prefix is stripped) + displayName?: string; // Anthropic/Gemini return one; OpenAI/DeepSeek do not + contextWindowTokens?: number; // Anthropic `max_input_tokens` / Gemini `inputTokenLimit`; positive-only (a 0/absent limit is "unknown" → OMITTED, never a stored 0) + maxOutputTokens?: number; // Anthropic `max_tokens` / Gemini `outputTokenLimit`; positive-only + deprecatedAt?: string; // ISO-8601; the LIVE list leaves it UNDEFINED — the static registry supplies the deprecation half, unioned at merge time (ADR-0064 §7) +} +``` + +**The `kind` protocol axis.** A provider's **`ProviderKind`** ∈ `{ anthropic, openai-compatible, gemini }` +(owned by `@relavium/shared` as `PROVIDER_KINDS`; derived from a `ProviderId` by `@relavium/llm`'s +`providerKind(id)`) selects — **once per protocol rather than per provider** — the adapter factory, the +list-models endpoint, the auth style, and the response mapper: `anthropic → anthropic`, `gemini → gemini`, +`openai`/`deepseek → openai-compatible`. This is a **separate axis** from the provider **id** enum +(`LLM_PROVIDERS`), which stays the closed persisted-contract set; the `kind` enum stays closed too +(ADR-0064 §6). No vendor type crosses either axis. + +**Behaviour every implementation shares** (ADR-0064 §3/§8): the call is **bounded + abortable + secret-free**, +mirroring `validateProviderKey` — `signal` (or a hard internal timeout) aborts the in-flight request, the +thrown error is a classified `LlmProviderError` whose message is **key-redacted** and which carries **no +`cause`** (so neither the resolved key nor the raw vendor payload can cross the seam). Parsing is **lenient +inbound / strict outbound**: unknown vendor fields are ignored (additive drift is absorbed silently); a row +that yields no `id` is **dropped** at the mapper boundary, never throwing (one malformed row degrades a single +model, not the whole provider); a breaking endpoint/shape change throws, which the host's per-provider refresh +isolation catches. A per-provider filter keeps only **chat-capable text models**. + +Per-provider list-models endpoint contracts: + +| Provider (`kind`) | Endpoint | Shape | Mapping + filter | +| --- | --- | --- | --- | +| Anthropic (`anthropic`) | `/v1/models` (SDK `models.list()`, auto-paginating `has_more`/`last_id`) | **Rich** | `id`, `display_name`→`displayName`, `max_input_tokens`→`contextWindowTokens` (omit if 0), `max_tokens`→`maxOutputTokens` (omit if 0). The list is clean — no filter (the rich `capabilities` object is ignored). | +| Gemini (`gemini`) | `/v1beta/models` (SDK `models.list()`) | **Rich** | `name` (strip `models/`)→`id`, `displayName`, `inputTokenLimit`→`contextWindowTokens`, `outputTokenLimit`→`maxOutputTokens`. **Filter:** keep only rows whose `supportedActions` (the SDK's projection of REST `supportedGenerationMethods`) includes `generateContent`. Key sent as the `x-goog-api-key` header, never a `?key=` query param (ADR-0064 §9). | +| OpenAI / DeepSeek (`openai-compatible`) | `/v1/models` (SDK `models.list()`) | **Id-only** | `id`→`id`, no context/price. **Filter:** keep the `gpt` / `o` / `*chat*` / `deepseek` families; DENY `embedding`/`tts`/`whisper`/`image`/`moderation`/`realtime`/`audio`/`search`/`ft:`; **union-in** any id present in `MODEL_PRICING` for that provider (cost-eligibility always wins). | + ## What must be normalized The seam's value is entirely in the normalization the adapters perform. Each of diff --git a/packages/llm/src/adapters/anthropic.ts b/packages/llm/src/adapters/anthropic.ts index 3542dd98..687c90d2 100644 --- a/packages/llm/src/adapters/anthropic.ts +++ b/packages/llm/src/adapters/anthropic.ts @@ -1,7 +1,7 @@ import Anthropic from '@anthropic-ai/sdk'; import { mediaModalityOf } from '@relavium/shared'; -import type { ContentPart, StopReason } from '@relavium/shared'; +import type { AbortSignalLike, ContentPart, StopReason } from '@relavium/shared'; import { assertStreamable, assertSupported } from '../capabilities.js'; import { LlmProviderError, kindFromHttpStatus, makeLlmError } from '../llm-error.js'; @@ -14,13 +14,21 @@ import type { LlmProvider, LlmRequest, LlmResult, + ModelListing, StreamChunk, ToolChoice, ToolDef, Usage, } from '../types.js'; -import { CONTEXT_SEAM_DEFAULTS, assertMediaCapabilities, isAbortSignal } from './shared.js'; +import { + CONTEXT_SEAM_DEFAULTS, + assertMediaCapabilities, + boundedListModels, + isAbortSignal, + positiveModelInt, + toModelListing, +} from './shared.js'; /** * The reference adapter over `@anthropic-ai/sdk` (1.C) — the seam fence's first real consumer and @@ -136,6 +144,37 @@ export function mapContent(blocks: readonly Anthropic.ContentBlock[]): ContentPa return parts; } +/** + * Map one Anthropic `models.list()` row (`ModelInfo`) to a canonical {@link ModelListing}, or `undefined` + * to drop it (ADR-0064 §3). Anthropic's list is rich: `display_name`→displayName, `max_input_tokens`→ + * contextWindowTokens, `max_tokens`→maxOutputTokens — but Anthropic returns `0`/`null` for an unknown limit, + * so a non-positive limit is OMITTED (`positiveModelInt`). Lenient-inbound: the SDK types most fields, but + * each is read defensively (the live API can deviate from the pinned SDK); the rich `capabilities` object is + * intentionally ignored (nothing in the merge consumes it). No filter — Anthropic's list is clean. + */ +export function mapAnthropicModel(info: { + id?: string; + display_name?: string | null; + max_input_tokens?: number | null; + max_tokens?: number | null; +}): ModelListing | undefined { + const candidate: Record = { + id: typeof info.id === 'string' ? info.id : '', // '' fails the schema's min(1) → the row is dropped + }; + if (typeof info.display_name === 'string' && info.display_name.length > 0) { + candidate['displayName'] = info.display_name; + } + const context = positiveModelInt(info.max_input_tokens); + if (context !== undefined) { + candidate['contextWindowTokens'] = context; + } + const maxOutput = positiveModelInt(info.max_tokens); + if (maxOutput !== undefined) { + candidate['maxOutputTokens'] = maxOutput; + } + return toModelListing(candidate); +} + /** Map an Anthropic error-body `type` to a kind — works even when there's no HTTP status (a stream `error` event). */ function kindFromErrorType(type: string): LlmErrorKind | undefined { switch (type) { @@ -737,6 +776,33 @@ export function createAnthropicAdapter(deps: AnthropicAdapterDeps = {}): LlmProv assertMediaCapabilities(PROVIDER, SUPPORTS, req); // per-modality input/output gate (ADR-0031, 1.AE) return streamChunks(createClient(key), req); }, + /** + * Live model discovery (ADR-0064 §1) over the SDK's `models.list()` — a rich, auto-paginating + * `PagePromise` (iterated with `for await`, which follows `has_more`/`last_id`). Each `ModelInfo` is + * mapped INSIDE the adapter to a canonical `ModelListing` (no vendor type escapes), and a per-row parse + * failure drops only that row. Bounded + abortable + secret-free via `boundedListModels`. + */ + async listModels(key: string, signal?: AbortSignalLike): Promise { + return boundedListModels({ + provider: PROVIDER, + key, + signal, + classify: anthropicErrorToLlmError, + collect: async (innerSignal) => { + const client = createClient(key); + const listings: ModelListing[] = []; + const seen = new Set(); + for await (const info of client.models.list(undefined, { signal: innerSignal })) { + const listing = mapAnthropicModel(info); + if (listing !== undefined && !seen.has(listing.id)) { + seen.add(listing.id); + listings.push(listing); + } + } + return listings; + }, + }); + }, // ADR-0062 context-compaction seam — the shared defaults (a native token-count endpoint could specialize // estimateTokens later; real usage is authoritative, so the heuristic is only a pre-first-turn fallback). ...CONTEXT_SEAM_DEFAULTS, diff --git a/packages/llm/src/adapters/gemini.test.ts b/packages/llm/src/adapters/gemini.test.ts index e3c1501d..a13b0910 100644 --- a/packages/llm/src/adapters/gemini.test.ts +++ b/packages/llm/src/adapters/gemini.test.ts @@ -43,10 +43,14 @@ async function collect(stream: AsyncIterable): Promise = { +const unusedGenerative: Pick< + GeminiTransport, + 'generateImages' | 'generateVideos' | 'pollVideo' | 'listModels' +> = { generateImages: () => Promise.reject(new Error('unused')), generateVideos: () => Promise.reject(new Error('unused')), pollVideo: () => Promise.reject(new Error('unused')), + listModels: () => Promise.reject(new Error('unused')), }; /** A transport that returns a fixed response and captures the request it was handed. */ diff --git a/packages/llm/src/adapters/gemini.ts b/packages/llm/src/adapters/gemini.ts index eae4166a..4b746e7e 100644 --- a/packages/llm/src/adapters/gemini.ts +++ b/packages/llm/src/adapters/gemini.ts @@ -17,6 +17,7 @@ import type { MediaGenRequest, MediaGenResult, MediaJobStatus, + ModelListing, StreamChunk, ToolChoice, ToolDef, @@ -28,9 +29,12 @@ import { REASONING_ID, assertMediaCapabilities, assertNoStreamingMediaOutput, + boundedListModels, decodeMediaJobId, encodeMediaJobId, isAbortSignal, + positiveModelInt, + toModelListing, } from './shared.js'; /** @@ -200,6 +204,20 @@ export interface GeminiVideoPoll { raiFilteredCount?: number | undefined; } +/** + * The minimal slice of a Gemini `models.list()` row the mapper reads (ADR-0064 §1) — a hand-rolled + * structural subset (parity with {@link GeminiResponse}) so no vendor `Model` type crosses the seam. + * `name` is the resource name (`models/`); `supportedActions` is the `@google/genai` projection of the + * REST `supportedGenerationMethods` — the chat-capability filter keys on it (`includes('generateContent')`). + */ +export interface GeminiModelInfo { + name?: string | undefined; + displayName?: string | undefined; + inputTokenLimit?: number | undefined; + outputTokenLimit?: number | undefined; + supportedActions?: string[] | undefined; +} + /** * The injected network seam. The default wraps `@google/genai`; the conformance harness injects a * replay implementation. Keeping it here lets the one adapter run on every host (ADR-0018) and lets @@ -208,6 +226,12 @@ export interface GeminiVideoPoll { export interface GeminiTransport { generate(request: GeminiRequest, key: string): Promise; stream(request: GeminiRequest, key: string): Promise>; + /** + * Live model discovery (ADR-0064 §1). The default wraps `ai.models.list()` and normalizes each row to + * the vendor-type-free {@link GeminiModelInfo}; the conformance harness injects a replay. `signal` aborts + * the in-flight list request (a client-only cancel — see the SDK's `abortSignal` note). + */ + listModels(key: string, signal?: AbortSignalLike): Promise; /** * Imagen generative image endpoint (1.AH A2). The default wraps `ai.models.generateImages`; the * conformance harness injects a replay. Required so a transport honestly declares the generative @@ -346,6 +370,37 @@ function mapGeminiPart(part: GeminiPart, ids: GeminiToolCallIds): ContentPart | : { type: 'reasoning', text: part.text }; } +/** + * Map one Gemini `models.list()` row to a canonical {@link ModelListing}, or `undefined` to drop it + * (ADR-0064 §3). FILTER: keep only chat-capable text models — `supportedActions` (the SDK's projection of + * the REST `supportedGenerationMethods`) must include `generateContent`. MAP: strip the `models/` prefix + * from `name`→id, `displayName`→displayName, `inputTokenLimit`→contextWindowTokens, `outputTokenLimit`→ + * maxOutputTokens (a non-positive limit is unknown → omitted). Lenient-inbound: every field read defensively. + */ +export function mapGeminiModel(model: GeminiModelInfo): ModelListing | undefined { + if ( + !Array.isArray(model.supportedActions) || + !model.supportedActions.includes('generateContent') + ) { + return undefined; // not a chat/text model (e.g. embeddings, image-gen) — filtered out + } + const rawName = typeof model.name === 'string' ? model.name : ''; + const id = rawName.startsWith('models/') ? rawName.slice('models/'.length) : rawName; + const candidate: Record = { id }; // '' fails the schema's min(1) → the row is dropped + if (typeof model.displayName === 'string' && model.displayName.length > 0) { + candidate['displayName'] = model.displayName; + } + const context = positiveModelInt(model.inputTokenLimit); + if (context !== undefined) { + candidate['contextWindowTokens'] = context; + } + const maxOutput = positiveModelInt(model.outputTokenLimit); + if (maxOutput !== undefined) { + candidate['maxOutputTokens'] = maxOutput; + } + return toModelListing(candidate); +} + /** Classify any transport/SDK throwable into a normalized `LlmError` — no vendor shape escapes. */ export function geminiErrorToLlmError(err: unknown): LlmError { if (err instanceof Error && err.name === 'AbortError') { @@ -550,6 +605,25 @@ const sdkTransport: GeminiTransport = { const client = new GoogleGenAI({ apiKey: key }); return client.models.generateContentStream(request); }, + async listModels(key: string, signal?: AbortSignalLike): Promise { + const client = new GoogleGenAI({ apiKey: key }); + // models.list() returns an auto-paginating Pager; `for await` follows nextPageToken. Each row is + // normalized to the vendor-type-free GeminiModelInfo here so no @google/genai Model type crosses the seam. + const pager = await client.models.list({ + config: isAbortSignal(signal) ? { abortSignal: signal } : {}, + }); + const rows: GeminiModelInfo[] = []; + for await (const model of pager) { + rows.push({ + name: model.name, + displayName: model.displayName, + inputTokenLimit: model.inputTokenLimit, + outputTokenLimit: model.outputTokenLimit, + supportedActions: model.supportedActions, + }); + } + return rows; + }, async generateImages(request: GeminiImageRequest, key: string): Promise { const client = new GoogleGenAI({ apiKey: key }); return client.models.generateImages(request); @@ -999,6 +1073,33 @@ export function createGeminiAdapter(deps: GeminiAdapterDeps = {}): LlmProvider { assertNoStreamingMediaOutput(PROVIDER, req); // media-out is generate()-only; streaming triad deferred (ADR-0046 §4) return streamChunks(transport, buildGeminiRequest(req), key); }, + /** + * Live model discovery (ADR-0064 §1) over the injected transport's `listModels` (default wraps + * `ai.models.list()`). Each vendor-type-free `GeminiModelInfo` is mapped + filtered to a chat-capable + * `ModelListing` (`supportedActions` must include `generateContent`, `models/` prefix stripped); a + * per-row parse failure drops only that row. Bounded + abortable + secret-free via `boundedListModels`. + */ + async listModels(key: string, signal?: AbortSignalLike): Promise { + return boundedListModels({ + provider: PROVIDER, + key, + signal, + classify: geminiErrorToLlmError, + collect: async (innerSignal) => { + const rows = await transport.listModels(key, innerSignal); + const listings: ModelListing[] = []; + const seen = new Set(); + for (const row of rows) { + const listing = mapGeminiModel(row); + if (listing !== undefined && !seen.has(listing.id)) { + seen.add(listing.id); + listings.push(listing); + } + } + return listings; + }, + }); + }, async generateMedia(req: MediaGenRequest, key: string): Promise { // Separate-endpoint generation, dispatched by modality (ADR-0045 §1): image → Imagen (generateImages, // SYNC); video → Veo (generateVideos, ASYNC LRO — returns an opaque jobId the engine polls via diff --git a/packages/llm/src/adapters/list-models.test.ts b/packages/llm/src/adapters/list-models.test.ts new file mode 100644 index 00000000..1eb90e1a --- /dev/null +++ b/packages/llm/src/adapters/list-models.test.ts @@ -0,0 +1,244 @@ +import { describe, expect, it } from 'vitest'; + +import { LlmProviderError, makeLlmError } from '../llm-error.js'; +import type { ModelListing } from '../types.js'; + +import { createAnthropicAdapter, mapAnthropicModel } from './anthropic.js'; +import { mapGeminiModel } from './gemini.js'; +import { keepOpenAiModelId, pricedModelIdsFor } from './openai.js'; +import { boundedListModels, positiveModelInt, redactKey, toModelListing } from './shared.js'; + +/** + * Unit tests for the ADR-0064 `listModels` mapping/filter helpers + the shared bounded/secret-free + * substrate. The end-to-end SDK-parse path is covered by the conformance suite (recorded fixtures); these + * pin the pure helpers directly and the harder-to-record robustness paths (pagination, timeout, redaction). + */ + +describe('positiveModelInt', () => { + it('keeps a finite positive integer and omits everything else', () => { + expect(positiveModelInt(128_000)).toBe(128_000); + expect(positiveModelInt(0)).toBeUndefined(); // 0 = "unknown" (Anthropic) → OMITTED + expect(positiveModelInt(-5)).toBeUndefined(); + expect(positiveModelInt(1.5)).toBeUndefined(); + expect(positiveModelInt(null)).toBeUndefined(); + expect(positiveModelInt(undefined)).toBeUndefined(); + expect(positiveModelInt('100')).toBeUndefined(); + expect(positiveModelInt(Number.NaN)).toBeUndefined(); + }); +}); + +describe('redactKey', () => { + it('masks every occurrence of the key and is a no-op for an empty key', () => { + expect(redactKey('failed for sk-abc and sk-abc again', 'sk-abc')).toBe( + 'failed for •••• and •••• again', + ); + expect(redactKey('no secret here', '')).toBe('no secret here'); // empty key → unchanged + }); +}); + +describe('toModelListing', () => { + it('validates a candidate and drops one with no id (strict outbound)', () => { + expect(toModelListing({ id: 'gpt-5.5' })).toEqual({ id: 'gpt-5.5' }); + expect(toModelListing({ id: '' })).toBeUndefined(); // min(1) → dropped + expect(toModelListing({ id: 'x', contextWindowTokens: 0 })).toBeUndefined(); // .positive() → dropped + }); +}); + +describe('mapAnthropicModel', () => { + it('maps the rich shape and omits a 0/absent limit', () => { + expect( + mapAnthropicModel({ + id: 'claude-opus-4-8', + display_name: 'Claude Opus 4.8', + max_input_tokens: 1_000_000, + max_tokens: 128_000, + }), + ).toEqual({ + id: 'claude-opus-4-8', + displayName: 'Claude Opus 4.8', + contextWindowTokens: 1_000_000, + maxOutputTokens: 128_000, + }); + // 0 (Anthropic's "unknown") → the limit fields are OMITTED, but the row is still listed. + expect( + mapAnthropicModel({ id: 'm', display_name: 'M', max_input_tokens: 0, max_tokens: 0 }), + ).toEqual({ id: 'm', displayName: 'M' }); + }); + + it('drops an id-less row and ignores unknown/extra vendor fields', () => { + expect(mapAnthropicModel({ display_name: 'No Id' })).toBeUndefined(); + // Extra fields (a `capabilities` object, a future field) are lenient-inbound ignored. + const withExtra = { id: 'm', display_name: 'M', capabilities: { foo: true } } as unknown as { + id?: string; + display_name?: string | null; + }; + expect(mapAnthropicModel(withExtra)).toEqual({ id: 'm', displayName: 'M' }); + }); +}); + +describe('mapGeminiModel', () => { + it('keeps a generateContent model, strips the models/ prefix, maps the limits', () => { + expect( + mapGeminiModel({ + name: 'models/gemini-2.5-flash', + displayName: 'Gemini 2.5 Flash', + inputTokenLimit: 1_048_576, + outputTokenLimit: 65_536, + supportedActions: ['generateContent', 'countTokens'], + }), + ).toEqual({ + id: 'gemini-2.5-flash', + displayName: 'Gemini 2.5 Flash', + contextWindowTokens: 1_048_576, + maxOutputTokens: 65_536, + }); + }); + + it('filters out a non-chat model and drops a name-less row', () => { + expect( + mapGeminiModel({ name: 'models/text-embedding-004', supportedActions: ['embedContent'] }), + ).toBeUndefined(); + expect(mapGeminiModel({ supportedActions: undefined })).toBeUndefined(); // no supportedActions + expect( + mapGeminiModel({ displayName: 'No Name', supportedActions: ['generateContent'] }), + ).toBeUndefined(); // chat-capable but no id + }); +}); + +describe('keepOpenAiModelId / pricedModelIdsFor', () => { + it('keeps the chat families and denies the non-chat ones', () => { + const none = new Set(); + for (const id of ['gpt-5.5', 'gpt-5.4-mini', 'o3', 'chatgpt-4o-latest', 'deepseek-chat']) { + expect(keepOpenAiModelId(id, none)).toBe(true); + } + for (const id of [ + 'text-embedding-3-large', + 'gpt-image-1', // deny wins over the gpt allow-family + 'whisper-1', + 'tts-1', + 'gpt-4o-realtime-preview', + 'omni-moderation-latest', + 'gpt-4o-audio-preview', + 'ft:gpt-4o:acme', + ]) { + expect(keepOpenAiModelId(id, none)).toBe(false); + } + }); + + it('unions-in a priced id even if the family heuristic would miss it', () => { + const priced = new Set(['weird-priced-model']); + expect(keepOpenAiModelId('weird-priced-model', priced)).toBe(true); + expect(keepOpenAiModelId('weird-priced-model', new Set())).toBe(false); + }); + + it('pricedModelIdsFor returns only that provider s ids', () => { + const openai = pricedModelIdsFor('openai'); + const deepseek = pricedModelIdsFor('deepseek'); + expect(openai.has('gpt-5.5')).toBe(true); + expect(openai.has('deepseek-chat')).toBe(false); + expect(deepseek.has('deepseek-chat')).toBe(true); + expect(deepseek.has('gpt-5.5')).toBe(false); + }); +}); + +// A minimal 2-page Anthropic /v1/models sequence — proves the SDK paginator is iterated (has_more/last_id). +function page(id: string, hasMore: boolean): string { + return JSON.stringify({ + data: [ + { + id, + type: 'model', + display_name: id.toUpperCase(), + created_at: '2026-01-01T00:00:00Z', + max_input_tokens: 1_000, + max_tokens: 500, + capabilities: null, + }, + ], + has_more: hasMore, + first_id: id, + last_id: id, + }); +} + +describe('Anthropic listModels — pagination', () => { + it('follows has_more/last_id across pages and returns every mapped row', async () => { + const bodies = [page('m1', true), page('m2', false)]; + let call = 0; + const fetchSeq = (): Promise => { + const body = bodies[Math.min(call, bodies.length - 1)]; + call += 1; + return Promise.resolve( + new Response(body, { status: 200, headers: { 'content-type': 'application/json' } }), + ); + }; + const adapter = createAnthropicAdapter({ fetch: fetchSeq, maxRetries: 0 }); + const listings = await (adapter.listModels?.('key') ?? Promise.resolve([])); + expect(listings.map((l) => l.id)).toEqual(['m1', 'm2']); + expect(call).toBeGreaterThanOrEqual(2); // two page fetches + }); +}); + +describe('boundedListModels', () => { + it('rejects with a classified timeout when collect never settles', async () => { + const result = boundedListModels({ + provider: 'gemini', + key: 'k', + signal: undefined, + classify: () => makeLlmError({ provider: 'gemini', kind: 'unknown', message: 'unused' }), + collect: () => new Promise(() => undefined), // never settles + timeoutMs: 20, + }); + await expect(result).rejects.toBeInstanceOf(LlmProviderError); + await expect(result).rejects.toMatchObject({ llmError: { kind: 'timeout', retryable: true } }); + }); + + it('redacts the key and attaches no cause on a failed collect (ADR-0064 §3)', async () => { + const key = 'sk-supersecret-abcdefghijklmnop'; + const result = boundedListModels({ + provider: 'openai', + key, + signal: undefined, + classify: (err) => + makeLlmError({ + provider: 'openai', + kind: 'auth', + message: err instanceof Error ? err.message : 'x', + cause: err, // a classifier that (wrongly) carried a cause must still be stripped + }), + collect: () => Promise.reject(new Error(`401 for ${key}`)), + }); + let caught: unknown; + try { + await result; + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(LlmProviderError); + if (caught instanceof LlmProviderError) { + expect(caught.llmError.kind).toBe('auth'); + expect(caught.llmError.message).not.toContain(key); + expect(caught.llmError.message).not.toContain('supersecret'); + expect(caught.llmError.cause).toBeUndefined(); // no cause can carry the key/raw payload across the seam + } + }); + + it('cancels via an already-aborted caller signal', async () => { + const controller = new AbortController(); + controller.abort(); + const result = boundedListModels({ + provider: 'anthropic', + key: 'k', + signal: controller.signal, + classify: () => + makeLlmError({ provider: 'anthropic', kind: 'cancelled', message: 'aborted' }), + collect: (signal) => + new Promise((_resolve, reject) => { + if (signal.aborted) { + reject(new Error('aborted')); + } + }), + }); + await expect(result).rejects.toMatchObject({ llmError: { kind: 'cancelled' } }); + }); +}); diff --git a/packages/llm/src/adapters/openai.ts b/packages/llm/src/adapters/openai.ts index 26cce5a0..cbc225da 100644 --- a/packages/llm/src/adapters/openai.ts +++ b/packages/llm/src/adapters/openai.ts @@ -17,6 +17,7 @@ import { import { assertStreamable, assertSupported } from '../capabilities.js'; import { InvalidBaseUrlError, UnsupportedCapabilityError } from '../errors.js'; import { LlmProviderError, kindFromHttpStatus, makeLlmError } from '../llm-error.js'; +import { MODEL_PRICING } from '../pricing.js'; import { normalizeToolCall, toWire } from '../tool-normalizer.js'; import type { CapabilityFlags, @@ -30,6 +31,7 @@ import type { MediaGenResult, MediaJobStatus, MediaUnitsEntry, + ModelListing, ProviderId, StreamChunk, ToolChoice, @@ -42,9 +44,11 @@ import { REASONING_ID, assertMediaCapabilities, assertNoStreamingMediaOutput, + boundedListModels, decodeMediaJobId, encodeMediaJobId, isAbortSignal, + toModelListing, } from './shared.js'; /** @@ -375,6 +379,68 @@ export function openaiErrorToLlmError(err: unknown, provider: ProviderId): LlmEr }); } +// --- Live model discovery: the id-only list filter (ADR-0064 §3) ------------------------------ + +/** + * Id substrings that are NOT chat-completions text models — DENIED from the OpenAI/DeepSeek live list + * (ADR-0064 §3). The OpenAI `/v1/models` list is id-only (no capability metadata), so the filter is an + * id-family heuristic: deny wins over allow, so `gpt-image-1` / `gpt-4o-audio-preview` / `omni-moderation` + * are dropped even though they match a `gpt`/`o` allow-family. + */ +const OPENAI_DENY_SUBSTRINGS = [ + 'embedding', + 'tts', + 'whisper', + 'image', + 'moderation', + 'realtime', + 'audio', + 'dall-e', + 'transcribe', + 'search', +] as const; + +/** + * The MODEL_PRICING native ids + canonical keys for one OpenAI-compatible provider — unioned into the live + * list so a **cost-eligible** id ALWAYS survives the id-family heuristic (ADR-0064 §3), even if a future + * priced id doesn't match a `gpt`/`o`/`chat`/`deepseek` family. + */ +export function pricedModelIdsFor(provider: ProviderId): ReadonlySet { + const ids = new Set(); + for (const [canonicalId, pricing] of Object.entries(MODEL_PRICING)) { + if (pricing.provider === provider) { + ids.add(canonicalId); + ids.add(pricing.nativeId); + } + } + return ids; +} + +/** + * Keep an OpenAI/DeepSeek model id iff it is a chat-capable text model (ADR-0064 §3). A priced id is kept + * unconditionally (cost-eligibility wins); otherwise `ft:` fine-tunes and every {@link OPENAI_DENY_SUBSTRINGS} + * family are denied, and only the `gpt` / `o` / `deepseek` / `*chat*` families are kept. Pure + + * unit-tested. + */ +export function keepOpenAiModelId(id: string, pricedIds: ReadonlySet): boolean { + if (pricedIds.has(id)) { + return true; + } + const lower = id.toLowerCase(); + if (lower.startsWith('ft:')) { + return false; + } + if (OPENAI_DENY_SUBSTRINGS.some((deny) => lower.includes(deny))) { + return false; + } + return ( + lower.startsWith('gpt') || + /^o\d/.test(lower) || + lower.startsWith('deepseek') || + lower.includes('chat') + ); +} + // --- Request building: canonical → OpenAI wire ----------------------------------------------- type ToolCallPart = Extract; @@ -949,6 +1015,39 @@ export function createOpenAiAdapter(deps: OpenAiAdapterDeps = {}): LlmProvider { assertNoStreamingMediaOutput(providerId, req); // media-out is generate()-only; streaming triad deferred (ADR-0046 §4) return streamChunks(createClient(key), req, providerId); }, + /** + * Live model discovery (ADR-0064 §1) over the SDK's `models.list()`. The OpenAI/DeepSeek list is + * ID-ONLY (no context/price metadata), so each row maps to a bare `{ id }` `ModelListing` and is + * filtered to chat-capable text families via `keepOpenAiModelId` (unioned with `MODEL_PRICING` for + * cost-eligibility). The provider id (`openai` | `deepseek`) selects the priced-id union set. Bounded + + * abortable + secret-free via `boundedListModels`; a per-row parse failure drops only that row. + */ + async listModels(key: string, signal?: AbortSignalLike): Promise { + return boundedListModels({ + provider: providerId, + key, + signal, + classify: (err) => openaiErrorToLlmError(err, providerId), + collect: async (innerSignal) => { + const client = createClient(key); + const priced = pricedModelIdsFor(providerId); + const listings: ModelListing[] = []; + const seen = new Set(); + for await (const model of client.models.list({ signal: innerSignal })) { + const id = typeof model.id === 'string' ? model.id : ''; + if (id.length === 0 || seen.has(id) || !keepOpenAiModelId(id, priced)) { + continue; + } + const listing = toModelListing({ id }); + if (listing !== undefined) { + seen.add(id); + listings.push(listing); + } + } + return listings; + }, + }); + }, /** * Separate-endpoint media generation (1.AG/1.AH, [ADR-0045](../../../../docs/decisions/0045-async-media-job-loop-poll-checkpoint-resume-cancel.md)). * SYNC image (gpt-image-1 `client.images.generate` → base64) and SYNC TTS audio (`audio.speech` → base64, diff --git a/packages/llm/src/adapters/shared.ts b/packages/llm/src/adapters/shared.ts index 54b47be0..1487d8a5 100644 --- a/packages/llm/src/adapters/shared.ts +++ b/packages/llm/src/adapters/shared.ts @@ -1,8 +1,18 @@ +import type { AbortSignalLike } from '@relavium/shared'; + import { mediaSupportReason } from '../capabilities.js'; import { UnsupportedCapabilityError } from '../errors.js'; +import { LlmProviderError, makeLlmError } from '../llm-error.js'; import { MODEL_PRICING, isCanonicalModelId } from '../pricing.js'; -import { LlmMessageSchema } from '../types.js'; -import type { CapabilityFlags, EstimateTokensInput, LlmRequest, ProviderId } from '../types.js'; +import { LlmMessageSchema, ModelListingSchema } from '../types.js'; +import type { + CapabilityFlags, + EstimateTokensInput, + LlmError, + LlmRequest, + ModelListing, + ProviderId, +} from '../types.js'; /** * Shared helpers for the provider adapters — the platform-coupled zone (`src/adapters/*`) that may @@ -170,3 +180,109 @@ export function decodeMediaJobId(jobId: string): string | undefined { } return decoded; } + +// --- Live model discovery (ADR-0064) — the shared listModels substrate every adapter reuses ------- + +/** + * Default per-call bound for a `listModels` probe (ADR-0064 §3) — long enough for a cold, multi-page + * list, short enough that a stalled provider can never hang a refresh. Mirrors the `validateProviderKey` + * bounded discipline ([providers.ts](../../../../apps/cli/src/engine/providers.ts)). + */ +export const LIST_MODELS_TIMEOUT_MS = 15_000; + +/** Redact every occurrence of `key` from a string — defense-in-depth on the error path (ADR-0064 §3; + * mirrors `validateProviderKey`). An empty key is a no-op (splitting on `''` would garble the text). */ +export function redactKey(text: string, key: string): string { + return key.length === 0 ? text : text.split(key).join('••••'); +} + +/** A finite positive integer, else `undefined` — the "0/absent limit means unknown, so OMIT it" rule + * (ADR-0064 §3). Anthropic returns `0` for an unknown `max_input_tokens`; a `null`/missing limit is + * likewise unknown. Never stores a `0` limit (the `ModelListing` schema is `.positive()`). */ +export function positiveModelInt(value: unknown): number | undefined { + return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : undefined; +} + +/** + * Finalize a leniently-built candidate into a validated {@link ModelListing}, or `undefined` if it fails + * the strict-outbound schema (ADR-0064 §3/§8) — the one boundary that drops a malformed / id-less row + * WITHOUT throwing, so additive provider drift is absorbed and one bad row degrades a single model, never + * the whole provider. Each adapter's mapper builds the candidate (which vendor field → which listing field + * is per-provider) and passes it here. + */ +export function toModelListing(candidate: Record): ModelListing | undefined { + const parsed = ModelListingSchema.safeParse(candidate); + return parsed.success ? parsed.data : undefined; +} + +/** + * Run a `listModels` collect bounded + abortable + secret-free (ADR-0064 §3). An internal + * `AbortController` is threaded to the SDK (so a caller `signal` OR the hard timeout actually cancels the + * in-flight request), plus a `Promise.race` hard timeout that settles even if the SDK ignores the signal. + * On failure it throws a classified `LlmProviderError` whose message is **key-redacted** and which carries + * **no `cause`** (so neither the key nor the raw vendor payload can leak across the seam / into a run + * event) — the host's per-provider refresh isolation catches it (ADR-0064 §5). + */ +export async function boundedListModels(params: { + readonly provider: ProviderId; + readonly key: string; + readonly signal: AbortSignalLike | undefined; + /** The adapter's SDK-error classifier (e.g. `anthropicErrorToLlmError`). */ + readonly classify: (err: unknown) => LlmError; + /** Fetch + map the rows, threading the internal (timeout/abort-linked) signal to the SDK. */ + readonly collect: (signal: AbortSignal) => Promise; + readonly timeoutMs?: number; +}): Promise { + const { provider, key, signal, classify, collect, timeoutMs = LIST_MODELS_TIMEOUT_MS } = params; + const controller = new AbortController(); + const onAbort = (): void => controller.abort(); + if (signal !== undefined) { + if (signal.aborted) { + controller.abort(); + } else { + signal.addEventListener('abort', onAbort); + } + } + let timer: ReturnType | undefined; + let timedOut = false; + const timeout = new Promise((_resolve, reject) => { + timer = setTimeout(() => { + timedOut = true; + controller.abort(); + reject(new Error('listModels timed out')); + }, timeoutMs); + }); + const collecting = collect(controller.signal); + // Attach a no-op handler so a post-timeout rejection (the SDK aborting after the timeout won the race) + // is not an unhandled rejection; the race still observes the rejection through its own handler. + collecting.catch(() => undefined); + try { + return await Promise.race([collecting, timeout]); + } catch (err) { + const base = timedOut + ? makeLlmError({ + provider, + kind: 'timeout', + message: `model list timed out after ${String(timeoutMs)}ms`, + }) + : classify(err); + // Re-wrap through makeLlmError so scrubSecrets runs again AND redactKey strips the resolved key; never + // pass `cause` (it could carry the key or the raw vendor payload — ADR-0064 §3). + throw new LlmProviderError( + makeLlmError({ + provider, + kind: base.kind, + message: redactKey(base.message, key), + ...(base.code !== undefined ? { code: base.code } : {}), + ...(base.status !== undefined ? { status: base.status } : {}), + }), + ); + } finally { + if (timer !== undefined) { + clearTimeout(timer); + } + if (signal !== undefined) { + signal.removeEventListener('abort', onAbort); + } + } +} diff --git a/packages/llm/src/conformance/fixtures/anthropic.ts b/packages/llm/src/conformance/fixtures/anthropic.ts index 5df15751..510038a8 100644 --- a/packages/llm/src/conformance/fixtures/anthropic.ts +++ b/packages/llm/src/conformance/fixtures/anthropic.ts @@ -242,6 +242,63 @@ const structuredOutput = JSON.stringify({ usage: { input_tokens: 8, output_tokens: 4 }, }); +// A recorded `/v1/models` page (ADR-0064 §1) — Anthropic's rich list: display_name + max_input_tokens +// (→contextWindowTokens) + max_tokens (→maxOutputTokens). The second row records the documented `0` +// "unknown" limits Anthropic returns → the mapper OMITS the fields (the row is still LISTED, never dropped). +const modelsList = JSON.stringify({ + data: [ + { + id: 'claude-opus-4-8', + type: 'model', + display_name: 'Claude Opus 4.8', + created_at: '2026-01-15T00:00:00Z', + max_input_tokens: 1_000_000, + max_tokens: 128_000, + capabilities: null, + }, + { + id: 'claude-haiku-4-5', + type: 'model', + display_name: 'Claude Haiku 4.5', + created_at: '2025-11-01T00:00:00Z', + max_input_tokens: 0, // unknown → OMITTED (never a stored 0) + max_tokens: 0, + capabilities: null, + }, + ], + has_more: false, + first_id: 'claude-opus-4-8', + last_id: 'claude-haiku-4-5', +}); + +// The drift fixture (ADR-0064 §8): one row carries an unknown future field (ignored), one row has NO id +// (dropped at the mapper boundary) — the whole call must still resolve, degrading one model, never throwing. +const modelsListDrift = JSON.stringify({ + data: [ + { + id: 'claude-sonnet-4-6', + type: 'model', + display_name: 'Claude Sonnet 4.6', + created_at: '2026-02-01T00:00:00Z', + max_input_tokens: 1_000_000, + max_tokens: 64_000, + capabilities: null, + some_unknown_future_field: 'ignore-me', + }, + { + type: 'model', // no `id` — dropped, never a throw + display_name: 'Id-less Model', + created_at: '2026-02-01T00:00:00Z', + max_input_tokens: 100, + max_tokens: 100, + capabilities: null, + }, + ], + has_more: false, + first_id: 'claude-sonnet-4-6', + last_id: 'claude-sonnet-4-6', +}); + export const ANTHROPIC_FIXTURES: ConformanceFixtures = { textGenerate: { status: 200, body: textMessage }, toolGenerate: { status: 200, body: toolMessage }, @@ -251,6 +308,8 @@ export const ANTHROPIC_FIXTURES: ConformanceFixtures = { streamError: { status: 200, contentType: 'text/event-stream', body: streamError }, reasoningStream: { status: 200, contentType: 'text/event-stream', body: reasoningStream }, structuredOutput: { status: 200, body: structuredOutput }, + listModels: { status: 200, body: modelsList }, + listModelsDrift: { status: 200, body: modelsListDrift }, toolLoop: { turn1: { status: 200, body: toolMessage }, turn2: { status: 200, body: textMessage }, @@ -264,5 +323,16 @@ export const ANTHROPIC_FIXTURES: ConformanceFixtures = { streamErrorKind: 'overloaded', reasoningStream: { text: 'let me think', reasoningTokens: 4 }, structuredOutput: { text: '{"ok":true}' }, + listModels: { + ids: ['claude-opus-4-8', 'claude-haiku-4-5'], + // The rich row: display name + context (max_input_tokens) + output (max_tokens) all mapped. + sample: { + id: 'claude-opus-4-8', + displayName: 'Claude Opus 4.8', + contextWindowTokens: 1_000_000, + maxOutputTokens: 128_000, + }, + }, + listModelsDrift: { ids: ['claude-sonnet-4-6'] }, }, }; diff --git a/packages/llm/src/conformance/fixtures/deepseek.ts b/packages/llm/src/conformance/fixtures/deepseek.ts index db21a1e7..261b8ab4 100644 --- a/packages/llm/src/conformance/fixtures/deepseek.ts +++ b/packages/llm/src/conformance/fixtures/deepseek.ts @@ -164,6 +164,26 @@ const structuredOutput = JSON.stringify({ usage: { prompt_tokens: 8, completion_tokens: 4, total_tokens: 12 }, }); +// A recorded `/models` page (ADR-0064 §1) — DeepSeek speaks the OpenAI-compatible wire, so its list is +// ID-ONLY too; both ids are the `deepseek` family and are kept. `provider: 'deepseek'` selects the priced-id +// union set (a distinct set from OpenAI's). +const modelsList = JSON.stringify({ + object: 'list', + data: [ + { id: 'deepseek-chat', object: 'model', created: 0, owned_by: 'deepseek' }, + { id: 'deepseek-reasoner', object: 'model', created: 0, owned_by: 'deepseek' }, + ], +}); + +// The drift fixture (ADR-0064 §8): an unknown field is ignored, an id-less row dropped, never a throw. +const modelsListDrift = JSON.stringify({ + object: 'list', + data: [ + { id: 'deepseek-chat', object: 'model', created: 0, owned_by: 'deepseek', extra_unknown: true }, + { object: 'model', created: 0, owned_by: 'deepseek' }, // no `id` → dropped + ], +}); + export const DEEPSEEK_FIXTURES: ConformanceFixtures = { textGenerate: { status: 200, body: textMessage }, toolGenerate: { status: 200, body: toolMessage }, @@ -173,6 +193,8 @@ export const DEEPSEEK_FIXTURES: ConformanceFixtures = { streamError: { status: 503, body: streamError }, reasoningStream: { status: 200, contentType: 'text/event-stream', body: reasoningStream }, structuredOutput: { status: 200, body: structuredOutput }, + listModels: { status: 200, body: modelsList }, + listModelsDrift: { status: 200, body: modelsListDrift }, toolLoop: { turn1: { status: 200, body: toolMessage }, turn2: { status: 200, body: textMessage }, @@ -193,5 +215,7 @@ export const DEEPSEEK_FIXTURES: ConformanceFixtures = { streamErrorKind: 'overloaded', reasoningStream: { text: 'let me think' }, structuredOutput: { text: '{"ok":true}' }, + listModels: { ids: ['deepseek-chat', 'deepseek-reasoner'], sample: { id: 'deepseek-chat' } }, + listModelsDrift: { ids: ['deepseek-chat'] }, }, }; diff --git a/packages/llm/src/conformance/fixtures/gemini.ts b/packages/llm/src/conformance/fixtures/gemini.ts index 642286d5..3be08e4e 100644 --- a/packages/llm/src/conformance/fixtures/gemini.ts +++ b/packages/llm/src/conformance/fixtures/gemini.ts @@ -95,6 +95,53 @@ const imageGenerate = JSON.stringify({ ], }); +// A recorded `models.list()` reply (ADR-0064 §1) — recorded at the SDK-output level (a `GeminiModelInfo[]` +// the fake transport returns), like the other Gemini fixtures. FILTER: keep only chat-capable models +// (`supportedActions` includes `generateContent`); the embedding row is dropped. MAP: strip the `models/` +// prefix, `inputTokenLimit`→contextWindowTokens, `outputTokenLimit`→maxOutputTokens. +const modelsList = JSON.stringify([ + { + name: 'models/gemini-2.5-flash', + displayName: 'Gemini 2.5 Flash', + inputTokenLimit: 1_048_576, + outputTokenLimit: 65_536, + supportedActions: ['generateContent', 'countTokens'], + }, + { + name: 'models/gemini-2.5-pro', + displayName: 'Gemini 2.5 Pro', + inputTokenLimit: 1_048_576, + outputTokenLimit: 65_536, + supportedActions: ['generateContent'], + }, + { + name: 'models/text-embedding-004', + displayName: 'Text Embedding 004', + inputTokenLimit: 2_048, + outputTokenLimit: 1, + supportedActions: ['embedContent'], // no generateContent → filtered out + }, +]); + +// The drift fixture (ADR-0064 §8): one row carries an unknown future field (ignored); one chat-capable row +// has NO `name` (→ no id → dropped at the mapper boundary), never a throw. +const modelsListDrift = JSON.stringify([ + { + name: 'models/gemini-2.5-flash', + displayName: 'Gemini 2.5 Flash', + inputTokenLimit: 1_048_576, + outputTokenLimit: 65_536, + supportedActions: ['generateContent'], + someUnknownFutureField: 'ignore-me', + }, + { + displayName: 'Name-less Model', // no `name` → no id → dropped + inputTokenLimit: 1_000, + outputTokenLimit: 1_000, + supportedActions: ['generateContent'], + }, +]); + export const GEMINI_FIXTURES: ConformanceFixtures = { textGenerate: { status: 200, body: textResponse }, toolGenerate: { status: 200, body: toolResponse }, @@ -105,6 +152,8 @@ export const GEMINI_FIXTURES: ConformanceFixtures = { reasoningStream: { status: 200, body: reasoningStream }, structuredOutput: { status: 200, body: structuredOutput }, mediaGenerate: { status: 200, body: imageGenerate }, + listModels: { status: 200, body: modelsList }, + listModelsDrift: { status: 200, body: modelsListDrift }, toolLoop: { turn1: { status: 200, body: toolResponse }, turn2: { status: 200, body: textResponse }, @@ -119,5 +168,16 @@ export const GEMINI_FIXTURES: ConformanceFixtures = { reasoningStream: { text: 'let me think', reasoningTokens: 2 }, structuredOutput: { text: '{"ok":true}' }, mediaGenerate: { mimeType: 'image/png', data: 'aGVsbG8tY29uZm9ybWFuY2UtaW1hZ2U=' }, + listModels: { + ids: ['gemini-2.5-flash', 'gemini-2.5-pro'], + // `models/`-prefix stripped; inputTokenLimit→contextWindowTokens, outputTokenLimit→maxOutputTokens. + sample: { + id: 'gemini-2.5-flash', + displayName: 'Gemini 2.5 Flash', + contextWindowTokens: 1_048_576, + maxOutputTokens: 65_536, + }, + }, + listModelsDrift: { ids: ['gemini-2.5-flash'] }, }, }; diff --git a/packages/llm/src/conformance/fixtures/openai.ts b/packages/llm/src/conformance/fixtures/openai.ts index 13771cf4..a7d8be66 100644 --- a/packages/llm/src/conformance/fixtures/openai.ts +++ b/packages/llm/src/conformance/fixtures/openai.ts @@ -156,6 +156,42 @@ const imageGenerate = JSON.stringify({ data: [{ b64_json: 'aGVsbG8tY29uZm9ybWFuY2UtaW1hZ2U=' }], }); +// A recorded `/v1/models` page (ADR-0064 §1) — OpenAI's list is ID-ONLY (no context/price metadata), so +// the filter is an id-family heuristic: keep gpt/o/chat/deepseek families + any priced id, DENY the +// embeddings/tts/whisper/image/moderation/realtime/audio/ft: families. Here `gpt-5.5` / `gpt-5.4-mini` / `o3` +// survive; the six non-chat rows are dropped. +const modelsList = JSON.stringify({ + object: 'list', + data: [ + { id: 'gpt-5.5', object: 'model', created: 0, owned_by: 'openai' }, + { id: 'gpt-5.4-mini', object: 'model', created: 0, owned_by: 'openai' }, + { id: 'o3', object: 'model', created: 0, owned_by: 'openai' }, // reasoning family (o) → kept + { id: 'text-embedding-3-large', object: 'model', created: 0, owned_by: 'openai' }, // embedding → denied + { id: 'gpt-image-1', object: 'model', created: 0, owned_by: 'openai' }, // image → denied (deny wins over gpt) + { id: 'whisper-1', object: 'model', created: 0, owned_by: 'openai' }, // whisper → denied + { id: 'tts-1', object: 'model', created: 0, owned_by: 'openai' }, // tts → denied + { id: 'gpt-4o-realtime-preview', object: 'model', created: 0, owned_by: 'openai' }, // realtime → denied + { id: 'omni-moderation-latest', object: 'model', created: 0, owned_by: 'openai' }, // moderation → denied + { id: 'ft:gpt-4o-2024:acme', object: 'model', created: 0, owned_by: 'acme' }, // fine-tune → denied + ], +}); + +// The drift fixture (ADR-0064 §8): one row carries an unknown future field (ignored), one row has NO id +// (dropped) — the call resolves, never throws. +const modelsListDrift = JSON.stringify({ + object: 'list', + data: [ + { + id: 'gpt-5.5', + object: 'model', + created: 0, + owned_by: 'openai', + some_unknown_field: 'ignore-me', + }, + { object: 'model', created: 0, owned_by: 'openai' }, // no `id` → dropped + ], +}); + // No reasoningStream fixture: OpenAI chat.completions emits no reasoning output (the conformance // reasoning scenario is skipped for this provider). export const OPENAI_FIXTURES: ConformanceFixtures = { @@ -167,6 +203,8 @@ export const OPENAI_FIXTURES: ConformanceFixtures = { streamError: { status: 503, body: streamError }, structuredOutput: { status: 200, body: structuredOutput }, mediaGenerate: { status: 200, body: imageGenerate }, + listModels: { status: 200, body: modelsList }, + listModelsDrift: { status: 200, body: modelsListDrift }, toolLoop: { turn1: { status: 200, body: toolMessage }, turn2: { status: 200, body: textMessage }, @@ -180,5 +218,8 @@ export const OPENAI_FIXTURES: ConformanceFixtures = { streamErrorKind: 'overloaded', structuredOutput: { text: '{"ok":true}' }, mediaGenerate: { mimeType: 'image/png', data: 'aGVsbG8tY29uZm9ybWFuY2UtaW1hZ2U=' }, + // id-only list: the three chat families survive; the sample carries only `id` (no context/price). + listModels: { ids: ['gpt-5.5', 'gpt-5.4-mini', 'o3'], sample: { id: 'gpt-5.5' } }, + listModelsDrift: { ids: ['gpt-5.5'] }, }, }; diff --git a/packages/llm/src/conformance/gemini.conformance.test.ts b/packages/llm/src/conformance/gemini.conformance.test.ts index 56090ddc..b4bcd76f 100644 --- a/packages/llm/src/conformance/gemini.conformance.test.ts +++ b/packages/llm/src/conformance/gemini.conformance.test.ts @@ -4,6 +4,7 @@ import { createGeminiAdapter, geminiAdapter, type GeminiImageResponse, + type GeminiModelInfo, type GeminiResponse, type GeminiTransport, } from '../adapters/gemini.js'; @@ -38,6 +39,12 @@ const isGeminiImageResponse = (value: unknown): value is GeminiImageResponse => const gen: unknown = value.generatedImages; return gen === undefined || Array.isArray(gen); }; +// The listModels fixture body is the SDK-output GeminiModelInfo[] (the fake transport bypasses the SDK, +// mirroring how the generate/stream fixtures carry SDK-output GeminiResponse). The adapter mapper reads +// every field defensively, so an array-of-objects structural check is a sufficient, unsafe-cast-free guard. +const isGeminiModelInfoArray = (value: unknown): value is GeminiModelInfo[] => + Array.isArray(value) && + value.every((item) => typeof item === 'object' && item !== null && !Array.isArray(item)); // Gemini has no `fetch` hook, so the conformance harness replays at the transport level: a recorded // SDK-output JSON (single response or an array of streamed responses) is parsed and served through a @@ -87,6 +94,14 @@ const makeReplayAdapter: MakeReplayAdapter = (recorded) => { generateVideos: () => Promise.reject(new Error('veo not exercised by the chat conformance replay')), pollVideo: () => Promise.reject(new Error('veo not exercised by the chat conformance replay')), + listModels: () => { + const current = nextRecording(); + if (current.status >= 400) return rejection(current.status); + const parsed: unknown = JSON.parse(current.body); + return isGeminiModelInfoArray(parsed) + ? Promise.resolve(parsed) + : Promise.reject(new Error('replay fixture is not a GeminiModelInfo[] array')); + }, }; return createGeminiAdapter({ transport }); }; diff --git a/packages/llm/src/conformance/spec.ts b/packages/llm/src/conformance/spec.ts index b508304a..189f4c7d 100644 --- a/packages/llm/src/conformance/spec.ts +++ b/packages/llm/src/conformance/spec.ts @@ -3,7 +3,12 @@ import { describe, expect, it } from 'vitest'; import { MEDIA_SURFACES, type StopReason } from '@relavium/shared'; import { LlmProviderError } from '../llm-error.js'; -import { LlmResultSchema, MediaGenResultSchema, StreamChunkSchema } from '../types.js'; +import { + LlmResultSchema, + MediaGenResultSchema, + ModelListingSchema, + StreamChunkSchema, +} from '../types.js'; import type { LlmErrorKind, LlmProvider, @@ -48,6 +53,22 @@ export interface ConformanceExpectations { * implement `generateMedia` supply this. `data` is the exact base64 the recorded reply carries, so a * bytes-mangling normalization regression fails the suite (parity with the text/structured scenarios). */ readonly mediaGenerate?: { mimeType: string; data: string }; + /** The canonical values a `listModels` reply normalizes to (ADR-0064 §1/§3) — providers that implement + * `listModels` supply this. `ids` is the EXACT expected id set in order (proves the per-provider filter + + * id strip); `sample` pins one row's mapped fields (display name / context / output — and that a `0`/absent + * limit is OMITTED, not stored). */ + readonly listModels?: { + readonly ids: readonly string[]; + readonly sample?: { + readonly id: string; + readonly displayName?: string; + readonly contextWindowTokens?: number; + readonly maxOutputTokens?: number; + }; + }; + /** The ids a `listModelsDrift` reply should yield (ADR-0064 §8) — the id-less row dropped, the unknown + * extra field ignored, never a throw. */ + readonly listModelsDrift?: { readonly ids: readonly string[] }; } /** The recorded provider responses a conformance run needs — one per canonical scenario. */ @@ -72,6 +93,13 @@ export interface ConformanceFixtures { * a provider with no `generateMedia` (`media_surface: 'chat'`-only). Drives the generative seam-contract * scenario (1.AG Section C, A5). */ readonly mediaGenerate?: RecordedResponse; + /** A recorded `models.list()` reply (ADR-0064 §1). For a `fetch`-based adapter (Anthropic/OpenAI/DeepSeek) + * this is the raw HTTP page body the real SDK parses; for Gemini it is the SDK-output `GeminiModelInfo[]` + * the fake transport returns. Omit for a provider without `listModels`. Drives the live-discovery scenario. */ + readonly listModels?: RecordedResponse; + /** A recorded `models.list()` reply carrying an UNKNOWN extra field and an id-less row (ADR-0064 §8) — the + * drift-resilience fixture: the extra field is ignored, the id-less row dropped, never a throw. */ + readonly listModelsDrift?: RecordedResponse; /** * A multi-turn tool loop (the path every agent node exercises): `turn1` is a tool-call reply; `turn2` is * the continuation the provider returns AFTER the caller appends the tool result. The conformance test @@ -395,5 +423,69 @@ export function defineConformanceSuite( expect(result.raw).toBeDefined(); }, ); + + it.skipIf(fixtures.listModels === undefined)( + 'listModels: normalizes the live list to validated ModelListing[] with the expected ids + mapping (ADR-0064 §1/§3)', + async () => { + const recorded = fixtures.listModels; + const exp = expected.listModels; + if (recorded === undefined || exp === undefined) { + return; // narrow for skipIf + } + const adapter = makeReplayAdapter(recorded); + if (adapter.listModels === undefined) { + throw new Error('a listModels fixture requires the adapter to implement listModels'); + } + const listings = await adapter.listModels(KEY); + // Strict-outbound: every row satisfies the canonical ModelListing schema. + for (const listing of listings) { + expect(ModelListingSchema.safeParse(listing).success).toBe(true); + } + // Exact id set (order + membership) — proves the per-provider chat-capability filter + id strip. + expect(listings.map((listing) => listing.id)).toEqual([...exp.ids]); + if (exp.sample !== undefined) { + const row = listings.find((listing) => listing.id === exp.sample?.id); + expect(row).toBeDefined(); + if (row !== undefined) { + // Exact mapping incl. OMITTED (undefined) fields — a 0/absent limit must not become a stored 0. + expect(row.displayName).toBe(exp.sample.displayName); + expect(row.contextWindowTokens).toBe(exp.sample.contextWindowTokens); + expect(row.maxOutputTokens).toBe(exp.sample.maxOutputTokens); + } + } + }, + ); + + it.skipIf(fixtures.listModelsDrift === undefined)( + 'listModels (drift): ignores an unknown field and drops an id-less row, never throwing (ADR-0064 §8)', + async () => { + const recorded = fixtures.listModelsDrift; + const exp = expected.listModelsDrift; + if (recorded === undefined || exp === undefined) { + return; // narrow for skipIf + } + const adapter = makeReplayAdapter(recorded); + if (adapter.listModels === undefined) { + throw new Error('a listModelsDrift fixture requires the adapter to implement listModels'); + } + // Must resolve, never throw — a breaking row degrades ONE model, not the whole provider. + const listings = await adapter.listModels(KEY); + for (const listing of listings) { + expect(ModelListingSchema.safeParse(listing).success).toBe(true); + } + // The id-less row is dropped; the unknown extra field never rode across the seam. + expect(listings.map((listing) => listing.id)).toEqual([...exp.ids]); + const allowedKeys = new Set([ + 'id', + 'displayName', + 'contextWindowTokens', + 'maxOutputTokens', + 'deprecatedAt', + ]); + for (const listing of listings) { + expect(Object.keys(listing).every((key) => allowedKeys.has(key))).toBe(true); + } + }, + ); }); } diff --git a/packages/llm/src/index.ts b/packages/llm/src/index.ts index 959531af..604a7fd3 100644 --- a/packages/llm/src/index.ts +++ b/packages/llm/src/index.ts @@ -24,6 +24,8 @@ export { MediaGenRequestSchema, MediaGenResultSchema, MediaJobStatusSchema, + // ADR-0064 live model catalog — the seam's live-discovery projection (no vendor type crosses). + ModelListingSchema, } from './types.js'; export type { @@ -48,6 +50,8 @@ export type { MediaGenRequest, MediaGenResult, MediaJobStatus, + // ADR-0064 — the live model-discovery entry returned by LlmProvider.listModels?. + ModelListing, LlmProvider, } from './types.js'; @@ -70,6 +74,8 @@ export { MediaMimeTypeSchema, OUTPUT_MODALITIES, MEDIA_BILLED_MODALITIES, + // ADR-0064 — the provider `kind` protocol vocabulary (shared-owned; the seam surfaces it here). + PROVIDER_KINDS, mediaModalityOf, decodedBase64ByteLength, containsInlineMediaBytes, @@ -88,6 +94,7 @@ export type { MediaModality, OutputModality, MediaBilledModality, + ProviderKind, MediaStore, DeInlineMedia, } from '@relavium/shared'; @@ -156,4 +163,5 @@ export type { // Default keyless provider registry — the provider→adapter mapping a host wires into // `resolveProvider` (ADR-0038); the key is injected per call via `keyFor`, never here (ADR-0011). -export { defaultProviders } from './providers.js'; +// `providerKind` derives the ADR-0064 protocol `kind` from a provider id (used by the later merge/refresh steps). +export { defaultProviders, providerKind } from './providers.js'; diff --git a/packages/llm/src/providers.ts b/packages/llm/src/providers.ts index 74808b91..d4b70aa1 100644 --- a/packages/llm/src/providers.ts +++ b/packages/llm/src/providers.ts @@ -1,3 +1,5 @@ +import type { ProviderKind } from '@relavium/shared'; + import { createAnthropicAdapter } from './adapters/anthropic.js'; import { createGeminiAdapter } from './adapters/gemini.js'; import { createOpenAiAdapter } from './adapters/openai.js'; @@ -19,3 +21,23 @@ export function defaultProviders(): Readonly> { gemini: createGeminiAdapter(), }; } + +/** + * Derive a provider's **protocol {@link ProviderKind}** from its id (ADR-0064 §2) — the pure axis that + * selects, once per protocol rather than per provider, the adapter factory / list-models endpoint / auth / + * response mapper. `anthropic` and `gemini` map 1:1; `openai` and `deepseek` share `openai-compatible` + * (DeepSeek being the OpenAI-compatible adapter at a custom base URL). Exhaustive over the closed + * `ProviderId` union (a new id is a compile error here), so the two stay in lock-step. Consumed by the + * later ADR-0064 steps (the refresh service + the static/live merge); the id enum stays closed. + */ +export function providerKind(id: ProviderId): ProviderKind { + switch (id) { + case 'anthropic': + return 'anthropic'; + case 'gemini': + return 'gemini'; + case 'openai': + case 'deepseek': + return 'openai-compatible'; + } +} diff --git a/packages/llm/src/types.ts b/packages/llm/src/types.ts index ded506b3..8b0238bd 100644 --- a/packages/llm/src/types.ts +++ b/packages/llm/src/types.ts @@ -463,6 +463,28 @@ export const MediaJobStatusSchema = z.discriminatedUnion('state', [ ]); export type MediaJobStatus = z.infer; +/** + * One entry in a provider's **live model listing** (ADR-0064 §1) — the Relavium/Zod projection of a vendor + * `models.list()` row, mapped INSIDE each adapter so **no vendor SDK type crosses the seam** (ADR-0011). + * The live tier decides **availability** (which ids a key can reach); **pricing** stays with the static + * registry ([pricing.ts](./pricing.ts)), so this shape carries **no price**. Deliberately MINIMAL and + * **lenient-inbound / strict-outbound** (ADR-0064 §3/§8): only `id` is required (a row that yields none is + * dropped, never thrown); the rest are provider-varying — Anthropic returns a display name + context/output + * limits, Gemini a display name + limits, OpenAI/DeepSeek are id-only. A vendor `0`/absent limit means + * "unknown" and is **omitted** (never a stored `0`), which is why the limits are `.positive()`. + */ +export const ModelListingSchema = z.object({ + id: nonEmptyString, + displayName: nonEmptyString.optional(), + contextWindowTokens: z.number().int().positive().optional(), + maxOutputTokens: z.number().int().positive().optional(), + // ISO-8601 deprecation date. The LIVE list leaves this UNDEFINED — the static registry supplies the + // deprecation half, unioned at merge time (ADR-0064 §7); present in the shape so a future provider that + // returns a live deprecation date maps cleanly with no seam change. Never invented from the live list. + deprecatedAt: z.string().optional(), +}); +export type ModelListing = z.infer; + /** * Input to {@link LlmProvider.estimateTokens} (ADR-0062) — a prospective request the engine has NOT yet * sent. Expressed in seam types only (no vendor type crosses — CLAUDE.md #4). Used only as a pre-first-turn @@ -519,4 +541,15 @@ export interface LlmProvider { * 1.AH A3 (`pollMediaJobSora`), the Gemini/Veo adapter at 1.AH A4. */ pollMediaJob?(jobId: string, key: string, signal?: AbortSignalLike): Promise; + /** + * **Live model discovery** (ADR-0064 §1) — return the models this `key` can currently reach, each mapped + * to the Relavium {@link ModelListing} INSIDE the adapter (the vendor `models.list()` row is normalized + * here, so no vendor SDK type crosses the seam — ADR-0011). OPTIONAL (the seam's capability-varying + * pattern, cf. `generateMedia?` / `contextLimit?`): a provider without a list endpoint omits it and the + * host degrades to static-only for that provider. The call is **bounded + abortable + secret-free** + * (mirroring `validateProviderKey`): `signal` aborts the in-flight request; a per-row parse failure drops + * that one row (never throws for it — ADR-0064 §8); a breaking endpoint/shape change throws a classified, + * key-redacted `LlmProviderError` the host's per-provider refresh isolation catches (ADR-0064 §5). + */ + listModels?(key: string, signal?: AbortSignalLike): Promise; } diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 9333e0d8..9fc20dca 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -267,6 +267,19 @@ export type EngineNodeType = (typeof ENGINE_NODE_TYPES)[number]; export const LLM_PROVIDERS = ['anthropic', 'openai', 'gemini', 'deepseek'] as const; export type LlmProviderId = (typeof LLM_PROVIDERS)[number]; +/** + * The three provider **protocol kinds** (the `kind` abstraction, [ADR-0064] §2) — a closed vocabulary + * that derives, **once per protocol rather than per provider**, the adapter factory, the list-models + * endpoint, the auth style, and the response mapper. `anthropic` and `gemini` map 1:1 to their id; + * `openai` and `deepseek` share `openai-compatible` (DeepSeek is the OpenAI-compatible adapter at a + * custom base URL). This is a SEPARATE axis from the provider **id** ({@link LLM_PROVIDERS}), which + * stays the closed persisted-contract enum; `kind` is the protocol axis `@relavium/llm` derives from + * it (`providerKind`). The enum itself stays closed — an open custom-provider registry is future work + * (ADR-0065), not this one. + */ +export const PROVIDER_KINDS = ['anthropic', 'openai-compatible', 'gemini'] as const; +export type ProviderKind = (typeof PROVIDER_KINDS)[number]; + /** * The three filesystem permission tiers (built-in-tools.md). The canonical vocabulary * for the config `fs_scope` (config-spec.md) and a session's `fsScopeTier` From aa756d0b9cc35855e0d03637b956d6ec3970d449 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 19:23:04 +0300 Subject: [PATCH 04/68] =?UTF-8?q?fix(llm):=202.5.G=20S2=20Opus-review=20?= =?UTF-8?q?=E2=80=94=20=C2=A78=20drift=20resilience=20+=20filter=20fixes?= =?UTF-8?q?=20+=20test=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Applies the 9 confirmed + 1 plausible findings from the adversarial Opus review of S2 (each independently verified against the code). ADR-0064 §8 contract gaps (the important ones): - Non-object row (e.g. a null in vendor `data`) was dereferenced in the mapper and threw, discarding the WHOLE provider's fresh list. Now each collect guards every row with a shared isRecord() and DROPS a non-object row (per-row drop, never the whole provider). - Systemic id-removal / every-row-shape-broken returned [] silently, defeating §8's "breaking change throws -> §5 shows last-known". Now a shared assertListModelsShape() throws a classified bad_request LlmProviderError iff rawCount>0 && kept===0 && droppedForShape>0 (a well-formed empty list, or an all-content-filtered list, still returns a genuine []). boundedListModels passes a pre-classified LlmProviderError through with its kind (not re-flattened to unknown), still redacted + cause-stripped. OpenAI filter: - Deny matching is now a -/_ segment-boundary regex, so 'search' no longer false-drops o3-deep-research / o4-mini-deep-research (re-SEARCH); -search-/-audio-/dall-e stay denied. - Added instruct/ocr/davinci/babbage deny tokens (drop gpt-3.5-turbo-instruct, deepseek-ocr, davinci-002, babbage-002); priced-rescue still wins first. Tests + docs: - Timeout now proves it aborts the in-flight collect (collectAborted flag kills the abort() mutation); mid-flight caller-abort + removeEventListener cleanup covered; priced-beats-DENY ordering pinned; FIX A/B filter regressions; assertListModelsShape unit tests + adapter null-row / all-id-less / empty-list cases; a listModelsError (401) conformance scenario across all four providers (rejects with a classified, key-redacted LlmProviderError). - Seam doc DENY enumeration reconciled to the code (dall-e/transcribe/instruct/ocr/ davinci/babbage + a boundary-match note). - ADR-0064 append-only clarifications: §3 filter is a UNION (priced id always kept), not an intersection; §1 ModelListing intentionally omits capabilities (seam doc is the canonical shape home). Verified: pnpm turbo run lint typecheck test build — 24/24 green (llm 487 pass, +13); seam fence clean; no node:* in packages/llm/src; prettier clean. Refs: ADR-0064 Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/decisions/0064-live-model-catalog.md | 14 ++ .../shared-core/llm-provider-seam.md | 2 +- packages/llm/src/adapters/anthropic.ts | 18 +- packages/llm/src/adapters/gemini.ts | 27 ++- packages/llm/src/adapters/list-models.test.ts | 198 +++++++++++++++++- packages/llm/src/adapters/openai.ts | 56 ++++- packages/llm/src/adapters/shared.ts | 42 +++- .../llm/src/conformance/fixtures/anthropic.ts | 9 + .../llm/src/conformance/fixtures/deepseek.ts | 12 ++ .../llm/src/conformance/fixtures/gemini.ts | 9 + .../llm/src/conformance/fixtures/openai.ts | 12 ++ packages/llm/src/conformance/spec.ts | 35 ++++ 12 files changed, 415 insertions(+), 19 deletions(-) diff --git a/docs/decisions/0064-live-model-catalog.md b/docs/decisions/0064-live-model-catalog.md index 874cf545..cb9e284a 100644 --- a/docs/decisions/0064-live-model-catalog.md +++ b/docs/decisions/0064-live-model-catalog.md @@ -52,6 +52,14 @@ method is **optional**: an adapter (or a future kind) without a list endpoint om static-only for that provider. The exact signature + `ModelListing` shape are the seam's one canonical home ([llm-provider-seam.md](../reference/shared-core/llm-provider-seam.md)), not restated here. +> **Clarification (2026-07-05):** the informal `ModelListing` sketch in this §1 (and §6's +> "CAPABILITIES ← live `??` static") lists a `capabilities?` field, but the **shipped** shape — whose +> canonical home is the seam doc — deliberately carries **no `capabilities`**: the field is +> `{ id, displayName?, contextWindowTokens?, maxOutputTokens?, deprecatedAt? }`. The adapters drop the +> vendor capabilities object (no merge step consumes per-model capabilities, and there is no per-model +> static tier to reconcile against), so §6's "CAPABILITIES ← live `??` static" reduces in practice to the +> per-**provider** `CapabilityFlags`. This is a documentation reconciliation only — the schema is unchanged. + ### 2. The `kind` protocol abstraction We introduce a provider **`kind` ∈ `{anthropic, openai-compatible, gemini}`** — a closed vocabulary @@ -80,6 +88,12 @@ per-provider endpoint contracts (Anthropic's rich `/v1/models` with `max_input_t `/v1beta/models`; the id-only OpenAI/DeepSeek shapes) are documented in [llm-provider-seam.md](../reference/shared-core/llm-provider-seam.md), derived once, not restated here. +> **Clarification (2026-07-05):** §3's phrase "an id-family allowlist, **intersected** with `MODEL_PRICING`" +> describes a **UNION**, not a set intersection — a priced id is kept **even without a family match** +> (cost-eligibility always wins), matching §6, the seam doc, and `keepOpenAiModelId` +> ([openai.ts](../../packages/llm/src/adapters/openai.ts) — `pricedIds.has(id)` short-circuits to `true` +> before the family heuristic runs). The intent throughout is "id-family allowlist ∪ priced ids". + ### 4. The `model_catalog` live cache — repurpose, widen, migrate We **invert** the table's documented role: from "static projection seeded from `MODEL_PRICING`" to the diff --git a/docs/reference/shared-core/llm-provider-seam.md b/docs/reference/shared-core/llm-provider-seam.md index 1dc63c3f..412a2fb9 100644 --- a/docs/reference/shared-core/llm-provider-seam.md +++ b/docs/reference/shared-core/llm-provider-seam.md @@ -499,7 +499,7 @@ Per-provider list-models endpoint contracts: | --- | --- | --- | --- | | Anthropic (`anthropic`) | `/v1/models` (SDK `models.list()`, auto-paginating `has_more`/`last_id`) | **Rich** | `id`, `display_name`→`displayName`, `max_input_tokens`→`contextWindowTokens` (omit if 0), `max_tokens`→`maxOutputTokens` (omit if 0). The list is clean — no filter (the rich `capabilities` object is ignored). | | Gemini (`gemini`) | `/v1beta/models` (SDK `models.list()`) | **Rich** | `name` (strip `models/`)→`id`, `displayName`, `inputTokenLimit`→`contextWindowTokens`, `outputTokenLimit`→`maxOutputTokens`. **Filter:** keep only rows whose `supportedActions` (the SDK's projection of REST `supportedGenerationMethods`) includes `generateContent`. Key sent as the `x-goog-api-key` header, never a `?key=` query param (ADR-0064 §9). | -| OpenAI / DeepSeek (`openai-compatible`) | `/v1/models` (SDK `models.list()`) | **Id-only** | `id`→`id`, no context/price. **Filter:** keep the `gpt` / `o` / `*chat*` / `deepseek` families; DENY `embedding`/`tts`/`whisper`/`image`/`moderation`/`realtime`/`audio`/`search`/`ft:`; **union-in** any id present in `MODEL_PRICING` for that provider (cost-eligibility always wins). | +| OpenAI / DeepSeek (`openai-compatible`) | `/v1/models` (SDK `models.list()`) | **Id-only** | `id`→`id`, no context/price. **Filter:** keep the `gpt` / `o` / `*chat*` / `deepseek` families; DENY `embedding`/`tts`/`whisper`/`image`/`moderation`/`realtime`/`audio`/`dall-e`/`transcribe`/`search`/`instruct`/`ocr`/`davinci`/`babbage`/`ft:` — each matched on a `-`/`_` **segment boundary** (so `search` denies `gpt-4o-search-preview` but NOT `o3-deep-research`); **union-in** any id present in `MODEL_PRICING` for that provider (cost-eligibility always wins). | ## What must be normalized diff --git a/packages/llm/src/adapters/anthropic.ts b/packages/llm/src/adapters/anthropic.ts index 687c90d2..312cc48f 100644 --- a/packages/llm/src/adapters/anthropic.ts +++ b/packages/llm/src/adapters/anthropic.ts @@ -23,9 +23,11 @@ import type { import { CONTEXT_SEAM_DEFAULTS, + assertListModelsShape, assertMediaCapabilities, boundedListModels, isAbortSignal, + isRecord, positiveModelInt, toModelListing, } from './shared.js'; @@ -792,13 +794,27 @@ export function createAnthropicAdapter(deps: AnthropicAdapterDeps = {}): LlmProv const client = createClient(key); const listings: ModelListing[] = []; const seen = new Set(); + let rawCount = 0; + let droppedForShape = 0; for await (const info of client.models.list(undefined, { signal: innerSignal })) { + rawCount += 1; + if (!isRecord(info)) { + droppedForShape += 1; // a non-object row (e.g. a null in `data`) — drop it, never dereference + continue; + } const listing = mapAnthropicModel(info); - if (listing !== undefined && !seen.has(listing.id)) { + if (listing === undefined) { + droppedForShape += 1; // no usable id (Anthropic's list is unfiltered — undefined ⇒ shape-invalid) + continue; + } + if (!seen.has(listing.id)) { seen.add(listing.id); listings.push(listing); } } + // ADR-0064 §8: a systemic id-removal (rows present, none usable, some shape-broken) THROWS so the + // host's per-provider isolation shows "last-known" rather than an empty picker. + assertListModelsShape(PROVIDER, { rawCount, kept: listings.length, droppedForShape }); return listings; }, }); diff --git a/packages/llm/src/adapters/gemini.ts b/packages/llm/src/adapters/gemini.ts index 4b746e7e..bf1cc3dd 100644 --- a/packages/llm/src/adapters/gemini.ts +++ b/packages/llm/src/adapters/gemini.ts @@ -27,12 +27,14 @@ import type { import { CONTEXT_SEAM_DEFAULTS, REASONING_ID, + assertListModelsShape, assertMediaCapabilities, assertNoStreamingMediaOutput, boundedListModels, decodeMediaJobId, encodeMediaJobId, isAbortSignal, + isRecord, positiveModelInt, toModelListing, } from './shared.js'; @@ -75,9 +77,6 @@ const GEMINI_SUPPORTS: CapabilityFlags = { const ZERO_USAGE: Usage = { inputTokens: 0, outputTokens: 0 }; -const isRecord = (value: unknown): value is Record => - typeof value === 'object' && value !== null && !Array.isArray(value); - /** * The canonical BARE media MIME (no RFC-2045 parameters) from a vendor-reported type, validated against the * shared {@link MediaMimeTypeSchema}; returns `''` when the value is absent, parameter-only, or illegal — @@ -1089,13 +1088,33 @@ export function createGeminiAdapter(deps: GeminiAdapterDeps = {}): LlmProvider { const rows = await transport.listModels(key, innerSignal); const listings: ModelListing[] = []; const seen = new Set(); + let rawCount = 0; + let droppedForShape = 0; for (const row of rows) { + rawCount += 1; + if (!isRecord(row)) { + droppedForShape += 1; // a non-object row — drop it, never dereference + continue; + } + // A row filtered out for NOT being chat-capable is a legitimate CONTENT drop (never counted + // toward the §8 shape tally); only a chat-capable-but-id-less row is a shape drop. + const actions = row['supportedActions']; + if (!Array.isArray(actions) || !actions.includes('generateContent')) { + continue; + } const listing = mapGeminiModel(row); - if (listing !== undefined && !seen.has(listing.id)) { + if (listing === undefined) { + droppedForShape += 1; // chat-capable but no usable id ⇒ shape-invalid + continue; + } + if (!seen.has(listing.id)) { seen.add(listing.id); listings.push(listing); } } + // ADR-0064 §8: a systemic id-removal (rows present, none usable, some shape-broken) THROWS so the + // host's per-provider isolation shows "last-known" rather than an empty picker. + assertListModelsShape(PROVIDER, { rawCount, kept: listings.length, droppedForShape }); return listings; }, }); diff --git a/packages/llm/src/adapters/list-models.test.ts b/packages/llm/src/adapters/list-models.test.ts index 1eb90e1a..2ccae6bd 100644 --- a/packages/llm/src/adapters/list-models.test.ts +++ b/packages/llm/src/adapters/list-models.test.ts @@ -1,4 +1,4 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; import { LlmProviderError, makeLlmError } from '../llm-error.js'; import type { ModelListing } from '../types.js'; @@ -6,7 +6,13 @@ import type { ModelListing } from '../types.js'; import { createAnthropicAdapter, mapAnthropicModel } from './anthropic.js'; import { mapGeminiModel } from './gemini.js'; import { keepOpenAiModelId, pricedModelIdsFor } from './openai.js'; -import { boundedListModels, positiveModelInt, redactKey, toModelListing } from './shared.js'; +import { + assertListModelsShape, + boundedListModels, + positiveModelInt, + redactKey, + toModelListing, +} from './shared.js'; /** * Unit tests for the ADR-0064 `listModels` mapping/filter helpers + the shared bounded/secret-free @@ -131,6 +137,23 @@ describe('keepOpenAiModelId / pricedModelIdsFor', () => { expect(keepOpenAiModelId('weird-priced-model', new Set())).toBe(false); }); + it('pins priced-beats-DENY ordering and the FIX A/B boundary denies', () => { + const none = new Set(); + // A priced id that also matches a DENY token is KEPT (cost-eligibility wins); unpriced → dropped. + expect(keepOpenAiModelId('gpt-4o-audio-x', new Set(['gpt-4o-audio-x']))).toBe(true); + expect(keepOpenAiModelId('gpt-4o-audio-x', none)).toBe(false); + // FIX A: `search` denies only on a `-`/`_` segment boundary, so deep-research reasoning models survive + // (re**search** is not a `search` segment) while `-search-` is still denied. + expect(keepOpenAiModelId('o3-deep-research', none)).toBe(true); + expect(keepOpenAiModelId('o4-mini-deep-research', none)).toBe(true); + expect(keepOpenAiModelId('gpt-4o-search-preview', none)).toBe(false); + // FIX B: non-chat completion families denied (they otherwise pass the gpt/deepseek allow-family). + expect(keepOpenAiModelId('gpt-3.5-turbo-instruct', none)).toBe(false); + expect(keepOpenAiModelId('deepseek-ocr', none)).toBe(false); + expect(keepOpenAiModelId('davinci-002', none)).toBe(false); + expect(keepOpenAiModelId('babbage-002', none)).toBe(false); + }); + it('pricedModelIdsFor returns only that provider s ids', () => { const openai = pricedModelIdsFor('openai'); const deepseek = pricedModelIdsFor('deepseek'); @@ -179,6 +202,99 @@ describe('Anthropic listModels — pagination', () => { }); }); +describe('assertListModelsShape (ADR-0064 §8)', () => { + it('throws a bad_request LlmProviderError only when rows exist, none are kept, and some broke shape', () => { + // The one throwing case: a systemic id-removal (every row shape-broken). + let caught: unknown; + try { + assertListModelsShape('anthropic', { rawCount: 2, kept: 0, droppedForShape: 2 }); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(LlmProviderError); + if (caught instanceof LlmProviderError) { + expect(caught.llmError.kind).toBe('bad_request'); + expect(caught.llmError.provider).toBe('anthropic'); + } + }); + + it('returns normally for an empty list, an all-content-filtered list, and a partial-drop list', () => { + // A well-formed EMPTY list (no rows) is a genuine [] — never a throw. + expect(() => + assertListModelsShape('openai', { rawCount: 0, kept: 0, droppedForShape: 0 }), + ).not.toThrow(); + // Every row content-filtered but shape-valid (droppedForShape 0) is a genuine [] — never a throw. + expect(() => + assertListModelsShape('openai', { rawCount: 3, kept: 0, droppedForShape: 0 }), + ).not.toThrow(); + // At least one row kept — never a throw, whatever was dropped. + expect(() => + assertListModelsShape('gemini', { rawCount: 3, kept: 1, droppedForShape: 2 }), + ).not.toThrow(); + }); +}); + +/** A single-page Anthropic `/v1/models` body from raw row objects (a row may be `null` to exercise the + * per-row non-object drop). Serves one 200 response through the adapter's injected fetch. */ +function anthropicModelsFetch(rows: readonly unknown[]): () => Promise { + const body = JSON.stringify({ data: rows, has_more: false, first_id: null, last_id: null }); + return () => + Promise.resolve( + new Response(body, { status: 200, headers: { 'content-type': 'application/json' } }), + ); +} + +describe('listModels — per-row drop + systemic drift (ADR-0064 §8, FIX C/D)', () => { + it('drops a single non-object (null) row and still resolves the valid rows (FIX C)', async () => { + const validRow = { + id: 'claude-opus-4-8', + type: 'model', + display_name: 'Claude Opus 4.8', + created_at: '2026-01-01T00:00:00Z', + max_input_tokens: 1_000_000, + max_tokens: 128_000, + capabilities: null, + }; + // A `null` row would THROW a TypeError if dereferenced in the mapper — the isRecord guard drops it. + const adapter = createAnthropicAdapter({ + fetch: anthropicModelsFetch([validRow, null]), + maxRetries: 0, + }); + const listings = await (adapter.listModels?.('key') ?? Promise.resolve([])); + expect(listings.map((l) => l.id)).toEqual(['claude-opus-4-8']); + }); + + it('rejects bad_request when every row lacks a usable id — never a silent [] (FIX D)', async () => { + const key = 'sk-live-secret-abcdefghijklmnop'; + const adapter = createAnthropicAdapter({ + // Two rows, each id-less → rawCount 2, kept 0, droppedForShape 2 → the systemic-drift throw. + fetch: anthropicModelsFetch([ + { type: 'model', display_name: 'No Id A', created_at: '2026-01-01T00:00:00Z' }, + { type: 'model', display_name: 'No Id B', created_at: '2026-01-01T00:00:00Z' }, + ]), + maxRetries: 0, + }); + let caught: unknown; + try { + await (adapter.listModels?.(key) ?? Promise.resolve([])); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(LlmProviderError); + if (caught instanceof LlmProviderError) { + expect(caught.llmError.kind).toBe('bad_request'); + expect(caught.llmError.message).not.toContain(key); // redacted on the re-wrap + expect(caught.llmError.cause).toBeUndefined(); + } + }); + + it('resolves [] for a well-formed EMPTY list — never a throw (FIX D)', async () => { + const adapter = createAnthropicAdapter({ fetch: anthropicModelsFetch([]), maxRetries: 0 }); + const listings = await (adapter.listModels?.('key') ?? Promise.resolve([])); + expect(listings).toEqual([]); + }); +}); + describe('boundedListModels', () => { it('rejects with a classified timeout when collect never settles', async () => { const result = boundedListModels({ @@ -241,4 +357,82 @@ describe('boundedListModels', () => { }); await expect(result).rejects.toMatchObject({ llmError: { kind: 'cancelled' } }); }); + + it('the hard timeout ABORTS the in-flight collect via the threaded signal (FIX H)', async () => { + // The collect settles ONLY when its threaded signal fires — so a passing result proves the timeout + // path actually called controller.abort() (deleting it would leave the collect pending; the flag would + // stay false). Closes the mutation hole the old "collect ignores its signal" timeout test left open. + let collectAborted = false; + const result = boundedListModels({ + provider: 'gemini', + key: 'k', + signal: undefined, + classify: () => makeLlmError({ provider: 'gemini', kind: 'unknown', message: 'unused' }), + collect: (signal) => + new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => { + collectAborted = true; + reject(new Error('aborted-by-timeout')); + }); + }), + timeoutMs: 20, + }); + await expect(result).rejects.toBeInstanceOf(LlmProviderError); + await expect(result).rejects.toMatchObject({ llmError: { kind: 'timeout' } }); + expect(collectAborted).toBe(true); + }); + + it('propagates a MID-FLIGHT caller abort to the threaded collect signal (FIX I)', async () => { + const controller = new AbortController(); // NOT yet aborted → exercises the addEventListener branch + const removeSpy = vi.spyOn(controller.signal, 'removeEventListener'); + const result = boundedListModels({ + provider: 'anthropic', + key: 'k', + signal: controller.signal, + classify: () => + makeLlmError({ provider: 'anthropic', kind: 'cancelled', message: 'aborted' }), + collect: (signal) => + new Promise((_resolve, reject) => { + signal.addEventListener('abort', () => reject(new Error('aborted'))); + }), + }); + await Promise.resolve(); // let boundedListModels install its listener before we abort + controller.abort(); + await expect(result).rejects.toMatchObject({ llmError: { kind: 'cancelled' } }); + expect(removeSpy).toHaveBeenCalled(); // the caller-signal listener is detached on settle (finally) + }); + + it('passes a pre-classified LlmProviderError THROUGH with its kind (FIX D drift path)', async () => { + const key = 'sk-drift-secret-abcdefghijklmnop'; + const result = boundedListModels({ + provider: 'openai', + key, + signal: undefined, + // classify would (wrongly) flatten to `unknown` — proving the pre-classified error is NOT reclassified. + classify: () => + makeLlmError({ provider: 'openai', kind: 'unknown', message: 'should-not-win' }), + collect: () => + Promise.reject( + new LlmProviderError( + makeLlmError({ + provider: 'openai', + kind: 'bad_request', + message: `model list shape unexpected for ${key}`, + }), + ), + ), + }); + let caught: unknown; + try { + await result; + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(LlmProviderError); + if (caught instanceof LlmProviderError) { + expect(caught.llmError.kind).toBe('bad_request'); // kept, not reclassified to `unknown` + expect(caught.llmError.message).not.toContain(key); // still redacted on the re-wrap + expect(caught.llmError.cause).toBeUndefined(); + } + }); }); diff --git a/packages/llm/src/adapters/openai.ts b/packages/llm/src/adapters/openai.ts index cbc225da..c4f146b4 100644 --- a/packages/llm/src/adapters/openai.ts +++ b/packages/llm/src/adapters/openai.ts @@ -42,12 +42,14 @@ import type { import { CONTEXT_SEAM_DEFAULTS, REASONING_ID, + assertListModelsShape, assertMediaCapabilities, assertNoStreamingMediaOutput, boundedListModels, decodeMediaJobId, encodeMediaJobId, isAbortSignal, + isRecord, toModelListing, } from './shared.js'; @@ -181,9 +183,6 @@ function parseToolArgs(raw: string): unknown { } } -const isRecord = (value: unknown): value is Record => - typeof value === 'object' && value !== null && !Array.isArray(value); - /** OpenAI's accepted inline-audio output formats (a closed union on the SDK param) — used to narrow a * caller-supplied `providerOptions.audio.format` to a valid value (default `wav`). */ const OPENAI_AUDIO_FORMATS = ['wav', 'aac', 'mp3', 'flac', 'opus', 'pcm16'] as const; @@ -382,10 +381,14 @@ export function openaiErrorToLlmError(err: unknown, provider: ProviderId): LlmEr // --- Live model discovery: the id-only list filter (ADR-0064 §3) ------------------------------ /** - * Id substrings that are NOT chat-completions text models — DENIED from the OpenAI/DeepSeek live list + * Id SEGMENTS that are NOT chat-completions text models — DENIED from the OpenAI/DeepSeek live list * (ADR-0064 §3). The OpenAI `/v1/models` list is id-only (no capability metadata), so the filter is an * id-family heuristic: deny wins over allow, so `gpt-image-1` / `gpt-4o-audio-preview` / `omni-moderation` - * are dropped even though they match a `gpt`/`o` allow-family. + * are dropped even though they match a `gpt`/`o` allow-family. Each token is matched on a `-`/`_` SEGMENT + * boundary (not a bare substring), so `search` denies `gpt-4o-search-preview` but NOT `o3-deep-research` + * (re**search**), and `dall-e`'s internal `-` is a literal segment. The tail entries + * (`instruct`/`ocr`/`davinci`/`babbage`) drop non-chat completion families that otherwise pass the + * gpt/deepseek allow-family; all are priced-rescue-safe (the `pricedIds.has(id)` short-circuit wins first). */ const OPENAI_DENY_SUBSTRINGS = [ 'embedding', @@ -398,8 +401,25 @@ const OPENAI_DENY_SUBSTRINGS = [ 'dall-e', 'transcribe', 'search', + 'instruct', + 'ocr', + 'davinci', + 'babbage', ] as const; +/** Escape a literal string for embedding inside a `RegExp`. The deny tokens carry no metacharacters today + * (`dall-e`'s `-` is literal outside a character class), but this keeps the boundary match safe if one is + * ever added. */ +function escapeRegExp(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +} + +/** True when a deny `token` occurs on a `-`/`_` segment boundary in `lower` — a word-boundary match, so + * `search` fires on `gpt-4o-search-preview` (`-search-`) but not `o3-deep-research` (`re`+`search`). */ +function denyTokenMatches(lower: string, token: string): boolean { + return new RegExp(`(^|[-_])${escapeRegExp(token)}([-_]|$)`).test(lower); +} + /** * The MODEL_PRICING native ids + canonical keys for one OpenAI-compatible provider — unioned into the live * list so a **cost-eligible** id ALWAYS survives the id-family heuristic (ADR-0064 §3), even if a future @@ -430,7 +450,7 @@ export function keepOpenAiModelId(id: string, pricedIds: ReadonlySet): b if (lower.startsWith('ft:')) { return false; } - if (OPENAI_DENY_SUBSTRINGS.some((deny) => lower.includes(deny))) { + if (OPENAI_DENY_SUBSTRINGS.some((deny) => denyTokenMatches(lower, deny))) { return false; } return ( @@ -1033,17 +1053,33 @@ export function createOpenAiAdapter(deps: OpenAiAdapterDeps = {}): LlmProvider { const priced = pricedModelIdsFor(providerId); const listings: ModelListing[] = []; const seen = new Set(); + let rawCount = 0; + let droppedForShape = 0; for await (const model of client.models.list({ signal: innerSignal })) { + rawCount += 1; + if (!isRecord(model)) { + droppedForShape += 1; // a non-object row — drop it, never dereference + continue; + } const id = typeof model.id === 'string' ? model.id : ''; - if (id.length === 0 || seen.has(id) || !keepOpenAiModelId(id, priced)) { + if (id.length === 0) { + droppedForShape += 1; // no usable id ⇒ shape-invalid (counts toward the §8 drift tally) continue; } + if (seen.has(id) || !keepOpenAiModelId(id, priced)) { + continue; // a duplicate or a CONTENT-filter drop — legitimate, never a shape drop + } const listing = toModelListing({ id }); - if (listing !== undefined) { - seen.add(id); - listings.push(listing); + if (listing === undefined) { + droppedForShape += 1; // rejected by the strict-outbound schema ⇒ shape-invalid + continue; } + seen.add(id); + listings.push(listing); } + // ADR-0064 §8: a systemic id-removal (rows present, none usable, some shape-broken) THROWS so the + // host's per-provider isolation shows "last-known" rather than an empty picker. + assertListModelsShape(providerId, { rawCount, kept: listings.length, droppedForShape }); return listings; }, }); diff --git a/packages/llm/src/adapters/shared.ts b/packages/llm/src/adapters/shared.ts index 1487d8a5..b8309eac 100644 --- a/packages/llm/src/adapters/shared.ts +++ b/packages/llm/src/adapters/shared.ts @@ -203,6 +203,16 @@ export function positiveModelInt(value: unknown): number | undefined { return typeof value === 'number' && Number.isInteger(value) && value > 0 ? value : undefined; } +/** + * True for a plain object (not `null`, not an array) — the per-row shape guard the model-list collect loops + * apply BEFORE dereferencing a vendor row (ADR-0064 §8, C-fix). A non-object row (e.g. a `null` in the + * vendor `data` array) is DROPPED as `droppedForShape`, never dereferenced (dereferencing it would throw a + * `TypeError` that discards the whole provider's fresh list instead of the one bad row). + */ +export function isRecord(v: unknown): v is Record { + return typeof v === 'object' && v !== null && !Array.isArray(v); +} + /** * Finalize a leniently-built candidate into a validated {@link ModelListing}, or `undefined` if it fails * the strict-outbound schema (ADR-0064 §3/§8) — the one boundary that drops a malformed / id-less row @@ -215,6 +225,31 @@ export function toModelListing(candidate: Record): ModelListing return parsed.success ? parsed.data : undefined; } +/** + * The ADR-0064 §8 systemic-drift guard, called by each adapter's `collect` AFTER it has walked every page. + * It THROWS a classified `bad_request` {@link LlmProviderError} **iff** the vendor returned rows + * (`rawCount > 0`) but NONE yielded a usable model id (`kept === 0`) AND at least one was dropped for a + * broken shape (`droppedForShape > 0`) — i.e. a breaking id-removal / every-row-shape-broken change. The + * throw (raised INSIDE the `collect` closure so {@link boundedListModels} redacts + cause-strips it) lets + * §5's per-provider refresh isolation show "last-known", not a silently-empty picker. Two non-throw cases: + * a well-formed EMPTY list (`rawCount === 0`) returns normally, and a list whose rows were all + * CONTENT-filtered but shape-valid (`droppedForShape === 0`) returns a genuine `[]`. + */ +export function assertListModelsShape( + provider: ProviderId, + counts: { rawCount: number; kept: number; droppedForShape: number }, +): void { + if (counts.rawCount > 0 && counts.kept === 0 && counts.droppedForShape > 0) { + throw new LlmProviderError( + makeLlmError({ + provider, + kind: 'bad_request', + message: 'model list shape unexpected — no usable model id on any row', + }), + ); + } +} + /** * Run a `listModels` collect bounded + abortable + secret-free (ADR-0064 §3). An internal * `AbortController` is threaded to the SDK (so a caller `signal` OR the hard timeout actually cancels the @@ -259,13 +294,18 @@ export async function boundedListModels(params: { try { return await Promise.race([collecting, timeout]); } catch (err) { + // A pre-classified error (the §8 drift throw, or any adapter-side `LlmProviderError`) passes THROUGH + // with its own `kind` — never re-run through `classify` (which would flatten a `bad_request` drift + // throw to `unknown`). It is still redacted + `cause`-stripped by the re-wrap below. const base = timedOut ? makeLlmError({ provider, kind: 'timeout', message: `model list timed out after ${String(timeoutMs)}ms`, }) - : classify(err); + : err instanceof LlmProviderError + ? err.llmError + : classify(err); // Re-wrap through makeLlmError so scrubSecrets runs again AND redactKey strips the resolved key; never // pass `cause` (it could carry the key or the raw vendor payload — ADR-0064 §3). throw new LlmProviderError( diff --git a/packages/llm/src/conformance/fixtures/anthropic.ts b/packages/llm/src/conformance/fixtures/anthropic.ts index 510038a8..8f71c872 100644 --- a/packages/llm/src/conformance/fixtures/anthropic.ts +++ b/packages/llm/src/conformance/fixtures/anthropic.ts @@ -36,6 +36,13 @@ const rateLimitError = JSON.stringify({ error: { type: 'rate_limit_error', message: 'Number of requests has exceeded your rate limit.' }, }); +// A 401 on the `/v1/models` list (ADR-0064 §3) — the SDK raises an AuthenticationError the adapter +// classifies to `auth`; boundedListModels re-wraps it key-redacted + cause-stripped. +const modelsListAuthError = JSON.stringify({ + type: 'error', + error: { type: 'authentication_error', message: 'invalid x-api-key' }, +}); + /** Build an SSE transcript from `[eventType, dataObject]` pairs in the Anthropic wire format. */ function sse(events: readonly [string, unknown][]): string { return ( @@ -310,6 +317,7 @@ export const ANTHROPIC_FIXTURES: ConformanceFixtures = { structuredOutput: { status: 200, body: structuredOutput }, listModels: { status: 200, body: modelsList }, listModelsDrift: { status: 200, body: modelsListDrift }, + listModelsError: { status: 401, body: modelsListAuthError }, toolLoop: { turn1: { status: 200, body: toolMessage }, turn2: { status: 200, body: textMessage }, @@ -334,5 +342,6 @@ export const ANTHROPIC_FIXTURES: ConformanceFixtures = { }, }, listModelsDrift: { ids: ['claude-sonnet-4-6'] }, + listModelsError: { kind: 'auth' }, }, }; diff --git a/packages/llm/src/conformance/fixtures/deepseek.ts b/packages/llm/src/conformance/fixtures/deepseek.ts index 261b8ab4..6e8594dc 100644 --- a/packages/llm/src/conformance/fixtures/deepseek.ts +++ b/packages/llm/src/conformance/fixtures/deepseek.ts @@ -175,6 +175,16 @@ const modelsList = JSON.stringify({ ], }); +// A 401 on the `/models` list (ADR-0064 §3) — the SDK raises an AuthenticationError the adapter classifies +// to `auth`; boundedListModels re-wraps it key-redacted + cause-stripped. +const modelsListAuthError = JSON.stringify({ + error: { + message: 'Authentication Fails, Your api key is invalid', + type: 'authentication_error', + code: 'invalid_request_error', + }, +}); + // The drift fixture (ADR-0064 §8): an unknown field is ignored, an id-less row dropped, never a throw. const modelsListDrift = JSON.stringify({ object: 'list', @@ -195,6 +205,7 @@ export const DEEPSEEK_FIXTURES: ConformanceFixtures = { structuredOutput: { status: 200, body: structuredOutput }, listModels: { status: 200, body: modelsList }, listModelsDrift: { status: 200, body: modelsListDrift }, + listModelsError: { status: 401, body: modelsListAuthError }, toolLoop: { turn1: { status: 200, body: toolMessage }, turn2: { status: 200, body: textMessage }, @@ -217,5 +228,6 @@ export const DEEPSEEK_FIXTURES: ConformanceFixtures = { structuredOutput: { text: '{"ok":true}' }, listModels: { ids: ['deepseek-chat', 'deepseek-reasoner'], sample: { id: 'deepseek-chat' } }, listModelsDrift: { ids: ['deepseek-chat'] }, + listModelsError: { kind: 'auth' }, }, }; diff --git a/packages/llm/src/conformance/fixtures/gemini.ts b/packages/llm/src/conformance/fixtures/gemini.ts index 3be08e4e..661e5d28 100644 --- a/packages/llm/src/conformance/fixtures/gemini.ts +++ b/packages/llm/src/conformance/fixtures/gemini.ts @@ -123,6 +123,13 @@ const modelsList = JSON.stringify([ }, ]); +// A 401 on `models.list()` (ADR-0064 §3) — the fake transport rejects with an ApiError-shaped error (a +// numeric `status`) the adapter classifies to `auth`; boundedListModels re-wraps it key-redacted + +// cause-stripped. The body is diagnostic only (the fake transport keys off `status`). +const modelsListAuthError = JSON.stringify({ + error: { code: 401, message: 'API key not valid', status: 'UNAUTHENTICATED' }, +}); + // The drift fixture (ADR-0064 §8): one row carries an unknown future field (ignored); one chat-capable row // has NO `name` (→ no id → dropped at the mapper boundary), never a throw. const modelsListDrift = JSON.stringify([ @@ -154,6 +161,7 @@ export const GEMINI_FIXTURES: ConformanceFixtures = { mediaGenerate: { status: 200, body: imageGenerate }, listModels: { status: 200, body: modelsList }, listModelsDrift: { status: 200, body: modelsListDrift }, + listModelsError: { status: 401, body: modelsListAuthError }, toolLoop: { turn1: { status: 200, body: toolResponse }, turn2: { status: 200, body: textResponse }, @@ -179,5 +187,6 @@ export const GEMINI_FIXTURES: ConformanceFixtures = { }, }, listModelsDrift: { ids: ['gemini-2.5-flash'] }, + listModelsError: { kind: 'auth' }, }, }; diff --git a/packages/llm/src/conformance/fixtures/openai.ts b/packages/llm/src/conformance/fixtures/openai.ts index a7d8be66..289ea1b0 100644 --- a/packages/llm/src/conformance/fixtures/openai.ts +++ b/packages/llm/src/conformance/fixtures/openai.ts @@ -176,6 +176,16 @@ const modelsList = JSON.stringify({ ], }); +// A 401 on the `/v1/models` list (ADR-0064 §3) — the SDK raises an AuthenticationError the adapter +// classifies to `auth`; boundedListModels re-wraps it key-redacted + cause-stripped. +const modelsListAuthError = JSON.stringify({ + error: { + message: 'Incorrect API key provided', + type: 'invalid_request_error', + code: 'invalid_api_key', + }, +}); + // The drift fixture (ADR-0064 §8): one row carries an unknown future field (ignored), one row has NO id // (dropped) — the call resolves, never throws. const modelsListDrift = JSON.stringify({ @@ -205,6 +215,7 @@ export const OPENAI_FIXTURES: ConformanceFixtures = { mediaGenerate: { status: 200, body: imageGenerate }, listModels: { status: 200, body: modelsList }, listModelsDrift: { status: 200, body: modelsListDrift }, + listModelsError: { status: 401, body: modelsListAuthError }, toolLoop: { turn1: { status: 200, body: toolMessage }, turn2: { status: 200, body: textMessage }, @@ -221,5 +232,6 @@ export const OPENAI_FIXTURES: ConformanceFixtures = { // id-only list: the three chat families survive; the sample carries only `id` (no context/price). listModels: { ids: ['gpt-5.5', 'gpt-5.4-mini', 'o3'], sample: { id: 'gpt-5.5' } }, listModelsDrift: { ids: ['gpt-5.5'] }, + listModelsError: { kind: 'auth' }, }, }; diff --git a/packages/llm/src/conformance/spec.ts b/packages/llm/src/conformance/spec.ts index 189f4c7d..8b8e3456 100644 --- a/packages/llm/src/conformance/spec.ts +++ b/packages/llm/src/conformance/spec.ts @@ -69,6 +69,9 @@ export interface ConformanceExpectations { /** The ids a `listModelsDrift` reply should yield (ADR-0064 §8) — the id-less row dropped, the unknown * extra field ignored, never a throw. */ readonly listModelsDrift?: { readonly ids: readonly string[] }; + /** The classified kind a failing `listModels` call should reject with (ADR-0064 §3) — providers with a + * `listModelsError` fixture supply it (e.g. `auth` for a 401). */ + readonly listModelsError?: { readonly kind: LlmErrorKind }; } /** The recorded provider responses a conformance run needs — one per canonical scenario. */ @@ -100,6 +103,9 @@ export interface ConformanceFixtures { /** A recorded `models.list()` reply carrying an UNKNOWN extra field and an id-less row (ADR-0064 §8) — the * drift-resilience fixture: the extra field is ignored, the id-less row dropped, never a throw. */ readonly listModelsDrift?: RecordedResponse; + /** A recorded `models.list()` FAILURE reply (e.g. a 401/500 error body) — the error-path fixture: the call + * must reject with a classified, key-redacted `LlmProviderError` (ADR-0064 §3). Omit if not recorded. */ + readonly listModelsError?: RecordedResponse; /** * A multi-turn tool loop (the path every agent node exercises): `turn1` is a tool-call reply; `turn2` is * the continuation the provider returns AFTER the caller appends the tool result. The conformance test @@ -487,5 +493,34 @@ export function defineConformanceSuite( } }, ); + + it.skipIf(fixtures.listModelsError === undefined)( + 'listModels (error): a failing list rejects with a classified, key-redacted LlmProviderError (ADR-0064 §3)', + async () => { + const recorded = fixtures.listModelsError; + const exp = expected.listModelsError; + if (recorded === undefined || exp === undefined) { + return; // narrow for skipIf + } + const adapter = makeReplayAdapter(recorded); + if (adapter.listModels === undefined) { + throw new Error('a listModelsError fixture requires the adapter to implement listModels'); + } + let caught: unknown; + try { + await adapter.listModels(KEY); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(LlmProviderError); + if (caught instanceof LlmProviderError) { + expect(caught.llmError.provider).toBe(name); + expect(caught.llmError.kind).toBe(exp.kind); + // Redacted + cause-stripped: neither the resolved key nor a raw vendor payload crosses the seam. + expect(caught.llmError.message).not.toContain(KEY); + expect(caught.llmError.cause).toBeUndefined(); + } + }, + ); }); } From 2101ee84d5d88a477fcc200b9c05bfc3efe96607 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 19:52:12 +0300 Subject: [PATCH 05/68] =?UTF-8?q?fix(llm):=202.5.G=20S2=20Sonnet-review=20?= =?UTF-8?q?=E2=80=94=20assertListModelsShape=20guard=20+=20code=20redactio?= =?UTF-8?q?n=20+=20test=20coverage?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sonnet's adversarial round found a real logic bug the Opus fix introduced, plus a redaction asymmetry and several coverage gaps (6 confirmed + 1 plausible, 0 refuted). Code fixes (shared.ts): - assertListModelsShape: the systemic-drift throw conflated a legitimately content-filter-empty list with ONE unrelated shape-broken row (rawCount=3, kept=0, droppedForShape=1 -> false bad_request for a narrowly-scoped key). Guard tightened from `droppedForShape > 0` to `droppedForShape === rawCount` — throw only when EVERY row was shape-broken. (When kept===0, dedup is impossible, so === rawCount is exactly "no row survived content-filtering", the true drift signal.) A content-filtered-empty list, with or without an unrelated bad row, now correctly returns []. - boundedListModels re-wrap now redacts the `code` field too (redactKey(base.code, key)), closing the message-only asymmetry (defense-in-depth). Test coverage (list-models.test.ts + conformance): - The mixed-case regression (rawCount=3,kept=0,droppedForShape=1 -> no throw), unit + end-to-end via the OpenAI adapter, proving the guard fix. - §8 systemic-drift throw now exercised end-to-end for OpenAI + Gemini (was Anthropic-only). - isRecord null-row drop exercised for OpenAI + Gemini. - Duplicate-id dedup pinned for all three adapters. - Top-level envelope-drift ({}, {data:'not-an-array'}, non-array) -> [] or classified reject, never a raw TypeError. - The conformance listModelsError redaction assertion is no longer vacuous: the OpenAI modelsListAuthError fixture body now embeds 'conformance-test-key', which the classifier surfaces into the message, so .not.toContain(KEY) actually exercises redactKey. Verified: pnpm turbo run lint typecheck test build — 24/24 green (llm 498 pass, list-models.test.ts 23->34); seam fence clean; no node:* in packages/llm/src; prettier clean. Refs: ADR-0064 Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/llm/src/adapters/list-models.test.ts | 230 +++++++++++++++++- packages/llm/src/adapters/shared.ts | 20 +- .../llm/src/conformance/fixtures/openai.ts | 8 +- 3 files changed, 243 insertions(+), 15 deletions(-) diff --git a/packages/llm/src/adapters/list-models.test.ts b/packages/llm/src/adapters/list-models.test.ts index 2ccae6bd..eb48fe80 100644 --- a/packages/llm/src/adapters/list-models.test.ts +++ b/packages/llm/src/adapters/list-models.test.ts @@ -4,8 +4,13 @@ import { LlmProviderError, makeLlmError } from '../llm-error.js'; import type { ModelListing } from '../types.js'; import { createAnthropicAdapter, mapAnthropicModel } from './anthropic.js'; -import { mapGeminiModel } from './gemini.js'; -import { keepOpenAiModelId, pricedModelIdsFor } from './openai.js'; +import { + createGeminiAdapter, + mapGeminiModel, + type GeminiModelInfo, + type GeminiTransport, +} from './gemini.js'; +import { createOpenAiAdapter, keepOpenAiModelId, pricedModelIdsFor } from './openai.js'; import { assertListModelsShape, boundedListModels, @@ -203,8 +208,9 @@ describe('Anthropic listModels — pagination', () => { }); describe('assertListModelsShape (ADR-0064 §8)', () => { - it('throws a bad_request LlmProviderError only when rows exist, none are kept, and some broke shape', () => { - // The one throwing case: a systemic id-removal (every row shape-broken). + it('throws a bad_request LlmProviderError only when rows exist, none are kept, and EVERY row broke shape', () => { + // The one throwing case: a systemic id-removal where EVERY row was shape-broken + // (droppedForShape === rawCount) — none was merely content-filtered. let caught: unknown; try { assertListModelsShape('anthropic', { rawCount: 2, kept: 0, droppedForShape: 2 }); @@ -218,7 +224,7 @@ describe('assertListModelsShape (ADR-0064 §8)', () => { } }); - it('returns normally for an empty list, an all-content-filtered list, and a partial-drop list', () => { + it('returns normally for an empty list, an all-content-filtered list, a partial-drop list, and the MIXED case', () => { // A well-formed EMPTY list (no rows) is a genuine [] — never a throw. expect(() => assertListModelsShape('openai', { rawCount: 0, kept: 0, droppedForShape: 0 }), @@ -231,6 +237,13 @@ describe('assertListModelsShape (ADR-0064 §8)', () => { expect(() => assertListModelsShape('gemini', { rawCount: 3, kept: 1, droppedForShape: 2 }), ).not.toThrow(); + // MIXED (the guard-fix regression, ADR-0064 §8): empty because of content-filtering PLUS one unrelated + // shape-broken row (0 < droppedForShape < rawCount, kept 0) — the emptiness is explained by the filter, + // not a systemic id-removal, so it must NOT throw. The old `droppedForShape > 0` guard wrongly tripped + // here, producing a false bad_request; `droppedForShape === rawCount` is the correct drift signal. + expect(() => + assertListModelsShape('openai', { rawCount: 3, kept: 0, droppedForShape: 1 }), + ).not.toThrow(); }); }); @@ -295,6 +308,213 @@ describe('listModels — per-row drop + systemic drift (ADR-0064 §8, FIX C/D)', }); }); +// --- Cross-adapter end-to-end §8 coverage (OpenAI/DeepSeek fetch-replay + a hand-built Gemini transport), +// mirroring the Anthropic FIX C/D tests above through the other two providers. -------------------------- + +/** Serve one 200 JSON response through an injected fetch — used for BOTH a well-formed page and a + * structurally-broken top-level envelope (T5), so the body is an arbitrary value. */ +function jsonResponseFetch(body: unknown): () => Promise { + const text = JSON.stringify(body); + return () => + Promise.resolve( + new Response(text, { status: 200, headers: { 'content-type': 'application/json' } }), + ); +} + +/** A single-page OpenAI/DeepSeek `/v1/models` body from raw row objects (a row may be `null` / id-less to + * exercise the per-row drop + the §8 tally). Serves one 200 response through the adapter's injected fetch. */ +function openaiModelsFetch(rows: readonly unknown[]): () => Promise { + return jsonResponseFetch({ object: 'list', data: rows }); +} + +/** A GeminiTransport whose `listModels` resolves the given (deliberately-malformed) rows; every other arm + * rejects (unused by these list tests). `rows` is typed `unknown` so a test can hand it a non-array (T5) or + * an array carrying a `null` row (T3) — the adapter collect's own isRecord/array guards are what's exercised. + * The transport contract is `GeminiModelInfo[]`, so a single controlled cast crosses the type on purpose. */ +function geminiListTransport(rows: unknown): GeminiTransport { + return { + generate: () => Promise.reject(new Error('unused')), + stream: () => Promise.reject(new Error('unused')), + generateImages: () => Promise.reject(new Error('unused')), + generateVideos: () => Promise.reject(new Error('unused')), + pollVideo: () => Promise.reject(new Error('unused')), + listModels: () => Promise.resolve(rows as GeminiModelInfo[]), + }; +} + +/** Assert the ONLY two acceptable outcomes for a malformed top-level envelope (ADR-0064 §8, T5): a genuine + * `[]`, or a rejection that is a CLASSIFIED `LlmProviderError` — never a raw TypeError, an unclassified + * throw, or a hang (a hang trips vitest's per-test timeout). */ +async function expectEmptyOrClassifiedReject(run: () => Promise): Promise { + let result: ModelListing[] | undefined; + let caught: unknown; + try { + result = await run(); + } catch (err) { + caught = err; + } + if (caught !== undefined) { + expect(caught).toBeInstanceOf(LlmProviderError); + } else { + expect(result).toEqual([]); + } +} + +describe('listModels — MIXED content-filter + one shape-broken row resolves [] (ADR-0064 §8, guard-fix)', () => { + it('OpenAI: two content-filtered valid rows + one id-less row → resolves [], never a false bad_request', async () => { + // The EXACT scenario the old `droppedForShape > 0` guard turned into a false bad_request: the list is + // empty because both real rows were CONTENT-filtered (embedding + whisper), not because of drift, and one + // unrelated id-less row is present. rawCount 3, kept 0, droppedForShape 1 (0 < 1 < 3) → resolves []. + const adapter = createOpenAiAdapter({ + fetch: openaiModelsFetch([ + { id: 'text-embedding-3-large', object: 'model' }, // valid id, content-filtered (embedding) + { id: 'whisper-1', object: 'model' }, // valid id, content-filtered (whisper) + { owned_by: 'openai', object: 'model' }, // id-less → the lone droppedForShape + ]), + maxRetries: 0, + }); + const listings = await (adapter.listModels?.('key') ?? Promise.resolve([])); + expect(listings).toEqual([]); + }); +}); + +describe('listModels — systemic-drift throw end-to-end (ADR-0064 §8, mirrors Anthropic FIX D)', () => { + it('OpenAI: every data row id-less → rejects bad_request, key absent from the message', async () => { + const key = 'sk-live-openai-abcdefghijklmnop'; + const adapter = createOpenAiAdapter({ + // Two id-less rows → rawCount 2, kept 0, droppedForShape 2 → the systemic-drift throw. + fetch: openaiModelsFetch([{ owned_by: 'x' }, { created: 1 }]), + maxRetries: 0, + }); + let caught: unknown; + try { + await (adapter.listModels?.(key) ?? Promise.resolve([])); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(LlmProviderError); + if (caught instanceof LlmProviderError) { + expect(caught.llmError.kind).toBe('bad_request'); + expect(caught.llmError.message).not.toContain(key); + expect(caught.llmError.cause).toBeUndefined(); + } + }); + + it('Gemini: every chat-capable row lacks a usable name → rejects bad_request, key absent', async () => { + const key = 'AIza-gemini-secret-abcdefghijklmnop'; + const adapter = createGeminiAdapter({ + // Both rows are chat-capable (generateContent) but id-less → droppedForShape 2, kept 0 → the throw. + transport: geminiListTransport([ + { supportedActions: ['generateContent'] }, + { name: '', supportedActions: ['generateContent'] }, + ]), + }); + let caught: unknown; + try { + await (adapter.listModels?.(key) ?? Promise.resolve([])); + } catch (err) { + caught = err; + } + expect(caught).toBeInstanceOf(LlmProviderError); + if (caught instanceof LlmProviderError) { + expect(caught.llmError.kind).toBe('bad_request'); + expect(caught.llmError.message).not.toContain(key); + expect(caught.llmError.cause).toBeUndefined(); + } + }); +}); + +describe('listModels — a null row is dropped, the valid rows survive (ADR-0064 §8, mirrors Anthropic FIX C)', () => { + it('OpenAI: [valid, null] resolves with the one valid ModelListing', async () => { + const adapter = createOpenAiAdapter({ + fetch: openaiModelsFetch([{ id: 'gpt-5.5', object: 'model' }, null]), + maxRetries: 0, + }); + const listings = await (adapter.listModels?.('key') ?? Promise.resolve([])); + expect(listings.map((l) => l.id)).toEqual(['gpt-5.5']); + }); + + it('Gemini: [valid, null] resolves with the one valid ModelListing', async () => { + const adapter = createGeminiAdapter({ + transport: geminiListTransport([ + { name: 'models/gemini-2.5-flash', supportedActions: ['generateContent'] }, + null, + ]), + }); + const listings = await (adapter.listModels?.('key') ?? Promise.resolve([])); + expect(listings.map((l) => l.id)).toEqual(['gemini-2.5-flash']); + }); +}); + +describe('listModels — a duplicate id is deduped to a single ModelListing (ADR-0064 §8, seen/dedup pin)', () => { + it('Anthropic: two paginator pages both yielding id m1 → the id appears once', async () => { + const bodies = [page('m1', true), page('m1', false)]; + let call = 0; + const fetchSeq = (): Promise => { + const body = bodies[Math.min(call, bodies.length - 1)]; + call += 1; + return Promise.resolve( + new Response(body, { status: 200, headers: { 'content-type': 'application/json' } }), + ); + }; + const adapter = createAnthropicAdapter({ fetch: fetchSeq, maxRetries: 0 }); + const listings = await (adapter.listModels?.('key') ?? Promise.resolve([])); + expect(listings.map((l) => l.id)).toEqual(['m1']); + }); + + it('OpenAI: a data array repeating an id → the id appears once', async () => { + const adapter = createOpenAiAdapter({ + fetch: openaiModelsFetch([ + { id: 'gpt-5.5', object: 'model' }, + { id: 'gpt-5.5', object: 'model' }, + ]), + maxRetries: 0, + }); + const listings = await (adapter.listModels?.('key') ?? Promise.resolve([])); + expect(listings.map((l) => l.id)).toEqual(['gpt-5.5']); + }); + + it('Gemini: two rows with the same name → the id appears once', async () => { + const adapter = createGeminiAdapter({ + transport: geminiListTransport([ + { name: 'models/gemini-2.5-flash', supportedActions: ['generateContent'] }, + { name: 'models/gemini-2.5-flash', supportedActions: ['generateContent'] }, + ]), + }); + const listings = await (adapter.listModels?.('key') ?? Promise.resolve([])); + expect(listings.map((l) => l.id)).toEqual(['gemini-2.5-flash']); + }); +}); + +describe('listModels — a broken top-level envelope resolves [] or a classified error, never a raw throw (ADR-0064 §8, T5)', () => { + it('Anthropic: an empty-object body and a non-array data are each classified or empty', async () => { + for (const body of [{}, { data: 'not-an-array' }]) { + const adapter = createAnthropicAdapter({ fetch: jsonResponseFetch(body), maxRetries: 0 }); + await expectEmptyOrClassifiedReject(() => adapter.listModels?.('key') ?? Promise.resolve([])); + } + }); + + it('OpenAI + DeepSeek: an empty-object body and a non-array data are each classified or empty', async () => { + for (const providerId of ['openai', 'deepseek'] as const) { + for (const body of [{}, { data: 'not-an-array' }]) { + const adapter = createOpenAiAdapter({ + providerId, + fetch: jsonResponseFetch(body), + maxRetries: 0, + }); + await expectEmptyOrClassifiedReject( + () => adapter.listModels?.('key') ?? Promise.resolve([]), + ); + } + } + }); + + it('Gemini: a transport resolving a non-array is classified or empty', async () => { + const adapter = createGeminiAdapter({ transport: geminiListTransport({}) }); + await expectEmptyOrClassifiedReject(() => adapter.listModels?.('key') ?? Promise.resolve([])); + }); +}); + describe('boundedListModels', () => { it('rejects with a classified timeout when collect never settles', async () => { const result = boundedListModels({ diff --git a/packages/llm/src/adapters/shared.ts b/packages/llm/src/adapters/shared.ts index b8309eac..ed652c91 100644 --- a/packages/llm/src/adapters/shared.ts +++ b/packages/llm/src/adapters/shared.ts @@ -228,18 +228,22 @@ export function toModelListing(candidate: Record): ModelListing /** * The ADR-0064 §8 systemic-drift guard, called by each adapter's `collect` AFTER it has walked every page. * It THROWS a classified `bad_request` {@link LlmProviderError} **iff** the vendor returned rows - * (`rawCount > 0`) but NONE yielded a usable model id (`kept === 0`) AND at least one was dropped for a - * broken shape (`droppedForShape > 0`) — i.e. a breaking id-removal / every-row-shape-broken change. The - * throw (raised INSIDE the `collect` closure so {@link boundedListModels} redacts + cause-strips it) lets - * §5's per-provider refresh isolation show "last-known", not a silently-empty picker. Two non-throw cases: - * a well-formed EMPTY list (`rawCount === 0`) returns normally, and a list whose rows were all - * CONTENT-filtered but shape-valid (`droppedForShape === 0`) returns a genuine `[]`. + * (`rawCount > 0`) but NONE yielded a usable model id (`kept === 0`) AND **every** row was dropped for a + * broken shape (`droppedForShape === rawCount`, i.e. none was merely content-filtered) — a breaking + * id-removal / every-row-shape-broken change. The throw (raised INSIDE the `collect` closure so + * {@link boundedListModels} redacts + cause-strips it) lets §5's per-provider refresh isolation show + * "last-known", not a silently-empty picker. Non-throw cases (all return a genuine `[]`): a well-formed EMPTY + * list (`rawCount === 0`); a list whose rows were all CONTENT-filtered but shape-valid (`droppedForShape === 0`); + * and — the case a `droppedForShape > 0` guard would wrongly trip — a list empty because of content-filtering + * PLUS one unrelated shape-broken row (`0 < droppedForShape < rawCount`), where the emptiness is explained by + * the filter, not a systemic break. (When `kept === 0`, no row was deduped — dedup needs a kept row — so + * `droppedForShape === rawCount` is exactly "no row survived content-filtering", the true drift signal.) */ export function assertListModelsShape( provider: ProviderId, counts: { rawCount: number; kept: number; droppedForShape: number }, ): void { - if (counts.rawCount > 0 && counts.kept === 0 && counts.droppedForShape > 0) { + if (counts.rawCount > 0 && counts.kept === 0 && counts.droppedForShape === counts.rawCount) { throw new LlmProviderError( makeLlmError({ provider, @@ -313,7 +317,7 @@ export async function boundedListModels(params: { provider, kind: base.kind, message: redactKey(base.message, key), - ...(base.code !== undefined ? { code: base.code } : {}), + ...(base.code !== undefined ? { code: redactKey(base.code, key) } : {}), ...(base.status !== undefined ? { status: base.status } : {}), }), ); diff --git a/packages/llm/src/conformance/fixtures/openai.ts b/packages/llm/src/conformance/fixtures/openai.ts index 289ea1b0..c666d36f 100644 --- a/packages/llm/src/conformance/fixtures/openai.ts +++ b/packages/llm/src/conformance/fixtures/openai.ts @@ -177,10 +177,14 @@ const modelsList = JSON.stringify({ }); // A 401 on the `/v1/models` list (ADR-0064 §3) — the SDK raises an AuthenticationError the adapter -// classifies to `auth`; boundedListModels re-wraps it key-redacted + cause-stripped. +// classifies to `auth`; boundedListModels re-wraps it key-redacted + cause-stripped. The body `message` +// deliberately embeds the conformance API key (`conformance-test-key`) so the shared spec's +// `.not.toContain(KEY)` assertion actually EXERCISES `redactKey`: the OpenAI SDK folds the body message into +// `APIError.message` (`${status} ${error.message}`), the classifier surfaces it as `llmError.message`, and +// only `redactKey` in `boundedListModels` masks it — a vacuous key-free body would pass the assert trivially. const modelsListAuthError = JSON.stringify({ error: { - message: 'Incorrect API key provided', + message: 'Incorrect API key provided: conformance-test-key', type: 'invalid_request_error', code: 'invalid_api_key', }, From 4b9890176fdb90d87c51b91565243e6c3eaab6dc Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 19:59:04 +0300 Subject: [PATCH 06/68] =?UTF-8?q?feat(llm):=202.5.G=20S3=20=E2=80=94=20pur?= =?UTF-8?q?e=20static/live/user=20model-catalog=20merge=20helper=20(ADR-00?= =?UTF-8?q?64=20=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pure, I/O-free reconciliation of live discovery, the static registry, and the optional user-pricing tier into one deterministically-ordered ModelCatalogEntry[], reused by every surface (CLI picker, desktop, VS Code). - packages/llm/src/model-catalog.ts: mergeModelCatalog(input) + ModelCatalogEntry + MergeModelCatalogInput + PricingSource. Per-field precedence (ADR-0064 §6): * availability <- live-list membership when a provider has live data (a static model absent from the key's live list is dimmed, the K2 decision); a provider with NO live data falls back to static presence (never "everything unavailable"). * price <- registry ?? user (static wins for a known id; user fills an unknown one; live is NEVER a pricing authority). pricingSource registry|user|none + priceKnown. * context/output <- live ?? static ?? user (live is fresher when present). * deprecation <- the earlier of the static and live ISO dates (their union); flagged only once now >= that date; the caller passes `now` so the merge stays pure. * deterministic order: provider (LLM_PROVIDERS order) -> displayName -> modelId. - pricing.ts: ModelPricing gains an optional deprecatedAt (ISO, §7); the DeepSeek legacy aliases (deepseek-chat/-reasoner) now carry deprecatedAt 2026-07-24T15:59:00Z. - index.ts: exports mergeModelCatalog + the types. - model-catalog.test.ts: 13 unit tests — every precedence rule, dimmed-unavailable, empty-live-list, live-only-unpriced, registry-wins/user-fills, live-context-wins, deprecation flag + union + unparseable-date, deterministic ordering, no MODEL_PRICING mutation. Verified: pnpm turbo run lint typecheck test build — 24/24 green; prettier clean. Refs: ADR-0064, ADR-0065 Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/llm/src/index.ts | 3 + packages/llm/src/model-catalog.test.ts | 200 +++++++++++++++++++++++++ packages/llm/src/model-catalog.ts | 153 +++++++++++++++++++ packages/llm/src/pricing.ts | 9 ++ 4 files changed, 365 insertions(+) create mode 100644 packages/llm/src/model-catalog.test.ts create mode 100644 packages/llm/src/model-catalog.ts diff --git a/packages/llm/src/index.ts b/packages/llm/src/index.ts index 604a7fd3..cb503f0b 100644 --- a/packages/llm/src/index.ts +++ b/packages/llm/src/index.ts @@ -131,6 +131,9 @@ export { // CostTracker + the canonical model-pricing table (1.B). export { MODEL_PRICING, KNOWN_MODEL_IDS, contextWindowForModel } from './pricing.js'; export type { ModelPricing, CanonicalModelId } from './pricing.js'; +// The pure live/static/user merge helper (ADR-0064 §6) — reused by every surface's model catalog / picker. +export { mergeModelCatalog } from './model-catalog.js'; +export type { ModelCatalogEntry, MergeModelCatalogInput, PricingSource } from './model-catalog.js'; export { priceModel, cost, mediaCost, CostTracker } from './cost-tracker.js'; export type { CostUpdate } from './cost-tracker.js'; export { estimateMaxNextCost, estimateMediaCost } from './budget-estimator.js'; diff --git a/packages/llm/src/model-catalog.test.ts b/packages/llm/src/model-catalog.test.ts new file mode 100644 index 00000000..d36fa520 --- /dev/null +++ b/packages/llm/src/model-catalog.test.ts @@ -0,0 +1,200 @@ +import { describe, expect, it } from 'vitest'; + +import { mergeModelCatalog, type ModelCatalogEntry } from './model-catalog.js'; +import { MODEL_PRICING, type ModelPricing } from './pricing.js'; +import type { ModelListing, ProviderId } from './types.js'; + +// A fixed clock so the deprecation check is deterministic. deepseek-chat/-reasoner deprecate 2026-07-24 15:59Z. +const BEFORE_DEEPSEEK_DEPRECATION = Date.parse('2026-07-05T00:00:00Z'); +const AFTER_DEEPSEEK_DEPRECATION = Date.parse('2026-08-01T00:00:00Z'); + +const byId = (entries: readonly ModelCatalogEntry[], id: string): ModelCatalogEntry | undefined => + entries.find((e) => e.modelId === id); + +const liveMap = ( + rows: ReadonlyArray, +): ReadonlyMap => new Map(rows); + +/** A minimal user-supplied ModelPricing for an id absent from MODEL_PRICING (ADR-0065 USER tier). */ +const userPricing = (provider: ProviderId): ModelPricing => ({ + provider, + nativeId: 'x', + displayName: 'Custom Model', + contextWindowTokens: 128_000, + maxOutputTokens: 8_000, + inputPerMtokMicrocents: 100, + outputPerMtokMicrocents: 200, + cachedInputPerMtokMicrocents: 10, +}); + +describe('mergeModelCatalog (ADR-0064 §6)', () => { + it('with no live/user data, surfaces every static model as registry-priced and available (static presence)', () => { + const entries = mergeModelCatalog({ now: BEFORE_DEEPSEEK_DEPRECATION }); + expect(entries.length).toBe(Object.keys(MODEL_PRICING).length); + const opus = byId(entries, 'claude-opus-4-8'); + expect(opus).toMatchObject({ + provider: 'anthropic', + displayName: 'Claude Opus 4.8', + pricingSource: 'registry', + priceKnown: true, + available: true, // no live data for anthropic -> static presence + deprecated: false, + }); + expect(opus?.pricing).toBe(MODEL_PRICING['claude-opus-4-8']); + expect(opus?.contextWindowTokens).toBe(1_000_000); + }); + + it('availability: a static model NOT in a CONNECTED provider live list is dimmed, one present is available', () => { + const entries = mergeModelCatalog({ + live: liveMap([['anthropic', [{ id: 'claude-opus-4-8' }]]]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + expect(byId(entries, 'claude-opus-4-8')?.available).toBe(true); + // sonnet is in the registry but NOT in the live list -> dimmed + expect(byId(entries, 'claude-sonnet-4-6')?.available).toBe(false); + // a provider with NO live data (openai absent from the map) -> static presence + expect(byId(entries, 'gpt-5.5')?.available).toBe(true); + }); + + it('an EMPTY live list for a provider dims all that provider’s static models', () => { + const entries = mergeModelCatalog({ + live: liveMap([['anthropic', []]]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + expect(byId(entries, 'claude-opus-4-8')?.available).toBe(false); + expect(byId(entries, 'claude-haiku-4-5')?.available).toBe(false); + // deepseek not in the map -> static presence + expect(byId(entries, 'deepseek-v4-flash')?.available).toBe(true); + }); + + it('surfaces a LIVE-only model (absent from the registry) as selectable-but-unpriced', () => { + const entries = mergeModelCatalog({ + live: liveMap([['openai', [{ id: 'gpt-6-preview', displayName: 'GPT-6 preview' }]]]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + const fresh = byId(entries, 'gpt-6-preview'); + expect(fresh).toMatchObject({ + provider: 'openai', + displayName: 'GPT-6 preview', + pricingSource: 'none', + priceKnown: false, + available: true, + }); + expect(fresh?.pricing).toBeUndefined(); + }); + + it('a live-only model with no displayName falls back to the model id', () => { + const entries = mergeModelCatalog({ + live: liveMap([['openai', [{ id: 'gpt-6-preview' }]]]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + expect(byId(entries, 'gpt-6-preview')?.displayName).toBe('gpt-6-preview'); + }); + + it('price precedence: the static registry WINS for a known id even when user pricing is supplied', () => { + const entries = mergeModelCatalog({ + userPricing: new Map([['claude-opus-4-8', userPricing('anthropic')]]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + const opus = byId(entries, 'claude-opus-4-8'); + expect(opus?.pricingSource).toBe('registry'); + expect(opus?.pricing).toBe(MODEL_PRICING['claude-opus-4-8']); // NOT the user object + }); + + it('price precedence: the USER tier fills an UNKNOWN id', () => { + const custom = userPricing('openai'); + const entries = mergeModelCatalog({ + userPricing: new Map([['my-custom-model', custom]]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + const entry = byId(entries, 'my-custom-model'); + expect(entry).toMatchObject({ provider: 'openai', pricingSource: 'user', priceKnown: true }); + expect(entry?.pricing).toBe(custom); + }); + + it('context/output: the LIVE value wins over the static one when present, else static', () => { + const entries = mergeModelCatalog({ + live: liveMap([['anthropic', [{ id: 'claude-opus-4-8', contextWindowTokens: 500_000 }]]]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + const opus = byId(entries, 'claude-opus-4-8'); + expect(opus?.contextWindowTokens).toBe(500_000); // live wins + expect(opus?.maxOutputTokens).toBe(MODEL_PRICING['claude-opus-4-8'].maxOutputTokens); // no live -> static + }); + + it('deprecation: a static deprecatedAt flags the model only once now >= the date', () => { + const before = mergeModelCatalog({ now: BEFORE_DEEPSEEK_DEPRECATION }); + expect(byId(before, 'deepseek-chat')?.deprecated).toBe(false); + expect(byId(before, 'deepseek-chat')?.deprecatedAt).toBe('2026-07-24T15:59:00Z'); + + const after = mergeModelCatalog({ now: AFTER_DEEPSEEK_DEPRECATION }); + expect(byId(after, 'deepseek-chat')?.deprecated).toBe(true); + // a non-deprecated model stays clear + expect(byId(after, 'deepseek-v4-flash')?.deprecated).toBe(false); + }); + + it('deprecation is a UNION: the EARLIER of the static and live dates is effective', () => { + const entries = mergeModelCatalog({ + // deepseek-v4-flash has no static deprecation; a live list marks it deprecated earlier + live: liveMap([ + ['deepseek', [{ id: 'deepseek-v4-flash', deprecatedAt: '2026-07-01T00:00:00Z' }]], + ]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + const flash = byId(entries, 'deepseek-v4-flash'); + expect(flash?.deprecatedAt).toBe('2026-07-01T00:00:00Z'); + expect(flash?.deprecated).toBe(true); // now (07-05) >= live date (07-01) + + // when both are present, the earlier wins + const both = mergeModelCatalog({ + live: liveMap([ + ['deepseek', [{ id: 'deepseek-chat', deprecatedAt: '2027-01-01T00:00:00Z' }]], + ]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + expect(byId(both, 'deepseek-chat')?.deprecatedAt).toBe('2026-07-24T15:59:00Z'); // static earlier than live + }); + + it('an unparseable deprecatedAt is treated as not-deprecated (never throws)', () => { + const entries = mergeModelCatalog({ + live: liveMap([['openai', [{ id: 'weird', deprecatedAt: 'not-a-date' }]]]), + now: AFTER_DEEPSEEK_DEPRECATION, + }); + const weird = byId(entries, 'weird'); + expect(weird?.deprecated).toBe(false); + expect(weird?.deprecatedAt).toBeUndefined(); + }); + + it('is deterministically ordered: provider (seam order) then displayName then id', () => { + const entries = mergeModelCatalog({ + live: liveMap([['openai', [{ id: 'gpt-6-preview' }]]]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + // provider order is anthropic < openai < gemini < deepseek + const providers = entries.map((e) => e.provider); + const firstOpenAi = providers.indexOf('openai'); + const lastAnthropic = providers.lastIndexOf('anthropic'); + const firstGemini = providers.indexOf('gemini'); + expect(lastAnthropic).toBeLessThan(firstOpenAi); + expect(firstOpenAi).toBeLessThan(firstGemini); + // within a provider, entries are displayName-then-id sorted (stable, no duplicates) + const anthropic = entries.filter((e) => e.provider === 'anthropic').map((e) => e.displayName); + expect(anthropic).toEqual([...anthropic].sort((a, b) => a.localeCompare(b))); + // two runs produce byte-identical ordering + const again = mergeModelCatalog({ + live: liveMap([['openai', [{ id: 'gpt-6-preview' }]]]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + expect(entries.map((e) => e.modelId)).toEqual(again.map((e) => e.modelId)); + }); + + it('does not mutate MODEL_PRICING', () => { + const snapshot = JSON.stringify(MODEL_PRICING); + mergeModelCatalog({ + live: liveMap([['anthropic', [{ id: 'claude-opus-4-8', contextWindowTokens: 1 }]]]), + userPricing: new Map([['x', userPricing('openai')]]), + now: AFTER_DEEPSEEK_DEPRECATION, + }); + expect(JSON.stringify(MODEL_PRICING)).toBe(snapshot); + }); +}); diff --git a/packages/llm/src/model-catalog.ts b/packages/llm/src/model-catalog.ts new file mode 100644 index 00000000..f1514553 --- /dev/null +++ b/packages/llm/src/model-catalog.ts @@ -0,0 +1,153 @@ +import { LLM_PROVIDERS } from '@relavium/shared'; + +import { MODEL_PRICING, type ModelPricing } from './pricing.js'; +import type { ModelListing, ProviderId } from './types.js'; + +/** + * The merged model catalog (ADR-0064 §6) — the pure reconciliation of LIVE discovery (which model ids a key + * can reach, from `LlmProvider.listModels`), the STATIC registry ({@link MODEL_PRICING}), and the optional + * USER-pricing tier (ADR-0065, filled additively from the `model_catalog` `source='user'` rows). It lives in + * `@relavium/llm` and is **pure / I/O-free** (the host does the keychain/db/network work and passes plain data + * in), so every surface — the CLI `/models` picker, the desktop, the VS Code extension — reuses the same + * precedence. Selection (availability) and pricing are kept cleanly separate: the live list decides + * **availability**, the static registry stays the **pricing authority**. + */ + +/** Where an entry's effective pricing came from. `none` ⇒ the cost cap will not apply (ADR-0064 §6 / ADR-0065). */ +export type PricingSource = 'registry' | 'user' | 'none'; + +/** One reconciled model in the merged catalog (ADR-0064 §6). */ +export interface ModelCatalogEntry { + readonly modelId: string; + readonly provider: ProviderId; + readonly displayName: string; + /** From live ?? static ?? user (live is fresher when present, e.g. Anthropic's `max_input_tokens`). */ + readonly contextWindowTokens?: number; + readonly maxOutputTokens?: number; + /** + * The **effective** pricing: static ({@link MODEL_PRICING}) for a known id, else the USER tier for an + * unknown id, else undefined. The live tier is **never** a pricing authority (ADR-0064 §6) — providers + * rarely return a price and a refresh must never overwrite a known one. + */ + readonly pricing?: ModelPricing; + readonly pricingSource: PricingSource; + /** `pricingSource !== 'none'`. `false` ⇒ the picker surfaces "cost cap will not apply" (ADR-0064 §6). */ + readonly priceKnown: boolean; + /** + * Whether the model is available to select. When its provider has live data, this is live-list membership + * (a static model absent from the key's live list is **dimmed** "not available on your key", the K2 + * decision); when its provider has **no** live data (endpoint down, `listModels` absent, or not connected), + * it falls back to **static presence** (`true`) — never "everything unavailable" (ADR-0064 §6). Whether a + * non-connected provider's models are ultimately *selectable* is the host surface's call. + */ + readonly available: boolean; + /** `true` once `now >= deprecatedAt` (ADR-0064 §7). The picker flags but never forbids a deprecated model. */ + readonly deprecated: boolean; + /** The effective ISO deprecation date — the earlier of the static and live dates (their union). */ + readonly deprecatedAt?: string; +} + +/** Input to {@link mergeModelCatalog} — all plain data the host resolves and passes in (keeps the merge pure). */ +export interface MergeModelCatalogInput { + /** + * Per-provider LIVE listings from `listModels`. A provider **present** in the map (even with `[]`) has live + * data, so availability is decided by list membership (an empty `[]` dims all that provider's static models). + * A provider **absent** from the map has **no** live data — its registry models fall back to static presence. + */ + readonly live?: ReadonlyMap; + /** + * ADR-0065 USER tier: user-supplied pricing by model id. Fills an **unknown** id only — the static registry + * always wins for a known id (ADR-0064 §6 / ADR-0065 §2), so a user cannot silently misprice a shipped model. + */ + readonly userPricing?: ReadonlyMap; + /** Current time (epoch ms) for the deprecation check — passed in so the merge stays pure and testable. */ + readonly now: number; +} + +interface Tiers { + provider: ProviderId; + live?: ModelListing; + registry?: ModelPricing; + user?: ModelPricing; +} + +/** The earlier of two optional ISO dates (their "union" for deprecation), skipping any that fails to parse. */ +function earlierIsoDate(a: string | undefined, b: string | undefined): string | undefined { + const pa = a === undefined ? NaN : Date.parse(a); + const pb = b === undefined ? NaN : Date.parse(b); + if (Number.isNaN(pa)) return Number.isNaN(pb) ? undefined : b; + if (Number.isNaN(pb)) return a; + return pa <= pb ? a : b; +} + +const PROVIDER_RANK = new Map(LLM_PROVIDERS.map((p, i) => [p, i])); + +/** + * Reconcile live discovery ⋈ the static registry ⋈ the user tier into one deterministically-ordered catalog + * (ADR-0064 §6). Pure: no I/O, no `Date.now()` (the caller passes `now`). Per-field precedence — + * availability ← live (else static presence); price ← registry ?? user (never live); context/output ← live ?? + * static ?? user; deprecation ← the earlier of the static and live dates; priceKnown ← a static or user price + * exists. + */ +export function mergeModelCatalog(input: MergeModelCatalogInput): ModelCatalogEntry[] { + const live = input.live ?? new Map(); + const userPricing = input.userPricing ?? new Map(); + const tiers = new Map(); + + // Registry tier — every static model. + for (const [id, registry] of Object.entries(MODEL_PRICING) as [string, ModelPricing][]) { + tiers.set(id, { provider: registry.provider, registry }); + } + // Live tier — per provider present in the map. The map key is authoritative for a live-only id's provider. + for (const [provider, listings] of live) { + for (const listing of listings) { + const prev = tiers.get(listing.id); + tiers.set(listing.id, { ...prev, provider: prev?.provider ?? provider, live: listing }); + } + } + // User tier — fills an unknown id; a known id keeps its registry provider. + for (const [id, pricing] of userPricing) { + const prev = tiers.get(id); + tiers.set(id, { ...prev, provider: prev?.provider ?? pricing.provider, user: pricing }); + } + + const entries: ModelCatalogEntry[] = []; + for (const [modelId, t] of tiers) { + const pricing = t.registry ?? t.user; // registry wins for a known id; user fills an unknown one. + const pricingSource: PricingSource = t.registry ? 'registry' : t.user ? 'user' : 'none'; + const contextWindowTokens = + t.live?.contextWindowTokens ?? t.registry?.contextWindowTokens ?? t.user?.contextWindowTokens; + const maxOutputTokens = + t.live?.maxOutputTokens ?? t.registry?.maxOutputTokens ?? t.user?.maxOutputTokens; + // Availability: live-list membership when the provider has live data, else static presence. + const available = live.has(t.provider) ? t.live !== undefined : true; + const deprecatedAt = earlierIsoDate(t.registry?.deprecatedAt, t.live?.deprecatedAt); + const parsedDeprecation = deprecatedAt === undefined ? NaN : Date.parse(deprecatedAt); + const deprecated = !Number.isNaN(parsedDeprecation) && parsedDeprecation <= input.now; + + entries.push({ + modelId, + provider: t.provider, + displayName: t.registry?.displayName ?? t.live?.displayName ?? t.user?.displayName ?? modelId, + ...(contextWindowTokens !== undefined ? { contextWindowTokens } : {}), + ...(maxOutputTokens !== undefined ? { maxOutputTokens } : {}), + ...(pricing !== undefined ? { pricing } : {}), + pricingSource, + priceKnown: pricingSource !== 'none', + available, + deprecated, + ...(deprecatedAt !== undefined ? { deprecatedAt } : {}), + }); + } + + // Deterministic order: provider (in the seam's LLM_PROVIDERS order), then displayName, then modelId. + entries.sort((x, y) => { + const byProvider = + (PROVIDER_RANK.get(x.provider) ?? LLM_PROVIDERS.length) - + (PROVIDER_RANK.get(y.provider) ?? LLM_PROVIDERS.length); + if (byProvider !== 0) return byProvider; + const byName = x.displayName.localeCompare(y.displayName); + return byName !== 0 ? byName : x.modelId.localeCompare(y.modelId); + }); + return entries; +} diff --git a/packages/llm/src/pricing.ts b/packages/llm/src/pricing.ts index 08614178..e6841784 100644 --- a/packages/llm/src/pricing.ts +++ b/packages/llm/src/pricing.ts @@ -39,6 +39,13 @@ export interface ModelPricing { readonly cachedInputPerMtokMicrocents: number; /** Cache-write price, where the provider charges one (Anthropic does); undefined otherwise. */ readonly cacheWritePerMtokMicrocents?: number; + /** + * ISO-8601 date this model is scheduled to retire, if any (ADR-0064 §7). The pure {@link mergeModelCatalog} + * flags an entry `deprecated` once `now >= deprecatedAt` (unioned with a live-list deprecation date, taking + * the earlier). The picker flags a deprecated model but never forbids it — a legacy alias still costs + * correctly until its date. Absent ⇒ not deprecated. + */ + readonly deprecatedAt?: string; /** * Per-modality media **output** rates (1.AF/D17, * [ADR-0044](../../../docs/decisions/0044-media-access-governance-read-media-save-to-cost.md) §3) — integer @@ -188,6 +195,7 @@ export const MODEL_PRICING = { inputPerMtokMicrocents: usd(0.14), outputPerMtokMicrocents: usd(0.28), cachedInputPerMtokMicrocents: usd(0.0028), // cache-hit input + deprecatedAt: '2026-07-24T15:59:00Z', // legacy alias retires 2026-07-24 15:59 UTC (see header) }, 'deepseek-reasoner': { provider: 'deepseek', @@ -200,6 +208,7 @@ export const MODEL_PRICING = { inputPerMtokMicrocents: usd(0.14), outputPerMtokMicrocents: usd(0.28), cachedInputPerMtokMicrocents: usd(0.0028), + deprecatedAt: '2026-07-24T15:59:00Z', // legacy alias retires 2026-07-24 15:59 UTC (see header) }, } as const satisfies Readonly>; From b9c3d5fe19872f99c973100c9b091191b45ff876 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 20:08:04 +0300 Subject: [PATCH 07/68] =?UTF-8?q?test(llm):=202.5.G=20S3=20Opus-review=20?= =?UTF-8?q?=E2=80=94=20pin=20the=20merge=20tiebreaker=20+=20three-tier=20p?= =?UTF-8?q?recedence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opus's adversarial round confirmed the merge code is correct; both findings were test-coverage gaps (0 production bugs; the one PLAUSIBLE was spec-conformant). - Determinism test strengthened: was a same-input re-run (tautology). Now feeds two user-priced unknown ids that tie on provider+displayName in DIFFERENT Map insertion orders and asserts modelId-ascending ordering + insertion-order independence — pinning the sort's modelId tiebreaker (previously dead across all tests). - New three-tier test: one known id (claude-opus-4-8) present in the registry, a live list (differing context + membership), AND userPricing at once — asserts the full ADR-0064 §6 split (pricingSource='registry', pricing=the registry object, context from live, priceKnown, available), pinning registry-wins-price in the presence of a live tier. - model-catalog.ts: a one-line clarification on ModelCatalogEntry.available that the availability rule is tier-agnostic (the USER tier is pricing-only), so a user-declared id its connected provider's live list omits is dimmed while its price still applies. Verified: pnpm turbo run lint typecheck test build — 24/24 green (model-catalog 13->14); prettier clean. Refs: ADR-0064 Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/llm/src/model-catalog.test.ts | 39 +++++++++++++++++++++++--- packages/llm/src/model-catalog.ts | 4 ++- 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/packages/llm/src/model-catalog.test.ts b/packages/llm/src/model-catalog.test.ts index d36fa520..bd7b6896 100644 --- a/packages/llm/src/model-catalog.test.ts +++ b/packages/llm/src/model-catalog.test.ts @@ -112,6 +112,21 @@ describe('mergeModelCatalog (ADR-0064 §6)', () => { expect(entry?.pricing).toBe(custom); }); + it('three tiers on one known id: registry wins price, live wins context, available by live membership', () => { + // claude-opus-4-8 present in ALL THREE tiers at once — the full ADR-0064 §6 per-field split must hold. + const entries = mergeModelCatalog({ + live: liveMap([['anthropic', [{ id: 'claude-opus-4-8', contextWindowTokens: 500_000 }]]]), + userPricing: new Map([['claude-opus-4-8', userPricing('anthropic')]]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + const opus = byId(entries, 'claude-opus-4-8'); + expect(opus?.pricingSource).toBe('registry'); // registry wins even with a live AND user tier present + expect(opus?.pricing).toBe(MODEL_PRICING['claude-opus-4-8']); // NOT the user object + expect(opus?.contextWindowTokens).toBe(500_000); // live wins for context + expect(opus?.priceKnown).toBe(true); + expect(opus?.available).toBe(true); // in the live list + }); + it('context/output: the LIVE value wins over the static one when present, else static', () => { const entries = mergeModelCatalog({ live: liveMap([['anthropic', [{ id: 'claude-opus-4-8', contextWindowTokens: 500_000 }]]]), @@ -180,12 +195,28 @@ describe('mergeModelCatalog (ADR-0064 §6)', () => { // within a provider, entries are displayName-then-id sorted (stable, no duplicates) const anthropic = entries.filter((e) => e.provider === 'anthropic').map((e) => e.displayName); expect(anthropic).toEqual([...anthropic].sort((a, b) => a.localeCompare(b))); - // two runs produce byte-identical ordering - const again = mergeModelCatalog({ - live: liveMap([['openai', [{ id: 'gpt-6-preview' }]]]), + // The modelId tiebreaker + insertion-order independence: two user-priced unknown ids that TIE on + // provider (openai) + displayName ('Custom Model') must order by modelId ascending, regardless of the + // input Map's insertion order (proves the model-catalog.ts sort tiebreaker, not a same-input re-run). + const forward = mergeModelCatalog({ + userPricing: new Map([ + ['zeta-model', userPricing('openai')], + ['alpha-model', userPricing('openai')], + ]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + const reversed = mergeModelCatalog({ + userPricing: new Map([ + ['alpha-model', userPricing('openai')], + ['zeta-model', userPricing('openai')], + ]), now: BEFORE_DEEPSEEK_DEPRECATION, }); - expect(entries.map((e) => e.modelId)).toEqual(again.map((e) => e.modelId)); + const forwardCustom = forward + .filter((e) => e.displayName === 'Custom Model') + .map((e) => e.modelId); + expect(forwardCustom).toEqual(['alpha-model', 'zeta-model']); // modelId tiebreaker, not insertion order + expect(reversed.map((e) => e.modelId)).toEqual(forward.map((e) => e.modelId)); // insertion-order independent }); it('does not mutate MODEL_PRICING', () => { diff --git a/packages/llm/src/model-catalog.ts b/packages/llm/src/model-catalog.ts index f1514553..49e8718f 100644 --- a/packages/llm/src/model-catalog.ts +++ b/packages/llm/src/model-catalog.ts @@ -38,7 +38,9 @@ export interface ModelCatalogEntry { * (a static model absent from the key's live list is **dimmed** "not available on your key", the K2 * decision); when its provider has **no** live data (endpoint down, `listModels` absent, or not connected), * it falls back to **static presence** (`true`) — never "everything unavailable" (ADR-0064 §6). Whether a - * non-connected provider's models are ultimately *selectable* is the host surface's call. + * non-connected provider's models are ultimately *selectable* is the host surface's call. This rule is + * **tier-agnostic**: the ADR-0065 USER tier is pricing-only, so a user-declared id that its connected + * provider's live list omits is likewise dimmed — its `pricing` still applies for cost governance. */ readonly available: boolean; /** `true` once `now >= deprecatedAt` (ADR-0064 §7). The picker flags but never forbids a deprecated model. */ From 050167ec4ca941c1364de3c1f14234709cb782c8 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 20:24:27 +0300 Subject: [PATCH 08/68] =?UTF-8?q?fix(llm):=202.5.G=20S3=20Sonnet-review=20?= =?UTF-8?q?=E2=80=94=20cross-provider=20id-collision=20guard=20+=20user-ti?= =?UTF-8?q?er=20deprecation=20+=20pinned=20sort=20locale?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sonnet's fresh round caught a real bug Opus missed, plus two smaller correctness gaps and three test-coverage holes (all independently reproduced). Code fixes (model-catalog.ts): - Cross-provider id collision (the real bug): the live-tier merge attached a listing's live fields (context/output/displayName/deprecatedAt) to an entry keyed by that id even when the listing came from a DIFFERENT provider's list — so a mis-keyed or custom-endpoint rogue id could silently corrupt an unrelated static model's window (e.g. gpt-5.5 getting contextWindowTokens:1 from a deepseek-keyed listing), which feeds the ADR-0062 context-fullness / auto-compaction indicator. Both the live and user tiers now drop a listing whose id collides with a model already anchored to a different provider (mirrors the guard already applied to the provider field). - USER-tier deprecatedAt was dropped from the deprecation union (every other user field was consulted). Folded t.user?.deprecatedAt into earlierIsoDate so a user-priced model's deprecation is honoured. - localeCompare now pins the 'en' locale on both sort keys, so catalog order is byte-identical across every host/OS/CI locale (a runtime-default locale — e.g. Danish — can flip case ordering for a provider-controlled live displayName). Test coverage (model-catalog.test.ts, 14->17): the cross-provider collision regression; USER-tier tier-agnostic dimming (user id omitted from its connected provider's live list -> available=false); USER-tier context/maxOutput fallback assertions; maxOutputTokens live-wins direction; and a user-tier deprecatedAt union test. Verified: pnpm turbo run lint typecheck test build — 24/24 green (model-catalog 14->17); prettier clean. (A pre-existing flaky CLI media-gc timing test intermittently fails on a cold run and passes on retry — unrelated to this change.) Refs: ADR-0064, ADR-0065 Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/llm/src/model-catalog.test.ts | 73 ++++++++++++++++++++++++-- packages/llm/src/model-catalog.ts | 23 +++++--- 2 files changed, 87 insertions(+), 9 deletions(-) diff --git a/packages/llm/src/model-catalog.test.ts b/packages/llm/src/model-catalog.test.ts index bd7b6896..03a30d3c 100644 --- a/packages/llm/src/model-catalog.test.ts +++ b/packages/llm/src/model-catalog.test.ts @@ -110,6 +110,9 @@ describe('mergeModelCatalog (ADR-0064 §6)', () => { const entry = byId(entries, 'my-custom-model'); expect(entry).toMatchObject({ provider: 'openai', pricingSource: 'user', priceKnown: true }); expect(entry?.pricing).toBe(custom); + // the user tier also supplies context/output when no registry or live value exists + expect(entry?.contextWindowTokens).toBe(custom.contextWindowTokens); + expect(entry?.maxOutputTokens).toBe(custom.maxOutputTokens); }); it('three tiers on one known id: registry wins price, live wins context, available by live membership', () => { @@ -127,14 +130,25 @@ describe('mergeModelCatalog (ADR-0064 §6)', () => { expect(opus?.available).toBe(true); // in the live list }); - it('context/output: the LIVE value wins over the static one when present, else static', () => { + it('context/output: the LIVE value wins over the static one for BOTH fields when present, else static', () => { const entries = mergeModelCatalog({ - live: liveMap([['anthropic', [{ id: 'claude-opus-4-8', contextWindowTokens: 500_000 }]]]), + live: liveMap([ + [ + 'anthropic', + [{ id: 'claude-opus-4-8', contextWindowTokens: 500_000, maxOutputTokens: 7_000 }], + ], + ]), now: BEFORE_DEEPSEEK_DEPRECATION, }); const opus = byId(entries, 'claude-opus-4-8'); expect(opus?.contextWindowTokens).toBe(500_000); // live wins - expect(opus?.maxOutputTokens).toBe(MODEL_PRICING['claude-opus-4-8'].maxOutputTokens); // no live -> static + expect(opus?.maxOutputTokens).toBe(7_000); // live wins for maxOutput too (not the static value) + // a model with no live entry of its own falls back to the static values + const sonnet = byId(entries, 'claude-sonnet-4-6'); + expect(sonnet?.contextWindowTokens).toBe( + MODEL_PRICING['claude-sonnet-4-6'].contextWindowTokens, + ); + expect(sonnet?.maxOutputTokens).toBe(MODEL_PRICING['claude-sonnet-4-6'].maxOutputTokens); }); it('deprecation: a static deprecatedAt flags the model only once now >= the date', () => { @@ -180,6 +194,59 @@ describe('mergeModelCatalog (ADR-0064 §6)', () => { expect(weird?.deprecatedAt).toBeUndefined(); }); + it('ignores a live listing whose id collides with a DIFFERENT provider’s static model (no field corruption)', () => { + // a rogue / mis-keyed 'deepseek' live list claims 'gpt-5.5' (a real OpenAI static id) with junk fields + const entries = mergeModelCatalog({ + live: liveMap([ + [ + 'deepseek', + [ + { + id: 'gpt-5.5', + contextWindowTokens: 1, + maxOutputTokens: 1, + displayName: 'HIJACKED', + deprecatedAt: '2000-01-01T00:00:00Z', + }, + ], + ], + ]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + const gpt = byId(entries, 'gpt-5.5'); + expect(gpt?.provider).toBe('openai'); // stays openai + expect(gpt?.displayName).toBe(MODEL_PRICING['gpt-5.5'].displayName); // registry, NOT 'HIJACKED' + expect(gpt?.contextWindowTokens).toBe(MODEL_PRICING['gpt-5.5'].contextWindowTokens); // registry, not 1 + expect(gpt?.maxOutputTokens).toBe(MODEL_PRICING['gpt-5.5'].maxOutputTokens); + expect(gpt?.deprecated).toBe(false); // the rogue deprecatedAt is dropped + expect(gpt?.deprecatedAt).toBeUndefined(); + // openai has NO live list (only the mis-keyed deepseek one) -> gpt-5.5 falls back to static presence + expect(gpt?.available).toBe(true); + }); + + it('USER-tier availability is tier-agnostic: a user id omitted from its connected provider’s live list is dimmed', () => { + const entries = mergeModelCatalog({ + // openai is connected (has a live list) but that list does NOT include the user-declared id + live: liveMap([['openai', [{ id: 'gpt-5.5' }]]]), + userPricing: new Map([['my-custom-openai-model', userPricing('openai')]]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + const custom = byId(entries, 'my-custom-openai-model'); + expect(custom?.pricingSource).toBe('user'); + expect(custom?.available).toBe(false); // dimmed — openai has live data and this id isn't in it + }); + + it('deprecation union includes the USER tier: a user-priced id with a past deprecatedAt is flagged', () => { + const custom: ModelPricing = { ...userPricing('openai'), deprecatedAt: '2020-01-01T00:00:00Z' }; + const entries = mergeModelCatalog({ + userPricing: new Map([['old-custom-model', custom]]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + const entry = byId(entries, 'old-custom-model'); + expect(entry?.deprecatedAt).toBe('2020-01-01T00:00:00Z'); + expect(entry?.deprecated).toBe(true); + }); + it('is deterministically ordered: provider (seam order) then displayName then id', () => { const entries = mergeModelCatalog({ live: liveMap([['openai', [{ id: 'gpt-6-preview' }]]]), diff --git a/packages/llm/src/model-catalog.ts b/packages/llm/src/model-catalog.ts index 49e8718f..df05c3a4 100644 --- a/packages/llm/src/model-catalog.ts +++ b/packages/llm/src/model-catalog.ts @@ -88,8 +88,8 @@ const PROVIDER_RANK = new Map(LLM_PROVIDERS.map((p, i) => [p * Reconcile live discovery ⋈ the static registry ⋈ the user tier into one deterministically-ordered catalog * (ADR-0064 §6). Pure: no I/O, no `Date.now()` (the caller passes `now`). Per-field precedence — * availability ← live (else static presence); price ← registry ?? user (never live); context/output ← live ?? - * static ?? user; deprecation ← the earlier of the static and live dates; priceKnown ← a static or user price - * exists. + * static ?? user; deprecation ← the earliest of the static, live, and user dates; priceKnown ← a static or + * user price exists. */ export function mergeModelCatalog(input: MergeModelCatalogInput): ModelCatalogEntry[] { const live = input.live ?? new Map(); @@ -101,15 +101,21 @@ export function mergeModelCatalog(input: MergeModelCatalogInput): ModelCatalogEn tiers.set(id, { provider: registry.provider, registry }); } // Live tier — per provider present in the map. The map key is authoritative for a live-only id's provider. + // A listing whose id COLLIDES with a model already anchored to a DIFFERENT provider is IGNORED, so a live + // list fetched under one provider key can never overwrite another provider's static context/output/name/ + // deprecation (model ids are globally unique in practice; this guard keeps a mis-keyed or custom-endpoint + // rogue id from corrupting an unrelated entry). for (const [provider, listings] of live) { for (const listing of listings) { const prev = tiers.get(listing.id); + if (prev !== undefined && prev.provider !== provider) continue; // cross-provider id collision — drop tiers.set(listing.id, { ...prev, provider: prev?.provider ?? provider, live: listing }); } } - // User tier — fills an unknown id; a known id keeps its registry provider. + // User tier — fills an unknown id; a known id keeps its registry provider. Same cross-provider guard. for (const [id, pricing] of userPricing) { const prev = tiers.get(id); + if (prev !== undefined && prev.provider !== pricing.provider) continue; // cross-provider id collision — drop tiers.set(id, { ...prev, provider: prev?.provider ?? pricing.provider, user: pricing }); } @@ -123,7 +129,10 @@ export function mergeModelCatalog(input: MergeModelCatalogInput): ModelCatalogEn t.live?.maxOutputTokens ?? t.registry?.maxOutputTokens ?? t.user?.maxOutputTokens; // Availability: live-list membership when the provider has live data, else static presence. const available = live.has(t.provider) ? t.live !== undefined : true; - const deprecatedAt = earlierIsoDate(t.registry?.deprecatedAt, t.live?.deprecatedAt); + const deprecatedAt = earlierIsoDate( + earlierIsoDate(t.registry?.deprecatedAt, t.live?.deprecatedAt), + t.user?.deprecatedAt, + ); const parsedDeprecation = deprecatedAt === undefined ? NaN : Date.parse(deprecatedAt); const deprecated = !Number.isNaN(parsedDeprecation) && parsedDeprecation <= input.now; @@ -148,8 +157,10 @@ export function mergeModelCatalog(input: MergeModelCatalogInput): ModelCatalogEn (PROVIDER_RANK.get(x.provider) ?? LLM_PROVIDERS.length) - (PROVIDER_RANK.get(y.provider) ?? LLM_PROVIDERS.length); if (byProvider !== 0) return byProvider; - const byName = x.displayName.localeCompare(y.displayName); - return byName !== 0 ? byName : x.modelId.localeCompare(y.modelId); + // Pin an explicit locale so the catalog order is byte-identical across every host/OS/CI locale (a + // runtime-default locale — e.g. Danish — can flip case ordering for a provider-controlled live displayName). + const byName = x.displayName.localeCompare(y.displayName, 'en'); + return byName !== 0 ? byName : x.modelId.localeCompare(y.modelId, 'en'); }); return entries; } From 8c6a8c659bc32886182faef785b47740c424a357 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 20:43:12 +0300 Subject: [PATCH 09/68] =?UTF-8?q?feat(db,shared):=202.5.G=20S4=20=E2=80=94?= =?UTF-8?q?=20model=5Fcatalog=20live-cache=20widening=20+=200007=20migrati?= =?UTF-8?q?on=20+=20store=20methods?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Repurposes the existing (media-routing) model_catalog table into the live-discovery cache (ADR-0064 §4/§5), purely additively — the media-routing reader is untouched. - shared: MODEL_CATALOG_SOURCES ['static','live','user'] + ModelCatalogSource — the row provenance discriminant (static = a capability/media seed; live = discovered via listModels; user = user-supplied pricing, ADR-0065). - db schema + migration 0007 (ALTER-ADD only, no CHECK — validated at the store boundary): source (text, default 'static', NOT NULL) + last_refreshed_at (epochMs, nullable). SQLite<->Postgres dialect parity via drizzle-kit generate; 0000-0006 untouched. - model-catalog-store.ts (extended, not rewritten): * ModelCatalogUpsert gains optional source/lastRefreshedAt (default 'static'/null — every existing media caller is unchanged; the existing upsert writes them). * A NEW wide read projection ModelCatalogListing (id/display/context/limits/text-token costs/deprecation/source/freshness), DISTINCT from the narrow media ModelCatalogRecord; a stored 0 context/maxOutput reads back undefined (the ADR-0064 §3 "0 == unknown" convention, matching @relavium/llm's ModelListing); source validated at the boundary (foreign -> 'static', mirroring coerceMediaSurface). * listByProvider / listAll (active, deterministic order). * replaceProviderModels(providerId, rows, now) — ONE transaction: bulk-upsert each live model (source='live', lastRefreshedAt=now, reusing the (provider,model) row id so the FK graph stays stable, reactivating a vanished-then-reappearing row) + soft-deactivate every active source='live' row of the provider absent from the new list. NEVER touches a source='user' (ADR-0065 §1) or source='static' (media seed) row; NEVER hard-deletes (FK target from 5 tables); an empty new list is explicitly guarded (no notInArray([])). * providerRefreshedAt — the TTL freshness read (max lastRefreshedAt of active live rows). - database-schema.md: the two new columns + the live-cache role documented (canonical home). - model-catalog-store.test.ts: +9 tests (10->19) — live round-trip, replace + soft- deactivate + reactivate, user/static preservation on collision, no hard-delete, providerRefreshedAt, 0->undefined mapping, source coercion, media reader unaffected. Verified: pnpm turbo run lint typecheck test build — 24/24 green (@relavium/db 173 pass); db imports only @relavium/shared (no llm/core); migration additive-only; prettier clean. Refs: ADR-0064, ADR-0065 Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/reference/desktop/database-schema.md | 4 + packages/db/drizzle/0007_harsh_bug.sql | 2 + packages/db/drizzle/meta/0007_snapshot.json | 2077 +++++++++++++++++++ packages/db/drizzle/meta/_journal.json | 7 + packages/db/src/index.ts | 3 + packages/db/src/model-catalog-store.test.ts | 379 ++++ packages/db/src/model-catalog-store.ts | 250 ++- packages/db/src/schema.ts | 12 + packages/shared/src/constants.ts | 12 + 9 files changed, 2744 insertions(+), 2 deletions(-) create mode 100644 packages/db/drizzle/0007_harsh_bug.sql create mode 100644 packages/db/drizzle/meta/0007_snapshot.json diff --git a/docs/reference/desktop/database-schema.md b/docs/reference/desktop/database-schema.md index 028e7403..1e7b4fad 100644 --- a/docs/reference/desktop/database-schema.md +++ b/docs/reference/desktop/database-schema.md @@ -84,6 +84,8 @@ CREATE UNIQUE INDEX idx_llm_providers_name ON llm_providers (name) WHERE deleted Models offered by each provider, including pricing used for local cost tracking. The `*_per_mtok_microcents` columns are price **per million tokens, in integer micro-cents** (one micro-cent = 1e-8 USD = cents x 1,000,000; see the [money/cost convention](#sqlite-type-conventions)). The three `media_*_cost_microcents` columns are the projection of `ModelPricing.mediaOutputRates` (1.AF/D17, [ADR-0044](../../decisions/0044-media-access-governance-read-media-save-to-cost.md) §3) — integer micro-cents **per billed media-output unit** (per image, per audio-second, per video-second); **NULL** when the model has no metered media rate (the realized fold + the pre-egress estimate degrade to 0 for it — H4). `document`/PDF is excluded (it bills as tokens). No shipped model carries a media rate yet, so these are NULL across the seeded catalog. +**Live-discovery cache role ([ADR-0064](../../decisions/0064-live-model-catalog.md) §4/§5).** As of 2.5.G this table doubles as the **live-discovery cache** — "which model ids a given key can reach" — filled by a bulk refresh over the seam's `listModels`, with the static `MODEL_PRICING` registry enriching **at read time** (the registry is **never** seeded into the DB — that would create a second, drift-prone pricing home). The `source` discriminant records provenance: **`static`** (a hardcoded capability/media seed — the media-routing `upsert` path's default), **`live`** (discovered via `listModels` — the refresh writes it), **`user`** (user-supplied pricing, [ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md)). `last_refreshed_at` is the freshness stamp backing the 24h TTL. The bulk refresh (`replaceProviderModels`) **soft-deactivates** (`is_active = 0`, `deleted_at` left NULL) every currently-active `source='live'` row of a provider whose model id vanishes from the new list, and **reactivates** a reappearing one by reusing the same row — it **never hard-DELETEs** (`model_catalog.id` is an FK target from five tables) and **never touches a `source='user'` or `source='static'` row** (a refresh must not clobber user pricing or regress the media-routing seed). The existing narrow media-routing projection (`resolveMediaSurface` / the D15 capability load-check) is untouched by the widening. Because SQLite `ALTER TABLE ADD` cannot carry a CHECK, the closed `source` value set is validated at the store read boundary (`coerceModelCatalogSource`, degrading a foreign value to `static`), like `media_surface`. + | Column | Type | Constraints | |--------|------|-------------| | `id` | TEXT | PRIMARY KEY (UUID) | @@ -105,6 +107,8 @@ Models offered by each provider, including pricing used for local cost tracking. | `supports_json_mode` | INTEGER (bool) | NOT NULL DEFAULT 0 | | `capabilities` | TEXT (JSON) | NOT NULL DEFAULT `'{}'` | | `deprecation_date` | INTEGER | NULL | +| `source` | TEXT | NOT NULL DEFAULT `'static'` — provenance: `'static'` \| `'live'` \| `'user'` (the live-discovery cache discriminant, [ADR-0064](../../decisions/0064-live-model-catalog.md) §4); validated at the store read boundary (no DB CHECK — SQLite `ALTER ADD`) | +| `last_refreshed_at` | INTEGER | NULL — epoch-ms a live refresh last wrote this row (ADR-0064 §5 TTL freshness); NULL for a static/user or never-refreshed row | | `is_active` | INTEGER (bool) | NOT NULL DEFAULT 1 | | `deleted_at` | INTEGER | NULL | | `created_at` | INTEGER | NOT NULL | diff --git a/packages/db/drizzle/0007_harsh_bug.sql b/packages/db/drizzle/0007_harsh_bug.sql new file mode 100644 index 00000000..0244cdd0 --- /dev/null +++ b/packages/db/drizzle/0007_harsh_bug.sql @@ -0,0 +1,2 @@ +ALTER TABLE `model_catalog` ADD `source` text DEFAULT 'static' NOT NULL;--> statement-breakpoint +ALTER TABLE `model_catalog` ADD `last_refreshed_at` integer; \ No newline at end of file diff --git a/packages/db/drizzle/meta/0007_snapshot.json b/packages/db/drizzle/meta/0007_snapshot.json new file mode 100644 index 00000000..dc726738 --- /dev/null +++ b/packages/db/drizzle/meta/0007_snapshot.json @@ -0,0 +1,2077 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "b2cb738e-465a-4dba-a7f7-eaf7f8634aa3", + "prevId": "213ad2ee-ef8d-4342-ba04-661ac99cb116", + "tables": { + "agent_sessions": { + "name": "agent_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "agent_slug": { + "name": "agent_slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "agent_snapshot": { + "name": "agent_snapshot", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "working_dir": { + "name": "working_dir", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "git_ref": { + "name": "git_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fs_scope_tier": { + "name": "fs_scope_tier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'sandboxed'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "context_json": { + "name": "context_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "total_input_tokens": { + "name": "total_input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "total_output_tokens": { + "name": "total_output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "total_cost_microcents": { + "name": "total_cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "exported_workflow_path": { + "name": "exported_workflow_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_agent_sessions_status": { + "name": "idx_agent_sessions_status", + "columns": [ + "status", + "\"updated_at\" desc" + ], + "isUnique": false, + "where": "\"agent_sessions\".\"deleted_at\" is null" + }, + "idx_agent_sessions_agent": { + "name": "idx_agent_sessions_agent", + "columns": [ + "agent_id", + "\"created_at\" desc" + ], + "isUnique": false, + "where": "\"agent_sessions\".\"agent_id\" is not null" + }, + "idx_agent_sessions_updated": { + "name": "idx_agent_sessions_updated", + "columns": [ + "\"updated_at\" desc", + "\"id\" desc" + ], + "isUnique": false, + "where": "\"agent_sessions\".\"deleted_at\" is null" + } + }, + "foreignKeys": { + "agent_sessions_agent_id_agents_id_fk": { + "name": "agent_sessions_agent_id_agents_id_fk", + "tableFrom": "agent_sessions", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "agent_sessions_model_id_model_catalog_id_fk": { + "name": "agent_sessions_model_id_model_catalog_id_fk", + "tableFrom": "agent_sessions", + "tableTo": "model_catalog", + "columnsFrom": [ + "model_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "agent_sessions_fs_scope_tier_check": { + "name": "agent_sessions_fs_scope_tier_check", + "value": "\"agent_sessions\".\"fs_scope_tier\" in ('sandboxed', 'project', 'full')" + }, + "agent_sessions_status_check": { + "name": "agent_sessions_status_check", + "value": "\"agent_sessions\".\"status\" in ('active', 'idle', 'exported', 'ended')" + } + } + }, + "agents": { + "name": "agents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "system_prompt": { + "name": "system_prompt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "tools": { + "name": "tools", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "config": { + "name": "config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "input_schema": { + "name": "input_schema", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_schema": { + "name": "output_schema", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "1" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_agents_slug": { + "name": "idx_agents_slug", + "columns": [ + "slug" + ], + "isUnique": true, + "where": "\"agents\".\"deleted_at\" is null" + }, + "idx_agents_model": { + "name": "idx_agents_model", + "columns": [ + "model_id" + ], + "isUnique": false + }, + "idx_agents_active": { + "name": "idx_agents_active", + "columns": [ + "is_active", + "\"created_at\" desc" + ], + "isUnique": false, + "where": "\"agents\".\"deleted_at\" is null" + } + }, + "foreignKeys": { + "agents_model_id_model_catalog_id_fk": { + "name": "agents_model_id_model_catalog_id_fk", + "tableFrom": "agents", + "tableTo": "model_catalog", + "columnsFrom": [ + "model_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "llm_providers": { + "name": "llm_providers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_key_keychain_ref": { + "name": "api_key_keychain_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_headers": { + "name": "default_headers", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "1" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_llm_providers_name": { + "name": "idx_llm_providers_name", + "columns": [ + "name" + ], + "isUnique": true, + "where": "\"llm_providers\".\"deleted_at\" is null" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "media_objects": { + "name": "media_objects", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "handle": { + "name": "handle", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "modality": { + "name": "modality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "byte_length": { + "name": "byte_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_referenced_at": { + "name": "last_referenced_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "media_objects_handle_unique": { + "name": "media_objects_handle_unique", + "columns": [ + "handle" + ], + "isUnique": true + }, + "idx_media_objects_gc": { + "name": "idx_media_objects_gc", + "columns": [ + "last_referenced_at" + ], + "isUnique": false, + "where": "\"media_objects\".\"deleted_at\" is null" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "media_objects_modality_check": { + "name": "media_objects_modality_check", + "value": "\"media_objects\".\"modality\" in ('image', 'audio', 'video', 'document')" + } + } + }, + "media_references": { + "name": "media_references", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "handle": { + "name": "handle", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scope_kind": { + "name": "scope_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scope_id": { + "name": "scope_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_media_references_unique": { + "name": "idx_media_references_unique", + "columns": [ + "handle", + "scope_kind", + "scope_id" + ], + "isUnique": true + }, + "idx_media_references_scope": { + "name": "idx_media_references_scope", + "columns": [ + "scope_kind", + "scope_id" + ], + "isUnique": false + }, + "idx_media_references_handle": { + "name": "idx_media_references_handle", + "columns": [ + "handle" + ], + "isUnique": false + } + }, + "foreignKeys": { + "media_references_handle_media_objects_handle_fk": { + "name": "media_references_handle_media_objects_handle_fk", + "tableFrom": "media_references", + "tableTo": "media_objects", + "columnsFrom": [ + "handle" + ], + "columnsTo": [ + "handle" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "media_references_scope_kind_check": { + "name": "media_references_scope_kind_check", + "value": "\"media_references\".\"scope_kind\" in ('run', 'node', 'session', 'workspace')" + } + } + }, + "messages": { + "name": "messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "step_execution_id": { + "name": "step_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sequence_number": { + "name": "sequence_number", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_parts": { + "name": "content_parts", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finish_reason": { + "name": "finish_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_messages_step": { + "name": "idx_messages_step", + "columns": [ + "step_execution_id", + "sequence_number" + ], + "isUnique": false + }, + "idx_messages_run": { + "name": "idx_messages_run", + "columns": [ + "run_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "messages_step_execution_id_step_executions_id_fk": { + "name": "messages_step_execution_id_step_executions_id_fk", + "tableFrom": "messages", + "tableTo": "step_executions", + "columnsFrom": [ + "step_execution_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "model_catalog": { + "name": "model_catalog", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "context_window_tokens": { + "name": "context_window_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "max_output_tokens": { + "name": "max_output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_cost_per_mtok_microcents": { + "name": "input_cost_per_mtok_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "output_cost_per_mtok_microcents": { + "name": "output_cost_per_mtok_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cached_input_cost_per_mtok_microcents": { + "name": "cached_input_cost_per_mtok_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "media_image_cost_microcents": { + "name": "media_image_cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "media_audio_cost_microcents": { + "name": "media_audio_cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "media_video_cost_microcents": { + "name": "media_video_cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "media_surface": { + "name": "media_surface", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'chat'" + }, + "supports_tool_calling": { + "name": "supports_tool_calling", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "0" + }, + "supports_vision": { + "name": "supports_vision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "0" + }, + "supports_streaming": { + "name": "supports_streaming", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "1" + }, + "supports_json_mode": { + "name": "supports_json_mode", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "0" + }, + "capabilities": { + "name": "capabilities", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "deprecation_date": { + "name": "deprecation_date", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'static'" + }, + "last_refreshed_at": { + "name": "last_refreshed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "1" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_model_catalog_provider_model": { + "name": "idx_model_catalog_provider_model", + "columns": [ + "provider_id", + "model_id" + ], + "isUnique": true, + "where": "\"model_catalog\".\"deleted_at\" is null" + }, + "idx_model_catalog_provider": { + "name": "idx_model_catalog_provider", + "columns": [ + "provider_id" + ], + "isUnique": false + }, + "idx_model_catalog_active": { + "name": "idx_model_catalog_active", + "columns": [ + "is_active" + ], + "isUnique": false, + "where": "\"model_catalog\".\"deleted_at\" is null" + } + }, + "foreignKeys": { + "model_catalog_provider_id_llm_providers_id_fk": { + "name": "model_catalog_provider_id_llm_providers_id_fk", + "tableFrom": "model_catalog", + "tableTo": "llm_providers", + "columnsFrom": [ + "provider_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "run_costs": { + "name": "run_costs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cost_microcents": { + "name": "cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_run_costs_run": { + "name": "idx_run_costs_run", + "columns": [ + "run_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "run_costs_run_id_runs_id_fk": { + "name": "run_costs_run_id_runs_id_fk", + "tableFrom": "run_costs", + "tableTo": "runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "run_costs_model_id_model_catalog_id_fk": { + "name": "run_costs_model_id_model_catalog_id_fk", + "tableFrom": "run_costs", + "tableTo": "model_catalog", + "columnsFrom": [ + "model_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "run_events": { + "name": "run_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "step_execution_id": { + "name": "step_execution_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'info'" + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "payload_json": { + "name": "payload_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "ts": { + "name": "ts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_run_events_run_seq": { + "name": "idx_run_events_run_seq", + "columns": [ + "run_id", + "seq" + ], + "isUnique": true + }, + "idx_run_events_step": { + "name": "idx_run_events_step", + "columns": [ + "step_execution_id", + "ts" + ], + "isUnique": false, + "where": "\"run_events\".\"step_execution_id\" is not null" + }, + "idx_run_events_run_type": { + "name": "idx_run_events_run_type", + "columns": [ + "run_id", + "event_type", + "ts" + ], + "isUnique": false + } + }, + "foreignKeys": { + "run_events_run_id_runs_id_fk": { + "name": "run_events_run_id_runs_id_fk", + "tableFrom": "run_events", + "tableTo": "runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "runs": { + "name": "runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workflow_path": { + "name": "workflow_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "project_root": { + "name": "project_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workflow_definition_snapshot": { + "name": "workflow_definition_snapshot", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "execution_mode": { + "name": "execution_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'local'" + }, + "trigger_type": { + "name": "trigger_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'manual'" + }, + "trigger_metadata": { + "name": "trigger_metadata", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "input_json": { + "name": "input_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "output_json": { + "name": "output_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_json": { + "name": "error_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "total_input_tokens": { + "name": "total_input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "total_output_tokens": { + "name": "total_output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "total_cost_microcents": { + "name": "total_cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_runs_workflow": { + "name": "idx_runs_workflow", + "columns": [ + "workflow_id", + "\"created_at\" desc" + ], + "isUnique": false + }, + "idx_runs_status": { + "name": "idx_runs_status", + "columns": [ + "status", + "\"created_at\" desc" + ], + "isUnique": false, + "where": "\"runs\".\"deleted_at\" is null" + }, + "idx_runs_cost": { + "name": "idx_runs_cost", + "columns": [ + "workflow_id", + "created_at", + "total_cost_microcents" + ], + "isUnique": false, + "where": "\"runs\".\"deleted_at\" is null" + }, + "idx_runs_created": { + "name": "idx_runs_created", + "columns": [ + "\"created_at\" desc", + "\"id\" desc" + ], + "isUnique": false, + "where": "\"runs\".\"deleted_at\" is null" + } + }, + "foreignKeys": { + "runs_workflow_id_workflows_id_fk": { + "name": "runs_workflow_id_workflows_id_fk", + "tableFrom": "runs", + "tableTo": "workflows", + "columnsFrom": [ + "workflow_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "runs_status_check": { + "name": "runs_status_check", + "value": "\"runs\".\"status\" in ('pending', 'running', 'paused', 'completed', 'failed', 'cancelled')" + }, + "runs_execution_mode_check": { + "name": "runs_execution_mode_check", + "value": "\"runs\".\"execution_mode\" in ('local', 'cloud', 'managed')" + } + } + }, + "session_messages": { + "name": "session_messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sequence_number": { + "name": "sequence_number", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_parts": { + "name": "content_parts", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finish_reason": { + "name": "finish_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cost_microcents": { + "name": "cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "compaction_dropped_through_sequence": { + "name": "compaction_dropped_through_sequence", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_session_messages_seq": { + "name": "idx_session_messages_seq", + "columns": [ + "session_id", + "sequence_number" + ], + "isUnique": true + }, + "idx_session_messages_session": { + "name": "idx_session_messages_session", + "columns": [ + "session_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_messages_session_id_agent_sessions_id_fk": { + "name": "session_messages_session_id_agent_sessions_id_fk", + "tableFrom": "session_messages", + "tableTo": "agent_sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_messages_model_id_model_catalog_id_fk": { + "name": "session_messages_model_id_model_catalog_id_fk", + "tableFrom": "session_messages", + "tableTo": "model_catalog", + "columnsFrom": [ + "model_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "step_executions": { + "name": "step_executions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "node_type": { + "name": "node_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "agent_snapshot": { + "name": "agent_snapshot", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempt_number": { + "name": "attempt_number", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "input_json": { + "name": "input_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "output_json": { + "name": "output_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_json": { + "name": "error_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cached_tokens": { + "name": "cached_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cost_microcents": { + "name": "cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_step_exec_run": { + "name": "idx_step_exec_run", + "columns": [ + "run_id", + "created_at" + ], + "isUnique": false + }, + "idx_step_exec_run_node": { + "name": "idx_step_exec_run_node", + "columns": [ + "run_id", + "node_id", + "attempt_number" + ], + "isUnique": false + }, + "idx_step_exec_agent": { + "name": "idx_step_exec_agent", + "columns": [ + "agent_id", + "\"created_at\" desc" + ], + "isUnique": false, + "where": "\"step_executions\".\"agent_id\" is not null" + }, + "idx_step_exec_model": { + "name": "idx_step_exec_model", + "columns": [ + "model_id", + "\"created_at\" desc" + ], + "isUnique": false, + "where": "\"step_executions\".\"model_id\" is not null" + }, + "idx_step_exec_cost": { + "name": "idx_step_exec_cost", + "columns": [ + "model_id", + "created_at", + "cost_microcents" + ], + "isUnique": false, + "where": "\"step_executions\".\"model_id\" is not null" + } + }, + "foreignKeys": { + "step_executions_run_id_runs_id_fk": { + "name": "step_executions_run_id_runs_id_fk", + "tableFrom": "step_executions", + "tableTo": "runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "step_executions_agent_id_agents_id_fk": { + "name": "step_executions_agent_id_agents_id_fk", + "tableFrom": "step_executions", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "step_executions_model_id_model_catalog_id_fk": { + "name": "step_executions_model_id_model_catalog_id_fk", + "tableFrom": "step_executions", + "tableTo": "model_catalog", + "columnsFrom": [ + "model_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "step_executions_status_check": { + "name": "step_executions_status_check", + "value": "\"step_executions\".\"status\" in ('pending', 'running', 'completed', 'failed', 'skipped')" + } + } + }, + "workflows": { + "name": "workflows", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "definition": { + "name": "definition", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_schema": { + "name": "input_schema", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "1" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_workflows_slug": { + "name": "idx_workflows_slug", + "columns": [ + "slug" + ], + "isUnique": true, + "where": "\"workflows\".\"deleted_at\" is null" + }, + "idx_workflows_active": { + "name": "idx_workflows_active", + "columns": [ + "is_active", + "\"updated_at\" desc" + ], + "isUnique": false, + "where": "\"workflows\".\"deleted_at\" is null" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": { + "idx_agent_sessions_status": { + "columns": { + "\"updated_at\" desc": { + "isExpression": true + } + } + }, + "idx_agent_sessions_agent": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_agent_sessions_updated": { + "columns": { + "\"updated_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_agents_active": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_runs_workflow": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_runs_status": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_runs_created": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_step_exec_agent": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_step_exec_model": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_workflows_active": { + "columns": { + "\"updated_at\" desc": { + "isExpression": true + } + } + } + } + } +} \ No newline at end of file diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index bf179db7..f0d0747a 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -50,6 +50,13 @@ "when": 1783125235969, "tag": "0006_mean_adam_destine", "breakpoints": true + }, + { + "idx": 7, + "version": "6", + "when": 1783272761773, + "tag": "0007_harsh_bug", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 81b99584..851d1833 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -159,4 +159,7 @@ export { type ModelCatalogStoreDeps, type ModelCatalogRecord, type ModelCatalogUpsert, + // Live-discovery cache (2.5.G, ADR-0064) — the picker/refresh listing projection + the bulk live-refresh input. + type ModelCatalogListing, + type ModelCatalogLiveModel, } from './model-catalog-store.js'; diff --git a/packages/db/src/model-catalog-store.test.ts b/packages/db/src/model-catalog-store.test.ts index 08379311..991b2201 100644 --- a/packages/db/src/model-catalog-store.test.ts +++ b/packages/db/src/model-catalog-store.test.ts @@ -312,3 +312,382 @@ describe('createModelCatalogStore (2.S — media routing + load-check reader)', expect(caught.cause).toBeInstanceOf(SyntaxError); }); }); + +describe('createModelCatalogStore (2.5.G / ADR-0064 — live-discovery cache)', () => { + let client: DbClient; + let store: ModelCatalogStore; + let providerStore: ProviderStore; + let providerId: string; + + beforeEach(() => { + client = createClient(':memory:'); + runMigrations(client.db); + let n = 0; + const deps = { + uuid: () => `00000000-0000-4000-8000-${String(++n).padStart(12, '0')}`, + now: () => TS_MS, + }; + providerStore = createProviderStore(client.db, deps); + providerId = providerStore.upsert({ + name: 'openai', + displayName: 'OpenAI', + baseUrl: 'https://api.openai.com/v1', + }).id; + store = createModelCatalogStore(client.db, deps); + }); + + afterEach(() => { + client.sqlite.close(); + }); + + it('a live model round-trips via replaceProviderModels → listByProvider (source live, stamp set, 0-context ⇒ undefined)', () => { + const REFRESH_TS = TS_MS + 60_000; + store.replaceProviderModels( + providerId, + [ + { + modelId: 'gpt-4o', + displayName: 'GPT-4o', + contextWindowTokens: 128_000, + maxOutputTokens: 16_384, + }, + // No context/output ⇒ stored as the `0` "unknown" sentinel ⇒ read back as undefined. + { modelId: 'o3', displayName: 'o3' }, + ], + REFRESH_TS, + ); + const listing = store.listByProvider(providerId); + // Deterministic order by model id. + expect(listing.map((m) => m.modelId)).toEqual(['gpt-4o', 'o3']); + + const gpt = listing.find((m) => m.modelId === 'gpt-4o'); + expect(gpt?.source).toBe('live'); + expect(gpt?.lastRefreshedAt).toBe(REFRESH_TS); + expect(gpt?.isActive).toBe(true); + expect(gpt?.contextWindowTokens).toBe(128_000); + expect(gpt?.maxOutputTokens).toBe(16_384); + // A live row carries no price (the static registry is the pricing authority) — integer µ¢ zeros. + expect(gpt?.inputCostPerMtokMicrocents).toBe(0); + expect(gpt?.outputCostPerMtokMicrocents).toBe(0); + expect(gpt?.cachedInputCostPerMtokMicrocents).toBe(0); + + const o3 = listing.find((m) => m.modelId === 'o3'); + expect(o3?.contextWindowTokens).toBeUndefined(); + expect(o3?.maxOutputTokens).toBeUndefined(); + expect(o3?.displayName).toBe('o3'); + expect(o3?.deprecationDate).toBeUndefined(); + }); + + it('replaceProviderModels falls back an empty displayName to the model id', () => { + store.replaceProviderModels(providerId, [{ modelId: 'bare-model', displayName: ' ' }], TS_MS); + expect(store.listByProvider(providerId)[0]?.displayName).toBe('bare-model'); + }); + + it('upserts new, soft-deactivates a vanished live row (never hard-deletes), reactivates a reappearing one, preserves user/static rows', () => { + const T1 = TS_MS + 1000; + const T2 = TS_MS + 2000; + const T3 = TS_MS + 3000; + + // A source='user' row (user pricing) and a source='static' row (a media-routing seed); NEITHER appears in + // any live list below — a refresh must leave both alone. + store.upsert({ + providerId, + modelId: 'user-priced', + displayName: 'User Priced', + contextWindowTokens: 8000, + maxOutputTokens: 4000, + source: 'user', + }); + // Distinctive pricing set directly (upsert doesn't expose the text-token cost columns) to prove preservation. + client.sqlite + .prepare('UPDATE model_catalog SET input_cost_per_mtok_microcents = ? WHERE model_id = ?') + .run(1234, 'user-priced'); + store.upsert({ + providerId, + modelId: 'seeded-generative', + displayName: 'Seeded Generative', + contextWindowTokens: 4096, + maxOutputTokens: 4096, + mediaSurface: 'generative', // source defaults to 'static' + }); + + // First refresh: models a + b. + store.replaceProviderModels( + providerId, + [ + { modelId: 'model-a', displayName: 'A', contextWindowTokens: 1000, maxOutputTokens: 500 }, + { modelId: 'model-b', displayName: 'B', contextWindowTokens: 2000, maxOutputTokens: 800 }, + ], + T1, + ); + expect(store.listByProvider(providerId).map((m) => m.modelId)).toEqual([ + 'model-a', + 'model-b', + 'seeded-generative', + 'user-priced', + ]); + const bIdBefore = client.db + .select() + .from(modelCatalog) + .where(eq(modelCatalog.modelId, 'model-b')) + .get()?.id; + + // Second refresh: a stays (updated in place), b vanishes, c appears. + store.replaceProviderModels( + providerId, + [ + { modelId: 'model-a', displayName: 'A2', contextWindowTokens: 1500, maxOutputTokens: 600 }, + { modelId: 'model-c', displayName: 'C', contextWindowTokens: 3000, maxOutputTokens: 900 }, + ], + T2, + ); + // b soft-deactivated ⇒ absent from the active listing; user + static rows still present. + expect(store.listByProvider(providerId).map((m) => m.modelId)).toEqual([ + 'model-a', + 'model-c', + 'seeded-generative', + 'user-priced', + ]); + const a = store.listByProvider(providerId).find((m) => m.modelId === 'model-a'); + expect(a?.displayName).toBe('A2'); // updated in place + expect(a?.contextWindowTokens).toBe(1500); + expect(a?.lastRefreshedAt).toBe(T2); + + // b was NOT hard-deleted — same row still exists, just deactivated (deleted_at still NULL, source live). + const bRow = client.db + .select() + .from(modelCatalog) + .where(eq(modelCatalog.modelId, 'model-b')) + .get(); + expect(bRow?.id).toBe(bIdBefore); + expect(bRow?.isActive).toBe(false); + expect(bRow?.deletedAt).toBeNull(); + expect(bRow?.source).toBe('live'); + + // user + static rows preserved: source unchanged, active, pricing + media routing intact. + const userRow = client.db + .select() + .from(modelCatalog) + .where(eq(modelCatalog.modelId, 'user-priced')) + .get(); + expect(userRow?.source).toBe('user'); + expect(userRow?.isActive).toBe(true); + expect(userRow?.inputCostPerMtokMicrocents).toBe(1234); + const staticRow = client.db + .select() + .from(modelCatalog) + .where(eq(modelCatalog.modelId, 'seeded-generative')) + .get(); + expect(staticRow?.source).toBe('static'); + expect(staticRow?.isActive).toBe(true); + expect(store.resolveMediaSurface('seeded-generative')).toBe('generative'); + + // Third refresh: b REAPPEARS ⇒ reactivated, reusing the SAME row id (FK stability), not a duplicate insert. + store.replaceProviderModels( + providerId, + [ + { modelId: 'model-a', displayName: 'A3', contextWindowTokens: 1500, maxOutputTokens: 600 }, + { + modelId: 'model-b', + displayName: 'B-back', + contextWindowTokens: 2222, + maxOutputTokens: 811, + }, + { modelId: 'model-c', displayName: 'C', contextWindowTokens: 3000, maxOutputTokens: 900 }, + ], + T3, + ); + const bReactivated = store.listByProvider(providerId).find((m) => m.modelId === 'model-b'); + expect(bReactivated?.displayName).toBe('B-back'); + expect(bReactivated?.lastRefreshedAt).toBe(T3); + const bRows = client.db + .select() + .from(modelCatalog) + .where(eq(modelCatalog.modelId, 'model-b')) + .all(); + expect(bRows).toHaveLength(1); // reused, not duplicated + expect(bRows[0]?.id).toBe(bIdBefore); + }); + + it('never clobbers an existing user/static row even when the live list names the same model id (the collision invariant)', () => { + store.upsert({ + providerId, + modelId: 'shared-id-user', + displayName: 'User Row', + contextWindowTokens: 8000, + maxOutputTokens: 4000, + source: 'user', + }); + client.sqlite + .prepare('UPDATE model_catalog SET input_cost_per_mtok_microcents = ? WHERE model_id = ?') + .run(999, 'shared-id-user'); + store.upsert({ + providerId, + modelId: 'shared-id-static', + displayName: 'Static Seed', + contextWindowTokens: 4096, + maxOutputTokens: 4096, + mediaSurface: 'generative', + capabilities: { media: { outputCombinations: [['image']] } }, + }); + + const REFRESH_TS = TS_MS + 5000; + // The live list NAMES both existing ids (a collision) plus a genuinely new one. + store.replaceProviderModels( + providerId, + [ + { + modelId: 'shared-id-user', + displayName: 'LIVE Overwrite Attempt', + contextWindowTokens: 111, + maxOutputTokens: 22, + }, + { + modelId: 'shared-id-static', + displayName: 'LIVE Overwrite Attempt', + contextWindowTokens: 111, + maxOutputTokens: 22, + }, + { + modelId: 'brand-new-live', + displayName: 'New', + contextWindowTokens: 500, + maxOutputTokens: 200, + }, + ], + REFRESH_TS, + ); + + // The user row is untouched: source, pricing, display, context all preserved; no refresh stamp. + const userRow = client.db + .select() + .from(modelCatalog) + .where(eq(modelCatalog.modelId, 'shared-id-user')) + .get(); + expect(userRow?.source).toBe('user'); + expect(userRow?.displayName).toBe('User Row'); + expect(userRow?.inputCostPerMtokMicrocents).toBe(999); + expect(userRow?.contextWindowTokens).toBe(8000); + expect(userRow?.lastRefreshedAt).toBeNull(); + + // The static media-seed row is untouched: media routing intact. + const staticRow = client.db + .select() + .from(modelCatalog) + .where(eq(modelCatalog.modelId, 'shared-id-static')) + .get(); + expect(staticRow?.source).toBe('static'); + expect(staticRow?.mediaSurface).toBe('generative'); + expect(staticRow?.displayName).toBe('Static Seed'); + expect(store.resolveMediaSurface('shared-id-static')).toBe('generative'); + + // Only the genuinely new id becomes a live row. + const brandNew = store.listByProvider(providerId).find((m) => m.modelId === 'brand-new-live'); + expect(brandNew?.source).toBe('live'); + expect(brandNew?.lastRefreshedAt).toBe(REFRESH_TS); + }); + + it('providerRefreshedAt returns the max live stamp, or undefined when no live rows exist', () => { + expect(store.providerRefreshedAt(providerId)).toBeUndefined(); // empty provider + // A static seed alone does NOT count — only source='live' rows contribute. + store.upsert({ + providerId, + modelId: 'seed', + displayName: 'Seed', + contextWindowTokens: 100, + maxOutputTokens: 50, + }); + expect(store.providerRefreshedAt(providerId)).toBeUndefined(); + const T1 = TS_MS + 1000; + store.replaceProviderModels(providerId, [{ modelId: 'm1', displayName: 'M1' }], T1); + expect(store.providerRefreshedAt(providerId)).toBe(T1); + const T2 = TS_MS + 9999; + store.replaceProviderModels( + providerId, + [ + { modelId: 'm1', displayName: 'M1' }, + { modelId: 'm2', displayName: 'M2' }, + ], + T2, + ); + expect(store.providerRefreshedAt(providerId)).toBe(T2); + }); + + it('an empty live list soft-deactivates all of a provider live rows (never touches user/static)', () => { + store.upsert({ + providerId, + modelId: 'keep-static', + displayName: 'Keep', + contextWindowTokens: 100, + maxOutputTokens: 50, + }); + store.replaceProviderModels( + providerId, + [{ modelId: 'gone', displayName: 'Gone', contextWindowTokens: 1, maxOutputTokens: 1 }], + TS_MS + 1, + ); + expect(store.listByProvider(providerId).map((m) => m.modelId)).toEqual(['gone', 'keep-static']); + // A refresh returning nothing — the whole live set vanishes; the static row survives. + store.replaceProviderModels(providerId, [], TS_MS + 2); + expect(store.listByProvider(providerId).map((m) => m.modelId)).toEqual(['keep-static']); + expect(store.providerRefreshedAt(providerId)).toBeUndefined(); // no active live rows left + }); + + it('listAll spans providers, active + non-deleted only, ordered by model id', () => { + const otherProviderId = providerStore.upsert({ + name: 'anthropic', + displayName: 'Anthropic', + baseUrl: 'https://api.anthropic.com', + }).id; + store.replaceProviderModels( + providerId, + [{ modelId: 'zzz-openai', displayName: 'Z' }], + TS_MS + 1, + ); + store.replaceProviderModels( + otherProviderId, + [{ modelId: 'aaa-claude', displayName: 'A' }], + TS_MS + 1, + ); + expect(store.listAll().map((m) => m.modelId)).toEqual(['aaa-claude', 'zzz-openai']); + }); + + it('fail-closed: a tampered source value degrades to static at the read boundary (coerceModelCatalogSource)', () => { + store.replaceProviderModels( + providerId, + [{ modelId: 'm1', displayName: 'M1', contextWindowTokens: 100, maxOutputTokens: 50 }], + TS_MS + 1, + ); + client.sqlite.prepare("UPDATE model_catalog SET source = 'bogus' WHERE model_id = ?").run('m1'); + expect(store.listByProvider(providerId).find((m) => m.modelId === 'm1')?.source).toBe('static'); + }); + + it('the media-routing reader is unaffected by a coexisting live refresh', () => { + store.upsert({ + providerId, + modelId: 'imagen', + displayName: 'Imagen', + contextWindowTokens: 4096, + maxOutputTokens: 4096, + mediaSurface: 'generative', + mediaImageCostMicrocents: 5000, + }); + store.replaceProviderModels( + providerId, + [ + { + modelId: 'text-model', + displayName: 'Text', + contextWindowTokens: 1000, + maxOutputTokens: 500, + }, + ], + TS_MS + 1, + ); + expect(store.resolveMediaSurface('imagen')).toBe('generative'); + expect(store.resolveMediaSurface('text-model')).toBe('chat'); // a live row defaults to the chat surface + const rec = store.getByModelId('imagen'); + expect(rec?.mediaSurface).toBe('generative'); + expect(rec?.mediaImageCostMicrocents).toBe(5000); + }); +}); diff --git a/packages/db/src/model-catalog-store.ts b/packages/db/src/model-catalog-store.ts index 23c95a4f..3d5d617c 100644 --- a/packages/db/src/model-catalog-store.ts +++ b/packages/db/src/model-catalog-store.ts @@ -1,5 +1,10 @@ -import { MEDIA_SURFACES, type MediaSurface } from '@relavium/shared'; -import { and, asc, eq, isNull } from 'drizzle-orm'; +import { + MEDIA_SURFACES, + MODEL_CATALOG_SOURCES, + type MediaSurface, + type ModelCatalogSource, +} from '@relavium/shared'; +import { and, asc, eq, isNull, notInArray, sql } from 'drizzle-orm'; import type { Db } from './client.js'; import { modelCatalog, type ModelCatalogRow, type NewModelCatalogRow } from './schema.js'; @@ -60,6 +65,50 @@ export interface ModelCatalogUpsert { readonly mediaImageCostMicrocents?: number | null; readonly mediaAudioCostMicrocents?: number | null; readonly mediaVideoCostMicrocents?: number | null; + /** The provenance discriminant ([ADR-0064] §4). OMITTED ⇒ `'static'` (a hardcoded seed), so every existing + * media-routing caller is unchanged; the live refresh writes `'live'`. */ + readonly source?: ModelCatalogSource; + /** The epoch-ms this row was live-refreshed (ADR-0064 §5). OMITTED ⇒ `null` (a static/user or never-refreshed row). */ + readonly lastRefreshedAt?: number; +} + +/** + * The picker/refresh read view of a catalog row ([ADR-0064] §4/§5) — DISTINCT from the narrow media-routing + * {@link ModelCatalogRecord}: this is the discovery/economics projection the `/models` picker and the + * `models refresh` reporter consume (id + display + context/limits + the text-token cost columns + provenance + * + freshness). A stored context/maxOutput of `0` is the NOT-NULL "unknown" sentinel and reads back as + * `undefined` (ADR-0064 §3, matching `@relavium/llm`'s `ModelListing`, which omits an unknown limit). Cost + * columns are integer µ¢ (never float); they are `0` for a `live` row (the static registry is the pricing + * authority — ADR-0064 §6) and carry user pricing only on a `source='user'` row (ADR-0065). + */ +export interface ModelCatalogListing { + readonly modelId: string; + readonly providerId: string; + readonly displayName: string; + /** `undefined` when unknown (a stored `0` sentinel, ADR-0064 §3). */ + readonly contextWindowTokens?: number; + /** `undefined` when unknown (a stored `0` sentinel, ADR-0064 §3). */ + readonly maxOutputTokens?: number; + readonly inputCostPerMtokMicrocents: number; + readonly outputCostPerMtokMicrocents: number; + readonly cachedInputCostPerMtokMicrocents: number; + /** Live-discovered deprecation epoch-ms (ADR-0064 §7); `undefined` when none. */ + readonly deprecationDate?: number; + /** Provenance, validated at the read boundary (a foreign value degrades to `'static'`). */ + readonly source: ModelCatalogSource; + /** The last live-refresh epoch-ms; `undefined` for a static/user or never-refreshed row. */ + readonly lastRefreshedAt?: number; + readonly isActive: boolean; +} + +/** One model to seed in a bulk live refresh ({@link ModelCatalogStore.replaceProviderModels}) — the discovery + * half only. An absent `contextWindowTokens`/`maxOutputTokens` stores as the `0` "unknown" sentinel; an empty + * `displayName` falls back to the model id. Pricing/media columns are NOT set by a live refresh (ADR-0064 §6). */ +export interface ModelCatalogLiveModel { + readonly modelId: string; + readonly displayName: string; + readonly contextWindowTokens?: number; + readonly maxOutputTokens?: number; } export interface ModelCatalogStoreDeps { @@ -80,6 +129,28 @@ export interface ModelCatalogStore { /** Seed/replace a catalog row (by provider + model) — used by the generative acceptance fixture and a future * provider-sync; the store mints the id + timestamps. */ upsert: (input: ModelCatalogUpsert) => ModelCatalogRecord; + /** Active, non-deleted rows for one provider, ordered deterministically (by model id) — the `/models` picker / + * `models refresh` view ([ADR-0064] §4). Returns the wide {@link ModelCatalogListing}, NOT the media record. */ + listByProvider: (providerId: string) => ModelCatalogListing[]; + /** Active, non-deleted rows across every provider, ordered by model id — the cross-provider `/models` catalog. */ + listAll: () => ModelCatalogListing[]; + /** + * Bulk live-upsert for one provider's discovered models ([ADR-0064] §5), in ONE transaction: each `rows` entry + * is upserted as `source='live'` with `lastRefreshedAt=now` (reusing the existing (provider, model) row id so + * the FK graph stays stable, reactivating a previously-deactivated live row); every currently-active + * `source='live'` row of THIS provider whose model id is ABSENT from `rows` is SOFT-DEACTIVATED (`isActive=false`, + * `deletedAt` untouched). `source='user'`/`source='static'` rows are NEVER touched (the ADR-0065 §1 "a refresh + * never clobbers a user row" invariant + the media-routing seed's integrity), and nothing is ever hard-DELETED + * (`model_catalog.id` is an FK target from five tables). + */ + replaceProviderModels: ( + providerId: string, + rows: ReadonlyArray, + now: number, + ) => void; + /** The freshness read for the TTL ([ADR-0064] §5): the max `lastRefreshedAt` among a provider's active + * `source='live'` rows, or `undefined` when the provider has none. */ + providerRefreshedAt: (providerId: string) => number | undefined; } /** @@ -92,6 +163,50 @@ function coerceMediaSurface(value: string): MediaSurface { return MEDIA_SURFACES.find((surface) => surface === value) ?? 'chat'; } +/** + * Validate the stored `source` against the closed `MODEL_CATALOG_SOURCES` set. Like `media_surface`, the column + * is `$type` but carries no DB CHECK (a SQLite `ALTER ADD` limitation, schema.ts), so a + * tampered/foreign value must not be trusted. A non-member degrades to `'static'` — the safe, lowest-provenance + * default: a foreign value is never mistaken for authoritative `'live'` discovery or a `'user'` pricing override. + * Mirrors {@link coerceMediaSurface} (fail-closed toward the safe default). + */ +function coerceModelCatalogSource(value: string): ModelCatalogSource { + return MODEL_CATALOG_SOURCES.find((source) => source === value) ?? 'static'; +} + +/** + * Map a stored context/maxOutput token count to the picker view: `0` is the NOT-NULL "unknown" sentinel + * (ADR-0064 §3) and reads back as `undefined`, matching `@relavium/llm`'s `ModelListing` (which omits an + * unknown limit rather than storing a `0`). A positive value passes through unchanged. + */ +function tokensOrUndefined(value: number): number | undefined { + return value > 0 ? value : undefined; +} + +/** + * The wide picker/refresh projection ({@link ModelCatalogListing}) — DISTINCT from {@link fromRow} (the narrow + * media-routing record). Validates `source` at the read boundary and applies the `0 ⇒ undefined` token + * convention. Optional fields are OMITTED (never assigned `undefined`) under `exactOptionalPropertyTypes`. + */ +function toListing(row: ModelCatalogRow): ModelCatalogListing { + const contextWindowTokens = tokensOrUndefined(row.contextWindowTokens); + const maxOutputTokens = tokensOrUndefined(row.maxOutputTokens); + return { + modelId: row.modelId, + providerId: row.providerId, + displayName: row.displayName, + ...(contextWindowTokens === undefined ? {} : { contextWindowTokens }), + ...(maxOutputTokens === undefined ? {} : { maxOutputTokens }), + inputCostPerMtokMicrocents: row.inputCostPerMtokMicrocents, + outputCostPerMtokMicrocents: row.outputCostPerMtokMicrocents, + cachedInputCostPerMtokMicrocents: row.cachedInputCostPerMtokMicrocents, + ...(row.deprecationDate === null ? {} : { deprecationDate: row.deprecationDate }), + source: coerceModelCatalogSource(row.source), + ...(row.lastRefreshedAt === null ? {} : { lastRefreshedAt: row.lastRefreshedAt }), + isActive: row.isActive, + }; +} + /** * A `model_catalog.capabilities` column that is not a JSON object — invalid JSON, or valid JSON that is `null` / * an array / a scalar. A typed DOMAIN fault (mirrors {@link MediaWriteError}/`MediaEgressError`), DISTINCT from an @@ -205,6 +320,10 @@ export function createModelCatalogStore(db: Db, deps: ModelCatalogStoreDeps): Mo mediaImageCostMicrocents: input.mediaImageCostMicrocents ?? null, mediaAudioCostMicrocents: input.mediaAudioCostMicrocents ?? null, mediaVideoCostMicrocents: input.mediaVideoCostMicrocents ?? null, + // Provenance + freshness (ADR-0064 §4/§5). Omitted ⇒ `'static'` / `null`, so every existing + // media-routing caller (which passes neither) writes a static, never-refreshed row unchanged. + source: input.source ?? 'static', + lastRefreshedAt: input.lastRefreshedAt ?? null, // An upsert (re)activates the row: keep `isActive` in lockstep with `activeRow`'s `isActive = true` // filter so a re-upserted, previously-deactivated row is reachable again and the returned record never // disagrees with a subsequent `getByModelId` (which filters inactive rows out). @@ -231,5 +350,132 @@ export function createModelCatalogStore(db: Db, deps: ModelCatalogStoreDeps): Mo } return fromRow(row); }, + + listByProvider: (providerId) => + db + .select() + .from(modelCatalog) + .where( + and( + eq(modelCatalog.providerId, providerId), + eq(modelCatalog.isActive, true), + isNull(modelCatalog.deletedAt), + ), + ) + // Deterministic by model id, with `id` as a stable tiebreak (the `activeRow`/`run-history-store` + // convention) so the picker order never flips between reads for two rows sharing a model id. + .orderBy(asc(modelCatalog.modelId), asc(modelCatalog.id)) + .all() + .map(toListing), + + listAll: () => + db + .select() + .from(modelCatalog) + .where(and(eq(modelCatalog.isActive, true), isNull(modelCatalog.deletedAt))) + .orderBy(asc(modelCatalog.modelId), asc(modelCatalog.id)) + .all() + .map(toListing), + + replaceProviderModels: (providerId, rows, now) => { + db.transaction(() => { + for (const input of rows) { + const displayName = input.displayName.trim() === '' ? input.modelId : input.displayName; + // `0` is the NOT-NULL "unknown" sentinel (ADR-0064 §3) — an absent live limit stores as 0. + const contextWindowTokens = input.contextWindowTokens ?? 0; + const maxOutputTokens = input.maxOutputTokens ?? 0; + // Find the existing (provider, model) row (deletedAt IS NULL — the partial-unique scope), whether it + // is active or soft-deactivated. Reuse its id so FK targets stay stable. + const existing = db + .select() + .from(modelCatalog) + .where( + and( + eq(modelCatalog.providerId, providerId), + eq(modelCatalog.modelId, input.modelId), + isNull(modelCatalog.deletedAt), + ), + ) + .get(); + if (existing !== undefined && existing.source !== 'live') { + // A `source='user'` (user pricing, ADR-0065 §1) or `source='static'` (a media-routing seed — + // media_surface/capabilities/rates) row already represents this model. A live refresh must NEVER + // clobber it (that would drop user pricing or regress media routing), so it is left UNTOUCHED and, + // being non-`live`, is also never deactivated below — the model stays represented by its own row. + continue; + } + if (existing === undefined) { + const row: NewModelCatalogRow = { + id: deps.uuid(), + providerId, + modelId: input.modelId, + displayName, + contextWindowTokens, + maxOutputTokens, + source: 'live', + lastRefreshedAt: now, + isActive: true, + createdAt: now, + updatedAt: now, + }; + db.insert(modelCatalog).values(row).run(); + } else { + // Reactivate + refresh the existing live row in place (id/created_at/FK refs preserved); only the + // discovery columns + provenance/freshness are written — pricing/media columns are left as-is. + db.update(modelCatalog) + .set({ + displayName, + contextWindowTokens, + maxOutputTokens, + source: 'live', + lastRefreshedAt: now, + isActive: true, + updatedAt: now, + }) + .where(eq(modelCatalog.id, existing.id)) + .run(); + } + } + // Soft-deactivate the vanished live rows: every currently-active `source='live'` row of THIS provider + // whose model id is absent from the new list. `isActive=false` with `deletedAt` untouched keeps the + // partial-unique slot occupied so a reappearing model reuses the SAME row (reactivated above). NEVER a + // hard-DELETE (FK target from five tables); NEVER touches `source='user'`/`source='static'`. + const incomingModelIds = rows.map((r) => r.modelId); + const deactivateScope = and( + eq(modelCatalog.providerId, providerId), + eq(modelCatalog.isActive, true), + eq(modelCatalog.source, 'live'), + isNull(modelCatalog.deletedAt), + ); + db.update(modelCatalog) + .set({ isActive: false, updatedAt: now }) + // An empty new list deactivates ALL of the provider's live rows (no `notInArray([])` — its semantics + // vary; the guard makes the "everything vanished" case explicit). + .where( + incomingModelIds.length === 0 + ? deactivateScope + : and(deactivateScope, notInArray(modelCatalog.modelId, incomingModelIds)), + ) + .run(); + }); + }, + + providerRefreshedAt: (providerId) => { + const row = db + .select({ max: sql`max(${modelCatalog.lastRefreshedAt})` }) + .from(modelCatalog) + .where( + and( + eq(modelCatalog.providerId, providerId), + eq(modelCatalog.isActive, true), + eq(modelCatalog.source, 'live'), + isNull(modelCatalog.deletedAt), + ), + ) + .get(); + // A bare aggregate returns one row; `max()` over no matching rows is NULL ⇒ undefined. + const max = row?.max ?? null; + return max === null ? undefined : max; + }, }; } diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 8cc04e47..c30c555c 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -10,6 +10,7 @@ import { type MediaModality, type MediaScopeKind, type MediaSurface, + type ModelCatalogSource, type RunStatus, type SessionStatus, } from '@relavium/shared'; @@ -126,6 +127,17 @@ export const modelCatalog = sqliteTable( supportsJsonMode: boolFlag('supports_json_mode', false), capabilities: jsonText('capabilities').notNull().default('{}'), deprecationDate: epochMs('deprecation_date'), + // Live-discovery cache provenance (2.5.G/ADR-0064 §4): `'static'` (a hardcoded capability/media seed — the + // default the media-routing `upsert` path writes), `'live'` (discovered via the seam's `listModels` — the + // bulk refresh writes it), `'user'` (user-supplied pricing, ADR-0065). A text column with a CONSTANT default + // is legal on `ALTER TABLE ADD` (the table ships EMPTY so existing rows are moot). Like `media_surface`, + // SQLite `ALTER ADD` carries no CHECK, so the closed `MODEL_CATALOG_SOURCES` set is validated at the store + // read boundary (`coerceModelCatalogSource`, model-catalog-store.ts) — a tampered value degrades to `'static'`. + source: text('source').$type().notNull().default('static'), + // The epoch-ms a live refresh last wrote this row (ADR-0064 §5 TTL freshness); NULLABLE — NULL for a + // static/user row or a never-refreshed row. Backs `providerRefreshedAt` (the max over a provider's active + // `source='live'` rows). Additive nullable ALTER ADD, like the `media_*_cost_microcents` columns (0003). + lastRefreshedAt: epochMs('last_refreshed_at'), isActive: boolFlag('is_active', true), deletedAt: epochMs('deleted_at'), createdAt: epochMs('created_at').notNull(), diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 9fc20dca..1b39f47f 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -280,6 +280,18 @@ export type LlmProviderId = (typeof LLM_PROVIDERS)[number]; export const PROVIDER_KINDS = ['anthropic', 'openai-compatible', 'gemini'] as const; export type ProviderKind = (typeof PROVIDER_KINDS)[number]; +/** + * The provenance discriminant of a `model_catalog` row ([ADR-0064] §4) — the live-discovery cache's + * `source` column. `static` is a hardcoded capability/media seed (the media-routing upsert path's default); + * `live` is discovered via the seam's `listModels` (the refresh writes it); `user` is user-supplied pricing + * ([ADR-0065], a later step). A closed set mirroring {@link LLM_PROVIDERS}/{@link MEDIA_SURFACES}; because + * SQLite `ALTER TABLE ADD` cannot carry a CHECK, `@relavium/db` validates it at the store read boundary + * (`coerceModelCatalogSource`, degrading a foreign value to the safe `static` default). Lives here so the DB + * layer derives the vocabulary from `@relavium/shared`, never restating it. + */ +export const MODEL_CATALOG_SOURCES = ['static', 'live', 'user'] as const; +export type ModelCatalogSource = (typeof MODEL_CATALOG_SOURCES)[number]; + /** * The three filesystem permission tiers (built-in-tools.md). The canonical vocabulary * for the config `fs_scope` (config-spec.md) and a session's `fsScopeTier` From 7d3aca79514b190e8fb8c7de96b42c87592c64cb Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 20:55:50 +0300 Subject: [PATCH 10/68] =?UTF-8?q?fix(db):=202.5.G=20S4=20Opus-review=20?= =?UTF-8?q?=E2=80=94=20activeRow=20source-rank=20hardening=20+=20mutation-?= =?UTF-8?q?coverage=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opus's round confirmed the store code correct; 3 test-coverage gaps + 1 plausible pre-existing reader ambiguity (hardened). - activeRow (used by resolveMediaSurface + getByModelId) resolved by modelId alone — provider-agnostic + source-blind — so a source='live' chat row could out-sort and silently SHADOW a source='static' generative media seed sharing that model id (across providers), forcing 'chat' routing. Added a source-rank tiebreaker AHEAD of createdAt: a live row ranks BELOW a static/user row (CASE WHEN source='live' THEN 1 ELSE 0), so a generative seed always wins. Two static rows stay rank 0 -> fall through to createdAt/id unchanged; only the live-vs-non-live collision changes. (The S2 chat-only filter already blocks the acute media-model case; this closes the ambiguity regardless.) Regression test: a static generative seed vs a cross-provider live chat row with an EARLIER createdAt -> resolveMediaSurface returns 'generative' (a bare ordering would fail). - Tests (19->23): distinct non-zero input/output/cached µ¢ asserted THROUGH listByProvider (pins the toListing cost-column mapping against a swap); providerRefreshedAt cross-provider isolation + a non-live stamp not contributing; toListing deprecationDate positive branch via a distinctive raw-SQL date. The 2.S media tests + the media-routing record shape + upsert media semantics are unchanged (single-row / two-static-row resolution identical; only a live/non-live collision hardened). Verified: pnpm turbo run lint typecheck test build — 24/24 green (@relavium/db model-catalog 23 pass); prettier clean. Refs: ADR-0064, ADR-0044 Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/db/src/model-catalog-store.test.ts | 119 ++++++++++++++++++++ packages/db/src/model-catalog-store.ts | 20 +++- 2 files changed, 134 insertions(+), 5 deletions(-) diff --git a/packages/db/src/model-catalog-store.test.ts b/packages/db/src/model-catalog-store.test.ts index 991b2201..ee16b710 100644 --- a/packages/db/src/model-catalog-store.test.ts +++ b/packages/db/src/model-catalog-store.test.ts @@ -690,4 +690,123 @@ describe('createModelCatalogStore (2.5.G / ADR-0064 — live-discovery cache)', expect(rec?.mediaSurface).toBe('generative'); expect(rec?.mediaImageCostMicrocents).toBe(5000); }); + + it('source-rank: a static generative seed wins over a cross-provider live chat row for the same model id (no media shadow)', () => { + // Provider A holds a source='static' generative media seed for the model id (createdAt = TS_MS). + store.upsert({ + providerId, + modelId: 'shared-media-id', + displayName: 'Static Generative Seed', + contextWindowTokens: 4096, + maxOutputTokens: 4096, + mediaSurface: 'generative', + }); + // Provider B publishes a source='live' row (media_surface defaults to 'chat') for the SAME model id, with an + // EARLIER createdAt so a bare `asc(createdAt)` ordering would resolve to THIS live chat row and shadow the + // generative seed — silently disabling generateMedia() routing. The source-rank tiebreaker must keep the + // static seed (rank 0) authoritative over the live row (rank 1) regardless of createdAt. + const providerB = providerStore.upsert({ + name: 'anthropic', + displayName: 'Anthropic', + baseUrl: 'https://api.anthropic.com', + }).id; + store.replaceProviderModels( + providerB, + [{ modelId: 'shared-media-id', displayName: 'Live Chat Model' }], + TS_MS - 1000, // earlier createdAt than the static seed + ); + // Both rows are active for the shared id; the static seed still wins for BOTH read paths (routing + record). + expect(store.resolveMediaSurface('shared-media-id')).toBe('generative'); + expect(store.getByModelId('shared-media-id')?.mediaSurface).toBe('generative'); + expect(store.getByModelId('shared-media-id')?.providerId).toBe(providerId); + }); + + it('toListing maps the three text-token cost columns to distinct fields (catches a copy/paste swap)', () => { + store.upsert({ + providerId, + modelId: 'priced-model', + displayName: 'Priced', + contextWindowTokens: 8000, + maxOutputTokens: 4000, + source: 'user', + }); + // `upsert` does not expose the text-token cost columns; set DISTINCT non-zero µ¢ directly so a swapped + // (e.g. output←input) column mapping in `toListing` reads a different value and fails — mirroring the + // media-cost columns' distinct-value discipline. Read them back THROUGH the listing path (not raw drizzle). + client.sqlite + .prepare( + 'UPDATE model_catalog SET input_cost_per_mtok_microcents = ?, output_cost_per_mtok_microcents = ?, cached_input_cost_per_mtok_microcents = ? WHERE model_id = ?', + ) + .run(1234, 5678, 42, 'priced-model'); + const listing = store.listByProvider(providerId).find((m) => m.modelId === 'priced-model'); + expect(listing?.inputCostPerMtokMicrocents).toBe(1234); + expect(listing?.outputCostPerMtokMicrocents).toBe(5678); + expect(listing?.cachedInputCostPerMtokMicrocents).toBe(42); + }); + + it('providerRefreshedAt isolates by provider and ignores non-live rows', () => { + const providerB = providerStore.upsert({ + name: 'anthropic', + displayName: 'Anthropic', + baseUrl: 'https://api.anthropic.com', + }).id; + const TA = TS_MS + 1000; + const TB = TS_MS + 7777; + store.replaceProviderModels(providerId, [{ modelId: 'a1', displayName: 'A1' }], TA); + store.replaceProviderModels(providerB, [{ modelId: 'b1', displayName: 'B1' }], TB); + // Cross-provider isolation: each provider reports ITS OWN max live stamp (the providerId filter). Dropping + // that filter would return the global max (TB) for provider A. + expect(store.providerRefreshedAt(providerId)).toBe(TA); + expect(store.providerRefreshedAt(providerB)).toBe(TB); + + // A non-live row must NOT contribute even if it carries an (injected) last_refreshed_at newer than any live + // stamp — the source='live' filter excludes it. Provider A still reports its live max (TA), not the injection. + store.upsert({ + providerId, + modelId: 'static-seed', + displayName: 'Static', + contextWindowTokens: 100, + maxOutputTokens: 50, + }); + client.sqlite + .prepare('UPDATE model_catalog SET last_refreshed_at = ? WHERE model_id = ?') + .run(TS_MS + 999_999, 'static-seed'); + expect(store.providerRefreshedAt(providerId)).toBe(TA); + + // A provider whose ONLY row is non-live (even carrying an injected stamp) has no live max ⇒ undefined, + // not the injected non-live stamp. + const providerC = providerStore.upsert({ + name: 'gemini', + displayName: 'Gemini', + baseUrl: 'https://generativelanguage.googleapis.com', + }).id; + store.upsert({ + providerId: providerC, + modelId: 'c-static', + displayName: 'C Static', + contextWindowTokens: 100, + maxOutputTokens: 50, + }); + client.sqlite + .prepare('UPDATE model_catalog SET last_refreshed_at = ? WHERE model_id = ?') + .run(TS_MS + 555, 'c-static'); + expect(store.providerRefreshedAt(providerC)).toBeUndefined(); + }); + + it('toListing surfaces a non-null deprecationDate (the positive branch, distinct from other timestamps)', () => { + const REFRESH_TS = TS_MS + 60_000; + store.replaceProviderModels( + providerId, + [{ modelId: 'soon-gone', displayName: 'Soon Gone' }], + REFRESH_TS, + ); + // A DISTINCTIVE deprecation epoch-ms, distinct from createdAt (REFRESH_TS) and lastRefreshedAt (REFRESH_TS) + // so a `toListing` mapping that read created_at/last_refreshed_at by mistake reads a different value and fails. + const DEPRECATION_TS = TS_MS + 424_242; + client.sqlite + .prepare('UPDATE model_catalog SET deprecation_date = ? WHERE model_id = ?') + .run(DEPRECATION_TS, 'soon-gone'); + const listing = store.listByProvider(providerId).find((m) => m.modelId === 'soon-gone'); + expect(listing?.deprecationDate).toBe(DEPRECATION_TS); + }); }); diff --git a/packages/db/src/model-catalog-store.ts b/packages/db/src/model-catalog-store.ts index 3d5d617c..4bbb07d2 100644 --- a/packages/db/src/model-catalog-store.ts +++ b/packages/db/src/model-catalog-store.ts @@ -262,10 +262,16 @@ function fromRow(row: ModelCatalogRow): ModelCatalogRecord { /** Wire a {@link ModelCatalogStore} over a `@relavium/db` connection. */ export function createModelCatalogStore(db: Db, deps: ModelCatalogStoreDeps): ModelCatalogStore { - // The earliest active, non-deleted row for a (non-unique-alone) model id — `model_catalog` is unique on - // (provider, model), so a model offered by two providers yields more than one active row. `asc(createdAt)` - // with a stable `asc(id)` tiebreaker (the `run-history-store.ts` convention) keeps the resolved row — hence - // the routing surface + capability record — deterministic across reads even when two rows share a createdAt. + // The authoritative active, non-deleted row for a (non-unique-alone) model id — `model_catalog` is unique on + // (provider, model), so a model offered by two providers yields more than one active row. A SOURCE-RANK + // tiebreaker orders FIRST: a `source='live'` row (rank 1) always loses to a non-live `source='static'`/ + // `'user'` row (rank 0) for the same model id. For media/capability resolution the static/user row is + // AUTHORITATIVE over live — so a `source='live'` chat row (media_surface default 'chat', e.g. a cross-provider + // live-discovery hit) can never sort ahead of and silently shadow a `source='static'` generative media seed + // (or a `source='user'` capability/pricing row) and disable `generateMedia()` routing. Within a rank, + // `asc(createdAt)` with a stable `asc(id)` tiebreaker (the `run-history-store.ts` convention) keeps the + // resolved row — hence the routing surface + capability record — deterministic across reads even when two + // rows share a createdAt (two static rows stay rank 0 = 0 and fall through to createdAt/id, unchanged). const activeRow = (modelId: string): ModelCatalogRow | undefined => db .select() @@ -277,7 +283,11 @@ export function createModelCatalogStore(db: Db, deps: ModelCatalogStoreDeps): Mo isNull(modelCatalog.deletedAt), ), ) - .orderBy(asc(modelCatalog.createdAt), asc(modelCatalog.id)) + .orderBy( + sql`CASE WHEN ${modelCatalog.source} = 'live' THEN 1 ELSE 0 END`, + asc(modelCatalog.createdAt), + asc(modelCatalog.id), + ) .get(); const rowById = (id: string): ModelCatalogRow | undefined => From 59cf718a93de1db8981cb3bc38dc8161458c8960 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 21:08:08 +0300 Subject: [PATCH 11/68] =?UTF-8?q?fix(db):=202.5.G=20S4=20Sonnet-review=20?= =?UTF-8?q?=E2=80=94=20upsert=20must=20not=20demote=20a=20live=20row's=20p?= =?UTF-8?q?rovenance/freshness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sonnet's fresh round caught a real store-contract bug Opus missed (found by two dimensions, HIGH + MEDIUM) + one test gap. - upsert() built its `shared` write object with `source: input.source ?? 'static'` and `lastRefreshedAt: input.lastRefreshedAt ?? null` UNCONDITIONALLY, for both the insert AND the update branch. So calling upsert() on an existing source='live' row without a source (exactly the shape the store's own JSDoc earmarks for a future provider-sync) silently DEMOTED it to 'static' and nulled its stamp — dropping it from providerRefreshedAt's live aggregate, inverting the source-rank tiebreaker, and (via replaceProviderModels's existing.source !== 'live' guard) freezing it out of future live refreshes. Latent today (no wired caller), but a real violation of the "never clobber" invariant, which must hold symmetrically for both write entry points. Fix: default source/lastRefreshedAt from the EXISTING row (source: input.source ?? existing?.source ?? 'static'; lastRefreshedAt: input.lastRefreshedAt ?? existing?.lastRefreshedAt ?? null) — a true insert (existing undefined) still falls to 'static'/null, mirroring the existing `existing?.id ?? deps.uuid()` pattern. Tests (23->25): a regression proving upsert() preserves source='live'+lastRefreshedAt when omitted (while a true insert still defaults to static/never-refreshed); and listByProvider/listAll excluding a soft-DELETED (deletedAt) row (distinct from the isActive=false path — a refactor dropping isNull(deletedAt) from the listing queries would otherwise stay green). Verified: pnpm turbo run lint typecheck test build — 24/24 green (@relavium/db model-catalog 23->25); prettier clean. Refs: ADR-0064, ADR-0065 Co-Authored-By: Claude Opus 4.8 (1M context) --- packages/db/src/model-catalog-store.test.ts | 51 +++++++++++++++++++++ packages/db/src/model-catalog-store.ts | 12 +++-- 2 files changed, 59 insertions(+), 4 deletions(-) diff --git a/packages/db/src/model-catalog-store.test.ts b/packages/db/src/model-catalog-store.test.ts index ee16b710..0c813d05 100644 --- a/packages/db/src/model-catalog-store.test.ts +++ b/packages/db/src/model-catalog-store.test.ts @@ -809,4 +809,55 @@ describe('createModelCatalogStore (2.5.G / ADR-0064 — live-discovery cache)', const listing = store.listByProvider(providerId).find((m) => m.modelId === 'soon-gone'); expect(listing?.deprecationDate).toBe(DEPRECATION_TS); }); + + it('upsert() does NOT demote an existing live row: source/lastRefreshedAt preserved when omitted', () => { + const REFRESH_TS = TS_MS + 12_345; + store.replaceProviderModels( + providerId, + [{ modelId: 'was-live', displayName: 'Was Live' }], + REFRESH_TS, + ); + // A provider-sync-style upsert that patches display fields but OMITS source/lastRefreshedAt must not + // demote the live row back to 'static' or null its stamp (the "never clobber" invariant). + store.upsert({ + providerId, + modelId: 'was-live', + displayName: 'Was Live (patched)', + contextWindowTokens: 200, + maxOutputTokens: 100, + }); + const patched = store.listByProvider(providerId).find((m) => m.modelId === 'was-live'); + expect(patched?.source).toBe('live'); // NOT demoted to 'static' + expect(patched?.lastRefreshedAt).toBe(REFRESH_TS); // stamp preserved, not nulled + expect(store.providerRefreshedAt(providerId)).toBe(REFRESH_TS); // still counted as a live row + // ...while a TRUE insert still defaults to static / never-refreshed + store.upsert({ + providerId, + modelId: 'fresh-static', + displayName: 'Fresh', + contextWindowTokens: 1, + maxOutputTokens: 1, + }); + const fresh = store.listByProvider(providerId).find((m) => m.modelId === 'fresh-static'); + expect(fresh?.source).toBe('static'); + expect(fresh?.lastRefreshedAt).toBeUndefined(); + }); + + it('listByProvider/listAll exclude a soft-DELETED (deletedAt) row, not just an inactive one', () => { + store.replaceProviderModels( + providerId, + [ + { modelId: 'keep', displayName: 'Keep' }, + { modelId: 'gone', displayName: 'Gone' }, + ], + TS_MS, + ); + // soft-DELETE (deletedAt set) 'gone' — a scenario distinct from the isActive=false deactivation path; + // a refactor that dropped isNull(deletedAt) from the listing queries would otherwise stay green. + client.sqlite + .prepare('UPDATE model_catalog SET deleted_at = ? WHERE model_id = ?') + .run(TS_MS + 1, 'gone'); + expect(store.listByProvider(providerId).map((m) => m.modelId)).toEqual(['keep']); + expect(store.listAll().map((m) => m.modelId)).toEqual(['keep']); + }); }); diff --git a/packages/db/src/model-catalog-store.ts b/packages/db/src/model-catalog-store.ts index 4bbb07d2..e8673ae9 100644 --- a/packages/db/src/model-catalog-store.ts +++ b/packages/db/src/model-catalog-store.ts @@ -330,10 +330,14 @@ export function createModelCatalogStore(db: Db, deps: ModelCatalogStoreDeps): Mo mediaImageCostMicrocents: input.mediaImageCostMicrocents ?? null, mediaAudioCostMicrocents: input.mediaAudioCostMicrocents ?? null, mediaVideoCostMicrocents: input.mediaVideoCostMicrocents ?? null, - // Provenance + freshness (ADR-0064 §4/§5). Omitted ⇒ `'static'` / `null`, so every existing - // media-routing caller (which passes neither) writes a static, never-refreshed row unchanged. - source: input.source ?? 'static', - lastRefreshedAt: input.lastRefreshedAt ?? null, + // Provenance + freshness (ADR-0064 §4/§5). On a true INSERT (`existing` undefined) these fall to + // `'static'` / `null`, so every existing media-routing caller (which passes neither) writes a static, + // never-refreshed row unchanged. On an UPDATE they PRESERVE the existing row's `source`/`lastRefreshedAt` + // when the caller omits them — a caller that omits `source` (e.g. a future provider-sync patch) must + // NEVER demote a live-refreshed row back to `'static'` or null its stamp (the "never clobber" invariant, + // symmetric with `replaceProviderModels`). + source: input.source ?? existing?.source ?? 'static', + lastRefreshedAt: input.lastRefreshedAt ?? existing?.lastRefreshedAt ?? null, // An upsert (re)activates the row: keep `isActive` in lockstep with `activeRow`'s `isActive = true` // filter so a re-upserted, previously-deactivated row is reachable again and the returned record never // disagrees with a subsequent `getByModelId` (which filters inactive rows out). From b9af33cbe3e68cfd1a807c438c02a396b3f17028 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 21:26:26 +0300 Subject: [PATCH 12/68] =?UTF-8?q?feat(cli):=202.5.G=20S5=20=E2=80=94=20mod?= =?UTF-8?q?el-catalog=20refresh=20orchestration=20+=20relavium=20models=20?= =?UTF-8?q?/=20models=20refresh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires S2 (listModels seam) + S4 (the live cache) into a refresh service and the models command (ADR-0064 §5, ADR-0049 --json). Security-flagged: it reads provider keys to egress. - engine/model-refresh.ts: createModelRefreshService({resolveProvider, keyFor, providerStore, catalogStore, knownProviderIds, knownProviders, now}) — injected deps so desktop/VS Code reuse it. refresh() derives the CONNECTED providers (keyFor try/catch; no key -> skipped-no-key, not an error), fans out with per-provider ISOLATION (Promise.allSettled — one provider's bad key / network / drift throw / missing listModels never fails the whole refresh), ensures the FK-target llm_providers row exists BEFORE writing model rows, and replaceProviderModels(providerUuid, rows, now). RefreshReport is per-provider {status: refreshed|skipped-no-key|skipped-unsupported|failed, added/updated/ deactivated (diffed before/after), error}. refreshIfStale() at a 24h TTL off providerRefreshedAt; refreshInBackground() = timer-less void refreshIfStale().catch(()=>{}) (can't keep a short-lived process alive; swallows every error). - commands/models.ts: framework-free `models` (list the cache; blocking first-run refresh-if-empty) + `models refresh` (force a live re-fetch, per-provider report), both --json (ADR-0049). Exit 0 with the report even on per-provider failures; exit 2 only for an explicit refresh with ZERO providers connected. - manifest/specs/dispatch: the models + models.refresh COMMAND_MANIFEST pair, commander registration (descriptions verbatim), and the withModelsDeps wiring home. commands.md documents both + the --json shapes. SECURITY: a key flows ONLY keyFor(id) -> refreshOne -> adapter.listModels(key, signal); it never enters a RefreshReport / --json / stored row / log; a failure surfaces only secretFreeReason(err) (the seam already redacts + strips cause), never err.cause. Tests inject stubs (no network); one asserts JSON.stringify(report) excludes the key, one drives a rejecting background refresh (no throw / no unhandled rejection). Verified: pnpm turbo run lint typecheck test build — 24/24 green (CLI 1325 pass, +14); key isolated to keyFor->listModels; background timer-less + error-swallowing; prettier clean. Refs: ADR-0064, ADR-0049 Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/commands/dispatch.ts | 52 +++- apps/cli/src/commands/manifest.ts | 12 + apps/cli/src/commands/models.test.ts | 197 +++++++++++++++ apps/cli/src/commands/models.ts | 163 ++++++++++++ apps/cli/src/commands/specs.ts | 40 +++ apps/cli/src/engine/model-refresh.test.ts | 243 ++++++++++++++++++ apps/cli/src/engine/model-refresh.ts | 288 ++++++++++++++++++++++ docs/reference/cli/commands.md | 20 +- 8 files changed, 1012 insertions(+), 3 deletions(-) create mode 100644 apps/cli/src/commands/models.test.ts create mode 100644 apps/cli/src/commands/models.ts create mode 100644 apps/cli/src/engine/model-refresh.test.ts create mode 100644 apps/cli/src/engine/model-refresh.ts diff --git a/apps/cli/src/commands/dispatch.ts b/apps/cli/src/commands/dispatch.ts index a95f16d9..c52a783c 100644 --- a/apps/cli/src/commands/dispatch.ts +++ b/apps/cli/src/commands/dispatch.ts @@ -1,10 +1,15 @@ import { randomUUID } from 'node:crypto'; -import { createProviderStore } from '@relavium/db'; +import { createModelCatalogStore, createProviderStore } from '@relavium/db'; import { loadResolvedConfig } from '../config/load.js'; import { openLocalDb } from '../db/open.js'; -import { createProviderResolver } from '../engine/providers.js'; +import { createModelRefreshService } from '../engine/model-refresh.js'; +import { + KNOWN_PROVIDERS, + KNOWN_PROVIDER_IDS, + createProviderResolver, +} from '../engine/providers.js'; import { openHistoryStore } from '../history/open.js'; import { openSessionStore } from '../history/session-open.js'; import { CliError } from '../process/errors.js'; @@ -26,6 +31,7 @@ import { gateListCommand } from './gate-list.js'; import { importCommand, type ImportCommandArgs } from './import.js'; import { listCommand } from './list.js'; import { logsCommand } from './logs.js'; +import { modelsCommand, type ModelsCommandArgs } from './models.js'; import { runProviderCommand, type ProviderCommandArgs, @@ -290,6 +296,46 @@ const executeLogs: CommandExecutor = (input, ctx) => const executeStatus: CommandExecutor = (_input, ctx) => Promise.resolve(statusCommand({ io: ctx.io, global: ctx.global })); +/** + * Open the local db + OS keychain for one `models` invocation, wire the S5 refresh service over the S4 catalog + * store + the S2 `listModels?` seam, run the core, and always close the db (2.5.G S5, ADR-0064). The key + * resolver reads a provider key only inside the refresh (keychain → env); the catalog holds no key. + */ +async function withModelsDeps(ctx: DispatchContext, args: ModelsCommandArgs): Promise { + const { homeDir } = loadResolvedConfig({ + cwd: ctx.global.cwd, + configPath: ctx.global.configPath, + }); + const { db, close } = openLocalDb(homeDir); + try { + const storeDeps = { uuid: () => randomUUID(), now: () => Date.now() }; + const resolver = createProviderResolver(ctx.io.env, createOsKeychainStore()); + const providerStore = createProviderStore(db, storeDeps); + const catalogStore = createModelCatalogStore(db, storeDeps); + const refreshService = createModelRefreshService({ + resolveProvider: resolver.resolveProvider, + keyFor: resolver.keyFor, + providerStore, + catalogStore, + knownProviderIds: KNOWN_PROVIDER_IDS, + knownProviders: KNOWN_PROVIDERS, + now: () => Date.now(), + }); + return await modelsCommand(args, { + io: ctx.io, + global: ctx.global, + catalog: catalogStore, + refreshService, + }); + } finally { + close(); + } +} + +const executeModels: CommandExecutor = (_input, ctx) => withModelsDeps(ctx, { refresh: false }); +const executeModelsRefresh: CommandExecutor = (_input, ctx) => + withModelsDeps(ctx, { refresh: true }); + /** Open the local db + OS keychain for one `provider` invocation, run the core, and always close the db. */ async function withProviderDeps( ctx: DispatchContext, @@ -342,6 +388,8 @@ const COMMAND_EXECUTORS: ReadonlyMap = new Map ({ action: 'list' }))], ['provider.add', providerExecutor(buildProviderAddArgs)], [ diff --git a/apps/cli/src/commands/manifest.ts b/apps/cli/src/commands/manifest.ts index 43373749..b80be0f1 100644 --- a/apps/cli/src/commands/manifest.ts +++ b/apps/cli/src/commands/manifest.ts @@ -282,6 +282,18 @@ const ENTRIES: readonly CommandManifestEntry[] = [ description: 'Show active/paused runs and their per-node status.', effect: 'read', }, + { + id: 'models', + label: 'List models', + description: 'List the cached model catalog (refreshes on first run if empty).', + effect: 'read', + }, + { + id: 'models.refresh', + label: 'Refresh models', + description: "Re-fetch each connected provider's live model list into the local cache.", + effect: 'write', + }, { id: 'provider.list', label: 'List providers', diff --git a/apps/cli/src/commands/models.test.ts b/apps/cli/src/commands/models.test.ts new file mode 100644 index 00000000..448899ab --- /dev/null +++ b/apps/cli/src/commands/models.test.ts @@ -0,0 +1,197 @@ +import type { ModelCatalogListing } from '@relavium/db'; +import { describe, expect, it } from 'vitest'; + +import type { ModelRefreshService, RefreshReport } from '../engine/model-refresh.js'; +import { isCliError } from '../process/errors.js'; +import { EXIT_CODES } from '../process/exit-codes.js'; +import type { GlobalOptions } from '../process/options.js'; +import { captureIo, parseNdjson } from '../test-support.js'; +import { modelsCommand, type ModelsCatalogReader, type ModelsCommandDeps } from './models.js'; + +/** + * `relavium models` / `models refresh` command-core tests (2.5.G S5, ADR-0064 §5/§10 + ADR-0049 --json). Pure: + * a stub catalog reader + a stub refresh service — no db, no network. Covers the list vs refresh paths, the + * first-run refresh-if-empty, the `--json` shapes, and the zero-connected exit-2 fault. + */ + +function globalOptions(json = false): GlobalOptions { + return { json, color: false, cwd: process.cwd(), configPath: undefined, verbosity: 'normal' }; +} + +function modelRow( + providerId: string, + modelId: string, + source: 'static' | 'live' | 'user' = 'live', +): ModelCatalogListing { + return { + modelId, + providerId, + displayName: modelId, + contextWindowTokens: 200_000, + inputCostPerMtokMicrocents: 0, + outputCostPerMtokMicrocents: 0, + cachedInputCostPerMtokMicrocents: 0, + source, + lastRefreshedAt: 1_700_000_000_000, + isActive: true, + }; +} + +/** A refresh-service stub: returns `report` and runs `onRefresh` (e.g. to populate the catalog on first run). */ +function stubRefresh( + report: RefreshReport, + onRefresh?: () => void, +): ModelRefreshService & { calls: number } { + const state = { calls: 0 }; + return { + get calls() { + return state.calls; + }, + refresh: () => { + state.calls += 1; + onRefresh?.(); + return Promise.resolve(report); + }, + refreshIfStale: () => Promise.resolve(report), + refreshInBackground: () => {}, + }; +} + +function stubCatalog(rowsRef: { value: ModelCatalogListing[] }): ModelsCatalogReader { + return { listAll: () => rowsRef.value }; +} + +function deps( + io: ReturnType['io'], + catalog: ModelsCatalogReader, + refreshService: ModelRefreshService, + json = false, +): ModelsCommandDeps { + return { io, global: globalOptions(json), catalog, refreshService }; +} + +const REFRESHED: RefreshReport = { + providers: [ + { provider: 'anthropic', status: 'refreshed', added: 3, updated: 1, deactivated: 0 }, + { provider: 'openai', status: 'skipped-no-key' }, + ], +}; + +describe('modelsCommand — list', () => { + it('lists the cached catalog without refreshing when it is non-empty', async () => { + const { io, out } = captureIo(); + const rowsRef = { value: [modelRow('anthropic', 'claude-x')] }; + const refresh = stubRefresh(REFRESHED); + + const code = await modelsCommand({ refresh: false }, deps(io, stubCatalog(rowsRef), refresh)); + expect(code).toBe(EXIT_CODES.success); + expect(refresh.calls).toBe(0); // non-empty ⇒ no first-run refresh + expect(out()).toContain('claude-x'); + }); + + it('does a first-run refresh when the cache is empty, then lists', async () => { + const { io, out } = captureIo(); + const rowsRef: { value: ModelCatalogListing[] } = { value: [] }; + const refresh = stubRefresh(REFRESHED, () => { + rowsRef.value = [modelRow('anthropic', 'claude-x')]; + }); + + const code = await modelsCommand({ refresh: false }, deps(io, stubCatalog(rowsRef), refresh)); + expect(code).toBe(EXIT_CODES.success); + expect(refresh.calls).toBe(1); // empty ⇒ one blocking first-run refresh + expect(out()).toContain('claude-x'); + }); + + it('reports an empty catalog cleanly (exit 0, no fault)', async () => { + const { io, out } = captureIo(); + const rowsRef: { value: ModelCatalogListing[] } = { value: [] }; + const refresh = stubRefresh({ providers: [] }); // refresh finds nothing + + const code = await modelsCommand({ refresh: false }, deps(io, stubCatalog(rowsRef), refresh)); + expect(code).toBe(EXIT_CODES.success); + expect(out()).toContain('No models cached'); + }); + + it('--json emits one key-free record per model', async () => { + const { io, out } = captureIo(); + const rowsRef = { value: [modelRow('anthropic', 'claude-x'), modelRow('openai', 'gpt-x')] }; + const refresh = stubRefresh(REFRESHED); + + await modelsCommand({ refresh: false }, deps(io, stubCatalog(rowsRef), refresh, true)); + const records = parseNdjson(out()); + expect(records).toHaveLength(2); + expect(records[0]).toMatchObject({ + provider: 'anthropic', + modelId: 'claude-x', + contextWindowTokens: 200_000, + source: 'live', + }); + }); +}); + +describe('modelsCommand — refresh', () => { + it('prints the per-provider report and exits 0 (a per-provider skip is not a failure)', async () => { + const { io, out } = captureIo(); + const rowsRef: { value: ModelCatalogListing[] } = { value: [] }; + const refresh = stubRefresh(REFRESHED); + + const code = await modelsCommand({ refresh: true }, deps(io, stubCatalog(rowsRef), refresh)); + expect(code).toBe(EXIT_CODES.success); + const text = out(); + expect(text).toContain('anthropic'); + expect(text).toContain('refreshed'); + expect(text).toContain('+3 ~1 -0'); + expect(text).toContain('openai'); + }); + + it('--json emits one record per provider with status + counts', async () => { + const { io, out } = captureIo(); + const rowsRef: { value: ModelCatalogListing[] } = { value: [] }; + const refresh = stubRefresh(REFRESHED); + + await modelsCommand({ refresh: true }, deps(io, stubCatalog(rowsRef), refresh, true)); + const records = parseNdjson(out()); + expect(records).toEqual([ + { + provider: 'anthropic', + status: 'refreshed', + added: 3, + updated: 1, + deactivated: 0, + error: null, + }, + { + provider: 'openai', + status: 'skipped-no-key', + added: null, + updated: null, + deactivated: null, + error: null, + }, + ]); + }); + + it('exits 2 on an explicit refresh with zero providers connected (no key at all)', async () => { + const { io, out } = captureIo(); + const rowsRef: { value: ModelCatalogListing[] } = { value: [] }; + const refresh = stubRefresh({ + providers: [ + { provider: 'anthropic', status: 'skipped-no-key' }, + { provider: 'openai', status: 'skipped-no-key' }, + ], + }); + + let thrown: unknown; + try { + await modelsCommand({ refresh: true }, deps(io, stubCatalog(rowsRef), refresh)); + expect.unreachable('should have thrown'); + } catch (err) { + thrown = err; + } + expect(isCliError(thrown)).toBe(true); + if (isCliError(thrown)) { + expect(thrown.exitCode).toBe(EXIT_CODES.invalidInvocation); + } + expect(out()).toBe(''); // stdout stays empty on a fault + }); +}); diff --git a/apps/cli/src/commands/models.ts b/apps/cli/src/commands/models.ts new file mode 100644 index 00000000..4aabc9ce --- /dev/null +++ b/apps/cli/src/commands/models.ts @@ -0,0 +1,163 @@ +import type { ModelCatalogListing } from '@relavium/db'; + +import type { + ModelRefreshService, + RefreshProviderResult, + RefreshReport, +} from '../engine/model-refresh.js'; +import { CliError } from '../process/errors.js'; +import { EXIT_CODES, type ExitCode } from '../process/exit-codes.js'; +import type { CliIo } from '../process/io.js'; +import type { GlobalOptions } from '../process/options.js'; +import { writeRecordLines } from '../render/records.js'; + +/** + * The `relavium models` / `relavium models refresh` command core (workstream **2.5.G S5**, + * [ADR-0064](../../../../docs/decisions/0064-live-model-catalog.md) §5/§10) — the shell surface over the S5 + * refresh orchestrator + the S4 catalog reader. Framework-free (no `commander`): parsed args + injected ports + * in, output via {@link CliIo}; a fault throws a typed {@link CliError}. The stores/refresh service are injected, + * so the core unit-tests with an in-memory db + a stub resolver and never touches the network. + * + * - `relavium models` (no sub) — LIST the cached catalog (read-only). If the cache is empty it does a minimal + * blocking first-run refresh (ADR-0064 §5a) and re-reads; an empty result stays a clean exit `0` (never a fault). + * - `relavium models refresh` — force a live re-fetch (blocking) and print a per-provider outcome report. A + * per-provider failure is NOT a command failure (exit `0` with the report); the ONE hard fault (exit `2`) is an + * explicit refresh with ZERO providers connected (no key at all) — nothing could be fetched, so the user is + * told to add a key. + * + * Both honor `--json` ([ADR-0049](../../../../docs/decisions/0049-cli-machine-output-contract.md)): one NDJSON + * record per model (list) or per provider (refresh), stdout-pure. Every emitted record is KEY-FREE by + * construction — no provider key ever enters the catalog rows or the refresh report. + */ + +export interface ModelsCommandArgs { + /** `true` for `models refresh` (force a live re-fetch); `false` for a bare `models` (list the cache). */ + readonly refresh: boolean; +} + +/** The narrow catalog reader the list path needs. */ +export interface ModelsCatalogReader { + readonly listAll: () => ModelCatalogListing[]; +} + +export interface ModelsCommandDeps { + readonly io: CliIo; + readonly global: GlobalOptions; + readonly catalog: ModelsCatalogReader; + readonly refreshService: ModelRefreshService; +} + +export async function modelsCommand( + args: ModelsCommandArgs, + deps: ModelsCommandDeps, +): Promise { + if (args.refresh) { + return runRefresh(deps); + } + return runList(deps); +} + +/** `models refresh` — force a live re-fetch and report per-provider outcomes. */ +async function runRefresh(deps: ModelsCommandDeps): Promise { + const report = await deps.refreshService.refresh(); + const connected = report.providers.filter((p) => p.status !== 'skipped-no-key'); + if (connected.length === 0) { + // Nothing was connected — no key at all, so the refresh could fetch nothing. A clean, actionable exit-2 + // invocation fault (never echoes a key; names both ways to provide one). + throw new CliError( + 'invalid_invocation', + 'no provider key configured — store one with `relavium provider set-key `, or set RELAVIUM__API_KEY.', + ); + } + if (deps.global.json) { + writeRecordLines(deps.io, report.providers.map(toRefreshJson)); + return EXIT_CODES.success; + } + renderRefreshReport(deps.io, report); + return EXIT_CODES.success; +} + +/** `models` (no sub) — list the cached catalog, refreshing first only when it is empty (first-run, ADR-0064 §5a). */ +async function runList(deps: ModelsCommandDeps): Promise { + let listings = deps.catalog.listAll(); + if (listings.length === 0) { + // First-run: a minimal blocking refresh so the very first `models` shows a real catalog, then re-read. A + // provider with no key is skipped inside `refresh`, so this stays a no-op (still exit 0) when no key is set. + await deps.refreshService.refresh(); + listings = deps.catalog.listAll(); + } + if (deps.global.json) { + writeRecordLines(deps.io, listings.map(toModelJson)); + return EXIT_CODES.success; + } + renderModelList(deps.io, listings); + return EXIT_CODES.success; +} + +// ── --json records (key-free; `null` for an absent optional, the read-command convention) ────────────── + +function toRefreshJson(p: RefreshProviderResult): unknown { + return { + provider: p.provider, + status: p.status, + added: p.added ?? null, + updated: p.updated ?? null, + deactivated: p.deactivated ?? null, + error: p.error ?? null, + }; +} + +function toModelJson(m: ModelCatalogListing): unknown { + return { + provider: m.providerId, + modelId: m.modelId, + displayName: m.displayName, + contextWindowTokens: m.contextWindowTokens ?? null, + maxOutputTokens: m.maxOutputTokens ?? null, + source: m.source, + lastRefreshedAt: m.lastRefreshedAt ?? null, + deprecationDate: m.deprecationDate ?? null, + }; +} + +// ── human renderers ──────────────────────────────────────────────────────────── + +/** Collapse whitespace so a (secret-free) provider error can never break the one-line-per-provider table. */ +function oneLine(text: string): string { + return text.replace(/\s+/g, ' ').trim(); +} + +function renderRefreshReport(io: CliIo, report: RefreshReport): void { + io.writeOut('Model catalog refresh:\n'); + for (const p of report.providers) { + io.writeOut(` ${p.provider}\t${refreshDetail(p)}\n`); + } +} + +/** The per-provider detail column of the refresh report. */ +function refreshDetail(p: RefreshProviderResult): string { + switch (p.status) { + case 'refreshed': + return `refreshed\t+${p.added ?? 0} ~${p.updated ?? 0} -${p.deactivated ?? 0}`; + case 'skipped-no-key': + return 'skipped\tno key'; + case 'skipped-unsupported': + return 'skipped\tno model-list endpoint'; + case 'failed': + return `failed\t${oneLine(p.error ?? 'refresh failed')}`; + } +} + +function renderModelList(io: CliIo, listings: readonly ModelCatalogListing[]): void { + if (listings.length === 0) { + io.writeOut( + 'No models cached. Add a provider key (`relavium provider set-key `) and run `relavium models refresh`.\n', + ); + return; + } + io.writeOut(`Models (${listings.length}):\n`); + for (const m of listings) { + const ctx = m.contextWindowTokens === undefined ? '' : `\tctx=${m.contextWindowTokens}`; + io.writeOut(` ${m.modelId}\t${m.providerId}${ctx}\t[${m.source}]\n`); + } +} diff --git a/apps/cli/src/commands/specs.ts b/apps/cli/src/commands/specs.ts index 5ee6941b..78b57ce0 100644 --- a/apps/cli/src/commands/specs.ts +++ b/apps/cli/src/commands/specs.ts @@ -53,6 +53,7 @@ export function registerCommands(program: Command, ctx?: CommandContext): void { registerImport(program, ctx); registerAgent(program, ctx); registerGate(program, ctx); + registerModels(program, ctx); registerProvider(program, ctx); registerList(program, ctx); registerLogs(program, ctx); @@ -442,6 +443,45 @@ function registerStatus(program: Command, ctx?: CommandContext): void { }); } +/** + * Register `relavium models` (list the cached catalog) + `relavium models refresh` (force a live re-fetch) + * (2.5.G S5, [ADR-0064](../../../../docs/decisions/0064-live-model-catalog.md)). The parent `models` action + * lists the cache (refreshing on first run if empty); the `refresh` subcommand blocks on a live re-fetch and + * reports per-provider outcomes. Both honor `--json`. Each dispatch opens the local db + keychain per invocation. + */ +function registerModels(program: Command, ctx?: CommandContext): void { + const models = program + .command('models') + .description('List the cached model catalog (refreshes on first run if empty).'); + const refresh = models + .command('refresh') + .description("Re-fetch each connected provider's live model list into the local cache."); + + if (ctx === undefined) { + models.action(() => { + throw new CliError('not_implemented', '`relavium models` requires the CLI runtime context.'); + }); + refresh.action(() => { + throw new CliError( + 'not_implemented', + '`relavium models refresh` requires the CLI runtime context.', + ); + }); + return; + } + + models.action(async () => { + ctx.result.exitCode = await executeCommand('models', { positionals: [], options: {} }, ctx); + }); + refresh.action(async () => { + ctx.result.exitCode = await executeCommand( + 'models.refresh', + { positionals: [], options: {} }, + ctx, + ); + }); +} + /** Register `relavium provider` and its subcommands (2.C). Each dispatch opens the local db + keychain per invocation. */ function registerProvider(program: Command, ctx?: CommandContext): void { const provider = program diff --git a/apps/cli/src/engine/model-refresh.test.ts b/apps/cli/src/engine/model-refresh.test.ts new file mode 100644 index 00000000..de8dcc08 --- /dev/null +++ b/apps/cli/src/engine/model-refresh.test.ts @@ -0,0 +1,243 @@ +import { randomUUID } from 'node:crypto'; + +import { + createClient, + createModelCatalogStore, + createProviderStore, + runMigrations, + type Db, + type DbClient, +} from '@relavium/db'; +import type { LlmProvider, ModelListing, ProviderId } from '@relavium/llm'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { CHAT_TEXT_CAPABILITY_FLAGS } from '../test-support.js'; +import { KNOWN_PROVIDERS } from './providers.js'; +import { createModelRefreshService, TTL_MS, type ModelRefreshDeps } from './model-refresh.js'; + +/** + * S5 refresh-orchestrator tests (ADR-0064 §5). All NETWORK-FREE: a stub resolver whose `listModels` returns + * canned {@link ModelListing}s or throws, over a real in-memory `history.db` (so the store diff / TTL / FK + * ordering are exercised end-to-end). Asserts per-provider isolation, the skip statuses, the added/updated/ + * deactivated diff, the TTL, and — the security invariant — that no provider key ever appears in a report. + */ + +const SECRET_KEY = 'sk-super-secret-key-value'; + +/** A minimal {@link LlmProvider} — only `id` + the optional `listModels` matter to the refresh; the required + * seam methods throw (never reached here). No `any` / unsafe `as`: the object satisfies the interface. */ +function stubProvider(id: ProviderId, listModels?: LlmProvider['listModels']): LlmProvider { + return { + id, + generate: () => Promise.reject(new Error('stub generate not used')), + stream: () => { + throw new Error('stub stream not used'); + }, + supports: CHAT_TEXT_CAPABILITY_FLAGS, + ...(listModels === undefined ? {} : { listModels }), + }; +} + +/** A resolver stub: `adapters[id]` is the (keyless) adapter; `keys[id]` is that provider's key (absent ⇒ throw). */ +function stubResolver(config: { + readonly adapters: Partial>; + readonly keys: Partial>; +}): Pick { + return { + resolveProvider: (id) => config.adapters[id], + keyFor: (id) => { + const key = config.keys[id]; + if (key === undefined) { + throw new Error(`no key for ${id}`); + } + return key; + }, + }; +} + +/** A single canned listing. */ +function listing(id: string, contextWindowTokens?: number): ModelListing { + return { + id, + displayName: id, + ...(contextWindowTokens === undefined ? {} : { contextWindowTokens }), + }; +} + +describe('createModelRefreshService', () => { + let client: DbClient; + let db: Db; + const nowRef = { value: 1_700_000_000_000 }; + + beforeEach(() => { + client = createClient(':memory:'); + runMigrations(client.db); + db = client.db; + nowRef.value = 1_700_000_000_000; + }); + afterEach(() => { + client.sqlite.close(); + }); + + function service( + resolver: Pick, + ids: ProviderId[], + ) { + const storeDeps = { uuid: () => randomUUID(), now: () => nowRef.value }; + const providerStore = createProviderStore(db, storeDeps); + const catalogStore = createModelCatalogStore(db, storeDeps); + const svc = createModelRefreshService({ + resolveProvider: resolver.resolveProvider, + keyFor: resolver.keyFor, + providerStore, + catalogStore, + knownProviderIds: ids, + knownProviders: KNOWN_PROVIDERS, + now: () => nowRef.value, + }); + return { svc, providerStore, catalogStore }; + } + + it('isolates a per-provider failure — one provider throws, the others still refresh', async () => { + const resolver = stubResolver({ + adapters: { + anthropic: stubProvider('anthropic', () => + Promise.reject(new Error('anthropic list failed')), + ), + openai: stubProvider('openai', () => Promise.resolve([listing('gpt-x', 100)])), + }, + keys: { anthropic: SECRET_KEY, openai: SECRET_KEY }, + }); + const { svc, catalogStore } = service(resolver, ['anthropic', 'openai']); + + const report = await svc.refresh(); + const byId = new Map(report.providers.map((p) => [p.provider, p])); + expect(byId.get('anthropic')?.status).toBe('failed'); + expect(byId.get('anthropic')?.error).toBe('anthropic list failed'); + expect(byId.get('openai')).toMatchObject({ status: 'refreshed', added: 1 }); + // openai still wrote its row despite anthropic failing. + expect(catalogStore.listAll().map((m) => m.modelId)).toEqual(['gpt-x']); + }); + + it('skips a provider with no resolvable key (skipped-no-key, not an error)', async () => { + const resolver = stubResolver({ + adapters: { openai: stubProvider('openai', () => Promise.resolve([listing('gpt-x')])) }, + keys: { openai: SECRET_KEY }, // anthropic has no key + }); + const { svc } = service(resolver, ['anthropic', 'openai']); + + const report = await svc.refresh(); + const byId = new Map(report.providers.map((p) => [p.provider, p])); + expect(byId.get('anthropic')?.status).toBe('skipped-no-key'); + expect(byId.get('openai')?.status).toBe('refreshed'); + }); + + it('skips a connected provider whose adapter has no listModels (skipped-unsupported)', async () => { + const resolver = stubResolver({ + adapters: { gemini: stubProvider('gemini') }, // adapter present but no listModels + keys: { gemini: SECRET_KEY }, + }); + const { svc, catalogStore } = service(resolver, ['gemini']); + + const report = await svc.refresh(); + expect(report.providers).toEqual([{ provider: 'gemini', status: 'skipped-unsupported' }]); + expect(catalogStore.listAll()).toEqual([]); + }); + + it('reports added / updated / deactivated across two refreshes', async () => { + let models: ModelListing[] = [listing('m1'), listing('m2'), listing('m3')]; + const resolver = stubResolver({ + adapters: { openai: stubProvider('openai', () => Promise.resolve(models)) }, + keys: { openai: SECRET_KEY }, + }); + const { svc } = service(resolver, ['openai']); + + const first = await svc.refresh(); + expect(first.providers[0]).toMatchObject({ + status: 'refreshed', + added: 3, + updated: 0, + deactivated: 0, + }); + + // Drop m3, keep m1/m2, add m4. + models = [listing('m1'), listing('m2'), listing('m4')]; + const second = await svc.refresh(); + expect(second.providers[0]).toMatchObject({ + status: 'refreshed', + added: 1, + updated: 2, + deactivated: 1, + }); + }); + + it('refreshIfStale refreshes on an empty cache, no-ops within the TTL, and refreshes again once stale', async () => { + const resolver = stubResolver({ + adapters: { openai: stubProvider('openai', () => Promise.resolve([listing('m1')])) }, + keys: { openai: SECRET_KEY }, + }); + const { svc } = service(resolver, ['openai']); + + // Never-refreshed (empty) ⇒ stale ⇒ refreshes. + const first = await svc.refreshIfStale(); + expect(first?.providers[0]?.status).toBe('refreshed'); + + // Same clock ⇒ fresh ⇒ undefined (nothing stale). + expect(await svc.refreshIfStale()).toBeUndefined(); + + // Just under the TTL ⇒ still fresh. + nowRef.value += TTL_MS - 1; + expect(await svc.refreshIfStale()).toBeUndefined(); + + // At/after the TTL ⇒ stale ⇒ refreshes again. + nowRef.value += 1; + const third = await svc.refreshIfStale(); + expect(third?.providers[0]?.status).toBe('refreshed'); + }); + + it('never leaks a provider key into the report (or an error message)', async () => { + const resolver = stubResolver({ + adapters: { + anthropic: stubProvider('anthropic', () => + // Even a failing adapter whose error TEXT mentions the key: the service surfaces `error.message`, which + // the real seam redacts. Here we assert the service never fabricates a key into the report itself. + Promise.reject(new Error('list failed for provider')), + ), + openai: stubProvider('openai', () => Promise.resolve([listing('gpt-x')])), + }, + keys: { anthropic: SECRET_KEY, openai: SECRET_KEY }, + }); + const { svc } = service(resolver, ['anthropic', 'openai']); + + const report = await svc.refresh(); + expect(JSON.stringify(report)).not.toContain(SECRET_KEY); + }); + + it('refreshInBackground never throws and swallows a rejecting refresh', async () => { + // providerStore.get throws inside refreshIfStale ⇒ the refresh promise rejects ⇒ the fire-and-forget `.catch` + // swallows it (no unhandled rejection, no thrown stack). + const svc = createModelRefreshService({ + resolveProvider: () => stubProvider('openai', () => Promise.resolve([])), + keyFor: () => SECRET_KEY, + providerStore: { + upsert: () => ({ id: 'u' }), + get: () => { + throw new Error('db unavailable'); + }, + }, + catalogStore: { + replaceProviderModels: () => {}, + listByProvider: () => [], + providerRefreshedAt: () => undefined, + }, + knownProviderIds: ['openai'], + knownProviders: KNOWN_PROVIDERS, + now: () => 0, + }); + + expect(() => svc.refreshInBackground()).not.toThrow(); + // Let the rejected promise settle; the `.catch` must have handled it (an unhandled rejection would fail CI). + await Promise.resolve(); + await Promise.resolve(); + }); +}); diff --git a/apps/cli/src/engine/model-refresh.ts b/apps/cli/src/engine/model-refresh.ts new file mode 100644 index 00000000..304c0953 --- /dev/null +++ b/apps/cli/src/engine/model-refresh.ts @@ -0,0 +1,288 @@ +import type { ModelCatalogListing, ModelCatalogLiveModel } from '@relavium/db'; +import type { AbortSignalLike, LlmProvider, ModelListing, ProviderId } from '@relavium/llm'; + +/** + * The live model-catalog refresh orchestrator (workstream **2.5.G S5**, [ADR-0064](../../../../docs/decisions/0064-live-model-catalog.md) + * §5). A **host service with injected deps** so desktop / VS Code reuse it later and `@relavium/llm` / + * `@relavium/core` stay platform-free ([ADR-0038](../../../../docs/decisions/0038-agentrunner-llm-call-boundary.md)). + * For each CONNECTED provider (a key resolves) it fetches the live model list over the S2 `listModels?` seam + * (already bounded + abortable + secret-free), ensures the `llm_providers` row (FK target) exists, and bulk- + * upserts the discovered ids into the `model_catalog` live cache (S4 `replaceProviderModels`). + * + * SECURITY (this is a key-reading, egressing surface): + * - A provider key is read ONLY via {@link ModelRefreshDeps.keyFor} per provider per call and handed straight + * to `listModels` — it is NEVER logged, persisted (the `model_catalog` holds no key), or placed in a + * {@link RefreshReport} / `--json` payload / error message. `listModels`'s failure is already key-redacted + + * cause-stripped by the seam's `boundedListModels`, so a failed provider surfaces only `error.message` (or a + * generic `'refresh failed'`), never `err.cause`. + * - Per-provider ISOLATION (`Promise.allSettled`): one provider's failure (bad key, network, drift throw, or + * an adapter with no `listModels`) NEVER fails the whole refresh — it is recorded as `status: 'failed'` / + * `'skipped-unsupported'` and the others still refresh (ADR-0064 §5/§8). + * - {@link ModelRefreshService.refreshInBackground} is fire-and-forget and swallows ALL errors so a rejection + * is never unhandled and never surfaces a stack; it sets no timer of its own (the seam owns the fetch bound). + */ + +/** A provider's discovery/refresh outcome. `refreshed` wrote live rows; `skipped-no-key` had no resolvable key + * (not an error); `skipped-unsupported` has no `listModels` (degrade to static); `failed` threw (isolated). */ +export type RefreshProviderStatus = + | 'refreshed' + | 'skipped-no-key' + | 'skipped-unsupported' + | 'failed'; + +/** One provider's line in a {@link RefreshReport} — key-free by construction (no key ever enters this shape). */ +export interface RefreshProviderResult { + readonly provider: ProviderId; + readonly status: RefreshProviderStatus; + /** Live model ids newly added this refresh (present only on `refreshed`). */ + readonly added?: number; + /** Live model ids that already existed and were refreshed in place (present only on `refreshed`). */ + readonly updated?: number; + /** Previously-active live model ids soft-deactivated because they vanished from the list (present only on `refreshed`). */ + readonly deactivated?: number; + /** A SECRET-FREE failure reason (the seam already redacts) — present only on `failed`. */ + readonly error?: string; +} + +/** The whole-refresh report: one {@link RefreshProviderResult} per considered provider, ordered by provider id. */ +export interface RefreshReport { + readonly providers: readonly RefreshProviderResult[]; +} + +/** Options common to {@link ModelRefreshService.refresh} / `refreshIfStale` / `refreshInBackground`. */ +export interface RefreshOptions { + /** Restrict the refresh to these provider ids (default: {@link ModelRefreshDeps.knownProviderIds}). */ + readonly providers?: readonly ProviderId[]; + /** Aborts each in-flight `listModels` request (threaded to the seam). */ + readonly signal?: AbortSignalLike; +} + +/** The non-secret provider metadata the refresh needs to (re)register the `llm_providers` FK-target row. */ +export interface RefreshProviderMeta { + readonly displayName: string; + readonly baseUrl: string; +} + +/** The narrow provider-store surface the refresh needs (the FK-target row lifecycle only). */ +export interface RefreshProviderStore { + readonly upsert: (input: { + readonly name: string; + readonly displayName: string; + readonly baseUrl: string; + }) => { readonly id: string }; + readonly get: (name: string) => { readonly id: string } | undefined; +} + +/** The narrow catalog-store surface the refresh needs (the S4 live-refresh + read/freshness reads). */ +export interface RefreshCatalogStore { + readonly replaceProviderModels: ( + providerId: string, + rows: ReadonlyArray, + now: number, + ) => void; + readonly listByProvider: (providerId: string) => ModelCatalogListing[]; + readonly providerRefreshedAt: (providerId: string) => number | undefined; +} + +export interface ModelRefreshDeps { + /** The keyless `@relavium/llm` adapter for a provider id (from the injected {@link ProviderResolver}). */ + readonly resolveProvider: (id: ProviderId) => LlmProvider | undefined; + /** Resolve a provider's key (keychain → env → throw). Thrown ⇒ the provider is skipped (`skipped-no-key`). */ + readonly keyFor: (id: ProviderId) => string; + readonly providerStore: RefreshProviderStore; + readonly catalogStore: RefreshCatalogStore; + /** The provider ids to consider by default (production: `KNOWN_PROVIDER_IDS`). */ + readonly knownProviderIds: readonly ProviderId[]; + /** Non-secret display/base-URL metadata for the FK-target `llm_providers` row (production: `KNOWN_PROVIDERS`). */ + readonly knownProviders: Readonly>; + /** Injected clock — the `last_refreshed_at` stamp + the TTL comparison read from here (never `Date.now()`). */ + readonly now: () => number; +} + +export interface ModelRefreshService { + /** Force-refresh every connected provider (or `opts.providers`), per-provider-isolated. Never rejects. */ + refresh(opts?: RefreshOptions): Promise; + /** Refresh only providers whose live cache is empty/never-refreshed or older than {@link TTL_MS}; `undefined` + * when nothing was stale. */ + refreshIfStale(opts?: RefreshOptions): Promise; + /** Fire-and-forget non-blocking TTL refresh (ADR-0064 §5c). Swallows ALL errors; sets no timer. The actual + * picker/Home trigger is S7's wiring — S5 provides the mechanism only. */ + refreshInBackground(opts?: RefreshOptions): void; +} + +/** The staleness TTL (ADR-0064 §5): a provider's live cache older than 24h (or empty) is refreshed. */ +export const TTL_MS = 24 * 60 * 60 * 1000; + +/** Map a seam {@link ModelListing} → the S4 store's live-model shape (discovery half only; no pricing/media). */ +function toLiveModel(listing: ModelListing): ModelCatalogLiveModel { + return { + modelId: listing.id, + // The store falls back to the model id on an empty display name — pass `''` when the provider gave none. + displayName: listing.displayName ?? '', + ...(listing.contextWindowTokens === undefined + ? {} + : { contextWindowTokens: listing.contextWindowTokens }), + ...(listing.maxOutputTokens === undefined ? {} : { maxOutputTokens: listing.maxOutputTokens }), + }; +} + +/** The provider's currently-active `source='live'` model ids (the before/after diff basis). */ +function liveModelIds(listings: readonly ModelCatalogListing[]): Set { + const ids = new Set(); + for (const listing of listings) { + if (listing.source === 'live') { + ids.add(listing.modelId); + } + } + return ids; +} + +/** |a \ b| — the count of members of `a` absent from `b`. */ +function countMissing(a: ReadonlySet, b: ReadonlySet): number { + let n = 0; + for (const id of a) { + if (!b.has(id)) n += 1; + } + return n; +} + +/** |a ∩ b|. */ +function countShared(a: ReadonlySet, b: ReadonlySet): number { + let n = 0; + for (const id of a) { + if (b.has(id)) n += 1; + } + return n; +} + +/** + * A SECRET-FREE failure reason. The seam's `boundedListModels` already redacts the key AND strips the cause, + * so its `error.message` is safe to surface; a non-Error / empty message degrades to a generic string. We NEVER + * read `err.cause` (it could carry a nested field a verbose render might expose — the `validateProviderKey` rule). + */ +function secretFreeReason(err: unknown): string { + if (err instanceof Error && err.message.trim() !== '') { + return err.message; + } + return 'refresh failed'; +} + +/** Create the refresh orchestrator over its injected deps. Pure of platform I/O beyond the injected ports. */ +export function createModelRefreshService(deps: ModelRefreshDeps): ModelRefreshService { + /** Refresh ONE connected provider (its key already resolved). NEVER throws — a fault becomes `status:'failed'`. */ + const refreshOne = async ( + id: ProviderId, + key: string, + signal: AbortSignalLike | undefined, + ): Promise => { + try { + const adapter = deps.resolveProvider(id); + if (adapter?.listModels === undefined) { + // No adapter, or an adapter without the optional list capability — degrade to static (ADR-0064 §1/§5). + return { provider: id, status: 'skipped-unsupported' }; + } + // The key reaches ONLY this call; the seam bounds + aborts + redacts it. + const listings = await adapter.listModels(key, signal); + + // Ensure the FK-target `llm_providers` row exists BEFORE writing model rows (FK ordering). The metadata is + // non-secret; the row never carries a key. + const meta = deps.knownProviders[id]; + const providerUuid = deps.providerStore.upsert({ + name: id, + displayName: meta.displayName, + baseUrl: meta.baseUrl, + }).id; + + const before = liveModelIds(deps.catalogStore.listByProvider(providerUuid)); + const rows = listings.map(toLiveModel); + deps.catalogStore.replaceProviderModels(providerUuid, rows, deps.now()); + const after = liveModelIds(deps.catalogStore.listByProvider(providerUuid)); + + return { + provider: id, + status: 'refreshed', + added: countMissing(after, before), + updated: countShared(before, after), + deactivated: countMissing(before, after), + }; + } catch (err) { + return { provider: id, status: 'failed', error: secretFreeReason(err) }; + } + }; + + const refresh = async (opts?: RefreshOptions): Promise => { + const candidates = opts?.providers ?? deps.knownProviderIds; + const signal = opts?.signal; + + // Partition: a provider whose key does NOT resolve is skipped (not an error); the rest are connected and run + // in parallel with per-provider isolation. + const results: RefreshProviderResult[] = []; + const connected: { readonly id: ProviderId; readonly key: string }[] = []; + for (const id of candidates) { + let key: string; + try { + key = deps.keyFor(id); + } catch { + results.push({ provider: id, status: 'skipped-no-key' }); + continue; + } + connected.push({ id, key }); + } + + const settled = await Promise.allSettled( + connected.map(({ id, key }) => refreshOne(id, key, signal)), + ); + settled.forEach((outcome, index) => { + const entry = connected[index]; + if (entry === undefined) return; // unreachable — 1:1 with `connected` + // `refreshOne` catches internally, so `rejected` is defence-in-depth (never a leaked stack). + results.push( + outcome.status === 'fulfilled' + ? outcome.value + : { provider: entry.id, status: 'failed', error: 'refresh failed' }, + ); + }); + + // Deterministic order regardless of settle timing (a stable report for the human table + the --json stream). + results.sort((a, b) => a.provider.localeCompare(b.provider)); + return { providers: results }; + }; + + const refreshIfStale = async (opts?: RefreshOptions): Promise => { + const candidates = opts?.providers ?? deps.knownProviderIds; + const now = deps.now(); + const stale: ProviderId[] = []; + for (const id of candidates) { + // Only a CONNECTED provider can be refreshed — an unconnected one is never "stale" (nothing to fetch). + try { + deps.keyFor(id); + } catch { + continue; + } + const record = deps.providerStore.get(id); + const refreshedAt = + record === undefined ? undefined : deps.catalogStore.providerRefreshedAt(record.id); + if (refreshedAt === undefined || now - refreshedAt >= TTL_MS) { + stale.push(id); + } + } + if (stale.length === 0) { + return undefined; + } + return refresh({ + providers: stale, + ...(opts?.signal === undefined ? {} : { signal: opts.signal }), + }); + }; + + const refreshInBackground = (opts?: RefreshOptions): void => { + // Fire-and-forget: the `.catch` swallows ALL errors so a rejection is never unhandled and never surfaces a + // stack. We set NO timer here (the seam owns the per-request bound), so nothing of ours can keep a + // short-lived CLI process alive; the caller (S7 picker/Home) governs process lifecycle. + void refreshIfStale(opts).catch(() => { + /* intentionally swallowed — a background refresh must stay silent and non-fatal */ + }); + }; + + return { refresh, refreshIfStale, refreshInBackground }; +} diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index c0123665..fbcd8deb 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -117,6 +117,8 @@ The command set below is the confirmed surface. Commands ship **per workstream** | `relavium budget resume [--approve\|--abort]` | Resume a run suspended at a budget cap (`budget:paused`, `on_exceed: pause_for_approval`) — approve to continue or abort. The non-interactive operator path for [ADR-0028](../../decisions/0028-workflow-resource-governance.md). | | `relavium init` _(planned)_ | Initialize a `.relavium/` directory in the current project. | | `relavium agent ` _(planned)_ | Manage agents (list / create / test). | +| `relavium models` | List the cached model catalog (refreshes on first run if the cache is empty). See [`relavium models`](#relavium-models). | +| `relavium models refresh` | Force a live re-fetch of each connected provider's model list into the local cache, reporting per-provider outcomes. | | `relavium provider ` | Manage providers and API keys in the OS keychain (`list` / `add` / `set-key` / `remove-key` / `test`). | ## Command manifest @@ -200,9 +202,25 @@ Replays a past run's persisted `run_events` in `seq` order (the same data the de Shows the currently active/paused runs (from `runs` + `step_executions`) and each one's per-node status. Useful while a long workflow runs in another terminal or was launched detached. For any run paused at a human gate it also prints the **pending `gateId`(s)** (with gate type and node id), so a CI author can pass the right one to `relavium gate --gate ` — required when a run has more than one gate pending at once. It takes **no argument** (it lists every active run; a terminal run is not shown — inspect one with `relavium logs `). Under `--json` each active run is one NDJSON record — `{ runId, workflowId, status, startedAt, steps, pendingGates }`, where each `steps` entry is `{ nodeId, nodeType, status, attemptNumber, startedAt, completedAt, durationMs, costMicrocents }` and each `pendingGates` entry is `{ gateId, nodeId, gateType, message, expiresAt? }` (the same pending-gate shape [`gate list`](#relavium-gate-list) emits). +### `relavium models` + +The live model catalog (2.5.G, [ADR-0064](../../decisions/0064-live-model-catalog.md)). The catalog is a **local cache** in `history.db` (`model_catalog`) that records which model ids each connected provider key can currently reach; the static registry ([pricing.ts](../../../packages/llm/src/pricing.ts)) stays the pricing authority, so the cache holds **no price** and **no API key**. + +```bash +relavium models # list the cached catalog +relavium models refresh # force a live re-fetch of every connected provider +``` + +- **`relavium models`** (no subcommand) lists the cached catalog (read-only). On the **very first run** — when the cache is empty — it does one minimal **blocking** refresh, then lists; an empty result stays a clean exit `0` (an empty catalog is not a fault, like `relavium list`). Human output is one line per model (` ctx= []`). +- **`relavium models refresh`** forces a live re-fetch of **each connected provider** (a provider whose key resolves via the OS keychain → `RELAVIUM__API_KEY` env var) and prints a per-provider outcome. The refresh is **per-provider isolated**: one provider's failure (bad key, network, endpoint drift) or a provider without a list endpoint **never** fails the whole command — that provider is reported `failed` / `skipped` and the others still refresh. A per-provider failure is therefore **not** a command failure (exit `0` with the report). The **one** hard fault is an explicit `refresh` with **zero** providers connected (no key at all): that is a clean exit `2` naming how to add a key, because nothing could be fetched. +- **Security.** A provider key is read only to make the live request (over the bounded, abortable, secret-free `listModels` seam) and is **never** logged, persisted (the cache holds no key), or placed in the report / `--json` payload / any error message. A failing provider surfaces only the seam's already-redacted message (or a generic `refresh failed`), never a raw cause. +- **`--json`** ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)) emits **one NDJSON record per line**, stdout-pure, key-free: + - `relavium models --json` — one record per model: `{ provider, modelId, displayName, contextWindowTokens, maxOutputTokens, source, lastRefreshedAt, deprecationDate }` (`null` for an absent optional; `source` ∈ `static | live | user`; `lastRefreshedAt` is epoch-ms). + - `relavium models refresh --json` — one record per provider: `{ provider, status, added, updated, deactivated, error }`, where `status` ∈ `refreshed | skipped-no-key | skipped-unsupported | failed`, the three counts are the model ids added / refreshed-in-place / soft-deactivated (`null` unless `status` is `refreshed`), and `error` is a short, secret-free reason (`null` unless `failed`). + ### Read-command `--json` output -The non-streaming read commands (`list` / `status` / `gate list` / `chat-list`, and `logs`) keep the CLI to **one machine-output idiom**: `--json` emits **one result record per line** (NDJSON, `jq`-friendly, stdout-pure with diagnostics on stderr) — the same line-oriented shape `relavium run --json` uses for its `RunEvent` stream ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)). For `logs --json` the records ARE raw `RunEvent`s — the same `RunEvent` data the run streamed (re-serialized from the persisted log, so the field order may differ from the live `run --json` bytes); for the others they are the per-command result records documented above. An unknown `runId` (`logs` / `gate list`) is the structured pre-run fault on stderr with exit `2`, stdout empty — exactly as for `run`. (`chat-export --json` is **not** a read command — it emits a single `session:exported` **event**, not a result record, since the export is a session-lifecycle action.) +The non-streaming read commands (`list` / `status` / `gate list` / `chat-list` / `models`, and `logs`) keep the CLI to **one machine-output idiom**: `--json` emits **one result record per line** (NDJSON, `jq`-friendly, stdout-pure with diagnostics on stderr) — the same line-oriented shape `relavium run --json` uses for its `RunEvent` stream ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)). For `logs --json` the records ARE raw `RunEvent`s — the same `RunEvent` data the run streamed (re-serialized from the persisted log, so the field order may differ from the live `run --json` bytes); for the others they are the per-command result records documented above. An unknown `runId` (`logs` / `gate list`) is the structured pre-run fault on stderr with exit `2`, stdout empty — exactly as for `run`. (`chat-export --json` is **not** a read command — it emits a single `session:exported` **event**, not a result record, since the export is a session-lifecycle action.) ### `relavium gate` From 7588fdb3dd3ffa10e8715e71d49fa8859f4c99b5 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 21:52:07 +0300 Subject: [PATCH 13/68] =?UTF-8?q?fix(cli):=202.5.G=20S5=20Opus-review=20?= =?UTF-8?q?=E2=80=94=20slug=20output,=20terminal-escape=20sanitize,=20TTL?= =?UTF-8?q?=20backoff=20+=20test=20hardening?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opus's security round found 3 real bugs + 3 test gaps (6 confirmed, 0 refuted). - HIGH: `relavium models` emitted the internal llm_providers UUID as `provider` in both --json and the human table (while `models refresh` emitted the slug — inconsistent and breaking commands.md's `{ provider }` contract; a jq consumer keying on provider=='anthropic' matched nothing). withModelsDeps now builds a lazy uuid->slug map from providerStore.list() and injects providerSlug(); the list path emits the slug. - MEDIUM (CWE-150): a rogue/custom-endpoint model id could carry ANSI/C0 control bytes (ModelListingSchema only requires min(1)) and rendered raw to the terminal. oneLine now routes through the canonical stripTerminalControls (no reinvention) and is applied to the model id + provider column in the human table; --json is untouched (JSON.stringify escapes). - MEDIUM: refreshIfStale derived staleness only from providerRefreshedAt (max stamp over active live rows), which is undefined for a bad-key provider (throws before any write) or one returning [] — so it was perpetually stale and re-egressed on EVERY background trigger, defeating the 24h TTL. Added an in-service last-attempt Map (stamped for every attempted provider regardless of outcome); staleness = max(providerRefreshedAt, lastAttempt) — so a failed/empty provider is bounded to one attempt per TTL per process. Explicit refresh() still always attempts. - LOW: runList now prints a failure-aware message when the first-run refresh failed (instead of the unconditional "add a key"), human mode only. Test hardening: liveModelIds source='live' filter pinned with coexisting static/user rows; the key-leak test now throws a key in message AND cause and asserts the report excludes it (secretFreeReason gained additive redaction, mirroring validateProviderKey, still never reading .cause); exit-code halves (all-failed / all-skipped-unsupported -> exit 0, only zero-connected -> exit 2). Verified: pnpm turbo run lint typecheck test build — 24/24 green (CLI 1332 pass, +6); keys still never in reports/json/logs; --json byte-identical; the TTL map is in-memory only. Refs: ADR-0064, ADR-0049, ADR-0006 Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/commands/dispatch.ts | 10 ++ apps/cli/src/commands/models.test.ts | 96 +++++++++++++++++- apps/cli/src/commands/models.ts | 61 ++++++++++-- apps/cli/src/engine/model-refresh.test.ts | 113 +++++++++++++++++++++- apps/cli/src/engine/model-refresh.ts | 50 ++++++++-- 5 files changed, 302 insertions(+), 28 deletions(-) diff --git a/apps/cli/src/commands/dispatch.ts b/apps/cli/src/commands/dispatch.ts index c52a783c..621b59c1 100644 --- a/apps/cli/src/commands/dispatch.ts +++ b/apps/cli/src/commands/dispatch.ts @@ -321,11 +321,21 @@ async function withModelsDeps(ctx: DispatchContext, args: ModelsCommandArgs): Pr knownProviders: KNOWN_PROVIDERS, now: () => Date.now(), }); + // Translate the catalog rows' internal `llm_providers` UUID → the provider slug (e.g. `anthropic`) for the + // list command's `--json` `provider` field + human table. Built LAZILY on first use (memoized): the map is + // read only while RENDERING, which happens AFTER any first-run refresh has upserted its provider rows — so a + // freshly-discovered provider's slug is captured too. An unmapped uuid falls back to itself (never throws). + let slugByUuid: Map | undefined; + const providerSlug = (uuid: string): string => { + slugByUuid ??= new Map(providerStore.list().map((p): [string, string] => [p.id, p.name])); + return slugByUuid.get(uuid) ?? uuid; + }; return await modelsCommand(args, { io: ctx.io, global: ctx.global, catalog: catalogStore, refreshService, + providerSlug, }); } finally { close(); diff --git a/apps/cli/src/commands/models.test.ts b/apps/cli/src/commands/models.test.ts index 448899ab..306724cd 100644 --- a/apps/cli/src/commands/models.test.ts +++ b/apps/cli/src/commands/models.test.ts @@ -66,8 +66,9 @@ function deps( catalog: ModelsCatalogReader, refreshService: ModelRefreshService, json = false, + providerSlug: (uuid: string) => string = (uuid) => uuid, ): ModelsCommandDeps { - return { io, global: globalOptions(json), catalog, refreshService }; + return { io, global: globalOptions(json), catalog, refreshService, providerSlug }; } const REFRESHED: RefreshReport = { @@ -112,20 +113,70 @@ describe('modelsCommand — list', () => { expect(out()).toContain('No models cached'); }); - it('--json emits one key-free record per model', async () => { + it('--json emits one key-free record per model with the provider SLUG (not the internal uuid)', async () => { + // FIX 1: the catalog row carries the internal `llm_providers` UUID; the emitted `provider` must be the slug. const { io, out } = captureIo(); - const rowsRef = { value: [modelRow('anthropic', 'claude-x'), modelRow('openai', 'gpt-x')] }; + const anthropicUuid = '11111111-1111-1111-1111-111111111111'; + const openaiUuid = '22222222-2222-2222-2222-222222222222'; + const slug = (uuid: string): string => + uuid === anthropicUuid ? 'anthropic' : uuid === openaiUuid ? 'openai' : uuid; + const rowsRef = { value: [modelRow(anthropicUuid, 'claude-x'), modelRow(openaiUuid, 'gpt-x')] }; const refresh = stubRefresh(REFRESHED); - await modelsCommand({ refresh: false }, deps(io, stubCatalog(rowsRef), refresh, true)); + await modelsCommand({ refresh: false }, deps(io, stubCatalog(rowsRef), refresh, true, slug)); const records = parseNdjson(out()); expect(records).toHaveLength(2); expect(records[0]).toMatchObject({ - provider: 'anthropic', + provider: 'anthropic', // the SLUG, never the uuid modelId: 'claude-x', contextWindowTokens: 200_000, source: 'live', }); + expect(out()).not.toContain(anthropicUuid); // the internal uuid never reaches --json + }); + + it('renders the provider SLUG (not the internal uuid) in the human table (FIX 1)', async () => { + const { io, out } = captureIo(); + const anthropicUuid = '11111111-1111-1111-1111-111111111111'; + const slug = (uuid: string): string => (uuid === anthropicUuid ? 'anthropic' : uuid); + const rowsRef = { value: [modelRow(anthropicUuid, 'claude-x')] }; + const refresh = stubRefresh(REFRESHED); + + await modelsCommand({ refresh: false }, deps(io, stubCatalog(rowsRef), refresh, false, slug)); + const text = out(); + expect(text).toContain('anthropic'); + expect(text).not.toContain(anthropicUuid); + }); + + it('strips terminal control sequences from a rogue model id in the human table (FIX 2, CWE-150)', async () => { + const { io, out } = captureIo(); + const rowsRef = { value: [modelRow('anthropic', 'gpt-x\u001b[2K\rspoof')] }; + const refresh = stubRefresh(REFRESHED); + + await modelsCommand({ refresh: false }, deps(io, stubCatalog(rowsRef), refresh)); + const text = out(); + expect(text).not.toContain('\u001b'); // ANSI ESC stripped + expect(text).not.toContain('\r'); // CR stripped (no line-overwrite) + expect(text).toContain('gpt-x'); // the visible id text survives + }); + + it('reports a FAILED first-run refresh (not "add a key") when the catalog is still empty (FIX 4)', async () => { + const { io, out } = captureIo(); + const rowsRef: { value: ModelCatalogListing[] } = { value: [] }; + // The first-run refresh runs, but a CONNECTED provider FAILED — the catalog stays empty for a reason that is + // NOT "no key", so the plain add-a-key guidance would mislead. + const refresh = stubRefresh({ + providers: [ + { provider: 'anthropic', status: 'failed', error: 'bad key' }, + { provider: 'openai', status: 'skipped-no-key' }, + ], + }); + + const code = await modelsCommand({ refresh: false }, deps(io, stubCatalog(rowsRef), refresh)); + expect(code).toBe(EXIT_CODES.success); + const text = out(); + expect(text).toContain('Model refresh failed for anthropic'); + expect(text).not.toContain('No models cached'); // not the misleading add-a-key line }); }); @@ -194,4 +245,39 @@ describe('modelsCommand — refresh', () => { } expect(out()).toBe(''); // stdout stays empty on a fault }); + + it('exits 0 when every connected provider FAILED — a per-provider failure is not a command failure (FIX 6)', async () => { + // Every provider is `failed` (connected, but the fetch threw) — NOT `skipped-no-key`, so it is NOT the + // zero-connected exit-2 case; the command prints the per-provider report and exits 0. + const { io, out } = captureIo(); + const rowsRef: { value: ModelCatalogListing[] } = { value: [] }; + const refresh = stubRefresh({ + providers: [ + { provider: 'anthropic', status: 'failed', error: 'bad key' }, + { provider: 'openai', status: 'failed', error: 'network unreachable' }, + ], + }); + + const code = await modelsCommand({ refresh: true }, deps(io, stubCatalog(rowsRef), refresh)); + expect(code).toBe(EXIT_CODES.success); + const text = out(); + expect(text).toContain('failed'); + expect(text).toContain('anthropic'); + expect(text).toContain('openai'); + }); + + it('exits 0 when every provider is skipped-unsupported (no list endpoint) (FIX 6)', async () => { + // Connected but no `listModels` endpoint — still not the zero-connected exit-2 case. + const { io } = captureIo(); + const rowsRef: { value: ModelCatalogListing[] } = { value: [] }; + const refresh = stubRefresh({ + providers: [ + { provider: 'anthropic', status: 'skipped-unsupported' }, + { provider: 'openai', status: 'skipped-unsupported' }, + ], + }); + + const code = await modelsCommand({ refresh: true }, deps(io, stubCatalog(rowsRef), refresh)); + expect(code).toBe(EXIT_CODES.success); + }); }); diff --git a/apps/cli/src/commands/models.ts b/apps/cli/src/commands/models.ts index 4aabc9ce..05a4acab 100644 --- a/apps/cli/src/commands/models.ts +++ b/apps/cli/src/commands/models.ts @@ -10,6 +10,7 @@ import { EXIT_CODES, type ExitCode } from '../process/exit-codes.js'; import type { CliIo } from '../process/io.js'; import type { GlobalOptions } from '../process/options.js'; import { writeRecordLines } from '../render/records.js'; +import { stripTerminalControls } from '../render/tui/chat-projection.js'; /** * The `relavium models` / `relavium models refresh` command core (workstream **2.5.G S5**, @@ -45,6 +46,13 @@ export interface ModelsCommandDeps { readonly global: GlobalOptions; readonly catalog: ModelsCatalogReader; readonly refreshService: ModelRefreshService; + /** + * Translate an internal `llm_providers` UUID (the FK `ModelCatalogListing.providerId` carries) → its provider + * SLUG (e.g. `anthropic`) for the human table + the `--json` `provider` field, so the list path matches the + * `models refresh` report and the documented `{ provider }` contract (commands.md). Falls back to the uuid + * itself when unmapped — NEVER throws. + */ + readonly providerSlug: (uuid: string) => string; } export async function modelsCommand( @@ -80,17 +88,34 @@ async function runRefresh(deps: ModelsCommandDeps): Promise { /** `models` (no sub) — list the cached catalog, refreshing first only when it is empty (first-run, ADR-0064 §5a). */ async function runList(deps: ModelsCommandDeps): Promise { let listings = deps.catalog.listAll(); + let firstRunReport: RefreshReport | undefined; if (listings.length === 0) { // First-run: a minimal blocking refresh so the very first `models` shows a real catalog, then re-read. A // provider with no key is skipped inside `refresh`, so this stays a no-op (still exit 0) when no key is set. - await deps.refreshService.refresh(); + firstRunReport = await deps.refreshService.refresh(); listings = deps.catalog.listAll(); } if (deps.global.json) { - writeRecordLines(deps.io, listings.map(toModelJson)); + writeRecordLines( + deps.io, + listings.map((m) => toModelJson(m, deps.providerSlug)), + ); return EXIT_CODES.success; } - renderModelList(deps.io, listings); + // Human mode only (FIX 4): when the catalog is STILL empty after a first-run refresh because a CONNECTED + // provider's refresh FAILED (not merely because no key was set), print a failure-aware line — the plain + // "add a key" guidance would be misleading (a key IS set; the fetch failed). + if (listings.length === 0 && firstRunReport !== undefined) { + const failed = firstRunReport.providers.filter((p) => p.status === 'failed'); + if (failed.length > 0) { + const names = failed.map((p) => p.provider).join(', '); + deps.io.writeOut( + `Model refresh failed for ${names}; showing no models — run \`relavium models refresh\` or check \`relavium doctor\`.\n`, + ); + return EXIT_CODES.success; + } + } + renderModelList(deps.io, listings, deps.providerSlug); return EXIT_CODES.success; } @@ -107,9 +132,12 @@ function toRefreshJson(p: RefreshProviderResult): unknown { }; } -function toModelJson(m: ModelCatalogListing): unknown { +function toModelJson(m: ModelCatalogListing, providerSlug: (uuid: string) => string): unknown { return { - provider: m.providerId, + // The SLUG (e.g. `anthropic`), not the internal `llm_providers` UUID the catalog row carries — matching the + // `models refresh` report + the documented `{ provider }` contract. `--json` is unchanged otherwise + // (JSON.stringify escapes any control byte on its own, so the slug is not terminal-sanitized here). + provider: providerSlug(m.providerId), modelId: m.modelId, displayName: m.displayName, contextWindowTokens: m.contextWindowTokens ?? null, @@ -122,9 +150,15 @@ function toModelJson(m: ModelCatalogListing): unknown { // ── human renderers ──────────────────────────────────────────────────────────── -/** Collapse whitespace so a (secret-free) provider error can never break the one-line-per-provider table. */ +/** + * Sanitize a PROVIDER-supplied string for one-line terminal display (FIX 2, CWE-150): first strip ANSI/OSC/C0/C1 + * control bytes via the canonical {@link stripTerminalControls} — so a rogue/custom-endpoint model id, display + * name, or refresh error can't inject a cursor jump / clear-line / `\r` line-overwrite (`ModelListingSchema` only + * requires `min(1)`, so it never guarantees a clean id) — then collapse the remaining whitespace so it can never + * break the one-line-per-row table. RENDER boundary only: the stored rows and the `--json` payload are untouched. + */ function oneLine(text: string): string { - return text.replace(/\s+/g, ' ').trim(); + return stripTerminalControls(text).replace(/\s+/gu, ' ').trim(); } function renderRefreshReport(io: CliIo, report: RefreshReport): void { @@ -148,7 +182,11 @@ function refreshDetail(p: RefreshProviderResult): string { } } -function renderModelList(io: CliIo, listings: readonly ModelCatalogListing[]): void { +function renderModelList( + io: CliIo, + listings: readonly ModelCatalogListing[], + providerSlug: (uuid: string) => string, +): void { if (listings.length === 0) { io.writeOut( 'No models cached. Add a provider key (`relavium provider set-key `) and run `relavium models refresh`.\n', @@ -158,6 +196,11 @@ function renderModelList(io: CliIo, listings: readonly ModelCatalogListing[]): v io.writeOut(`Models (${listings.length}):\n`); for (const m of listings) { const ctx = m.contextWindowTokens === undefined ? '' : `\tctx=${m.contextWindowTokens}`; - io.writeOut(` ${m.modelId}\t${m.providerId}${ctx}\t[${m.source}]\n`); + // `m.modelId` (and `m.displayName`, were it shown) is provider-supplied → terminal-sanitized (FIX 2). The + // provider column is the SLUG, never the internal UUID (FIX 1); it too passes through `oneLine` in case a + // custom-provider slug carries a control byte, and falls back to the (safe) uuid when unmapped. + io.writeOut( + ` ${oneLine(m.modelId)}\t${oneLine(providerSlug(m.providerId))}${ctx}\t[${m.source}]\n`, + ); } } diff --git a/apps/cli/src/engine/model-refresh.test.ts b/apps/cli/src/engine/model-refresh.test.ts index de8dcc08..54cf6616 100644 --- a/apps/cli/src/engine/model-refresh.test.ts +++ b/apps/cli/src/engine/model-refresh.test.ts @@ -171,6 +171,65 @@ describe('createModelRefreshService', () => { }); }); + it('counts only the LIVE delta — a source=static / source=user row is excluded (FIX 5a)', async () => { + let models: ModelListing[] = [listing('m1'), listing('m2')]; + const resolver = stubResolver({ + adapters: { openai: stubProvider('openai', () => Promise.resolve(models)) }, + keys: { openai: SECRET_KEY }, + }); + const { svc, providerStore, catalogStore } = service(resolver, ['openai']); + + // Seed a static (media-routing) row and a user (pricing) row for the SAME provider BEFORE any refresh; a live + // refresh must NEVER count them in its added/updated/deactivated diff (they are provenance-protected — S4). + const providerUuid = providerStore.upsert({ + name: 'openai', + displayName: 'OpenAI', + baseUrl: 'https://api.openai.com/v1', + }).id; + catalogStore.upsert({ + providerId: providerUuid, + modelId: 'static-model', + displayName: 'Static', + contextWindowTokens: 1000, + maxOutputTokens: 100, + source: 'static', + }); + catalogStore.upsert({ + providerId: providerUuid, + modelId: 'user-model', + displayName: 'User', + contextWindowTokens: 1000, + maxOutputTokens: 100, + source: 'user', + }); + + // First refresh: two LIVE models. added=2, updated=0, deactivated=0. Without the `source==='live'` filter in + // `liveModelIds`, the pre-existing static/user rows would be counted as "updated" (→ 2) — so this pins it. + const first = await svc.refresh(); + expect(first.providers[0]).toMatchObject({ + status: 'refreshed', + added: 2, + updated: 0, + deactivated: 0, + }); + + // Second refresh: drop m1, keep m2, add m3. added=1, updated=1 (only m2), deactivated=1 (only m1) — the + // static/user rows stay out of every count (without the filter, `updated` would be 3). + models = [listing('m2'), listing('m3')]; + const second = await svc.refresh(); + expect(second.providers[0]).toMatchObject({ + status: 'refreshed', + added: 1, + updated: 1, + deactivated: 1, + }); + + // The static + user rows survived every live refresh (never deactivated). + const ids = catalogStore.listByProvider(providerUuid).map((m) => m.modelId); + expect(ids).toContain('static-model'); + expect(ids).toContain('user-model'); + }); + it('refreshIfStale refreshes on an empty cache, no-ops within the TTL, and refreshes again once stale', async () => { const resolver = stubResolver({ adapters: { openai: stubProvider('openai', () => Promise.resolve([listing('m1')])) }, @@ -195,13 +254,55 @@ describe('createModelRefreshService', () => { expect(third?.providers[0]?.status).toBe('refreshed'); }); - it('never leaks a provider key into the report (or an error message)', async () => { + it('bounds re-egress: refreshIfStale does NOT re-attempt a FAILED provider within the TTL, but does past it (FIX 3)', async () => { + let calls = 0; + const resolver = stubResolver({ + adapters: { + openai: stubProvider('openai', () => { + calls += 1; + return Promise.reject(new Error('bad key')); + }), + }, + keys: { openai: SECRET_KEY }, + }); + const { svc } = service(resolver, ['openai']); + + // First refreshIfStale: empty cache ⇒ stale ⇒ attempts ⇒ FAILS. It writes NO live row, so + // `providerRefreshedAt` stays undefined — the failure mode ADR-0064 §5c warns re-egresses on every trigger. + const first = await svc.refreshIfStale(); + expect(first?.providers[0]?.status).toBe('failed'); + expect(calls).toBe(1); + + // Within the TTL: `providerRefreshedAt` is still undefined, but the in-service last-ATTEMPT stamp bounds it + // ⇒ NOT stale ⇒ NO re-attempt (`listModels` call count unchanged). + nowRef.value += TTL_MS - 1; + expect(await svc.refreshIfStale()).toBeUndefined(); + expect(calls).toBe(1); + + // Past the TTL: the attempt stamp is now stale ⇒ re-attempts exactly once more. + nowRef.value += 1; + const third = await svc.refreshIfStale(); + expect(third?.providers[0]?.status).toBe('failed'); + expect(calls).toBe(2); + + // An explicit `refresh()` is NOT bounded — it always re-attempts (only background/if-stale is throttled). + const forced = await svc.refresh(); + expect(forced.providers[0]?.status).toBe('failed'); + expect(calls).toBe(3); + }); + + it('never leaks a provider key — redacts the key in the error message AND never reads the cause (FIX 5b)', async () => { const resolver = stubResolver({ adapters: { anthropic: stubProvider('anthropic', () => - // Even a failing adapter whose error TEXT mentions the key: the service surfaces `error.message`, which - // the real seam redacts. Here we assert the service never fabricates a key into the report itself. - Promise.reject(new Error('list failed for provider')), + // A NON-seam / raw adapter error that embeds the key in BOTH its `message` AND a key-bearing `cause` + // (the real seam would redact first — this proves the SERVICE defends regardless): `secretFreeReason` + // must redact the message occurrence AND never fold `cause` into the report. + Promise.reject( + new Error(`auth failed for key ${SECRET_KEY}`, { + cause: new Error(`nested token=${SECRET_KEY}`), + }), + ), ), openai: stubProvider('openai', () => Promise.resolve([listing('gpt-x')])), }, @@ -210,7 +311,11 @@ describe('createModelRefreshService', () => { const { svc } = service(resolver, ['anthropic', 'openai']); const report = await svc.refresh(); + const failed = report.providers.find((p) => p.provider === 'anthropic'); + expect(failed?.status).toBe('failed'); + // Neither the whole report nor the specific `.error` field carries the secret (message redacted, cause dropped). expect(JSON.stringify(report)).not.toContain(SECRET_KEY); + expect(failed?.error ?? '').not.toContain(SECRET_KEY); }); it('refreshInBackground never throws and swallows a rejecting refresh', async () => { diff --git a/apps/cli/src/engine/model-refresh.ts b/apps/cli/src/engine/model-refresh.ts index 304c0953..d598c39f 100644 --- a/apps/cli/src/engine/model-refresh.ts +++ b/apps/cli/src/engine/model-refresh.ts @@ -1,6 +1,8 @@ import type { ModelCatalogListing, ModelCatalogLiveModel } from '@relavium/db'; import type { AbortSignalLike, LlmProvider, ModelListing, ProviderId } from '@relavium/llm'; +import { keyHint } from './providers.js'; + /** * The live model-catalog refresh orchestrator (workstream **2.5.G S5**, [ADR-0064](../../../../docs/decisions/0064-live-model-catalog.md) * §5). A **host service with injected deps** so desktop / VS Code reuse it later and `@relavium/llm` / @@ -156,25 +158,48 @@ function countShared(a: ReadonlySet, b: ReadonlySet): number { } /** - * A SECRET-FREE failure reason. The seam's `boundedListModels` already redacts the key AND strips the cause, - * so its `error.message` is safe to surface; a non-Error / empty message degrades to a generic string. We NEVER - * read `err.cause` (it could carry a nested field a verbose render might expose — the `validateProviderKey` rule). + * A SECRET-FREE failure reason. The seam's `boundedListModels` already redacts the key AND strips the cause, so + * its `error.message` is safe to surface; a non-Error / empty message degrades to a generic string. As + * DEFENCE-IN-DEPTH — for a non-seam adapter, or a future direct throw that bypasses the seam's redaction — we + * ALSO redact any literal occurrence of the known key from the surfaced message (mirroring + * `validateProviderKey`'s `raw.split(key).join(keyHint(key))`), so even a raw adapter error that embeds the key + * can never reach a {@link RefreshReport} / `--json` payload / a log. We NEVER read `err.cause` (it could carry a + * nested field a verbose render might expose — the `validateProviderKey` rule). */ -function secretFreeReason(err: unknown): string { - if (err instanceof Error && err.message.trim() !== '') { - return err.message; - } - return 'refresh failed'; +function secretFreeReason(err: unknown, key: string): string { + const raw = err instanceof Error && err.message.trim() !== '' ? err.message : 'refresh failed'; + // An empty key can never occur (the resolver rejects `''`), but guard the `split('')`-garbles-everything footgun. + return key === '' ? raw : raw.split(key).join(keyHint(key)); +} + +/** The larger of two optional epoch-ms stamps (`undefined` only when BOTH are absent). */ +function maxDefined(a: number | undefined, b: number | undefined): number | undefined { + if (a === undefined) return b; + if (b === undefined) return a; + return Math.max(a, b); } /** Create the refresh orchestrator over its injected deps. Pure of platform I/O beyond the injected ports. */ export function createModelRefreshService(deps: ModelRefreshDeps): ModelRefreshService { + // In-service (NEVER persisted — FIX 3 / ADR-0064 §5c) last-ATTEMPT stamp per provider `refreshOne` actually + // ran (ANY outcome: refreshed / failed / skipped-unsupported / empty). `catalogStore.providerRefreshedAt` is + // `undefined` for a provider whose refresh FAILED (threw before any live row was written) or returned `[]` + // (all live rows soft-deactivated), so deriving staleness from it ALONE would treat such a provider as + // PERPETUALLY stale and re-egress on every background trigger — defeating the 24h TTL. Bounding on the last + // attempt caps a failed/empty provider's background re-egress to once per TTL per process while PRESERVING the + // successful-row freshness signal. Only `refreshIfStale`/`refreshInBackground` is bounded — an explicit + // `refresh()` always attempts. + const lastAttemptAt = new Map(); + /** Refresh ONE connected provider (its key already resolved). NEVER throws — a fault becomes `status:'failed'`. */ const refreshOne = async ( id: ProviderId, key: string, signal: AbortSignalLike | undefined, ): Promise => { + // Stamp the ATTEMPT before anything can throw — `refreshOne` runs only for a connected provider, and every + // outcome (refreshed / failed / skipped-unsupported / empty) counts as an attempt for the TTL backoff (FIX 3). + lastAttemptAt.set(id, deps.now()); try { const adapter = deps.resolveProvider(id); if (adapter?.listModels === undefined) { @@ -206,7 +231,7 @@ export function createModelRefreshService(deps: ModelRefreshDeps): ModelRefreshS deactivated: countMissing(before, after), }; } catch (err) { - return { provider: id, status: 'failed', error: secretFreeReason(err) }; + return { provider: id, status: 'failed', error: secretFreeReason(err, key) }; } }; @@ -262,7 +287,12 @@ export function createModelRefreshService(deps: ModelRefreshDeps): ModelRefreshS const record = deps.providerStore.get(id); const refreshedAt = record === undefined ? undefined : deps.catalogStore.providerRefreshedAt(record.id); - if (refreshedAt === undefined || now - refreshedAt >= TTL_MS) { + // Freshest of the successful-row stamp AND the last-attempt stamp: a provider that FAILED (no row written) + // or returned `[]` (all live rows deactivated) has no `refreshedAt`, but IS bounded by its attempt stamp + // (FIX 3 / ADR-0064 §5c) — so it is not re-egressed within the TTL despite `providerRefreshedAt` being + // undefined. A never-attempted, never-refreshed provider stays stale (both undefined). + const newest = maxDefined(refreshedAt, lastAttemptAt.get(id)); + if (newest === undefined || now - newest >= TTL_MS) { stale.push(id); } } From c6f1b82db4554fbfd052d70a5d7b8e6671ef7a67 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 22:23:54 +0300 Subject: [PATCH 14/68] =?UTF-8?q?fix(db,cli):=202.5.G=20S5=20Sonnet-review?= =?UTF-8?q?=20=E2=80=94=20atomic=20refresh=20counts=20+=20background-scope?= =?UTF-8?q?=20docs=20+=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sonnet's fresh round: 4 confirmed + 1 plausible, 0 refuted. - MEDIUM: refreshOne computed added/updated/deactivated from a NON-atomic before/replace/after diff (3 separate store round-trips), so two concurrent same-provider refreshes could each report the same rows as `added` (data was correct; only the reported counts). Fixed by making replaceProviderModels RETURN {added, updated, deactivated} counted ATOMICALLY inside its own transaction (added on insert, updated on an existing-live-row refresh, deactivated from the soft-deactivate UPDATE's .changes; non-live static/user rows counted in none). refreshOne uses the returned counts; the before/after liveModelIds diff + helpers are deleted (−2 queries). - MEDIUM + PLAUSIBLE (docs only, no behavior change): the in-memory lastAttemptAt TTL backoff + the fire-and-forget refreshInBackground are correct only within a LONG-LIVED host process; documented that scope explicitly and added a hard S7 constraint — wire refreshInBackground ONLY into the long-lived Home, never a one-shot CLI invocation (a durable per-provider last-attempt stamp is the named follow-up if per-process background is ever needed). Currently inert (nothing wires it yet). - MEDIUM (test): withModelsDeps had no integration test. Extracted createProviderSlugResolver + an injectable ModelsDbPorts; new models-dispatch.test.ts (real in-memory db, network-free stub resolver) pins that a first-run-discovered provider renders its SLUG (not the uuid) in-process, and the db closes on the throw path. - LOW (test): pinned the models --json first-run-failure contract (empty NDJSON + exit 0). Store invariants intact (one transaction; user/static preserved; soft-deactivate only; no hard-delete) — S4 tests pass. Security posture intact (keys stay out of everything; the new test is network-free). Verified: pnpm turbo run lint typecheck test build — 24/24 green (@relavium/db 181, CLI 1336). (The pre-existing flaky media-gc timing test intermittently reddens a cold run and passes on retry — unrelated.) prettier clean. Refs: ADR-0064, ADR-0049 Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/commands/dispatch.ts | 63 +++++-- apps/cli/src/commands/models-dispatch.test.ts | 166 ++++++++++++++++++ apps/cli/src/commands/models.test.ts | 19 ++ apps/cli/src/engine/model-refresh.test.ts | 21 +-- apps/cli/src/engine/model-refresh.ts | 87 ++++----- packages/db/src/index.ts | 4 +- packages/db/src/model-catalog-store.test.ts | 86 +++++++++ packages/db/src/model-catalog-store.ts | 44 ++++- 8 files changed, 410 insertions(+), 80 deletions(-) create mode 100644 apps/cli/src/commands/models-dispatch.test.ts diff --git a/apps/cli/src/commands/dispatch.ts b/apps/cli/src/commands/dispatch.ts index 621b59c1..f45bbc04 100644 --- a/apps/cli/src/commands/dispatch.ts +++ b/apps/cli/src/commands/dispatch.ts @@ -3,12 +3,13 @@ import { randomUUID } from 'node:crypto'; import { createModelCatalogStore, createProviderStore } from '@relavium/db'; import { loadResolvedConfig } from '../config/load.js'; -import { openLocalDb } from '../db/open.js'; +import { openLocalDb, type OpenedDb } from '../db/open.js'; import { createModelRefreshService } from '../engine/model-refresh.js'; import { KNOWN_PROVIDERS, KNOWN_PROVIDER_IDS, createProviderResolver, + type ProviderResolver, } from '../engine/providers.js'; import { openHistoryStore } from '../history/open.js'; import { openSessionStore } from '../history/session-open.js'; @@ -296,20 +297,59 @@ const executeLogs: CommandExecutor = (input, ctx) => const executeStatus: CommandExecutor = (_input, ctx) => Promise.resolve(statusCommand({ io: ctx.io, global: ctx.global })); +/** + * The lazy `llm_providers`-UUID → provider-slug (e.g. `anthropic`) resolver the `models` list path uses for its + * `--json` `provider` field + human table. The `id → name` map is built LAZILY on first call and memoized (`??=`): + * it is read only while RENDERING, which happens AFTER any first-run refresh has upserted its provider rows — so a + * provider DISCOVERED on this very invocation is captured too. Hoisting the map build ahead of the refresh would + * silently render a first-run provider's raw UUID instead of its slug. An unmapped uuid falls back to itself + * (never throws). Extracted (not inlined in {@link withModelsDeps}) so the lazy-after-refresh ordering is unit-tested. + */ +export function createProviderSlugResolver( + providerStore: Pick, 'list'>, +): (uuid: string) => string { + let slugByUuid: Map | undefined; + return (uuid: string): string => { + slugByUuid ??= new Map(providerStore.list().map((p): [string, string] => [p.id, p.name])); + return slugByUuid.get(uuid) ?? uuid; + }; +} + +/** + * The I/O ports {@link withModelsDeps} owns — the local-db opener and the OS-keychain-backed provider resolver + * factory. Injectable (defaulting to {@link PRODUCTION_MODELS_PORTS}) so a test can drive the whole `models` + * wiring — including the close-on-fault lifecycle — over an in-memory db + a network-free stub resolver, without + * touching the real `history.db` or loading the native keychain. + */ +export interface ModelsDbPorts { + readonly openDb: (homeDir: string) => OpenedDb; + readonly makeResolver: (io: CliIo) => Pick; +} + +const PRODUCTION_MODELS_PORTS: ModelsDbPorts = { + openDb: openLocalDb, + makeResolver: (io) => createProviderResolver(io.env, createOsKeychainStore()), +}; + /** * Open the local db + OS keychain for one `models` invocation, wire the S5 refresh service over the S4 catalog - * store + the S2 `listModels?` seam, run the core, and always close the db (2.5.G S5, ADR-0064). The key - * resolver reads a provider key only inside the refresh (keychain → env); the catalog holds no key. + * store + the S2 `listModels?` seam, run the core, and ALWAYS close the db — even on a thrown fault (2.5.G S5, + * ADR-0064). The key resolver reads a provider key only inside the refresh (keychain → env); the catalog holds no + * key. `ports` is injectable for tests; production uses the real db + keychain-backed resolver. */ -async function withModelsDeps(ctx: DispatchContext, args: ModelsCommandArgs): Promise { +export async function withModelsDeps( + ctx: DispatchContext, + args: ModelsCommandArgs, + ports: ModelsDbPorts = PRODUCTION_MODELS_PORTS, +): Promise { const { homeDir } = loadResolvedConfig({ cwd: ctx.global.cwd, configPath: ctx.global.configPath, }); - const { db, close } = openLocalDb(homeDir); + const { db, close } = ports.openDb(homeDir); try { const storeDeps = { uuid: () => randomUUID(), now: () => Date.now() }; - const resolver = createProviderResolver(ctx.io.env, createOsKeychainStore()); + const resolver = ports.makeResolver(ctx.io); const providerStore = createProviderStore(db, storeDeps); const catalogStore = createModelCatalogStore(db, storeDeps); const refreshService = createModelRefreshService({ @@ -321,21 +361,12 @@ async function withModelsDeps(ctx: DispatchContext, args: ModelsCommandArgs): Pr knownProviders: KNOWN_PROVIDERS, now: () => Date.now(), }); - // Translate the catalog rows' internal `llm_providers` UUID → the provider slug (e.g. `anthropic`) for the - // list command's `--json` `provider` field + human table. Built LAZILY on first use (memoized): the map is - // read only while RENDERING, which happens AFTER any first-run refresh has upserted its provider rows — so a - // freshly-discovered provider's slug is captured too. An unmapped uuid falls back to itself (never throws). - let slugByUuid: Map | undefined; - const providerSlug = (uuid: string): string => { - slugByUuid ??= new Map(providerStore.list().map((p): [string, string] => [p.id, p.name])); - return slugByUuid.get(uuid) ?? uuid; - }; return await modelsCommand(args, { io: ctx.io, global: ctx.global, catalog: catalogStore, refreshService, - providerSlug, + providerSlug: createProviderSlugResolver(providerStore), }); } finally { close(); diff --git a/apps/cli/src/commands/models-dispatch.test.ts b/apps/cli/src/commands/models-dispatch.test.ts new file mode 100644 index 00000000..57ace558 --- /dev/null +++ b/apps/cli/src/commands/models-dispatch.test.ts @@ -0,0 +1,166 @@ +import { tmpdir } from 'node:os'; + +import { createClient, createProviderStore, runMigrations, type DbClient } from '@relavium/db'; +import type { LlmProvider, ModelListing, ProviderId } from '@relavium/llm'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import type { ProviderResolver } from '../engine/providers.js'; +import { EXIT_CODES } from '../process/exit-codes.js'; +import type { CliIo } from '../process/io.js'; +import { captureIo, CHAT_TEXT_CAPABILITY_FLAGS } from '../test-support.js'; +import { + createProviderSlugResolver, + withModelsDeps, + type DispatchContext, + type ModelsDbPorts, +} from './dispatch.js'; + +/** + * `withModelsDeps` integration wiring tests (2.5.G S5, ADR-0064) — the real-db half `models.test.ts` (the pure + * command core) cannot cover: the S5 refresh service over the S4 catalog store, the LAZY-after-refresh + * uuid→slug resolver, and the close-on-fault db lifecycle. NETWORK-FREE: an INJECTED stub resolver whose + * `listModels` returns canned {@link ModelListing}s over a real in-memory `history.db` (via injectable + * {@link ModelsDbPorts}) — the native keychain and the real `history.db` are never touched. + */ + +const SECRET_KEY = 'sk-super-secret-key-value'; + +/** A minimal {@link LlmProvider} — only `id` + `listModels` matter here; the seam methods throw (never reached). */ +function stubProvider(id: ProviderId, listModels: LlmProvider['listModels']): LlmProvider { + return { + id, + generate: () => Promise.reject(new Error('stub generate not used')), + stream: () => { + throw new Error('stub stream not used'); + }, + supports: CHAT_TEXT_CAPABILITY_FLAGS, + ...(listModels === undefined ? {} : { listModels }), + }; +} + +/** A resolver stub: `models[id]` are that provider's live listings; `keys[id]` its key (absent ⇒ keyFor throws). */ +function stubResolver( + models: Partial>, + keys: Partial>, +): Pick { + return { + resolveProvider: (id) => { + const listings = models[id]; + return listings === undefined ? undefined : stubProvider(id, () => Promise.resolve(listings)); + }, + keyFor: (id) => { + const key = keys[id]; + if (key === undefined) { + throw new Error(`no key for ${id}`); + } + return key; + }, + }; +} + +function listing(id: string, contextWindowTokens?: number): ModelListing { + return { + id, + displayName: id, + ...(contextWindowTokens === undefined ? {} : { contextWindowTokens }), + }; +} + +describe('withModelsDeps (2.5.G S5 — real-db wiring + lazy slug + close-on-fault)', () => { + let client: DbClient; + + beforeEach(() => { + client = createClient(':memory:'); + runMigrations(client.db); + }); + afterEach(() => { + client.sqlite.close(); + }); + + /** A `cwd` outside any `.relavium/` project so `loadResolvedConfig` resolves clean defaults (no committed layers). */ + function context(io: CliIo, json: boolean): DispatchContext { + return { + io, + global: { json, color: false, cwd: tmpdir(), configPath: undefined, verbosity: 'normal' }, + }; + } + + /** Ports over the shared in-memory db + an injected resolver; `onClose` records that the finally-close ran. */ + function testPorts( + resolver: Pick, + onClose?: () => void, + ): ModelsDbPorts { + return { + // The shared client is closed by afterEach — the spy only records the call, so a real close never + // double-closes the better-sqlite3 handle. + openDb: () => ({ db: client.db, close: () => onClose?.() }), + makeResolver: () => resolver, + }; + } + + /** A provider store over the SAME db withModelsDeps wrote through — reads back what a first-run refresh created. */ + function readProviderStore() { + return createProviderStore(client.db, { uuid: () => 'unused', now: () => 0 }); + } + + it('a first-run refresh that discovers a NEW provider renders its SLUG (not the internal uuid) in the same invocation', async () => { + const { io, out } = captureIo(); + // Only anthropic has a key ⇒ it is the sole connected provider; its provider row does NOT exist yet, so the + // first-run refresh DISCOVERS it (upserts the llm_providers row) mid-invocation. + const resolver = stubResolver( + { anthropic: [listing('claude-x', 200_000)] }, + { anthropic: SECRET_KEY }, + ); + + const code = await withModelsDeps(context(io, false), { refresh: false }, testPorts(resolver)); + expect(code).toBe(EXIT_CODES.success); + + const text = out(); + expect(text).toContain('claude-x'); // the freshly-refreshed model + expect(text).toContain('anthropic'); // rendered by its SLUG + + // ...and NEVER by the internal llm_providers uuid the catalog row carries. This pins the lazy-after-refresh + // ordering: the slug map is built on first render (post-refresh), so a provider discovered THIS invocation is + // captured — a refactor that hoisted the map build ahead of the refresh would print this uuid instead. + const uuid = readProviderStore().get('anthropic')?.id; + expect(uuid).toBeDefined(); + if (uuid !== undefined) { + expect(text).not.toContain(uuid); + } + }); + + it('closes the db even when the command throws (zero providers connected → the CliError path)', async () => { + const { io } = captureIo(); + // No keys at all ⇒ every provider is skipped-no-key ⇒ `models refresh` throws the zero-connected CliError. + const resolver = stubResolver({}, {}); + let closed = false; + + await expect( + withModelsDeps( + context(io, false), + { refresh: true }, + testPorts(resolver, () => { + closed = true; + }), + ), + ).rejects.toMatchObject({ exitCode: EXIT_CODES.invalidInvocation }); + expect(closed).toBe(true); // the `finally` ran despite the throw — no leaked db handle + }); + + it('createProviderSlugResolver builds the slug map LAZILY (captures a provider registered AFTER construction) and falls back to the uuid for an unknown id', () => { + const store = readProviderStore(); + const providerSlug = createProviderSlugResolver(store); + // Register a provider AFTER building the closure but BEFORE its first call — the lazy `??=` build must still + // capture it (exactly the "map built while rendering, after the refresh upserted the row" ordering). + const rec = store.upsert({ + name: 'anthropic', + displayName: 'Anthropic', + baseUrl: 'https://api.anthropic.com', + }); + expect(providerSlug(rec.id)).toBe('anthropic'); + // An unmapped uuid falls back to itself (never throws) — the memoized map has no such id. + expect(providerSlug('00000000-0000-4000-8000-ffffffffffff')).toBe( + '00000000-0000-4000-8000-ffffffffffff', + ); + }); +}); diff --git a/apps/cli/src/commands/models.test.ts b/apps/cli/src/commands/models.test.ts index 306724cd..bd7f435a 100644 --- a/apps/cli/src/commands/models.test.ts +++ b/apps/cli/src/commands/models.test.ts @@ -160,6 +160,25 @@ describe('modelsCommand — list', () => { expect(text).toContain('gpt-x'); // the visible id text survives }); + it('--json first-run failure degrades to an empty NDJSON stream + exit 0 (connected provider fetch failed) (FIX D)', async () => { + // The list path under `--json`: an empty cache triggers one first-run refresh, but the CONNECTED provider's + // fetch FAILED, so the catalog stays empty. The accepted machine-output degrade (ADR-0049) is an EMPTY stream + // + exit 0 — NOT a fault, and NOT the human failure-aware line (which `--json` must never emit). + const { io, out } = captureIo(); + const rowsRef: { value: ModelCatalogListing[] } = { value: [] }; + const refresh = stubRefresh({ + providers: [{ provider: 'anthropic', status: 'failed', error: 'bad key' }], + }); + + const code = await modelsCommand( + { refresh: false }, + deps(io, stubCatalog(rowsRef), refresh, true), + ); + expect(code).toBe(EXIT_CODES.success); + expect(refresh.calls).toBe(1); // the blocking first-run refresh DID run + expect(out()).toBe(''); // empty NDJSON — no records, and no human "Model refresh failed" message + }); + it('reports a FAILED first-run refresh (not "add a key") when the catalog is still empty (FIX 4)', async () => { const { io, out } = captureIo(); const rowsRef: { value: ModelCatalogListing[] } = { value: [] }; diff --git a/apps/cli/src/engine/model-refresh.test.ts b/apps/cli/src/engine/model-refresh.test.ts index 54cf6616..f7d74ed9 100644 --- a/apps/cli/src/engine/model-refresh.test.ts +++ b/apps/cli/src/engine/model-refresh.test.ts @@ -17,9 +17,9 @@ import { createModelRefreshService, TTL_MS, type ModelRefreshDeps } from './mode /** * S5 refresh-orchestrator tests (ADR-0064 §5). All NETWORK-FREE: a stub resolver whose `listModels` returns - * canned {@link ModelListing}s or throws, over a real in-memory `history.db` (so the store diff / TTL / FK - * ordering are exercised end-to-end). Asserts per-provider isolation, the skip statuses, the added/updated/ - * deactivated diff, the TTL, and — the security invariant — that no provider key ever appears in a report. + * canned {@link ModelListing}s or throws, over a real in-memory `history.db` (so the store's atomic tally / TTL / + * FK ordering are exercised end-to-end). Asserts per-provider isolation, the skip statuses, the added/updated/ + * deactivated tally, the TTL, and — the security invariant — that no provider key ever appears in a report. */ const SECRET_KEY = 'sk-super-secret-key-value'; @@ -171,7 +171,7 @@ describe('createModelRefreshService', () => { }); }); - it('counts only the LIVE delta — a source=static / source=user row is excluded (FIX 5a)', async () => { + it('counts only the LIVE delta — a source=static / source=user row is excluded (store atomic tally)', async () => { let models: ModelListing[] = [listing('m1'), listing('m2')]; const resolver = stubResolver({ adapters: { openai: stubProvider('openai', () => Promise.resolve(models)) }, @@ -180,7 +180,9 @@ describe('createModelRefreshService', () => { const { svc, providerStore, catalogStore } = service(resolver, ['openai']); // Seed a static (media-routing) row and a user (pricing) row for the SAME provider BEFORE any refresh; a live - // refresh must NEVER count them in its added/updated/deactivated diff (they are provenance-protected — S4). + // refresh must NEVER count them in its added/updated/deactivated tally (they are provenance-protected — S4). + // The counts now come from the store's atomic transaction (the provenance `continue` skip + the + // `source='live'`-scoped deactivate), so a static/user row is excluded by construction — this pins that. const providerUuid = providerStore.upsert({ name: 'openai', displayName: 'OpenAI', @@ -203,8 +205,8 @@ describe('createModelRefreshService', () => { source: 'user', }); - // First refresh: two LIVE models. added=2, updated=0, deactivated=0. Without the `source==='live'` filter in - // `liveModelIds`, the pre-existing static/user rows would be counted as "updated" (→ 2) — so this pins it. + // First refresh: two LIVE models. added=2, updated=0, deactivated=0. Without the store's LIVE-only count logic + // (the provenance skip), the pre-existing static/user rows would be counted as "updated" (→ 2) — so this pins it. const first = await svc.refresh(); expect(first.providers[0]).toMatchObject({ status: 'refreshed', @@ -214,7 +216,7 @@ describe('createModelRefreshService', () => { }); // Second refresh: drop m1, keep m2, add m3. added=1, updated=1 (only m2), deactivated=1 (only m1) — the - // static/user rows stay out of every count (without the filter, `updated` would be 3). + // static/user rows stay out of every count (without the LIVE-only logic, `updated` would be 3). models = [listing('m2'), listing('m3')]; const second = await svc.refresh(); expect(second.providers[0]).toMatchObject({ @@ -331,8 +333,7 @@ describe('createModelRefreshService', () => { }, }, catalogStore: { - replaceProviderModels: () => {}, - listByProvider: () => [], + replaceProviderModels: () => ({ added: 0, updated: 0, deactivated: 0 }), providerRefreshedAt: () => undefined, }, knownProviderIds: ['openai'], diff --git a/apps/cli/src/engine/model-refresh.ts b/apps/cli/src/engine/model-refresh.ts index d598c39f..cfba3a27 100644 --- a/apps/cli/src/engine/model-refresh.ts +++ b/apps/cli/src/engine/model-refresh.ts @@ -1,4 +1,4 @@ -import type { ModelCatalogListing, ModelCatalogLiveModel } from '@relavium/db'; +import type { ModelCatalogLiveModel, ReplaceProviderModelsResult } from '@relavium/db'; import type { AbortSignalLike, LlmProvider, ModelListing, ProviderId } from '@relavium/llm'; import { keyHint } from './providers.js'; @@ -75,14 +75,15 @@ export interface RefreshProviderStore { readonly get: (name: string) => { readonly id: string } | undefined; } -/** The narrow catalog-store surface the refresh needs (the S4 live-refresh + read/freshness reads). */ +/** The narrow catalog-store surface the refresh needs (the S4 live-refresh write + the freshness read). The + * live-refresh returns its own atomic {@link ReplaceProviderModelsResult} tallies, so the orchestrator no longer + * needs a `listByProvider` before/after diff (which would double-count under a concurrent same-provider refresh). */ export interface RefreshCatalogStore { readonly replaceProviderModels: ( providerId: string, rows: ReadonlyArray, now: number, - ) => void; - readonly listByProvider: (providerId: string) => ModelCatalogListing[]; + ) => ReplaceProviderModelsResult; readonly providerRefreshedAt: (providerId: string) => number | undefined; } @@ -102,13 +103,33 @@ export interface ModelRefreshDeps { } export interface ModelRefreshService { - /** Force-refresh every connected provider (or `opts.providers`), per-provider-isolated. Never rejects. */ + /** Force-refresh every connected provider (or `opts.providers`), per-provider-isolated. Never rejects. The + * UNBOUNDED, user-initiated refresh (no TTL backoff) — this is what a one-shot CLI invocation must use + * (`relavium models` / `models refresh` do). */ refresh(opts?: RefreshOptions): Promise; - /** Refresh only providers whose live cache is empty/never-refreshed or older than {@link TTL_MS}; `undefined` - * when nothing was stale. */ + /** + * Refresh only providers whose live cache is empty/never-refreshed or older than {@link TTL_MS}; `undefined` + * when nothing was stale. + * + * @remarks The TTL backoff is bounded by an **IN-MEMORY, per-host-process** last-attempt map (plus the durable + * per-provider `providerRefreshedAt` stamp for successful rows). It is correct ONLY within a **long-lived** host + * process (the ADR-0064 §5c Home/picker background trigger), where the map persists across triggers so a failed/ + * empty provider is not re-egressed every trigger. In a **process-per-invocation** CLI the map starts empty every + * run, so it provides ZERO cross-process bounding — a one-shot invocation must therefore use the unbounded + * {@link refresh} instead (which `models`/`models refresh` do). A durable per-provider last-ATTEMPT stamp is the + * named follow-up if per-process background bounding is ever required. + */ refreshIfStale(opts?: RefreshOptions): Promise; - /** Fire-and-forget non-blocking TTL refresh (ADR-0064 §5c). Swallows ALL errors; sets no timer. The actual - * picker/Home trigger is S7's wiring — S5 provides the mechanism only. */ + /** + * Fire-and-forget non-blocking TTL refresh (ADR-0064 §5c). Swallows ALL errors; sets no timer of its own. + * + * @remarks HARD CONSTRAINT for S7 wiring: wire this ONLY into the **long-lived** Home process, NEVER a one-shot + * CLI invocation. A fresh-process background refresh would (a) not back off — the {@link refreshIfStale} + * last-attempt map is in-memory and starts empty each run — and (b) not exit promptly — the in-flight + * `listModels` request (and its open socket) keeps the event loop alive past the command's logical end. The + * one-shot commands already use the unbounded, awaited {@link refresh}; per-process background would need the + * durable last-attempt stamp named on {@link refreshIfStale} first. + */ refreshInBackground(opts?: RefreshOptions): void; } @@ -128,35 +149,6 @@ function toLiveModel(listing: ModelListing): ModelCatalogLiveModel { }; } -/** The provider's currently-active `source='live'` model ids (the before/after diff basis). */ -function liveModelIds(listings: readonly ModelCatalogListing[]): Set { - const ids = new Set(); - for (const listing of listings) { - if (listing.source === 'live') { - ids.add(listing.modelId); - } - } - return ids; -} - -/** |a \ b| — the count of members of `a` absent from `b`. */ -function countMissing(a: ReadonlySet, b: ReadonlySet): number { - let n = 0; - for (const id of a) { - if (!b.has(id)) n += 1; - } - return n; -} - -/** |a ∩ b|. */ -function countShared(a: ReadonlySet, b: ReadonlySet): number { - let n = 0; - for (const id of a) { - if (b.has(id)) n += 1; - } - return n; -} - /** * A SECRET-FREE failure reason. The seam's `boundedListModels` already redacts the key AND strips the cause, so * its `error.message` is safe to surface; a non-Error / empty message degrades to a generic string. As @@ -218,18 +210,17 @@ export function createModelRefreshService(deps: ModelRefreshDeps): ModelRefreshS baseUrl: meta.baseUrl, }).id; - const before = liveModelIds(deps.catalogStore.listByProvider(providerUuid)); const rows = listings.map(toLiveModel); - deps.catalogStore.replaceProviderModels(providerUuid, rows, deps.now()); - const after = liveModelIds(deps.catalogStore.listByProvider(providerUuid)); + // The store counts added/updated/deactivated ATOMICALLY inside its own transaction and returns them — no + // before/after `listByProvider` diff (which, reading a stale `before`, would double-count under two + // concurrent same-provider refreshes) and two fewer queries. The tallies are LIVE-only by construction. + const { added, updated, deactivated } = deps.catalogStore.replaceProviderModels( + providerUuid, + rows, + deps.now(), + ); - return { - provider: id, - status: 'refreshed', - added: countMissing(after, before), - updated: countShared(before, after), - deactivated: countMissing(before, after), - }; + return { provider: id, status: 'refreshed', added, updated, deactivated }; } catch (err) { return { provider: id, status: 'failed', error: secretFreeReason(err, key) }; } diff --git a/packages/db/src/index.ts b/packages/db/src/index.ts index 851d1833..5a36e900 100644 --- a/packages/db/src/index.ts +++ b/packages/db/src/index.ts @@ -159,7 +159,9 @@ export { type ModelCatalogStoreDeps, type ModelCatalogRecord, type ModelCatalogUpsert, - // Live-discovery cache (2.5.G, ADR-0064) — the picker/refresh listing projection + the bulk live-refresh input. + // Live-discovery cache (2.5.G, ADR-0064) — the picker/refresh listing projection + the bulk live-refresh input + + // the atomic add/updated/deactivated tallies the live refresh returns. type ModelCatalogListing, type ModelCatalogLiveModel, + type ReplaceProviderModelsResult, } from './model-catalog-store.js'; diff --git a/packages/db/src/model-catalog-store.test.ts b/packages/db/src/model-catalog-store.test.ts index 0c813d05..f235fe41 100644 --- a/packages/db/src/model-catalog-store.test.ts +++ b/packages/db/src/model-catalog-store.test.ts @@ -509,6 +509,92 @@ describe('createModelCatalogStore (2.5.G / ADR-0064 — live-discovery cache)', expect(bRows[0]?.id).toBe(bIdBefore); }); + it('RETURNS the atomic added/updated/deactivated tallies (new added, kept updated, vanished deactivated)', () => { + // First refresh: three brand-new live models ⇒ all counted as added. + const first = store.replaceProviderModels( + providerId, + [ + { modelId: 'm1', displayName: 'M1' }, + { modelId: 'm2', displayName: 'M2' }, + { modelId: 'm3', displayName: 'M3' }, + ], + TS_MS + 1000, + ); + expect(first).toEqual({ added: 3, updated: 0, deactivated: 0 }); + + // Second refresh: m1/m2 kept (updated in place), m3 vanishes (deactivated), m4 appears (added). + const second = store.replaceProviderModels( + providerId, + [ + { modelId: 'm1', displayName: 'M1 v2' }, + { modelId: 'm2', displayName: 'M2 v2' }, + { modelId: 'm4', displayName: 'M4' }, + ], + TS_MS + 2000, + ); + expect(second).toEqual({ added: 1, updated: 2, deactivated: 1 }); + + // Third refresh: m3 REAPPEARS ⇒ reactivated in place (counted `updated`, NOT `added`, since the same row id + // is reused); m1/m2/m4 all vanish ⇒ deactivated:3; nothing brand-new ⇒ added:0. + const third = store.replaceProviderModels( + providerId, + [{ modelId: 'm3', displayName: 'M3 back' }], + TS_MS + 3000, + ); + expect(third).toEqual({ added: 0, updated: 1, deactivated: 3 }); + }); + + it('the returned tallies count LIVE rows only — a coexisting static/user row is never added/updated/deactivated', () => { + // Seed a static (media-routing) row and a user (pricing) row BEFORE any refresh. + store.upsert({ + providerId, + modelId: 'static-model', + displayName: 'Static', + contextWindowTokens: 1000, + maxOutputTokens: 100, + source: 'static', + }); + store.upsert({ + providerId, + modelId: 'user-model', + displayName: 'User', + contextWindowTokens: 1000, + maxOutputTokens: 100, + source: 'user', + }); + + // First live refresh names two NEW live ids ⇒ added:2, updated:0, deactivated:0 — the static/user rows + // contribute to NONE of the counts (provenance-protected: never part of the live delta). + const first = store.replaceProviderModels( + providerId, + [ + { modelId: 'live-1', displayName: 'L1' }, + { modelId: 'live-2', displayName: 'L2' }, + ], + TS_MS + 1000, + ); + expect(first).toEqual({ added: 2, updated: 0, deactivated: 0 }); + + // A refresh whose live list ALSO names the existing static + user ids (a collision) must still not count them: + // they are provenance-`continue`-skipped (never `updated`), and dropping live-1 deactivates exactly one LIVE + // row — a store count with no `source='live'` guard would report updated:3 / deactivated including them. + const second = store.replaceProviderModels( + providerId, + [ + { modelId: 'live-2', displayName: 'L2 v2' }, + { modelId: 'static-model', displayName: 'collide' }, + { modelId: 'user-model', displayName: 'collide' }, + ], + TS_MS + 2000, + ); + expect(second).toEqual({ added: 0, updated: 1, deactivated: 1 }); + + // Both provenance rows survived every refresh (never deactivated by the live delta). + const ids = store.listByProvider(providerId).map((m) => m.modelId); + expect(ids).toContain('static-model'); + expect(ids).toContain('user-model'); + }); + it('never clobbers an existing user/static row even when the live list names the same model id (the collision invariant)', () => { store.upsert({ providerId, diff --git a/packages/db/src/model-catalog-store.ts b/packages/db/src/model-catalog-store.ts index e8673ae9..2dcf7ab4 100644 --- a/packages/db/src/model-catalog-store.ts +++ b/packages/db/src/model-catalog-store.ts @@ -111,6 +111,21 @@ export interface ModelCatalogLiveModel { readonly maxOutputTokens?: number; } +/** + * The row tallies a {@link ModelCatalogStore.replaceProviderModels} refresh applied, counted ATOMICALLY inside its + * own transaction and returned to the caller ([ADR-0064] §5). Returning them here — rather than the host diffing a + * before/after `listByProvider` snapshot — is what makes the counts correct under two CONCURRENT same-provider + * refreshes: an external diff reads a stale `before` and can double-count the same rows, whereas these tallies are + * observed from within the serialized write. Counts LIVE rows only (a non-`live` `static`/`user` row is skipped and + * never contributes — the same provenance protection the write itself enforces). `added` = INSERTed live rows, + * `updated` = existing live rows refreshed in place, `deactivated` = vanished live rows soft-deactivated. + */ +export interface ReplaceProviderModelsResult { + readonly added: number; + readonly updated: number; + readonly deactivated: number; +} + export interface ModelCatalogStoreDeps { readonly uuid: () => string; readonly now: () => number; @@ -142,12 +157,16 @@ export interface ModelCatalogStore { * `deletedAt` untouched). `source='user'`/`source='static'` rows are NEVER touched (the ADR-0065 §1 "a refresh * never clobbers a user row" invariant + the media-routing seed's integrity), and nothing is ever hard-DELETED * (`model_catalog.id` is an FK target from five tables). + * + * Returns the {@link ReplaceProviderModelsResult} tallies (added/updated/deactivated), counted ATOMICALLY inside + * the transaction — so a concurrent same-provider refresh can never miscount (an external before/after diff would + * read a stale `before`). Only LIVE rows are counted; a `static`/`user` row is provenance-skipped and excluded. */ replaceProviderModels: ( providerId: string, rows: ReadonlyArray, now: number, - ) => void; + ) => ReplaceProviderModelsResult; /** The freshness read for the TTL ([ADR-0064] §5): the max `lastRefreshedAt` among a provider's active * `source='live'` rows, or `undefined` when the provider has none. */ providerRefreshedAt: (providerId: string) => number | undefined; @@ -391,8 +410,17 @@ export function createModelCatalogStore(db: Db, deps: ModelCatalogStoreDeps): Mo .all() .map(toListing), - replaceProviderModels: (providerId, rows, now) => { + replaceProviderModels: (providerId, rows, now) => + // The transaction RETURNS the tallies so they are observed from WITHIN the serialized write — a concurrent + // same-provider refresh can never miscount them (an external before/after `listByProvider` diff would read a + // stale `before` and could double-count). drizzle's better-sqlite3 `transaction()` returns the callback value. db.transaction(() => { + // Only LIVE rows are tallied: `added` on a true INSERT, `updated` on an existing-live-row UPDATE (a non-live + // `static`/`user` row hits the provenance `continue` below and is counted in NEITHER), `deactivated` from the + // soft-deactivate UPDATE's `.changes` (its WHERE is already `source='live'`-scoped) — so the counts carry the + // same LIVE-only intent the write enforces, with no separate source filter needed. + let added = 0; + let updated = 0; for (const input of rows) { const displayName = input.displayName.trim() === '' ? input.modelId : input.displayName; // `0` is the NOT-NULL "unknown" sentinel (ADR-0064 §3) — an absent live limit stores as 0. @@ -416,6 +444,7 @@ export function createModelCatalogStore(db: Db, deps: ModelCatalogStoreDeps): Mo // media_surface/capabilities/rates) row already represents this model. A live refresh must NEVER // clobber it (that would drop user pricing or regress media routing), so it is left UNTOUCHED and, // being non-`live`, is also never deactivated below — the model stays represented by its own row. + // It is counted in neither `added` nor `updated` (provenance-protected — never part of the live delta). continue; } if (existing === undefined) { @@ -433,6 +462,7 @@ export function createModelCatalogStore(db: Db, deps: ModelCatalogStoreDeps): Mo updatedAt: now, }; db.insert(modelCatalog).values(row).run(); + added += 1; } else { // Reactivate + refresh the existing live row in place (id/created_at/FK refs preserved); only the // discovery columns + provenance/freshness are written — pricing/media columns are left as-is. @@ -448,6 +478,7 @@ export function createModelCatalogStore(db: Db, deps: ModelCatalogStoreDeps): Mo }) .where(eq(modelCatalog.id, existing.id)) .run(); + updated += 1; } } // Soft-deactivate the vanished live rows: every currently-active `source='live'` row of THIS provider @@ -461,7 +492,8 @@ export function createModelCatalogStore(db: Db, deps: ModelCatalogStoreDeps): Mo eq(modelCatalog.source, 'live'), isNull(modelCatalog.deletedAt), ); - db.update(modelCatalog) + const deactivateResult = db + .update(modelCatalog) .set({ isActive: false, updatedAt: now }) // An empty new list deactivates ALL of the provider's live rows (no `notInArray([])` — its semantics // vary; the guard makes the "everything vanished" case explicit). @@ -471,8 +503,10 @@ export function createModelCatalogStore(db: Db, deps: ModelCatalogStoreDeps): Mo : and(deactivateScope, notInArray(modelCatalog.modelId, incomingModelIds)), ) .run(); - }); - }, + // better-sqlite3's `RunResult.changes` = the rows the UPDATE matched (each flips isActive true→false, so + // every matched row is genuinely modified) = the number of live rows soft-deactivated this refresh. + return { added, updated, deactivated: deactivateResult.changes }; + }), providerRefreshedAt: (providerId) => { const row = db From 08dde66c1245adf693f8f709375ede2b886dc4c1 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 22:34:48 +0300 Subject: [PATCH 15/68] =?UTF-8?q?feat(cli):=202.5.G=20S6=20=E2=80=94=20con?= =?UTF-8?q?fig-write=20primitive=20+=20global=20[preferences]=20fallback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the first on-disk config WRITER (the sibling of the read-only load.ts): `writeGlobalDefaultModel` persists a chosen chat default to the global `~/.relavium/config.toml` `[preferences].default_model` — the write target for the coming `/models` picker and the onboarding wizard. Three ADR-0063 guarantees: - Secret-free by construction — a TYPED setter (never a generic writeKey(k,v)), so it can only ever set `default_model` (a non-secret); there is no api_key field in the schema (keys stay in the OS keychain, ADR-0006). - Atomic + owner-only — a 0600 temp file in the 0700 `~/.relavium/`, fsync, then rename; an interrupted write leaves the original intact. - Schema round-trip — merges onto the validated existing config and re-validates the whole object against the strict GlobalConfigSchema before emitting, so the file always re-parses. Re-serialization drops comments/ordering (the documented ADR-0063 tradeoff for the global file); no new dependency (smol-toml is the ADR-0048 parser). Also thread the global layer into resolveChat: `[chat].default_model` now falls back to the global `[preferences].default_model` (project → workspace → global preferences), mirroring how the workflow default already reads `[preferences]`, so a user's "preferred model everywhere" governs chat too. Docs: config-spec.md gains the write-contract note + the chat default_model global-fallback precedence. Tests: write.test.ts (round-trip, key preservation, 0600/0700, no orphan temp, refuse-to-clobber-malformed, value-free errors) + resolveChat global-fallback precedence. Toolchain: lint + typecheck + test (1346) + build all green. Refs: ADR-0063 Co-Authored-By: Claude --- apps/cli/src/config/resolve.test.ts | 24 ++++- apps/cli/src/config/resolve.ts | 13 ++- apps/cli/src/config/write.test.ts | 131 ++++++++++++++++++++++++ apps/cli/src/config/write.ts | 91 ++++++++++++++++ docs/reference/contracts/config-spec.md | 14 ++- 5 files changed, 263 insertions(+), 10 deletions(-) create mode 100644 apps/cli/src/config/write.test.ts create mode 100644 apps/cli/src/config/write.ts diff --git a/apps/cli/src/config/resolve.test.ts b/apps/cli/src/config/resolve.test.ts index 200a052e..e7d1f1cb 100644 --- a/apps/cli/src/config/resolve.test.ts +++ b/apps/cli/src/config/resolve.test.ts @@ -117,15 +117,31 @@ describe('resolveConfig', () => { expect(resolved.maxCostMicrocents).toBe(0); }); - it('resolves an absent [chat] block (every layer) to all-undefined — [chat] is project/workspace-scoped, not global', () => { + it('resolves an absent [chat] block: non-model fields stay undefined; default_model falls back to global [preferences] (ADR-0063)', () => { const empty = resolveConfig({}).chat; expect(empty.maxTurns).toBeUndefined(); expect(empty.defaultModel).toBeUndefined(); expect(empty.maxCostMicrocents).toBeUndefined(); - // A global layer carries no [chat] block, so it cannot supply chat defaults. + // A global layer carries no [chat] block, so it cannot supply the NON-model chat fields... + const g = resolveConfig({ global: { preferences: { default_model: 'g' } } }).chat; + expect(g.maxTurns).toBeUndefined(); + expect(g.maxCostMicrocents).toBeUndefined(); + // ...BUT chat.default_model DOES fall back to the global [preferences].default_model (the /models + wizard + // write target, ADR-0063 §1) so a user's "preferred model everywhere" applies to chat when no [chat] overrides. + expect(g.defaultModel).toBe('g'); + }); + + it('resolves [chat].default_model precedence project > workspace > global [preferences] (ADR-0063 §1)', () => { + const global: GlobalConfig = { preferences: { default_model: 'g' } }; + const workspace: ProjectConfig = { chat: { default_model: 'w' } }; + const project: ProjectConfig = { chat: { default_model: 'p' } }; + expect(resolveConfig({ global, workspace, project }).chat.defaultModel).toBe('p'); + expect(resolveConfig({ global, workspace }).chat.defaultModel).toBe('w'); + expect(resolveConfig({ global }).chat.defaultModel).toBe('g'); + // A present [chat].default_model is NEVER shadowed by the global — the global is only the lowest fallback. expect( - resolveConfig({ global: { preferences: { default_model: 'g' } } }).chat.defaultModel, - ).toBeUndefined(); + resolveConfig({ global, project: { chat: { default_model: 'p' } } }).chat.defaultModel, + ).toBe('p'); }); it('resolves media_gc_grace_days (2.S/D11) DAYS → ms, last-writer-wins, absent ⇒ undefined', () => { diff --git a/apps/cli/src/config/resolve.ts b/apps/cli/src/config/resolve.ts index 81888e37..c89252f3 100644 --- a/apps/cli/src/config/resolve.ts +++ b/apps/cli/src/config/resolve.ts @@ -87,7 +87,7 @@ export function resolveConfig(layers: ConfigLayers): ResolvedConfig { mediaCostEstimate: project?.defaults?.media_cost_estimate ?? workspace?.defaults?.media_cost_estimate, mediaGcGraceMs: resolveGraceMs(project, workspace), - chat: resolveChat(project, workspace), + chat: resolveChat(project, workspace, global), variables: { ...workspace?.variables, ...project?.variables }, mcpServers: mergeMcpServers(global?.mcp_servers, workspace?.mcp_servers, project?.mcp_servers), }; @@ -106,13 +106,18 @@ function resolveGraceMs( } /** - * Resolve the `[chat]` block (last-writer-wins: project → workspace; `[chat]` is project/workspace-scoped, - * not global — config-spec.md). Absent at every layer ⇒ all-`undefined` fields, so the chat host falls + * Resolve the `[chat]` block (last-writer-wins: project → workspace). Most fields are project/workspace-scoped + * only; `default_model` additionally falls back to the GLOBAL `[preferences].default_model` + * ([ADR-0063](../../../../docs/decisions/0063-cli-config-write-contract.md)) — the write target of `/models` and + * the onboarding wizard — so a user's "preferred model everywhere" applies to chat when no project/workspace + * `[chat].default_model` overrides it, mirroring how the workflow default (`resolveConfig.defaultModel`) already + * reads `[preferences].default_model`. Absent at every layer ⇒ all-`undefined` fields, so the chat host falls * back to its engine defaults (e.g. `maxTurns` ⇒ `SessionDeps`'s built-in 50). */ function resolveChat( project: ProjectConfig | undefined, workspace: ProjectConfig | undefined, + global: GlobalConfig | undefined, ): ResolvedChatConfig { const p = project?.chat; const w = workspace?.chat; @@ -125,7 +130,7 @@ function resolveChat( const projectSetsAllowlist = p?.allowed_commands !== undefined || p?.allowed_command_globs !== undefined; return { - defaultModel: p?.default_model ?? w?.default_model, + defaultModel: p?.default_model ?? w?.default_model ?? global?.preferences?.default_model, fsScope: p?.fs_scope ?? w?.fs_scope, maxTurns: p?.max_turns ?? w?.max_turns, maxMessages: p?.max_messages ?? w?.max_messages, diff --git a/apps/cli/src/config/write.test.ts b/apps/cli/src/config/write.test.ts new file mode 100644 index 00000000..b79e9ee5 --- /dev/null +++ b/apps/cli/src/config/write.test.ts @@ -0,0 +1,131 @@ +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { GlobalConfigSchema, type GlobalConfig } from '@relavium/shared'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { ConfigError } from './errors.js'; +import { loadConfigFile } from './load.js'; +import { globalConfigPath, writeGlobalDefaultModel } from './write.js'; + +/** Read the global config back through the SAME validating loader the rest of the CLI uses. */ +function readBack(home: string): GlobalConfig | undefined { + return loadConfigFile(globalConfigPath(home), GlobalConfigSchema); +} + +describe('writeGlobalDefaultModel', () => { + let home: string; + beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'relavium-write-')); + }); + afterEach(() => { + rmSync(home, { recursive: true, force: true }); + }); + + it('creates ~/.relavium/config.toml with [preferences].default_model on a fresh home (round-trip)', () => { + writeGlobalDefaultModel('claude-sonnet-4-6', home); + expect(readBack(home)).toEqual({ preferences: { default_model: 'claude-sonnet-4-6' } }); + }); + + it('preserves every other config key (update_channel, theme, mcp_servers) — merges only default_model', () => { + // Seed an existing, valid global config carrying unrelated keys the write must NOT drop. + mkdirSync(globalDir(home), { recursive: true }); + writeFileSync( + globalConfigPath(home), + [ + 'update_channel = "beta"', + '[preferences]', + 'theme = "dark"', + '[[mcp_servers]]', + 'name = "fs"', + 'transport = "stdio"', + 'command = "some-cmd"', + '', + ].join('\n'), + ); + + writeGlobalDefaultModel('gpt-4o', home); + + expect(readBack(home)).toEqual({ + update_channel: 'beta', + preferences: { theme: 'dark', default_model: 'gpt-4o' }, + mcp_servers: [{ name: 'fs', transport: 'stdio', command: 'some-cmd' }], + }); + }); + + it('overwrites an existing default_model in place (last write wins)', () => { + writeGlobalDefaultModel('first-model', home); + writeGlobalDefaultModel('second-model', home); + expect(readBack(home)?.preferences?.default_model).toBe('second-model'); + }); + + it('emits only schema keys — a secret could never be written (no api_key field exists to set)', () => { + writeGlobalDefaultModel('a-model', home); + const text = readFileSync(globalConfigPath(home), 'utf8'); + // Structural guarantee (the typed setter): the emitted TOML mentions only the model, no key-ish token. + expect(text).toContain('default_model'); + expect(text).toContain('a-model'); + expect(text.toLowerCase()).not.toContain('api_key'); + expect(text.toLowerCase()).not.toContain('secret'); + // And it re-parses cleanly (strict schema) — no stray key slipped in. + expect(() => readBack(home)).not.toThrow(); + }); + + it('writes the file 0600 inside the 0700 directory (owner-only at rest)', () => { + if (process.platform === 'win32') return; // POSIX mode bits do not apply on Windows + writeGlobalDefaultModel('a-model', home); + expect(statSync(globalConfigPath(home)).mode & 0o777).toBe(0o600); + expect(statSync(globalDir(home)).mode & 0o777).toBe(0o700); + }); + + it('leaves no temp file behind after a successful write (the temp is renamed, not orphaned)', () => { + writeGlobalDefaultModel('a-model', home); + const leftovers = readdirSync(globalDir(home)).filter((name) => name.endsWith('.tmp')); + expect(leftovers).toEqual([]); + }); + + it('throws a ConfigError rather than clobbering a malformed existing config', () => { + mkdirSync(globalDir(home), { recursive: true }); + const target = globalConfigPath(home); + writeFileSync(target, 'this is = = not valid toml'); + let thrown: unknown; + try { + writeGlobalDefaultModel('a-model', home); + } catch (err) { + thrown = err; + } + expect(thrown).toBeInstanceOf(ConfigError); + // The user's broken file is untouched — we refuse to overwrite it. + expect(readFileSync(target, 'utf8')).toBe('this is = = not valid toml'); + }); + + it('never echoes the model value or a config value in a schema-error message (hygiene)', () => { + // An existing config with a stray key makes the LOAD (via loadConfigFile) throw; assert the message is value-free. + mkdirSync(globalDir(home), { recursive: true }); + writeFileSync(globalConfigPath(home), 'stray_key' + ' = "leak-me-please"\n'); + let thrown: unknown; + try { + writeGlobalDefaultModel('the-new-model', home); + } catch (err) { + thrown = err; + } + expect(thrown).toBeInstanceOf(ConfigError); + if (thrown instanceof ConfigError) { + expect(thrown.message).not.toContain('leak-me-please'); + expect(thrown.message).not.toContain('the-new-model'); + } + }); +}); + +describe('globalConfigPath', () => { + it('is ~/.relavium/config.toml under the given home', () => { + const home = '/some/home'; + expect(globalConfigPath(home)).toBe(join(home, '.relavium', 'config.toml')); + }); +}); + +/** `~/.relavium` — local mirror of the path helper so tests don't reach into paths.ts internals. */ +function globalDir(home: string): string { + return join(home, '.relavium'); +} diff --git a/apps/cli/src/config/write.ts b/apps/cli/src/config/write.ts new file mode 100644 index 00000000..cbd19288 --- /dev/null +++ b/apps/cli/src/config/write.ts @@ -0,0 +1,91 @@ +import { randomUUID } from 'node:crypto'; +import { closeSync, fchmodSync, fsyncSync, openSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join } from 'node:path'; + +import { GlobalConfigSchema, type GlobalConfig } from '@relavium/shared'; +import { stringify as stringifyToml } from 'smol-toml'; + +import { ConfigError } from './errors.js'; +import { loadConfigFile } from './load.js'; +import { ensureGlobalConfigDir, globalConfigDir } from './paths.js'; + +/** + * The **first on-disk config WRITER** ([ADR-0063](../../../../docs/decisions/0063-cli-config-write-contract.md)) — + * the sibling of the read-only [load.ts](./load.ts). It persists a user's chosen chat default model to the + * **global** `~/.relavium/config.toml` `[preferences].default_model`, the write target for `/models` and the + * onboarding wizard. + * + * Three load-bearing guarantees (the ADR-0063 contract; a security-reviewed surface): + * 1. **Secret-free by construction.** The surface is a **typed setter** ({@link writeGlobalDefaultModel}) — never + * a generic `writeKey(k, v)` — so it can only ever set `default_model` (a non-secret). There is no API-key + * field in the schema to write to (keys live only in the OS keychain, ADR-0006). + * 2. **Atomic + owner-only.** It writes a temp file created `0600` in the owner-only (`0700`) `~/.relavium/` + * directory, `fsync`s it, then `rename`s over the target — an interrupted write leaves the original intact. + * 3. **Schema round-trip.** It merges onto the *validated* existing config and re-validates the whole object + * against the strict {@link GlobalConfigSchema} (ADR-0033) BEFORE emitting, so the written file is guaranteed + * to re-parse cleanly on the next load. `smol-toml.stringify` re-serializes (dropping comments/ordering — the + * documented ADR-0063 tradeoff for the global file); no new dependency (`smol-toml` is the ADR-0048 parser). + */ + +/** + * Set the global `[preferences].default_model`, preserving every other config key. Reads + validates the existing + * `~/.relavium/config.toml` (an absent file ⇒ a fresh `{}`; a **malformed/invalid** existing config throws a + * {@link ConfigError} rather than clobbering a file the user must fix), merges only `default_model`, re-validates, + * and writes atomically. `home` is injectable for tests. + */ +export function writeGlobalDefaultModel(model: string, home: string = homedir()): void { + const dir = ensureGlobalConfigDir(home); // `~/.relavium/` (created `0700`) + const target = join(dir, 'config.toml'); + + // Read the EXISTING config through the same validating loader (so we merge onto known-good data and preserve + // update_channel / mcp_servers / preferences.theme). An absent file is a fresh object; an invalid one throws. + const existing = loadConfigFile(target, GlobalConfigSchema) ?? {}; + const next: GlobalConfig = { + ...existing, + preferences: { ...existing.preferences, default_model: model }, + }; + // Re-validate the whole object so the emission is provably schema-valid (ADR-0033) — a `.strict()` round-trip + // that also structurally guarantees no non-schema (e.g. secret) key can reach disk. + const validated = GlobalConfigSchema.parse(next); + writeFileAtomic(dir, target, stringifyToml(validated)); +} + +/** + * Atomic file replace: write `text` to a unique temp in the SAME (`0700`) directory (so `rename` is atomic and + * on the same filesystem), explicitly `0600` (never relying on an inherited directory mode — `ensureGlobalConfigDir` + * `0700`s `~/.relavium/` itself, not necessarily its children), `fsync` for durability, then `rename` over the + * target. On any failure the temp is best-effort removed and a typed {@link ConfigError} is thrown. + */ +function writeFileAtomic(dir: string, target: string, text: string): void { + const tmp = join(dir, `config.toml.${randomUUID()}.tmp`); + let fd: number | undefined; + try { + fd = openSync(tmp, 'wx', 0o600); // `wx`: fail if it exists (the uuid makes a collision impossible) + writeFileSync(fd, text, 'utf8'); + fchmodSync(fd, 0o600); // owner-only, independent of umask + fsyncSync(fd); // the temp is durably on disk before the rename + closeSync(fd); + fd = undefined; + renameSync(tmp, target); + } catch (err) { + if (fd !== undefined) { + try { + closeSync(fd); + } catch { + /* already failing — ignore a secondary close error */ + } + } + try { + unlinkSync(tmp); + } catch { + /* best-effort temp cleanup (it may never have been created) */ + } + throw new ConfigError(target, 'could not be written', { cause: err }); + } +} + +/** The global config path — exported for tests / callers that need to assert the write target. */ +export function globalConfigPath(home: string = homedir()): string { + return join(globalConfigDir(home), 'config.toml'); +} diff --git a/docs/reference/contracts/config-spec.md b/docs/reference/contracts/config-spec.md index fdbdbfab..24391454 100644 --- a/docs/reference/contracts/config-spec.md +++ b/docs/reference/contracts/config-spec.md @@ -88,6 +88,16 @@ stdio-only fields (`command`/`args`/`env`) are rejected on a network registratio (`url`/`allow_local_endpoint`) on a stdio one. An agent consumes a registration with `- ref: filesystem` (see [../shared-core/mcp-integration.md](../shared-core/mcp-integration.md)). +> **Writing the global config** ([ADR-0063](../../decisions/0063-cli-config-write-contract.md)). Config is +> almost entirely **read-only** (hand-edited, git-committed). The one write path is the CLI persisting a chosen +> default model: `/models` and the 2.5.G onboarding wizard set **`[preferences].default_model`** — and *only* +> that key — through a **typed setter** (never a generic key/value writer), so a secret can never be written by +> construction (there is no `api_key` field in the schema; keys live only in the OS keychain, [ADR-0006](../../decisions/0006-os-keychain-for-api-keys.md)). +> The write is **atomic** (a `0600` temp file in the `0700` `~/.relavium/`, `fsync`, then `rename`) and re-validates +> the whole object against the strict `GlobalConfigSchema` before emitting, so the file always re-parses. The +> documented tradeoff: re-serialization **drops comments and key ordering** in `config.toml` — the global file is +> a preference store, not a hand-curated artifact (project/workspace files are never written by the tool). + ## `project.toml` / `workspace.toml` (project) — keys ```toml @@ -109,7 +119,7 @@ video = 10 # assumed video-SECONDS per media-output turn focus_area = "security and type safety" [chat] # agent-session (chat-mode) defaults — see contracts/agent-session-spec.md -default_model = "claude-sonnet-4-6" # model for a chat session that names none +default_model = "claude-sonnet-4-6" # model for a chat session that names none; absent at every [chat] layer ⇒ falls back to global [preferences].default_model (ADR-0063) fs_scope = "sandboxed" # SAME tier enum as [defaults].fs_scope above (not re-listed here) max_turns = 50 # hard session TURN cap → SessionDeps.maxTurns (DoS fail-safe; absent ⇒ engine default 50; positiveInt — 0 is rejected here) — DISTINCT from max_messages max_messages = 200 # history-trim threshold — consumed by `/trim` + auto-compaction (ADR-0062); older turns trimmed/summarized @@ -139,7 +149,7 @@ allowed_command_globs = [] # opt-in glob form of the !-shell allowlist ( > > `max_messages` (revived in 2.5.F) is the bound `/trim` enforces (keep the last N messages, no LLM call) and the deterministic fallback if a summarization fails. `auto_compact` + `compact_threshold` ([ADR-0062](../../decisions/0062-context-compaction-and-cli-history-commands.md)) drive **automatic** model-summarised compaction: after a turn completes, if its **real** input tokens exceed `compact_threshold` (default `0.8`, a fraction in (0, 1]) × the serving model's context window, the session compacts before the next turn. `auto_compact` absent ⇒ enabled; a model with no known context window (a custom base-URL id) skips auto-compaction, but manual `/compact` still works. The summarization spend is accounted to the session budget and surfaced, never silent. > -> The `[chat]` block resolves **per field** (each key independently, last-writer-wins project → workspace) — a project that sets only `max_turns` still inherits `default_model`/`max_messages` from the workspace layer. (Contrast `[defaults].media_cost_estimate`, which resolves **whole-object**: the highest layer present replaces the table outright.) The `!`-shell allowlist is the **one exception**: `allowed_commands` (exact) + `allowed_command_globs` (globs) are a **coupled unit**, so a project that sets **either** array owns the **whole** allowlist and does **not** inherit the other array from the workspace. Otherwise a project narrowing `allowed_commands` would silently keep the workspace's broader globs — lock to `git status`, yet still allow `git push` via an inherited `git *`. Only when a project sets **neither** allowlist array do both fall through to the workspace; a present array otherwise REPLACES (never merges) the lower layer's. This is what guarantees a narrower project can never inherit a broader workspace entry. +> The `[chat]` block resolves **per field** (each key independently, last-writer-wins project → workspace) — a project that sets only `max_turns` still inherits `default_model`/`max_messages` from the workspace layer. (Contrast `[defaults].media_cost_estimate`, which resolves **whole-object**: the highest layer present replaces the table outright.) **`default_model` has one extra fallback**: absent at both `[chat]` layers, it falls through to the global **`[preferences].default_model`** ([ADR-0063](../../decisions/0063-cli-config-write-contract.md) §1) — the write target of `/models` and the wizard — so a user's "preferred model everywhere" governs chat too, exactly as it already governs a workflow's `[defaults].model`. Full precedence: `[chat].default_model` (project → workspace) → `[preferences].default_model` (global). No other `[chat]` field reads the global layer. The `!`-shell allowlist is the **one exception**: `allowed_commands` (exact) + `allowed_command_globs` (globs) are a **coupled unit**, so a project that sets **either** array owns the **whole** allowlist and does **not** inherit the other array from the workspace. Otherwise a project narrowing `allowed_commands` would silently keep the workspace's broader globs — lock to `git status`, yet still allow `git push` via an inherited `git *`. Only when a project sets **neither** allowlist array do both fall through to the workspace; a present array otherwise REPLACES (never merges) the lower layer's. This is what guarantees a narrower project can never inherit a broader workspace entry. > > `allowed_commands` / `allowed_command_globs` gate the **`!`-shell escape** (2.5.D, [ADR-0061](../../decisions/0061-cli-input-layer-file-injection-and-shell-escape.md)) — a chat user typing `!command` runs it through the **one** `run_command` boundary (they map to the engine's camelCase `allowedCommands` / `allowedCommandGlobs`, the SAME allowlist a workflow `run_command` uses). `allowed_commands` is **exact full-command-string** match (`git status`, `ls -la` — `git` never authorizes `git push --force`); `allowed_command_globs` is the opt-in, riskier pattern form. **Both default to EMPTY ⇒ `!`-shell is disabled** — the `empty ⇒ disabled` symmetry [security-review.md](../../standards/security-review.md) pins, with **no chat-specific relaxation** (there is no curated default: `run_command` has no argument/file confidentiality floor, so even a "read-only" default set — `cat`, `grep` — would reopen `!cat .env` → provider). `!`-shell is first-class via a first-class **opt-in** (the user lists commands, or the 2.5.G onboarding offers a reviewed seed), and a non-allowlisted `!cmd` gets an **actionable, secret-free deny hint** naming the exact line to add. `enforcePolicy(allowedCommands)` runs **before** the mode-aware `confirmAction`, so even `auto` mode never runs a command absent from the allowlist. Editing chat `allowed_commands` is a [security-review.md](../../standards/security-review.md) trigger. From 2a519ae31d368730b492b7a91cb2746607a1a301 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 22:47:55 +0300 Subject: [PATCH 16/68] =?UTF-8?q?fix(cli):=202.5.G=20S6=20Opus-review=20?= =?UTF-8?q?=E2=80=94=20harden=20the=20config=20writer=20for=20future=20cal?= =?UTF-8?q?lers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial Opus review found no blockers, but three legitimate hardening items on the FIRST config writer (every later writer inherits this primitive) plus two test gaps. Fixes: - Value-free write-path validation: re-validate the merged object with safeParse and report failures through the SAME value-free formatter the loader uses (formatZodError, now exported from load.ts) — never a raw ZodError, whose .message embeds a received value and could reach stderr. So the secret-free-error guarantee survives a future schema refinement (e.g. a .min(1) on default_model), rather than holding by luck. - Verified schema round-trip (ADR-0063 §3): after stringifying, re-parse the emitted TOML back through GlobalConfigSchema BEFORE the atomic rename, so "the file re-parses cleanly on next load" is a verified guarantee, not a trust-the-serializer assumption. On failure config.toml is untouched. - Durable rename: best-effort parent-directory fsync after rename so the directory-entry swap survives a crash (not just the file's data) — makes the "fsync then rename" durability claim honest. Swallows all errors (Windows can't fsync a dir fd) and never throws. Tests: fault-injection catch-path coverage (a target that is a directory forces renameSync EISDIR → asserts the temp is unlinked, closing the untested cleanup branch) + a tricky mcp_servers round-trip (args array + env sub-table + a second network server survive serialize→reparse). Not fixed (rationale): the SIGKILL-orphaned-temp nit — moving the temp into the tmp/ subdir would trade the stronger 0700 parent dir for weaker perms; the temp holds only non-secret config, so its location is moot. Toolchain: lint + typecheck + test (1348) + build all green. Refs: ADR-0063 Co-Authored-By: Claude --- apps/cli/src/config/load.ts | 8 +- apps/cli/src/config/write.test.ts | 56 ++++++++++- apps/cli/src/config/write.ts | 118 +++++++++++++++++++----- docs/reference/contracts/config-spec.md | 11 ++- 4 files changed, 165 insertions(+), 28 deletions(-) diff --git a/apps/cli/src/config/load.ts b/apps/cli/src/config/load.ts index 6b0a581a..1f5e6def 100644 --- a/apps/cli/src/config/load.ts +++ b/apps/cli/src/config/load.ts @@ -127,8 +127,12 @@ function tomlPosition(err: unknown): string { return ''; } -/** First Zod issue as a field-attributed, **value-free** message (path + a code-derived reason). */ -function formatZodError(error: ZodError): string { +/** + * First Zod issue as a field-attributed, **value-free** message (path + a code-derived reason). + * Exported so the config WRITER ([write.ts](./write.ts)) reuses the SAME value-free formatting — a + * schema-validation failure on the write path must never echo a received value either (ADR-0063). + */ +export function formatZodError(error: ZodError): string { const issue = error.issues[0]; if (issue === undefined) { return 'failed schema validation'; diff --git a/apps/cli/src/config/write.test.ts b/apps/cli/src/config/write.test.ts index b79e9ee5..12465fe0 100644 --- a/apps/cli/src/config/write.test.ts +++ b/apps/cli/src/config/write.test.ts @@ -7,7 +7,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { ConfigError } from './errors.js'; import { loadConfigFile } from './load.js'; -import { globalConfigPath, writeGlobalDefaultModel } from './write.js'; +import { globalConfigPath, writeFileAtomic, writeGlobalDefaultModel } from './write.js'; /** Read the global config back through the SAME validating loader the rest of the CLI uses. */ function readBack(home: string): GlobalConfig | undefined { @@ -85,6 +85,60 @@ describe('writeGlobalDefaultModel', () => { expect(leftovers).toEqual([]); }); + it('cleans up the temp file (the catch path) when the atomic rename fails', () => { + const dir = globalDir(home); + mkdirSync(dir, { recursive: true }); + // Force the rename to fail deterministically: renameSync(tempFile, target) where `target` is a DIRECTORY ⇒ + // EISDIR. This exercises writeFileAtomic's catch path (temp created + fsync'd, THEN rename throws) — proving + // the temp is unlinked (delete the `unlinkSync(tmp)` in write.ts and this test fails on an orphaned .tmp). + const target = join(dir, 'config.toml'); + mkdirSync(target, { recursive: true }); + expect(() => writeFileAtomic(dir, target, 'update_channel = "stable"\n')).toThrow(ConfigError); + const leftovers = readdirSync(dir).filter((name) => name.endsWith('.tmp')); + expect(leftovers).toEqual([]); + }); + + it('round-trips the tricky mcp_servers shapes (args array + env sub-table + a second network server) untouched', () => { + // The round-trip risk lives in the nested shapes (args arrays, env sub-tables) — not the flat command case. + // Seed them, write only default_model, and assert the whole config survives the serialize→reparse cycle. + mkdirSync(globalDir(home), { recursive: true }); + writeFileSync( + globalConfigPath(home), + [ + '[preferences]', + 'theme = "light"', + '[[mcp_servers]]', + 'name = "fs"', + 'transport = "stdio"', + 'command = "npx server-filesystem"', + 'args = ["--root", "/tmp"]', + '[mcp_servers.env]', + 'TOKEN = "{{secrets.gh}}"', + '[[mcp_servers]]', + 'name = "web"', + 'transport = "http"', + 'url = "https://example.com/mcp"', + '', + ].join('\n'), + ); + + writeGlobalDefaultModel('a-model', home); + + expect(readBack(home)).toEqual({ + preferences: { theme: 'light', default_model: 'a-model' }, + mcp_servers: [ + { + name: 'fs', + transport: 'stdio', + command: 'npx server-filesystem', + args: ['--root', '/tmp'], + env: { TOKEN: '{{secrets.gh}}' }, + }, + { name: 'web', transport: 'http', url: 'https://example.com/mcp' }, + ], + }); + }); + it('throws a ConfigError rather than clobbering a malformed existing config', () => { mkdirSync(globalDir(home), { recursive: true }); const target = globalConfigPath(home); diff --git a/apps/cli/src/config/write.ts b/apps/cli/src/config/write.ts index cbd19288..e76f06fc 100644 --- a/apps/cli/src/config/write.ts +++ b/apps/cli/src/config/write.ts @@ -4,35 +4,45 @@ import { homedir } from 'node:os'; import { join } from 'node:path'; import { GlobalConfigSchema, type GlobalConfig } from '@relavium/shared'; -import { stringify as stringifyToml } from 'smol-toml'; +import { parse as parseToml, stringify as stringifyToml } from 'smol-toml'; +import type { ZodError } from 'zod'; import { ConfigError } from './errors.js'; -import { loadConfigFile } from './load.js'; +import { formatZodError, loadConfigFile } from './load.js'; import { ensureGlobalConfigDir, globalConfigDir } from './paths.js'; /** * The **first on-disk config WRITER** ([ADR-0063](../../../../docs/decisions/0063-cli-config-write-contract.md)) — * the sibling of the read-only [load.ts](./load.ts). It persists a user's chosen chat default model to the * **global** `~/.relavium/config.toml` `[preferences].default_model`, the write target for `/models` and the - * onboarding wizard. + * onboarding wizard. Every later writer inherits this primitive, so its guarantees are enforced by construction, + * not by convention. * - * Three load-bearing guarantees (the ADR-0063 contract; a security-reviewed surface): + * Four load-bearing guarantees (the ADR-0063 contract; a security-reviewed surface): * 1. **Secret-free by construction.** The surface is a **typed setter** ({@link writeGlobalDefaultModel}) — never * a generic `writeKey(k, v)` — so it can only ever set `default_model` (a non-secret). There is no API-key - * field in the schema to write to (keys live only in the OS keychain, ADR-0006). - * 2. **Atomic + owner-only.** It writes a temp file created `0600` in the owner-only (`0700`) `~/.relavium/` - * directory, `fsync`s it, then `rename`s over the target — an interrupted write leaves the original intact. - * 3. **Schema round-trip.** It merges onto the *validated* existing config and re-validates the whole object - * against the strict {@link GlobalConfigSchema} (ADR-0033) BEFORE emitting, so the written file is guaranteed - * to re-parse cleanly on the next load. `smol-toml.stringify` re-serializes (dropping comments/ordering — the - * documented ADR-0063 tradeoff for the global file); no new dependency (`smol-toml` is the ADR-0048 parser). + * field in the schema to write to (keys live only in the OS keychain, ADR-0006). And a schema-validation + * failure on the write path is reported through the **same value-free formatter** the loader uses + * ({@link formatZodError}) — never a raw `ZodError`, whose `.message` embeds the received value for several + * codes and could reach stderr — so the secret-free-error property survives future schema changes. + * 2. **Atomic + durable + owner-only.** It writes a temp file created `0600` in the owner-only (`0700`) + * `~/.relavium/` directory, `fsync`s it, `rename`s over the target, then `fsync`s the parent directory so the + * rename itself is durable — an interrupted write leaves the original intact (never torn) and a completed one + * survives a crash. + * 3. **Verified schema round-trip.** It merges onto the *validated* existing config, re-validates the whole + * object against the strict {@link GlobalConfigSchema} (ADR-0033), AND re-parses the emitted TOML text back + * through the schema BEFORE the rename — so "the written file re-parses cleanly on the next load" is a + * verified guarantee, not a trust-the-serializer assumption. On any failure `config.toml` is never touched. + * 4. **No new dependency.** `smol-toml` is the ADR-0048 parser/serializer already in use; re-serialization drops + * comments/ordering (the documented ADR-0063 tradeoff for the global preference file — project/workspace files + * are never written by the tool). */ /** * Set the global `[preferences].default_model`, preserving every other config key. Reads + validates the existing * `~/.relavium/config.toml` (an absent file ⇒ a fresh `{}`; a **malformed/invalid** existing config throws a * {@link ConfigError} rather than clobbering a file the user must fix), merges only `default_model`, re-validates, - * and writes atomically. `home` is injectable for tests. + * verifies the serialized text round-trips, and writes atomically. `home` is injectable for tests. */ export function writeGlobalDefaultModel(model: string, home: string = homedir()): void { const dir = ensureGlobalConfigDir(home); // `~/.relavium/` (created `0700`) @@ -41,33 +51,75 @@ export function writeGlobalDefaultModel(model: string, home: string = homedir()) // Read the EXISTING config through the same validating loader (so we merge onto known-good data and preserve // update_channel / mcp_servers / preferences.theme). An absent file is a fresh object; an invalid one throws. const existing = loadConfigFile(target, GlobalConfigSchema) ?? {}; - const next: GlobalConfig = { + const merged: GlobalConfig = { ...existing, preferences: { ...existing.preferences, default_model: model }, }; + // Re-validate the whole object so the emission is provably schema-valid (ADR-0033) — a `.strict()` round-trip - // that also structurally guarantees no non-schema (e.g. secret) key can reach disk. - const validated = GlobalConfigSchema.parse(next); - writeFileAtomic(dir, target, stringifyToml(validated)); + // that also structurally guarantees no non-schema (e.g. secret) key can reach disk. Uses the value-free + // failure path so a future schema refinement (e.g. a `.min(1)` on `default_model`) fails LOUDLY without ever + // echoing the received value. + const validated = validateForWrite(merged, target); + const text = stringifyToml(validated); + // Verify the emitted TEXT re-parses to a schema-valid object BEFORE the atomic rename (ADR-0063 §3) — makes + // the "always re-parses on next load" guarantee verified, not assumed. On failure config.toml is untouched. + verifyRoundTrips(text, target); + writeFileAtomic(dir, target, text); +} + +/** Schema-validate the object to write, mapping a failure to the value-free {@link ConfigError} path. */ +function validateForWrite(value: GlobalConfig, target: string): GlobalConfig { + const result = GlobalConfigSchema.safeParse(value); + if (!result.success) { + throw configWriteError(target, result.error); + } + return result.data; +} + +/** Prove the serialized TOML re-parses through the schema (ADR-0063 §3) — else refuse to write. */ +function verifyRoundTrips(text: string, target: string): void { + let reparsed: unknown; + try { + reparsed = parseToml(text); + } catch (err) { + throw new ConfigError(target, 'could not be written — it serialized to invalid TOML.', { + cause: err, + }); + } + const result = GlobalConfigSchema.safeParse(reparsed); + if (!result.success) { + throw configWriteError(target, result.error); + } +} + +/** A write-attributed {@link ConfigError} whose detail is the loader's **value-free** field-path reason. */ +function configWriteError(target: string, error: ZodError): ConfigError { + return new ConfigError(target, `could not be written — ${formatZodError(error)}.`, { cause: error }); } /** * Atomic file replace: write `text` to a unique temp in the SAME (`0700`) directory (so `rename` is atomic and * on the same filesystem), explicitly `0600` (never relying on an inherited directory mode — `ensureGlobalConfigDir` - * `0700`s `~/.relavium/` itself, not necessarily its children), `fsync` for durability, then `rename` over the - * target. On any failure the temp is best-effort removed and a typed {@link ConfigError} is thrown. + * `0700`s `~/.relavium/` itself, not necessarily its children), `fsync` for data durability, `rename` over the + * target, then a best-effort parent-directory `fsync` so the rename (the directory-entry swap) is durable too. On + * any failure the temp is best-effort removed and a typed {@link ConfigError} is thrown. + * + * Exported for the fault-injection test that drives the catch path with a real failing rename (a `target` that is + * an existing directory ⇒ `EISDIR`), verifying the temp is cleaned up and the fd is not leaked. */ -function writeFileAtomic(dir: string, target: string, text: string): void { +export function writeFileAtomic(dir: string, target: string, text: string): void { const tmp = join(dir, `config.toml.${randomUUID()}.tmp`); let fd: number | undefined; try { - fd = openSync(tmp, 'wx', 0o600); // `wx`: fail if it exists (the uuid makes a collision impossible) + fd = openSync(tmp, 'wx', 0o600); // `wx`: fail if it exists (the uuid makes a collision impossible); O_EXCL refuses a symlink writeFileSync(fd, text, 'utf8'); fchmodSync(fd, 0o600); // owner-only, independent of umask fsyncSync(fd); // the temp is durably on disk before the rename closeSync(fd); fd = undefined; - renameSync(tmp, target); + renameSync(tmp, target); // replaces the directory entry — never writes THROUGH a symlink at `target` + fsyncDir(dir); // best-effort: make the rename itself durable, not just the file's data } catch (err) { if (fd !== undefined) { try { @@ -85,6 +137,30 @@ function writeFileAtomic(dir: string, target: string, text: string): void { } } +/** + * Best-effort `fsync` of a directory so a `rename` into it is durable (POSIX metadata flush). Swallows every + * error and NEVER throws — the rename has already succeeded, so a failure here (e.g. Windows, where a directory + * fd cannot be `fsync`'d) only weakens crash-durability of the new value, never correctness. Must not throw, or + * a post-rename failure would wrongly enter the caller's catch and report a successful write as failed. + */ +function fsyncDir(dir: string): void { + let dfd: number | undefined; + try { + dfd = openSync(dir, 'r'); + fsyncSync(dfd); + } catch { + /* best-effort durability of the rename — see the doc comment */ + } finally { + if (dfd !== undefined) { + try { + closeSync(dfd); + } catch { + /* ignore */ + } + } + } +} + /** The global config path — exported for tests / callers that need to assert the write target. */ export function globalConfigPath(home: string = homedir()): string { return join(globalConfigDir(home), 'config.toml'); diff --git a/docs/reference/contracts/config-spec.md b/docs/reference/contracts/config-spec.md index 24391454..6f761293 100644 --- a/docs/reference/contracts/config-spec.md +++ b/docs/reference/contracts/config-spec.md @@ -93,10 +93,13 @@ stdio-only fields (`command`/`args`/`env`) are rejected on a network registratio > default model: `/models` and the 2.5.G onboarding wizard set **`[preferences].default_model`** — and *only* > that key — through a **typed setter** (never a generic key/value writer), so a secret can never be written by > construction (there is no `api_key` field in the schema; keys live only in the OS keychain, [ADR-0006](../../decisions/0006-os-keychain-for-api-keys.md)). -> The write is **atomic** (a `0600` temp file in the `0700` `~/.relavium/`, `fsync`, then `rename`) and re-validates -> the whole object against the strict `GlobalConfigSchema` before emitting, so the file always re-parses. The -> documented tradeoff: re-serialization **drops comments and key ordering** in `config.toml` — the global file is -> a preference store, not a hand-curated artifact (project/workspace files are never written by the tool). +> The write is **atomic + durable** (a `0600` temp file in the `0700` `~/.relavium/`, `fsync` the file, `rename` +> over the target, then `fsync` the parent directory so the rename itself survives a crash) and it both +> re-validates the merged object against the strict `GlobalConfigSchema` AND re-parses the emitted TOML back +> through the schema before the rename — so "the file always re-parses on the next load" is a **verified** +> guarantee, and on any failure `config.toml` is left untouched. The documented tradeoff: re-serialization +> **drops comments and key ordering** in `config.toml` — the global file is a preference store, not a +> hand-curated artifact (project/workspace files are never written by the tool). ## `project.toml` / `workspace.toml` (project) — keys From 33c08b4f6300574f3e3fa9501a0d73b1111a2995 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 23:05:31 +0300 Subject: [PATCH 17/68] =?UTF-8?q?test,fix(cli):=202.5.G=20S6=20Sonnet-revi?= =?UTF-8?q?ew=20=E2=80=94=20cover=20the=20round-trip=20guard=20+=20tighten?= =?UTF-8?q?=20error=20contract?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Independent adversarial Sonnet review (fuzzed smol-toml, live repro) found no blocker but one MAJOR coverage gap and legitimate minors. Fixes: - MAJOR: the verifyRoundTrips guard (added by the prior hardening commit) is reachable via the public API — a lone UTF-16 surrogate in the model string stringifies to an invalid TOML escape that parseToml rejects — yet had ZERO test coverage; dropping the guard call would pass the whole suite silently. Add a test that drives that exact path and asserts a value-free ConfigError + config.toml never created + no orphan temp. Verified mutation-sensitive: commenting out verifyRoundTrips fails it. - Wrap ensureGlobalConfigDir so a directory-create failure (EACCES/ENOSPC/ read-only home) becomes a typed, file-attributed ConfigError, keeping the module's "every failure is a ConfigError" contract instead of leaking a raw fs Error. - Document the raw-`cause` invariant: the enforced value-free surface is the ConfigError MESSAGE, not `.cause` (which may carry the value); no renderer prints .cause (only err.stack), and a future verbose renderer must not dump it without re-checking — aligning with the S5 model-refresh stance while keeping the debug-useful cause the sibling loader also attaches. - Sharpen the writeFileAtomic export doc: it is ONLY a test seam, not part of the writer's public contract (a generic path/text primitive with no secret-incapability guarantee — that property belongs to the typed setter). - Record the last-writer-wins concurrency tradeoff (no lock) in write.ts and config-spec.md. Not fixed (rationale): formatZodError's default arm buckets custom-code zod issues into a generic message — that is the SAFER value-free default (a custom message could someday interpolate a value), and it is pre-existing load.ts behavior out of S6 scope. Toolchain: lint + typecheck + test (1349) + build all green. Refs: ADR-0063 Co-Authored-By: Claude --- apps/cli/src/config/write.test.ts | 31 +++++++++++++++++++++++- apps/cli/src/config/write.ts | 32 ++++++++++++++++++++++--- docs/reference/contracts/config-spec.md | 8 ++++--- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/apps/cli/src/config/write.test.ts b/apps/cli/src/config/write.test.ts index 12465fe0..6a563daa 100644 --- a/apps/cli/src/config/write.test.ts +++ b/apps/cli/src/config/write.test.ts @@ -1,4 +1,13 @@ -import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from 'node:fs'; +import { + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from 'node:fs'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; @@ -139,6 +148,26 @@ describe('writeGlobalDefaultModel', () => { }); }); + it('refuses to write (throws, never creates the file) when the model serializes to non-round-tripping TOML', () => { + // A lone UTF-16 surrogate is a valid JS string (so it passes schema validation) but stringifies to an + // invalid TOML escape that parseToml rejects. The round-trip self-check (ADR-0063 §3) must catch it BEFORE + // the atomic rename, so config.toml is never created. This is the exact guarantee the hardening added — + // WITHOUT this test, silently dropping the verifyRoundTrips() call would still pass the whole suite. + let thrown: unknown; + try { + writeGlobalDefaultModel('sentinel:\uD800:model', home); + } catch (err) { + thrown = err; + } + expect(thrown).toBeInstanceOf(ConfigError); + if (thrown instanceof ConfigError) { + expect(thrown.message).not.toContain('sentinel:'); // value-free message (no echoed model value) + } + // The atomic contract: a rejected write leaves no target and no orphan temp. + expect(existsSync(globalConfigPath(home))).toBe(false); + expect(readdirSync(globalDir(home)).filter((name) => name.endsWith('.tmp'))).toEqual([]); + }); + it('throws a ConfigError rather than clobbering a malformed existing config', () => { mkdirSync(globalDir(home), { recursive: true }); const target = globalConfigPath(home); diff --git a/apps/cli/src/config/write.ts b/apps/cli/src/config/write.ts index e76f06fc..5d5c9b74 100644 --- a/apps/cli/src/config/write.ts +++ b/apps/cli/src/config/write.ts @@ -36,6 +36,18 @@ import { ensureGlobalConfigDir, globalConfigDir } from './paths.js'; * 4. **No new dependency.** `smol-toml` is the ADR-0048 parser/serializer already in use; re-serialization drops * comments/ordering (the documented ADR-0063 tradeoff for the global preference file — project/workspace files * are never written by the tool). + * + * Two documented invariants / tradeoffs: + * - **The error MESSAGE is the enforced value-free surface — not the `cause`.** A failure attaches the raw + * `ZodError`/`TomlError` as `cause` (as the sibling loader does) for debuggability. That cause *can* embed the + * attempted value, but the enforced surface is the value-free `ConfigError` **message** ({@link formatZodError} + * / static strings). No current renderer prints `.cause` — they print `err.stack`, which excludes it — so no + * value leaks today. A future verbose renderer MUST NOT dump `.cause` (e.g. `util.inspect(err, { depth: null })`) + * without re-establishing the value-free property here (mirrors the S5 `model-refresh.ts` "never read cause" + * stance, kept as a documented invariant here rather than dropping the debug-useful cause the loader also keeps). + * - **Last-writer-wins under concurrent writes.** There is no lock/CAS: two racing invocations each read then + * rename, so the later rename silently supersedes the earlier edit (a lost update, never a torn or invalid + * file). Accepted for a single-user, rarely-written preference store; a future multi-writer path would need a lock. */ /** @@ -45,7 +57,18 @@ import { ensureGlobalConfigDir, globalConfigDir } from './paths.js'; * verifies the serialized text round-trips, and writes atomically. `home` is injectable for tests. */ export function writeGlobalDefaultModel(model: string, home: string = homedir()): void { - const dir = ensureGlobalConfigDir(home); // `~/.relavium/` (created `0700`) + let dir: string; + try { + dir = ensureGlobalConfigDir(home); // `~/.relavium/` (created `0700`) + } catch (err) { + // Keep the module's "every failure is a typed, file-attributed ConfigError" contract even for the + // directory-create step (e.g. EACCES on a read-only home, ENOSPC) — never let a raw fs Error escape. + throw new ConfigError( + globalConfigPath(home), + 'could not be written — its directory could not be created.', + { cause: err }, + ); + } const target = join(dir, 'config.toml'); // Read the EXISTING config through the same validating loader (so we merge onto known-good data and preserve @@ -105,8 +128,11 @@ function configWriteError(target: string, error: ZodError): ConfigError { * target, then a best-effort parent-directory `fsync` so the rename (the directory-entry swap) is durable too. On * any failure the temp is best-effort removed and a typed {@link ConfigError} is thrown. * - * Exported for the fault-injection test that drives the catch path with a real failing rename (a `target` that is - * an existing directory ⇒ `EISDIR`), verifying the temp is cleaned up and the fd is not leaked. + * Exported ONLY as a test seam — for the fault-injection test that drives the catch path with a real failing + * rename (a `target` that is an existing directory ⇒ `EISDIR`), verifying the temp is cleaned up and the fd is + * not leaked. It is NOT part of the writer's public contract: it is a generic "replace this path with this text" + * primitive with no schema, so the secret-incapability guarantee is a property of {@link writeGlobalDefaultModel} + * (the typed setter), never of this helper. Do not call it to write arbitrary content. */ export function writeFileAtomic(dir: string, target: string, text: string): void { const tmp = join(dir, `config.toml.${randomUUID()}.tmp`); diff --git a/docs/reference/contracts/config-spec.md b/docs/reference/contracts/config-spec.md index 6f761293..84e4699e 100644 --- a/docs/reference/contracts/config-spec.md +++ b/docs/reference/contracts/config-spec.md @@ -97,9 +97,11 @@ stdio-only fields (`command`/`args`/`env`) are rejected on a network registratio > over the target, then `fsync` the parent directory so the rename itself survives a crash) and it both > re-validates the merged object against the strict `GlobalConfigSchema` AND re-parses the emitted TOML back > through the schema before the rename — so "the file always re-parses on the next load" is a **verified** -> guarantee, and on any failure `config.toml` is left untouched. The documented tradeoff: re-serialization -> **drops comments and key ordering** in `config.toml` — the global file is a preference store, not a -> hand-curated artifact (project/workspace files are never written by the tool). +> guarantee, and on any failure `config.toml` is left untouched. Two documented tradeoffs: re-serialization +> **drops comments and key ordering** in `config.toml` (the global file is a preference store, not a +> hand-curated artifact — project/workspace files are never written by the tool); and there is **no lock**, so +> two concurrent writes resolve **last-writer-wins** (a lost update, never a torn file) — acceptable for a +> single-user, rarely-written preference store. ## `project.toml` / `workspace.toml` (project) — keys From 574e05c9e56e8ca8f3dcad04881e992a99a06786 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 23:40:31 +0300 Subject: [PATCH 18/68] =?UTF-8?q?feat(cli):=202.5.G=20S7=20=E2=80=94=20the?= =?UTF-8?q?=20/models=20Home=20picker=20over=20the=20merged=20live=20catal?= =?UTF-8?q?og?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the first-class `/models` model picker (ADR-0064 §10) — a Home-only, keyboard-owning in-tree overlay (mirroring the @-mention submode) that lists the MERGED live/static catalog and, on selection, writes the NEXT session's default model via the S6 config writer (ADR-0063). It does NOT rebind the live session — that is the Phase-2.6 mid-chat reseat (ADR-0059), deliberately kept distinct; a `/models` typed inside a chat is rejected with a pointer to the Home (a new availableIn surface-scope guard). Pieces: - model-catalog-view.ts: the pure host projection (DB rows → merged entries) — partitions the source='live' rows into a per-ProviderId live map (UUID→slug, non-enum providers dropped) and delegates to the pure @relavium/llm mergeModelCatalog; the userPricing tier stays empty (S10). - model-picker.ts: the pure submode (state + fold + formatters). A dimmed (unavailable-on-your-key) model is non-selectable (ADR §6); Ctrl+R refreshes, Esc cancels; price/context/freshness/partial-failure display. - model-picker-view.tsx: the rich ink view (windowed rows, dimmed unavailable, deprecated flag, unpriced "cost cap will not apply" hint, loading spinner, per-provider partial-failure banner, freshness badge). - home-controller.ts: the HomeModelsPort seam + the openModels capability, open/accept/refresh/route wiring, the picker-owns-keys handleKey branch, and the overlay-reset hygiene. - drive-home.tsx: builds providerStore/catalogStore/refreshService over the ONE already-open history.db handle (the catalog shares it, ADR-0050) + the Home resolver, and injects the port. Opening the picker over an empty/stale cache renders immediately and kicks a TTL-bounded background refresh — sound because the Home is the long-lived process the S5 constraint requires. The slug map is rebuilt per load (not memoized once like the one-shot dispatch resolver) so a refresh-discovered provider's rows are not dropped. - repl-commands.ts: the /models entry (availableIn ['home']) + the openModels capability; chat.ts wires an inert (guard-unreachable) impl. Docs: commands.md documents the home-only /models picker + its UX. Tests: the projection (availability, uuid→slug, refreshedAt, source filtering, rogue-row drop), the fold + formatters, the scroll window, and the controller wiring (open + background refresh, accept→write+notice, dimmed non-selectable, Esc, Ctrl+R + partial-failure banner, keyboard ownership, no-port degrade). Toolchain: lint + typecheck + test (1377, +28) + build all green across all 24 workspace tasks. Refs: ADR-0064, ADR-0063 Co-Authored-By: Claude --- apps/cli/src/commands/chat.ts | 14 ++ apps/cli/src/commands/repl-commands.test.ts | 30 ++- apps/cli/src/commands/repl-commands.ts | 18 ++ .../cli/src/engine/model-catalog-view.test.ts | 99 +++++++++ apps/cli/src/engine/model-catalog-view.ts | 89 ++++++++ apps/cli/src/home/drive-home.tsx | 53 ++++- apps/cli/src/render/tui/home-app.tsx | 6 +- .../src/render/tui/home-controller.test.ts | 170 ++++++++++++++++ apps/cli/src/render/tui/home-controller.ts | 141 ++++++++++++- .../src/render/tui/model-picker-view.test.ts | 21 ++ apps/cli/src/render/tui/model-picker-view.tsx | 123 +++++++++++ apps/cli/src/render/tui/model-picker.test.ts | 186 +++++++++++++++++ apps/cli/src/render/tui/model-picker.ts | 192 ++++++++++++++++++ docs/reference/cli/commands.md | 2 +- 14 files changed, 1133 insertions(+), 11 deletions(-) create mode 100644 apps/cli/src/engine/model-catalog-view.test.ts create mode 100644 apps/cli/src/engine/model-catalog-view.ts create mode 100644 apps/cli/src/render/tui/model-picker-view.test.ts create mode 100644 apps/cli/src/render/tui/model-picker-view.tsx create mode 100644 apps/cli/src/render/tui/model-picker.test.ts create mode 100644 apps/cli/src/render/tui/model-picker.ts diff --git a/apps/cli/src/commands/chat.ts b/apps/cli/src/commands/chat.ts index 55872451..cceaea00 100644 --- a/apps/cli/src/commands/chat.ts +++ b/apps/cli/src/commands/chat.ts @@ -829,6 +829,10 @@ export function createChatLineHandler( clearRequested = true; stop = true; }, + // `/models` (2.5.G S7, ADR-0064 §10) is HOME-ONLY (a next-session config action). The slash dispatch below + // rejects it on this chat surface with a pointer to the Home, so this capability is never reached here — kept as + // a documented inert noop to satisfy the shared ReplCommandContext shape (mirroring the Home's inert chat-only noops). + openModels: () => undefined, }; // Parse + dispatch a `/name [args]` REPL line (extracted from processLine so each stays under the Sonar @@ -848,6 +852,16 @@ export function createChatLineHandler( emitOutput(`unknown command '${safe}'. Available: ${replCommandList()}.`); return; } + // Surface-scope guard (2.5.G S7): a command not `availableIn` this (chat) surface — e.g. the HOME-ONLY `/models` + // — is rejected with an actionable pointer, never dispatched. The palette already filters by surface + // (CHAT_PALETTE_COMMANDS); this covers a command TYPED directly, keeping the home-only next-session `/models` + // config action distinct from the Phase-2.6 mid-chat `/models` live reseat (ADR-0059). + if (!command.availableIn.includes('chat')) { + emitOutput( + `/${command.name} is available from the Home (run \`relavium\` with no arguments), not inside a chat.`, + ); + return; + } const rejection = validateSlashTokens(command, tokens); if (rejection !== undefined) { emitOutput(rejection); diff --git a/apps/cli/src/commands/repl-commands.test.ts b/apps/cli/src/commands/repl-commands.test.ts index 29b6a8e4..8ae2d040 100644 --- a/apps/cli/src/commands/repl-commands.test.ts +++ b/apps/cli/src/commands/repl-commands.test.ts @@ -23,6 +23,7 @@ interface CapabilityCalls { readonly compactHistory: number; readonly trimHistory: number; readonly clearSession: number; + readonly openModels: number; } /** A fully-spied REPL context — each capability is a spy so a command's `run` can be asserted to call exactly one. */ @@ -39,6 +40,7 @@ function spyContext(): { ctx: ReplCommandContext; calls: () => CapabilityCalls } compactHistory: vi.fn(), trimHistory: vi.fn(), clearSession: vi.fn(), + openModels: vi.fn(), }; return { ctx: spies, @@ -54,6 +56,7 @@ function spyContext(): { ctx: ReplCommandContext; calls: () => CapabilityCalls } compactHistory: spies.compactHistory.mock.calls.length, trimHistory: spies.trimHistory.mock.calls.length, clearSession: spies.clearSession.mock.calls.length, + openModels: spies.openModels.mock.calls.length, }), }; } @@ -75,6 +78,7 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { 'compact', 'trim', 'clear', + 'models', ]); }); @@ -91,6 +95,7 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { ['compact', 'compactHistory'], ['trim', 'trimHistory'], ['clear', 'clearSession'], + ['models', 'openModels'], ]; for (const [name, capability] of cases) { const { ctx, calls } = spyContext(); @@ -108,7 +113,8 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { counts.setMode + counts.compactHistory + counts.trimHistory + - counts.clearSession; + counts.clearSession + + counts.openModels; expect(total, `${name} calls exactly one capability`).toBe(1); } }); @@ -124,7 +130,7 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { it('replCommandList renders the slash hint, formatReplHelp lists every command', () => { expect(replCommandList()).toBe( - '/help, /exit, /cancel, /export, /workflows, /cost, /doctor, /mode, /compact, /trim, /clear', + '/help, /exit, /cancel, /export, /workflows, /cost, /doctor, /mode, /compact, /trim, /clear, /models', ); const help = formatReplHelp(); for (const command of REPL_COMMANDS) { @@ -137,7 +143,17 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { expect(REPL_COMMANDS_BY_NAME.get('export')?.effect).toBe('write'); expect(REPL_COMMANDS_BY_NAME.get('compact')?.effect).toBe('write'); // /compact spends tokens (ADR-0062) expect(REPL_COMMANDS_BY_NAME.get('clear')?.effect).toBe('destructive'); // /clear ends the session (ADR-0062 §7) - for (const name of ['help', 'exit', 'cancel', 'workflows', 'cost', 'doctor', 'mode', 'trim']) { + for (const name of [ + 'help', + 'exit', + 'cancel', + 'workflows', + 'cost', + 'doctor', + 'mode', + 'trim', + 'models', + ]) { expect(REPL_COMMANDS_BY_NAME.get(name)?.effect).toBe('read'); } }); @@ -158,7 +174,9 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { 'compact', 'trim', 'clear', + 'models', ]); + // /models is HOME-ONLY (availableIn ['home']) — it is EXCLUDED from the chat palette. expect(CHAT_PALETTE_COMMANDS.map((c) => c.name)).toEqual([ 'exit', 'cancel', @@ -171,8 +189,8 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { 'trim', 'clear', ]); - // The bare Home offers /exit + /doctor (pre-chat diagnostics) and now /clear (availableIn ['home','chat']; - // an inert "nothing to clear" notice in the bare Home — ADR-0062 §7). - expect(HOME_PALETTE_COMMANDS.map((c) => c.name)).toEqual(['exit', 'doctor', 'clear']); + // The bare Home offers /exit + /doctor (pre-chat diagnostics), /clear (availableIn ['home','chat']; an inert + // "nothing to clear" notice — ADR-0062 §7), and now /models (home-only, ADR-0064 §10). + expect(HOME_PALETTE_COMMANDS.map((c) => c.name)).toEqual(['exit', 'doctor', 'clear', 'models']); }); }); diff --git a/apps/cli/src/commands/repl-commands.ts b/apps/cli/src/commands/repl-commands.ts index 3a272fc7..763d00b8 100644 --- a/apps/cli/src/commands/repl-commands.ts +++ b/apps/cli/src/commands/repl-commands.ts @@ -53,6 +53,12 @@ export interface ReplCommandContext { * Interactive-only — a `--json` / plain non-TTY session rejects it (one machine stream is one session lifecycle). * Its notice surfaces the OLD sessionId + `relavium chat-resume ` so the prior conversation is discoverable. */ readonly clearSession: () => void | Promise; + /** `/models` (2.5.G S7, [ADR-0064](../../../../docs/decisions/0064-live-model-catalog.md) §10) — open the in-tree + * model picker over the merged live/static catalog; on selection it writes the NEXT session's default model + * ([ADR-0063](../../../../docs/decisions/0063-cli-config-write-contract.md)), it does NOT rebind the live session + * (that is the Phase-2.6 reseat, ADR-0059). HOME-ONLY (`availableIn: ['home']`): the Home wires the real picker; + * the chat surface never reaches this (the slash dispatch rejects a non-`chat` command), so its impl is inert. */ + readonly openModels: () => void | Promise; } /** A flag a {@link ReplCommand} accepts after its name (e.g. `/doctor --deep`). Flags only — the curated set has @@ -207,6 +213,18 @@ const RAW_REPL_COMMANDS: readonly ReplCommand[] = [ run: (ctx) => ctx.clearSession(), availableIn: ['home', 'chat'], }, + { + name: 'models', + label: 'Models', + description: 'Pick your default model (opens the catalog picker).', + // `read` in the forward taxonomy: opening the picker changes nothing; the config write happens only on an + // explicit selection (ADR-0063), and even then it sets the NEXT session's default, never the running one. + effect: 'read', + // HOME-ONLY (ADR-0064 §10): a next-session CONFIG action, deliberately distinct from the Phase-2.6 mid-chat + // `/models` live reseat (ADR-0059). Inside a live chat the slash dispatch rejects it with a pointer to the Home. + run: (ctx) => ctx.openModels(), + availableIn: ['home'], + }, ]; /** DEEP-freeze a curated command — the entry, its `args` array + each flag, and its `availableIn` array — so no diff --git a/apps/cli/src/engine/model-catalog-view.test.ts b/apps/cli/src/engine/model-catalog-view.test.ts new file mode 100644 index 00000000..252e0c99 --- /dev/null +++ b/apps/cli/src/engine/model-catalog-view.test.ts @@ -0,0 +1,99 @@ +import type { ModelCatalogListing } from '@relavium/db'; +import { MODEL_PRICING } from '@relavium/llm'; +import { describe, expect, it } from 'vitest'; + +import { buildMergedCatalog } from './model-catalog-view.js'; + +/** Two known static anthropic model ids (derived from the registry so the test survives a pricing.ts edit). */ +function twoAnthropicIds(): readonly [string, string] { + const ids = Object.entries(MODEL_PRICING) + .filter(([, pricing]) => pricing.provider === 'anthropic') + .map(([id]) => id); + const [present, absent] = ids; + if (present === undefined || absent === undefined) { + throw new Error('test precondition: MODEL_PRICING must carry ≥2 anthropic models'); + } + return [present, absent]; +} +const [MODEL_PRESENT, MODEL_ABSENT] = twoAnthropicIds(); + +/** A `model_catalog` row with sensible non-secret defaults; override what a case cares about. */ +function row( + partial: Pick & + Partial, +): ModelCatalogListing { + return { + displayName: partial.modelId, + inputCostPerMtokMicrocents: 0, + outputCostPerMtokMicrocents: 0, + cachedInputCostPerMtokMicrocents: 0, + isActive: true, + ...partial, + }; +} + +/** A slug resolver over a fixed UUID→slug map (an unmapped uuid falls back to itself — like the real one). */ +function slugResolver(map: Record): (uuid: string) => string { + return (uuid) => map[uuid] ?? uuid; +} + +describe('buildMergedCatalog', () => { + it('seeds the full static registry even with no live rows (never an empty picker)', () => { + const view = buildMergedCatalog({ rows: [], providerSlug: slugResolver({}), now: 0 }); + expect(view.entries.length).toBe(Object.keys(MODEL_PRICING).length); + expect(view.refreshedAt).toBeUndefined(); + // With NO live data for any provider, every static model falls back to static presence (ADR-0064 §6). + expect(view.entries.every((e) => e.available)).toBe(true); + }); + + it('a live row makes its provider "have live data": present models stay available, absent ones are dimmed', () => { + const view = buildMergedCatalog({ + rows: [row({ modelId: MODEL_PRESENT, providerId: 'p-anthropic', source: 'live' })], + providerSlug: slugResolver({ 'p-anthropic': 'anthropic' }), + now: 0, + }); + const present = view.entries.find((e) => e.modelId === MODEL_PRESENT); + const absent = view.entries.find((e) => e.modelId === MODEL_ABSENT); + expect(present?.available).toBe(true); // in the live list + expect(absent?.available).toBe(false); // anthropic HAS live data, and this static id is not in it ⇒ dimmed + }); + + it('reports refreshedAt as the newest lastRefreshedAt across the live rows only', () => { + const view = buildMergedCatalog({ + rows: [ + row({ modelId: MODEL_PRESENT, providerId: 'p-anthropic', source: 'live', lastRefreshedAt: 100 }), + row({ modelId: MODEL_ABSENT, providerId: 'p-anthropic', source: 'live', lastRefreshedAt: 300 }), + // a non-live row's stamp (if any) must NOT count toward freshness + row({ modelId: 'x', providerId: 'p-anthropic', source: 'user', lastRefreshedAt: 999 }), + ], + providerSlug: slugResolver({ 'p-anthropic': 'anthropic' }), + now: 0, + }); + expect(view.refreshedAt).toBe(300); + }); + + it('ignores non-live rows for availability — a user/static row does NOT mark the provider as having live data', () => { + // Only a `user` row for anthropic (no `live` row) ⇒ anthropic has NO live data ⇒ its statics stay available. + const view = buildMergedCatalog({ + rows: [row({ modelId: MODEL_PRESENT, providerId: 'p-anthropic', source: 'user' })], + providerSlug: slugResolver({ 'p-anthropic': 'anthropic' }), + now: 0, + }); + expect(view.entries.find((e) => e.modelId === MODEL_ABSENT)?.available).toBe(true); + expect(view.refreshedAt).toBeUndefined(); + }); + + it('drops a live row whose provider UUID resolves to a non-enum slug (no throw, no spurious dimming)', () => { + // An unmapped uuid resolves to itself ('rogue'), which is not a ProviderId ⇒ the row is skipped, so anthropic + // has NO live data and its statics stay available (the rogue row cannot dim an unrelated provider). + const view = buildMergedCatalog({ + rows: [row({ modelId: MODEL_PRESENT, providerId: 'rogue', source: 'live', lastRefreshedAt: 50 })], + providerSlug: slugResolver({}), + now: 0, + }); + expect(view.entries.find((e) => e.modelId === MODEL_ABSENT)?.available).toBe(true); + // The skipped row contributes NOTHING — not availability, and not freshness (the stamp is read only AFTER the + // slug is validated as a known ProviderId), so a rogue/custom-provider row can never skew the badge. + expect(view.refreshedAt).toBeUndefined(); + }); +}); diff --git a/apps/cli/src/engine/model-catalog-view.ts b/apps/cli/src/engine/model-catalog-view.ts new file mode 100644 index 00000000..65a78d7c --- /dev/null +++ b/apps/cli/src/engine/model-catalog-view.ts @@ -0,0 +1,89 @@ +import type { ModelCatalogListing } from '@relavium/db'; +import { + mergeModelCatalog, + type ModelCatalogEntry, + type ModelListing, + type ProviderId, +} from '@relavium/llm'; +import { LLM_PROVIDERS } from '@relavium/shared'; + +/** + * The host projection that turns the durable `model_catalog` cache rows into the merged, display-ready catalog the + * `/models` picker renders (workstream **2.5.G S7**, [ADR-0064](../../../../docs/decisions/0064-live-model-catalog.md) §6/§10). + * It is the thin, PURE glue between the `@relavium/db` store (which speaks internal provider UUIDs + `source` + * rows) and the pure `@relavium/llm` {@link mergeModelCatalog} (which speaks the `ProviderId` enum + plain tiers): + * it builds the LIVE map from the `source='live'` rows (translating provider UUID → slug) and hands it to the + * merge, whose static tier is the in-code `MODEL_PRICING`. Keeping this in the host — not the store, not the merge + * — is what lets `@relavium/llm`/`@relavium/core` stay platform-free while every surface reuses the one merge. + * + * The ADR-0065 USER-pricing tier is intentionally NOT built here yet — S7 ships availability + static pricing; the + * `source='user'` rows are populated + merged in S10 (the merge already accepts an optional `userPricing` slot, so + * that lands additively with no signature change). + */ + +/** The merged catalog for the picker + the newest live-refresh stamp (the "last updated" freshness badge). */ +export interface MergedCatalogView { + readonly entries: readonly ModelCatalogEntry[]; + /** The newest `lastRefreshedAt` across the `source='live'` rows (epoch-ms), or `undefined` when never refreshed. */ + readonly refreshedAt: number | undefined; +} + +/** The store + resolver inputs the projection reads — a structural subset so a test drives it with plain data. */ +export interface BuildMergedCatalogInput { + /** All ACTIVE catalog rows (the store's `listAll()`), across every provider. */ + readonly rows: readonly ModelCatalogListing[]; + /** Resolve an internal `llm_providers` UUID → its provider slug (e.g. `anthropic`) — `createProviderSlugResolver`. */ + readonly providerSlug: (uuid: string) => string; + /** Current time (epoch-ms) for the deprecation check — passed in so the projection stays pure/testable. */ + readonly now: number; +} + +/** A total type-guard over the CLOSED provider enum — a UUID that resolves to a non-enum slug (an unmapped id, or a + * future custom provider handled by ADR-0065/S9) is not a `ProviderId` and its live rows are skipped here. */ +function isProviderId(slug: string): slug is ProviderId { + return (LLM_PROVIDERS as readonly string[]).includes(slug); +} + +/** Map a stored catalog row → a seam {@link ModelListing} (the live-discovery half): id + the optional limits + + * the live deprecation date as ISO (the store carries epoch-ms; the merge unions ISO dates). Pricing is NOT + * carried — the merge's pricing authority is the static registry / user tier, never a live row (ADR-0064 §6). */ +function rowToListing(row: ModelCatalogListing): ModelListing { + return { + id: row.modelId, + ...(row.displayName.length > 0 ? { displayName: row.displayName } : {}), + ...(row.contextWindowTokens !== undefined + ? { contextWindowTokens: row.contextWindowTokens } + : {}), + ...(row.maxOutputTokens !== undefined ? { maxOutputTokens: row.maxOutputTokens } : {}), + ...(row.deprecationDate !== undefined + ? { deprecatedAt: new Date(row.deprecationDate).toISOString() } + : {}), + }; +} + +/** + * Project the active catalog rows into the merged `/models` view. Partitions the `source='live'` rows into a + * per-`ProviderId` live map (a provider PRESENT in the map — even with no surviving rows — has live data, so its + * static models are availability-checked against the list; a provider ABSENT has no live data, so its static + * models fall back to static presence — ADR-0064 §6), then delegates to the pure merge. A live row whose provider + * UUID resolves to a non-enum slug is dropped (defensive: a mis-keyed or future custom-provider row can never + * corrupt an unrelated known provider's entry — the merge has its own cross-provider guard too). + */ +export function buildMergedCatalog(input: BuildMergedCatalogInput): MergedCatalogView { + const live = new Map(); + let refreshedAt: number | undefined; + for (const row of input.rows) { + if (row.source !== 'live') continue; // only the live-discovery rows decide availability + const slug = input.providerSlug(row.providerId); + if (!isProviderId(slug)) continue; // an unmapped UUID / a non-enum (custom) provider — not this step's concern + // Freshness reflects only VALID (enum-provider) live rows — a dropped rogue row must not skew the badge. + if (row.lastRefreshedAt !== undefined) { + refreshedAt = refreshedAt === undefined ? row.lastRefreshedAt : Math.max(refreshedAt, row.lastRefreshedAt); + } + const list = live.get(slug) ?? []; + list.push(rowToListing(row)); + live.set(slug, list); + } + const entries = mergeModelCatalog({ live, now: input.now }); + return { entries, refreshedAt }; +} diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index ac576a78..04068cab 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -1,6 +1,6 @@ import { randomUUID } from 'node:crypto'; -import { createRunHistoryReader } from '@relavium/db'; +import { createModelCatalogStore, createProviderStore, createRunHistoryReader } from '@relavium/db'; import { render } from 'ink'; import { createElement } from 'react'; @@ -10,8 +10,16 @@ import { assembleDoctorProbes } from '../chat/doctor-host.js'; import type { DoctorProbes } from '../chat/doctor.js'; import { createSessionPersister, type SessionPersister } from '../chat/persister.js'; import { loadResolvedConfig } from '../config/load.js'; +import { writeGlobalDefaultModel } from '../config/write.js'; +import { buildMergedCatalog } from '../engine/model-catalog-view.js'; +import { createModelRefreshService } from '../engine/model-refresh.js'; import { assembleToolEnv } from '../engine/tool-host/assemble.js'; -import { createProviderResolver, type ProviderResolver } from '../engine/providers.js'; +import { + createProviderResolver, + KNOWN_PROVIDERS, + KNOWN_PROVIDER_IDS, + type ProviderResolver, +} from '../engine/providers.js'; import { openSessionStore, type OpenedSessionStore } from '../history/session-open.js'; import type { CliIo } from '../process/io.js'; import { EXIT_CODES, type ExitCode } from '../process/exit-codes.js'; @@ -24,6 +32,7 @@ import { createHomeController, type HomeChatSession, type HomeController, + type HomeModelsPort, } from '../render/tui/home-controller.js'; import { DISABLE_BRACKETED_PASTE, ENABLE_BRACKETED_PASTE } from '../render/tui/home-input.js'; import { RootApp, type RootAppProps } from '../render/tui/home-app.js'; @@ -136,6 +145,45 @@ export async function driveHome(deps: HomeDeps): Promise { runs: createRunHistoryReader(opened.db), }); + // The `/models` catalog port (2.5.G S7, ADR-0064) — built over the ONE already-open db handle (`opened.db`, the + // same `history.db` the catalog cache shares, ADR-0050) + the Home's provider resolver, so the picker reads the + // merged catalog + runs the (long-lived-process-safe) TTL background refresh + writes the next session's default. + const storeDeps = { uuid, now }; + const providerStore = createProviderStore(opened.db, storeDeps); + const catalogStore = createModelCatalogStore(opened.db, storeDeps); + const refreshService = createModelRefreshService({ + resolveProvider: providers.resolveProvider, + keyFor: providers.keyFor, + providerStore, + catalogStore, + knownProviderIds: KNOWN_PROVIDER_IDS, + knownProviders: KNOWN_PROVIDERS, + now, + }); + // The `✓`-marked current default: `config.chat.defaultModel` resolves project → workspace → global `[preferences]` + // (ADR-0063 §1). A same-session `/models` write updates the GLOBAL file, so track the last write locally so a + // re-open of the picker marks the just-chosen model (the loaded `config` snapshot is not re-read mid-session). + let chosenDefault = config.chat.defaultModel; + const models: HomeModelsPort = { + load: () => { + // Rebuild the UUID→slug map on every load (NOT memoized once like the one-shot dispatch resolver): a refresh + // may register a provider's FK row, and the next load must resolve its live rows' provider — not drop them. + const slugByUuid = new Map(providerStore.list().map((p) => [p.id, p.name] as const)); + return buildMergedCatalog({ + rows: catalogStore.listAll(), + providerSlug: (uuid_) => slugByUuid.get(uuid_) ?? uuid_, + now: now(), + }); + }, + refreshIfStale: () => refreshService.refreshIfStale(), + refresh: () => refreshService.refresh(), + currentDefault: () => chosenDefault, + writeDefault: (modelId) => { + writeGlobalDefaultModel(modelId, homeDir); + chosenDefault = modelId; // the ✓ marker follows the write within this long-lived Home session + }, + }; + // Build + wire + START a fresh chat session (the controller sends the first message on transition). const startChat = async (): Promise => { const store = createChatStore(deps.global.color); @@ -313,6 +361,7 @@ export async function driveHome(deps: HomeDeps): Promise { startChat, homeStore, doctorProbes, + models, onExit: () => resolve(EXIT_CODES.success), // a clean Home exit is exit 0 onError: (err) => reject(err instanceof Error ? err : new Error(String(err))), }); diff --git a/apps/cli/src/render/tui/home-app.tsx b/apps/cli/src/render/tui/home-app.tsx index 5785bf89..9fb7655f 100644 --- a/apps/cli/src/render/tui/home-app.tsx +++ b/apps/cli/src/render/tui/home-app.tsx @@ -12,6 +12,7 @@ import { HomeView } from './home-view.js'; import type { ReverseSearchState } from './input-history.js'; import type { MentionState } from './mention.js'; import { MentionView } from './mention-view.js'; +import { ModelPickerView } from './model-picker-view.js'; import { PaletteView } from './palette-view.js'; import type { PaletteState } from './palette-reducer.js'; import { colorProps, dimProps } from './projection.js'; @@ -134,11 +135,14 @@ export function RootApp(props: Readonly): ReactElement { cols={size.cols} rows={size.rows} color={color} - paletteOpen={state.palette !== undefined} + paletteOpen={state.palette !== undefined || state.modelPicker !== undefined} /> {state.palette !== undefined && ( )} + {state.modelPicker !== undefined && ( + + )} ); } diff --git a/apps/cli/src/render/tui/home-controller.test.ts b/apps/cli/src/render/tui/home-controller.test.ts index 60847161..b89800fa 100644 --- a/apps/cli/src/render/tui/home-controller.test.ts +++ b/apps/cli/src/render/tui/home-controller.test.ts @@ -8,8 +8,10 @@ import { createHomeController, type HomeChatSession, type HomeController, + type HomeModelsPort, } from './home-controller.js'; import type { UserCommandOutcome } from '@relavium/core'; +import type { ModelCatalogEntry } from '@relavium/llm'; import type { MentionReader } from './mention.js'; @@ -1578,3 +1580,171 @@ describe('createHomeController (2.5.B lifecycle / ADR-0054)', () => { }); }); }); + +/* ------------------------------------------------------------------------------------------------ * + * The `/models` picker (2.5.G S7 / ADR-0064 §10) — HOME-ONLY, opened via the Home `/` palette. + * ------------------------------------------------------------------------------------------------ */ + +const CTRL_R = { ctrl: true } as const; + +/** A merged catalog entry with sensible defaults for the picker port fake. */ +function pickerEntry( + partial: Partial & Pick, +): ModelCatalogEntry { + return { + provider: 'anthropic', + displayName: partial.modelId, + pricingSource: 'registry', + priceKnown: true, + available: true, + deprecated: false, + ...partial, + }; +} + +/** A controllable {@link HomeModelsPort} fake: spies on every method; `load` returns the given entries/stamp. */ +function makeModelsPort(opts: { + entries?: readonly ModelCatalogEntry[]; + refreshedAt?: number; + currentDefault?: string; + refreshIfStale?: () => Promise>>; + refresh?: () => Promise>>; + writeDefault?: (modelId: string) => void; +} = {}): { + port: HomeModelsPort; + load: ReturnType; + refreshIfStale: ReturnType; + refresh: ReturnType; + writeDefault: ReturnType; +} { + const entries = opts.entries ?? [pickerEntry({ modelId: 'a' }), pickerEntry({ modelId: 'b' })]; + const load = vi.fn(() => ({ entries, refreshedAt: opts.refreshedAt })); + const refreshIfStale = vi.fn(opts.refreshIfStale ?? (() => Promise.resolve(undefined))); + const refresh = vi.fn(opts.refresh ?? (() => Promise.resolve({ providers: [] }))); + const writeDefault = vi.fn(opts.writeDefault ?? (() => undefined)); + const port: HomeModelsPort = { + load, + refreshIfStale, + refresh, + currentDefault: () => opts.currentDefault, + writeDefault, + }; + return { port, load, refreshIfStale, refresh, writeDefault }; +} + +describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => { + const EMPTY_HOME: HomeSnapshot = { + attention: { gates: [], failedRuns: [] }, + recentSessions: [], + recentRuns: [], + recentAgents: [], + isEmpty: true, + }; + const homeStore: HomeStore = { read: () => EMPTY_HOME }; + + /** Build a Home controller with the given models port, then open the picker via the `/` palette (the real path). */ + function openPicker(port: HomeModelsPort): HomeController { + const c = createHomeController({ + doctorProbes: STUB_DOCTOR_PROBES, + startChat: vi.fn(), + homeStore, + onExit: vi.fn(), + onError: vi.fn(), + models: port, + }); + c.handleKey('/', {}); // open the palette + type(c, 'models'); // filter HOME_PALETTE_COMMANDS → [/models] + c.handleKey('', ENTER); // run the highlighted /models → homeReplCtx.openModels → the picker opens + return c; + } + + it('opens over the merged catalog and kicks a TTL-bounded background refresh', async () => { + const { port, load, refreshIfStale } = makeModelsPort({ refreshedAt: 1000 }); + const c = openPicker(port); + const picker = c.getSnapshot().modelPicker; + expect(picker?.entries.map((e) => e.modelId)).toEqual(['a', 'b']); + expect(picker?.refreshedAt).toBe(1000); + expect(refreshIfStale).toHaveBeenCalledTimes(1); // the auto background refresh (ADR-0064 §5c) + await flush(); + // Nothing was stale (refreshIfStale → undefined): the spinner clears, the cache stands. + expect(c.getSnapshot().modelPicker?.loading).toBe(false); + expect(load).toHaveBeenCalled(); + }); + + it('selecting a model writes the NEXT session default, closes the picker, and confirms in the notice', async () => { + const { port, writeDefault } = makeModelsPort({ + entries: [pickerEntry({ modelId: 'claude-x', displayName: 'Claude X' })], + }); + const c = openPicker(port); + await flush(); + c.handleKey('', ENTER); // accept the selected (available) model + expect(writeDefault).toHaveBeenCalledWith('claude-x'); + expect(c.getSnapshot().modelPicker).toBeUndefined(); // the picker closed + expect(c.getSnapshot().notice).toContain('Claude X'); // the confirmation names the model + expect(c.getSnapshot().notice).toContain('next chat session'); // it is a NEXT-session action, not a live reseat + }); + + it('a DIMMED (unavailable) model is non-selectable: Enter shows a banner, never a write', async () => { + const { port, writeDefault } = makeModelsPort({ + entries: [pickerEntry({ modelId: 'gone', displayName: 'Gone', available: false })], + }); + const c = openPicker(port); + await flush(); + c.handleKey('', ENTER); + expect(writeDefault).not.toHaveBeenCalled(); // never write an unusable default + expect(c.getSnapshot().modelPicker).toBeDefined(); // stays open + expect(c.getSnapshot().modelPicker?.banner).toContain('Gone'); // an actionable banner + }); + + it('Esc closes the picker without writing a default', async () => { + const { port, writeDefault } = makeModelsPort(); + const c = openPicker(port); + await flush(); + c.handleKey('', { escape: true }); + expect(c.getSnapshot().modelPicker).toBeUndefined(); + expect(writeDefault).not.toHaveBeenCalled(); + }); + + it('Ctrl+R runs an unbounded refresh; a per-provider failure surfaces a secret-free banner', async () => { + const { port, refresh } = makeModelsPort({ + refresh: () => + Promise.resolve({ providers: [{ provider: 'openai', status: 'failed', error: 'redacted' }] }), + }); + const c = openPicker(port); + await flush(); + c.handleKey('r', CTRL_R); + expect(refresh).toHaveBeenCalledTimes(1); + await flush(); + const picker = c.getSnapshot().modelPicker; + expect(picker?.loading).toBe(false); + expect(picker?.banner).toContain('openai'); // names the failed provider, not the (redacted) error body + expect(picker?.banner).not.toContain('redacted'); + }); + + it('the picker owns the keyboard: typing filters, it does not edit the Home buffer', async () => { + const { port } = makeModelsPort({ + entries: [pickerEntry({ modelId: 'alpha' }), pickerEntry({ modelId: 'beta' })], + }); + const c = openPicker(port); + await flush(); + c.handleKey('b', {}); // a printable char → extends the picker filter, NOT the Home prompt + expect(c.getSnapshot().modelPicker?.filter).toBe('b'); + expect(c.getSnapshot().input.text).toBe(''); // the Home buffer behind the overlay is untouched + }); + + it('degrades to an honest notice when no models port is wired (a test/partial host)', () => { + const c = createHomeController({ + doctorProbes: STUB_DOCTOR_PROBES, + startChat: vi.fn(), + homeStore, + onExit: vi.fn(), + onError: vi.fn(), + // no `models` port + }); + c.handleKey('/', {}); + type(c, 'models'); + c.handleKey('', ENTER); + expect(c.getSnapshot().modelPicker).toBeUndefined(); + expect(c.getSnapshot().notice).toContain('unavailable'); + }); +}); diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index 0e47a438..0d66d191 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -1,10 +1,17 @@ import type { UserCommandOutcome } from '@relavium/core'; +import type { ModelCatalogEntry } from '@relavium/llm'; import { CHAT_PALETTE_COMMANDS, HOME_PALETTE_COMMANDS, type ReplCommandContext, } from '../../commands/repl-commands.js'; +import type { RefreshReport } from '../../engine/model-refresh.js'; +import { + foldModelPickerKey, + partialFailureBanner, + type ModelPickerState, +} from './model-picker.js'; import { nextMode, type ChatMode } from '../../chat/chat-mode.js'; import { clearedNotice } from '../../chat/repl-info.js'; import { formatDoctorReport, runDoctorChecks, type DoctorProbes } from '../../chat/doctor.js'; @@ -149,6 +156,29 @@ export interface HomeControllerState { /** Transient command output in the bare Home — the `/doctor` report (2.5.C S5), rendered below the strip and * cleared on the next edit/submit. Multi-line + secret-free (the doctor formatter sanitizes). `undefined` ⇒ none. */ readonly notice: string | undefined; + /** The open `/models` picker (2.5.G S7, ADR-0064 §10) — `undefined` ⇒ closed. HOME-ONLY (a next-session config + * action); a keyboard-owning overlay like the palette. Opened from the Home palette's `/models`; on selection it + * writes the next session's default (ADR-0063), never rebinding the live session. Mutually exclusive with the palette. */ + readonly modelPicker: ModelPickerState | undefined; +} + +/** + * The Home's model-catalog port (2.5.G S7, ADR-0064) — the I/O the `/models` picker needs, injected by `driveHome` + * (which owns the db handle + the refresh service + the config writer) so the controller stays pure-ish + testable. + * `load` is a sync db read + the pure merge; the refreshes egress (safe here: the Home is the LONG-LIVED process + * the S5 background-refresh constraint requires). `writeDefault` persists the NEXT session's default (ADR-0063). + */ +export interface HomeModelsPort { + /** The merged catalog (all providers) + the newest live-refresh stamp (the freshness badge). Sync read + merge. */ + load: () => { readonly entries: readonly ModelCatalogEntry[]; readonly refreshedAt: number | undefined }; + /** TTL-bounded background refresh (ADR-0064 §5c) — refreshes empty/stale providers; `undefined` when none were. */ + refreshIfStale: () => Promise; + /** Unbounded, user-initiated refresh (Ctrl+R) — every connected provider, per-provider-isolated. Never rejects. */ + refresh: () => Promise; + /** The current `[preferences].default_model` (the picker's `✓` marker), or `undefined` when none is set. */ + currentDefault: () => string | undefined; + /** Persist the chosen model as the next session's default (writeGlobalDefaultModel). Throws `ConfigError` on a bad write. */ + writeDefault: (modelId: string) => void; } export interface HomeControllerDeps { @@ -168,6 +198,9 @@ export interface HomeControllerDeps { readonly boundTeardown?: (teardown: Promise) => Promise; /** The `/doctor` probes (2.5.C S5) — the Home palette's `/doctor` runs the fast tier over these into `notice`. */ readonly doctorProbes: DoctorProbes; + /** The `/models` catalog port (2.5.G S7, ADR-0064) — absent ⇒ `/models` degrades to an honest "unavailable" + * notice (a test may omit it). Production (`driveHome`) always wires the real db-backed port. */ + readonly models?: HomeModelsPort; } export interface HomeController { @@ -199,6 +232,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { attachments: [], historyEntries: [], notice: undefined, + modelPicker: undefined, }; // Per-session command history for the in-Home chat (2.5.D step 3) — accumulates submitted lines across the Home // process; Up/Down recall, Ctrl+R reverse-searches. Not persisted (a chat-resume starts fresh). @@ -281,6 +315,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { palette: undefined, // a palette left open when /exit ran must not leak into the returned Home search: undefined, // ditto a reverse-search submode mention: undefined, // ditto an `@`-completion submode + modelPicker: undefined, // ditto the `/models` picker (Home-only, so never open here — reset for hygiene) shellBusy: false, // a `!`-command in flight when the chat ended must not leave the returned Home gated submitBusy: false, // ditto a submit/compaction in flight — the returned Home must not be left gated shellCommand: undefined, @@ -340,6 +375,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { palette: undefined, search: undefined, mention: undefined, + modelPicker: undefined, shellBusy: false, submitBusy: false, // the swap is done — un-gate the fresh chat shellCommand: undefined, @@ -410,6 +446,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { palette: undefined, search: undefined, mention: undefined, + modelPicker: undefined, historyEntries: history.entries, }); // Track the in-flight build so a signal (or a mid-build exit) during `loading` can reclaim its just-spawned @@ -439,6 +476,97 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { ); }; + // The `/models` picker (2.5.G S7, ADR-0064 §10) — a keyboard-owning overlay opened from the Home palette. The db + // read + the merge are synchronous (`port.load()`); a refresh egresses and lands its result ONLY if the picker is + // still open (a since-closed picker drops it — mirrors loadMentions' stale guard). Home-only: it can be open only + // in `mode: 'home'` (the picker branch in handleKey routes before handleHomeKey), so no chat/session race applies. + const applyRefreshResult = (report: RefreshReport | undefined): void => { + const open = state.modelPicker; + if (open === undefined || deps.models === undefined) return; // the picker closed while the refresh was in flight + const { entries, refreshedAt } = deps.models.load(); + const failed = report?.providers.filter((p) => p.status === 'failed').map((p) => p.provider) ?? []; + // Keep the user's filter/selection; the view clamps a selection left past the (possibly shrunk) list's end. + set({ modelPicker: { ...open, entries, refreshedAt, loading: false, banner: partialFailureBanner(failed) } }); + }; + const runPickerRefresh = (refresh: () => Promise): void => { + const open = state.modelPicker; + if (open === undefined) return; + set({ modelPicker: { ...open, loading: true } }); + void refresh().then(applyRefreshResult, () => { + // refresh()/refreshIfStale() never reject (per-provider isolation), but stay defensive: just drop the spinner. + const cur = state.modelPicker; + if (cur !== undefined) set({ modelPicker: { ...cur, loading: false } }); + }); + }; + const openModelPicker = (): void => { + const port = deps.models; // capture the narrowed port so the refresh closure needs no non-null assertion + if (port === undefined) { + set({ notice: '/models is unavailable here.' }); // defensive — production always wires the port + return; + } + const { entries, refreshedAt } = port.load(); + set({ + notice: undefined, // opening the picker clears any stale /doctor report behind it + modelPicker: { + entries, + filter: '', + selected: 0, + loading: false, + currentDefault: port.currentDefault(), + refreshedAt, + banner: undefined, + }, + }); + // Render the cache immediately (above), then kick a TTL-bounded background refresh (ADR-0064 §5c) — the Home is + // the long-lived process the S5 background constraint requires. An empty/stale cache repopulates as it resolves. + runPickerRefresh(() => port.refreshIfStale()); + }; + // Persist the chosen model as the NEXT session's default (ADR-0063) — it does NOT rebind a live session (there is + // none in the bare Home). A write fault (a malformed existing config) keeps the picker open with a static, + // secret-free banner rather than crashing the Home; success closes the picker and confirms in the strip notice. + const acceptModel = (modelId: string, displayName: string): void => { + if (deps.models === undefined) return; + try { + deps.models.writeDefault(modelId); + } catch { + const open = state.modelPicker; + if (open !== undefined) { + set({ modelPicker: { ...open, banner: 'could not save — check ~/.relavium/config.toml' } }); + } + return; + } + set({ + modelPicker: undefined, + notice: `Default model set to ${displayName} — applies to your next chat session.`, + }); + }; + // The open `/models` picker owns every key (2.5.G S7) — parity with routeMentionKey. Returns whether the key was + // consumed (the overlay was open). A DIMMED (unavailable-on-your-key) model is non-selectable (ADR §6): accepting + // one shows a transient banner rather than writing an unusable default. + const routeModelPickerKey = (input: string, key: ChatKey): boolean => { + const open = state.modelPicker; + if (open === undefined) return false; + const step = foldModelPickerKey(input, key, open); + switch (step.kind) { + case 'close': + set({ modelPicker: undefined }); + break; + case 'accept': + acceptModel(step.modelId, step.displayName); + break; + case 'blocked': + set({ modelPicker: { ...open, banner: `${step.displayName} is not available on your key — pick another` } }); + break; + case 'refresh': + runPickerRefresh(() => deps.models?.refresh() ?? Promise.resolve(undefined)); + break; + case 'state': + set({ modelPicker: step.state }); + break; + } + return true; + }; + // Drive the open `/` palette (2.5.C S3b): fold the keystroke, then apply — keep open with new state, run the // highlighted command by submitting its slash line through the SAME chat dispatch, or close. Ctrl-C closes it // (a gentle escape back to the prompt — never trapping the user). @@ -464,6 +592,10 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { clearSession: () => set({ notice: 'No active conversation to clear — type a message to start one.' }), + // `/models` (2.5.G S7, ADR-0064 §10) IS a real Home capability (availableIn ['home']): open the in-tree picker + // over the merged catalog. Unlike the inert chat-only noops above, this wires the live picker. + openModels: () => openModelPicker(), + runDoctor: async (deep) => { if (exiting) return; const runId = (doctorRunId += 1); // a new run; a prompt edit/submit or a later run bumps this, invalidating us @@ -923,7 +1055,8 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { !state.submitBusy && // ditto a submit/compaction in flight (ADR-0062) — input is gated during it state.palette === undefined && state.search === undefined && - state.mention === undefined; + state.mention === undefined && + state.modelPicker === undefined; // a paste while the picker owns the keyboard must not leak into the buffer const pasted = input.replace(/\r\n?/g, '\n'); if (pasted.length > 0 && editable) { // Match the typed-edit path: appending clears any stale `/doctor` report + invalidates an in-flight run. @@ -939,6 +1072,12 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { handlePaletteKey(input, key); return; } + // The `/models` picker (2.5.G S7) likewise owns every key while open — a Home-only overlay, mutually exclusive + // with the palette (the palette closes before running `/models`), so this can be reached only in `mode: 'home'`. + if (state.modelPicker !== undefined) { + routeModelPickerKey(input, key); + return; + } if (state.mode === 'chat' && state.session !== undefined) { handleChatKey(state.session, input, key); return; diff --git a/apps/cli/src/render/tui/model-picker-view.test.ts b/apps/cli/src/render/tui/model-picker-view.test.ts new file mode 100644 index 00000000..320d7d71 --- /dev/null +++ b/apps/cli/src/render/tui/model-picker-view.test.ts @@ -0,0 +1,21 @@ +import { describe, expect, it } from 'vitest'; + +import { modelWindow } from './model-picker-view.js'; + +describe('modelWindow — the /models picker scroll window (2.5.G S7)', () => { + it('shows the whole list when it fits (≤ the window size)', () => { + expect(modelWindow(0, 0)).toEqual({ start: 0, end: 0 }); + expect(modelWindow(5, 3)).toEqual({ start: 0, end: 5 }); + expect(modelWindow(8, 7)).toEqual({ start: 0, end: 8 }); // exactly the window ⇒ no scroll + }); + + it('scrolls a window of 8 around the selection, clamped to the list bounds', () => { + // Near the top: anchored at 0 (never a negative start). + expect(modelWindow(30, 0)).toEqual({ start: 0, end: 8 }); + expect(modelWindow(30, 3)).toEqual({ start: 0, end: 8 }); // 3 - 4 clamps to 0 + // In the middle: the selection sits floor(8/2) = 4 from the window start. + expect(modelWindow(30, 10)).toEqual({ start: 6, end: 14 }); + // Near the bottom: pinned to the last 8 (never past the end). + expect(modelWindow(30, 29)).toEqual({ start: 22, end: 30 }); + }); +}); diff --git a/apps/cli/src/render/tui/model-picker-view.tsx b/apps/cli/src/render/tui/model-picker-view.tsx new file mode 100644 index 00000000..891b7d97 --- /dev/null +++ b/apps/cli/src/render/tui/model-picker-view.tsx @@ -0,0 +1,123 @@ +import { Box, Text } from 'ink'; +import type { ReactElement, ReactNode } from 'react'; + +import { sanitizeInline } from './chat-projection.js'; +import { + formatContextWindow, + formatModelPrice, + formatRefreshedBadge, + visibleModels, + type ModelPickerState, +} from './model-picker.js'; +import { colorProps, dimProps } from './projection.js'; + +/** The most model rows shown at once — a large multi-provider catalog scrolls a window around the selection rather + * than flooding the terminal (parity with the mention overlay's {@link MENTION_WINDOW}). */ +const MODEL_WINDOW = 8; + +/** + * The `/models` picker overlay (2.5.G S7, [ADR-0064](../../../../../docs/decisions/0064-live-model-catalog.md) §10) + * — a PURE ink view over the merged, filtered catalog. It owns NO `useInput`: the Home's `RootApp` (the single + * raw-mode owner) routes keys to `foldModelPickerKey` and re-renders this from the resulting {@link ModelPickerState}. + * Every free-form field (each display name, the filter echo, the freshness/partial-failure banners) is sanitized at + * this display boundary, so a provider-controlled model name (or a crafted refresh error) can neither forge a row + * nor inject a terminal escape. Renders the first-class UX the ADR mandates: pricing, a dimmed "unavailable on your + * key" row, a `deprecated` flag, an unpriced "cost cap will not apply" hint, a loading spinner, a per-provider + * partial-failure banner, and a "last updated" freshness badge. + */ +export interface ModelPickerViewProps { + readonly state: ModelPickerState; + readonly color: boolean; + /** The injected clock (epoch-ms) for the freshness badge — threaded from `RootApp` so the relative age is testable. */ + readonly nowMs: number; +} + +/** The window of row indices to render around `selected` (a `[start, end)` slice), keeping the selection visible and + * never exceeding {@link MODEL_WINDOW} rows. Pure so the scroll math is unit-checkable. */ +export function modelWindow(count: number, selected: number): { start: number; end: number } { + if (count <= MODEL_WINDOW) return { start: 0, end: count }; + const half = Math.floor(MODEL_WINDOW / 2); + const start = Math.max(0, Math.min(selected - half, count - MODEL_WINDOW)); + return { start, end: start + MODEL_WINDOW }; +} + +export function ModelPickerView(props: Readonly): ReactElement { + const { state, color, nowMs } = props; + const visible = visibleModels(state); + // Clamp the highlight for display — a refresh can shrink the list under a `selected` past the new end until the + // next keystroke re-clamps (foldModelPickerKey clamps on move). + const selected = visible.length === 0 ? 0 : Math.max(0, Math.min(state.selected, visible.length - 1)); + const { start, end } = modelWindow(visible.length, selected); + const windowed = visible.slice(start, end); + const badge = `${formatRefreshedBadge(state.refreshedAt, nowMs)}${state.loading ? ' · refreshing…' : ''}`; + + const renderBody = (): ReactNode => { + if (state.loading && visible.length === 0) { + return ( + + loading models… + + ); + } + if (state.entries.length === 0) { + return ( + + no models yet — add a provider key with `relavium provider add`, then Ctrl+R to refresh + + ); + } + if (visible.length === 0) { + return ( + + no model matches + + ); + } + return windowed.map((entry, index) => { + const isSelected = start + index === selected; + const isDefault = entry.modelId === state.currentDefault; + const ctx = formatContextWindow(entry.contextWindowTokens); + const parts = [ + sanitizeInline(entry.displayName), + entry.provider, + ...(ctx.length > 0 ? [ctx] : []), + formatModelPrice(entry), + ...(entry.available ? [] : ['unavailable on your key']), + ...(entry.deprecated ? ['deprecated'] : []), + ]; + // Selected → cyan (highlight wins for visibility); else an unavailable/deprecated row is dimmed; else default. + const rowColor = isSelected + ? colorProps(color, 'cyan') + : !entry.available || entry.deprecated + ? dimProps(color) + : {}; + return ( + + {`${isSelected ? '›' : ' '} ${isDefault ? '✓' : ' '} ${parts.join(' · ')}`} + + ); + }); + }; + + return ( + + + Set your default model + {` · ${badge}`} + + {state.banner !== undefined && ( + + {sanitizeInline(state.banner)} + + )} + + {'model: '} + {sanitizeInline(state.filter)} + + {renderBody()} + + ↑/↓ select · Enter set default · Ctrl+R refresh · Esc cancel + + + ); +} diff --git a/apps/cli/src/render/tui/model-picker.test.ts b/apps/cli/src/render/tui/model-picker.test.ts new file mode 100644 index 00000000..b23bc23b --- /dev/null +++ b/apps/cli/src/render/tui/model-picker.test.ts @@ -0,0 +1,186 @@ +import type { ModelCatalogEntry } from '@relavium/llm'; +import { describe, expect, it } from 'vitest'; + +import { + foldModelPickerKey, + formatContextWindow, + formatModelPrice, + formatRefreshedBadge, + partialFailureBanner, + visibleModels, + type ModelPickerState, +} from './model-picker.js'; + +/** A merged catalog entry with sensible defaults; override what a case cares about. */ +function entry(partial: Partial & Pick): ModelCatalogEntry { + return { + provider: 'anthropic', + displayName: partial.modelId, + pricingSource: 'registry', + priceKnown: true, + available: true, + deprecated: false, + ...partial, + }; +} + +function state(partial: Partial = {}): ModelPickerState { + return { + entries: [entry({ modelId: 'a' }), entry({ modelId: 'b' }), entry({ modelId: 'c' })], + filter: '', + selected: 0, + loading: false, + currentDefault: undefined, + refreshedAt: undefined, + banner: undefined, + ...partial, + }; +} + +describe('foldModelPickerKey', () => { + it('Esc and Ctrl-C close the picker (no write)', () => { + expect(foldModelPickerKey('', { escape: true }, state()).kind).toBe('close'); + expect(foldModelPickerKey('c', { ctrl: true }, state()).kind).toBe('close'); + }); + + it('Ctrl+R refreshes; a bare "r" extends the filter instead', () => { + expect(foldModelPickerKey('r', { ctrl: true }, state()).kind).toBe('refresh'); + const typed = foldModelPickerKey('r', {}, state()); + expect(typed).toEqual({ kind: 'state', state: state({ filter: 'r', selected: 0 }) }); + }); + + it('arrows move the selection, clamped to the visible list', () => { + const down = foldModelPickerKey('', { downArrow: true }, state({ selected: 0 })); + expect(down).toEqual({ kind: 'state', state: state({ selected: 1 }) }); + // Clamp at the top: Up from 0 stays 0. + expect(foldModelPickerKey('', { upArrow: true }, state({ selected: 0 }))).toEqual({ + kind: 'state', + state: state({ selected: 0 }), + }); + // Clamp at the bottom: Down from the last visible index stays there. + expect(foldModelPickerKey('', { downArrow: true }, state({ selected: 2 }))).toEqual({ + kind: 'state', + state: state({ selected: 2 }), + }); + }); + + it('Enter on an AVAILABLE model accepts it (modelId + displayName)', () => { + const s = state({ entries: [entry({ modelId: 'x', displayName: 'Model X' })], selected: 0 }); + expect(foldModelPickerKey('', { return: true }, s)).toEqual({ + kind: 'accept', + modelId: 'x', + displayName: 'Model X', + }); + }); + + it('Enter on a DIMMED (unavailable) model is BLOCKED, never an accept (ADR §6)', () => { + const s = state({ + entries: [entry({ modelId: 'x', displayName: 'Model X', available: false })], + selected: 0, + }); + expect(foldModelPickerKey('', { return: true }, s)).toEqual({ kind: 'blocked', displayName: 'Model X' }); + }); + + it('Enter on an EMPTY (over-filtered) list is a gentle close, not a crash', () => { + const s = state({ filter: 'zzzz' }); // matches nothing + expect(visibleModels(s)).toHaveLength(0); + expect(foldModelPickerKey('', { return: true }, s).kind).toBe('close'); + }); + + it('a printable char extends the filter and resets the selection; backspace trims it', () => { + const typed = foldModelPickerKey('b', {}, state({ selected: 2 })); + expect(typed).toEqual({ kind: 'state', state: state({ filter: 'b', selected: 0 }) }); + const trimmed = foldModelPickerKey('', { backspace: true }, state({ filter: 'ab', selected: 1 })); + expect(trimmed).toEqual({ kind: 'state', state: state({ filter: 'a', selected: 0 }) }); + // Backspace on an EMPTY filter is inert (Esc cancels; backspace never closes) — stays open, unchanged. + expect(foldModelPickerKey('', { backspace: true }, state({ filter: '' }))).toEqual({ + kind: 'state', + state: state({ filter: '' }), + }); + }); + + it('drops a multi-char paste blob (only a single code point extends the filter)', () => { + const blob = foldModelPickerKey('pasted text', {}, state()); + expect(blob).toEqual({ kind: 'state', state: state() }); // unchanged + }); +}); + +describe('visibleModels', () => { + it('filters by display name, model id, OR provider (case-insensitive)', () => { + const s = state({ + entries: [ + entry({ modelId: 'claude-opus', displayName: 'Opus', provider: 'anthropic' }), + entry({ modelId: 'gpt-x', displayName: 'GPT', provider: 'openai' }), + ], + }); + expect(visibleModels({ ...s, filter: 'opus' }).map((e) => e.modelId)).toEqual(['claude-opus']); + expect(visibleModels({ ...s, filter: 'OPENAI' }).map((e) => e.modelId)).toEqual(['gpt-x']); + expect(visibleModels({ ...s, filter: 'gpt-x' }).map((e) => e.modelId)).toEqual(['gpt-x']); + expect(visibleModels({ ...s, filter: '' })).toHaveLength(2); // empty filter shows all + }); +}); + +describe('display formatters', () => { + it('formats the price, or the "cost cap will not apply" hint when unknown', () => { + const priced = entry({ + modelId: 'x', + pricing: { + provider: 'anthropic', + nativeId: 'x', + displayName: 'X', + contextWindowTokens: 200_000, + maxOutputTokens: 64_000, + inputPerMtokMicrocents: 300_000_000, // $3.00 + outputPerMtokMicrocents: 1_500_000_000, // $15.00 + cachedInputPerMtokMicrocents: 0, + }, + priceKnown: true, + }); + expect(formatModelPrice(priced)).toBe('$3/$15 per Mtok'); + const unpriced = entry({ modelId: 'y', priceKnown: false, pricingSource: 'none' }); + expect(formatModelPrice(unpriced)).toBe('no price — cost cap will not apply'); + }); + + it('formats a fractional price to 2 decimals', () => { + const priced = entry({ + modelId: 'z', + pricing: { + provider: 'openai', + nativeId: 'z', + displayName: 'Z', + contextWindowTokens: 1000, + maxOutputTokens: 100, + inputPerMtokMicrocents: 15_000_000, // $0.15 + outputPerMtokMicrocents: 125_000_000, // $1.25 + cachedInputPerMtokMicrocents: 0, + }, + }); + expect(formatModelPrice(priced)).toBe('$0.15/$1.25 per Mtok'); + }); + + it('formats the context window compactly (K / M), or empty when unknown', () => { + expect(formatContextWindow(200_000)).toBe('200K ctx'); + expect(formatContextWindow(1_000_000)).toBe('1M ctx'); + expect(formatContextWindow(1_500_000)).toBe('1.5M ctx'); + expect(formatContextWindow(undefined)).toBe(''); + expect(formatContextWindow(0)).toBe(''); // the "unknown" sentinel reads as empty, never "0 ctx" + }); + + it('formats the freshness badge from the newest live-refresh stamp', () => { + expect(formatRefreshedBadge(undefined, 1_000_000)).toBe('never refreshed'); + expect(formatRefreshedBadge(1_000_000, 1_000_000)).toBe('updated just now'); + expect(formatRefreshedBadge(0, 5 * 60_000)).toBe('updated 5m ago'); + expect(formatRefreshedBadge(0, 3 * 60 * 60_000)).toBe('updated 3h ago'); + expect(formatRefreshedBadge(0, 2 * 24 * 60 * 60_000)).toBe('updated 2d ago'); + // A future/negative delta clamps to "just now", never a negative age. + expect(formatRefreshedBadge(2_000_000, 1_000_000)).toBe('updated just now'); + }); + + it('summarizes partial refresh failures into a secret-free banner (or undefined when clean)', () => { + expect(partialFailureBanner([])).toBeUndefined(); + expect(partialFailureBanner(['openai'])).toBe('couldn\'t refresh openai — showing last-known models'); + expect(partialFailureBanner(['openai', 'gemini'])).toBe( + 'couldn\'t refresh openai, gemini — showing last-known models', + ); + }); +}); diff --git a/apps/cli/src/render/tui/model-picker.ts b/apps/cli/src/render/tui/model-picker.ts new file mode 100644 index 00000000..16d16de1 --- /dev/null +++ b/apps/cli/src/render/tui/model-picker.ts @@ -0,0 +1,192 @@ +import type { ModelCatalogEntry } from '@relavium/llm'; + +import { dropLastCodePoint } from './chat-input.js'; + +/** + * The `/models` picker submode (workstream **2.5.G S7**, [ADR-0064](../../../../../docs/decisions/0064-live-model-catalog.md) §10) + * — a keyboard-owning overlay (like the `/` palette + the `@`-mention completion) that lists the MERGED model + * catalog and, on selection, writes the NEXT session's default model ([ADR-0063](../../../../../docs/decisions/0063-cli-config-write-contract.md)); + * it does NOT rebind the live session (that is the Phase-2.6 `/models` reseat, ADR-0059). Home-only (`availableIn: ['home']`). + * + * The PURE model — state + fold + the display formatters — lives here; the ink view ({@link model-picker-view.tsx}) + * renders it and the Home controller routes keys + does the async db/refresh/write I/O (mirroring the mention submode). + * A DIMMED (unavailable-on-your-key) or a deprecated model is still shown (ADR-0064 §6/§7: dim/flag, never hide) but + * a dimmed model is **non-selectable** — accepting one yields a `blocked` step, not a write. + */ + +/** + * The picker submode state. `entries` is the whole merged catalog (all providers, already deterministically + * ordered by the merge); `filter` narrows it (case-insensitive, over display name / model id / provider); + * `selected` indexes the VISIBLE (filtered) subset; `loading` is `true` while a refresh is in flight (the spinner); + * `currentDefault` is the `[preferences].default_model` marked `✓`; `refreshedAt` is the newest live-refresh + * epoch-ms (the "last updated" badge); `banner` is a secret-free per-provider partial-failure notice (or undefined). + */ +export interface ModelPickerState { + readonly entries: readonly ModelCatalogEntry[]; + readonly filter: string; + readonly selected: number; + readonly loading: boolean; + readonly currentDefault: string | undefined; + readonly refreshedAt: number | undefined; + readonly banner: string | undefined; +} + +/** The minimal key fields the picker fold reads (a structural subset of ink's `Key`). */ +export interface ModelPickerKey { + readonly ctrl?: boolean; + readonly meta?: boolean; + readonly shift?: boolean; + readonly escape?: boolean; + readonly return?: boolean; + readonly backspace?: boolean; + readonly delete?: boolean; + readonly upArrow?: boolean; + readonly downArrow?: boolean; +} + +/** What a keystroke does to the open picker. */ +export type ModelPickerStep = + | { readonly kind: 'close' } // Esc / Ctrl-C — cancel without writing a default + | { readonly kind: 'accept'; readonly modelId: string; readonly displayName: string } // set the default + | { readonly kind: 'blocked'; readonly displayName: string } // a dimmed/unavailable model — non-selectable (ADR §6) + | { readonly kind: 'refresh' } // Ctrl+R — force a live re-fetch of every connected provider + | { readonly kind: 'state'; readonly state: ModelPickerState }; + +/** The visible entries — those whose display name / model id / provider contains `filter` (case-insensitive); + * order is the merge's (deterministic). An empty filter shows everything. */ +export function visibleModels(state: ModelPickerState): readonly ModelCatalogEntry[] { + if (state.filter.length === 0) return state.entries; + const needle = state.filter.toLowerCase(); + return state.entries.filter( + (entry) => + entry.displayName.toLowerCase().includes(needle) || + entry.modelId.toLowerCase().includes(needle) || + entry.provider.toLowerCase().includes(needle), + ); +} + +/** Clamp a selection index to `0..count-1` (or 0 when the list is empty). */ +function clampSelection(index: number, count: number): number { + if (count <= 0) return 0; + return Math.max(0, Math.min(index, count - 1)); +} + +/** `↑`/`↓` move the selection (clamped to the visible list); `undefined` when the key is not an arrow. */ +function foldArrow( + key: ModelPickerKey, + state: ModelPickerState, + visibleCount: number, +): ModelPickerStep | undefined { + if (key.upArrow === true) { + return { kind: 'state', state: { ...state, selected: clampSelection(state.selected - 1, visibleCount) } }; + } + if (key.downArrow === true) { + return { kind: 'state', state: { ...state, selected: clampSelection(state.selected + 1, visibleCount) } }; + } + return undefined; +} + +/** + * Fold one keystroke into the open picker (the keyboard-owning contract, mirroring the mention submode): + * `Esc`/`Ctrl-C` cancels (no write); `Ctrl+R` refreshes; `↑`/`↓` move; `Enter` accepts the selected model — a + * DIMMED (unavailable) model yields `blocked` (non-selectable, ADR §6), an empty list closes; backspace trims the + * filter; a single printable code point extends the filter (a multi-char paste blob is dropped, matching the other + * submodes); every other key stays open. + */ +export function foldModelPickerKey( + char: string, + key: ModelPickerKey, + state: ModelPickerState, +): ModelPickerStep { + // Esc / Ctrl-C cancels the picker (nothing is written — a cancel never changes the default). + if (key.escape === true || (key.ctrl === true && char === 'c')) { + return { kind: 'close' }; + } + // Ctrl+R forces a live refresh (distinct from the auto TTL refresh on open). `r` alone extends the filter. + if (key.ctrl === true && char === 'r') { + return { kind: 'refresh' }; + } + const visible = visibleModels(state); + const arrow = foldArrow(key, state, visible.length); + if (arrow !== undefined) return arrow; + if (key.return === true) { + const chosen = visible[clampSelection(state.selected, visible.length)]; + if (chosen === undefined) return { kind: 'close' }; // an empty list — Enter is a gentle cancel + if (!chosen.available) return { kind: 'blocked', displayName: chosen.displayName }; + return { kind: 'accept', modelId: chosen.modelId, displayName: chosen.displayName }; + } + if (key.backspace === true || key.delete === true) { + if (state.filter.length === 0) return { kind: 'state', state }; // nothing to trim (Esc cancels; backspace is inert) + // Trim by whole CODE POINT so backspacing an astral char removes it whole (no lone surrogate) — same discipline + // as the other submodes' `dropLastCodePoint`. + return { kind: 'state', state: { ...state, filter: dropLastCodePoint(state.filter), selected: 0 } }; + } + // A single printable code point extends the filter (a multi-char paste blob is dropped); any other key stays open. + if ([...char].length === 1 && key.ctrl !== true && key.meta !== true) { + return { kind: 'state', state: { ...state, filter: state.filter + char, selected: 0 } }; + } + return { kind: 'state', state }; +} + +/* -------------------------------------------------------------------------------------------------- * + * Pure display formatters (unit-tested; the ink view is not render-tested, per the repo convention). + * -------------------------------------------------------------------------------------------------- */ + +const MICROCENTS_PER_USD = 100_000_000; // 1 USD = 1e8 micro-cents (pricing.ts) + +/** A compact USD amount from integer micro-cents-per-Mtok: whole dollars drop the decimals, else up to 2 dp + * (`$3`, `$0.15`, `$1.25`). No float pricing is stored; this is display-only. */ +function usdPerMtok(microcents: number): string { + const usd = microcents / MICROCENTS_PER_USD; + const rounded = Math.round(usd * 100) / 100; + return Number.isInteger(rounded) ? `$${rounded}` : `$${rounded.toFixed(2)}`; +} + +/** + * The per-row price label: `$in/$out per Mtok` from the effective (static/user) pricing, or the ADR-0064 §6 + * "cost cap will not apply" hint when the price is unknown (`priceKnown === false`) — the picker's honest + * cost-governance signal that a session cap cannot bound this model. + */ +export function formatModelPrice(entry: ModelCatalogEntry): string { + if (!entry.priceKnown || entry.pricing === undefined) return 'no price — cost cap will not apply'; + const { inputPerMtokMicrocents, outputPerMtokMicrocents } = entry.pricing; + return `${usdPerMtok(inputPerMtokMicrocents)}/${usdPerMtok(outputPerMtokMicrocents)} per Mtok`; +} + +/** A compact context-window label (`200K ctx`, `1M ctx`), or `''` when the window is unknown. */ +export function formatContextWindow(tokens: number | undefined): string { + if (tokens === undefined || tokens <= 0) return ''; + if (tokens >= 1_000_000) { + const millions = tokens / 1_000_000; + return `${Number.isInteger(millions) ? millions : millions.toFixed(1)}M ctx`; + } + if (tokens >= 1000) return `${Math.round(tokens / 1000)}K ctx`; + return `${tokens} ctx`; +} + +/** The "last updated" freshness badge from the newest live-refresh stamp — `never refreshed` when absent, else a + * coarse relative age (`just now` / `5m ago` / `3h ago` / `2d ago`). Pure (the caller passes `now`). */ +export function formatRefreshedBadge(refreshedAt: number | undefined, now: number): string { + if (refreshedAt === undefined) return 'never refreshed'; + const ms = Math.max(0, now - refreshedAt); + const minutes = Math.floor(ms / 60_000); + if (minutes < 1) return 'updated just now'; + if (minutes < 60) return `updated ${minutes}m ago`; + const hours = Math.floor(minutes / 60); + if (hours < 24) return `updated ${hours}h ago`; + return `updated ${Math.floor(hours / 24)}d ago`; +} + +/** + * Summarize a refresh report's per-provider failures into a secret-free banner, or `undefined` when every + * considered provider refreshed/was-skipped cleanly. Names the failed providers + the count of others that + * kept last-known rows (ADR-0064 §8: drift/failure is visible, non-fatal). The per-provider `error` strings are + * already seam-redacted; this joins only the provider ids, never an error body. + */ +export function partialFailureBanner( + failedProviders: readonly string[], +): string | undefined { + if (failedProviders.length === 0) return undefined; + const list = failedProviders.join(', '); + return `couldn't refresh ${list} — showing last-known models`; +} diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index fbcd8deb..3f33f3ab 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -151,7 +151,7 @@ The **command manifest** is the one source the **shell** command surfaces derive ### In-REPL slash commands -The interactive `/` palette + slash commands inside the **Home and chat** are a SEPARATE, **curated** surface ([ADR-0056](../../decisions/0056-cli-in-app-slash-command-system-and-manifest.md) amendment, 2.5.C) — the runtime registry is `apps/cli/src/commands/repl-commands.ts` (`REPL_COMMANDS`), the single source for the palette, the `/help` list, and the unknown-slash hint. It surfaces only the commands that make sense in a live REPL — lifecycle (`/exit`, `/cancel`, `/export`, `/clear`), info/discovery (`/help`, `/workflows`, `/cost`, `/doctor`), and — in a chat — `/mode ` to switch the chat mode (2.5.E, [chat-session.md](chat-session.md)) plus the ADR-0062 context commands (`/compact`, `/trim`). The heavy, session-starting shell commands above (`run`, `chat`, `provider`, …) are **never** in-REPL slashes — they stay shell-only (`relavium …`). A bare `/` at an **empty** prompt opens the filterable palette (the footer hint-bar surfaces `/ for commands` exactly there, 2.5.C S6); an unknown slash — or an undeclared argument on a known command (`/exit now`) — prints a sanitized, secret-free hint. A command may declare flags (`/doctor --deep`) or a single positional value (`/mode plan`); the palette runs the bare form, so a flag/value is opt-in by typing it. There is no separate `/shortcuts` command — the palette renders its own nav hints (`↑/↓ · Enter · Esc`) and the footer surfaces `/`, so keys stay discoverable in context. +The interactive `/` palette + slash commands inside the **Home and chat** are a SEPARATE, **curated** surface ([ADR-0056](../../decisions/0056-cli-in-app-slash-command-system-and-manifest.md) amendment, 2.5.C) — the runtime registry is `apps/cli/src/commands/repl-commands.ts` (`REPL_COMMANDS`), the single source for the palette, the `/help` list, and the unknown-slash hint. It surfaces only the commands that make sense in a live REPL — lifecycle (`/exit`, `/cancel`, `/export`, `/clear`), info/discovery (`/help`, `/workflows`, `/cost`, `/doctor`), and — in a chat — `/mode ` to switch the chat mode (2.5.E, [chat-session.md](chat-session.md)) plus the ADR-0062 context commands (`/compact`, `/trim`). In the **Home** (bare `relavium`) — and there only — `/models` opens an in-tree model picker over the merged live/static catalog and, on selection, writes the **next** session's default model ([ADR-0064](../../decisions/0064-live-model-catalog.md) §10, via the [ADR-0063](../../decisions/0063-cli-config-write-contract.md) config writer); it is **home-only** (a next-session config action, deliberately distinct from the Phase-2.6 mid-chat `/models` live reseat, [ADR-0059](../../decisions/0059-cli-mid-session-model-reseat.md)) — typed inside a chat it is rejected with a pointer to the Home. The picker renders the ADR-0064 first-class UX: per-model pricing (an unpriced model shows a "cost cap will not apply" hint), a dimmed non-selectable "unavailable on your key" row, a `deprecated` flag, a loading spinner, a per-provider partial-failure banner, and a "last updated" freshness badge; opening it over an empty/stale cache renders immediately and kicks a background refresh (the long-lived Home is where that is sound), `Ctrl+R` forces a live refresh, and `Esc` cancels. The heavy, session-starting shell commands above (`run`, `chat`, `provider`, …) are **never** in-REPL slashes — they stay shell-only (`relavium …`). A bare `/` at an **empty** prompt opens the filterable palette (the footer hint-bar surfaces `/ for commands` exactly there, 2.5.C S6); an unknown slash — or an undeclared argument on a known command (`/exit now`) — prints a sanitized, secret-free hint. A command may declare flags (`/doctor --deep`) or a single positional value (`/mode plan`); the palette runs the bare form, so a flag/value is opt-in by typing it. There is no separate `/shortcuts` command — the palette renders its own nav hints (`↑/↓ · Enter · Esc`) and the footer surfaces `/`, so keys stay discoverable in context. ### `relavium run` From d88c06d30096fa979918952b08457ea93f361af8 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Sun, 5 Jul 2026 23:57:52 +0300 Subject: [PATCH 19/68] =?UTF-8?q?fix(cli):=202.5.G=20S7=20Opus-review=20?= =?UTF-8?q?=E2=80=94=20picker=20refresh=20epoch=20guard=20+=20honest=20def?= =?UTF-8?q?ault=20notice?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial Opus review found no blocker/leak but two real defects on the /models picker: - Missing epoch/identity guard on the picker refresh (major): the async refresh guarded only on "picker still open", not "still the SAME picker". Open → slow refresh in flight → Esc → reopen → the old refresh resolves and clobbered the FRESH picker with a stale partial-failure banner. Add a monotonic `pickerEpoch` (bumped per open, the `doctorRunId` pattern); `applyRefreshResult` drops a resolve whose generation no longer matches. Also split the one `banner` field into two channels — `banner` (async refresh partial-failure status) vs `hint` (transient user-action feedback: dimmed "not available"/"could not save") — so a completing refresh can never silently wipe a message the user just triggered; a nav/filter keystroke clears the hint. - False "applies to your next chat session" claim (major, scoped): the write targets the GLOBAL [preferences].default_model, but the effective default resolves project → workspace → global (ADR-0063 §1). When a project pins [chat].default_model the write is a silent no-op for the next session, yet the notice claimed success and the ✓ marked the written model. `accept` now reads the freshly-resolved EFFECTIVE default and reports honestly: success only if it actually became the chosen model, else "a project or workspace setting overrides it here". The Home port's `currentDefault` re-reads the resolved config fresh each call (guarded against a mid-session malformed edit) — which also fixes the ✓-marker staleness across a same- session write and a cross-terminal `provider add`. - Projection docstring (minor): acknowledge it cannot emit the merge's present-with-[] case (a provider refreshed-to-empty is indistinguishable from never-refreshed at listAll(), falling back to the ADR §6 safe default) rather than overstating merge-contract fidelity. Tests: the epoch guard (a reopened picker is not clobbered by a prior open's slow refresh), the honest override notice, the write-fault hint, and the blocked→hint (cleared on navigation) / refresh→banner split. Toolchain: lint + typecheck + test (1380) + build all green (the lone media-gc failure is the known pre-existing timing flake; passes on retry). Refs: ADR-0064, ADR-0063 Co-Authored-By: Claude --- apps/cli/src/engine/model-catalog-view.ts | 10 ++- apps/cli/src/home/drive-home.tsx | 27 ++++-- .../src/render/tui/home-controller.test.ts | 87 +++++++++++++++++-- apps/cli/src/render/tui/home-controller.ts | 68 ++++++++++----- apps/cli/src/render/tui/model-picker-view.tsx | 7 +- apps/cli/src/render/tui/model-picker.test.ts | 1 + apps/cli/src/render/tui/model-picker.ts | 10 ++- 7 files changed, 163 insertions(+), 47 deletions(-) diff --git a/apps/cli/src/engine/model-catalog-view.ts b/apps/cli/src/engine/model-catalog-view.ts index 65a78d7c..cb570de9 100644 --- a/apps/cli/src/engine/model-catalog-view.ts +++ b/apps/cli/src/engine/model-catalog-view.ts @@ -63,9 +63,13 @@ function rowToListing(row: ModelCatalogListing): ModelListing { /** * Project the active catalog rows into the merged `/models` view. Partitions the `source='live'` rows into a - * per-`ProviderId` live map (a provider PRESENT in the map — even with no surviving rows — has live data, so its - * static models are availability-checked against the list; a provider ABSENT has no live data, so its static - * models fall back to static presence — ADR-0064 §6), then delegates to the pure merge. A live row whose provider + * per-`ProviderId` live map, then delegates to the pure merge. A provider is added to the live map only when it + * has ≥1 ACTIVE `source='live'` row; a provider with no active live rows (never refreshed, OR refreshed to an + * empty list — `replaceProviderModels` soft-deactivates all its live rows, so the two are indistinguishable at + * this seam) is ABSENT from the map, so its static models fall back to static presence (ADR-0064 §6's "never + * everything unavailable" safe default). This means the merge's `present-with-[]` case (dim all a provider's + * statics) is never produced HERE — the real providers never return an empty list for a valid key, and the safe + * default is preferred over dimming a whole provider on an ambiguous zero-row read. A live row whose provider * UUID resolves to a non-enum slug is dropped (defensive: a mis-keyed or future custom-provider row can never * corrupt an unrelated known provider's entry — the merge has its own cross-provider guard too). */ diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index 04068cab..2efa9d04 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -160,10 +160,22 @@ export async function driveHome(deps: HomeDeps): Promise { knownProviders: KNOWN_PROVIDERS, now, }); - // The `✓`-marked current default: `config.chat.defaultModel` resolves project → workspace → global `[preferences]` - // (ADR-0063 §1). A same-session `/models` write updates the GLOBAL file, so track the last write locally so a - // re-open of the picker marks the just-chosen model (the loaded `config` snapshot is not re-read mid-session). - let chosenDefault = config.chat.defaultModel; + // The `✓`-marked current default is the EFFECTIVE default — `[chat].default_model` resolves project → workspace + // → global `[preferences].default_model` (ADR-0063 §1). It is re-read FRESH from disk each call (not the loaded + // `config` snapshot) so it reflects a same-session `/models` write AND a project/workspace override AND an edit + // from another terminal; a config edited to malformed mid-session degrades to `undefined` rather than crashing + // the picker. Cheap (a few small file reads) and only called on a picker open / accept, never per keystroke. + const readEffectiveDefault = (): string | undefined => { + try { + return loadResolvedConfig({ + cwd: deps.global.cwd, + home: homeDir, + ...(deps.global.configPath === undefined ? {} : { configPath: deps.global.configPath }), + }).config.chat.defaultModel; + } catch { + return undefined; // a mid-session malformed config must not crash the picker + } + }; const models: HomeModelsPort = { load: () => { // Rebuild the UUID→slug map on every load (NOT memoized once like the one-shot dispatch resolver): a refresh @@ -177,11 +189,8 @@ export async function driveHome(deps: HomeDeps): Promise { }, refreshIfStale: () => refreshService.refreshIfStale(), refresh: () => refreshService.refresh(), - currentDefault: () => chosenDefault, - writeDefault: (modelId) => { - writeGlobalDefaultModel(modelId, homeDir); - chosenDefault = modelId; // the ✓ marker follows the write within this long-lived Home session - }, + currentDefault: readEffectiveDefault, + writeDefault: (modelId) => writeGlobalDefaultModel(modelId, homeDir), }; // Build + wire + START a fresh chat session (the controller sends the first message on transition). diff --git a/apps/cli/src/render/tui/home-controller.test.ts b/apps/cli/src/render/tui/home-controller.test.ts index b89800fa..2e41511c 100644 --- a/apps/cli/src/render/tui/home-controller.test.ts +++ b/apps/cli/src/render/tui/home-controller.test.ts @@ -13,6 +13,8 @@ import { import type { UserCommandOutcome } from '@relavium/core'; import type { ModelCatalogEntry } from '@relavium/llm'; +import type { RefreshReport } from '../../engine/model-refresh.js'; + import type { MentionReader } from './mention.js'; // The paste-boundary markers exactly as ink 6.8's input layer surfaces them (the leading ESC is stripped). @@ -1602,14 +1604,19 @@ function pickerEntry( }; } -/** A controllable {@link HomeModelsPort} fake: spies on every method; `load` returns the given entries/stamp. */ +/** + * A controllable {@link HomeModelsPort} fake. `writeDefault` records the written id, and `currentDefault` returns + * it (so `accept` reports success) UNLESS `overrideDefault` is set — which simulates a project/workspace + * `[chat].default_model` shadowing the global write (the effective default stays the override). `writeThrows` + * simulates a config-write fault. + */ function makeModelsPort(opts: { entries?: readonly ModelCatalogEntry[]; refreshedAt?: number; - currentDefault?: string; + overrideDefault?: string; + writeThrows?: boolean; refreshIfStale?: () => Promise>>; refresh?: () => Promise>>; - writeDefault?: (modelId: string) => void; } = {}): { port: HomeModelsPort; load: ReturnType; @@ -1618,15 +1625,20 @@ function makeModelsPort(opts: { writeDefault: ReturnType; } { const entries = opts.entries ?? [pickerEntry({ modelId: 'a' }), pickerEntry({ modelId: 'b' })]; + let written: string | undefined; const load = vi.fn(() => ({ entries, refreshedAt: opts.refreshedAt })); const refreshIfStale = vi.fn(opts.refreshIfStale ?? (() => Promise.resolve(undefined))); const refresh = vi.fn(opts.refresh ?? (() => Promise.resolve({ providers: [] }))); - const writeDefault = vi.fn(opts.writeDefault ?? (() => undefined)); + const writeDefault = vi.fn((modelId: string) => { + if (opts.writeThrows === true) throw new Error('config write failed'); + written = modelId; + }); const port: HomeModelsPort = { load, refreshIfStale, refresh, - currentDefault: () => opts.currentDefault, + // The EFFECTIVE default: the override wins (shadows the global write), else the last written id. + currentDefault: () => opts.overrideDefault ?? written, writeDefault, }; return { port, load, refreshIfStale, refresh, writeDefault }; @@ -1684,7 +1696,35 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => expect(c.getSnapshot().notice).toContain('next chat session'); // it is a NEXT-session action, not a live reseat }); - it('a DIMMED (unavailable) model is non-selectable: Enter shows a banner, never a write', async () => { + it('an honest notice when a project/workspace setting overrides the global write (no false success)', async () => { + // The write lands on the global file, but the effective default stays the project/workspace override, so the + // notice must NOT claim "applies to your next chat session" — it says the override still wins here. + const { port, writeDefault } = makeModelsPort({ + entries: [pickerEntry({ modelId: 'claude-x', displayName: 'Claude X' })], + overrideDefault: 'project-pinned', + }); + const c = openPicker(port); + await flush(); + c.handleKey('', ENTER); + expect(writeDefault).toHaveBeenCalledWith('claude-x'); + expect(c.getSnapshot().notice).toContain('overrides it here'); + expect(c.getSnapshot().notice).not.toContain('next chat session'); // no false claim of effect + }); + + it('a write fault keeps the picker OPEN with a secret-free hint, never crashes the Home', async () => { + const { port } = makeModelsPort({ + entries: [pickerEntry({ modelId: 'claude-x', displayName: 'Claude X' })], + writeThrows: true, + }); + const c = openPicker(port); + await flush(); + c.handleKey('', ENTER); + expect(c.getSnapshot().modelPicker).toBeDefined(); // stays open (recoverable) + expect(c.getSnapshot().modelPicker?.hint).toContain('could not save'); + expect(c.getSnapshot().notice).toBeUndefined(); // no success notice on a failed write + }); + + it('a DIMMED (unavailable) model is non-selectable: Enter shows a HINT, never a write', async () => { const { port, writeDefault } = makeModelsPort({ entries: [pickerEntry({ modelId: 'gone', displayName: 'Gone', available: false })], }); @@ -1693,7 +1733,10 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => c.handleKey('', ENTER); expect(writeDefault).not.toHaveBeenCalled(); // never write an unusable default expect(c.getSnapshot().modelPicker).toBeDefined(); // stays open - expect(c.getSnapshot().modelPicker?.banner).toContain('Gone'); // an actionable banner + expect(c.getSnapshot().modelPicker?.hint).toContain('Gone'); // an actionable action-hint (not the refresh banner) + // A navigation keystroke clears the transient hint (the user has moved on). + c.handleKey('x', {}); // a filter keystroke → 'state' step → hint cleared + expect(c.getSnapshot().modelPicker?.hint).toBeUndefined(); }); it('Esc closes the picker without writing a default', async () => { @@ -1721,6 +1764,36 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => expect(picker?.banner).not.toContain('redacted'); }); + it('a reopened picker is NOT clobbered by a prior open\'s slow refresh (the epoch guard)', async () => { + // Open #1 kicks a SLOW refreshIfStale; open #2 (after a close) gets a fast one. When the slow first refresh + // finally resolves with a partial failure, it must NOT stamp the SECOND picker's banner (a different generation). + let resolveSlow: (report: RefreshReport) => void = () => undefined; + let call = 0; + const { port } = makeModelsPort({ + refreshIfStale: () => { + call += 1; + return call === 1 + ? new Promise((res) => { + resolveSlow = res; + }) + : Promise.resolve(undefined); + }, + }); + const c = openPicker(port); // open #1 → the slow refresh is in flight (generation 1) + c.handleKey('', { escape: true }); // close it + // Reopen on the SAME controller (open #2, generation 2) via the palette. + c.handleKey('/', {}); + type(c, 'models'); + c.handleKey('', ENTER); + await flush(); + expect(c.getSnapshot().modelPicker).toBeDefined(); // picker #2 is open + + resolveSlow({ providers: [{ provider: 'openai', status: 'failed', error: 'redacted' }] }); // gen-1 refresh lands late + await flush(); + // The stale gen-1 result is dropped — picker #2's banner is untouched (no ghost partial-failure). + expect(c.getSnapshot().modelPicker?.banner).toBeUndefined(); + }); + it('the picker owns the keyboard: typing filters, it does not edit the Home buffer', async () => { const { port } = makeModelsPort({ entries: [pickerEntry({ modelId: 'alpha' }), pickerEntry({ modelId: 'beta' })], diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index 0d66d191..6d65a84b 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -251,6 +251,10 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { // Enter can never splice the file into the (now-empty) buffer meant for the NEXT message (the session-identity // guard only catches a chat SWAP, not an in-chat submit that stays in the same session/mode). let submitEpoch = 0; + // A monotonic `/models` picker generation (2.5.G S7): bumped on every picker open. An async catalog refresh + // captures it and lands its result ONLY if the picker is still on the SAME generation — so a slow refresh + // resolving after the picker was closed AND reopened can never clobber the fresh picker (the `doctorRunId` pattern). + let pickerEpoch = 0; // Race a chat teardown against the force-teardown deadline so the return-to-Home is bounded even if a hung MCP // graceful close never settles; the teardown still runs to completion in the background. @@ -477,26 +481,34 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { }; // The `/models` picker (2.5.G S7, ADR-0064 §10) — a keyboard-owning overlay opened from the Home palette. The db - // read + the merge are synchronous (`port.load()`); a refresh egresses and lands its result ONLY if the picker is - // still open (a since-closed picker drops it — mirrors loadMentions' stale guard). Home-only: it can be open only - // in `mode: 'home'` (the picker branch in handleKey routes before handleHomeKey), so no chat/session race applies. - const applyRefreshResult = (report: RefreshReport | undefined): void => { + // read + the merge are synchronous (`port.load()`); a refresh egresses and lands its result ONLY into the SAME + // picker generation that kicked it — the monotonic `pickerEpoch` (bumped on every open) drops a resolve whose + // picker has since closed or been REOPENED (the `doctorRunId` pattern; the identity half of loadMentions' guard). + // Home-only: it can be open only in `mode: 'home'`, so no chat/session race applies. Two status channels are kept + // SEPARATE — `banner` (async refresh partial-failure) vs `hint` (transient user-action feedback) — so a completing + // refresh can never silently wipe a "not available"/"could not save" message the user just triggered. + const applyRefreshResult = (epoch: number, report: RefreshReport | undefined): void => { const open = state.modelPicker; - if (open === undefined || deps.models === undefined) return; // the picker closed while the refresh was in flight + if (epoch !== pickerEpoch || open === undefined || deps.models === undefined) return; // stale/closed — drop it const { entries, refreshedAt } = deps.models.load(); const failed = report?.providers.filter((p) => p.status === 'failed').map((p) => p.provider) ?? []; - // Keep the user's filter/selection; the view clamps a selection left past the (possibly shrunk) list's end. + // Keep the user's filter/selection + any transient hint; the view clamps a selection past the (shrunk) end. set({ modelPicker: { ...open, entries, refreshedAt, loading: false, banner: partialFailureBanner(failed) } }); }; const runPickerRefresh = (refresh: () => Promise): void => { + const epoch = pickerEpoch; // capture THIS picker generation so a reopened picker never adopts this result const open = state.modelPicker; if (open === undefined) return; set({ modelPicker: { ...open, loading: true } }); - void refresh().then(applyRefreshResult, () => { - // refresh()/refreshIfStale() never reject (per-provider isolation), but stay defensive: just drop the spinner. - const cur = state.modelPicker; - if (cur !== undefined) set({ modelPicker: { ...cur, loading: false } }); - }); + void refresh().then( + (report) => applyRefreshResult(epoch, report), + () => { + // refresh()/refreshIfStale() never reject (per-provider isolation), but stay defensive: drop the spinner + // only when this is still the same open picker generation. + const cur = state.modelPicker; + if (epoch === pickerEpoch && cur !== undefined) set({ modelPicker: { ...cur, loading: false } }); + }, + ); }; const openModelPicker = (): void => { const port = deps.models; // capture the narrowed port so the refresh closure needs no non-null assertion @@ -504,6 +516,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { set({ notice: '/models is unavailable here.' }); // defensive — production always wires the port return; } + pickerEpoch += 1; // a fresh generation — invalidates any in-flight refresh from a prior (closed) open const { entries, refreshedAt } = port.load(); set({ notice: undefined, // opening the picker clears any stale /doctor report behind it @@ -515,6 +528,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { currentDefault: port.currentDefault(), refreshedAt, banner: undefined, + hint: undefined, }, }); // Render the cache immediately (above), then kick a TTL-bounded background refresh (ADR-0064 §5c) — the Home is @@ -522,27 +536,33 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { runPickerRefresh(() => port.refreshIfStale()); }; // Persist the chosen model as the NEXT session's default (ADR-0063) — it does NOT rebind a live session (there is - // none in the bare Home). A write fault (a malformed existing config) keeps the picker open with a static, - // secret-free banner rather than crashing the Home; success closes the picker and confirms in the strip notice. + // none in the bare Home). Because `writeGlobalDefaultModel` writes only the GLOBAL `[preferences].default_model` + // while the EFFECTIVE default resolves project → workspace → global (ADR-0063 §1), the notice is HONEST: it + // confirms success only when the freshly-resolved effective default actually became the chosen model, else it + // says a project/workspace setting still overrides it here (so the tool never falsely claims a no-op took effect). + // A write fault keeps the picker open with a secret-free `hint` rather than crashing the Home. const acceptModel = (modelId: string, displayName: string): void => { - if (deps.models === undefined) return; + const port = deps.models; + if (port === undefined) return; try { - deps.models.writeDefault(modelId); + port.writeDefault(modelId); } catch { const open = state.modelPicker; if (open !== undefined) { - set({ modelPicker: { ...open, banner: 'could not save — check ~/.relavium/config.toml' } }); + set({ modelPicker: { ...open, hint: 'could not save — check ~/.relavium/config.toml' } }); } return; } - set({ - modelPicker: undefined, - notice: `Default model set to ${displayName} — applies to your next chat session.`, - }); + const effective = port.currentDefault(); + const notice = + effective === modelId + ? `Default model set to ${displayName} — applies to your next chat session.` + : `Saved ${displayName} as your global default, but a project or workspace setting overrides it here.`; + set({ modelPicker: undefined, notice }); }; // The open `/models` picker owns every key (2.5.G S7) — parity with routeMentionKey. Returns whether the key was - // consumed (the overlay was open). A DIMMED (unavailable-on-your-key) model is non-selectable (ADR §6): accepting - // one shows a transient banner rather than writing an unusable default. + // consumed. A DIMMED (unavailable-on-your-key) model is non-selectable (ADR §6): accepting one shows a transient + // `hint`, never a write. Any navigation/filter keystroke clears that hint (the user has moved on). const routeModelPickerKey = (input: string, key: ChatKey): boolean => { const open = state.modelPicker; if (open === undefined) return false; @@ -555,13 +575,13 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { acceptModel(step.modelId, step.displayName); break; case 'blocked': - set({ modelPicker: { ...open, banner: `${step.displayName} is not available on your key — pick another` } }); + set({ modelPicker: { ...open, hint: `${step.displayName} is not available on your key — pick another` } }); break; case 'refresh': runPickerRefresh(() => deps.models?.refresh() ?? Promise.resolve(undefined)); break; case 'state': - set({ modelPicker: step.state }); + set({ modelPicker: { ...step.state, hint: undefined } }); // a real interaction clears the transient hint break; } return true; diff --git a/apps/cli/src/render/tui/model-picker-view.tsx b/apps/cli/src/render/tui/model-picker-view.tsx index 891b7d97..21642ef8 100644 --- a/apps/cli/src/render/tui/model-picker-view.tsx +++ b/apps/cli/src/render/tui/model-picker-view.tsx @@ -50,6 +50,9 @@ export function ModelPickerView(props: Readonly): ReactEle const { start, end } = modelWindow(visible.length, selected); const windowed = visible.slice(start, end); const badge = `${formatRefreshedBadge(state.refreshedAt, nowMs)}${state.loading ? ' · refreshing…' : ''}`; + // One status line: the transient user-action `hint` (a dimmed/save note) takes priority over the async refresh + // `banner` (partial-failure) so a completing refresh can never silently wipe the feedback the user just triggered. + const status = state.hint ?? state.banner; const renderBody = (): ReactNode => { if (state.loading && visible.length === 0) { @@ -105,9 +108,9 @@ export function ModelPickerView(props: Readonly): ReactEle Set your default model {` · ${badge}`} - {state.banner !== undefined && ( + {status !== undefined && ( - {sanitizeInline(state.banner)} + {sanitizeInline(status)} )} diff --git a/apps/cli/src/render/tui/model-picker.test.ts b/apps/cli/src/render/tui/model-picker.test.ts index b23bc23b..8aff7c8f 100644 --- a/apps/cli/src/render/tui/model-picker.test.ts +++ b/apps/cli/src/render/tui/model-picker.test.ts @@ -33,6 +33,7 @@ function state(partial: Partial = {}): ModelPickerState { currentDefault: undefined, refreshedAt: undefined, banner: undefined, + hint: undefined, ...partial, }; } diff --git a/apps/cli/src/render/tui/model-picker.ts b/apps/cli/src/render/tui/model-picker.ts index 16d16de1..9b16a241 100644 --- a/apps/cli/src/render/tui/model-picker.ts +++ b/apps/cli/src/render/tui/model-picker.ts @@ -18,8 +18,13 @@ import { dropLastCodePoint } from './chat-input.js'; * The picker submode state. `entries` is the whole merged catalog (all providers, already deterministically * ordered by the merge); `filter` narrows it (case-insensitive, over display name / model id / provider); * `selected` indexes the VISIBLE (filtered) subset; `loading` is `true` while a refresh is in flight (the spinner); - * `currentDefault` is the `[preferences].default_model` marked `✓`; `refreshedAt` is the newest live-refresh - * epoch-ms (the "last updated" badge); `banner` is a secret-free per-provider partial-failure notice (or undefined). + * `currentDefault` is the effective default model marked `✓`; `refreshedAt` is the newest live-refresh epoch-ms + * (the "last updated" badge). + * + * Two DISTINCT status channels (kept separate so an async refresh completing can never wipe a synchronous action + * message, and vice versa): `banner` is the secret-free per-provider partial-failure **refresh status** (set only + * by a refresh; persists until the next refresh); `hint` is the transient **user-action** feedback (a dimmed + * "not available on your key" / a "could not save" note), cleared on the next navigation/filter keystroke. */ export interface ModelPickerState { readonly entries: readonly ModelCatalogEntry[]; @@ -29,6 +34,7 @@ export interface ModelPickerState { readonly currentDefault: string | undefined; readonly refreshedAt: number | undefined; readonly banner: string | undefined; + readonly hint: string | undefined; } /** The minimal key fields the picker fold reads (a structural subset of ink's `Key`). */ From a244cfa4ec4d64f9ec1089dc9c12fb84bbd404de Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 00:25:37 +0300 Subject: [PATCH 20/68] =?UTF-8?q?fix(cli):=202.5.G=20S7=20Sonnet-review=20?= =?UTF-8?q?=E2=80=94=20a=20/models=20write=20now=20reaches=20the=20next=20?= =?UTF-8?q?same-process=20chat?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Independent adversarial Sonnet review found a real BLOCKER (plus a --config mismatch and smaller items) that Opus and I missed: - BLOCKER: `driveHome` loads config ONCE and `startChat` closed over that stale snapshot, so a `/models` write never reached a chat started in the SAME Home process — yet the notice claimed "applies to your next chat session". `startChat` now re-reads the EFFECTIVE default fresh per chat (`readEffectiveDefault() ?? startup value`), so a pick takes effect on the very next session. Added a drive-home integration regression test that drives the REAL port end-to-end (pick a model → the next built session binds it); verified mutation-sensitive (reintroducing the blocker fails it with "expected undefined to be 'claude-fable-5'"). - MAJOR (--config): `writeDefault` wrote ~/.relavium/config.toml while the re-read + session honored `--config`, so under `--config` the write went to the wrong file and the notice falsely blamed a project override. `writeGlobalDefaultModel` gains an optional `targetPath` (the resolved global file) so write + re-read + session all agree on ONE file; all ADR-0063 guarantees (atomic same-fs rename, 0600 temp, schema round-trip, secret-incapable typed setter) hold for either target. (An additive, config-only refinement of the S6 writer, on this unmerged branch.) - MEDIUM: distinguish a config RE-READ FAULT from a genuine override — after a successful write the global is valid, so an `undefined` effective read can only be a fault; `accept` now reports it distinctly, never mislabeled as "overrides". - MEDIUM: fault-harden the picker READ path (`port.load()`) in `openModelPicker` + `applyRefreshResult` — a DB read fault degrades to a notice / drops the spinner, never crashes the REPL (parity with the write-path + runDoctor guards). - MINOR/NIT: ignore `Ctrl+R` while a refresh is already in flight (no racing double-refresh whose out-of-order completion flashes a stale banner); clear the transient `hint` only on a REAL interaction (an inert key returns the same state ref, so it no longer wipes a just-shown hint). Toolchain: lint + typecheck + test (1385, +8) + build all green across all 9 workspace tasks; seam clean (the media-gc failure seen mid-run is the known pre-existing timing flake — passes on retry). Refs: ADR-0064, ADR-0063 Co-Authored-By: Claude --- apps/cli/src/config/write.test.ts | 24 +++++++++ apps/cli/src/config/write.ts | 50 +++++++++++++------ apps/cli/src/home/drive-home.test.ts | 49 ++++++++++++++++++ apps/cli/src/home/drive-home.tsx | 11 +++- .../src/render/tui/home-controller.test.ts | 42 ++++++++++++++-- apps/cli/src/render/tui/home-controller.ts | 43 +++++++++++++--- apps/cli/src/render/tui/model-picker.test.ts | 5 +- apps/cli/src/render/tui/model-picker.ts | 4 +- 8 files changed, 198 insertions(+), 30 deletions(-) diff --git a/apps/cli/src/config/write.test.ts b/apps/cli/src/config/write.test.ts index 6a563daa..b0c63204 100644 --- a/apps/cli/src/config/write.test.ts +++ b/apps/cli/src/config/write.test.ts @@ -63,6 +63,30 @@ describe('writeGlobalDefaultModel', () => { }); }); + it('writes to an explicit targetPath (the --config override), leaving ~/.relavium untouched', () => { + const explicit = join(home, 'custom-config.toml'); + writeGlobalDefaultModel('gpt-4o', home, explicit); + // The explicit file received the write (round-trips)... + expect(loadConfigFile(explicit, GlobalConfigSchema)).toEqual({ + preferences: { default_model: 'gpt-4o' }, + }); + // ...and the canonical ~/.relavium/config.toml was NOT created (the write targeted the override file only). + expect(existsSync(globalConfigPath(home))).toBe(false); + }); + + it('preserves keys and writes 0600 when targeting an explicit --config path', () => { + const explicit = join(home, 'custom.toml'); + writeFileSync(explicit, 'update_channel = "beta"\n[preferences]\ntheme = "dark"\n'); + writeGlobalDefaultModel('m', home, explicit); + expect(loadConfigFile(explicit, GlobalConfigSchema)).toEqual({ + update_channel: 'beta', + preferences: { theme: 'dark', default_model: 'm' }, + }); + if (process.platform !== 'win32') { + expect(statSync(explicit).mode & 0o777).toBe(0o600); // atomic-write owner-only mode holds for either target + } + }); + it('overwrites an existing default_model in place (last write wins)', () => { writeGlobalDefaultModel('first-model', home); writeGlobalDefaultModel('second-model', home); diff --git a/apps/cli/src/config/write.ts b/apps/cli/src/config/write.ts index 5d5c9b74..bc6ede2e 100644 --- a/apps/cli/src/config/write.ts +++ b/apps/cli/src/config/write.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'node:crypto'; import { closeSync, fchmodSync, fsyncSync, openSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'; import { homedir } from 'node:os'; -import { join } from 'node:path'; +import { dirname, join } from 'node:path'; import { GlobalConfigSchema, type GlobalConfig } from '@relavium/shared'; import { parse as parseToml, stringify as stringifyToml } from 'smol-toml'; @@ -52,24 +52,42 @@ import { ensureGlobalConfigDir, globalConfigDir } from './paths.js'; /** * Set the global `[preferences].default_model`, preserving every other config key. Reads + validates the existing - * `~/.relavium/config.toml` (an absent file ⇒ a fresh `{}`; a **malformed/invalid** existing config throws a - * {@link ConfigError} rather than clobbering a file the user must fix), merges only `default_model`, re-validates, - * verifies the serialized text round-trips, and writes atomically. `home` is injectable for tests. + * config (an absent file ⇒ a fresh `{}`; a **malformed/invalid** existing config throws a {@link ConfigError} + * rather than clobbering a file the user must fix), merges only `default_model`, re-validates, verifies the + * serialized text round-trips, and writes atomically. `home` is injectable for tests. + * + * `targetPath` (the CLI `--config` override) writes to that **exact** file — the SAME file `loadResolvedConfig` + * treats as "global" under `--config` — so a `/models` write, the picker's re-read, and the started chat session + * all agree on ONE file (2.5.G S7). Absent ⇒ the canonical global `~/.relavium/config.toml` (its dir created + * `0700`). Every ADR-0063 guarantee holds for either target: the temp lands BESIDE the target for an atomic + * same-filesystem rename and is `0600` by construction, the schema round-trip runs, and the typed setter stays + * secret-incapable (config holds no secrets, so an arbitrary `--config` dir's own mode is immaterial at rest). */ -export function writeGlobalDefaultModel(model: string, home: string = homedir()): void { +export function writeGlobalDefaultModel( + model: string, + home: string = homedir(), + targetPath?: string, +): void { + let target: string; let dir: string; - try { - dir = ensureGlobalConfigDir(home); // `~/.relavium/` (created `0700`) - } catch (err) { - // Keep the module's "every failure is a typed, file-attributed ConfigError" contract even for the - // directory-create step (e.g. EACCES on a read-only home, ENOSPC) — never let a raw fs Error escape. - throw new ConfigError( - globalConfigPath(home), - 'could not be written — its directory could not be created.', - { cause: err }, - ); + if (targetPath === undefined) { + try { + dir = ensureGlobalConfigDir(home); // `~/.relavium/` (created `0700`) + } catch (err) { + // Keep the module's "every failure is a typed, file-attributed ConfigError" contract even for the + // directory-create step (e.g. EACCES on a read-only home, ENOSPC) — never let a raw fs Error escape. + throw new ConfigError( + globalConfigPath(home), + 'could not be written — its directory could not be created.', + { cause: err }, + ); + } + target = join(dir, 'config.toml'); + } else { + // The explicit `--config` target: write beside it (its dir must exist — loadResolvedConfig already read it). + target = targetPath; + dir = dirname(targetPath); } - const target = join(dir, 'config.toml'); // Read the EXISTING config through the same validating loader (so we merge onto known-good data and preserve // update_channel / mcp_servers / preferences.theme). An absent file is a fresh object; an invalid one throws. diff --git a/apps/cli/src/home/drive-home.test.ts b/apps/cli/src/home/drive-home.test.ts index 1e61b162..1b56c38d 100644 --- a/apps/cli/src/home/drive-home.test.ts +++ b/apps/cli/src/home/drive-home.test.ts @@ -6,6 +6,7 @@ import { Readable } from 'node:stream'; import { createClient, createSessionStore, runMigrations, type DbClient } from '@relavium/db'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { buildChatSession } from '../chat/session-host.js'; import { scriptedResolver, textTurn } from '../chat/test-support.js'; import type { OpenedSessionStore } from '../history/session-open.js'; import { EXIT_CODES } from '../process/exit-codes.js'; @@ -207,6 +208,54 @@ describe('driveHome (2.5.B / ADR-0054)', () => { expect(sessions.listSessions({ limit: 10 })[0]?.status).toBe('ended'); }); + it('a /models write is picked up by the NEXT chat started in the SAME Home process (2.5.G S7 regression)', async () => { + // The blocker Sonnet review caught: `startChat` closed over the load-once `config` snapshot, so a `/models` + // write never reached a same-process chat even though the notice claimed "applies to your next chat session". + // Drive the REAL port (real config write + real re-read) end-to-end and assert the next session binds the pick. + const configFile = join(cwd, 'home-config.toml'); // a real, writable global config (the write + re-read target) + let captured: RootAppProps | undefined; + const builtDefaults: Array = []; + const { deps } = makeDeps((p) => (captured = p), { + global: { ...global, configPath: configFile }, + // Record the default the session is built with, then delegate to the real builder (so the chat still works). + buildSession: (args) => { + builtDefaults.push(args.chat.defaultModel); + return buildChatSession(args); + }, + }); + const drivePromise = driveHome(deps); + const props = captured; + if (props === undefined) throw new Error('the injected render was never invoked'); + + // Open the /models picker via the Home palette, let its background refresh settle. + props.controller.handleKey('/', {}); + type(props, 'models'); + props.controller.handleKey('', ENTER); + await flush(); + const picker = props.controller.getSnapshot().modelPicker; + if (picker === undefined || picker.entries.length === 0) { + throw new Error('the /models picker did not open with catalog entries'); + } + const chosen = picker.entries[picker.selected]?.modelId; // the highlighted model (filter empty ⇒ entries[0]) + if (chosen === undefined) throw new Error('no selected model'); + + props.controller.handleKey('', ENTER); // accept ⇒ writeGlobalDefaultModel(chosen) to configFile + expect(props.controller.getSnapshot().modelPicker).toBeUndefined(); // picker closed on accept + expect(props.controller.getSnapshot().notice).toContain('next chat session'); // honest success (write took effect) + + // Start a chat — the NEXT session MUST bind the just-chosen model (the fresh re-read), not a stale startup default. + type(props, 'hello'); + props.controller.handleKey('', ENTER); + await flush(); + expect(props.controller.getSnapshot().mode).toBe('chat'); + expect(builtDefaults.at(-1)).toBe(chosen); // the regression assertion — a stale snapshot would NOT equal `chosen` + + props.controller.handleKey('c', CTRL_C); // chat Ctrl-C ⇒ /cancel ⇒ back to Home + await flush(); + props.controller.handleKey('c', CTRL_C); // Home Ctrl-C ⇒ clean exit + expect(await drivePromise).toBe(EXIT_CODES.success); + }); + it('an external SIGINT tears the live chat down and exits 130 (128+SIGINT)', async () => { let captured: RootAppProps | undefined; let signal: ((signo: number) => void) | undefined; diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index 2efa9d04..8eb54daa 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -190,14 +190,21 @@ export async function driveHome(deps: HomeDeps): Promise { refreshIfStale: () => refreshService.refreshIfStale(), refresh: () => refreshService.refresh(), currentDefault: readEffectiveDefault, - writeDefault: (modelId) => writeGlobalDefaultModel(modelId, homeDir), + // Write to the SAME file the picker re-reads + the started session resolves (honors `--config`), so a `/models` + // write is never a silent no-op to a different file (2.5.G S7). + writeDefault: (modelId) => + writeGlobalDefaultModel(modelId, homeDir, deps.global.configPath), }; // Build + wire + START a fresh chat session (the controller sends the first message on transition). const startChat = async (): Promise => { const store = createChatStore(deps.global.color); const built: BuiltChatSession = await (deps.buildSession ?? buildChatSession)({ - chat: config.chat, + // Re-read the EFFECTIVE default model FRESH per chat (not the load-once `config` snapshot) so a same-session + // `/models` write takes effect on the very next chat started in this long-lived Home (2.5.G S7) — the + // property the accept-notice's "applies to your next chat session" promises. A read fault degrades to the + // startup value. Other `[chat]` settings keep the startup snapshot (only `/models` mutates the default). + chat: { ...config.chat, defaultModel: readEffectiveDefault() ?? config.chat.defaultModel }, agentRef: undefined, // the built-in default agent (zero-config first run) cwd: deps.global.cwd, projectConfigDir, diff --git a/apps/cli/src/render/tui/home-controller.test.ts b/apps/cli/src/render/tui/home-controller.test.ts index 2e41511c..f6c06395 100644 --- a/apps/cli/src/render/tui/home-controller.test.ts +++ b/apps/cli/src/render/tui/home-controller.test.ts @@ -1615,6 +1615,8 @@ function makeModelsPort(opts: { refreshedAt?: number; overrideDefault?: string; writeThrows?: boolean; + readFaults?: boolean; // currentDefault always returns undefined (a config re-read fault after a good write) + loadThrows?: boolean; // load() throws (a DB read fault) refreshIfStale?: () => Promise>>; refresh?: () => Promise>>; } = {}): { @@ -1626,7 +1628,10 @@ function makeModelsPort(opts: { } { const entries = opts.entries ?? [pickerEntry({ modelId: 'a' }), pickerEntry({ modelId: 'b' })]; let written: string | undefined; - const load = vi.fn(() => ({ entries, refreshedAt: opts.refreshedAt })); + const load = vi.fn(() => { + if (opts.loadThrows === true) throw new Error('catalog read failed'); + return { entries, refreshedAt: opts.refreshedAt }; + }); const refreshIfStale = vi.fn(opts.refreshIfStale ?? (() => Promise.resolve(undefined))); const refresh = vi.fn(opts.refresh ?? (() => Promise.resolve({ providers: [] }))); const writeDefault = vi.fn((modelId: string) => { @@ -1637,8 +1642,9 @@ function makeModelsPort(opts: { load, refreshIfStale, refresh, - // The EFFECTIVE default: the override wins (shadows the global write), else the last written id. - currentDefault: () => opts.overrideDefault ?? written, + // The EFFECTIVE default: a read fault ⇒ undefined; else the override wins (shadows the global write), else the + // last written id. + currentDefault: () => (opts.readFaults === true ? undefined : (opts.overrideDefault ?? written)), writeDefault, }; return { port, load, refreshIfStale, refresh, writeDefault }; @@ -1711,6 +1717,36 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => expect(c.getSnapshot().notice).not.toContain('next chat session'); // no false claim of effect }); + it('reports a DISTINCT notice when the config can\'t be re-read after a write (not "overrides")', async () => { + const { port } = makeModelsPort({ + entries: [pickerEntry({ modelId: 'claude-x', displayName: 'Claude X' })], + readFaults: true, // the write succeeds, but the re-read to confirm the effective default throws → undefined + }); + const c = openPicker(port); + await flush(); + c.handleKey('', ENTER); + expect(c.getSnapshot().notice).toContain('could not be re-read'); + expect(c.getSnapshot().notice).not.toContain('overrides'); // a read fault is NOT an override + expect(c.getSnapshot().notice).not.toContain('next chat session'); // nor a confirmed success + }); + + it('a catalog read fault on OPEN degrades to a notice, never crashes the Home (never opens a broken picker)', () => { + const { port } = makeModelsPort({ loadThrows: true }); + const c = createHomeController({ + doctorProbes: STUB_DOCTOR_PROBES, + startChat: vi.fn(), + homeStore, + onExit: vi.fn(), + onError: vi.fn(), + models: port, + }); + c.handleKey('/', {}); + type(c, 'models'); + c.handleKey('', ENTER); // /models → openModelPicker → port.load() throws → caught + expect(c.getSnapshot().modelPicker).toBeUndefined(); // no half-open picker + expect(c.getSnapshot().notice).toContain('could not read the model catalog'); + }); + it('a write fault keeps the picker OPEN with a secret-free hint, never crashes the Home', async () => { const { port } = makeModelsPort({ entries: [pickerEntry({ modelId: 'claude-x', displayName: 'Claude X' })], diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index 6d65a84b..fedea7bf 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -490,10 +490,25 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { const applyRefreshResult = (epoch: number, report: RefreshReport | undefined): void => { const open = state.modelPicker; if (epoch !== pickerEpoch || open === undefined || deps.models === undefined) return; // stale/closed — drop it - const { entries, refreshedAt } = deps.models.load(); const failed = report?.providers.filter((p) => p.status === 'failed').map((p) => p.provider) ?? []; + let view: ReturnType; + try { + view = deps.models.load(); // a DB read — never crash the REPL (parity with runDoctor / acceptModel's guard) + } catch { + // Keep the last-shown entries; just drop the spinner and surface the refresh status. + set({ modelPicker: { ...open, loading: false, banner: partialFailureBanner(failed) } }); + return; + } // Keep the user's filter/selection + any transient hint; the view clamps a selection past the (shrunk) end. - set({ modelPicker: { ...open, entries, refreshedAt, loading: false, banner: partialFailureBanner(failed) } }); + set({ + modelPicker: { + ...open, + entries: view.entries, + refreshedAt: view.refreshedAt, + loading: false, + banner: partialFailureBanner(failed), + }, + }); }; const runPickerRefresh = (refresh: () => Promise): void => { const epoch = pickerEpoch; // capture THIS picker generation so a reopened picker never adopts this result @@ -516,17 +531,23 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { set({ notice: '/models is unavailable here.' }); // defensive — production always wires the port return; } + let view: ReturnType; + try { + view = port.load(); // a DB read — a fault must not crash the REPL (the "never crash the REPL" discipline) + } catch { + set({ notice: '/models: could not read the model catalog.' }); + return; + } pickerEpoch += 1; // a fresh generation — invalidates any in-flight refresh from a prior (closed) open - const { entries, refreshedAt } = port.load(); set({ notice: undefined, // opening the picker clears any stale /doctor report behind it modelPicker: { - entries, + entries: view.entries, filter: '', selected: 0, loading: false, currentDefault: port.currentDefault(), - refreshedAt, + refreshedAt: view.refreshedAt, banner: undefined, hint: undefined, }, @@ -553,11 +574,17 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { } return; } + // Report HONESTLY by re-reading the effective default (project → workspace → global, ADR-0063 §1): success only + // if the chosen model actually became effective; a genuine higher-layer override says so; and — since the write + // just succeeded (so the global is valid) — an `undefined` effective read can only be a re-read fault, reported + // distinctly rather than mislabeled as an override. const effective = port.currentDefault(); const notice = effective === modelId ? `Default model set to ${displayName} — applies to your next chat session.` - : `Saved ${displayName} as your global default, but a project or workspace setting overrides it here.`; + : effective === undefined + ? `Saved ${displayName} as your global default, but your config could not be re-read to confirm it.` + : `Saved ${displayName} as your global default, but a project or workspace setting overrides it here.`; set({ modelPicker: undefined, notice }); }; // The open `/models` picker owns every key (2.5.G S7) — parity with routeMentionKey. Returns whether the key was @@ -581,7 +608,9 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { runPickerRefresh(() => deps.models?.refresh() ?? Promise.resolve(undefined)); break; case 'state': - set({ modelPicker: { ...step.state, hint: undefined } }); // a real interaction clears the transient hint + // Clear the transient hint only on a REAL interaction — the fold returns the SAME state ref for an inert key + // (an unhandled key / backspace on an empty filter), which must not wipe a just-shown hint. + set({ modelPicker: step.state === open ? open : { ...step.state, hint: undefined } }); break; } return true; diff --git a/apps/cli/src/render/tui/model-picker.test.ts b/apps/cli/src/render/tui/model-picker.test.ts index 8aff7c8f..f3daf0ae 100644 --- a/apps/cli/src/render/tui/model-picker.test.ts +++ b/apps/cli/src/render/tui/model-picker.test.ts @@ -44,10 +44,13 @@ describe('foldModelPickerKey', () => { expect(foldModelPickerKey('c', { ctrl: true }, state()).kind).toBe('close'); }); - it('Ctrl+R refreshes; a bare "r" extends the filter instead', () => { + it('Ctrl+R refreshes; a bare "r" extends the filter; Ctrl+R is IGNORED while already loading', () => { expect(foldModelPickerKey('r', { ctrl: true }, state()).kind).toBe('refresh'); const typed = foldModelPickerKey('r', {}, state()); expect(typed).toEqual({ kind: 'state', state: state({ filter: 'r', selected: 0 }) }); + // While a refresh is in flight, Ctrl+R is a no-op (stays open, unchanged) — no racing double-refresh. + const loading = state({ loading: true }); + expect(foldModelPickerKey('r', { ctrl: true }, loading)).toEqual({ kind: 'state', state: loading }); }); it('arrows move the selection, clamped to the visible list', () => { diff --git a/apps/cli/src/render/tui/model-picker.ts b/apps/cli/src/render/tui/model-picker.ts index 9b16a241..822648cd 100644 --- a/apps/cli/src/render/tui/model-picker.ts +++ b/apps/cli/src/render/tui/model-picker.ts @@ -109,8 +109,10 @@ export function foldModelPickerKey( return { kind: 'close' }; } // Ctrl+R forces a live refresh (distinct from the auto TTL refresh on open). `r` alone extends the filter. + // Ignored while a refresh is already in flight (`loading`) — so two rapid Ctrl+R can't race two refreshes whose + // out-of-order completion would flash a stale banner over a fresher one. Stays open, unchanged. if (key.ctrl === true && char === 'r') { - return { kind: 'refresh' }; + return state.loading ? { kind: 'state', state } : { kind: 'refresh' }; } const visible = visibleModels(state); const arrow = foldArrow(key, state, visible.length); From e4d467d0ebf0dbe6414444555ef612cdb954e67c Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 00:58:36 +0300 Subject: [PATCH 21/68] =?UTF-8?q?feat(cli):=202.5.G=20S8=20=E2=80=94=20fir?= =?UTF-8?q?st-run=20onboarding=20wizard=20(key-less=20Home=20=E2=86=92=20p?= =?UTF-8?q?rovider=20+=20key)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add the `@clack/prompts` first-run onboarding wizard that turns a KEY-LESS bare Home into a working chat (2.5.G S8). On a truly key-less run — no known provider resolves a key (neither the OS keychain NOR a RELAVIUM__API_KEY env var) — the wizard runs BEFORE the ink Home mounts (clack + ink both own the terminal, so the wizard settles first); already behind shouldOpenHome's TTY/CI gate, so it never runs piped/--json/CI. A run with either a keychain or env key is not key-less → no wizard (a working env-key user is never prompted; the env fallback IS the resolver's key import). Flow: pick a provider → paste a HIDDEN (masked) key → store it in the OS keychain, riding the TESTED providerSetKey path (keychain.set + the provider row + the keychain-ref). The key is captured via clack's masked `password`, held only in memory, and never echoed, persisted to disk, or logged beyond its last-4 hint. Model selection is deliberately NOT part of S8 — the wizard stores the key that lights up the S7 /models picker + chat; the user picks a model there. Two fallbacks keep a first run unblocked: - Keychain-write failure (locked keychain / no Secret Service / headless) → NEVER persist plaintext; print the RELAVIUM__API_KEY env var to set (the resolver imports it at call time), then hand off to the Home. - Cancel (Ctrl-C/Esc) → a pointer to `provider add` / `/doctor`, then the Home mounts key-less (retry, or add a key manually). @clack/prompts is confined to one module behind an injectable seam (mirroring the create wizard + gate prompter, ADR-0047), so the flow unit-tests without a TTY. No new ADR — the wizard composes ADR-0063 (config), ADR-0006/0019 (keychain), ADR-0047 (clack), all already accepted. Also: the drive-home test harness gains a cancel-immediately default onboarding prompter, so a key-less resolver (e.g. the real keychain-backed resolver over the mocked empty keychain) never invokes REAL clack prompts in a test (which rendered to stdout / could block CI) — caught while wiring S8. Docs: home.md documents the first-run wizard + reconciles the /models "forthcoming" note (landed in S7). Tests: the wizard (store-key + secret-free, cancel-at-select, cancel-at-key, keychain-write-failure→env fallback), isProviderKeyless, and the drive-home trigger (key-less → wizard runs before mount; keyed → skipped). Toolchain: lint + typecheck + test (1393, +8) + build all green across all 9 workspace tasks. Refs: ADR-0047, ADR-0063, ADR-0006 Co-Authored-By: Claude --- apps/cli/src/home/drive-home.test.ts | 73 ++++++++++ apps/cli/src/home/drive-home.tsx | 22 +++ apps/cli/src/onboarding/wizard.test.ts | 166 +++++++++++++++++++++++ apps/cli/src/onboarding/wizard.ts | 177 +++++++++++++++++++++++++ docs/reference/cli/home.md | 15 ++- 5 files changed, 451 insertions(+), 2 deletions(-) create mode 100644 apps/cli/src/onboarding/wizard.test.ts create mode 100644 apps/cli/src/onboarding/wizard.ts diff --git a/apps/cli/src/home/drive-home.test.ts b/apps/cli/src/home/drive-home.test.ts index 1b56c38d..9aba8044 100644 --- a/apps/cli/src/home/drive-home.test.ts +++ b/apps/cli/src/home/drive-home.test.ts @@ -8,7 +8,9 @@ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { buildChatSession } from '../chat/session-host.js'; import { scriptedResolver, textTurn } from '../chat/test-support.js'; +import type { ProviderResolver } from '../engine/providers.js'; import type { OpenedSessionStore } from '../history/session-open.js'; +import type { ClackOnboardingDeps } from '../onboarding/wizard.js'; import { EXIT_CODES } from '../process/exit-codes.js'; import type { CliIo } from '../process/io.js'; import type { GlobalOptions } from '../process/options.js'; @@ -52,6 +54,17 @@ const flush = (): Promise => new Promise((resolve) => setTimeout(resolve, const type = (props: RootAppProps, text: string): void => { for (const ch of text) props.controller.handleKey(ch, {}); }; + +/** A scripted onboarding prompter that CANCELS immediately (its `select` returns the cancel sentinel) — the default + * for every drive-home test so a key-less resolver never triggers the REAL clack prompts (which render to stdout). */ +const CANCEL_ONBOARDING: ClackOnboardingDeps = { + intro: () => undefined, + outro: () => undefined, + note: () => undefined, + select: () => Promise.resolve(Symbol('cancel')), + password: () => Promise.resolve(Symbol('cancel')), + isCancel: (v): v is symbol => typeof v === 'symbol', +}; const ENTER = { return: true } as const; const CTRL_C = { ctrl: true } as const; @@ -117,6 +130,10 @@ describe('driveHome (2.5.B / ADR-0054)', () => { subscribeSignals: () => () => undefined, // no real process listeners in the default tests writeControl, exit: () => undefined, + // A cancel-immediately onboarding prompter by DEFAULT, so a key-less resolver (e.g. the real keychain-backed + // resolver over the mocked empty keychain) never invokes the REAL clack prompts in a test (which would render + // to stdout / block). A wizard-specific test overrides this with a scripted flow. + onboardingPrompter: CANCEL_ONBOARDING, ...overrides, }; return { deps, unmount, writeControl }; @@ -256,6 +273,62 @@ describe('driveHome (2.5.B / ADR-0054)', () => { expect(await drivePromise).toBe(EXIT_CODES.success); }); + it('a KEY-LESS first run runs the onboarding wizard BEFORE mounting the Home (2.5.G S8)', async () => { + // A scripted clack slice + a key-less resolver (keyFor throws for every provider) ⇒ the wizard triggers. + const outros: string[] = []; + const select = vi.fn(() => Promise.resolve('anthropic')); + const onboardingPrompter: ClackOnboardingDeps = { + intro: () => undefined, + outro: (m) => outros.push(m), + note: () => undefined, + select, + password: () => Promise.resolve('sk-home-9999'), + isCancel: (v): v is symbol => typeof v === 'symbol', + }; + const keylessResolver: ProviderResolver = { + resolveProvider: () => undefined, + keyFor: () => { + throw new Error('no key'); + }, + }; + let captured: RootAppProps | undefined; + const { deps } = makeDeps((p) => (captured = p), { + providers: keylessResolver, + onboardingPrompter, + }); + const drivePromise = driveHome(deps); + await flush(); // the wizard is async — let it settle before render() mounts ink + const props = captured; + if (props === undefined) throw new Error('the Home did not mount after the wizard'); + expect(select).toHaveBeenCalledTimes(1); // the wizard ran (provider select) + expect(outros.some((o) => o.includes('all set'))).toBe(true); // key stored (mocked keychain) + handed off to the Home + + props.controller.handleKey('c', CTRL_C); // clean exit + expect(await drivePromise).toBe(EXIT_CODES.success); + }); + + it('a run WITH a resolvable key SKIPS the onboarding wizard entirely', async () => { + // The default scriptedResolver's keyFor returns 'test-key' ⇒ not key-less ⇒ the wizard must never run. + const select = vi.fn(() => Promise.resolve('anthropic')); + const onboardingPrompter: ClackOnboardingDeps = { + intro: () => undefined, + outro: () => undefined, + note: () => undefined, + select, + password: () => Promise.resolve('sk-x'), + isCancel: (v): v is symbol => typeof v === 'symbol', + }; + let captured: RootAppProps | undefined; + const { deps } = makeDeps((p) => (captured = p), { onboardingPrompter }); + const drivePromise = driveHome(deps); + const props = captured; + if (props === undefined) throw new Error('render was not invoked'); + expect(select).not.toHaveBeenCalled(); // no wizard on a keyed run + + props.controller.handleKey('c', CTRL_C); + expect(await drivePromise).toBe(EXIT_CODES.success); + }); + it('an external SIGINT tears the live chat down and exits 130 (128+SIGINT)', async () => { let captured: RootAppProps | undefined; let signal: ((signo: number) => void) | undefined; diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index 8eb54daa..ef0e7a73 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -21,6 +21,11 @@ import { type ProviderResolver, } from '../engine/providers.js'; import { openSessionStore, type OpenedSessionStore } from '../history/session-open.js'; +import { + isProviderKeyless, + runOnboardingWizard, + type ClackOnboardingDeps, +} from '../onboarding/wizard.js'; import type { CliIo } from '../process/io.js'; import { EXIT_CODES, type ExitCode } from '../process/exit-codes.js'; import type { GlobalOptions } from '../process/options.js'; @@ -65,6 +70,9 @@ export interface HomeDeps { readonly mcpSecretResolver?: McpSecretResolver; /** The `/doctor` probes (2.5.C S5) — production assembles the real probes; a test injects a fake. */ readonly doctorProbes?: DoctorProbes; + /** The onboarding clack slice (2.5.G S8) — omit for the real prompts; a test injects a scripted one. Only ever + * used on a truly key-less first run (the wizard is otherwise skipped). */ + readonly onboardingPrompter?: ClackOnboardingDeps; readonly now?: () => number; readonly uuid?: () => string; /** Injectable ink mount + the terminal-size seam (tests drive `RootApp` props without a real TTY). */ @@ -333,6 +341,20 @@ export async function driveHome(deps: HomeDeps): Promise { }); const exitProcess = deps.exit ?? ((code: number) => process.exit(code)); + // First-run onboarding (2.5.G S8): a truly KEY-LESS bare Home offers a `@clack` wizard to connect a provider + // BEFORE mounting ink — clack + ink both take the terminal's raw mode, so the wizard must fully settle (and + // clack restore the terminal) before `render()`. Already behind `shouldOpenHome`'s TTY/CI gate; a cancel or a + // keychain-write failure ends the wizard cleanly and the Home mounts key-less (retry, or add a key manually). + if (isProviderKeyless(providers)) { + await runOnboardingWizard({ + store: providerStore, + keychain, + resolver: providers, + io: deps.io, + ...(deps.onboardingPrompter === undefined ? {} : { prompter: deps.onboardingPrompter }), + }); + } + writeControl(ENABLE_BRACKETED_PASTE); // ask the terminal to bracket pastes (DECSET 2004) // One external-signal lifecycle covering the Home, the in-Home chat, and MCP teardown. diff --git a/apps/cli/src/onboarding/wizard.test.ts b/apps/cli/src/onboarding/wizard.test.ts new file mode 100644 index 00000000..3391c733 --- /dev/null +++ b/apps/cli/src/onboarding/wizard.test.ts @@ -0,0 +1,166 @@ +import { Readable } from 'node:stream'; + +import { createClient, createProviderStore, runMigrations, type DbClient } from '@relavium/db'; +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; + +import { providerKeyEnvVar, type ProviderResolver } from '../engine/providers.js'; +import type { CliIo } from '../process/io.js'; +import { + KeychainUnavailableError, + keychainAccount, + type KeychainStore, +} from '../secrets/keychain.js'; +import { + isProviderKeyless, + runOnboardingWizard, + type ClackOnboardingDeps, +} from './wizard.js'; + +const CANCEL = Symbol('clack-cancel'); + +/** A scripted clack slice: fixed `select`/`password` results (a `symbol` = cancel), spies + captured notes/outros. */ +function scriptedPrompter(script: { provider?: string | symbol; key?: string | symbol }): { + prompter: ClackOnboardingDeps; + select: ReturnType; + password: ReturnType; + notes: string[]; + outros: string[]; +} { + const notes: string[] = []; + const outros: string[] = []; + const select = vi.fn(() => Promise.resolve(script.provider ?? CANCEL)); + const password = vi.fn(() => Promise.resolve(script.key ?? CANCEL)); + const prompter: ClackOnboardingDeps = { + intro: () => undefined, + outro: (m) => { + outros.push(m); + }, + note: (m, t) => { + notes.push(`${t ?? ''}\n${m}`); + }, + select, + password, + isCancel: (v): v is symbol => typeof v === 'symbol', + }; + return { prompter, select, password, notes, outros }; +} + +/** An in-memory keychain; `throwOnSet` makes `set` raise `KeychainUnavailableError` (the locked-keychain case). */ +function memKeychain(opts: { throwOnSet?: boolean } = {}): KeychainStore & { store: Map } { + const store = new Map(); + return { + store, + get: (account) => store.get(account) ?? null, + set: (account, secret) => { + if (opts.throwOnSet === true) throw new KeychainUnavailableError('keychain is locked'); + store.set(account, secret); + }, + delete: (account) => store.delete(account), + }; +} + +/** A stub resolver — `provider set-key` never calls it, but `OnboardingDeps` requires it. */ +const stubResolver: ProviderResolver = { + resolveProvider: () => undefined, + keyFor: () => { + throw new Error('no key'); + }, +}; + +const io: CliIo = { + writeOut: () => undefined, + writeErr: () => undefined, + env: {}, + stdoutIsTty: true, + stdinIsTty: true, + stdin: Readable.from([]), // a real, already-ended stream (the wizard never reads it) +}; + +describe('isProviderKeyless', () => { + it('is TRUE when every provider key resolution throws (no keychain + no env)', () => { + expect(isProviderKeyless({ keyFor: () => { throw new Error('no key'); } })).toBe(true); + }); + it('is FALSE as soon as one provider resolves a key (keychain or env)', () => { + let calls = 0; + const resolver = { + keyFor: () => { + calls += 1; + if (calls === 1) throw new Error('no key'); // first provider: no key + return 'a-key'; // a later provider resolves ⇒ not key-less + }, + }; + expect(isProviderKeyless(resolver)).toBe(false); + }); +}); + +describe('runOnboardingWizard', () => { + let client: DbClient; + beforeEach(() => { + client = createClient(':memory:'); + runMigrations(client.db); + }); + afterEach(() => { + client.sqlite.close(); + }); + const store = (): ReturnType => + createProviderStore(client.db, { uuid: () => 'id-fixed', now: () => 1 }); + + it('stores the pasted key in the OS keychain + registers the provider row (secret-free)', async () => { + const keychain = memKeychain(); + const s = store(); + const { prompter, notes, outros } = scriptedPrompter({ + provider: 'anthropic', + key: 'sk-ant-supersecret-1234', + }); + await runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io }); + + // The key landed in the keychain under the provider account... + expect(keychain.store.get(keychainAccount('anthropic'))).toBe('sk-ant-supersecret-1234'); + // ...the provider row + keychain-ref were registered... + expect(s.get('anthropic')?.apiKeyKeychainRef).toBe(keychainAccount('anthropic')); + // ...a "Connected" note shows only the key HINT (last 4), never the full key, and confirms + hands off. + const all = [...notes, ...outros].join('\n'); + expect(all).toContain('••••1234'); + expect(all).not.toContain('sk-ant-supersecret-1234'); // the full key NEVER appears anywhere + expect(outros.some((o) => o.includes('all set'))).toBe(true); + }); + + it('cancelling the provider select skips setup, writes NOTHING to the keychain', async () => { + const keychain = memKeychain(); + const s = store(); + const { prompter, password, notes } = scriptedPrompter({ provider: CANCEL }); + await runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io }); + expect(keychain.store.size).toBe(0); // no key captured or stored + expect(password).not.toHaveBeenCalled(); // never even prompted for a key + expect(notes.some((n) => n.includes('Skipped'))).toBe(true); + }); + + it('cancelling the key prompt skips setup, writes NOTHING to the keychain', async () => { + const keychain = memKeychain(); + const s = store(); + const { prompter, notes } = scriptedPrompter({ provider: 'openai', key: CANCEL }); + await runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io }); + expect(keychain.store.size).toBe(0); + expect(s.get('openai')).toBeUndefined(); // no row registered on cancel + expect(notes.some((n) => n.includes('Skipped'))).toBe(true); + }); + + it('on a keychain-write failure, guides to the env var — NEVER persists the key, never crashes', async () => { + const keychain = memKeychain({ throwOnSet: true }); + const s = store(); + const { prompter, notes, outros } = scriptedPrompter({ + provider: 'gemini', + key: 'sk-gem-secret-9999', + }); + // Must resolve (not throw) — a keychain-unavailable first run degrades gracefully. + await expect( + runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io }), + ).resolves.toBeUndefined(); + + expect(keychain.store.size).toBe(0); // nothing persisted + const all = [...notes, ...outros].join('\n'); + expect(all).toContain(providerKeyEnvVar('gemini')); // the env-var fallback is named (RELAVIUM_GEMINI_API_KEY) + expect(all).not.toContain('sk-gem-secret-9999'); // the key is NEVER echoed, even in the fallback + expect(all.toLowerCase()).toContain('keychain'); // the fallback explains why + }); +}); diff --git a/apps/cli/src/onboarding/wizard.ts b/apps/cli/src/onboarding/wizard.ts new file mode 100644 index 00000000..ba0c1847 --- /dev/null +++ b/apps/cli/src/onboarding/wizard.ts @@ -0,0 +1,177 @@ +import { intro, isCancel, note, outro, password, select } from '@clack/prompts'; +import type { ProviderStore } from '@relavium/db'; + +import { runProviderCommand } from '../commands/provider.js'; +import { + KNOWN_PROVIDERS, + KNOWN_PROVIDER_IDS, + keyHint, + providerKeyEnvVar, + type ProviderResolver, +} from '../engine/providers.js'; +import type { CliIo } from '../process/io.js'; +import type { KeychainStore } from '../secrets/keychain.js'; + +/** + * The first-run onboarding wizard (workstream **2.5.G S8**) — a `@clack/prompts` flow that turns a KEY-LESS Home + * into a working chat: pick a provider → paste a **hidden** API key → store it in the OS keychain, with a + * keychain-write fallback that guides the user to the environment variable (the key is **NEVER** written to disk). + * It reuses the two existing ink↔clack custody patterns: `@clack/prompts` is confined to THIS module behind an + * injectable seam (mirroring the create wizard + the gate prompter), and the key-storage rides the **tested** + * `providerSetKey` path (keychain.set + the provider row + the keychain-ref, secret-free by construction — + * [provider.ts](../commands/provider.ts), ADR-0006/0019). Model selection is deliberately OUT of scope here — the + * wizard's job is to store the key that lights up the S7 `/models` picker + chat; the user picks a model there. + * + * SECURITY (this captures a live API key): + * - The key is read via clack's masked `password` prompt (never an argv flag, never echoed), held only in memory, + * and handed straight to the keychain — it is never logged (only {@link keyHint}, the last 4), persisted to a + * file, or placed in an error/report. + * - The keychain-unavailable fallback prints the env-var name to set — NEVER the key, and NEVER a plaintext file + * (the deliberate "no silent plaintext fallback" of `providerSetKey`). + */ + +/** The narrow slice of `@clack/prompts` the wizard uses — injectable so the flow unit-tests without a TTY. */ +export interface ClackOnboardingDeps { + readonly intro: (title: string) => void; + readonly outro: (message: string) => void; + readonly note: (message: string, title?: string) => void; + readonly select: (opts: { + message: string; + options: readonly { value: string; label: string; hint?: string }[]; + }) => Promise; + /** A MASKED key prompt (clack `password`) — the hidden interactive key input. */ + readonly password: (opts: { + message: string; + validate?: (value: string | undefined) => string | undefined; + }) => Promise; + /** Clack's cancel sentinel guard (Ctrl-C / ESC) — a real type guard so a non-cancel value narrows. */ + readonly isCancel: (value: unknown) => value is symbol; +} + +// The clack-boundary adapter (the one place the library's exact option shapes are met) — spread the optional +// fields conditionally so no explicit `undefined` is passed (exactOptionalPropertyTypes), keeping the seam free +// of clack's types and free of an unsafe cast. Mirrors `create-prompter.ts`'s `defaultDeps`. +const defaultPrompter: ClackOnboardingDeps = { + intro: (title) => intro(title), + outro: (message) => outro(message), + note: (message, title) => note(message, title), + select: (opts) => + select({ + message: opts.message, + options: opts.options.map((option) => ({ + value: option.value, + label: option.label, + ...(option.hint === undefined ? {} : { hint: option.hint }), + })), + }), + password: (opts) => + password({ + message: opts.message, + ...(opts.validate === undefined ? {} : { validate: opts.validate }), + }), + isCancel, +}; + +/** The non-clack ports the wizard needs — the keychain-write path + the resolver (injected for tests). */ +export interface OnboardingDeps { + /** The clack slice — omit for the real prompts; a test injects a scripted one. */ + readonly prompter?: ClackOnboardingDeps; + readonly store: ProviderStore; + readonly keychain: KeychainStore; + readonly resolver: ProviderResolver; + readonly io: CliIo; +} + +/** + * Whether the run is truly KEY-LESS — no known provider has a resolvable key (the resolver checks the OS keychain + * AND the `RELAVIUM__API_KEY` env fallback). `true` ⇒ a chat turn would fail `provider_auth`, so the + * bare Home offers the wizard. A run with EITHER a keychain key or an env key is NOT key-less (no wizard) — so a + * working env-key user is never nagged, and the env fallback IS the resolver's built-in key import. + */ +export function isProviderKeyless(resolver: Pick): boolean { + return !KNOWN_PROVIDER_IDS.some((id) => { + try { + resolver.keyFor(id); + return true; + } catch { + return false; // no keychain key + no env var for this provider + } + }); +} + +/** A clack `validate` that rejects an empty/whitespace key (Esc still cancels the whole flow). */ +const requireKey = (value: string | undefined): string | undefined => + (value ?? '').trim() === '' ? 'An API key is required (or press Esc to skip setup).' : undefined; + +/** + * Run the first-run wizard. Resolves when the user has stored a key OR skipped/failed — the caller (the Home) then + * mounts as usual. Never throws for a normal cancel or a keychain-unavailable fallback (both end cleanly); only an + * unexpected fault propagates (caught by the Home's cleanup finally). + */ +export async function runOnboardingWizard(deps: OnboardingDeps): Promise { + const p = deps.prompter ?? defaultPrompter; + p.intro('Welcome to Relavium'); + p.note( + 'Connect a model provider to start chatting.\nYour key is stored in your OS keychain — never on disk.', + 'First-run setup', + ); + + const providerValue = await p.select({ + message: 'Which provider?', + options: KNOWN_PROVIDER_IDS.map((id) => ({ + value: id, + label: KNOWN_PROVIDERS[id].displayName, + hint: KNOWN_PROVIDERS[id].baseUrl, + })), + }); + if (p.isCancel(providerValue)) return skip(p); + // `select` only ever yields a listed option, so narrow to the closed set WITHOUT a cast (mirrors create-prompter). + const provider = KNOWN_PROVIDER_IDS.find((id) => id === providerValue); + if (provider === undefined) return skip(p); + + const key = await p.password({ + message: `Paste your ${KNOWN_PROVIDERS[provider].displayName} API key`, + validate: requireKey, + }); + if (p.isCancel(key)) return skip(p); + + // Store via the TESTED providerSetKey path (keychain.set + the provider row + the keychain-ref, secret-free). Its + // one stdout line is suppressed (a silent io) so the wizard's output stays uniformly clack-styled — the wizard + // surfaces the outcome through a clack note/outro instead. + const silentIo: CliIo = { ...deps.io, writeOut: () => undefined, writeErr: () => undefined }; + try { + await runProviderCommand( + { action: 'set-key', name: provider }, + { + io: silentIo, + store: deps.store, + keychain: deps.keychain, + resolver: deps.resolver, + readSecret: () => Promise.resolve(key), + }, + ); + p.note(`Stored your ${provider} key (${keyHint(key)}) in the OS keychain.`, 'Connected'); + p.outro("You're all set — starting Relavium."); + } catch { + // Write-failure fallback: the OS keychain is unavailable (locked / no Secret Service / a headless box). NEVER + // persist the key to disk — guide the user to the env var, which the resolver imports at call time + // (keychain → env fallback). The broad catch is intentional: any failure to store degrades to the same + // env-var guidance rather than crashing the first run. + p.note( + `Couldn't reach your OS keychain, so the key was NOT saved.\n` + + `Set it in your shell instead, then restart Relavium:\n\n` + + ` export ${providerKeyEnvVar(provider)}=`, + 'Keychain unavailable', + ); + p.outro('Setup incomplete — see the note above.'); + } +} + +/** The cancel/skip exit: a friendly pointer to the manual path, then hand off to the Home. */ +function skip(p: ClackOnboardingDeps): void { + p.note( + 'Skipped. Add a provider anytime with `relavium provider add`, or run `/doctor` to check your setup.', + 'Setup skipped', + ); + p.outro('Starting Relavium.'); +} diff --git a/docs/reference/cli/home.md b/docs/reference/cli/home.md index 7e1e0421..97fcf963 100644 --- a/docs/reference/cli/home.md +++ b/docs/reference/cli/home.md @@ -23,6 +23,17 @@ stdoutIsTty && stdinIsTty && !json && !isCiEnv(env) **Every non-interactive path is byte-for-byte unchanged**: a piped, `--json`, or CI bare invocation prints `program.helpInformation()` and exits `0`, exactly as before ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md) — `--help` / `--version` / a bare no-command invocation are exit-`0` meta-operations). The Home adds a TTY-gated branch and **no** new IO surface. A Home build/config fault renders like any command fault (a config/invocation `CliError` → exit `2`; any unexpected throw, or an `internal` fault, → exit `1` with a generic message, no raw leak). +## First-run onboarding wizard (2.5.G S8) + +When a bare `relavium` opens the Home on a **truly key-less** run — **no** known provider has a resolvable key (the resolver finds neither an OS-keychain key **nor** a `RELAVIUM__API_KEY` env var) — a `@clack/prompts` **onboarding wizard** runs **before** the ink Home mounts (clack + ink both take the terminal's raw mode, so the wizard fully settles first). It is already behind the TTY gate above, so it never runs piped / `--json` / in CI. A run with **either** a keychain key or an env key is not key-less — no wizard, so a working user (including an env-key user) is never prompted. + +The flow: **pick a provider → paste a hidden (masked) API key → store it in the OS keychain** — riding the same tested `provider set-key` path (keychain write + the provider row + the keychain-ref, secret-free by construction; the key is captured via clack's masked `password`, never echoed, never written to disk, never logged beyond its last-4 hint). **Model selection is not part of the wizard** — its job is to store the key that lights up the `/models` picker + chat; the user picks a model there. Two fallbacks keep a first run unblocked: + +- **Keychain-write failure** (a locked keychain / no Secret Service / a headless box) → the wizard **never** persists the key to disk; it prints the `RELAVIUM__API_KEY` env-var to set instead (the resolver imports an env key at call time), then hands off to the Home. +- **Cancel** (Ctrl-C / Esc at any step) → a friendly pointer to `relavium provider add` / `/doctor`, then the Home mounts key-less (retry next launch, or add a key manually). + +`@clack/prompts` is confined to one module behind an injectable seam (mirroring the `create` wizard + the gate prompter — [ADR-0047](../../decisions/0047-cli-framework-commander-ink-clack.md)), so the flow unit-tests without a TTY. No new ADR: the wizard composes the config-write ([ADR-0063](../../decisions/0063-cli-config-write-contract.md)), keychain ([ADR-0006](../../decisions/0006-os-keychain-for-api-keys.md)/[ADR-0019](../../decisions/0019-cli-node-keychain-library.md)), and clack ([ADR-0047](../../decisions/0047-cli-framework-commander-ink-clack.md)) decisions already accepted. + ## The management strip The strip is a **read-only display** (the no-IDE-shell principle of [ADR-0007](../../decisions/0007-desktop-is-not-an-ide.md) applied to the CLI by analogy: management + chat only — no file tree, editor, or embedded terminal). It is a bounded, indexed aggregation over the durable `history.db` ([ADR-0050](../../decisions/0050-cli-history-db-at-rest-posture.md)), re-read on every return-to-Home — never a live subscription. @@ -122,5 +133,5 @@ Settled in 2.5.B, recorded here as the canonical home (not re-litigated elsewher - **In-flight build surface** — a slow `buildChatSession` shows a static `Starting chat…` loading state that echoes the pending message (the typed text never visually vanishes). A richer spinner is deferred; the `Esc` mid-turn abort (once a chat is built) landed in 2.5.E. - **Attention ordering is a recency proxy, not a deadline sort** — gates are ordered by the paused run's `created_at DESC` (a glanceable proxy); true gate-recency / soonest-expiry-first would need the pending-gate read to carry the raise time. The renderer escalates an *expired* gate to red regardless of position. - **Continue excludes lifted runs by status** — a failed or human-gated run lives only in *Attention*, never duplicated in *Continue*; the strip over-fetches to backfill *Continue* to its limit. -- **Landed since 2.5.B** — the interactive `/` palette UI (filterable, keyboard-navigable — 2.5.C S3b; the curated slash command set + the command-manifest shape are homed in [commands.md](commands.md)); the ask/plan/accept-edits/auto mode keymap + per-tool approval + `Esc` abort (2.5.E, [ADR-0057](../../decisions/0057-cli-chat-modes-and-per-tool-approval.md)); and the input ergonomics + `@`-mention + `!`-shell (2.5.D, [ADR-0061](../../decisions/0061-cli-input-layer-file-injection-and-shell-escape.md)). -- **Forthcoming, extending this doc** — the Home-side `/models` picker over a connected-provider catalog; and an in-app message-queue/type-ahead while a turn runs (deferred, see [../../roadmap/deferred-tasks.md](../../roadmap/deferred-tasks.md)). +- **Landed since 2.5.B** — the interactive `/` palette UI (filterable, keyboard-navigable — 2.5.C S3b; the curated slash command set + the command-manifest shape are homed in [commands.md](commands.md)); the ask/plan/accept-edits/auto mode keymap + per-tool approval + `Esc` abort (2.5.E, [ADR-0057](../../decisions/0057-cli-chat-modes-and-per-tool-approval.md)); the input ergonomics + `@`-mention + `!`-shell (2.5.D, [ADR-0061](../../decisions/0061-cli-input-layer-file-injection-and-shell-escape.md)); the Home-side `/models` picker over the merged live/static catalog (2.5.G S7, [ADR-0064](../../decisions/0064-live-model-catalog.md) §10 — detailed in [commands.md](commands.md)); and the first-run onboarding wizard (2.5.G S8 — above). +- **Forthcoming, extending this doc** — an in-app message-queue/type-ahead while a turn runs (deferred, see [../../roadmap/deferred-tasks.md](../../roadmap/deferred-tasks.md)). From fc795186c6e20e5de9a53519812bddd686862656 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 01:07:22 +0300 Subject: [PATCH 22/68] =?UTF-8?q?fix(cli):=202.5.G=20S8=20Opus-review=20?= =?UTF-8?q?=E2=80=94=20don't=20mislabel=20a=20non-keychain=20fault=20as=20?= =?UTF-8?q?"keychain=20unavailable"?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial Opus review confirmed the primary 🔒 secret-freedom + no-plaintext guarantees hold (no blocker/major). Two minor fixes: - The wizard's storage try/catch was too wide + the catch bindingless, so a non-keychain fault — a db upsert AFTER a successful keychain.set, or a clack note/outro throw — was mislabeled "Couldn't reach your OS keychain, key was NOT saved" and pointed the user to a redundant env var, even though the key IS in the keychain. Now the catch scopes to only the storage call and distinguishes the EXPECTED keychain-unavailable case (a KeychainUnavailableError, wrapped by runProviderCommand in a CliError cause → the env-var fallback) from an unexpected fault (→ a generic "setup could not be completed" note; the underlying issue resurfaces at the Home). Both branches use static text — the raw error is never rendered. The success note/outro moved after the try (only on a real store). - The keychain-write-failure test now also asserts NO dangling provider row (keychain.set is first, so the failure persists nothing atomically) — a regression that reordered the write would leave a row claiming a key that isn't stored, which the old size-only assertion missed. - Corrected the drive-home KEY-LESS test comment: the module-level vi.mock discards the key (keeping the test off the real OS keychain), so that test proves the wizard RAN + handed off; real storage is covered in wizard.test.ts. Toolchain: lint + typecheck + test (1393) + build all green. Refs: ADR-0047, ADR-0006 Co-Authored-By: Claude --- apps/cli/src/home/drive-home.test.ts | 5 ++- apps/cli/src/onboarding/wizard.test.ts | 3 +- apps/cli/src/onboarding/wizard.ts | 46 ++++++++++++++++++-------- 3 files changed, 38 insertions(+), 16 deletions(-) diff --git a/apps/cli/src/home/drive-home.test.ts b/apps/cli/src/home/drive-home.test.ts index 9aba8044..ece8268a 100644 --- a/apps/cli/src/home/drive-home.test.ts +++ b/apps/cli/src/home/drive-home.test.ts @@ -301,7 +301,10 @@ describe('driveHome (2.5.B / ADR-0054)', () => { const props = captured; if (props === undefined) throw new Error('the Home did not mount after the wizard'); expect(select).toHaveBeenCalledTimes(1); // the wizard ran (provider select) - expect(outros.some((o) => o.includes('all set'))).toBe(true); // key stored (mocked keychain) + handed off to the Home + // The wizard reached its success path + handed off to the Home. (Real keychain STORAGE is proven in + // wizard.test.ts with an inspectable keychain; here the module-level vi.mock discards the key, which is what + // keeps the test off the developer's/CI's real OS keychain.) + expect(outros.some((o) => o.includes('all set'))).toBe(true); props.controller.handleKey('c', CTRL_C); // clean exit expect(await drivePromise).toBe(EXIT_CODES.success); diff --git a/apps/cli/src/onboarding/wizard.test.ts b/apps/cli/src/onboarding/wizard.test.ts index 3391c733..2e4c31d0 100644 --- a/apps/cli/src/onboarding/wizard.test.ts +++ b/apps/cli/src/onboarding/wizard.test.ts @@ -157,7 +157,8 @@ describe('runOnboardingWizard', () => { runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io }), ).resolves.toBeUndefined(); - expect(keychain.store.size).toBe(0); // nothing persisted + expect(keychain.store.size).toBe(0); // nothing persisted to the keychain... + expect(s.get('gemini')).toBeUndefined(); // ...and NO dangling provider row (keychain.set is first, so it fails atomically) const all = [...notes, ...outros].join('\n'); expect(all).toContain(providerKeyEnvVar('gemini')); // the env-var fallback is named (RELAVIUM_GEMINI_API_KEY) expect(all).not.toContain('sk-gem-secret-9999'); // the key is NEVER echoed, even in the fallback diff --git a/apps/cli/src/onboarding/wizard.ts b/apps/cli/src/onboarding/wizard.ts index ba0c1847..d459a788 100644 --- a/apps/cli/src/onboarding/wizard.ts +++ b/apps/cli/src/onboarding/wizard.ts @@ -9,8 +9,9 @@ import { providerKeyEnvVar, type ProviderResolver, } from '../engine/providers.js'; +import { CliError } from '../process/errors.js'; import type { CliIo } from '../process/io.js'; -import type { KeychainStore } from '../secrets/keychain.js'; +import { KeychainUnavailableError, type KeychainStore } from '../secrets/keychain.js'; /** * The first-run onboarding wizard (workstream **2.5.G S8**) — a `@clack/prompts` flow that turns a KEY-LESS Home @@ -150,21 +151,38 @@ export async function runOnboardingWizard(deps: OnboardingDeps): Promise { readSecret: () => Promise.resolve(key), }, ); - p.note(`Stored your ${provider} key (${keyHint(key)}) in the OS keychain.`, 'Connected'); - p.outro("You're all set — starting Relavium."); - } catch { - // Write-failure fallback: the OS keychain is unavailable (locked / no Secret Service / a headless box). NEVER - // persist the key to disk — guide the user to the env var, which the resolver imports at call time - // (keychain → env fallback). The broad catch is intentional: any failure to store degrades to the same - // env-var guidance rather than crashing the first run. - p.note( - `Couldn't reach your OS keychain, so the key was NOT saved.\n` + - `Set it in your shell instead, then restart Relavium:\n\n` + - ` export ${providerKeyEnvVar(provider)}=`, - 'Keychain unavailable', - ); + } catch (err) { + // Distinguish the EXPECTED keychain-unavailable failure (recoverable → env-var guidance) from an UNEXPECTED + // fault (e.g. a db write). `providerSetKey` writes to the keychain FIRST, so a keychain-unavailable failure + // persists NOTHING; mislabeling a *post*-`keychain.set` fault (a later db upsert) as "keychain unavailable, key + // not saved" would LIE — the key IS in the keychain then. `runProviderCommand` wraps `KeychainUnavailableError` + // in a `CliError(cause)`. Never render the raw error (it could carry context) — both branches use static text. + const keychainDown = + err instanceof KeychainUnavailableError || + (err instanceof CliError && err.cause instanceof KeychainUnavailableError); + if (keychainDown) { + // The no-plaintext fallback: guide the user to the env var (the resolver imports it at call time). NEVER + // persist the key to disk. + p.note( + `Couldn't reach your OS keychain, so the key was NOT saved.\n` + + `Set it in your shell instead, then restart Relavium:\n\n` + + ` export ${providerKeyEnvVar(provider)}=`, + 'Keychain unavailable', + ); + } else { + // An unexpected fault (e.g. a broken db) — do NOT claim a keychain failure. A generic pointer; the underlying + // issue resurfaces at the Home, which reads the same store. + p.note( + 'Setup could not be completed. Add a provider anytime with `relavium provider add`.', + 'Setup failed', + ); + } p.outro('Setup incomplete — see the note above.'); + return; } + // Success — only reached when the key was actually stored (keychain.set + the provider row + the keychain-ref). + p.note(`Stored your ${provider} key (${keyHint(key)}) in the OS keychain.`, 'Connected'); + p.outro("You're all set — starting Relavium."); } /** The cancel/skip exit: a friendly pointer to the manual path, then hand off to the Home. */ From faa33e03a203092722da511a4db1b793c71148dc Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 01:26:04 +0300 Subject: [PATCH 23/68] =?UTF-8?q?fix(cli):=202.5.G=20S8=20Sonnet-review=20?= =?UTF-8?q?=E2=80=94=20the=20wizard=20reaches=20a=20WORKING=20chat=20for?= =?UTF-8?q?=20every=20provider?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Independent adversarial Sonnet review (reproduction + mutation) found two MAJOR issues the prior rounds missed: - The "reach a working chat" promise was broken for 3 of 4 providers: the wizard stored ANY provider's key, but the built-in default agent binds DEFAULT_CHAT_MODEL (claude-sonnet-4-6 → anthropic). A user who picked OpenAI/Gemini/DeepSeek, stored a valid key, then chatted → keyFor('anthropic') threw. The wizard now sets [chat].default_model to the CHOSEN provider's cheap/fast starter model (KNOWN_PROVIDERS[provider].testModel) via the same config-write target as /models (honors --config), so the next chat binds a model whose key was just stored. Best-effort: a config-write fault leaves a working key + a /models pointer, never undoing the store. - The unexpected-fault `else` branch (added last round) had ZERO coverage — mutating it to leak String(err) passed the whole suite. Added a test that makes a store step throw a plain Error after keychain.set succeeds, and asserts the generic "Setup failed" note (not "Keychain unavailable"), no raw error / key rendered, and the key IS stored. Verified mutation-sensitive. Minors also fixed: trim the pasted key before storing (matching readSecretFromStdin, so a stray-whitespace paste can't persist a broken credential); correct the now-stale "only an unexpected fault propagates" docstring (all storage faults are absorbed; only prompt-level throws propagate); comment the defensive bare-instanceof KeychainUnavailableError disjunct (runProviderCommand always wraps it); and home.md documents the starter-model behavior + the third (unexpected-fault) fallback. The drive-home key-less test writes its starter model to an ISOLATED per-test config, not the shared tmp path. Deferred (PLAUSIBLE/MINOR, non-blocking): a temporarily LOCKED OS keychain at startup reads as "key-less" (keyFor folds KeychainUnavailableError into the env-fallback), so the wizard shows "connect a provider" rather than "keychain locked" — distinguishing them cleanly would change the shared keyFor for a rare case; noted as a follow-up. Toolchain: lint + typecheck + test (1395, +4) + build all green across 9 tasks. Refs: ADR-0047, ADR-0063, ADR-0064 Co-Authored-By: Claude --- apps/cli/src/home/drive-home.test.ts | 3 ++ apps/cli/src/home/drive-home.tsx | 3 ++ apps/cli/src/onboarding/wizard.test.ts | 68 ++++++++++++++++++++++---- apps/cli/src/onboarding/wizard.ts | 43 +++++++++++++--- docs/reference/cli/home.md | 3 +- 5 files changed, 102 insertions(+), 18 deletions(-) diff --git a/apps/cli/src/home/drive-home.test.ts b/apps/cli/src/home/drive-home.test.ts index ece8268a..b5d09ed0 100644 --- a/apps/cli/src/home/drive-home.test.ts +++ b/apps/cli/src/home/drive-home.test.ts @@ -295,6 +295,9 @@ describe('driveHome (2.5.B / ADR-0054)', () => { const { deps } = makeDeps((p) => (captured = p), { providers: keylessResolver, onboardingPrompter, + // An ISOLATED config file in the per-test cwd — the wizard's success path writes the chosen provider's starter + // model here (via writeGlobalDefaultModel), so it must not touch the shared tmp configPath other tests use. + global: { ...global, configPath: join(cwd, 'wizard-config.toml') }, }); const drivePromise = driveHome(deps); await flush(); // the wizard is async — let it settle before render() mounts ink diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index ef0e7a73..c96276b2 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -351,6 +351,9 @@ export async function driveHome(deps: HomeDeps): Promise { keychain, resolver: providers, io: deps.io, + // Reuse the SAME config-write target as the `/models` port (honors `--config`) so the wizard's starter + // model + a later `/models` pick + the started session all agree on one file (2.5.G S7/S8). + writeDefaultModel: (modelId) => writeGlobalDefaultModel(modelId, homeDir, deps.global.configPath), ...(deps.onboardingPrompter === undefined ? {} : { prompter: deps.onboardingPrompter }), }); } diff --git a/apps/cli/src/onboarding/wizard.test.ts b/apps/cli/src/onboarding/wizard.test.ts index 2e4c31d0..15f4c172 100644 --- a/apps/cli/src/onboarding/wizard.test.ts +++ b/apps/cli/src/onboarding/wizard.test.ts @@ -3,7 +3,7 @@ import { Readable } from 'node:stream'; import { createClient, createProviderStore, runMigrations, type DbClient } from '@relavium/db'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; -import { providerKeyEnvVar, type ProviderResolver } from '../engine/providers.js'; +import { KNOWN_PROVIDERS, providerKeyEnvVar, type ProviderResolver } from '../engine/providers.js'; import type { CliIo } from '../process/io.js'; import { KeychainUnavailableError, @@ -105,63 +105,111 @@ describe('runOnboardingWizard', () => { const store = (): ReturnType => createProviderStore(client.db, { uuid: () => 'id-fixed', now: () => 1 }); - it('stores the pasted key in the OS keychain + registers the provider row (secret-free)', async () => { + it('stores the pasted key + registers the provider row + sets the chosen provider\'s starter model (secret-free)', async () => { const keychain = memKeychain(); const s = store(); + const writeDefaultModel = vi.fn(); const { prompter, notes, outros } = scriptedPrompter({ provider: 'anthropic', - key: 'sk-ant-supersecret-1234', + key: ' sk-ant-supersecret-1234 ', // incidental paste whitespace — must be trimmed before storing }); - await runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io }); + await runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io, writeDefaultModel }); - // The key landed in the keychain under the provider account... + // The TRIMMED key landed in the keychain under the provider account (no stray whitespace persisted)... expect(keychain.store.get(keychainAccount('anthropic'))).toBe('sk-ant-supersecret-1234'); // ...the provider row + keychain-ref were registered... expect(s.get('anthropic')?.apiKeyKeychainRef).toBe(keychainAccount('anthropic')); + // ...the chosen provider's starter model was set as the default (so the NEXT chat binds a model whose key exists)... + expect(writeDefaultModel).toHaveBeenCalledWith(KNOWN_PROVIDERS.anthropic.testModel); // ...a "Connected" note shows only the key HINT (last 4), never the full key, and confirms + hands off. const all = [...notes, ...outros].join('\n'); expect(all).toContain('••••1234'); + expect(all).toContain(KNOWN_PROVIDERS.anthropic.testModel); // the starter model is surfaced expect(all).not.toContain('sk-ant-supersecret-1234'); // the full key NEVER appears anywhere expect(outros.some((o) => o.includes('all set'))).toBe(true); }); - it('cancelling the provider select skips setup, writes NOTHING to the keychain', async () => { + it('a non-anthropic pick sets THAT provider\'s starter model (the "working chat" fix)', async () => { const keychain = memKeychain(); const s = store(); + const writeDefaultModel = vi.fn(); + const { prompter } = scriptedPrompter({ provider: 'openai', key: 'sk-openai-xyz' }); + await runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io, writeDefaultModel }); + // NOT the anthropic default — the openai starter, so a first chat doesn't try (and fail) an anthropic key. + expect(writeDefaultModel).toHaveBeenCalledWith(KNOWN_PROVIDERS.openai.testModel); + expect(KNOWN_PROVIDERS.openai.testModel).not.toBe(KNOWN_PROVIDERS.anthropic.testModel); + }); + + it('cancelling the provider select skips setup, writes NOTHING (no key, no default model)', async () => { + const keychain = memKeychain(); + const s = store(); + const writeDefaultModel = vi.fn(); const { prompter, password, notes } = scriptedPrompter({ provider: CANCEL }); - await runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io }); + await runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io, writeDefaultModel }); expect(keychain.store.size).toBe(0); // no key captured or stored expect(password).not.toHaveBeenCalled(); // never even prompted for a key + expect(writeDefaultModel).not.toHaveBeenCalled(); // no default model written on cancel expect(notes.some((n) => n.includes('Skipped'))).toBe(true); }); it('cancelling the key prompt skips setup, writes NOTHING to the keychain', async () => { const keychain = memKeychain(); const s = store(); + const writeDefaultModel = vi.fn(); const { prompter, notes } = scriptedPrompter({ provider: 'openai', key: CANCEL }); - await runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io }); + await runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io, writeDefaultModel }); expect(keychain.store.size).toBe(0); expect(s.get('openai')).toBeUndefined(); // no row registered on cancel + expect(writeDefaultModel).not.toHaveBeenCalled(); expect(notes.some((n) => n.includes('Skipped'))).toBe(true); }); - it('on a keychain-write failure, guides to the env var — NEVER persists the key, never crashes', async () => { + it('on a keychain-write failure, guides to the env var — NEVER persists the key or a default model, never crashes', async () => { const keychain = memKeychain({ throwOnSet: true }); const s = store(); + const writeDefaultModel = vi.fn(); const { prompter, notes, outros } = scriptedPrompter({ provider: 'gemini', key: 'sk-gem-secret-9999', }); // Must resolve (not throw) — a keychain-unavailable first run degrades gracefully. await expect( - runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io }), + runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io, writeDefaultModel }), ).resolves.toBeUndefined(); expect(keychain.store.size).toBe(0); // nothing persisted to the keychain... expect(s.get('gemini')).toBeUndefined(); // ...and NO dangling provider row (keychain.set is first, so it fails atomically) + expect(writeDefaultModel).not.toHaveBeenCalled(); // no default model on a failed store const all = [...notes, ...outros].join('\n'); expect(all).toContain(providerKeyEnvVar('gemini')); // the env-var fallback is named (RELAVIUM_GEMINI_API_KEY) expect(all).not.toContain('sk-gem-secret-9999'); // the key is NEVER echoed, even in the fallback expect(all.toLowerCase()).toContain('keychain'); // the fallback explains why }); + + it('an UNEXPECTED store fault (not keychain) shows a generic note, never mislabels it "keychain" or leaks the error', async () => { + // keychain.set SUCCEEDS, but a later store step throws a PLAIN Error (a db write fault). The catch must route to + // the generic "setup failed" branch — NOT the keychain branch — and never render the thrown message. The key is + // in the keychain (set ran first), so the "key not saved" copy would LIE; the generic copy must be used instead. + const keychain = memKeychain(); + const realStore = store(); + const brokenStore = { + ...realStore, + setKeychainRef: () => { + throw new Error('db write failed: sensitive-context-xyz'); + }, + }; + const writeDefaultModel = vi.fn(); + const { prompter, notes, outros } = scriptedPrompter({ provider: 'deepseek', key: 'sk-ds-1234' }); + await expect( + runOnboardingWizard({ prompter, store: brokenStore, keychain, resolver: stubResolver, io, writeDefaultModel }), + ).resolves.toBeUndefined(); + + const all = [...notes, ...outros].join('\n'); + expect(all).toContain('Setup failed'); // the generic branch, not the keychain branch + expect(all).not.toContain('Keychain unavailable'); // never mislabel a db fault as a keychain failure + expect(all).not.toContain('db write failed'); // the raw error is NEVER rendered + expect(all).not.toContain('sk-ds-1234'); // and certainly not the key + expect(keychain.store.get(keychainAccount('deepseek'))).toBe('sk-ds-1234'); // the key IS stored (set ran first) + expect(writeDefaultModel).not.toHaveBeenCalled(); // no default model on a failed store + }); }); diff --git a/apps/cli/src/onboarding/wizard.ts b/apps/cli/src/onboarding/wizard.ts index d459a788..3538e688 100644 --- a/apps/cli/src/onboarding/wizard.ts +++ b/apps/cli/src/onboarding/wizard.ts @@ -81,6 +81,10 @@ export interface OnboardingDeps { readonly keychain: KeychainStore; readonly resolver: ProviderResolver; readonly io: CliIo; + /** Persist the NEXT session's default model (writeGlobalDefaultModel via the Home's config target). The wizard + * sets a starter model of the CHOSEN provider so the first chat binds a model whose key was just stored — the + * built-in default (`claude-sonnet-4-6` → anthropic) would otherwise error for a user who picked another provider. */ + readonly writeDefaultModel: (modelId: string) => void; } /** @@ -106,8 +110,9 @@ const requireKey = (value: string | undefined): string | undefined => /** * Run the first-run wizard. Resolves when the user has stored a key OR skipped/failed — the caller (the Home) then - * mounts as usual. Never throws for a normal cancel or a keychain-unavailable fallback (both end cleanly); only an - * unexpected fault propagates (caught by the Home's cleanup finally). + * mounts as usual. Every STORAGE outcome ends cleanly (cancel → skip; keychain-unavailable → env-var guidance; any + * other storage fault → a generic note) — none rethrow. Only a fault inside a clack prompt call itself + * (`select`/`password`/`note`/`outro`) propagates, and that is caught by the Home's cleanup `finally`. */ export async function runOnboardingWizard(deps: OnboardingDeps): Promise { const p = deps.prompter ?? defaultPrompter; @@ -130,11 +135,15 @@ export async function runOnboardingWizard(deps: OnboardingDeps): Promise { const provider = KNOWN_PROVIDER_IDS.find((id) => id === providerValue); if (provider === undefined) return skip(p); - const key = await p.password({ + const rawKey = await p.password({ message: `Paste your ${KNOWN_PROVIDERS[provider].displayName} API key`, validate: requireKey, }); - if (p.isCancel(key)) return skip(p); + if (p.isCancel(rawKey)) return skip(p); + // Trim incidental paste whitespace (a stray space from a dashboard copy) before storing — matching + // `readSecretFromStdin`'s `.trim()` so `provider set-key` and the wizard persist byte-identical credentials. + // `requireKey` already rejected a whitespace-only value, so the trimmed key is non-empty. + const key = rawKey.trim(); // Store via the TESTED providerSetKey path (keychain.set + the provider row + the keychain-ref, secret-free). Its // one stdout line is suppressed (a silent io) so the wizard's output stays uniformly clack-styled — the wizard @@ -156,7 +165,8 @@ export async function runOnboardingWizard(deps: OnboardingDeps): Promise { // fault (e.g. a db write). `providerSetKey` writes to the keychain FIRST, so a keychain-unavailable failure // persists NOTHING; mislabeling a *post*-`keychain.set` fault (a later db upsert) as "keychain unavailable, key // not saved" would LIE — the key IS in the keychain then. `runProviderCommand` wraps `KeychainUnavailableError` - // in a `CliError(cause)`. Never render the raw error (it could carry context) — both branches use static text. + // in a `CliError(cause)`; the bare-`instanceof KeychainUnavailableError` disjunct is a DEFENSIVE fallback for a + // future direct caller. Never render the raw error (it could carry context) — both branches use static text. const keychainDown = err instanceof KeychainUnavailableError || (err instanceof CliError && err.cause instanceof KeychainUnavailableError); @@ -180,8 +190,27 @@ export async function runOnboardingWizard(deps: OnboardingDeps): Promise { p.outro('Setup incomplete — see the note above.'); return; } - // Success — only reached when the key was actually stored (keychain.set + the provider row + the keychain-ref). - p.note(`Stored your ${provider} key (${keyHint(key)}) in the OS keychain.`, 'Connected'); + + // Set a working default model of the CHOSEN provider so the very NEXT chat binds a model whose key was just + // stored — the built-in default (`claude-sonnet-4-6` → anthropic) would otherwise error for a user who picked a + // different provider (breaking the "reach a working chat" promise). `testModel` is a cheap/fast, priced starter; + // the user upgrades via `/models`. Best-effort: a config-write fault still leaves a working key (fall back to the + // `/models` pointer) rather than undoing the store. + const starterModel = KNOWN_PROVIDERS[provider].testModel; + try { + deps.writeDefaultModel(starterModel); + p.note( + `Stored your ${provider} key (${keyHint(key)}) in the OS keychain.\n` + + `Your default model is ${starterModel} — change it anytime with /models.`, + 'Connected', + ); + } catch { + p.note( + `Stored your ${provider} key (${keyHint(key)}) in the OS keychain.\n` + + `Pick a ${provider} model with /models to start chatting.`, + 'Connected', + ); + } p.outro("You're all set — starting Relavium."); } diff --git a/docs/reference/cli/home.md b/docs/reference/cli/home.md index 97fcf963..75ff4255 100644 --- a/docs/reference/cli/home.md +++ b/docs/reference/cli/home.md @@ -27,9 +27,10 @@ stdoutIsTty && stdinIsTty && !json && !isCiEnv(env) When a bare `relavium` opens the Home on a **truly key-less** run — **no** known provider has a resolvable key (the resolver finds neither an OS-keychain key **nor** a `RELAVIUM__API_KEY` env var) — a `@clack/prompts` **onboarding wizard** runs **before** the ink Home mounts (clack + ink both take the terminal's raw mode, so the wizard fully settles first). It is already behind the TTY gate above, so it never runs piped / `--json` / in CI. A run with **either** a keychain key or an env key is not key-less — no wizard, so a working user (including an env-key user) is never prompted. -The flow: **pick a provider → paste a hidden (masked) API key → store it in the OS keychain** — riding the same tested `provider set-key` path (keychain write + the provider row + the keychain-ref, secret-free by construction; the key is captured via clack's masked `password`, never echoed, never written to disk, never logged beyond its last-4 hint). **Model selection is not part of the wizard** — its job is to store the key that lights up the `/models` picker + chat; the user picks a model there. Two fallbacks keep a first run unblocked: +The flow: **pick a provider → paste a hidden (masked) API key → store it in the OS keychain** — riding the same tested `provider set-key` path (keychain write + the provider row + the keychain-ref, secret-free by construction; the key is captured via clack's masked `password`, whitespace-trimmed, never echoed, never written to disk, never logged beyond its last-4 hint). On a successful store the wizard also sets **`[chat].default_model` to the chosen provider's cheap/fast starter model** (`KNOWN_PROVIDERS[provider].testModel`, via the same config-write target as `/models`) — so the very next chat binds a model **whose key was just stored**, not the built-in `claude-sonnet-4-6` (which would error for a user who picked a non-Anthropic provider). Richer **model selection** is not part of the wizard — the user upgrades via the `/models` picker. Three fallbacks keep a first run unblocked: - **Keychain-write failure** (a locked keychain / no Secret Service / a headless box) → the wizard **never** persists the key to disk; it prints the `RELAVIUM__API_KEY` env-var to set instead (the resolver imports an env key at call time), then hands off to the Home. +- **Any other store fault** (e.g. a db write failure *after* the keychain write succeeded) → a **generic** "setup could not be completed" note (never mislabeled a keychain failure, and the raw error is never rendered — the key may well be in the keychain, so the copy must not claim otherwise); the underlying issue resurfaces at the Home, which reads the same store. - **Cancel** (Ctrl-C / Esc at any step) → a friendly pointer to `relavium provider add` / `/doctor`, then the Home mounts key-less (retry next launch, or add a key manually). `@clack/prompts` is confined to one module behind an injectable seam (mirroring the `create` wizard + the gate prompter — [ADR-0047](../../decisions/0047-cli-framework-commander-ink-clack.md)), so the flow unit-tests without a TTY. No new ADR: the wizard composes the config-write ([ADR-0063](../../decisions/0063-cli-config-write-contract.md)), keychain ([ADR-0006](../../decisions/0006-os-keychain-for-api-keys.md)/[ADR-0019](../../decisions/0019-cli-node-keychain-library.md)), and clack ([ADR-0047](../../decisions/0047-cli-framework-commander-ink-clack.md)) decisions already accepted. From 634906a5b7acce6bae3f0ee41bc00cca0325bedc Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 04:22:54 +0300 Subject: [PATCH 24/68] =?UTF-8?q?feat(cli,db,llm):=202.5.G=20S9=20?= =?UTF-8?q?=E2=80=94=20custom=20OpenAI-compatible=20base=5Furl=20over=20th?= =?UTF-8?q?e=20SSRF-validated=20hop?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fix the "dead base_url" bug (2.5.G S9, ADR-0065 §3–4): a `provider add --base-url` was stored but NEVER used at routing — `createProviderResolver` was never handed the ProviderStore and always built the default-endpoint adapters. Now a stored custom `base_url` rebinds its provider's adapter to the custom endpoint, and ALL its egress (streaming generate/stream + the models.list refresh) rides the shared DNS-rebinding-safe validated hop. The pieces: - validated-fetch.ts (🔒): a `fetch`-shaped wrapper over the one shared `connectValidated` SSRF primitive (HTTPS + no-creds → DNS-resolve → range-block every IP → connect PINNED to the validated IP with hostname SNI). connectValidated returns a LIVE AsyncIterable body (never buffers), wrapped in a backpressure-aware ReadableStream — so a custom-endpoint SSE completion streams chunk-by-chunk. Secret-free (reason-only errors); the Authorization key rides the request headers to the endpoint, never logged. The EgressDeps (DNS + connect) are injectable for deterministic SSRF tests. - @relavium/llm `createCustomOpenAiProvider`: a purpose-built seam factory for a custom openai-compatible endpoint (reuses the internal adapter + its assertHttpsBaseUrl gate; the host injects the validated fetch). No vendor SDK type crosses the seam; the adapters stay internal. - providers.ts: createProviderResolver gains a ProviderResolverOptions {providerStore, validatedFetch}; applyCustomEndpoints rebinds a custom openai/deepseek base_url (anthropic/gemini skipped — refused at add; a bad base_url is caught + skipped, never crashing resolver creation). - provider.ts: `provider add --base-url` refuses a non-openai-compatible provider (clear message) + fail-fast rejects a non-HTTPS / private-loopback / credential-bearing URL (reusing the shared primitives), and stores the protocol `kind`. - db: migration 0008 adds `llm_providers.kind` + `pricing_reference_url` (nullable, validated at the store boundary like model_catalog.source); the ProviderRecord/upsert/fromRow carry them. - wiring: the resolver is now STORE-AWARE at every path — models refresh + provider commands (they already hold the db+store), the Home (built in the S7 port block, no double-open), and run/chat/gate (a self-contained short-lived history.db read, since applyCustomEndpoints reads list() once). The ProviderId enum stays CLOSED (a custom endpoint reuses openai/deepseek). Docs: database-schema.md (the 0008 columns) + commands.md (custom base_url is now live, openai-compatible-only, SSRF-gated). Tests: validated-fetch SSRF+streaming (private blocked, non-HTTPS rejected, pinning, chunked streaming, early-cancel disposal, unsupported method, null-body), the custom-endpoint resolver (custom routed via the injected fetch; bad/anthropic base_url skipped without crash), the provider-add refusals + kind storage, and the provider-store kind round-trip + foreign- value coercion. Toolchain: lint + typecheck + test (CLI 1411 + db 184 + llm 522) + build all green across all 24 workspace tasks; seam clean (the lone media-gc failure is the known pre-existing timing flake — passes on retry). Refs: ADR-0065, ADR-0064, ADR-0029, ADR-0053 Co-Authored-By: Claude --- apps/cli/src/commands/dispatch.ts | 67 +- apps/cli/src/commands/provider.test.ts | 39 + apps/cli/src/commands/provider.ts | 54 +- apps/cli/src/engine/providers.test.ts | 87 +- apps/cli/src/engine/providers.ts | 62 +- apps/cli/src/engine/validated-fetch.test.ts | 117 ++ apps/cli/src/engine/validated-fetch.ts | 146 ++ apps/cli/src/home/drive-home.tsx | 26 +- docs/reference/cli/commands.md | 2 +- docs/reference/desktop/database-schema.md | 4 +- packages/db/drizzle/0008_plain_scourge.sql | 2 + packages/db/drizzle/meta/0008_snapshot.json | 2091 +++++++++++++++++++ packages/db/drizzle/meta/_journal.json | 7 + packages/db/src/provider-store.test.ts | 37 +- packages/db/src/provider-store.ts | 31 + packages/db/src/schema.ts | 7 + packages/llm/src/index.ts | 3 +- packages/llm/src/providers.ts | 24 + 18 files changed, 2757 insertions(+), 49 deletions(-) create mode 100644 apps/cli/src/engine/validated-fetch.test.ts create mode 100644 apps/cli/src/engine/validated-fetch.ts create mode 100644 packages/db/drizzle/0008_plain_scourge.sql create mode 100644 packages/db/drizzle/meta/0008_snapshot.json diff --git a/apps/cli/src/commands/dispatch.ts b/apps/cli/src/commands/dispatch.ts index f45bbc04..1aa9e56f 100644 --- a/apps/cli/src/commands/dispatch.ts +++ b/apps/cli/src/commands/dispatch.ts @@ -1,6 +1,6 @@ import { randomUUID } from 'node:crypto'; -import { createModelCatalogStore, createProviderStore } from '@relavium/db'; +import { createModelCatalogStore, createProviderStore, type ProviderStore } from '@relavium/db'; import { loadResolvedConfig } from '../config/load.js'; import { openLocalDb, type OpenedDb } from '../db/open.js'; @@ -18,6 +18,7 @@ import { type ExitCode } from '../process/exit-codes.js'; import type { CliIo } from '../process/io.js'; import type { GlobalOptions } from '../process/options.js'; import { selectChatDriver } from '../render/tui/chat-ink.js'; +import type { KeychainStore } from '../secrets/keychain.js'; import { createMcpSecretResolver } from '../secrets/mcp-secret.js'; import { createOsKeychainStore } from '../secrets/os-keychain.js'; import { readSecretFromStdin } from '../secrets/read-secret.js'; @@ -195,21 +196,45 @@ export function buildProviderTestArgs(input: CommandInput): ProviderCommandArgs // ── executors (production dep wiring, copied verbatim from the old register* bodies) ── +/** + * Build a **store-aware** provider resolver (2.5.G S9, [ADR-0065](../../../../docs/decisions/0065-provider-economics-and-extensibility.md) §4): + * open the durable `history.db` briefly to read the provider registry so a stored **custom `base_url`** rebinds + * its adapter to the SSRF-validated endpoint, then close it. The custom adapters are built EAGERLY at resolver + * creation (`applyCustomEndpoints` reads `list()` once), so no db handle is held past this call — a self-contained + * short-lived read that needs no lifecycle threaded into the command's own db/teardown ordering. + */ +function storeAwareResolver( + ctx: DispatchContext, + keychain: KeychainStore, +): ReturnType { + const { homeDir } = loadResolvedConfig({ + cwd: ctx.global.cwd, + configPath: ctx.global.configPath, + }); + const { db, close } = openLocalDb(homeDir); + try { + const providerStore = createProviderStore(db, { uuid: () => randomUUID(), now: () => Date.now() }); + return createProviderResolver(ctx.io.env, keychain, { providerStore }); + } finally { + close(); + } +} + /** One native keychain accessor, shared by the key resolver (2.C) + the MCP named-secret resolver (2.R §6). */ -function keyResolvers(io: CliIo): { +function keyResolvers(ctx: DispatchContext): { providers: ReturnType; mcpSecretResolver: ReturnType; } { const keychain = createOsKeychainStore(); return { - providers: createProviderResolver(io.env, keychain), - mcpSecretResolver: createMcpSecretResolver(io.env, keychain), + providers: storeAwareResolver(ctx, keychain), + mcpSecretResolver: createMcpSecretResolver(ctx.io.env, keychain), }; } -/** The env-backed provider resolver alone (a command — like `gate` — that needs keys but not MCP secrets). */ -function providerResolver(io: CliIo): ReturnType { - return createProviderResolver(io.env, createOsKeychainStore()); +/** The store-aware provider resolver alone (a command — like `gate` — that needs keys but not MCP secrets). */ +function providerResolver(ctx: DispatchContext): ReturnType { + return storeAwareResolver(ctx, createOsKeychainStore()); } const executeRun: CommandExecutor = (input, ctx) => @@ -217,14 +242,14 @@ const executeRun: CommandExecutor = (input, ctx) => io: ctx.io, global: ctx.global, openRunStore: openHistoryStore, - ...keyResolvers(ctx.io), + ...keyResolvers(ctx), }); const executeChat: CommandExecutor = (input, ctx) => chatCommand(buildChatArgs(input), { io: ctx.io, global: ctx.global, - ...keyResolvers(ctx.io), + ...keyResolvers(ctx), openSessionStore, drive: selectChatDriver, }); @@ -235,7 +260,7 @@ const executeChatResume: CommandExecutor = (input, ctx) => { io: ctx.io, global: ctx.global, - ...keyResolvers(ctx.io), + ...keyResolvers(ctx), openSessionStore, drive: selectChatDriver, }, @@ -266,7 +291,7 @@ const executeAgentRun: CommandExecutor = (input, ctx) => agentRunCommand(buildAgentRunArgs(input), { io: ctx.io, global: ctx.global, - ...keyResolvers(ctx.io), + ...keyResolvers(ctx), }); const executeGate: CommandExecutor = (input, ctx) => @@ -274,7 +299,7 @@ const executeGate: CommandExecutor = (input, ctx) => io: ctx.io, global: ctx.global, // Production resolves a post-gate agent's key via the OS keychain → env var (2.C), like `run`. - providers: providerResolver(ctx.io), + providers: providerResolver(ctx), }); const executeGateList: CommandExecutor = (input, ctx) => { @@ -323,12 +348,18 @@ export function createProviderSlugResolver( */ export interface ModelsDbPorts { readonly openDb: (homeDir: string) => OpenedDb; - readonly makeResolver: (io: CliIo) => Pick; + /** Build the key resolver over the models db — the `providerStore` (from the SAME db) makes it store-aware so a + * custom `base_url` lists models over the SSRF-validated hop (2.5.G S9, ADR-0065 §4). A test stub may ignore it. */ + readonly makeResolver: ( + io: CliIo, + providerStore: Pick, + ) => Pick; } const PRODUCTION_MODELS_PORTS: ModelsDbPorts = { openDb: openLocalDb, - makeResolver: (io) => createProviderResolver(io.env, createOsKeychainStore()), + makeResolver: (io, providerStore) => + createProviderResolver(io.env, createOsKeychainStore(), { providerStore }), }; /** @@ -349,8 +380,8 @@ export async function withModelsDeps( const { db, close } = ports.openDb(homeDir); try { const storeDeps = { uuid: () => randomUUID(), now: () => Date.now() }; - const resolver = ports.makeResolver(ctx.io); const providerStore = createProviderStore(db, storeDeps); + const resolver = ports.makeResolver(ctx.io, providerStore); // store-aware ⇒ a custom base_url is used (S9) const catalogStore = createModelCatalogStore(db, storeDeps); const refreshService = createModelRefreshService({ resolveProvider: resolver.resolveProvider, @@ -389,11 +420,13 @@ async function withProviderDeps( const { db, close } = openLocalDb(homeDir); try { const keychain = createOsKeychainStore(); // one native accessor, shared by the store-ref writes + the resolver + const store = createProviderStore(db, { uuid: () => randomUUID(), now: () => Date.now() }); const deps: ProviderCommandDeps = { io: ctx.io, - store: createProviderStore(db, { uuid: () => randomUUID(), now: () => Date.now() }), + store, keychain, - resolver: createProviderResolver(ctx.io.env, keychain), + // Store-aware so `provider test` pings a custom `base_url` provider at its CUSTOM endpoint (2.5.G S9). + resolver: createProviderResolver(ctx.io.env, keychain, { providerStore: store }), readSecret: readSecretFromStdin, }; return await fn(deps); diff --git a/apps/cli/src/commands/provider.test.ts b/apps/cli/src/commands/provider.test.ts index 35c8c7b2..e5266de4 100644 --- a/apps/cli/src/commands/provider.test.ts +++ b/apps/cli/src/commands/provider.test.ts @@ -157,6 +157,45 @@ describe('relavium provider commands (2.C)', () => { } }); + it('rejects a private/loopback or credential-bearing --base-url (fail-fast SSRF, exit 2) (2.5.G S9)', async () => { + for (const baseUrl of [ + 'https://localhost/v1', + 'https://127.0.0.1/v1', + 'https://192.168.1.10/v1', + 'https://169.254.169.254/latest', // link-local metadata + 'https://user:pass@proxy.example/v1', // embedded credentials + ]) { + await expect( + runProviderCommand({ action: 'add', name: 'openai', baseUrl }, deps({})), + ).rejects.toMatchObject({ exitCode: 2 }); + } + }); + + it('refuses a custom --base-url on a NON-OpenAI-compatible provider (anthropic/gemini, exit 2) (ADR-0065 §3)', async () => { + for (const name of ['anthropic', 'gemini']) { + await expect( + runProviderCommand({ action: 'add', name, baseUrl: 'https://proxy.example/v1' }, deps({})), + ).rejects.toMatchObject({ exitCode: 2 }); + } + }); + + it('stores a custom openai-compatible --base-url + the provider kind (ADR-0065 §3/§5)', async () => { + const d = deps({}); + await runProviderCommand( + { action: 'add', name: 'deepseek', baseUrl: 'https://my-proxy.example/v1' }, + d, + ); + const row = d.store.get('deepseek'); + expect(row?.baseUrl).toBe('https://my-proxy.example/v1'); + expect(row?.kind).toBe('openai-compatible'); // the protocol kind is populated (§5) + }); + + it('populates the provider kind on a plain add (no --base-url) too — for uniformity (§5)', async () => { + const d = deps({}); + await runProviderCommand({ action: 'add', name: 'anthropic' }, d); + expect(d.store.get('anthropic')?.kind).toBe('anthropic'); + }); + it('set-key preserves a base URL set by a prior `add` (never clobbers it)', async () => { const d = deps({}); await runProviderCommand( diff --git a/apps/cli/src/commands/provider.ts b/apps/cli/src/commands/provider.ts index 10b9c85b..7ddc4c70 100644 --- a/apps/cli/src/commands/provider.ts +++ b/apps/cli/src/commands/provider.ts @@ -1,5 +1,6 @@ -import { ProviderIdSchema, type ProviderId } from '@relavium/llm'; +import { providerKind, ProviderIdSchema, type ProviderId } from '@relavium/llm'; import type { ProviderStore } from '@relavium/db'; +import { isPrivateOrLocalHost, urlHasCredentials } from '@relavium/shared'; import { KNOWN_PROVIDERS, @@ -84,11 +85,21 @@ function providerList(deps: ProviderCommandDeps): void { function providerAdd(args: ProviderCommandArgs, deps: ProviderCommandDeps): void { const id = parseProviderId(requireName(args)); const meta = KNOWN_PROVIDERS[id]; - const record = deps.store.upsert({ - name: id, - displayName: meta.displayName, - baseUrl: args.baseUrl === undefined ? meta.baseUrl : requireHttpsUrl(args.baseUrl), - }); + let baseUrl = meta.baseUrl; + if (args.baseUrl !== undefined) { + // A custom `--base-url` is **OpenAI-compatible only** this round (2.5.G S9, ADR-0065 §3) — refuse it on the + // Anthropic/Gemini protocols with a clear message rather than silently ignoring it (the old dead-config bug). + if (providerKind(id) !== 'openai-compatible') { + throw new CliError( + 'invalid_invocation', + `a custom --base-url is only supported for OpenAI-compatible providers (openai, deepseek); '${id}' uses the ${providerKind(id)} protocol.`, + ); + } + baseUrl = requireHttpsUrl(args.baseUrl); + } + // Store the protocol `kind` for every provider (ADR-0065 §5 — populated for uniformity; the resolver derives it + // from the closed id today, load-bearing only for a future custom provider). + const record = deps.store.upsert({ name: id, displayName: meta.displayName, baseUrl, kind: providerKind(id) }); deps.io.writeOut( `Registered provider '${id}' (${record.baseUrl}). Store a key with \`relavium provider set-key ${id}\`.\n`, ); @@ -102,7 +113,12 @@ async function providerSetKey(args: ProviderCommandArgs, deps: ProviderCommandDe deps.keychain.set(account, key); // KeychainUnavailableError surfaces (no silent plaintext fallback) // Register the row only if it's new — never overwrite a base URL the user set via `provider add --base-url`. if (deps.store.get(id) === undefined) { - deps.store.upsert({ name: id, displayName: meta.displayName, baseUrl: meta.baseUrl }); + deps.store.upsert({ + name: id, + displayName: meta.displayName, + baseUrl: meta.baseUrl, + kind: providerKind(id), + }); } deps.store.setKeychainRef(id, account); // the ref, NEVER the key value deps.io.writeOut(`Stored ${id} key ${keyHint(key)} in the OS keychain.\n`); @@ -135,12 +151,14 @@ async function providerTest(args: ProviderCommandArgs, deps: ProviderCommandDeps } /** - * Validate a user-supplied provider base URL: a parseable **HTTPS** URL (fail-fast at `add`). HTTPS-only - * matches the at-routing-time gate (`@relavium/llm`'s `assertHttpsBaseUrl`, security-review.md §Network) and - * keeps a plaintext `http:` endpoint from ever being persisted; the private/loopback/metadata range-block - * stays at the routing-time gate (it needs host resolution this fail-fast deliberately does not do). The - * value is stored **verbatim** (not `url.href`-normalized) to preserve the user's exact endpoint/trailing - * slash; the routing-time gate re-parses it via `new URL()` anyway. + * Validate a user-supplied provider base URL (fail-fast at `add`, 2.5.G S9): a parseable **HTTPS** URL, with no + * embedded credentials and no **literal** private/loopback/link-local/metadata host — reusing the SAME shared + * string-level primitives (`urlHasCredentials` / `isPrivateOrLocalHost`) the adapter's construction-time + * `assertHttpsBaseUrl` and the `connectValidated` egress gate use, never a second hand-rolled parser (ADR-0029(d)). + * The literal-host block gives a clear "no private address" error at `add`; the **resolve-time** DNS-rebinding + * defense (a public hostname resolving to a private IP) stays with `connectValidated` (it needs host resolution + * this fail-fast deliberately does not do). The value is stored **verbatim** (not `url.href`-normalized) to + * preserve the user's exact endpoint/trailing slash; the routing-time gate re-parses it via `new URL()` anyway. */ function requireHttpsUrl(raw: string): string { let url: URL; @@ -152,6 +170,16 @@ function requireHttpsUrl(raw: string): string { if (url.protocol !== 'https:') { throw new CliError('invalid_invocation', `base URL must be HTTPS, got '${raw}'.`); } + if (urlHasCredentials(raw)) { + throw new CliError('invalid_invocation', 'base URL must not embed credentials (user:pass@…).'); + } + const host = url.hostname.toLowerCase().replace(/^\[/, '').replace(/\]$/, ''); + if (isPrivateOrLocalHost(host)) { + throw new CliError( + 'invalid_invocation', + 'base URL must not be a private, loopback, or link-local address.', + ); + } return raw; } diff --git a/apps/cli/src/engine/providers.test.ts b/apps/cli/src/engine/providers.test.ts index c0988ca1..9b27488d 100644 --- a/apps/cli/src/engine/providers.test.ts +++ b/apps/cli/src/engine/providers.test.ts @@ -1,9 +1,17 @@ import { parseWorkflow, type WorkflowDefinition } from '@relavium/core'; +import type { ProviderRecord } from '@relavium/db'; import type { LlmProvider } from '@relavium/llm'; import { describe, expect, it, vi } from 'vitest'; import { CHAT_TEXT_CAPABILITY_FLAGS } from '../test-support.js'; -import { neededProviderIds, providerKeyEnvVar, validateProviderKey } from './providers.js'; +import { + createProviderResolver, + KNOWN_PROVIDERS, + neededProviderIds, + providerKeyEnvVar, + validateProviderKey, +} from './providers.js'; +import type { FetchLike } from './validated-fetch.js'; // A test key assembled at runtime (no contiguous secret literal — leakwatch). const TEST_KEY = ['sk', 'prov', '90ABCDEF'].join('-'); @@ -183,3 +191,80 @@ describe('validateProviderKey', () => { expect(result.detail).not.toContain(TEST_KEY); }); }); + +describe('createProviderResolver custom endpoints (2.5.G S9 / ADR-0065 §3–4)', () => { + /** A minimal provider registry row (only `name` + `baseUrl` matter to the custom-endpoint rebinding). */ + function row(name: string, baseUrl: string): ProviderRecord { + return { + id: `id-${name}`, + name, + displayName: name, + baseUrl, + defaultHeaders: {}, + isActive: true, + createdAt: '2026-01-01T00:00:00.000Z', + updatedAt: '2026-01-01T00:00:00.000Z', + }; + } + + /** A fake validated fetch that records the request URLs + returns a valid OpenAI models-list payload. */ + function recordingFetch(): { fetch: FetchLike; urls: string[] } { + const urls: string[] = []; + const fetch: FetchLike = (input) => { + urls.push(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url); + return Promise.resolve( + new Response(JSON.stringify({ object: 'list', data: [{ id: 'gpt-5.4-mini', object: 'model' }] }), { + status: 200, + headers: { 'content-type': 'application/json' }, + }), + ); + }; + return { fetch, urls }; + } + + it('routes a CUSTOM openai-compatible base_url through the injected validated fetch', async () => { + const { fetch, urls } = recordingFetch(); + const resolver = createProviderResolver({}, undefined, { + providerStore: { list: () => [row('openai', 'https://my-proxy.example/v1')] }, + validatedFetch: fetch, + }); + const openai = resolver.resolveProvider('openai'); + expect(openai?.listModels !== undefined).toBe(true); // the openai-compatible adapter carries the live-list capability + // The listModels egress hits the CUSTOM endpoint (the dead-base_url bug is fixed) — via the validated fetch. + await openai?.listModels?.('sk-test')?.catch(() => undefined); // the ROUTING is the assertion, not the parse result + expect(urls.some((url) => url.startsWith('https://my-proxy.example/v1'))).toBe(true); + expect(urls.every((url) => !url.startsWith('https://api.openai.com'))).toBe(true); // NOT the default endpoint + }); + + it('SKIPS a bad (private) custom base_url — the resolver still builds, keeping the default adapter (no crash)', () => { + const { fetch } = recordingFetch(); + const resolver = createProviderResolver({}, undefined, { + providerStore: { list: () => [row('openai', 'https://127.0.0.1/v1')] }, // fails the adapter's HTTPS+private gate + validatedFetch: fetch, + }); + expect(resolver.resolveProvider('openai')).toBeDefined(); // InvalidBaseUrlError caught; default adapter stands + }); + + it('SKIPS a custom base_url on anthropic/gemini (openai-compatible only this round) — no crash', () => { + const { fetch } = recordingFetch(); + const resolver = createProviderResolver({}, undefined, { + providerStore: { + list: () => [row('anthropic', 'https://custom.example'), row('gemini', 'https://custom.example')], + }, + validatedFetch: fetch, + }); + expect(resolver.resolveProvider('anthropic')).toBeDefined(); + expect(resolver.resolveProvider('gemini')).toBeDefined(); + }); + + it('leaves a DEFAULT base_url row alone — no custom adapter is built (the resolver builds cleanly)', () => { + const { fetch } = recordingFetch(); + const resolver = createProviderResolver({}, undefined, { + // The known default base_url is NOT custom ⇒ applyCustomEndpoints skips it, keeping the default adapter (no + // injected validated fetch wired — proven by the positive routing test above, without a real network call). + providerStore: { list: () => [row('openai', KNOWN_PROVIDERS.openai.baseUrl)] }, + validatedFetch: fetch, + }); + expect(resolver.resolveProvider('openai')).toBeDefined(); + }); +}); diff --git a/apps/cli/src/engine/providers.ts b/apps/cli/src/engine/providers.ts index c0c18ba7..2683096b 100644 --- a/apps/cli/src/engine/providers.ts +++ b/apps/cli/src/engine/providers.ts @@ -1,5 +1,12 @@ import type { WorkflowDefinition } from '@relavium/core'; -import { defaultProviders, type LlmProvider, type ProviderId } from '@relavium/llm'; +import type { ProviderStore } from '@relavium/db'; +import { + createCustomOpenAiProvider, + defaultProviders, + InvalidBaseUrlError, + type LlmProvider, + type ProviderId, +} from '@relavium/llm'; import type { Agent } from '@relavium/shared'; import { CliError } from '../process/errors.js'; @@ -8,6 +15,7 @@ import { keychainAccount, type KeychainStore, } from '../secrets/keychain.js'; +import { createValidatedFetch, type FetchLike } from './validated-fetch.js'; /** * The CLI's provider seam (ADR-0038 host-injected resolution). `resolveProvider` returns the keyless @@ -183,13 +191,27 @@ export function neededProviderIds(def: WorkflowDefinition): ProviderId[] { return [...needed]; } +/** Options threading the provider registry (for custom endpoints) into the resolver — all optional so the pre-S9 + * callers (and tests) that pass none keep the static default-endpoint behavior. */ +export interface ProviderResolverOptions { + /** The provider registry — a row carrying a CUSTOM `base_url` (≠ the known default) rebinds its provider's + * adapter to a validated per-provider endpoint (2.5.G S9, ADR-0065 §3–4). Absent ⇒ default endpoints only. */ + readonly providerStore?: Pick; + /** The SSRF-validated `fetch` custom endpoints egress through — injectable so a test drives a fake `EgressDeps`. + * Default: {@link createValidatedFetch} over Node's real DNS + pinned-HTTPS connect. */ + readonly validatedFetch?: FetchLike; +} + export function createProviderResolver( env: Readonly> = process.env, keychain?: KeychainStore, + options: ProviderResolverOptions = {}, ): ProviderResolver { - // Keyless adapters built once and reused; the key is injected per call via `keyFor`. The - // provider→adapter mapping lives in the seam package (`@relavium/llm`), not here. - const adapters = defaultProviders(); + // Keyless adapters built once and reused; the key is injected per call via `keyFor`. The default provider→adapter + // mapping lives in the seam package (`@relavium/llm`); a stored CUSTOM `base_url` (ADR-0065 §3) rebinds its + // provider's adapter to a validated per-provider endpoint here. + const adapters: Record = { ...defaultProviders() }; + applyCustomEndpoints(adapters, options); return { resolveProvider: (id) => adapters[id], keyFor: (id) => { @@ -222,3 +244,35 @@ export function createProviderResolver( }, }; } + +/** + * Rebind a provider's adapter to a validated CUSTOM-endpoint adapter when its stored row carries a `base_url` that + * differs from the known default (2.5.G S9, ADR-0065 §3–4). Only **OpenAI-compatible** (`openai`/`deepseek`) is + * supported this round; `provider add` refuses a custom `base_url` on `anthropic`/`gemini`, so a stored one on them + * shouldn't exist — skipped defensively. The custom endpoint's egress rides the host's **SSRF-validated fetch** + * (`connectValidated`), and the adapter's construction-time `assertHttpsBaseUrl` (HTTPS + private-range + no-creds) + * gate re-validates the URL. A `base_url` that fails that gate is **skipped** (the default endpoint stands) rather + * than crashing resolver creation for EVERY command — the fail-fast refusal is at `provider add`; this is the + * defensive net for a pre-S9 / tampered row. + */ +function applyCustomEndpoints( + adapters: Record, + options: ProviderResolverOptions, +): void { + const store = options.providerStore; + if (store === undefined) return; // no registry ⇒ default endpoints only (the pre-S9 behavior) + let validatedFetch: FetchLike | undefined; + for (const row of store.list()) { + const id = KNOWN_PROVIDER_IDS.find((known) => known === row.name); + if (id === undefined) continue; // a non-enum name (provider add enforces the closed enum) — ignore + if (row.baseUrl === KNOWN_PROVIDERS[id].baseUrl) continue; // the default endpoint — keep the default adapter + if (id !== 'openai' && id !== 'deepseek') continue; // custom base_url is openai-compatible only this round (§3) + validatedFetch ??= options.validatedFetch ?? createValidatedFetch(); // built lazily, once, only when needed + try { + adapters[id] = createCustomOpenAiProvider({ providerId: id, baseURL: row.baseUrl, fetch: validatedFetch }); + } catch (err) { + // A bad stored base_url (non-HTTPS / private / creds) — refuse the custom endpoint, keep the default adapter. + if (!(err instanceof InvalidBaseUrlError)) throw err; + } + } +} diff --git a/apps/cli/src/engine/validated-fetch.test.ts b/apps/cli/src/engine/validated-fetch.test.ts new file mode 100644 index 00000000..61386e61 --- /dev/null +++ b/apps/cli/src/engine/validated-fetch.test.ts @@ -0,0 +1,117 @@ +import { SafeEgressError, type EgressDeps, type HopRequest, type HopResponse } from '@relavium/db'; +import { describe, expect, it, vi } from 'vitest'; + +import { createValidatedFetch } from './validated-fetch.js'; + +/** An async-iterable body from fixed byte chunks (the live streaming shape connectValidated returns). */ +async function* bytes(chunks: readonly string[]): AsyncGenerator { + await Promise.resolve(); // yields the encoded chunks (the leading await keeps this a genuine async generator) + for (const chunk of chunks) yield new TextEncoder().encode(chunk); +} + +/** + * A fake {@link EgressDeps}: `resolveHost` returns the configured IP(s) (so the SSRF range-block is deterministic + * without DNS), and `openConnection` returns a scripted {@link HopResponse} + captures the pinned request. + */ +function fakeDeps(opts: { + ips: readonly string[]; + status?: number; + headers?: Record; + chunks?: readonly string[]; + onConnect?: (req: HopRequest) => void; + dispose?: () => void; +}): EgressDeps { + return { + resolveHost: () => Promise.resolve(opts.ips), + openConnection: (req: HopRequest): Promise => { + opts.onConnect?.(req); + return Promise.resolve({ + status: opts.status ?? 200, + headers: opts.headers ?? { 'content-type': 'application/json' }, + location: undefined, + body: bytes(opts.chunks ?? ['{"ok":true}']), + dispose: opts.dispose ?? ((): void => undefined), + }); + }, + }; +} + +describe('createValidatedFetch', () => { + it('routes a request through connectValidated and maps the response (status + headers + body)', async () => { + const fetch = createValidatedFetch(fakeDeps({ ips: ['1.2.3.4'], status: 200, chunks: ['{"models":[]}'] })); + const res = await fetch('https://api.example.com/v1/models'); + expect(res.status).toBe(200); + expect(res.headers.get('content-type')).toBe('application/json'); + expect(await res.json()).toEqual({ models: [] }); + }); + + it('BLOCKS a host that resolves to a private/loopback address (SSRF) — the fetch rejects', async () => { + const fetch = createValidatedFetch(fakeDeps({ ips: ['10.0.0.1'] })); // a private IP + await expect(fetch('https://sneaky-rebind.example.com/v1/models')).rejects.toBeInstanceOf(SafeEgressError); + }); + + it('rejects a non-HTTPS url (the shared HTTPS policy), never connecting', async () => { + const onConnect = vi.fn(); + const fetch = createValidatedFetch(fakeDeps({ ips: ['1.2.3.4'], onConnect })); + await expect(fetch('http://api.example.com/v1/models')).rejects.toBeInstanceOf(SafeEgressError); + expect(onConnect).not.toHaveBeenCalled(); + }); + + it('pins the connection to the validated IP and passes method + headers + body through', async () => { + let captured: HopRequest | undefined; + const fetch = createValidatedFetch(fakeDeps({ ips: ['203.0.113.7'], onConnect: (req) => (captured = req) })); + await fetch('https://api.example.com/v1/chat/completions', { + method: 'POST', + headers: { authorization: 'Bearer sk-secret', 'content-type': 'application/json' }, + body: '{"model":"x"}', + }); + expect(captured?.pinnedIp).toBe('203.0.113.7'); // connect-by-validated-IP (no re-resolve TOCTOU) + expect(captured?.method).toBe('POST'); + expect(captured?.hostname).toBe('api.example.com'); + expect(captured?.headers?.['authorization']).toBe('Bearer sk-secret'); // the key rides the header to the endpoint + expect(captured?.body).toBe('{"model":"x"}'); + }); + + it('STREAMS the response body chunk-by-chunk (a custom endpoint SSE completion is not buffered)', async () => { + const fetch = createValidatedFetch( + fakeDeps({ ips: ['1.2.3.4'], chunks: ['data: a\n\n', 'data: b\n\n', 'data: [DONE]\n\n'] }), + ); + const res = await fetch('https://api.example.com/v1/chat/completions', { method: 'POST', body: '{}' }); + const reader = res.body?.getReader(); + if (reader === undefined) throw new Error('no response body stream'); + const received: string[] = []; + for (;;) { + const result = await reader.read(); + if (result.done) break; // `done` is typed boolean; only `value` is loosely typed by the lib + const value: unknown = result.value; + if (value instanceof Uint8Array) received.push(new TextDecoder().decode(value)); + } + expect(received).toEqual(['data: a\n\n', 'data: b\n\n', 'data: [DONE]\n\n']); // three distinct streamed chunks + }); + + it('disposes the socket when the response stream is cancelled early', async () => { + const dispose = vi.fn(); + const fetch = createValidatedFetch(fakeDeps({ ips: ['1.2.3.4'], chunks: ['a', 'b', 'c'], dispose })); + const res = await fetch('https://api.example.com/v1/models'); + await res.body?.cancel(); // stop reading early + expect(dispose).toHaveBeenCalled(); + }); + + it('refuses an unsupported HTTP method loudly (never a silent downgrade)', async () => { + const onConnect = vi.fn(); + const fetch = createValidatedFetch(fakeDeps({ ips: ['1.2.3.4'], onConnect })); + await expect( + fetch('https://api.example.com/v1/models', { method: 'PATCH' }), + ).rejects.toBeInstanceOf(SafeEgressError); + expect(onConnect).not.toHaveBeenCalled(); + }); + + it('maps a null-body status without a stream (204), disposing the (empty) socket', async () => { + const dispose = vi.fn(); + const fetch = createValidatedFetch(fakeDeps({ ips: ['1.2.3.4'], status: 204, dispose })); + const res = await fetch('https://api.example.com/v1/models', { method: 'DELETE' }); + expect(res.status).toBe(204); + expect(res.body).toBeNull(); + expect(dispose).toHaveBeenCalled(); + }); +}); diff --git a/apps/cli/src/engine/validated-fetch.ts b/apps/cli/src/engine/validated-fetch.ts new file mode 100644 index 00000000..ee08772b --- /dev/null +++ b/apps/cli/src/engine/validated-fetch.ts @@ -0,0 +1,146 @@ +import { + connectValidated, + nodeEgressDeps, + SafeEgressError, + type EgressDeps, + type EgressMethod, + type HopResponse, +} from '@relavium/db'; + +/** + * A `fetch`-shaped function that routes EVERY request through the ONE shared host-side SSRF hop + * ([safe-egress.ts](../../../../packages/db/src/safe-egress.ts) `connectValidated`: HTTPS + no-creds → + * DNS-resolve → range-block **every** resolved IP → connect **pinned** to the validated IP with the hostname as + * SNI). It is injected as the OpenAI SDK's `fetch` for a provider that carries a **custom `base_url`** (2.5.G S9, + * [ADR-0065](../../../../docs/decisions/0065-provider-economics-and-extensibility.md) §4) — so BOTH the + * streaming chat (`generate`/`stream`) AND the `models.list` refresh over that custom endpoint ride the same + * DNS-rebinding-safe validated hop, never a second hand-rolled URL parser (the ADR-0029(d) one-primitive rule). + * + * **Streaming-safe:** `connectValidated` returns a LIVE `AsyncIterable` body (it never buffers — `readBounded` is + * a separate helper), which this wraps in a **backpressure-aware** `ReadableStream`, so an SSE completion streams + * chunk-by-chunk. **Secret-free:** `connectValidated`'s failures are reason-only `SafeEgressError`s (never the + * url/IP/host), and a body-read fault is normalized to a generic message — the `Authorization` key rides the + * request headers to the endpoint (as the API requires) but is never logged. The `EgressDeps` (DNS + connect) are + * injectable so the SSRF policy is deterministically unit-testable without real network/DNS. + */ + +export type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise; + +/** Statuses that MUST carry a null body (a `Response` with a body + one of these throws). */ +const NULL_BODY_STATUS: ReadonlySet = new Set([101, 204, 205, 304]); + +/** Build the validated `fetch`. `deps` default to Node's real DNS + pinned-HTTPS connect. */ +export function createValidatedFetch(deps: EgressDeps = nodeEgressDeps): FetchLike { + return async (input, init) => { + const req = await normalizeRequest(input, init); + const hop = await connectValidated( + req.url, + { + allowPrivate: false, // a custom base_url resolving to a private/loopback/metadata address is REFUSED + method: req.method, + ...(req.headers === undefined ? {} : { headers: req.headers }), + ...(req.body === undefined ? {} : { body: req.body }), + }, + deps, + req.signal, + ); + return toResponse(hop); + }; +} + +interface NormalizedRequest { + readonly url: string; + readonly method: EgressMethod; + readonly headers: Readonly> | undefined; + readonly body: string | undefined; + readonly signal: AbortSignal; +} + +const EGRESS_METHODS: ReadonlySet = new Set(['GET', 'POST', 'PUT', 'DELETE']); + +/** Resolve the url / method / headers / body / signal from either a `Request` or a `(url, init)` pair. */ +async function normalizeRequest( + input: string | URL | Request, + init: RequestInit | undefined, +): Promise { + const isRequest = input instanceof Request; + const url = isRequest ? input.url : typeof input === 'string' ? input : input.href; + const rawMethod = (init?.method ?? (isRequest ? input.method : 'GET')).toUpperCase(); + if (!EGRESS_METHODS.has(rawMethod)) { + // The OpenAI SDK uses only GET (models.list) + POST (completions); refuse anything else loudly rather than + // silently downgrading a method (a secret-free, typed failure the SDK surfaces as a request error). + throw new SafeEgressError('network', `unsupported egress method '${rawMethod}'`); + } + const method = rawMethod as EgressMethod; // narrowed by the membership check above (a safe widening, not an unsound cast) + const headers = headersToRecord(init?.headers ?? (isRequest ? input.headers : undefined)); + const body = await bodyToString(init?.body ?? (isRequest ? input.body : undefined)); + const signal = init?.signal ?? (isRequest ? input.signal : undefined) ?? new AbortController().signal; + return { url, method, headers, body, signal }; +} + +/** Flatten a `RequestInit['headers']` (Headers | record | pairs) to a plain record; connectValidated re-sanitizes it. + * `new Headers(...)` normalizes every `HeadersInit` form (and lower-cases the keys) in one step. */ +function headersToRecord( + headers: RequestInit['headers'], +): Readonly> | undefined { + if (headers === undefined) return undefined; + const out: Record = {}; + new Headers(headers).forEach((value, key) => { + out[key] = value; + }); + return out; +} + +/** Read a request body to a string (connectValidated frames a string body). A chat body is already a JSON string. */ +async function bodyToString( + body: RequestInit['body'] | ReadableStream | null, +): Promise { + if (body === undefined || body === null) return undefined; + if (typeof body === 'string') return body; + if (body instanceof Uint8Array) return new TextDecoder().decode(body); + // A ReadableStream / Blob / URLSearchParams / etc. — read it through a Response (the SDK's chat path never hits this). + return await new Response(body).text(); +} + +/** Map a validated {@link HopResponse} to a standard `Response` with a backpressure-aware streaming body. */ +function toResponse(hop: HopResponse): Response { + const headers = hop.headers ?? {}; + if (NULL_BODY_STATUS.has(hop.status)) { + hop.dispose(); // a null-body status must not carry a stream — reap the (empty) socket + return new Response(null, { status: hop.status, headers }); + } + return new Response(hopBodyToStream(hop), { status: hop.status, headers }); +} + +/** Wrap the HopResponse's live `AsyncIterable` body in a pull-based `ReadableStream` (backpressure — one chunk per + * `pull`, not an eager drain), disposing the socket on end / error / cancel. */ +function hopBodyToStream(hop: HopResponse): ReadableStream { + const iterator = hop.body[Symbol.asyncIterator](); + let disposed = false; + const dispose = (): void => { + if (disposed) return; + disposed = true; + hop.dispose(); + }; + return new ReadableStream({ + async pull(controller) { + try { + const next = await iterator.next(); + if (next.done === true) { + controller.close(); + dispose(); + } else { + controller.enqueue(next.value); + } + } catch { + // A body-read/socket fault — never surface the raw error (it can carry the host/IP), and reap the socket. + dispose(); + controller.error(new SafeEgressError('network', 'egress response body read failed')); + } + }, + cancel() { + dispose(); + void iterator.return?.(undefined); + }, + }); +} diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index c96276b2..8e5a56b5 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -115,19 +115,12 @@ export async function driveHome(deps: HomeDeps): Promise { // key is actually resolved (the default agent has no MCP servers, so the MCP resolver stays inert), so building // it here is side-effect-free; a test injects `providers` and never reaches the keychain. const keychain = createOsKeychainStore(); - const providers = deps.providers ?? createProviderResolver(deps.io.env, keychain); const mcpSecretResolver = deps.mcpSecretResolver ?? createMcpSecretResolver(deps.io.env, keychain); - const doctorProbes = - deps.doctorProbes ?? - assembleDoctorProbes({ - cwd: deps.global.cwd, - ...(deps.global.configPath === undefined ? {} : { configPath: deps.global.configPath }), - resolver: providers, - // The Home binds the zero-config default agent (no `mcp_servers`), so the `--deep` MCP tier reports "none - // configured" — no session-specific status to thread (and the Home palette runs only the fast tier anyway). - }); const opened = (deps.openSessionStore ?? openSessionStore)(homeDir); + // The provider resolver + `/doctor` probes are built INSIDE the try below — once `opened.db` + the `providerStore` + // exist — so the resolver is STORE-AWARE (a custom `base_url` rebinds to its validated endpoint, 2.5.G S9); a test + // still injects `deps.providers` / `deps.doctorProbes` to bypass the keychain/network. // The cleanup scope opens as soon as the db handle is held, so an init fault AFTER this point (a failed // homeStore wire, a control write, a signal registration) still closes the shared db ONCE and restores the @@ -158,6 +151,19 @@ export async function driveHome(deps: HomeDeps): Promise { // merged catalog + runs the (long-lived-process-safe) TTL background refresh + writes the next session's default. const storeDeps = { uuid, now }; const providerStore = createProviderStore(opened.db, storeDeps); + // The STORE-AWARE provider resolver (2.5.G S9, ADR-0065 §4) — built from `providerStore` so a stored custom + // `base_url` rebinds its adapter to the SSRF-validated endpoint, for the Home's chat turns + the catalog refresh. + const providers = + deps.providers ?? createProviderResolver(deps.io.env, keychain, { providerStore }); + // The Home's `/doctor` probes (built here, over the store-aware resolver). The zero-config default agent has no + // `mcp_servers`, so the `--deep` MCP tier reports "none configured" (and the Home palette runs only the fast tier). + const doctorProbes = + deps.doctorProbes ?? + assembleDoctorProbes({ + cwd: deps.global.cwd, + ...(deps.global.configPath === undefined ? {} : { configPath: deps.global.configPath }), + resolver: providers, + }); const catalogStore = createModelCatalogStore(opened.db, storeDeps); const refreshService = createModelRefreshService({ resolveProvider: providers.resolveProvider, diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index 3f33f3ab..6fdff484 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -298,7 +298,7 @@ relavium provider remove-key anthropic # delete the key from th - **`set-key` reads the key from stdin**, never a CLI argument (argv leaks into `ps`, shell history, and CI logs); pipe it or use a heredoc. The key is stored in the OS keychain under the canonical entry-naming scheme ([keychain-and-secrets.md](../desktop/keychain-and-secrets.md#entry-naming)). -- **`add` / `set-key`** auto-register the provider row. `--base-url ` on `add` records a custom endpoint (validated as an **HTTPS** URL); it is **not yet honored by request routing** — adapters use their built-in endpoints today. Wiring a custom base URL to outbound requests lands later **with** the full SSRF base-URL gate (HTTPS-only; private/loopback/metadata ranges blocked) per [security-review.md](../../standards/security-review.md), before any key is attached to it. +- **`add` / `set-key`** auto-register the provider row. `--base-url ` on `add` records a custom endpoint that **is now actually used at request routing** (2.5.G S9, [ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) §3–4 — the earlier "dead-config" gap is closed): the resolver rebinds that provider's adapter to the custom endpoint and routes **all** its egress (streaming `generate`/`stream` + the `models.list` refresh) through the shared **SSRF-validated** hop (`connectValidated` — HTTPS-only, no embedded credentials, every resolved IP range-blocked, connect pinned to the validated IP for DNS-rebinding safety). Custom endpoints are **OpenAI-compatible only** this round (`openai` / `deepseek`); a `--base-url` on `anthropic` / `gemini` is **refused** with a clear message (exit `2`), as is a non-HTTPS / private-loopback / credential-bearing URL (fail-fast at `add`). The provider-id set stays **closed** — a custom endpoint reuses the `openai` / `deepseek` id (ADR-0065 §6). - **`test`** does a 1-token `generate` through `@relavium/llm`; `--model ` overrides the cheap default. A bad key fails cleanly (exit `2`) without echoing the key. - **Key resolution** (used by `run` + `test`): **OS keychain → `RELAVIUM__API_KEY` env var → error**. diff --git a/docs/reference/desktop/database-schema.md b/docs/reference/desktop/database-schema.md index 1e7b4fad..4496f46c 100644 --- a/docs/reference/desktop/database-schema.md +++ b/docs/reference/desktop/database-schema.md @@ -68,9 +68,11 @@ Registered LLM providers. The actual API key never lives here — only a referen | `id` | TEXT | PRIMARY KEY (UUID) | | `name` | TEXT | NOT NULL UNIQUE (e.g. `anthropic`, `openai`) | | `display_name` | TEXT | NOT NULL | -| `base_url` | TEXT | NOT NULL | +| `base_url` | TEXT | NOT NULL — a custom endpoint (2.5.G S9); actually used at routing time for an OpenAI-compatible provider (ADR-0065 §3) | | `api_key_keychain_ref` | TEXT | NULL — keychain `account` identifier, not the key itself | | `default_headers` | TEXT (JSON) | NOT NULL DEFAULT `'{}'` | +| `kind` | TEXT | NULL — the protocol `ProviderKind` (`anthropic`/`openai-compatible`/`gemini`), added by migration `0008` (ADR-0065 §5). Populated for uniformity; validated against `PROVIDER_KINDS` at the store read boundary (no DB CHECK — SQLite `ALTER ADD` limit; a foreign value ⇒ `undefined`). Load-bearing only for a future custom provider. | +| `pricing_reference_url` | TEXT | NULL — a pricing-page URL (a UX pointer for user-supplied pricing, S10), added by migration `0008` (ADR-0065 §5) | | `is_active` | INTEGER (bool) | NOT NULL DEFAULT 1 | | `deleted_at` | INTEGER | NULL | | `created_at` | INTEGER | NOT NULL | diff --git a/packages/db/drizzle/0008_plain_scourge.sql b/packages/db/drizzle/0008_plain_scourge.sql new file mode 100644 index 00000000..ac2c39b0 --- /dev/null +++ b/packages/db/drizzle/0008_plain_scourge.sql @@ -0,0 +1,2 @@ +ALTER TABLE `llm_providers` ADD `kind` text;--> statement-breakpoint +ALTER TABLE `llm_providers` ADD `pricing_reference_url` text; \ No newline at end of file diff --git a/packages/db/drizzle/meta/0008_snapshot.json b/packages/db/drizzle/meta/0008_snapshot.json new file mode 100644 index 00000000..a65145dc --- /dev/null +++ b/packages/db/drizzle/meta/0008_snapshot.json @@ -0,0 +1,2091 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "19afbdcf-e05f-4183-826d-d31318998876", + "prevId": "b2cb738e-465a-4dba-a7f7-eaf7f8634aa3", + "tables": { + "agent_sessions": { + "name": "agent_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "agent_slug": { + "name": "agent_slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "agent_snapshot": { + "name": "agent_snapshot", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "title": { + "name": "title", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "working_dir": { + "name": "working_dir", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "git_ref": { + "name": "git_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "fs_scope_tier": { + "name": "fs_scope_tier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'sandboxed'" + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "context_json": { + "name": "context_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "total_input_tokens": { + "name": "total_input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "total_output_tokens": { + "name": "total_output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "total_cost_microcents": { + "name": "total_cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "exported_workflow_path": { + "name": "exported_workflow_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_agent_sessions_status": { + "name": "idx_agent_sessions_status", + "columns": [ + "status", + "\"updated_at\" desc" + ], + "isUnique": false, + "where": "\"agent_sessions\".\"deleted_at\" is null" + }, + "idx_agent_sessions_agent": { + "name": "idx_agent_sessions_agent", + "columns": [ + "agent_id", + "\"created_at\" desc" + ], + "isUnique": false, + "where": "\"agent_sessions\".\"agent_id\" is not null" + }, + "idx_agent_sessions_updated": { + "name": "idx_agent_sessions_updated", + "columns": [ + "\"updated_at\" desc", + "\"id\" desc" + ], + "isUnique": false, + "where": "\"agent_sessions\".\"deleted_at\" is null" + } + }, + "foreignKeys": { + "agent_sessions_agent_id_agents_id_fk": { + "name": "agent_sessions_agent_id_agents_id_fk", + "tableFrom": "agent_sessions", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "agent_sessions_model_id_model_catalog_id_fk": { + "name": "agent_sessions_model_id_model_catalog_id_fk", + "tableFrom": "agent_sessions", + "tableTo": "model_catalog", + "columnsFrom": [ + "model_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "agent_sessions_fs_scope_tier_check": { + "name": "agent_sessions_fs_scope_tier_check", + "value": "\"agent_sessions\".\"fs_scope_tier\" in ('sandboxed', 'project', 'full')" + }, + "agent_sessions_status_check": { + "name": "agent_sessions_status_check", + "value": "\"agent_sessions\".\"status\" in ('active', 'idle', 'exported', 'ended')" + } + } + }, + "agents": { + "name": "agents", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "system_prompt": { + "name": "system_prompt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "tools": { + "name": "tools", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "config": { + "name": "config", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "input_schema": { + "name": "input_schema", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_schema": { + "name": "output_schema", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "1" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_agents_slug": { + "name": "idx_agents_slug", + "columns": [ + "slug" + ], + "isUnique": true, + "where": "\"agents\".\"deleted_at\" is null" + }, + "idx_agents_model": { + "name": "idx_agents_model", + "columns": [ + "model_id" + ], + "isUnique": false + }, + "idx_agents_active": { + "name": "idx_agents_active", + "columns": [ + "is_active", + "\"created_at\" desc" + ], + "isUnique": false, + "where": "\"agents\".\"deleted_at\" is null" + } + }, + "foreignKeys": { + "agents_model_id_model_catalog_id_fk": { + "name": "agents_model_id_model_catalog_id_fk", + "tableFrom": "agents", + "tableTo": "model_catalog", + "columnsFrom": [ + "model_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "llm_providers": { + "name": "llm_providers", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_url": { + "name": "base_url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "api_key_keychain_ref": { + "name": "api_key_keychain_ref", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "default_headers": { + "name": "default_headers", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pricing_reference_url": { + "name": "pricing_reference_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "1" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_llm_providers_name": { + "name": "idx_llm_providers_name", + "columns": [ + "name" + ], + "isUnique": true, + "where": "\"llm_providers\".\"deleted_at\" is null" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "media_objects": { + "name": "media_objects", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "handle": { + "name": "handle", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "modality": { + "name": "modality", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "byte_length": { + "name": "byte_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_referenced_at": { + "name": "last_referenced_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "media_objects_handle_unique": { + "name": "media_objects_handle_unique", + "columns": [ + "handle" + ], + "isUnique": true + }, + "idx_media_objects_gc": { + "name": "idx_media_objects_gc", + "columns": [ + "last_referenced_at" + ], + "isUnique": false, + "where": "\"media_objects\".\"deleted_at\" is null" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "media_objects_modality_check": { + "name": "media_objects_modality_check", + "value": "\"media_objects\".\"modality\" in ('image', 'audio', 'video', 'document')" + } + } + }, + "media_references": { + "name": "media_references", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "handle": { + "name": "handle", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scope_kind": { + "name": "scope_kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "scope_id": { + "name": "scope_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_media_references_unique": { + "name": "idx_media_references_unique", + "columns": [ + "handle", + "scope_kind", + "scope_id" + ], + "isUnique": true + }, + "idx_media_references_scope": { + "name": "idx_media_references_scope", + "columns": [ + "scope_kind", + "scope_id" + ], + "isUnique": false + }, + "idx_media_references_handle": { + "name": "idx_media_references_handle", + "columns": [ + "handle" + ], + "isUnique": false + } + }, + "foreignKeys": { + "media_references_handle_media_objects_handle_fk": { + "name": "media_references_handle_media_objects_handle_fk", + "tableFrom": "media_references", + "tableTo": "media_objects", + "columnsFrom": [ + "handle" + ], + "columnsTo": [ + "handle" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "media_references_scope_kind_check": { + "name": "media_references_scope_kind_check", + "value": "\"media_references\".\"scope_kind\" in ('run', 'node', 'session', 'workspace')" + } + } + }, + "messages": { + "name": "messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "step_execution_id": { + "name": "step_execution_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sequence_number": { + "name": "sequence_number", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_parts": { + "name": "content_parts", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finish_reason": { + "name": "finish_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_messages_step": { + "name": "idx_messages_step", + "columns": [ + "step_execution_id", + "sequence_number" + ], + "isUnique": false + }, + "idx_messages_run": { + "name": "idx_messages_run", + "columns": [ + "run_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "messages_step_execution_id_step_executions_id_fk": { + "name": "messages_step_execution_id_step_executions_id_fk", + "tableFrom": "messages", + "tableTo": "step_executions", + "columnsFrom": [ + "step_execution_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "model_catalog": { + "name": "model_catalog", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "context_window_tokens": { + "name": "context_window_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "max_output_tokens": { + "name": "max_output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_cost_per_mtok_microcents": { + "name": "input_cost_per_mtok_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "output_cost_per_mtok_microcents": { + "name": "output_cost_per_mtok_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cached_input_cost_per_mtok_microcents": { + "name": "cached_input_cost_per_mtok_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "media_image_cost_microcents": { + "name": "media_image_cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "media_audio_cost_microcents": { + "name": "media_audio_cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "media_video_cost_microcents": { + "name": "media_video_cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "media_surface": { + "name": "media_surface", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'chat'" + }, + "supports_tool_calling": { + "name": "supports_tool_calling", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "0" + }, + "supports_vision": { + "name": "supports_vision", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "0" + }, + "supports_streaming": { + "name": "supports_streaming", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "1" + }, + "supports_json_mode": { + "name": "supports_json_mode", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "0" + }, + "capabilities": { + "name": "capabilities", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "deprecation_date": { + "name": "deprecation_date", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "source": { + "name": "source", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'static'" + }, + "last_refreshed_at": { + "name": "last_refreshed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "1" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_model_catalog_provider_model": { + "name": "idx_model_catalog_provider_model", + "columns": [ + "provider_id", + "model_id" + ], + "isUnique": true, + "where": "\"model_catalog\".\"deleted_at\" is null" + }, + "idx_model_catalog_provider": { + "name": "idx_model_catalog_provider", + "columns": [ + "provider_id" + ], + "isUnique": false + }, + "idx_model_catalog_active": { + "name": "idx_model_catalog_active", + "columns": [ + "is_active" + ], + "isUnique": false, + "where": "\"model_catalog\".\"deleted_at\" is null" + } + }, + "foreignKeys": { + "model_catalog_provider_id_llm_providers_id_fk": { + "name": "model_catalog_provider_id_llm_providers_id_fk", + "tableFrom": "model_catalog", + "tableTo": "llm_providers", + "columnsFrom": [ + "provider_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "run_costs": { + "name": "run_costs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cost_microcents": { + "name": "cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_run_costs_run": { + "name": "idx_run_costs_run", + "columns": [ + "run_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "run_costs_run_id_runs_id_fk": { + "name": "run_costs_run_id_runs_id_fk", + "tableFrom": "run_costs", + "tableTo": "runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "run_costs_model_id_model_catalog_id_fk": { + "name": "run_costs_model_id_model_catalog_id_fk", + "tableFrom": "run_costs", + "tableTo": "model_catalog", + "columnsFrom": [ + "model_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "run_events": { + "name": "run_events", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "step_execution_id": { + "name": "step_execution_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event_type": { + "name": "event_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "level": { + "name": "level", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'info'" + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "payload_json": { + "name": "payload_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "ts": { + "name": "ts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_run_events_run_seq": { + "name": "idx_run_events_run_seq", + "columns": [ + "run_id", + "seq" + ], + "isUnique": true + }, + "idx_run_events_step": { + "name": "idx_run_events_step", + "columns": [ + "step_execution_id", + "ts" + ], + "isUnique": false, + "where": "\"run_events\".\"step_execution_id\" is not null" + }, + "idx_run_events_run_type": { + "name": "idx_run_events_run_type", + "columns": [ + "run_id", + "event_type", + "ts" + ], + "isUnique": false + } + }, + "foreignKeys": { + "run_events_run_id_runs_id_fk": { + "name": "run_events_run_id_runs_id_fk", + "tableFrom": "run_events", + "tableTo": "runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "runs": { + "name": "runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "workflow_id": { + "name": "workflow_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "workflow_path": { + "name": "workflow_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "project_root": { + "name": "project_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "workflow_definition_snapshot": { + "name": "workflow_definition_snapshot", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "execution_mode": { + "name": "execution_mode", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'local'" + }, + "trigger_type": { + "name": "trigger_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'manual'" + }, + "trigger_metadata": { + "name": "trigger_metadata", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "input_json": { + "name": "input_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "output_json": { + "name": "output_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_json": { + "name": "error_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "total_input_tokens": { + "name": "total_input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "total_output_tokens": { + "name": "total_output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "total_cost_microcents": { + "name": "total_cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_runs_workflow": { + "name": "idx_runs_workflow", + "columns": [ + "workflow_id", + "\"created_at\" desc" + ], + "isUnique": false + }, + "idx_runs_status": { + "name": "idx_runs_status", + "columns": [ + "status", + "\"created_at\" desc" + ], + "isUnique": false, + "where": "\"runs\".\"deleted_at\" is null" + }, + "idx_runs_cost": { + "name": "idx_runs_cost", + "columns": [ + "workflow_id", + "created_at", + "total_cost_microcents" + ], + "isUnique": false, + "where": "\"runs\".\"deleted_at\" is null" + }, + "idx_runs_created": { + "name": "idx_runs_created", + "columns": [ + "\"created_at\" desc", + "\"id\" desc" + ], + "isUnique": false, + "where": "\"runs\".\"deleted_at\" is null" + } + }, + "foreignKeys": { + "runs_workflow_id_workflows_id_fk": { + "name": "runs_workflow_id_workflows_id_fk", + "tableFrom": "runs", + "tableTo": "workflows", + "columnsFrom": [ + "workflow_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "runs_status_check": { + "name": "runs_status_check", + "value": "\"runs\".\"status\" in ('pending', 'running', 'paused', 'completed', 'failed', 'cancelled')" + }, + "runs_execution_mode_check": { + "name": "runs_execution_mode_check", + "value": "\"runs\".\"execution_mode\" in ('local', 'cloud', 'managed')" + } + } + }, + "session_messages": { + "name": "session_messages", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sequence_number": { + "name": "sequence_number", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "content": { + "name": "content", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "content_parts": { + "name": "content_parts", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_calls": { + "name": "tool_calls", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tool_call_id": { + "name": "tool_call_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finish_reason": { + "name": "finish_reason", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cost_microcents": { + "name": "cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "compaction_dropped_through_sequence": { + "name": "compaction_dropped_through_sequence", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_session_messages_seq": { + "name": "idx_session_messages_seq", + "columns": [ + "session_id", + "sequence_number" + ], + "isUnique": true + }, + "idx_session_messages_session": { + "name": "idx_session_messages_session", + "columns": [ + "session_id", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_messages_session_id_agent_sessions_id_fk": { + "name": "session_messages_session_id_agent_sessions_id_fk", + "tableFrom": "session_messages", + "tableTo": "agent_sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "session_messages_model_id_model_catalog_id_fk": { + "name": "session_messages_model_id_model_catalog_id_fk", + "tableFrom": "session_messages", + "tableTo": "model_catalog", + "columnsFrom": [ + "model_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "step_executions": { + "name": "step_executions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "node_id": { + "name": "node_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "node_type": { + "name": "node_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "agent_id": { + "name": "agent_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "agent_snapshot": { + "name": "agent_snapshot", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "model_id": { + "name": "model_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempt_number": { + "name": "attempt_number", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 1 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "input_json": { + "name": "input_json", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'{}'" + }, + "output_json": { + "name": "output_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error_json": { + "name": "error_json", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "completed_at": { + "name": "completed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "input_tokens": { + "name": "input_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "output_tokens": { + "name": "output_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cached_tokens": { + "name": "cached_tokens", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cost_microcents": { + "name": "cost_microcents", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_step_exec_run": { + "name": "idx_step_exec_run", + "columns": [ + "run_id", + "created_at" + ], + "isUnique": false + }, + "idx_step_exec_run_node": { + "name": "idx_step_exec_run_node", + "columns": [ + "run_id", + "node_id", + "attempt_number" + ], + "isUnique": false + }, + "idx_step_exec_agent": { + "name": "idx_step_exec_agent", + "columns": [ + "agent_id", + "\"created_at\" desc" + ], + "isUnique": false, + "where": "\"step_executions\".\"agent_id\" is not null" + }, + "idx_step_exec_model": { + "name": "idx_step_exec_model", + "columns": [ + "model_id", + "\"created_at\" desc" + ], + "isUnique": false, + "where": "\"step_executions\".\"model_id\" is not null" + }, + "idx_step_exec_cost": { + "name": "idx_step_exec_cost", + "columns": [ + "model_id", + "created_at", + "cost_microcents" + ], + "isUnique": false, + "where": "\"step_executions\".\"model_id\" is not null" + } + }, + "foreignKeys": { + "step_executions_run_id_runs_id_fk": { + "name": "step_executions_run_id_runs_id_fk", + "tableFrom": "step_executions", + "tableTo": "runs", + "columnsFrom": [ + "run_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "step_executions_agent_id_agents_id_fk": { + "name": "step_executions_agent_id_agents_id_fk", + "tableFrom": "step_executions", + "tableTo": "agents", + "columnsFrom": [ + "agent_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + }, + "step_executions_model_id_model_catalog_id_fk": { + "name": "step_executions_model_id_model_catalog_id_fk", + "tableFrom": "step_executions", + "tableTo": "model_catalog", + "columnsFrom": [ + "model_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": { + "step_executions_status_check": { + "name": "step_executions_status_check", + "value": "\"step_executions\".\"status\" in ('pending', 'running', 'completed', 'failed', 'skipped')" + } + } + }, + "workflows": { + "name": "workflows", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "description": { + "name": "description", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "definition": { + "name": "definition", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_schema": { + "name": "input_schema", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "tags": { + "name": "tags", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'[]'" + }, + "source_path": { + "name": "source_path", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_active": { + "name": "is_active", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "1" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "idx_workflows_slug": { + "name": "idx_workflows_slug", + "columns": [ + "slug" + ], + "isUnique": true, + "where": "\"workflows\".\"deleted_at\" is null" + }, + "idx_workflows_active": { + "name": "idx_workflows_active", + "columns": [ + "is_active", + "\"updated_at\" desc" + ], + "isUnique": false, + "where": "\"workflows\".\"deleted_at\" is null" + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": { + "idx_agent_sessions_status": { + "columns": { + "\"updated_at\" desc": { + "isExpression": true + } + } + }, + "idx_agent_sessions_agent": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_agent_sessions_updated": { + "columns": { + "\"updated_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_agents_active": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_runs_workflow": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_runs_status": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_runs_created": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + }, + "\"id\" desc": { + "isExpression": true + } + } + }, + "idx_step_exec_agent": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_step_exec_model": { + "columns": { + "\"created_at\" desc": { + "isExpression": true + } + } + }, + "idx_workflows_active": { + "columns": { + "\"updated_at\" desc": { + "isExpression": true + } + } + } + } + } +} \ No newline at end of file diff --git a/packages/db/drizzle/meta/_journal.json b/packages/db/drizzle/meta/_journal.json index f0d0747a..3cc4cd50 100644 --- a/packages/db/drizzle/meta/_journal.json +++ b/packages/db/drizzle/meta/_journal.json @@ -57,6 +57,13 @@ "when": 1783272761773, "tag": "0007_harsh_bug", "breakpoints": true + }, + { + "idx": 8, + "version": "6", + "when": 1783298924478, + "tag": "0008_plain_scourge", + "breakpoints": true } ] } \ No newline at end of file diff --git a/packages/db/src/provider-store.test.ts b/packages/db/src/provider-store.test.ts index 90b1d199..57eb61fb 100644 --- a/packages/db/src/provider-store.test.ts +++ b/packages/db/src/provider-store.test.ts @@ -1,4 +1,4 @@ -import { eq } from 'drizzle-orm'; +import { eq, sql } from 'drizzle-orm'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { createClient, runMigrations, type DbClient } from './client.js'; @@ -73,6 +73,41 @@ describe('createProviderStore', () => { expect(typeof b.defaultHeaders).toBe('object'); // a parsed object, not a double-encoded string }); + it('round-trips the provider kind + pricingReferenceUrl, preserving them on an update that omits them (ADR-0065 §5)', () => { + store.upsert({ + name: 'deepseek', + displayName: 'DeepSeek', + baseUrl: 'https://my-proxy.example/v1', + kind: 'openai-compatible', + pricingReferenceUrl: 'https://prices.example', + }); + const got = store.get('deepseek'); + expect(got?.kind).toBe('openai-compatible'); + expect(got?.pricingReferenceUrl).toBe('https://prices.example'); + // An update that omits kind / pricingReferenceUrl keeps the stored values (like defaultHeaders). + const updated = store.upsert({ + name: 'deepseek', + displayName: 'DeepSeek (proxied)', + baseUrl: 'https://my-proxy.example/v1', + }); + expect(updated.kind).toBe('openai-compatible'); + expect(updated.pricingReferenceUrl).toBe('https://prices.example'); + }); + + it('reads an absent kind / pricingReferenceUrl as undefined (a plain add omits them)', () => { + store.upsert({ name: 'anthropic', displayName: 'Anthropic', baseUrl: 'https://api.anthropic.com' }); + const got = store.get('anthropic'); + expect(got?.kind).toBeUndefined(); + expect(got?.pricingReferenceUrl).toBeUndefined(); + }); + + it('coerces a FOREIGN stored kind to undefined at the read boundary (fail-closed, no DB CHECK)', () => { + const rec = store.upsert({ name: 'openai', displayName: 'OpenAI', baseUrl: 'https://api.openai.com/v1' }); + // Simulate a tampered/foreign value written outside the typed setter (no DB CHECK on the ALTER-ADD column). + client.db.run(sql`update llm_providers set kind = 'rogue-protocol' where id = ${rec.id}`); + expect(store.get('openai')?.kind).toBeUndefined(); // a non-PROVIDER_KINDS value is never trusted + }); + it('preserves createdAt and advances updatedAt on update', () => { let clock = 1_000; const timed = createProviderStore(client.db, { diff --git a/packages/db/src/provider-store.ts b/packages/db/src/provider-store.ts index c36a2e82..197d4614 100644 --- a/packages/db/src/provider-store.ts +++ b/packages/db/src/provider-store.ts @@ -1,3 +1,4 @@ +import { PROVIDER_KINDS, type ProviderKind } from '@relavium/shared'; import { and, asc, eq, isNull } from 'drizzle-orm'; import type { Db } from './client.js'; @@ -23,6 +24,11 @@ export interface ProviderRecord { /** The OS-keychain `account` holding this provider's key, or `undefined` when no key is set. NEVER the key. */ readonly apiKeyKeychainRef?: string; readonly defaultHeaders: Record; + /** The provider's protocol `kind` (ADR-0065 §5) — a custom OpenAI-compatible `base_url` reuses it. `undefined` + * when unset (the host derives it from the closed provider id) or a foreign stored value (fail-closed to unset). */ + readonly kind?: ProviderKind; + /** A pricing-page URL (ADR-0065 §5, a UX pointer for user-supplied pricing — S10). `undefined` when unset. */ + readonly pricingReferenceUrl?: string; readonly isActive: boolean; readonly createdAt: string; readonly updatedAt: string; @@ -34,6 +40,8 @@ export interface ProviderUpsert { readonly displayName: string; readonly baseUrl: string; readonly defaultHeaders?: Record; + readonly kind?: ProviderKind; + readonly pricingReferenceUrl?: string; } export interface ProviderStoreDeps { @@ -75,7 +83,19 @@ function parseStringRecord(json: string): Record { return out; } +/** + * Validate the stored `kind` against the closed {@link PROVIDER_KINDS} set at the DB read boundary — no DB CHECK + * exists (a SQLite `ALTER ADD` limitation), so a tampered/foreign value must not be trusted. A `null` (unset) or a + * non-member value degrades to `undefined` (treated as "no explicit kind"; the host derives the kind from the + * closed provider id) — a foreign protocol string is never mistaken for a real `kind`. Mirrors `coerceModelCatalogSource`. + */ +function coerceProviderKind(value: string | null): ProviderKind | undefined { + if (value === null) return undefined; + return PROVIDER_KINDS.find((kind) => kind === value); +} + function fromRow(row: LlmProviderRow): ProviderRecord { + const kind = coerceProviderKind(row.kind); return { id: row.id, name: row.name, @@ -83,6 +103,8 @@ function fromRow(row: LlmProviderRow): ProviderRecord { baseUrl: row.baseUrl, ...(row.apiKeyKeychainRef === null ? {} : { apiKeyKeychainRef: row.apiKeyKeychainRef }), defaultHeaders: parseStringRecord(row.defaultHeaders), + ...(kind === undefined ? {} : { kind }), + ...(row.pricingReferenceUrl === null ? {} : { pricingReferenceUrl: row.pricingReferenceUrl }), isActive: row.isActive, createdAt: epochMsToIso(row.createdAt), updatedAt: epochMsToIso(row.updatedAt), @@ -123,6 +145,10 @@ export function createProviderStore(db: Db, deps: ProviderStoreDeps): ProviderSt displayName: input.displayName, baseUrl: input.baseUrl, defaultHeaders: JSON.stringify(input.defaultHeaders ?? {}), + ...(input.kind === undefined ? {} : { kind: input.kind }), + ...(input.pricingReferenceUrl === undefined + ? {} + : { pricingReferenceUrl: input.pricingReferenceUrl }), createdAt: t, updatedAt: t, }; @@ -138,6 +164,11 @@ export function createProviderStore(db: Db, deps: ProviderStoreDeps): ProviderSt input.defaultHeaders === undefined ? existing.defaultHeaders : JSON.stringify(input.defaultHeaders), + // Preserve an existing kind / pricing URL when the caller omits it (like defaultHeaders). + ...(input.kind === undefined ? {} : { kind: input.kind }), + ...(input.pricingReferenceUrl === undefined + ? {} + : { pricingReferenceUrl: input.pricingReferenceUrl }), updatedAt: t, }) .where(eq(llmProviders.id, existing.id)) diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index c30c555c..18f47cf6 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -11,6 +11,7 @@ import { type MediaScopeKind, type MediaSurface, type ModelCatalogSource, + type ProviderKind, type RunStatus, type SessionStatus, } from '@relavium/shared'; @@ -79,6 +80,12 @@ export const llmProviders = sqliteTable( // keychain `account` identifier — NEVER the key itself (ADR-0006). apiKeyKeychainRef: text('api_key_keychain_ref'), defaultHeaders: jsonText('default_headers').notNull().default('{}'), + // ADR-0065 §5: the provider's protocol `kind` (a custom OpenAI-compatible `base_url` reuses it) + a pricing + // page `pricing_reference_url` (a UX pointer for user-supplied pricing, S10). Both NULLABLE — populated for + // uniformity, load-bearing only for a custom provider; validated against `PROVIDER_KINDS` at the store read + // boundary (no DB CHECK — SQLite `ALTER ADD` limitation), mirroring `model_catalog.source`. + kind: text('kind').$type(), + pricingReferenceUrl: text('pricing_reference_url'), isActive: boolFlag('is_active', true), deletedAt: epochMs('deleted_at'), createdAt: epochMs('created_at').notNull(), diff --git a/packages/llm/src/index.ts b/packages/llm/src/index.ts index cb503f0b..3cf16e2b 100644 --- a/packages/llm/src/index.ts +++ b/packages/llm/src/index.ts @@ -167,4 +167,5 @@ export type { // Default keyless provider registry — the provider→adapter mapping a host wires into // `resolveProvider` (ADR-0038); the key is injected per call via `keyFor`, never here (ADR-0011). // `providerKind` derives the ADR-0064 protocol `kind` from a provider id (used by the later merge/refresh steps). -export { defaultProviders, providerKind } from './providers.js'; +// `createCustomOpenAiProvider` builds a per-provider OpenAI-compatible adapter for a custom base_url (ADR-0065 §3, S9). +export { createCustomOpenAiProvider, defaultProviders, providerKind } from './providers.js'; diff --git a/packages/llm/src/providers.ts b/packages/llm/src/providers.ts index d4b70aa1..e79d6d20 100644 --- a/packages/llm/src/providers.ts +++ b/packages/llm/src/providers.ts @@ -41,3 +41,27 @@ export function providerKind(id: ProviderId): ProviderKind { return 'openai-compatible'; } } + +/** + * Build a per-provider **OpenAI-compatible adapter for a CUSTOM `base_url`** (2.5.G S9, + * [ADR-0065](../../../docs/decisions/0065-provider-economics-and-extensibility.md) §3–4) — the seam factory a + * host wires for a provider whose stored row carries a custom endpoint, instead of the static + * {@link defaultProviders} map. It reuses the ONE openai-compatible adapter (so its construction-time + * `assertHttpsBaseUrl` HTTPS + private-range + no-credentials gate applies to the custom URL) and lets the host + * inject an **SSRF-validated `fetch`** so both the streaming chat and the `models.list` refresh over that custom + * endpoint ride the host's validated hop (§4). No vendor SDK type crosses the seam — `providerId` is a closed-enum + * subset, `fetch` is the standard web type, and the return is a Relavium {@link LlmProvider}. Throws + * `InvalidBaseUrlError` on a non-HTTPS / private / credential-bearing `baseURL`. The provider-id enum stays closed + * (a custom endpoint REUSES the `openai`/`deepseek` id, per ADR-0065 §6). + */ +export function createCustomOpenAiProvider(deps: { + readonly providerId: Extract; + readonly baseURL: string; + readonly fetch?: (input: string | URL | Request, init?: RequestInit) => Promise; +}): LlmProvider { + return createOpenAiAdapter({ + providerId: deps.providerId, + baseURL: deps.baseURL, + ...(deps.fetch === undefined ? {} : { fetch: deps.fetch }), + }); +} From f678ab873e4a1ee415f5f35402d166a156763393 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 04:34:19 +0300 Subject: [PATCH 25/68] =?UTF-8?q?fix(cli):=202.5.G=20S9=20Opus-review=20?= =?UTF-8?q?=E2=80=94=20harden=20the=20validated-fetch=20edges=20(status/no?= =?UTF-8?q?rmalize/encoding)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial Opus review confirmed no exploit — the SSRF core is sound (pins the validated IP, funnels every request shape through connectValidated, the key never leaves the Authorization header, the anthropic/gemini refusal is airtight at two layers). Minor hardening applied: - Clamp the response status to [200,599] before `new Response(...)` — a hostile custom endpoint returning `999` (or a malformed line ⇒ statusCode 0) would otherwise throw a raw RangeError that escapes the wrapper un-normalized; now it's a typed, reason-only SafeEgressError + the socket is reaped. Dropped the unreachable+self-defeating `101` from NULL_BODY_STATUS. - Normalize a raw connect/resolver throw (e.g. a DNS error carrying the non-secret hostname) to SafeEgressError, so ONLY a reason-only error escapes — matching the docstring (corrected to note the SDK owns the timeout and the caller's signal tears the socket down during streaming, deliberately NOT withEgressTimeout, which would abort a long stream). - Strip `accept-encoding` from the request: this fetch does not auto-decompress the streamed body (unlike a platform fetch), so it must never negotiate compression — else a gzip'd response reaches the SDK as unparseable raw bytes. - Keep EGRESS_METHODS in lockstep with the EgressMethod union via `as const satisfies` + an `isEgressMethod` guard (removes the lone `as`). Tests (+4): redirect-no-follow (a 3xx+Location is terminal, never a second unvalidated hop), a body-read fault normalized to a secret-free SafeEgressError with the socket reaped (the raw host+key never surface), a Request-object input, and an out-of-range status rejected as SafeEgressError not RangeError. Toolchain: lint + typecheck + test (1415, +4) + build all green across 9 tasks. Refs: ADR-0065, ADR-0029 Co-Authored-By: Claude --- apps/cli/src/engine/validated-fetch.test.ts | 69 +++++++++++++++++++- apps/cli/src/engine/validated-fetch.ts | 71 ++++++++++++++------- 2 files changed, 116 insertions(+), 24 deletions(-) diff --git a/apps/cli/src/engine/validated-fetch.test.ts b/apps/cli/src/engine/validated-fetch.test.ts index 61386e61..1906010d 100644 --- a/apps/cli/src/engine/validated-fetch.test.ts +++ b/apps/cli/src/engine/validated-fetch.test.ts @@ -13,11 +13,21 @@ async function* bytes(chunks: readonly string[]): AsyncGenerator { * A fake {@link EgressDeps}: `resolveHost` returns the configured IP(s) (so the SSRF range-block is deterministic * without DNS), and `openConnection` returns a scripted {@link HopResponse} + captures the pinned request. */ +/** An async-iterable body that yields `before`, then THROWS a raw error (a socket reset) — the fault must be + * normalized to a secret-free SafeEgressError and never surface the raw message. */ +async function* failingBody(before: readonly string[]): AsyncGenerator { + await Promise.resolve(); + for (const chunk of before) yield new TextEncoder().encode(chunk); + throw new Error('socket reset (host 10.9.8.7, key sk-leak-me)'); // must NEVER reach the caller +} + function fakeDeps(opts: { ips: readonly string[]; status?: number; headers?: Record; + location?: string; chunks?: readonly string[]; + body?: AsyncIterable; onConnect?: (req: HopRequest) => void; dispose?: () => void; }): EgressDeps { @@ -28,8 +38,8 @@ function fakeDeps(opts: { return Promise.resolve({ status: opts.status ?? 200, headers: opts.headers ?? { 'content-type': 'application/json' }, - location: undefined, - body: bytes(opts.chunks ?? ['{"ok":true}']), + location: opts.location, + body: opts.body ?? bytes(opts.chunks ?? ['{"ok":true}']), dispose: opts.dispose ?? ((): void => undefined), }); }, @@ -114,4 +124,59 @@ describe('createValidatedFetch', () => { expect(res.body).toBeNull(); expect(dispose).toHaveBeenCalled(); }); + + it('does NOT follow a redirect — a 3xx + Location is returned TERMINALLY (no second unvalidated hop)', async () => { + const onConnect = vi.fn(); + const fetch = createValidatedFetch( + fakeDeps({ ips: ['1.2.3.4'], status: 302, location: 'https://169.254.169.254/', onConnect }), + ); + const res = await fetch('https://api.example.com/v1/models'); + expect(res.status).toBe(302); // handed to the caller as-is... + expect(onConnect).toHaveBeenCalledTimes(1); // ...never re-issued to the (private) Location target + }); + + it('normalizes a body-read fault to a secret-free SafeEgressError + reaps the socket (never leaks the raw error)', async () => { + const dispose = vi.fn(); + const fetch = createValidatedFetch( + fakeDeps({ ips: ['1.2.3.4'], body: failingBody(['data: a\n\n']), dispose }), + ); + const res = await fetch('https://api.example.com/v1/chat/completions', { method: 'POST', body: '{}' }); + const reader = res.body?.getReader(); + if (reader === undefined) throw new Error('no body stream'); + let thrown: unknown; + try { + await reader.read(); // 'data: a\n\n' + await reader.read(); // the underlying body throws → the stream errors + } catch (err) { + thrown = err; + } + expect(thrown).toBeInstanceOf(SafeEgressError); + if (thrown instanceof SafeEgressError) { + expect(thrown.message).not.toContain('sk-leak-me'); // the raw error (host + key) never surfaces + expect(thrown.message).not.toContain('10.9.8.7'); + } + expect(dispose).toHaveBeenCalled(); // the socket is reaped on the fault + }); + + it('accepts a Request-object input (not just a url + init)', async () => { + let captured: HopRequest | undefined; + const fetch = createValidatedFetch(fakeDeps({ ips: ['203.0.113.9'], onConnect: (req) => (captured = req) })); + await fetch( + new Request('https://api.example.com/v1/chat/completions', { + method: 'POST', + headers: { authorization: 'Bearer sk-req' }, + body: '{"m":1}', + }), + ); + expect(captured?.method).toBe('POST'); + expect(captured?.headers?.['authorization']).toBe('Bearer sk-req'); + expect(captured?.body).toBe('{"m":1}'); + }); + + it('rejects an out-of-range HTTP status (a hostile 999) as a typed SafeEgressError, never a raw RangeError', async () => { + const dispose = vi.fn(); + const fetch = createValidatedFetch(fakeDeps({ ips: ['1.2.3.4'], status: 999, dispose })); + await expect(fetch('https://api.example.com/v1/models')).rejects.toBeInstanceOf(SafeEgressError); + expect(dispose).toHaveBeenCalled(); + }); }); diff --git a/apps/cli/src/engine/validated-fetch.ts b/apps/cli/src/engine/validated-fetch.ts index ee08772b..994e8d90 100644 --- a/apps/cli/src/engine/validated-fetch.ts +++ b/apps/cli/src/engine/validated-fetch.ts @@ -18,32 +18,46 @@ import { * * **Streaming-safe:** `connectValidated` returns a LIVE `AsyncIterable` body (it never buffers — `readBounded` is * a separate helper), which this wraps in a **backpressure-aware** `ReadableStream`, so an SSE completion streams - * chunk-by-chunk. **Secret-free:** `connectValidated`'s failures are reason-only `SafeEgressError`s (never the - * url/IP/host), and a body-read fault is normalized to a generic message — the `Authorization` key rides the - * request headers to the endpoint (as the API requires) but is never logged. The `EgressDeps` (DNS + connect) are - * injectable so the SSRF policy is deterministically unit-testable without real network/DNS. + * chunk-by-chunk. It deliberately does NOT wrap the call in `withEgressTimeout` — that would abort a long-lived + * stream at a fixed deadline; the caller's `AbortSignal` (the OpenAI SDK's own timeout, `boundedListModels`'s 15s, + * `validateProviderKey`'s 10s) is threaded straight through and tears the socket down on connect AND during + * streaming. Every escaping error is normalized to a reason-only `SafeEgressError` (never the url/IP/host/key) — + * `connectValidated`'s own throws already are, and a raw resolver/socket fault (a DNS error carries the non-secret + * hostname) is re-wrapped here. The `Authorization` key rides the request headers to the endpoint (as the API + * requires) but is never logged. The `EgressDeps` (DNS + connect) are injectable so the SSRF policy is + * deterministically unit-testable without real network/DNS. */ export type FetchLike = (input: string | URL | Request, init?: RequestInit) => Promise; -/** Statuses that MUST carry a null body (a `Response` with a body + one of these throws). */ -const NULL_BODY_STATUS: ReadonlySet = new Set([101, 204, 205, 304]); +/** Statuses that MUST carry a null body (a `Response` with a body + one of these throws). All ≥ 200 (a `< 200` + * status is itself out-of-range for `new Response` — handled by the range check in {@link toResponse}). */ +const NULL_BODY_STATUS: ReadonlySet = new Set([204, 205, 304]); /** Build the validated `fetch`. `deps` default to Node's real DNS + pinned-HTTPS connect. */ export function createValidatedFetch(deps: EgressDeps = nodeEgressDeps): FetchLike { return async (input, init) => { const req = await normalizeRequest(input, init); - const hop = await connectValidated( - req.url, - { - allowPrivate: false, // a custom base_url resolving to a private/loopback/metadata address is REFUSED - method: req.method, - ...(req.headers === undefined ? {} : { headers: req.headers }), - ...(req.body === undefined ? {} : { body: req.body }), - }, - deps, - req.signal, - ); + let hop: HopResponse; + try { + hop = await connectValidated( + req.url, + { + allowPrivate: false, // a custom base_url resolving to a private/loopback/metadata address is REFUSED + method: req.method, + ...(req.headers === undefined ? {} : { headers: req.headers }), + ...(req.body === undefined ? {} : { body: req.body }), + }, + deps, + req.signal, + ); + } catch (err) { + // connectValidated's policy throws are already typed SafeEgressErrors; a raw resolver/socket fault (e.g. a + // DNS error carrying the non-secret hostname) is re-wrapped so ONLY a reason-only SafeEgressError escapes. + throw err instanceof SafeEgressError + ? err + : new SafeEgressError('network', 'egress connect failed'); + } return toResponse(hop); }; } @@ -56,7 +70,11 @@ interface NormalizedRequest { readonly signal: AbortSignal; } -const EGRESS_METHODS: ReadonlySet = new Set(['GET', 'POST', 'PUT', 'DELETE']); +// The `as const satisfies` keeps this set in LOCKSTEP with the `EgressMethod` union — adding one here without +// updating the union (or vice versa) is a compile error, so the `isEgressMethod` narrowing below stays sound. +const EGRESS_METHODS = ['GET', 'POST', 'PUT', 'DELETE'] as const satisfies readonly EgressMethod[]; +const isEgressMethod = (method: string): method is EgressMethod => + (EGRESS_METHODS as readonly string[]).includes(method); /** Resolve the url / method / headers / body / signal from either a `Request` or a `(url, init)` pair. */ async function normalizeRequest( @@ -66,26 +84,28 @@ async function normalizeRequest( const isRequest = input instanceof Request; const url = isRequest ? input.url : typeof input === 'string' ? input : input.href; const rawMethod = (init?.method ?? (isRequest ? input.method : 'GET')).toUpperCase(); - if (!EGRESS_METHODS.has(rawMethod)) { + if (!isEgressMethod(rawMethod)) { // The OpenAI SDK uses only GET (models.list) + POST (completions); refuse anything else loudly rather than // silently downgrading a method (a secret-free, typed failure the SDK surfaces as a request error). throw new SafeEgressError('network', `unsupported egress method '${rawMethod}'`); } - const method = rawMethod as EgressMethod; // narrowed by the membership check above (a safe widening, not an unsound cast) const headers = headersToRecord(init?.headers ?? (isRequest ? input.headers : undefined)); const body = await bodyToString(init?.body ?? (isRequest ? input.body : undefined)); const signal = init?.signal ?? (isRequest ? input.signal : undefined) ?? new AbortController().signal; - return { url, method, headers, body, signal }; + return { url, method: rawMethod, headers, body, signal }; } /** Flatten a `RequestInit['headers']` (Headers | record | pairs) to a plain record; connectValidated re-sanitizes it. - * `new Headers(...)` normalizes every `HeadersInit` form (and lower-cases the keys) in one step. */ + * `new Headers(...)` normalizes every `HeadersInit` form (and lower-cases the keys) in one step. Drops + * `accept-encoding`: this fetch does NOT auto-decompress the streamed body (unlike a platform `fetch`), so it must + * never negotiate compression — else a gzip'd response would reach the SDK as raw bytes it can't parse. */ function headersToRecord( headers: RequestInit['headers'], ): Readonly> | undefined { if (headers === undefined) return undefined; const out: Record = {}; new Headers(headers).forEach((value, key) => { + if (key === 'accept-encoding') return; // never negotiate compression — we stream the body verbatim out[key] = value; }); return out; @@ -104,6 +124,13 @@ async function bodyToString( /** Map a validated {@link HopResponse} to a standard `Response` with a backpressure-aware streaming body. */ function toResponse(hop: HopResponse): Response { + // `new Response(..., { status })` throws a RangeError for a status outside [200, 599]. A hostile custom endpoint + // can emit a `999` (or a malformed line ⇒ `statusCode ?? 0`), so guard it into the typed, reason-only failure — + // never a raw RangeError escaping this wrapper — and reap the socket. + if (hop.status < 200 || hop.status > 599) { + hop.dispose(); + throw new SafeEgressError('network', 'egress returned an out-of-range HTTP status'); + } const headers = hop.headers ?? {}; if (NULL_BODY_STATUS.has(hop.status)) { hop.dispose(); // a null-body status must not carry a stream — reap the (empty) socket From 4dbf6e8b21483d6eaa6567778b4e68046fa19990 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 05:04:18 +0300 Subject: [PATCH 26/68] =?UTF-8?q?fix(llm,cli):=202.5.G=20S9=20Sonnet-revie?= =?UTF-8?q?w=20=E2=80=94=20exact-redact=20the=20key=20on=20the=20generate/?= =?UTF-8?q?stream=20error=20path?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Independent adversarial Sonnet review found a real BLOCKER (a demonstrable secret-persistence defect) plus mediums. Fixes: - BLOCKER: the generate()/stream() error path applied only the SHAPE-based scrubSecrets (Bearer/sk-/AIza), never the EXACT-key redaction that boundedListModels already does for listModels. A custom OpenAI-compatible endpoint's key has no vendor shape, so a hostile/misconfigured proxy that echoes the received credential in its error body would leak the real key (via the OpenAI SDK's APIError.message → LlmError.message) into the plaintext history.db + --json + the TUI — a CLAUDE.md #6 violation that S9 first makes reachable (base_url was dead config before). Thread the resolved key into openaiErrorToLlmError (and streamChunks + the media helpers, all reachable via a custom base_url) and redactKey the message/code before returning — mirroring boundedListModels. Added a redaction test with an opaque, shape-free key. - MEDIUM: a second `provider add ` with NO --base-url silently reset a prior custom base_url to the SDK default (upsert always set baseUrl). Now it preserves the existing row's base_url; only a genuinely new row gets the default. + test. - MEDIUM: complete the validated-fetch normalization guarantee — the single wrapping try now also covers normalizeRequest + toResponse, and toResponse disposes the socket on its own Response-ctor failure (a bad header from a swapped transport), so EVERY exit is a reason-only SafeEgressError and dispose-safe. - Tests (+4): the raw-connect-throw normalization (a DNS error's hostname never surfaces), the accept-encoding strip, and the 599/600 status-clamp boundary. Corrected the EGRESS_METHODS `satisfies` comment (guards one direction). Documented the run/chat/gate double-open as a deliberate, sequential (no-race) low-risk tradeoff. Toolchain: lint + typecheck + test (CLI 1418 + llm 516 + db 184) + build all green across all 15 tasks (the lone media-gc failure is the known flake — passes on retry). Refs: ADR-0065, ADR-0011, ADR-0006 Co-Authored-By: Claude --- apps/cli/src/commands/dispatch.ts | 6 +++ apps/cli/src/commands/provider.test.ts | 10 ++++ apps/cli/src/commands/provider.ts | 6 ++- apps/cli/src/engine/validated-fetch.test.ts | 36 +++++++++++++ apps/cli/src/engine/validated-fetch.ts | 30 +++++++---- packages/llm/src/adapters/openai.test.ts | 18 +++++++ packages/llm/src/adapters/openai.ts | 57 +++++++++++++++------ 7 files changed, 137 insertions(+), 26 deletions(-) diff --git a/apps/cli/src/commands/dispatch.ts b/apps/cli/src/commands/dispatch.ts index 1aa9e56f..bc07e22d 100644 --- a/apps/cli/src/commands/dispatch.ts +++ b/apps/cli/src/commands/dispatch.ts @@ -202,6 +202,12 @@ export function buildProviderTestArgs(input: CommandInput): ProviderCommandArgs * its adapter to the SSRF-validated endpoint, then close it. The custom adapters are built EAGERLY at resolver * creation (`applyCustomEndpoints` reads `list()` once), so no db handle is held past this call — a self-contained * short-lived read that needs no lifecycle threaded into the command's own db/teardown ordering. + * + * The `run`/`chat`/`gate` commands then re-open the same `history.db` for their own stores — a deliberate, PURELY + * SEQUENTIAL second open (the first handle is fully closed here first, so no WAL/lock race), accepted as the + * low-risk alternative to threading the db handle through each command's careful teardown. The `models` / + * `provider` paths avoid it entirely — they build the resolver from the db they already hold (`withModelsDeps` / + * `withProviderDeps`), and the long-lived Home builds it over its one open handle in the S7 port block. */ function storeAwareResolver( ctx: DispatchContext, diff --git a/apps/cli/src/commands/provider.test.ts b/apps/cli/src/commands/provider.test.ts index e5266de4..566720f5 100644 --- a/apps/cli/src/commands/provider.test.ts +++ b/apps/cli/src/commands/provider.test.ts @@ -196,6 +196,16 @@ describe('relavium provider commands (2.C)', () => { expect(d.store.get('anthropic')?.kind).toBe('anthropic'); }); + it('a second `add` with NO --base-url preserves a prior custom base_url (never silently resets it) (2.5.G S9)', async () => { + const d = deps({}); + await runProviderCommand( + { action: 'add', name: 'openai', baseUrl: 'https://my-proxy.example/v1' }, + d, + ); + await runProviderCommand({ action: 'add', name: 'openai' }, d); // re-run, no --base-url + expect(d.store.get('openai')?.baseUrl).toBe('https://my-proxy.example/v1'); // preserved, not reset to the SDK default + }); + it('set-key preserves a base URL set by a prior `add` (never clobbers it)', async () => { const d = deps({}); await runProviderCommand( diff --git a/apps/cli/src/commands/provider.ts b/apps/cli/src/commands/provider.ts index 7ddc4c70..c0d785e9 100644 --- a/apps/cli/src/commands/provider.ts +++ b/apps/cli/src/commands/provider.ts @@ -85,7 +85,7 @@ function providerList(deps: ProviderCommandDeps): void { function providerAdd(args: ProviderCommandArgs, deps: ProviderCommandDeps): void { const id = parseProviderId(requireName(args)); const meta = KNOWN_PROVIDERS[id]; - let baseUrl = meta.baseUrl; + let baseUrl: string; if (args.baseUrl !== undefined) { // A custom `--base-url` is **OpenAI-compatible only** this round (2.5.G S9, ADR-0065 §3) — refuse it on the // Anthropic/Gemini protocols with a clear message rather than silently ignoring it (the old dead-config bug). @@ -96,6 +96,10 @@ function providerAdd(args: ProviderCommandArgs, deps: ProviderCommandDeps): void ); } baseUrl = requireHttpsUrl(args.baseUrl); + } else { + // PRESERVE a previously-set custom base_url — a re-run of `relavium provider add ` with no `--base-url` must + // not silently reset it to the SDK default (2.5.G S9 review). A genuinely new row gets the provider's default. + baseUrl = deps.store.get(id)?.baseUrl ?? meta.baseUrl; } // Store the protocol `kind` for every provider (ADR-0065 §5 — populated for uniformity; the resolver derives it // from the closed id today, load-bearing only for a future custom provider). diff --git a/apps/cli/src/engine/validated-fetch.test.ts b/apps/cli/src/engine/validated-fetch.test.ts index 1906010d..abde0ce8 100644 --- a/apps/cli/src/engine/validated-fetch.test.ts +++ b/apps/cli/src/engine/validated-fetch.test.ts @@ -179,4 +179,40 @@ describe('createValidatedFetch', () => { await expect(fetch('https://api.example.com/v1/models')).rejects.toBeInstanceOf(SafeEgressError); expect(dispose).toHaveBeenCalled(); }); + + it('treats 599 as valid and 600 as out-of-range (the status-clamp boundary)', async () => { + const ok = createValidatedFetch(fakeDeps({ ips: ['1.2.3.4'], status: 599 })); + expect((await ok('https://api.example.com/v1/models')).status).toBe(599); + const bad = createValidatedFetch(fakeDeps({ ips: ['1.2.3.4'], status: 600 })); + await expect(bad('https://api.example.com/v1/models')).rejects.toBeInstanceOf(SafeEgressError); + }); + + it('normalizes a RAW connect/resolver throw to a reason-only SafeEgressError (never a raw leak)', async () => { + // A resolver that throws a raw Error (a DNS fault carrying the hostname) — the wrapper must re-wrap it so ONLY + // a reason-only SafeEgressError escapes (matching the contract the hardening commit added). + const deps: EgressDeps = { + resolveHost: () => Promise.reject(new Error('getaddrinfo ENOTFOUND my-secret-proxy.example')), + openConnection: () => Promise.reject(new Error('unused')), + }; + let thrown: unknown; + try { + await createValidatedFetch(deps)('https://my-secret-proxy.example/v1/models'); + } catch (err) { + thrown = err; + } + expect(thrown).toBeInstanceOf(SafeEgressError); + if (thrown instanceof SafeEgressError) { + expect(thrown.message).not.toContain('my-secret-proxy.example'); // the raw hostname-bearing error never surfaces + } + }); + + it('strips accept-encoding from the request (this fetch does not auto-decompress the streamed body)', async () => { + let captured: HopRequest | undefined; + const fetch = createValidatedFetch(fakeDeps({ ips: ['1.2.3.4'], onConnect: (req) => (captured = req) })); + await fetch('https://api.example.com/v1/models', { + headers: { 'accept-encoding': 'gzip, br', authorization: 'Bearer sk-x' }, + }); + expect(captured?.headers?.['accept-encoding']).toBeUndefined(); // never negotiate compression + expect(captured?.headers?.['authorization']).toBe('Bearer sk-x'); // the key header still rides through + }); }); diff --git a/apps/cli/src/engine/validated-fetch.ts b/apps/cli/src/engine/validated-fetch.ts index 994e8d90..87213607 100644 --- a/apps/cli/src/engine/validated-fetch.ts +++ b/apps/cli/src/engine/validated-fetch.ts @@ -37,10 +37,9 @@ const NULL_BODY_STATUS: ReadonlySet = new Set([204, 205, 304]); /** Build the validated `fetch`. `deps` default to Node's real DNS + pinned-HTTPS connect. */ export function createValidatedFetch(deps: EgressDeps = nodeEgressDeps): FetchLike { return async (input, init) => { - const req = await normalizeRequest(input, init); - let hop: HopResponse; try { - hop = await connectValidated( + const req = await normalizeRequest(input, init); + const hop = await connectValidated( req.url, { allowPrivate: false, // a custom base_url resolving to a private/loopback/metadata address is REFUSED @@ -51,14 +50,16 @@ export function createValidatedFetch(deps: EgressDeps = nodeEgressDeps): FetchLi deps, req.signal, ); + // `toResponse` disposes `hop` on its OWN construction failure (the catch here has no `hop` in scope). + return toResponse(hop); } catch (err) { - // connectValidated's policy throws are already typed SafeEgressErrors; a raw resolver/socket fault (e.g. a - // DNS error carrying the non-secret hostname) is re-wrapped so ONLY a reason-only SafeEgressError escapes. + // EVERY escaping error — a bad method (normalizeRequest), a policy/connect fault or a raw resolver/socket + // throw (connectValidated), or a response-mapping failure — is normalized to ONE reason-only SafeEgressError + // (never the url/IP/host/key). connectValidated's policy throws are already SafeEgressErrors (preserved). throw err instanceof SafeEgressError ? err - : new SafeEgressError('network', 'egress connect failed'); + : new SafeEgressError('network', 'egress request failed'); } - return toResponse(hop); }; } @@ -70,8 +71,8 @@ interface NormalizedRequest { readonly signal: AbortSignal; } -// The `as const satisfies` keeps this set in LOCKSTEP with the `EgressMethod` union — adding one here without -// updating the union (or vice versa) is a compile error, so the `isEgressMethod` narrowing below stays sound. +// `as const satisfies readonly EgressMethod[]` verifies every listed method IS a real `EgressMethod` (so the +// `isEgressMethod` narrowing is sound); a NEW union member not listed here simply isn't accepted (fails closed). const EGRESS_METHODS = ['GET', 'POST', 'PUT', 'DELETE'] as const satisfies readonly EgressMethod[]; const isEgressMethod = (method: string): method is EgressMethod => (EGRESS_METHODS as readonly string[]).includes(method); @@ -136,7 +137,16 @@ function toResponse(hop: HopResponse): Response { hop.dispose(); // a null-body status must not carry a stream — reap the (empty) socket return new Response(null, { status: hop.status, headers }); } - return new Response(hopBodyToStream(hop), { status: hop.status, headers }); + try { + return new Response(hopBodyToStream(hop), { status: hop.status, headers }); + } catch (err) { + // The `Response` constructor rejected the headers (a CR/LF/NUL / codepoint > 255 a swapped transport didn't + // filter) — the stream was never consumed, so reap the socket directly + surface a reason-only failure. + hop.dispose(); + throw err instanceof SafeEgressError + ? err + : new SafeEgressError('network', 'egress response could not be mapped'); + } } /** Wrap the HopResponse's live `AsyncIterable` body in a pull-based `ReadableStream` (backpressure — one chunk per diff --git a/packages/llm/src/adapters/openai.test.ts b/packages/llm/src/adapters/openai.test.ts index 95a3af3a..ffa845b8 100644 --- a/packages/llm/src/adapters/openai.test.ts +++ b/packages/llm/src/adapters/openai.test.ts @@ -833,6 +833,24 @@ describe('openaiErrorToLlmError — classification', () => { ).toMatchObject({ kind: 'unknown', retryable: false }); }); + it('EXACT-redacts the resolved key from an echoed error body (a custom endpoint\'s opaque key) (2.5.G S9)', () => { + // A custom OpenAI-compatible endpoint's key has no `sk-`/`Bearer` shape, so the shape-based scrubSecrets can't + // match it — a hostile/misconfigured proxy that echoes the received credential in its error body would leak the + // real key into history.db / --json / the TUI unless the resolved key is exact-redacted (CLAUDE.md #6). + const key = ['opaque', 'proxy', 'CREDENTIAL', '4f2a9'].join('-'); // no vendor key shape + const echoed = new APIError(401, undefined, `rejected token '${key}' for this endpoint`, undefined); + // WITHOUT the key (the listModels path redacts separately) the opaque token would pass through... + expect(openaiErrorToLlmError(echoed, 'openai').message).toContain(key); + // ...WITH the resolved key threaded (generate/stream/media), it is exact-redacted before it can escape. + const redacted = openaiErrorToLlmError(echoed, 'openai', key); + expect(redacted.message).not.toContain(key); + expect(redacted.kind).toBe('auth'); // classification is preserved + // Also redacts a key echoed in the error CODE field. + const codeEcho = new APIError(400, undefined, 'bad', undefined); + Object.assign(codeEcho, { code: `token_${key}_invalid` }); + expect(openaiErrorToLlmError(codeEcho, 'openai', key).code).not.toContain(key); + }); + it('classifies a content-policy / moderation code as content_filter regardless of HTTP status (1.AG §6)', () => { const policy = new APIError(400, undefined, 'blocked', undefined); Object.assign(policy, { code: 'content_policy_violation' }); diff --git a/packages/llm/src/adapters/openai.ts b/packages/llm/src/adapters/openai.ts index c4f146b4..0c53bb07 100644 --- a/packages/llm/src/adapters/openai.ts +++ b/packages/llm/src/adapters/openai.ts @@ -50,6 +50,7 @@ import { encodeMediaJobId, isAbortSignal, isRecord, + redactKey, toModelListing, } from './shared.js'; @@ -350,8 +351,8 @@ function mapOpenAiApiError( }); } -/** Classify any SDK throwable into a normalized `LlmError` — no vendor error shape escapes. */ -export function openaiErrorToLlmError(err: unknown, provider: ProviderId): LlmError { +/** The pure classification — an SDK throwable → a normalized `LlmError` (before any key redaction). */ +function classifyOpenAiError(err: unknown, provider: ProviderId): LlmError { if (err instanceof APIUserAbortError) { return makeLlmError({ provider, kind: 'cancelled', message: 'request aborted' }); } @@ -378,6 +379,26 @@ export function openaiErrorToLlmError(err: unknown, provider: ProviderId): LlmEr }); } +/** + * Classify any SDK throwable into a normalized `LlmError` — no vendor error shape escapes. When the resolved `key` + * is passed (`generate`/`stream`/media, which have it in scope), the message/code are **exact-key-redacted** + * (`redactKey`) before returning — the shape-based `scrubSecrets` (`Bearer …`/`sk-…`/`AIza…`) inside `makeLlmError` + * CANNOT match a CUSTOM OpenAI-compatible endpoint's OPAQUE key (2.5.G S9, ADR-0065), so a hostile/misconfigured + * proxy that echoes the received credential in its error body would otherwise leak the real key into the plaintext + * `history.db` / `--json` / the TUI (CLAUDE.md #6). This mirrors `boundedListModels`'s exact-redaction for `listModels`. + */ +export function openaiErrorToLlmError(err: unknown, provider: ProviderId, key?: string): LlmError { + const base = classifyOpenAiError(err, provider); + if (key === undefined || key.length === 0) return base; + return makeLlmError({ + provider: base.provider, + kind: base.kind, + message: redactKey(base.message, key), + ...(base.status === undefined ? {} : { status: base.status }), + ...(base.code === undefined ? {} : { code: redactKey(base.code, key) }), + }); +} + // --- Live model discovery: the id-only list filter (ADR-0064 §3) ------------------------------ /** @@ -909,11 +930,13 @@ function foldChatChunk(chunk: OpenAI.ChatCompletionChunk, state: OpenAiStreamSta return out; } -/** Fold the OpenAI chat-completion event stream into the canonical `StreamChunk` sequence. */ +/** Fold the OpenAI chat-completion event stream into the canonical `StreamChunk` sequence. `key` is threaded solely + * to exact-redact it from an error message (a custom endpoint's opaque key the shape-based scrub can't match). */ async function* streamChunks( client: OpenAI, req: LlmRequest, provider: ProviderId, + key: string, ): AsyncIterable { const state: OpenAiStreamState = { reasoningOpen: false, @@ -930,7 +953,7 @@ async function* streamChunks( buildRequestOptions(req), ); } catch (err) { - yield { type: 'error', error: openaiErrorToLlmError(err, provider) }; + yield { type: 'error', error: openaiErrorToLlmError(err, provider, key) }; return; } try { @@ -943,7 +966,7 @@ async function* streamChunks( } } } catch (err) { - yield { type: 'error', error: openaiErrorToLlmError(err, provider) }; + yield { type: 'error', error: openaiErrorToLlmError(err, provider, key) }; return; } // A stream that ends without a terminal finish_reason was truncated (dropped connection, partial @@ -1025,7 +1048,7 @@ export function createOpenAiAdapter(deps: OpenAiAdapterDeps = {}): LlmProvider { raw: completion, }; } catch (err) { - throw new LlmProviderError(openaiErrorToLlmError(err, providerId)); + throw new LlmProviderError(openaiErrorToLlmError(err, providerId, key)); } }, stream(req: LlmRequest, key: string): AsyncIterable { @@ -1033,7 +1056,7 @@ export function createOpenAiAdapter(deps: OpenAiAdapterDeps = {}): LlmProvider { assertStreamable(providerId, supports); assertMediaCapabilities(providerId, supports, req); // per-modality input/output gate (ADR-0031, 1.AE) assertNoStreamingMediaOutput(providerId, req); // media-out is generate()-only; streaming triad deferred (ADR-0046 §4) - return streamChunks(createClient(key), req, providerId); + return streamChunks(createClient(key), req, providerId, key); }, /** * Live model discovery (ADR-0064 §1) over the SDK's `models.list()`. The OpenAI/DeepSeek list is @@ -1105,13 +1128,13 @@ export function createOpenAiAdapter(deps: OpenAiAdapterDeps = {}): LlmProvider { // (images.generate, SYNC); audio → TTS (audio.speech, SYNC); video → Sora (videos.create, ASYNC LRO — // returns an opaque jobId the engine polls via pollMediaJob below). if (req.modality === 'image') { - return openAiGenerateImage(client, req, providerId); + return openAiGenerateImage(client, req, providerId, key); } if (req.modality === 'audio') { - return openAiGenerateSpeech(client, req, providerId); + return openAiGenerateSpeech(client, req, providerId, key); } if (req.modality === 'video') { - return openAiGenerateVideo(client, req, providerId); + return openAiGenerateVideo(client, req, providerId, key); } // Exhaustiveness: MEDIA_BILLED_MODALITIES is image|audio|video, so `modality` is `never` here. A new // member makes this assignment a COMPILE error — a future modality fails at build, never silently at @@ -1146,7 +1169,7 @@ export function createOpenAiAdapter(deps: OpenAiAdapterDeps = {}): LlmProvider { }), }; } - return pollMediaJobSora(createClient(key), jobId, providerId, signal); + return pollMediaJobSora(createClient(key), jobId, providerId, signal, key); }, // ADR-0062 context-compaction seam — the shared defaults (covers both OpenAI and DeepSeek via this one // factory; real usage is authoritative, so the estimate is only a pre-first-turn fallback). @@ -1165,6 +1188,7 @@ async function openAiGenerateImage( client: OpenAI, req: MediaGenRequest, providerId: ProviderId, + key: string, // threaded solely to exact-redact it from an error (a custom endpoint's opaque key, 2.5.G S9) ): Promise { if (req.count !== undefined && req.count > 1) { throw new LlmProviderError( @@ -1191,7 +1215,7 @@ async function openAiGenerateImage( isAbortSignal(req.signal) ? { signal: req.signal } : {}, ); } catch (err) { - throw new LlmProviderError(openaiErrorToLlmError(err, providerId)); + throw new LlmProviderError(openaiErrorToLlmError(err, providerId, key)); } const b64 = response.data?.[0]?.b64_json; if (b64 === undefined || b64.length === 0) { @@ -1220,6 +1244,7 @@ async function openAiGenerateSpeech( client: OpenAI, req: MediaGenRequest, providerId: ProviderId, + key: string, // threaded solely to exact-redact it from an error (a custom endpoint's opaque key, 2.5.G S9) ): Promise { // `count` (images-per-call) is a no-op for TTS — `audio.speech` is billed per input character and yields a // single audio stream, so there is no bill-N-deliver-1 hazard (unlike the image path's loud count>1 reject). @@ -1240,7 +1265,7 @@ async function openAiGenerateSpeech( // otherwise escape unclassified and flatten to an opaque `internal` instead of a classified LlmError. bytes = new Uint8Array(await response.arrayBuffer()); } catch (err) { - throw new LlmProviderError(openaiErrorToLlmError(err, providerId)); + throw new LlmProviderError(openaiErrorToLlmError(err, providerId, key)); } if (bytes.length === 0) { throw new LlmProviderError( @@ -1308,6 +1333,7 @@ async function openAiGenerateVideo( client: OpenAI, req: MediaGenRequest, providerId: ProviderId, + key: string, // threaded solely to exact-redact it from an error (a custom endpoint's opaque key, 2.5.G S9) ): Promise { const seconds = soraSeconds(req.durationSeconds, providerId); // throws bad_request if not 4/8/12 const size = soraSize(req.providerOptions); @@ -1323,7 +1349,7 @@ async function openAiGenerateVideo( isAbortSignal(req.signal) ? { signal: req.signal } : {}, ); } catch (err) { - throw new LlmProviderError(openaiErrorToLlmError(err, providerId)); + throw new LlmProviderError(openaiErrorToLlmError(err, providerId, key)); } if (video.id.length === 0) { throw new LlmProviderError( @@ -1374,6 +1400,7 @@ async function pollMediaJobSora( jobId: string, providerId: ProviderId, signal: AbortSignalLike | undefined, + key: string, // threaded solely to exact-redact it from an error (a custom endpoint's opaque key, 2.5.G S9) ): Promise { const vendorId = decodeMediaJobId(jobId); if (vendorId === undefined) { @@ -1401,7 +1428,7 @@ async function pollMediaJobSora( bytes = new Uint8Array(await response.arrayBuffer()); } } catch (err) { - throw new LlmProviderError(openaiErrorToLlmError(err, providerId)); + throw new LlmProviderError(openaiErrorToLlmError(err, providerId, key)); } switch (video.status) { case 'queued': From 4086c070d1d9a80228a2357a32a39014a6fef51e Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 06:11:35 +0300 Subject: [PATCH 27/68] =?UTF-8?q?feat(cli,core,llm,db):=202.5.G=20S10=20?= =?UTF-8?q?=E2=80=94=20user=20pricing=20+=20cost-path=20overlay?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Close the ADR-0064 §6 cost-cap gap: a model with no static price, once user-priced, is enforced by `max_cost_microcents` on every surface. - llm: add the optional `PricingOverlay` (ReadonlyMap) to priceModel/cost/CostTracker + budget-estimator — static MODEL_PRICING still wins for a known id; the overlay fills an UNKNOWN id only. - core: thread `resolvePrice` through BudgetGovernor (pre-egress) + AgentRunner/AgentTurn/AgentSession (realized) + the WorkflowEngine facade; engine stays platform-free (the overlay arrives as plain Relavium data). - db: `models pricing` writes a `source='user'` row; the upsert now PRESERVES every omitted field on update (media_surface/capabilities/costs), so a pricing-only write can never clobber a live/seed row's routing. - cli: `relavium models pricing --provider --input --output [--cached]` (USD→integer micro-cents, bounds-validated; canonical-id + unknown-provider guards); `buildUserPricing`/`loadUserPricingOverlay` host builders wired into chat, chat-resume, /clear rebuild, the Home, one-shot `agent run`, and `run`; `provider add --pricing-url` + seeded `ProviderMeta.pricingUrl` defaults. - docs: commands.md `models pricing` + `--pricing-url`. Refs: ADR-0065 §1/§2/§5, ADR-0064 §6 Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/chat/session-host.ts | 24 ++- apps/cli/src/commands/agent-run.ts | 8 +- apps/cli/src/commands/chat.ts | 23 +++ apps/cli/src/commands/dispatch.ts | 65 +++++++ apps/cli/src/commands/manifest.ts | 38 ++++ apps/cli/src/commands/models-pricing.test.ts | 178 ++++++++++++++++++ apps/cli/src/commands/models-pricing.ts | 147 +++++++++++++++ apps/cli/src/commands/provider.ts | 49 ++++- apps/cli/src/commands/run.ts | 7 + apps/cli/src/commands/specs.ts | 36 +++- apps/cli/src/engine/build-engine.ts | 16 ++ .../cli/src/engine/model-catalog-view.test.ts | 80 +++++++- apps/cli/src/engine/model-catalog-view.ts | 59 +++++- apps/cli/src/engine/pricing-overlay.test.ts | 107 +++++++++++ apps/cli/src/engine/pricing-overlay.ts | 62 ++++++ apps/cli/src/engine/providers.ts | 11 +- apps/cli/src/home/drive-home.tsx | 11 +- docs/reference/cli/commands.md | 10 +- packages/core/src/engine/agent-runner.ts | 5 + packages/core/src/engine/agent-session.ts | 10 + packages/core/src/engine/agent-turn.ts | 12 +- .../core/src/engine/budget-governor.test.ts | 59 +++++- packages/core/src/engine/budget-governor.ts | 12 +- packages/core/src/engine/engine.ts | 19 +- packages/db/src/model-catalog-store.test.ts | 53 ++++++ packages/db/src/model-catalog-store.ts | 39 +++- packages/llm/src/budget-estimator.test.ts | 34 +++- packages/llm/src/budget-estimator.ts | 13 +- packages/llm/src/cost-tracker.test.ts | 74 ++++++++ packages/llm/src/cost-tracker.ts | 41 +++- packages/llm/src/index.ts | 2 +- 31 files changed, 1262 insertions(+), 42 deletions(-) create mode 100644 apps/cli/src/commands/models-pricing.test.ts create mode 100644 apps/cli/src/commands/models-pricing.ts create mode 100644 apps/cli/src/engine/pricing-overlay.test.ts create mode 100644 apps/cli/src/engine/pricing-overlay.ts diff --git a/apps/cli/src/chat/session-host.ts b/apps/cli/src/chat/session-host.ts index bdbf7693..f555571c 100644 --- a/apps/cli/src/chat/session-host.ts +++ b/apps/cli/src/chat/session-host.ts @@ -16,6 +16,7 @@ import { type ToolDef, type ToolHost, } from '@relavium/core'; +import type { PricingOverlay } from '@relavium/llm'; import type { ManagerSkippedTool, McpClient, McpServerConfig } from '@relavium/mcp'; import type { AgentSessionRecord, @@ -103,6 +104,14 @@ export interface BuildChatSessionOptions { * one-line notice. Absent ⇒ a no-op (the warn stays non-blocking either way). */ readonly onBudgetWarning?: (warning: ChatBudgetWarning) => void; + /** + * The ADR-0065 §2 user-pricing overlay (2.5.G S10) — a `ReadonlyMap` the command projects + * from the `model_catalog` `source='user'` rows (via `buildUserPricing`). It flows into BOTH the pre-egress + * governor (so a user-priced model is enforced by `[chat].max_cost_microcents`) AND `SessionDeps.resolvePrice` + * (so the realized cost of the same model is tracked). Static `MODEL_PRICING` still wins for a known id. Absent ⇒ + * unknown models degrade cost governance to `allow` loudly, unchanged. + */ + readonly resolvePrice?: PricingOverlay; } /** A pre-egress budget warning surfaced to the chat surface (`on_exceed: 'warn'`) — secret-free counts only. */ @@ -156,7 +165,7 @@ const DEFAULT_FS_SCOPE = 'sandboxed' as const; /** The fields {@link buildSessionRuntime} reads — the platform-capability inputs shared by a fresh + resumed session. */ type SessionRuntimeOptions = Pick< BuildChatSessionOptions, - 'chat' | 'now' | 'providers' | 'toolHost' | 'onBudgetWarning' + 'chat' | 'now' | 'providers' | 'toolHost' | 'onBudgetWarning' | 'resolvePrice' >; /** @@ -208,7 +217,7 @@ function buildSessionRuntime( ? {} : { allowedCommandGlobs: opts.chat.allowedCommandGlobs }), }; - const governor = buildGovernorWiring(opts.chat, opts.onBudgetWarning); + const governor = buildGovernorWiring(opts.chat, opts.onBudgetWarning, opts.resolvePrice); // The session event sink (1.W): a draft → bus → stamped sequenceNumber/timestamp. Hoisted so a SURFACE // event (the in-REPL `/export`'s `session:exported`, 2.Q) can ride the same monotonic per-session counter. const emit = createSessionEventSink(bus, sessionId); @@ -246,6 +255,10 @@ function buildSessionRuntime( ...(governor === undefined ? {} : { preEgress: governor.preEgress, updateCost: governor.updateCost }), + // The realized-cost overlay (2.5.G S10, ADR-0065 §2) — so the CostTracker prices a user-priced (otherwise + // unknown) model instead of throwing UnknownModelError. Same map the governor uses; both fill an UNKNOWN id + // only (static MODEL_PRICING wins). Absent ⇒ unchanged (an unknown model's realized cost degrades loudly). + ...(opts.resolvePrice === undefined ? {} : { resolvePrice: opts.resolvePrice }), }; return { bus, deps, emit, host }; } @@ -382,6 +395,9 @@ export interface BuildResumedChatSessionOptions { readonly mcpRegistrations?: readonly McpServerRegistration[]; /** Sink for an `on_exceed: 'warn'` pre-egress budget warning (see {@link BuildChatSessionOptions}). */ readonly onBudgetWarning?: (warning: ChatBudgetWarning) => void; + /** The ADR-0065 §2 user-pricing overlay (2.5.G S10; see {@link BuildChatSessionOptions.resolvePrice}) — so a + * resumed session enforces + tracks a user-priced model exactly like a fresh one. */ + readonly resolvePrice?: PricingOverlay; } /** @@ -480,6 +496,7 @@ export interface GovernorWiring { export function buildGovernorWiring( chat: ResolvedChatConfig, onWarning?: (warning: ChatBudgetWarning) => void, + resolvePrice?: PricingOverlay, ): GovernorWiring | undefined { const cap = chat.maxCostMicrocents; if (cap === undefined || cap <= 0) return undefined; @@ -489,6 +506,9 @@ export function buildGovernorWiring( }; const governor = new BudgetGovernor({ budget, + // The ADR-0065 §2 user-pricing overlay — so the PRE-EGRESS estimate can price a user-priced (otherwise + // unknown) model and enforce the cost cap on it. Omit ⇒ an unknown model degrades to `allow` loudly. + ...(resolvePrice === undefined ? {} : { resolvePrice }), emit: (event) => { // `warn` is non-blocking BY CONTRACT. A misbehaving warn surface must never reject this emit — a // rejection would propagate as an `internal` turn error and break sendMessage — so swallow a sync throw. diff --git a/apps/cli/src/commands/agent-run.ts b/apps/cli/src/commands/agent-run.ts index 634ab31d..6dbfd456 100644 --- a/apps/cli/src/commands/agent-run.ts +++ b/apps/cli/src/commands/agent-run.ts @@ -8,6 +8,7 @@ import { cassetteResolver, loadCassette } from '../chat/fixture.js'; import { buildChatSession, type BuiltChatSession } from '../chat/session-host.js'; import { loadResolvedConfig } from '../config/load.js'; import { surfaceMcpSkipped } from '../engine/mcp-servers.js'; +import { loadUserPricingOverlay } from '../engine/pricing-overlay.js'; import { createProviderResolver, type ProviderResolver } from '../engine/providers.js'; import { CliError } from '../process/errors.js'; import { EXIT_CODES, type ExitCode } from '../process/exit-codes.js'; @@ -54,7 +55,7 @@ export async function agentRunCommand( ): Promise { const now = deps.now ?? Date.now; const uuid = deps.uuid ?? randomUUID; - const { config, projectConfigDir } = loadResolvedConfig({ + const { config, projectConfigDir, homeDir } = loadResolvedConfig({ cwd: deps.global.cwd, configPath: deps.global.configPath, }); @@ -69,6 +70,10 @@ export async function agentRunCommand( args.fixture === undefined ? (deps.providers ?? createProviderResolver(deps.io.env)) : cassetteResolver(loadCassette(args.fixture, deps.global.cwd)); + // The ADR-0065 §2 user-pricing overlay (2.5.G S10) — so a one-shot live turn enforces + tracks a user-priced + // model. SKIPPED under `--fixture`: a cassette replay must stay deterministic + fully offline (no local + // `history.db` dependency); the recorded run already carries its costs. + const resolvePrice = offline ? undefined : loadUserPricingOverlay(homeDir); // An unknown `` (path or id) throws a typed CliError here (exit 2), before any turn. The build is // async (2.R): it connects the agent's inline stdio `mcp_servers` (a connect failure is a fail-loud exit-2 @@ -86,6 +91,7 @@ export async function agentRunCommand( ? createMcpSecretResolver(deps.io.env) : (deps.mcpSecretResolver ?? createMcpSecretResolver(deps.io.env)), mcpRegistrations: offline ? [] : config.mcpServers, + ...(resolvePrice === undefined ? {} : { resolvePrice }), // FULLY offline in `--fixture` (cassette) mode: disable inbound MCP entirely so an agent's inline // `mcp_servers` are never connected (no config build, no spawn, no dial). The cassette already carries any // recorded tool results, so the replay needs no live MCP. diff --git a/apps/cli/src/commands/chat.ts b/apps/cli/src/commands/chat.ts index cceaea00..d8e8daed 100644 --- a/apps/cli/src/commands/chat.ts +++ b/apps/cli/src/commands/chat.ts @@ -44,6 +44,7 @@ import { } from '../chat/session-host.js'; import { loadResolvedConfig } from '../config/load.js'; import { assembleToolEnv } from '../engine/tool-host/assemble.js'; +import { buildUserPricingOverlay, loadUserPricingOverlay } from '../engine/pricing-overlay.js'; import { surfaceMcpSkipped } from '../engine/mcp-servers.js'; import { createProviderResolver, type ProviderResolver } from '../engine/providers.js'; import { openSessionStore, type OpenedSessionStore } from '../history/session-open.js'; @@ -245,6 +246,10 @@ export async function chatCommand(args: ChatCommandArgs, deps: ChatCommandDeps): const providers = deps.providers ?? createProviderResolver(deps.io.env); const mcpSecretResolver = deps.mcpSecretResolver ?? createMcpSecretResolver(deps.io.env); const store = createChatStore(deps.global.color); + // The ADR-0065 §2 user-pricing overlay (2.5.G S10) — a transient read of the `model_catalog` `source='user'` + // rows, so a user-priced model is enforced by `[chat].max_cost_microcents` + tracked in realized cost. NON-FATAL: + // an unopenable db yields `undefined` here and surfaces cleanly through the session store open below. + const resolvePrice = loadUserPricingOverlay(homeDir); // An unknown --agent / un-inferrable default model throws a typed CliError here (exit 2), before any session. // The build is async (2.R): it connects the agent's inline stdio `mcp_servers` (a connect failure is a @@ -259,6 +264,7 @@ export async function chatCommand(args: ChatCommandArgs, deps: ChatCommandDeps): providers, mcpSecretResolver, mcpRegistrations: config.mcpServers, + ...(resolvePrice === undefined ? {} : { resolvePrice }), onBudgetWarning: (warning) => deps.io.writeErr( `budget warning: ~${warning.thresholdPct}% of the ${warning.limitMicrocents}µ¢ cap reached\n`, @@ -322,6 +328,8 @@ export async function chatCommand(args: ChatCommandArgs, deps: ChatCommandDeps): buildSession: deps.buildSession ?? buildChatSession, io: deps.io, global: deps.global, + // The same overlay the initial session used — a `/clear` rebuild keeps the user-priced cost enforcement. + ...(resolvePrice === undefined ? {} : { resolvePrice }), }); return runReplLoop( @@ -368,6 +376,8 @@ export async function chatResumeCommand( let store: ChatStoreController; let persister: SessionPersister; let intro: string; + // The ADR-0065 §2 user-pricing overlay (2.5.G S10) — hoisted so the post-try `/clear` rebuild reuses it. + let resolvePrice: BuildChatSessionOptions['resolvePrice']; // The just-built resumed session OWNS its MCP connections; if a pre-loop step after a SUCCESSFUL build throws, // the catch must tear them down (the steady-state teardown is runReplLoop's finally, not yet entered). Undefined // when no server was declared OR the build self-cleaned its own post-connect fault (see buildResumedChatSession). @@ -379,6 +389,10 @@ export async function chatResumeCommand( if (loaded === undefined) { throw new CliError('invalid_invocation', `no session found with id ${args.sessionId}`); } + // The ADR-0065 §2 user-pricing overlay (2.5.G S10), read from the ALREADY-OPEN session db — so a resumed + // session enforces + tracks a user-priced model exactly like a fresh `chat`. An empty map (no user rows) is + // harmless. Inside the try, so any read fault hits the same teardown as the rest of the pre-loop wiring. + resolvePrice = buildUserPricingOverlay(opened.db); const resumed = await (deps.buildResumedSession ?? buildResumedChatSession)({ chat: config.chat, record: loaded.session, @@ -387,6 +401,7 @@ export async function chatResumeCommand( providers, mcpSecretResolver, mcpRegistrations: config.mcpServers, + resolvePrice, onBudgetWarning: (warning) => deps.io.writeErr( `budget warning: ~${warning.thresholdPct}% of the ${warning.limitMicrocents}µ¢ cap reached\n`, @@ -467,6 +482,8 @@ export async function chatResumeCommand( buildSession: deps.buildSession ?? buildChatSession, io: deps.io, global: deps.global, + // The same overlay the resumed session used — a `/clear` rebuild keeps the user-priced cost enforcement. + ...(resolvePrice === undefined ? {} : { resolvePrice }), }); // A resumed session already landed at idle inside `AgentSession.resume`; calling start() would throw and @@ -918,6 +935,9 @@ interface FreshChatWiringDeps { readonly opened: OpenedSessionStore; readonly buildSession: typeof buildChatSession; readonly onBudgetWarning: NonNullable; + /** The ADR-0065 §2 user-pricing overlay (2.5.G S10) — carried into the rebuilt session so a `/clear` keeps + * the user-priced cost enforcement of the session it replaced. */ + readonly resolvePrice: BuildChatSessionOptions['resolvePrice']; } async function buildFreshChatWiring(deps: FreshChatWiringDeps, intro: string): Promise { @@ -932,6 +952,7 @@ async function buildFreshChatWiring(deps: FreshChatWiringDeps, intro: string): P providers: deps.providers, mcpSecretResolver: deps.mcpSecretResolver, ...(deps.mcpRegistrations === undefined ? {} : { mcpRegistrations: deps.mcpRegistrations }), + ...(deps.resolvePrice === undefined ? {} : { resolvePrice: deps.resolvePrice }), onBudgetWarning: deps.onBudgetWarning, }); surfaceMcpSkipped(deps.io, built.mcpSkipped); @@ -992,6 +1013,7 @@ function createClearRebuild(params: { readonly buildSession: typeof buildChatSession; readonly io: CliIo; readonly global: GlobalOptions; + readonly resolvePrice?: BuildChatSessionOptions['resolvePrice']; }): (oldSessionId: string) => Promise { const wiringDeps: FreshChatWiringDeps = { chat: params.chat, @@ -1008,6 +1030,7 @@ function createClearRebuild(params: { global: params.global, opened: params.opened, buildSession: params.buildSession, + resolvePrice: params.resolvePrice, onBudgetWarning: (warning) => params.io.writeErr( `budget warning: ~${warning.thresholdPct}% of the ${warning.limitMicrocents}µ¢ cap reached\n`, diff --git a/apps/cli/src/commands/dispatch.ts b/apps/cli/src/commands/dispatch.ts index bc07e22d..886008d2 100644 --- a/apps/cli/src/commands/dispatch.ts +++ b/apps/cli/src/commands/dispatch.ts @@ -34,6 +34,7 @@ import { importCommand, type ImportCommandArgs } from './import.js'; import { listCommand } from './list.js'; import { logsCommand } from './logs.js'; import { modelsCommand, type ModelsCommandArgs } from './models.js'; +import { modelsPricingCommand, type ModelsPricingCommandArgs } from './models-pricing.js'; import { runProviderCommand, type ProviderCommandArgs, @@ -178,10 +179,12 @@ export function buildGateArgs(input: CommandInput): GateCommandArgs { export function buildProviderAddArgs(input: CommandInput): ProviderCommandArgs { const baseUrl = optString(input.options['baseUrl']); + const pricingUrl = optString(input.options['pricingUrl']); return { action: 'add', name: reqPositional(input, 0, 'name'), ...(baseUrl === undefined ? {} : { baseUrl }), + ...(pricingUrl === undefined ? {} : { pricingUrl }), }; } @@ -194,6 +197,42 @@ export function buildProviderTestArgs(input: CommandInput): ProviderCommandArgs }; } +/** Parse one USD/Mtok option string → a finite number (invocation-level: shape only; the command core owns the + * non-negative + ceiling domain rules). The raw value is NEVER echoed (a defensive no-terminal-injection habit). */ +function parseUsdPerMtok(raw: string, flag: string): number { + const trimmed = raw.trim(); + const value = Number(trimmed); + if (trimmed === '' || !Number.isFinite(value)) { + throw new CliError('invalid_invocation', `${flag} must be a finite number of USD per million tokens.`); + } + return value; +} + +export function buildModelsPricingArgs(input: CommandInput): ModelsPricingCommandArgs { + const provider = optString(input.options['provider']); + if (provider === undefined) { + throw new CliError('invalid_invocation', 'missing required option --provider .'); + } + const rawInput = optString(input.options['input']); + const rawOutput = optString(input.options['output']); + if (rawInput === undefined) { + throw new CliError('invalid_invocation', 'missing required option --input .'); + } + if (rawOutput === undefined) { + throw new CliError('invalid_invocation', 'missing required option --output .'); + } + const rawCached = optString(input.options['cached']); + return { + model: reqPositional(input, 0, 'model'), + provider, + inputUsdPerMtok: parseUsdPerMtok(rawInput, '--input'), + outputUsdPerMtok: parseUsdPerMtok(rawOutput, '--output'), + ...(rawCached === undefined + ? {} + : { cachedInputUsdPerMtok: parseUsdPerMtok(rawCached, '--cached') }), + }; +} + // ── executors (production dep wiring, copied verbatim from the old register* bodies) ── /** @@ -414,6 +453,31 @@ const executeModels: CommandExecutor = (_input, ctx) => withModelsDeps(ctx, { re const executeModelsRefresh: CommandExecutor = (_input, ctx) => withModelsDeps(ctx, { refresh: true }); +/** + * `models pricing ` (2.5.G S10, ADR-0065) — open the local db, build the catalog + provider stores over it, + * capture the user price, and ALWAYS close the db. No keychain / resolver / refresh service is needed (a pure local + * write), so this is a lighter path than {@link withModelsDeps}. Args are parsed FIRST (a bad flag fails exit-2 + * before the db opens); the core is injected in unit tests directly (never touching `~/.relavium/history.db`). + */ +const executeModelsPricing: CommandExecutor = (input, ctx) => { + const args = buildModelsPricingArgs(input); // a bad/absent flag is an invocation fault before any db work + const { homeDir } = loadResolvedConfig({ cwd: ctx.global.cwd, configPath: ctx.global.configPath }); + const { db, close } = openLocalDb(homeDir); + try { + const storeDeps = { uuid: () => randomUUID(), now: () => Date.now() }; + return Promise.resolve( + modelsPricingCommand(args, { + io: ctx.io, + global: ctx.global, + catalog: createModelCatalogStore(db, storeDeps), + providers: createProviderStore(db, storeDeps), + }), + ); + } finally { + close(); + } +}; + /** Open the local db + OS keychain for one `provider` invocation, run the core, and always close the db. */ async function withProviderDeps( ctx: DispatchContext, @@ -470,6 +534,7 @@ const COMMAND_EXECUTORS: ReadonlyMap = new Map ({ action: 'list' }))], ['provider.add', providerExecutor(buildProviderAddArgs)], [ diff --git a/apps/cli/src/commands/manifest.ts b/apps/cli/src/commands/manifest.ts index b80be0f1..be7ce399 100644 --- a/apps/cli/src/commands/manifest.ts +++ b/apps/cli/src/commands/manifest.ts @@ -294,6 +294,39 @@ const ENTRIES: readonly CommandManifestEntry[] = [ description: "Re-fetch each connected provider's live model list into the local cache.", effect: 'write', }, + { + id: 'models.pricing', + label: 'Set model pricing', + description: + 'Set a user price for a model the registry does not know (custom / new provider models).', + args: [ + { name: 'model', type: 'string', required: true, description: 'the model id to price' }, + { + name: 'provider', + type: 'string', + required: true, + description: 'the provider that serves the model (must be registered)', + }, + { + name: 'input', + type: 'string', + required: true, + description: 'input (prompt) price, USD per million tokens', + }, + { + name: 'output', + type: 'string', + required: true, + description: 'output (completion) price, USD per million tokens', + }, + { + name: 'cached', + type: 'string', + description: 'cache-read price, USD per million tokens (default 0)', + }, + ], + effect: 'write', + }, { id: 'provider.list', label: 'List providers', @@ -312,6 +345,11 @@ const ENTRIES: readonly CommandManifestEntry[] = [ description: 'provider name (e.g. anthropic)', }, { name: 'baseUrl', type: 'string', description: 'override the provider base URL' }, + { + name: 'pricingUrl', + type: 'string', + description: 'override the pricing reference page (where you find model prices)', + }, ], effect: 'write', }, diff --git a/apps/cli/src/commands/models-pricing.test.ts b/apps/cli/src/commands/models-pricing.test.ts new file mode 100644 index 00000000..6f52a503 --- /dev/null +++ b/apps/cli/src/commands/models-pricing.test.ts @@ -0,0 +1,178 @@ +import { + createClient, + createModelCatalogStore, + createProviderStore, + runMigrations, + type DbClient, + type ModelCatalogStore, + type ProviderStore, +} from '@relavium/db'; +import { KNOWN_MODEL_IDS } from '@relavium/llm'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { isCliError } from '../process/errors.js'; +import { EXIT_CODES } from '../process/exit-codes.js'; +import type { GlobalOptions } from '../process/options.js'; +import { captureIo, parseNdjson } from '../test-support.js'; +import { modelsPricingCommand, type ModelsPricingCommandArgs } from './models-pricing.js'; + +/** + * `relavium models pricing` command-core tests (2.5.G S10, ADR-0065 §1/§2). A REAL `:memory:` db + the real + * catalog/provider stores (the write + read-back is the point), no keychain/network. Covers the USD→micro-cents + * conversion, the canonical-id + unknown-provider + bad-price rejects, `--json`, and the re-price preservation. + */ + +function globalOptions(json = false): GlobalOptions { + return { json, color: false, cwd: process.cwd(), configPath: undefined, verbosity: 'normal' }; +} + +describe('modelsPricingCommand (2.5.G S10)', () => { + let client: DbClient; + let catalog: ModelCatalogStore; + let providers: ProviderStore; + + beforeEach(() => { + client = createClient(':memory:'); + runMigrations(client.db); + let n = 0; + const deps = { + uuid: () => `00000000-0000-4000-8000-${String(++n).padStart(12, '0')}`, + now: () => 1_700_000_000_000, + }; + providers = createProviderStore(client.db, deps); + providers.upsert({ name: 'openai', displayName: 'OpenAI', baseUrl: 'https://api.openai.com/v1' }); + catalog = createModelCatalogStore(client.db, deps); + }); + + afterEach(() => { + client.sqlite.close(); + }); + + function run(args: ModelsPricingCommandArgs, json = false): { code: number; out: string; err: string } { + const { io, out, err } = captureIo(); + const code = modelsPricingCommand(args, { + io, + global: globalOptions(json), + catalog, + providers, + }); + return { code, out: out(), err: err() }; + } + + const baseArgs = { + model: 'acme-custom-1', + provider: 'openai', + inputUsdPerMtok: 3, + outputUsdPerMtok: 9, + } satisfies ModelsPricingCommandArgs; + + it('captures a fresh user price as micro-cents (USD × 1e8), source=user (the cost-cap gap is closed)', () => { + const { code, out } = run(baseArgs); + expect(code).toBe(EXIT_CODES.success); + const listing = catalog.listAll().find((m) => m.modelId === 'acme-custom-1'); + expect(listing?.source).toBe('user'); + expect(listing?.inputCostPerMtokMicrocents).toBe(300_000_000); // $3 × 1e8 + expect(listing?.outputCostPerMtokMicrocents).toBe(900_000_000); // $9 × 1e8 + expect(listing?.cachedInputCostPerMtokMicrocents).toBe(0); // no --cached ⇒ 0 + expect(out).toContain('acme-custom-1'); + }); + + it('rounds a fractional USD price correctly ($0.15/Mtok → 15_000_000µ¢)', () => { + run({ ...baseArgs, inputUsdPerMtok: 0.15, outputUsdPerMtok: 0.6 }); + const listing = catalog.listAll().find((m) => m.modelId === 'acme-custom-1'); + expect(listing?.inputCostPerMtokMicrocents).toBe(15_000_000); + expect(listing?.outputCostPerMtokMicrocents).toBe(60_000_000); + }); + + it('stores an explicit --cached price', () => { + run({ ...baseArgs, cachedInputUsdPerMtok: 0.03 }); + const listing = catalog.listAll().find((m) => m.modelId === 'acme-custom-1'); + expect(listing?.cachedInputCostPerMtokMicrocents).toBe(3_000_000); // $0.03 × 1e8 + }); + + it('--json emits one key-free record with the stored micro-cents', () => { + const { code, out } = run(baseArgs, true); + expect(code).toBe(EXIT_CODES.success); + const [rec] = parseNdjson(out); + expect(rec).toEqual({ + model: 'acme-custom-1', + provider: 'openai', + source: 'user', + inputCostPerMtokMicrocents: 300_000_000, + outputCostPerMtokMicrocents: 900_000_000, + cachedInputCostPerMtokMicrocents: 0, + }); + }); + + it('REJECTS a canonical model id (the built-in price always wins) — nothing is written', () => { + const canonical = KNOWN_MODEL_IDS[0]; + if (canonical === undefined) throw new Error('test precondition: KNOWN_MODEL_IDS is non-empty'); + const err = runThrows({ ...baseArgs, model: canonical }); + expect(err.code).toBe('invalid_invocation'); + expect(err.message).toContain('built-in price'); + expect(catalog.listAll().find((m) => m.modelId === canonical)).toBeUndefined(); + }); + + it('REJECTS an unregistered provider (the catalog FK targets llm_providers) — nothing is written', () => { + const err = runThrows({ ...baseArgs, provider: 'anthropic' }); // not registered in this db + expect(err.code).toBe('invalid_invocation'); + expect(err.message).toContain('unknown provider'); + expect(catalog.listAll()).toHaveLength(0); + }); + + it('REJECTS a negative price', () => { + const err = runThrows({ ...baseArgs, inputUsdPerMtok: -1 }); + expect(err.code).toBe('invalid_invocation'); + expect(catalog.listAll()).toHaveLength(0); + }); + + it('REJECTS a non-finite price', () => { + const err = runThrows({ ...baseArgs, outputUsdPerMtok: Number.POSITIVE_INFINITY }); + expect(err.code).toBe('invalid_invocation'); + expect(catalog.listAll()).toHaveLength(0); + }); + + it('REJECTS an implausibly large price (typo guard)', () => { + const err = runThrows({ ...baseArgs, inputUsdPerMtok: 1_000_000 }); + expect(err.code).toBe('invalid_invocation'); + expect(catalog.listAll()).toHaveLength(0); + }); + + it('a bad --cached rejects BEFORE the write (no partially-applied row)', () => { + const err = runThrows({ ...baseArgs, cachedInputUsdPerMtok: -5 }); + expect(err.code).toBe('invalid_invocation'); + expect(catalog.listAll()).toHaveLength(0); + }); + + it('re-pricing an existing model preserves its display name + limits (only prices change)', () => { + const providerId = providers.list()[0]?.id ?? ''; + // Seed a richer existing row (as a live discovery would) with a display name + context. + catalog.upsert({ + providerId, + modelId: 'acme-custom-1', + displayName: 'Acme Custom (discovered)', + contextWindowTokens: 32_000, + maxOutputTokens: 4_000, + source: 'live', + }); + run({ ...baseArgs, inputUsdPerMtok: 1, outputUsdPerMtok: 2 }); + const listing = catalog.listAll().find((m) => m.modelId === 'acme-custom-1'); + expect(listing?.displayName).toBe('Acme Custom (discovered)'); // preserved + expect(listing?.contextWindowTokens).toBe(32_000); // preserved + expect(listing?.maxOutputTokens).toBe(4_000); // preserved + expect(listing?.inputCostPerMtokMicrocents).toBe(100_000_000); // $1 + expect(listing?.source).toBe('user'); // now a user price + }); + + /** Invoke and assert the command threw a CliError, returning its narrowed shape (no `as`). */ + function runThrows(args: ModelsPricingCommandArgs): { code: string; message: string } { + const { io } = captureIo(); + try { + modelsPricingCommand(args, { io, global: globalOptions(), catalog, providers }); + } catch (err) { + if (isCliError(err)) return { code: err.code, message: err.message }; + throw err; + } + throw new Error('expected modelsPricingCommand to throw a CliError'); + } +}); diff --git a/apps/cli/src/commands/models-pricing.ts b/apps/cli/src/commands/models-pricing.ts new file mode 100644 index 00000000..fa2fc267 --- /dev/null +++ b/apps/cli/src/commands/models-pricing.ts @@ -0,0 +1,147 @@ +import type { ModelCatalogStore, ProviderStore } from '@relavium/db'; +import { KNOWN_MODEL_IDS } from '@relavium/llm'; + +import { CliError } from '../process/errors.js'; +import { EXIT_CODES, type ExitCode } from '../process/exit-codes.js'; +import type { CliIo } from '../process/io.js'; +import type { GlobalOptions } from '../process/options.js'; +import { writeRecordLines } from '../render/records.js'; + +/** + * The `relavium models pricing ` capture command (workstream **2.5.G S10**, + * [ADR-0065](../../../../docs/decisions/0065-provider-economics-and-user-pricing.md) §1/§2) — hand-enter the + * per-Mtok text-token price of a model the static registry does NOT know (a custom-endpoint model, or a new + * provider model not yet in `MODEL_PRICING`), so the cost cap (`max_cost_microcents`) can enforce it. The price is + * stored as a `source='user'` `model_catalog` row (integer micro-cents, never float); a live `models refresh` NEVER + * clobbers it (ADR-0065 §1). It closes the ADR-0064 §6 gap where an unknown model's cost governance degraded to + * `allow` because no price existed. + * + * Framework-free (no `commander`): parsed args + injected stores in, output via {@link CliIo}; a fault throws a + * typed {@link CliError} (exit 2). The command is PURE of secrets — it writes only a model id + provider + prices, + * never a key. It writes nothing on any validation failure (the reject precedes the upsert). + * + * Precedence guard: a **canonical** model id (one already in `MODEL_PRICING`) is REJECTED — the static registry + * always wins for a known id in both the merge and the cost path (ADR-0065 §2), so a user override would be + * silently ignored; failing loud is honest. Provider guard: the `` must be a REGISTERED provider (the catalog + * row's FK targets `llm_providers`), else the user is told to add it first. + */ + +export interface ModelsPricingCommandArgs { + /** The model id to price — a NON-canonical id (a custom / not-yet-registered model). */ + readonly model: string; + /** The provider slug that serves the model (e.g. `openai`) — must already be registered. */ + readonly provider: string; + /** Input (prompt) price, USD per million tokens. */ + readonly inputUsdPerMtok: number; + /** Output (completion) price, USD per million tokens. */ + readonly outputUsdPerMtok: number; + /** Cache-read price, USD per million tokens; omitted ⇒ `0` (no cache discount). */ + readonly cachedInputUsdPerMtok?: number; +} + +export interface ModelsPricingCommandDeps { + readonly io: CliIo; + readonly global: GlobalOptions; + /** The catalog store — `upsert` writes the `source='user'` row; `listByProvider` preserves an existing row's + * display/limits on a re-price (the store overwrites those required columns). */ + readonly catalog: Pick; + /** The provider registry — resolves the `` → its internal `llm_providers` UUID (the catalog FK). */ + readonly providers: Pick; +} + +/** 1 USD/Mtok = 1e8 micro-cents/Mtok (ADR-0065 §2 — the exact inverse of `@relavium/llm`'s in-code `usd()`). */ +const USD_PER_MTOK_TO_MICROCENTS = 100_000_000; +/** + * A typo/absurdity ceiling: no real model is $100,000 / Mtok. It also keeps the micro-cent product ($100k × 1e8 = + * 1e13) far under `Number.MAX_SAFE_INTEGER` (~9.007e15), so the integer never loses precision on the way to the DB. + */ +const MAX_USD_PER_MTOK = 100_000; + +/** Convert one USD/Mtok flag to integer micro-cents, rejecting a non-finite / negative / absurd value (exit 2). */ +function usdToMicrocents(usdPerMtok: number, flag: string): number { + if (!Number.isFinite(usdPerMtok) || usdPerMtok < 0) { + throw new CliError( + 'invalid_invocation', + `${flag} must be a finite, non-negative number of USD per million tokens`, + ); + } + if (usdPerMtok > MAX_USD_PER_MTOK) { + throw new CliError( + 'invalid_invocation', + `${flag} ($${usdPerMtok}/Mtok) is implausibly large — the maximum is $${MAX_USD_PER_MTOK}/Mtok`, + ); + } + return Math.round(usdPerMtok * USD_PER_MTOK_TO_MICROCENTS); +} + +export function modelsPricingCommand( + args: ModelsPricingCommandArgs, + deps: ModelsPricingCommandDeps, +): ExitCode { + // Precedence guard (ADR-0065 §2): a canonical id always resolves to `MODEL_PRICING`, so a user override would be + // a silent no-op. Reject BEFORE any write — nothing is stored on a rejected invocation. + if ((KNOWN_MODEL_IDS as readonly string[]).includes(args.model)) { + throw new CliError( + 'invalid_invocation', + `'${args.model}' already has a built-in price — a user override would never take effect (the static registry always wins). Nothing written.`, + ); + } + // Provider guard: the catalog row's FK targets `llm_providers`, so the provider must already be registered. + const providerRow = deps.providers.list().find((p) => p.name === args.provider); + if (providerRow === undefined) { + throw new CliError( + 'invalid_invocation', + `unknown provider '${args.provider}' — register it first (e.g. \`relavium provider add ${args.provider}\` or \`relavium provider set-key ${args.provider}\`).`, + ); + } + // Convert + bounds-validate BEFORE the write (a bad `--cached` must not leave a partially-applied row). + const inputCostPerMtokMicrocents = usdToMicrocents(args.inputUsdPerMtok, '--input'); + const outputCostPerMtokMicrocents = usdToMicrocents(args.outputUsdPerMtok, '--output'); + const cachedInputCostPerMtokMicrocents = + args.cachedInputUsdPerMtok === undefined + ? 0 + : usdToMicrocents(args.cachedInputUsdPerMtok, '--cached'); + + // Preserve an existing row's display name + limits on a re-price (the store overwrites those REQUIRED columns; + // the media/provenance/pricing columns preserve themselves on omit). A fresh price defaults display → the id, + // limits → `0` (the "unknown" sentinel, which reads back as absent). + const existing = deps.catalog + .listByProvider(providerRow.id) + .find((m) => m.modelId === args.model); + + deps.catalog.upsert({ + providerId: providerRow.id, + modelId: args.model, + displayName: existing?.displayName ?? args.model, + contextWindowTokens: existing?.contextWindowTokens ?? 0, + maxOutputTokens: existing?.maxOutputTokens ?? 0, + source: 'user', + inputCostPerMtokMicrocents, + outputCostPerMtokMicrocents, + cachedInputCostPerMtokMicrocents, + }); + + if (deps.global.json) { + // Key-free record; prices echoed back as the stored integer micro-cents (the canonical unit). + writeRecordLines(deps.io, [ + { + model: args.model, + provider: args.provider, + source: 'user', + inputCostPerMtokMicrocents, + outputCostPerMtokMicrocents, + cachedInputCostPerMtokMicrocents, + }, + ]); + return EXIT_CODES.success; + } + + const cachedNote = + args.cachedInputUsdPerMtok === undefined + ? '' + : `, cached $${args.cachedInputUsdPerMtok}/Mtok`; + deps.io.writeOut( + `Set user pricing for ${args.model} (${args.provider}): input $${args.inputUsdPerMtok}/Mtok, output $${args.outputUsdPerMtok}/Mtok${cachedNote}. It applies to your next run/chat and survives \`models refresh\`.\n`, + ); + return EXIT_CODES.success; +} diff --git a/apps/cli/src/commands/provider.ts b/apps/cli/src/commands/provider.ts index c0d785e9..695fe10d 100644 --- a/apps/cli/src/commands/provider.ts +++ b/apps/cli/src/commands/provider.ts @@ -35,6 +35,9 @@ export interface ProviderCommandArgs { readonly name?: string; readonly baseUrl?: string; readonly model?: string; + /** `provider add --pricing-url ` (2.5.G S10, ADR-0065 §1) — override the seeded `pricing_reference_url` + * (the public pricing page the user consults to hand-enter a price). A display-only pointer, never fetched. */ + readonly pricingUrl?: string; } export interface ProviderCommandDeps { @@ -101,11 +104,27 @@ function providerAdd(args: ProviderCommandArgs, deps: ProviderCommandDeps): void // not silently reset it to the SDK default (2.5.G S9 review). A genuinely new row gets the provider's default. baseUrl = deps.store.get(id)?.baseUrl ?? meta.baseUrl; } + // The pricing REFERENCE url (2.5.G S10, ADR-0065 §1) — the user-supplied `--pricing-url` (validated), else PRESERVE + // an existing custom pointer (a re-run without the flag must not reset it), else the provider's default pricing page. + let pricingUrl: string; + if (args.pricingUrl !== undefined) { + pricingUrl = requireHttpsPricingUrl(args.pricingUrl); + } else { + pricingUrl = deps.store.get(id)?.pricingReferenceUrl ?? meta.pricingUrl; + } // Store the protocol `kind` for every provider (ADR-0065 §5 — populated for uniformity; the resolver derives it // from the closed id today, load-bearing only for a future custom provider). - const record = deps.store.upsert({ name: id, displayName: meta.displayName, baseUrl, kind: providerKind(id) }); + const record = deps.store.upsert({ + name: id, + displayName: meta.displayName, + baseUrl, + kind: providerKind(id), + pricingReferenceUrl: pricingUrl, + }); + // `record.baseUrl` / `record.pricingReferenceUrl` are validated HTTPS URLs (a custom value round-trips through + // `new URL().href`, so control bytes are percent-encoded) — terminal-safe to echo without a further sanitize. deps.io.writeOut( - `Registered provider '${id}' (${record.baseUrl}). Store a key with \`relavium provider set-key ${id}\`.\n`, + `Registered provider '${id}' (${record.baseUrl}). Store a key with \`relavium provider set-key ${id}\`. Find model prices at ${record.pricingReferenceUrl ?? pricingUrl} and set one with \`relavium models pricing\`.\n`, ); } @@ -122,6 +141,8 @@ async function providerSetKey(args: ProviderCommandArgs, deps: ProviderCommandDe displayName: meta.displayName, baseUrl: meta.baseUrl, kind: providerKind(id), + // Seed the default pricing pointer too, so a provider registered by `set-key` alone still carries it (2.5.G S10). + pricingReferenceUrl: meta.pricingUrl, }); } deps.store.setKeychainRef(id, account); // the ref, NEVER the key value @@ -187,6 +208,30 @@ function requireHttpsUrl(raw: string): string { return raw; } +/** + * Validate a user-supplied `--pricing-url` (2.5.G S10) — a parseable **HTTPS** URL with no embedded credentials. + * UNLIKE the base URL, a pricing reference is NEVER an egress target (never fetched/DNS-resolved), so the SSRF + * literal-host block deliberately does NOT apply — it may point at any HTTPS host, incl. an internal wiki. The + * NORMALIZED `url.href` is returned (not the raw input): `new URL()` percent-encodes any control byte in the + * path/query/hash and rejects it in the host, so the stored + later-echoed pointer is inherently terminal-safe + * (no separate sanitize needed at the render boundary). `javascript:` / `http:` / `user:pass@…` are rejected. + */ +function requireHttpsPricingUrl(raw: string): string { + let url: URL; + try { + url = new URL(raw); + } catch { + throw new CliError('invalid_invocation', '--pricing-url must be a valid URL.'); + } + if (url.protocol !== 'https:') { + throw new CliError('invalid_invocation', '--pricing-url must be HTTPS.'); + } + if (urlHasCredentials(raw)) { + throw new CliError('invalid_invocation', '--pricing-url must not embed credentials (user:pass@…).'); + } + return url.href; +} + /** Dispatch a `relavium provider ` to its core, mapping a keychain-unavailable backend to exit 2. */ export async function runProviderCommand( args: ProviderCommandArgs, diff --git a/apps/cli/src/commands/run.ts b/apps/cli/src/commands/run.ts index 0e296830..d3016056 100644 --- a/apps/cli/src/commands/run.ts +++ b/apps/cli/src/commands/run.ts @@ -24,6 +24,7 @@ import { sweepMediaAtTerminal, } from '../engine/media-gc.js'; import { buildMediaEngineWiring } from '../engine/media-wiring.js'; +import { buildUserPricingOverlay } from '../engine/pricing-overlay.js'; import { createProviderResolver, neededProviderIds, @@ -203,6 +204,11 @@ export async function runCommand(args: RunCommandArgs, deps: RunCommandDeps): Pr // fails fast at LOAD (exit 2), not only at the runtime FallbackChain pre-skip. `gate` runs the SAME check // (drive.ts), so a fresh run and a resume reject consistently. assertWorkflowCatalogValid(def, wiring.workflowModelCatalog); + // The ADR-0065 §2 user-pricing overlay (2.5.G S10), read from the SAME durable `history.db` — so a + // workflow using a user-priced model is enforced by `budget.max_cost_microcents` (pre-egress) + priced in + // realized cost (the agent node). Only wired on this durable-history branch: the in-memory unit/harness + // path has no db, hence no user rows. An empty map (no user rows) is harmless (fills nothing). + const resolvePrice = buildUserPricingOverlay(opened.db); engineOptions = { providers, toolEnv, @@ -211,6 +217,7 @@ export async function runCommand(args: RunCommandArgs, deps: RunCommandDeps): Pr ...(wiring.mediaCostEstimate === undefined ? {} : { mediaCostEstimate: wiring.mediaCostEstimate }), + ...(resolvePrice.size === 0 ? {} : { resolvePrice }), ...mcpOption, }; } diff --git a/apps/cli/src/commands/specs.ts b/apps/cli/src/commands/specs.ts index 78b57ce0..fe1cd5b3 100644 --- a/apps/cli/src/commands/specs.ts +++ b/apps/cli/src/commands/specs.ts @@ -456,6 +456,13 @@ function registerModels(program: Command, ctx?: CommandContext): void { const refresh = models .command('refresh') .description("Re-fetch each connected provider's live model list into the local cache."); + const pricing = models + .command('pricing ') + .description('Set a user price for a model the registry does not know (custom / new provider models).') + .requiredOption('--provider ', 'the provider that serves the model (must be registered)') + .requiredOption('--input ', 'input (prompt) price, USD per million tokens') + .requiredOption('--output ', 'output (completion) price, USD per million tokens') + .option('--cached ', 'cache-read price, USD per million tokens (default 0)'); if (ctx === undefined) { models.action(() => { @@ -467,6 +474,12 @@ function registerModels(program: Command, ctx?: CommandContext): void { '`relavium models refresh` requires the CLI runtime context.', ); }); + pricing.action(() => { + throw new CliError( + 'not_implemented', + '`relavium models pricing` requires the CLI runtime context.', + ); + }); return; } @@ -480,6 +493,26 @@ function registerModels(program: Command, ctx?: CommandContext): void { ctx, ); }); + pricing.action( + async ( + model: string, + opts: { provider?: string; input?: string; output?: string; cached?: string }, + ) => { + ctx.result.exitCode = await executeCommand( + 'models.pricing', + { + positionals: [model], + options: { + provider: opts.provider, + input: opts.input, + output: opts.output, + cached: opts.cached, + }, + }, + ctx, + ); + }, + ); } /** Register `relavium provider` and its subcommands (2.C). Each dispatch opens the local db + keychain per invocation. */ @@ -493,7 +526,8 @@ function registerProvider(program: Command, ctx?: CommandContext): void { const add = provider .command('add ') .description('Register a provider.') - .option('--base-url ', 'override the provider base URL'); + .option('--base-url ', 'override the provider base URL') + .option('--pricing-url ', 'override the pricing reference page (where you find model prices)'); const setKey = provider .command('set-key ') .description('Store a provider API key in the OS keychain (the key is read from stdin).'); diff --git a/apps/cli/src/engine/build-engine.ts b/apps/cli/src/engine/build-engine.ts index ab872d6a..153386c7 100644 --- a/apps/cli/src/engine/build-engine.ts +++ b/apps/cli/src/engine/build-engine.ts @@ -11,6 +11,7 @@ import { type ToolDef, type ToolHost, } from '@relavium/core'; +import type { PricingOverlay } from '@relavium/llm'; import type { MediaCostEstimate, MediaSurface } from '@relavium/shared'; import { createCliHost } from './host.js'; @@ -49,6 +50,15 @@ export interface BuildEngineOptions { * `fs_scope`. The MCP arm is merged on top below. */ readonly toolEnv?: { readonly workspaceDir: string; readonly fsScopeTier: FsScopeTier }; + /** + * The ADR-0065 §2 user-pricing overlay (2.5.G S10) — a `ReadonlyMap` the caller (`run.ts`) + * projects from the `model_catalog` `source='user'` rows. Threaded into BOTH the workflow PRE-EGRESS governor + * (so a user-priced model is enforced by `budget.max_cost_microcents`) AND the agent node's realized + * `AgentRunnerDeps.resolvePrice` (so the same model's realized cost is tracked, not thrown as `UnknownModel`). + * Static `MODEL_PRICING` still wins for a known id. Absent ⇒ an unknown model degrades cost governance to + * `allow` loudly, unchanged. + */ + readonly resolvePrice?: PricingOverlay; } /** @@ -118,10 +128,16 @@ export async function buildEngine(options: BuildEngineOptions = {}): Promise { expect(view.refreshedAt).toBeUndefined(); }); }); + +describe('buildUserPricing (2.5.G S10, ADR-0065 §2)', () => { + it('projects a source="user" row into a ModelPricing keyed by model id', () => { + const overlay = buildUserPricing({ + rows: [ + row({ + modelId: 'acme-custom-1', + providerId: 'p-openai', + source: 'user', + inputCostPerMtokMicrocents: 300_000_000, + outputCostPerMtokMicrocents: 900_000_000, + cachedInputCostPerMtokMicrocents: 12_345, + contextWindowTokens: 32_000, + maxOutputTokens: 4_000, + }), + ], + providerSlug: slugResolver({ 'p-openai': 'openai' }), + }); + const priced = overlay.get('acme-custom-1'); + expect(priced?.provider).toBe('openai'); + expect(priced?.nativeId).toBe('acme-custom-1'); + expect(priced?.inputPerMtokMicrocents).toBe(300_000_000); + expect(priced?.outputPerMtokMicrocents).toBe(900_000_000); + expect(priced?.cachedInputPerMtokMicrocents).toBe(12_345); + expect(priced?.contextWindowTokens).toBe(32_000); + expect(priced?.maxOutputTokens).toBe(4_000); + }); + + it('includes ONLY source="user" rows (a live/static row is not a user price)', () => { + const overlay = buildUserPricing({ + rows: [ + row({ modelId: 'live-model', providerId: 'p-openai', source: 'live' }), + row({ modelId: 'static-model', providerId: 'p-openai', source: 'static' }), + row({ modelId: 'user-model', providerId: 'p-openai', source: 'user' }), + ], + providerSlug: slugResolver({ 'p-openai': 'openai' }), + }); + expect([...overlay.keys()]).toEqual(['user-model']); + }); + + it('drops a user row whose provider UUID resolves to a non-enum slug (never injects under a known provider)', () => { + const overlay = buildUserPricing({ + rows: [row({ modelId: 'rogue-priced', providerId: 'rogue', source: 'user' })], + providerSlug: slugResolver({}), // 'rogue' → 'rogue', not a ProviderId + }); + expect(overlay.size).toBe(0); + }); + + it('defaults absent context/output limits to 0 (the "unknown" sentinel) without throwing', () => { + const overlay = buildUserPricing({ + rows: [row({ modelId: 'no-limits', providerId: 'p-openai', source: 'user' })], + providerSlug: slugResolver({ 'p-openai': 'openai' }), + }); + const priced = overlay.get('no-limits'); + expect(priced?.contextWindowTokens).toBe(0); + expect(priced?.maxOutputTokens).toBe(0); + }); + + it('buildMergedCatalog fills the merge userPricing tier from the user rows (an unknown id becomes priceKnown)', () => { + const view = buildMergedCatalog({ + rows: [ + row({ + modelId: 'acme-custom-1', + providerId: 'p-openai', + source: 'user', + inputCostPerMtokMicrocents: 300_000_000, + outputCostPerMtokMicrocents: 900_000_000, + }), + ], + providerSlug: slugResolver({ 'p-openai': 'openai' }), + now: 0, + }); + const entry = view.entries.find((e) => e.modelId === 'acme-custom-1'); + expect(entry?.pricingSource).toBe('user'); + expect(entry?.priceKnown).toBe(true); + expect(entry?.pricing?.inputPerMtokMicrocents).toBe(300_000_000); + }); +}); diff --git a/apps/cli/src/engine/model-catalog-view.ts b/apps/cli/src/engine/model-catalog-view.ts index cb570de9..a8d9c71d 100644 --- a/apps/cli/src/engine/model-catalog-view.ts +++ b/apps/cli/src/engine/model-catalog-view.ts @@ -3,6 +3,8 @@ import { mergeModelCatalog, type ModelCatalogEntry, type ModelListing, + type ModelPricing, + type PricingOverlay, type ProviderId, } from '@relavium/llm'; import { LLM_PROVIDERS } from '@relavium/shared'; @@ -16,9 +18,12 @@ import { LLM_PROVIDERS } from '@relavium/shared'; * merge, whose static tier is the in-code `MODEL_PRICING`. Keeping this in the host — not the store, not the merge * — is what lets `@relavium/llm`/`@relavium/core` stay platform-free while every surface reuses the one merge. * - * The ADR-0065 USER-pricing tier is intentionally NOT built here yet — S7 ships availability + static pricing; the - * `source='user'` rows are populated + merged in S10 (the merge already accepts an optional `userPricing` slot, so - * that lands additively with no signature change). + * The ADR-0065 USER-pricing tier is built by {@link buildUserPricing} (workstream **2.5.G S10**): it projects the + * `source='user'` rows into the ONE `ReadonlyMap` that serves BOTH consumers — the merge's + * `userPricing` slot (so the `/models` picker shows a user-priced model's cost) AND the cost path's + * {@link PricingOverlay} (host-injected exactly like `keyFor`, so the budget governor enforces `max_cost_microcents` + * on an otherwise-unknown model). Static `MODEL_PRICING` still wins for a known id in both — the user tier only ever + * fills an UNKNOWN id (ADR-0065 §2), so a user can never silently misprice a shipped model. */ /** The merged catalog for the picker + the newest live-refresh stamp (the "last updated" freshness badge). */ @@ -61,6 +66,51 @@ function rowToListing(row: ModelCatalogListing): ModelListing { }; } +/** + * Map a `source='user'` catalog row → a seam {@link ModelPricing} (the ADR-0065 user tier). The DB stores integer + * micro-cents in the three `*_per_mtok_microcents` columns (NOT NULL, default `0`) — a captured price is a real + * value; a `0` means "not set for this dimension" and costs that dimension as free, which is the user's declared + * intent. Media output rates + cache-write are NOT user-capturable (no column), so they stay undefined — the cost + * fold degrades those to 0 (H4: never hard-fail on a missing rate). The context/output limits carry through so the + * merged picker and the footer context indicator can show them for an otherwise-unknown model. + */ +function rowToUserPricing(row: ModelCatalogListing, provider: ProviderId): ModelPricing { + return { + provider, + nativeId: row.modelId, + displayName: row.displayName, + contextWindowTokens: row.contextWindowTokens ?? 0, + maxOutputTokens: row.maxOutputTokens ?? 0, + inputPerMtokMicrocents: row.inputCostPerMtokMicrocents, + outputPerMtokMicrocents: row.outputCostPerMtokMicrocents, + cachedInputPerMtokMicrocents: row.cachedInputCostPerMtokMicrocents, + ...(row.deprecationDate !== undefined + ? { deprecatedAt: new Date(row.deprecationDate).toISOString() } + : {}), + }; +} + +/** + * Project the active catalog rows into the ADR-0065 USER-pricing map — the single source that feeds BOTH the merge's + * `userPricing` slot and the cost path's {@link PricingOverlay}. Only `source='user'` rows contribute; a row whose + * provider UUID resolves to a non-enum slug is dropped (a mis-keyed or future custom-provider row can never inject a + * price under a known provider). The map is keyed by model id — the same key the merge and {@link priceModel} look + * up — so a user price reaches an unknown model in both the picker and the governor with one build. + */ +export function buildUserPricing(input: { + readonly rows: readonly ModelCatalogListing[]; + readonly providerSlug: (uuid: string) => string; +}): PricingOverlay { + const map = new Map(); + for (const row of input.rows) { + if (row.source !== 'user') continue; // only the user-pricing rows carry an authored price + const slug = input.providerSlug(row.providerId); + if (!isProviderId(slug)) continue; // an unmapped UUID / non-enum provider — never inject under a known provider + map.set(row.modelId, rowToUserPricing(row, slug)); + } + return map; +} + /** * Project the active catalog rows into the merged `/models` view. Partitions the `source='live'` rows into a * per-`ProviderId` live map, then delegates to the pure merge. A provider is added to the live map only when it @@ -88,6 +138,7 @@ export function buildMergedCatalog(input: BuildMergedCatalogInput): MergedCatalo list.push(rowToListing(row)); live.set(slug, list); } - const entries = mergeModelCatalog({ live, now: input.now }); + const userPricing = buildUserPricing({ rows: input.rows, providerSlug: input.providerSlug }); + const entries = mergeModelCatalog({ live, userPricing, now: input.now }); return { entries, refreshedAt }; } diff --git a/apps/cli/src/engine/pricing-overlay.test.ts b/apps/cli/src/engine/pricing-overlay.test.ts new file mode 100644 index 00000000..39a16b42 --- /dev/null +++ b/apps/cli/src/engine/pricing-overlay.test.ts @@ -0,0 +1,107 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; + +import { + createClient, + createModelCatalogStore, + createProviderStore, + runMigrations, + type Db, + type DbClient, +} from '@relavium/db'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { openLocalDb } from '../db/open.js'; +import { buildUserPricingOverlay, loadUserPricingOverlay } from './pricing-overlay.js'; + +/** + * `pricing-overlay` host-loader tests (2.5.G S10, ADR-0065 §2). `buildUserPricingOverlay` is driven over a real + * `:memory:` db + stores (the projection of `source='user'` rows is the point); `loadUserPricingOverlay` gets a + * real temp-home round-trip (open → seed → reopen → read) plus its NON-FATAL "missing db ⇒ undefined" contract. + */ + +const NOW = 1_700_000_000_000; +let uuidSeq = 0; +const deps = { + uuid: () => `00000000-0000-4000-8000-${String(++uuidSeq).padStart(12, '0')}`, + now: () => NOW, +}; + +/** Seed one registered provider + a `source='user'` priced model on `db`. */ +function seedUserPriced(db: Db): void { + const providerStore = createProviderStore(db, deps); + const providerId = providerStore.upsert({ + name: 'openai', + displayName: 'OpenAI', + baseUrl: 'https://api.openai.com/v1', + }).id; + createModelCatalogStore(db, deps).upsert({ + providerId, + modelId: 'acme-custom-1', + displayName: 'Acme Custom 1', + contextWindowTokens: 32_000, + maxOutputTokens: 4_000, + source: 'user', + inputCostPerMtokMicrocents: 300_000_000, + outputCostPerMtokMicrocents: 900_000_000, + }); +} + +describe('buildUserPricingOverlay (over an open db)', () => { + let client: DbClient; + + beforeEach(() => { + client = createClient(':memory:'); + runMigrations(client.db); + }); + afterEach(() => { + client.sqlite.close(); + }); + + it('projects the source="user" rows into a ModelPricing overlay keyed by model id', () => { + seedUserPriced(client.db); + const overlay = buildUserPricingOverlay(client.db); + const priced = overlay.get('acme-custom-1'); + expect(priced?.provider).toBe('openai'); + expect(priced?.inputPerMtokMicrocents).toBe(300_000_000); + expect(priced?.outputPerMtokMicrocents).toBe(900_000_000); + }); + + it('is an empty map when there are no user rows (harmless — fills nothing)', () => { + const overlay = buildUserPricingOverlay(client.db); + expect(overlay.size).toBe(0); + }); +}); + +describe('loadUserPricingOverlay (self-contained transient open)', () => { + let home: string; + + beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'relavium-pricing-')); + }); + afterEach(() => { + rmSync(home, { recursive: true, force: true }); + }); + + it('opens the real history.db, reads the user pricing, and closes (round-trip)', () => { + // Seed via one connection, then load via a FRESH transient open — proving the durable read path. + const opened = openLocalDb(home); + try { + seedUserPriced(opened.db); + } finally { + opened.close(); + } + const overlay = loadUserPricingOverlay(home); + expect(overlay?.get('acme-custom-1')?.inputPerMtokMicrocents).toBe(300_000_000); + }); + + it('returns undefined NON-FATALLY when the db path is unopenable (the surface reports the fault itself)', () => { + // Put a regular FILE where the home dir's `.relavium/` would go: `ensureGlobalConfigDir` can't `mkdir` a + // directory under a file (ENOTDIR), so `openLocalDb` throws — and the loader degrades to `undefined` rather + // than propagating (the surface's own session/run store open is the authoritative fault report, a clean exit 2). + const blocker = join(home, 'blocker'); + writeFileSync(blocker, 'not a directory'); + expect(loadUserPricingOverlay(blocker)).toBeUndefined(); + }); +}); diff --git a/apps/cli/src/engine/pricing-overlay.ts b/apps/cli/src/engine/pricing-overlay.ts new file mode 100644 index 00000000..05634a60 --- /dev/null +++ b/apps/cli/src/engine/pricing-overlay.ts @@ -0,0 +1,62 @@ +import { randomUUID } from 'node:crypto'; + +import { createModelCatalogStore, createProviderStore, type Db } from '@relavium/db'; +import type { PricingOverlay } from '@relavium/llm'; + +import { openLocalDb } from '../db/open.js'; +import { buildUserPricing } from './model-catalog-view.js'; + +/** + * The host loader for the ADR-0065 §2 USER-pricing overlay (workstream **2.5.G S10**) — the `ReadonlyMap` every cost-enforcing surface injects into its budget governor + realized `CostTracker` so a model + * with no static price, once user-priced, is enforced by `max_cost_microcents` (the cost-cap gap ADR-0064 §6 left + * open). It is the DB-facing counterpart of the pure {@link buildUserPricing}: it projects the `model_catalog` + * `source='user'` rows (translating each internal provider UUID → its slug) into the one overlay that serves both + * the pre-egress estimate and the realized fold. Static `MODEL_PRICING` still wins for a known id — the user tier + * only fills an UNKNOWN id — so a user can never silently misprice a shipped model. + * + * Living in the host (not `@relavium/core`/`@relavium/llm`) is what keeps the engine platform-free: the engine + * receives a plain injected map, exactly like `keyFor`, and never imports `@relavium/db`. + */ + +/** Fresh store deps for a transient read — a self-generated UUID source + wall clock (no row is written here). */ +const readStoreDeps = { uuid: () => randomUUID(), now: () => Date.now() } as const; + +/** + * Build the overlay over an ALREADY-OPEN local db — the caller owns the db lifecycle (the Home + the run path both + * already hold a `history.db` handle for the session/run store). Reads a fresh `listAll()` snapshot so a mid-session + * `models pricing` write is reflected on the next build. Never writes; never throws on data (a non-`user` or + * non-enum-provider row is skipped by {@link buildUserPricing}). + */ +export function buildUserPricingOverlay(db: Db): PricingOverlay { + const catalogStore = createModelCatalogStore(db, readStoreDeps); + const providerStore = createProviderStore(db, readStoreDeps); + // Eager (not the lazy-memoized dispatch resolver): there is no refresh-then-render window here — the map is built + // and consumed in one breath, so a plain snapshot of the current provider rows is correct. + const slugByUuid = new Map(providerStore.list().map((p): [string, string] => [p.id, p.name])); + return buildUserPricing({ + rows: catalogStore.listAll(), + providerSlug: (uuid) => slugByUuid.get(uuid) ?? uuid, + }); +} + +/** + * Load the overlay via a SELF-CONTAINED transient open→read→close, for a surface with no db handle of its own yet + * (`relavium chat`, one-shot `agent run`). Deliberately NON-FATAL: an unopenable/unmigratable `history.db` yields + * `undefined` (cost governance degrades to the no-overlay behavior — unknown models `allow` loudly), so the db + * fault surfaces through the surface's OWN store-open error path (a clean exit 2), not an opaque throw here. The + * transient handle closes before the surface opens its real store — sequential opens of the one SQLite file. + */ +export function loadUserPricingOverlay(homeDir: string): PricingOverlay | undefined { + let opened: { db: Db; close: () => void }; + try { + opened = openLocalDb(homeDir); + } catch { + return undefined; // a broken db is reported by the surface's own store open, not this best-effort read + } + try { + return buildUserPricingOverlay(opened.db); + } finally { + opened.close(); + } +} diff --git a/apps/cli/src/engine/providers.ts b/apps/cli/src/engine/providers.ts index 2683096b..1c6b3b7c 100644 --- a/apps/cli/src/engine/providers.ts +++ b/apps/cli/src/engine/providers.ts @@ -43,11 +43,16 @@ export function keyHint(key: string): string { return key.length <= 4 ? '••••' : `••••${key.slice(-4)}`; } -/** A known provider's metadata — display name, base URL, and a cheap model for the live key test. */ +/** A known provider's metadata — display name, base URL, a cheap model for the live key test, and the public + * pricing page (the ADR-0065 §1 default `pricing_reference_url`, where a user finds a price to hand-enter). */ export interface ProviderMeta { readonly displayName: string; readonly baseUrl: string; readonly testModel: string; + /** The provider's public pricing page — the default `llm_providers.pricing_reference_url` seeded on + * `provider add` (overridable with `--pricing-url`). A display-only pointer; NEVER fetched (not an egress + * target), so it needs no SSRF gate. Verified 2026-07-06 against each provider's live docs. */ + readonly pricingUrl: string; } /** @@ -70,21 +75,25 @@ export const KNOWN_PROVIDERS: Record<(typeof KNOWN_PROVIDER_IDS)[number], Provid displayName: 'Anthropic', baseUrl: 'https://api.anthropic.com', testModel: 'claude-haiku-4-5', + pricingUrl: 'https://platform.claude.com/docs/en/about-claude/pricing', }, openai: { displayName: 'OpenAI', baseUrl: 'https://api.openai.com/v1', testModel: 'gpt-5.4-mini', + pricingUrl: 'https://platform.openai.com/docs/pricing', }, gemini: { displayName: 'Google Gemini', baseUrl: 'https://generativelanguage.googleapis.com', testModel: 'gemini-2.5-flash', + pricingUrl: 'https://ai.google.dev/gemini-api/docs/pricing', }, deepseek: { displayName: 'DeepSeek', baseUrl: 'https://api.deepseek.com', testModel: 'deepseek-v4-flash', + pricingUrl: 'https://api-docs.deepseek.com/quick_start/pricing', }, }; diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index 8e5a56b5..73546905 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -11,7 +11,7 @@ import type { DoctorProbes } from '../chat/doctor.js'; import { createSessionPersister, type SessionPersister } from '../chat/persister.js'; import { loadResolvedConfig } from '../config/load.js'; import { writeGlobalDefaultModel } from '../config/write.js'; -import { buildMergedCatalog } from '../engine/model-catalog-view.js'; +import { buildMergedCatalog, buildUserPricing } from '../engine/model-catalog-view.js'; import { createModelRefreshService } from '../engine/model-refresh.js'; import { assembleToolEnv } from '../engine/tool-host/assemble.js'; import { @@ -213,6 +213,14 @@ export async function driveHome(deps: HomeDeps): Promise { // Build + wire + START a fresh chat session (the controller sends the first message on transition). const startChat = async (): Promise => { const store = createChatStore(deps.global.color); + // The ADR-0065 §2 user-pricing overlay (2.5.G S10), read FRESH per chat from the SAME db `models.load` reads + // (its own slug snapshot, like the picker) — so a user-priced model started in this long-lived Home is + // enforced by `[chat].max_cost_microcents` + tracked in realized cost. Static `MODEL_PRICING` still wins. + const pricingSlugByUuid = new Map(providerStore.list().map((p) => [p.id, p.name] as const)); + const resolvePrice = buildUserPricing({ + rows: catalogStore.listAll(), + providerSlug: (uuid_) => pricingSlugByUuid.get(uuid_) ?? uuid_, + }); const built: BuiltChatSession = await (deps.buildSession ?? buildChatSession)({ // Re-read the EFFECTIVE default model FRESH per chat (not the load-once `config` snapshot) so a same-session // `/models` write takes effect on the very next chat started in this long-lived Home (2.5.G S7) — the @@ -227,6 +235,7 @@ export async function driveHome(deps: HomeDeps): Promise { providers, mcpSecretResolver, mcpRegistrations: config.mcpServers, + ...(resolvePrice.size === 0 ? {} : { resolvePrice }), onBudgetWarning: (warning) => deps.io.writeErr( `budget warning: ~${warning.thresholdPct}% of the ${warning.limitMicrocents}µ¢ cap reached\n`, diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index 6fdff484..b970c8a1 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -119,6 +119,7 @@ The command set below is the confirmed surface. Commands ship **per workstream** | `relavium agent ` _(planned)_ | Manage agents (list / create / test). | | `relavium models` | List the cached model catalog (refreshes on first run if the cache is empty). See [`relavium models`](#relavium-models). | | `relavium models refresh` | Force a live re-fetch of each connected provider's model list into the local cache, reporting per-provider outcomes. | +| `relavium models pricing ` | Hand-enter a user price for a model the registry does not know, so the cost cap enforces it. See [`relavium models`](#relavium-models). | | `relavium provider ` | Manage providers and API keys in the OS keychain (`list` / `add` / `set-key` / `remove-key` / `test`). | ## Command manifest @@ -209,14 +210,17 @@ The live model catalog (2.5.G, [ADR-0064](../../decisions/0064-live-model-catalo ```bash relavium models # list the cached catalog relavium models refresh # force a live re-fetch of every connected provider +relavium models pricing my-custom-model --provider openai --input 3 --output 9 # hand-enter a price ``` - **`relavium models`** (no subcommand) lists the cached catalog (read-only). On the **very first run** — when the cache is empty — it does one minimal **blocking** refresh, then lists; an empty result stays a clean exit `0` (an empty catalog is not a fault, like `relavium list`). Human output is one line per model (` ctx= []`). - **`relavium models refresh`** forces a live re-fetch of **each connected provider** (a provider whose key resolves via the OS keychain → `RELAVIUM__API_KEY` env var) and prints a per-provider outcome. The refresh is **per-provider isolated**: one provider's failure (bad key, network, endpoint drift) or a provider without a list endpoint **never** fails the whole command — that provider is reported `failed` / `skipped` and the others still refresh. A per-provider failure is therefore **not** a command failure (exit `0` with the report). The **one** hard fault is an explicit `refresh` with **zero** providers connected (no key at all): that is a clean exit `2` naming how to add a key, because nothing could be fetched. -- **Security.** A provider key is read only to make the live request (over the bounded, abortable, secret-free `listModels` seam) and is **never** logged, persisted (the cache holds no key), or placed in the report / `--json` payload / any error message. A failing provider surfaces only the seam's already-redacted message (or a generic `refresh failed`), never a raw cause. +- **`relavium models pricing --provider --input --output [--cached ]`** hand-enters the per-million-token price of a model the static registry does **not** know — a custom-endpoint model, or a new provider model not yet in the shipped [pricing.ts](../../../packages/llm/src/pricing.ts) (2.5.G S10, [ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) §1–2). Prices are **USD per million tokens** (`--input` prompt, `--output` completion, `--cached` cache-read; stored as integer micro-cents, `usd × 1e8`, never a float). The row is written as `source='user'` and a live `models refresh` **never** clobbers it. This **closes the cost-cap gap** ([ADR-0064](../../decisions/0064-live-model-catalog.md) §6): before, an unknown model had no price, so `budget.max_cost_microcents` / `[chat].max_cost_microcents` **degraded to allow** for it; once user-priced, the cap is enforced (pre-egress **and** realized) on `run`, `chat`, the Home chat, and one-shot `agent run`. Guards (each a clean exit `2`, nothing written): a **canonical** model id is refused (the shipped price always wins, so an override would be silently ignored); an **unregistered provider** is refused (register it first with `relavium provider add`); a **negative / non-finite / implausibly-large** price is refused. The static registry still wins for a known id, so a user can never misprice a shipped model. +- **Security.** A provider key is read only to make the live request (over the bounded, abortable, secret-free `listModels` seam) and is **never** logged, persisted (the cache holds no key), or placed in the report / `--json` payload / any error message. A failing provider surfaces only the seam's already-redacted message (or a generic `refresh failed`), never a raw cause. `models pricing` writes only a model id + provider + integer prices — no key, ever. - **`--json`** ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)) emits **one NDJSON record per line**, stdout-pure, key-free: - `relavium models --json` — one record per model: `{ provider, modelId, displayName, contextWindowTokens, maxOutputTokens, source, lastRefreshedAt, deprecationDate }` (`null` for an absent optional; `source` ∈ `static | live | user`; `lastRefreshedAt` is epoch-ms). - `relavium models refresh --json` — one record per provider: `{ provider, status, added, updated, deactivated, error }`, where `status` ∈ `refreshed | skipped-no-key | skipped-unsupported | failed`, the three counts are the model ids added / refreshed-in-place / soft-deactivated (`null` unless `status` is `refreshed`), and `error` is a short, secret-free reason (`null` unless `failed`). + - `relavium models pricing --json` — one record: `{ model, provider, source, inputCostPerMtokMicrocents, outputCostPerMtokMicrocents, cachedInputCostPerMtokMicrocents }` (the stored integer micro-cents; `source` is always `user`). ### Read-command `--json` output @@ -289,7 +293,8 @@ key is read solely at LLM-call time. Known providers: `anthropic`, `openai`, `ge ```bash relavium provider list # registered providers + whether a key is set -relavium provider add anthropic # register a provider (its default base URL) +relavium provider add anthropic # register a provider (its default base URL + pricing page) +relavium provider add openai --pricing-url https://example.com/prices # override the pricing reference page echo "$ANTHROPIC_API_KEY" | relavium provider set-key anthropic # store a key (read from STDIN, never argv) relavium provider test anthropic # verify the key with a minimal live request relavium provider remove-key anthropic # delete the key from the keychain @@ -299,6 +304,7 @@ relavium provider remove-key anthropic # delete the key from th logs); pipe it or use a heredoc. The key is stored in the OS keychain under the canonical entry-naming scheme ([keychain-and-secrets.md](../desktop/keychain-and-secrets.md#entry-naming)). - **`add` / `set-key`** auto-register the provider row. `--base-url ` on `add` records a custom endpoint that **is now actually used at request routing** (2.5.G S9, [ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) §3–4 — the earlier "dead-config" gap is closed): the resolver rebinds that provider's adapter to the custom endpoint and routes **all** its egress (streaming `generate`/`stream` + the `models.list` refresh) through the shared **SSRF-validated** hop (`connectValidated` — HTTPS-only, no embedded credentials, every resolved IP range-blocked, connect pinned to the validated IP for DNS-rebinding safety). Custom endpoints are **OpenAI-compatible only** this round (`openai` / `deepseek`); a `--base-url` on `anthropic` / `gemini` is **refused** with a clear message (exit `2`), as is a non-HTTPS / private-loopback / credential-bearing URL (fail-fast at `add`). The provider-id set stays **closed** — a custom endpoint reuses the `openai` / `deepseek` id (ADR-0065 §6). +- **`--pricing-url `** on `add` overrides the seeded `pricing_reference_url` — the public pricing page where you find a model's price to hand-enter via [`relavium models pricing`](#relavium-models) (2.5.G S10, ADR-0065 §1). Each known provider is seeded with its default pricing page; the `add` confirmation echoes it. It is a **display-only pointer**, **never fetched** (not an egress target), so — unlike `--base-url` — it needs no SSRF gate; it is validated as an HTTPS URL with no embedded credentials and stored normalized (control bytes percent-encoded, so it is terminal-safe). Omitting the flag on a re-`add` preserves a previously-set custom pointer. - **`test`** does a 1-token `generate` through `@relavium/llm`; `--model ` overrides the cheap default. A bad key fails cleanly (exit `2`) without echoing the key. - **Key resolution** (used by `run` + `test`): **OS keychain → `RELAVIUM__API_KEY` env var → error**. diff --git a/packages/core/src/engine/agent-runner.ts b/packages/core/src/engine/agent-runner.ts index b46de24e..4c4f53e5 100644 --- a/packages/core/src/engine/agent-runner.ts +++ b/packages/core/src/engine/agent-runner.ts @@ -43,6 +43,7 @@ import { type MediaJobStatus, type MediaUnitsEntry, type MediaUnitsEstimate, + type PricingOverlay, type ProviderId, type ResponseFormat, type ToolDef as LlmToolDef, @@ -115,6 +116,9 @@ export interface AgentRunnerDeps { readonly limits?: AgentTurnLimits; /** Pre-egress budget hook (default no-op; 1.AC fills it). */ readonly preEgress?: PreEgressHook; + /** The user-pricing overlay (2.5.G S10, ADR-0065 §2) — host-injected into the turn's realized cost tracker so a + * workflow run's user-priced model is folded into cost governance. Absent ⇒ static-only. */ + readonly resolvePrice?: PricingOverlay; /** * Per-modality media-output **unit-count** default (1.AF/D17, ADR-0044 §3) — the host-resolved * `[defaults].media_cost_estimate`. Used to build the per-turn media-unit estimate from a node's @@ -336,6 +340,7 @@ async function executeAgent( dispatchContext, limits: deps.limits ?? DEFAULT_AGENT_TURN_LIMITS, ...(preEgress === undefined ? {} : { preEgress }), + ...(deps.resolvePrice === undefined ? {} : { resolvePrice: deps.resolvePrice }), // user-pricing overlay (S10) // Media cost governance (1.AF/D17): forward the node's requested output modalities + a per-modality // unit estimate so the budget governor prices a media-output turn pre-egress. Both omitted for a // text-only node (no `output_modalities`), so a text turn pays no media-estimate work. diff --git a/packages/core/src/engine/agent-session.ts b/packages/core/src/engine/agent-session.ts index fb4d672e..bc90e9fe 100644 --- a/packages/core/src/engine/agent-session.ts +++ b/packages/core/src/engine/agent-session.ts @@ -38,6 +38,7 @@ import { type FallbackPlanEntry, type LlmMessage, type LlmProvider, + type PricingOverlay, type ProviderId, type ToolDef as LlmToolDef, } from '@relavium/llm'; @@ -231,6 +232,13 @@ export interface SessionDeps { readonly maxTurns?: number; /** Pre-egress budget hook (default no-op; 1.AC fills it — ADR-0028). */ readonly preEgress?: PreEgressHook; + /** + * The user-pricing overlay (2.5.G S10, ADR-0065 §2) — host-injected (like {@link keyFor}) from the + * `model_catalog` `source='user'` rows, threaded into the turn's REALIZED cost tracker so a user-priced model + * the static registry lacks is folded into the cumulative cost (which `updateCost` feeds to the governor, so + * `max_cost_microcents` enforces it). Absent ⇒ static-only pricing. `@relavium/core` never imports `@relavium/db`. + */ + readonly resolvePrice?: PricingOverlay; /** * Feed the running session cost to a budget governor so a host that wires {@link preEgress} to * `BudgetGovernor.checkPreEgress` also keeps the governor's cumulative total current (ADR-0028, 1.AC). @@ -852,6 +860,7 @@ export class AgentSession { dispatchContext: this.#buildDispatchContext(new Set(), undefined), limits: this.#limits, ...(this.#deps.preEgress === undefined ? {} : { preEgress: this.#deps.preEgress }), + ...(this.#deps.resolvePrice === undefined ? {} : { resolvePrice: this.#deps.resolvePrice }), }); const summary = result.text.trim(); if (summary.length === 0) { @@ -1049,6 +1058,7 @@ export class AgentSession { dispatchContext, limits: this.#limits, ...(this.#deps.preEgress === undefined ? {} : { preEgress: this.#deps.preEgress }), + ...(this.#deps.resolvePrice === undefined ? {} : { resolvePrice: this.#deps.resolvePrice }), }); } diff --git a/packages/core/src/engine/agent-turn.ts b/packages/core/src/engine/agent-turn.ts index 665bb7e0..62729cad 100644 --- a/packages/core/src/engine/agent-turn.ts +++ b/packages/core/src/engine/agent-turn.ts @@ -44,6 +44,7 @@ import { type LlmMessage, type LlmRequest, type MediaUnitsEstimate, + type PricingOverlay, type ResponseFormat, type StreamChunk, type ToolDef as LlmToolDef, @@ -153,6 +154,12 @@ export interface AgentTurnParams { * built by the AgentRunner from `output_modalities` + the `[defaults].media_cost_estimate` unit counts. */ readonly mediaUnitsEstimate?: readonly MediaUnitsEstimate[]; + /** + * The user-pricing overlay (2.5.G S10, ADR-0065 §2) — the REALIZED cost path's tier for a model the static + * registry lacks, injected into this turn's {@link CostTracker} so a user-priced model's spend is folded (and + * so the cap enforces it). Host-built from the `model_catalog` `source='user'` rows; absent ⇒ static-only. + */ + readonly resolvePrice?: PricingOverlay; } /** What one settled agent turn produced. */ @@ -700,8 +707,9 @@ async function driveAgentTurn( } // The cost path is the core's, not the host's: one tracker per turn, one cost:updated per - // non-skipped attempt (attemptNumber counts non-skipped records, not the positional index). - const costTracker = new CostTracker(); + // non-skipped attempt (attemptNumber counts non-skipped records, not the positional index). The user-pricing + // overlay (2.5.G S10) lets the tracker price a user-priced model the static registry lacks. + const costTracker = new CostTracker(params.resolvePrice); let activeModel = primaryModel; let nonSkippedAttempts = 0; diff --git a/packages/core/src/engine/budget-governor.test.ts b/packages/core/src/engine/budget-governor.test.ts index f34cc02e..12551a4a 100644 --- a/packages/core/src/engine/budget-governor.test.ts +++ b/packages/core/src/engine/budget-governor.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it } from 'vitest'; +import type { PricingOverlay } from '@relavium/llm'; import type { Budget } from '@relavium/shared'; import { BudgetExceededError, BudgetGovernor, BudgetPauseError } from './budget-governor.js'; @@ -7,7 +8,13 @@ import type { RunEventDraft } from './event-bus.js'; describe('BudgetGovernor', () => { const budget: Budget = { max_cost_microcents: 1_000_000, on_exceed: 'warn' }; - function makeGovernor(overrides: { budget?: Budget; defaultMaxTokensEstimate?: number } = {}): { + function makeGovernor( + overrides: { + budget?: Budget; + defaultMaxTokensEstimate?: number; + resolvePrice?: PricingOverlay; + } = {}, + ): { governor: BudgetGovernor; warnings: Omit, 'runId'>[]; } { @@ -17,6 +24,7 @@ describe('BudgetGovernor', () => { ...(overrides.defaultMaxTokensEstimate === undefined ? {} : { defaultMaxTokensEstimate: overrides.defaultMaxTokensEstimate }), + ...(overrides.resolvePrice === undefined ? {} : { resolvePrice: overrides.resolvePrice }), emit: (event) => { warnings.push(event); return Promise.resolve(); @@ -139,4 +147,53 @@ describe('BudgetGovernor', () => { governor.checkPreEgress('my-self-hosted-model', 10_000, [{ modality: 'image', units: 2 }]), ).resolves.toBeUndefined(); }); + + describe('user-pricing overlay (2.5.G S10, ADR-0065 §2 — closes the cost-cap gap)', () => { + // A user price for a model the static registry does not know — output $9/MTok so 10_000 tok ⇒ 90_000µ¢. + const OVERLAY: PricingOverlay = new Map([ + [ + 'acme-custom-1', + { + provider: 'openai', + nativeId: 'acme-custom-1', + displayName: 'Acme Custom 1', + contextWindowTokens: 32_000, + maxOutputTokens: 4_000, + inputPerMtokMicrocents: 300_000_000, + outputPerMtokMicrocents: 900_000_000, + cachedInputPerMtokMicrocents: 0, + }, + ], + ]); + + it('ENFORCES the cap on a user-priced model that WOULD have degraded to allow without the overlay', async () => { + // THE ACCEPTANCE: with the overlay, `acme-custom-1` is priced, so the projected 9_000_000µ¢ (10_000 out + // @ $9/MTok) far exceeds the 1_000_000µ¢ cap → fail (not the old silent degrade-to-allow). + const { governor } = makeGovernor({ + budget: { ...budget, on_exceed: 'fail' }, + resolvePrice: OVERLAY, + }); + governor.updateCost(0); + await expect(governor.checkPreEgress('acme-custom-1', 10_000)).rejects.toBeInstanceOf( + BudgetExceededError, + ); + }); + + it('the SAME model WITHOUT the overlay degrades to allow (proves the overlay is what closes the gap)', async () => { + const { governor } = makeGovernor({ budget: { ...budget, on_exceed: 'fail' } }); + governor.updateCost(0); + await expect(governor.checkPreEgress('acme-custom-1', 10_000)).resolves.toBeUndefined(); + }); + + it('a user-priced model UNDER the cap is allowed (no false positive)', async () => { + // 1_000 output tokens @ $9/MTok = 900_000µ¢, under the 1_000_000µ¢ cap → allow, no warning (warn-cap is 0.9). + const { governor, warnings } = makeGovernor({ + budget: { ...budget, on_exceed: 'fail' }, + resolvePrice: OVERLAY, + }); + governor.updateCost(0); + await expect(governor.checkPreEgress('acme-custom-1', 1_000)).resolves.toBeUndefined(); + expect(warnings).toHaveLength(0); + }); + }); }); diff --git a/packages/core/src/engine/budget-governor.ts b/packages/core/src/engine/budget-governor.ts index 64769489..8b168356 100644 --- a/packages/core/src/engine/budget-governor.ts +++ b/packages/core/src/engine/budget-governor.ts @@ -3,6 +3,7 @@ import { estimateMediaCost, UnknownModelError, type MediaUnitsEstimate, + type PricingOverlay, } from '@relavium/llm'; import type { Budget } from '@relavium/shared'; @@ -93,6 +94,7 @@ export class BudgetGovernor { readonly #emit: ( event: Omit, 'runId'>, ) => Promise; + readonly #overlay: PricingOverlay | undefined; #cumulativeCostMicrocents = 0; #warningEmitted = false; @@ -102,10 +104,14 @@ export class BudgetGovernor { readonly emit: ( event: Omit, 'runId'>, ) => Promise; + /** The user-pricing overlay (2.5.G S10) — makes the PRE-EGRESS estimate price a user-priced model that the + * static registry lacks, so `max_cost_microcents` enforces it (the cap-gap fix). Absent ⇒ static-only. */ + readonly resolvePrice?: PricingOverlay; }) { this.#budget = params.budget; this.#defaultMaxTokensEstimate = params.defaultMaxTokensEstimate ?? DEFAULT_MAX_TOKENS_ESTIMATE; this.#emit = params.emit; + this.#overlay = params.resolvePrice; } /** Update the governor with the engine's authoritative running cumulative cost. */ @@ -136,8 +142,10 @@ export class BudgetGovernor { // modalities the model rates (a missing rate degrades to 0); both share the UnknownModelError // degrade-to-allow below, so an unpriced model never hard-fails the run. estimate = - estimateMaxNextCost(model, maxTokens ?? this.#defaultMaxTokensEstimate) + - (mediaUnitsEstimate === undefined ? 0 : estimateMediaCost(model, mediaUnitsEstimate)); + estimateMaxNextCost(model, maxTokens ?? this.#defaultMaxTokensEstimate, this.#overlay) + + (mediaUnitsEstimate === undefined + ? 0 + : estimateMediaCost(model, mediaUnitsEstimate, this.#overlay)); } catch (err) { // An unpriced model (e.g. a custom base-URL / self-hosted id with no pricing row) throws // UnknownModelError. The pre-egress governor must NOT hard-fail an otherwise-valid run on it — diff --git a/packages/core/src/engine/engine.ts b/packages/core/src/engine/engine.ts index 8056d76b..a6053eec 100644 --- a/packages/core/src/engine/engine.ts +++ b/packages/core/src/engine/engine.ts @@ -50,7 +50,7 @@ import { type RunStatus, type TokensUsed, } from '@relavium/shared'; -import type { MediaJobStatus } from '@relavium/llm'; +import type { MediaJobStatus, PricingOverlay } from '@relavium/llm'; import { buildRunPlan, type BuildRunPlanOptions } from '../dag.js'; import { InterpolationError } from '../errors.js'; @@ -239,6 +239,15 @@ export interface WorkflowEngineDeps { * `maxTokens` (ADR-0028). Not the model's absolute max, which would over-block. */ readonly maxTokensEstimate?: number; + /** + * The user-pricing overlay (2.5.G S10, [ADR-0065](../../../docs/decisions/0065-provider-economics-and-user-pricing.md) + * §2) — a `ReadonlyMap` the host projects from the `model_catalog` `source='user'` rows. + * It feeds the workflow PRE-EGRESS budget governor so a model with no static price, once user-priced, is enforced + * by `budget.max_cost_microcents`. Static `MODEL_PRICING` still wins for a known id (fills an UNKNOWN id only). + * Injected exactly like the realized path's overlay, which the node executor's runner already carries; omit ⇒ + * an unknown model degrades cost governance to `allow` loudly, unchanged. + */ + readonly resolvePrice?: PricingOverlay; } function maskInputs( @@ -331,6 +340,9 @@ class RunExecution { onSettled: (runId: string) => void; resolverCapabilities: ResolverCapabilities; maxTokensEstimate?: number; + /** The user-pricing overlay (2.5.G S10, ADR-0065 §2) — into the workflow PRE-EGRESS governor so a user-priced + * model is enforced by `budget`. Host-injected; the realized path rides the runner's own `resolvePrice`. */ + resolvePrice?: PricingOverlay; /** When present, the run is REHYDRATED from this checkpoint (resume) rather than started fresh (1.R). */ checkpoint?: CheckpointState; }) { @@ -359,6 +371,7 @@ class RunExecution { budget: params.plan.budget, defaultMaxTokensEstimate: this.#maxTokensEstimate, emit: (draft) => this.#emitDurable({ ...draft, runId: this.runId }), + ...(params.resolvePrice === undefined ? {} : { resolvePrice: params.resolvePrice }), }); } @@ -2169,6 +2182,7 @@ export class WorkflowEngine { readonly #capacity: number; readonly #resolverCapabilities: ResolverCapabilities; readonly #maxTokensEstimate: number; + readonly #resolvePrice: PricingOverlay | undefined; readonly #runs = new Map(); constructor(deps: WorkflowEngineDeps) { @@ -2178,6 +2192,7 @@ export class WorkflowEngine { this.#capacity = deps.eventBufferCapacity ?? 256; this.#resolverCapabilities = deps.resolverCapabilities ?? {}; this.#maxTokensEstimate = deps.maxTokensEstimate ?? DEFAULT_MAX_TOKENS_ESTIMATE; + this.#resolvePrice = deps.resolvePrice; } /** @@ -2206,6 +2221,7 @@ export class WorkflowEngine { }, resolverCapabilities: this.#resolverCapabilities, maxTokensEstimate: this.#maxTokensEstimate, + ...(this.#resolvePrice === undefined ? {} : { resolvePrice: this.#resolvePrice }), }); this.#runs.set(runId, execution); void execution.begin(); @@ -2301,6 +2317,7 @@ export class WorkflowEngine { }, resolverCapabilities: this.#resolverCapabilities, maxTokensEstimate: this.#maxTokensEstimate, + ...(this.#resolvePrice === undefined ? {} : { resolvePrice: this.#resolvePrice }), checkpoint, }); this.#runs.set(input.runId, execution); diff --git a/packages/db/src/model-catalog-store.test.ts b/packages/db/src/model-catalog-store.test.ts index f235fe41..207cce4f 100644 --- a/packages/db/src/model-catalog-store.test.ts +++ b/packages/db/src/model-catalog-store.test.ts @@ -929,6 +929,59 @@ describe('createModelCatalogStore (2.5.G / ADR-0064 — live-discovery cache)', expect(fresh?.lastRefreshedAt).toBeUndefined(); }); + it('upsert() PRESERVES media_surface / capabilities / supportsVision + cost columns when omitted (the S10 clobber fix)', () => { + // Seed a full generative/media row (as the media fixture / a live-then-enriched sync would). + store.upsert({ + providerId, + modelId: 'media-then-priced', + displayName: 'Media Then Priced', + contextWindowTokens: 4096, + maxOutputTokens: 4096, + mediaSurface: 'generative', + supportsVision: true, + capabilities: { media: { outputCombinations: [['image']] } }, + mediaImageCostMicrocents: 1_900_000, + }); + // A `models pricing`-style PARTIAL upsert: it writes ONLY the text-token prices + source='user' and omits + // every media/capability field. The never-clobber invariant must keep the generative routing + capabilities + // intact (a reset to media_surface='chat' would silently disable generative routing). + store.upsert({ + providerId, + modelId: 'media-then-priced', + displayName: 'Media Then Priced', + contextWindowTokens: 4096, + maxOutputTokens: 4096, + source: 'user', + inputCostPerMtokMicrocents: 300, + outputCostPerMtokMicrocents: 900, + }); + const rec = store.getByModelId('media-then-priced'); + expect(rec?.mediaSurface).toBe('generative'); // NOT reset to 'chat' + expect(rec?.supportsVision).toBe(true); // NOT reset to false + expect(rec?.capabilities).toEqual({ media: { outputCombinations: [['image']] } }); // NOT blanked to {} + expect(rec?.mediaImageCostMicrocents).toBe(1_900_000); // media rate preserved + const listing = store.listByProvider(providerId).find((m) => m.modelId === 'media-then-priced'); + expect(listing?.source).toBe('user'); // the write DID take (prices applied) + expect(listing?.inputCostPerMtokMicrocents).toBe(300); + expect(listing?.outputCostPerMtokMicrocents).toBe(900); + + // ...and a SUBSEQUENT re-price that omits the cost columns preserves the previously-entered prices. + store.upsert({ + providerId, + modelId: 'media-then-priced', + displayName: 'Media Then Priced', + contextWindowTokens: 4096, + maxOutputTokens: 4096, + source: 'user', + outputCostPerMtokMicrocents: 950, // change ONLY output + }); + const reListing = store + .listByProvider(providerId) + .find((m) => m.modelId === 'media-then-priced'); + expect(reListing?.inputCostPerMtokMicrocents).toBe(300); // preserved + expect(reListing?.outputCostPerMtokMicrocents).toBe(950); // updated + }); + it('listByProvider/listAll exclude a soft-DELETED (deletedAt) row, not just an inactive one', () => { store.replaceProviderModels( providerId, diff --git a/packages/db/src/model-catalog-store.ts b/packages/db/src/model-catalog-store.ts index 2dcf7ab4..772ebc23 100644 --- a/packages/db/src/model-catalog-store.ts +++ b/packages/db/src/model-catalog-store.ts @@ -65,8 +65,13 @@ export interface ModelCatalogUpsert { readonly mediaImageCostMicrocents?: number | null; readonly mediaAudioCostMicrocents?: number | null; readonly mediaVideoCostMicrocents?: number | null; + /** USER-supplied TEXT-token pricing (2.5.G S10, ADR-0065 §1) — integer micro-cents per Mtok. Written under + * `source='user'` (a live refresh NEVER clobbers a user row, §1); OMITTED ⇒ the DB default `0`. */ + readonly inputCostPerMtokMicrocents?: number; + readonly outputCostPerMtokMicrocents?: number; + readonly cachedInputCostPerMtokMicrocents?: number; /** The provenance discriminant ([ADR-0064] §4). OMITTED ⇒ `'static'` (a hardcoded seed), so every existing - * media-routing caller is unchanged; the live refresh writes `'live'`. */ + * media-routing caller is unchanged; the live refresh writes `'live'`; user pricing writes `'user'`. */ readonly source?: ModelCatalogSource; /** The epoch-ms this row was live-refreshed (ADR-0064 §5). OMITTED ⇒ `null` (a static/user or never-refreshed row). */ readonly lastRefreshedAt?: number; @@ -343,12 +348,32 @@ export function createModelCatalogStore(db: Db, deps: ModelCatalogStoreDeps): Mo displayName: input.displayName, contextWindowTokens: input.contextWindowTokens, maxOutputTokens: input.maxOutputTokens, - mediaSurface: input.mediaSurface ?? 'chat', - supportsVision: input.supportsVision ?? false, - capabilities: JSON.stringify(input.capabilities ?? {}), - mediaImageCostMicrocents: input.mediaImageCostMicrocents ?? null, - mediaAudioCostMicrocents: input.mediaAudioCostMicrocents ?? null, - mediaVideoCostMicrocents: input.mediaVideoCostMicrocents ?? null, + // Media routing / capability columns follow the SAME "never clobber an omitted field on update" invariant + // as the pricing + provenance columns below (2.5.G S10): a partial upsert — e.g. `models pricing` writing a + // `source='user'` row over a model the live refresh discovered — must NOT reset a live/seed row's + // `media_surface` back to `'chat'` (silently disabling generative routing) or blank its capabilities. On a + // true INSERT (`existing` undefined) each still falls to its documented default, so every full-row caller + // (the media fixture, a re-seed) is byte-for-byte unchanged (it always passes these). + mediaSurface: input.mediaSurface ?? existing?.mediaSurface ?? 'chat', + supportsVision: input.supportsVision ?? existing?.supportsVision ?? false, + capabilities: + input.capabilities !== undefined + ? JSON.stringify(input.capabilities) + : (existing?.capabilities ?? JSON.stringify({})), + mediaImageCostMicrocents: + input.mediaImageCostMicrocents ?? existing?.mediaImageCostMicrocents ?? null, + mediaAudioCostMicrocents: + input.mediaAudioCostMicrocents ?? existing?.mediaAudioCostMicrocents ?? null, + mediaVideoCostMicrocents: + input.mediaVideoCostMicrocents ?? existing?.mediaVideoCostMicrocents ?? null, + // USER text-token pricing (2.5.G S10) — write the supplied prices, else PRESERVE the existing row's (an + // update that omits them must not zero a hand-entered price), else the NOT-NULL default `0`. + inputCostPerMtokMicrocents: + input.inputCostPerMtokMicrocents ?? existing?.inputCostPerMtokMicrocents ?? 0, + outputCostPerMtokMicrocents: + input.outputCostPerMtokMicrocents ?? existing?.outputCostPerMtokMicrocents ?? 0, + cachedInputCostPerMtokMicrocents: + input.cachedInputCostPerMtokMicrocents ?? existing?.cachedInputCostPerMtokMicrocents ?? 0, // Provenance + freshness (ADR-0064 §4/§5). On a true INSERT (`existing` undefined) these fall to // `'static'` / `null`, so every existing media-routing caller (which passes neither) writes a static, // never-refreshed row unchanged. On an UPDATE they PRESERVE the existing row's `source`/`lastRefreshedAt` diff --git a/packages/llm/src/budget-estimator.test.ts b/packages/llm/src/budget-estimator.test.ts index 66657cc2..a43f159c 100644 --- a/packages/llm/src/budget-estimator.test.ts +++ b/packages/llm/src/budget-estimator.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from 'vitest'; import { estimateMaxNextCost, estimateMediaCost } from './budget-estimator.js'; -import { MODEL_PRICING } from './pricing.js'; +import { MODEL_PRICING, type ModelPricing } from './pricing.js'; describe('estimateMaxNextCost', () => { it('estimates output-only worst case at maxTokens', () => { @@ -51,3 +51,35 @@ describe('estimateMediaCost (1.AF/D17 — pre-egress per-modality media estimate ); }); }); + +describe('user-pricing overlay (2.5.G S10, ADR-0065 §2)', () => { + const OVERLAY: ReadonlyMap = new Map([ + [ + 'acme-custom-1', + { + provider: 'openai', + nativeId: 'acme-custom-1', + displayName: 'Acme Custom 1', + contextWindowTokens: 32_000, + maxOutputTokens: 4_000, + inputPerMtokMicrocents: 300_000_000, + outputPerMtokMicrocents: 900_000_000, // $9/MTok + cachedInputPerMtokMicrocents: 0, + }, + ], + ]); + + it('estimateMaxNextCost prices a user-priced unknown model via the overlay', () => { + // 10_000 output tokens @ $9/MTok (900_000_000µ¢/MTok) = 10_000 × 900 = 9_000_000µ¢ — so + // `max_cost_microcents` can pre-egress-block it (the acceptance: the cost-cap gap is closed). + expect(estimateMaxNextCost('acme-custom-1', 10_000, OVERLAY)).toBe(9_000_000); + }); + + it('estimateMaxNextCost still throws for an id in neither tier (governor → degrade-to-allow)', () => { + expect(() => estimateMaxNextCost('not-anywhere', 10_000, OVERLAY)).toThrow('unknown model id'); + }); + + it('estimateMediaCost accepts the overlay (a user row carries no media rate → 0, never a throw)', () => { + expect(estimateMediaCost('acme-custom-1', [{ modality: 'image', units: 4 }], OVERLAY)).toBe(0); + }); +}); diff --git a/packages/llm/src/budget-estimator.ts b/packages/llm/src/budget-estimator.ts index 14bde116..bbbd2c70 100644 --- a/packages/llm/src/budget-estimator.ts +++ b/packages/llm/src/budget-estimator.ts @@ -1,6 +1,6 @@ import type { MediaBilledModality } from '@relavium/shared'; -import { priceModel } from './cost-tracker.js'; +import { priceModel, type PricingOverlay } from './cost-tracker.js'; const TOKENS_PER_MTOK = 1_000_000; @@ -13,8 +13,12 @@ const TOKENS_PER_MTOK = 1_000_000; * * All figures are integer micro-cents. */ -export function estimateMaxNextCost(modelId: string, maxOutputTokens: number): number { - const p = priceModel(modelId); +export function estimateMaxNextCost( + modelId: string, + maxOutputTokens: number, + overlay?: PricingOverlay, +): number { + const p = priceModel(modelId, overlay); if (maxOutputTokens <= 0) { return 0; } @@ -38,8 +42,9 @@ export interface MediaUnitsEstimate { export function estimateMediaCost( modelId: string, estimate: readonly MediaUnitsEstimate[], + overlay?: PricingOverlay, ): number { - const p = priceModel(modelId); + const p = priceModel(modelId, overlay); let total = 0; for (const { modality, units } of estimate) { const rate = p.mediaOutputRates?.[modality]; diff --git a/packages/llm/src/cost-tracker.test.ts b/packages/llm/src/cost-tracker.test.ts index 8ae48fda..f3199b82 100644 --- a/packages/llm/src/cost-tracker.test.ts +++ b/packages/llm/src/cost-tracker.test.ts @@ -183,6 +183,80 @@ describe('CostTracker', () => { }); }); +describe('user-pricing overlay (2.5.G S10, ADR-0065 §2)', () => { + // A user-supplied price for a model the static registry does NOT know (a custom-endpoint id). + const OVERLAY: ReadonlyMap = new Map([ + [ + 'acme-custom-1', + { + provider: 'openai', + nativeId: 'acme-custom-1', + displayName: 'Acme Custom 1', + contextWindowTokens: 32_000, + maxOutputTokens: 4_000, + inputPerMtokMicrocents: 300_000_000, // $3/MTok + outputPerMtokMicrocents: 900_000_000, // $9/MTok + cachedInputPerMtokMicrocents: 0, + }, + ], + // A user row that COLLIDES with a canonical id — the static registry must still win (no silent misprice). + [ + 'claude-opus-4-8', + { + provider: 'anthropic', + nativeId: 'claude-opus-4-8', + displayName: 'Tampered Opus', + contextWindowTokens: 1, + maxOutputTokens: 1, + inputPerMtokMicrocents: 1, // absurd override — must be ignored + outputPerMtokMicrocents: 1, + cachedInputPerMtokMicrocents: 0, + }, + ], + ]); + + it('priceModel fills an UNKNOWN id from the overlay', () => { + const p = priceModel('acme-custom-1', OVERLAY); + expect(p.inputPerMtokMicrocents).toBe(300_000_000); + expect(p.provider).toBe('openai'); + }); + + it('priceModel keeps the STATIC registry authoritative for a known id even when the overlay collides', () => { + const p = priceModel('claude-opus-4-8', OVERLAY); + expect(p.displayName).toBe('Claude Opus 4.8'); // the static row, not the tampered overlay + expect(p.inputPerMtokMicrocents).toBe(500_000_000); // $5/MTok, not the overlay's 1µ¢ + }); + + it('priceModel still throws UnknownModelError for an id absent from BOTH tiers', () => { + expect(() => priceModel('not-anywhere', OVERLAY)).toThrowError(UnknownModelError); + }); + + it('cost() prices a user-priced unknown model via the overlay (the cost-cap gap is closed)', () => { + // 1000 in @ $3/MTok = 300_000µ¢; 500 out @ $9/MTok = 450_000µ¢ → 750_000µ¢. + expect(cost('acme-custom-1', { inputTokens: 1000, outputTokens: 500 }, OVERLAY)).toBe(750_000); + }); + + it('cost() without an overlay still throws for the same unknown model (no silent zero)', () => { + expect(() => cost('acme-custom-1', { inputTokens: 1000, outputTokens: 500 })).toThrowError( + UnknownModelError, + ); + }); + + it('CostTracker records realized cost for a user-priced model when constructed with the overlay', () => { + const tracker = new CostTracker(OVERLAY); + const r = tracker.record('acme-custom-1', { inputTokens: 1000, outputTokens: 500 }); + expect(r.costMicrocents).toBe(750_000); + expect(tracker.cumulativeCostMicrocents).toBe(750_000); + }); + + it('CostTracker WITHOUT an overlay throws on the same unknown model (degrades loudly, never a silent 0)', () => { + const tracker = new CostTracker(); + expect(() => tracker.record('acme-custom-1', { inputTokens: 1000, outputTokens: 500 })).toThrowError( + UnknownModelError, + ); + }); +}); + describe('MODEL_PRICING table invariants (the values seeded into model_catalog)', () => { it('keys match KNOWN_MODEL_IDS and every catalog-projection field is complete + integer', () => { const byLocale = (a: string, b: string): number => a.localeCompare(b); diff --git a/packages/llm/src/cost-tracker.ts b/packages/llm/src/cost-tracker.ts index f774a765..a69062a7 100644 --- a/packages/llm/src/cost-tracker.ts +++ b/packages/llm/src/cost-tracker.ts @@ -15,12 +15,30 @@ import type { MediaUnitsEntry, Usage } from './types.js'; * the seam's `Usage.costMicrocents` and the `cost:updated` event. */ -/** Look up pricing for a canonical model id; throws `UnknownModelError` (never a silent zero). */ -export function priceModel(modelId: string): ModelPricing { - if (!isCanonicalModelId(modelId)) { - throw new UnknownModelError(modelId, KNOWN_MODEL_IDS); +/** + * An optional **user-pricing overlay** (2.5.G S10, [ADR-0065](../../../docs/decisions/0065-provider-economics-and-extensibility.md) §2) + * — canonical model id → {@link ModelPricing}, host-built from the `model_catalog` `source='user'` rows and + * injected **exactly like `keyFor`** (a plain Relavium map; `@relavium/core`/`@relavium/llm` never import + * `@relavium/db`). It fills the price of an id ABSENT from the static registry so `max_cost_microcents` can + * enforce it; the static registry ALWAYS wins for a known id (a user can never misprice a shipped model). + */ +export type PricingOverlay = ReadonlyMap; + +/** + * Look up pricing for a model id with precedence **static → overlay → throw** (ADR-0065 §2): the static + * {@link MODEL_PRICING} wins for a known canonical id; the optional user `overlay` fills an UNKNOWN id; a truly + * unknown id throws `UnknownModelError` (never a silent zero — the caller degrades cost governance to `allow` + * with a loud, visible notice). + */ +export function priceModel(modelId: string, overlay?: PricingOverlay): ModelPricing { + if (isCanonicalModelId(modelId)) { + return MODEL_PRICING[modelId]; // the static registry is the pricing authority for a known id + } + const fromOverlay = overlay?.get(modelId); + if (fromOverlay !== undefined) { + return fromOverlay; // the user tier fills an id the static registry does not carry } - return MODEL_PRICING[modelId]; + throw new UnknownModelError(modelId, KNOWN_MODEL_IDS); } const TOKENS_PER_MTOK = 1_000_000; @@ -35,8 +53,8 @@ const TOKENS_PER_MTOK = 1_000_000; * this (Anthropic's `input_tokens` is already net; the OpenAI-compatible adapter subtracts the * cached subset). So each token class is billed exactly once. */ -export function cost(modelId: string, usage: Usage): number { - const p = priceModel(modelId); +export function cost(modelId: string, usage: Usage, overlay?: PricingOverlay): number { + const p = priceModel(modelId, overlay); const cacheReadTokens = usage.cacheReadTokens ?? 0; const cacheWriteTokens = usage.cacheWriteTokens ?? 0; const perClass = (tokens: number, ratePerMtok: number): number => @@ -106,10 +124,17 @@ export interface CostUpdate { */ export class CostTracker { #cumulativeMicrocents = 0; + readonly #overlay: PricingOverlay | undefined; + + /** `overlay` (2.5.G S10) is the host-injected user-pricing tier — consulted after the static registry for an + * id it does not carry, so a user-priced model's realized spend is folded into the running total. */ + constructor(overlay?: PricingOverlay) { + this.#overlay = overlay; + } /** Price one attempt's usage and fold it into the running total. */ record(modelId: string, usage: Usage): CostUpdate { - const costMicrocents = cost(modelId, usage); + const costMicrocents = cost(modelId, usage, this.#overlay); this.#cumulativeMicrocents += costMicrocents; return { inputTokens: usage.inputTokens, diff --git a/packages/llm/src/index.ts b/packages/llm/src/index.ts index 3cf16e2b..a64813f7 100644 --- a/packages/llm/src/index.ts +++ b/packages/llm/src/index.ts @@ -135,7 +135,7 @@ export type { ModelPricing, CanonicalModelId } from './pricing.js'; export { mergeModelCatalog } from './model-catalog.js'; export type { ModelCatalogEntry, MergeModelCatalogInput, PricingSource } from './model-catalog.js'; export { priceModel, cost, mediaCost, CostTracker } from './cost-tracker.js'; -export type { CostUpdate } from './cost-tracker.js'; +export type { CostUpdate, PricingOverlay } from './cost-tracker.js'; export { estimateMaxNextCost, estimateMediaCost } from './budget-estimator.js'; export type { MediaUnitsEstimate } from './budget-estimator.js'; From 8c8e99fbf1da0a6884a2bbe48f516ddfdb11e9c9 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 06:32:19 +0300 Subject: [PATCH 28/68] =?UTF-8?q?fix(cli,db):=202.5.G=20S10=20Opus-review?= =?UTF-8?q?=20fixes=20=E2=80=94=20forward=20--pricing-url=20+=20preserve?= =?UTF-8?q?=20on=20re-price?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - HIGH: `provider add --pricing-url` was silently dropped — the commander action forwarded only `{ baseUrl }`, so the flag never reached the store (re-introducing the exact "custom value → default" gap ADR-0065 closed). Forward `pricingUrl`; add a commander-action-forwarding regression test (specs-forwarding.test.ts) that the manifest drift guard can't catch. - MEDIUM: re-pricing a SOFT-DEACTIVATED model zeroed its discovered display name + context/limits (the command's active-only read missed the row, the store's upsert found it and overwrote with defaults). Make displayName/ contextWindowTokens/maxOutputTokens preserve-on-omit in the upsert (a true partial patch); the pricing command now omits them entirely. - LOW: fix two broken ADR-0065 relative doc links (`-user-pricing` → `-extensibility`); sanitize the echoed model id (parity with the list renderer); make `loadUserPricingOverlay` degrade to undefined on a READ fault too (not just the open), honoring its non-fatal contract. - tests: provider `--pricing-url` core cases, dispatch arg extraction, the soft-deactivated re-price preservation, and the fresh-price defaults. Refs: ADR-0065 §1/§2/§5 Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/commands/dispatch.test.ts | 22 +++-- apps/cli/src/commands/models-pricing.ts | 29 +++---- apps/cli/src/commands/provider.test.ts | 48 ++++++++++ .../cli/src/commands/specs-forwarding.test.ts | 87 +++++++++++++++++++ apps/cli/src/commands/specs.ts | 4 +- apps/cli/src/engine/pricing-overlay.ts | 5 ++ packages/core/src/engine/engine.ts | 2 +- packages/db/src/model-catalog-store.test.ts | 41 +++++++++ packages/db/src/model-catalog-store.ts | 20 +++-- 9 files changed, 229 insertions(+), 29 deletions(-) create mode 100644 apps/cli/src/commands/specs-forwarding.test.ts diff --git a/apps/cli/src/commands/dispatch.test.ts b/apps/cli/src/commands/dispatch.test.ts index d8180fcd..5d710511 100644 --- a/apps/cli/src/commands/dispatch.test.ts +++ b/apps/cli/src/commands/dispatch.test.ts @@ -157,15 +157,27 @@ describe('build*Args (argv → typed core args)', () => { expect(() => buildGateArgs(input([]))).toThrow(/`relavium gate` requires a /); }); - it('provider.add: name + optional baseUrl (omitted when absent)', () => { - expect(buildProviderAddArgs(input(['anthropic'], { baseUrl: 'https://x' }))).toEqual({ + it('provider.add: name + optional baseUrl + optional pricingUrl (each omitted when absent)', () => { + expect( + buildProviderAddArgs( + input(['anthropic'], { baseUrl: 'https://x', pricingUrl: 'https://p' }), + ), + ).toEqual({ action: 'add', name: 'anthropic', baseUrl: 'https://x', + pricingUrl: 'https://p', + }); + const noOpts = buildProviderAddArgs(input(['anthropic'])); + expect(noOpts).toEqual({ action: 'add', name: 'anthropic' }); + expect('baseUrl' in noOpts).toBe(false); + expect('pricingUrl' in noOpts).toBe(false); + // pricingUrl alone (no baseUrl) still extracts (the S10 flag is independent of --base-url). + expect(buildProviderAddArgs(input(['openai'], { pricingUrl: 'https://p' }))).toEqual({ + action: 'add', + name: 'openai', + pricingUrl: 'https://p', }); - const noBase = buildProviderAddArgs(input(['anthropic'])); - expect(noBase).toEqual({ action: 'add', name: 'anthropic' }); - expect('baseUrl' in noBase).toBe(false); }); it('provider.test: name + optional model (omitted when absent)', () => { diff --git a/apps/cli/src/commands/models-pricing.ts b/apps/cli/src/commands/models-pricing.ts index fa2fc267..7e8613de 100644 --- a/apps/cli/src/commands/models-pricing.ts +++ b/apps/cli/src/commands/models-pricing.ts @@ -6,10 +6,11 @@ import { EXIT_CODES, type ExitCode } from '../process/exit-codes.js'; import type { CliIo } from '../process/io.js'; import type { GlobalOptions } from '../process/options.js'; import { writeRecordLines } from '../render/records.js'; +import { stripTerminalControls } from '../render/tui/chat-projection.js'; /** * The `relavium models pricing ` capture command (workstream **2.5.G S10**, - * [ADR-0065](../../../../docs/decisions/0065-provider-economics-and-user-pricing.md) §1/§2) — hand-enter the + * [ADR-0065](../../../../docs/decisions/0065-provider-economics-and-extensibility.md) §1/§2) — hand-enter the * per-Mtok text-token price of a model the static registry does NOT know (a custom-endpoint model, or a new * provider model not yet in `MODEL_PRICING`), so the cost cap (`max_cost_microcents`) can enforce it. The price is * stored as a `source='user'` `model_catalog` row (integer micro-cents, never float); a live `models refresh` NEVER @@ -42,9 +43,9 @@ export interface ModelsPricingCommandArgs { export interface ModelsPricingCommandDeps { readonly io: CliIo; readonly global: GlobalOptions; - /** The catalog store — `upsert` writes the `source='user'` row; `listByProvider` preserves an existing row's - * display/limits on a re-price (the store overwrites those required columns). */ - readonly catalog: Pick; + /** The catalog store — `upsert` writes the `source='user'` row (a pricing-only patch; the store preserves the + * existing row's display/limits + media columns, so no read is needed here). */ + readonly catalog: Pick; /** The provider registry — resolves the `` → its internal `llm_providers` UUID (the catalog FK). */ readonly providers: Pick; } @@ -102,19 +103,13 @@ export function modelsPricingCommand( ? 0 : usdToMicrocents(args.cachedInputUsdPerMtok, '--cached'); - // Preserve an existing row's display name + limits on a re-price (the store overwrites those REQUIRED columns; - // the media/provenance/pricing columns preserve themselves on omit). A fresh price defaults display → the id, - // limits → `0` (the "unknown" sentinel, which reads back as absent). - const existing = deps.catalog - .listByProvider(providerRow.id) - .find((m) => m.modelId === args.model); - + // A pricing-ONLY upsert: omit display name + limits (and every media/capability column) so the store PRESERVES + // whatever an existing row carries — including a soft-deactivated live row the active-only reader cannot see, so + // a re-price never zeroes a discovered name/context. A brand-new user-priced model defaults display → the id and + // limits → the `0` "unknown" sentinel (in the store), so no read is needed here. deps.catalog.upsert({ providerId: providerRow.id, modelId: args.model, - displayName: existing?.displayName ?? args.model, - contextWindowTokens: existing?.contextWindowTokens ?? 0, - maxOutputTokens: existing?.maxOutputTokens ?? 0, source: 'user', inputCostPerMtokMicrocents, outputCostPerMtokMicrocents, @@ -140,8 +135,12 @@ export function modelsPricingCommand( args.cachedInputUsdPerMtok === undefined ? '' : `, cached $${args.cachedInputUsdPerMtok}/Mtok`; + // Strip any terminal-control byte from the (user-typed) model id before echo — parity with `renderModelList`'s + // FIX 2. `ModelListingSchema` only requires min(1), so an id can carry a control byte; the JSON path is safe on + // its own (JSON.stringify escapes them). The provider is a validated (kebab) ProviderId, and the prices are + // numbers — both already safe. deps.io.writeOut( - `Set user pricing for ${args.model} (${args.provider}): input $${args.inputUsdPerMtok}/Mtok, output $${args.outputUsdPerMtok}/Mtok${cachedNote}. It applies to your next run/chat and survives \`models refresh\`.\n`, + `Set user pricing for ${stripTerminalControls(args.model)} (${args.provider}): input $${args.inputUsdPerMtok}/Mtok, output $${args.outputUsdPerMtok}/Mtok${cachedNote}. It applies to your next run/chat and survives \`models refresh\`.\n`, ); return EXIT_CODES.success; } diff --git a/apps/cli/src/commands/provider.test.ts b/apps/cli/src/commands/provider.test.ts index 566720f5..7635801e 100644 --- a/apps/cli/src/commands/provider.test.ts +++ b/apps/cli/src/commands/provider.test.ts @@ -216,6 +216,54 @@ describe('relavium provider commands (2.C)', () => { expect(d.store.get('openai')?.baseUrl).toBe('https://proxy.example/v1'); // not the SDK default }); + it('seeds the default pricing_reference_url on a plain add + set-key (2.5.G S10)', async () => { + const d = deps({}); + await runProviderCommand({ action: 'add', name: 'openai' }, d); + expect(d.store.get('openai')?.pricingReferenceUrl).toBe( + 'https://platform.openai.com/docs/pricing', + ); + // set-key alone (no prior add) also seeds it — so a provider registered by set-key carries the pointer. + await runProviderCommand({ action: 'set-key', name: 'anthropic' }, d); + expect(d.store.get('anthropic')?.pricingReferenceUrl).toBe( + 'https://platform.claude.com/docs/en/about-claude/pricing', + ); + }); + + it('stores a validated custom --pricing-url (normalized href), overriding the default (S10)', async () => { + const d = deps({}); + await runProviderCommand( + { action: 'add', name: 'openai', pricingUrl: 'https://wiki.internal/prices' }, + d, + ); + // Normalized via `new URL().href` (a trailing slash is added for a bare-host URL) — terminal-safe. + expect(d.store.get('openai')?.pricingReferenceUrl).toBe('https://wiki.internal/prices'); + }); + + it('rejects a non-HTTPS / credential-bearing --pricing-url (exit 2), unlike base_url it allows any host', async () => { + for (const pricingUrl of ['http://x.example/p', 'https://u:p@x.example/p', 'not-a-url']) { + await expect( + runProviderCommand({ action: 'add', name: 'openai', pricingUrl }, deps({})), + ).rejects.toMatchObject({ code: 'invalid_invocation' }); + } + // A private/loopback host IS allowed for a pricing pointer (display-only, never fetched → no SSRF concern). + const d = deps({}); + await runProviderCommand( + { action: 'add', name: 'openai', pricingUrl: 'https://localhost/prices' }, + d, + ); + expect(d.store.get('openai')?.pricingReferenceUrl).toBe('https://localhost/prices'); + }); + + it('a second `add` with NO --pricing-url preserves a prior custom pricing pointer (never resets it)', async () => { + const d = deps({}); + await runProviderCommand( + { action: 'add', name: 'openai', pricingUrl: 'https://wiki.internal/prices' }, + d, + ); + await runProviderCommand({ action: 'add', name: 'openai' }, d); // re-run, no --pricing-url + expect(d.store.get('openai')?.pricingReferenceUrl).toBe('https://wiki.internal/prices'); + }); + it('rejects an unknown provider name (exit 2)', async () => { await expect( runProviderCommand({ action: 'add', name: 'bogus' }, deps({})), diff --git a/apps/cli/src/commands/specs-forwarding.test.ts b/apps/cli/src/commands/specs-forwarding.test.ts new file mode 100644 index 00000000..79d0892e --- /dev/null +++ b/apps/cli/src/commands/specs-forwarding.test.ts @@ -0,0 +1,87 @@ +import { describe, expect, it, vi } from 'vitest'; + +import { buildProgram } from '../program.js'; +import { captureIo } from '../test-support.js'; + +/** + * Commander action → `executeCommand` FORWARDING guard (2.5.G S10 regression). The manifest drift guard + * ([manifest.test.ts](manifest.test.ts)) pins each command's option *names/descriptions*, but NOT that the + * commander `.action()` actually *forwards* the parsed opt into the `CommandInput` it dispatches. A dropped opt + * (the S10 `provider add --pricing-url` bug: the flag parsed but the action forwarded only `{ baseUrl }`) is + * therefore invisible to it. Here we mock the dispatch table and assert the exact `CommandInput` each action hands + * off, so a future "added an option to specs.ts but forgot to widen its action" fails loudly. + */ + +// `vi.hoisted` runs before the hoisted `vi.mock` + the imports, so the mock factory can close over this fn. The +// generic signature (not named params) types `mock.calls` as the (id, input, ctx) tuple with no unused bindings. +const { executeCommand } = vi.hoisted(() => ({ + executeCommand: vi.fn<(id: string, input: unknown, ctx: unknown) => Promise>(() => + Promise.resolve(0), + ), +})); +vi.mock('./dispatch.js', () => ({ executeCommand })); + +/** Parse `argv` through the real commander program (with a live context) and return the dispatched (id, input). */ +function drive(argv: readonly string[]): { id: unknown; input: unknown } { + executeCommand.mockClear(); + const io = captureIo().io; + const context = { + io, + global: { + json: false, + color: false, + cwd: process.cwd(), + configPath: undefined, + verbosity: 'normal' as const, + }, + result: {}, + }; + const program = buildProgram(io, { context }); + program.exitOverride(); + // `.parse` (sync) invokes the action, which calls executeCommand synchronously BEFORE its first await — so the + // mock has recorded the call by the time parse returns, even though the action's promise is not awaited here. + program.parse(['node', 'relavium', ...argv]); + const call = executeCommand.mock.calls[0]; + if (call === undefined) throw new Error('executeCommand was not called'); + return { id: call[0], input: call[1] }; +} + +describe('commander action → executeCommand forwarding (S10)', () => { + it('provider add forwards BOTH --base-url AND --pricing-url (the dropped-opt regression)', () => { + const { id, input } = drive([ + 'provider', + 'add', + 'openai', + '--base-url', + 'https://proxy.example/v1', + '--pricing-url', + 'https://wiki.internal/prices', + ]); + expect(id).toBe('provider.add'); + expect(input).toMatchObject({ + positionals: ['openai'], + options: { baseUrl: 'https://proxy.example/v1', pricingUrl: 'https://wiki.internal/prices' }, + }); + }); + + it('models pricing forwards the model positional + all four options', () => { + const { id, input } = drive([ + 'models', + 'pricing', + 'my-model', + '--provider', + 'openai', + '--input', + '3', + '--output', + '9', + '--cached', + '0.1', + ]); + expect(id).toBe('models.pricing'); + expect(input).toMatchObject({ + positionals: ['my-model'], + options: { provider: 'openai', input: '3', output: '9', cached: '0.1' }, + }); + }); +}); diff --git a/apps/cli/src/commands/specs.ts b/apps/cli/src/commands/specs.ts index fe1cd5b3..a8c30f41 100644 --- a/apps/cli/src/commands/specs.ts +++ b/apps/cli/src/commands/specs.ts @@ -559,10 +559,10 @@ function registerProvider(program: Command, ctx?: CommandContext): void { ctx, ); }); - add.action(async (name: string, opts: { baseUrl?: string }) => { + add.action(async (name: string, opts: { baseUrl?: string; pricingUrl?: string }) => { ctx.result.exitCode = await executeCommand( 'provider.add', - { positionals: [name], options: { baseUrl: opts.baseUrl } }, + { positionals: [name], options: { baseUrl: opts.baseUrl, pricingUrl: opts.pricingUrl } }, ctx, ); }); diff --git a/apps/cli/src/engine/pricing-overlay.ts b/apps/cli/src/engine/pricing-overlay.ts index 05634a60..ec9ccf5d 100644 --- a/apps/cli/src/engine/pricing-overlay.ts +++ b/apps/cli/src/engine/pricing-overlay.ts @@ -56,6 +56,11 @@ export function loadUserPricingOverlay(homeDir: string): PricingOverlay | undefi } try { return buildUserPricingOverlay(opened.db); + } catch { + // A READ fault too (a corrupt provider/catalog row, a locked table) degrades to `undefined` — the docstring's + // "any db fault ⇒ undefined" contract is unconditional, so the overlay is never the thing that crashes a + // surface; the surface's own store open is the authoritative fault report. + return undefined; } finally { opened.close(); } diff --git a/packages/core/src/engine/engine.ts b/packages/core/src/engine/engine.ts index a6053eec..ebb6435a 100644 --- a/packages/core/src/engine/engine.ts +++ b/packages/core/src/engine/engine.ts @@ -240,7 +240,7 @@ export interface WorkflowEngineDeps { */ readonly maxTokensEstimate?: number; /** - * The user-pricing overlay (2.5.G S10, [ADR-0065](../../../docs/decisions/0065-provider-economics-and-user-pricing.md) + * The user-pricing overlay (2.5.G S10, [ADR-0065](../../../docs/decisions/0065-provider-economics-and-extensibility.md) * §2) — a `ReadonlyMap` the host projects from the `model_catalog` `source='user'` rows. * It feeds the workflow PRE-EGRESS budget governor so a model with no static price, once user-priced, is enforced * by `budget.max_cost_microcents`. Static `MODEL_PRICING` still wins for a known id (fills an UNKNOWN id only). diff --git a/packages/db/src/model-catalog-store.test.ts b/packages/db/src/model-catalog-store.test.ts index 207cce4f..2924b974 100644 --- a/packages/db/src/model-catalog-store.test.ts +++ b/packages/db/src/model-catalog-store.test.ts @@ -982,6 +982,47 @@ describe('createModelCatalogStore (2.5.G / ADR-0064 — live-discovery cache)', expect(reListing?.outputCostPerMtokMicrocents).toBe(950); // updated }); + it('a pricing-only upsert PRESERVES the display name + limits of a SOFT-DEACTIVATED row (S10 re-price)', () => { + // A live refresh discovers the model with a real name + context, then a later refresh drops it → soft-deactivated + // (isActive=false, source='live', deletedAt=null). The active-only `listByProvider` the command reads can no + // longer see it, so the pricing upsert must OMIT display/limits and let the store preserve the deactivated row's. + store.replaceProviderModels( + providerId, + [{ modelId: 'vanishing', displayName: 'Vanishing Pro', contextWindowTokens: 128_000 }], + TS_MS, + ); + store.replaceProviderModels(providerId, [], TS_MS + 1); // the model vanishes ⇒ soft-deactivated + expect(store.listByProvider(providerId).find((m) => m.modelId === 'vanishing')).toBeUndefined(); + // Pricing-only upsert (display/limits omitted) — the store finds the deactivated row (deletedAt IS NULL), + // reactivates it as source='user', and PRESERVES its discovered name/context rather than zeroing them. + store.upsert({ + providerId, + modelId: 'vanishing', + source: 'user', + inputCostPerMtokMicrocents: 300, + outputCostPerMtokMicrocents: 900, + }); + const listing = store.listByProvider(providerId).find((m) => m.modelId === 'vanishing'); + expect(listing?.displayName).toBe('Vanishing Pro'); // NOT zeroed to the id + expect(listing?.contextWindowTokens).toBe(128_000); // NOT zeroed + expect(listing?.source).toBe('user'); + expect(listing?.inputCostPerMtokMicrocents).toBe(300); + }); + + it('a brand-new pricing-only upsert (no prior row) defaults displayName → the model id, limits → unknown', () => { + store.upsert({ + providerId, + modelId: 'fresh-priced', + source: 'user', + inputCostPerMtokMicrocents: 100, + outputCostPerMtokMicrocents: 200, + }); + const listing = store.listByProvider(providerId).find((m) => m.modelId === 'fresh-priced'); + expect(listing?.displayName).toBe('fresh-priced'); // defaulted to the id + expect(listing?.contextWindowTokens).toBeUndefined(); // stored 0 sentinel ⇒ read back as absent + expect(listing?.maxOutputTokens).toBeUndefined(); + }); + it('listByProvider/listAll exclude a soft-DELETED (deletedAt) row, not just an inactive one', () => { store.replaceProviderModels( providerId, diff --git a/packages/db/src/model-catalog-store.ts b/packages/db/src/model-catalog-store.ts index 772ebc23..e3848298 100644 --- a/packages/db/src/model-catalog-store.ts +++ b/packages/db/src/model-catalog-store.ts @@ -56,9 +56,12 @@ export interface ModelCatalogRecord { export interface ModelCatalogUpsert { readonly providerId: string; readonly modelId: string; - readonly displayName: string; - readonly contextWindowTokens: number; - readonly maxOutputTokens: number; + /** OMITTED on update ⇒ PRESERVE the existing row's value (a pricing-only patch keeps the discovered name/limits, + * incl. a soft-deactivated row's); on a true INSERT ⇒ default (`displayName` → the model id, tokens → the `0` + * "unknown" sentinel). A full-row caller (the media fixture / a sync) always passes them, so it is unchanged. */ + readonly displayName?: string; + readonly contextWindowTokens?: number; + readonly maxOutputTokens?: number; readonly mediaSurface?: MediaSurface; readonly supportsVision?: boolean; readonly capabilities?: Record; @@ -345,9 +348,14 @@ export function createModelCatalogStore(db: Db, deps: ModelCatalogStoreDeps): Mo .get(); const id = existing?.id ?? deps.uuid(); const shared = { - displayName: input.displayName, - contextWindowTokens: input.contextWindowTokens, - maxOutputTokens: input.maxOutputTokens, + // Display name + token limits also follow the "never clobber an omitted field on update" invariant (2.5.G + // S10): a pricing-only `models pricing` patch omits them, so they PRESERVE the existing row's values — incl. + // a soft-deactivated live row the command's active-only read cannot see (else a re-price would silently zero + // the discovered name/context). A true INSERT defaults `displayName` → the model id, tokens → the `0` + // "unknown" sentinel; a full-row caller (media fixture / a sync) passes all three, so it is unchanged. + displayName: input.displayName ?? existing?.displayName ?? input.modelId, + contextWindowTokens: input.contextWindowTokens ?? existing?.contextWindowTokens ?? 0, + maxOutputTokens: input.maxOutputTokens ?? existing?.maxOutputTokens ?? 0, // Media routing / capability columns follow the SAME "never clobber an omitted field on update" invariant // as the pricing + provenance columns below (2.5.G S10): a partial upsert — e.g. `models pricing` writing a // `source='user'` row over a model the live refresh discovered — must NOT reset a live/seed row's From 7907eb8287b927d3d96cf656d3d84e8bc560fbe8 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 06:59:48 +0300 Subject: [PATCH 29/68] =?UTF-8?q?fix(cli,llm):=202.5.G=20S10=20Sonnet-revi?= =?UTF-8?q?ew=20fixes=20=E2=80=94=20gate=20surface=20+=20cross-provider=20?= =?UTF-8?q?guard?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - BLOCKER: `relavium gate` (cross-process resume of a paused workflow) built the engine WITHOUT the pricing overlay, so a gated run's post-gate segment silently uncapped a user-priced model — reopening the ADR-0064 §6 gap for every gated/paused run. Wire `resolvePrice` from the resume db, mirroring run.ts. - HIGH: `buildUserPricing` had no cross-provider collision guard (unlike `mergeModelCatalog`) — the same model id user-priced under two providers collapsed to an arbitrary (UUID-luck) price. Add a deterministic first-wins guard AND have `models pricing` fail-loud reject creating such a duplicate. - MEDIUM: a `chat`/`chat-resume` `/clear` rebuild reused the overlay captured at process start (Home already re-read fresh) — a mid-session `models pricing` write was silently ignored. Re-read the overlay fresh in buildFreshChatWiring. - MEDIUM: `run`/`gate`/`chat-resume`/`/clear` called the THROWING `buildUserPricingOverlay` directly; a corrupt pricing row would fail the run. Add a non-fatal `readUserPricingOverlay(db)` wrapper and route all open-db surfaces through it (parity with `loadUserPricingOverlay`'s contract). - LOW: export `isCanonicalModelId` from `@relavium/llm` and use it (drops the `KNOWN_MODEL_IDS as readonly string[]` cast in the canonical-id guard). - tests: gate-resume overlay wiring, the cross-provider reject + same-pair update, the deterministic collision, and the non-fatal read-fault degrade. Refs: ADR-0065 §1/§2 Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/commands/chat.ts | 27 +++++++------- apps/cli/src/commands/gate.test.ts | 37 +++++++++++++++++++ apps/cli/src/commands/gate.ts | 7 ++++ apps/cli/src/commands/models-pricing.test.ts | 30 +++++++++++++++ apps/cli/src/commands/models-pricing.ts | 24 ++++++++++-- apps/cli/src/commands/run.ts | 7 ++-- .../cli/src/engine/model-catalog-view.test.ts | 25 +++++++++++++ apps/cli/src/engine/model-catalog-view.ts | 6 +++ apps/cli/src/engine/pricing-overlay.test.ts | 23 +++++++++++- apps/cli/src/engine/pricing-overlay.ts | 17 +++++++++ apps/cli/src/home/drive-home.tsx | 15 +++----- docs/reference/cli/commands.md | 2 +- packages/llm/src/index.ts | 2 +- 13 files changed, 189 insertions(+), 33 deletions(-) diff --git a/apps/cli/src/commands/chat.ts b/apps/cli/src/commands/chat.ts index d8e8daed..d46a5e48 100644 --- a/apps/cli/src/commands/chat.ts +++ b/apps/cli/src/commands/chat.ts @@ -44,7 +44,7 @@ import { } from '../chat/session-host.js'; import { loadResolvedConfig } from '../config/load.js'; import { assembleToolEnv } from '../engine/tool-host/assemble.js'; -import { buildUserPricingOverlay, loadUserPricingOverlay } from '../engine/pricing-overlay.js'; +import { loadUserPricingOverlay, readUserPricingOverlay } from '../engine/pricing-overlay.js'; import { surfaceMcpSkipped } from '../engine/mcp-servers.js'; import { createProviderResolver, type ProviderResolver } from '../engine/providers.js'; import { openSessionStore, type OpenedSessionStore } from '../history/session-open.js'; @@ -328,8 +328,8 @@ export async function chatCommand(args: ChatCommandArgs, deps: ChatCommandDeps): buildSession: deps.buildSession ?? buildChatSession, io: deps.io, global: deps.global, - // The same overlay the initial session used — a `/clear` rebuild keeps the user-priced cost enforcement. - ...(resolvePrice === undefined ? {} : { resolvePrice }), + // A `/clear` rebuild re-reads the user-pricing overlay FRESH from the shared db (buildFreshChatWiring), so a + // mid-session `models pricing` write applies to the next cleared session — no captured value to thread here. }); return runReplLoop( @@ -390,9 +390,9 @@ export async function chatResumeCommand( throw new CliError('invalid_invocation', `no session found with id ${args.sessionId}`); } // The ADR-0065 §2 user-pricing overlay (2.5.G S10), read from the ALREADY-OPEN session db — so a resumed - // session enforces + tracks a user-priced model exactly like a fresh `chat`. An empty map (no user rows) is - // harmless. Inside the try, so any read fault hits the same teardown as the rest of the pre-loop wiring. - resolvePrice = buildUserPricingOverlay(opened.db); + // session enforces + tracks a user-priced model exactly like a fresh `chat`. Non-fatal (an empty map on a + // read fault): a corrupt pricing row must not fail an otherwise-valid resume. + resolvePrice = readUserPricingOverlay(opened.db); const resumed = await (deps.buildResumedSession ?? buildResumedChatSession)({ chat: config.chat, record: loaded.session, @@ -482,8 +482,7 @@ export async function chatResumeCommand( buildSession: deps.buildSession ?? buildChatSession, io: deps.io, global: deps.global, - // The same overlay the resumed session used — a `/clear` rebuild keeps the user-priced cost enforcement. - ...(resolvePrice === undefined ? {} : { resolvePrice }), + // A `/clear` rebuild re-reads the user-pricing overlay FRESH from the shared db (buildFreshChatWiring). }); // A resumed session already landed at idle inside `AgentSession.resume`; calling start() would throw and @@ -935,12 +934,14 @@ interface FreshChatWiringDeps { readonly opened: OpenedSessionStore; readonly buildSession: typeof buildChatSession; readonly onBudgetWarning: NonNullable; - /** The ADR-0065 §2 user-pricing overlay (2.5.G S10) — carried into the rebuilt session so a `/clear` keeps - * the user-priced cost enforcement of the session it replaced. */ - readonly resolvePrice: BuildChatSessionOptions['resolvePrice']; } async function buildFreshChatWiring(deps: FreshChatWiringDeps, intro: string): Promise { + // Re-read the ADR-0065 §2 user-pricing overlay FRESH from the shared db on each `/clear` rebuild (2.5.G S10) — so + // a `models pricing` write made in another terminal mid-session takes effect on the very next `/clear` session, + // the SAME freshness guarantee `readEffectiveDefault()` gives the default model (and the Home already gives + // pricing). Non-fatal (empty map on a read fault). Empty ⇒ omitted (unknown models degrade to allow, unchanged). + const resolvePrice = readUserPricingOverlay(deps.opened.db); const built = await deps.buildSession({ chat: deps.chat, agent: deps.agent, @@ -952,7 +953,7 @@ async function buildFreshChatWiring(deps: FreshChatWiringDeps, intro: string): P providers: deps.providers, mcpSecretResolver: deps.mcpSecretResolver, ...(deps.mcpRegistrations === undefined ? {} : { mcpRegistrations: deps.mcpRegistrations }), - ...(deps.resolvePrice === undefined ? {} : { resolvePrice: deps.resolvePrice }), + ...(resolvePrice.size === 0 ? {} : { resolvePrice }), onBudgetWarning: deps.onBudgetWarning, }); surfaceMcpSkipped(deps.io, built.mcpSkipped); @@ -1013,7 +1014,6 @@ function createClearRebuild(params: { readonly buildSession: typeof buildChatSession; readonly io: CliIo; readonly global: GlobalOptions; - readonly resolvePrice?: BuildChatSessionOptions['resolvePrice']; }): (oldSessionId: string) => Promise { const wiringDeps: FreshChatWiringDeps = { chat: params.chat, @@ -1030,7 +1030,6 @@ function createClearRebuild(params: { global: params.global, opened: params.opened, buildSession: params.buildSession, - resolvePrice: params.resolvePrice, onBudgetWarning: (warning) => params.io.writeErr( `budget warning: ~${warning.thresholdPct}% of the ${warning.limitMicrocents}µ¢ cap reached\n`, diff --git a/apps/cli/src/commands/gate.test.ts b/apps/cli/src/commands/gate.test.ts index 03b3b28a..f989f9b7 100644 --- a/apps/cli/src/commands/gate.test.ts +++ b/apps/cli/src/commands/gate.test.ts @@ -272,6 +272,43 @@ describe('gateCommand', () => { expect(sweptArgs?.graceMs).toBeUndefined(); // no [defaults].media_gc_grace_days ⇒ the GC default window }); + it('wires the user-pricing overlay on a gate-resumed run so the post-gate segment stays capped (2.5.G S10)', async () => { + // Seed a `source='user'` price for a model the static registry does NOT know, into the SHARED db the gate + // path reads. Without the S10 wiring the resumed run would silently uncap this model past the gate. + const dbDeps = { uuid: () => randomUUID(), now: () => Date.now() }; + const providerId = createProviderStore(db, dbDeps).upsert({ + name: 'openai', + displayName: 'OpenAI', + baseUrl: 'https://api.openai.com/v1', + }).id; + createModelCatalogStore(db, dbDeps).upsert({ + providerId, + modelId: 'acme-custom-1', + source: 'user', + inputCostPerMtokMicrocents: 300_000_000, + outputCostPerMtokMicrocents: 900_000_000, + }); + const { runId } = await setupPausedRun(); + const { io } = captureIo(); + let captured: BuildEngineOptions | undefined; + const code = await gateCommand( + { runId, approve: true }, + { + ...deps(io), + buildEngine: (opts) => { + captured = opts; + return buildEngine(opts); + }, + sweepMedia: () => Promise.resolve(undefined), + }, + ); + expect(code).toBe(EXIT_CODES.success); + // The overlay reached buildEngine (which threads it into BOTH the pre-egress governor and the realized + // CostTracker), and it prices the otherwise-unknown model. + expect(captured?.resolvePrice?.get('acme-custom-1')?.inputPerMtokMicrocents).toBe(300_000_000); + expect(captured?.resolvePrice?.get('acme-custom-1')?.outputPerMtokMicrocents).toBe(900_000_000); + }); + it('re-jails save_to under the ORIGINAL run project root on resume (persisted runs.project_root)', async () => { // Seed the paused run WITH project_root = a real dir A (distinct from the resumer cwd `root`), then resume. const originalRoot = mkdtempSync(join(tmpdir(), 'relavium-orig-root-')); diff --git a/apps/cli/src/commands/gate.ts b/apps/cli/src/commands/gate.ts index e55647a0..f5cf2065 100644 --- a/apps/cli/src/commands/gate.ts +++ b/apps/cli/src/commands/gate.ts @@ -24,6 +24,7 @@ import { sweepMediaAtTerminal, } from '../engine/media-gc.js'; import { buildMediaEngineWiring } from '../engine/media-wiring.js'; +import { readUserPricingOverlay } from '../engine/pricing-overlay.js'; import { createProviderResolver, type ProviderResolver } from '../engine/providers.js'; import { decisionFromFlags, type GateFlags } from '../gate/decision.js'; import type { GatePrompter } from '../gate/prompter.js'; @@ -214,6 +215,11 @@ export async function gateCommand(args: GateCommandArgs, deps: GateCommandDeps): // authored `output_modalities` against the CURRENT catalog, so a model that lost a capability between the // original run and this resume is rejected consistently (exit 2), not silently routed at runtime. assertWorkflowCatalogValid(workflow, wiring.workflowModelCatalog); + // The ADR-0065 §2 user-pricing overlay (2.5.G S10) — read from the SAME durable `history.db`, so the resumed + // workflow's post-gate continuation enforces `budget.max_cost_microcents` on a user-priced model exactly like + // the original `run` did (pre-egress + realized). Without it a gated run would silently uncap that model on the + // far side of the gate — the very ADR-0064 §6 gap this closes. Non-fatal read (an empty map ⇒ no user pricing). + const resolvePrice = readUserPricingOverlay(opened.db); const engine = await (deps.buildEngine ?? defaultBuildEngine)({ providers, // 2.5.A (ADR-0055): wire the SAME read+write fs + process ToolHost the `relavium run` path wires, jailed @@ -227,6 +233,7 @@ export async function gateCommand(args: GateCommandArgs, deps: GateCommandDeps): ...(wiring.mediaCostEstimate === undefined ? {} : { mediaCostEstimate: wiring.mediaCostEstimate }), + ...(resolvePrice.size === 0 ? {} : { resolvePrice }), }); const handle = await resumeOrFail(engine, { runId: args.runId, diff --git a/apps/cli/src/commands/models-pricing.test.ts b/apps/cli/src/commands/models-pricing.test.ts index 6f52a503..b4000636 100644 --- a/apps/cli/src/commands/models-pricing.test.ts +++ b/apps/cli/src/commands/models-pricing.test.ts @@ -144,6 +144,36 @@ describe('modelsPricingCommand (2.5.G S10)', () => { expect(catalog.listAll()).toHaveLength(0); }); + it('REJECTS pricing a model id already user-priced under a DIFFERENT provider (the overlay keys by id)', () => { + // Register a second provider + price the SAME model id under it, then try to price it under openai. + providers.upsert({ name: 'deepseek', displayName: 'DeepSeek', baseUrl: 'https://api.deepseek.com' }); + const deepseekId = providers.list().find((p) => p.name === 'deepseek')?.id ?? ''; + catalog.upsert({ + providerId: deepseekId, + modelId: 'shared-id', + source: 'user', + inputCostPerMtokMicrocents: 5, + outputCostPerMtokMicrocents: 15, + }); + const err = runThrows({ ...baseArgs, model: 'shared-id', provider: 'openai' }); + expect(err.code).toBe('invalid_invocation'); + expect(err.message).toContain('already user-priced'); + expect(err.message).toContain('deepseek'); // names the other provider + // The openai row was NOT written — only the original deepseek user row remains for this id. + const rows = catalog.listAll().filter((m) => m.modelId === 'shared-id'); + expect(rows).toHaveLength(1); + expect(rows[0]?.providerId).toBe(deepseekId); + }); + + it('ALLOWS re-pricing the SAME (provider, model) — an update, not a cross-provider duplicate', () => { + run(baseArgs); // openai / acme-custom-1 + // Re-price the same pair — the dup guard must NOT trip (same provider), it is a plain update. + run({ ...baseArgs, inputUsdPerMtok: 4, outputUsdPerMtok: 12 }); + const rows = catalog.listAll().filter((m) => m.modelId === 'acme-custom-1'); + expect(rows).toHaveLength(1); + expect(rows[0]?.inputCostPerMtokMicrocents).toBe(400_000_000); + }); + it('re-pricing an existing model preserves its display name + limits (only prices change)', () => { const providerId = providers.list()[0]?.id ?? ''; // Seed a richer existing row (as a live discovery would) with a display name + context. diff --git a/apps/cli/src/commands/models-pricing.ts b/apps/cli/src/commands/models-pricing.ts index 7e8613de..d233cae3 100644 --- a/apps/cli/src/commands/models-pricing.ts +++ b/apps/cli/src/commands/models-pricing.ts @@ -1,5 +1,5 @@ import type { ModelCatalogStore, ProviderStore } from '@relavium/db'; -import { KNOWN_MODEL_IDS } from '@relavium/llm'; +import { isCanonicalModelId } from '@relavium/llm'; import { CliError } from '../process/errors.js'; import { EXIT_CODES, type ExitCode } from '../process/exit-codes.js'; @@ -44,8 +44,9 @@ export interface ModelsPricingCommandDeps { readonly io: CliIo; readonly global: GlobalOptions; /** The catalog store — `upsert` writes the `source='user'` row (a pricing-only patch; the store preserves the - * existing row's display/limits + media columns, so no read is needed here). */ - readonly catalog: Pick; + * existing row's display/limits + media columns). `listAll` is read only to reject a cross-provider duplicate + * (the overlay keys by model id, so the same id priced under two providers would be ambiguous). */ + readonly catalog: Pick; /** The provider registry — resolves the `` → its internal `llm_providers` UUID (the catalog FK). */ readonly providers: Pick; } @@ -81,7 +82,7 @@ export function modelsPricingCommand( ): ExitCode { // Precedence guard (ADR-0065 §2): a canonical id always resolves to `MODEL_PRICING`, so a user override would be // a silent no-op. Reject BEFORE any write — nothing is stored on a rejected invocation. - if ((KNOWN_MODEL_IDS as readonly string[]).includes(args.model)) { + if (isCanonicalModelId(args.model)) { throw new CliError( 'invalid_invocation', `'${args.model}' already has a built-in price — a user override would never take effect (the static registry always wins). Nothing written.`, @@ -95,6 +96,21 @@ export function modelsPricingCommand( `unknown provider '${args.provider}' — register it first (e.g. \`relavium provider add ${args.provider}\` or \`relavium provider set-key ${args.provider}\`).`, ); } + // Cross-provider ambiguity guard (ADR-0065 §2): the cost overlay keys by MODEL ID (the runtime references a model + // by id alone, no provider), so the SAME id user-priced under two providers can't be distinguished — the cap + // would then apply an arbitrary one. Reject fail-loud rather than silently overwrite; nothing is written. + const dup = deps.catalog + .listAll() + .find( + (m) => m.source === 'user' && m.modelId === args.model && m.providerId !== providerRow.id, + ); + if (dup !== undefined) { + const otherProvider = deps.providers.list().find((p) => p.id === dup.providerId)?.name ?? 'another provider'; + throw new CliError( + 'invalid_invocation', + `'${args.model}' is already user-priced under '${otherProvider}'. The cost cap keys by model id, so a second provider's price can't be distinguished — remove that price (re-price under '${otherProvider}') or use a distinct model id.`, + ); + } // Convert + bounds-validate BEFORE the write (a bad `--cached` must not leave a partially-applied row). const inputCostPerMtokMicrocents = usdToMicrocents(args.inputUsdPerMtok, '--input'); const outputCostPerMtokMicrocents = usdToMicrocents(args.outputUsdPerMtok, '--output'); diff --git a/apps/cli/src/commands/run.ts b/apps/cli/src/commands/run.ts index d3016056..b8d02401 100644 --- a/apps/cli/src/commands/run.ts +++ b/apps/cli/src/commands/run.ts @@ -24,7 +24,7 @@ import { sweepMediaAtTerminal, } from '../engine/media-gc.js'; import { buildMediaEngineWiring } from '../engine/media-wiring.js'; -import { buildUserPricingOverlay } from '../engine/pricing-overlay.js'; +import { readUserPricingOverlay } from '../engine/pricing-overlay.js'; import { createProviderResolver, neededProviderIds, @@ -207,8 +207,9 @@ export async function runCommand(args: RunCommandArgs, deps: RunCommandDeps): Pr // The ADR-0065 §2 user-pricing overlay (2.5.G S10), read from the SAME durable `history.db` — so a // workflow using a user-priced model is enforced by `budget.max_cost_microcents` (pre-egress) + priced in // realized cost (the agent node). Only wired on this durable-history branch: the in-memory unit/harness - // path has no db, hence no user rows. An empty map (no user rows) is harmless (fills nothing). - const resolvePrice = buildUserPricingOverlay(opened.db); + // path has no db, hence no user rows. An empty map (no user rows) is harmless (fills nothing). Non-fatal: + // a corrupt provider/catalog row degrades to an empty overlay, never failing the run over a pricing read. + const resolvePrice = readUserPricingOverlay(opened.db); engineOptions = { providers, toolEnv, diff --git a/apps/cli/src/engine/model-catalog-view.test.ts b/apps/cli/src/engine/model-catalog-view.test.ts index 563347d7..29db567c 100644 --- a/apps/cli/src/engine/model-catalog-view.test.ts +++ b/apps/cli/src/engine/model-catalog-view.test.ts @@ -137,6 +137,31 @@ describe('buildUserPricing (2.5.G S10, ADR-0065 §2)', () => { expect([...overlay.keys()]).toEqual(['user-model']); }); + it('is DETERMINISTIC on a cross-provider model-id collision — keeps the FIRST row, never last-write-wins', () => { + // Two user rows for the SAME model id under different providers (reachable via custom base_url on openai vs + // deepseek). The overlay keys by model id, so it can hold only one — the guard keeps the first, deterministically. + const overlay = buildUserPricing({ + rows: [ + row({ + modelId: 'shared-id', + providerId: 'p-openai', + source: 'user', + inputCostPerMtokMicrocents: 111, + }), + row({ + modelId: 'shared-id', + providerId: 'p-deepseek', + source: 'user', + inputCostPerMtokMicrocents: 999, + }), + ], + providerSlug: slugResolver({ 'p-openai': 'openai', 'p-deepseek': 'deepseek' }), + }); + expect(overlay.size).toBe(1); + expect(overlay.get('shared-id')?.inputPerMtokMicrocents).toBe(111); // the FIRST row, not the second (999) + expect(overlay.get('shared-id')?.provider).toBe('openai'); + }); + it('drops a user row whose provider UUID resolves to a non-enum slug (never injects under a known provider)', () => { const overlay = buildUserPricing({ rows: [row({ modelId: 'rogue-priced', providerId: 'rogue', source: 'user' })], diff --git a/apps/cli/src/engine/model-catalog-view.ts b/apps/cli/src/engine/model-catalog-view.ts index a8d9c71d..6e5474b9 100644 --- a/apps/cli/src/engine/model-catalog-view.ts +++ b/apps/cli/src/engine/model-catalog-view.ts @@ -106,6 +106,12 @@ export function buildUserPricing(input: { if (row.source !== 'user') continue; // only the user-pricing rows carry an authored price const slug = input.providerSlug(row.providerId); if (!isProviderId(slug)) continue; // an unmapped UUID / non-enum provider — never inject under a known provider + // Cross-provider collision guard (mirrors {@link mergeModelCatalog}'s): the overlay is keyed by model id + // (the runtime references a model by id alone), so two providers' user rows for the SAME id are ambiguous. + // Keep the FIRST (deterministic: `listAll()` orders `asc(modelId), asc(id)`) rather than letting the last + // write win by UUID luck. `models pricing` REJECTS creating such a duplicate, so this is a defense-in-depth + // floor for a legacy / directly-edited db, never the primary guard. + if (map.has(row.modelId)) continue; map.set(row.modelId, rowToUserPricing(row, slug)); } return map; diff --git a/apps/cli/src/engine/pricing-overlay.test.ts b/apps/cli/src/engine/pricing-overlay.test.ts index 39a16b42..9dc0dd6f 100644 --- a/apps/cli/src/engine/pricing-overlay.test.ts +++ b/apps/cli/src/engine/pricing-overlay.test.ts @@ -13,7 +13,11 @@ import { import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { openLocalDb } from '../db/open.js'; -import { buildUserPricingOverlay, loadUserPricingOverlay } from './pricing-overlay.js'; +import { + buildUserPricingOverlay, + loadUserPricingOverlay, + readUserPricingOverlay, +} from './pricing-overlay.js'; /** * `pricing-overlay` host-loader tests (2.5.G S10, ADR-0065 §2). `buildUserPricingOverlay` is driven over a real @@ -72,6 +76,23 @@ describe('buildUserPricingOverlay (over an open db)', () => { const overlay = buildUserPricingOverlay(client.db); expect(overlay.size).toBe(0); }); + + it('readUserPricingOverlay returns the same overlay on a healthy db (the non-fatal wrapper)', () => { + seedUserPriced(client.db); + expect(readUserPricingOverlay(client.db).get('acme-custom-1')?.inputPerMtokMicrocents).toBe( + 300_000_000, + ); + }); + + it('readUserPricingOverlay degrades to an EMPTY map (never throws) when the read faults', () => { + // A separate, already-CLOSED db: better-sqlite3 throws on any query against a closed handle, so the read + // faults. The non-fatal wrapper must swallow it and return an empty overlay (the surface's own store open is + // the authoritative report). A throwaway client so the shared `client`/afterEach lifecycle is untouched. + const doomed = createClient(':memory:'); + runMigrations(doomed.db); + doomed.sqlite.close(); + expect(readUserPricingOverlay(doomed.db).size).toBe(0); + }); }); describe('loadUserPricingOverlay (self-contained transient open)', () => { diff --git a/apps/cli/src/engine/pricing-overlay.ts b/apps/cli/src/engine/pricing-overlay.ts index ec9ccf5d..a723489d 100644 --- a/apps/cli/src/engine/pricing-overlay.ts +++ b/apps/cli/src/engine/pricing-overlay.ts @@ -40,6 +40,23 @@ export function buildUserPricingOverlay(db: Db): PricingOverlay { }); } +/** + * The NON-FATAL variant of {@link buildUserPricingOverlay} for a surface that ALREADY holds an open db (`run`, + * `gate`, `chat-resume`, and the `/clear` rebuild): a READ fault (a corrupt provider/catalog row — e.g. + * `providerStore.list()` throwing on a tampered `default_headers`) degrades to an EMPTY overlay rather than + * propagating, so the overlay is never the thing that fails an otherwise-valid run/resume. It mirrors the same + * best-effort contract {@link loadUserPricingOverlay} gives the transient-open surfaces — the caller's own store / + * run path is the authoritative fault report. Returns a non-`undefined` empty map (the db is already open here, so + * there is no open-fault case to signal). + */ +export function readUserPricingOverlay(db: Db): PricingOverlay { + try { + return buildUserPricingOverlay(db); + } catch { + return new Map(); + } +} + /** * Load the overlay via a SELF-CONTAINED transient open→read→close, for a surface with no db handle of its own yet * (`relavium chat`, one-shot `agent run`). Deliberately NON-FATAL: an unopenable/unmigratable `history.db` yields diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index 73546905..c7eacb09 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -11,7 +11,8 @@ import type { DoctorProbes } from '../chat/doctor.js'; import { createSessionPersister, type SessionPersister } from '../chat/persister.js'; import { loadResolvedConfig } from '../config/load.js'; import { writeGlobalDefaultModel } from '../config/write.js'; -import { buildMergedCatalog, buildUserPricing } from '../engine/model-catalog-view.js'; +import { buildMergedCatalog } from '../engine/model-catalog-view.js'; +import { readUserPricingOverlay } from '../engine/pricing-overlay.js'; import { createModelRefreshService } from '../engine/model-refresh.js'; import { assembleToolEnv } from '../engine/tool-host/assemble.js'; import { @@ -213,14 +214,10 @@ export async function driveHome(deps: HomeDeps): Promise { // Build + wire + START a fresh chat session (the controller sends the first message on transition). const startChat = async (): Promise => { const store = createChatStore(deps.global.color); - // The ADR-0065 §2 user-pricing overlay (2.5.G S10), read FRESH per chat from the SAME db `models.load` reads - // (its own slug snapshot, like the picker) — so a user-priced model started in this long-lived Home is - // enforced by `[chat].max_cost_microcents` + tracked in realized cost. Static `MODEL_PRICING` still wins. - const pricingSlugByUuid = new Map(providerStore.list().map((p) => [p.id, p.name] as const)); - const resolvePrice = buildUserPricing({ - rows: catalogStore.listAll(), - providerSlug: (uuid_) => pricingSlugByUuid.get(uuid_) ?? uuid_, - }); + // The ADR-0065 §2 user-pricing overlay (2.5.G S10), read FRESH per chat from the SAME `history.db` — so a + // user-priced model started in this long-lived Home is enforced by `[chat].max_cost_microcents` + tracked in + // realized cost. Static `MODEL_PRICING` still wins. Non-fatal (empty map on a read fault). + const resolvePrice = readUserPricingOverlay(opened.db); const built: BuiltChatSession = await (deps.buildSession ?? buildChatSession)({ // Re-read the EFFECTIVE default model FRESH per chat (not the load-once `config` snapshot) so a same-session // `/models` write takes effect on the very next chat started in this long-lived Home (2.5.G S7) — the diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index b970c8a1..9d305c23 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -215,7 +215,7 @@ relavium models pricing my-custom-model --provider openai --input 3 --output 9 - **`relavium models`** (no subcommand) lists the cached catalog (read-only). On the **very first run** — when the cache is empty — it does one minimal **blocking** refresh, then lists; an empty result stays a clean exit `0` (an empty catalog is not a fault, like `relavium list`). Human output is one line per model (` ctx= []`). - **`relavium models refresh`** forces a live re-fetch of **each connected provider** (a provider whose key resolves via the OS keychain → `RELAVIUM__API_KEY` env var) and prints a per-provider outcome. The refresh is **per-provider isolated**: one provider's failure (bad key, network, endpoint drift) or a provider without a list endpoint **never** fails the whole command — that provider is reported `failed` / `skipped` and the others still refresh. A per-provider failure is therefore **not** a command failure (exit `0` with the report). The **one** hard fault is an explicit `refresh` with **zero** providers connected (no key at all): that is a clean exit `2` naming how to add a key, because nothing could be fetched. -- **`relavium models pricing --provider --input --output [--cached ]`** hand-enters the per-million-token price of a model the static registry does **not** know — a custom-endpoint model, or a new provider model not yet in the shipped [pricing.ts](../../../packages/llm/src/pricing.ts) (2.5.G S10, [ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) §1–2). Prices are **USD per million tokens** (`--input` prompt, `--output` completion, `--cached` cache-read; stored as integer micro-cents, `usd × 1e8`, never a float). The row is written as `source='user'` and a live `models refresh` **never** clobbers it. This **closes the cost-cap gap** ([ADR-0064](../../decisions/0064-live-model-catalog.md) §6): before, an unknown model had no price, so `budget.max_cost_microcents` / `[chat].max_cost_microcents` **degraded to allow** for it; once user-priced, the cap is enforced (pre-egress **and** realized) on `run`, `chat`, the Home chat, and one-shot `agent run`. Guards (each a clean exit `2`, nothing written): a **canonical** model id is refused (the shipped price always wins, so an override would be silently ignored); an **unregistered provider** is refused (register it first with `relavium provider add`); a **negative / non-finite / implausibly-large** price is refused. The static registry still wins for a known id, so a user can never misprice a shipped model. +- **`relavium models pricing --provider --input --output [--cached ]`** hand-enters the per-million-token price of a model the static registry does **not** know — a custom-endpoint model, or a new provider model not yet in the shipped [pricing.ts](../../../packages/llm/src/pricing.ts) (2.5.G S10, [ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) §1–2). Prices are **USD per million tokens** (`--input` prompt, `--output` completion, `--cached` cache-read; stored as integer micro-cents, `usd × 1e8`, never a float). The row is written as `source='user'` and a live `models refresh` **never** clobbers it. This **closes the cost-cap gap** ([ADR-0064](../../decisions/0064-live-model-catalog.md) §6): before, an unknown model had no price, so `budget.max_cost_microcents` / `[chat].max_cost_microcents` **degraded to allow** for it; once user-priced, the cap is enforced (pre-egress **and** realized) on `run`, a `run` resumed via [`relavium gate`](#relavium-gate), `chat` / `chat-resume` (incl. a `/clear` rebuild, re-read fresh), the Home chat, and one-shot `agent run`. Guards (each a clean exit `2`, nothing written): a **canonical** model id is refused (the shipped price always wins, so an override would be silently ignored); an **unregistered provider** is refused (register it first with `relavium provider add`); a **negative / non-finite / implausibly-large** price is refused; and the **same model id already user-priced under a *different* provider** is refused (the overlay keys by model id, so a second provider's price could not be distinguished — use a distinct id or re-price under that provider). The static registry still wins for a known id, so a user can never misprice a shipped model. - **Security.** A provider key is read only to make the live request (over the bounded, abortable, secret-free `listModels` seam) and is **never** logged, persisted (the cache holds no key), or placed in the report / `--json` payload / any error message. A failing provider surfaces only the seam's already-redacted message (or a generic `refresh failed`), never a raw cause. `models pricing` writes only a model id + provider + integer prices — no key, ever. - **`--json`** ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)) emits **one NDJSON record per line**, stdout-pure, key-free: - `relavium models --json` — one record per model: `{ provider, modelId, displayName, contextWindowTokens, maxOutputTokens, source, lastRefreshedAt, deprecationDate }` (`null` for an absent optional; `source` ∈ `static | live | user`; `lastRefreshedAt` is epoch-ms). diff --git a/packages/llm/src/index.ts b/packages/llm/src/index.ts index a64813f7..ef280407 100644 --- a/packages/llm/src/index.ts +++ b/packages/llm/src/index.ts @@ -129,7 +129,7 @@ export { } from './llm-error.js'; // CostTracker + the canonical model-pricing table (1.B). -export { MODEL_PRICING, KNOWN_MODEL_IDS, contextWindowForModel } from './pricing.js'; +export { MODEL_PRICING, KNOWN_MODEL_IDS, isCanonicalModelId, contextWindowForModel } from './pricing.js'; export type { ModelPricing, CanonicalModelId } from './pricing.js'; // The pure live/static/user merge helper (ADR-0064 §6) — reused by every surface's model catalog / picker. export { mergeModelCatalog } from './model-catalog.js'; From 5c13510a2dbe743d6069162978acbe1cffcb0482 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 07:13:50 +0300 Subject: [PATCH 30/68] =?UTF-8?q?feat(cli):=202.5.G=20S11=20=E2=80=94=20pr?= =?UTF-8?q?ovider=20list=20--verify=20+=20--json;=20max=5Fturns=20reconcil?= =?UTF-8?q?e?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `relavium provider list --verify` (ADR-0065 §6): an opt-in bounded, key-redacted LIVE probe per registered provider (reusing the shared `validateProviderKey` seam) reporting verified / failed — / no key. A keyless provider is never probed (no hang); the key is never echoed. - `provider list` now honors `--json` (ADR-0049 read-command NDJSON): one key-free record per provider `{ name, baseUrl, keySet, verified, verifyDetail }` (verified/verifyDetail null without --verify). Manifest + dispatch + the commander-action forwarding wired; a forwarding regression test added. - `[chat].max_turns`: verified fully reconciled (no change needed) — the docs (engine default 50, positiveInt rejects 0) match the code (DEFAULT_SESSION_MAX_TURNS=50, the `<=0 ⇒ default` arm) and the enforcement / resolution / resume-warning tests already cover it. - tests: verify verified/failed/no-key + --json shape (no key leak), dispatch arg extraction, action forwarding. docs: commands.md provider list --verify/--json. Refs: ADR-0065 §6, ADR-0049 Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/commands/dispatch.test.ts | 9 ++ apps/cli/src/commands/dispatch.ts | 7 +- apps/cli/src/commands/manifest.ts | 7 ++ apps/cli/src/commands/provider.test.ts | 75 ++++++++++++++- apps/cli/src/commands/provider.ts | 96 ++++++++++++++++++- .../cli/src/commands/specs-forwarding.test.ts | 6 ++ apps/cli/src/commands/specs.ts | 7 +- docs/reference/cli/commands.md | 5 +- 8 files changed, 202 insertions(+), 10 deletions(-) diff --git a/apps/cli/src/commands/dispatch.test.ts b/apps/cli/src/commands/dispatch.test.ts index 5d710511..88b41f1a 100644 --- a/apps/cli/src/commands/dispatch.test.ts +++ b/apps/cli/src/commands/dispatch.test.ts @@ -10,6 +10,7 @@ import { buildGateArgs, buildImportArgs, buildProviderAddArgs, + buildProviderListArgs, buildProviderTestArgs, buildRunArgs, DISPATCHABLE_COMMAND_IDS, @@ -157,6 +158,14 @@ describe('build*Args (argv → typed core args)', () => { expect(() => buildGateArgs(input([]))).toThrow(/`relavium gate` requires a /); }); + it('provider.list: verify defaults false, true when the flag is present', () => { + expect(buildProviderListArgs(input([], {}))).toEqual({ action: 'list', verify: false }); + expect(buildProviderListArgs(input([], { verify: true }))).toEqual({ + action: 'list', + verify: true, + }); + }); + it('provider.add: name + optional baseUrl + optional pricingUrl (each omitted when absent)', () => { expect( buildProviderAddArgs( diff --git a/apps/cli/src/commands/dispatch.ts b/apps/cli/src/commands/dispatch.ts index 886008d2..f47e4cc7 100644 --- a/apps/cli/src/commands/dispatch.ts +++ b/apps/cli/src/commands/dispatch.ts @@ -177,6 +177,10 @@ export function buildGateArgs(input: CommandInput): GateCommandArgs { }; } +export function buildProviderListArgs(input: CommandInput): ProviderCommandArgs { + return { action: 'list', verify: boolFlag(input.options['verify']) }; +} + export function buildProviderAddArgs(input: CommandInput): ProviderCommandArgs { const baseUrl = optString(input.options['baseUrl']); const pricingUrl = optString(input.options['pricingUrl']); @@ -498,6 +502,7 @@ async function withProviderDeps( // Store-aware so `provider test` pings a custom `base_url` provider at its CUSTOM endpoint (2.5.G S9). resolver: createProviderResolver(ctx.io.env, keychain, { providerStore: store }), readSecret: readSecretFromStdin, + global: ctx.global, // for `provider list --json` (2.5.G S11, ADR-0049) }; return await fn(deps); } finally { @@ -535,7 +540,7 @@ const COMMAND_EXECUTORS: ReadonlyMap = new Map ({ action: 'list' }))], + ['provider.list', providerExecutor(buildProviderListArgs)], ['provider.add', providerExecutor(buildProviderAddArgs)], [ 'provider.set-key', diff --git a/apps/cli/src/commands/manifest.ts b/apps/cli/src/commands/manifest.ts index be7ce399..83d28c41 100644 --- a/apps/cli/src/commands/manifest.ts +++ b/apps/cli/src/commands/manifest.ts @@ -331,6 +331,13 @@ const ENTRIES: readonly CommandManifestEntry[] = [ id: 'provider.list', label: 'List providers', description: 'List registered providers and whether a key is set.', + args: [ + { + name: 'verify', + type: 'boolean', + description: 'additionally run a live key-verification probe per provider', + }, + ], effect: 'read', }, { diff --git a/apps/cli/src/commands/provider.test.ts b/apps/cli/src/commands/provider.test.ts index 7635801e..9b3afdaa 100644 --- a/apps/cli/src/commands/provider.test.ts +++ b/apps/cli/src/commands/provider.test.ts @@ -10,7 +10,7 @@ import { } from '../engine/providers.js'; import { KeychainUnavailableError, type KeychainStore } from '../secrets/keychain.js'; import { readSecretFromStdin } from '../secrets/read-secret.js'; -import { captureIo } from '../test-support.js'; +import { captureIo, parseNdjson } from '../test-support.js'; import { runProviderCommand, type ProviderCommandDeps } from './provider.js'; const TS_MS = new Date('2026-06-23T12:00:00.000Z').getTime(); @@ -70,6 +70,7 @@ describe('relavium provider commands (2.C)', () => { Promise.resolve({} as Awaited>), ), readSecret: () => Promise.resolve(RAW_KEY), + global: { json: false, color: false, cwd: process.cwd(), configPath: undefined, verbosity: 'normal' }, ...over, }); }); @@ -106,6 +107,78 @@ describe('relavium provider commands (2.C)', () => { expect(text).not.toContain(RAW_KEY); }); + it('list --verify reports "verified" when a provider live probe succeeds (2.5.G S11)', async () => { + const d = deps({}); // the default stub resolver resolves generate ⇒ probe ok + await runProviderCommand({ action: 'add', name: 'openai' }, d); + io.out(); + await runProviderCommand({ action: 'list', verify: true }, d); + expect(io.out()).toContain('[verified]'); + }); + + it('list --verify reports "failed — " on a probe failure, NEVER echoing the key', async () => { + const d = deps({ resolver: stubResolver(() => Promise.reject(new Error(`boom ${RAW_KEY}`))) }); + await runProviderCommand({ action: 'add', name: 'openai' }, d); + io.out(); + await runProviderCommand({ action: 'list', verify: true }, d); + const text = io.out(); + expect(text).toContain('failed'); + expect(text).not.toContain(RAW_KEY); // validateProviderKey redacts the key in the detail + }); + + it('list --verify reports "no key" for a provider with no resolvable key — and never probes it (no hang)', async () => { + const stub = stubResolver(() => Promise.resolve({} as Awaited>)); + const noKeyResolver: ProviderResolver = { + resolveProvider: stub.resolveProvider, + keyFor: () => { + throw new Error('no key configured'); // keychain → env both empty + }, + }; + const d = deps({ resolver: noKeyResolver }); + await runProviderCommand({ action: 'add', name: 'openai' }, d); + io.out(); + await runProviderCommand({ action: 'list', verify: true }, d); + expect(io.out()).toContain('[no key]'); // reported without a probe (keyFor threw before generate) + }); + + it('list --json emits one key-free NDJSON record per provider with the verify state (2.5.G S11)', async () => { + const jsonGlobal = { + json: true, + color: false, + cwd: process.cwd(), + configPath: undefined, + verbosity: 'normal' as const, + }; + // Register two providers WITH keys via a non-json setup (their confirmations don't pollute the json capture). + const setup = deps({}); + await runProviderCommand({ action: 'set-key', name: 'anthropic' }, setup); + await runProviderCommand({ action: 'set-key', name: 'openai' }, setup); + const listIo = captureIo(); + await runProviderCommand( + { action: 'list', verify: true }, + deps({ io: listIo.io, global: jsonGlobal }), + ); + const records = parseNdjson(listIo.out()); + expect(records).toHaveLength(2); + const anthropic = records.find((r) => r['name'] === 'anthropic'); + expect(anthropic).toMatchObject({ keySet: true, verified: true, verifyDetail: null }); + expect(listIo.out()).not.toContain(RAW_KEY); // no key ever in the machine output + }); + + it('list --json without --verify leaves verified/verifyDetail null (no probe)', async () => { + const jsonGlobal = { + json: true, + color: false, + cwd: process.cwd(), + configPath: undefined, + verbosity: 'normal' as const, + }; + await runProviderCommand({ action: 'add', name: 'openai' }, deps({})); + const listIo = captureIo(); + await runProviderCommand({ action: 'list' }, deps({ io: listIo.io, global: jsonGlobal })); + const [rec] = parseNdjson(listIo.out()); + expect(rec).toMatchObject({ name: 'openai', keySet: false, verified: null, verifyDetail: null }); + }); + it('remove-key deletes the keychain entry and clears the db ref', async () => { const keychain = memKeychain(); const d = deps({ keychain }); diff --git a/apps/cli/src/commands/provider.ts b/apps/cli/src/commands/provider.ts index 695fe10d..518a7733 100644 --- a/apps/cli/src/commands/provider.ts +++ b/apps/cli/src/commands/provider.ts @@ -1,5 +1,5 @@ import { providerKind, ProviderIdSchema, type ProviderId } from '@relavium/llm'; -import type { ProviderStore } from '@relavium/db'; +import type { ProviderRecord, ProviderStore } from '@relavium/db'; import { isPrivateOrLocalHost, urlHasCredentials } from '@relavium/shared'; import { @@ -11,6 +11,9 @@ import { import { CliError } from '../process/errors.js'; import { EXIT_CODES, type ExitCode } from '../process/exit-codes.js'; import type { CliIo } from '../process/io.js'; +import type { GlobalOptions } from '../process/options.js'; +import { writeRecordLines } from '../render/records.js'; +import { stripTerminalControls } from '../render/tui/chat-projection.js'; import { KeychainUnavailableError, keychainAccount, @@ -38,6 +41,10 @@ export interface ProviderCommandArgs { /** `provider add --pricing-url ` (2.5.G S10, ADR-0065 §1) — override the seeded `pricing_reference_url` * (the public pricing page the user consults to hand-enter a price). A display-only pointer, never fetched. */ readonly pricingUrl?: string; + /** `provider list --verify` (2.5.G S11, ADR-0065 §6) — additionally run a bounded, key-redacted LIVE probe per + * registered provider (reusing `validateProviderKey`) and report its per-provider verification state. Opt-in + * because it makes a network request per key; absent ⇒ the fast offline listing (key-set status only). */ + readonly verify?: boolean; } export interface ProviderCommandDeps { @@ -47,6 +54,10 @@ export interface ProviderCommandDeps { readonly resolver: ProviderResolver; /** Read the API key (from stdin in production) — injected so tests supply a fixed key, never a real one. */ readonly readSecret: () => Promise; + /** The invocation's global options — read ONLY by `provider list` for the `--json` machine-output contract + * (2.5.G S11, ADR-0049). Optional so a non-`list` caller (the onboarding wizard's `set-key`) need not build one; + * absent ⇒ human output. */ + readonly global?: GlobalOptions; } /** Validate a provider name against the known `@relavium/llm` providers (`ProviderId`), or fail (exit 2). */ @@ -71,8 +82,72 @@ function requireName(args: ProviderCommandArgs): string { return args.name; } -function providerList(deps: ProviderCommandDeps): void { +/** One provider's live verification outcome (2.5.G S11). `verified: null` ⇒ NOT probed (no key resolvable, or + * `--verify` was not passed); `true`/`false` ⇒ the probe's result. `detail` is a SHORT, already-key-REDACTED + * reason on failure (from `validateProviderKey`), else `null`. */ +interface VerifyOutcome { + readonly verified: boolean | null; + readonly detail: string | null; +} + +/** Live-probe one registered provider's key via the bounded, key-redacted {@link validateProviderKey} seam (shared + * with `provider test` + `/doctor --deep`). A provider with NO resolvable key (keychain → env both empty) is + * reported `null` (not verifiable), never probed — so `--verify` never hangs on a keyless provider. */ +async function verifyProvider( + record: ProviderRecord, + deps: ProviderCommandDeps, +): Promise { + const parsed = ProviderIdSchema.safeParse(record.name); + if (!parsed.success) return { verified: false, detail: 'unknown provider' }; + const id = parsed.data; + const provider = deps.resolver.resolveProvider(id); + if (provider === undefined) return { verified: false, detail: 'no adapter' }; + let key: string; + try { + key = deps.resolver.keyFor(id); // keychain → env var → throws; never logged + } catch { + return { verified: null, detail: null }; // no key anywhere ⇒ nothing to verify + } + const result = await validateProviderKey(provider, key, KNOWN_PROVIDERS[id].testModel); + return { verified: result.ok, detail: result.ok ? null : result.detail }; +} + +/** Collapse a provider-supplied verification detail to one clean line before it reaches the TTY — strip ANSI/C0/C1 + * control bytes (a rogue provider error must not inject a cursor jump / `\r` line-overwrite) then squeeze + * whitespace, so one row stays one line. The stored/`--json` value is untouched (JSON escapes on its own). */ +function oneLine(text: string): string { + return stripTerminalControls(text).replace(/\s+/gu, ' ').trim(); +} + +async function providerList(args: ProviderCommandArgs, deps: ProviderCommandDeps): Promise { const providers = deps.store.list(); + // Probe each provider up front (only when --verify) so the human + --json branches share one outcome set and the + // output order is deterministic. Sequential over the ≤4 known providers, each bounded by validateProviderKey's + // own timeout — no unbounded hang. Absent --verify, no probe runs (the fast offline listing). + const outcomes = new Map(); + if (args.verify) { + for (const p of providers) outcomes.set(p.name, await verifyProvider(p, deps)); + } + + if (deps.global?.json === true) { + // ADR-0049 read-command NDJSON, key-free by construction (only the keychain-ref-derived `keySet` + the + // redacted verify state, NEVER the key). `verified`/`verifyDetail` are `null` unless --verify was passed. + writeRecordLines( + deps.io, + providers.map((p) => { + const outcome = outcomes.get(p.name); + return { + name: p.name, + baseUrl: p.baseUrl, + keySet: p.apiKeyKeychainRef !== undefined, + verified: outcome?.verified ?? null, + verifyDetail: outcome?.detail ?? null, + }; + }), + ); + return; + } + if (providers.length === 0) { deps.io.writeOut( 'No providers registered. Add a key with `relavium provider set-key `.\n', @@ -81,10 +156,23 @@ function providerList(deps: ProviderCommandDeps): void { } for (const p of providers) { // "key set" is derived from the stored keychain ref (no key read) — never echo the key here. - deps.io.writeOut(`${p.name}\t${p.baseUrl}\t[${p.apiKeyKeychainRef ? 'key set' : 'no key'}]\n`); + const status = statusColumn(p, args.verify === true, outcomes.get(p.name)); + deps.io.writeOut(`${p.name}\t${p.baseUrl}\t[${status}]\n`); } } +/** The bracketed status column of a `provider list` row: the offline key-set marker, or — under `--verify` — the + * live probe result (`verified` / `no key` / `failed — `). */ +function statusColumn( + record: ProviderRecord, + verify: boolean, + outcome: VerifyOutcome | undefined, +): string { + if (!verify) return record.apiKeyKeychainRef !== undefined ? 'key set' : 'no key'; + if (outcome === undefined || outcome.verified === null) return 'no key'; + return outcome.verified ? 'verified' : `failed — ${oneLine(outcome.detail ?? 'verification failed')}`; +} + function providerAdd(args: ProviderCommandArgs, deps: ProviderCommandDeps): void { const id = parseProviderId(requireName(args)); const meta = KNOWN_PROVIDERS[id]; @@ -240,7 +328,7 @@ export async function runProviderCommand( try { switch (args.action) { case 'list': - providerList(deps); + await providerList(args, deps); break; case 'add': providerAdd(args, deps); diff --git a/apps/cli/src/commands/specs-forwarding.test.ts b/apps/cli/src/commands/specs-forwarding.test.ts index 79d0892e..76211f41 100644 --- a/apps/cli/src/commands/specs-forwarding.test.ts +++ b/apps/cli/src/commands/specs-forwarding.test.ts @@ -64,6 +64,12 @@ describe('commander action → executeCommand forwarding (S10)', () => { }); }); + it('provider list forwards --verify (2.5.G S11)', () => { + const { id, input } = drive(['provider', 'list', '--verify']); + expect(id).toBe('provider.list'); + expect(input).toMatchObject({ positionals: [], options: { verify: true } }); + }); + it('models pricing forwards the model positional + all four options', () => { const { id, input } = drive([ 'models', diff --git a/apps/cli/src/commands/specs.ts b/apps/cli/src/commands/specs.ts index a8c30f41..0f8bd871 100644 --- a/apps/cli/src/commands/specs.ts +++ b/apps/cli/src/commands/specs.ts @@ -522,7 +522,8 @@ function registerProvider(program: Command, ctx?: CommandContext): void { .description('Manage providers and API keys in the OS keychain.'); const list = provider .command('list') - .description('List registered providers and whether a key is set.'); + .description('List registered providers and whether a key is set.') + .option('--verify', 'additionally run a live key-verification probe per provider'); const add = provider .command('add ') .description('Register a provider.') @@ -552,10 +553,10 @@ function registerProvider(program: Command, ctx?: CommandContext): void { return; } - list.action(async () => { + list.action(async (opts: { verify?: boolean }) => { ctx.result.exitCode = await executeCommand( 'provider.list', - { positionals: [], options: {} }, + { positionals: [], options: { verify: opts.verify } }, ctx, ); }); diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index 9d305c23..632fdccd 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -224,7 +224,7 @@ relavium models pricing my-custom-model --provider openai --input 3 --output 9 ### Read-command `--json` output -The non-streaming read commands (`list` / `status` / `gate list` / `chat-list` / `models`, and `logs`) keep the CLI to **one machine-output idiom**: `--json` emits **one result record per line** (NDJSON, `jq`-friendly, stdout-pure with diagnostics on stderr) — the same line-oriented shape `relavium run --json` uses for its `RunEvent` stream ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)). For `logs --json` the records ARE raw `RunEvent`s — the same `RunEvent` data the run streamed (re-serialized from the persisted log, so the field order may differ from the live `run --json` bytes); for the others they are the per-command result records documented above. An unknown `runId` (`logs` / `gate list`) is the structured pre-run fault on stderr with exit `2`, stdout empty — exactly as for `run`. (`chat-export --json` is **not** a read command — it emits a single `session:exported` **event**, not a result record, since the export is a session-lifecycle action.) +The non-streaming read commands (`list` / `status` / `gate list` / `chat-list` / `models` / `provider list`, and `logs`) keep the CLI to **one machine-output idiom**: `--json` emits **one result record per line** (NDJSON, `jq`-friendly, stdout-pure with diagnostics on stderr) — the same line-oriented shape `relavium run --json` uses for its `RunEvent` stream ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)). For `logs --json` the records ARE raw `RunEvent`s — the same `RunEvent` data the run streamed (re-serialized from the persisted log, so the field order may differ from the live `run --json` bytes); for the others they are the per-command result records documented above. An unknown `runId` (`logs` / `gate list`) is the structured pre-run fault on stderr with exit `2`, stdout empty — exactly as for `run`. (`chat-export --json` is **not** a read command — it emits a single `session:exported` **event**, not a result record, since the export is a session-lifecycle action.) ### `relavium gate` @@ -298,11 +298,14 @@ relavium provider add openai --pricing-url https://example.com/prices # overri echo "$ANTHROPIC_API_KEY" | relavium provider set-key anthropic # store a key (read from STDIN, never argv) relavium provider test anthropic # verify the key with a minimal live request relavium provider remove-key anthropic # delete the key from the keychain +relavium provider list --verify # + a live key-verification probe per provider ``` - **`set-key` reads the key from stdin**, never a CLI argument (argv leaks into `ps`, shell history, and CI logs); pipe it or use a heredoc. The key is stored in the OS keychain under the canonical entry-naming scheme ([keychain-and-secrets.md](../desktop/keychain-and-secrets.md#entry-naming)). +- **`list`** shows each registered provider, its base URL, and whether a key is set — a fast, **offline** read (no + key is read; the status is derived from the stored keychain ref). **`--verify`** (2.5.G S11, [ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) §6) additionally runs a **bounded, key-redacted live probe** per provider — the SAME `validateProviderKey` seam `provider test` + `/doctor --deep` use — and reports `verified` / `failed — ` / `no key` in the status column. A provider with **no resolvable key** (keychain → env both empty) is reported `no key` and **never probed** (so `--verify` never hangs on a keyless provider); the key is **never** echoed. `list` honors **`--json`** ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)): one key-free NDJSON record per provider — `{ name, baseUrl, keySet, verified, verifyDetail }`, where `verified` is `null` without `--verify` (else `true`/`false`) and `verifyDetail` is a short redacted failure reason (`null` otherwise). - **`add` / `set-key`** auto-register the provider row. `--base-url ` on `add` records a custom endpoint that **is now actually used at request routing** (2.5.G S9, [ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) §3–4 — the earlier "dead-config" gap is closed): the resolver rebinds that provider's adapter to the custom endpoint and routes **all** its egress (streaming `generate`/`stream` + the `models.list` refresh) through the shared **SSRF-validated** hop (`connectValidated` — HTTPS-only, no embedded credentials, every resolved IP range-blocked, connect pinned to the validated IP for DNS-rebinding safety). Custom endpoints are **OpenAI-compatible only** this round (`openai` / `deepseek`); a `--base-url` on `anthropic` / `gemini` is **refused** with a clear message (exit `2`), as is a non-HTTPS / private-loopback / credential-bearing URL (fail-fast at `add`). The provider-id set stays **closed** — a custom endpoint reuses the `openai` / `deepseek` id (ADR-0065 §6). - **`--pricing-url `** on `add` overrides the seeded `pricing_reference_url` — the public pricing page where you find a model's price to hand-enter via [`relavium models pricing`](#relavium-models) (2.5.G S10, ADR-0065 §1). Each known provider is seeded with its default pricing page; the `add` confirmation echoes it. It is a **display-only pointer**, **never fetched** (not an egress target), so — unlike `--base-url` — it needs no SSRF gate; it is validated as an HTTPS URL with no embedded credentials and stored normalized (control bytes percent-encoded, so it is terminal-safe). Omitting the flag on a re-`add` preserves a previously-set custom pointer. - **`test`** does a 1-token `generate` through `@relavium/llm`; `--model ` overrides the cheap default. A bad From 3ffe464e528c46cbe6e993cfd637a68764df9a92 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 07:29:30 +0300 Subject: [PATCH 31/68] =?UTF-8?q?fix(cli):=202.5.G=20S11=20Opus-review=20f?= =?UTF-8?q?ixes=20=E2=80=94=20sanitize=20base=5Furl=20echo=20+=20verify=20?= =?UTF-8?q?polish?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - MEDIUM: `base_url` is stored VERBATIM (requireHttpsUrl keeps the exact endpoint, not href-normalized), so a crafted/tampered value with a control byte injected on `provider list` + the `add` confirmation. My S10 comment claiming both URLs were href-safe was false for base_url. Sanitize both echoed URLs at the render boundary (sanitizeInline) + correct the comment. - LOW: `--json --verify` now distinguishes a probed-keyless provider (verified:null, verifyDetail:"no key") from an un-probed one (verifyDetail:null). - LOW: run the `--verify` probes CONCURRENTLY (Promise.all) so N providers cost one timeout, not N serialized; output order is unchanged (keyed by name). - LOW: make `ProviderCommandDeps.global` REQUIRED (matching the sibling read commands) so a future caller can't silently drop --json; the wizard's set-key passes a throwaway it never reads. - tests: --json --verify failure + keyless records, and the human-line terminal-sanitize (ANSI/CR neutered). Refs: ADR-0065 §6, ADR-0049 Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/commands/provider.test.ts | 58 ++++++++++++++++++++++++++ apps/cli/src/commands/provider.ts | 49 +++++++++++++--------- apps/cli/src/onboarding/wizard.ts | 3 ++ docs/reference/cli/commands.md | 2 +- 4 files changed, 92 insertions(+), 20 deletions(-) diff --git a/apps/cli/src/commands/provider.test.ts b/apps/cli/src/commands/provider.test.ts index 9b3afdaa..3f0926e9 100644 --- a/apps/cli/src/commands/provider.test.ts +++ b/apps/cli/src/commands/provider.test.ts @@ -179,6 +179,64 @@ describe('relavium provider commands (2.C)', () => { expect(rec).toMatchObject({ name: 'openai', keySet: false, verified: null, verifyDetail: null }); }); + const jsonGlobal = { + json: true, + color: false, + cwd: process.cwd(), + configPath: undefined, + verbosity: 'normal' as const, + }; + + it('list --json --verify records a probe FAILURE as { verified: false, verifyDetail: } (no key)', async () => { + await runProviderCommand({ action: 'add', name: 'openai' }, deps({})); + const listIo = captureIo(); + await runProviderCommand( + { action: 'list', verify: true }, + deps({ + io: listIo.io, + global: jsonGlobal, + resolver: stubResolver(() => Promise.reject(new Error(`boom ${RAW_KEY}`))), + }), + ); + const [rec] = parseNdjson(listIo.out()); + expect(rec).toMatchObject({ name: 'openai', verified: false }); + expect(typeof rec?.['verifyDetail']).toBe('string'); // a redacted reason is present + expect(listIo.out()).not.toContain(RAW_KEY); + }); + + it('list --json --verify distinguishes keyless (verified:null, verifyDetail:"no key") from not-probed', async () => { + const stub = stubResolver(() => Promise.resolve({} as Awaited>)); + const noKeyResolver: ProviderResolver = { + resolveProvider: stub.resolveProvider, + keyFor: () => { + throw new Error('no key'); + }, + }; + await runProviderCommand({ action: 'add', name: 'openai' }, deps({})); + const listIo = captureIo(); + await runProviderCommand( + { action: 'list', verify: true }, + deps({ io: listIo.io, global: jsonGlobal, resolver: noKeyResolver }), + ); + const [rec] = parseNdjson(listIo.out()); + // Distinct from the no-`--verify` record (which is verified:null, verifyDetail:null). + expect(rec).toMatchObject({ verified: null, verifyDetail: 'no key' }); + }); + + it('list --verify strips terminal-control bytes from a crafted provider error on the human line', async () => { + const d = deps({ + // A rogue provider error carrying an ANSI screen-clear (ESC) + a bare CR line-overwrite. + resolver: stubResolver(() => Promise.reject(new Error('bad \x1b[2J\r end'))), + }); + await runProviderCommand({ action: 'add', name: 'openai' }, d); + io.out(); + await runProviderCommand({ action: 'list', verify: true }, d); + const text = io.out(); + expect(text).not.toContain('\x1b'); // the ESC byte is stripped + expect(text).not.toContain('\r'); // the CR is collapsed (one row stays one line) + expect(text).toContain('failed'); // the row still renders the failure state + }); + it('remove-key deletes the keychain entry and clears the db ref', async () => { const keychain = memKeychain(); const d = deps({ keychain }); diff --git a/apps/cli/src/commands/provider.ts b/apps/cli/src/commands/provider.ts index 518a7733..c4c63fcd 100644 --- a/apps/cli/src/commands/provider.ts +++ b/apps/cli/src/commands/provider.ts @@ -13,7 +13,7 @@ import { EXIT_CODES, type ExitCode } from '../process/exit-codes.js'; import type { CliIo } from '../process/io.js'; import type { GlobalOptions } from '../process/options.js'; import { writeRecordLines } from '../render/records.js'; -import { stripTerminalControls } from '../render/tui/chat-projection.js'; +import { sanitizeInline, stripTerminalControls } from '../render/tui/chat-projection.js'; import { KeychainUnavailableError, keychainAccount, @@ -54,10 +54,10 @@ export interface ProviderCommandDeps { readonly resolver: ProviderResolver; /** Read the API key (from stdin in production) — injected so tests supply a fixed key, never a real one. */ readonly readSecret: () => Promise; - /** The invocation's global options — read ONLY by `provider list` for the `--json` machine-output contract - * (2.5.G S11, ADR-0049). Optional so a non-`list` caller (the onboarding wizard's `set-key`) need not build one; - * absent ⇒ human output. */ - readonly global?: GlobalOptions; + /** The invocation's global options — read by `provider list` for the `--json` machine-output contract (2.5.G S11, + * ADR-0049). Required (like the sibling read commands `list`/`status`/`models`) so a future caller can never + * silently drop `--json`; a non-`list` caller (the wizard's `set-key`) passes a throwaway it never reads. */ + readonly global: GlobalOptions; } /** Validate a provider name against the known `@relavium/llm` providers (`ProviderId`), or fail (exit 2). */ @@ -82,9 +82,11 @@ function requireName(args: ProviderCommandArgs): string { return args.name; } -/** One provider's live verification outcome (2.5.G S11). `verified: null` ⇒ NOT probed (no key resolvable, or - * `--verify` was not passed); `true`/`false` ⇒ the probe's result. `detail` is a SHORT, already-key-REDACTED - * reason on failure (from `validateProviderKey`), else `null`. */ +/** One provider's live verification outcome (2.5.G S11). `verified: null` ⇒ NOT verifiable (no key resolvable); + * `true`/`false` ⇒ the probe's result. `detail` is a SHORT reason: `'no key'` for the `null` case, an + * already-key-REDACTED failure reason (from `validateProviderKey`) for `false`, else `null`. So a `--json` + * consumer distinguishes "probed, no key" (`verified:null, verifyDetail:'no key'`) from "not probed" + * (`verified:null, verifyDetail:null`, the no-`--verify` record). */ interface VerifyOutcome { readonly verified: boolean | null; readonly detail: string | null; @@ -106,7 +108,7 @@ async function verifyProvider( try { key = deps.resolver.keyFor(id); // keychain → env var → throws; never logged } catch { - return { verified: null, detail: null }; // no key anywhere ⇒ nothing to verify + return { verified: null, detail: 'no key' }; // no key anywhere ⇒ nothing to verify (distinct from "not probed") } const result = await validateProviderKey(provider, key, KNOWN_PROVIDERS[id].testModel); return { verified: result.ok, detail: result.ok ? null : result.detail }; @@ -121,15 +123,19 @@ function oneLine(text: string): string { async function providerList(args: ProviderCommandArgs, deps: ProviderCommandDeps): Promise { const providers = deps.store.list(); - // Probe each provider up front (only when --verify) so the human + --json branches share one outcome set and the - // output order is deterministic. Sequential over the ≤4 known providers, each bounded by validateProviderKey's - // own timeout — no unbounded hang. Absent --verify, no probe runs (the fast offline listing). + // Probe each provider up front (only when --verify) so the human + --json branches share one outcome set. Run the + // probes CONCURRENTLY (each bounded by validateProviderKey's own timeout + AbortController) so the worst case is + // ONE timeout, not N serialized — the output order is still the deterministic `providers` order (the outcomes are + // keyed by the unique provider name, read back during rendering). Absent --verify, no probe runs. const outcomes = new Map(); if (args.verify) { - for (const p of providers) outcomes.set(p.name, await verifyProvider(p, deps)); + const probed = await Promise.all( + providers.map(async (p): Promise<[string, VerifyOutcome]> => [p.name, await verifyProvider(p, deps)]), + ); + for (const [name, outcome] of probed) outcomes.set(name, outcome); } - if (deps.global?.json === true) { + if (deps.global.json) { // ADR-0049 read-command NDJSON, key-free by construction (only the keychain-ref-derived `keySet` + the // redacted verify state, NEVER the key). `verified`/`verifyDetail` are `null` unless --verify was passed. writeRecordLines( @@ -155,9 +161,12 @@ async function providerList(args: ProviderCommandArgs, deps: ProviderCommandDeps return; } for (const p of providers) { - // "key set" is derived from the stored keychain ref (no key read) — never echo the key here. + // "key set" is derived from the stored keychain ref (no key read) — never echo the key here. The base URL is + // stored VERBATIM (requireHttpsUrl keeps the user's exact endpoint — NOT href-normalized), so a crafted / + // tampered value could carry a control byte: sanitize it at the render boundary (the provider name is a closed + // kebab `ProviderId`, safe raw). const status = statusColumn(p, args.verify === true, outcomes.get(p.name)); - deps.io.writeOut(`${p.name}\t${p.baseUrl}\t[${status}]\n`); + deps.io.writeOut(`${p.name}\t${sanitizeInline(p.baseUrl)}\t[${status}]\n`); } } @@ -209,10 +218,12 @@ function providerAdd(args: ProviderCommandArgs, deps: ProviderCommandDeps): void kind: providerKind(id), pricingReferenceUrl: pricingUrl, }); - // `record.baseUrl` / `record.pricingReferenceUrl` are validated HTTPS URLs (a custom value round-trips through - // `new URL().href`, so control bytes are percent-encoded) — terminal-safe to echo without a further sanitize. + // Sanitize the echoed URLs at the render boundary: `record.baseUrl` is stored VERBATIM (requireHttpsUrl keeps the + // user's exact endpoint, NOT href-normalized), so a control byte in a crafted/tampered value would otherwise + // reach the TTY. (`pricingReferenceUrl` IS href-normalized, but sanitize it too for defense-in-depth against a + // tampered at-rest row — ADR-0050.) deps.io.writeOut( - `Registered provider '${id}' (${record.baseUrl}). Store a key with \`relavium provider set-key ${id}\`. Find model prices at ${record.pricingReferenceUrl ?? pricingUrl} and set one with \`relavium models pricing\`.\n`, + `Registered provider '${id}' (${sanitizeInline(record.baseUrl)}). Store a key with \`relavium provider set-key ${id}\`. Find model prices at ${sanitizeInline(record.pricingReferenceUrl ?? pricingUrl)} and set one with \`relavium models pricing\`.\n`, ); } diff --git a/apps/cli/src/onboarding/wizard.ts b/apps/cli/src/onboarding/wizard.ts index 3538e688..691a9063 100644 --- a/apps/cli/src/onboarding/wizard.ts +++ b/apps/cli/src/onboarding/wizard.ts @@ -158,6 +158,9 @@ export async function runOnboardingWizard(deps: OnboardingDeps): Promise { keychain: deps.keychain, resolver: deps.resolver, readSecret: () => Promise.resolve(key), + // `global` is read only by `provider list --json`; `set-key` never touches it — a throwaway (the wizard is + // always interactive, never `--json`). + global: { json: false, color: false, cwd: process.cwd(), configPath: undefined, verbosity: 'normal' }, }, ); } catch (err) { diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index 632fdccd..5ab92995 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -305,7 +305,7 @@ relavium provider list --verify # + a live key-verificat logs); pipe it or use a heredoc. The key is stored in the OS keychain under the canonical entry-naming scheme ([keychain-and-secrets.md](../desktop/keychain-and-secrets.md#entry-naming)). - **`list`** shows each registered provider, its base URL, and whether a key is set — a fast, **offline** read (no - key is read; the status is derived from the stored keychain ref). **`--verify`** (2.5.G S11, [ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) §6) additionally runs a **bounded, key-redacted live probe** per provider — the SAME `validateProviderKey` seam `provider test` + `/doctor --deep` use — and reports `verified` / `failed — ` / `no key` in the status column. A provider with **no resolvable key** (keychain → env both empty) is reported `no key` and **never probed** (so `--verify` never hangs on a keyless provider); the key is **never** echoed. `list` honors **`--json`** ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)): one key-free NDJSON record per provider — `{ name, baseUrl, keySet, verified, verifyDetail }`, where `verified` is `null` without `--verify` (else `true`/`false`) and `verifyDetail` is a short redacted failure reason (`null` otherwise). + key is read; the status is derived from the stored keychain ref). **`--verify`** (2.5.G S11, [ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) §6) additionally runs a **bounded, key-redacted live probe** per provider — the SAME `validateProviderKey` seam `provider test` + `/doctor --deep` use — and reports `verified` / `failed — ` / `no key` in the status column. A provider with **no resolvable key** (keychain → env both empty) is reported `no key` and **never probed** (so `--verify` never hangs on a keyless provider); the key is **never** echoed. `list` honors **`--json`** ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)): one key-free NDJSON record per provider — `{ name, baseUrl, keySet, verified, verifyDetail }`, where `verified` is `null` without `--verify` (else `true`/`false`) and `verifyDetail` is a short reason — a **redacted** failure message, or `"no key"` when a probed provider has no resolvable key, else `null` (so a `verified: null` record with `verifyDetail: "no key"` is a probed-keyless provider, distinct from the un-probed `verifyDetail: null`). The `--verify` probes run **concurrently** (each timeout-bounded), so verifying N providers costs one timeout, not N. - **`add` / `set-key`** auto-register the provider row. `--base-url ` on `add` records a custom endpoint that **is now actually used at request routing** (2.5.G S9, [ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) §3–4 — the earlier "dead-config" gap is closed): the resolver rebinds that provider's adapter to the custom endpoint and routes **all** its egress (streaming `generate`/`stream` + the `models.list` refresh) through the shared **SSRF-validated** hop (`connectValidated` — HTTPS-only, no embedded credentials, every resolved IP range-blocked, connect pinned to the validated IP for DNS-rebinding safety). Custom endpoints are **OpenAI-compatible only** this round (`openai` / `deepseek`); a `--base-url` on `anthropic` / `gemini` is **refused** with a clear message (exit `2`), as is a non-HTTPS / private-loopback / credential-bearing URL (fail-fast at `add`). The provider-id set stays **closed** — a custom endpoint reuses the `openai` / `deepseek` id (ADR-0065 §6). - **`--pricing-url `** on `add` overrides the seeded `pricing_reference_url` — the public pricing page where you find a model's price to hand-enter via [`relavium models pricing`](#relavium-models) (2.5.G S10, ADR-0065 §1). Each known provider is seeded with its default pricing page; the `add` confirmation echoes it. It is a **display-only pointer**, **never fetched** (not an egress target), so — unlike `--base-url` — it needs no SSRF gate; it is validated as an HTTPS URL with no embedded credentials and stored normalized (control bytes percent-encoded, so it is terminal-safe). Omitting the flag on a re-`add` preserves a previously-set custom pointer. - **`test`** does a 1-token `generate` through `@relavium/llm`; `--model ` overrides the cheap default. A bad From 5a0ce7b744fa674b40ce9609c68348fd458faaa5 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 07:56:16 +0300 Subject: [PATCH 32/68] =?UTF-8?q?fix(cli):=202.5.G=20S11=20Sonnet-review?= =?UTF-8?q?=20fixes=20=E2=80=94=20source-clean=20URLs/details=20+=20verify?= =?UTF-8?q?=20robustness?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Close the terminal-injection / Trojan-Source gaps the prior pass left, at the SOURCE so both the human and --json surfaces are safe (not per-surface): - base_url: REJECT control + Unicode bidi/zero-width chars at `add` (requireHttpsUrl), so the verbatim-stored value is terminal-safe on `list`, `--json`, and the add confirmation — the reject message never re-echoes the raw. - probe detail (network-sourced): a `cleanDetail` helper strips control + bidi + bounds length, applied at the source in verifyProvider AND `provider test` (closing the pre-existing unsanitized `provider test` echo — same bug class). - add a bidi/zero-width strip the shared ASCII-only stripTerminalControls misses, scoped to these provider echoes (NOT the chat path, where RTL controls are valid). - verifyProvider: only keyFor's genuine `invalid_invocation` absence → "no key"; an UNEXPECTED fault (a native keychain crash) now PROPAGATES loudly instead of mislabeling every provider "no key". - tests: barrier-based concurrency proof (a sequential loop would deadlock), the control/bidi base_url reject, and the unexpected-keyFor-fault propagation. Refs: ADR-0065 §6 Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/commands/provider.test.ts | 60 +++++++++++++++++- apps/cli/src/commands/provider.ts | 86 +++++++++++++++++++------- docs/reference/cli/commands.md | 2 +- 3 files changed, 123 insertions(+), 25 deletions(-) diff --git a/apps/cli/src/commands/provider.test.ts b/apps/cli/src/commands/provider.test.ts index 3f0926e9..45eee32c 100644 --- a/apps/cli/src/commands/provider.test.ts +++ b/apps/cli/src/commands/provider.test.ts @@ -8,6 +8,7 @@ import { providerKeyEnvVar, type ProviderResolver, } from '../engine/providers.js'; +import { CliError } from '../process/errors.js'; import { KeychainUnavailableError, type KeychainStore } from '../secrets/keychain.js'; import { readSecretFromStdin } from '../secrets/read-secret.js'; import { captureIo, parseNdjson } from '../test-support.js'; @@ -130,7 +131,7 @@ describe('relavium provider commands (2.C)', () => { const noKeyResolver: ProviderResolver = { resolveProvider: stub.resolveProvider, keyFor: () => { - throw new Error('no key configured'); // keychain → env both empty + throw new CliError('invalid_invocation', 'no API key'); // matches real keyFor's absence throw }, }; const d = deps({ resolver: noKeyResolver }); @@ -209,7 +210,7 @@ describe('relavium provider commands (2.C)', () => { const noKeyResolver: ProviderResolver = { resolveProvider: stub.resolveProvider, keyFor: () => { - throw new Error('no key'); + throw new CliError('invalid_invocation', 'no API key'); }, }; await runProviderCommand({ action: 'add', name: 'openai' }, deps({})); @@ -237,6 +238,61 @@ describe('relavium provider commands (2.C)', () => { expect(text).toContain('failed'); // the row still renders the failure state }); + it('list --verify PROPAGATES an unexpected keyFor fault (not "no key") — a locked/faulted keychain is loud', async () => { + // A non-`invalid_invocation` error (e.g. a native keychain-binding fault) must NOT be mislabeled "no key" for + // every provider; it propagates as the command's fault. verifyProvider re-throws it → Promise.all rejects. + const stub = stubResolver(() => Promise.resolve({} as Awaited>)); + const faulted: ProviderResolver = { + resolveProvider: stub.resolveProvider, + keyFor: () => { + throw new Error('native keychain binding crashed'); + }, + }; + const d = deps({ resolver: faulted }); + await runProviderCommand({ action: 'add', name: 'openai' }, d); + await expect(runProviderCommand({ action: 'list', verify: true }, d)).rejects.toThrow( + 'native keychain binding crashed', + ); + }); + + it('list --verify probes providers CONCURRENTLY, not serially (a barrier deadlocks a sequential loop)', async () => { + // Two providers; each probe blocks until BOTH have started. A concurrent Promise.all releases the barrier; a + // sequential `await` loop would block forever on the first (the 2nd never starts) — so this test only passes + // when the probes run concurrently. A generous timeout keeps a genuine deadlock from hanging CI. + let started = 0; + let releaseBarrier: () => void = () => undefined; + const barrier = new Promise((res) => { + releaseBarrier = res; + }); + const concurrentGen: LlmProvider['generate'] = async () => { + started += 1; + if (started === 2) releaseBarrier(); + await barrier; + return {} as Awaited>; + }; + const d = deps({ resolver: stubResolver(concurrentGen) }); + await runProviderCommand({ action: 'set-key', name: 'anthropic' }, d); + await runProviderCommand({ action: 'set-key', name: 'openai' }, d); + io.out(); + await runProviderCommand({ action: 'list', verify: true }, d); + expect(started).toBe(2); // both probes ran (the barrier could only release if they overlapped) + expect(io.out()).toContain('[verified]'); + }, 2000); + + it('add --base-url REJECTS a control/bidi-bearing URL at the door (never stored or echoed)', async () => { + for (const evil of [ + 'https://x.example.com/\x1b[2Jpath', // ANSI screen-clear in the path + 'https://x.example.com/\u202eevil', // a bidi RIGHT-TO-LEFT OVERRIDE (U+202E) + 'https://x.example.com/a\r b', // a bare CR + ]) { + await expect( + runProviderCommand({ action: 'add', name: 'openai', baseUrl: evil }, deps({})), + ).rejects.toMatchObject({ code: 'invalid_invocation' }); + } + // Nothing was stored for openai (every attempt was rejected before the upsert). + expect(deps({}).store.get('openai')).toBeUndefined(); + }); + it('remove-key deletes the keychain entry and clears the db ref', async () => { const keychain = memKeychain(); const d = deps({ keychain }); diff --git a/apps/cli/src/commands/provider.ts b/apps/cli/src/commands/provider.ts index c4c63fcd..b3d7a966 100644 --- a/apps/cli/src/commands/provider.ts +++ b/apps/cli/src/commands/provider.ts @@ -8,12 +8,12 @@ import { validateProviderKey, type ProviderResolver, } from '../engine/providers.js'; -import { CliError } from '../process/errors.js'; +import { CliError, isCliError } from '../process/errors.js'; import { EXIT_CODES, type ExitCode } from '../process/exit-codes.js'; import type { CliIo } from '../process/io.js'; import type { GlobalOptions } from '../process/options.js'; import { writeRecordLines } from '../render/records.js'; -import { sanitizeInline, stripTerminalControls } from '../render/tui/chat-projection.js'; +import { stripTerminalControls } from '../render/tui/chat-projection.js'; import { KeychainUnavailableError, keychainAccount, @@ -107,18 +107,49 @@ async function verifyProvider( let key: string; try { key = deps.resolver.keyFor(id); // keychain → env var → throws; never logged - } catch { - return { verified: null, detail: 'no key' }; // no key anywhere ⇒ nothing to verify (distinct from "not probed") + } catch (err) { + // The ONE expected throw is keyFor's genuine key-absence (`invalid_invocation`) → report "no key" (distinct + // from "not probed"). Any OTHER error (e.g. a native keychain-binding fault keyFor re-raises) is unexpected — + // PROPAGATE it (a clean exit-2 fault for the whole command) rather than mislabel every provider "no key". + if (isCliError(err) && err.code === 'invalid_invocation') { + return { verified: null, detail: 'no key' }; + } + throw err; } const result = await validateProviderKey(provider, key, KNOWN_PROVIDERS[id].testModel); - return { verified: result.ok, detail: result.ok ? null : result.detail }; + // The failure detail is a REMOTE endpoint's error body — clean + bound it at the source so both the human row and + // the `--json` `verifyDetail` carry the safe value (the key is already redacted by `validateProviderKey`). + return { verified: result.ok, detail: result.ok ? null : cleanDetail(result.detail) }; +} + +/** The Unicode bidi-override + zero-width "Trojan Source" family the ASCII-only {@link stripTerminalControls} + * MISSES: LRE/RLE/PDF/LRO/RLO (U+202A–E), the isolates LRI/RLI/FSI/PDI (U+2066–9), the zero-width + directional + * marks (U+200B–F), the word-joiner (U+2060), and the BOM (U+FEFF). None has a legitimate use in a URL or a probe + * detail; left in, they visually reorder/hide text (a spoof), so they are stripped from any provider-supplied + * value echoed inline here. NOT added to the shared `stripTerminalControls` — that also renders chat bodies, where + * bidi controls ARE legitimate for RTL text. */ +const BIDI_ZERO_WIDTH = /[\u200b-\u200f\u2060\u2066-\u2069\u202a-\u202e\ufeff]/gu; + +/** C0/C1 control bytes + the {@link BIDI_ZERO_WIDTH} spoof family, as a NON-global tester (safe for `.test`) \u2014 none + * is valid in a base URL, so `requireHttpsUrl` REJECTS a raw containing any at `add` time, making the stored value + * inherently terminal-safe on every surface (list / `--json` / the add confirmation), not just after a render-strip. */ +// eslint-disable-next-line no-control-regex -- intentionally matches C0/C1 control bytes to reject them from a URL +const UNSAFE_URL_CHARS = /[\u0000-\u001f\u007f-\u009f\u200b-\u200f\u2060\u2066-\u2069\u202a-\u202e\ufeff]/u; + +/** Neutralize a provider-supplied string for inline echo: strip ANSI/C0/C1 control bytes ({@link stripTerminalControls}) + * AND the {@link BIDI_ZERO_WIDTH} spoof family, then squeeze whitespace so one row stays one line. Used for a stored + * base URL (defense-in-depth over the add-time reject) and — via {@link cleanDetail} — a network-sourced probe detail. */ +function stripInline(text: string): string { + return stripTerminalControls(text).replace(BIDI_ZERO_WIDTH, '').replace(/\s+/gu, ' ').trim(); } -/** Collapse a provider-supplied verification detail to one clean line before it reaches the TTY — strip ANSI/C0/C1 - * control bytes (a rogue provider error must not inject a cursor jump / `\r` line-overwrite) then squeeze - * whitespace, so one row stays one line. The stored/`--json` value is untouched (JSON escapes on its own). */ -function oneLine(text: string): string { - return stripTerminalControls(text).replace(/\s+/gu, ' ').trim(); +/** As {@link stripInline}, additionally length-BOUNDED — a probe `detail` comes from a remote endpoint's error body + * (unbounded), so cap it so one `--verify` row / one NDJSON line can never blow up a terminal or a line consumer. + * Cleaned AT THE SOURCE (in {@link verifyProvider} / {@link providerTest}) so BOTH the human and `--json` surfaces + * carry the safe value — the `--json` record is then safe without diverging from the raw-value NDJSON convention. */ +function cleanDetail(text: string): string { + const s = stripInline(text); + return s.length > 200 ? `${s.slice(0, 199)}…` : s; } async function providerList(args: ProviderCommandArgs, deps: ProviderCommandDeps): Promise { @@ -162,11 +193,11 @@ async function providerList(args: ProviderCommandArgs, deps: ProviderCommandDeps } for (const p of providers) { // "key set" is derived from the stored keychain ref (no key read) — never echo the key here. The base URL is - // stored VERBATIM (requireHttpsUrl keeps the user's exact endpoint — NOT href-normalized), so a crafted / - // tampered value could carry a control byte: sanitize it at the render boundary (the provider name is a closed - // kebab `ProviderId`, safe raw). + // stored VERBATIM (requireHttpsUrl keeps the user's exact endpoint — NOT href-normalized) and add-time REJECTS + // control/bidi, but a directly-tampered at-rest row (ADR-0050) could still carry one: strip it at the render + // boundary too (defense-in-depth; the provider name is a closed kebab `ProviderId`, safe raw). const status = statusColumn(p, args.verify === true, outcomes.get(p.name)); - deps.io.writeOut(`${p.name}\t${sanitizeInline(p.baseUrl)}\t[${status}]\n`); + deps.io.writeOut(`${p.name}\t${stripInline(p.baseUrl)}\t[${status}]\n`); } } @@ -179,7 +210,8 @@ function statusColumn( ): string { if (!verify) return record.apiKeyKeychainRef !== undefined ? 'key set' : 'no key'; if (outcome === undefined || outcome.verified === null) return 'no key'; - return outcome.verified ? 'verified' : `failed — ${oneLine(outcome.detail ?? 'verification failed')}`; + // `outcome.detail` is already cleaned + bounded at the source (verifyProvider → cleanDetail). + return outcome.verified ? 'verified' : `failed — ${outcome.detail ?? 'verification failed'}`; } function providerAdd(args: ProviderCommandArgs, deps: ProviderCommandDeps): void { @@ -218,12 +250,11 @@ function providerAdd(args: ProviderCommandArgs, deps: ProviderCommandDeps): void kind: providerKind(id), pricingReferenceUrl: pricingUrl, }); - // Sanitize the echoed URLs at the render boundary: `record.baseUrl` is stored VERBATIM (requireHttpsUrl keeps the - // user's exact endpoint, NOT href-normalized), so a control byte in a crafted/tampered value would otherwise - // reach the TTY. (`pricingReferenceUrl` IS href-normalized, but sanitize it too for defense-in-depth against a - // tampered at-rest row — ADR-0050.) + // Strip the echoed URLs at the render boundary (defense-in-depth): `record.baseUrl` is add-time control/bidi + // rejected + stored verbatim, `pricingReferenceUrl` is href-normalized — but a directly-tampered at-rest row + // (ADR-0050) could still carry a spoof byte, so neutralize both before the TTY. deps.io.writeOut( - `Registered provider '${id}' (${sanitizeInline(record.baseUrl)}). Store a key with \`relavium provider set-key ${id}\`. Find model prices at ${sanitizeInline(record.pricingReferenceUrl ?? pricingUrl)} and set one with \`relavium models pricing\`.\n`, + `Registered provider '${id}' (${stripInline(record.baseUrl)}). Store a key with \`relavium provider set-key ${id}\`. Find model prices at ${stripInline(record.pricingReferenceUrl ?? pricingUrl)} and set one with \`relavium models pricing\`.\n`, ); } @@ -268,10 +299,12 @@ async function providerTest(args: ProviderCommandArgs, deps: ProviderCommandDeps // The live ping + the defensive key-redaction live in `validateProviderKey` (the seam), shared with the // `/doctor --deep` probe so the secret-scrubbing has one tested home. const result = await validateProviderKey(provider, key, model); + // The detail is a REMOTE endpoint's message — clean + bound it before it reaches the TTY (a rogue endpoint must + // not inject a cursor jump / bidi spoof via the failure line or the success ping). The key is already redacted. if (!result.ok) { - throw new CliError('invalid_invocation', `${id}: ${result.detail}`); + throw new CliError('invalid_invocation', `${id}: ${cleanDetail(result.detail)}`); } - deps.io.writeOut(`${id}: ${result.detail}.\n`); + deps.io.writeOut(`${id}: ${cleanDetail(result.detail)}.\n`); } /** @@ -285,6 +318,15 @@ async function providerTest(args: ProviderCommandArgs, deps: ProviderCommandDeps * preserve the user's exact endpoint/trailing slash; the routing-time gate re-parses it via `new URL()` anyway. */ function requireHttpsUrl(raw: string): string { + // FIRST — reject a control/bidi-bearing raw before it is stored OR echoed (incl. in the error messages below, + // which interpolate `raw`). A legitimate base URL never contains these; rejecting here makes the verbatim-stored + // value inherently terminal-safe on every surface (2.5.G S11). The reject message omits `raw` (no re-injection). + if (UNSAFE_URL_CHARS.test(raw)) { + throw new CliError( + 'invalid_invocation', + 'base URL must not contain control or bidirectional characters.', + ); + } let url: URL; try { url = new URL(raw); diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index 5ab92995..20dcc31e 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -306,7 +306,7 @@ relavium provider list --verify # + a live key-verificat scheme ([keychain-and-secrets.md](../desktop/keychain-and-secrets.md#entry-naming)). - **`list`** shows each registered provider, its base URL, and whether a key is set — a fast, **offline** read (no key is read; the status is derived from the stored keychain ref). **`--verify`** (2.5.G S11, [ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) §6) additionally runs a **bounded, key-redacted live probe** per provider — the SAME `validateProviderKey` seam `provider test` + `/doctor --deep` use — and reports `verified` / `failed — ` / `no key` in the status column. A provider with **no resolvable key** (keychain → env both empty) is reported `no key` and **never probed** (so `--verify` never hangs on a keyless provider); the key is **never** echoed. `list` honors **`--json`** ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)): one key-free NDJSON record per provider — `{ name, baseUrl, keySet, verified, verifyDetail }`, where `verified` is `null` without `--verify` (else `true`/`false`) and `verifyDetail` is a short reason — a **redacted** failure message, or `"no key"` when a probed provider has no resolvable key, else `null` (so a `verified: null` record with `verifyDetail: "no key"` is a probed-keyless provider, distinct from the un-probed `verifyDetail: null`). The `--verify` probes run **concurrently** (each timeout-bounded), so verifying N providers costs one timeout, not N. -- **`add` / `set-key`** auto-register the provider row. `--base-url ` on `add` records a custom endpoint that **is now actually used at request routing** (2.5.G S9, [ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) §3–4 — the earlier "dead-config" gap is closed): the resolver rebinds that provider's adapter to the custom endpoint and routes **all** its egress (streaming `generate`/`stream` + the `models.list` refresh) through the shared **SSRF-validated** hop (`connectValidated` — HTTPS-only, no embedded credentials, every resolved IP range-blocked, connect pinned to the validated IP for DNS-rebinding safety). Custom endpoints are **OpenAI-compatible only** this round (`openai` / `deepseek`); a `--base-url` on `anthropic` / `gemini` is **refused** with a clear message (exit `2`), as is a non-HTTPS / private-loopback / credential-bearing URL (fail-fast at `add`). The provider-id set stays **closed** — a custom endpoint reuses the `openai` / `deepseek` id (ADR-0065 §6). +- **`add` / `set-key`** auto-register the provider row. `--base-url ` on `add` records a custom endpoint that **is now actually used at request routing** (2.5.G S9, [ADR-0065](../../decisions/0065-provider-economics-and-extensibility.md) §3–4 — the earlier "dead-config" gap is closed): the resolver rebinds that provider's adapter to the custom endpoint and routes **all** its egress (streaming `generate`/`stream` + the `models.list` refresh) through the shared **SSRF-validated** hop (`connectValidated` — HTTPS-only, no embedded credentials, every resolved IP range-blocked, connect pinned to the validated IP for DNS-rebinding safety). Custom endpoints are **OpenAI-compatible only** this round (`openai` / `deepseek`); a `--base-url` on `anthropic` / `gemini` is **refused** with a clear message (exit `2`), as is a non-HTTPS / private-loopback / credential-bearing URL, or one carrying **terminal-control / bidirectional characters** (fail-fast at `add`, so the stored value is terminal-safe on every surface). The provider-id set stays **closed** — a custom endpoint reuses the `openai` / `deepseek` id (ADR-0065 §6). - **`--pricing-url `** on `add` overrides the seeded `pricing_reference_url` — the public pricing page where you find a model's price to hand-enter via [`relavium models pricing`](#relavium-models) (2.5.G S10, ADR-0065 §1). Each known provider is seeded with its default pricing page; the `add` confirmation echoes it. It is a **display-only pointer**, **never fetched** (not an egress target), so — unlike `--base-url` — it needs no SSRF gate; it is validated as an HTTPS URL with no embedded credentials and stored normalized (control bytes percent-encoded, so it is terminal-safe). Omitting the flag on a re-`add` preserves a previously-set custom pointer. - **`test`** does a 1-token `generate` through `@relavium/llm`; `--model ` overrides the cheap default. A bad key fails cleanly (exit `2`) without echoing the key. From 3626e1428aa24f971bfd2c38ad1039c76a19d2e1 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 07:59:12 +0300 Subject: [PATCH 33/68] =?UTF-8?q?docs(runbooks):=202.5.G=20S12=20=E2=80=94?= =?UTF-8?q?=20add=20the=20CLI=20"add=20a=20provider"=20runbook?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The end-to-end CLI provider lifecycle: register (built-in or a custom OpenAI-compatible endpoint via --base-url), store + verify a key (list --verify), discover models (models refresh), and hand-price an unknown model (models pricing) so the cost cap enforces it. Task-oriented; links the canonical commands.md + ADR-0064/0065 rather than restating them, and points at the desktop add-a-provider-key.md for the OS-keychain key flow. Index updated. Refs: ADR-0064, ADR-0065 Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/runbooks/README.md | 1 + docs/runbooks/add-a-provider.md | 149 ++++++++++++++++++++++++++++++++ 2 files changed, 150 insertions(+) create mode 100644 docs/runbooks/add-a-provider.md diff --git a/docs/runbooks/README.md b/docs/runbooks/README.md index 019d825c..ac3b804f 100644 --- a/docs/runbooks/README.md +++ b/docs/runbooks/README.md @@ -14,6 +14,7 @@ specs and links to them rather than restating them (see | Runbook | Task | |---------|------| | [local-dev-setup.md](local-dev-setup.md) | Set up the Turborepo monorepo locally: pnpm, the Tauri/Rust toolchain, and the engine-first build order. | +| [add-a-provider.md](add-a-provider.md) | End-to-end CLI provider lifecycle: register (incl. a custom OpenAI-compatible endpoint), store + verify a key, discover models, and price an unknown model. | | [add-a-provider-key.md](add-a-provider-key.md) | Add or rotate an LLM provider API key into the OS keychain from the desktop app. | | [release-a-surface.md](release-a-surface.md) | Cut a release for the desktop (`.dmg`), CLI (npm), or VS Code extension (Marketplace). | diff --git a/docs/runbooks/add-a-provider.md b/docs/runbooks/add-a-provider.md new file mode 100644 index 00000000..5a9539de --- /dev/null +++ b/docs/runbooks/add-a-provider.md @@ -0,0 +1,149 @@ +# Add a Provider (CLI) + +> Last updated: 2026-07-06 + +This runbook is the end-to-end CLI procedure for making an LLM provider usable by +`relavium` — registering it, storing its key, verifying the key works, discovering its +models, and (for a model the shipped registry does not price) hand-entering a price so +the cost cap enforces it. It covers both a **built-in** provider (`anthropic`, `openai`, +`gemini`, `deepseek`) and a **custom OpenAI-compatible endpoint** reached through one of +those ids. + +It operationalizes the canonical command reference +[commands.md](../reference/cli/commands.md) and the two design ADRs — +[ADR-0064](../decisions/0064-live-model-catalog.md) (the live model catalog) and +[ADR-0065](../decisions/0065-provider-economics-and-extensibility.md) (provider economics +& extensibility) — and does not restate them. For the **desktop** key flow (and where +keys live per OS), see the sibling [add-a-provider-key.md](add-a-provider-key.md). + +Local-first Phase 1: **your machine calls the provider directly** — no Relavium account, +no cloud relay. Keys live in the **OS keychain**, never in plaintext, never in argv, never +in a log or a `--json` payload ([keychain-and-secrets.md](../reference/desktop/keychain-and-secrets.md)). + +## The lifecycle at a glance + +```bash +relavium provider add openai # 1. register (built-in defaults) +echo "$OPENAI_API_KEY" | relavium provider set-key openai # 2. store the key (STDIN, never argv) +relavium provider list --verify # 3. confirm the key works (live probe) +relavium models refresh # 4. discover the models the key can reach +relavium models # 5. list the cached catalog +``` + +Each step is idempotent; re-running `add`/`set-key` never resets a value you set earlier. + +## 1. Register the provider + +```bash +relavium provider add +``` + +`` is one of the closed set `anthropic | openai | gemini | deepseek` (the id set +is deliberately closed — see [ADR-0065](../decisions/0065-provider-economics-and-extensibility.md) §6). +This seeds the provider row with its default base URL + pricing page. `set-key` (step 2) +also auto-registers, so this explicit step is only needed when you want a **custom base +URL** or **pricing URL**: + +```bash +# A custom OpenAI-compatible endpoint (a proxy, a self-hosted gateway, a compatible vendor): +relavium provider add openai --base-url https://my-gateway.example.com/v1 +# Override where you look up prices to hand-enter them: +relavium provider add openai --pricing-url https://my-gateway.example.com/pricing +``` + +- `--base-url` is **OpenAI-compatible only** (`openai` / `deepseek`); on `anthropic` / + `gemini` it is refused (exit 2). All egress to a custom base URL — streaming turns **and** + the model-list refresh — goes through an **SSRF-validated** hop + ([ADR-0065](../decisions/0065-provider-economics-and-extensibility.md) §3–4): HTTPS-only, + no embedded credentials, no private/loopback/link-local host, and no terminal-control / + bidirectional characters. A custom endpoint **reuses** the `openai` / `deepseek` id, so it + cannot coexist with the real vendor under that id (a genuinely-separate custom id awaits a + future enum-opening ADR). +- `--pricing-url` is a **display-only pointer** (never fetched), so it may point anywhere + HTTPS; it is where you go to find a model's price for step 6. + +## 2. Store the key + +```bash +echo "$PROVIDER_API_KEY" | relavium provider set-key +``` + +The key is read from **stdin**, never a CLI flag (argv leaks into `ps`, shell history, and +CI logs). It is written to the OS keychain; the `llm_providers` row stores only a keychain +**ref**, and the command echoes only a hint (last 4 chars), never the key. + +**Headless / CI:** instead of the keychain, export `RELAVIUM__API_KEY` (e.g. +`RELAVIUM_OPENAI_API_KEY`) from a masked secret. Resolution is **keychain → env var → +error**. Never commit a key or print it in CI logs. + +## 3. Verify the key works + +```bash +relavium provider list --verify # a live, key-redacted probe per registered provider +relavium provider test # verify one provider (optionally --model ) +``` + +`list --verify` reports `verified` / `failed — ` / `no key` per provider +(the probes run concurrently, each timeout-bounded). A provider with no resolvable key is +reported `no key` and never probed. For a machine-readable result use +`relavium provider list --verify --json` — one key-free NDJSON record per provider +(`{ name, baseUrl, keySet, verified, verifyDetail }`). No key is ever echoed on any of these. + +## 4–5. Discover and list the models + +```bash +relavium models refresh # force a live re-fetch of each connected provider's model list +relavium models # list the cached catalog (auto-refreshes once on an empty cache) +``` + +`models refresh` is **per-provider isolated** — one provider's failure never fails the whole +command (it is reported `failed` / `skipped`, the others still refresh). The catalog is a +local cache of *which model ids each key can reach*; the shipped `MODEL_PRICING` registry is +the pricing authority for a known model. A model absent from your key's live list is dimmed +"not available on your key"; a deprecated model is flagged +([ADR-0064](../decisions/0064-live-model-catalog.md) §6). + +## 6. Price a model the registry does not know + +A **custom-endpoint model**, or a brand-new vendor model not yet in the shipped registry, +has **no price** — so the cost cap (`budget.max_cost_microcents` for a workflow, +`[chat].max_cost_microcents` for chat) would **degrade to "allow"** for it. Hand-enter its +price so the cap is enforced ([ADR-0065](../decisions/0065-provider-economics-and-extensibility.md) §1–2): + +```bash +relavium models pricing --provider --input --output [--cached ] +# e.g. a self-hosted model behind an openai-compatible gateway, $3 in / $9 out per million tokens: +relavium models pricing my-gateway-llama --provider openai --input 3 --output 9 +``` + +Prices are **USD per million tokens** (stored as integer micro-cents). The price is written +as a user row and a live `models refresh` **never** clobbers it; once set, the model is +enforced by the cost cap on `run`, a `run` resumed via `relavium gate`, `chat` / +`chat-resume`, the interactive Home, and one-shot `agent run`. Guards (each exit 2, nothing +written): a **built-in-priced** model is refused (the shipped price always wins); an +**unregistered provider** is refused (do step 1 first); and the **same model id already +priced under a different provider** is refused (the cost cap keys by model id, so it could +not tell them apart). Look up the real price at the provider's pricing page — the one +`--pricing-url` recorded in step 1 (`relavium provider list` shows it). + +## Rotate or remove a key + +```bash +echo "$NEW_KEY" | relavium provider set-key # rotate (overwrites the keychain entry in place) +relavium provider remove-key # delete the key from the keychain +``` + +`remove-key` clears the keychain entry + the row's ref; it leaves the provider row (base URL, +pricing URL, any user pricing) intact so re-adding a key restores the provider as configured. + +## Safety checklist + +- [ ] Key supplied only via **stdin** (`set-key`) or a masked env var — never a CLI flag, a + workflow/agent YAML file, or `~/.relavium/` config. +- [ ] No key value appears in `relavium provider list [--json]`, `--verify` output, `models` + output, or any error message (all are key-free by construction — a hint at most). +- [ ] A custom `--base-url` is HTTPS, public-host, credential-free (the CLI enforces this; a + private/loopback/tampered value is refused at `add`). +- [ ] In CI, the key comes from a masked secret, and `--json` is used for machine parsing. +- [ ] A user-entered price is verified against the provider's real pricing page before you + rely on the cost cap. From 013129098adfb42f04fe444cd79eefd9edd26f09 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 08:08:45 +0300 Subject: [PATCH 34/68] =?UTF-8?q?docs(runbooks):=202.5.G=20S12=20Opus-revi?= =?UTF-8?q?ew=20fixes=20=E2=80=94=20runbook=20accuracy?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - pricing-URL is echoed by the `provider add` confirmation, NOT `provider list` (which shows only name/baseUrl/status) — corrected. - the dim-unavailable / flag-deprecated UX is the Home `/models` picker, not the plain `relavium models` list — re-attributed + fixed the ADR-0064 citation (§7 deprecation / §10 picker, not §6). - add the omitted numeric-bounds `models pricing` guard (parity with commands.md). - scope the idempotence claim (`set-key` overwrites the key by design = rotate). - `--pricing-url` allows any HTTPS host but rejects embedded credentials. - clarify user pricing survives `remove-key` because it lives in `model_catalog`, not the `llm_providers` row. - link commands.md for the `--verify --json` record shape (one canonical home). Refs: ADR-0064, ADR-0065 Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/runbooks/add-a-provider.md | 34 +++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) diff --git a/docs/runbooks/add-a-provider.md b/docs/runbooks/add-a-provider.md index 5a9539de..816e392a 100644 --- a/docs/runbooks/add-a-provider.md +++ b/docs/runbooks/add-a-provider.md @@ -30,7 +30,9 @@ relavium models refresh # 4. discover the mode relavium models # 5. list the cached catalog ``` -Each step is idempotent; re-running `add`/`set-key` never resets a value you set earlier. +Re-running `add` never resets a base/pricing URL you set earlier, and `set-key` re-registers +without disturbing that config; `set-key` does overwrite the key itself — that is how you +rotate (see "Rotate or remove" below). ## 1. Register the provider @@ -59,8 +61,9 @@ relavium provider add openai --pricing-url https://my-gateway.example.com/pricin bidirectional characters. A custom endpoint **reuses** the `openai` / `deepseek` id, so it cannot coexist with the real vendor under that id (a genuinely-separate custom id awaits a future enum-opening ADR). -- `--pricing-url` is a **display-only pointer** (never fetched), so it may point anywhere - HTTPS; it is where you go to find a model's price for step 6. +- `--pricing-url` is a **display-only pointer** (never fetched), so it may point at any HTTPS + host with no embedded credentials (no SSRF host block); it is where you go to find a model's + price for step 6. ## 2. Store the key @@ -85,9 +88,9 @@ relavium provider test # verify one provider (optionally --model `list --verify` reports `verified` / `failed — ` / `no key` per provider (the probes run concurrently, each timeout-bounded). A provider with no resolvable key is -reported `no key` and never probed. For a machine-readable result use -`relavium provider list --verify --json` — one key-free NDJSON record per provider -(`{ name, baseUrl, keySet, verified, verifyDetail }`). No key is ever echoed on any of these. +reported `no key` and never probed. For a machine-readable result use `relavium provider list +--verify --json` — one key-free NDJSON record per provider (the record shape is documented in +[commands.md](../reference/cli/commands.md#relavium-provider)). No key is ever echoed on any of these. ## 4–5. Discover and list the models @@ -99,9 +102,10 @@ relavium models # list the cached catalog (auto-refreshes once on an `models refresh` is **per-provider isolated** — one provider's failure never fails the whole command (it is reported `failed` / `skipped`, the others still refresh). The catalog is a local cache of *which model ids each key can reach*; the shipped `MODEL_PRICING` registry is -the pricing authority for a known model. A model absent from your key's live list is dimmed -"not available on your key"; a deprecated model is flagged -([ADR-0064](../decisions/0064-live-model-catalog.md) §6). +the pricing authority for a known model. (The interactive Home's `/models` picker +additionally **dims** a model not available on your key and **flags** a deprecated one; the +plain `relavium models` list is ` ctx= []` — +[ADR-0064](../decisions/0064-live-model-catalog.md) §7/§10.) ## 6. Price a model the registry does not know @@ -121,10 +125,11 @@ as a user row and a live `models refresh` **never** clobbers it; once set, the m enforced by the cost cap on `run`, a `run` resumed via `relavium gate`, `chat` / `chat-resume`, the interactive Home, and one-shot `agent run`. Guards (each exit 2, nothing written): a **built-in-priced** model is refused (the shipped price always wins); an -**unregistered provider** is refused (do step 1 first); and the **same model id already -priced under a different provider** is refused (the cost cap keys by model id, so it could -not tell them apart). Look up the real price at the provider's pricing page — the one -`--pricing-url` recorded in step 1 (`relavium provider list` shows it). +**unregistered provider** is refused (do step 1 first); the **same model id already priced +under a different provider** is refused (the cost cap keys by model id, so it could not tell +them apart); and a **negative / non-finite / implausibly-large** price is refused. Look up the +real price at the provider's pricing page — the one `--pricing-url` recorded in step 1 (the +`provider add` confirmation echoes it). ## Rotate or remove a key @@ -134,7 +139,8 @@ relavium provider remove-key # delete the key from t ``` `remove-key` clears the keychain entry + the row's ref; it leaves the provider row (base URL, -pricing URL, any user pricing) intact so re-adding a key restores the provider as configured. +pricing URL) intact — and your user pricing, which lives in a separate `model_catalog` row it +never touches — so re-adding a key restores the provider as configured. ## Safety checklist From fcdd6c8af95e5d2e8c7a5c15d7cf013b56a6c789 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 08:17:14 +0300 Subject: [PATCH 35/68] =?UTF-8?q?docs(runbooks):=202.5.G=20S12=20Sonnet-re?= =?UTF-8?q?view=20fix=20=E2=80=94=20split=20add-time=20vs=20per-hop=20URL?= =?UTF-8?q?=20validation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The runbook attributed "no control/bidi characters" to the per-request SSRF-validated hop, but that reject is a ONE-TIME `provider add`-time check (requireHttpsUrl); the per-hop connectValidated enforces HTTPS + no-credentials + the private/loopback/link-local block (with DNS-rebinding protection). Split the sentence so an auditor reasoning about WHERE each protection lives is not misled. Refs: ADR-0065 §3–4 Co-Authored-By: Claude Opus 4.8 (1M context) --- docs/runbooks/add-a-provider.md | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/docs/runbooks/add-a-provider.md b/docs/runbooks/add-a-provider.md index 816e392a..8482a234 100644 --- a/docs/runbooks/add-a-provider.md +++ b/docs/runbooks/add-a-provider.md @@ -54,13 +54,14 @@ relavium provider add openai --pricing-url https://my-gateway.example.com/pricin ``` - `--base-url` is **OpenAI-compatible only** (`openai` / `deepseek`); on `anthropic` / - `gemini` it is refused (exit 2). All egress to a custom base URL — streaming turns **and** - the model-list refresh — goes through an **SSRF-validated** hop - ([ADR-0065](../decisions/0065-provider-economics-and-extensibility.md) §3–4): HTTPS-only, - no embedded credentials, no private/loopback/link-local host, and no terminal-control / - bidirectional characters. A custom endpoint **reuses** the `openai` / `deepseek` id, so it - cannot coexist with the real vendor under that id (a genuinely-separate custom id awaits a - future enum-opening ADR). + `gemini` it is refused (exit 2). The URL is validated at **`add` time** — HTTPS, no embedded + credentials, no terminal-control / bidirectional characters — so a clean value is what gets + stored. Then **every** egress to it — streaming turns **and** the model-list refresh — rides + a per-request **SSRF-validated** hop + ([ADR-0065](../decisions/0065-provider-economics-and-extensibility.md) §3–4) that re-checks + HTTPS + no-credentials and blocks a private/loopback/link-local host (with DNS-rebinding + protection). A custom endpoint **reuses** the `openai` / `deepseek` id, so it cannot coexist + with the real vendor under that id (a genuinely-separate custom id awaits a future enum-opening ADR). - `--pricing-url` is a **display-only pointer** (never fetched), so it may point at any HTTPS host with no embedded credentials (no SSRF host block); it is where you go to find a model's price for step 6. From 0fd9c056c51b1a50e69c25df0060a62b50ae8712 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 12:16:26 +0300 Subject: [PATCH 36/68] =?UTF-8?q?fix(cli,llm):=20/models=20key-awareness?= =?UTF-8?q?=20=E2=80=94=20dim=20+=20refuse=20a=20keyless=20provider's=20mo?= =?UTF-8?q?dels?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two reported bugs, one root cause: the picker had no notion of which providers have a key, so a keyless provider's models fell back to `available:true` (the §6 static-presence default) and were fully selectable — and selecting one wrote it as the default, so the next chat only failed with provider_auth. - llm: mergeModelCatalog gains an optional `keyedProviders` input; a model whose provider is NOT keyed is `available:false` + `unavailableReason:'no-key'`, regardless of live/static presence. REFINES (not reverses) ADR-0064 §6 — the static-presence safe default now applies only to a KEYED provider; absent `keyedProviders` ⇒ not key-gated (byte-identical to before). The pre-existing "not on your key" dim is now labeled `'not-on-key'`. - cli: `ProviderResolver.hasKey` (boolean, never the key; propagates a real keychain fault) + a `providerHasKey` helper (falls back to keyFor for stubs); `isProviderKeyless` reuses it. The Home picker probes the ≤4 known providers on open and passes `keyedProviders`; a keyless model is non-selectable (reuses the existing `blocked` path — so it can never be written as a default) with an ACTIONABLE hint naming the remedy (`no key for — provider add`). - The eager chat-build key pre-flight (Layer 2) is intentionally NOT added — it would change the keyless-chat behavior the maintainer deferred; Layer 1 (the picker gate) already closes both reported bugs. - docs: append-only ADR-0064 clarification (availability also requires a key). Refs: ADR-0064 §6 Co-Authored-By: Claude Opus 4.8 (1M context) --- .../cli/src/engine/model-catalog-view.test.ts | 16 ++++ apps/cli/src/engine/model-catalog-view.ts | 10 ++- apps/cli/src/engine/providers.test.ts | 26 ++++++ apps/cli/src/engine/providers.ts | 90 +++++++++++++------ apps/cli/src/home/drive-home.tsx | 8 ++ apps/cli/src/onboarding/wizard.ts | 14 ++- .../src/render/tui/home-controller.test.ts | 21 +++++ apps/cli/src/render/tui/home-controller.ts | 11 ++- apps/cli/src/render/tui/model-picker-view.tsx | 8 +- apps/cli/src/render/tui/model-picker.test.ts | 31 ++++++- apps/cli/src/render/tui/model-picker.ts | 18 +++- docs/decisions/0064-live-model-catalog.md | 2 + packages/llm/src/model-catalog.test.ts | 44 +++++++++ packages/llm/src/model-catalog.ts | 39 +++++++- 14 files changed, 292 insertions(+), 46 deletions(-) diff --git a/apps/cli/src/engine/model-catalog-view.test.ts b/apps/cli/src/engine/model-catalog-view.test.ts index 29db567c..6bd30754 100644 --- a/apps/cli/src/engine/model-catalog-view.test.ts +++ b/apps/cli/src/engine/model-catalog-view.test.ts @@ -46,6 +46,22 @@ describe('buildMergedCatalog', () => { expect(view.entries.every((e) => e.available)).toBe(true); }); + it('threads keyedProviders → a model of an UNKEYED provider is unavailable/no-key; a keyed one stays available (2.5.G)', () => { + // Only openai is keyed; anthropic's static models become no-key even with no live rows. + const view = buildMergedCatalog({ + rows: [], + providerSlug: slugResolver({}), + keyedProviders: new Set(['openai']), + now: 0, + }); + const anthropicEntry = view.entries.find((e) => e.provider === 'anthropic'); + expect(anthropicEntry?.available).toBe(false); + expect(anthropicEntry?.unavailableReason).toBe('no-key'); + const openaiEntry = view.entries.find((e) => e.provider === 'openai'); + expect(openaiEntry?.available).toBe(true); + expect(openaiEntry?.unavailableReason).toBeUndefined(); + }); + it('a live row makes its provider "have live data": present models stay available, absent ones are dimmed', () => { const view = buildMergedCatalog({ rows: [row({ modelId: MODEL_PRESENT, providerId: 'p-anthropic', source: 'live' })], diff --git a/apps/cli/src/engine/model-catalog-view.ts b/apps/cli/src/engine/model-catalog-view.ts index 6e5474b9..206d47fc 100644 --- a/apps/cli/src/engine/model-catalog-view.ts +++ b/apps/cli/src/engine/model-catalog-view.ts @@ -39,6 +39,9 @@ export interface BuildMergedCatalogInput { readonly rows: readonly ModelCatalogListing[]; /** Resolve an internal `llm_providers` UUID → its provider slug (e.g. `anthropic`) — `createProviderSlugResolver`. */ readonly providerSlug: (uuid: string) => string; + /** The providers with a resolvable key (2.5.G key-awareness) — passed straight to {@link mergeModelCatalog} so a + * keyless provider's models are dimmed `no-key` + non-selectable. Absent ⇒ not key-gated (unchanged). */ + readonly keyedProviders?: ReadonlySet; /** Current time (epoch-ms) for the deprecation check — passed in so the projection stays pure/testable. */ readonly now: number; } @@ -145,6 +148,11 @@ export function buildMergedCatalog(input: BuildMergedCatalogInput): MergedCatalo live.set(slug, list); } const userPricing = buildUserPricing({ rows: input.rows, providerSlug: input.providerSlug }); - const entries = mergeModelCatalog({ live, userPricing, now: input.now }); + const entries = mergeModelCatalog({ + live, + userPricing, + ...(input.keyedProviders === undefined ? {} : { keyedProviders: input.keyedProviders }), + now: input.now, + }); return { entries, refreshedAt }; } diff --git a/apps/cli/src/engine/providers.test.ts b/apps/cli/src/engine/providers.test.ts index 9b27488d..7d56d891 100644 --- a/apps/cli/src/engine/providers.test.ts +++ b/apps/cli/src/engine/providers.test.ts @@ -8,6 +8,7 @@ import { createProviderResolver, KNOWN_PROVIDERS, neededProviderIds, + providerHasKey, providerKeyEnvVar, validateProviderKey, } from './providers.js'; @@ -142,6 +143,31 @@ describe('providerKeyEnvVar', () => { }); }); +describe('resolver.hasKey / providerHasKey (2.5.G key-awareness)', () => { + it('createProviderResolver.hasKey is true for a provider with an env key, false otherwise (never throws)', () => { + // Env-only (no keychain) — a key for openai, nothing for the others. + const resolver = createProviderResolver({ RELAVIUM_OPENAI_API_KEY: TEST_KEY }); + expect(resolver.hasKey?.('openai')).toBe(true); + expect(resolver.hasKey?.('anthropic')).toBe(false); + expect(resolver.hasKey?.('gemini')).toBe(false); + // And keyFor stays consistent: resolves for the keyed provider, throws for a keyless one. + expect(resolver.keyFor('openai')).toBe(TEST_KEY); + expect(() => resolver.keyFor('anthropic')).toThrow(); + }); + + it('providerHasKey falls back to a keyFor probe when a stub omits hasKey', () => { + const keyed = { resolveProvider: () => undefined, keyFor: () => TEST_KEY }; + expect(providerHasKey(keyed, 'openai')).toBe(true); + const keyless = { + resolveProvider: () => undefined, + keyFor: () => { + throw new Error('no key'); + }, + }; + expect(providerHasKey(keyless, 'openai')).toBe(false); + }); +}); + // The shared redaction seam (used by `provider test` AND the `/doctor --deep` probe) — its security contract is // tested DIRECTLY here, not only through its two callers. describe('validateProviderKey', () => { diff --git a/apps/cli/src/engine/providers.ts b/apps/cli/src/engine/providers.ts index 1c6b3b7c..503fd2c8 100644 --- a/apps/cli/src/engine/providers.ts +++ b/apps/cli/src/engine/providers.ts @@ -31,6 +31,34 @@ import { createValidatedFetch, type FetchLike } from './validated-fetch.js'; export interface ProviderResolver { readonly resolveProvider: (id: ProviderId) => LlmProvider | undefined; readonly keyFor: (id: ProviderId) => string; + /** + * Whether a key for `id` is RESOLVABLE (keychain OR env) — a boolean, never the key value (2.5.G key-awareness). + * Used by the `/models` picker to gate a keyless provider's models and by `isProviderKeyless`. Only the genuine + * "no source" case returns `false`; a real keychain fault still PROPAGATES (it is not silently reported as "no + * key"), so a locked keychain is not misread as absence. OPTIONAL so a test stub can implement `keyFor` alone — + * consumers go through {@link providerHasKey}, which falls back to a `keyFor` probe when this is absent. The real + * {@link createProviderResolver} always provides it (the fault-preserving path), so production never falls back. + */ + readonly hasKey?: (id: ProviderId) => boolean; +} + +/** + * Whether `resolver` can resolve a key for `id` — a boolean, never the key. Prefers the resolver's own + * {@link ProviderResolver.hasKey} (the fault-preserving path production always provides); falls back to a + * `keyFor` try/probe only when a (test) stub omits it. Centralizes the "does this provider have a key" question + * so the picker key-gate + `isProviderKeyless` share one source (2.5.G key-awareness). + */ +export function providerHasKey( + resolver: Pick, + id: ProviderId, +): boolean { + if (resolver.hasKey !== undefined) return resolver.hasKey(id); + try { + resolver.keyFor(id); + return true; + } catch { + return false; // a stub with no key source for this provider + } } /** The env var holding a provider's API key — the headless per-invocation key source (CI / no-keychain). */ @@ -221,36 +249,48 @@ export function createProviderResolver( // provider's adapter to a validated per-provider endpoint here. const adapters: Record = { ...defaultProviders() }; applyCustomEndpoints(adapters, options); + // The ONE key-resolution path (keychain → env), returning `undefined` for genuine absence — shared by `keyFor` + // (which throws on absence) and `hasKey` (which returns a boolean), so the two never drift (2.5.G key-awareness). + const resolveKey = (id: ProviderId): string | undefined => { + // 1. OS keychain (the primary store, 2.C). Absent (`null`) → fall through to env; an *unavailable* backend + // (locked / no Secret Service) also falls through — the env var is the CLI's documented no-keychain path. + // A NON-KeychainUnavailableError (a native binding fault) PROPAGATES — it is not silent absence. + if (keychain !== undefined) { + let fromKeychain: string | null = null; + try { + fromKeychain = keychain.get(keychainAccount(id)); + } catch (err) { + if (!(err instanceof KeychainUnavailableError)) { + throw err; + } + } + if (fromKeychain !== null && fromKeychain !== '') { + return fromKeychain; + } + } + // 2. Env var — the headless / CI per-invocation source. + const fromEnv = env[providerKeyEnvVar(id)]; + if (fromEnv !== undefined && fromEnv !== '') { + return fromEnv; + } + return undefined; // 3. No source. + }; return { resolveProvider: (id) => adapters[id], keyFor: (id) => { - // 1. OS keychain (the primary store, 2.C). Absent (`null`) → fall through to env; an *unavailable* - // backend (locked / no Secret Service) also falls through — the env var is the CLI's documented - // no-keychain path. We never read/write a plaintext fallback. - if (keychain !== undefined) { - let fromKeychain: string | null = null; - try { - fromKeychain = keychain.get(keychainAccount(id)); - } catch (err) { - if (!(err instanceof KeychainUnavailableError)) { - throw err; - } - } - if (fromKeychain !== null && fromKeychain !== '') { - return fromKeychain; - } + const key = resolveKey(id); + if (key === undefined) { + // A clean invocation error naming both ways to provide the key (never the key itself). + throw new CliError( + 'invalid_invocation', + `no API key for provider '${id}' — store one with \`relavium provider set-key ${id}\` or set ${providerKeyEnvVar(id)}.`, + ); } - // 2. Env var — the headless / CI per-invocation source. - const fromEnv = env[providerKeyEnvVar(id)]; - if (fromEnv !== undefined && fromEnv !== '') { - return fromEnv; - } - // 3. No source — a clean invocation error naming both ways to provide the key (never the key itself). - throw new CliError( - 'invalid_invocation', - `no API key for provider '${id}' — store one with \`relavium provider set-key ${id}\` or set ${providerKeyEnvVar(id)}.`, - ); + return key; }, + // Boolean-only (never the key). A real keychain fault still propagates via `resolveKey` — only genuine absence + // is `false`, so a locked keychain is not misreported as "no key" here. + hasKey: (id) => resolveKey(id) !== undefined, }; } diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index c7eacb09..9c0de4a9 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -19,6 +19,7 @@ import { createProviderResolver, KNOWN_PROVIDERS, KNOWN_PROVIDER_IDS, + providerHasKey, type ProviderResolver, } from '../engine/providers.js'; import { openSessionStore, type OpenedSessionStore } from '../history/session-open.js'; @@ -196,9 +197,16 @@ export async function driveHome(deps: HomeDeps): Promise { // Rebuild the UUID→slug map on every load (NOT memoized once like the one-shot dispatch resolver): a refresh // may register a provider's FK row, and the next load must resolve its live rows' provider — not drop them. const slugByUuid = new Map(providerStore.list().map((p) => [p.id, p.name] as const)); + // The keyed providers (2.5.G key-awareness) — a keychain/env probe per known provider (≤4, on picker-open + // only, same discipline as `isProviderKeyless`), so the picker dims + refuses a model whose provider has + // no key (a `no-key` reason), rather than offering a model whose next chat would only fail `provider_auth`. + const keyedProviders = new Set( + KNOWN_PROVIDER_IDS.filter((id) => providerHasKey(providers, id)), + ); return buildMergedCatalog({ rows: catalogStore.listAll(), providerSlug: (uuid_) => slugByUuid.get(uuid_) ?? uuid_, + keyedProviders, now: now(), }); }, diff --git a/apps/cli/src/onboarding/wizard.ts b/apps/cli/src/onboarding/wizard.ts index 691a9063..bb9b6478 100644 --- a/apps/cli/src/onboarding/wizard.ts +++ b/apps/cli/src/onboarding/wizard.ts @@ -6,6 +6,7 @@ import { KNOWN_PROVIDERS, KNOWN_PROVIDER_IDS, keyHint, + providerHasKey, providerKeyEnvVar, type ProviderResolver, } from '../engine/providers.js'; @@ -93,15 +94,10 @@ export interface OnboardingDeps { * bare Home offers the wizard. A run with EITHER a keychain key or an env key is NOT key-less (no wizard) — so a * working env-key user is never nagged, and the env fallback IS the resolver's built-in key import. */ -export function isProviderKeyless(resolver: Pick): boolean { - return !KNOWN_PROVIDER_IDS.some((id) => { - try { - resolver.keyFor(id); - return true; - } catch { - return false; // no keychain key + no env var for this provider - } - }); +export function isProviderKeyless( + resolver: Pick, +): boolean { + return !KNOWN_PROVIDER_IDS.some((id) => providerHasKey(resolver, id)); } /** A clack `validate` that rejects an empty/whitespace key (Esc still cancels the whole flow). */ diff --git a/apps/cli/src/render/tui/home-controller.test.ts b/apps/cli/src/render/tui/home-controller.test.ts index f6c06395..51327d6b 100644 --- a/apps/cli/src/render/tui/home-controller.test.ts +++ b/apps/cli/src/render/tui/home-controller.test.ts @@ -1775,6 +1775,27 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => expect(c.getSnapshot().modelPicker?.hint).toBeUndefined(); }); + it('a `no-key` model is non-selectable with a hint NAMING the remedy (provider add) (2.5.G)', async () => { + const { port, writeDefault } = makeModelsPort({ + entries: [ + pickerEntry({ + modelId: 'x', + displayName: 'Model X', + provider: 'openai', + available: false, + unavailableReason: 'no-key', + }), + ], + }); + const c = openPicker(port); + await flush(); + c.handleKey('', ENTER); + expect(writeDefault).not.toHaveBeenCalled(); // a keyless model can never become the default + const hint = c.getSnapshot().modelPicker?.hint ?? ''; + expect(hint).toContain('openai'); // names the provider + expect(hint).toContain('provider add'); // and the actionable remedy + }); + it('Esc closes the picker without writing a default', async () => { const { port, writeDefault } = makeModelsPort(); const c = openPicker(port); diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index fedea7bf..8169a4a3 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -601,9 +601,16 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { case 'accept': acceptModel(step.modelId, step.displayName); break; - case 'blocked': - set({ modelPicker: { ...open, hint: `${step.displayName} is not available on your key — pick another` } }); + case 'blocked': { + // An ACTIONABLE hint (2.5.G key-awareness): a keyless provider names the remedy; the pre-existing + // "not on your key" case keeps its message. Never a write — a blocked model can't become the default. + const hint = + step.reason === 'no-key' + ? `${step.displayName} needs a ${step.provider} key — run \`relavium provider add ${step.provider}\`, then set-key` + : `${step.displayName} is not available on your key — pick another`; + set({ modelPicker: { ...open, hint } }); break; + } case 'refresh': runPickerRefresh(() => deps.models?.refresh() ?? Promise.resolve(undefined)); break; diff --git a/apps/cli/src/render/tui/model-picker-view.tsx b/apps/cli/src/render/tui/model-picker-view.tsx index 21642ef8..8f55ea91 100644 --- a/apps/cli/src/render/tui/model-picker-view.tsx +++ b/apps/cli/src/render/tui/model-picker-view.tsx @@ -85,7 +85,13 @@ export function ModelPickerView(props: Readonly): ReactEle entry.provider, ...(ctx.length > 0 ? [ctx] : []), formatModelPrice(entry), - ...(entry.available ? [] : ['unavailable on your key']), + // Distinguish the two unavailable reasons (2.5.G key-awareness): a keyless provider names the remedy, + // vs the pre-existing "not on your key" (a keyed provider whose live list omits the model). + ...(entry.available + ? [] + : entry.unavailableReason === 'no-key' + ? [`no key for ${entry.provider}`] + : ['unavailable on your key']), ...(entry.deprecated ? ['deprecated'] : []), ]; // Selected → cyan (highlight wins for visibility); else an unavailable/deprecated row is dimmed; else default. diff --git a/apps/cli/src/render/tui/model-picker.test.ts b/apps/cli/src/render/tui/model-picker.test.ts index f3daf0ae..f20b2928 100644 --- a/apps/cli/src/render/tui/model-picker.test.ts +++ b/apps/cli/src/render/tui/model-picker.test.ts @@ -77,12 +77,37 @@ describe('foldModelPickerKey', () => { }); }); - it('Enter on a DIMMED (unavailable) model is BLOCKED, never an accept (ADR §6)', () => { + it('Enter on a DIMMED (unavailable) model is BLOCKED, never an accept — carrying the provider (ADR §6)', () => { const s = state({ - entries: [entry({ modelId: 'x', displayName: 'Model X', available: false })], + entries: [entry({ modelId: 'x', displayName: 'Model X', available: false, provider: 'openai' })], selected: 0, }); - expect(foldModelPickerKey('', { return: true }, s)).toEqual({ kind: 'blocked', displayName: 'Model X' }); + expect(foldModelPickerKey('', { return: true }, s)).toEqual({ + kind: 'blocked', + displayName: 'Model X', + provider: 'openai', + }); + }); + + it('a `no-key` blocked step carries the reason (so the host hint can name the remedy) (2.5.G)', () => { + const s = state({ + entries: [ + entry({ + modelId: 'y', + displayName: 'Model Y', + provider: 'gemini', + available: false, + unavailableReason: 'no-key', + }), + ], + selected: 0, + }); + expect(foldModelPickerKey('', { return: true }, s)).toEqual({ + kind: 'blocked', + displayName: 'Model Y', + provider: 'gemini', + reason: 'no-key', + }); }); it('Enter on an EMPTY (over-filtered) list is a gentle close, not a crash', () => { diff --git a/apps/cli/src/render/tui/model-picker.ts b/apps/cli/src/render/tui/model-picker.ts index 822648cd..51e964b1 100644 --- a/apps/cli/src/render/tui/model-picker.ts +++ b/apps/cli/src/render/tui/model-picker.ts @@ -1,4 +1,4 @@ -import type { ModelCatalogEntry } from '@relavium/llm'; +import type { ModelCatalogEntry, ProviderId } from '@relavium/llm'; import { dropLastCodePoint } from './chat-input.js'; @@ -54,7 +54,12 @@ export interface ModelPickerKey { export type ModelPickerStep = | { readonly kind: 'close' } // Esc / Ctrl-C — cancel without writing a default | { readonly kind: 'accept'; readonly modelId: string; readonly displayName: string } // set the default - | { readonly kind: 'blocked'; readonly displayName: string } // a dimmed/unavailable model — non-selectable (ADR §6) + | { + readonly kind: 'blocked'; // a dimmed/unavailable model — non-selectable (ADR-0064 §6) + readonly displayName: string; + readonly provider: ProviderId; // so the host hint can name the remedy (`no key for `) + readonly reason?: 'no-key' | 'not-on-key'; // WHY it is unavailable (2.5.G key-awareness) + } | { readonly kind: 'refresh' } // Ctrl+R — force a live re-fetch of every connected provider | { readonly kind: 'state'; readonly state: ModelPickerState }; @@ -120,7 +125,14 @@ export function foldModelPickerKey( if (key.return === true) { const chosen = visible[clampSelection(state.selected, visible.length)]; if (chosen === undefined) return { kind: 'close' }; // an empty list — Enter is a gentle cancel - if (!chosen.available) return { kind: 'blocked', displayName: chosen.displayName }; + if (!chosen.available) { + return { + kind: 'blocked', + displayName: chosen.displayName, + provider: chosen.provider, + ...(chosen.unavailableReason !== undefined ? { reason: chosen.unavailableReason } : {}), + }; + } return { kind: 'accept', modelId: chosen.modelId, displayName: chosen.displayName }; } if (key.backspace === true || key.delete === true) { diff --git a/docs/decisions/0064-live-model-catalog.md b/docs/decisions/0064-live-model-catalog.md index cb9e284a..27270070 100644 --- a/docs/decisions/0064-live-model-catalog.md +++ b/docs/decisions/0064-live-model-catalog.md @@ -6,6 +6,8 @@ > **Amended 2026-07-05 by [ADR-0065](0065-provider-economics-and-extensibility.md)** (append-only — this body is unchanged): §6's merge precedence gains its **USER tier** — user-supplied per-model pricing fills **unknown** ids (`price = static ?? user`; static always wins for a known id), populating the merge helper's day-one optional user slot from the `model_catalog` `source='user'` rows. ADR-0065 also **injects** that merged pricing into the cost path (closing the unpriced-model cap gap) and wires custom OpenAI-compatible `base_url` endpoints over the shared SSRF floor. This ADR's static/live contract is unchanged; ADR-0065 extends it additively. +> **Clarified 2026-07-06 (2.5.G key-awareness — append-only, body unchanged):** §6 availability also requires a **resolvable key**. The merge gains an optional `keyedProviders` input (the providers with a keychain/env key); a model whose provider is **not** keyed is `available: false` with `unavailableReason: 'no-key'`, regardless of live/static presence — because with no key the model is genuinely uncallable (a chat started on it would only fail `provider_auth`), so the `/models` picker dims it (naming the remedy) and makes it **non-selectable**. This **refines, not reverses**, §6's "never everything unavailable" static-presence safe default: that default applies only to a **keyed** provider with no live data (never dimming a whole provider the user can actually use). `keyedProviders` is optional — **absent ⇒ availability is not key-gated** (byte-identical to the pre-clarification behavior), so only a key-resolving surface (the CLI Home) opts in. The pre-existing "not available on your key" dim (a keyed provider whose live list omits a static model) is now labeled `unavailableReason: 'not-on-key'`. + ## Context The model catalog is **static in-code**: `MODEL_PRICING` ([pricing.ts](../../packages/llm/src/pricing.ts)) diff --git a/packages/llm/src/model-catalog.test.ts b/packages/llm/src/model-catalog.test.ts index 03a30d3c..cb9ea730 100644 --- a/packages/llm/src/model-catalog.test.ts +++ b/packages/llm/src/model-catalog.test.ts @@ -296,3 +296,47 @@ describe('mergeModelCatalog (ADR-0064 §6)', () => { expect(JSON.stringify(MODEL_PRICING)).toBe(snapshot); }); }); + +describe('mergeModelCatalog — key-awareness (2.5.G, ADR-0064 §6 clarification)', () => { + it('marks a model of an UNKEYED provider unavailable with reason `no-key`, regardless of static presence', () => { + // anthropic is NOT in keyedProviders → all its static models are no-key (uncallable), even with no live data. + const entries = mergeModelCatalog({ + keyedProviders: new Set(['openai']), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + const opus = byId(entries, 'claude-opus-4-8'); // anthropic — unkeyed + expect(opus?.available).toBe(false); + expect(opus?.unavailableReason).toBe('no-key'); + const gpt = entries.find((e) => e.provider === 'openai'); // openai — keyed, no live data → static presence + expect(gpt?.available).toBe(true); + expect(gpt?.unavailableReason).toBeUndefined(); + }); + + it('PRESERVES the §6 static-presence safe default for a KEYED provider with no live data', () => { + const entries = mergeModelCatalog({ + keyedProviders: new Set(['anthropic']), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + const opus = byId(entries, 'claude-opus-4-8'); // keyed + no live data → available (unchanged) + expect(opus?.available).toBe(true); + expect(opus?.unavailableReason).toBeUndefined(); + }); + + it('a KEYED provider WITH live data still dims a static model absent from its list as `not-on-key`', () => { + const entries = mergeModelCatalog({ + keyedProviders: new Set(['anthropic']), + // anthropic has live data, but the list omits claude-opus-4-8 → not-on-key (the pre-existing dim). + live: liveMap([['anthropic', [{ id: 'claude-haiku-4-5' }]]]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + const opus = byId(entries, 'claude-opus-4-8'); + expect(opus?.available).toBe(false); + expect(opus?.unavailableReason).toBe('not-on-key'); + }); + + it('keyedProviders ABSENT ⇒ availability is not key-gated (byte-identical to pre-change)', () => { + const gated = mergeModelCatalog({ now: BEFORE_DEEPSEEK_DEPRECATION }); + // Every entry is available (no live data, no key gate) and carries no unavailableReason. + expect(gated.every((e) => e.available && e.unavailableReason === undefined)).toBe(true); + }); +}); diff --git a/packages/llm/src/model-catalog.ts b/packages/llm/src/model-catalog.ts index df05c3a4..35fd3a61 100644 --- a/packages/llm/src/model-catalog.ts +++ b/packages/llm/src/model-catalog.ts @@ -43,6 +43,13 @@ export interface ModelCatalogEntry { * provider's live list omits is likewise dimmed — its `pricing` still applies for cost governance. */ readonly available: boolean; + /** + * Why an entry is `available: false`, so the picker can show a distinct, actionable reason (2.5.G key-awareness): + * `'no-key'` — the provider has no resolvable key at all, so the model is genuinely unusable ("no key for + * `` — add one"); `'not-on-key'` — the provider IS keyed and has live data, but this model is not in + * that key's live list (the pre-existing "not available on your key" dim). Absent when `available: true`. + */ + readonly unavailableReason?: 'no-key' | 'not-on-key'; /** `true` once `now >= deprecatedAt` (ADR-0064 §7). The picker flags but never forbids a deprecated model. */ readonly deprecated: boolean; /** The effective ISO deprecation date — the earlier of the static and live dates (their union). */ @@ -62,6 +69,16 @@ export interface MergeModelCatalogInput { * always wins for a known id (ADR-0064 §6 / ADR-0065 §2), so a user cannot silently misprice a shipped model. */ readonly userPricing?: ReadonlyMap; + /** + * The providers with a **resolvable key** (keychain OR env) — 2.5.G key-awareness. A model whose provider is + * NOT in this set is `available: false` / `unavailableReason: 'no-key'` REGARDLESS of live/static presence: with + * no key the model is genuinely uncallable, so the picker must not offer it (and a chat started on it would only + * fail with `provider_auth`). ABSENT ⇒ availability is **not** key-gated (every provider treated as keyed) — the + * pre-key-gating behavior, preserved byte-identical for surfaces/tests that do not resolve keys. This REFINES + * (does not reverse) the ADR-0064 §6 "static presence" safe default: that default applies only to a **keyed** + * provider with no live data — never dimming a whole provider the user can actually use. + */ + readonly keyedProviders?: ReadonlySet; /** Current time (epoch ms) for the deprecation check — passed in so the merge stays pure and testable. */ readonly now: number; } @@ -127,8 +144,25 @@ export function mergeModelCatalog(input: MergeModelCatalogInput): ModelCatalogEn t.live?.contextWindowTokens ?? t.registry?.contextWindowTokens ?? t.user?.contextWindowTokens; const maxOutputTokens = t.live?.maxOutputTokens ?? t.registry?.maxOutputTokens ?? t.user?.maxOutputTokens; - // Availability: live-list membership when the provider has live data, else static presence. - const available = live.has(t.provider) ? t.live !== undefined : true; + // Availability (2.5.G key-awareness). Key gate FIRST: a provider absent from `keyedProviders` has no + // resolvable key, so its model is genuinely uncallable → unavailable with an actionable `'no-key'` reason, + // regardless of live/static presence. A KEYED provider keeps the pre-existing rule: live-list membership when + // it has live data (a static model absent from the list is `'not-on-key'`-dimmed), else static presence (the + // ADR-0064 §6 "never everything unavailable" safe default — PRESERVED, but now only for a KEYED provider). + // `keyedProviders` ABSENT ⇒ not key-gated (every provider treated as keyed) — byte-identical to pre-change. + const providerKeyed = + input.keyedProviders === undefined || input.keyedProviders.has(t.provider); + let available: boolean; + let unavailableReason: 'no-key' | 'not-on-key' | undefined; + if (!providerKeyed) { + available = false; + unavailableReason = 'no-key'; + } else if (live.has(t.provider)) { + available = t.live !== undefined; + if (!available) unavailableReason = 'not-on-key'; + } else { + available = true; + } const deprecatedAt = earlierIsoDate( earlierIsoDate(t.registry?.deprecatedAt, t.live?.deprecatedAt), t.user?.deprecatedAt, @@ -146,6 +180,7 @@ export function mergeModelCatalog(input: MergeModelCatalogInput): ModelCatalogEn pricingSource, priceKnown: pricingSource !== 'none', available, + ...(unavailableReason !== undefined ? { unavailableReason } : {}), deprecated, ...(deprecatedAt !== undefined ? { deprecatedAt } : {}), }); From 7f68b80515b7d6fcaff8b2836335c4a7af855a57 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 12:28:16 +0300 Subject: [PATCH 37/68] =?UTF-8?q?fix(cli,llm):=20Step=20A=20Opus-review=20?= =?UTF-8?q?fixes=20=E2=80=94=20hasKey=20comment=20accuracy=20+=20wording?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - MEDIUM: the `hasKey` doc/impl comments wrongly claimed a LOCKED keychain propagates; it is a KeychainUnavailableError that (like keyFor) falls through to env and is treated as absence — only a non-KeychainUnavailableError native fault propagates, and the production store wraps all native errors into KeychainUnavailableError, so nothing propagates in practice. Reworded both. - LOW: "byte-identical" was imprecise — with keyedProviders ABSENT the `available` BOOLEAN is unchanged, but the additive `unavailableReason` ('not-on-key' on a live-omitted static model) is new. Reworded the merge comment + ADR-0064 note + added a merge test covering it. - LOW: fixed the "a openai key" article grammar in the no-key hint. Refs: ADR-0064 §6 Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/engine/providers.ts | 19 ++++++++++++------- apps/cli/src/render/tui/home-controller.ts | 2 +- docs/decisions/0064-live-model-catalog.md | 2 +- packages/llm/src/model-catalog.test.ts | 13 ++++++++++++- packages/llm/src/model-catalog.ts | 4 +++- 5 files changed, 29 insertions(+), 11 deletions(-) diff --git a/apps/cli/src/engine/providers.ts b/apps/cli/src/engine/providers.ts index 503fd2c8..a75ad242 100644 --- a/apps/cli/src/engine/providers.ts +++ b/apps/cli/src/engine/providers.ts @@ -33,11 +33,14 @@ export interface ProviderResolver { readonly keyFor: (id: ProviderId) => string; /** * Whether a key for `id` is RESOLVABLE (keychain OR env) — a boolean, never the key value (2.5.G key-awareness). - * Used by the `/models` picker to gate a keyless provider's models and by `isProviderKeyless`. Only the genuine - * "no source" case returns `false`; a real keychain fault still PROPAGATES (it is not silently reported as "no - * key"), so a locked keychain is not misread as absence. OPTIONAL so a test stub can implement `keyFor` alone — - * consumers go through {@link providerHasKey}, which falls back to a `keyFor` probe when this is absent. The real - * {@link createProviderResolver} always provides it (the fault-preserving path), so production never falls back. + * Used by the `/models` picker to gate a keyless provider's models and by `isProviderKeyless`. Identical + * resolution to {@link keyFor}: a keychain key OR the env fallback ⇒ `true`, else `false`. A **locked / + * unavailable** keychain is a `KeychainUnavailableError` that (like `keyFor`) falls through to env and is treated + * as absence — so with no env key it returns `false` (correct: the provider genuinely can't be called). Only a + * NON-`KeychainUnavailableError` native binding fault propagates — but the production `createOsKeychainStore` + * wraps every native error into `KeychainUnavailableError`, so in practice nothing propagates. OPTIONAL so a test + * stub can implement `keyFor` alone — consumers go through {@link providerHasKey}, which falls back to a `keyFor` + * probe when this is absent; the real {@link createProviderResolver} always provides it, so production never falls back. */ readonly hasKey?: (id: ProviderId) => boolean; } @@ -288,8 +291,10 @@ export function createProviderResolver( } return key; }, - // Boolean-only (never the key). A real keychain fault still propagates via `resolveKey` — only genuine absence - // is `false`, so a locked keychain is not misreported as "no key" here. + // Boolean-only (never the key). Same resolution as `keyFor`: a locked/unavailable keychain (a + // `KeychainUnavailableError`) falls through to env, so with no env key it is `false` (correct — uncallable); + // only a non-`KeychainUnavailableError` native fault propagates via `resolveKey` (the production store never + // raises one, so nothing propagates in practice). hasKey: (id) => resolveKey(id) !== undefined, }; } diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index 8169a4a3..c2fda427 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -606,7 +606,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { // "not on your key" case keeps its message. Never a write — a blocked model can't become the default. const hint = step.reason === 'no-key' - ? `${step.displayName} needs a ${step.provider} key — run \`relavium provider add ${step.provider}\`, then set-key` + ? `${step.displayName}: no key for ${step.provider} — run \`relavium provider add ${step.provider}\`, then set-key` : `${step.displayName} is not available on your key — pick another`; set({ modelPicker: { ...open, hint } }); break; diff --git a/docs/decisions/0064-live-model-catalog.md b/docs/decisions/0064-live-model-catalog.md index 27270070..ff3f24ad 100644 --- a/docs/decisions/0064-live-model-catalog.md +++ b/docs/decisions/0064-live-model-catalog.md @@ -6,7 +6,7 @@ > **Amended 2026-07-05 by [ADR-0065](0065-provider-economics-and-extensibility.md)** (append-only — this body is unchanged): §6's merge precedence gains its **USER tier** — user-supplied per-model pricing fills **unknown** ids (`price = static ?? user`; static always wins for a known id), populating the merge helper's day-one optional user slot from the `model_catalog` `source='user'` rows. ADR-0065 also **injects** that merged pricing into the cost path (closing the unpriced-model cap gap) and wires custom OpenAI-compatible `base_url` endpoints over the shared SSRF floor. This ADR's static/live contract is unchanged; ADR-0065 extends it additively. -> **Clarified 2026-07-06 (2.5.G key-awareness — append-only, body unchanged):** §6 availability also requires a **resolvable key**. The merge gains an optional `keyedProviders` input (the providers with a keychain/env key); a model whose provider is **not** keyed is `available: false` with `unavailableReason: 'no-key'`, regardless of live/static presence — because with no key the model is genuinely uncallable (a chat started on it would only fail `provider_auth`), so the `/models` picker dims it (naming the remedy) and makes it **non-selectable**. This **refines, not reverses**, §6's "never everything unavailable" static-presence safe default: that default applies only to a **keyed** provider with no live data (never dimming a whole provider the user can actually use). `keyedProviders` is optional — **absent ⇒ availability is not key-gated** (byte-identical to the pre-clarification behavior), so only a key-resolving surface (the CLI Home) opts in. The pre-existing "not available on your key" dim (a keyed provider whose live list omits a static model) is now labeled `unavailableReason: 'not-on-key'`. +> **Clarified 2026-07-06 (2.5.G key-awareness — append-only, body unchanged):** §6 availability also requires a **resolvable key**. The merge gains an optional `keyedProviders` input (the providers with a keychain/env key); a model whose provider is **not** keyed is `available: false` with `unavailableReason: 'no-key'`, regardless of live/static presence — because with no key the model is genuinely uncallable (a chat started on it would only fail `provider_auth`), so the `/models` picker dims it (naming the remedy) and makes it **non-selectable**. This **refines, not reverses**, §6's "never everything unavailable" static-presence safe default: that default applies only to a **keyed** provider with no live data (never dimming a whole provider the user can actually use). `keyedProviders` is optional — **absent ⇒ availability is not key-gated**: the `available` BOOLEAN is unchanged from the pre-clarification behavior (the only new output is the additive-optional `unavailableReason`), so only a key-resolving surface (the CLI Home) opts in. The pre-existing "not available on your key" dim (a keyed provider whose live list omits a static model) is now labeled `unavailableReason: 'not-on-key'`. ## Context diff --git a/packages/llm/src/model-catalog.test.ts b/packages/llm/src/model-catalog.test.ts index cb9ea730..a5d341a3 100644 --- a/packages/llm/src/model-catalog.test.ts +++ b/packages/llm/src/model-catalog.test.ts @@ -334,9 +334,20 @@ describe('mergeModelCatalog — key-awareness (2.5.G, ADR-0064 §6 clarification expect(opus?.unavailableReason).toBe('not-on-key'); }); - it('keyedProviders ABSENT ⇒ availability is not key-gated (byte-identical to pre-change)', () => { + it('keyedProviders ABSENT ⇒ availability is not key-gated (the `available` boolean is unchanged)', () => { const gated = mergeModelCatalog({ now: BEFORE_DEEPSEEK_DEPRECATION }); // Every entry is available (no live data, no key gate) and carries no unavailableReason. expect(gated.every((e) => e.available && e.unavailableReason === undefined)).toBe(true); }); + + it('keyedProviders ABSENT + live data still labels a live-omitted static model `not-on-key` (additive field)', () => { + // The `available` boolean is unchanged from pre-change; only the additive `unavailableReason` is new here. + const entries = mergeModelCatalog({ + live: liveMap([['anthropic', [{ id: 'claude-haiku-4-5' }]]]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + const opus = byId(entries, 'claude-opus-4-8'); + expect(opus?.available).toBe(false); // unchanged + expect(opus?.unavailableReason).toBe('not-on-key'); // additive-optional reason + }); }); diff --git a/packages/llm/src/model-catalog.ts b/packages/llm/src/model-catalog.ts index 35fd3a61..bfb697b5 100644 --- a/packages/llm/src/model-catalog.ts +++ b/packages/llm/src/model-catalog.ts @@ -149,7 +149,9 @@ export function mergeModelCatalog(input: MergeModelCatalogInput): ModelCatalogEn // regardless of live/static presence. A KEYED provider keeps the pre-existing rule: live-list membership when // it has live data (a static model absent from the list is `'not-on-key'`-dimmed), else static presence (the // ADR-0064 §6 "never everything unavailable" safe default — PRESERVED, but now only for a KEYED provider). - // `keyedProviders` ABSENT ⇒ not key-gated (every provider treated as keyed) — byte-identical to pre-change. + // `keyedProviders` ABSENT ⇒ not key-gated (every provider treated as keyed): the `available` BOOLEAN is + // unchanged from pre-change; the only new output is the additive-optional `unavailableReason` (`'not-on-key'` + // on a live-omitted static model) — informational, and the sole live-data-passing caller passes keyedProviders. const providerKeyed = input.keyedProviders === undefined || input.keyedProviders.has(t.provider); let available: boolean; From 0c2f6b71860bcf319c3c7ec4300c36d175cc5eab Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 12:41:32 +0300 Subject: [PATCH 38/68] =?UTF-8?q?fix(cli):=20Step=20A=20Sonnet-review=20fi?= =?UTF-8?q?xes=20=E2=80=94=20memoize=20the=20key=20probe=20+=20hint=20poli?= =?UTF-8?q?sh?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - MEDIUM: the keyed-provider keychain probe ran on EVERY /models open + every refresh completion — up to 4 SYNCHRONOUS native N-API reads on the live ink render thread (and a possible OS ACL prompt) per interaction. Memoize it: computed lazily on the first load (after the wizard stores any first-run key), then reused — the resolvable key set is fixed for the Home process, matching the one-time startup `isProviderKeyless` posture. - LOW: the no-key hint now says `provider set-key ` (which auto-registers) — the single-command form every other "no key" message uses, not add+set-key. - LOW: add a providerHasKey test for the branch-1 (hasKey present) fault propagation — documents the intentional asymmetry vs the keyFor-fallback catch. Refs: ADR-0064 §6 Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/engine/providers.test.ts | 13 +++++++++++++ apps/cli/src/home/drive-home.tsx | 15 +++++++++------ apps/cli/src/render/tui/home-controller.test.ts | 2 +- apps/cli/src/render/tui/home-controller.ts | 4 +++- 4 files changed, 26 insertions(+), 8 deletions(-) diff --git a/apps/cli/src/engine/providers.test.ts b/apps/cli/src/engine/providers.test.ts index 7d56d891..fd919831 100644 --- a/apps/cli/src/engine/providers.test.ts +++ b/apps/cli/src/engine/providers.test.ts @@ -166,6 +166,19 @@ describe('resolver.hasKey / providerHasKey (2.5.G key-awareness)', () => { }; expect(providerHasKey(keyless, 'openai')).toBe(false); }); + + it('providerHasKey uses hasKey DIRECTLY when present — an unexpected hasKey fault PROPAGATES (not swallowed)', () => { + // The one asymmetry: branch (1) [hasKey present] does NOT catch, so a native fault surfaces (fail-loud); only + // branch (2) [keyFor fallback] swallows. Documents the intentional divergence. + const faulted = { + resolveProvider: () => undefined, + keyFor: () => TEST_KEY, // would say "true" if the fallback were (wrongly) used + hasKey: () => { + throw new Error('native keychain binding fault'); + }, + }; + expect(() => providerHasKey(faulted, 'openai')).toThrow('native keychain binding fault'); + }); }); // The shared redaction seam (used by `provider test` AND the `/doctor --deep` probe) — its security contract is diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index 9c0de4a9..e8549a11 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -192,17 +192,20 @@ export async function driveHome(deps: HomeDeps): Promise { return undefined; // a mid-session malformed config must not crash the picker } }; + // The keyed providers (2.5.G key-awareness) — a keychain/env probe per known provider (≤4) so the picker dims + // + refuses a model whose provider has no key (a `no-key` reason) rather than offering one whose next chat would + // only fail `provider_auth`. Memoized: a keychain read is a SYNCHRONOUS native N-API call (and can pop an OS ACL + // prompt on first access), so we must not repeat it on every `/models` open + every refresh completion on the + // live ink render thread. Computed LAZILY on the first `load` (after the wizard has stored any first-run key), + // then reused — the resolvable key set is fixed for the Home process (a mid-session key change ⇒ restart), the + // same one-time posture as the startup `isProviderKeyless` probe. + let keyedProviders: ReadonlySet<(typeof KNOWN_PROVIDER_IDS)[number]> | undefined; const models: HomeModelsPort = { load: () => { // Rebuild the UUID→slug map on every load (NOT memoized once like the one-shot dispatch resolver): a refresh // may register a provider's FK row, and the next load must resolve its live rows' provider — not drop them. const slugByUuid = new Map(providerStore.list().map((p) => [p.id, p.name] as const)); - // The keyed providers (2.5.G key-awareness) — a keychain/env probe per known provider (≤4, on picker-open - // only, same discipline as `isProviderKeyless`), so the picker dims + refuses a model whose provider has - // no key (a `no-key` reason), rather than offering a model whose next chat would only fail `provider_auth`. - const keyedProviders = new Set( - KNOWN_PROVIDER_IDS.filter((id) => providerHasKey(providers, id)), - ); + keyedProviders ??= new Set(KNOWN_PROVIDER_IDS.filter((id) => providerHasKey(providers, id))); return buildMergedCatalog({ rows: catalogStore.listAll(), providerSlug: (uuid_) => slugByUuid.get(uuid_) ?? uuid_, diff --git a/apps/cli/src/render/tui/home-controller.test.ts b/apps/cli/src/render/tui/home-controller.test.ts index 51327d6b..29689458 100644 --- a/apps/cli/src/render/tui/home-controller.test.ts +++ b/apps/cli/src/render/tui/home-controller.test.ts @@ -1793,7 +1793,7 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => expect(writeDefault).not.toHaveBeenCalled(); // a keyless model can never become the default const hint = c.getSnapshot().modelPicker?.hint ?? ''; expect(hint).toContain('openai'); // names the provider - expect(hint).toContain('provider add'); // and the actionable remedy + expect(hint).toContain('provider set-key'); // and the actionable single-command remedy }); it('Esc closes the picker without writing a default', async () => { diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index c2fda427..09b2ad50 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -606,7 +606,9 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { // "not on your key" case keeps its message. Never a write — a blocked model can't become the default. const hint = step.reason === 'no-key' - ? `${step.displayName}: no key for ${step.provider} — run \`relavium provider add ${step.provider}\`, then set-key` + ? // `set-key` alone auto-registers a known provider (no prior `add` needed) — the single-command form + // every other "no key" message uses. + `${step.displayName}: no key for ${step.provider} — run \`relavium provider set-key ${step.provider}\`` : `${step.displayName} is not available on your key — pick another`; set({ modelPicker: { ...open, hint } }); break; From d3c65c076275b4a24790b59e2d9b70243bcb38a0 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 12:50:43 +0300 Subject: [PATCH 39/68] feat(cli): wizard live key-validation with a retry UX MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first-run wizard stored the pasted key WITHOUT any live check — a bad/expired key was silently accepted and only surfaced as a confusing provider_auth on the first chat turn. Now it validates the key before storing, with a retry-not-hard- fail flow. - providers: `validateProviderKey` gains a `reason: 'ok'|'auth'|'network'|'other'` discriminant, classified from the seam's own `LlmProviderError.llmError.kind` (never a string heuristic) — so the wizard branches on the CAUSE. Additive field; `provider test` / `/doctor --deep` are unaffected. - wizard: after the key is entered, a bounded key-redacted live probe (the same seam `provider test` uses). On failure, a clean select — auth → default "Enter a new key"; network/offline → default "Save it anyway" (a bare Enter, via the new `select.initialValue`), plus "Skip setup". A retry re-prompts and re-validates; the key is stored ONLY after the loop, so a bad key never reaches the keychain. The "Connected" note is honest: "Verified and stored" vs "Saved … couldn't be verified — run /doctor". A "Checking your key…" spinner (optional seam method). Secret-free throughout (only the redacted detail + keyHint). - tests: auth→retry→good (stores the 2nd key, "Verified"); auth→continue (stores the 1st, "couldn't be verified", no leak); network→continue; retry→Esc→skip; skip; + the reason-classification unit test. Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/engine/providers.test.ts | 35 ++++++- apps/cli/src/engine/providers.ts | 46 ++++++++- apps/cli/src/onboarding/wizard.test.ts | 134 +++++++++++++++++++++++++ apps/cli/src/onboarding/wizard.ts | 115 +++++++++++++++++++-- 4 files changed, 313 insertions(+), 17 deletions(-) diff --git a/apps/cli/src/engine/providers.test.ts b/apps/cli/src/engine/providers.test.ts index fd919831..5f25a96c 100644 --- a/apps/cli/src/engine/providers.test.ts +++ b/apps/cli/src/engine/providers.test.ts @@ -1,6 +1,6 @@ import { parseWorkflow, type WorkflowDefinition } from '@relavium/core'; import type { ProviderRecord } from '@relavium/db'; -import type { LlmProvider } from '@relavium/llm'; +import { LlmProviderError, type LlmProvider } from '@relavium/llm'; import { describe, expect, it, vi } from 'vitest'; import { CHAT_TEXT_CAPABILITY_FLAGS } from '../test-support.js'; @@ -187,10 +187,39 @@ describe('validateProviderKey', () => { it('reports ok with the test model on a successful ping', async () => { const generate = vi.fn().mockResolvedValue({}); const result = await validateProviderKey(fakeProvider(generate), TEST_KEY, 'm-test'); - expect(result).toEqual({ ok: true, detail: 'key works (m-test)' }); + expect(result).toEqual({ ok: true, detail: 'key works (m-test)', reason: 'ok' }); expect(generate).toHaveBeenCalledWith(expect.anything(), TEST_KEY); // the key reached generate, not the detail }); + it('classifies the failure CAUSE from the seam error kind — auth vs network vs other (2.5.G S8)', async () => { + const err = (kind: 'auth' | 'timeout' | 'bad_request'): LlmProviderError => + new LlmProviderError({ kind, retryable: false, provider: 'anthropic', message: `${kind} boom` }); + const auth = await validateProviderKey( + fakeProvider(vi.fn().mockRejectedValue(err('auth'))), + TEST_KEY, + 'm-test', + ); + expect(auth).toMatchObject({ ok: false, reason: 'auth' }); // a rejected key → auth (retry with a new key) + const net = await validateProviderKey( + fakeProvider(vi.fn().mockRejectedValue(err('timeout'))), + TEST_KEY, + 'm-test', + ); + expect(net.reason).toBe('network'); // transient/offline → network (continue-anyway is sane) + const other = await validateProviderKey( + fakeProvider(vi.fn().mockRejectedValue(err('bad_request'))), + TEST_KEY, + 'm-test', + ); + expect(other.reason).toBe('other'); + const plain = await validateProviderKey( + fakeProvider(vi.fn().mockRejectedValue(new Error('not a seam error'))), + TEST_KEY, + 'm-test', + ); + expect(plain.reason).toBe('other'); // a non-LlmProviderError throw → other + }); + it('REDACTS the key from a failing-ping message (never the full key, keeps the last-4 hint)', async () => { const generate = vi .fn() @@ -211,7 +240,7 @@ describe('validateProviderKey', () => { it('guards an empty key (the split("") footgun) without calling generate', async () => { const generate = vi.fn(); const result = await validateProviderKey(fakeProvider(generate), '', 'm-test'); - expect(result).toEqual({ ok: false, detail: 'key test failed — (no key)' }); + expect(result).toEqual({ ok: false, detail: 'key test failed — (no key)', reason: 'other' }); expect(generate).not.toHaveBeenCalled(); }); diff --git a/apps/cli/src/engine/providers.ts b/apps/cli/src/engine/providers.ts index a75ad242..4b3db270 100644 --- a/apps/cli/src/engine/providers.ts +++ b/apps/cli/src/engine/providers.ts @@ -4,6 +4,7 @@ import { createCustomOpenAiProvider, defaultProviders, InvalidBaseUrlError, + LlmProviderError, type LlmProvider, type ProviderId, } from '@relavium/llm'; @@ -132,11 +133,23 @@ export const KNOWN_PROVIDERS: Record<(typeof KNOWN_PROVIDER_IDS)[number], Provid * enough that a stalled provider can never hang `provider test` or `/doctor --deep`. */ export const VALIDATE_KEY_TIMEOUT_MS = 10_000; -/** The outcome of a {@link validateProviderKey} probe — `ok` plus a secret-free `detail` line. */ +/** + * Why a {@link validateProviderKey} probe ended — so a caller can branch on the CAUSE, not by string-matching + * `detail` (2.5.G S8 wizard). `'auth'` — the key was rejected (a bad key; retrying with a new key is the remedy); + * `'network'` — a timeout / transport / overloaded / rate-limit (offline/transient; "continue anyway" is the sane + * default so an offline first-run isn't blocked); `'other'` — a non-auth request fault or an unexpected throw. + * Sourced from the seam's own `LlmProviderError.llmError.kind`, never a heuristic — a Relavium classification, not + * a message; it carries no provider text. + */ +export type ValidationReason = 'ok' | 'auth' | 'network' | 'other'; + +/** The outcome of a {@link validateProviderKey} probe — `ok` plus a secret-free `detail` line + a {@link ValidationReason}. */ export interface ProviderKeyValidation { readonly ok: boolean; /** Secret-free: `key works ()` on success, or `key test failed — ` on failure. */ readonly detail: string; + /** The cause discriminant (2.5.G S8) — `'ok'` on success, else `'auth'` / `'network'` / `'other'`. */ + readonly reason: ValidationReason; } /** @@ -162,14 +175,14 @@ export async function validateProviderKey( // and garble the message (no secret leaks — the key is empty — but the detail becomes nonsense). All current // callers resolve a non-empty key (createProviderResolver rejects `''`); this closes the footgun at the seam. if (key.length === 0) { - return { ok: false, detail: 'key test failed — (no key)' }; + return { ok: false, detail: 'key test failed — (no key)', reason: 'other' }; } const controller = new AbortController(); let timer: ReturnType | undefined; const timeout = new Promise((resolve) => { timer = setTimeout(() => { controller.abort(); - resolve({ ok: false, detail: `key test failed — timeout (${timeoutMs}ms)` }); + resolve({ ok: false, detail: `key test failed — timeout (${timeoutMs}ms)`, reason: 'network' }); }, timeoutMs); }); const probe = (async (): Promise => { @@ -183,10 +196,16 @@ export async function validateProviderKey( }, key, ); - return { ok: true, detail: `key works (${model})` }; + return { ok: true, detail: `key works (${model})`, reason: 'ok' }; } catch (err) { const raw = err instanceof Error ? err.message : String(err); - return { ok: false, detail: `key test failed — ${raw.split(key).join(keyHint(key))}` }; + return { + ok: false, + detail: `key test failed — ${raw.split(key).join(keyHint(key))}`, + // Classify from the seam's own error kind (never a string heuristic): a rejected key is `auth` (retry with + // a new key); a timeout/transport/overloaded/rate-limit is `network` (continue-anyway is sane); else `other`. + reason: classifyValidationFailure(err), + }; } })(); try { @@ -196,6 +215,23 @@ export async function validateProviderKey( } } +/** Map a probe throw → a {@link ValidationReason}, from the seam's `LlmProviderError.llmError.kind` (never a string + * heuristic). A non-`LlmProviderError` (or an unclassified kind) is `'other'`. No provider text is read. */ +function classifyValidationFailure(err: unknown): ValidationReason { + if (!(err instanceof LlmProviderError)) return 'other'; + switch (err.llmError.kind) { + case 'auth': + return 'auth'; + case 'timeout': + case 'transport': + case 'overloaded': + case 'rate_limit': + return 'network'; + default: + return 'other'; // bad_request / content_filter / cancelled / unknown + } +} + /** * The provider ids whose key is **guaranteed** needed by a parsed workflow — the **primary** * `provider` (the authored `agent.provider`, never derived from the model) of every inline agent diff --git a/apps/cli/src/onboarding/wizard.test.ts b/apps/cli/src/onboarding/wizard.test.ts index 15f4c172..39453737 100644 --- a/apps/cli/src/onboarding/wizard.test.ts +++ b/apps/cli/src/onboarding/wizard.test.ts @@ -10,6 +10,7 @@ import { keychainAccount, type KeychainStore, } from '../secrets/keychain.js'; +import type { ProviderKeyValidation } from '../engine/providers.js'; import { isProviderKeyless, runOnboardingWizard, @@ -18,6 +19,34 @@ import { const CANCEL = Symbol('clack-cancel'); +/** A prompter that drains QUEUES of `select`/`password` results (for the retry flow) — a bare no-op spinner. */ +function queuedPrompter( + selects: (string | symbol)[], + passwords: (string | symbol)[], +): { prompter: ClackOnboardingDeps; notes: string[]; outros: string[] } { + const notes: string[] = []; + const outros: string[] = []; + const prompter: ClackOnboardingDeps = { + intro: () => undefined, + outro: (m) => { + outros.push(m); + }, + note: (m, t) => { + notes.push(`${t ?? ''}\n${m}`); + }, + select: () => Promise.resolve(selects.shift() ?? CANCEL), + password: () => Promise.resolve(passwords.shift() ?? CANCEL), + isCancel: (v): v is symbol => typeof v === 'symbol', + spinner: () => ({ start: () => undefined, stop: () => undefined }), + }; + return { prompter, notes, outros }; +} + +/** A scripted live-validation port draining a queue of outcomes (defaulting to ok when exhausted). */ +function validateSeq(results: ProviderKeyValidation[]): (id: string, key: string) => Promise { + return () => Promise.resolve(results.shift() ?? { ok: true, detail: 'ok', reason: 'ok' }); +} + /** A scripted clack slice: fixed `select`/`password` results (a `symbol` = cancel), spies + captured notes/outros. */ function scriptedPrompter(script: { provider?: string | symbol; key?: string | symbol }): { prompter: ClackOnboardingDeps; @@ -212,4 +241,109 @@ describe('runOnboardingWizard', () => { expect(keychain.store.get(keychainAccount('deepseek'))).toBe('sk-ds-1234'); // the key IS stored (set ran first) expect(writeDefaultModel).not.toHaveBeenCalled(); // no default model on a failed store }); + + // ── Live key validation + retry UX (2.5.G S8) ───────────────────────────────────────────── + it('a bad key (auth) → RETRY → a good key: stores the SECOND key, note says Verified', async () => { + const keychain = memKeychain(); + const s = store(); + const writeDefaultModel = vi.fn(); + const { prompter, notes } = queuedPrompter( + ['openai', 'retry'], // provider, then the retry-decision + ['sk-bad-first', 'sk-good-second'], // initial key, then the re-entered key + ); + await runOnboardingWizard({ + prompter, + store: s, + keychain, + resolver: stubResolver, + io, + writeDefaultModel, + validate: validateSeq([ + { ok: false, detail: 'key test failed — invalid_api_key', reason: 'auth' }, + { ok: true, detail: 'key works', reason: 'ok' }, + ]), + }); + // The SECOND (re-entered) key is what landed — a bad key never reaches the keychain. + expect(keychain.store.get(keychainAccount('openai'))).toBe('sk-good-second'); + const all = notes.join('\n'); + expect(all).toContain('Verified and stored'); + expect(all).not.toContain('sk-bad-first'); + expect(all).not.toContain('sk-good-second'); + }); + + it('a bad key (auth) → CONTINUE anyway: stores the FIRST key, note says couldn\'t be verified (secret-free)', async () => { + const keychain = memKeychain(); + const s = store(); + const writeDefaultModel = vi.fn(); + const { prompter, notes } = queuedPrompter(['openai', 'continue'], ['sk-unverified-key']); + await runOnboardingWizard({ + prompter, + store: s, + keychain, + resolver: stubResolver, + io, + writeDefaultModel, + validate: validateSeq([{ ok: false, detail: 'key test failed — invalid_api_key', reason: 'auth' }]), + }); + expect(keychain.store.get(keychainAccount('openai'))).toBe('sk-unverified-key'); // consciously accepted + const all = notes.join('\n'); + expect(all).toContain("couldn't be verified"); + expect(all).not.toContain('sk-unverified-key'); // never echoed, even on the continue path + }); + + it('a NETWORK failure → default Continue (offline first-run isn\'t blocked): stores the key', async () => { + const keychain = memKeychain(); + const s = store(); + const writeDefaultModel = vi.fn(); + const { prompter } = queuedPrompter(['gemini', 'continue'], ['sk-offline-key']); + await runOnboardingWizard({ + prompter, + store: s, + keychain, + resolver: stubResolver, + io, + writeDefaultModel, + validate: validateSeq([{ ok: false, detail: 'key test failed — timeout (10000ms)', reason: 'network' }]), + }); + expect(keychain.store.get(keychainAccount('gemini'))).toBe('sk-offline-key'); + expect(writeDefaultModel).toHaveBeenCalledWith(KNOWN_PROVIDERS.gemini.testModel); + }); + + it('a bad key → RETRY → Esc on the re-prompt: SKIPS, keychain empty', async () => { + const keychain = memKeychain(); + const s = store(); + const writeDefaultModel = vi.fn(); + const { prompter, notes } = queuedPrompter(['openai', 'retry'], ['sk-bad', CANCEL]); + await runOnboardingWizard({ + prompter, + store: s, + keychain, + resolver: stubResolver, + io, + writeDefaultModel, + validate: validateSeq([{ ok: false, detail: 'key test failed — invalid_api_key', reason: 'auth' }]), + }); + expect(keychain.store.size).toBe(0); + expect(writeDefaultModel).not.toHaveBeenCalled(); + expect(notes.some((n) => n.includes('Skipped'))).toBe(true); + }); + + it('a bad key → SKIP choice: keychain empty, no default written', async () => { + const keychain = memKeychain(); + const s = store(); + const writeDefaultModel = vi.fn(); + const { prompter, notes } = queuedPrompter(['openai', 'skip'], ['sk-bad']); + await runOnboardingWizard({ + prompter, + store: s, + keychain, + resolver: stubResolver, + io, + writeDefaultModel, + validate: validateSeq([{ ok: false, detail: 'key test failed — invalid_api_key', reason: 'auth' }]), + }); + expect(keychain.store.size).toBe(0); + expect(writeDefaultModel).not.toHaveBeenCalled(); + expect(notes.some((n) => n.includes('Skipped'))).toBe(true); + }); }); diff --git a/apps/cli/src/onboarding/wizard.ts b/apps/cli/src/onboarding/wizard.ts index bb9b6478..5ccc7840 100644 --- a/apps/cli/src/onboarding/wizard.ts +++ b/apps/cli/src/onboarding/wizard.ts @@ -1,5 +1,6 @@ -import { intro, isCancel, note, outro, password, select } from '@clack/prompts'; +import { intro, isCancel, note, outro, password, select, spinner } from '@clack/prompts'; import type { ProviderStore } from '@relavium/db'; +import type { ProviderId } from '@relavium/llm'; import { runProviderCommand } from '../commands/provider.js'; import { @@ -8,6 +9,8 @@ import { keyHint, providerHasKey, providerKeyEnvVar, + validateProviderKey, + type ProviderKeyValidation, type ProviderResolver, } from '../engine/providers.js'; import { CliError } from '../process/errors.js'; @@ -40,6 +43,8 @@ export interface ClackOnboardingDeps { readonly select: (opts: { message: string; options: readonly { value: string; label: string; hint?: string }[]; + /** Pre-highlight an option so a bare Enter takes it (2.5.G S8 — the "press Enter to continue" affordance). */ + initialValue?: string; }) => Promise; /** A MASKED key prompt (clack `password`) — the hidden interactive key input. */ readonly password: (opts: { @@ -48,6 +53,8 @@ export interface ClackOnboardingDeps { }) => Promise; /** Clack's cancel sentinel guard (Ctrl-C / ESC) — a real type guard so a non-cancel value narrows. */ readonly isCancel: (value: unknown) => value is symbol; + /** An optional progress spinner for the live key-check moment (2.5.G S8) — omit ⇒ no spinner (a test no-op). */ + readonly spinner?: () => { start: (message?: string) => void; stop: (message?: string) => void }; } // The clack-boundary adapter (the one place the library's exact option shapes are met) — spread the optional @@ -65,6 +72,7 @@ const defaultPrompter: ClackOnboardingDeps = { label: option.label, ...(option.hint === undefined ? {} : { hint: option.hint }), })), + ...(opts.initialValue === undefined ? {} : { initialValue: opts.initialValue }), }), password: (opts) => password({ @@ -72,6 +80,10 @@ const defaultPrompter: ClackOnboardingDeps = { ...(opts.validate === undefined ? {} : { validate: opts.validate }), }), isCancel, + spinner: () => { + const sp = spinner(); + return { start: (message) => sp.start(message), stop: (message) => sp.stop(message) }; + }, }; /** The non-clack ports the wizard needs — the keychain-write path + the resolver (injected for tests). */ @@ -86,6 +98,13 @@ export interface OnboardingDeps { * sets a starter model of the CHOSEN provider so the first chat binds a model whose key was just stored — the * built-in default (`claude-sonnet-4-6` → anthropic) would otherwise error for a user who picked another provider. */ readonly writeDefaultModel: (modelId: string) => void; + /** + * LIVE-validate a just-entered key (2.5.G S8) — injected for tests (no network). Absent ⇒ the real bounded, + * key-redacted {@link validateProviderKey} probe against the provider's cheap `testModel`. The wizard uses the + * result's `reason` to branch the retry UX (auth → re-enter; network → continue-anyway). A resolver with no + * adapter (a test stub) validates as `ok` so the flow never blocks on an un-probeable provider. + */ + readonly validate?: (id: ProviderId, key: string) => Promise; } /** @@ -141,6 +160,14 @@ export async function runOnboardingWizard(deps: OnboardingDeps): Promise { // `requireKey` already rejected a whitespace-only value, so the trimmed key is non-empty. const key = rawKey.trim(); + // LIVE-validate the key BEFORE storing it (2.5.G S8), with a retry-not-hard-fail UX so a fat-fingered paste is + // instantly recoverable and a bad key never lands in the keychain. `verified` is threaded into the note so the + // "connected" copy stays honest on the continue-anyway path. `keyToStore` is what we finally persist (the + // originally-typed or a re-entered key). A `null` return ⇒ the user skipped mid-retry. + const outcome = await validateWithRetry(p, deps, provider, key); + if (outcome === null) return; // skip already surfaced its note/outro + const { keyToStore, verified } = outcome; + // Store via the TESTED providerSetKey path (keychain.set + the provider row + the keychain-ref, secret-free). Its // one stdout line is suppressed (a silent io) so the wizard's output stays uniformly clack-styled — the wizard // surfaces the outcome through a clack note/outro instead. @@ -153,7 +180,7 @@ export async function runOnboardingWizard(deps: OnboardingDeps): Promise { store: deps.store, keychain: deps.keychain, resolver: deps.resolver, - readSecret: () => Promise.resolve(key), + readSecret: () => Promise.resolve(keyToStore), // `global` is read only by `provider list --json`; `set-key` never touches it — a throwaway (the wizard is // always interactive, never `--json`). global: { json: false, color: false, cwd: process.cwd(), configPath: undefined, verbosity: 'normal' }, @@ -196,23 +223,93 @@ export async function runOnboardingWizard(deps: OnboardingDeps): Promise { // the user upgrades via `/models`. Best-effort: a config-write fault still leaves a working key (fall back to the // `/models` pointer) rather than undoing the store. const starterModel = KNOWN_PROVIDERS[provider].testModel; + // Honest copy: a verified key is "Verified and stored"; a consciously-accepted (network/continue-anyway) key is + // "Saved … couldn't be verified" so the user knows to re-check — never claim verification we didn't do. + const storedLine = verified + ? `Verified and stored your ${provider} key (${keyHint(keyToStore)}) in the OS keychain.` + : `Saved your ${provider} key (${keyHint(keyToStore)}) — it couldn't be verified now. Run /doctor to re-check.`; try { deps.writeDefaultModel(starterModel); p.note( - `Stored your ${provider} key (${keyHint(key)}) in the OS keychain.\n` + - `Your default model is ${starterModel} — change it anytime with /models.`, + `${storedLine}\nYour default model is ${starterModel} — change it anytime with /models.`, 'Connected', ); } catch { - p.note( - `Stored your ${provider} key (${keyHint(key)}) in the OS keychain.\n` + - `Pick a ${provider} model with /models to start chatting.`, - 'Connected', - ); + p.note(`${storedLine}\nPick a ${provider} model with /models to start chatting.`, 'Connected'); } p.outro("You're all set — starting Relavium."); } +/** + * Live-validate a key with a retry loop (2.5.G S8). Returns `{ keyToStore, verified }` once the user has a key to + * persist — verified (probe ok), or consciously accepted despite a failure ("save it anyway"). Returns `null` when + * the user skips (a note/outro is already shown, mirroring the top-level flow). Never throws for a bad key; only a + * clack-prompt fault propagates (caught by the Home's cleanup). Secret-free: the key is never echoed — only the + * redacted `detail`. A bad key never reaches the keychain (storage happens only after this resolves). + */ +async function validateWithRetry( + p: ClackOnboardingDeps, + deps: OnboardingDeps, + provider: ProviderId, + key: string, +): Promise<{ keyToStore: string; verified: boolean } | null> { + const validate = + deps.validate ?? + (async (id: ProviderId, k: string): Promise => { + const adapter = deps.resolver.resolveProvider(id); + // No adapter (a test stub) ⇒ can't probe → don't block the flow; treat as verified. + if (adapter === undefined) return { ok: true, detail: 'skipped', reason: 'ok' }; + return validateProviderKey(adapter, k, KNOWN_PROVIDERS[id].testModel); + }); + + let keyToStore = key; + for (;;) { + const sp = p.spinner?.(); + sp?.start(`Checking your ${KNOWN_PROVIDERS[provider].displayName} key…`); + const res = await validate(provider, keyToStore); + sp?.stop(res.ok ? 'Key verified.' : 'Key check finished.'); + if (res.ok) return { keyToStore, verified: true }; + + // The order + the pre-highlighted (bare-Enter) option depend on the CAUSE: an offline/transient `network` + // failure defaults to "save it anyway" (don't block an offline first-run); a rejected key defaults to + // "re-enter". `res.detail` is already key-redacted, so it is safe to show. + const isNetwork = res.reason === 'network'; + const choice = await p.select({ + message: isNetwork + ? `Couldn't reach ${KNOWN_PROVIDERS[provider].displayName} to verify — you may be offline.` + : `That key didn't work — ${res.detail}.`, + options: isNetwork + ? [ + { value: 'continue', label: 'Save it anyway', hint: 'verify later with /doctor' }, + { value: 'retry', label: 'Enter a different key' }, + { value: 'skip', label: 'Skip setup' }, + ] + : [ + { value: 'retry', label: 'Enter a new key' }, + { value: 'continue', label: 'Save it anyway', hint: 'fix it later with /doctor' }, + { value: 'skip', label: 'Skip setup' }, + ], + initialValue: isNetwork ? 'continue' : 'retry', + }); + if (p.isCancel(choice) || choice === 'skip') { + skip(p); + return null; + } + if (choice === 'continue') return { keyToStore, verified: false }; + + // 'retry' — re-prompt for a key; Esc here also skips. The new key loops back through validation. + const again = await p.password({ + message: `Paste your ${KNOWN_PROVIDERS[provider].displayName} API key`, + validate: requireKey, + }); + if (p.isCancel(again)) { + skip(p); + return null; + } + keyToStore = again.trim(); + } +} + /** The cancel/skip exit: a friendly pointer to the manual path, then hand off to the Home. */ function skip(p: ClackOnboardingDeps): void { p.note( From 3ccd5898cef90afb7a7e96c21932911929844460 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 13:00:57 +0300 Subject: [PATCH 40/68] =?UTF-8?q?fix(cli):=20Step=20B=20Opus-review=20fixe?= =?UTF-8?q?s=20=E2=80=94=20DRY=20the=20network=20bucket=20+=20spinner=20sa?= =?UTF-8?q?fety?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - LOW: `classifyValidationFailure` now derives the `network` (continue-anyway) bucket from the seam's `isRetryable` instead of hand-re-listing the retryable kinds — so a new transient kind can't drift into the wrong bucket. - LOW: wrap the live probe in try/finally so the "Checking your key…" spinner always stops even if a (future injected) `validate` rejects — no runaway spinner over the Home; the throw then surfaces cleanly to the Home's cleanup. - LOW: clarify the default-validate no-adapter branch is a non-production (stub-only) path — in production every first-run key is genuinely probed. - LOW: the retry tests now capture the select opts and assert the cause-based bare-Enter default (auth→retry, network→continue) + that the (redacted) detail flows into the select message — covering the wizard's composition, not just the seam's redaction. Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/engine/providers.ts | 18 ++++------- apps/cli/src/onboarding/wizard.test.ts | 41 +++++++++++++++++++++----- apps/cli/src/onboarding/wizard.ts | 16 ++++++++-- 3 files changed, 52 insertions(+), 23 deletions(-) diff --git a/apps/cli/src/engine/providers.ts b/apps/cli/src/engine/providers.ts index 4b3db270..4d8c4c29 100644 --- a/apps/cli/src/engine/providers.ts +++ b/apps/cli/src/engine/providers.ts @@ -4,6 +4,7 @@ import { createCustomOpenAiProvider, defaultProviders, InvalidBaseUrlError, + isRetryable, LlmProviderError, type LlmProvider, type ProviderId, @@ -216,20 +217,13 @@ export async function validateProviderKey( } /** Map a probe throw → a {@link ValidationReason}, from the seam's `LlmProviderError.llmError.kind` (never a string - * heuristic). A non-`LlmProviderError` (or an unclassified kind) is `'other'`. No provider text is read. */ + * heuristic). A non-`LlmProviderError` is `'other'`. The `'network'` (continue-anyway) bucket IS exactly the seam's + * RETRYABLE (transient) set — bound to `isRetryable` so a new transient kind can't drift out of it. No text read. */ function classifyValidationFailure(err: unknown): ValidationReason { if (!(err instanceof LlmProviderError)) return 'other'; - switch (err.llmError.kind) { - case 'auth': - return 'auth'; - case 'timeout': - case 'transport': - case 'overloaded': - case 'rate_limit': - return 'network'; - default: - return 'other'; // bad_request / content_filter / cancelled / unknown - } + const kind = err.llmError.kind; + if (kind === 'auth') return 'auth'; // a rejected key — re-entering is the remedy + return isRetryable(kind) ? 'network' : 'other'; // transient (timeout/transport/overloaded/rate_limit) ⇒ network } /** diff --git a/apps/cli/src/onboarding/wizard.test.ts b/apps/cli/src/onboarding/wizard.test.ts index 39453737..4b8d8505 100644 --- a/apps/cli/src/onboarding/wizard.test.ts +++ b/apps/cli/src/onboarding/wizard.test.ts @@ -19,13 +19,21 @@ import { const CANCEL = Symbol('clack-cancel'); -/** A prompter that drains QUEUES of `select`/`password` results (for the retry flow) — a bare no-op spinner. */ +/** A prompter that drains QUEUES of `select`/`password` results (for the retry flow) — a bare no-op spinner. It + * CAPTURES each `select`'s `{ message, initialValue }` so a test can assert the retry copy is redaction-safe and + * the bare-Enter default branches by cause. */ function queuedPrompter( selects: (string | symbol)[], passwords: (string | symbol)[], -): { prompter: ClackOnboardingDeps; notes: string[]; outros: string[] } { +): { + prompter: ClackOnboardingDeps; + notes: string[]; + outros: string[]; + selectCalls: { message: string; initialValue?: string }[]; +} { const notes: string[] = []; const outros: string[] = []; + const selectCalls: { message: string; initialValue?: string }[] = []; const prompter: ClackOnboardingDeps = { intro: () => undefined, outro: (m) => { @@ -34,12 +42,18 @@ function queuedPrompter( note: (m, t) => { notes.push(`${t ?? ''}\n${m}`); }, - select: () => Promise.resolve(selects.shift() ?? CANCEL), + select: (opts) => { + selectCalls.push({ + message: opts.message, + ...(opts.initialValue === undefined ? {} : { initialValue: opts.initialValue }), + }); + return Promise.resolve(selects.shift() ?? CANCEL); + }, password: () => Promise.resolve(passwords.shift() ?? CANCEL), isCancel: (v): v is symbol => typeof v === 'symbol', spinner: () => ({ start: () => undefined, stop: () => undefined }), }; - return { prompter, notes, outros }; + return { prompter, notes, outros, selectCalls }; } /** A scripted live-validation port draining a queue of outcomes (defaulting to ok when exhausted). */ @@ -275,7 +289,7 @@ describe('runOnboardingWizard', () => { const keychain = memKeychain(); const s = store(); const writeDefaultModel = vi.fn(); - const { prompter, notes } = queuedPrompter(['openai', 'continue'], ['sk-unverified-key']); + const { prompter, notes, selectCalls } = queuedPrompter(['openai', 'continue'], ['sk-unverified-key']); await runOnboardingWizard({ prompter, store: s, @@ -283,19 +297,28 @@ describe('runOnboardingWizard', () => { resolver: stubResolver, io, writeDefaultModel, - validate: validateSeq([{ ok: false, detail: 'key test failed — invalid_api_key', reason: 'auth' }]), + // A PLANTED secret in the detail must never reach the select MESSAGE (the wizard surfaces res.detail there). + validate: validateSeq([ + { ok: false, detail: 'key test failed — LEAKED-sk-unverified-key rejected', reason: 'auth' }, + ]), }); expect(keychain.store.get(keychainAccount('openai'))).toBe('sk-unverified-key'); // consciously accepted const all = notes.join('\n'); expect(all).toContain("couldn't be verified"); - expect(all).not.toContain('sk-unverified-key'); // never echoed, even on the continue path + expect(all).not.toContain('sk-unverified-key'); // never echoed in a note, even on the continue path + // The retry select is the AUTH branch → its bare-Enter default is 'retry' (re-enter a key)… + const retrySelect = selectCalls[1]; // [0] is the provider select + expect(retrySelect?.initialValue).toBe('retry'); + // …and whatever detail the seam produced flows through the select message unmodified — so if it ever carried a + // secret the wizard would echo it. (Here the detail IS the redacted value; the seam's own test proves redaction.) + expect(retrySelect?.message).toContain('LEAKED-sk-unverified-key'); // detail is surfaced verbatim (already redacted at the seam) }); it('a NETWORK failure → default Continue (offline first-run isn\'t blocked): stores the key', async () => { const keychain = memKeychain(); const s = store(); const writeDefaultModel = vi.fn(); - const { prompter } = queuedPrompter(['gemini', 'continue'], ['sk-offline-key']); + const { prompter, selectCalls } = queuedPrompter(['gemini', 'continue'], ['sk-offline-key']); await runOnboardingWizard({ prompter, store: s, @@ -307,6 +330,8 @@ describe('runOnboardingWizard', () => { }); expect(keychain.store.get(keychainAccount('gemini'))).toBe('sk-offline-key'); expect(writeDefaultModel).toHaveBeenCalledWith(KNOWN_PROVIDERS.gemini.testModel); + // The NETWORK branch pre-highlights 'continue' so a bare Enter saves anyway (an offline first-run isn't blocked). + expect(selectCalls[1]?.initialValue).toBe('continue'); }); it('a bad key → RETRY → Esc on the re-prompt: SKIPS, keychain empty', async () => { diff --git a/apps/cli/src/onboarding/wizard.ts b/apps/cli/src/onboarding/wizard.ts index 5ccc7840..b4540c94 100644 --- a/apps/cli/src/onboarding/wizard.ts +++ b/apps/cli/src/onboarding/wizard.ts @@ -257,7 +257,10 @@ async function validateWithRetry( deps.validate ?? (async (id: ProviderId, k: string): Promise => { const adapter = deps.resolver.resolveProvider(id); - // No adapter (a test stub) ⇒ can't probe → don't block the flow; treat as verified. + // A NON-PRODUCTION path only: the real `createProviderResolver` returns an adapter for every `ProviderId`, so + // this `undefined` branch is reachable ONLY by a test stub (`resolveProvider: () => undefined`). There it + // can't probe, so it treats the key as ok to avoid blocking the scripted flow; in production every first-run + // key is genuinely probed, so the "Verified and stored" copy is never shown for an un-probed key. if (adapter === undefined) return { ok: true, detail: 'skipped', reason: 'ok' }; return validateProviderKey(adapter, k, KNOWN_PROVIDERS[id].testModel); }); @@ -266,8 +269,15 @@ async function validateWithRetry( for (;;) { const sp = p.spinner?.(); sp?.start(`Checking your ${KNOWN_PROVIDERS[provider].displayName} key…`); - const res = await validate(provider, keyToStore); - sp?.stop(res.ok ? 'Key verified.' : 'Key check finished.'); + // The spinner MUST stop even if the probe rejects unexpectedly (validateProviderKey never rejects, but a future + // injected `validate` might) — otherwise a runaway spinner interval would sit over the ink-hosted Home while the + // throw propagates to the Home's cleanup. The `finally` stops it; the throw then surfaces cleanly there. + let res: ProviderKeyValidation; + try { + res = await validate(provider, keyToStore); + } finally { + sp?.stop('Key check finished.'); + } if (res.ok) return { keyToStore, verified: true }; // The order + the pre-highlighted (bare-Enter) option depend on the CAUSE: an offline/transient `network` From ca4e2ca7de6fb51cd989a0da5f9c2744ae283b78 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 13:16:38 +0300 Subject: [PATCH 41/68] =?UTF-8?q?fix(cli):=20Step=20B=20Sonnet-review=20fi?= =?UTF-8?q?xes=20=E2=80=94=20neutral=20copy,=20discriminated=20union,=20do?= =?UTF-8?q?c=20+=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - MEDIUM: the retry copy no longer asserts "the key is bad" for a non-auth/ non-network cause — a 402 (billing) or 400 (retired test model) shows a neutral "Couldn't verify your key ()"; the network branch says "offline or it's busy" (covers rate-limit/overloaded too). The redacted detail (which names the specific reason) is the evidence. - MEDIUM: update the wizard's SECURITY doc block — the key is now sent LIVE to its provider once (the redacted probe) BEFORE storage, and a "save it anyway" key is stored UNVERIFIED (the note says so). - MEDIUM/LOW tests: a rejecting `validate` stops the spinner (finally) + propagates (no swallow); a 2+-iteration retry threads the last key; the "other" bucket copy; the timeout `reason: 'network'` assertion. - LOW: `ProviderKeyValidation` is now a DISCRIMINATED union (reason pinned to ok); an exhaustive `choice !== 'retry'` guard fails loud on a future unhandled option; the classification-test fixture's `retryable` matches the real makeLlmError. Co-Authored-By: Claude Opus 4.8 (1M context) --- apps/cli/src/engine/providers.test.ts | 9 +++- apps/cli/src/engine/providers.ts | 18 ++++---- apps/cli/src/onboarding/wizard.test.ts | 57 ++++++++++++++++++++++++++ apps/cli/src/onboarding/wizard.ts | 37 +++++++++++------ 4 files changed, 98 insertions(+), 23 deletions(-) diff --git a/apps/cli/src/engine/providers.test.ts b/apps/cli/src/engine/providers.test.ts index 5f25a96c..c6129406 100644 --- a/apps/cli/src/engine/providers.test.ts +++ b/apps/cli/src/engine/providers.test.ts @@ -192,8 +192,14 @@ describe('validateProviderKey', () => { }); it('classifies the failure CAUSE from the seam error kind — auth vs network vs other (2.5.G S8)', async () => { + // `retryable` set to match the real `makeLlmError` (timeout IS retryable) so the fixture is a possible value. const err = (kind: 'auth' | 'timeout' | 'bad_request'): LlmProviderError => - new LlmProviderError({ kind, retryable: false, provider: 'anthropic', message: `${kind} boom` }); + new LlmProviderError({ + kind, + retryable: kind === 'timeout', + provider: 'anthropic', + message: `${kind} boom`, + }); const auth = await validateProviderKey( fakeProvider(vi.fn().mockRejectedValue(err('auth'))), TEST_KEY, @@ -256,6 +262,7 @@ describe('validateProviderKey', () => { const result = await validateProviderKey(fakeProvider(generate), TEST_KEY, 'm-test', 5); expect(result.ok).toBe(false); expect(result.detail).toContain('timeout'); + expect(result.reason).toBe('network'); // a timeout classifies as network (continue-anyway in the wizard) expect(result.detail).not.toContain(TEST_KEY); }); }); diff --git a/apps/cli/src/engine/providers.ts b/apps/cli/src/engine/providers.ts index 4d8c4c29..2b6e14f8 100644 --- a/apps/cli/src/engine/providers.ts +++ b/apps/cli/src/engine/providers.ts @@ -144,14 +144,14 @@ export const VALIDATE_KEY_TIMEOUT_MS = 10_000; */ export type ValidationReason = 'ok' | 'auth' | 'network' | 'other'; -/** The outcome of a {@link validateProviderKey} probe — `ok` plus a secret-free `detail` line + a {@link ValidationReason}. */ -export interface ProviderKeyValidation { - readonly ok: boolean; - /** Secret-free: `key works ()` on success, or `key test failed — ` on failure. */ - readonly detail: string; - /** The cause discriminant (2.5.G S8) — `'ok'` on success, else `'auth'` / `'network'` / `'other'`. */ - readonly reason: ValidationReason; -} +/** + * The outcome of a {@link validateProviderKey} probe — a DISCRIMINATED union so `reason` is pinned to `ok` at the + * type level (the "typed, discriminated" standard): success is always `reason: 'ok'`; a failure is never `'ok'`. + * `detail` is secret-free (`key works ()` on success, `key test failed — ` on failure). + */ +export type ProviderKeyValidation = + | { readonly ok: true; readonly detail: string; readonly reason: 'ok' } + | { readonly ok: false; readonly detail: string; readonly reason: Exclude }; /** * Validate a provider key with a minimal live request (`maxTokens: 1` 'ping'). Returns a RESULT so the caller @@ -219,7 +219,7 @@ export async function validateProviderKey( /** Map a probe throw → a {@link ValidationReason}, from the seam's `LlmProviderError.llmError.kind` (never a string * heuristic). A non-`LlmProviderError` is `'other'`. The `'network'` (continue-anyway) bucket IS exactly the seam's * RETRYABLE (transient) set — bound to `isRetryable` so a new transient kind can't drift out of it. No text read. */ -function classifyValidationFailure(err: unknown): ValidationReason { +function classifyValidationFailure(err: unknown): Exclude { if (!(err instanceof LlmProviderError)) return 'other'; const kind = err.llmError.kind; if (kind === 'auth') return 'auth'; // a rejected key — re-entering is the remedy diff --git a/apps/cli/src/onboarding/wizard.test.ts b/apps/cli/src/onboarding/wizard.test.ts index 4b8d8505..3b9c4ef8 100644 --- a/apps/cli/src/onboarding/wizard.test.ts +++ b/apps/cli/src/onboarding/wizard.test.ts @@ -371,4 +371,61 @@ describe('runOnboardingWizard', () => { expect(writeDefaultModel).not.toHaveBeenCalled(); expect(notes.some((n) => n.includes('Skipped'))).toBe(true); }); + + it('threads keyToStore across MULTIPLE retries (retry → retry → good stores the THIRD key)', async () => { + const keychain = memKeychain(); + const { prompter } = queuedPrompter(['openai', 'retry', 'retry'], ['sk-bad1', 'sk-bad2', 'sk-good3']); + await runOnboardingWizard({ + prompter, + store: store(), + keychain, + resolver: stubResolver, + io, + writeDefaultModel: vi.fn(), + validate: validateSeq([ + { ok: false, detail: 'x', reason: 'auth' }, + { ok: false, detail: 'x', reason: 'auth' }, + { ok: true, detail: 'ok', reason: 'ok' }, + ]), + }); + expect(keychain.store.get(keychainAccount('openai'))).toBe('sk-good3'); // the LAST-entered key threaded through + }); + + it('an "other" (non-transient) failure uses neutral "Couldn\'t verify" copy + a retry default, not "offline"', async () => { + const { prompter, selectCalls } = queuedPrompter(['openai', 'skip'], ['sk-x']); + await runOnboardingWizard({ + prompter, + store: store(), + keychain: memKeychain(), + resolver: stubResolver, + io, + writeDefaultModel: vi.fn(), + validate: validateSeq([{ ok: false, detail: 'bad_request: model retired', reason: 'other' }]), + }); + const msg = selectCalls[1]?.message ?? ''; + expect(msg).toContain("Couldn't verify"); // never asserts "the key is bad" for a non-auth/non-network cause + expect(msg).not.toContain('offline'); + expect(selectCalls[1]?.initialValue).toBe('retry'); // non-transient → re-enter is the default + }); + + it('a REJECTING validate STOPS the spinner (finally) + PROPAGATES (no runaway spinner, no swallow)', async () => { + const stops: number[] = []; + const { prompter } = queuedPrompter(['openai'], ['sk-x']); + const spied: ClackOnboardingDeps = { + ...prompter, + spinner: () => ({ start: () => undefined, stop: () => stops.push(1) }), + }; + await expect( + runOnboardingWizard({ + prompter: spied, + store: store(), + keychain: memKeychain(), + resolver: stubResolver, + io, + writeDefaultModel: vi.fn(), + validate: () => Promise.reject(new Error('probe boom')), + }), + ).rejects.toThrow('probe boom'); // propagates (not swallowed) to the Home's cleanup + expect(stops.length).toBe(1); // the finally stopped the spinner despite the rejection + }); }); diff --git a/apps/cli/src/onboarding/wizard.ts b/apps/cli/src/onboarding/wizard.ts index b4540c94..cfd1bba3 100644 --- a/apps/cli/src/onboarding/wizard.ts +++ b/apps/cli/src/onboarding/wizard.ts @@ -28,9 +28,14 @@ import { KeychainUnavailableError, type KeychainStore } from '../secrets/keychai * wizard's job is to store the key that lights up the S7 `/models` picker + chat; the user picks a model there. * * SECURITY (this captures a live API key): - * - The key is read via clack's masked `password` prompt (never an argv flag, never echoed), held only in memory, - * and handed straight to the keychain — it is never logged (only {@link keyHint}, the last 4), persisted to a - * file, or placed in an error/report. + * - The key is read via clack's masked `password` prompt (never an argv flag, never echoed) and held only in + * memory. Before it is stored, it is sent LIVE to the provider ONCE (the bounded, key-redacted + * {@link validateProviderKey} `maxTokens:1` ping) to verify it — the key crosses the network only to its own + * provider, over the adapter's own transport, and is never logged/persisted/echoed on that path. + * - It is then handed to the OS keychain. On a validation FAILURE the user may consciously "save it anyway" — the + * key is then stored UNVERIFIED and the note says so (never claiming a verification we didn't do). + * - The only key material ever surfaced is {@link keyHint} (the last 4); the redacted probe `detail` is the only + * failure text shown. Nothing is written to a file/log/error/report. * - The keychain-unavailable fallback prints the env-var name to set — NEVER the key, and NEVER a plaintext file * (the deliberate "no silent plaintext fallback" of `providerSetKey`). */ @@ -280,15 +285,19 @@ async function validateWithRetry( } if (res.ok) return { keyToStore, verified: true }; - // The order + the pre-highlighted (bare-Enter) option depend on the CAUSE: an offline/transient `network` - // failure defaults to "save it anyway" (don't block an offline first-run); a rejected key defaults to - // "re-enter". `res.detail` is already key-redacted, so it is safe to show. - const isNetwork = res.reason === 'network'; + // The order + the pre-highlighted (bare-Enter) option depend on the CAUSE: a TRANSIENT (`network` — offline / + // timeout / rate-limit / overloaded) failure defaults to "save it anyway" (don't block an offline first-run); + // a non-transient failure (a rejected key, a billing/account issue, or an unexpected fault) defaults to + // "re-enter". `res.detail` (already key-redacted, and it names the specific reason, e.g. `invalid_api_key`) is + // the neutral evidence — we do NOT assert "the key is bad" for the non-network case, since a 402/400 may mean + // billing or a stale test model rather than a wrong key. + const isTransient = res.reason === 'network'; + const displayName = KNOWN_PROVIDERS[provider].displayName; const choice = await p.select({ - message: isNetwork - ? `Couldn't reach ${KNOWN_PROVIDERS[provider].displayName} to verify — you may be offline.` - : `That key didn't work — ${res.detail}.`, - options: isNetwork + message: isTransient + ? `Couldn't reach ${displayName} to verify — you may be offline or it's busy (${res.detail}).` + : `Couldn't verify your ${displayName} key (${res.detail}).`, + options: isTransient ? [ { value: 'continue', label: 'Save it anyway', hint: 'verify later with /doctor' }, { value: 'retry', label: 'Enter a different key' }, @@ -299,17 +308,19 @@ async function validateWithRetry( { value: 'continue', label: 'Save it anyway', hint: 'fix it later with /doctor' }, { value: 'skip', label: 'Skip setup' }, ], - initialValue: isNetwork ? 'continue' : 'retry', + initialValue: isTransient ? 'continue' : 'retry', }); if (p.isCancel(choice) || choice === 'skip') { skip(p); return null; } if (choice === 'continue') return { keyToStore, verified: false }; + // clack's `select` only ever yields a listed value; fail loud if a future option is added without a handler. + if (choice !== 'retry') throw new Error(`unexpected wizard choice: ${String(choice)}`); // 'retry' — re-prompt for a key; Esc here also skips. The new key loops back through validation. const again = await p.password({ - message: `Paste your ${KNOWN_PROVIDERS[provider].displayName} API key`, + message: `Paste your ${displayName} API key`, validate: requireKey, }); if (p.isCancel(again)) { From 6e5b8726c1108db5ab79c88918fbb42edc5a2a45 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 13:40:36 +0300 Subject: [PATCH 42/68] =?UTF-8?q?docs,test(cli,skills):=202.5.G=20Step=20C?= =?UTF-8?q?=20=E2=80=94=20document=20data-driven=20provider=20onboarding?= =?UTF-8?q?=20+=20lock-step=20guard=20test?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers question #2 ("does a new provider auto-appear in the wizard?"). The wizard, `provider add`/`test`, `/doctor --deep`, and the `/models` Home key-gate are ALREADY data-driven off `KNOWN_PROVIDER_IDS`/`KNOWN_PROVIDERS` — no `wizard.ts` edit is needed for a new provider. The gate is registration in two lists, so this documents the invariant and pins it with a guard test rather than changing behavior: - add-llm-adapter skill: new step 7 (CLI registration — the two homes + the lock-step rule), an Inputs `Provider id` clarification (canonical home = `LLM_PROVIDERS`), a Done-criteria bullet, and a Common-pitfall entry. Renumbered steps 8–10. - providers.ts: extend the `KNOWN_PROVIDERS`/`KNOWN_PROVIDER_IDS` doc comments to name the wizard + `/models` key-gate as data-driven consumers and spell out the `LLM_PROVIDERS`↔`KNOWN_PROVIDER_IDS` lock-step (the Step-A latent coupling: a seam id missing from `KNOWN_PROVIDER_IDS` is silently mis-dimmed `no-key` in the Home). - providers.test.ts: a guard test pinning `KNOWN_PROVIDER_IDS` and `LLM_PROVIDERS` as equal sets (the compiler enforces only `⊆`), plus a non-empty-`testModel` assertion — a missed registration is now a red CI run, not a runtime mis-dim. Toolchain green: 1481 tests, all lint/typecheck/test tasks pass. Refs: ADR-0064, ADR-0065 Co-Authored-By: Claude --- .claude/skills/add-llm-adapter/SKILL.md | 15 ++++++++++---- apps/cli/src/engine/providers.test.ts | 24 ++++++++++++++++++++++ apps/cli/src/engine/providers.ts | 27 +++++++++++++++++++------ 3 files changed, 56 insertions(+), 10 deletions(-) diff --git a/.claude/skills/add-llm-adapter/SKILL.md b/.claude/skills/add-llm-adapter/SKILL.md index b278dd7d..7add05eb 100644 --- a/.claude/skills/add-llm-adapter/SKILL.md +++ b/.claude/skills/add-llm-adapter/SKILL.md @@ -22,7 +22,7 @@ Integrate a new model provider into `packages/llm` (`@relavium/llm`) as a thin a ## Inputs | Input | Description | |-------|-------------| -| Provider id | The literal added to `LlmProvider.id` (`'anthropic' \| 'openai' \| 'gemini' \| 'deepseek' \| …`) — a plain string union, never a vendor enum. | +| Provider id | The literal added to the closed `ProviderId` enum — its **canonical home is `LLM_PROVIDERS`** in `packages/shared/src/constants.ts` (`ProviderId = (typeof LLM_PROVIDERS)[number]`, the `ProviderIdSchema` `z.enum`), never a vendor enum. The CLI mirrors it in `KNOWN_PROVIDER_IDS`/`KNOWN_PROVIDERS` (step 7). A genuinely-new arbitrary id **opens** the closed enum — a deliberate supersede per [ADR-0065](../../../docs/decisions/0065-provider-economics-and-extensibility.md) §6, not a silent edit. | | SDK / transport | The official TS SDK to wrap, or `openai` + a custom `baseURL` if OpenAI-compatible. An SDK dependency stays strictly inside `packages/llm/src/adapters/*`. | | Capabilities | `{ tools, streaming, parallelToolCalls, vision, promptCache, reasoning }` — what this provider genuinely supports, for the `supports` capability flags. | | Model ids + pricing | Canonical model ids this adapter serves and their per-token input/output (and cache) prices, for the pricing table. | @@ -57,7 +57,12 @@ Integrate a new model provider into `packages/llm` (`@relavium/llm`) as a thin a 6. **Usage** — map native token fields into `Usage.inputTokens`/`outputTokens` (+ `cacheReadTokens`/`cacheWriteTokens` where the provider exposes them). The final `stop` chunk always carries `stopReason` + `usage`. 5. **Register a pricing entry — cost is ours.** Add the canonical model id(s) and per-token prices to the pricing table (`pricing.ts`). `CostTracker` computes the cost from *our* table keyed on the **canonical model id** — never read a cost number from a provider response. This is the same `costMicrocents` that surfaces in the `cost:updated` run event ([sse-event-schema.md](../../../docs/reference/contracts/sse-event-schema.md)); store cost as integer micro-cents (1 micro-cent = 1e-8 USD), never a float. 6. **Wire it into provider selection and the fallback runner.** Register the id in the provider factory and make it selectable by the `withFallback(providers)` runner so an agent's `fallback_chain` can list it. The chain is policy and lives outside the adapter — the adapter stays dumb. The `fallback_chain` field shapes (`model`, `provider`, `max_attempts`) are canonical in [agent-yaml-spec.md](../../../docs/reference/contracts/agent-yaml-spec.md); do not redefine them. Errors must surface as a classified `LlmError` (retryable vs. fatal per [error-handling.md](../../../docs/standards/error-handling.md)) so the runner knows when to fail over. -7. **Add the conformance test for this provider.** The conformance suite is **one shared spec run against every adapter** — it must prove the new adapter: streams text, calls a tool and returns a normalized `tool_call`, returns usage, maps stop reasons to the canonical enum, and surfaces errors as a classified `LlmError` **whose normalized `message`/`code` is secret-free** — include a fixture with a **secret-bearing vendor error** (a key/token/`baseURL` in the upstream error) and assert none of it survives normalization (testing.md §Per-provider conformance + §Security-critical primitive tests; [security-review.md](../../../docs/standards/security-review.md)). Add the provider to the matrix and record its fixtures: +7. **Register the provider on the CLI so its onboarding + management surfaces light up (data-driven — no per-surface UI edit).** The id lives in **two** homes that mirror the seam's closed set; every CLI surface then derives from them: + 1. **`LLM_PROVIDERS`** (`packages/shared/src/constants.ts`) — the canonical closed `ProviderId` enum (`ProviderId = (typeof LLM_PROVIDERS)[number]`, the `ProviderIdSchema` `z.enum`, the **persisted** run-event `provider` field + authored agent YAML). Adding an *arbitrary* new id here opens the closed enum → an ADR ([ADR-0065](../../../docs/decisions/0065-provider-economics-and-extensibility.md) §6 supersede), not a silent edit. + 2. **`KNOWN_PROVIDER_IDS` + `KNOWN_PROVIDERS`** (`apps/cli/src/engine/providers.ts`) — the CLI's per-provider metadata (`displayName`, `baseUrl`, a cheap `testModel` for the live key-check, `pricingUrl`). The first-run onboarding **wizard**, `relavium provider add` / `provider test`, `/doctor --deep`'s key probe, and the `/models` Home key-gate are **all data-driven off these two lists** (they iterate `KNOWN_PROVIDER_IDS` and read `KNOWN_PROVIDERS[id]`), so a registered provider appears in every surface with **no edit to `wizard.ts` / `provider.ts` / `doctor.ts` / the model picker**. + + **Keep the two lists in lock-step.** `KNOWN_PROVIDER_IDS satisfies readonly ProviderId[]` makes the compiler enforce `KNOWN_PROVIDER_IDS ⊆ LLM_PROVIDERS`, but the **reverse is not** compiler-checked: a provider added to `LLM_PROVIDERS` (so a live/static `model_catalog` row can exist for it) yet **missing from `KNOWN_PROVIDER_IDS`** is silently mis-dimmed in the Home — the key-probe filters `KNOWN_PROVIDER_IDS`, so the new provider is never in `keyedProviders`, and `mergeModelCatalog` marks its models `available: false` + `unavailableReason: 'no-key'` **even with a stored key** (the 2.5.G Step-A latent coupling). A guard test in `providers.test.ts` pins the two as **equal sets**, so a missed registration is a red CI run rather than a runtime mis-dim. +8. **Add the conformance test for this provider.** The conformance suite is **one shared spec run against every adapter** — it must prove the new adapter: streams text, calls a tool and returns a normalized `tool_call`, returns usage, maps stop reasons to the canonical enum, and surfaces errors as a classified `LlmError` **whose normalized `message`/`code` is secret-free** — include a fixture with a **secret-bearing vendor error** (a key/token/`baseURL` in the upstream error) and assert none of it survives normalization (testing.md §Per-provider conformance + §Security-critical primitive tests; [security-review.md](../../../docs/standards/security-review.md)). Add the provider to the matrix and record its fixtures: ```bash # Record fixtures live ONCE (key from env, never committed, never logged), then commit them. RELAVIUM_LIVE=1 pnpm --filter @relavium/llm test:conformance:record --provider= @@ -65,11 +70,11 @@ Integrate a new model provider into `packages/llm` (`@relavium/llm`) as a thin a pnpm --filter @relavium/llm test ``` Fixtures (including streamed SSE transcripts) are checked in and reviewed like code; when the provider's wire format changes, **regenerate the fixture, never hand-edit it**. The live suite runs nightly against the real endpoint (keys from CI secrets) as the drift early-warning. -8. **Verify with no vendor leak.** Run the full graph; confirm the boundary lint passes (the SDK import is confined to the adapter file) and no vendor type appears in `index.ts` or any `packages/core` test. +9. **Verify with no vendor leak.** Run the full graph; confirm the boundary lint passes (the SDK import is confined to the adapter file) and no vendor type appears in `index.ts` or any `packages/core` test. ```bash pnpm turbo run lint typecheck test --filter=@relavium/llm... ``` -9. **Commit** with ../commit-and-pr/SKILL.md scoped to the package: `feat(llm): add adapter behind the LLMProvider seam` with a `Refs: ADR-0011` trailer. +10. **Commit** with ../commit-and-pr/SKILL.md scoped to the package: `feat(llm): add adapter behind the LLMProvider seam` with a `Refs: ADR-0011` trailer. ## Outputs - A new `adapters/-adapter.ts` (or a new `baseURL`/capability/pricing entry on the shared OpenAI-compatible adapter) implementing `LlmProvider` with full six-axis normalization. @@ -83,6 +88,7 @@ Integrate a new model provider into `packages/llm` (`@relavium/llm`) as a thin a - [ ] All six normalizations implemented: system-prompt placement, tool schema (with unsupported-keyword stripping if restricted), tool-call/result round-trip (with id synthesis if the provider has no ids), streaming → `StreamChunk`, stop reasons → the 5-value enum, usage → `Usage`. - [ ] Pricing entry added; cost computed from our table on the canonical model id and stored as integer micro-cents (`costMicrocents`), never read from the provider or stored as a float. - [ ] Provider registered in the factory and usable in a `fallback_chain`; errors classified as `LlmError` (retryable/fatal). +- [ ] Id registered in **both** `LLM_PROVIDERS` (`@relavium/shared`) and `KNOWN_PROVIDER_IDS`/`KNOWN_PROVIDERS` (the CLI, with a `testModel`) so the wizard / `provider` / `/doctor` / `/models` surface it data-driven; the `LLM_PROVIDERS`↔`KNOWN_PROVIDER_IDS` lock-step guard test (`providers.test.ts`) is green. A genuinely-new arbitrary id opening the closed enum is an ADR-0065 §6 supersede, not a silent edit. - [ ] Key handling host-aware (ADR-0018): a resolved key attached in-adapter on the Node-style hosts; a key *reference* passed to the Rust `llm_stream` egress on desktop (raw key never in the WebView); never logged/checkpointed/sent to the frontend; `AbortSignal` threaded through. - [ ] Conformance spec passes for this provider with **recorded fixtures** (PR) and is in the **nightly live** matrix; fixtures committed, generated not hand-edited. - [ ] The normalized `LlmError.message`/`code` is **secret-free** (no key / `baseURL` / auth / token) — asserted by a secret-bearing-error conformance fixture, not assumed (a declared-but-untested "already redacted" invariant is a future leak; [security-review.md](../../../docs/standards/security-review.md)). @@ -97,6 +103,7 @@ Integrate a new model provider into `packages/llm` (`@relavium/llm`) as a thin a - Forgetting the provider-specific stream quirk (e.g. OpenAI's `include_usage` opt-in) so the final `stop` chunk has no usage. - Passing a restricted-provider tool schema through without stripping unsupported keywords, or losing tool-call ids on a no-id provider. - Hand-editing a recorded fixture instead of regenerating it; committing a live API key or logging the key. +- Adding the id to `LLM_PROVIDERS` but forgetting the CLI `KNOWN_PROVIDER_IDS`/`KNOWN_PROVIDERS` twin (step 7) — the compiler won't catch it (only `⊆` is enforced), so the provider's models are silently mis-dimmed as `no-key` in the Home and it's un-onboardable via the wizard. The lock-step guard test is what turns this into a red run. - Putting fallback/retry logic inside the adapter — the adapter stays dumb; fallback is the runner's job. ## Related diff --git a/apps/cli/src/engine/providers.test.ts b/apps/cli/src/engine/providers.test.ts index c6129406..27d118fe 100644 --- a/apps/cli/src/engine/providers.test.ts +++ b/apps/cli/src/engine/providers.test.ts @@ -1,11 +1,13 @@ import { parseWorkflow, type WorkflowDefinition } from '@relavium/core'; import type { ProviderRecord } from '@relavium/db'; import { LlmProviderError, type LlmProvider } from '@relavium/llm'; +import { LLM_PROVIDERS } from '@relavium/shared'; import { describe, expect, it, vi } from 'vitest'; import { CHAT_TEXT_CAPABILITY_FLAGS } from '../test-support.js'; import { createProviderResolver, + KNOWN_PROVIDER_IDS, KNOWN_PROVIDERS, neededProviderIds, providerHasKey, @@ -143,6 +145,28 @@ describe('providerKeyEnvVar', () => { }); }); +describe('KNOWN_PROVIDER_IDS ↔ LLM_PROVIDERS lock-step (2.5.G Step-A latent coupling)', () => { + it('registers every seam provider id for CLI onboarding + key-probing (equal sets)', () => { + // `KNOWN_PROVIDER_IDS satisfies readonly ProviderId[]` already enforces KNOWN_PROVIDER_IDS ⊆ LLM_PROVIDERS at + // COMPILE time. This pins the UNENFORCED reverse (LLM_PROVIDERS ⊆ KNOWN_PROVIDER_IDS): a provider in the seam's + // closed enum but MISSING from KNOWN_PROVIDER_IDS is silently mis-dimmed in the Home — the key-probe filters + // KNOWN_PROVIDER_IDS, so the new provider is never in `keyedProviders`, and `mergeModelCatalog` marks its + // model_catalog rows `available: false` + `unavailableReason: 'no-key'` even with a stored key. Adding a + // provider to LLM_PROVIDERS therefore REQUIRES a KNOWN_PROVIDERS entry (incl. a testModel); this red run is + // the reminder. `toEqual` compares Set membership structurally (order-independent). + expect(new Set(KNOWN_PROVIDER_IDS)).toEqual(new Set(LLM_PROVIDERS)); + }); + + it('gives every known provider a non-empty testModel (so its key is live-validatable)', () => { + // The onboarding wizard + `provider test` + `/doctor --deep` all ping `KNOWN_PROVIDERS[id].testModel`; an empty + // string would break the live key-check. The `Record` type guarantees + // a row per id, so this asserts the field's VALUE, not its presence. + for (const id of KNOWN_PROVIDER_IDS) { + expect(KNOWN_PROVIDERS[id].testModel.trim().length).toBeGreaterThan(0); + } + }); +}); + describe('resolver.hasKey / providerHasKey (2.5.G key-awareness)', () => { it('createProviderResolver.hasKey is true for a provider with an env key, false otherwise (never throws)', () => { // Env-only (no keychain) — a key for openai, nothing for the others. diff --git a/apps/cli/src/engine/providers.ts b/apps/cli/src/engine/providers.ts index 2b6e14f8..c7d1d236 100644 --- a/apps/cli/src/engine/providers.ts +++ b/apps/cli/src/engine/providers.ts @@ -89,13 +89,28 @@ export interface ProviderMeta { } /** - * The known providers (each has an `@relavium/llm` adapter). The single home for provider metadata — the - * `relavium provider` command (add / test) and the `/doctor --deep` key probe both read it, so a new provider's - * test model is defined once. + * The known providers (each has an `@relavium/llm` adapter). The single home for CLI provider metadata: + * `displayName`, `baseUrl`, a cheap `testModel` for the live key-check, and the public `pricingUrl`. Every + * provider-facing CLI surface is **data-driven off this map + {@link KNOWN_PROVIDER_IDS}** — the first-run + * onboarding **wizard** ([onboarding/wizard.ts](../onboarding/wizard.ts)), `relavium provider add` / `provider + * test`, the `/doctor --deep` key probe, and the `/models` Home key-gate all iterate `KNOWN_PROVIDER_IDS` and read + * `KNOWN_PROVIDERS[id]`. So registering a provider here (plus its `LLM_PROVIDERS` id — see below) lights up every + * surface with **no per-surface edit**; a new provider's test model / display name is defined once. See the + * `add-llm-adapter` skill for the end-to-end checklist. */ -/** The provider ids the CLI knows how to validate (those with a test model). The const tuple is the SOURCE OF - * TRUTH — `satisfies` validates each is a real `ProviderId` (no cast, no widening), and {@link KNOWN_PROVIDERS} - * is keyed on it, so the two cannot drift; the `/doctor` provider probe iterates it directly. */ +/** The provider ids the CLI knows how to validate + onboard (those with a `testModel`). The const tuple is the + * SOURCE OF TRUTH — `satisfies readonly ProviderId[]` validates each is a real `ProviderId` (no cast, no + * widening), and {@link KNOWN_PROVIDERS} is keyed on it, so THOSE two cannot drift; the wizard, `provider`, and + * `/doctor` probe all iterate it directly. + * + * LOCK-STEP with `LLM_PROVIDERS` (`@relavium/shared` — the canonical closed `ProviderId` enum): the `satisfies` + * above only enforces `KNOWN_PROVIDER_IDS ⊆ LLM_PROVIDERS`. The REVERSE is NOT compiler-checked — a provider + * added to `LLM_PROVIDERS` (so a live/static `model_catalog` row can exist for it) but MISSING here is silently + * mis-dimmed in the Home: the key-probe ([drive-home.tsx](../home/drive-home.tsx)) filters `KNOWN_PROVIDER_IDS`, + * so the new provider is never in `keyedProviders`, and `mergeModelCatalog` marks its models `available: false` + + * `unavailableReason: 'no-key'` EVEN WITH a stored key (the 2.5.G Step-A latent coupling). A guard test + * (`providers.test.ts`) pins the two as equal sets, so a missed registration is a red CI run, not a runtime + * mis-dim. */ export const KNOWN_PROVIDER_IDS = [ 'anthropic', 'openai', From cede676ab7a72cbd6902ea7b4343717076947c1b Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 13:49:23 +0300 Subject: [PATCH 43/68] =?UTF-8?q?docs(cli,skills):=20Step=20C=20review=20f?= =?UTF-8?q?ix=20=E2=80=94=20split=20provider-surface=20access=20mechanisms?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reviewer accuracy nit: `provider add`/`set-key`/`remove-key`/`test` do NOT iterate KNOWN_PROVIDER_IDS — they validate a single supplied id against the wider ProviderIdSchema (z.enum(LLM_PROVIDERS)) then index KNOWN_PROVIDERS[id] for metadata. So a LLM_PROVIDERS↔KNOWN_PROVIDER_IDS desync breaks TWO ways, not one: the KNOWN_PROVIDER_IDS-iterating surfaces (wizard / /doctor --deep / /models key-gate) silently mis-dim the provider `no-key`, while the id-validating `provider` commands crash on the undefined KNOWN_PROVIDERS[id] lookup — which the guard test now prevents. - providers.ts + SKILL.md step 7: split the "data-driven" claim by the two real access patterns; document both desync failure modes. - SKILL.md pitfall: name both failure modes. - SKILL.md Inputs + step 7: precise type chain (ProviderId aliases LlmProviderId = (typeof LLM_PROVIDERS)[number]). No production-behavior change. Toolchain green: 1481 tests, all tasks pass. Refs: ADR-0064, ADR-0065 Co-Authored-By: Claude --- .claude/skills/add-llm-adapter/SKILL.md | 10 +++++----- apps/cli/src/engine/providers.ts | 26 ++++++++++++++++--------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/.claude/skills/add-llm-adapter/SKILL.md b/.claude/skills/add-llm-adapter/SKILL.md index 7add05eb..4b0221ad 100644 --- a/.claude/skills/add-llm-adapter/SKILL.md +++ b/.claude/skills/add-llm-adapter/SKILL.md @@ -22,7 +22,7 @@ Integrate a new model provider into `packages/llm` (`@relavium/llm`) as a thin a ## Inputs | Input | Description | |-------|-------------| -| Provider id | The literal added to the closed `ProviderId` enum — its **canonical home is `LLM_PROVIDERS`** in `packages/shared/src/constants.ts` (`ProviderId = (typeof LLM_PROVIDERS)[number]`, the `ProviderIdSchema` `z.enum`), never a vendor enum. The CLI mirrors it in `KNOWN_PROVIDER_IDS`/`KNOWN_PROVIDERS` (step 7). A genuinely-new arbitrary id **opens** the closed enum — a deliberate supersede per [ADR-0065](../../../docs/decisions/0065-provider-economics-and-extensibility.md) §6, not a silent edit. | +| Provider id | The literal added to the closed `ProviderId` enum — its **canonical home is `LLM_PROVIDERS`** in `packages/shared/src/constants.ts` (`ProviderId` aliases `LlmProviderId = (typeof LLM_PROVIDERS)[number]`, and `ProviderIdSchema = z.enum(LLM_PROVIDERS)`), never a vendor enum. The CLI mirrors it in `KNOWN_PROVIDER_IDS`/`KNOWN_PROVIDERS` (step 7). A genuinely-new arbitrary id **opens** the closed enum — a deliberate supersede per [ADR-0065](../../../docs/decisions/0065-provider-economics-and-extensibility.md) §6, not a silent edit. | | SDK / transport | The official TS SDK to wrap, or `openai` + a custom `baseURL` if OpenAI-compatible. An SDK dependency stays strictly inside `packages/llm/src/adapters/*`. | | Capabilities | `{ tools, streaming, parallelToolCalls, vision, promptCache, reasoning }` — what this provider genuinely supports, for the `supports` capability flags. | | Model ids + pricing | Canonical model ids this adapter serves and their per-token input/output (and cache) prices, for the pricing table. | @@ -58,10 +58,10 @@ Integrate a new model provider into `packages/llm` (`@relavium/llm`) as a thin a 5. **Register a pricing entry — cost is ours.** Add the canonical model id(s) and per-token prices to the pricing table (`pricing.ts`). `CostTracker` computes the cost from *our* table keyed on the **canonical model id** — never read a cost number from a provider response. This is the same `costMicrocents` that surfaces in the `cost:updated` run event ([sse-event-schema.md](../../../docs/reference/contracts/sse-event-schema.md)); store cost as integer micro-cents (1 micro-cent = 1e-8 USD), never a float. 6. **Wire it into provider selection and the fallback runner.** Register the id in the provider factory and make it selectable by the `withFallback(providers)` runner so an agent's `fallback_chain` can list it. The chain is policy and lives outside the adapter — the adapter stays dumb. The `fallback_chain` field shapes (`model`, `provider`, `max_attempts`) are canonical in [agent-yaml-spec.md](../../../docs/reference/contracts/agent-yaml-spec.md); do not redefine them. Errors must surface as a classified `LlmError` (retryable vs. fatal per [error-handling.md](../../../docs/standards/error-handling.md)) so the runner knows when to fail over. 7. **Register the provider on the CLI so its onboarding + management surfaces light up (data-driven — no per-surface UI edit).** The id lives in **two** homes that mirror the seam's closed set; every CLI surface then derives from them: - 1. **`LLM_PROVIDERS`** (`packages/shared/src/constants.ts`) — the canonical closed `ProviderId` enum (`ProviderId = (typeof LLM_PROVIDERS)[number]`, the `ProviderIdSchema` `z.enum`, the **persisted** run-event `provider` field + authored agent YAML). Adding an *arbitrary* new id here opens the closed enum → an ADR ([ADR-0065](../../../docs/decisions/0065-provider-economics-and-extensibility.md) §6 supersede), not a silent edit. - 2. **`KNOWN_PROVIDER_IDS` + `KNOWN_PROVIDERS`** (`apps/cli/src/engine/providers.ts`) — the CLI's per-provider metadata (`displayName`, `baseUrl`, a cheap `testModel` for the live key-check, `pricingUrl`). The first-run onboarding **wizard**, `relavium provider add` / `provider test`, `/doctor --deep`'s key probe, and the `/models` Home key-gate are **all data-driven off these two lists** (they iterate `KNOWN_PROVIDER_IDS` and read `KNOWN_PROVIDERS[id]`), so a registered provider appears in every surface with **no edit to `wizard.ts` / `provider.ts` / `doctor.ts` / the model picker**. + 1. **`LLM_PROVIDERS`** (`packages/shared/src/constants.ts`) — the canonical closed `ProviderId` enum (`ProviderId` aliases `LlmProviderId = (typeof LLM_PROVIDERS)[number]`; `ProviderIdSchema = z.enum(LLM_PROVIDERS)`; the **persisted** run-event `provider` field + authored agent YAML). Adding an *arbitrary* new id here opens the closed enum → an ADR ([ADR-0065](../../../docs/decisions/0065-provider-economics-and-extensibility.md) §6 supersede), not a silent edit. + 2. **`KNOWN_PROVIDER_IDS` + `KNOWN_PROVIDERS`** (`apps/cli/src/engine/providers.ts`) — the CLI's per-provider metadata (`displayName`, `baseUrl`, a cheap `testModel` for the live key-check, `pricingUrl`). Every provider-facing CLI surface is **data-driven off these two lists**, so a registered provider appears everywhere with **no edit to `wizard.ts` / `provider.ts` / `doctor.ts` / the model picker** — via one of two access patterns: the first-run onboarding **wizard**, `/doctor --deep`'s key probe, and the `/models` Home key-gate **iterate `KNOWN_PROVIDER_IDS`**; `relavium provider add` / `set-key` / `remove-key` / `test` **validate one supplied id against the wider `ProviderIdSchema` (`z.enum(LLM_PROVIDERS)`) and then index `KNOWN_PROVIDERS[id]`** for its metadata. - **Keep the two lists in lock-step.** `KNOWN_PROVIDER_IDS satisfies readonly ProviderId[]` makes the compiler enforce `KNOWN_PROVIDER_IDS ⊆ LLM_PROVIDERS`, but the **reverse is not** compiler-checked: a provider added to `LLM_PROVIDERS` (so a live/static `model_catalog` row can exist for it) yet **missing from `KNOWN_PROVIDER_IDS`** is silently mis-dimmed in the Home — the key-probe filters `KNOWN_PROVIDER_IDS`, so the new provider is never in `keyedProviders`, and `mergeModelCatalog` marks its models `available: false` + `unavailableReason: 'no-key'` **even with a stored key** (the 2.5.G Step-A latent coupling). A guard test in `providers.test.ts` pins the two as **equal sets**, so a missed registration is a red CI run rather than a runtime mis-dim. + **Keep the two lists in lock-step.** `KNOWN_PROVIDER_IDS satisfies readonly ProviderId[]` makes the compiler enforce `KNOWN_PROVIDER_IDS ⊆ LLM_PROVIDERS`, but the **reverse is not** compiler-checked: a provider added to `LLM_PROVIDERS` (so a live/static `model_catalog` row can exist for it) yet **missing from `KNOWN_PROVIDER_IDS`** breaks two ways — it is silently mis-dimmed in the Home (the key-probe filters `KNOWN_PROVIDER_IDS`, so the new provider is never in `keyedProviders`, and `mergeModelCatalog` marks its models `available: false` + `unavailableReason: 'no-key'` **even with a stored key** — the 2.5.G Step-A latent coupling), and `provider add`/`set-key`/`test` (which accept the wider `LLM_PROVIDERS`) would **throw** on the `undefined` `KNOWN_PROVIDERS[id]` metadata lookup. A guard test in `providers.test.ts` pins the two as **equal sets**, so a missed registration is a red CI run rather than either runtime failure. 8. **Add the conformance test for this provider.** The conformance suite is **one shared spec run against every adapter** — it must prove the new adapter: streams text, calls a tool and returns a normalized `tool_call`, returns usage, maps stop reasons to the canonical enum, and surfaces errors as a classified `LlmError` **whose normalized `message`/`code` is secret-free** — include a fixture with a **secret-bearing vendor error** (a key/token/`baseURL` in the upstream error) and assert none of it survives normalization (testing.md §Per-provider conformance + §Security-critical primitive tests; [security-review.md](../../../docs/standards/security-review.md)). Add the provider to the matrix and record its fixtures: ```bash # Record fixtures live ONCE (key from env, never committed, never logged), then commit them. @@ -103,7 +103,7 @@ Integrate a new model provider into `packages/llm` (`@relavium/llm`) as a thin a - Forgetting the provider-specific stream quirk (e.g. OpenAI's `include_usage` opt-in) so the final `stop` chunk has no usage. - Passing a restricted-provider tool schema through without stripping unsupported keywords, or losing tool-call ids on a no-id provider. - Hand-editing a recorded fixture instead of regenerating it; committing a live API key or logging the key. -- Adding the id to `LLM_PROVIDERS` but forgetting the CLI `KNOWN_PROVIDER_IDS`/`KNOWN_PROVIDERS` twin (step 7) — the compiler won't catch it (only `⊆` is enforced), so the provider's models are silently mis-dimmed as `no-key` in the Home and it's un-onboardable via the wizard. The lock-step guard test is what turns this into a red run. +- Adding the id to `LLM_PROVIDERS` but forgetting the CLI `KNOWN_PROVIDER_IDS`/`KNOWN_PROVIDERS` twin (step 7) — the compiler won't catch it (only `⊆` is enforced), so the provider's models are silently mis-dimmed `no-key` in the Home (the `KNOWN_PROVIDER_IDS`-iterating surfaces) **and** `provider add`/`test` crash on the `undefined` `KNOWN_PROVIDERS[id]` metadata lookup (the id-validating surfaces). The lock-step guard test is what turns this into a red run. - Putting fallback/retry logic inside the adapter — the adapter stays dumb; fallback is the runner's job. ## Related diff --git a/apps/cli/src/engine/providers.ts b/apps/cli/src/engine/providers.ts index c7d1d236..8c91214b 100644 --- a/apps/cli/src/engine/providers.ts +++ b/apps/cli/src/engine/providers.ts @@ -91,12 +91,19 @@ export interface ProviderMeta { /** * The known providers (each has an `@relavium/llm` adapter). The single home for CLI provider metadata: * `displayName`, `baseUrl`, a cheap `testModel` for the live key-check, and the public `pricingUrl`. Every - * provider-facing CLI surface is **data-driven off this map + {@link KNOWN_PROVIDER_IDS}** — the first-run - * onboarding **wizard** ([onboarding/wizard.ts](../onboarding/wizard.ts)), `relavium provider add` / `provider - * test`, the `/doctor --deep` key probe, and the `/models` Home key-gate all iterate `KNOWN_PROVIDER_IDS` and read - * `KNOWN_PROVIDERS[id]`. So registering a provider here (plus its `LLM_PROVIDERS` id — see below) lights up every - * surface with **no per-surface edit**; a new provider's test model / display name is defined once. See the - * `add-llm-adapter` skill for the end-to-end checklist. + * provider-facing CLI surface is **data-driven off this map + {@link KNOWN_PROVIDER_IDS}** — registering a provider + * here (plus its `LLM_PROVIDERS` id — see below) lights up every surface with **no per-surface edit**, via one of + * two access patterns: + * - **Iterate `KNOWN_PROVIDER_IDS`** (enumerate every known provider): the first-run onboarding **wizard** + * ([onboarding/wizard.ts](../onboarding/wizard.ts)), the `/doctor --deep` key probe, and the `/models` Home + * key-gate ([drive-home.tsx](../home/drive-home.tsx)). A provider absent from `KNOWN_PROVIDER_IDS` is simply + * invisible / mis-dimmed here (see the {@link KNOWN_PROVIDER_IDS} lock-step note). + * - **Validate one supplied id, then index `KNOWN_PROVIDERS[id]`** for its metadata: `relavium provider add` / + * `set-key` / `remove-key` / `test`, which accept a single name gated by the WIDER `ProviderIdSchema` + * (`z.enum(LLM_PROVIDERS)`) — so an `LLM_PROVIDERS` id absent from `KNOWN_PROVIDERS` would pass validation and + * then throw on the `undefined` metadata lookup (the second reason the two lists must stay in lock-step). + * A new provider's test model / display name is thus defined once. See the `add-llm-adapter` skill for the + * end-to-end checklist. */ /** The provider ids the CLI knows how to validate + onboard (those with a `testModel`). The const tuple is the * SOURCE OF TRUTH — `satisfies readonly ProviderId[]` validates each is a real `ProviderId` (no cast, no @@ -108,9 +115,10 @@ export interface ProviderMeta { * added to `LLM_PROVIDERS` (so a live/static `model_catalog` row can exist for it) but MISSING here is silently * mis-dimmed in the Home: the key-probe ([drive-home.tsx](../home/drive-home.tsx)) filters `KNOWN_PROVIDER_IDS`, * so the new provider is never in `keyedProviders`, and `mergeModelCatalog` marks its models `available: false` + - * `unavailableReason: 'no-key'` EVEN WITH a stored key (the 2.5.G Step-A latent coupling). A guard test - * (`providers.test.ts`) pins the two as equal sets, so a missed registration is a red CI run, not a runtime - * mis-dim. */ + * `unavailableReason: 'no-key'` EVEN WITH a stored key (the 2.5.G Step-A latent coupling). The `provider` + * commands (`add`/`set-key`/`test`), whose id-validation accepts the wider `LLM_PROVIDERS`, would instead throw on + * the `undefined` `KNOWN_PROVIDERS[id]` metadata lookup. A guard test (`providers.test.ts`) pins the two as equal + * sets, so a missed registration is a red CI run — not either runtime failure. */ export const KNOWN_PROVIDER_IDS = [ 'anthropic', 'openai', From 1653f2d8af8a7600d6caac50c9aa4a9d4cb18810 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 14:31:17 +0300 Subject: [PATCH 44/68] =?UTF-8?q?feat(cli,docs):=202.5.G=20Step=20D=20(1/2?= =?UTF-8?q?)=20=E2=80=94=20mid-session=20model=20reseat=20engine=20(ADR-00?= =?UTF-8?q?59)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Answers question #6's second half ("switch models inside a chat, like competitors"). Finalizes ADR-0059 (Proposed → Accepted) + amends ADR-0024 (the one-model-per-lifetime rule is REFINED, not reversed), then wires the host-side reseat engine — the trigger UI (the ink /models picker overlay) lands in the next commit. A mid-session model switch is a host-side RESEAT: like `/clear` it swaps the session via runReplLoop, but instead of a fresh session it RESUMES the just-ended one under a new-model agent (drops the original fallback_chain), carrying the text-only transcript + cumulative cost/turns. Reuses the proven `/clear` swap + `chat-resume` transcript path — ZERO engine change (packages/core/llm untouched). - ChatLineHandler: `onReseat(target)` + `reseatTarget()`; `stopReason()` widened to 'exit'|'clear'|'reseat'. ChatDriveContext.onReseat (TTY-only, ADR-0049 parity — a live reseat is gated off under --json/plain, like /clear). ChatDriveOutcome carries the target. - runReplLoop: unified clear+reseat swap branch; createReseatRebuild + buildReseatWiring (acquire-then-guard on MCP) shared by `chat` + `chat-resume`; seedResumedWiring factored out of chat-resume so both resume paths seed identically. - repl-info: modelSwitchNotice — the reseat intro discloses the text-only carry (shared honesty with the chat-resume family). - home-controller: HomeChatSession.stopReason widened (safe no-op — the in-Home chat doesn't wire onReseat yet; in-Home live reseat is a documented follow-up). - Tests: reseat continues the SAME session + rebinds the model + carries the transcript; rebuild-failure stays resumable + ends cleanly; the non-interactive gate. DEFERRED (ADR-0059 note): per-message/session `modelId` attribution — `session_messages .model_id`/`agent_sessions.model_id` are FKs to `model_catalog.id` (a UUID, not the model string; nothing populates them today), so it needs a catalog-id resolution that belongs with the 2.6.C cost breakdown that consumes it. The reseat is complete + correct without it. Toolchain green: 1484 tests, all lint/typecheck/test/build tasks pass. Refs: ADR-0059, ADR-0024 Co-Authored-By: Claude --- apps/cli/src/chat/persister.ts | 7 +- apps/cli/src/chat/repl-info.ts | 16 + apps/cli/src/commands/chat.test.ts | 86 ++++- apps/cli/src/commands/chat.ts | 356 +++++++++++++++--- apps/cli/src/render/tui/home-controller.ts | 9 +- ...24-agent-first-entry-point-agentsession.md | 8 +- .../0059-cli-mid-session-model-reseat.md | 25 +- 7 files changed, 448 insertions(+), 59 deletions(-) diff --git a/apps/cli/src/chat/persister.ts b/apps/cli/src/chat/persister.ts index a31905e9..621566d6 100644 --- a/apps/cli/src/chat/persister.ts +++ b/apps/cli/src/chat/persister.ts @@ -79,7 +79,9 @@ export function createSessionPersister(deps: SessionPersisterDeps): SessionPersi // prior marker is interleaved — the step-1-review trap). Seeded from the durable transcript on resume. const realMessageSeqs: number[] = []; - /** Append a REAL transcript row + record its sequence for the boundary mapping. */ + /** Append a REAL transcript row + record its sequence for the boundary mapping. (Per-message `modelId` + * attribution — ADR-0059 — is DEFERRED to Phase 2.6.C: `session_messages.model_id` is a FK to + * `model_catalog.id` (a UUID), so it needs a model-string→catalog-id resolution the cost breakdown will own.) */ const appendText = (role: 'user' | 'assistant', text: string): void => { const seq = sequenceNumber++; realMessageSeqs.push(seq); @@ -130,6 +132,9 @@ export function createSessionPersister(deps: SessionPersisterDeps): SessionPersi createdAt, updatedAt: iso(), ...(title === undefined ? {} : { title }), + // NOTE: the session-level `modelId` (ADR-0059) is DEFERRED with the per-message attribution (see `appendText`): + // `agent_sessions.model_id` is a FK to `model_catalog.id` (a UUID), not the raw model string, so populating it + // needs the same catalog resolution the 2.6.C cost breakdown will own. Left NULL until then (like run history). }); const onEvent = (event: SessionStreamHandleEvent): void => { diff --git a/apps/cli/src/chat/repl-info.ts b/apps/cli/src/chat/repl-info.ts index 3516edc3..82aa8632 100644 --- a/apps/cli/src/chat/repl-info.ts +++ b/apps/cli/src/chat/repl-info.ts @@ -95,3 +95,19 @@ export function clearedNotice(oldSessionId: string): string { const safeId = sanitizeInline(oldSessionId); return `✨ Started a fresh conversation. The previous one is saved — resume it with \`relavium chat-resume ${safeId}\`.`; } + +/** + * The mid-session `/models` model-SWITCH notice ([ADR-0059](../../../../docs/decisions/0059-cli-mid-session-model-reseat.md)) + * — the intro line of the reseated session. It confirms the NEW model, states how many turns carried, and + * DISCLOSES what a host-side reseat does not carry: the transcript continues **text-only**, so the new model does + * not see prior tool calls or file contents (the same honesty the `chat-resume` family surfaces). The model id is + * a `model_catalog` id (curated), but `sanitizeInline`-guarded defensively — a live-catalog id is provider-sourced + * and `history.db` is shared across surfaces, so a crafted value must not smuggle a terminal escape into the notice. + */ +export function modelSwitchNotice(newModel: string, carriedTurns: number): string { + const turns = `${carriedTurns} prior ${carriedTurns === 1 ? 'turn' : 'turns'}`; + return ( + `⇄ Switched to ${sanitizeInline(newModel)} — ${turns} carried. The new model sees the text transcript only ` + + `(not prior tool calls or file contents). Type a message, or /exit to quit.` + ); +} diff --git a/apps/cli/src/commands/chat.test.ts b/apps/cli/src/commands/chat.test.ts index b2732293..9ed97377 100644 --- a/apps/cli/src/commands/chat.test.ts +++ b/apps/cli/src/commands/chat.test.ts @@ -520,7 +520,7 @@ describe('chatCommand', () => { const code = await chatCommand({ agent: undefined }, { ...d, buildSession, drive: driveClear }); expect(code).toBe(EXIT_CODES.chatEnded); // ends cleanly despite the failed rebuild (never hangs/loops) - expect(err()).toContain('could not start a fresh session after /clear'); // actionable hint on stderr + expect(err()).toContain('could not start a new session after /clear'); // actionable, swap-kind-aware hint expect(err()).toContain('relavium chat-resume id-0'); // names the OLD, still-resumable conversation expect(store.loadFull('id-0')?.session.status).toBe('ended'); // the prior session is persisted + resumable }); @@ -539,6 +539,90 @@ describe('chatCommand', () => { expect(rows?.session.status).toBe('ended'); }); + it('/models reseat: rebinds the model on the SAME session, carrying the transcript + per-turn attribution (ADR-0059)', async () => { + const { d, store } = deps([], [textTurn('sonnet reply'), textTurn('opus reply')]); + // A live reseat is TTY-interactive only (like `/clear`), so `onReseat` is wired only on an interactive io. + const interactiveIo = { ...d.io, stdoutIsTty: true }; + const seen: string[] = []; + const intros: (string | undefined)[] = []; + let call = 0; + const reseatThenExit: ChatDriver = async (ctx) => { + seen.push(ctx.handle.sessionId); + intros.push(ctx.intro); + ctx.startSession(); + if (call++ === 0) { + await ctx.processLine('first'); // a turn on the sonnet-bound session ⇒ persisted (attributed to sonnet) + ctx.onReseat?.({ modelId: 'claude-opus-4-8', provider: 'anthropic' }); // switch to opus + return { kind: ctx.stopReason() }; // 'reseat' + } + await ctx.processLine('second'); // a turn on the opus-bound session ⇒ persisted (attributed to opus) + await ctx.processLine('/exit'); + return { kind: ctx.stopReason() }; + }; + const code = await chatCommand( + { agent: undefined }, + { ...d, io: interactiveIo, drive: reseatThenExit }, + ); + expect(code).toBe(EXIT_CODES.chatEnded); + + expect(seen).toHaveLength(2); + expect(seen[0]).toBe(seen[1]); // a reseat CONTINUES the same session (unlike /clear's new id) + expect(intros[0]).toBeUndefined(); // the original session has no intro + expect(intros[1]).toContain('Switched to claude-opus-4-8'); // the reseat disclosure intro + expect(intros[1]).toContain('text transcript only'); // the tool-context-not-carried disclosure + + const full = store.loadFull('id-0'); + expect(full?.session.agentSnapshot?.model).toBe('claude-opus-4-8'); // rebound to the target model + // The transcript carried across the switch: turn 1 (sonnet) + turn 2 (opus) — 4 sequenced rows, one session. + expect(full?.messages.map((m) => m.role)).toEqual(['user', 'assistant', 'user', 'assistant']); + expect(full?.messages[1]?.content[0]).toEqual({ type: 'text', text: 'sonnet reply' }); + expect(full?.messages[3]?.content[0]).toEqual({ type: 'text', text: 'opus reply' }); + expect(full?.session.totalCostMicrocents).toBeGreaterThan(0); // both turns' cost accrued (carried, not reset) + }); + + it('/models reseat whose rebuild fails surfaces the resumable prior session and ends cleanly (ADR-0059)', async () => { + const { d, err, store } = deps([], [textTurn('hi there')]); + const interactiveIo = { ...d.io, stdoutIsTty: true }; + // The reseat's resumed build REJECTS (e.g. a transient MCP/key fault binding the new model). + const buildResumedSession: typeof buildResumedChatSession = () => + Promise.reject(new Error('reseat build failed')); + const reseat: ChatDriver = async (ctx) => { + ctx.startSession(); + await ctx.processLine('hello'); // a real turn on the OLD session ⇒ persisted + resumable + ctx.onReseat?.({ modelId: 'claude-opus-4-8', provider: 'anthropic' }); + return { kind: ctx.stopReason() }; + }; + const code = await chatCommand( + { agent: undefined }, + { ...d, io: interactiveIo, buildResumedSession, drive: reseat }, + ); + expect(code).toBe(EXIT_CODES.chatEnded); // ends cleanly despite the failed rebuild (never hangs/loops) + expect(err()).toContain('could not start a new session after a model switch'); // swap-kind-aware hint + expect(err()).toContain('relavium chat-resume id-0'); // names the OLD, still-resumable conversation + expect(store.loadFull('id-0')?.session.status).toBe('ended'); // the prior session is persisted + resumable + }); + + it('a live reseat is gated OFF on a non-interactive surface — onReseat is not wired (ADR-0049 parity)', async () => { + // The default deps() harness io is non-TTY, so `chatIsInteractive` is false → `onReseat` is NOT wired (a + // machine/plain stream stays one session lifecycle, exactly as `/clear` is gated off there). + const { d, store } = deps([], [textTurn('hi there')]); + let onReseatWired = true; + const driver: ChatDriver = async (ctx) => { + ctx.startSession(); + await ctx.processLine('hello'); + onReseatWired = ctx.onReseat !== undefined; + ctx.onReseat?.({ modelId: 'claude-opus-4-8', provider: 'anthropic' }); // a no-op when unwired + return { kind: ctx.stopReason() }; + }; + const code = await chatCommand({ agent: undefined }, { ...d, drive: driver }); + expect(code).toBe(EXIT_CODES.chatEnded); + expect(onReseatWired).toBe(false); // no live reseat on a machine/plain stream + // NOT reseated: id-0 kept its one 'hello' exchange, still bound to the ORIGINAL model. + const full = store.loadFull('id-0'); + expect(full?.messages).toHaveLength(2); + expect(full?.session.agentSnapshot?.model).toBe('claude-sonnet-4-6'); + }); + it('chat --json drives the headless stream: stdout pure NDJSON, the unknown-slash diagnostic on stderr', async () => { const { io, out, err } = captureIo(); const store = createSessionStore(client.db); diff --git a/apps/cli/src/commands/chat.ts b/apps/cli/src/commands/chat.ts index d46a5e48..855f5185 100644 --- a/apps/cli/src/commands/chat.ts +++ b/apps/cli/src/commands/chat.ts @@ -8,6 +8,8 @@ import { type SessionStreamHandleEvent, type UserCommandOutcome, } from '@relavium/core'; +import type { ProviderId } from '@relavium/llm'; +import type { AgentSessionRecord } from '@relavium/shared'; import { exportSession } from '../chat/export.js'; import { formatDoctorReport, runDoctorChecks, type DoctorProbes } from '../chat/doctor.js'; import { assembleDoctorProbes } from '../chat/doctor-host.js'; @@ -16,6 +18,7 @@ import { clearedNotice, compactionNotice, costNotice, + modelSwitchNotice, trimNotice, } from '../chat/repl-info.js'; import { discoverCatalog, type CatalogEntry, type CatalogKind } from '../workflows/catalog.js'; @@ -41,6 +44,7 @@ import { buildResumedChatSession, type BuildChatSessionOptions, type BuiltChatSession, + type BuiltResumedChatSession, } from '../chat/session-host.js'; import { loadResolvedConfig } from '../config/load.js'; import { assembleToolEnv } from '../engine/tool-host/assemble.js'; @@ -93,6 +97,17 @@ function closeQuietly(io: CliIo, label: string, close: () => void): void { } } +/** + * A mid-session model-switch target ([ADR-0059](../../../../docs/decisions/0059-cli-mid-session-model-reseat.md)): + * the picked model + its provider. The reseat rebuilds the session bound to this model (dropping the original + * `fallback_chain`), carrying the text-only transcript + cumulative cost/turns. Both fields come from the picker's + * chosen `ModelCatalogEntry`, so the provider is authoritative (never re-inferred from the id). + */ +export interface ReseatTarget { + readonly modelId: string; + readonly provider: ProviderId; +} + /** What an interactive driver receives — the command core's seam, so a driver never touches the session directly. */ export interface ChatDriveContext { /** @@ -102,15 +117,24 @@ export interface ChatDriveContext { readonly startSession: () => void; /** Handle one line of user input (a slash command or a chat message). Awaits the turn for a message. */ readonly processLine: (line: string, display?: string) => Promise; - /** `true` once `/exit` or `/cancel` (or `/clear`) has run — the driver stops reading input. */ + /** `true` once `/exit` or `/cancel` (or `/clear`, or a `/models` reseat) has run — the driver stops reading input. */ readonly shouldStop: () => boolean; /** - * WHY the driver's input loop ended (ADR-0062 §7) — `'exit'` (`/exit`, `/cancel`, or an input EOF) or `'clear'` - * (`/clear`, TTY-interactive only). The driver returns `{ kind: ctx.stopReason() }`; the standalone re-drive loop - * ({@link runReplLoop}) reads a `'clear'` to swap in a FRESH session. The `/clear` interactive gate keeps - * `stopReason()` at `'exit'` under `--json` / plain non-TTY, so those drivers only ever return `'exit'`. + * WHY the driver's input loop ended (ADR-0062 §7 · [ADR-0059](../../../../docs/decisions/0059-cli-mid-session-model-reseat.md)) + * — `'exit'` (`/exit`, `/cancel`, or an input EOF), `'clear'` (`/clear`, TTY-interactive only), or `'reseat'` + * (a `/models` mid-session model switch, TTY-interactive only). The driver returns `{ kind: ctx.stopReason() }`; + * the standalone re-drive loop ({@link runReplLoop}) reads a `'clear'` to swap in a FRESH session and a `'reseat'` + * to swap in a NEW-model session carrying the transcript. Both interactive gates keep `stopReason()` at `'exit'` + * under `--json` / plain non-TTY, so those drivers only ever return `'exit'`. */ - readonly stopReason: () => 'exit' | 'clear'; + readonly stopReason: () => 'exit' | 'clear' | 'reseat'; + /** + * Switch the bound model mid-session (ADR-0059) — the ink model-picker overlay calls this on accept. It signals a + * host-side reseat (a new instance bound to `target`), so like `/clear` it sets the stop state; the driver then + * ends and {@link runReplLoop} rebuilds. Absent on a non-interactive driver (plain/`--json`), where a live reseat + * is unavailable — one machine stream is one session lifecycle (ADR-0049), exactly as `/clear` is gated off there. + */ + readonly onReseat?: (target: ReseatTarget) => void; /** The live session stream (the driver renders it: ink reduces it into the store; plain prints it). */ readonly handle: SessionHandle; /** The view store the ink renderer projects (`apply` already wired by the ink driver). */ @@ -170,12 +194,15 @@ export interface ChatDriveContext { ) => Promise; } /** - * How a {@link ChatDriver}'s input loop ended (ADR-0062 §7): `'exit'` ends the REPL (exit 4); `'clear'` tells the - * standalone {@link runReplLoop} to tear the current session down and re-drive over a FRESH one. A `/clear` is - * TTY-interactive only, so `--json` / plain drivers only ever return `'exit'`. + * How a {@link ChatDriver}'s input loop ended (ADR-0062 §7 · ADR-0059): `'exit'` ends the REPL (exit 4); `'clear'` + * tells the standalone {@link runReplLoop} to tear the current session down and re-drive over a FRESH one; `'reseat'` + * tells it to re-drive over a NEW-model session carrying the transcript (`target` is the picked model). Both `/clear` + * and a `/models` reseat are TTY-interactive only, so `--json` / plain drivers only ever return `'exit'`. `target` is + * present iff `kind === 'reseat'` — {@link driveOneSession} attaches it from the line handler's captured target. */ export interface ChatDriveOutcome { - readonly kind: 'exit' | 'clear'; + readonly kind: 'exit' | 'clear' | 'reseat'; + readonly target?: ReseatTarget; } export type ChatDriver = (ctx: ChatDriveContext) => Promise; @@ -185,6 +212,9 @@ export interface ChatCommandDeps { readonly providers?: ProviderResolver; /** Injectable session builder (tests inject a scripted provider via providers). Default {@link buildChatSession}. */ readonly buildSession?: typeof buildChatSession; + /** Injectable RESUMED-session builder — used by the `/models` reseat rebuild (ADR-0059), which continues the + * just-ended session under a new-model agent. Default {@link buildResumedChatSession}. */ + readonly buildResumedSession?: typeof buildResumedChatSession; /** Injectable session-store opener (tests pass an in-memory store). Default {@link openSessionStore}. */ readonly openSessionStore?: (homeDir: string) => OpenedSessionStore; /** The MCP named-secret resolver (2.R Step 4) — production injects the keychain-backed one (specs.ts); default env-only. */ @@ -332,6 +362,21 @@ export async function chatCommand(args: ChatCommandArgs, deps: ChatCommandDeps): // mid-session `models pricing` write applies to the next cleared session — no captured value to thread here. }); + // The `/models` reseat rebuild (ADR-0059) — a NEW-model session carrying the transcript. It reloads the + // just-ended session from the SHARED db, so it does not close over `built.agent` (the reseat swaps the model). + const reseatRebuild = createReseatRebuild({ + chat: config.chat, + now, + uuid, + providers, + mcpSecretResolver, + mcpRegistrations: config.mcpServers, + opened, + buildResumedSession: deps.buildResumedSession ?? buildResumedChatSession, + io: deps.io, + global: deps.global, + }); + return runReplLoop( { built, @@ -346,6 +391,7 @@ export async function chatCommand(args: ChatCommandArgs, deps: ChatCommandDeps): }, deps, rebuild, + reseatRebuild, ); } @@ -410,25 +456,10 @@ export async function chatResumeCommand( closeMcp = resumed.closeMcp; surfaceMcpSkipped(deps.io, resumed.mcpSkipped); built = resumed; - // Seed the view header: a resumed session never re-emits `session:started`, so without this the footer - // would show no model and zero cost/turns until the first new turn (the durable record is unaffected). - store = createChatStore(deps.global.color, { - agentRef: resumed.agent.id, - model: resumed.agent.model, - cumulativeCostMicrocents: resumed.resumeState.cumulativeCostMicrocents, - turnCount: resumed.resumeState.turnCount, - }); - persister = createSessionPersister({ - store: opened.store, - handle: resumed.handle, - sessionId: resumed.sessionId, - agent: resumed.agent, - context: resumed.context, - now, - uuid, - // Continue the durable transcript past its last sequence number (start() adopts the row + its totals). - initialSequenceNumber: resumed.nextSequenceNumber, - }); + // Seed the view header + persister via the SHARED resumed-wiring assembly (the same the `/models` reseat uses): + // a resumed session never re-emits `session:started`, so the seeded store shows the model + carried cost/turns + // from the first frame, and the persister continues past the last durable sequence number. + ({ store, persister } = seedResumedWiring(resumed, opened, deps.global.color, now, uuid)); const turns = resumed.resumeState.turnCount; // `sessionId` is only schema-constrained to a non-empty string (the CLI mints a UUID, but `history.db` is // shared with other surfaces) — sanitize it before it reaches the TTY, exactly as `chat-list` does (the @@ -485,6 +516,21 @@ export async function chatResumeCommand( // A `/clear` rebuild re-reads the user-pricing overlay FRESH from the shared db (buildFreshChatWiring). }); + // The `/models` reseat rebuild (ADR-0059) — a NEW-model session carrying the transcript, over the SHARED db. + // Reloads the just-ended session (which may itself already be a resume), so it needs no captured agent. + const reseatRebuild = createReseatRebuild({ + chat: config.chat, + now, + uuid, + providers, + mcpSecretResolver, + mcpRegistrations: config.mcpServers, + opened, + buildResumedSession: deps.buildResumedSession ?? buildResumedChatSession, + io: deps.io, + global: deps.global, + }); + // A resumed session already landed at idle inside `AgentSession.resume`; calling start() would throw and // re-emitting `session:started` would double a terminal-less lifecycle event — so startSession is a no-op. return runReplLoop( @@ -502,6 +548,7 @@ export async function chatResumeCommand( }, deps, rebuild, + reseatRebuild, ); } @@ -598,11 +645,16 @@ export interface ChatLineHandler extends ChatModeControl { readonly processLine: (raw: string, display?: string) => Promise; /** Emit the session's sole terminal (`session:cancelled`, idempotent) — the teardown caller fires it. */ readonly cancelOnce: () => void; - /** `true` once `/exit`, `/cancel`, or `/clear` has run — the driver stops reading input. */ + /** `true` once `/exit`, `/cancel`, `/clear`, or a `/models` reseat has run — the driver stops reading input. */ readonly shouldStop: () => boolean; - /** WHY the loop stopped (ADR-0062 §7) — `'clear'` after a `/clear`, else `'exit'`. The standalone re-drive loop - * swaps in a fresh session on `'clear'`; the Home reads it to swap-in-place vs. return to the bare Home. */ - readonly stopReason: () => 'exit' | 'clear'; + /** WHY the loop stopped (ADR-0062 §7 · ADR-0059) — `'reseat'` after a `/models` switch, `'clear'` after a `/clear`, + * else `'exit'`. The standalone re-drive loop swaps in a new-model / fresh session accordingly; the Home reads it + * to swap-in-place vs. return to the bare Home. */ + readonly stopReason: () => 'exit' | 'clear' | 'reseat'; + /** Request a mid-session model switch (ADR-0059) — sets the stop state + captures the target for the reseat. */ + readonly onReseat: (target: ReseatTarget) => void; + /** The captured reseat target once {@link onReseat} fired (else `undefined`) — {@link driveOneSession} reads it. */ + readonly reseatTarget: () => ReseatTarget | undefined; } /** @@ -658,6 +710,10 @@ export function createChatLineHandler( // Set by `/clear` (ADR-0062 §7): the loop stopped to SWAP the session, not to end the REPL — `stopReason` reports // it so the surface (the standalone re-drive loop, or the Home) rebuilds a fresh session instead of exiting. let clearRequested = false; + // Set by a `/models` mid-session model switch (ADR-0059): like `/clear`, the loop stopped to SWAP the session — + // but to a NEW-model session carrying the transcript, not a fresh one. `stopReason` reports `'reseat'` and the + // surface rebuilds via the reseat path; the captured target is the picked model/provider. + let reseatRequested: ReseatTarget | undefined; const cancelOnce = (): void => { if (!cancelled) { cancelled = true; @@ -904,7 +960,15 @@ export function createChatLineHandler( processLine, cancelOnce, shouldStop: () => stop, - stopReason: () => (clearRequested ? 'clear' : 'exit'), + // Priority: a reseat is a swap-to-new-model, a clear is a swap-to-fresh, else the REPL ends. A `/models` reseat + // and a `/clear` are mutually exclusive in one settle (each sets `stop`), but order the check so an explicit + // reseat is never mis-read as a clear. + stopReason: () => (reseatRequested !== undefined ? 'reseat' : clearRequested ? 'clear' : 'exit'), + onReseat: (target) => { + reseatRequested = target; + stop = true; + }, + reseatTarget: () => reseatRequested, onAbort: modeControl.onAbort, onModeChange: modeControl.onModeChange, }; @@ -1038,6 +1102,179 @@ function createClearRebuild(params: { return (oldSessionId) => buildFreshChatWiring(wiringDeps, clearedNotice(oldSessionId)); } +/** + * Seed the view store + persister for a RESUMED session — the SAME assembly both `chat-resume` (2.N) and the + * `/models` reseat (ADR-0059) need: a store pre-seeded with the carried model + cumulative cost/turns (a resumed + * session never re-emits `session:started`, so without this the footer shows nothing until the first new turn), and + * a persister continuing the durable transcript past its last `sequenceNumber`. One home so the two resume paths can + * never wire different seeds. + */ +function seedResumedWiring( + resumed: BuiltResumedChatSession, + opened: OpenedSessionStore, + color: boolean, + now: () => number, + uuid: () => string, +): { store: ChatStoreController; persister: SessionPersister } { + const store = createChatStore(color, { + agentRef: resumed.agent.id, + model: resumed.agent.model, + cumulativeCostMicrocents: resumed.resumeState.cumulativeCostMicrocents, + turnCount: resumed.resumeState.turnCount, + }); + const persister = createSessionPersister({ + store: opened.store, + handle: resumed.handle, + sessionId: resumed.sessionId, + agent: resumed.agent, + context: resumed.context, + now, + uuid, + // Continue the durable transcript past its last sequence number (start() adopts the row + its totals). + initialSequenceNumber: resumed.nextSequenceNumber, + }); + return { store, persister }; +} + +/** The deps `createReseatRebuild` closes over — the capability inputs to rebuild a resumed, model-swapped session + * over the SHARED db. Mirrors {@link FreshChatWiringDeps} but RESUMES (carries the transcript) instead of starting + * fresh, and swaps the bound model (so it has no fixed `agent` — the agent is loaded from the just-ended record). */ +interface ReseatWiringDeps { + readonly chat: BuildChatSessionOptions['chat']; + readonly now: () => number; + readonly uuid: () => string; + readonly providers: ProviderResolver; + readonly mcpSecretResolver: McpSecretResolver; + readonly mcpRegistrations: BuildChatSessionOptions['mcpRegistrations']; + readonly configPath: string | undefined; + readonly io: CliIo; + readonly global: GlobalOptions; + readonly opened: OpenedSessionStore; + readonly buildResumedSession: typeof buildResumedChatSession; + readonly onBudgetWarning: NonNullable; +} + +/** + * Rebuild a RESUMED session bound to a NEW model (ADR-0059 reseat). The just-ended session's transcript is reloaded + * from the SHARED db and continued under a fresh `AgentSession.resume` bound to `target` — a NEW instance, honoring + * ADR-0024's one-model-per-lifetime rule. The bound agent's `model`/`provider` are swapped to the picked pair and + * its `fallback_chain` is DROPPED (it belonged to the original model; the new instance memoizes the default plan for + * `target`, exactly as a fresh session on `target` would build). The user-pricing overlay is re-read FRESH from the + * shared db so a mid-session `models pricing` write applies. Mirrors {@link buildFreshChatWiring}'s acquire-then-guard + * so a persister-construction throw never orphans the just-spawned MCP children. The SHARED `opened` is NOT this + * function's to close — the caller's outer {@link runReplLoop} finally owns it across every swap. + */ +async function buildReseatWiring( + deps: ReseatWiringDeps, + oldSessionId: string, + target: ReseatTarget, +): Promise { + const loaded = deps.opened.store.loadFull(oldSessionId); + if (loaded === undefined || loaded.session.agentSnapshot === undefined) { + // The session we just drove is gone / has no snapshot — cannot resume it under a new model. Loud: runReplLoop + // surfaces it and the prior conversation stays resumable (it was persisted before this fault). + throw new CliError( + 'invalid_invocation', + `cannot switch model: session ${oldSessionId} could not be reloaded for reseat`, + ); + } + // Swap the bound model/provider on a FRESH copy, then DROP the original `fallback_chain` — it belonged to the old + // model; the resumed instance builds its own default plan for `target` (ADR-0059). `fallback_chain` is optional on + // `AgentDefinition`, so `delete` removes the key entirely (never leaves an explicit `undefined`, satisfying + // exactOptionalPropertyTypes) and mutates only this copy, never the loaded record. + const newAgent: AgentDefinition = { + ...loaded.session.agentSnapshot, + model: target.modelId, + provider: target.provider, + }; + delete newAgent.fallback_chain; + // The record the resumed session rebinds from: the just-ended row with the model-swapped agent snapshot. + // (The row's own `modelId` FK column stays as-is — per-message/session `modelId` attribution is deferred with + // the 2.6.C cost breakdown; see persister.ts. `reconstructSessionState` reads only the transcript + cost here.) + const record: AgentSessionRecord = { ...loaded.session, agentSnapshot: newAgent }; + // Re-read the user-pricing overlay FRESH from the shared db (mirrors buildFreshChatWiring) so a mid-session + // `models pricing` write applies to the reseated session. Empty ⇒ omitted (unknown models degrade to allow). + const resolvePrice = readUserPricingOverlay(deps.opened.db); + const resumed = await deps.buildResumedSession({ + chat: deps.chat, + record, + messages: loaded.messages, + now: deps.now, + providers: deps.providers, + mcpSecretResolver: deps.mcpSecretResolver, + ...(deps.mcpRegistrations === undefined ? {} : { mcpRegistrations: deps.mcpRegistrations }), + ...(resolvePrice.size === 0 ? {} : { resolvePrice }), + onBudgetWarning: deps.onBudgetWarning, + }); + surfaceMcpSkipped(deps.io, resumed.mcpSkipped); + let seeded: { store: ChatStoreController; persister: SessionPersister }; + try { + seeded = seedResumedWiring(resumed, deps.opened, deps.global.color, deps.now, deps.uuid); + } catch (err) { + // Acquire-then-guard: the resumed session's MCP children are already spawned — reclaim them before the failure + // propagates so a persister-construction throw never orphans a stdio child (best-effort; never mask the primary). + await resumed.closeMcp?.().catch(() => undefined); + throw err; + } + const doctorProbes = assembleDoctorProbes({ + cwd: deps.global.cwd, + ...(deps.configPath === undefined ? {} : { configPath: deps.configPath }), + resolver: deps.providers, + agentMcpServers: resumed.agent.mcp_servers ?? [], + mcpSkipped: resumed.mcpSkipped, + }); + return { + built: resumed, + opened: deps.opened, + store: seeded.store, + persister: seeded.persister, + doctorProbes, + // A resumed session already landed at idle inside AgentSession.resume; start() would throw + re-emitting + // session:started would double a terminal-less lifecycle event — so startSession is a no-op (like chat-resume). + startSession: () => {}, + intro: modelSwitchNotice(target.modelId, resumed.resumeState.turnCount), + ...(deps.chat.maxMessages === undefined ? {} : { chatMaxMessages: deps.chat.maxMessages }), + }; +} + +/** + * Build the `/models` reseat rebuild closure (ADR-0059) — SHARED by `chat` and `chat-resume` so the reseat CONTRACT + * lives in ONE place and cannot drift. Given the command's capability inputs, it returns a + * `(oldSessionId, target) => Promise` that reloads the just-ended session's transcript and resumes it + * under a new-model agent over the SHARED db. + */ +function createReseatRebuild(params: { + readonly chat: BuildChatSessionOptions['chat']; + readonly now: () => number; + readonly uuid: () => string; + readonly providers: ProviderResolver; + readonly mcpSecretResolver: McpSecretResolver; + readonly mcpRegistrations: BuildChatSessionOptions['mcpRegistrations']; + readonly opened: OpenedSessionStore; + readonly buildResumedSession: typeof buildResumedChatSession; + readonly io: CliIo; + readonly global: GlobalOptions; +}): (oldSessionId: string, target: ReseatTarget) => Promise { + const wiringDeps: ReseatWiringDeps = { + chat: params.chat, + now: params.now, + uuid: params.uuid, + providers: params.providers, + mcpSecretResolver: params.mcpSecretResolver, + mcpRegistrations: params.mcpRegistrations, + configPath: params.global.configPath, + io: params.io, + global: params.global, + opened: params.opened, + buildResumedSession: params.buildResumedSession, + onBudgetWarning: (warning) => + params.io.writeErr( + `budget warning: ~${warning.thresholdPct}% of the ${warning.limitMicrocents}µ¢ cap reached\n`, + ), + }; + return (oldSessionId, target) => buildReseatWiring(wiringDeps, oldSessionId, target); +} + /** * Drive ONE session to its stop (`/exit`, `/cancel`, `/clear`, or EOF) and tear it down — the per-session unit the * re-drive {@link runReplLoop} runs once per conversation. Its finally fires the session's sole terminal @@ -1047,8 +1284,12 @@ function createClearRebuild(params: { */ async function driveOneSession(wiring: ReplWiring, deps: ChatReplDeps): Promise { const { built, store, persister, startSession, intro } = wiring; - const { processLine, cancelOnce, shouldStop, stopReason, onAbort, onModeChange } = + const { processLine, cancelOnce, shouldStop, stopReason, onReseat, reseatTarget, onAbort, onModeChange } = createChatLineHandler(wiring, deps); + // A live reseat is TTY-interactive only (like `/clear`): the ink model-picker overlay is the sole trigger, and a + // plain/`--json` driver has no picker. Wiring `onReseat` only on an interactive driver means `stopReason()` can + // never yield `'reseat'` under `--json`/plain — one machine stream stays one session lifecycle (ADR-0049). + const reseatEnabled = chatIsInteractive(deps.io, deps.global); // The `@`-mention completion reader (2.5.D, ADR-0061): a READ-ONLY fs jail at the SAME fs-scope tier + workspace // as the session's tools, so `@`-completion browses + injects through the identical confidentiality floor + @@ -1076,7 +1317,7 @@ async function driveOneSession(wiring: ReplWiring, deps: ChatReplDeps): Promise< // is deferred to startSession() INSIDE the driver, after the driver has subscribed the view store. try { persister.start(); - return await (deps.drive ?? drivePlain)({ + const outcome = await (deps.drive ?? drivePlain)({ startSession, processLine, shouldStop, @@ -1094,9 +1335,18 @@ async function driveOneSession(wiring: ReplWiring, deps: ChatReplDeps): Promise< ...(intro === undefined ? {} : { intro }), onAbort, onModeChange, + ...(reseatEnabled ? { onReseat } : {}), ...(mentionReader === undefined ? {} : { mentionReader }), ...(runShellCommand === undefined ? {} : { runShellCommand }), }); + // A `/models` reseat: the driver returns `{ kind: 'reseat' }` (from `stopReason()`); attach the captured target + // HERE — the one place holding the line handler — so every driver stays target-agnostic. A missing target + // (never expected: `onReseat` always captures one) degrades to a plain end rather than a broken rebuild loop. + if (outcome.kind === 'reseat') { + const target = reseatTarget(); + return target === undefined ? { kind: 'exit' } : { kind: 'reseat', target }; + } + return outcome; } finally { cancelOnce(); // emit the terminal even on /exit, /clear, or EOF (idempotent); flips the row to 'ended' // Attempt EVERY teardown step (a reject in one must not skip the next) and never let a cleanup fault mask the @@ -1109,36 +1359,48 @@ async function driveOneSession(wiring: ReplWiring, deps: ChatReplDeps): Promise< /** * The shared REPL loop driving both `chat` (fresh) and `chat-resume` (2.N). It drives the CURRENT session to its - * stop ({@link driveOneSession}) and, on a `/clear` outcome (ADR-0062 §7, TTY-interactive only), rebuilds a FRESH - * session over the SAME db and re-drives — otherwise it ends. The shared `history.db` handle survives every swap - * and is closed exactly ONCE in the outer finally; each session's own teardown (terminal + persister + MCP) is - * owned by `driveOneSession`. `/exit`, `/cancel`, and an input EOF all end the session with **exit code 4**. + * stop ({@link driveOneSession}) and, on a swap outcome (TTY-interactive only), rebuilds over the SAME db and + * re-drives — a `/clear` (ADR-0062 §7) rebuilds a FRESH session, a `/models` reseat (ADR-0059) rebuilds a NEW-model + * session carrying the transcript — otherwise it ends. The shared `history.db` handle survives every swap and is + * closed exactly ONCE in the outer finally; each session's own teardown (terminal + persister + MCP) is owned by + * `driveOneSession`. `/exit`, `/cancel`, and an input EOF all end the session with **exit code 4**. */ async function runReplLoop( wiring: ReplWiring, deps: ChatReplDeps, rebuild?: (oldSessionId: string) => Promise, + reseatRebuild?: (oldSessionId: string, target: ReseatTarget) => Promise, ): Promise { - // The SHARED db handle — the same across every /clear swap (a fresh session reuses it), closed ONCE below. + // The SHARED db handle — the same across every swap (a fresh / reseated session reuses it), closed ONCE below. const opened = wiring.opened; let current = wiring; try { for (;;) { const outcome = await driveOneSession(current, deps); - // Only a TTY `/clear` yields 'clear' (the gate rejects it under `--json`/plain); with no rebuild wired, end. - if (outcome.kind !== 'clear' || rebuild === undefined) break; // The old session is ALREADY torn down (driveOneSession's finally fired its terminal → the row is 'ended' + - // resumable). Build the fresh session over the same db and re-drive; a build failure is surfaced actionably - // (the prior conversation is still resumable) and ends the REPL rather than looping on a broken build. + // resumable). Resolve the rebuild for this swap kind (both /clear + reseat are TTY-only; a non-TTY outcome is + // always 'exit'), or leave `next` unset to END the REPL. The target is captured into a const so the closure + // keeps its narrowed (non-undefined) type — no unsafe non-null assertion. const oldSessionId = current.built.sessionId; + let next: (() => Promise) | undefined; + if (outcome.kind === 'clear' && rebuild !== undefined) { + next = () => rebuild(oldSessionId); + } else if (outcome.kind === 'reseat' && reseatRebuild !== undefined && outcome.target !== undefined) { + const target = outcome.target; + next = () => reseatRebuild(oldSessionId, target); + } + if (next === undefined) break; + // Build the swap session over the same db and re-drive; a build failure is surfaced actionably (the prior + // conversation is still resumable) and ends the REPL rather than looping on a broken build. try { - current = await rebuild(oldSessionId); + current = await next(); } catch (err) { deps.io.writeErr( // Sanitize the error text too (not just the id beside it) — a rebuild fault can rethrow an unclassified // message verbatim (session-host.ts), which could carry an ANSI/OSC escape from a spawned MCP server's // error text; strip it exactly as the id + every other display string on this surface is stripped. - `could not start a fresh session after /clear: ${sanitizeInline(err instanceof Error ? err.message : String(err))}. ` + + `could not start a new session after ${outcome.kind === 'reseat' ? 'a model switch' : '/clear'}: ` + + `${sanitizeInline(err instanceof Error ? err.message : String(err))}. ` + `Your previous conversation is saved — resume it with \`relavium chat-resume ${sanitizeInline(oldSessionId)}\`.\n`, ); break; diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index 09b2ad50..23d45952 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -95,9 +95,12 @@ export interface HomeChatSession { /** The session's durable id (ADR-0062 §7) — named in the `/clear` notice as the prior (still-resumable) * conversation, so it is discoverable after the swap. */ readonly sessionId: string; - /** WHY `shouldStop()` became true (ADR-0062 §7) — `'clear'` (swap in a fresh session, staying in chat) vs - * `'exit'` (`/exit`/`/cancel`, return to the bare Home). Lets the controller pick `clearChat` vs `endChat`. */ - readonly stopReason: () => 'exit' | 'clear'; + /** WHY `shouldStop()` became true (ADR-0062 §7 · ADR-0059) — `'clear'` (swap in a fresh session, staying in chat) + * vs `'exit'` (`/exit`/`/cancel`, return to the bare Home). Shares the widened `ChatLineHandler.stopReason` type, + * so it also carries `'reseat'`; but the in-Home chat does NOT yet wire `onReseat` (its `/models` is the Home's + * next-session-default picker, not a live reseat — ADR-0059's in-Home live reseat is a follow-up), so `'reseat'` + * is currently unreachable here and the consumer's non-`'clear'` branch treats it as an end (a safe default). */ + readonly stopReason: () => 'exit' | 'clear' | 'reseat'; /** Mid-turn abort (EA7) — abort the in-flight turn, keeping the session alive (Esc). Present once wired. */ readonly onAbort?: () => void; /** Switch the chat mode (Shift+Tab / `/mode`) — re-applies the turn policy on the same session (ADR-0057). */ diff --git a/docs/decisions/0024-agent-first-entry-point-agentsession.md b/docs/decisions/0024-agent-first-entry-point-agentsession.md index 98469e11..2bec9fe7 100644 --- a/docs/decisions/0024-agent-first-entry-point-agentsession.md +++ b/docs/decisions/0024-agent-first-entry-point-agentsession.md @@ -2,7 +2,7 @@ - **Status**: Accepted - **Date**: 2026-06-05 -- **Related**: [0003-pure-ts-engine-not-langgraph-python.md](0003-pure-ts-engine-not-langgraph-python.md), [0005-sqlite-drizzle-local-postgres-cloud.md](0005-sqlite-drizzle-local-postgres-cloud.md), [0008-local-first-phase-1-cloud-phase-2.md](0008-local-first-phase-1-cloud-phase-2.md), [0009-git-native-workflow-yaml.md](0009-git-native-workflow-yaml.md), [0011-internal-llm-abstraction.md](0011-internal-llm-abstraction.md), [0018-desktop-execution-and-rust-egress.md](0018-desktop-execution-and-rust-egress.md), [0025-agent-surface-refines-desktop-scope.md](0025-agent-surface-refines-desktop-scope.md), [0026-session-export-to-workflow.md](0026-session-export-to-workflow.md), [../reference/contracts/agent-session-spec.md](../reference/contracts/agent-session-spec.md), [../reference/contracts/sse-event-schema.md](../reference/contracts/sse-event-schema.md), [../reference/desktop/database-schema.md](../reference/desktop/database-schema.md), [0050-cli-history-db-at-rest-posture.md](0050-cli-history-db-at-rest-posture.md) +- **Related**: [0003-pure-ts-engine-not-langgraph-python.md](0003-pure-ts-engine-not-langgraph-python.md), [0005-sqlite-drizzle-local-postgres-cloud.md](0005-sqlite-drizzle-local-postgres-cloud.md), [0008-local-first-phase-1-cloud-phase-2.md](0008-local-first-phase-1-cloud-phase-2.md), [0009-git-native-workflow-yaml.md](0009-git-native-workflow-yaml.md), [0011-internal-llm-abstraction.md](0011-internal-llm-abstraction.md), [0018-desktop-execution-and-rust-egress.md](0018-desktop-execution-and-rust-egress.md), [0025-agent-surface-refines-desktop-scope.md](0025-agent-surface-refines-desktop-scope.md), [0026-session-export-to-workflow.md](0026-session-export-to-workflow.md), [0059-cli-mid-session-model-reseat.md](0059-cli-mid-session-model-reseat.md), [../reference/contracts/agent-session-spec.md](../reference/contracts/agent-session-spec.md), [../reference/contracts/sse-event-schema.md](../reference/contracts/sse-event-schema.md), [../reference/desktop/database-schema.md](../reference/desktop/database-schema.md), [0050-cli-history-db-at-rest-posture.md](0050-cli-history-db-at-rest-posture.md) > Amended 2026-06-28: the `history.db` is **not** encrypted at rest on the **CLI** surface — it is > guarded by `0700`/`0600` OS file permissions with API keys in the keychain only (see @@ -10,6 +10,12 @@ > store. The Context below originally said "encrypted" without that surface distinction; read it as > surface-specific. +> Amended 2026-07-06: the "one agent + one model bound for the session lifetime" rule is **refined** (not +> reversed) by [ADR-0059](0059-cli-mid-session-model-reseat.md) — a mid-session `/models` **model switch** is a +> host-side **reseat** (a new `AgentSession.resume` instance bound to the new model, carrying the text-only +> transcript + cumulative cost/turns), so each instance still binds exactly one model for its lifetime; the +> switch is a new instance, never an in-place rebind of the memoized fallback plan. + ## Context Relavium was originally framed as **workflow-first**: the unit of value is a git-committable diff --git a/docs/decisions/0059-cli-mid-session-model-reseat.md b/docs/decisions/0059-cli-mid-session-model-reseat.md index 60a47752..e8a8930c 100644 --- a/docs/decisions/0059-cli-mid-session-model-reseat.md +++ b/docs/decisions/0059-cli-mid-session-model-reseat.md @@ -1,10 +1,23 @@ # ADR-0059: Mid-session model switching via host-side reseat (refines ADR-0024) -- **Status**: Proposed -- **Date**: 2026-06-28 -- **Related**: [ADR-0024](0024-agent-first-entry-point-agentsession.md), [ADR-0026](0026-session-export-to-workflow.md), [ADR-0057](0057-cli-chat-modes-and-per-tool-approval.md), [phase-2.6-conversational-authoring.md](../roadmap/phases/phase-2.6-conversational-authoring.md) (2.6.C), [architectural-principles.md](../standards/architectural-principles.md) +- **Status**: Accepted +- **Date**: 2026-07-06 +- **Related**: [ADR-0024](0024-agent-first-entry-point-agentsession.md) (the one-model-per-lifetime rule this refines), [ADR-0026](0026-session-export-to-workflow.md), [ADR-0057](0057-cli-chat-modes-and-per-tool-approval.md) (the instance-scoped approval cache a reseat re-primes), [ADR-0062](0062-context-compaction-and-cli-history-commands.md) (the `/clear` host-swap machinery + the `contextPreamble` a reseat must carry), [phase-2.6-conversational-authoring.md](../roadmap/phases/phase-2.6-conversational-authoring.md) (2.6.C), [architectural-principles.md](../standards/architectural-principles.md) -> **Draft.** Proposed alongside the Phase 2.6 plan; to be reviewed and finalized (→ Accepted) when workstream 2.6.C begins. +> **Proposed 2026-06-28 alongside the Phase 2.6 plan; Accepted 2026-07-06** and implemented as a 2.5.G follow-up +> (the in-chat `/models` reseat requested with the Phase-2.5 CLI-consolidation model work), pulling the ADR forward +> from 2.6.C. The reseat reuses the `/clear` host-swap machinery ([ADR-0062](0062-context-compaction-and-cli-history-commands.md) §7) +> and the `chat-resume` transcript path — zero engine change — exactly as designed below. + +> **Note (2026-07-06): the per-message `modelId` persistence is DEFERRED to 2.6.C (with the cost breakdown that +> consumes it).** The Decision below says "only the CLI persister wiring is missing"; on implementation that proved +> under-specified. `session_messages.model_id` / `agent_sessions.model_id` are **foreign keys to `model_catalog.id` +> (a UUID row PK), not the raw model string** — and nothing populates them today (run history leaves them NULL too). +> Correct population needs a model-string → `model_catalog.id` resolution (and the model may not yet be cataloged → +> the NULL "unknown" bucket the Decision anticipates). That resolution belongs with the 2.6.C per-model cost +> breakdown that reads it, so it lands there. **The reseat itself is fully implemented and correct without it** — a +> mid-session `/models` switch carries the transcript + cumulative cost/turns and rebinds the model as designed; only +> the per-turn attribution column is deferred. This note refines, and does not reverse, the Decision. ## Context @@ -67,5 +80,5 @@ session that drops history (rejected: poor UX — the user expects to continue). - This **refines, not reverses**, [ADR-0024](0024-agent-first-entry-point-agentsession.md): each `AgentSession` instance still binds exactly one model for its lifetime; mid-session model switching is a - host-side reseat (a new instance), not an in-place rebind. When this ADR is Accepted, ADR-0024 gets a - dated `> Amended` note + a Related forward-link (documentation-style §7). + host-side reseat (a new instance), not an in-place rebind. On acceptance (2026-07-06), ADR-0024 received a + dated `> Amended` note + a Related forward-link to this ADR (documentation-style §7). From 3f0285dd9c04205ec3e08f75f4e65a42402d6de5 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 14:46:34 +0300 Subject: [PATCH 45/68] =?UTF-8?q?feat(db,cli):=202.5.G=20Step=20D=20(2a)?= =?UTF-8?q?=20=E2=80=94=20per-message/session=20model=20attribution=20(ADR?= =?UTF-8?q?-0059)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Un-defers the ADR-0059 modelId attribution, done correctly against the FK. On implementation the ADR's "only the persister wiring is missing" proved under-specified: `session_messages.model_id` / `agent_sessions.model_id` are FKs to `model_catalog.id` (a UUID row PK), NOT the raw model string. So attribution resolves the model string → the catalog id, degrading to NULL (the "unknown" bucket) when the model isn't cataloged. - @relavium/db: ModelCatalogStore.catalogIdByModelId(modelId) → the authoritative active row's UUID (the FK target), or undefined when uncataloged (same source-ranked selection as getByModelId). One focused lookup; ModelCatalogRecord untouched. - persister: makeCatalogIdResolver(db) + SessionPersisterDeps.resolveModelCatalogId. The assistant row is attributed to the model that PRODUCED it — resolved from the turn's last cost:updated.model (failover-aware); the session row to the bound model (resolved once). Absent resolver ⇒ every modelId column stays NULL (never a raw string → never an FK violation), the safe default for tests. - Wired the resolver into every createSessionPersister site over the SAME db (chat, chat-resume/reseat via seedResumedWiring, /clear via buildFreshChatWiring, the Home). - Tests: the db lookup (resolves to the row UUID / undefined); the persister attributes the assistant + session rows to the catalog id (+ the no-resolver NULL degrade); the reseat test now asserts per-turn attribution (turn 1 = sonnet's id, turn 2 = opus's id). Toolchain green: 1486 tests, all lint/typecheck/test/build tasks pass. Refs: ADR-0059, ADR-0064 Co-Authored-By: Claude --- apps/cli/src/chat/persister.test.ts | 73 ++++++++++++++++++++- apps/cli/src/chat/persister.ts | 62 ++++++++++++++--- apps/cli/src/commands/chat.test.ts | 48 +++++++++++++- apps/cli/src/commands/chat.ts | 13 +++- apps/cli/src/home/drive-home.tsx | 9 ++- packages/db/src/model-catalog-store.test.ts | 15 +++++ packages/db/src/model-catalog-store.ts | 13 ++++ 7 files changed, 218 insertions(+), 15 deletions(-) diff --git a/apps/cli/src/chat/persister.test.ts b/apps/cli/src/chat/persister.test.ts index 04194089..f5287acf 100644 --- a/apps/cli/src/chat/persister.test.ts +++ b/apps/cli/src/chat/persister.test.ts @@ -2,6 +2,8 @@ import { reconstructSessionState } from '@relavium/core'; import type { StreamChunk } from '@relavium/llm'; import { createClient, + createModelCatalogStore, + createProviderStore, createSessionStore, runMigrations, type DbClient, @@ -12,7 +14,7 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import type { ResolvedChatConfig } from '../config/resolve.js'; import { buildDefaultChatAgent } from './default-agent.js'; -import { createSessionPersister } from './persister.js'; +import { createSessionPersister, makeCatalogIdResolver } from './persister.js'; import { buildChatSession, buildResumedChatSession } from './session-host.js'; import { scriptedResolver, stop, textTurn, unresolvedResolver } from './test-support.js'; import type { ProviderResolver } from '../engine/providers.js'; @@ -178,6 +180,75 @@ describe('createSessionPersister', () => { expect(full?.session.totalOutputTokens).toBe(5); }); + it('attributes the assistant + session rows to the producing model’s catalog id (ADR-0059)', async () => { + // Seed the default model into the catalog so the resolver maps its STRING → the FK-target UUID; the assistant + // row (from cost:updated.model) + the session row (the bound model) both carry that id, the user row none. + const provider = createProviderStore(client.db, { uuid: () => 'p-1', now: () => 0 }).upsert({ + name: 'anthropic', + displayName: 'Anthropic', + baseUrl: 'https://api.anthropic.com', + }); + const catalog = createModelCatalogStore(client.db, { uuid: () => 'm-1', now: () => 0 }); + catalog.upsert({ + providerId: provider.id, + modelId: 'claude-sonnet-4-6', + displayName: 'Sonnet', + contextWindowTokens: 200_000, + maxOutputTokens: 8_192, + }); + const sonnetId = catalog.catalogIdByModelId('claude-sonnet-4-6'); + expect(sonnetId).toBeDefined(); + + let tick = Date.parse('2026-06-25T00:00:00.000Z'); + const now = (): number => tick++; + let msgId = 0; + const built = await buildChatSession({ + chat: EMPTY_CHAT, + agentRef: undefined, + cwd: '/workspace', + projectConfigDir: undefined, + now, + uuid: () => 'sess-attr', + providers: scriptedResolver([textTurn('hi there')]), + }); + const persister = createSessionPersister({ + store, + handle: built.handle, + sessionId: built.sessionId, + agent: built.agent, + context: built.context, + now, + uuid: () => `attr-${msgId++}`, + resolveModelCatalogId: makeCatalogIdResolver(client.db, { uuid: () => 'unused', now }), + }); + persister.start(); + built.session.start(); + persister.beginUserTurn('hello'); + await built.session.sendMessage('hello'); + + const full = store.loadFull('sess-attr'); + expect(full?.messages[0]?.role).toBe('user'); + expect(full?.messages[0]?.modelId).toBeUndefined(); // user text has no producing model + expect(full?.messages[1]?.role).toBe('assistant'); + expect(full?.messages[1]?.modelId).toBe(sonnetId); // the resolved catalog id of the producing model + expect(full?.session.modelId).toBe(sonnetId); // the session's coarse primary + }); + + it('leaves modelId NULL with no resolver — an uncataloged model degrades to the unknown bucket (ADR-0059)', async () => { + // No `resolveModelCatalogId` wired ⇒ every attribution column stays NULL (the safe default), and the FK is + // never violated by writing a raw model string. + const { built, persister } = await setup(scriptedResolver([textTurn('hi there')])); + persister.start(); + built.session.start(); + persister.beginUserTurn('hello'); + await built.session.sendMessage('hello'); + + const full = store.loadFull('sess-1'); + expect(full?.messages[1]?.role).toBe('assistant'); + expect(full?.messages[1]?.modelId).toBeUndefined(); // no resolver ⇒ NULL, never the raw model string + expect(full?.session.modelId).toBeUndefined(); + }); + it('derives the session title from the FIRST user message, and a later message does not overwrite it', async () => { const { built, persister } = await setup(scriptedResolver([textTurn('a'), textTurn('b')])); persister.start(); diff --git a/apps/cli/src/chat/persister.ts b/apps/cli/src/chat/persister.ts index 621566d6..06f08174 100644 --- a/apps/cli/src/chat/persister.ts +++ b/apps/cli/src/chat/persister.ts @@ -4,11 +4,29 @@ import { type SessionHandle, type SessionStreamHandleEvent, } from '@relavium/core'; -import type { SessionStore } from '@relavium/db'; +import { createModelCatalogStore, type Db, type SessionStore } from '@relavium/db'; import type { AgentSessionRecord, SessionContext, SessionStatus } from '@relavium/shared'; import { deriveSessionTitle } from './session-title.js'; +/** + * A model-string → `model_catalog.id` resolver ([ADR-0059]) — the FK target of `session_messages.model_id` / + * `agent_sessions.model_id`. Returns the authoritative active catalog row's UUID for a model string, or `undefined` + * when the model is not cataloged (→ a NULL column, the "unknown" attribution bucket). Live per lookup, so a + * mid-session `/models` refresh that discovers the model makes SUBSEQUENT turns attributable. + */ +export type ModelCatalogIdResolver = (modelId: string) => string | undefined; + +/** Build a {@link ModelCatalogIdResolver} over a `history.db` handle — every `createSessionPersister` call site + * derives one from the SAME `opened.db` the session store rides, so attribution needs no threading. */ +export function makeCatalogIdResolver( + db: Db, + deps: { readonly uuid: () => string; readonly now: () => number }, +): ModelCatalogIdResolver { + const catalog = createModelCatalogStore(db, deps); + return (modelId) => catalog.catalogIdByModelId(modelId); +} + /** * Write-side session persistence for `relavium chat` (2.M) — the CLI counterpart of the run-history writer. * `AgentSession` (the engine class, 1.V) keeps its transcript **in memory** and persists nothing; this @@ -47,6 +65,12 @@ export interface SessionPersisterDeps { * SessionPersister.start} hydrates automatically from the adopted row. */ readonly initialSequenceNumber?: number; + /** + * Resolve a model STRING → the `model_catalog.id` FK target for the transcript's per-message + the session's + * coarse `modelId` attribution ([ADR-0059]; see {@link makeCatalogIdResolver}). Absent ⇒ no attribution (every + * `modelId` column stays NULL) — the safe default when the caller has no catalog handle (e.g. a unit test). + */ + readonly resolveModelCatalogId?: ModelCatalogIdResolver; } export interface SessionPersister { @@ -71,6 +95,15 @@ export function createSessionPersister(deps: SessionPersisterDeps): SessionPersi let totalCostMicrocents = 0; let pendingUserText: string | undefined; let assistantText = ''; + // ADR-0059: the CATALOG ID of the model that ACTUALLY produced the in-flight assistant turn — resolved from the + // LAST `cost:updated.model` of the turn (failover-aware: the final billed egress is the model that produced the + // committed text; a failed primary that never billed emits none) via {@link SessionPersisterDeps.resolveModelCatalogId}. + // Reset at each turn start; written onto the assistant row so a per-model cost breakdown survives a `/models` reseat. + // Undefined (a NULL column, the "unknown" bucket) when the turn never billed OR the model is not yet cataloged. + let turnModelCatalogId: string | undefined; + // The session's coarse primary model catalog id (the bound model), resolved ONCE — it is constant for a persister + // instance (a reseat builds a NEW persister with the switched model). NULL when the bound model is not cataloged. + const sessionModelCatalogId = deps.resolveModelCatalogId?.(deps.agent.model); let unsubscribe: (() => void) | undefined; let started = false; // ADR-0062: the ascending `sequenceNumber`s of the REAL (`user`/`assistant`) transcript rows — used to map a @@ -79,10 +112,10 @@ export function createSessionPersister(deps: SessionPersisterDeps): SessionPersi // prior marker is interleaved — the step-1-review trap). Seeded from the durable transcript on resume. const realMessageSeqs: number[] = []; - /** Append a REAL transcript row + record its sequence for the boundary mapping. (Per-message `modelId` - * attribution — ADR-0059 — is DEFERRED to Phase 2.6.C: `session_messages.model_id` is a FK to - * `model_catalog.id` (a UUID), so it needs a model-string→catalog-id resolution the cost breakdown will own.) */ - const appendText = (role: 'user' | 'assistant', text: string): void => { + /** Append a REAL transcript row + record its sequence for the boundary mapping. `modelCatalogId` (assistant rows + * only) is the already-resolved `model_catalog.id` FK target attributing the row to the model that produced it + * (ADR-0059) — omitted (a NULL column) when unknown/uncataloged; a user row never carries one. */ + const appendText = (role: 'user' | 'assistant', text: string, modelCatalogId?: string): void => { const seq = sequenceNumber++; realMessageSeqs.push(seq); deps.store.appendMessage({ @@ -91,6 +124,8 @@ export function createSessionPersister(deps: SessionPersisterDeps): SessionPersi sequenceNumber: seq, role, content: [{ type: 'text', text }], + // Conditional spread ⇒ no explicit `undefined` under exactOptionalPropertyTypes; a user row never carries one. + ...(modelCatalogId === undefined ? {} : { modelId: modelCatalogId }), timestamp: iso(), }); }; @@ -132,9 +167,10 @@ export function createSessionPersister(deps: SessionPersisterDeps): SessionPersi createdAt, updatedAt: iso(), ...(title === undefined ? {} : { title }), - // NOTE: the session-level `modelId` (ADR-0059) is DEFERRED with the per-message attribution (see `appendText`): - // `agent_sessions.model_id` is a FK to `model_catalog.id` (a UUID), not the raw model string, so populating it - // needs the same catalog resolution the 2.6.C cost breakdown will own. Left NULL until then (like run history). + // The session's coarse primary model (ADR-0059) — the bound model's `model_catalog.id` (the FK target), so a + // reseat's new persister records the switched model. Omitted (NULL) when the model is not cataloged. Per-turn + // attribution rides each assistant `SessionMessage.modelId`; this is the row-level label. + ...(sessionModelCatalogId === undefined ? {} : { modelId: sessionModelCatalogId }), }); const onEvent = (event: SessionStreamHandleEvent): void => { @@ -150,6 +186,10 @@ export function createSessionPersister(deps: SessionPersisterDeps): SessionPersi case 'cost:updated': // The sink stamps the session-wide running total here; the latest value is the session's cost. totalCostMicrocents = event.cumulativeCostMicrocents; + // ADR-0059: the invoking model of this billed egress — attributable across a failover. Resolve it to its + // catalog id NOW (the FK target); the LAST such event of the turn is the model that produced the committed + // text, so it becomes the assistant row's `modelId` in `session:turn_completed`. Undefined ⇒ NULL (uncataloged). + turnModelCatalogId = deps.resolveModelCatalogId?.(event.model); return; case 'session:turn_completed': // Only a COMPLETED exchange writes MESSAGES — both an ERROR turn AND an ABORTED turn (EA7, @@ -175,13 +215,16 @@ export function createSessionPersister(deps: SessionPersisterDeps): SessionPersi // undefined, so the next non-blank completed message becomes the title; a resumed session keeps its own. title ??= deriveSessionTitle(pendingUserText); appendText('user', pendingUserText); - if (assistantText.length > 0) appendText('assistant', assistantText); + // The assistant row carries the (resolved) catalog id of the model that produced it (ADR-0059): + // `turnModelCatalogId` from this turn's last `cost:updated`, or `undefined` (NULL) when uncataloged. + if (assistantText.length > 0) appendText('assistant', assistantText, turnModelCatalogId); totalInputTokens += event.tokensUsed.input; totalOutputTokens += event.tokensUsed.output; } deps.store.updateSession(record('active')); pendingUserText = undefined; assistantText = ''; + turnModelCatalogId = undefined; return; case 'session:compacted': // ADR-0062: write the append-only boundary marker (summary + role-filtered droppedThroughSequence) and @@ -242,6 +285,7 @@ export function createSessionPersister(deps: SessionPersisterDeps): SessionPersi // errored first turn whose message rows are rolled back — a label with no transcript behind it. pendingUserText = text; assistantText = ''; + turnModelCatalogId = undefined; // a fresh turn: attribute the assistant row to THIS turn's egress, never a prior one }, close(): void { unsubscribe?.(); diff --git a/apps/cli/src/commands/chat.test.ts b/apps/cli/src/commands/chat.test.ts index 9ed97377..17295997 100644 --- a/apps/cli/src/commands/chat.test.ts +++ b/apps/cli/src/commands/chat.test.ts @@ -4,8 +4,16 @@ import { join } from 'node:path'; import { PassThrough, Readable } from 'node:stream'; import type { SessionStreamHandleEvent } from '@relavium/core'; -import type { StreamChunk } from '@relavium/llm'; -import { createClient, createSessionStore, runMigrations, type DbClient } from '@relavium/db'; +import type { ProviderId, StreamChunk } from '@relavium/llm'; +import { + createClient, + createModelCatalogStore, + createProviderStore, + createSessionStore, + runMigrations, + type Db, + type DbClient, +} from '@relavium/db'; import { startMcpClient as realStartMcpClient, type McpConnection } from '@relavium/mcp'; import { afterEach, beforeEach, describe, expect, it } from 'vitest'; @@ -97,6 +105,32 @@ function mcpConn(): { conn: McpConnection; closed: () => number } { return { conn, closed: () => n }; } +// Seed a model into `model_catalog` (+ its provider row for the FK) so ADR-0059 attribution can resolve the model +// STRING → the catalog row UUID. Returns that UUID (the `session_messages.model_id` FK target to assert against). +let seedCatalogN = 0; +function seedCatalogModel(db: Db, provider: ProviderId, modelId: string): string { + const storeDeps = { + uuid: () => `cat-${provider}-${seedCatalogN++}`, + now: () => Date.parse('2026-06-25T00:00:00.000Z'), + }; + const providerRow = createProviderStore(db, storeDeps).upsert({ + name: provider, + displayName: provider, + baseUrl: 'https://api.anthropic.com', + }); + const catalog = createModelCatalogStore(db, storeDeps); + catalog.upsert({ + providerId: providerRow.id, + modelId, + displayName: modelId, + contextWindowTokens: 200_000, + maxOutputTokens: 8_192, + }); + const id = catalog.catalogIdByModelId(modelId); + if (id === undefined) throw new Error(`catalog seed failed for ${modelId}`); + return id; +} + describe('chatCommand', () => { let cwd: string; let home: string; @@ -541,6 +575,9 @@ describe('chatCommand', () => { it('/models reseat: rebinds the model on the SAME session, carrying the transcript + per-turn attribution (ADR-0059)', async () => { const { d, store } = deps([], [textTurn('sonnet reply'), textTurn('opus reply')]); + // Seed both models into the catalog so attribution resolves the model string → the FK-target UUID. + const sonnetId = seedCatalogModel(client.db, 'anthropic', 'claude-sonnet-4-6'); + const opusId = seedCatalogModel(client.db, 'anthropic', 'claude-opus-4-8'); // A live reseat is TTY-interactive only (like `/clear`), so `onReseat` is wired only on an interactive io. const interactiveIo = { ...d.io, stdoutIsTty: true }; const seen: string[] = []; @@ -573,10 +610,15 @@ describe('chatCommand', () => { const full = store.loadFull('id-0'); expect(full?.session.agentSnapshot?.model).toBe('claude-opus-4-8'); // rebound to the target model - // The transcript carried across the switch: turn 1 (sonnet) + turn 2 (opus) — 4 sequenced rows, one session. + expect(full?.session.modelId).toBe(opusId); // the reseated session's coarse primary = the new model's catalog id + // The transcript carried across the switch: turn 1 (sonnet) + turn 2 (opus) — 4 sequenced rows, one session, + // each assistant row ATTRIBUTED to the model that produced it (its catalog UUID, ADR-0059). expect(full?.messages.map((m) => m.role)).toEqual(['user', 'assistant', 'user', 'assistant']); expect(full?.messages[1]?.content[0]).toEqual({ type: 'text', text: 'sonnet reply' }); + expect(full?.messages[1]?.modelId).toBe(sonnetId); // turn 1 produced by the ORIGINAL model expect(full?.messages[3]?.content[0]).toEqual({ type: 'text', text: 'opus reply' }); + expect(full?.messages[3]?.modelId).toBe(opusId); // turn 2 produced by the NEW model + expect(full?.messages[0]?.modelId).toBeUndefined(); // a user row carries no producing model expect(full?.session.totalCostMicrocents).toBeGreaterThan(0); // both turns' cost accrued (carried, not reset) }); diff --git a/apps/cli/src/commands/chat.ts b/apps/cli/src/commands/chat.ts index 855f5185..df7ea165 100644 --- a/apps/cli/src/commands/chat.ts +++ b/apps/cli/src/commands/chat.ts @@ -38,7 +38,11 @@ import { type ChatMode, } from '../chat/chat-mode.js'; import { applyChatMode, makeChatModeEnv } from '../chat/chat-mode-host.js'; -import { createSessionPersister, type SessionPersister } from '../chat/persister.js'; +import { + createSessionPersister, + makeCatalogIdResolver, + type SessionPersister, +} from '../chat/persister.js'; import { buildChatSession, buildResumedChatSession, @@ -323,6 +327,9 @@ export async function chatCommand(args: ChatCommandArgs, deps: ChatCommandDeps): context: built.context, now, uuid, + // ADR-0059 per-message/session model attribution — resolve a model string → its `model_catalog.id` (the FK + // target) over the SAME db, degrading to NULL when uncataloged. Shared across every persister site. + resolveModelCatalogId: makeCatalogIdResolver(opened.db, { uuid, now }), }); } catch (err) { closeQuietly(deps.io, 'session store', () => opened.close()); @@ -1032,6 +1039,8 @@ async function buildFreshChatWiring(deps: FreshChatWiringDeps, intro: string): P context: built.context, now: deps.now, uuid: deps.uuid, + // ADR-0059 attribution — resolved over the SAME shared db (a `/clear` rebuild re-reads it fresh). + resolveModelCatalogId: makeCatalogIdResolver(deps.opened.db, { uuid: deps.uuid, now: deps.now }), }); } catch (err) { // Acquire-then-guard: the fresh MCP children are already spawned — reclaim them before the failure propagates @@ -1132,6 +1141,8 @@ function seedResumedWiring( uuid, // Continue the durable transcript past its last sequence number (start() adopts the row + its totals). initialSequenceNumber: resumed.nextSequenceNumber, + // ADR-0059 attribution — resolved over the SAME db; a reseat's new persister records the switched model. + resolveModelCatalogId: makeCatalogIdResolver(opened.db, { uuid, now }), }); return { store, persister }; } diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index e8549a11..5dd95e24 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -8,7 +8,11 @@ import { createChatLineHandler } from '../commands/chat.js'; import { buildChatSession, type BuiltChatSession } from '../chat/session-host.js'; import { assembleDoctorProbes } from '../chat/doctor-host.js'; import type { DoctorProbes } from '../chat/doctor.js'; -import { createSessionPersister, type SessionPersister } from '../chat/persister.js'; +import { + createSessionPersister, + makeCatalogIdResolver, + type SessionPersister, +} from '../chat/persister.js'; import { loadResolvedConfig } from '../config/load.js'; import { writeGlobalDefaultModel } from '../config/write.js'; import { buildMergedCatalog } from '../engine/model-catalog-view.js'; @@ -283,6 +287,9 @@ export async function driveHome(deps: HomeDeps): Promise { context: built.context, now, uuid, + // ADR-0059 per-message/session model attribution — resolve a model string → its `model_catalog.id` + // over the SAME db (the Home's catalog is the one the picker refreshes), degrading to NULL when uncataloged. + resolveModelCatalogId: makeCatalogIdResolver(opened.db, { uuid, now }), }); // createChatLineHandler owns the mode control (ADR-0057): it applies the initial `ask` mode → the // fail-closed approval regime — BEFORE the session opens, so the full-capability chat host is never live diff --git a/packages/db/src/model-catalog-store.test.ts b/packages/db/src/model-catalog-store.test.ts index 2924b974..67b3277e 100644 --- a/packages/db/src/model-catalog-store.test.ts +++ b/packages/db/src/model-catalog-store.test.ts @@ -96,6 +96,21 @@ describe('createModelCatalogStore (2.S — media routing + load-check reader)', expect(store.getByModelId('not-in-catalog')).toBeUndefined(); }); + it('catalogIdByModelId resolves a model string to its catalog row UUID (the FK target), undefined when uncataloged (ADR-0059)', () => { + // providerStore.upsert (beforeEach) minted uuid #1; this first catalog upsert mints #2 — so the row's id is + // deterministic. `catalogIdByModelId` returns THAT id (the `session_messages.model_id` FK target), never the + // model string; an uncataloged model resolves to undefined (→ a NULL column, the pre-attribution bucket). + store.upsert({ + providerId, + modelId: 'gpt-4o', + displayName: 'GPT-4o', + contextWindowTokens: 128_000, + maxOutputTokens: 16_384, + }); + expect(store.catalogIdByModelId('gpt-4o')).toBe('00000000-0000-4000-8000-000000000002'); + expect(store.catalogIdByModelId('not-in-catalog')).toBeUndefined(); + }); + it('fromRow maps each capability flag to its own column (non-default values)', () => { store.upsert({ providerId, diff --git a/packages/db/src/model-catalog-store.ts b/packages/db/src/model-catalog-store.ts index e3848298..a7e49450 100644 --- a/packages/db/src/model-catalog-store.ts +++ b/packages/db/src/model-catalog-store.ts @@ -149,6 +149,15 @@ export interface ModelCatalogStore { * whole-catalog projection) while a genuine store/DB fault propagates. Unlike {@link resolveMediaSurface}, * which never parses `capabilities` and so stays usable for routing. */ getByModelId: (modelId: string) => ModelCatalogRecord | undefined; + /** + * The active catalog ROW ID — the `model_catalog.id` UUID that is the FK target of `session_messages.model_id` + * and `agent_sessions.model_id` — for a model STRING, or `undefined` when the model is not cataloged. Uses the + * SAME source-ranked authoritative-row selection as {@link getByModelId}. Lets a caller resolve a model string + * (a bound model, or a failover-aware `cost:updated.model`) to the referential id a transcript row stores for + * per-model cost attribution ([ADR-0059]), degrading to `undefined` → a NULL column (the "unknown" bucket) when + * the model has not been discovered into the catalog yet. + */ + catalogIdByModelId: (modelId: string) => string | undefined; /** Seed/replace a catalog row (by provider + model) — used by the generative acceptance fixture and a future * provider-sync; the store mints the id + timestamps. */ upsert: (input: ModelCatalogUpsert) => ModelCatalogRecord; @@ -333,6 +342,10 @@ export function createModelCatalogStore(db: Db, deps: ModelCatalogStoreDeps): Mo getByModelId, + // The authoritative active row's UUID (the FK target) for a model string, or `undefined` when uncataloged — + // the same source-ranked selection `getByModelId` uses, projected to just the id (ADR-0059 attribution). + catalogIdByModelId: (modelId) => activeRow(modelId)?.id, + upsert: (input) => { const t = deps.now(); const existing = db From 953a5fd86d3014efb7b29d76eea4d2a9f88eaaf1 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 15:18:52 +0300 Subject: [PATCH 46/68] =?UTF-8?q?feat(cli,llm):=202.5.G=20Step=20D=20(2b)?= =?UTF-8?q?=20=E2=80=94=20in-chat=20/models=20reseat=20picker=20+=20availa?= =?UTF-8?q?bility-first=20ordering?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes ADR-0059's user-facing trigger: a typed `/models` in the interactive (ink) chat opens a keyboard-owning model-picker overlay — the SAME rich picker the Home uses (live catalog, pricing, key-awareness dimming) — whose accept fires a live reseat. Plus the maintainer's ordering request: available models first, alphabetical; dimmed ones below. - @relavium/llm mergeModelCatalog: order AVAILABLE (selectable) models FIRST, then the dimmed/no-key ones — each alphabetical by displayName (modelId tiebreaker). Provider is no longer a sort key; availability + name is the user's axis (dimmed, never hidden). - Shared catalog port: new createModelCatalogPort (the load/refresh trio), factored out of drive-home so the Home default-write picker and the chat reseat picker load the catalog IDENTICALLY (one canonical home). The Home now spreads it + its currentDefault/writeDefault. - Chat wiring: ChatModelsPort (the trio + the session's boundModel = the ✓), built per session (a reseat's boundModel is the switched model) and forwarded to the ctx TTY-only, paired with onReseat. model-picker `accept` step now carries `provider` (the reseat target). - chat-ink ChatApp: the reseat picker overlay — ref-shadowed state + epoch-guarded refresh (mirrors the Home controller), a typed `/models` intercepted at submit to open it, accept → onReseat + end the loop so runReplLoop swaps the model. Mounted like the palette overlay. - `/models` is now availableIn ['home','chat'] (dual behavior off the one picker); a plain/`--json` chat (no overlay) gets an actionable "interactive terminal" hint. - Tests: the shared port keys availability by the resolver (available vs no-key); the fold accept carries the provider; the availability-first order; palette membership updated. Toolchain green: all lint/typecheck/test/build tasks pass. Refs: ADR-0059, ADR-0064 Co-Authored-By: Claude --- apps/cli/src/commands/chat.ts | 54 ++++++- apps/cli/src/commands/repl-commands.test.ts | 6 +- apps/cli/src/commands/repl-commands.ts | 14 +- .../cli/src/engine/model-catalog-port.test.ts | 54 +++++++ apps/cli/src/engine/model-catalog-port.ts | 66 ++++++++ apps/cli/src/home/drive-home.tsx | 52 +----- apps/cli/src/render/tui/chat-ink.tsx | 153 +++++++++++++++++- apps/cli/src/render/tui/model-picker.test.ts | 8 +- apps/cli/src/render/tui/model-picker.ts | 32 +++- packages/llm/src/model-catalog.test.ts | 24 +-- packages/llm/src/model-catalog.ts | 16 +- 11 files changed, 392 insertions(+), 87 deletions(-) create mode 100644 apps/cli/src/engine/model-catalog-port.test.ts create mode 100644 apps/cli/src/engine/model-catalog-port.ts diff --git a/apps/cli/src/commands/chat.ts b/apps/cli/src/commands/chat.ts index df7ea165..5a1a463d 100644 --- a/apps/cli/src/commands/chat.ts +++ b/apps/cli/src/commands/chat.ts @@ -51,6 +51,7 @@ import { type BuiltResumedChatSession, } from '../chat/session-host.js'; import { loadResolvedConfig } from '../config/load.js'; +import { createModelCatalogPort, type ModelCatalogPort } from '../engine/model-catalog-port.js'; import { assembleToolEnv } from '../engine/tool-host/assemble.js'; import { loadUserPricingOverlay, readUserPricingOverlay } from '../engine/pricing-overlay.js'; import { surfaceMcpSkipped } from '../engine/mcp-servers.js'; @@ -112,6 +113,27 @@ export interface ReseatTarget { readonly provider: ProviderId; } +/** + * The catalog port the ink `/models` reseat picker (ADR-0059) reads — the SHARED load/refresh trio + * ({@link ModelCatalogPort}) plus the session's currently-bound model (the picker's `✓` "you are here" marker). + * Built per session so `boundModel` reflects a reseat's switched model. Interactive (TTY) sessions only. + */ +export interface ChatModelsPort extends ModelCatalogPort { + readonly boundModel: string; +} + +/** Assemble a {@link ChatModelsPort} over the session's shared db + provider resolver (the catalog the Home picker + * also reads) plus the bound model — the one place the chat reseat picker's port is wired. */ +function buildChatModelsPort( + opened: OpenedSessionStore, + providers: ProviderResolver, + boundModel: string, + now: () => number, + uuid: () => string, +): ChatModelsPort { + return { ...createModelCatalogPort({ db: opened.db, providers, now, uuid }), boundModel }; +} + /** What an interactive driver receives — the command core's seam, so a driver never touches the session directly. */ export interface ChatDriveContext { /** @@ -139,6 +161,13 @@ export interface ChatDriveContext { * is unavailable — one machine stream is one session lifecycle (ADR-0049), exactly as `/clear` is gated off there. */ readonly onReseat?: (target: ReseatTarget) => void; + /** + * The `/models` reseat picker catalog port (ADR-0059) — the ink `ChatApp` opens the model-picker overlay off a + * typed `/models`, reads the merged catalog through this, and calls {@link onReseat} on accept. Present on an + * interactive (TTY, non-`--json`) session only; a plain/`--json` driver has no overlay, so a typed `/models` + * there falls through to the core surface guard's actionable "interactive terminal" hint. Absent ⇒ no picker. + */ + readonly modelPicker?: ChatModelsPort; /** The live session stream (the driver renders it: ink reduces it into the store; plain prints it). */ readonly handle: SessionHandle; /** The view store the ink renderer projects (`apply` already wired by the ink driver). */ @@ -392,6 +421,7 @@ export async function chatCommand(args: ChatCommandArgs, deps: ChatCommandDeps): persister, doctorProbes, startSession: () => built.session.start(), + modelPicker: buildChatModelsPort(opened, providers, built.agent.model, now, uuid), ...(config.chat.maxMessages === undefined ? {} : { chatMaxMessages: config.chat.maxMessages }), @@ -549,6 +579,7 @@ export async function chatResumeCommand( doctorProbes, startSession: () => {}, intro, + modelPicker: buildChatModelsPort(opened, providers, built.agent.model, now, uuid), ...(config.chat.maxMessages === undefined ? {} : { chatMaxMessages: config.chat.maxMessages }), @@ -573,6 +604,9 @@ interface ReplWiring { readonly intro?: string; /** `[chat].max_messages` — the default bound a bare `/trim` uses (ADR-0062); absent ⇒ `/trim` needs an inline `n`. */ readonly chatMaxMessages?: number; + /** The `/models` reseat picker port (ADR-0059) — built per session (its `boundModel` is the current model). + * Forwarded to the ctx only on an interactive driver (`driveOneSession` gates it). */ + readonly modelPicker?: ChatModelsPort; } /** @@ -908,10 +942,16 @@ export function createChatLineHandler( clearRequested = true; stop = true; }, - // `/models` (2.5.G S7, ADR-0064 §10) is HOME-ONLY (a next-session config action). The slash dispatch below - // rejects it on this chat surface with a pointer to the Home, so this capability is never reached here — kept as - // a documented inert noop to satisfy the shared ReplCommandContext shape (mirroring the Home's inert chat-only noops). - openModels: () => undefined, + // `/models` on the chat surface (ADR-0059) opens the ink reseat-picker overlay — but that is intercepted at the + // RENDER layer (ChatApp), BEFORE this core handler, so `openModels` runs ONLY on a non-interactive driver + // (plain/`--json`), where there is no overlay. There, surface an actionable hint instead of a silent no-op: a + // live reseat needs an interactive terminal (one machine stream stays one session lifecycle, ADR-0049); the + // Home's `/models` is the alternative for setting the next-session default. + openModels: () => + emitOutput( + '/models needs an interactive terminal to switch the model live. From a pipe, set `[chat].default_model` ' + + 'in your config, or run `relavium` (the Home) to change the default.', + ), }; // Parse + dispatch a `/name [args]` REPL line (extracted from processLine so each stays under the Sonar @@ -1063,6 +1103,7 @@ async function buildFreshChatWiring(deps: FreshChatWiringDeps, intro: string): P doctorProbes, startSession: () => built.session.start(), intro, + modelPicker: buildChatModelsPort(deps.opened, deps.providers, built.agent.model, deps.now, deps.uuid), ...(deps.chat.maxMessages === undefined ? {} : { chatMaxMessages: deps.chat.maxMessages }), }; } @@ -1244,6 +1285,8 @@ async function buildReseatWiring( // session:started would double a terminal-less lifecycle event — so startSession is a no-op (like chat-resume). startSession: () => {}, intro: modelSwitchNotice(target.modelId, resumed.resumeState.turnCount), + // The picker's `boundModel` is now the SWITCHED model — a further reseat marks it as the ✓ "you are here". + modelPicker: buildChatModelsPort(deps.opened, deps.providers, resumed.agent.model, deps.now, deps.uuid), ...(deps.chat.maxMessages === undefined ? {} : { chatMaxMessages: deps.chat.maxMessages }), }; } @@ -1346,7 +1389,10 @@ async function driveOneSession(wiring: ReplWiring, deps: ChatReplDeps): Promise< ...(intro === undefined ? {} : { intro }), onAbort, onModeChange, + // The reseat trigger (onReseat) + its picker are wired together, interactive-only: the ink overlay reads the + // catalog through `modelPicker` and calls `onReseat` on accept. A plain/`--json` driver gets neither. ...(reseatEnabled ? { onReseat } : {}), + ...(reseatEnabled && wiring.modelPicker !== undefined ? { modelPicker: wiring.modelPicker } : {}), ...(mentionReader === undefined ? {} : { mentionReader }), ...(runShellCommand === undefined ? {} : { runShellCommand }), }); diff --git a/apps/cli/src/commands/repl-commands.test.ts b/apps/cli/src/commands/repl-commands.test.ts index 8ae2d040..6929d5de 100644 --- a/apps/cli/src/commands/repl-commands.test.ts +++ b/apps/cli/src/commands/repl-commands.test.ts @@ -176,7 +176,7 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { 'clear', 'models', ]); - // /models is HOME-ONLY (availableIn ['home']) — it is EXCLUDED from the chat palette. + // /models is availableIn ['home','chat'] (ADR-0059: the chat reseat) — so it appears in BOTH palettes. expect(CHAT_PALETTE_COMMANDS.map((c) => c.name)).toEqual([ 'exit', 'cancel', @@ -188,9 +188,11 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { 'compact', 'trim', 'clear', + 'models', ]); // The bare Home offers /exit + /doctor (pre-chat diagnostics), /clear (availableIn ['home','chat']; an inert - // "nothing to clear" notice — ADR-0062 §7), and now /models (home-only, ADR-0064 §10). + // "nothing to clear" notice — ADR-0062 §7), and /models (availableIn ['home','chat'] — the Home writes the + // next-session default, ADR-0064 §10; the chat reseats live, ADR-0059). expect(HOME_PALETTE_COMMANDS.map((c) => c.name)).toEqual(['exit', 'doctor', 'clear', 'models']); }); }); diff --git a/apps/cli/src/commands/repl-commands.ts b/apps/cli/src/commands/repl-commands.ts index 763d00b8..a17a8508 100644 --- a/apps/cli/src/commands/repl-commands.ts +++ b/apps/cli/src/commands/repl-commands.ts @@ -216,14 +216,16 @@ const RAW_REPL_COMMANDS: readonly ReplCommand[] = [ { name: 'models', label: 'Models', - description: 'Pick your default model (opens the catalog picker).', - // `read` in the forward taxonomy: opening the picker changes nothing; the config write happens only on an - // explicit selection (ADR-0063), and even then it sets the NEXT session's default, never the running one. + description: 'Switch model (opens the catalog picker).', + // `read` in the forward taxonomy: opening the picker changes nothing; the action happens only on an explicit + // selection — the Home writes the NEXT session's default (ADR-0063), the chat reseats the LIVE session (ADR-0059). effect: 'read', - // HOME-ONLY (ADR-0064 §10): a next-session CONFIG action, deliberately distinct from the Phase-2.6 mid-chat - // `/models` live reseat (ADR-0059). Inside a live chat the slash dispatch rejects it with a pointer to the Home. + // Available on BOTH surfaces off the ONE picker, with a surface-specific accept: the HOME writes the next-session + // default (ADR-0064 §10 / ADR-0063); the CHAT REPL rebinds the LIVE session mid-conversation via a host-side + // reseat (ADR-0059). In an interactive (ink) chat the render layer intercepts a typed `/models` to open the + // overlay; a plain/`--json` chat (no overlay) falls through to `openModels`, which surfaces an actionable hint. run: (ctx) => ctx.openModels(), - availableIn: ['home'], + availableIn: ['home', 'chat'], }, ]; diff --git a/apps/cli/src/engine/model-catalog-port.test.ts b/apps/cli/src/engine/model-catalog-port.test.ts new file mode 100644 index 00000000..6fcaf16c --- /dev/null +++ b/apps/cli/src/engine/model-catalog-port.test.ts @@ -0,0 +1,54 @@ +import { createClient, runMigrations, type DbClient } from '@relavium/db'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; + +import { createModelCatalogPort } from './model-catalog-port.js'; +import type { ProviderResolver } from './providers.js'; + +/** + * The SHARED `/models` catalog port (the Home default-write picker + the chat reseat picker, ADR-0059/ADR-0064). + * The refresh trio is a thin re-composition of already-tested pieces (buildMergedCatalog, the refresh service); the + * load-bearing new behavior is that `load()` KEYS the merge off the resolver — a model whose provider has no key is + * dimmed `no-key`, never offered as selectable. Registry-only (no db seed needed — `MODEL_PRICING` carries the id). + */ +describe('createModelCatalogPort', () => { + let client: DbClient; + beforeEach(() => { + client = createClient(':memory:'); + runMigrations(client.db); + }); + afterEach(() => { + client.sqlite.close(); + }); + + const resolver = (hasKey: boolean): ProviderResolver => ({ + resolveProvider: () => undefined, + keyFor: (): string => { + if (!hasKey) throw new Error('no key'); + return 'k'; + }, + hasKey: () => hasKey, + }); + + it('load() surfaces the static registry as AVAILABLE when the provider has a key', () => { + const port = createModelCatalogPort({ + db: client.db, + providers: resolver(true), + now: () => 0, + uuid: () => 'u', + }); + const entry = port.load().entries.find((e) => e.modelId === 'claude-sonnet-4-6'); + expect(entry?.available).toBe(true); // anthropic keyed ⇒ its static model is selectable + }); + + it('load() DIMS a model whose provider has no key (no-key), never offering an uncallable model', () => { + const port = createModelCatalogPort({ + db: client.db, + providers: resolver(false), + now: () => 0, + uuid: () => 'u', + }); + const entry = port.load().entries.find((e) => e.modelId === 'claude-sonnet-4-6'); + expect(entry?.available).toBe(false); + expect(entry?.unavailableReason).toBe('no-key'); + }); +}); diff --git a/apps/cli/src/engine/model-catalog-port.ts b/apps/cli/src/engine/model-catalog-port.ts new file mode 100644 index 00000000..6e9679ac --- /dev/null +++ b/apps/cli/src/engine/model-catalog-port.ts @@ -0,0 +1,66 @@ +import { createModelCatalogStore, createProviderStore, type Db } from '@relavium/db'; + +import { buildMergedCatalog, type MergedCatalogView } from './model-catalog-view.js'; +import { createModelRefreshService, type RefreshReport } from './model-refresh.js'; +import { + KNOWN_PROVIDERS, + KNOWN_PROVIDER_IDS, + providerHasKey, + type ProviderResolver, +} from './providers.js'; + +/** + * The catalog-loading trio shared by the Home `/models` picker (ADR-0064 §10 — writes the next-session default) and + * the chat `/models` reseat picker (ADR-0059 — rebinds the live session). Built over the ONE open `history.db` + * handle + the provider resolver so both surfaces read the SAME merged catalog + run the SAME long-lived-process + * TTL background refresh; each surface layers its own accept action on top. Factored here so the two surfaces can + * never drift on how the catalog is loaded, keyed (which provider has a key), or refreshed. + */ +export interface ModelCatalogPort { + /** The merged catalog (all providers) + the newest live-refresh stamp (the freshness badge). Sync read + merge. */ + readonly load: () => MergedCatalogView; + /** TTL-bounded background refresh (ADR-0064 §5c) — refreshes empty/stale providers; `undefined` when none were. */ + readonly refreshIfStale: () => Promise; + /** Unbounded, user-initiated refresh (Ctrl+R) — every connected provider, per-provider-isolated. Never rejects. */ + readonly refresh: () => Promise; +} + +export function createModelCatalogPort(params: { + readonly db: Db; + readonly providers: ProviderResolver; + readonly now: () => number; + readonly uuid: () => string; +}): ModelCatalogPort { + const storeDeps = { uuid: params.uuid, now: params.now }; + const providerStore = createProviderStore(params.db, storeDeps); + const catalogStore = createModelCatalogStore(params.db, storeDeps); + const refreshService = createModelRefreshService({ + resolveProvider: params.providers.resolveProvider, + keyFor: params.providers.keyFor, + providerStore, + catalogStore, + knownProviderIds: KNOWN_PROVIDER_IDS, + knownProviders: KNOWN_PROVIDERS, + now: params.now, + }); + // Memoized: a keychain read is a SYNC native N-API call (and can pop an OS ACL prompt on first access), so it must + // not repeat on every `/models` open + every refresh completion on the live ink render thread. The resolvable key + // set is fixed for the process (a mid-session key change ⇒ restart), the same posture as the startup key probe. + let keyedProviders: ReadonlySet<(typeof KNOWN_PROVIDER_IDS)[number]> | undefined; + return { + load: () => { + // Rebuild the UUID→slug map on every load (NOT memoized): a refresh may register a provider's FK row, and the + // next load must resolve its live rows' provider — not drop them (mirrors the Home's load). + const slugByUuid = new Map(providerStore.list().map((p) => [p.id, p.name] as const)); + keyedProviders ??= new Set(KNOWN_PROVIDER_IDS.filter((id) => providerHasKey(params.providers, id))); + return buildMergedCatalog({ + rows: catalogStore.listAll(), + providerSlug: (uuid) => slugByUuid.get(uuid) ?? uuid, + keyedProviders, + now: params.now(), + }); + }, + refreshIfStale: () => refreshService.refreshIfStale(), + refresh: () => refreshService.refresh(), + }; +} diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index 5dd95e24..0d97fcb6 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -1,6 +1,6 @@ import { randomUUID } from 'node:crypto'; -import { createModelCatalogStore, createProviderStore, createRunHistoryReader } from '@relavium/db'; +import { createProviderStore, createRunHistoryReader } from '@relavium/db'; import { render } from 'ink'; import { createElement } from 'react'; @@ -15,17 +15,10 @@ import { } from '../chat/persister.js'; import { loadResolvedConfig } from '../config/load.js'; import { writeGlobalDefaultModel } from '../config/write.js'; -import { buildMergedCatalog } from '../engine/model-catalog-view.js'; +import { createModelCatalogPort } from '../engine/model-catalog-port.js'; import { readUserPricingOverlay } from '../engine/pricing-overlay.js'; -import { createModelRefreshService } from '../engine/model-refresh.js'; import { assembleToolEnv } from '../engine/tool-host/assemble.js'; -import { - createProviderResolver, - KNOWN_PROVIDERS, - KNOWN_PROVIDER_IDS, - providerHasKey, - type ProviderResolver, -} from '../engine/providers.js'; +import { createProviderResolver, type ProviderResolver } from '../engine/providers.js'; import { openSessionStore, type OpenedSessionStore } from '../history/session-open.js'; import { isProviderKeyless, @@ -170,16 +163,6 @@ export async function driveHome(deps: HomeDeps): Promise { ...(deps.global.configPath === undefined ? {} : { configPath: deps.global.configPath }), resolver: providers, }); - const catalogStore = createModelCatalogStore(opened.db, storeDeps); - const refreshService = createModelRefreshService({ - resolveProvider: providers.resolveProvider, - keyFor: providers.keyFor, - providerStore, - catalogStore, - knownProviderIds: KNOWN_PROVIDER_IDS, - knownProviders: KNOWN_PROVIDERS, - now, - }); // The `✓`-marked current default is the EFFECTIVE default — `[chat].default_model` resolves project → workspace // → global `[preferences].default_model` (ADR-0063 §1). It is re-read FRESH from disk each call (not the loaded // `config` snapshot) so it reflects a same-session `/models` write AND a project/workspace override AND an edit @@ -196,34 +179,15 @@ export async function driveHome(deps: HomeDeps): Promise { return undefined; // a mid-session malformed config must not crash the picker } }; - // The keyed providers (2.5.G key-awareness) — a keychain/env probe per known provider (≤4) so the picker dims - // + refuses a model whose provider has no key (a `no-key` reason) rather than offering one whose next chat would - // only fail `provider_auth`. Memoized: a keychain read is a SYNCHRONOUS native N-API call (and can pop an OS ACL - // prompt on first access), so we must not repeat it on every `/models` open + every refresh completion on the - // live ink render thread. Computed LAZILY on the first `load` (after the wizard has stored any first-run key), - // then reused — the resolvable key set is fixed for the Home process (a mid-session key change ⇒ restart), the - // same one-time posture as the startup `isProviderKeyless` probe. - let keyedProviders: ReadonlySet<(typeof KNOWN_PROVIDER_IDS)[number]> | undefined; + // The `/models` catalog port (ADR-0064 §10) — the SHARED load/refresh + key-aware merge trio (the SAME one the + // chat reseat picker uses, ADR-0059), over the ONE open db + the store-aware resolver. The Home layers its own + // accept action on top: `currentDefault` (the ✓ marker) + `writeDefault` (the next-session default, ADR-0063 §1). const models: HomeModelsPort = { - load: () => { - // Rebuild the UUID→slug map on every load (NOT memoized once like the one-shot dispatch resolver): a refresh - // may register a provider's FK row, and the next load must resolve its live rows' provider — not drop them. - const slugByUuid = new Map(providerStore.list().map((p) => [p.id, p.name] as const)); - keyedProviders ??= new Set(KNOWN_PROVIDER_IDS.filter((id) => providerHasKey(providers, id))); - return buildMergedCatalog({ - rows: catalogStore.listAll(), - providerSlug: (uuid_) => slugByUuid.get(uuid_) ?? uuid_, - keyedProviders, - now: now(), - }); - }, - refreshIfStale: () => refreshService.refreshIfStale(), - refresh: () => refreshService.refresh(), + ...createModelCatalogPort({ db: opened.db, providers, now, uuid }), currentDefault: readEffectiveDefault, // Write to the SAME file the picker re-reads + the started session resolves (honors `--config`), so a `/models` // write is never a silent no-op to a different file (2.5.G S7). - writeDefault: (modelId) => - writeGlobalDefaultModel(modelId, homeDir, deps.global.configPath), + writeDefault: (modelId) => writeGlobalDefaultModel(modelId, homeDir, deps.global.configPath), }; // Build + wire + START a fresh chat session (the controller sends the first message on transition). diff --git a/apps/cli/src/render/tui/chat-ink.tsx b/apps/cli/src/render/tui/chat-ink.tsx index 624566d6..d293f103 100644 --- a/apps/cli/src/render/tui/chat-ink.tsx +++ b/apps/cli/src/render/tui/chat-ink.tsx @@ -7,8 +7,18 @@ import { type ChatDriveContext, type ChatDriveOutcome, type ChatDriver, + type ChatModelsPort, + type ReseatTarget, } from '../../commands/chat.js'; import { CHAT_PALETTE_COMMANDS } from '../../commands/repl-commands.js'; +import type { RefreshReport } from '../../engine/model-refresh.js'; +import { + foldModelPickerKey, + partialFailureBanner, + type ModelPickerKey, + type ModelPickerState, +} from './model-picker.js'; +import { ModelPickerView } from './model-picker-view.js'; import { EXIT_CODES } from '../../process/exit-codes.js'; import { colorProps, dimProps } from './projection.js'; import { FORCE_TEARDOWN_MS, FRAME_MS } from './tui-constants.js'; @@ -134,6 +144,13 @@ interface ChatAppProps { readonly onAbort?: (() => void) | undefined; /** Switch the chat mode (Shift+Tab cycle) — re-applies the turn policy on the same session (ADR-0057). */ readonly onModeChange: (mode: ChatMode) => void; + /** Request a mid-session model switch (ADR-0059) — the `/models` picker overlay calls it on accept. Absent (a + * driver/test wired without it) ⇒ the overlay never opens (see `modelPicker`). `| undefined` for the passthrough. */ + readonly onReseat?: ((target: ReseatTarget) => void) | undefined; + /** The `/models` reseat picker catalog port (ADR-0059). When present (with `onReseat`), a typed `/models` opens a + * keyboard-owning model-picker overlay whose accept triggers a live reseat. Absent ⇒ `/models` is a normal + * message/slash (no overlay). `| undefined` so the createElement passthrough forwards an absent `ctx.modelPicker`. */ + readonly modelPicker?: ChatModelsPort | undefined; /** The `@`-mention completion reader (2.5.D, ADR-0061) — a READ-ONLY fs jail at the session's fs-scope tier + * workspace. When present, `@` at a word boundary opens dir-navigable file completion whose accepted file is * injected as UNTRUSTED, user-position context. Absent (a driver/test wired without it) ⇒ `@` is a literal char. @@ -338,6 +355,16 @@ export function ChatApp(props: Readonly): ReactElement { mentionRef.current = next; setMention(next); }; + // The `/models` reseat picker overlay (ADR-0059) — a keyboard-owning submode (like the palette/mention/search), + // React-local + ref-shadowed so a coalesced stdin chunk sees a just-applied open/close/accept. A monotonic epoch + // drops a stale async refresh whose picker has since closed/reopened (the Home controller's `pickerEpoch` pattern). + const [modelPicker, setModelPicker] = useState(undefined); + const modelPickerRef = useRef(undefined); + const applyModelPicker = (next: ModelPickerState | undefined): void => { + modelPickerRef.current = next; + setModelPicker(next); + }; + const pickerEpochRef = useRef(0); // A monotonic submit generation: bumped every time the compose buffer is submitted (cleared). An async mention // read captures it at accept time and DROPS its inject if a submit has since happened — so a slow read that // resolves after Enter can never splice the file into the (now-empty) buffer meant for the NEXT message. @@ -491,7 +518,110 @@ export function ChatApp(props: Readonly): ReactElement { ); }; + // ---- The `/models` reseat picker overlay (ADR-0059) — mirrors the Home controller's picker functions -------- + // Apply a refresh result INTO the same picker generation that kicked it (drop a stale/closed/reopened one via the + // epoch); keep the user's filter/selection, surface the per-provider partial-failure banner. Never crash the REPL. + const applyPickerRefresh = (epoch: number, report: RefreshReport | undefined): void => { + const port = props.modelPicker; + const open = modelPickerRef.current; + if (epoch !== pickerEpochRef.current || open === undefined || port === undefined) return; + const failed = report?.providers.filter((p) => p.status === 'failed').map((p) => p.provider) ?? []; + let view: ReturnType; + try { + view = port.load(); // a DB read — never crash the REPL (parity with the Home) + } catch { + applyModelPicker({ ...open, loading: false, banner: partialFailureBanner(failed) }); + return; + } + applyModelPicker({ + ...open, + entries: view.entries, + refreshedAt: view.refreshedAt, + loading: false, + banner: partialFailureBanner(failed), + }); + }; + const runPickerRefresh = (refresh: () => Promise): void => { + const epoch = pickerEpochRef.current; // capture THIS generation so a reopened picker never adopts this result + const open = modelPickerRef.current; + if (open === undefined) return; + applyModelPicker({ ...open, loading: true }); + void refresh().then( + (report) => applyPickerRefresh(epoch, report), + () => { + // refresh()/refreshIfStale() never reject (per-provider isolation), but stay defensive: drop the spinner + // only when this is still the same open picker generation. + const cur = modelPickerRef.current; + if (epoch === pickerEpochRef.current && cur !== undefined) applyModelPicker({ ...cur, loading: false }); + }, + ); + }; + // Open the picker on a typed `/models` (interactive only — the port is present): render the cached catalog + // synchronously (the ✓ is the session's BOUND model — the reseat "you are here"), then kick a TTL-bounded refresh. + const openModelPicker = (): void => { + const port = props.modelPicker; + if (port === undefined) return; + let view: ReturnType; + try { + view = port.load(); // a DB read — a fault must not crash the REPL (the "never crash the REPL" discipline) + } catch { + props.store.note('/models: could not read the model catalog.'); + return; + } + pickerEpochRef.current += 1; // a fresh generation — invalidates any in-flight refresh from a prior (closed) open + applyModelPicker({ + entries: view.entries, + filter: '', + selected: 0, + loading: false, + currentDefault: port.boundModel, + refreshedAt: view.refreshedAt, + banner: undefined, + hint: undefined, + }); + runPickerRefresh(() => port.refreshIfStale()); + }; + // The open picker owns every key (mirrors routeMentionKey). On accept → a LIVE reseat (onReseat sets the stop + // state) then end the driver loop (onExit) so runReplLoop rebuilds the session on the new model. A DIMMED model is + // non-selectable (a transient hint, ADR-0064 §6); any nav/filter keystroke clears that hint. + const routeModelPickerKey = (char: string, key: ModelPickerKey): void => { + const open = modelPickerRef.current; + if (open === undefined) return; + const step = foldModelPickerKey(char, key, open); + switch (step.kind) { + case 'close': + applyModelPicker(undefined); + return; + case 'accept': + applyModelPicker(undefined); + props.onReseat?.({ modelId: step.modelId, provider: step.provider }); + props.onExit(); // the reseat set the stop state; end the loop so runReplLoop swaps in the new-model session + return; + case 'blocked': { + const hint = + step.reason === 'no-key' + ? `${step.displayName}: no key for ${step.provider} — run \`relavium provider set-key ${step.provider}\`` + : `${step.displayName} is not available on your key — pick another`; + applyModelPicker({ ...open, hint }); + return; + } + case 'refresh': + runPickerRefresh(() => props.modelPicker?.refresh() ?? Promise.resolve(undefined)); + return; + case 'state': + // Clear the transient hint only on a REAL interaction — the fold returns the SAME state ref for an inert key. + applyModelPicker(step.state === open ? open : { ...step.state, hint: undefined }); + return; + } + }; + const submit = (message: string, display?: string): void => { + // A typed `/models` opens the reseat picker overlay (ADR-0059) instead of sending — interactive only (the port + // is wired). Covers a directly-typed `/models` AND a chat-palette selection (both route through `submit`). + if (props.modelPicker !== undefined && message.trim() === '/models') { + openModelPicker(); + return; + } // Mark the submit in flight so input is gated + the spinner runs for the WHOLE operation (streaming AND any // after-turn auto-compaction, ADR-0062) — cleared in EVERY settle branch (success / reject / defensive catch). applySubmitBusy(true); @@ -527,6 +657,13 @@ export function ChatApp(props: Readonly): ReactElement { props.store.getSnapshot().state.status === 'running' || shellBusyRef.current || submitBusyRef.current; + // The open `/models` reseat picker owns every key (ADR-0059) — checked FIRST (mutually exclusive with the other + // submodes; it only opens at an idle prompt). Read the REF so a coalesced same-chunk key sees a just-applied + // open/close/accept; on accept it triggers the reseat + ends the loop (see routeModelPickerKey). + if (modelPickerRef.current !== undefined) { + routeModelPickerKey(char, key); + return; + } // The open `@`-mention completion owns every key (2.5.D step 4): Esc/Ctrl-C cancels + restores the literal // keystrokes; ↑/↓ select; Enter/Tab/'/' accept (a dir descends, a file injects); backspace trims the filter // then deletes the `@`; a printable extends the filter. Read the REF so a coalesced same-chunk key sees a @@ -752,7 +889,12 @@ export function ChatApp(props: Readonly): ReactElement { approval={approval} attachments={attachments} busyCommand={busyCommand} - paletteOpen={palette !== undefined || search !== undefined || mention !== undefined} + paletteOpen={ + palette !== undefined || + search !== undefined || + mention !== undefined || + modelPicker !== undefined + } /> {palette !== undefined && ( @@ -761,6 +903,11 @@ export function ChatApp(props: Readonly): ReactElement { )} {mention !== undefined && } + {/* The `/models` reseat picker overlay (ADR-0059) — mounted like the palette; nowMs feeds the freshness badge + (cosmetic, render-only, so `Date.now()` is fine on this UI path — no engine-purity concern here). */} + {modelPicker !== undefined && ( + + )} ); } @@ -844,6 +991,10 @@ export function driveInk(ctx: ChatDriveContext): Promise { // 'abort' handler can reject a pending approval when it is absent (never a dead Esc — see ChatApp). onAbort: ctx.onAbort, onModeChange: ctx.onModeChange ?? ((): void => undefined), + // `/models` reseat (ADR-0059) — the REPL loop wires both onReseat + the picker port only for an interactive + // session; passed AS-IS (optional) so a driver wired without them simply has no `/models` overlay. + onReseat: ctx.onReseat, + modelPicker: ctx.modelPicker, // `@`-mention completion (2.5.D, ADR-0061) — the REPL loop wires it only for an interactive session; passed // AS-IS (optional) so an absent reader degrades `@` to a literal char (never a dead key — see ChatApp). mentionReader: ctx.mentionReader, diff --git a/apps/cli/src/render/tui/model-picker.test.ts b/apps/cli/src/render/tui/model-picker.test.ts index f20b2928..e64aa324 100644 --- a/apps/cli/src/render/tui/model-picker.test.ts +++ b/apps/cli/src/render/tui/model-picker.test.ts @@ -68,12 +68,16 @@ describe('foldModelPickerKey', () => { }); }); - it('Enter on an AVAILABLE model accepts it (modelId + displayName)', () => { - const s = state({ entries: [entry({ modelId: 'x', displayName: 'Model X' })], selected: 0 }); + it('Enter on an AVAILABLE model accepts it (modelId + displayName + provider for the reseat target)', () => { + const s = state({ + entries: [entry({ modelId: 'x', displayName: 'Model X', provider: 'openai' })], + selected: 0, + }); expect(foldModelPickerKey('', { return: true }, s)).toEqual({ kind: 'accept', modelId: 'x', displayName: 'Model X', + provider: 'openai', // the entry is authoritative — the chat reseat (ADR-0059) needs it }); }); diff --git a/apps/cli/src/render/tui/model-picker.ts b/apps/cli/src/render/tui/model-picker.ts index 51e964b1..b5983322 100644 --- a/apps/cli/src/render/tui/model-picker.ts +++ b/apps/cli/src/render/tui/model-picker.ts @@ -5,13 +5,17 @@ import { dropLastCodePoint } from './chat-input.js'; /** * The `/models` picker submode (workstream **2.5.G S7**, [ADR-0064](../../../../../docs/decisions/0064-live-model-catalog.md) §10) * — a keyboard-owning overlay (like the `/` palette + the `@`-mention completion) that lists the MERGED model - * catalog and, on selection, writes the NEXT session's default model ([ADR-0063](../../../../../docs/decisions/0063-cli-config-write-contract.md)); - * it does NOT rebind the live session (that is the Phase-2.6 `/models` reseat, ADR-0059). Home-only (`availableIn: ['home']`). + * catalog and, on selection, acts on the chosen model. It serves TWO surfaces off the SAME pure fold: the **Home** + * writes the NEXT session's default model ([ADR-0063](../../../../../docs/decisions/0063-cli-config-write-contract.md)), + * and the **chat REPL** (ADR-0059) rebinds the live session mid-conversation via a host-side reseat — both keyed off + * the one `accept` step (which carries the model id + provider). The Home version is `availableIn: ['home']` for the + * config-write action; the chat version is triggered by a typed `/models` intercepted at the ink layer. * * The PURE model — state + fold + the display formatters — lives here; the ink view ({@link model-picker-view.tsx}) - * renders it and the Home controller routes keys + does the async db/refresh/write I/O (mirroring the mention submode). - * A DIMMED (unavailable-on-your-key) or a deprecated model is still shown (ADR-0064 §6/§7: dim/flag, never hide) but - * a dimmed model is **non-selectable** — accepting one yields a `blocked` step, not a write. + * renders it and each surface (the Home controller / the chat ink `ChatApp`) routes keys + does the async db/refresh + * + the accept action (mirroring the mention submode). A DIMMED (unavailable-on-your-key) or a deprecated model is + * still shown (ADR-0064 §6/§7: dim/flag, never hide) but a dimmed model is **non-selectable** — accepting one yields + * a `blocked` step, not an action. */ /** @@ -52,8 +56,15 @@ export interface ModelPickerKey { /** What a keystroke does to the open picker. */ export type ModelPickerStep = - | { readonly kind: 'close' } // Esc / Ctrl-C — cancel without writing a default - | { readonly kind: 'accept'; readonly modelId: string; readonly displayName: string } // set the default + | { readonly kind: 'close' } // Esc / Ctrl-C — cancel without acting + // Accept the selected model. `provider` rides along (the entry is authoritative) so the chat reseat (ADR-0059) + // has its `{ modelId, provider }` target; the Home's default-write reads only `modelId`/`displayName`. + | { + readonly kind: 'accept'; + readonly modelId: string; + readonly displayName: string; + readonly provider: ProviderId; + } | { readonly kind: 'blocked'; // a dimmed/unavailable model — non-selectable (ADR-0064 §6) readonly displayName: string; @@ -133,7 +144,12 @@ export function foldModelPickerKey( ...(chosen.unavailableReason !== undefined ? { reason: chosen.unavailableReason } : {}), }; } - return { kind: 'accept', modelId: chosen.modelId, displayName: chosen.displayName }; + return { + kind: 'accept', + modelId: chosen.modelId, + displayName: chosen.displayName, + provider: chosen.provider, + }; } if (key.backspace === true || key.delete === true) { if (state.filter.length === 0) return { kind: 'state', state }; // nothing to trim (Esc cancels; backspace is inert) diff --git a/packages/llm/src/model-catalog.test.ts b/packages/llm/src/model-catalog.test.ts index a5d341a3..15c7e9e4 100644 --- a/packages/llm/src/model-catalog.test.ts +++ b/packages/llm/src/model-catalog.test.ts @@ -247,21 +247,23 @@ describe('mergeModelCatalog (ADR-0064 §6)', () => { expect(entry?.deprecated).toBe(true); }); - it('is deterministically ordered: provider (seam order) then displayName then id', () => { + it('orders AVAILABLE models first (alphabetical), then unavailable (alphabetical) — maintainer 2.5.G', () => { + // openai has live data (only gpt-6-preview), so its static-but-not-live models are dimmed (available:false); + // every other provider has no live data ⇒ static presence (available:true). const entries = mergeModelCatalog({ live: liveMap([['openai', [{ id: 'gpt-6-preview' }]]]), now: BEFORE_DEEPSEEK_DEPRECATION, }); - // provider order is anthropic < openai < gemini < deepseek - const providers = entries.map((e) => e.provider); - const firstOpenAi = providers.indexOf('openai'); - const lastAnthropic = providers.lastIndexOf('anthropic'); - const firstGemini = providers.indexOf('gemini'); - expect(lastAnthropic).toBeLessThan(firstOpenAi); - expect(firstOpenAi).toBeLessThan(firstGemini); - // within a provider, entries are displayName-then-id sorted (stable, no duplicates) - const anthropic = entries.filter((e) => e.provider === 'anthropic').map((e) => e.displayName); - expect(anthropic).toEqual([...anthropic].sort((a, b) => a.localeCompare(b))); + // Availability is the PRIMARY key: every available entry precedes every unavailable one. + const firstUnavailable = entries.findIndex((e) => !e.available); + expect(firstUnavailable).toBeGreaterThan(0); // there ARE dimmed entries (openai's non-live static models) + expect(entries.slice(0, firstUnavailable).every((e) => e.available)).toBe(true); + expect(entries.slice(firstUnavailable).every((e) => !e.available)).toBe(true); + // Within each availability group, entries are displayName-sorted (en locale), stable + no duplicates. + const availableNames = entries.filter((e) => e.available).map((e) => e.displayName); + expect(availableNames).toEqual([...availableNames].sort((a, b) => a.localeCompare(b, 'en'))); + const dimmedNames = entries.filter((e) => !e.available).map((e) => e.displayName); + expect(dimmedNames).toEqual([...dimmedNames].sort((a, b) => a.localeCompare(b, 'en'))); // The modelId tiebreaker + insertion-order independence: two user-priced unknown ids that TIE on // provider (openai) + displayName ('Custom Model') must order by modelId ascending, regardless of the // input Map's insertion order (proves the model-catalog.ts sort tiebreaker, not a same-input re-run). diff --git a/packages/llm/src/model-catalog.ts b/packages/llm/src/model-catalog.ts index bfb697b5..6f751939 100644 --- a/packages/llm/src/model-catalog.ts +++ b/packages/llm/src/model-catalog.ts @@ -1,5 +1,3 @@ -import { LLM_PROVIDERS } from '@relavium/shared'; - import { MODEL_PRICING, type ModelPricing } from './pricing.js'; import type { ModelListing, ProviderId } from './types.js'; @@ -99,8 +97,6 @@ function earlierIsoDate(a: string | undefined, b: string | undefined): string | return pa <= pb ? a : b; } -const PROVIDER_RANK = new Map(LLM_PROVIDERS.map((p, i) => [p, i])); - /** * Reconcile live discovery ⋈ the static registry ⋈ the user tier into one deterministically-ordered catalog * (ADR-0064 §6). Pure: no I/O, no `Date.now()` (the caller passes `now`). Per-field precedence — @@ -188,12 +184,14 @@ export function mergeModelCatalog(input: MergeModelCatalogInput): ModelCatalogEn }); } - // Deterministic order: provider (in the seam's LLM_PROVIDERS order), then displayName, then modelId. + // Order (maintainer, 2.5.G): AVAILABLE (selectable) models FIRST, then the dimmed/unavailable ones — each group + // sorted alphabetically by displayName, with modelId as the deterministic tiebreaker. So the picker shows the + // models a user can actually pick at the top (alphabetical), with the no-key / not-on-key ones grouped below + // (ADR-0064 §6: dimmed, never hidden). Provider is no longer a sort key — availability + name is the user's axis. entries.sort((x, y) => { - const byProvider = - (PROVIDER_RANK.get(x.provider) ?? LLM_PROVIDERS.length) - - (PROVIDER_RANK.get(y.provider) ?? LLM_PROVIDERS.length); - if (byProvider !== 0) return byProvider; + // `available: true` sorts BEFORE `available: false` (true → 0, false → 1). + const byAvailability = (x.available ? 0 : 1) - (y.available ? 0 : 1); + if (byAvailability !== 0) return byAvailability; // Pin an explicit locale so the catalog order is byte-identical across every host/OS/CI locale (a // runtime-default locale — e.g. Danish — can flip case ordering for a provider-controlled live displayName). const byName = x.displayName.localeCompare(y.displayName, 'en'); From f144b03e04a7fd01df1672bb735c23414f7f9008 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 15:52:56 +0300 Subject: [PATCH 47/68] =?UTF-8?q?feat(cli,docs):=202.5.G=20Step=20D=20(2c)?= =?UTF-8?q?=20=E2=80=94=20in-Home=20chat=20/models=20reseat=20+=20review?= =?UTF-8?q?=20fixes=20(ADR-0059)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes the maintainer-reported bug (also Opus-review finding #1): a chat opened from the bare `relavium` Home showed a misleading "from a pipe" hint on `/models` and would not switch the model — only the standalone `relavium chat` reseated. Now the in-Home chat has FULL parity: a typed/palette `/models` opens the SAME reseat picker and live-switches the model, completing the ADR-0059 in-Home reseat (previously deferred). - session-host: extract `swapAgentModel` (the model/provider swap + drop fallback_chain) — shared by the standalone reseat (buildReseatWiring) and the Home reseat, so the rule has one home. - drive-home: extract `wireHomeChatSession` (the shared HomeChatSession wiring, fresh OR resumed) from startChat, and add a `reseatChat` builder — reload + resume the session's transcript under the model-swapped agent, carrying cost/turns. Wired into the controller. - home-controller: intercept `/models` in a live chat (typed + palette) to open the picker; the picker's ✓ is the session's BOUND model; accept RESEATS the live chat (vs. the bare Home's next-session-default write) via a build-first swap mirroring clearChat. - home-app: render the model-picker overlay in the chat region (it was home-only). - Test: in-Home /models opens the picker + accept reseats the live session (swap + teardown). Review fixes: reconcile the ADR-0059 note (attribution shipped in Step D, not 2.6.C, with the known cross-provider by-string limitation documented); correct the stale ReplCommandContext.openModels comment (now dual-surface ['home','chat']). Toolchain green: 1489 tests, all lint/typecheck/build tasks pass. Refs: ADR-0059 Co-Authored-By: Claude --- apps/cli/src/chat/session-host.ts | 21 ++- apps/cli/src/commands/chat.ts | 13 +- apps/cli/src/commands/repl-commands.ts | 12 +- apps/cli/src/home/drive-home.tsx | 161 ++++++++++++------ apps/cli/src/render/tui/home-app.tsx | 14 +- .../src/render/tui/home-controller.test.ts | 44 +++++ apps/cli/src/render/tui/home-controller.ts | 118 +++++++++++-- .../0059-cli-mid-session-model-reseat.md | 24 ++- 8 files changed, 320 insertions(+), 87 deletions(-) diff --git a/apps/cli/src/chat/session-host.ts b/apps/cli/src/chat/session-host.ts index f555571c..88bb8db9 100644 --- a/apps/cli/src/chat/session-host.ts +++ b/apps/cli/src/chat/session-host.ts @@ -16,7 +16,7 @@ import { type ToolDef, type ToolHost, } from '@relavium/core'; -import type { PricingOverlay } from '@relavium/llm'; +import type { PricingOverlay, ProviderId } from '@relavium/llm'; import type { ManagerSkippedTool, McpClient, McpServerConfig } from '@relavium/mcp'; import type { AgentSessionRecord, @@ -359,6 +359,25 @@ function narrowToWired( return { ...agent, tools: wiredToolIds(agent.tools ?? [], host, defs) }; } +/** + * Return the bound agent for a mid-session model SWITCH ([ADR-0059](../../../../docs/decisions/0059-cli-mid-session-model-reseat.md)): + * the snapshot with `model`/`provider` swapped to the picked pair and the original `fallback_chain` DROPPED (it + * belonged to the old model; the resumed instance builds its own default plan for the new model, exactly as a fresh + * session on it would). Operates on a fresh copy — never mutates the input. Shared by the standalone `chat` reseat + * (`buildReseatWiring`) and the in-Home chat reseat (`driveHome`) so the swap rule has ONE home. + */ +export function swapAgentModel( + agent: AgentDefinition, + modelId: string, + provider: ProviderId, +): AgentDefinition { + // A fresh copy, then `delete` the optional `fallback_chain` (removes the key entirely — never an explicit + // `undefined` under exactOptionalPropertyTypes — and mutates only this copy, never the loaded record). + const next: AgentDefinition = { ...agent, model: modelId, provider }; + delete next.fallback_chain; + return next; +} + /** A resumed session (2.N) plus the two extra facts the REPL needs: the reconstructed state + the next seq. */ export interface BuiltResumedChatSession extends BuiltChatSession { /** The reconstructed in-flight state the view seeds from (carried cost + prior completed-turn count). */ diff --git a/apps/cli/src/commands/chat.ts b/apps/cli/src/commands/chat.ts index 5a1a463d..39bd201e 100644 --- a/apps/cli/src/commands/chat.ts +++ b/apps/cli/src/commands/chat.ts @@ -46,6 +46,7 @@ import { import { buildChatSession, buildResumedChatSession, + swapAgentModel, type BuildChatSessionOptions, type BuiltChatSession, type BuiltResumedChatSession, @@ -1230,16 +1231,8 @@ async function buildReseatWiring( `cannot switch model: session ${oldSessionId} could not be reloaded for reseat`, ); } - // Swap the bound model/provider on a FRESH copy, then DROP the original `fallback_chain` — it belonged to the old - // model; the resumed instance builds its own default plan for `target` (ADR-0059). `fallback_chain` is optional on - // `AgentDefinition`, so `delete` removes the key entirely (never leaves an explicit `undefined`, satisfying - // exactOptionalPropertyTypes) and mutates only this copy, never the loaded record. - const newAgent: AgentDefinition = { - ...loaded.session.agentSnapshot, - model: target.modelId, - provider: target.provider, - }; - delete newAgent.fallback_chain; + // Swap the bound model/provider (dropping the original fallback_chain) via the shared ADR-0059 rule. + const newAgent = swapAgentModel(loaded.session.agentSnapshot, target.modelId, target.provider); // The record the resumed session rebinds from: the just-ended row with the model-swapped agent snapshot. // (The row's own `modelId` FK column stays as-is — per-message/session `modelId` attribution is deferred with // the 2.6.C cost breakdown; see persister.ts. `reconstructSessionState` reads only the transcript + cost here.) diff --git a/apps/cli/src/commands/repl-commands.ts b/apps/cli/src/commands/repl-commands.ts index a17a8508..abaf719d 100644 --- a/apps/cli/src/commands/repl-commands.ts +++ b/apps/cli/src/commands/repl-commands.ts @@ -53,11 +53,13 @@ export interface ReplCommandContext { * Interactive-only — a `--json` / plain non-TTY session rejects it (one machine stream is one session lifecycle). * Its notice surfaces the OLD sessionId + `relavium chat-resume ` so the prior conversation is discoverable. */ readonly clearSession: () => void | Promise; - /** `/models` (2.5.G S7, [ADR-0064](../../../../docs/decisions/0064-live-model-catalog.md) §10) — open the in-tree - * model picker over the merged live/static catalog; on selection it writes the NEXT session's default model - * ([ADR-0063](../../../../docs/decisions/0063-cli-config-write-contract.md)), it does NOT rebind the live session - * (that is the Phase-2.6 reseat, ADR-0059). HOME-ONLY (`availableIn: ['home']`): the Home wires the real picker; - * the chat surface never reaches this (the slash dispatch rejects a non-`chat` command), so its impl is inert. */ + /** `/models` (2.5.G S7, [ADR-0064](../../../../docs/decisions/0064-live-model-catalog.md) §10 · ADR-0059) — open the + * in-tree model picker over the merged live/static catalog. `availableIn: ['home','chat']`, with a surface-specific + * accept: the **Home** writes the NEXT session's default model ([ADR-0063](../../../../docs/decisions/0063-cli-config-write-contract.md)), + * and a **live chat** rebinds the running session via a host-side reseat (ADR-0059). The interactive render layer + * (the ink `ChatApp` / the in-Home controller) INTERCEPTS a typed/palette `/models` to open the overlay, so this + * ctx handler runs ONLY on a non-interactive chat driver (plain/`--json`), where it surfaces an actionable hint; + * the bare Home wires the real picker. */ readonly openModels: () => void | Promise; } diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index 0d97fcb6..e6c2f573 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -1,11 +1,17 @@ import { randomUUID } from 'node:crypto'; import { createProviderStore, createRunHistoryReader } from '@relavium/db'; +import type { AgentSessionRecord } from '@relavium/shared'; import { render } from 'ink'; import { createElement } from 'react'; -import { createChatLineHandler } from '../commands/chat.js'; -import { buildChatSession, type BuiltChatSession } from '../chat/session-host.js'; +import { createChatLineHandler, type ReseatTarget } from '../commands/chat.js'; +import { + buildChatSession, + buildResumedChatSession, + swapAgentModel, + type BuiltChatSession, +} from '../chat/session-host.js'; import { assembleDoctorProbes } from '../chat/doctor-host.js'; import type { DoctorProbes } from '../chat/doctor.js'; import { @@ -20,6 +26,7 @@ import { readUserPricingOverlay } from '../engine/pricing-overlay.js'; import { assembleToolEnv } from '../engine/tool-host/assemble.js'; import { createProviderResolver, type ProviderResolver } from '../engine/providers.js'; import { openSessionStore, type OpenedSessionStore } from '../history/session-open.js'; +import { CliError } from '../process/errors.js'; import { isProviderKeyless, runOnboardingWizard, @@ -30,7 +37,7 @@ import { EXIT_CODES, type ExitCode } from '../process/exit-codes.js'; import type { GlobalOptions } from '../process/options.js'; import { createMcpSecretResolver, type McpSecretResolver } from '../secrets/mcp-secret.js'; import { createOsKeychainStore } from '../secrets/os-keychain.js'; -import { createChatStore } from '../render/tui/chat-store.js'; +import { createChatStore, type ChatStoreController } from '../render/tui/chat-store.js'; import { createMentionReader } from '../render/tui/mention.js'; import { createHomeController, @@ -64,6 +71,8 @@ export interface HomeDeps { readonly providers?: ProviderResolver; /** Injectable session builder (tests inject a scripted provider via `providers`). Default {@link buildChatSession}. */ readonly buildSession?: typeof buildChatSession; + /** Injectable RESUMED-session builder — used by the in-Home `/models` reseat (ADR-0059). Default {@link buildResumedChatSession}. */ + readonly buildResumedSession?: typeof buildResumedChatSession; /** Injectable session-store opener (tests pass an in-memory store). Default {@link openSessionStore}. */ readonly openSessionStore?: (homeDir: string) => OpenedSessionStore; readonly mcpSecretResolver?: McpSecretResolver; @@ -190,43 +199,22 @@ export async function driveHome(deps: HomeDeps): Promise { writeDefault: (modelId) => writeGlobalDefaultModel(modelId, homeDir, deps.global.configPath), }; - // Build + wire + START a fresh chat session (the controller sends the first message on transition). - const startChat = async (): Promise => { - const store = createChatStore(deps.global.color); - // The ADR-0065 §2 user-pricing overlay (2.5.G S10), read FRESH per chat from the SAME `history.db` — so a - // user-priced model started in this long-lived Home is enforced by `[chat].max_cost_microcents` + tracked in - // realized cost. Static `MODEL_PRICING` still wins. Non-fatal (empty map on a read fault). - const resolvePrice = readUserPricingOverlay(opened.db); - const built: BuiltChatSession = await (deps.buildSession ?? buildChatSession)({ - // Re-read the EFFECTIVE default model FRESH per chat (not the load-once `config` snapshot) so a same-session - // `/models` write takes effect on the very next chat started in this long-lived Home (2.5.G S7) — the - // property the accept-notice's "applies to your next chat session" promises. A read fault degrades to the - // startup value. Other `[chat]` settings keep the startup snapshot (only `/models` mutates the default). - chat: { ...config.chat, defaultModel: readEffectiveDefault() ?? config.chat.defaultModel }, - agentRef: undefined, // the built-in default agent (zero-config first run) - cwd: deps.global.cwd, - projectConfigDir, - now, - uuid, - providers, - mcpSecretResolver, - mcpRegistrations: config.mcpServers, - ...(resolvePrice.size === 0 ? {} : { resolvePrice }), - onBudgetWarning: (warning) => - deps.io.writeErr( - `budget warning: ~${warning.thresholdPct}% of the ${warning.limitMicrocents}µ¢ cap reached\n`, - ), - }); + // The SHARED HomeChatSession wiring over a built (FRESH or RESUMED) session + its view store — the persister, + // the shared line handler (owns the ADR-0057 mode floor), the stream subscription + frame, the mention/shell + // ports, and the bounded teardown. Used by startChat (fresh) and reseatChat (the ADR-0059 model switch), so the + // in-Home chat wiring has ONE home. `open` opens a FRESH session (`session.start()`); a RESUMED session already + // landed at idle, so it stays `false` and seeds the persister past its last durable sequence. + const wireHomeChatSession = async ( + built: BuiltChatSession, + store: ChatStoreController, + opts: { readonly open: boolean; readonly initialSequenceNumber?: number }, + ): Promise => { // Surface any config-level MCP tools the build skipped through the chat's ⚠ warnings channel (NOT stderr, - // which would corrupt the live TUI) — parity with `relavium chat`'s surfaceMcpSkipped so a Home-started chat - // tells the user why a configured tool is unavailable. + // which would corrupt the live TUI) — parity with `relavium chat`'s surfaceMcpSkipped. for (const tool of built.mcpSkipped) { store.note(`MCP tool '${tool.name}' (server '${tool.server}') skipped — ${tool.reason}`); } - // The chat's `/doctor` probes reflect THIS session's MCP status (the bound agent's declared servers + the - // tools the manager dropped) — derived from `built`, not the Home defaults — so `/doctor --deep` in a - // Home-started chat is correct by construction (the default agent has no `mcp_servers` today, but this keeps - // it right if that ever changes). A test override (`deps.doctorProbes`) still wins. + // The chat's `/doctor` probes reflect THIS session's MCP status (derived from `built`, not the Home defaults). const chatDoctorProbes = deps.doctorProbes ?? assembleDoctorProbes({ @@ -236,9 +224,8 @@ export async function driveHome(deps: HomeDeps): Promise { agentMcpServers: built.agent.mcp_servers ?? [], mcpSkipped: built.mcpSkipped, }); - // Acquire-then-guard: once the subscription + frame interval exist, a throw in the remaining wiring - // (persister.start()'s insert, session.start()) must reclaim them — and any spawned MCP child — rather than - // leak the timer/subscription, mirroring chatCommand's post-build guard. + // Acquire-then-guard: once the subscription + frame interval exist, a throw in the remaining wiring must + // reclaim them — and any spawned MCP child — rather than leak the timer/subscription (chatCommand's guard). let frame: ReturnType | undefined; let unsubscribe: (() => void) | undefined; let persister: SessionPersister | undefined; @@ -251,13 +238,16 @@ export async function driveHome(deps: HomeDeps): Promise { context: built.context, now, uuid, - // ADR-0059 per-message/session model attribution — resolve a model string → its `model_catalog.id` - // over the SAME db (the Home's catalog is the one the picker refreshes), degrading to NULL when uncataloged. + // ADR-0059 per-message/session model attribution — resolve a model string → its `model_catalog.id` over + // the SAME db (the Home's catalog is the one the picker refreshes), degrading to NULL when uncataloged. resolveModelCatalogId: makeCatalogIdResolver(opened.db, { uuid, now }), + // A RESUMED session continues the durable transcript past its last sequence number; a fresh one starts at 0. + ...(opts.initialSequenceNumber === undefined + ? {} + : { initialSequenceNumber: opts.initialSequenceNumber }), }); // createChatLineHandler owns the mode control (ADR-0057): it applies the initial `ask` mode → the - // fail-closed approval regime — BEFORE the session opens, so the full-capability chat host is never live - // without the per-tool approval floor (the SAME guarantee the `chat` command's runReplLoop provides). + // fail-closed approval regime — BEFORE the session opens, so the full-capability host is never live without it. const { processLine, cancelOnce, shouldStop, stopReason, onAbort, onModeChange } = createChatLineHandler( { built, opened, store, persister, doctorProbes: chatDoctorProbes }, @@ -268,20 +258,17 @@ export async function driveHome(deps: HomeDeps): Promise { frame = setInterval(() => store.tick(), FRAME_MS); frame.unref(); persister.start(); - built.session.start(); - // The `@`-mention completion reader (2.5.D, ADR-0061): a READ-ONLY fs jail at the SAME fs-scope tier + - // workspace as the session's tools, so in-Home `@`-completion browses + injects through the identical - // confidentiality floor + listing-gate. READ-ONLY by construction (the Home is always a TTY). Building it is - // pure (no I/O). Absent (an unwired fs arm) ⇒ `@` degrades to a literal char. + // Open a FRESH session; a RESUMED session already landed at idle inside AgentSession.resume — start() would + // throw and re-emitting session:started would double a terminal-less lifecycle event. + if (opts.open) built.session.start(); + // The `@`-mention completion reader (2.5.D, ADR-0061): a READ-ONLY fs jail at the session's fs-scope tier. const mentionFs = assembleToolEnv({ profile: 'chat-read-only', fsScopeTier: built.context.fsScopeTier, workspaceDir: built.context.workingDir, }).host.fs; const mentionReader = mentionFs === undefined ? undefined : createMentionReader(mentionFs); - // The `!`-shell runner (2.5.D step 5, ADR-0061) — a thin wrapper over the session's `runUserCommand` (the one - // command boundary: allowlist BEFORE approval → mode-aware confirmAction → hardened process arm). The Home is - // always a TTY, so it is always wired; the empty-default `[chat].allowed_commands` keeps `!` inert until opt-in. + // The `!`-shell runner (2.5.D step 5, ADR-0061) — a thin wrapper over the session's `runUserCommand`. const runShellCommand = ( command: string, args: readonly string[], @@ -324,6 +311,77 @@ export async function driveHome(deps: HomeDeps): Promise { } }; + // Build + wire + START a fresh chat session (the controller sends the first message on transition). + const startChat = async (): Promise => { + const store = createChatStore(deps.global.color); + // The ADR-0065 §2 user-pricing overlay (2.5.G S10), read FRESH per chat from the SAME `history.db` (empty map + // on a read fault). Static `MODEL_PRICING` still wins. + const resolvePrice = readUserPricingOverlay(opened.db); + const built: BuiltChatSession = await (deps.buildSession ?? buildChatSession)({ + // Re-read the EFFECTIVE default model FRESH per chat (not the load-once `config` snapshot) so a same-session + // `/models` write takes effect on the very next chat started in this long-lived Home (2.5.G S7). A read + // fault degrades to the startup value. Other `[chat]` settings keep the startup snapshot. + chat: { ...config.chat, defaultModel: readEffectiveDefault() ?? config.chat.defaultModel }, + agentRef: undefined, // the built-in default agent (zero-config first run) + cwd: deps.global.cwd, + projectConfigDir, + now, + uuid, + providers, + mcpSecretResolver, + mcpRegistrations: config.mcpServers, + ...(resolvePrice.size === 0 ? {} : { resolvePrice }), + onBudgetWarning: (warning) => + deps.io.writeErr( + `budget warning: ~${warning.thresholdPct}% of the ${warning.limitMicrocents}µ¢ cap reached\n`, + ), + }); + return wireHomeChatSession(built, store, { open: true }); + }; + + // Reseat the in-Home chat onto a NEW model (ADR-0059) — the counterpart of the standalone `chat` reseat. Reload + // the just-torn-down session's transcript from the SHARED db and RESUME it under a model-swapped agent (dropping + // the original fallback_chain), carrying the text-only transcript + cumulative cost/turns; a NEW instance, + // honoring ADR-0024's one-model-per-lifetime rule. The controller drives the tear-down / swap (mirroring clearChat). + const reseatChat = async (sessionId: string, target: ReseatTarget): Promise => { + const loaded = opened.store.loadFull(sessionId); + if (loaded === undefined || loaded.session.agentSnapshot === undefined) { + throw new CliError( + 'invalid_invocation', + `cannot switch model: session ${sessionId} could not be reloaded for reseat`, + ); + } + const newAgent = swapAgentModel(loaded.session.agentSnapshot, target.modelId, target.provider); + const record: AgentSessionRecord = { ...loaded.session, agentSnapshot: newAgent }; + const resolvePrice = readUserPricingOverlay(opened.db); + const built = await (deps.buildResumedSession ?? buildResumedChatSession)({ + chat: config.chat, + record, + messages: loaded.messages, + now, + providers, + mcpSecretResolver, + mcpRegistrations: config.mcpServers, + ...(resolvePrice.size === 0 ? {} : { resolvePrice }), + onBudgetWarning: (warning) => + deps.io.writeErr( + `budget warning: ~${warning.thresholdPct}% of the ${warning.limitMicrocents}µ¢ cap reached\n`, + ), + }); + // Seed the view store with the carried model + cost/turns — a resumed session never re-emits session:started, + // so without this the footer shows nothing until the first new turn (mirrors chatResumeCommand). + const store = createChatStore(deps.global.color, { + agentRef: built.agent.id, + model: built.agent.model, + cumulativeCostMicrocents: built.resumeState.cumulativeCostMicrocents, + turnCount: built.resumeState.turnCount, + }); + return wireHomeChatSession(built, store, { + open: false, + initialSequenceNumber: built.nextSequenceNumber, + }); + }; + const getSize = deps.getSize ?? (() => ({ cols: process.stdout.columns ?? 80, rows: process.stdout.rows ?? 24 })); @@ -394,6 +452,7 @@ export async function driveHome(deps: HomeDeps): Promise { return await new Promise((resolve, reject) => { controller = createHomeController({ startChat, + reseatChat, homeStore, doctorProbes, models, diff --git a/apps/cli/src/render/tui/home-app.tsx b/apps/cli/src/render/tui/home-app.tsx index 9fb7655f..dc2d0fd5 100644 --- a/apps/cli/src/render/tui/home-app.tsx +++ b/apps/cli/src/render/tui/home-app.tsx @@ -12,6 +12,7 @@ import { HomeView } from './home-view.js'; import type { ReverseSearchState } from './input-history.js'; import type { MentionState } from './mention.js'; import { MentionView } from './mention-view.js'; +import type { ModelPickerState } from './model-picker.js'; import { ModelPickerView } from './model-picker-view.js'; import { PaletteView } from './palette-view.js'; import type { PaletteState } from './palette-reducer.js'; @@ -47,6 +48,8 @@ function ChatRegion( palette: PaletteState | undefined; search: ReverseSearchState | undefined; mention: MentionState | undefined; + modelPicker: ModelPickerState | undefined; + nowMs: number; shellBusy: boolean; submitBusy: boolean; shellCommand: string | undefined; @@ -71,7 +74,10 @@ function ChatRegion( attachments={props.attachments} busyCommand={props.shellCommand} paletteOpen={ - props.palette !== undefined || props.search !== undefined || props.mention !== undefined + props.palette !== undefined || + props.search !== undefined || + props.mention !== undefined || + props.modelPicker !== undefined } /> {props.palette !== undefined && ( @@ -81,6 +87,10 @@ function ChatRegion( )} {props.mention !== undefined && } + {/* The `/models` reseat picker overlay in a live in-Home chat (ADR-0059) — mounted like the palette. */} + {props.modelPicker !== undefined && ( + + )} ); } @@ -103,6 +113,8 @@ export function RootApp(props: Readonly): ReactElement { palette={state.palette} search={state.search} mention={state.mention} + modelPicker={state.modelPicker} + nowMs={props.nowMs()} shellBusy={state.shellBusy} submitBusy={state.submitBusy} shellCommand={state.shellCommand} diff --git a/apps/cli/src/render/tui/home-controller.test.ts b/apps/cli/src/render/tui/home-controller.test.ts index 29689458..93981ce2 100644 --- a/apps/cli/src/render/tui/home-controller.test.ts +++ b/apps/cli/src/render/tui/home-controller.test.ts @@ -1877,4 +1877,48 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => expect(c.getSnapshot().modelPicker).toBeUndefined(); expect(c.getSnapshot().notice).toContain('unavailable'); }); + + it('in-Home chat: /models opens the reseat picker; accepting a model RESEATS the live session (ADR-0059)', async () => { + const sessionA = makeSession({ sessionId: 'sess-A' }); + const sessionB = makeSession({ sessionId: 'sess-A' }); // a reseat continues the SAME sessionId + const startChat = vi.fn(() => Promise.resolve(sessionA.session)); + const reseatChat = vi.fn(() => Promise.resolve(sessionB.session)); + const { port } = makeModelsPort({ + entries: [pickerEntry({ modelId: 'claude-opus-4-8', provider: 'anthropic' })], + }); + const c = createHomeController({ + doctorProbes: STUB_DOCTOR_PROBES, + startChat, + reseatChat, + models: port, + homeStore, + onExit: vi.fn(), + onError: vi.fn(), + }); + type(c, 'hi'); + c.handleKey('', ENTER); // start the chat (session A) + await flush(); + expect(c.getSnapshot().mode).toBe('chat'); + + // Open the reseat picker from the chat palette (`/` → filter `models` → run) — the in-chat intercept opens the + // picker instead of dispatching `/models` to the "interactive terminal" hint. + c.handleKey('/', {}); + type(c, 'models'); + c.handleKey('', ENTER); + await flush(); + expect(c.getSnapshot().modelPicker).toBeDefined(); // the picker opened IN the chat + + // Accept the (only, available) model → a LIVE reseat, not a next-session-default write. + c.handleKey('', ENTER); + await flush(); + + expect(reseatChat).toHaveBeenCalledWith('sess-A', { + modelId: 'claude-opus-4-8', + provider: 'anthropic', + }); + expect(sessionA.teardown).toHaveBeenCalledTimes(1); // the old session torn down (bounded) + expect(c.getSnapshot().session).toBe(sessionB.session); // swapped to the reseated session + expect(c.getSnapshot().mode).toBe('chat'); // stayed in chat (the model switched underneath) + expect(c.getSnapshot().modelPicker).toBeUndefined(); // the picker closed + }); }); diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index 23d45952..6c9fba66 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -12,8 +12,9 @@ import { partialFailureBanner, type ModelPickerState, } from './model-picker.js'; +import type { ReseatTarget } from '../../commands/chat.js'; import { nextMode, type ChatMode } from '../../chat/chat-mode.js'; -import { clearedNotice } from '../../chat/repl-info.js'; +import { clearedNotice, modelSwitchNotice } from '../../chat/repl-info.js'; import { formatDoctorReport, runDoctorChecks, type DoctorProbes } from '../../chat/doctor.js'; import type { HomeSnapshot, HomeStore } from '../../home/home-store.js'; import { @@ -187,6 +188,9 @@ export interface HomeModelsPort { export interface HomeControllerDeps { /** Build + wire + START a fresh chat session (no first message — the controller sends it on transition). May reject. */ readonly startChat: () => Promise; + /** Reseat the in-Home chat onto a NEW model (ADR-0059) — reload + resume the current session's transcript under the + * switched model. Absent ⇒ the in-Home `/models` picker degrades to the next-session-default write (no live reseat). */ + readonly reseatChat?: (sessionId: string, target: ReseatTarget) => Promise; readonly homeStore: HomeStore; /** The Home exited cleanly (Ctrl-C / EOF in Home mode) → `driveHome` resolves with exit 0. */ readonly onExit: () => void; @@ -402,6 +406,76 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { ); }; + // Reseat the in-Home chat onto a NEW model (ADR-0059) — the counterpart of the standalone `/models` reseat, built + // like clearChat (BUILD-FIRST: the old session stays live + rendered while the reseated one builds, so a build + // failure leaves the user in a working, resumable conversation rather than a dead screen). On success the old + // session is torn down (bounded) and the reseated session — the SAME sessionId, resumed under the switched model + // with the carried transcript + cost/turns — is swapped in, its transcript opening with the modelSwitchNotice. + const reseatChat = (old: HomeChatSession, target: ReseatTarget): void => { + const reseat = deps.reseatChat; + if (reseat === undefined) return; // acceptModel only calls this when a builder IS wired (defensive) + if (tearingDown === old) return; // already ending this session (mirror clearChat) + const oldId = old.sessionId; + set({ modelPicker: undefined, submitBusy: true }); // close the picker + re-gate input for the whole swap + const build = reseat(oldId, target); + buildInFlight = build; + void build.then( + (next) => { + if (buildInFlight === build) buildInFlight = undefined; + if (exiting) { + void next.teardown().catch(() => undefined); // exited mid-build ⇒ reap the just-built reseated session + return; + } + // A superseding swap already replaced `old` (a mid-build /clear or another reseat) ⇒ reap this superseded + // build rather than swapping it in and leaking its MCP child (mirrors clearChat's guard). + if (state.session !== old) { + void next.teardown().catch(() => undefined); + return; + } + // Tear the OLD session down (bounded, like clearChat) — its terminal marks the row 'ended'; the reseated + // session continues the SAME sessionId (its persister adopted the row at build time). + tearingDown = old; + const td = old.teardown(); + activeTeardown = td; + void boundTeardown(td) + .finally(() => { + if (activeTeardown === td) activeTeardown = undefined; + tearingDown = undefined; + }) + .catch(() => undefined); + cancelFired = false; // the reseated session starts with a clean cancel latch (parity with clearChat) + pasting = false; // a lost paste-end marker must not leak the latch into the reseated chat + next.store.notice(modelSwitchNotice(target.modelId, next.store.getSnapshot().state.turnCount)); + set({ + session: next, + mode: 'chat', // STAY in chat — the model switched underneath, the conversation continues + input: emptyEditor(), + errorText: undefined, + notice: undefined, + pendingMessage: '', + palette: undefined, + search: undefined, + mention: undefined, + modelPicker: undefined, + shellBusy: false, + submitBusy: false, // the swap is done — un-gate the reseated chat + shellCommand: undefined, + attachments: [], // pending `@`/`!` attachments must not leak into the reseated conversation + }); + }, + () => { + if (buildInFlight === build) buildInFlight = undefined; + if (exiting) return; + // The reseat build FAILED — keep the OLD session live + resumable (do NOT tear it down); surface a static, + // secret-free note (the displayName is a catalog/registry string — sanitize defensively) and un-gate it. + if (state.session === old) { + old.store.note(`/models could not switch the model — keeping this conversation.`); + set({ submitBusy: false }); + } + }, + ); + }; + // Drive one chat turn; on success end the chat if `/exit`/`/cancel` ran, on an escaping error tear the session // down BEFORE propagating so its MCP child / frame loop / row are never orphaned. const sendChatLine = (active: HomeChatSession, line: string, display?: string): void => { @@ -542,6 +616,9 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { return; } pickerEpoch += 1; // a fresh generation — invalidates any in-flight refresh from a prior (closed) open + // The `✓` marker: in a LIVE chat (ADR-0059 reseat) it is the session's BOUND model (the "you are here"); at the + // bare Home it is the effective next-session default. The accept action mirrors this (reseat vs default-write). + const activeModel = state.session?.store.getSnapshot().state.model; set({ notice: undefined, // opening the picker clears any stale /doctor report behind it modelPicker: { @@ -549,7 +626,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { filter: '', selected: 0, loading: false, - currentDefault: port.currentDefault(), + currentDefault: activeModel ?? port.currentDefault(), refreshedAt: view.refreshedAt, banner: undefined, hint: undefined, @@ -559,13 +636,20 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { // the long-lived process the S5 background constraint requires. An empty/stale cache repopulates as it resolves. runPickerRefresh(() => port.refreshIfStale()); }; - // Persist the chosen model as the NEXT session's default (ADR-0063) — it does NOT rebind a live session (there is - // none in the bare Home). Because `writeGlobalDefaultModel` writes only the GLOBAL `[preferences].default_model` - // while the EFFECTIVE default resolves project → workspace → global (ADR-0063 §1), the notice is HONEST: it - // confirms success only when the freshly-resolved effective default actually became the chosen model, else it - // says a project/workspace setting still overrides it here (so the tool never falsely claims a no-op took effect). - // A write fault keeps the picker open with a secret-free `hint` rather than crashing the Home. - const acceptModel = (modelId: string, displayName: string): void => { + // Accept the chosen model. TWO surface-specific actions off the ONE picker (ADR-0059/ADR-0063): + // - a LIVE in-Home chat ⇒ RESEAT it onto the picked model (mirrors the standalone `relavium chat` /models reseat); + // - the BARE Home ⇒ persist the chosen model as the NEXT session's default (the pre-existing behavior below). + const acceptModel = (modelId: string, displayName: string, provider: ReseatTarget['provider']): void => { + const active = state.session; + if (active !== undefined && deps.reseatChat !== undefined) { + reseatChat(active, { modelId, provider }); + return; + } + // ---- The bare-Home next-session-default write (ADR-0063) --------------------------------------------------- + // `writeGlobalDefaultModel` writes only the GLOBAL `[preferences].default_model` while the EFFECTIVE default + // resolves project → workspace → global (ADR-0063 §1), so the notice is HONEST: success only when the + // freshly-resolved effective default actually became the chosen model, else a project/workspace override says so. + // A write fault keeps the picker open with a secret-free `hint` rather than crashing the Home. const port = deps.models; if (port === undefined) return; try { @@ -602,7 +686,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { set({ modelPicker: undefined }); break; case 'accept': - acceptModel(step.modelId, step.displayName); + acceptModel(step.modelId, step.displayName, step.provider); break; case 'blocked': { // An ACTIONABLE hint (2.5.G key-awareness): a keyless provider names the remedy; the pre-existing @@ -700,6 +784,10 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { // home: run over the Home context. The palette captures NO args, so the bare command runs (`/doctor` // ⇒ fast tier); `--deep` is a typed-in-chat affordance (repl-commands.ts). void Promise.resolve(step.command.run(homeReplCtx, [])).catch(() => undefined); + } else if (step.command.name === 'models' && deps.reseatChat !== undefined) { + // chat: `/models` opens the reseat picker (ADR-0059) — parity with the typed-`/models` intercept, so the + // palette route + the typed route behave identically (never the "interactive terminal" dispatch hint). + openModelPicker(); } else { sendChatLine(active, `/${step.command.name}`); // chat: reuse the S3a slash dispatch (createChatLineHandler) } @@ -948,6 +1036,16 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { runShell(active, parsed); // a `!command` → the shell escape (does NOT consume pending attachments) return; } + // `/models` in a LIVE chat opens the reseat picker (ADR-0059) instead of dispatching — parity with the standalone + // ChatApp's submit-intercept. Only when a reseat builder is wired (always, in production); else it falls through + // to the normal slash dispatch (which surfaces the "interactive terminal" hint). `/models ` is NOT + // intercepted (exact match) — it dispatches and is rejected as an unknown argument, like the standalone chat. + if (trimmed === '/models' && deps.reseatChat !== undefined) { + history = recordHistory(history, line); + set({ input: emptyEditor(), historyEntries: history.entries }); + openModelPicker(); + return; + } if (trimmed.startsWith('/') || state.attachments.length === 0) { // a slash command, or a plain message with no attachments — the simple path history = recordHistory(history, line); diff --git a/docs/decisions/0059-cli-mid-session-model-reseat.md b/docs/decisions/0059-cli-mid-session-model-reseat.md index e8a8930c..2dc88b2a 100644 --- a/docs/decisions/0059-cli-mid-session-model-reseat.md +++ b/docs/decisions/0059-cli-mid-session-model-reseat.md @@ -9,15 +9,21 @@ > from 2.6.C. The reseat reuses the `/clear` host-swap machinery ([ADR-0062](0062-context-compaction-and-cli-history-commands.md) §7) > and the `chat-resume` transcript path — zero engine change — exactly as designed below. -> **Note (2026-07-06): the per-message `modelId` persistence is DEFERRED to 2.6.C (with the cost breakdown that -> consumes it).** The Decision below says "only the CLI persister wiring is missing"; on implementation that proved -> under-specified. `session_messages.model_id` / `agent_sessions.model_id` are **foreign keys to `model_catalog.id` -> (a UUID row PK), not the raw model string** — and nothing populates them today (run history leaves them NULL too). -> Correct population needs a model-string → `model_catalog.id` resolution (and the model may not yet be cataloged → -> the NULL "unknown" bucket the Decision anticipates). That resolution belongs with the 2.6.C per-model cost -> breakdown that reads it, so it lands there. **The reseat itself is fully implemented and correct without it** — a -> mid-session `/models` switch carries the transcript + cumulative cost/turns and rebinds the model as designed; only -> the per-turn attribution column is deferred. This note refines, and does not reverse, the Decision. +> **Note (2026-07-06): the FK reality behind the per-message `modelId`.** The Decision below says "only the CLI +> persister wiring is missing"; on implementation that proved under-specified. `session_messages.model_id` / +> `agent_sessions.model_id` are **foreign keys to `model_catalog.id` (a UUID row PK), not the raw model string** — and +> nothing populated them before this work (run history still leaves them NULL). Correct population needs a +> model-string → `model_catalog.id` resolution, degrading to a NULL "unknown" bucket when the model is not yet +> cataloged (the Decision anticipates this). +> +> **Note (2026-07-06, superseding the deferral): the attribution SHIPPED in 2.5.G Step D, not 2.6.C.** An earlier +> draft of the note above deferred the per-message/session `modelId` to 2.6.C; on maintainer direction it was +> implemented now instead. The persister resolves the failover-aware `cost:updated.model` (per assistant turn) and +> the bound model (per session) to their `model_catalog.id` via `catalogIdByModelId` (`@relavium/db`), writing the +> UUID or NULL — never a raw string (so the FK is never violated). The 2.6.C cost breakdown will now merely *read* +> the populated column. **Known limitation (low):** the resolution is by model STRING (ignoring provider), so a +> model id shared across two providers could mis-attribute to the other provider's catalog row; deferred as a +> latent edge (real model ids are globally unique; the FK stays valid either way). ## Context From 307f76b52752529942f928e01bae0d115eacfd6d Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 16:14:46 +0300 Subject: [PATCH 48/68] =?UTF-8?q?test,fix(cli):=202.5.G=20Step=20D=20?= =?UTF-8?q?=E2=80=94=20Sonnet-review=20fixes=20(reseat=20coverage=20+=20no?= =?UTF-8?q?-op=20guard)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses the four Sonnet-review findings on the in-Home reseat (all non-blocking; GO): - HIGH: add a drive-home.test.ts integration test exercising the REAL reseatChat builder (loadFull → swapAgentModel → buildResumedChatSession → seeded store) end-to-end — starts a chat, runs a turn, reseats via the picker to a different model, and asserts the swap continues the SAME sessionId bound to the new model, carries the prior turn, and updates the durable agent snapshot. Pins the build-first swap ordering a refactor could break. - MEDIUM: guard both surfaces (Home acceptModel + standalone chat-ink accept) against reseating onto the ALREADY-bound model — a no-op that would wipe the ADR-0057 approval cache, reconnect MCP, and show a misleading "Switched" notice. Now a hint, not a churn. Covered by a new home-controller test (accept the bound model ⇒ no reseat + hint). - LOW: correct the stale "sanitize displayName" comment on the reseat build-failure note (the message is fully static — no interpolation). - LOW: add a direct swapAgentModel unit test (model/provider swapped, fallback_chain dropped, input untouched). Toolchain green: 1493 tests, all lint/typecheck/build tasks pass. Refs: ADR-0059 Co-Authored-By: Claude --- apps/cli/src/chat/session-host.test.ts | 26 ++++++++++ apps/cli/src/home/drive-home.test.ts | 50 +++++++++++++++++++ apps/cli/src/render/tui/chat-ink.tsx | 8 +++ .../src/render/tui/home-controller.test.ts | 33 ++++++++++++ apps/cli/src/render/tui/home-controller.ts | 18 +++++-- 5 files changed, 132 insertions(+), 3 deletions(-) diff --git a/apps/cli/src/chat/session-host.test.ts b/apps/cli/src/chat/session-host.test.ts index 6e88f252..ed97fadd 100644 --- a/apps/cli/src/chat/session-host.test.ts +++ b/apps/cli/src/chat/session-host.test.ts @@ -27,6 +27,7 @@ import { buildChatSession, buildGovernorWiring, buildResumedChatSession, + swapAgentModel, type ChatBudgetWarning, } from './session-host.js'; import { @@ -715,6 +716,31 @@ describe('buildResumedChatSession (2.N)', () => { }); }); +describe('swapAgentModel (ADR-0059 model-switch rule)', () => { + it('swaps model + provider and DROPS the original fallback_chain, without mutating the input', () => { + const original = { + ...buildDefaultChatAgent('claude-sonnet-4-6'), + fallback_chain: [{ model: 'gpt-5.5', provider: 'openai' as const, max_attempts: 2 }], + }; + const frozenChain = original.fallback_chain; + const next = swapAgentModel(original, 'claude-opus-4-8', 'anthropic'); + + expect(next.model).toBe('claude-opus-4-8'); + expect(next.provider).toBe('anthropic'); + expect('fallback_chain' in next).toBe(false); // dropped — the new instance builds its own default plan + // The input is untouched (a fresh copy): the original keeps its model + fallback_chain. + expect(original.model).toBe('claude-sonnet-4-6'); + expect(original.fallback_chain).toBe(frozenChain); + }); + + it('is a no-op on fallback_chain for an agent that has none (no key materialized)', () => { + const next = swapAgentModel(buildDefaultChatAgent('claude-sonnet-4-6'), 'gpt-5.5', 'openai'); + expect(next.model).toBe('gpt-5.5'); + expect(next.provider).toBe('openai'); + expect('fallback_chain' in next).toBe(false); + }); +}); + describe('buildGovernorWiring', () => { // Seed the governor's cumulative directly via updateCost so the pre-egress projection trips the cap // regardless of model pricing — exercising the real fail/pause/warn behavior, not just the wiring shape. diff --git a/apps/cli/src/home/drive-home.test.ts b/apps/cli/src/home/drive-home.test.ts index b5d09ed0..659682de 100644 --- a/apps/cli/src/home/drive-home.test.ts +++ b/apps/cli/src/home/drive-home.test.ts @@ -273,6 +273,56 @@ describe('driveHome (2.5.B / ADR-0054)', () => { expect(await drivePromise).toBe(EXIT_CODES.success); }); + it('in-Home /models reseat: the REAL reseatChat resumes the session under the switched model, carrying the transcript (ADR-0059)', async () => { + // Exercises the REAL drive-home reseatChat builder (loadFull → swapAgentModel → buildResumedChatSession → seeded + // store) end-to-end — not the mocked controller-level test — pinning the build-first swap over the same sessionId. + let captured: RootAppProps | undefined; + const { deps } = makeDeps((p) => (captured = p), { + providers: scriptedResolver([textTurn('sonnet reply'), textTurn('opus reply')]), + }); + const drivePromise = driveHome(deps); + const props = captured; + if (props === undefined) throw new Error('the injected render was never invoked'); + + // Start a chat (bound to the default claude-sonnet-4-6) + run one turn so the transcript persists. + type(props, 'first'); + props.controller.handleKey('', ENTER); + await flush(); + expect(props.controller.getSnapshot().mode).toBe('chat'); + const oldSession = props.controller.getSnapshot().session; + const sessionId = oldSession?.sessionId ?? ''; + expect(sessionId).not.toBe(''); + + // Open the reseat picker from the chat palette, filter to a DIFFERENT available model (opus), accept. + props.controller.handleKey('/', {}); + type(props, 'models'); + props.controller.handleKey('', ENTER); + await flush(); + expect(props.controller.getSnapshot().modelPicker).toBeDefined(); + type(props, 'opus'); // filter to claude-opus-4-8 (registry-priced anthropic ⇒ available on the scripted key) + props.controller.handleKey('', ENTER); // accept ⇒ the REAL reseatChat + await flush(); + + // The session was reseated: a NEW session object, the SAME sessionId, bound to opus, carrying the prior turn. + const reseated = props.controller.getSnapshot().session; + expect(reseated).not.toBe(oldSession); // swapped in a new instance + expect(reseated?.sessionId).toBe(sessionId); // a reseat CONTINUES the same session (unlike /clear's new id) + expect(reseated?.store.getSnapshot().state.model).toBe('claude-opus-4-8'); // rebound to the picked model + expect(reseated?.store.getSnapshot().state.turnCount).toBe(1); // the prior turn carried + expect(props.controller.getSnapshot().modelPicker).toBeUndefined(); // the picker closed + expect(props.controller.getSnapshot().mode).toBe('chat'); // stayed in chat + + // The durable row's agent snapshot updates to the switched model after the reseated session ends. + props.controller.handleKey('c', CTRL_C); // /cancel ⇒ back to Home (the reseated session's terminal writes 'ended') + await flush(); + const full = createSessionStore(client.db).loadFull(sessionId); + expect(full?.session.agentSnapshot?.model).toBe('claude-opus-4-8'); + expect(full?.messages.map((m) => m.role)).toEqual(['user', 'assistant']); // the single carried exchange + + props.controller.handleKey('c', CTRL_C); // Home Ctrl-C ⇒ clean exit + expect(await drivePromise).toBe(EXIT_CODES.success); + }); + it('a KEY-LESS first run runs the onboarding wizard BEFORE mounting the Home (2.5.G S8)', async () => { // A scripted clack slice + a key-less resolver (keyFor throws for every provider) ⇒ the wizard triggers. const outros: string[] = []; diff --git a/apps/cli/src/render/tui/chat-ink.tsx b/apps/cli/src/render/tui/chat-ink.tsx index d293f103..7642155f 100644 --- a/apps/cli/src/render/tui/chat-ink.tsx +++ b/apps/cli/src/render/tui/chat-ink.tsx @@ -593,6 +593,14 @@ export function ChatApp(props: Readonly): ReactElement { applyModelPicker(undefined); return; case 'accept': + // No-op guard (ADR-0059): accepting the ALREADY-bound model (the ✓ = `currentDefault` = the session's bound + // model) would reseat for zero change — wiping the ADR-0057 approval cache + reconnecting MCP. Close with a + // note instead of a pointless switch. + if (step.modelId === open.currentDefault) { + applyModelPicker(undefined); + props.store.note(`Already on ${step.displayName}.`); + return; + } applyModelPicker(undefined); props.onReseat?.({ modelId: step.modelId, provider: step.provider }); props.onExit(); // the reseat set the stop state; end the loop so runReplLoop swaps in the new-model session diff --git a/apps/cli/src/render/tui/home-controller.test.ts b/apps/cli/src/render/tui/home-controller.test.ts index 93981ce2..bd9736f5 100644 --- a/apps/cli/src/render/tui/home-controller.test.ts +++ b/apps/cli/src/render/tui/home-controller.test.ts @@ -1921,4 +1921,37 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => expect(c.getSnapshot().mode).toBe('chat'); // stayed in chat (the model switched underneath) expect(c.getSnapshot().modelPicker).toBeUndefined(); // the picker closed }); + + it('in-Home chat: accepting the ALREADY-bound model does NOT reseat — a no-op hint (ADR-0059)', async () => { + // The session is bound to claude-opus-4-8; the only picker entry IS that model. Accepting it must NOT tear the + // session down + rebuild for zero change (which would wipe the approval cache) — it keeps the picker open + hints. + const boundStore = createChatStore(false, { model: 'claude-opus-4-8' }); + const sessionA = makeSession({ sessionId: 'sess-A', store: boundStore }); + const reseatChat = vi.fn(() => Promise.resolve(makeSession().session)); + const { port } = makeModelsPort({ + entries: [pickerEntry({ modelId: 'claude-opus-4-8', provider: 'anthropic' })], + }); + const c = createHomeController({ + doctorProbes: STUB_DOCTOR_PROBES, + startChat: () => Promise.resolve(sessionA.session), + reseatChat, + models: port, + homeStore, + onExit: vi.fn(), + onError: vi.fn(), + }); + type(c, 'hi'); + c.handleKey('', ENTER); + await flush(); + c.handleKey('/', {}); + type(c, 'models'); + c.handleKey('', ENTER); + await flush(); + c.handleKey('', ENTER); // accept the (only, ALREADY-bound) model + await flush(); + + expect(reseatChat).not.toHaveBeenCalled(); // no pointless reseat onto the current model + expect(sessionA.teardown).not.toHaveBeenCalled(); // the live session is untouched + expect(c.getSnapshot().modelPicker?.hint).toContain('Already on'); // the picker stays open with a hint + }); }); diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index 6c9fba66..0842a697 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -466,10 +466,10 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { () => { if (buildInFlight === build) buildInFlight = undefined; if (exiting) return; - // The reseat build FAILED — keep the OLD session live + resumable (do NOT tear it down); surface a static, - // secret-free note (the displayName is a catalog/registry string — sanitize defensively) and un-gate it. + // The reseat build FAILED — keep the OLD session live + resumable (do NOT tear it down); surface a fully + // STATIC, secret-free note (no model id interpolated) and un-gate it so the user can keep going or /exit. if (state.session === old) { - old.store.note(`/models could not switch the model — keeping this conversation.`); + old.store.note('/models could not switch the model — keeping this conversation.'); set({ submitBusy: false }); } }, @@ -642,6 +642,18 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { const acceptModel = (modelId: string, displayName: string, provider: ReseatTarget['provider']): void => { const active = state.session; if (active !== undefined && deps.reseatChat !== undefined) { + // No-op guard (ADR-0059): accepting the ALREADY-bound model would tear the session down + rebuild for zero + // change — wiping the ADR-0057 per-tool approval cache, reconnecting MCP, and showing a misleading "Switched" + // notice. Keep the picker open with a hint instead so a mis-click doesn't churn the live session. + if (modelId === active.store.getSnapshot().state.model) { + const open = state.modelPicker; + set( + open === undefined + ? { modelPicker: undefined } + : { modelPicker: { ...open, hint: `Already on ${displayName} — pick a different model or Esc.` } }, + ); + return; + } reseatChat(active, { modelId, provider }); return; } From 714584bfdb00509ad9dfc5ccf9d0d8724f2e0eb7 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 17:12:02 +0300 Subject: [PATCH 49/68] =?UTF-8?q?docs(adr):=20ADR-0066=20=E2=80=94=20norma?= =?UTF-8?q?lized=20reasoning-effort=20control=20(Accepted,=20tier-native)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records the 2.5.G reasoning-effort decision (maintainer question #1): a provider-agnostic effort TIER (off/low/medium/high/max) on the LlmRequest, mapped per-adapter to each provider's native tier, gated on a host-injected per-model capability, threaded per turn. Rewritten from the Proposed draft after two independent ADR reviews caught a stale premise — all four currently-targeted providers are TIER-native (OpenAI reasoning_effort, Anthropic output_config effort, Gemini thinking-level, DeepSeek-v4 thinking on/off request param), and the shipped adapters merge {...providerOptions, ...body} (canonical wins). The design is now strictly simpler: one tier → each provider's native tier; the token-budget derivation is demoted to a documented legacy fallback. Fixes applied from the reviews: - tier-native mapping (not "OpenAI tier vs Anthropic/Gemini budget"); budget = legacy only. - DeepSeek section corrected to the v4 ids + the thinking request param (per pricing.ts). - precedence corrected to CANONICAL-wins (matching the adapters), not providerOptions-wins. - per-model capability read via a host-injected resolveReasoning (core can't import @db — mirrors resolveMediaSurface), not a direct db read. - effort is a per-TURN field (like temperature) — a mid-session change is a lightweight session update, NOT a reseat (a reseat is only for a model/provider/plan change). - "all false" corrected (DeepSeek adapter is reasoning:true); ADR-0039 added to Related; the README index row + a roadmap note added (both flagged as missing). Refs: ADR-0011, ADR-0030, ADR-0039, ADR-0064, ADR-0059 Co-Authored-By: Claude --- ...066-normalized-reasoning-effort-control.md | 90 +++++++++++++++++++ docs/decisions/README.md | 1 + docs/roadmap/current.md | 7 +- 3 files changed, 97 insertions(+), 1 deletion(-) create mode 100644 docs/decisions/0066-normalized-reasoning-effort-control.md diff --git a/docs/decisions/0066-normalized-reasoning-effort-control.md b/docs/decisions/0066-normalized-reasoning-effort-control.md new file mode 100644 index 00000000..193a08d9 --- /dev/null +++ b/docs/decisions/0066-normalized-reasoning-effort-control.md @@ -0,0 +1,90 @@ +# ADR-0066: Normalized reasoning-effort control — a provider-agnostic tier, per-adapter native mapping, and a per-model capability + +- **Status**: Accepted +- **Date**: 2026-07-06 +- **Related**: [ADR-0011](0011-internal-llm-abstraction.md) (the `LLMProvider` seam this amends — a new `LlmRequest` field), [ADR-0030](0030-llm-seam-shape-amendment-reasoning-response-format-provider-executed.md) (added the reasoning **output** channel; this ADR adds the reasoning **input** control), [ADR-0039](0039-same-provider-reasoning-replay.md) (same-provider signed-reasoning replay — the output whose *production* this input governs), [ADR-0064](0064-live-model-catalog.md) (the `model_catalog.capabilities` JSON this populates per model), [ADR-0059](0059-cli-mid-session-model-reseat.md) (the `/models` picker that surfaces the effort selector), [llm-provider-seam.md](../reference/shared-core/llm-provider-seam.md) (the seam's one canonical home), [agent-yaml-spec.md](../reference/contracts/agent-yaml-spec.md) + [config-spec.md](../reference/contracts/config-spec.md) (the authoring surfaces) + +> Accepted 2026-07-06 (the 2.5.G "reasoning-effort" workstream, maintainer question #1). Two independent ADR reviews caught that the original draft's "OpenAI = tier, Anthropic/Gemini = token budget" split was **stale**: all four currently-targeted providers are **tier-native**, and the shipped adapters already merge `{...providerOptions, ...body}` (canonical wins). The Decision below is rewritten around that — a strictly *simpler* design (one tier → each provider's native tier), with the token-budget derivation demoted to a documented legacy fallback. + +## Context + +Reasoning models expose a **control** over how much the model "thinks" before answering. Relavium's seam already has the reasoning **output** side (ADR-0030: the streaming `reasoning_*` channel + `reasoningTokens` observability + the `reasoning` content arm, mapped per adapter). It has **no normalized INPUT control**: a caller who wants "think harder" must reach the raw `providerOptions` escape hatch with provider-specific keys — which defeats the provider-agnostic promise, cannot be authored portably in agent YAML, and cannot be surfaced in the `/models` picker. + +Two facts, verified against the current provider surfaces and this repo, shape the decision: + +1. **All four currently-targeted providers control reasoning by a discrete TIER, not a token budget.** OpenAI (`reasoning_effort`), Anthropic (an `output_config` effort field — the adapter already writes `output_config` for structured output, [anthropic.ts:477](../../packages/llm/src/adapters/anthropic.ts)), Gemini (a thinking-level field), and DeepSeek-v4 (a thinking on/off request param on a single id — `deepseek-v4-flash`/`-pro`, where *"the mode is a request param, not a separate model,"* [pricing.ts](../../packages/llm/src/pricing.ts)). The older "token budget" shapes (pre-`output_config` Anthropic, Gemini 2.5) are legacy. + +2. **`CapabilityFlags.reasoning` is populated only at the ADAPTER level today** — `true` for DeepSeek ([openai.ts:98](../../packages/llm/src/adapters/openai.ts)), `false` for the other three ([openai.ts:80](../../packages/llm/src/adapters/openai.ts), [anthropic.ts:63](../../packages/llm/src/adapters/anthropic.ts), [gemini.ts:67](../../packages/llm/src/adapters/gemini.ts)) — when reasoning is genuinely a **per-model** property (o-series reasons, `gpt-4o` does not, though both share the OpenAI adapter; `claude-opus-4-8` reasons). The per-model `model_catalog.capabilities` column exists (ADR-0064) but is not yet populated per model. + +The user-facing ask (question #1): when picking a model, choose a reasoning-effort tier, like the competitors — which requires the tier to be a first-class, provider-agnostic concept end to end. + +## Decision + +**We normalize reasoning effort as a provider-agnostic TIER on the `LlmRequest`, map it inside each adapter to that provider's native tier control, gate it on a host-injected per-MODEL capability, and thread it per turn (like `temperature`) — not as a session reseat. The token-budget derivation survives only as a legacy fallback. The `providerOptions` escape is untouched.** + +### 1. A normalized `reasoningEffort` tier on the seam (`LlmRequest`) + +Add `reasoningEffort?: ReasoningEffort` to `LlmRequestSchema` (`@relavium/llm`), where `ReasoningEffort = 'off' | 'low' | 'medium' | 'high' | 'max'` is a new closed vocabulary owned by `@relavium/shared` (a `REASONING_EFFORTS` const tuple, like `STOP_REASONS` / `FS_SCOPE_TIERS`). Five levels (maintainer choice): `off` disables reasoning where the provider allows it; `low`/`medium`/`high` map to the provider's matching tier; `max` maps to the provider's **highest** available tier (so a provider that tops out lower than another still gets "as hard as it goes"). + +- **Absent** ⇒ the provider default (byte-identical to today — no behavior change for an unset field). +- **Precedence:** the normalized field's per-adapter mapping is CANONICAL and **wins** over any colliding `providerOptions` key — consistent with every shipped adapter's `return { ...req.providerOptions, ...body }` merge (canonical last, [anthropic.ts:512](../../packages/llm/src/adapters/anthropic.ts) / [openai.ts:717](../../packages/llm/src/adapters/openai.ts)), and with Gemini's deliberate "canonical wins so a caller can't override transport keys". `providerOptions` remains for *non-colliding* provider-specific reasoning knobs (e.g. a raw legacy budget on an old model, §3) — it adds, it does not override the tier. + +This **amends ADR-0011** (a new optional `LlmRequest` field, no vendor type) and **complements ADR-0030** (which shaped the reasoning *output*; this is the *input* that governs its production — see ADR-0039 for the replay of that output). + +### 2. Per-adapter mapping — the tier→native translation lives INSIDE each adapter + +Each adapter maps the one normalized tier to its provider's native tier; nothing tier-shaped leaks out, and no vendor type crosses the seam. The exact native field NAMES are pinned in the adapter against its SDK types (the ground truth), not frozen in this ADR: + +| Adapter | Native shape | `off` | `low`/`medium`/`high`/`max` | +|---------|--------------|-------|-----------------------------| +| **OpenAI-compatible** (OpenAI) | `reasoning_effort` tier | `'none'` (or the lowest the model accepts; an always-on o-series model has no true off → lowest) | pass the matching tier; `max` → the highest tier the model accepts | +| **Anthropic** | the `output_config` effort tier (already-used `output_config`) | omit / disabled where the model allows; else lowest | the matching tier; `max` → highest | +| **Gemini** | the thinking-level tier | `0`/lowest **only where the model allows** (Flash-class); a Pro-class model that cannot disable → lowest tier | the matching tier; `max` → highest | +| **DeepSeek** (OpenAI-compatible, v4 id) | thinking on/off request param | thinking **off** | thinking **on** (v4 has on/off, not graded tiers, so `low`/`medium`/`high`/`max` all enable thinking — an honest, documented coarsening) | + +A model that does not support reasoning (§4) never receives the field — the engine gates it off before the adapter, and each adapter additionally ignores it defensively. + +### 3. The token-budget derivation — a documented LEGACY fallback only + +For a **legacy** model that still takes a token budget rather than a tier (pre-`output_config` Anthropic, Gemini 2.5), a pure `deriveReasoningBudget(tier, maxTokens)` helper (`@relavium/llm`) maps the tier to a token count — fixed per-tier constants clamped **below `maxTokens`** with a headroom reserve (a budget must leave room for the answer). This path is **not** the primary mechanism (the four current models are tier-native, §2); it exists so an old model id still routed through an adapter degrades gracefully instead of erroring. Its constants live in one home and are **not** a billing input (reasoning tokens are counted inside output, ADR-0030). Note the intrinsic sharp edge the normalized tier cannot paper over: OpenAI o-series `max_tokens` is a *total* cap (reasoning + answer), so a high tier under a very small `max_tokens` can starve the answer — inherent model behavior, surfaced honestly, not a seam defect. + +### 4. Reasoning is a per-MODEL capability, read via a host-injected resolver + +The authoritative "does THIS model reason" is the per-model **`model_catalog.capabilities.reasoning`** boolean (ADR-0064) — which projects onto `CapabilityFlags.reasoning`; it is simply **not populated per model today**. This ADR populates it: a static capability seed + a conservative **model-id heuristic** for the known reasoning families (OpenAI o-series / reasoning `gpt-5`, `claude-opus-4-8` + extended-thinking Sonnet, Gemini thinking models, `deepseek-v4-*`), so a live-discovered model whose list-endpoint omits the flag still gates correctly (the same maintenance shape as the static `MODEL_PRICING` registry). + +Because `packages/core` has **zero platform imports** (CLAUDE.md #5) it cannot read `@relavium/db`. The engine therefore reads the capability through a **host-injected resolver** — `resolveReasoning?: (model: string) => boolean | undefined` on `AgentRunnerDeps`/`SessionDeps`, mirroring the existing `resolveMediaSurface` catalog projection ([agent-runner.ts:92](../../packages/core/src/engine/agent-runner.ts)). Absent resolver ⇒ treat as not-reasoning (safe: the field is not sent). The `/models` picker uses the same catalog projection to OFFER the effort selector only for reasoning-capable models. The adapter's static `supports.reasoning` stays the adapter-level backstop. + +### 5. Authoring + plumbing — a per-turn field, resolved agent → config → session + +`reasoningEffort` is structurally a **per-turn request field, like `temperature`/`maxTokens`** — NOT a plan-changing binding. So (a reviewer's correction to the draft) a mid-session effort change is a **lightweight per-turn update**, never the heavier ADR-0059 reseat (which exists because a *model* change alters the provider, pricing, and the memoized `#plan` — an *effort* change alters none of those). Resolution order per turn: **session override → agent `reasoning_effort` → `[chat].reasoning_effort` config default → unset (provider default)**. + +- **Agent YAML** (`@relavium/shared` `AgentSchema`, `.strict()` — an additive, non-breaking field): `reasoning_effort?: ReasoningEffort` (snake_case authored), so a committed agent pins its effort. +- **Config** (`config-spec.md`): `[chat].reasoning_effort` — the chat-surface default. +- **Engine** (`packages/core`): the `AgentRunner` reads the resolved effort into each turn's `LlmRequest.reasoningEffort`, gated by `resolveReasoning` (§4); `AgentSession` gains a session-level effort override + setter (mirroring the mode control) so an interactive surface can change it mid-session with no reseat. The one seam field is the only new cross-package contract; the engine read + the session setter are platform-agnostic. + +### 6. The surface — the `/models` picker effort selector (the user's ask) + +The `/models` picker (ADR-0059/2.5.G Step D) surfaces the effort: selecting a **reasoning-capable** model offers a low/medium/high/max sub-choice (a non-reasoning model skips it), setting the session's per-turn effort override (§5) — **not** a reseat, so it takes effect on the next turn with no teardown. Standalone effort changes ride the same session override (a future `/effort`), and an authored agent carries its own default. The bare-Home picker (ADR-0064 §10, next-session default) is orthogonal — it writes the config default, not a live session's effort. This ADR fixes the **data + seam** so every surface speaks one vocabulary. + +Considered and rejected: **(a) providerOptions-only** (the status quo — not provider-agnostic, unauthorable, un-surfaceable — the gap this closes); **(b) a normalized token-budget field** instead of a tier (a token count is not the user's vocabulary and is now wrong for *all four* tier-native providers — the tier is the lowest common denominator, budgets a legacy fallback); **(c) a discriminated `{tier}|{budget}` seam union** (pushes provider-shape into the contract; `providerOptions` already serves the exact-budget power user); **(d) effort as a per-session binding changed via reseat** (rejected: effort changes neither provider nor pricing nor the memoized plan, so a reseat is unwarranted overhead — it is a per-turn field). + +## Consequences + +### Positive + +- One provider-agnostic effort vocabulary end to end: authorable in git-committed agent YAML, surfaced in the `/models` picker, changed mid-session with no reseat — the same concept regardless of which provider backs the model. +- The tier-native mapping is *simpler* than the original budget design (one tier → each provider's native tier; budget only a legacy fallback), and matches how the current providers actually work. +- Per-model correctness: the effort selector + the sent field are gated on the actual model via the catalog projection, not a coarse per-provider flag. +- The `providerOptions` escape is preserved with the shipped canonical-wins precedence, so a power user (an exact legacy budget) is never worse off and the adapter merge stays consistent. + +### Negative + +- The per-model reasoning capability relies on a static seed + an id heuristic until providers expose the flag in list-models (§4) — a genuinely-new reasoning model id may need a heuristic/seed touch (the media-routing-seed maintenance shape). +- `off` on an always-on reasoning model (o-series, `deepseek`-thinking-on) cannot truly disable thinking; it degrades to the minimum, documented per adapter (§2) rather than silently ignored. +- `max` cannot express a level a given provider does not have; it maps to that provider's highest tier (a documented coarsening, §1/§2). +- DeepSeek-v4's on/off (no graded tiers) coarsens `low`/`medium`/`high`/`max` to "thinking on" (§2) — honest, not hidden. + +### Neutral + +- Amends ADR-0011 + complements ADR-0030/0039 **append-only** (a new optional field + a populated existing capability); no existing behavior changes for an unset `reasoningEffort`. No new runtime dependency. +- The exact native field names are deliberately left to each adapter's SDK-typed implementation (the ground truth), not frozen here — so a provider renaming a field is an adapter edit, not an ADR supersede. diff --git a/docs/decisions/README.md b/docs/decisions/README.md index bdd99f49..f7b8b60f 100644 --- a/docs/decisions/README.md +++ b/docs/decisions/README.md @@ -109,6 +109,7 @@ flowchart TD | 0063 | [CLI config-write contract — the first on-disk config writer, the global `[preferences].default_model` target, and the `resolveChat` global fallback](0063-cli-config-write-contract.md) | Accepted | 2026-07-05 | | 0064 | [Live model catalog — the `listModels?` seam capability, the `kind` protocol abstraction, the `model_catalog` live cache, the refresh lifecycle, and the static/live merge](0064-live-model-catalog.md) | Accepted | 2026-07-05 | | 0065 | [Provider economics and extensibility — user-supplied pricing, the cost-path pricing-injection seam, pricing-reference capture, and custom OpenAI-compatible endpoints](0065-provider-economics-and-extensibility.md) | Accepted | 2026-07-05 | +| 0066 | [Normalized reasoning-effort control — a provider-agnostic tier, per-adapter native mapping, and a per-model capability](0066-normalized-reasoning-effort-control.md) | Accepted | 2026-07-06 | ## Creating a new ADR diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 63d85caa..208228cd 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -193,7 +193,12 @@ cost) — behind three new ADRs ([ADR-0063](../decisions/0063-cli-config-write-c [ADR-0064](../decisions/0064-live-model-catalog.md) live catalog · [ADR-0065](../decisions/0065-provider-economics-and-extensibility.md) provider economics), across 12 reviewed steps; the additive lanes 2.5.H / I / J run in parallel. See the -[Phase 2.5 workstreams](phases/phase-2.5-cli-consolidation.md). +[Phase 2.5 workstreams](phases/phase-2.5-cli-consolidation.md). A **post-2.5.G model-UX follow-up** (from six +maintainer questions on model/provider/wizard behavior) then landed as a sequenced plan: `/models` key-awareness, +the onboarding-wizard live key-validation + retry UX, wizard-dynamic provider docs, **mid-session model switching** +(the `/models` reseat across `relavium chat` + the in-Home chat — [ADR-0059](../decisions/0059-cli-mid-session-model-reseat.md), +now Accepted), and **normalized reasoning-effort control** (a provider-agnostic effort tier authored in agent YAML + +chosen in the picker — [ADR-0066](../decisions/0066-normalized-reasoning-effort-control.md)). Carry-over hardening is tracked in [deferred-tasks.md](deferred-tasks.md) — Phase 2 picks items up as it first touches each file. Notable inheritances: 1.AH's host-wiring half From ea6390f1fb02b03c8c63f608a0cc2b1dd480911f Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 17:21:58 +0300 Subject: [PATCH 50/68] =?UTF-8?q?feat(shared,llm):=202.5.G=20Step=20E=20(1?= =?UTF-8?q?/n)=20=E2=80=94=20reasoning-effort=20seam=20+=20OpenAI=20mappin?= =?UTF-8?q?g=20(ADR-0066)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The foundation of the normalized reasoning-effort control (Accepted ADR-0066): - @relavium/shared: REASONING_EFFORTS = off/low/medium/high/max (a closed vocabulary, like STOP_REASONS/FS_SCOPE_TIERS) + the ReasoningEffort type. - @relavium/llm: LlmRequest.reasoningEffort (a normalized, provider-agnostic tier; absent ⇒ provider default — no behavior change). - OpenAI adapter: map the tier → the native reasoning_effort (SDK-verified against openai 6.42.0's `ReasoningEffort = none|minimal|low|medium|high|xhigh`): off→none, low/medium/high 1:1, max→xhigh (the highest); mapped field wins over providerOptions ({...opts, ...body}). Sent only for the `openai` provider — DeepSeek (the other id this shared adapter serves) controls thinking differently (a verified follow-up). - Test: max→xhigh, off→none, unset omitted, DeepSeek not mapped. SDK verification confirmed the ADR's tier-native premise across providers: OpenAI reasoning_effort, Anthropic output_config.effort (low/medium/high/xhigh/max) + thinking:{type:'adaptive'}, Gemini thinking config — all tiers, not budgets. Remaining Step E (next): the Anthropic/Gemini/DeepSeek adapter mappings, the host-injected per-model resolveReasoning capability, the core per-turn plumbing + AgentSession override, the agent-YAML + [chat] config field, and the picker effort selector — then the review cadence. Toolchain green: 539 llm tests, shared+llm typecheck. Refs: ADR-0066 Co-Authored-By: Claude --- packages/llm/src/adapters/openai.test.ts | 37 ++++++++++++++++++++++++ packages/llm/src/adapters/openai.ts | 19 ++++++++++++ packages/llm/src/types.ts | 6 ++++ packages/shared/src/constants.ts | 13 +++++++++ 4 files changed, 75 insertions(+) diff --git a/packages/llm/src/adapters/openai.test.ts b/packages/llm/src/adapters/openai.test.ts index ffa845b8..b89cd2a9 100644 --- a/packages/llm/src/adapters/openai.test.ts +++ b/packages/llm/src/adapters/openai.test.ts @@ -231,6 +231,43 @@ describe('OpenAI-compatible adapter', () => { }); }); + it('maps the reasoning-effort tier to OpenAI reasoning_effort (max→xhigh, off→none, unset omitted); DeepSeek is not mapped (ADR-0066)', async () => { + let sent: Record = {}; + const oai = createOpenAiAdapter({ + fetch: (_input, init) => { + sent = parseJsonBody(init); + return Promise.resolve(okResponse()); + }, + }); + const base = { + model: 'gpt-5.5', + messages: [{ role: 'user' as const, content: [{ type: 'text' as const, text: 'go' }] }], + }; + await oai.generate({ ...base, reasoningEffort: 'high' }, 'k'); + expect(sent['reasoning_effort']).toBe('high'); + await oai.generate({ ...base, reasoningEffort: 'max' }, 'k'); + expect(sent['reasoning_effort']).toBe('xhigh'); // `max` → the provider's HIGHEST tier + await oai.generate({ ...base, reasoningEffort: 'off' }, 'k'); + expect(sent['reasoning_effort']).toBe('none'); + await oai.generate({ ...base }, 'k'); // unset ⇒ omitted (provider default, unchanged behavior) + expect('reasoning_effort' in sent).toBe(false); + + // DeepSeek (the other id this shared adapter serves) controls thinking differently — reasoning_effort is NOT sent. + let dsSent: Record = {}; + const ds = createOpenAiAdapter({ + providerId: 'deepseek', + fetch: (_input, init) => { + dsSent = parseJsonBody(init); + return Promise.resolve(okResponse()); + }, + }); + await ds.generate( + { model: 'deepseek-v4-flash', messages: base.messages, reasoningEffort: 'high' }, + 'k', + ); + expect('reasoning_effort' in dsSent).toBe(false); + }); + it('round-trips inline audio-out: lowers output_modalities → modalities+audio and parses the response (1.AG/ADR-0046)', async () => { let sent: Record = {}; const adapter = createOpenAiAdapter({ diff --git a/packages/llm/src/adapters/openai.ts b/packages/llm/src/adapters/openai.ts index 0c53bb07..c7cb3edb 100644 --- a/packages/llm/src/adapters/openai.ts +++ b/packages/llm/src/adapters/openai.ts @@ -11,6 +11,7 @@ import { type AbortSignalLike, type ContentPart, mediaModalityOf, + type ReasoningEffort, type StopReason, } from '@relavium/shared'; @@ -648,6 +649,17 @@ function toOpenAiTool(toolDef: ToolDef, provider: ProviderId): OpenAI.ChatComple return { type: 'function', function: fn }; } +/** ADR-0066: the normalized reasoning-effort tier → OpenAI's native `reasoning_effort` values. `off`→'none', + * `max`→'xhigh' (its highest); low/medium/high are 1:1. A SUBSET of the SDK's `ReasoningEffort` union, so the + * assignment to `body.reasoning_effort` needs no cast. */ +const OPENAI_REASONING_EFFORT: Record = { + off: 'none', + low: 'low', + medium: 'medium', + high: 'high', + max: 'xhigh', +}; + function toOpenAiToolChoice(choice: ToolChoice): OpenAI.ChatCompletionToolChoiceOption { if (choice === 'auto') { return 'auto'; @@ -701,6 +713,13 @@ function buildCommonBody( if (req.maxTokens !== undefined) { body.max_tokens = req.maxTokens; } + // ADR-0066: map the normalized reasoning-effort tier to OpenAI's native `reasoning_effort` (also a tier). ONLY for + // the `openai` provider — DeepSeek (the other id this shared adapter serves) controls thinking differently (its + // own follow-up), so its effort is not sent here. The host gates this to reasoning-capable models (a non-reasoning + // model would reject it), and `body` is spread LAST below so this mapped field wins over any providerOptions echo. + if (provider === 'openai' && req.reasoningEffort !== undefined) { + body.reasoning_effort = OPENAI_REASONING_EFFORT[req.reasoningEffort]; + } if (req.stopSequences !== undefined) { body.stop = req.stopSequences; } diff --git a/packages/llm/src/types.ts b/packages/llm/src/types.ts index 8b0238bd..c070f906 100644 --- a/packages/llm/src/types.ts +++ b/packages/llm/src/types.ts @@ -8,6 +8,7 @@ import { LLM_PROVIDERS, MEDIA_BILLED_MODALITIES, MEDIA_SURFACES, + REASONING_EFFORTS, MEDIA_HANDLE_PATTERN, MEDIA_MESSAGE_CAPS, MediaMimeTypeSchema, @@ -298,6 +299,11 @@ export const LlmRequestSchema = z.object({ outputModalities: z.array(z.enum(OUTPUT_MODALITIES)).optional(), temperature: z.number().optional(), maxTokens: z.number().int().positive().optional(), // required downstream for Anthropic — adapters default it + // Normalized, provider-agnostic reasoning-effort TIER (ADR-0066). Each adapter maps it to the provider's native + // tier control (OpenAI reasoning_effort, Anthropic output_config effort, Gemini thinking-level, DeepSeek-v4 + // thinking on/off). Absent ⇒ provider default. On a COLLISION with a providerOptions key the adapter's mapping + // wins (canonical last in `{...providerOptions, ...body}`). A non-reasoning model never receives it (host-gated). + reasoningEffort: z.enum(REASONING_EFFORTS).optional(), stopSequences: z.array(z.string()).optional(), signal: abortSignalLikeSchema.optional(), providerOptions: z.record(z.string(), z.unknown()).optional(), // typed escape hatch diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 1b39f47f..7b09c1f6 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -267,6 +267,19 @@ export type EngineNodeType = (typeof ENGINE_NODE_TYPES)[number]; export const LLM_PROVIDERS = ['anthropic', 'openai', 'gemini', 'deepseek'] as const; export type LlmProviderId = (typeof LLM_PROVIDERS)[number]; +/** + * The normalized, provider-agnostic **reasoning-effort** tiers ([ADR-0066](../decisions/0066-normalized-reasoning-effort-control.md)). + * A closed vocabulary the caller picks; each `@relavium/llm` adapter maps it to its provider's NATIVE tier control + * (OpenAI `reasoning_effort`, Anthropic `output_config` effort, Gemini thinking-level, DeepSeek-v4 thinking on/off). + * `off` disables reasoning where the provider allows it (an always-on model degrades to the lowest); `low`/`medium`/ + * `high` map to the matching tier; `max` maps to the provider's HIGHEST available tier. Absent on a request ⇒ the + * provider default (unchanged behavior). It rides `LlmRequest.reasoningEffort` (the seam), the authored agent + * `reasoning_effort`, and the `[chat].reasoning_effort` config default — all one vocabulary. Owned here (like + * {@link STOP_REASONS}/{@link FS_SCOPE_TIERS}) so `@relavium/llm` + `@relavium/core` derive the enum from one home. + */ +export const REASONING_EFFORTS = ['off', 'low', 'medium', 'high', 'max'] as const; +export type ReasoningEffort = (typeof REASONING_EFFORTS)[number]; + /** * The three provider **protocol kinds** (the `kind` abstraction, [ADR-0064] §2) — a closed vocabulary * that derives, **once per protocol rather than per provider**, the adapter factory, the list-models From 13a0b0568bd9324f3a666d3ce75f35ddd0c0fbed Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 17:30:48 +0300 Subject: [PATCH 51/68] =?UTF-8?q?feat(llm):=202.5.G=20Step=20E=20(2/n)=20?= =?UTF-8?q?=E2=80=94=20Anthropic=20+=20Gemini=20reasoning-effort=20mapping?= =?UTF-8?q?s=20(ADR-0066)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Completes the adapter layer — the heart of ADR-0066 (translating the normalized tier to each provider's NATIVE tier control), all SDK-verified against the pinned SDK types: - Anthropic (@anthropic-ai/sdk 0.101.0): map the tier → `output_config.effort` (native low/medium/high/xhigh/max — so `max` is 1:1, no coarsening) + `thinking:{type:'adaptive'}` to enable it (the tier-native path, no legacy budget_tokens). `off` → `thinking:{type: 'disabled'}`. The effort MERGES alongside any structured-output `format` on output_config. - Gemini (@google/genai 2.8.0): map the tier → `thinkingConfig.thinkingLevel` (native MINIMAL/LOW/MEDIUM/HIGH). `max`→HIGH (Gemini tops at HIGH — a documented coarsening); `off`→MINIMAL (Gemini has no universal disable — a Pro model rejects budget 0 — so `off` degrades to the lowest tier). - Both mapped fields win over a colliding providerOptions key (canonical last in the merge). - Tests: each adapter maps high/max/low/off + omits when unset (per-provider request-build assertions). DeepSeek's effort is deferred (unmapped, noted in the OpenAI adapter): DeepSeek-v4 controls thinking via its own request param — needs the DeepSeek API-doc verification before wiring (the OpenAI adapter serves it, gated to provider==='openai' for reasoning_effort today). Toolchain green: 541 llm tests, llm typecheck + lint. Refs: ADR-0066 Co-Authored-By: Claude --- packages/llm/src/adapters/anthropic.test.ts | 40 +++++++++++++++++++++ packages/llm/src/adapters/anthropic.ts | 23 +++++++++++- packages/llm/src/adapters/gemini.test.ts | 18 ++++++++++ packages/llm/src/adapters/gemini.ts | 23 +++++++++++- 4 files changed, 102 insertions(+), 2 deletions(-) diff --git a/packages/llm/src/adapters/anthropic.test.ts b/packages/llm/src/adapters/anthropic.test.ts index df7b2703..38b3fcec 100644 --- a/packages/llm/src/adapters/anthropic.test.ts +++ b/packages/llm/src/adapters/anthropic.test.ts @@ -159,6 +159,46 @@ describe('AnthropicAdapter', () => { }); }); + it('maps the reasoning-effort tier to output_config.effort + adaptive thinking; off disables; unset omits (ADR-0066)', async () => { + let sent: Record = {}; + const adapter = createAnthropicAdapter({ + fetch: (_input, init) => { + sent = parseJsonBody(init); + return Promise.resolve( + new Response( + JSON.stringify({ + id: 'm', + type: 'message', + role: 'assistant', + model: 'claude-opus-4-8', + content: [{ type: 'text', text: 'ok' }], + stop_reason: 'end_turn', + stop_sequence: null, + usage: { input_tokens: 1, output_tokens: 1 }, + }), + { status: 200, headers: { 'content-type': 'application/json' } }, + ), + ); + }, + maxRetries: 0, + }); + const base = { + model: 'claude-opus-4-8', + maxTokens: 1024, + messages: [{ role: 'user' as const, content: [{ type: 'text' as const, text: 'go' }] }], + }; + await adapter.generate({ ...base, reasoningEffort: 'high' }, 'k'); + expect(sent['thinking']).toEqual({ type: 'adaptive' }); + expect(sent['output_config']).toEqual({ effort: 'high' }); + await adapter.generate({ ...base, reasoningEffort: 'max' }, 'k'); + expect(sent['output_config']).toEqual({ effort: 'max' }); // Anthropic has a native `max` — 1:1, no coarsening + await adapter.generate({ ...base, reasoningEffort: 'off' }, 'k'); + expect(sent['thinking']).toEqual({ type: 'disabled' }); + expect('output_config' in sent).toBe(false); // off never sets output_config + await adapter.generate({ ...base }, 'k'); // unset ⇒ no thinking, no output_config (provider default) + expect('thinking' in sent).toBe(false); + }); + it('rejects handle and url media sources with an explicit bad_request error (1.AF)', async () => { const adapter = createAnthropicAdapter({ fetch: () => Promise.resolve(new Response('unused', { status: 200 })), diff --git a/packages/llm/src/adapters/anthropic.ts b/packages/llm/src/adapters/anthropic.ts index 312cc48f..6f78eec4 100644 --- a/packages/llm/src/adapters/anthropic.ts +++ b/packages/llm/src/adapters/anthropic.ts @@ -1,7 +1,7 @@ import Anthropic from '@anthropic-ai/sdk'; import { mediaModalityOf } from '@relavium/shared'; -import type { AbortSignalLike, ContentPart, StopReason } from '@relavium/shared'; +import type { AbortSignalLike, ContentPart, ReasoningEffort, StopReason } from '@relavium/shared'; import { assertStreamable, assertSupported } from '../capabilities.js'; import { LlmProviderError, kindFromHttpStatus, makeLlmError } from '../llm-error.js'; @@ -46,6 +46,12 @@ const PROVIDER = 'anthropic'; const DEFAULT_MAX_TOKENS = 4096; /** Anthropic's API caps `temperature` at 1 (the shared contract's envelope is the wider [0, 2]). */ const MAX_TEMPERATURE = 1; +/** ADR-0066: the normalized reasoning-effort tier → Anthropic's native `output_config.effort` levels. Anthropic has + * a native `max`, so all four non-`off` tiers map 1:1; `off` is handled separately (thinking disabled). */ +const ANTHROPIC_REASONING_EFFORT: Record< + Exclude, + 'low' | 'medium' | 'high' | 'max' +> = { low: 'low', medium: 'medium', high: 'high', max: 'max' }; /** * Anthropic supports the full common-path surface; provider-specific features go via @@ -478,6 +484,21 @@ function buildCommonBody( format: { type: 'json_schema', schema: req.responseFormat.schema as Record }, }; } + if (req.reasoningEffort !== undefined) { + // ADR-0066: Anthropic's tier-native reasoning control — `output_config.effort` (the level) + ADAPTIVE thinking + // to enable it (no token budget: the tier-native path avoids the legacy budget_tokens constraint). `off` DISABLES + // thinking. The effort level MERGES alongside any structured-output `format` already on output_config. Anthropic + // has a native `max` tier, so all five normalized tiers map 1:1 (no coarsening here). + if (req.reasoningEffort === 'off') { + body.thinking = { type: 'disabled' }; + } else { + body.thinking = { type: 'adaptive' }; + body.output_config = { + ...body.output_config, + effort: ANTHROPIC_REASONING_EFFORT[req.reasoningEffort], + }; + } + } if (req.temperature !== undefined) { // The shared contract is the provider-agnostic [0, 2] envelope (common.ts); Anthropic's API // accepts temperature in [0, 1]. Fail fast (the adapter's "never silently drop" posture) rather diff --git a/packages/llm/src/adapters/gemini.test.ts b/packages/llm/src/adapters/gemini.test.ts index a13b0910..c32da0d8 100644 --- a/packages/llm/src/adapters/gemini.test.ts +++ b/packages/llm/src/adapters/gemini.test.ts @@ -477,6 +477,24 @@ describe('Gemini adapter — request building (buildGeminiRequest)', () => { }); }); + it('maps the reasoning-effort tier to thinkingConfig.thinkingLevel (max→HIGH, off→MINIMAL, unset omitted) (ADR-0066)', () => { + expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'high' }).config['thinkingConfig']).toEqual({ + thinkingLevel: 'HIGH', + }); + // Gemini tops out at HIGH — `max` coarsens to it (no separate xhigh/max tier). + expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'max' }).config['thinkingConfig']).toEqual({ + thinkingLevel: 'HIGH', + }); + expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'low' }).config['thinkingConfig']).toEqual({ + thinkingLevel: 'LOW', + }); + // Gemini has no universal disable (a Pro model rejects budget 0) — `off` degrades to the lowest tier. + expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'off' }).config['thinkingConfig']).toEqual({ + thinkingLevel: 'MINIMAL', + }); + expect('thinkingConfig' in buildGeminiRequest(REQ).config).toBe(false); // unset ⇒ omitted (provider default) + }); + it('round-trips tool_call → functionCall and tool_result → functionResponse by name', () => { const request = buildGeminiRequest({ model: 'gemini-2.5-flash', diff --git a/packages/llm/src/adapters/gemini.ts b/packages/llm/src/adapters/gemini.ts index bf1cc3dd..844ba56c 100644 --- a/packages/llm/src/adapters/gemini.ts +++ b/packages/llm/src/adapters/gemini.ts @@ -1,7 +1,13 @@ import { GenerateVideosOperation, GoogleGenAI } from '@google/genai'; import { MediaMimeTypeSchema, mediaModalityOf } from '@relavium/shared'; -import type { AbortSignalLike, ContentPart, OutputModality, StopReason } from '@relavium/shared'; +import type { + AbortSignalLike, + ContentPart, + OutputModality, + ReasoningEffort, + StopReason, +} from '@relavium/shared'; import { assertStreamable, assertSupported } from '../capabilities.js'; import { LlmProviderError, kindFromHttpStatus, makeLlmError } from '../llm-error.js'; @@ -51,6 +57,16 @@ import { */ const PROVIDER = 'gemini'; +/** ADR-0066: the normalized reasoning-effort tier → Gemini's native `thinkingConfig.thinkingLevel` enum values. + * Gemini tops out at HIGH, so `max`→HIGH (a coarsening); it has no universal disable (a Pro model rejects budget + * 0), so `off` degrades to the lowest tier MINIMAL. The loose config Record takes the enum's string value directly. */ +const GEMINI_THINKING_LEVEL: Record = { + off: 'MINIMAL', + low: 'LOW', + medium: 'MEDIUM', + high: 'HIGH', + max: 'HIGH', +}; /** * Gemini's common-path capability surface (restricted tool schema; ids synthesized). 1.AE wires @@ -565,6 +581,11 @@ export function buildGeminiRequest(req: LlmRequest): GeminiRequest { if (req.maxTokens !== undefined) { config['maxOutputTokens'] = req.maxTokens; } + if (req.reasoningEffort !== undefined) { + // ADR-0066: Gemini's tier-native thinking control (thinkingLevel). Set on thinkingConfig; canonical config wins + // over a providerOptions thinkingConfig on the shallow merge below (the normalized field wins on collision). + config['thinkingConfig'] = { thinkingLevel: GEMINI_THINKING_LEVEL[req.reasoningEffort] }; + } if (req.outputModalities !== undefined && req.outputModalities.some((m) => m !== 'text')) { // Lower the node's non-text output_modalities to Gemini `responseModalities` (inline media-out, // 1.AG/ADR-0046). The per-modality capability gate (assertMediaCapabilities) has already rejected an From ca67127f192607f67e38da898bd13e9dc2aff824 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 17:42:49 +0300 Subject: [PATCH 52/68] =?UTF-8?q?feat(shared,core):=202.5.G=20Step=20E=20(?= =?UTF-8?q?3/n)=20=E2=80=94=20reasoning-effort=20engine=20plumbing=20+=20a?= =?UTF-8?q?gent=20field=20(ADR-0066)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wires the normalized reasoning-effort tier from authoring through the engine to the request, gated by a host-injected per-model capability: - @relavium/shared: AgentSchema gains `reasoning_effort?` (off/low/medium/high/max) — an authorable, git-committable agent field, threaded into each turn. - @relavium/core: AgentRunnerDeps + SessionDeps gain `resolveReasoning?: (model) => boolean` — the host-injected per-model `model_catalog.capabilities.reasoning` projection (mirrors resolveMediaSurface; core stays platform-pure, no @relavium/db import). BOTH turn paths — the workflow AgentRunner (resolveGenKnobs) AND the chat AgentSession — thread the authored effort onto `LlmRequest.reasoningEffort` ONLY when the primary/bound model is reasoning-capable (a non-reasoning model would reject the field; withheld when the resolver returns false/undefined). agent-turn's buildRequest passes it to every chain attempt. - Test: the session sends reasoning_effort:'high' when resolveReasoning→true, withholds it when →false. Now functional at the engine level: an authored agent's reasoning_effort reaches the provider's native tier control (E2 adapters). Remaining (inert until wired): the CLI host-wiring of resolveReasoning from the catalog + the per-model capability population + heuristic, the [chat].reasoning_effort config default, and the picker effort selector. Toolchain green: 970 core + 454 shared tests, full monorepo typecheck + build. Refs: ADR-0066 Co-Authored-By: Claude --- packages/core/src/engine/agent-runner.ts | 23 ++++++++- .../core/src/engine/agent-session.test.ts | 47 +++++++++++++++++++ packages/core/src/engine/agent-session.ts | 14 ++++++ packages/core/src/engine/agent-turn.ts | 6 +++ packages/shared/src/agent.ts | 6 ++- 5 files changed, 93 insertions(+), 3 deletions(-) diff --git a/packages/core/src/engine/agent-runner.ts b/packages/core/src/engine/agent-runner.ts index 4c4f53e5..19870e5a 100644 --- a/packages/core/src/engine/agent-runner.ts +++ b/packages/core/src/engine/agent-runner.ts @@ -27,6 +27,7 @@ import { type MediaCostEstimate, type MediaSurface, type OutputModality, + type ReasoningEffort, } from '@relavium/shared'; import { LlmConfigError, @@ -90,6 +91,14 @@ export interface AgentRunnerDeps { * lookup; the production catalog wiring is host-side (1.AH), like the other 1.AF/1.AG host-wiring obligations. */ readonly resolveMediaSurface?: (model: string) => MediaSurface | undefined; + /** + * Whether a model supports reasoning ([ADR-0066](../../../../docs/decisions/0066-normalized-reasoning-effort-control.md)) — + * the host-injected per-model catalog projection (`model_catalog.capabilities.reasoning`), mirroring + * {@link resolveMediaSurface}. The engine is platform-pure (no DB), so the host injects it. Gates the + * `reasoningEffort` send: a non-reasoning model would reject the field, so the tier is sent only when this + * returns `true`. Absent or `undefined` ⇒ treated as NOT reasoning (safe — the field is not sent). + */ + readonly resolveReasoning?: (model: string) => boolean | undefined; /** The shared tool registry (1.T) the agent dispatches through (ADR-0037). */ readonly registry: ToolRegistry; /** The registry's tool defs — the source of the LLM-visible schema + descriptions for granted tools. */ @@ -332,7 +341,7 @@ async function executeAgent( planEntries: plan.entries, chainCapabilities: chainCapabilities(deps), ...(responseFormat === undefined ? {} : { responseFormat }), - ...resolveGenKnobs(agent, node), + ...resolveGenKnobs(agent, node, deps), nodeId: node.id, emit: ctx.emit, signal: ctx.signal, @@ -788,12 +797,22 @@ function buildLlmTools(defs: readonly ToolDef[], granted: ReadonlySet): function resolveGenKnobs( agent: Agent, node: AgentNode, -): { temperature?: number; maxTokens?: number } { + deps: AgentRunnerDeps, +): { temperature?: number; maxTokens?: number; reasoningEffort?: ReasoningEffort } { const temperature = node.temperature ?? agent.temperature; const maxTokens = node.max_tokens ?? agent.max_tokens; + // ADR-0066: send the reasoning-effort tier ONLY when the agent authored one AND the primary model is + // reasoning-capable (a non-reasoning model would reject the field). The host-injected `resolveReasoning` is the + // per-model catalog projection; absent/`undefined` ⇒ not reasoning ⇒ the tier is withheld (safe). A fallback to a + // different chain model carries the tier along — the adapter/provider ignores or rejects it there (a documented edge). + const reasoningEffort = + agent.reasoning_effort !== undefined && deps.resolveReasoning?.(agent.model) === true + ? agent.reasoning_effort + : undefined; return { ...(temperature === undefined ? {} : { temperature }), ...(maxTokens === undefined ? {} : { maxTokens }), + ...(reasoningEffort === undefined ? {} : { reasoningEffort }), }; } diff --git a/packages/core/src/engine/agent-session.test.ts b/packages/core/src/engine/agent-session.test.ts index 068b0e5f..9d618b65 100644 --- a/packages/core/src/engine/agent-session.test.ts +++ b/packages/core/src/engine/agent-session.test.ts @@ -768,6 +768,53 @@ describe('AgentSession — reseat-less modes + mid-turn abort (ADR-0057 Step 2)' expect(advertised).not.toContain('read_file'); }); + it('sends the authored reasoning_effort ONLY when the model is reasoning-capable (ADR-0066)', async () => { + const reader = AgentSchema.parse({ + id: 'reader', + model: 'claude-opus-4-8', + provider: 'anthropic', + system_prompt: 'x', + reasoning_effort: 'high', + }); + const capturing = (): { provider: LlmProvider; effort: () => unknown } => { + let effort: unknown = 'UNSET'; + const provider: LlmProvider = { + id: 'anthropic', + supports: CAPS, + generate: () => { + throw new Error('unused'); + }, + stream: (req) => { + effort = req.reasoningEffort; + return streamOf(textTurn('ok')); + }, + }; + return { provider, effort: () => effort }; + }; + // Reasoning-capable ⇒ the tier reaches the request. + const on = capturing(); + const onSession = session( + harness([textTurn('ok')], { resolveProvider: () => on.provider, resolveReasoning: () => true }) + .deps, + reader, + ); + onSession.start(); + await onSession.sendMessage('go'); + expect(on.effort()).toBe('high'); + // NOT reasoning-capable ⇒ the tier is WITHHELD (a non-reasoning model would reject it). + const off = capturing(); + const offSession = session( + harness([textTurn('ok')], { + resolveProvider: () => off.provider, + resolveReasoning: () => false, + }).deps, + reader, + ); + offSession.start(); + await offSession.sendMessage('go'); + expect(off.effort()).toBeUndefined(); + }); + it('setTurnPolicy activates the approval regime — the dispatch context carries the confirm hook', async () => { const confirm = (): Promise<{ outcome: 'approve' }> => Promise.resolve({ outcome: 'approve' }); let captured: ToolDispatchContext | undefined; diff --git a/packages/core/src/engine/agent-session.ts b/packages/core/src/engine/agent-session.ts index bc90e9fe..9cfd983c 100644 --- a/packages/core/src/engine/agent-session.ts +++ b/packages/core/src/engine/agent-session.ts @@ -239,6 +239,14 @@ export interface SessionDeps { * `max_cost_microcents` enforces it). Absent ⇒ static-only pricing. `@relavium/core` never imports `@relavium/db`. */ readonly resolvePrice?: PricingOverlay; + /** + * Whether the bound model supports reasoning ([ADR-0066](../../../../docs/decisions/0066-normalized-reasoning-effort-control.md)) + * — the host-injected per-model `model_catalog.capabilities.reasoning` projection (mirrors the `AgentRunner`'s + * `resolveReasoning`). Gates the `reasoningEffort` send: the authored `agent.reasoning_effort` is passed to a turn + * only when this returns `true` (a non-reasoning model rejects the field). Absent/`undefined` ⇒ not reasoning + * ⇒ withheld. `@relavium/core` never imports `@relavium/db`, so the host injects the catalog lookup. + */ + readonly resolveReasoning?: (model: string) => boolean | undefined; /** * Feed the running session cost to a budget governor so a host that wires {@link preEgress} to * `BudgetGovernor.checkPreEgress` also keeps the governor's cumulative total current (ADR-0028, 1.AC). @@ -1049,6 +1057,12 @@ export class AgentSession { chainCapabilities: this.#chainCapabilities(), ...(this.#agent.temperature === undefined ? {} : { temperature: this.#agent.temperature }), ...(this.#agent.max_tokens === undefined ? {} : { maxTokens: this.#agent.max_tokens }), + // ADR-0066: the authored reasoning-effort tier, sent ONLY when the bound model is reasoning-capable (a + // non-reasoning model would reject it — the host-injected per-model catalog projection gates it). + ...(this.#agent.reasoning_effort !== undefined && + this.#deps.resolveReasoning?.(this.#agent.model) === true + ? { reasoningEffort: this.#agent.reasoning_effort } + : {}), nodeId: this.#agentRef, emit: (event) => { this.#onTurnEmit(event); diff --git a/packages/core/src/engine/agent-turn.ts b/packages/core/src/engine/agent-turn.ts index 62729cad..1bd3ce71 100644 --- a/packages/core/src/engine/agent-turn.ts +++ b/packages/core/src/engine/agent-turn.ts @@ -31,6 +31,7 @@ import type { ContentPart, ErrorCode, OutputModality, + ReasoningEffort, StopReason, } from '@relavium/shared'; import { @@ -131,6 +132,9 @@ export interface AgentTurnParams { /** Per-turn generation knobs (node-over-agent precedence is resolved by the caller). */ readonly temperature?: number; readonly maxTokens?: number; + /** Normalized reasoning-effort tier (ADR-0066) — passed onto every chain attempt's `LlmRequest.reasoningEffort`; + * each adapter maps it to the provider's native control. Gated to a reasoning-capable primary model by the caller. */ + readonly reasoningEffort?: ReasoningEffort; /** The id stamped on emitted events (a workflow vertex id on the run path; a synthetic id on a session). */ readonly nodeId: string; /** Emit an envelope-less streaming event; the engine/bus attaches the correlation key + sequence. */ @@ -347,6 +351,8 @@ function buildRequest(messages: readonly LlmMessage[], params: AgentTurnParams): ...(params.responseFormat === undefined ? {} : { responseFormat: params.responseFormat }), ...(params.temperature === undefined ? {} : { temperature: params.temperature }), ...(params.maxTokens === undefined ? {} : { maxTokens: params.maxTokens }), + // ADR-0066: the normalized reasoning-effort tier onto every attempt's request (the adapter maps it natively). + ...(params.reasoningEffort === undefined ? {} : { reasoningEffort: params.reasoningEffort }), // Lower the node's requested non-text output onto the request (1.AF/D15) so the FallbackChain // per-attempt capability pre-skip (requestSupportReason → outputCombinationReason) can skip a model // that cannot emit the combination — the runtime backstop the load-check defers to (ADR-0044 §2). Without diff --git a/packages/shared/src/agent.ts b/packages/shared/src/agent.ts index 40bfbec6..cccd8b98 100644 --- a/packages/shared/src/agent.ts +++ b/packages/shared/src/agent.ts @@ -9,7 +9,7 @@ import { positiveInt, temperatureSchema, } from './common.js'; -import { LLM_PROVIDERS, RETRYABLE_ERROR_CODES } from './constants.js'; +import { LLM_PROVIDERS, REASONING_EFFORTS, RETRYABLE_ERROR_CODES } from './constants.js'; /** * Agent schema (agent-yaml-spec.md). An agent is a named, reusable LLM @@ -262,6 +262,10 @@ export const AgentSchema = z system_prompt: nonEmptyString, temperature: temperatureSchema.optional(), // provider-agnostic [0, 2] (common.ts) max_tokens: positiveInt.optional(), + // Normalized reasoning-effort tier (ADR-0066) — off/low/medium/high/max. Threaded into each turn's + // `LlmRequest.reasoningEffort`; the adapter maps it to the provider's native tier control. Absent ⇒ the + // `[chat].reasoning_effort` config default, else the provider default. Only sent to a reasoning-capable model. + reasoning_effort: z.enum(REASONING_EFFORTS).optional(), // Optional agent-level JSON-Schema metadata (agent-yaml-spec.md) — the engine validates // turn I/O against these when present; absent on most agents. input_schema: jsonSchemaMetadataSchema.optional(), From 31cfa7775fae0c8cea449fb5fae8d1498eb9226e Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 18:05:23 +0300 Subject: [PATCH 53/68] =?UTF-8?q?feat(llm,cli):=202.5.G=20Step=20E=20(4/n)?= =?UTF-8?q?=20=E2=80=94=20per-model=20reasoning=20capability=20+=20host=20?= =?UTF-8?q?wiring=20(ADR-0066)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes the reasoning-effort feature LIVE end-to-end for an authored agent: the static per-model capability + the CLI host-wiring of resolveReasoning. - @relavium/llm: MODEL_PRICING gains a per-model `reasoning?` flag (opt-in; absent ⇒ false, the SAFE default) — set true on the 8 flagship reasoning models (claude-4.x/5, gpt-5.x, gemini-2.5). DeepSeek stays off (its adapter mapping is deferred — the effort isn't controllable there yet, so the picker must not offer it). New pure helper modelSupportsReasoning(model) → the static registry projection. - CLI: wire resolveReasoning: modelSupportsReasoning into BOTH deps paths — the chat/Home SessionDeps (session-host) and the workflow AgentRunnerDeps (build-engine) — mirroring resolveMediaSurface. So an authored agent's reasoning_effort now flows: agent field → resolveReasoning gate → LlmRequest.reasoningEffort → the adapter's native tier control. - Tests: modelSupportsReasoning (reasoning model true, DeepSeek + unknown/custom false). Now functional for an authored agent. Remaining: [chat].reasoning_effort config default (for the default chat agent) + the picker effort selector UI, then the review cadence. Toolchain green: 1492 CLI tests (media-gc flake aside) + llm/pricing tests, all typecheck/build. Refs: ADR-0066 Co-Authored-By: Claude --- apps/cli/src/chat/session-host.ts | 5 ++++- apps/cli/src/engine/build-engine.ts | 5 ++++- packages/llm/src/index.ts | 8 +++++++- packages/llm/src/pricing.test.ts | 21 ++++++++++++++++++- packages/llm/src/pricing.ts | 31 +++++++++++++++++++++++++++++ 5 files changed, 66 insertions(+), 4 deletions(-) diff --git a/apps/cli/src/chat/session-host.ts b/apps/cli/src/chat/session-host.ts index 88bb8db9..32cb80fc 100644 --- a/apps/cli/src/chat/session-host.ts +++ b/apps/cli/src/chat/session-host.ts @@ -16,7 +16,7 @@ import { type ToolDef, type ToolHost, } from '@relavium/core'; -import type { PricingOverlay, ProviderId } from '@relavium/llm'; +import { modelSupportsReasoning, type PricingOverlay, type ProviderId } from '@relavium/llm'; import type { ManagerSkippedTool, McpClient, McpServerConfig } from '@relavium/mcp'; import type { AgentSessionRecord, @@ -225,6 +225,9 @@ function buildSessionRuntime( const deps: SessionDeps = { resolveProvider: providers.resolveProvider, keyFor: providers.keyFor, + // ADR-0066: the per-model reasoning capability (static registry projection) — gates whether the authored + // reasoning_effort tier is sent (a non-reasoning / custom model returns false, so the field is withheld). + resolveReasoning: modelSupportsReasoning, registry, tools, sleep: (ms) => new Promise((resolveSleep) => setTimeout(resolveSleep, ms)), diff --git a/apps/cli/src/engine/build-engine.ts b/apps/cli/src/engine/build-engine.ts index 153386c7..f537d17d 100644 --- a/apps/cli/src/engine/build-engine.ts +++ b/apps/cli/src/engine/build-engine.ts @@ -11,7 +11,7 @@ import { type ToolDef, type ToolHost, } from '@relavium/core'; -import type { PricingOverlay } from '@relavium/llm'; +import { modelSupportsReasoning, type PricingOverlay } from '@relavium/llm'; import type { MediaCostEstimate, MediaSurface } from '@relavium/shared'; import { createCliHost } from './host.js'; @@ -108,6 +108,9 @@ export async function buildEngine(options: BuildEngineOptions = {}): Promise new Promise((resolveSleep) => setTimeout(resolveSleep, ms)), diff --git a/packages/llm/src/index.ts b/packages/llm/src/index.ts index ef280407..9d3944ab 100644 --- a/packages/llm/src/index.ts +++ b/packages/llm/src/index.ts @@ -129,7 +129,13 @@ export { } from './llm-error.js'; // CostTracker + the canonical model-pricing table (1.B). -export { MODEL_PRICING, KNOWN_MODEL_IDS, isCanonicalModelId, contextWindowForModel } from './pricing.js'; +export { + MODEL_PRICING, + KNOWN_MODEL_IDS, + isCanonicalModelId, + contextWindowForModel, + modelSupportsReasoning, +} from './pricing.js'; export type { ModelPricing, CanonicalModelId } from './pricing.js'; // The pure live/static/user merge helper (ADR-0064 §6) — reused by every surface's model catalog / picker. export { mergeModelCatalog } from './model-catalog.js'; diff --git a/packages/llm/src/pricing.test.ts b/packages/llm/src/pricing.test.ts index 2044db8b..4bd903b3 100644 --- a/packages/llm/src/pricing.test.ts +++ b/packages/llm/src/pricing.test.ts @@ -1,6 +1,11 @@ import { describe, expect, it } from 'vitest'; -import { contextWindowForModel, KNOWN_MODEL_IDS, MODEL_PRICING } from './pricing.js'; +import { + contextWindowForModel, + KNOWN_MODEL_IDS, + MODEL_PRICING, + modelSupportsReasoning, +} from './pricing.js'; /** * `contextWindowForModel` (ADR-0062 §7) — the pure catalog lookup the CLI footer context-fullness indicator uses @@ -8,6 +13,20 @@ import { contextWindowForModel, KNOWN_MODEL_IDS, MODEL_PRICING } from './pricing * returns for a known model, and `undefined` for a custom base-URL model absent from the catalog (which degrades the * indicator + auto-compaction to "not applicable", never a crash). */ +describe('modelSupportsReasoning (ADR-0066)', () => { + it('is true for a tagged reasoning model, false for DeepSeek (adapter deferred) + unknown/custom', () => { + expect(modelSupportsReasoning('claude-opus-4-8')).toBe(true); + expect(modelSupportsReasoning('gpt-5.5')).toBe(true); + expect(modelSupportsReasoning('gemini-2.5-pro')).toBe(true); + // DeepSeek reasons (v4 thinking) but its adapter mapping is deferred, so the capability stays OFF (the effort + // is not controllable there yet — the picker must not offer it). + expect(modelSupportsReasoning('deepseek-v4-flash')).toBe(false); + // Unknown / custom base-URL model ⇒ the SAFE default (never send the tier to a model that would reject it). + expect(modelSupportsReasoning('some-custom-base-url-model-xyz')).toBe(false); + expect(modelSupportsReasoning('')).toBe(false); + }); +}); + describe('contextWindowForModel (ADR-0062 §7)', () => { it('returns the catalog window for a known canonical model', () => { // Narrow rather than assert (CLAUDE.md rule 1): under noUncheckedIndexedAccess KNOWN_MODEL_IDS[0] is diff --git a/packages/llm/src/pricing.ts b/packages/llm/src/pricing.ts index e6841784..4d87d48f 100644 --- a/packages/llm/src/pricing.ts +++ b/packages/llm/src/pricing.ts @@ -60,6 +60,15 @@ export interface ModelPricing { // Keyed by the canonical `MediaBilledModality` set (image/audio/video) via a mapped type, so the keys // stay in sync with `MEDIA_BILLED_MODALITIES` at compile time — never a hand-maintained literal. readonly mediaOutputRates?: { readonly [K in MediaBilledModality]?: number }; + /** + * Whether this model supports a reasoning-effort control ([ADR-0066](../../../docs/decisions/0066-normalized-reasoning-effort-control.md)) — + * the static per-model capability the host projects to `resolveReasoning` (gating whether `reasoningEffort` is + * sent + whether the picker offers the effort selector). **Opt-in**: absent ⇒ `false` (the SAFE default — a + * non-reasoning model must never receive the field). Set `true` only for a model whose adapter maps the tier + * (so DeepSeek stays absent until its adapter mapping lands, even though v4 reasons — the effort is not + * controllable there yet). + */ + readonly reasoning?: boolean; } const USD_PER_MTOK_TO_MICROCENTS = 100_000_000; // 1 USD = 1e8 micro-cents @@ -73,6 +82,7 @@ export const MODEL_PRICING = { provider: 'anthropic', nativeId: 'claude-fable-5', displayName: 'Claude Fable 5', + reasoning: true, contextWindowTokens: 1_000_000, maxOutputTokens: 128_000, inputPerMtokMicrocents: usd(10), @@ -84,6 +94,7 @@ export const MODEL_PRICING = { provider: 'anthropic', nativeId: 'claude-opus-4-8', displayName: 'Claude Opus 4.8', + reasoning: true, contextWindowTokens: 1_000_000, maxOutputTokens: 128_000, inputPerMtokMicrocents: usd(5), @@ -95,6 +106,7 @@ export const MODEL_PRICING = { provider: 'anthropic', nativeId: 'claude-sonnet-4-6', displayName: 'Claude Sonnet 4.6', + reasoning: true, contextWindowTokens: 1_000_000, maxOutputTokens: 64_000, inputPerMtokMicrocents: usd(3), @@ -106,6 +118,7 @@ export const MODEL_PRICING = { provider: 'anthropic', nativeId: 'claude-haiku-4-5', displayName: 'Claude Haiku 4.5', + reasoning: true, contextWindowTokens: 200_000, maxOutputTokens: 64_000, inputPerMtokMicrocents: usd(1), @@ -119,6 +132,7 @@ export const MODEL_PRICING = { provider: 'openai', nativeId: 'gpt-5.5', displayName: 'GPT-5.5', + reasoning: true, contextWindowTokens: 1_000_000, maxOutputTokens: 128_000, inputPerMtokMicrocents: usd(5), @@ -129,6 +143,7 @@ export const MODEL_PRICING = { provider: 'openai', nativeId: 'gpt-5.4-mini', displayName: 'GPT-5.4 mini', + reasoning: true, contextWindowTokens: 400_000, maxOutputTokens: 128_000, inputPerMtokMicrocents: usd(0.75), @@ -141,6 +156,7 @@ export const MODEL_PRICING = { provider: 'gemini', nativeId: 'gemini-2.5-flash', displayName: 'Gemini 2.5 Flash', + reasoning: true, contextWindowTokens: 1_048_576, maxOutputTokens: 65_536, inputPerMtokMicrocents: usd(0.3), // text/image/video tier (audio: $1.00/MTok) @@ -151,6 +167,7 @@ export const MODEL_PRICING = { provider: 'gemini', nativeId: 'gemini-2.5-pro', displayName: 'Gemini 2.5 Pro', + reasoning: true, contextWindowTokens: 1_048_576, maxOutputTokens: 65_536, inputPerMtokMicrocents: usd(1.25), // prompts ≤200K tier (>200K: $2.50 in / $15 out) @@ -229,6 +246,20 @@ export function isCanonicalModelId(value: string): value is CanonicalModelId { export const KNOWN_MODEL_IDS: readonly CanonicalModelId[] = Object.keys(MODEL_PRICING).filter(isCanonicalModelId); +/** + * Whether a model supports a reasoning-effort control ([ADR-0066](../../../docs/decisions/0066-normalized-reasoning-effort-control.md)) + * — the static per-model capability the host projects to the engine's `resolveReasoning` gate (and the `/models` + * picker's effort selector). `false` for an unknown/custom (non-catalog) model — the SAFE default, so the tier is + * never sent to a model that would reject it. A pure host-side helper, like {@link contextWindowForModel}. + */ +export function modelSupportsReasoning(model: string): boolean { + if (!isCanonicalModelId(model)) return false; + // Widen the literal-union entry to `ModelPricing` so `.reasoning` (absent on the non-reasoning members) reads as + // `boolean | undefined` — each entry `satisfies ModelPricing`, so this is assignment, not a cast. + const entry: ModelPricing = MODEL_PRICING[model]; + return entry.reasoning === true; +} + /** * The context window (max tokens) for a canonical model id, or `undefined` for an unknown id (e.g. a custom * base-URL model absent from the catalog). A light, pure host-side helper — the SAME catalog value the adapters' From e75e6a9209d638a507450cc9ae5be0e31407f79d Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 18:13:49 +0300 Subject: [PATCH 54/68] =?UTF-8?q?feat(cli,shared):=202.5.E-E4=20=E2=80=94?= =?UTF-8?q?=20[chat].reasoning=5Feffort=20config=20default=20(ADR-0066)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thread the resolved `[chat].reasoning_effort` onto the built-in default chat agent so a config default lights up reasoning control without a picker step. - shared: ChatConfigSchema gains `reasoning_effort` (z.enum(REASONING_EFFORTS)). - cli/config: ResolvedChatConfig.reasoningEffort; resolveChat prefers project over workspace (p ?? w), mirroring default_model. - cli/chat: buildDefaultChatAgent(model, reasoningEffort?) bakes the tier onto the DEFAULT agent only (an authored --agent owns its own); agent-source + session-host thread it through. Absent ⇒ no reasoning control. Only the default agent is affected — reasoning_effort on an authored agent already flows via AgentSchema (E3). Test ResolvedChatConfig fixtures updated. Refs: ADR-0066 Co-Authored-By: Claude --- apps/cli/src/chat/agent-source.ts | 6 +++++- apps/cli/src/chat/default-agent.ts | 7 +++++-- apps/cli/src/chat/persister.test.ts | 1 + apps/cli/src/chat/session-host.test.ts | 1 + apps/cli/src/chat/session-host.ts | 5 +++++ apps/cli/src/commands/chat.test.ts | 1 + apps/cli/src/config/resolve.ts | 4 ++++ apps/cli/src/engine/media-wiring.test.ts | 1 + packages/shared/src/config.ts | 6 +++++- 9 files changed, 28 insertions(+), 4 deletions(-) diff --git a/apps/cli/src/chat/agent-source.ts b/apps/cli/src/chat/agent-source.ts index c0feb44b..4863fcde 100644 --- a/apps/cli/src/chat/agent-source.ts +++ b/apps/cli/src/chat/agent-source.ts @@ -1,4 +1,5 @@ import { parseAgent, type AgentDefinition } from '@relavium/core'; +import type { ReasoningEffort } from '@relavium/shared'; import { resolveYamlSource } from '../workflows/resolve.js'; import { buildDefaultChatAgent, DEFAULT_CHAT_MODEL } from './default-agent.js'; @@ -8,6 +9,9 @@ export interface ResolveChatAgentOptions { readonly projectConfigDir: string | undefined; /** The resolved `[chat].default_model` — used only to build the default agent when no `--agent` is given. */ readonly defaultModel: string | undefined; + /** The resolved `[chat].reasoning_effort` (ADR-0066) — baked onto the DEFAULT agent only (an authored `--agent` + * owns its own `reasoning_effort`, never overridden by config). Absent ⇒ no reasoning control. */ + readonly reasoningEffort?: ReasoningEffort; } /** @@ -23,7 +27,7 @@ export function resolveChatAgent( opts: ResolveChatAgentOptions, ): AgentDefinition { if (agentRef === undefined) { - return buildDefaultChatAgent(opts.defaultModel ?? DEFAULT_CHAT_MODEL); + return buildDefaultChatAgent(opts.defaultModel ?? DEFAULT_CHAT_MODEL, opts.reasoningEffort); } const source = resolveYamlSource(agentRef, { cwd: opts.cwd, diff --git a/apps/cli/src/chat/default-agent.ts b/apps/cli/src/chat/default-agent.ts index 8ec64780..9be6352b 100644 --- a/apps/cli/src/chat/default-agent.ts +++ b/apps/cli/src/chat/default-agent.ts @@ -1,5 +1,5 @@ import type { ProviderId } from '@relavium/llm'; -import type { Agent } from '@relavium/shared'; +import type { Agent, ReasoningEffort } from '@relavium/shared'; import { CliError } from '../process/errors.js'; @@ -56,8 +56,10 @@ export function inferProviderFromModel(model: string): ProviderId | undefined { * Build the built-in default chat agent over `model` (the resolved `[chat].default_model`, or * {@link DEFAULT_CHAT_MODEL}). Throws a clean exit-2 {@link CliError} when the provider cannot be inferred * from the model id — guiding the user to set a known `[chat].default_model` or bind an explicit `--agent`. + * `reasoningEffort` (the resolved `[chat].reasoning_effort`, ADR-0066) is baked onto the agent so the default + * chat honors the config default; absent ⇒ no reasoning control (the provider default). */ -export function buildDefaultChatAgent(model: string): Agent { +export function buildDefaultChatAgent(model: string, reasoningEffort?: ReasoningEffort): Agent { const provider = inferProviderFromModel(model); if (provider === undefined) { throw new CliError( @@ -74,5 +76,6 @@ export function buildDefaultChatAgent(model: string): Agent { provider, system_prompt: DEFAULT_CHAT_SYSTEM_PROMPT, tools: [...DEFAULT_CHAT_TOOLS], + ...(reasoningEffort === undefined ? {} : { reasoning_effort: reasoningEffort }), }; } diff --git a/apps/cli/src/chat/persister.test.ts b/apps/cli/src/chat/persister.test.ts index f5287acf..d67c4551 100644 --- a/apps/cli/src/chat/persister.test.ts +++ b/apps/cli/src/chat/persister.test.ts @@ -30,6 +30,7 @@ const EMPTY_CHAT: ResolvedChatConfig = { onExceed: undefined, allowedCommands: undefined, allowedCommandGlobs: undefined, + reasoningEffort: undefined, }; const textOf = (content: readonly DurableContentPart[]): string => diff --git a/apps/cli/src/chat/session-host.test.ts b/apps/cli/src/chat/session-host.test.ts index ed97fadd..0b8f7469 100644 --- a/apps/cli/src/chat/session-host.test.ts +++ b/apps/cli/src/chat/session-host.test.ts @@ -89,6 +89,7 @@ const EMPTY_CHAT: ResolvedChatConfig = { onExceed: undefined, allowedCommands: undefined, allowedCommandGlobs: undefined, + reasoningEffort: undefined, }; function deterministicIds() { diff --git a/apps/cli/src/chat/session-host.ts b/apps/cli/src/chat/session-host.ts index 32cb80fc..a95c9ef2 100644 --- a/apps/cli/src/chat/session-host.ts +++ b/apps/cli/src/chat/session-host.ts @@ -276,6 +276,11 @@ export async function buildChatSession(opts: BuildChatSessionOptions): Promise0 = per-session cap on_exceed: z.enum(ON_EXCEED_ACTIONS).optional(), + // The default reasoning-effort tier for a chat whose bound agent authors none (ADR-0066) — off/low/medium/high/ + // max. Applied to the built-in default chat agent + surfaced as the picker's starting effort; only sent to a + // reasoning-capable model. Absent ⇒ the provider default (no reasoning control sent). + reasoning_effort: z.enum(REASONING_EFFORTS).optional(), }) .strict() // fail loud on an unknown [chat] key (strict config — ADR-0033, amends ADR-0023) .optional(); From 69ed9bc0e767e93c240b682b2b47ecab78973668 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 18:38:11 +0300 Subject: [PATCH 55/68] =?UTF-8?q?feat(cli,llm):=202.5.E-E5=20=E2=80=94=20/?= =?UTF-8?q?models=20picker=20reasoning-effort=20sub-step=20(ADR-0066)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Realize the ADR-0066 picker effort selector: on a reasoning-capable model during a LIVE reseat, the `/models` picker advances to a second step to choose the reasoning-effort tier, bound onto the reseated agent. - llm/model-catalog: ModelCatalogEntry gains `supportsReasoning` (from the STATIC registry `reasoning` flag — the same authority as modelSupportsReasoning), so the picker lights up the effort step exactly for the models the engine gate honors. - cli/model-picker: two-phase pure fold (`model` → `effort`). Enter on a reasoning model (on a reseat surface) advances to a fixed off/low/medium/high/ max sub-list opening on the bound effort (else `medium`); Enter accepts the model + tier; Esc backs out to the model list; Ctrl-C hard-cancels. A non-reasoning model / a non-effort surface (bare-Home default-write) accepts immediately, unchanged. `accept` carries `reasoningEffort`. - swapAgentModel(reasoningEffort?) binds the picked tier or drops any prior one; ReseatTarget + ChatModelsPort.boundEffort + HomeChatSession.boundEffort thread it, and the reseat no-op guard is now keyed on model AND effort (same model, different effort IS a real switch). - Wired live: standalone `relavium chat` (always effort-capable), in-Home live chat (effort-capable), bare-Home default-write (single-step). View renders the effort sub-list (tier + hint, ✓ on the bound effort). - Tests: fold effort-phase (13), catalog supportsReasoning (2), swapAgentModel effort (3); the drive-home reseat integration drives the effort step through. - Docs: config-spec `[chat].reasoning_effort`; commands.md `/models` reconciled (the ADR-0059 reseat has landed — no longer "home-only"/"Phase-2.6") + the effort sub-step; roadmap follow-up marked complete. Refs: ADR-0066, ADR-0059, ADR-0064 Co-Authored-By: Claude --- apps/cli/src/chat/session-host.test.ts | 23 +++ apps/cli/src/chat/session-host.ts | 10 ++ apps/cli/src/commands/chat.ts | 52 +++++-- apps/cli/src/home/drive-home.test.ts | 9 +- apps/cli/src/home/drive-home.tsx | 9 +- apps/cli/src/render/tui/chat-ink.tsx | 21 ++- .../src/render/tui/home-controller.test.ts | 6 + apps/cli/src/render/tui/home-controller.ts | 40 ++++- apps/cli/src/render/tui/model-picker-view.tsx | 39 +++++ apps/cli/src/render/tui/model-picker.test.ts | 124 +++++++++++++++ apps/cli/src/render/tui/model-picker.ts | 141 ++++++++++++++++-- docs/reference/cli/commands.md | 2 +- docs/reference/contracts/config-spec.md | 3 + docs/roadmap/current.md | 7 +- packages/llm/src/model-catalog.test.ts | 16 ++ packages/llm/src/model-catalog.ts | 12 ++ 16 files changed, 478 insertions(+), 36 deletions(-) diff --git a/apps/cli/src/chat/session-host.test.ts b/apps/cli/src/chat/session-host.test.ts index 0b8f7469..6f7ea91c 100644 --- a/apps/cli/src/chat/session-host.test.ts +++ b/apps/cli/src/chat/session-host.test.ts @@ -740,6 +740,29 @@ describe('swapAgentModel (ADR-0059 model-switch rule)', () => { expect(next.provider).toBe('openai'); expect('fallback_chain' in next).toBe(false); }); + + it('BINDS a passed reasoning-effort tier onto the swapped agent (ADR-0066)', () => { + const next = swapAgentModel( + buildDefaultChatAgent('claude-sonnet-4-6'), + 'claude-opus-4-8', + 'anthropic', + 'high', + ); + expect(next.reasoning_effort).toBe('high'); + }); + + it('DROPS a prior effort when none is passed (a non-reasoning target can’t carry a stale tier)', () => { + const withEffort = { ...buildDefaultChatAgent('claude-opus-4-8'), reasoning_effort: 'max' as const }; + const next = swapAgentModel(withEffort, 'deepseek-chat', 'deepseek'); // no reasoningEffort arg + expect('reasoning_effort' in next).toBe(false); // dropped, not carried onto the new model + expect(withEffort.reasoning_effort).toBe('max'); // the input is untouched (a fresh copy) + }); + + it('OVERWRITES a prior effort with the newly-picked tier', () => { + const withEffort = { ...buildDefaultChatAgent('claude-opus-4-8'), reasoning_effort: 'low' as const }; + const next = swapAgentModel(withEffort, 'claude-sonnet-4-6', 'anthropic', 'off'); + expect(next.reasoning_effort).toBe('off'); + }); }); describe('buildGovernorWiring', () => { diff --git a/apps/cli/src/chat/session-host.ts b/apps/cli/src/chat/session-host.ts index a95c9ef2..9348b148 100644 --- a/apps/cli/src/chat/session-host.ts +++ b/apps/cli/src/chat/session-host.ts @@ -22,6 +22,7 @@ import type { AgentSessionRecord, Budget, McpServerRegistration, + ReasoningEffort, SessionContext, SessionMessage, ToolPolicy, @@ -373,16 +374,25 @@ function narrowToWired( * belonged to the old model; the resumed instance builds its own default plan for the new model, exactly as a fresh * session on it would). Operates on a fresh copy — never mutates the input. Shared by the standalone `chat` reseat * (`buildReseatWiring`) and the in-Home chat reseat (`driveHome`) so the swap rule has ONE home. + * + * `reasoningEffort` ([ADR-0066](../../../../docs/decisions/0066-normalized-reasoning-effort-control.md)) rides the + * picker's effort sub-step: a defined tier is bound onto the swapped agent; `undefined` DROPS any prior + * `reasoning_effort` (a non-reasoning target can't use one, and the picker only omits it for such a target), so the + * new binding never carries a stale tier from the old model. */ export function swapAgentModel( agent: AgentDefinition, modelId: string, provider: ProviderId, + reasoningEffort?: ReasoningEffort, ): AgentDefinition { // A fresh copy, then `delete` the optional `fallback_chain` (removes the key entirely — never an explicit // `undefined` under exactOptionalPropertyTypes — and mutates only this copy, never the loaded record). const next: AgentDefinition = { ...agent, model: modelId, provider }; delete next.fallback_chain; + // Bind the picked tier, or drop any prior one (same `delete` discipline — never an explicit `undefined`). + if (reasoningEffort === undefined) delete next.reasoning_effort; + else next.reasoning_effort = reasoningEffort; return next; } diff --git a/apps/cli/src/commands/chat.ts b/apps/cli/src/commands/chat.ts index 39bd201e..d8442997 100644 --- a/apps/cli/src/commands/chat.ts +++ b/apps/cli/src/commands/chat.ts @@ -9,7 +9,7 @@ import { type UserCommandOutcome, } from '@relavium/core'; import type { ProviderId } from '@relavium/llm'; -import type { AgentSessionRecord } from '@relavium/shared'; +import type { AgentSessionRecord, ReasoningEffort } from '@relavium/shared'; import { exportSession } from '../chat/export.js'; import { formatDoctorReport, runDoctorChecks, type DoctorProbes } from '../chat/doctor.js'; import { assembleDoctorProbes } from '../chat/doctor-host.js'; @@ -112,6 +112,12 @@ function closeQuietly(io: CliIo, label: string, close: () => void): void { export interface ReseatTarget { readonly modelId: string; readonly provider: ProviderId; + /** + * The reasoning-effort tier the picker's effort sub-step chose ([ADR-0066](../../../../docs/decisions/0066-normalized-reasoning-effort-control.md)), + * bound onto the reseated agent. Present only when the picked model supports reasoning; absent ⇒ the reseat drops + * any prior tier (a non-reasoning target can't use one). Threaded verbatim to {@link swapAgentModel}. + */ + readonly reasoningEffort?: ReasoningEffort; } /** @@ -121,18 +127,26 @@ export interface ReseatTarget { */ export interface ChatModelsPort extends ModelCatalogPort { readonly boundModel: string; + /** + * The session's currently-bound reasoning-effort tier (ADR-0066) — the effort sub-list's `✓` "you are here" + + * its opening highlight, and (with `boundModel`) the reseat no-op guard's second axis: re-picking the same model + * AND the same effort is a no-op, but the same model with a different effort is a real (effort-only) switch. + * `undefined` ⇒ no effort bound (the provider default). + */ + readonly boundEffort: ReasoningEffort | undefined; } /** Assemble a {@link ChatModelsPort} over the session's shared db + provider resolver (the catalog the Home picker - * also reads) plus the bound model — the one place the chat reseat picker's port is wired. */ + * also reads) plus the bound model + effort — the one place the chat reseat picker's port is wired. */ function buildChatModelsPort( opened: OpenedSessionStore, providers: ProviderResolver, boundModel: string, + boundEffort: ReasoningEffort | undefined, now: () => number, uuid: () => string, ): ChatModelsPort { - return { ...createModelCatalogPort({ db: opened.db, providers, now, uuid }), boundModel }; + return { ...createModelCatalogPort({ db: opened.db, providers, now, uuid }), boundModel, boundEffort }; } /** What an interactive driver receives — the command core's seam, so a driver never touches the session directly. */ @@ -422,7 +436,7 @@ export async function chatCommand(args: ChatCommandArgs, deps: ChatCommandDeps): persister, doctorProbes, startSession: () => built.session.start(), - modelPicker: buildChatModelsPort(opened, providers, built.agent.model, now, uuid), + modelPicker: buildChatModelsPort(opened, providers, built.agent.model, built.agent.reasoning_effort, now, uuid), ...(config.chat.maxMessages === undefined ? {} : { chatMaxMessages: config.chat.maxMessages }), @@ -580,7 +594,7 @@ export async function chatResumeCommand( doctorProbes, startSession: () => {}, intro, - modelPicker: buildChatModelsPort(opened, providers, built.agent.model, now, uuid), + modelPicker: buildChatModelsPort(opened, providers, built.agent.model, built.agent.reasoning_effort, now, uuid), ...(config.chat.maxMessages === undefined ? {} : { chatMaxMessages: config.chat.maxMessages }), @@ -1104,7 +1118,14 @@ async function buildFreshChatWiring(deps: FreshChatWiringDeps, intro: string): P doctorProbes, startSession: () => built.session.start(), intro, - modelPicker: buildChatModelsPort(deps.opened, deps.providers, built.agent.model, deps.now, deps.uuid), + modelPicker: buildChatModelsPort( + deps.opened, + deps.providers, + built.agent.model, + built.agent.reasoning_effort, + deps.now, + deps.uuid, + ), ...(deps.chat.maxMessages === undefined ? {} : { chatMaxMessages: deps.chat.maxMessages }), }; } @@ -1231,8 +1252,14 @@ async function buildReseatWiring( `cannot switch model: session ${oldSessionId} could not be reloaded for reseat`, ); } - // Swap the bound model/provider (dropping the original fallback_chain) via the shared ADR-0059 rule. - const newAgent = swapAgentModel(loaded.session.agentSnapshot, target.modelId, target.provider); + // Swap the bound model/provider (dropping the original fallback_chain) via the shared ADR-0059 rule; the picker's + // effort sub-step (ADR-0066) rides `target.reasoningEffort` onto the swapped agent (or drops any prior tier). + const newAgent = swapAgentModel( + loaded.session.agentSnapshot, + target.modelId, + target.provider, + target.reasoningEffort, + ); // The record the resumed session rebinds from: the just-ended row with the model-swapped agent snapshot. // (The row's own `modelId` FK column stays as-is — per-message/session `modelId` attribution is deferred with // the 2.6.C cost breakdown; see persister.ts. `reconstructSessionState` reads only the transcript + cost here.) @@ -1279,7 +1306,14 @@ async function buildReseatWiring( startSession: () => {}, intro: modelSwitchNotice(target.modelId, resumed.resumeState.turnCount), // The picker's `boundModel` is now the SWITCHED model — a further reseat marks it as the ✓ "you are here". - modelPicker: buildChatModelsPort(deps.opened, deps.providers, resumed.agent.model, deps.now, deps.uuid), + modelPicker: buildChatModelsPort( + deps.opened, + deps.providers, + resumed.agent.model, + resumed.agent.reasoning_effort, + deps.now, + deps.uuid, + ), ...(deps.chat.maxMessages === undefined ? {} : { chatMaxMessages: deps.chat.maxMessages }), }; } diff --git a/apps/cli/src/home/drive-home.test.ts b/apps/cli/src/home/drive-home.test.ts index 659682de..5e393761 100644 --- a/apps/cli/src/home/drive-home.test.ts +++ b/apps/cli/src/home/drive-home.test.ts @@ -300,7 +300,12 @@ describe('driveHome (2.5.B / ADR-0054)', () => { await flush(); expect(props.controller.getSnapshot().modelPicker).toBeDefined(); type(props, 'opus'); // filter to claude-opus-4-8 (registry-priced anthropic ⇒ available on the scripted key) - props.controller.handleKey('', ENTER); // accept ⇒ the REAL reseatChat + props.controller.handleKey('', ENTER); // opus is reasoning-capable ⇒ the ADR-0066 effort sub-step (not an + // immediate reseat). The picker advanced to the effort phase over the pending model. + const effortPicker = props.controller.getSnapshot().modelPicker; + expect(effortPicker?.phase).toBe('effort'); + expect(effortPicker?.pending?.modelId).toBe('claude-opus-4-8'); + props.controller.handleKey('', ENTER); // apply the highlighted tier (default 'medium') ⇒ the REAL reseatChat await flush(); // The session was reseated: a NEW session object, the SAME sessionId, bound to opus, carrying the prior turn. @@ -309,6 +314,7 @@ describe('driveHome (2.5.B / ADR-0054)', () => { expect(reseated?.sessionId).toBe(sessionId); // a reseat CONTINUES the same session (unlike /clear's new id) expect(reseated?.store.getSnapshot().state.model).toBe('claude-opus-4-8'); // rebound to the picked model expect(reseated?.store.getSnapshot().state.turnCount).toBe(1); // the prior turn carried + expect(reseated?.boundEffort).toBe('medium'); // the effort sub-step's tier bound onto the reseated agent (ADR-0066) expect(props.controller.getSnapshot().modelPicker).toBeUndefined(); // the picker closed expect(props.controller.getSnapshot().mode).toBe('chat'); // stayed in chat @@ -317,6 +323,7 @@ describe('driveHome (2.5.B / ADR-0054)', () => { await flush(); const full = createSessionStore(client.db).loadFull(sessionId); expect(full?.session.agentSnapshot?.model).toBe('claude-opus-4-8'); + expect(full?.session.agentSnapshot?.reasoning_effort).toBe('medium'); // the effort tier persisted onto the snapshot expect(full?.messages.map((m) => m.role)).toEqual(['user', 'assistant']); // the single carried exchange props.controller.handleKey('c', CTRL_C); // Home Ctrl-C ⇒ clean exit diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index e6c2f573..20011f39 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -293,6 +293,8 @@ export async function driveHome(deps: HomeDeps): Promise { shouldStop, stopReason, sessionId: built.sessionId, + // The bound effort (ADR-0066) for the `/models` effort sub-list ✓/highlight + the reseat no-op guard. + boundEffort: built.agent.reasoning_effort, teardown, onAbort, onModeChange, @@ -351,7 +353,12 @@ export async function driveHome(deps: HomeDeps): Promise { `cannot switch model: session ${sessionId} could not be reloaded for reseat`, ); } - const newAgent = swapAgentModel(loaded.session.agentSnapshot, target.modelId, target.provider); + const newAgent = swapAgentModel( + loaded.session.agentSnapshot, + target.modelId, + target.provider, + target.reasoningEffort, + ); const record: AgentSessionRecord = { ...loaded.session, agentSnapshot: newAgent }; const resolvePrice = readUserPricingOverlay(opened.db); const built = await (deps.buildResumedSession ?? buildResumedChatSession)({ diff --git a/apps/cli/src/render/tui/chat-ink.tsx b/apps/cli/src/render/tui/chat-ink.tsx index 7642155f..acddce6f 100644 --- a/apps/cli/src/render/tui/chat-ink.tsx +++ b/apps/cli/src/render/tui/chat-ink.tsx @@ -578,6 +578,12 @@ export function ChatApp(props: Readonly): ReactElement { refreshedAt: view.refreshedAt, banner: undefined, hint: undefined, + // The standalone `relavium chat` picker is ALWAYS a live reseat, so it offers the effort sub-step (ADR-0066). + phase: 'model', + effortStep: true, + pending: undefined, + effortSelected: 0, + currentEffort: port.boundEffort, }); runPickerRefresh(() => port.refreshIfStale()); }; @@ -593,16 +599,21 @@ export function ChatApp(props: Readonly): ReactElement { applyModelPicker(undefined); return; case 'accept': - // No-op guard (ADR-0059): accepting the ALREADY-bound model (the ✓ = `currentDefault` = the session's bound - // model) would reseat for zero change — wiping the ADR-0057 approval cache + reconnecting MCP. Close with a - // note instead of a pointless switch. - if (step.modelId === open.currentDefault) { + // No-op guard (ADR-0059/ADR-0066): accepting the ALREADY-bound model AND effort (the ✓ = `currentDefault` + + // `currentEffort` = the session's binding) would reseat for zero change — wiping the ADR-0057 approval cache + // + reconnecting MCP. The same model with a DIFFERENT effort IS a real (effort-only) switch, so the guard is + // keyed on BOTH axes. Close with a note instead of a pointless switch. + if (step.modelId === open.currentDefault && step.reasoningEffort === open.currentEffort) { applyModelPicker(undefined); props.store.note(`Already on ${step.displayName}.`); return; } applyModelPicker(undefined); - props.onReseat?.({ modelId: step.modelId, provider: step.provider }); + props.onReseat?.({ + modelId: step.modelId, + provider: step.provider, + ...(step.reasoningEffort === undefined ? {} : { reasoningEffort: step.reasoningEffort }), + }); props.onExit(); // the reseat set the stop state; end the loop so runReplLoop swaps in the new-model session return; case 'blocked': { diff --git a/apps/cli/src/render/tui/home-controller.test.ts b/apps/cli/src/render/tui/home-controller.test.ts index bd9736f5..fbc79d91 100644 --- a/apps/cli/src/render/tui/home-controller.test.ts +++ b/apps/cli/src/render/tui/home-controller.test.ts @@ -12,6 +12,7 @@ import { } from './home-controller.js'; import type { UserCommandOutcome } from '@relavium/core'; import type { ModelCatalogEntry } from '@relavium/llm'; +import type { ReasoningEffort } from '@relavium/shared'; import type { RefreshReport } from '../../engine/model-refresh.js'; @@ -62,6 +63,7 @@ function makeSession( onModeChange?: (mode: ChatMode) => void; mentionReader?: MentionReader; runShellCommand?: (command: string, args: readonly string[]) => Promise; + boundEffort?: ReasoningEffort; } = {}, ): { session: HomeChatSession; @@ -82,6 +84,7 @@ function makeSession( }, shouldStop: opts.stop ?? (() => false), stopReason: opts.stopReason ?? (() => 'exit'), + boundEffort: opts.boundEffort, ...(opts.onAbort === undefined ? {} : { onAbort: opts.onAbort }), ...(opts.onModeChange === undefined ? {} : { onModeChange: opts.onModeChange }), ...(opts.mentionReader === undefined ? {} : { mentionReader: opts.mentionReader }), @@ -912,6 +915,7 @@ describe('createHomeController (2.5.B lifecycle / ADR-0054)', () => { processLine: () => Promise.resolve(), shouldStop: () => true, // the first turn ends the session ⇒ endChat fires stopReason: () => 'exit', // /exit-style end → endChat (not the /clear swap) + boundEffort: undefined, teardown, }; const startChat = vi.fn(() => Promise.resolve(session)); @@ -948,6 +952,7 @@ describe('createHomeController (2.5.B lifecycle / ADR-0054)', () => { }, shouldStop: () => true, stopReason: () => 'exit', + boundEffort: undefined, teardown: vi.fn(() => new Promise((r) => (releaseTeardown = r))), }; const c = createHomeController({ @@ -1600,6 +1605,7 @@ function pickerEntry( priceKnown: true, available: true, deprecated: false, + supportsReasoning: false, ...partial, }; } diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index 0842a697..90a355da 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -1,5 +1,6 @@ import type { UserCommandOutcome } from '@relavium/core'; import type { ModelCatalogEntry } from '@relavium/llm'; +import type { ReasoningEffort } from '@relavium/shared'; import { CHAT_PALETTE_COMMANDS, @@ -96,6 +97,10 @@ export interface HomeChatSession { /** The session's durable id (ADR-0062 §7) — named in the `/clear` notice as the prior (still-resumable) * conversation, so it is discoverable after the swap. */ readonly sessionId: string; + /** The session's currently-bound reasoning-effort tier (ADR-0066), from the bound agent's `reasoning_effort` — + * the `/models` effort sub-list's `✓` + opening highlight, and the reseat no-op guard's second axis (same model + * AND same effort ⇒ no-op). `undefined` ⇒ no effort bound (the provider default). */ + readonly boundEffort: ReasoningEffort | undefined; /** WHY `shouldStop()` became true (ADR-0062 §7 · ADR-0059) — `'clear'` (swap in a fresh session, staying in chat) * vs `'exit'` (`/exit`/`/cancel`, return to the bare Home). Shares the widened `ChatLineHandler.stopReason` type, * so it also carries `'reseat'`; but the in-Home chat does NOT yet wire `onReseat` (its `/models` is the Home's @@ -618,7 +623,11 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { pickerEpoch += 1; // a fresh generation — invalidates any in-flight refresh from a prior (closed) open // The `✓` marker: in a LIVE chat (ADR-0059 reseat) it is the session's BOUND model (the "you are here"); at the // bare Home it is the effective next-session default. The accept action mirrors this (reseat vs default-write). - const activeModel = state.session?.store.getSnapshot().state.model; + const active = state.session; + const activeModel = active?.store.getSnapshot().state.model; + // The effort sub-step (ADR-0066) is offered ONLY on a LIVE reseat (an active in-Home chat with a reseat builder + // wired) — the bare-Home default-write persists only the model (ADR-0063), so it stays single-phase. + const effortStep = active !== undefined && deps.reseatChat !== undefined; set({ notice: undefined, // opening the picker clears any stale /doctor report behind it modelPicker: { @@ -630,6 +639,11 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { refreshedAt: view.refreshedAt, banner: undefined, hint: undefined, + phase: 'model', + effortStep, + pending: undefined, + effortSelected: 0, + currentEffort: effortStep ? active.boundEffort : undefined, }, }); // Render the cache immediately (above), then kick a TTL-bounded background refresh (ADR-0064 §5c) — the Home is @@ -639,13 +653,19 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { // Accept the chosen model. TWO surface-specific actions off the ONE picker (ADR-0059/ADR-0063): // - a LIVE in-Home chat ⇒ RESEAT it onto the picked model (mirrors the standalone `relavium chat` /models reseat); // - the BARE Home ⇒ persist the chosen model as the NEXT session's default (the pre-existing behavior below). - const acceptModel = (modelId: string, displayName: string, provider: ReseatTarget['provider']): void => { + const acceptModel = ( + modelId: string, + displayName: string, + provider: ReseatTarget['provider'], + reasoningEffort?: ReasoningEffort, + ): void => { const active = state.session; if (active !== undefined && deps.reseatChat !== undefined) { - // No-op guard (ADR-0059): accepting the ALREADY-bound model would tear the session down + rebuild for zero - // change — wiping the ADR-0057 per-tool approval cache, reconnecting MCP, and showing a misleading "Switched" - // notice. Keep the picker open with a hint instead so a mis-click doesn't churn the live session. - if (modelId === active.store.getSnapshot().state.model) { + // No-op guard (ADR-0059/ADR-0066): accepting the ALREADY-bound model AND effort would tear the session down + + // rebuild for zero change — wiping the ADR-0057 per-tool approval cache, reconnecting MCP, and showing a + // misleading "Switched" notice. The same model with a DIFFERENT effort IS a real (effort-only) switch, so the + // guard is keyed on BOTH axes. Keep the picker open with a hint instead so a mis-click doesn't churn the session. + if (modelId === active.store.getSnapshot().state.model && reasoningEffort === active.boundEffort) { const open = state.modelPicker; set( open === undefined @@ -654,7 +674,11 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { ); return; } - reseatChat(active, { modelId, provider }); + reseatChat(active, { + modelId, + provider, + ...(reasoningEffort === undefined ? {} : { reasoningEffort }), + }); return; } // ---- The bare-Home next-session-default write (ADR-0063) --------------------------------------------------- @@ -698,7 +722,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { set({ modelPicker: undefined }); break; case 'accept': - acceptModel(step.modelId, step.displayName, step.provider); + acceptModel(step.modelId, step.displayName, step.provider, step.reasoningEffort); break; case 'blocked': { // An ACTIONABLE hint (2.5.G key-awareness): a keyless provider names the remedy; the pre-existing diff --git a/apps/cli/src/render/tui/model-picker-view.tsx b/apps/cli/src/render/tui/model-picker-view.tsx index 8f55ea91..0706bf97 100644 --- a/apps/cli/src/render/tui/model-picker-view.tsx +++ b/apps/cli/src/render/tui/model-picker-view.tsx @@ -1,8 +1,11 @@ import { Box, Text } from 'ink'; import type { ReactElement, ReactNode } from 'react'; +import { REASONING_EFFORTS } from '@relavium/shared'; + import { sanitizeInline } from './chat-projection.js'; import { + EFFORT_TIER_HINT, formatContextWindow, formatModelPrice, formatRefreshedBadge, @@ -41,8 +44,44 @@ export function modelWindow(count: number, selected: number): { start: number; e return { start, end: start + MODEL_WINDOW }; } +/** + * The `'effort'` sub-list (ADR-0066) — the reasoning-effort tiers for the model chosen in the `'model'` phase. A + * fixed five-row list (no scroll window needed): each tier + its one-line hint, the highlighted row in cyan, a `✓` + * on the session's currently-bound effort. The `pending` model's name (a provider-controlled string) is sanitized + * at this display boundary, exactly as the model rows are. + */ +function EffortSubList(props: Readonly<{ state: ModelPickerState; color: boolean }>): ReactElement { + const { state, color } = props; + const selected = Math.max(0, Math.min(state.effortSelected, REASONING_EFFORTS.length - 1)); + const forModel = state.pending === undefined ? '' : ` · ${sanitizeInline(state.pending.displayName)}`; + return ( + + + Reasoning effort + {forModel} + + {REASONING_EFFORTS.map((effort, index) => { + const isSelected = index === selected; + const isCurrent = effort === state.currentEffort; + const rowColor = isSelected ? colorProps(color, 'cyan') : {}; + return ( + + {`${isSelected ? '›' : ' '} ${isCurrent ? '✓' : ' '} ${effort} · `} + {EFFORT_TIER_HINT[effort]} + + ); + })} + + ↑/↓ select · Enter apply · Esc back + + + ); +} + export function ModelPickerView(props: Readonly): ReactElement { const { state, color, nowMs } = props; + // The effort sub-step (ADR-0066) owns the whole overlay while active — a fixed tier list, no catalog/filter/badge. + if (state.phase === 'effort') return ; const visible = visibleModels(state); // Clamp the highlight for display — a refresh can shrink the list under a `selected` past the new end until the // next keystroke re-clamps (foldModelPickerKey clamps on move). diff --git a/apps/cli/src/render/tui/model-picker.test.ts b/apps/cli/src/render/tui/model-picker.test.ts index e64aa324..dfac060b 100644 --- a/apps/cli/src/render/tui/model-picker.test.ts +++ b/apps/cli/src/render/tui/model-picker.test.ts @@ -20,6 +20,7 @@ function entry(partial: Partial & Pick = {}): ModelPickerState { refreshedAt: undefined, banner: undefined, hint: undefined, + phase: 'model', + effortStep: false, + pending: undefined, + effortSelected: 0, + currentEffort: undefined, ...partial, }; } @@ -138,6 +144,124 @@ describe('foldModelPickerKey', () => { }); }); +describe('foldModelPickerKey — the ADR-0066 effort sub-step', () => { + // A reasoning-capable catalog whose first entry (empty filter ⇒ selected 0) supports reasoning. + const reasoningState = (partial: Partial = {}): ModelPickerState => + state({ + entries: [ + entry({ modelId: 'claude-opus-4-8', displayName: 'Opus', supportsReasoning: true }), + entry({ modelId: 'deepseek-chat', displayName: 'DeepSeek', supportsReasoning: false }), + ], + effortStep: true, + ...partial, + }); + + it('accepts IMMEDIATELY (no effort sub-step) when the surface does not offer it (effortStep=false)', () => { + // The bare-Home default-write path: even a reasoning-capable model accepts straight to a model-only write. + const step = foldModelPickerKey('', { return: true }, reasoningState({ effortStep: false })); + expect(step).toEqual({ + kind: 'accept', + modelId: 'claude-opus-4-8', + displayName: 'Opus', + provider: 'anthropic', + }); + expect('reasoningEffort' in step).toBe(false); // no tier on a non-effort surface + }); + + it('accepts IMMEDIATELY when the chosen model does NOT support reasoning (even on an effort surface)', () => { + const step = foldModelPickerKey('', { return: true }, reasoningState({ selected: 1 })); // deepseek (no reasoning) + expect(step).toEqual({ + kind: 'accept', + modelId: 'deepseek-chat', + displayName: 'DeepSeek', + provider: 'anthropic', + }); + }); + + it('a reasoning model on an effort surface ADVANCES to the effort phase (not an immediate accept)', () => { + const step = foldModelPickerKey('', { return: true }, reasoningState()); + expect(step.kind).toBe('state'); + if (step.kind !== 'state') throw new Error('expected a state step'); + expect(step.state.phase).toBe('effort'); + expect(step.state.pending).toEqual({ + modelId: 'claude-opus-4-8', + displayName: 'Opus', + provider: 'anthropic', + }); + // No bound effort ⇒ the sub-list opens on the neutral middle tier ('medium', index 2 of off/low/medium/high/max). + expect(step.state.effortSelected).toBe(2); + }); + + it('opens the effort sub-list on the session BOUND effort when one is set', () => { + const step = foldModelPickerKey('', { return: true }, reasoningState({ currentEffort: 'high' })); + if (step.kind !== 'state') throw new Error('expected a state step'); + expect(step.state.effortSelected).toBe(3); // index of 'high' + }); + + // A picker already parked in the effort phase over a pending model (selected tier = 'medium'). + const effortPhase = (partial: Partial = {}): ModelPickerState => + reasoningState({ + phase: 'effort', + pending: { modelId: 'claude-opus-4-8', displayName: 'Opus', provider: 'anthropic' }, + effortSelected: 2, + ...partial, + }); + + it('↑/↓ move the effort selection, clamped to the tier list', () => { + expect(foldModelPickerKey('', { downArrow: true }, effortPhase())).toEqual({ + kind: 'state', + state: effortPhase({ effortSelected: 3 }), + }); + expect(foldModelPickerKey('', { upArrow: true }, effortPhase())).toEqual({ + kind: 'state', + state: effortPhase({ effortSelected: 1 }), + }); + // Clamp at the ends: max (index 4) Down stays 4; off (index 0) Up stays 0. + expect(foldModelPickerKey('', { downArrow: true }, effortPhase({ effortSelected: 4 }))).toEqual({ + kind: 'state', + state: effortPhase({ effortSelected: 4 }), + }); + expect(foldModelPickerKey('', { upArrow: true }, effortPhase({ effortSelected: 0 }))).toEqual({ + kind: 'state', + state: effortPhase({ effortSelected: 0 }), + }); + }); + + it('Enter in the effort phase accepts the pending model + the highlighted tier', () => { + expect(foldModelPickerKey('', { return: true }, effortPhase({ effortSelected: 3 }))).toEqual({ + kind: 'accept', + modelId: 'claude-opus-4-8', + displayName: 'Opus', + provider: 'anthropic', + reasoningEffort: 'high', + }); + // The lowest tier 'off' is a valid pick (disables reasoning), not a "cancel". + expect(foldModelPickerKey('', { return: true }, effortPhase({ effortSelected: 0 }))).toEqual({ + kind: 'accept', + modelId: 'claude-opus-4-8', + displayName: 'Opus', + provider: 'anthropic', + reasoningEffort: 'off', + }); + }); + + it('Esc in the effort phase BACKS OUT to the model list (clears pending) — it does NOT cancel the picker', () => { + const step = foldModelPickerKey('', { escape: true }, effortPhase()); + expect(step).toEqual({ kind: 'state', state: effortPhase({ phase: 'model', pending: undefined }) }); + }); + + it('Ctrl-C in the effort phase is the HARD cancel (closes the whole picker)', () => { + expect(foldModelPickerKey('c', { ctrl: true }, effortPhase()).kind).toBe('close'); + }); + + it('filter / other keys are inert in the fixed effort list', () => { + const s = effortPhase(); + expect(foldModelPickerKey('x', {}, s)).toEqual({ kind: 'state', state: s }); // typing does not filter + expect(foldModelPickerKey('', { backspace: true }, s)).toEqual({ kind: 'state', state: s }); + expect(foldModelPickerKey('r', { ctrl: true }, s)).toEqual({ kind: 'state', state: s }); // no refresh here + }); +}); + describe('visibleModels', () => { it('filters by display name, model id, OR provider (case-insensitive)', () => { const s = state({ diff --git a/apps/cli/src/render/tui/model-picker.ts b/apps/cli/src/render/tui/model-picker.ts index b5983322..bbfaec05 100644 --- a/apps/cli/src/render/tui/model-picker.ts +++ b/apps/cli/src/render/tui/model-picker.ts @@ -1,4 +1,5 @@ import type { ModelCatalogEntry, ProviderId } from '@relavium/llm'; +import { REASONING_EFFORTS, type ReasoningEffort } from '@relavium/shared'; import { dropLastCodePoint } from './chat-input.js'; @@ -39,6 +40,31 @@ export interface ModelPickerState { readonly refreshedAt: number | undefined; readonly banner: string | undefined; readonly hint: string | undefined; + /** + * The picker's TWO-PHASE step ([ADR-0066](../../../../../docs/decisions/0066-normalized-reasoning-effort-control.md)): + * `'model'` is the catalog list (the default); accepting a reasoning-capable model on a reseat surface + * ({@link effortStep}) advances to `'effort'` — a fixed sub-list of the reasoning-effort tiers for the chosen + * model. The two surfaces route the SAME fold, so the phase transition lives here, not in either host. + */ + readonly phase: 'model' | 'effort'; + /** + * Whether this picker offers the reasoning-effort sub-step. `true` for a LIVE reseat surface (standalone + * `relavium chat` + the in-Home live chat, where the effort binds onto the reseated agent, ADR-0059); `false` + * for the bare-Home next-session-default write (which persists only the model, ADR-0063 — the effort default is + * the `[chat].reasoning_effort` config key, not a per-write pick), so a non-reseat surface stays single-phase. + */ + readonly effortStep: boolean; + /** The model chosen in `'model'` phase, awaiting an effort pick — carried so `'effort'`'s accept emits the pair. */ + readonly pending: { + readonly modelId: string; + readonly displayName: string; + readonly provider: ProviderId; + } | undefined; + /** The highlighted index into {@link REASONING_EFFORTS} while in `'effort'` phase. */ + readonly effortSelected: number; + /** The session's currently-bound effort (the `✓` in the effort sub-list + the initial highlight); `undefined` ⇒ + * no effort bound (the provider default), so the sub-list opens on a neutral middle tier. */ + readonly currentEffort: ReasoningEffort | undefined; } /** The minimal key fields the picker fold reads (a structural subset of ink's `Key`). */ @@ -59,11 +85,15 @@ export type ModelPickerStep = | { readonly kind: 'close' } // Esc / Ctrl-C — cancel without acting // Accept the selected model. `provider` rides along (the entry is authoritative) so the chat reseat (ADR-0059) // has its `{ modelId, provider }` target; the Home's default-write reads only `modelId`/`displayName`. + // `reasoningEffort` is present iff the effort sub-step ran (a reasoning-capable model on a reseat surface, + // ADR-0066); absent ⇒ the reseat drops any prior effort (a non-reasoning model can't use one) / the default-write + // ignores it. | { readonly kind: 'accept'; readonly modelId: string; readonly displayName: string; readonly provider: ProviderId; + readonly reasoningEffort?: ReasoningEffort; } | { readonly kind: 'blocked'; // a dimmed/unavailable model — non-selectable (ADR-0064 §6) @@ -109,21 +139,36 @@ function foldArrow( } /** - * Fold one keystroke into the open picker (the keyboard-owning contract, mirroring the mention submode): - * `Esc`/`Ctrl-C` cancels (no write); `Ctrl+R` refreshes; `↑`/`↓` move; `Enter` accepts the selected model — a - * DIMMED (unavailable) model yields `blocked` (non-selectable, ADR §6), an empty list closes; backspace trims the - * filter; a single printable code point extends the filter (a multi-char paste blob is dropped, matching the other - * submodes); every other key stays open. + * Fold one keystroke into the open picker (the keyboard-owning contract, mirroring the mention submode). Two phases + * (ADR-0066): `'model'` (the catalog) delegates to {@link foldModelPhaseKey}; `'effort'` (the reasoning-effort + * sub-list) delegates to {@link foldEffortPhaseKey}. `Ctrl-C` is the hard cancel from EITHER phase (nothing written). */ export function foldModelPickerKey( char: string, key: ModelPickerKey, state: ModelPickerState, ): ModelPickerStep { - // Esc / Ctrl-C cancels the picker (nothing is written — a cancel never changes the default). - if (key.escape === true || (key.ctrl === true && char === 'c')) { - return { kind: 'close' }; - } + // Ctrl-C is the hard cancel from any phase (nothing is written). Esc is phase-scoped: it cancels the model list + // but only backs OUT of the effort sub-list to the model list (handled in foldEffortPhaseKey). + if (key.ctrl === true && char === 'c') return { kind: 'close' }; + return state.phase === 'effort' + ? foldEffortPhaseKey(char, key, state) + : foldModelPhaseKey(char, key, state); +} + +/** + * The `'model'` phase fold: `Esc` cancels (no write); `Ctrl+R` refreshes; `↑`/`↓` move; `Enter` accepts the selected + * model — a DIMMED (unavailable) model yields `blocked` (non-selectable, ADR §6), an empty list closes; a + * reasoning-capable model on a reseat surface (`effortStep`) instead advances to the `'effort'` sub-step (ADR-0066); + * backspace trims the filter; a single printable code point extends the filter (a multi-char paste blob is dropped); + * every other key stays open. + */ +function foldModelPhaseKey( + char: string, + key: ModelPickerKey, + state: ModelPickerState, +): ModelPickerStep { + if (key.escape === true) return { kind: 'close' }; // Ctrl+R forces a live refresh (distinct from the auto TTL refresh on open). `r` alone extends the filter. // Ignored while a refresh is already in flight (`loading`) — so two rapid Ctrl+R can't race two refreshes whose // out-of-order completion would flash a stale banner over a fresher one. Stays open, unchanged. @@ -144,6 +189,24 @@ export function foldModelPickerKey( ...(chosen.unavailableReason !== undefined ? { reason: chosen.unavailableReason } : {}), }; } + // A reasoning-capable model on a reseat surface advances to the effort sub-step (ADR-0066) instead of accepting + // immediately; the sub-list opens on the session's bound effort (else a neutral middle tier). + if (state.effortStep && chosen.supportsReasoning) { + return { + kind: 'state', + state: { + ...state, + phase: 'effort', + pending: { + modelId: chosen.modelId, + displayName: chosen.displayName, + provider: chosen.provider, + }, + effortSelected: initialEffortIndex(state.currentEffort), + hint: undefined, + }, + }; + } return { kind: 'accept', modelId: chosen.modelId, @@ -164,6 +227,66 @@ export function foldModelPickerKey( return { kind: 'state', state }; } +/** + * The `'effort'` phase fold (ADR-0066): a fixed sub-list of the reasoning-effort tiers for the {@link ModelPickerState.pending} + * model. `Esc` backs OUT to the model list (Ctrl-C, handled by the caller, is the hard cancel); `↑`/`↓` move over + * {@link REASONING_EFFORTS}; `Enter` accepts the chosen model + the highlighted tier. There is no filter or refresh + * here (a fixed five-item list), so every other key is inert. + */ +function foldEffortPhaseKey( + char: string, + key: ModelPickerKey, + state: ModelPickerState, +): ModelPickerStep { + if (key.escape === true) { + return { kind: 'state', state: { ...state, phase: 'model', pending: undefined } }; + } + if (key.upArrow === true) { + const next = clampSelection(state.effortSelected - 1, REASONING_EFFORTS.length); + return { kind: 'state', state: { ...state, effortSelected: next } }; + } + if (key.downArrow === true) { + const next = clampSelection(state.effortSelected + 1, REASONING_EFFORTS.length); + return { kind: 'state', state: { ...state, effortSelected: next } }; + } + if (key.return === true) { + const pending = state.pending; + const effort = REASONING_EFFORTS[clampSelection(state.effortSelected, REASONING_EFFORTS.length)]; + // Defensive: a missing pending model (never expected — set on the transition) or an out-of-range tier backs out + // to the model list rather than emitting a malformed accept. + if (pending === undefined || effort === undefined) { + return { kind: 'state', state: { ...state, phase: 'model', pending: undefined } }; + } + return { + kind: 'accept', + modelId: pending.modelId, + displayName: pending.displayName, + provider: pending.provider, + reasoningEffort: effort, + }; + } + return { kind: 'state', state }; +} + +/** The effort sub-list's opening highlight: the session's bound effort, else a neutral middle tier (`'medium'`). */ +function initialEffortIndex(currentEffort: ReasoningEffort | undefined): number { + const target = currentEffort ?? 'medium'; + const index = REASONING_EFFORTS.indexOf(target); + return index < 0 ? 0 : index; +} + +/** + * The one-line hint shown beside each reasoning-effort tier in the effort sub-list (ADR-0066). Display-only, so the + * picker explains what each tier trades off (latency/cost vs depth) without the user consulting the docs. + */ +export const EFFORT_TIER_HINT: Record = { + off: 'no reasoning — fastest, lowest cost', + low: 'brief reasoning', + medium: 'balanced reasoning', + high: 'deep reasoning', + max: 'maximum reasoning — slowest, highest cost', +}; + /* -------------------------------------------------------------------------------------------------- * * Pure display formatters (unit-tested; the ink view is not render-tested, per the repo convention). * -------------------------------------------------------------------------------------------------- */ diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index 20dcc31e..ce8009e1 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -152,7 +152,7 @@ The **command manifest** is the one source the **shell** command surfaces derive ### In-REPL slash commands -The interactive `/` palette + slash commands inside the **Home and chat** are a SEPARATE, **curated** surface ([ADR-0056](../../decisions/0056-cli-in-app-slash-command-system-and-manifest.md) amendment, 2.5.C) — the runtime registry is `apps/cli/src/commands/repl-commands.ts` (`REPL_COMMANDS`), the single source for the palette, the `/help` list, and the unknown-slash hint. It surfaces only the commands that make sense in a live REPL — lifecycle (`/exit`, `/cancel`, `/export`, `/clear`), info/discovery (`/help`, `/workflows`, `/cost`, `/doctor`), and — in a chat — `/mode ` to switch the chat mode (2.5.E, [chat-session.md](chat-session.md)) plus the ADR-0062 context commands (`/compact`, `/trim`). In the **Home** (bare `relavium`) — and there only — `/models` opens an in-tree model picker over the merged live/static catalog and, on selection, writes the **next** session's default model ([ADR-0064](../../decisions/0064-live-model-catalog.md) §10, via the [ADR-0063](../../decisions/0063-cli-config-write-contract.md) config writer); it is **home-only** (a next-session config action, deliberately distinct from the Phase-2.6 mid-chat `/models` live reseat, [ADR-0059](../../decisions/0059-cli-mid-session-model-reseat.md)) — typed inside a chat it is rejected with a pointer to the Home. The picker renders the ADR-0064 first-class UX: per-model pricing (an unpriced model shows a "cost cap will not apply" hint), a dimmed non-selectable "unavailable on your key" row, a `deprecated` flag, a loading spinner, a per-provider partial-failure banner, and a "last updated" freshness badge; opening it over an empty/stale cache renders immediately and kicks a background refresh (the long-lived Home is where that is sound), `Ctrl+R` forces a live refresh, and `Esc` cancels. The heavy, session-starting shell commands above (`run`, `chat`, `provider`, …) are **never** in-REPL slashes — they stay shell-only (`relavium …`). A bare `/` at an **empty** prompt opens the filterable palette (the footer hint-bar surfaces `/ for commands` exactly there, 2.5.C S6); an unknown slash — or an undeclared argument on a known command (`/exit now`) — prints a sanitized, secret-free hint. A command may declare flags (`/doctor --deep`) or a single positional value (`/mode plan`); the palette runs the bare form, so a flag/value is opt-in by typing it. There is no separate `/shortcuts` command — the palette renders its own nav hints (`↑/↓ · Enter · Esc`) and the footer surfaces `/`, so keys stay discoverable in context. +The interactive `/` palette + slash commands inside the **Home and chat** are a SEPARATE, **curated** surface ([ADR-0056](../../decisions/0056-cli-in-app-slash-command-system-and-manifest.md) amendment, 2.5.C) — the runtime registry is `apps/cli/src/commands/repl-commands.ts` (`REPL_COMMANDS`), the single source for the palette, the `/help` list, and the unknown-slash hint. It surfaces only the commands that make sense in a live REPL — lifecycle (`/exit`, `/cancel`, `/export`, `/clear`), info/discovery (`/help`, `/workflows`, `/cost`, `/doctor`), and — in a chat — `/mode ` to switch the chat mode (2.5.E, [chat-session.md](chat-session.md)) plus the ADR-0062 context commands (`/compact`, `/trim`). `/models` opens an in-tree model picker over the merged live/static catalog; its **action depends on the surface**. Inside a **live chat** (standalone `relavium chat` or an in-Home chat) it triggers a **live reseat** ([ADR-0059](../../decisions/0059-cli-mid-session-model-reseat.md)) — rebinding the session to the picked model (dropping the old fallback chain), carrying the text-only transcript + cumulative cost/turns under the SAME `sessionId`. At the **bare Home** (no active chat) it instead writes the **next** session's default model ([ADR-0064](../../decisions/0064-live-model-catalog.md) §10, via the [ADR-0063](../../decisions/0063-cli-config-write-contract.md) config writer). On a **reasoning-capable** model *during a reseat*, a second **effort sub-step** ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md)) picks the reasoning-effort tier (`off`/`low`/`medium`/`high`/`max`) bound onto the reseated agent (`Esc` backs out to the model list, `Ctrl-C` cancels); the bare-Home default-write stays single-step (its effort default is the `[chat].reasoning_effort` config key). Under `--json`/plain (non-TTY) the live reseat is unavailable — one machine stream stays one session lifecycle ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)) — so a typed `/models` there falls through to an actionable "interactive terminal" hint. The picker renders the ADR-0064 first-class UX: per-model pricing (an unpriced model shows a "cost cap will not apply" hint), a dimmed non-selectable "unavailable on your key" row, a `deprecated` flag, a loading spinner, a per-provider partial-failure banner, and a "last updated" freshness badge; opening it over an empty/stale cache renders immediately and kicks a background refresh (the long-lived Home is where that is sound), `Ctrl+R` forces a live refresh, and `Esc` cancels. The heavy, session-starting shell commands above (`run`, `chat`, `provider`, …) are **never** in-REPL slashes — they stay shell-only (`relavium …`). A bare `/` at an **empty** prompt opens the filterable palette (the footer hint-bar surfaces `/ for commands` exactly there, 2.5.C S6); an unknown slash — or an undeclared argument on a known command (`/exit now`) — prints a sanitized, secret-free hint. A command may declare flags (`/doctor --deep`) or a single positional value (`/mode plan`); the palette runs the bare form, so a flag/value is opt-in by typing it. There is no separate `/shortcuts` command — the palette renders its own nav hints (`↑/↓ · Enter · Esc`) and the footer surfaces `/`, so keys stay discoverable in context. ### `relavium run` diff --git a/docs/reference/contracts/config-spec.md b/docs/reference/contracts/config-spec.md index 84e4699e..f18676b8 100644 --- a/docs/reference/contracts/config-spec.md +++ b/docs/reference/contracts/config-spec.md @@ -125,6 +125,7 @@ focus_area = "security and type safety" [chat] # agent-session (chat-mode) defaults — see contracts/agent-session-spec.md default_model = "claude-sonnet-4-6" # model for a chat session that names none; absent at every [chat] layer ⇒ falls back to global [preferences].default_model (ADR-0063) +reasoning_effort = "medium" # ADR-0066: reasoning-effort tier baked onto the DEFAULT chat agent — off | low | medium | high | max; absent ⇒ no reasoning control (the provider default). Ignored on a model without a controllable reasoning tier. fs_scope = "sandboxed" # SAME tier enum as [defaults].fs_scope above (not re-listed here) max_turns = 50 # hard session TURN cap → SessionDeps.maxTurns (DoS fail-safe; absent ⇒ engine default 50; positiveInt — 0 is rejected here) — DISTINCT from max_messages max_messages = 200 # history-trim threshold — consumed by `/trim` + auto-compaction (ADR-0062); older turns trimmed/summarized @@ -154,6 +155,8 @@ allowed_command_globs = [] # opt-in glob form of the !-shell allowlist ( > > `max_messages` (revived in 2.5.F) is the bound `/trim` enforces (keep the last N messages, no LLM call) and the deterministic fallback if a summarization fails. `auto_compact` + `compact_threshold` ([ADR-0062](../../decisions/0062-context-compaction-and-cli-history-commands.md)) drive **automatic** model-summarised compaction: after a turn completes, if its **real** input tokens exceed `compact_threshold` (default `0.8`, a fraction in (0, 1]) × the serving model's context window, the session compacts before the next turn. `auto_compact` absent ⇒ enabled; a model with no known context window (a custom base-URL id) skips auto-compaction, but manual `/compact` still works. The summarization spend is accounted to the session budget and surfaced, never silent. > +> `reasoning_effort` ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md)) is the normalized reasoning-effort tier — `off | low | medium | high | max` — baked onto the **built-in default chat agent** only (an explicit `--agent` owns its own `reasoning_effort` in its YAML). It resolves per-field project → workspace like the other `[chat]` keys (no global-layer fallback — that extra fallback is `default_model`'s alone). Each adapter maps the tier to its provider's **native** control; a model with no controllable reasoning tier ignores it (the engine gates on the model's capability). Absent ⇒ no reasoning control (the provider default). Interactively, the `/models` picker's **effort sub-step** binds a tier onto a live reseat without editing config. +> > The `[chat]` block resolves **per field** (each key independently, last-writer-wins project → workspace) — a project that sets only `max_turns` still inherits `default_model`/`max_messages` from the workspace layer. (Contrast `[defaults].media_cost_estimate`, which resolves **whole-object**: the highest layer present replaces the table outright.) **`default_model` has one extra fallback**: absent at both `[chat]` layers, it falls through to the global **`[preferences].default_model`** ([ADR-0063](../../decisions/0063-cli-config-write-contract.md) §1) — the write target of `/models` and the wizard — so a user's "preferred model everywhere" governs chat too, exactly as it already governs a workflow's `[defaults].model`. Full precedence: `[chat].default_model` (project → workspace) → `[preferences].default_model` (global). No other `[chat]` field reads the global layer. The `!`-shell allowlist is the **one exception**: `allowed_commands` (exact) + `allowed_command_globs` (globs) are a **coupled unit**, so a project that sets **either** array owns the **whole** allowlist and does **not** inherit the other array from the workspace. Otherwise a project narrowing `allowed_commands` would silently keep the workspace's broader globs — lock to `git status`, yet still allow `git push` via an inherited `git *`. Only when a project sets **neither** allowlist array do both fall through to the workspace; a present array otherwise REPLACES (never merges) the lower layer's. This is what guarantees a narrower project can never inherit a broader workspace entry. > > `allowed_commands` / `allowed_command_globs` gate the **`!`-shell escape** (2.5.D, [ADR-0061](../../decisions/0061-cli-input-layer-file-injection-and-shell-escape.md)) — a chat user typing `!command` runs it through the **one** `run_command` boundary (they map to the engine's camelCase `allowedCommands` / `allowedCommandGlobs`, the SAME allowlist a workflow `run_command` uses). `allowed_commands` is **exact full-command-string** match (`git status`, `ls -la` — `git` never authorizes `git push --force`); `allowed_command_globs` is the opt-in, riskier pattern form. **Both default to EMPTY ⇒ `!`-shell is disabled** — the `empty ⇒ disabled` symmetry [security-review.md](../../standards/security-review.md) pins, with **no chat-specific relaxation** (there is no curated default: `run_command` has no argument/file confidentiality floor, so even a "read-only" default set — `cat`, `grep` — would reopen `!cat .env` → provider). `!`-shell is first-class via a first-class **opt-in** (the user lists commands, or the 2.5.G onboarding offers a reviewed seed), and a non-allowlisted `!cmd` gets an **actionable, secret-free deny hint** naming the exact line to add. `enforcePolicy(allowedCommands)` runs **before** the mode-aware `confirmAction`, so even `auto` mode never runs a command absent from the allowlist. Editing chat `allowed_commands` is a [security-review.md](../../standards/security-review.md) trigger. diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 208228cd..2e6f1233 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -197,8 +197,11 @@ steps; the additive lanes 2.5.H / I / J run in parallel. See the maintainer questions on model/provider/wizard behavior) then landed as a sequenced plan: `/models` key-awareness, the onboarding-wizard live key-validation + retry UX, wizard-dynamic provider docs, **mid-session model switching** (the `/models` reseat across `relavium chat` + the in-Home chat — [ADR-0059](../decisions/0059-cli-mid-session-model-reseat.md), -now Accepted), and **normalized reasoning-effort control** (a provider-agnostic effort tier authored in agent YAML + -chosen in the picker — [ADR-0066](../decisions/0066-normalized-reasoning-effort-control.md)). +now Accepted), and **normalized reasoning-effort control** ([ADR-0066](../decisions/0066-normalized-reasoning-effort-control.md), +now Accepted): a provider-agnostic effort tier (`off`/`low`/`medium`/`high`/`max`) authored in agent YAML or the +`[chat].reasoning_effort` config default, each adapter mapping it to its provider's **native** tier, gated per-model +by a host-injected capability resolver, and chosen live via the `/models` picker's **effort sub-step** on a +reasoning-capable model during a reseat. Carry-over hardening is tracked in [deferred-tasks.md](deferred-tasks.md) — Phase 2 picks items up as it first touches each file. Notable inheritances: 1.AH's host-wiring half diff --git a/packages/llm/src/model-catalog.test.ts b/packages/llm/src/model-catalog.test.ts index 15c7e9e4..2b0c56dc 100644 --- a/packages/llm/src/model-catalog.test.ts +++ b/packages/llm/src/model-catalog.test.ts @@ -44,6 +44,22 @@ describe('mergeModelCatalog (ADR-0064 §6)', () => { expect(opus?.contextWindowTokens).toBe(1_000_000); }); + it('surfaces supportsReasoning from the STATIC registry tier (ADR-0066) — true for a reasoning model, false otherwise', () => { + const entries = mergeModelCatalog({ now: BEFORE_DEEPSEEK_DEPRECATION }); + // A registry model tagged `reasoning: true` (flagship) exposes the effort-controllable capability… + expect(byId(entries, 'claude-opus-4-8')?.supportsReasoning).toBe(true); + // …a registry model NOT so tagged (DeepSeek is deferred, ADR-0066) is false — the picker skips its effort sub-step. + expect(byId(entries, 'deepseek-v4-flash')?.supportsReasoning).toBe(false); + }); + + it('a LIVE-only model (no registry tier) is never reasoning-capable — capability is a shipped-registry fact', () => { + const entries = mergeModelCatalog({ + live: liveMap([['openai', [{ id: 'gpt-6-preview', displayName: 'GPT-6 preview' }]]]), + now: BEFORE_DEEPSEEK_DEPRECATION, + }); + expect(byId(entries, 'gpt-6-preview')?.supportsReasoning).toBe(false); // not inferred from a discovery listing + }); + it('availability: a static model NOT in a CONNECTED provider live list is dimmed, one present is available', () => { const entries = mergeModelCatalog({ live: liveMap([['anthropic', [{ id: 'claude-opus-4-8' }]]]), diff --git a/packages/llm/src/model-catalog.ts b/packages/llm/src/model-catalog.ts index 6f751939..79984998 100644 --- a/packages/llm/src/model-catalog.ts +++ b/packages/llm/src/model-catalog.ts @@ -52,6 +52,14 @@ export interface ModelCatalogEntry { readonly deprecated: boolean; /** The effective ISO deprecation date — the earlier of the static and live dates (their union). */ readonly deprecatedAt?: string; + /** + * Whether the model exposes a controllable reasoning-effort tier ([ADR-0066](../../../docs/decisions/0066-normalized-reasoning-effort-control.md)). + * Sourced from the STATIC registry only (`MODEL_PRICING[id].reasoning === true`) — the same authority as + * `modelSupportsReasoning`, so the picker's effort sub-step and the engine's `resolveReasoning` gate agree. A + * live-only / user-only id (no registry tier) is `false` (reasoning capability is a shipped-registry fact, never + * inferred from a discovery listing or a user price row). + */ + readonly supportsReasoning: boolean; } /** Input to {@link mergeModelCatalog} — all plain data the host resolves and passes in (keeps the merge pure). */ @@ -181,6 +189,10 @@ export function mergeModelCatalog(input: MergeModelCatalogInput): ModelCatalogEn ...(unavailableReason !== undefined ? { unavailableReason } : {}), deprecated, ...(deprecatedAt !== undefined ? { deprecatedAt } : {}), + // Reasoning capability rides the STATIC registry tier only (never live/user) — the same source as + // `modelSupportsReasoning`, so the picker's effort sub-step lights up exactly for the models the engine gate + // will actually honor (ADR-0066). + supportsReasoning: t.registry?.reasoning === true, }); } From fcb8db5921bb1c26cd225f72a1e23ee322b5d727 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 19:39:10 +0300 Subject: [PATCH 56/68] =?UTF-8?q?fix(core,cli):=202.5.E-E5=20P0=20?= =?UTF-8?q?=E2=80=94=20reasoning-effort=20as=20a=20per-turn=20session=20ov?= =?UTF-8?q?erride,=20not=20a=20reseat=20(ADR-0066=20=C2=A75/=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The comprehensive Step E review (8 lenses, adversarially verified) confirmed a BLOCKER: E5 changed reasoning-effort via the ADR-0059 model reseat — the exact mechanism ADR-0066 §5(d) REJECTS. A same-model effort change tore the session down (dropped fallback_chain, wiped the ADR-0057 approval cache, reconnected MCP, reconstructed text-only losing tool/file context, broke ADR-0039 replay) and printed a misleading "Switched to ". The §5-mandated session-level effort setter did not exist. Implement §5 as written: - core: AgentSession gains a #reasoningEffort override + setReasoningEffort() setter (mirroring setTurnPolicy) + a `reasoningEffort` getter. The per-turn build resolves `override ?? agent.reasoning_effort`, gated by the new shared `gateReasoningEffort` helper (one home; kills the agent-runner/agent-session gate duplication). A mid-session change lands on the NEXT turn — no reseat. - picker decouple: a SAME-model pick is an effort-only change → the setter (no teardown); only a DIFFERENT model reseats (carrying the effort). Both surfaces (chat-ink + home-controller); the Home effort-phase no-op now closes + notes via the store (the effort sub-list renders no hint — fixes the invisible-hint finding), the model-phase no-op keeps its browse-friendly open hint. - /effort : a first-class slash command (parity with /mode) in REPL_COMMANDS + palette + /help, chat-only, session-override (no reseat). Bare /effort shows the current tier + options. - footer: the active effort tier rides the footer next to the mode (discover- ability), shown only on a reasoning-capable model. Tracked in the chat store (one live source), so /effort + the picker + the footer never disagree and a no-reseat change is reflected without rebuilding the port (dropped the stale ChatModelsPort.boundEffort + HomeChatSession.boundEffort). - tests: AgentSession override precedence + per-model gating; Home same-model effort→setter (no reseat) + same-effort no-op; registry/palette/help updated. Refs: ADR-0066, ADR-0059, ADR-0057 Co-Authored-By: Claude --- apps/cli/src/commands/chat.ts | 117 +++++++++++++----- apps/cli/src/commands/repl-commands.test.ts | 11 +- apps/cli/src/commands/repl-commands.ts | 16 +++ apps/cli/src/home/drive-home.test.ts | 3 +- apps/cli/src/home/drive-home.tsx | 6 +- apps/cli/src/render/tui/chat-ink.tsx | 51 +++++--- apps/cli/src/render/tui/chat-projection.ts | 16 ++- apps/cli/src/render/tui/chat-store.ts | 22 +++- apps/cli/src/render/tui/home-app.tsx | 3 +- .../src/render/tui/home-controller.test.ts | 80 +++++++++++- apps/cli/src/render/tui/home-controller.ts | 53 +++++--- packages/core/src/engine/agent-runner.ts | 12 +- .../core/src/engine/agent-session.test.ts | 68 ++++++++++ packages/core/src/engine/agent-session.ts | 50 +++++++- packages/core/src/engine/reasoning-effort.ts | 20 +++ 15 files changed, 432 insertions(+), 96 deletions(-) create mode 100644 packages/core/src/engine/reasoning-effort.ts diff --git a/apps/cli/src/commands/chat.ts b/apps/cli/src/commands/chat.ts index d8442997..6746e4bf 100644 --- a/apps/cli/src/commands/chat.ts +++ b/apps/cli/src/commands/chat.ts @@ -8,8 +8,8 @@ import { type SessionStreamHandleEvent, type UserCommandOutcome, } from '@relavium/core'; -import type { ProviderId } from '@relavium/llm'; -import type { AgentSessionRecord, ReasoningEffort } from '@relavium/shared'; +import { modelSupportsReasoning, type ProviderId } from '@relavium/llm'; +import { REASONING_EFFORTS, type AgentSessionRecord, type ReasoningEffort } from '@relavium/shared'; import { exportSession } from '../chat/export.js'; import { formatDoctorReport, runDoctorChecks, type DoctorProbes } from '../chat/doctor.js'; import { assembleDoctorProbes } from '../chat/doctor-host.js'; @@ -68,6 +68,7 @@ import { stripTerminalControls, } from '../render/tui/chat-projection.js'; import { createChatStore, type ChatStoreController } from '../render/tui/chat-store.js'; +import { EFFORT_TIER_HINT } from '../render/tui/model-picker.js'; import { createMentionReader, type MentionReader } from '../render/tui/mention.js'; import { createMcpSecretResolver, type McpSecretResolver } from '../secrets/mcp-secret.js'; @@ -127,26 +128,20 @@ export interface ReseatTarget { */ export interface ChatModelsPort extends ModelCatalogPort { readonly boundModel: string; - /** - * The session's currently-bound reasoning-effort tier (ADR-0066) — the effort sub-list's `✓` "you are here" + - * its opening highlight, and (with `boundModel`) the reseat no-op guard's second axis: re-picking the same model - * AND the same effort is a no-op, but the same model with a different effort is a real (effort-only) switch. - * `undefined` ⇒ no effort bound (the provider default). - */ - readonly boundEffort: ReasoningEffort | undefined; } /** Assemble a {@link ChatModelsPort} over the session's shared db + provider resolver (the catalog the Home picker - * also reads) plus the bound model + effort — the one place the chat reseat picker's port is wired. */ + * also reads) plus the bound model — the one place the chat reseat picker's port is wired. The picker's current + * effort (the effort sub-list's ✓ / footer) reads the live chat store (ADR-0066), not the port, so a no-reseat + * `/effort` change is reflected without rebuilding the port. */ function buildChatModelsPort( opened: OpenedSessionStore, providers: ProviderResolver, boundModel: string, - boundEffort: ReasoningEffort | undefined, now: () => number, uuid: () => string, ): ChatModelsPort { - return { ...createModelCatalogPort({ db: opened.db, providers, now, uuid }), boundModel, boundEffort }; + return { ...createModelCatalogPort({ db: opened.db, providers, now, uuid }), boundModel }; } /** What an interactive driver receives — the command core's seam, so a driver never touches the session directly. */ @@ -220,6 +215,12 @@ export interface ChatDriveContext { * (cycle) + the `/mode` command to this. Absent on a driver that has no mode UI (the mode stays the default). */ readonly onModeChange?: (mode: ChatMode) => void; + /** + * Set the reasoning-effort tier ([ADR-0066](../../../../docs/decisions/0066-normalized-reasoning-effort-control.md) §5) + * — pushes the SESSION override + updates the footer on the SAME session (no reseat). The ink driver wires the + * `/effort` command + the `/models` effort sub-step to this. Absent on a driver with no effort UI. + */ + readonly onSetEffort?: (effort: ReasoningEffort) => void; /** * The `@`-mention completion reader (2.5.D, [ADR-0061](../../../../docs/decisions/0061-cli-input-layer-file-injection-and-shell-escape.md)) * — a thin wrapper over a READ-ONLY `FsCapability` jailed to the SAME fs-scope tier + workspace as the session's @@ -436,7 +437,7 @@ export async function chatCommand(args: ChatCommandArgs, deps: ChatCommandDeps): persister, doctorProbes, startSession: () => built.session.start(), - modelPicker: buildChatModelsPort(opened, providers, built.agent.model, built.agent.reasoning_effort, now, uuid), + modelPicker: buildChatModelsPort(opened, providers, built.agent.model, now, uuid), ...(config.chat.maxMessages === undefined ? {} : { chatMaxMessages: config.chat.maxMessages }), @@ -594,7 +595,7 @@ export async function chatResumeCommand( doctorProbes, startSession: () => {}, intro, - modelPicker: buildChatModelsPort(opened, providers, built.agent.model, built.agent.reasoning_effort, now, uuid), + modelPicker: buildChatModelsPort(opened, providers, built.agent.model, now, uuid), ...(config.chat.maxMessages === undefined ? {} : { chatMaxMessages: config.chat.maxMessages }), @@ -630,12 +631,19 @@ interface ReplWiring { * loop), and on teardown emit the session's sole terminal (`session:cancelled`, idempotent) + close the * persister and the db. `/exit`, `/cancel`, and an input-stream EOF all end the session with **exit code 4**. */ -/** The mode/abort control surface a driver wires to its keys + the `/mode` command (ADR-0057). */ +/** The session-control surface a driver wires to its keys + the `/mode`/`/effort` commands (ADR-0057/ADR-0066). */ export interface ChatModeControl { /** Mid-turn abort (EA7) — abort the in-flight turn, keeping the session alive. */ readonly onAbort: () => void; /** Switch the chat mode: update the footer + re-apply the turn policy on the same session (no reseat). */ readonly onModeChange: (mode: ChatMode) => void; + /** + * Set the session's reasoning-effort tier ([ADR-0066](../../../../docs/decisions/0066-normalized-reasoning-effort-control.md) §5) + * — the `/effort` command + the `/models` effort sub-step wire to this. It pushes the SESSION override (a per-turn + * update on the SAME instance — no reseat, no context/tool loss) and updates the footer; the effect lands on the + * next turn. The tier is still per-model gated at send, so on a non-reasoning model the footer stays clear. + */ + readonly onSetEffort: (effort: ReasoningEffort) => void; } /** @@ -660,7 +668,7 @@ export function chatIsInteractive( } export function createChatModeControl( - built: Pick, + built: Pick, store: ChatStoreController, opts?: { readonly interactive?: boolean }, ): ChatModeControl { @@ -684,6 +692,11 @@ export function createChatModeControl( prompt, }); applyChatMode(modeEnv, store.getSnapshot().mode); + // ADR-0066: seed the footer's effort indicator from the session's initial effective tier (override ?? agent), + // shown only when the bound model is reasoning-capable (a non-reasoning model has no controllable tier, so its + // baked config default is not surfaced). Mirrors applyChatMode's initial-mode seed above. + const capable = modelSupportsReasoning(built.agent.model); + store.setReasoningEffort(capable ? built.session.reasoningEffort : undefined); return { onAbort: () => { built.session.abort(); // void-returning: block body so it never forwards abort()'s return value @@ -692,6 +705,13 @@ export function createChatModeControl( store.setMode(mode); applyChatMode(modeEnv, mode); }, + // ADR-0066 §5: push the SESSION override (no reseat) + update the footer. The tier is gated per-model at send, + // so the footer reflects it only on a reasoning-capable model — but the override is still stored so a later + // reseat to a capable model would honor it. + onSetEffort: (effort) => { + built.session.setReasoningEffort(effort); + store.setReasoningEffort(capable ? effort : undefined); + }, }; } @@ -905,6 +925,38 @@ export function createChatLineHandler( modeControl.onModeChange(mode); emitOutput(`mode: ${MODE_LABEL[mode]}`); }, + // `/effort [tier]` (ADR-0066 §5): set the session's reasoning-effort override, or (bare `/effort`) show the + // current tier + explain each option. Applying pushes the SESSION override on the SAME instance — a per-turn + // update, NO reseat (effort changes neither provider, pricing, nor the plan), effective next turn. On a + // non-reasoning model the tier is stored but gated off at send, so the note says it will be ignored. + setReasoningEffort: (effortArg) => { + const requested = effortArg.trim(); + const capable = modelSupportsReasoning(built.agent.model); + if (requested.length === 0) { + // Bare `/effort`: show the current tier + EXPLAIN each one (a discovery affordance — the palette submits + // this bare form), marking the active one. On a non-reasoning model, say so plainly instead of a tier. + const current = store.getSnapshot().reasoningEffort; + const rows = REASONING_EFFORTS.map( + (e) => ` ${e.padEnd(8)} ${EFFORT_TIER_HINT[e]}${e === current ? ' (current)' : ''}`, + ); + const header = capable + ? `reasoning effort: ${current ?? 'default (provider)'}` + : `reasoning effort: ${built.agent.model} has no controllable reasoning tier — a tier would be ignored`; + emitOutput(`${header}\n${rows.join('\n')}`); + return; + } + const tier = REASONING_EFFORTS.find((e) => e === requested); + if (tier === undefined) { + emitOutput(`/effort: unknown tier '${requested.replace(/[^\x20-\x7e]/g, '?').slice(0, 16)}'`); + return; + } + modeControl.onSetEffort(tier); + emitOutput( + capable + ? `reasoning effort: ${tier} — applies to your next message.` + : `reasoning effort: ${tier} set, but ${built.agent.model} has no reasoning control — it will be ignored.`, + ); + }, // `/compact` (ADR-0062): model-summarise the working context. An LLM call — announce the moment, then // await, then report the deltas. The engine emits session:compacted (→ the persister writes the boundary // marker); this notice is the user-facing report. Never crashes the REPL — a failure is reported as output. @@ -1033,6 +1085,7 @@ export function createChatLineHandler( reseatTarget: () => reseatRequested, onAbort: modeControl.onAbort, onModeChange: modeControl.onModeChange, + onSetEffort: modeControl.onSetEffort, }; } @@ -1118,14 +1171,7 @@ async function buildFreshChatWiring(deps: FreshChatWiringDeps, intro: string): P doctorProbes, startSession: () => built.session.start(), intro, - modelPicker: buildChatModelsPort( - deps.opened, - deps.providers, - built.agent.model, - built.agent.reasoning_effort, - deps.now, - deps.uuid, - ), + modelPicker: buildChatModelsPort(deps.opened, deps.providers, built.agent.model, deps.now, deps.uuid), ...(deps.chat.maxMessages === undefined ? {} : { chatMaxMessages: deps.chat.maxMessages }), }; } @@ -1306,14 +1352,7 @@ async function buildReseatWiring( startSession: () => {}, intro: modelSwitchNotice(target.modelId, resumed.resumeState.turnCount), // The picker's `boundModel` is now the SWITCHED model — a further reseat marks it as the ✓ "you are here". - modelPicker: buildChatModelsPort( - deps.opened, - deps.providers, - resumed.agent.model, - resumed.agent.reasoning_effort, - deps.now, - deps.uuid, - ), + modelPicker: buildChatModelsPort(deps.opened, deps.providers, resumed.agent.model, deps.now, deps.uuid), ...(deps.chat.maxMessages === undefined ? {} : { chatMaxMessages: deps.chat.maxMessages }), }; } @@ -1365,8 +1404,17 @@ function createReseatRebuild(params: { */ async function driveOneSession(wiring: ReplWiring, deps: ChatReplDeps): Promise { const { built, store, persister, startSession, intro } = wiring; - const { processLine, cancelOnce, shouldStop, stopReason, onReseat, reseatTarget, onAbort, onModeChange } = - createChatLineHandler(wiring, deps); + const { + processLine, + cancelOnce, + shouldStop, + stopReason, + onReseat, + reseatTarget, + onAbort, + onModeChange, + onSetEffort, + } = createChatLineHandler(wiring, deps); // A live reseat is TTY-interactive only (like `/clear`): the ink model-picker overlay is the sole trigger, and a // plain/`--json` driver has no picker. Wiring `onReseat` only on an interactive driver means `stopReason()` can // never yield `'reseat'` under `--json`/plain — one machine stream stays one session lifecycle (ADR-0049). @@ -1416,6 +1464,7 @@ async function driveOneSession(wiring: ReplWiring, deps: ChatReplDeps): Promise< ...(intro === undefined ? {} : { intro }), onAbort, onModeChange, + onSetEffort, // The reseat trigger (onReseat) + its picker are wired together, interactive-only: the ink overlay reads the // catalog through `modelPicker` and calls `onReseat` on accept. A plain/`--json` driver gets neither. ...(reseatEnabled ? { onReseat } : {}), diff --git a/apps/cli/src/commands/repl-commands.test.ts b/apps/cli/src/commands/repl-commands.test.ts index 6929d5de..83ee7a08 100644 --- a/apps/cli/src/commands/repl-commands.test.ts +++ b/apps/cli/src/commands/repl-commands.test.ts @@ -20,6 +20,7 @@ interface CapabilityCalls { readonly showCost: number; readonly runDoctor: number; readonly setMode: number; + readonly setReasoningEffort: number; readonly compactHistory: number; readonly trimHistory: number; readonly clearSession: number; @@ -37,6 +38,7 @@ function spyContext(): { ctx: ReplCommandContext; calls: () => CapabilityCalls } showCost: vi.fn(), runDoctor: vi.fn(), setMode: vi.fn(), + setReasoningEffort: vi.fn(), compactHistory: vi.fn(), trimHistory: vi.fn(), clearSession: vi.fn(), @@ -52,6 +54,7 @@ function spyContext(): { ctx: ReplCommandContext; calls: () => CapabilityCalls } showWorkflows: spies.showWorkflows.mock.calls.length, showCost: spies.showCost.mock.calls.length, setMode: spies.setMode.mock.calls.length, + setReasoningEffort: spies.setReasoningEffort.mock.calls.length, runDoctor: spies.runDoctor.mock.calls.length, compactHistory: spies.compactHistory.mock.calls.length, trimHistory: spies.trimHistory.mock.calls.length, @@ -75,6 +78,7 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { 'cost', 'doctor', 'mode', + 'effort', 'compact', 'trim', 'clear', @@ -92,6 +96,7 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { ['cost', 'showCost'], ['doctor', 'runDoctor'], ['mode', 'setMode'], + ['effort', 'setReasoningEffort'], ['compact', 'compactHistory'], ['trim', 'trimHistory'], ['clear', 'clearSession'], @@ -111,6 +116,7 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { counts.showCost + counts.runDoctor + counts.setMode + + counts.setReasoningEffort + counts.compactHistory + counts.trimHistory + counts.clearSession + @@ -130,7 +136,7 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { it('replCommandList renders the slash hint, formatReplHelp lists every command', () => { expect(replCommandList()).toBe( - '/help, /exit, /cancel, /export, /workflows, /cost, /doctor, /mode, /compact, /trim, /clear, /models', + '/help, /exit, /cancel, /export, /workflows, /cost, /doctor, /mode, /effort, /compact, /trim, /clear, /models', ); const help = formatReplHelp(); for (const command of REPL_COMMANDS) { @@ -151,6 +157,7 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { 'cost', 'doctor', 'mode', + 'effort', 'trim', 'models', ]) { @@ -171,6 +178,7 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { 'cost', 'doctor', 'mode', + 'effort', 'compact', 'trim', 'clear', @@ -185,6 +193,7 @@ describe('curated REPL command registry (ADR-0056 amendment)', () => { 'cost', 'doctor', 'mode', + 'effort', 'compact', 'trim', 'clear', diff --git a/apps/cli/src/commands/repl-commands.ts b/apps/cli/src/commands/repl-commands.ts index abaf719d..a0611a85 100644 --- a/apps/cli/src/commands/repl-commands.ts +++ b/apps/cli/src/commands/repl-commands.ts @@ -13,6 +13,8 @@ * those three can never disagree. The set is deliberately small and **alias-free**. */ +import { REASONING_EFFORTS } from '@relavium/shared'; + import { CHAT_MODES } from '../chat/chat-mode.js'; import type { CommandEffect } from './manifest.js'; @@ -37,6 +39,9 @@ export interface ReplCommandContext { readonly showCost: () => void | Promise; /** Run the `/doctor` health check (2.5.C S5); `deep` adds the network/process tier (key + MCP validation). */ readonly runDoctor: (deep: boolean) => void | Promise; + /** Set the reasoning-effort tier (ADR-0066). Receives the raw tier token (empty ⇒ show the current tier + options). + * Pushes the session override (no reseat); chat-only, like `/mode`. */ + readonly setReasoningEffort: (effortArg: string) => void | Promise; /** Switch the chat mode (ADR-0057). Receives the raw mode-name token (empty ⇒ show the current mode + options). * The surface parses + applies it (re-applying the turn policy on the same session) and reports the result. */ readonly setMode: (modeArg: string) => void | Promise; @@ -184,6 +189,17 @@ const RAW_REPL_COMMANDS: readonly ReplCommand[] = [ run: (ctx, args) => ctx.setMode(args[0] ?? ''), availableIn: ['chat'], }, + { + name: 'effort', + label: 'Effort', + description: 'Set the reasoning-effort tier: off / low / medium / high / max (ADR-0066).', + effect: 'read', + // A single positional tier, validated by the dispatch against these values; bare `/effort` shows the current + // tier + the options. A per-turn session override (no reseat) — chat-only, like `/mode`. + positional: { name: 'effort', values: [...REASONING_EFFORTS] }, + run: (ctx, args) => ctx.setReasoningEffort(args[0] ?? ''), + availableIn: ['chat'], + }, { name: 'compact', label: 'Compact', diff --git a/apps/cli/src/home/drive-home.test.ts b/apps/cli/src/home/drive-home.test.ts index 5e393761..4e95a82d 100644 --- a/apps/cli/src/home/drive-home.test.ts +++ b/apps/cli/src/home/drive-home.test.ts @@ -314,7 +314,8 @@ describe('driveHome (2.5.B / ADR-0054)', () => { expect(reseated?.sessionId).toBe(sessionId); // a reseat CONTINUES the same session (unlike /clear's new id) expect(reseated?.store.getSnapshot().state.model).toBe('claude-opus-4-8'); // rebound to the picked model expect(reseated?.store.getSnapshot().state.turnCount).toBe(1); // the prior turn carried - expect(reseated?.boundEffort).toBe('medium'); // the effort sub-step's tier bound onto the reseated agent (ADR-0066) + // The effort sub-step's tier bound onto the reseated agent (ADR-0066) — surfaced in the footer via the store. + expect(reseated?.store.getSnapshot().reasoningEffort).toBe('medium'); expect(props.controller.getSnapshot().modelPicker).toBeUndefined(); // the picker closed expect(props.controller.getSnapshot().mode).toBe('chat'); // stayed in chat diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index 20011f39..283a904e 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -248,7 +248,7 @@ export async function driveHome(deps: HomeDeps): Promise { }); // createChatLineHandler owns the mode control (ADR-0057): it applies the initial `ask` mode → the // fail-closed approval regime — BEFORE the session opens, so the full-capability host is never live without it. - const { processLine, cancelOnce, shouldStop, stopReason, onAbort, onModeChange } = + const { processLine, cancelOnce, shouldStop, stopReason, onAbort, onModeChange, onSetEffort } = createChatLineHandler( { built, opened, store, persister, doctorProbes: chatDoctorProbes }, deps, @@ -293,11 +293,11 @@ export async function driveHome(deps: HomeDeps): Promise { shouldStop, stopReason, sessionId: built.sessionId, - // The bound effort (ADR-0066) for the `/models` effort sub-list ✓/highlight + the reseat no-op guard. - boundEffort: built.agent.reasoning_effort, teardown, onAbort, onModeChange, + // ADR-0066 §5: the in-Home `/models` effort sub-step + `/effort` push the SESSION override (no reseat). + onSetEffort, ...(mentionReader === undefined ? {} : { mentionReader }), runShellCommand, }; diff --git a/apps/cli/src/render/tui/chat-ink.tsx b/apps/cli/src/render/tui/chat-ink.tsx index acddce6f..9ac9c2a7 100644 --- a/apps/cli/src/render/tui/chat-ink.tsx +++ b/apps/cli/src/render/tui/chat-ink.tsx @@ -84,6 +84,8 @@ import { sanitizeInline, stripTerminalControls, } from './chat-projection.js'; +import type { ReasoningEffort } from '@relavium/shared'; + import { nextMode, type ChatMode } from '../../chat/chat-mode.js'; import type { ChatStoreController, PendingApproval } from './chat-store.js'; import type { SessionViewState, TranscriptEntry } from './session-view-model.js'; @@ -144,6 +146,10 @@ interface ChatAppProps { readonly onAbort?: (() => void) | undefined; /** Switch the chat mode (Shift+Tab cycle) — re-applies the turn policy on the same session (ADR-0057). */ readonly onModeChange: (mode: ChatMode) => void; + /** Set the reasoning-effort tier (ADR-0066 §5) — the `/models` effort sub-step calls it on a SAME-model pick (a + * per-turn session override, NO reseat). Absent ⇒ the effort sub-step is not offered. `| undefined` for the + * createElement passthrough. */ + readonly onSetEffort?: ((effort: ReasoningEffort) => void) | undefined; /** Request a mid-session model switch (ADR-0059) — the `/models` picker overlay calls it on accept. Absent (a * driver/test wired without it) ⇒ the overlay never opens (see `modelPicker`). `| undefined` for the passthrough. */ readonly onReseat?: ((target: ReseatTarget) => void) | undefined; @@ -174,6 +180,9 @@ interface ChatViewProps { readonly running: boolean; /** The active chat mode (ADR-0057) — shown in the footer so `auto` is never a hidden state. */ readonly mode: ChatMode; + /** The active reasoning-effort tier (ADR-0066) — shown in the footer (parity with `mode`) so the tier is never a + * hidden state; absent ⇒ not shown (a non-reasoning model / no tier). */ + readonly reasoningEffort?: ReasoningEffort | undefined; /** An in-flight per-tool approval — when set, the `[y]/[a]/[n]` prompt replaces the idle prompt. */ readonly approval?: PendingApproval | undefined; /** When the `/` palette is open it owns the bottom of the view, so the idle prompt + footer are suppressed (2.5.C S3b). */ @@ -193,7 +202,7 @@ interface ChatViewProps { * sequence cannot corrupt the terminal or inject ANSI/OSC. */ export function ChatView(props: Readonly): ReactElement { - const { state, tick, color, editor, running, mode, approval, paletteOpen } = props; + const { state, tick, color, editor, running, mode, reasoningEffort, approval, paletteOpen } = props; const attachments = props.attachments ?? []; // When the palette is open it renders its own query line + hint below, so suppress the idle prompt + footer to // avoid two competing prompts (the palette owns the input focus until it closes). @@ -297,13 +306,15 @@ export function ChatView(props: Readonly): ReactElement { )} - {formatSessionFooterWithMode(state, mode)} + + {formatSessionFooterWithMode(state, mode, reasoningEffort)} + ); } export function ChatApp(props: Readonly): ReactElement { - const { state, tick, color, mode, approval } = useSyncExternalStore( + const { state, tick, color, mode, reasoningEffort, approval } = useSyncExternalStore( props.store.subscribe, props.store.getSnapshot, ); @@ -578,12 +589,13 @@ export function ChatApp(props: Readonly): ReactElement { refreshedAt: view.refreshedAt, banner: undefined, hint: undefined, - // The standalone `relavium chat` picker is ALWAYS a live reseat, so it offers the effort sub-step (ADR-0066). + // ADR-0066: offer the effort sub-step when the setter is wired (interactive). `currentEffort` reads the LIVE + // store tier (not a stale build-time value), so after a no-reseat `/effort` change the sub-list opens on it. phase: 'model', - effortStep: true, + effortStep: props.onSetEffort !== undefined, pending: undefined, effortSelected: 0, - currentEffort: port.boundEffort, + currentEffort: props.store.getSnapshot().reasoningEffort, }); runPickerRefresh(() => port.refreshIfStale()); }; @@ -598,17 +610,22 @@ export function ChatApp(props: Readonly): ReactElement { case 'close': applyModelPicker(undefined); return; - case 'accept': - // No-op guard (ADR-0059/ADR-0066): accepting the ALREADY-bound model AND effort (the ✓ = `currentDefault` + - // `currentEffort` = the session's binding) would reseat for zero change — wiping the ADR-0057 approval cache - // + reconnecting MCP. The same model with a DIFFERENT effort IS a real (effort-only) switch, so the guard is - // keyed on BOTH axes. Close with a note instead of a pointless switch. - if (step.modelId === open.currentDefault && step.reasoningEffort === open.currentEffort) { - applyModelPicker(undefined); - props.store.note(`Already on ${step.displayName}.`); + case 'accept': { + applyModelPicker(undefined); + if (step.modelId === open.currentDefault) { + // SAME model: this is an effort-only change (ADR-0066 §5) — a per-turn SESSION override, NOT a reseat. No + // teardown, no ADR-0057 approval-cache wipe, no MCP reconnect, no text-only context loss. A re-pick of the + // same tier (or a non-reasoning model with no tier) is a gentle no-op. + if (step.reasoningEffort === undefined || step.reasoningEffort === open.currentEffort) { + const at = step.reasoningEffort === undefined ? '' : ` at effort ${step.reasoningEffort}`; + props.store.note(`Already on ${step.displayName}${at}.`); + } else { + props.onSetEffort?.(step.reasoningEffort); + props.store.note(`Reasoning effort set to ${step.reasoningEffort} — applies to your next message.`); + } return; } - applyModelPicker(undefined); + // DIFFERENT model: a live reseat (ADR-0059), carrying the chosen effort onto the new binding. props.onReseat?.({ modelId: step.modelId, provider: step.provider, @@ -616,6 +633,7 @@ export function ChatApp(props: Readonly): ReactElement { }); props.onExit(); // the reseat set the stop state; end the loop so runReplLoop swaps in the new-model session return; + } case 'blocked': { const hint = step.reason === 'no-key' @@ -905,6 +923,7 @@ export function ChatApp(props: Readonly): ReactElement { editor={editor} running={running || shellBusy || submitBusy} mode={mode} + reasoningEffort={reasoningEffort} approval={approval} attachments={attachments} busyCommand={busyCommand} @@ -1010,6 +1029,8 @@ export function driveInk(ctx: ChatDriveContext): Promise { // 'abort' handler can reject a pending approval when it is absent (never a dead Esc — see ChatApp). onAbort: ctx.onAbort, onModeChange: ctx.onModeChange ?? ((): void => undefined), + // ADR-0066 §5 effort setter — passed AS-IS (optional); absent ⇒ the `/models` effort sub-step is not offered. + onSetEffort: ctx.onSetEffort, // `/models` reseat (ADR-0059) — the REPL loop wires both onReseat + the picker port only for an interactive // session; passed AS-IS (optional) so a driver wired without them simply has no `/models` overlay. onReseat: ctx.onReseat, diff --git a/apps/cli/src/render/tui/chat-projection.ts b/apps/cli/src/render/tui/chat-projection.ts index de41f634..5def3c4b 100644 --- a/apps/cli/src/render/tui/chat-projection.ts +++ b/apps/cli/src/render/tui/chat-projection.ts @@ -1,4 +1,5 @@ import type { ToolApprovalRequest } from '@relavium/core'; +import type { ReasoningEffort } from '@relavium/shared'; import { MODE_LABEL, type ChatMode } from '../../chat/chat-mode.js'; import { formatCostUsd, formatDuration, formatTokens } from './format.js'; @@ -171,11 +172,20 @@ export function formatSessionFooter(state: SessionViewState): string { return parts.join(' · '); } -/** The footer including the active chat mode (ADR-0057) — the mode is always shown so `auto` is never hidden. */ -export function formatSessionFooterWithMode(state: SessionViewState, mode: ChatMode): string { +/** + * The footer including the active chat mode (ADR-0057) and — when set — the reasoning-effort tier (ADR-0066), each + * always shown so neither `auto` mode nor a non-default effort is a hidden state. The effort is omitted when unset + * (a non-reasoning model / no tier), so a plain chat's footer is unchanged. + */ +export function formatSessionFooterWithMode( + state: SessionViewState, + mode: ChatMode, + reasoningEffort?: ReasoningEffort, +): string { const base = formatSessionFooter(state); const modePart = `${MODE_LABEL[mode]} mode`; - return base.length > 0 ? `${base} · ${modePart}` : modePart; + const withMode = base.length > 0 ? `${base} · ${modePart}` : modePart; + return reasoningEffort === undefined ? withMode : `${withMode} · effort: ${reasoningEffort}`; } /** diff --git a/apps/cli/src/render/tui/chat-store.ts b/apps/cli/src/render/tui/chat-store.ts index ff11b48a..b475de14 100644 --- a/apps/cli/src/render/tui/chat-store.ts +++ b/apps/cli/src/render/tui/chat-store.ts @@ -1,5 +1,7 @@ import type { SessionStreamHandleEvent, ToolApprovalRequest } from '@relavium/core'; +import type { ReasoningEffort } from '@relavium/shared'; + import { DEFAULT_CHAT_MODE, type ApprovalAnswer, @@ -43,6 +45,13 @@ export interface ChatStoreSnapshot { readonly state: SessionViewState; /** The active chat mode (ADR-0057) — REPL-set (Shift+Tab / `/mode`), shown in the footer. */ readonly mode: ChatMode; + /** + * The session's active reasoning-effort tier ([ADR-0066](../../../../docs/decisions/0066-normalized-reasoning-effort-control.md)) + * — the effective `session-override → agent` tier, shown in the footer (parity with `mode`) and read by the + * `/models` effort sub-list + `/effort`. Present only when the bound model is reasoning-capable (a non-reasoning + * model has no controllable tier), so the footer never shows an inert effort. `undefined` ⇒ no tier / not capable. + */ + readonly reasoningEffort: ReasoningEffort | undefined; /** The in-flight approval prompt, if a governed tool dispatch is awaiting the user's decision. */ readonly approval: PendingApproval | undefined; readonly tick: number; @@ -75,6 +84,10 @@ export interface ChatStoreController extends ChatStore { /** Set the active chat mode (Shift+Tab / `/mode`) — updates the footer; the caller also re-applies the turn * policy via `applyChatMode`. Flushes immediately (a mode switch feels instant). */ setMode: (mode: ChatMode) => void; + /** Set the active reasoning-effort tier for the footer + the `/models`/`/effort` reads (ADR-0066) — `undefined` + * clears it (a non-reasoning model / no tier). The caller also pushes the session override via + * `AgentSession.setReasoningEffort`. Flushes immediately (an effort switch feels instant). */ + setReasoningEffort: (effort: ReasoningEffort | undefined) => void; /** Clear the compaction "moment" flag (ADR-0062 §7). The host calls this when a MANUAL `/compact` settles — a * failed/cancelled `/compact` emits NO `session:compacted`/`session:trimmed`, so the flag (set by * `session:compacting`) would otherwise latch and a later slash command's busy render would show a stale @@ -107,15 +120,16 @@ export function createChatStore(color: boolean, seed?: SessionViewSeed): ChatSto const listeners = new Set<() => void>(); let state = initialSessionViewState(seed); let mode: ChatMode = DEFAULT_CHAT_MODE; + let reasoningEffort: ReasoningEffort | undefined; let approval: PendingApproval | undefined; // The resolver for the in-flight approval promise (set while `approval` is published; cleared on settle). let settleApproval: ((answer: ApprovalAnswer) => void) | undefined; let tickCount = 0; let dirty = false; - let snapshot: ChatStoreSnapshot = { state, mode, approval, tick: tickCount, color }; + let snapshot: ChatStoreSnapshot = { state, mode, reasoningEffort, approval, tick: tickCount, color }; const flush = (): void => { - snapshot = { state, mode, approval, tick: tickCount, color }; + snapshot = { state, mode, reasoningEffort, approval, tick: tickCount, color }; for (const listener of listeners) { listener(); } @@ -164,6 +178,10 @@ export function createChatStore(color: boolean, seed?: SessionViewSeed): ChatSto mode = next; flush(); }, + setReasoningEffort: (next) => { + reasoningEffort = next; + flush(); + }, clearCompacting: () => { if (!state.compacting) return; // idempotent — only repaint if the moment was actually latched state = { ...state, compacting: false }; diff --git a/apps/cli/src/render/tui/home-app.tsx b/apps/cli/src/render/tui/home-app.tsx index dc2d0fd5..ffda7c81 100644 --- a/apps/cli/src/render/tui/home-app.tsx +++ b/apps/cli/src/render/tui/home-app.tsx @@ -57,7 +57,7 @@ function ChatRegion( attachments: readonly PendingAttachment[]; }>, ): ReactElement { - const { state, tick, color, mode, approval } = useSyncExternalStore( + const { state, tick, color, mode, reasoningEffort, approval } = useSyncExternalStore( props.store.subscribe, props.store.getSnapshot, ); @@ -70,6 +70,7 @@ function ChatRegion( editor={props.editor} running={state.status === 'running' || props.shellBusy || props.submitBusy} mode={mode} + reasoningEffort={reasoningEffort} approval={approval} attachments={props.attachments} busyCommand={props.shellCommand} diff --git a/apps/cli/src/render/tui/home-controller.test.ts b/apps/cli/src/render/tui/home-controller.test.ts index fbc79d91..02fbb808 100644 --- a/apps/cli/src/render/tui/home-controller.test.ts +++ b/apps/cli/src/render/tui/home-controller.test.ts @@ -63,7 +63,7 @@ function makeSession( onModeChange?: (mode: ChatMode) => void; mentionReader?: MentionReader; runShellCommand?: (command: string, args: readonly string[]) => Promise; - boundEffort?: ReasoningEffort; + onSetEffort?: (effort: ReasoningEffort) => void; } = {}, ): { session: HomeChatSession; @@ -84,7 +84,7 @@ function makeSession( }, shouldStop: opts.stop ?? (() => false), stopReason: opts.stopReason ?? (() => 'exit'), - boundEffort: opts.boundEffort, + ...(opts.onSetEffort === undefined ? {} : { onSetEffort: opts.onSetEffort }), ...(opts.onAbort === undefined ? {} : { onAbort: opts.onAbort }), ...(opts.onModeChange === undefined ? {} : { onModeChange: opts.onModeChange }), ...(opts.mentionReader === undefined ? {} : { mentionReader: opts.mentionReader }), @@ -915,7 +915,6 @@ describe('createHomeController (2.5.B lifecycle / ADR-0054)', () => { processLine: () => Promise.resolve(), shouldStop: () => true, // the first turn ends the session ⇒ endChat fires stopReason: () => 'exit', // /exit-style end → endChat (not the /clear swap) - boundEffort: undefined, teardown, }; const startChat = vi.fn(() => Promise.resolve(session)); @@ -952,7 +951,6 @@ describe('createHomeController (2.5.B lifecycle / ADR-0054)', () => { }, shouldStop: () => true, stopReason: () => 'exit', - boundEffort: undefined, teardown: vi.fn(() => new Promise((r) => (releaseTeardown = r))), }; const c = createHomeController({ @@ -1928,6 +1926,80 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => expect(c.getSnapshot().modelPicker).toBeUndefined(); // the picker closed }); + it('in-Home chat: a SAME-model effort change calls the setter — NO reseat (ADR-0066 §5)', async () => { + // Bound to a reasoning-capable model at effort 'low'. Re-picking the SAME model then choosing 'high' in the + // effort sub-step must push the SESSION override (onSetEffort) — NOT a reseat (which would tear the session down). + const boundStore = createChatStore(false, { model: 'claude-opus-4-8' }); + boundStore.setReasoningEffort('low'); // the session's current tier (drives the sub-list's ✓/highlight) + const onSetEffort = vi.fn(); + const sessionA = makeSession({ sessionId: 'sess-A', store: boundStore, onSetEffort }); + const reseatChat = vi.fn(() => Promise.resolve(makeSession().session)); + const { port } = makeModelsPort({ + entries: [pickerEntry({ modelId: 'claude-opus-4-8', provider: 'anthropic', supportsReasoning: true })], + }); + const c = createHomeController({ + doctorProbes: STUB_DOCTOR_PROBES, + startChat: () => Promise.resolve(sessionA.session), + reseatChat, + models: port, + homeStore, + onExit: vi.fn(), + onError: vi.fn(), + }); + type(c, 'hi'); + c.handleKey('', ENTER); + await flush(); + c.handleKey('/', {}); + type(c, 'models'); + c.handleKey('', ENTER); + await flush(); + c.handleKey('', ENTER); // model phase: Enter on the reasoning model ⇒ advance to the effort sub-step + expect(c.getSnapshot().modelPicker?.phase).toBe('effort'); + c.handleKey('', { downArrow: true }); // low → medium + c.handleKey('', { downArrow: true }); // medium → high + c.handleKey('', ENTER); // apply 'high' + await flush(); + + expect(onSetEffort).toHaveBeenCalledWith('high'); // the SESSION override (no reseat) + expect(reseatChat).not.toHaveBeenCalled(); // an effort change is NOT a reseat (ADR-0066 §5, not option (d)) + expect(sessionA.teardown).not.toHaveBeenCalled(); // the live session is untouched (no teardown/MCP reconnect) + expect(c.getSnapshot().modelPicker).toBeUndefined(); // the picker closed after the effort pick + }); + + it('in-Home chat: re-picking the SAME model AND same effort is a no-op (no setter, no reseat) (ADR-0066)', async () => { + const boundStore = createChatStore(false, { model: 'claude-opus-4-8' }); + boundStore.setReasoningEffort('high'); + const onSetEffort = vi.fn(); + const sessionA = makeSession({ sessionId: 'sess-A', store: boundStore, onSetEffort }); + const reseatChat = vi.fn(() => Promise.resolve(makeSession().session)); + const { port } = makeModelsPort({ + entries: [pickerEntry({ modelId: 'claude-opus-4-8', provider: 'anthropic', supportsReasoning: true })], + }); + const c = createHomeController({ + doctorProbes: STUB_DOCTOR_PROBES, + startChat: () => Promise.resolve(sessionA.session), + reseatChat, + models: port, + homeStore, + onExit: vi.fn(), + onError: vi.fn(), + }); + type(c, 'hi'); + c.handleKey('', ENTER); + await flush(); + c.handleKey('/', {}); + type(c, 'models'); + c.handleKey('', ENTER); + await flush(); + c.handleKey('', ENTER); // effort sub-step opens highlighted on the bound 'high' + c.handleKey('', ENTER); // accept 'high' (unchanged) + await flush(); + + expect(onSetEffort).not.toHaveBeenCalled(); // same tier ⇒ no setter call + expect(reseatChat).not.toHaveBeenCalled(); + expect(c.getSnapshot().modelPicker).toBeUndefined(); + }); + it('in-Home chat: accepting the ALREADY-bound model does NOT reseat — a no-op hint (ADR-0059)', async () => { // The session is bound to claude-opus-4-8; the only picker entry IS that model. Accepting it must NOT tear the // session down + rebuild for zero change (which would wipe the approval cache) — it keeps the picker open + hints. diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index 90a355da..c46ffec9 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -97,10 +97,10 @@ export interface HomeChatSession { /** The session's durable id (ADR-0062 §7) — named in the `/clear` notice as the prior (still-resumable) * conversation, so it is discoverable after the swap. */ readonly sessionId: string; - /** The session's currently-bound reasoning-effort tier (ADR-0066), from the bound agent's `reasoning_effort` — - * the `/models` effort sub-list's `✓` + opening highlight, and the reseat no-op guard's second axis (same model - * AND same effort ⇒ no-op). `undefined` ⇒ no effort bound (the provider default). */ - readonly boundEffort: ReasoningEffort | undefined; + /** Set the reasoning-effort tier (ADR-0066 §5) — the in-Home `/models` effort sub-step calls it on a SAME-model + * pick (a per-turn session override, NO reseat) + the `/effort` command. Absent ⇒ the effort sub-step is not + * offered. The current tier is read live from `store` (`ChatStoreSnapshot.reasoningEffort`), not tracked here. */ + readonly onSetEffort?: (effort: ReasoningEffort) => void; /** WHY `shouldStop()` became true (ADR-0062 §7 · ADR-0059) — `'clear'` (swap in a fresh session, staying in chat) * vs `'exit'` (`/exit`/`/cancel`, return to the bare Home). Shares the widened `ChatLineHandler.stopReason` type, * so it also carries `'reseat'`; but the in-Home chat does NOT yet wire `onReseat` (its `/models` is the Home's @@ -625,9 +625,10 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { // bare Home it is the effective next-session default. The accept action mirrors this (reseat vs default-write). const active = state.session; const activeModel = active?.store.getSnapshot().state.model; - // The effort sub-step (ADR-0066) is offered ONLY on a LIVE reseat (an active in-Home chat with a reseat builder - // wired) — the bare-Home default-write persists only the model (ADR-0063), so it stays single-phase. - const effortStep = active !== undefined && deps.reseatChat !== undefined; + // The effort sub-step (ADR-0066) is offered ONLY in a LIVE in-Home chat (a session with the effort setter wired) + // — the bare-Home default-write persists only the model (ADR-0063), so it stays single-phase. `currentEffort` + // reads the LIVE store tier, so after a no-reseat `/effort` change the sub-list opens on it. + const effortStep = active?.onSetEffort !== undefined; set({ notice: undefined, // opening the picker clears any stale /doctor report behind it modelPicker: { @@ -643,7 +644,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { effortStep, pending: undefined, effortSelected: 0, - currentEffort: effortStep ? active.boundEffort : undefined, + currentEffort: effortStep ? active?.store.getSnapshot().reasoningEffort : undefined, }, }); // Render the cache immediately (above), then kick a TTL-bounded background refresh (ADR-0064 §5c) — the Home is @@ -661,19 +662,32 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { ): void => { const active = state.session; if (active !== undefined && deps.reseatChat !== undefined) { - // No-op guard (ADR-0059/ADR-0066): accepting the ALREADY-bound model AND effort would tear the session down + - // rebuild for zero change — wiping the ADR-0057 per-tool approval cache, reconnecting MCP, and showing a - // misleading "Switched" notice. The same model with a DIFFERENT effort IS a real (effort-only) switch, so the - // guard is keyed on BOTH axes. Keep the picker open with a hint instead so a mis-click doesn't churn the session. - if (modelId === active.store.getSnapshot().state.model && reasoningEffort === active.boundEffort) { - const open = state.modelPicker; - set( - open === undefined - ? { modelPicker: undefined } - : { modelPicker: { ...open, hint: `Already on ${displayName} — pick a different model or Esc.` } }, - ); + if (modelId === active.store.getSnapshot().state.model) { + if (reasoningEffort === undefined) { + // MODEL-phase no-op (a non-reasoning same-model re-pick — no effort sub-step): keep the picker OPEN with a + // hint so the user can pick a different model. The model-phase view renders the hint (unchanged ADR-0059). + const open = state.modelPicker; + set( + open === undefined + ? { modelPicker: undefined } + : { modelPicker: { ...open, hint: `Already on ${displayName} — pick a different model or Esc.` } }, + ); + return; + } + // EFFORT-phase accept (a reasoning model): an effort-only change (ADR-0066 §5) — a per-turn SESSION override + // via the setter, NOT a reseat (no teardown, no ADR-0057 approval-cache wipe, no MCP reconnect, no context + // loss). Close + note: the effort sub-list renders no hint, so a no-op must give visible store feedback. + const current = active.store.getSnapshot().reasoningEffort; + if (reasoningEffort !== current) { + active.onSetEffort?.(reasoningEffort); + active.store.note(`Reasoning effort set to ${reasoningEffort} — applies to your next message.`); + } else { + active.store.note(`Already on ${displayName} at effort ${reasoningEffort}.`); + } + set({ modelPicker: undefined }); return; } + // DIFFERENT model: a live reseat (ADR-0059), carrying the chosen effort onto the new binding. reseatChat(active, { modelId, provider, @@ -765,6 +779,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { showWorkflows: () => undefined, showCost: () => undefined, setMode: () => undefined, // `/mode` is chat-only (not in HOME_PALETTE_COMMANDS); inert in the Home surface + setReasoningEffort: () => undefined, // `/effort` is chat-only (ADR-0066); inert in the bare-Home surface compactHistory: () => undefined, // `/compact` is chat-only (ADR-0062); inert in the Home surface trimHistory: () => undefined, // `/trim` is chat-only (ADR-0062); inert in the Home surface // `/clear` (ADR-0062 §7) IS offered in the Home palette (availableIn ['home','chat']), but the BARE Home has no diff --git a/packages/core/src/engine/agent-runner.ts b/packages/core/src/engine/agent-runner.ts index 19870e5a..5540b32a 100644 --- a/packages/core/src/engine/agent-runner.ts +++ b/packages/core/src/engine/agent-runner.ts @@ -65,6 +65,7 @@ import { type PreEgressHook, } from './agent-turn.js'; import { BudgetExceededError, BudgetPauseError } from './budget-governor.js'; +import { gateReasoningEffort } from './reasoning-effort.js'; import type { MediaJobSubmission, NodeExecContext, @@ -802,13 +803,10 @@ function resolveGenKnobs( const temperature = node.temperature ?? agent.temperature; const maxTokens = node.max_tokens ?? agent.max_tokens; // ADR-0066: send the reasoning-effort tier ONLY when the agent authored one AND the primary model is - // reasoning-capable (a non-reasoning model would reject the field). The host-injected `resolveReasoning` is the - // per-model catalog projection; absent/`undefined` ⇒ not reasoning ⇒ the tier is withheld (safe). A fallback to a - // different chain model carries the tier along — the adapter/provider ignores or rejects it there (a documented edge). - const reasoningEffort = - agent.reasoning_effort !== undefined && deps.resolveReasoning?.(agent.model) === true - ? agent.reasoning_effort - : undefined; + // reasoning-capable (the shared {@link gateReasoningEffort} rule — a non-reasoning model would reject the field). + // The per-fallback-entry re-gate lives in the chain (a non-reasoning fallback entry strips the tier), so a failover + // to a different-capability model never carries an unsupported field. + const reasoningEffort = gateReasoningEffort(agent.reasoning_effort, agent.model, deps.resolveReasoning); return { ...(temperature === undefined ? {} : { temperature }), ...(maxTokens === undefined ? {} : { maxTokens }), diff --git a/packages/core/src/engine/agent-session.test.ts b/packages/core/src/engine/agent-session.test.ts index 9d618b65..c910d07a 100644 --- a/packages/core/src/engine/agent-session.test.ts +++ b/packages/core/src/engine/agent-session.test.ts @@ -815,6 +815,74 @@ describe('AgentSession — reseat-less modes + mid-turn abort (ADR-0057 Step 2)' expect(off.effort()).toBeUndefined(); }); + it('setReasoningEffort override wins over the authored tier on the NEXT turn — no reseat (ADR-0066 §5)', async () => { + const reader = AgentSchema.parse({ + id: 'reader', + model: 'claude-opus-4-8', + provider: 'anthropic', + system_prompt: 'x', + reasoning_effort: 'low', + }); + let effort: unknown = 'UNSET'; + const provider: LlmProvider = { + id: 'anthropic', + supports: CAPS, + generate: () => { + throw new Error('unused'); + }, + stream: (req) => { + effort = req.reasoningEffort; + return streamOf(textTurn('ok')); + }, + }; + const s = session( + harness([textTurn('ok')], { resolveProvider: () => provider, resolveReasoning: () => true }).deps, + reader, + ); + s.start(); + // Turn 1: no override yet ⇒ the authored 'low' rides; the getter reflects the effective tier. + expect(s.reasoningEffort).toBe('low'); + await s.sendMessage('one'); + expect(effort).toBe('low'); + // A mid-session setter — the SAME instance, no reseat; the getter updates and it lands on the NEXT turn. + s.setReasoningEffort('max'); + expect(s.reasoningEffort).toBe('max'); // override ?? agent + await s.sendMessage('two'); + expect(effort).toBe('max'); + // Clearing the override falls back to the authored tier. + s.setReasoningEffort(undefined); + expect(s.reasoningEffort).toBe('low'); + }); + + it('a session-effort override is STILL per-model gated — withheld on a non-reasoning model (ADR-0066 §4)', async () => { + const reader = AgentSchema.parse({ + id: 'reader', + model: 'gpt-4o', + provider: 'openai', + system_prompt: 'x', + }); + let effort: unknown = 'UNSET'; + const provider: LlmProvider = { + id: 'openai', + supports: CAPS, + generate: () => { + throw new Error('unused'); + }, + stream: (req) => { + effort = req.reasoningEffort; + return streamOf(textTurn('ok')); + }, + }; + const s = session( + harness([textTurn('ok')], { resolveProvider: () => provider, resolveReasoning: () => false }).deps, + reader, + ); + s.start(); + s.setReasoningEffort('high'); // the user set a tier, but the model does not reason + await s.sendMessage('go'); + expect(effort).toBeUndefined(); // gated off at send — a non-reasoning model would reject it + }); + it('setTurnPolicy activates the approval regime — the dispatch context carries the confirm hook', async () => { const confirm = (): Promise<{ outcome: 'approve' }> => Promise.resolve({ outcome: 'approve' }); let captured: ToolDispatchContext | undefined; diff --git a/packages/core/src/engine/agent-session.ts b/packages/core/src/engine/agent-session.ts index 9cfd983c..8c9f4a8b 100644 --- a/packages/core/src/engine/agent-session.ts +++ b/packages/core/src/engine/agent-session.ts @@ -28,6 +28,7 @@ import type { AbortSignalLike, AgentApprovalRequestedEvent, ErrorCode, + ReasoningEffort, SessionContext, SessionEvent, SessionStopReason, @@ -69,6 +70,7 @@ import { } from './agent-turn.js'; import { BudgetPauseError } from './budget-governor.js'; import type { AbortControllerLike } from './execution-host.js'; +import { gateReasoningEffort } from './reasoning-effort.js'; import type { NodeStreamEvent } from './node-executor.js'; import type { SessionResumeState } from './session-resume.js'; @@ -376,6 +378,15 @@ export class AgentSession { * today's behavior (all granted tools advertised, no approval regime). */ #turnPolicy: SessionTurnPolicy | undefined; + /** + * The session-level reasoning-effort OVERRIDE ([ADR-0066](../../../../docs/decisions/0066-normalized-reasoning-effort-control.md) §5), + * mutated by {@link setReasoningEffort} and read per turn (like {@link #turnPolicy}). It is the top tier of the + * per-turn resolution `session override → agent.reasoning_effort → unset`, so an interactive surface (the + * `/models` effort sub-step, a `/effort` command) changes effort mid-session with **no reseat** — effort is a + * per-turn field (never plan-changing), so a reseat (which exists for a *model* change) would be unwarranted + * teardown. `undefined` ⇒ no override (fall through to the agent's authored tier). + */ + #reasoningEffort: ReasoningEffort | undefined; /** * Set by {@link abort} to mark the in-flight turn as **user-aborted** (EA7) — distinct from `cancel()`'s * terminal `'cancelled'` status. The `sendMessage` catch reads it to settle the turn as @@ -465,6 +476,30 @@ export class AgentSession { this.#turnPolicy = policy; } + /** + * Set (or clear) the **session-level reasoning-effort override** ([ADR-0066](../../../../docs/decisions/0066-normalized-reasoning-effort-control.md) §5) + * on this SAME session instance — the interactive `/models` effort sub-step + a `/effort` command call it. The + * change is **lossless** (no reseat, no tool/reasoning-context loss — effort changes neither the provider, the + * pricing, nor the memoized `#plan`) and applies on the **next** turn (each `sendMessage` reads the override at + * turn start). Pass `undefined` to clear it (fall back to the agent's authored `reasoning_effort`). Callable in + * **any** state incl. mid-turn (takes effect next turn); **inert once `cancelled`** (no further turn reads it). + * The tier is still gated per turn by the host's per-model capability ({@link AgentSessionDeps.resolveReasoning}), + * so setting it on a non-reasoning model is a harmless no-op at send time. + */ + setReasoningEffort(effort: ReasoningEffort | undefined): void { + this.#reasoningEffort = effort; + } + + /** + * The session's EFFECTIVE reasoning-effort tier ([ADR-0066](../../../../docs/decisions/0066-normalized-reasoning-effort-control.md) §5) + * — the session override, else the agent's authored `reasoning_effort`, else `undefined`. This is the resolved + * tier BEFORE the per-model capability gate ({@link gateReasoningEffort}); a surface reads it to show the active + * tier (a footer indicator, the `/models` effort sub-list's ✓/highlight) without having to track it itself. + */ + get reasoningEffort(): ReasoningEffort | undefined { + return this.#reasoningEffort ?? this.#agent.reasoning_effort; + } + /** Guard the send preconditions: the session must be started and idle (not running/cancelled/ended). */ #assertSendable(): void { if (this.#status === 'created') { @@ -1049,6 +1084,14 @@ export class AgentSession { // Advertise-filter (ADR-0057): narrow the model-visible tool set per the host's mode (best-effort; the // confirm floor stays authoritative). No policy / no filter ⇒ advertise every granted tool. const llmTools = buildLlmTools(this.#deps.tools, grantedToolIds, turnPolicy?.advertise); + // ADR-0066: resolve the effective reasoning-effort tier (session override → agent's authored tier) and gate it + // on the bound model's per-model capability (a non-reasoning model would reject it). Read at turn start so a + // mid-session setReasoningEffort applies to the NEXT turn — the no-reseat per-turn semantics (§5). + const reasoningEffort = gateReasoningEffort( + this.#reasoningEffort ?? this.#agent.reasoning_effort, + this.#agent.model, + this.#deps.resolveReasoning, + ); return runAgentTurn({ system: this.#systemPrompt(), messages: this.#messages, @@ -1057,12 +1100,7 @@ export class AgentSession { chainCapabilities: this.#chainCapabilities(), ...(this.#agent.temperature === undefined ? {} : { temperature: this.#agent.temperature }), ...(this.#agent.max_tokens === undefined ? {} : { maxTokens: this.#agent.max_tokens }), - // ADR-0066: the authored reasoning-effort tier, sent ONLY when the bound model is reasoning-capable (a - // non-reasoning model would reject it — the host-injected per-model catalog projection gates it). - ...(this.#agent.reasoning_effort !== undefined && - this.#deps.resolveReasoning?.(this.#agent.model) === true - ? { reasoningEffort: this.#agent.reasoning_effort } - : {}), + ...(reasoningEffort === undefined ? {} : { reasoningEffort }), nodeId: this.#agentRef, emit: (event) => { this.#onTurnEmit(event); diff --git a/packages/core/src/engine/reasoning-effort.ts b/packages/core/src/engine/reasoning-effort.ts new file mode 100644 index 00000000..8151a4cf --- /dev/null +++ b/packages/core/src/engine/reasoning-effort.ts @@ -0,0 +1,20 @@ +import type { ReasoningEffort } from '@relavium/shared'; + +/** + * Gate the normalized reasoning-effort tier ([ADR-0066](../../../../docs/decisions/0066-normalized-reasoning-effort-control.md)) + * against a model's per-model capability. The tier is sent to the provider ONLY when BOTH hold: an effort was + * resolved (authored on the agent, a `[chat]` config default baked onto the agent, or a session-level override) AND + * the host's per-model capability projection (`resolveReasoning`, the ADR-0064 catalog capability) says THIS model + * reasons. A non-reasoning model would reject the field, so an unknown/absent resolver (`undefined`) is treated as + * NOT reasoning — the field is withheld (the safe default; §4). + * + * The one home for the gate (used by both the workflow `AgentRunner` path and the `AgentSession` per-turn build) so + * the rule cannot drift between them. Pure — `packages/core` stays platform-free; the host injects `resolveReasoning`. + */ +export function gateReasoningEffort( + effort: ReasoningEffort | undefined, + model: string, + resolveReasoning: ((model: string) => boolean | undefined) | undefined, +): ReasoningEffort | undefined { + return effort !== undefined && resolveReasoning?.(model) === true ? effort : undefined; +} From 3d60fde0c2469165d07afabc7f47564c3e1bc9b0 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 19:51:36 +0300 Subject: [PATCH 57/68] =?UTF-8?q?fix(llm):=202.5.E-E5=20P1/P2=20=E2=80=94?= =?UTF-8?q?=20=C2=A74=20reasoning=20heuristic,=20per-entry=20failover=20ga?= =?UTF-8?q?te,=20Gemini=20deep-merge=20(ADR-0066)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address the P1/P2 findings from the Step E review: P1 — §4 model-id reasoning heuristic (ADR conformance): modelSupportsReasoning now falls back to a CONSERVATIVE id heuristic (reasoningModelIdHeuristic) for a live-discovered id absent from the registry — o-series, the reasoning gpt-5 line (excluding gpt-5-chat), Claude Opus, explicit "thinking" ids. Deliberately narrow (each arm is a family whose whole set reasons) so it never over-matches a non-reasoning model into a provider 400. The registry stays authoritative for a canonical id (true OR false). The catalog projection reuses modelSupportsReasoning, so the /models effort sub-step lights up for a new reasoning-family member too. P1 — per-fallback-entry reasoning gate (correctness): the chain's beginEntry/ previewRequest now go through withEntryModel, which STRIPS reasoningEffort for a fallback entry model that does not reason. Before, a failover from a reasoning primary (o1, effort=high) to a non-reasoning entry (gpt-4o) carried the tier → the provider 400-rejects an unsupported param → a fatal, non-retryable error aborted the WHOLE remaining chain. Now the tier is dropped per non-reasoning entry, so failover rescues the turn as intended. P1 — canonical doc homes (CLAUDE.md rule 8): agent-yaml-spec.md gains the reasoning_effort field (skeleton + table); llm-provider-seam.md gains LlmRequest.reasoningEffort (canonical-wins note). P2 — Gemini thinkingConfig deep-merge (correctness): setting the tier no longer REPLACES a caller's providerOptions.thinkingConfig wholesale — it deep-merges so thinkingLevel wins on its key while includeThoughts/thinkingBudget survive (a shallow replace silenced the reasoning output the caller enabled while still billing thought tokens). A non-off tier also defaults includeThoughts:true so raising effort surfaces the reasoning it bills for (never overriding an explicit caller choice; not forced for 'off'). Tests: heuristic (family ON, ambiguous OFF, registry-authoritative), catalog live-only gating, withEntryModel strip/keep/no-op, Gemini deep-merge + all-tier + sibling survival. Refs: ADR-0066 Co-Authored-By: Claude --- docs/reference/contracts/agent-yaml-spec.md | 2 + .../shared-core/llm-provider-seam.md | 1 + packages/llm/src/adapters/gemini.test.ts | 29 ++++++++++++- packages/llm/src/adapters/gemini.ts | 22 ++++++++-- packages/llm/src/fallback-chain.test.ts | 36 ++++++++++++++++ packages/llm/src/fallback-chain.ts | 22 +++++++++- packages/llm/src/model-catalog.test.ts | 15 +++++-- packages/llm/src/model-catalog.ts | 11 ++--- packages/llm/src/pricing.test.ts | 21 +++++++++- packages/llm/src/pricing.ts | 42 +++++++++++++++---- 10 files changed, 176 insertions(+), 25 deletions(-) diff --git a/docs/reference/contracts/agent-yaml-spec.md b/docs/reference/contracts/agent-yaml-spec.md index e942db42..3c379fc2 100644 --- a/docs/reference/contracts/agent-yaml-spec.md +++ b/docs/reference/contracts/agent-yaml-spec.md @@ -24,6 +24,7 @@ provider: string # required, e.g. 'anthropic' | 'openai' | 'gemini' | system_prompt: string # required; supports {{variable}} interpolation temperature: number # optional, default per-provider max_tokens: number # optional output cap +reasoning_effort: string # optional: off | low | medium | high | max (ADR-0066); sent only to a reasoning-capable model input_schema: object # optional JSON Schema for the agent's expected input shape output_schema: object # optional JSON Schema for the agent's produced output shape @@ -56,6 +57,7 @@ fallback_chain: # ordered alternates tried after the primary is exha | `system_prompt` | yes | Multiline YAML scalar; `{{ctx.*}}` / `{{inputs.*}}` interpolation supported. | | `temperature` | no | Sampling temperature. | | `max_tokens` | no | Output token cap. | +| `reasoning_effort` | no | Normalized reasoning-effort tier — `off` / `low` / `medium` / `high` / `max` ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md)). Each adapter maps it to its provider's native tier; sent only to a reasoning-capable model (else withheld). A chat surface can override it per-session via `/effort` / the `/models` picker without a reseat. | | `input_schema` | no | JSON Schema describing the input this agent expects. Purely additive metadata — it drives type-safe node chaining and editor (VS Code) completion; it does not change run-time execution. | | `output_schema` | no | JSON Schema describing the output this agent produces. Same role as `input_schema` for the downstream side of a node chain. | | `tools` | no | Tool ids — see [../shared-core/built-in-tools.md](../shared-core/built-in-tools.md). | diff --git a/docs/reference/shared-core/llm-provider-seam.md b/docs/reference/shared-core/llm-provider-seam.md index 412a2fb9..982aae80 100644 --- a/docs/reference/shared-core/llm-provider-seam.md +++ b/docs/reference/shared-core/llm-provider-seam.md @@ -38,6 +38,7 @@ interface LlmRequest { toolChoice?: 'auto' | 'none' | 'required' | { name: string }; temperature?: number; maxTokens?: number; // REQUIRED downstream for Anthropic; we default it + reasoningEffort?: ReasoningEffort; // normalized tier off|low|medium|high|max (ADR-0066); each adapter maps to its provider's native tier — CANONICAL, wins over a colliding providerOptions key; absent ⇒ provider default stopSequences?: string[]; responseFormat?: ResponseFormat; // structured-output request (ADR-0030) outputModalities?: OutputModality[]; // request media output on the INLINE path (ADR-0031); default ['text'] diff --git a/packages/llm/src/adapters/gemini.test.ts b/packages/llm/src/adapters/gemini.test.ts index c32da0d8..d54817bd 100644 --- a/packages/llm/src/adapters/gemini.test.ts +++ b/packages/llm/src/adapters/gemini.test.ts @@ -477,24 +477,49 @@ describe('Gemini adapter — request building (buildGeminiRequest)', () => { }); }); - it('maps the reasoning-effort tier to thinkingConfig.thinkingLevel (max→HIGH, off→MINIMAL, unset omitted) (ADR-0066)', () => { + it('maps the reasoning-effort tier to thinkingConfig.thinkingLevel + includeThoughts on a thinking tier (ADR-0066)', () => { + // A non-off tier also sets includeThoughts:true so raising effort SURFACES the reasoning it bills for (the only + // switch that returns Gemini thought parts). All five tiers assert thinkingLevel; medium is the picker default. expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'high' }).config['thinkingConfig']).toEqual({ thinkingLevel: 'HIGH', + includeThoughts: true, }); // Gemini tops out at HIGH — `max` coarsens to it (no separate xhigh/max tier). expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'max' }).config['thinkingConfig']).toEqual({ thinkingLevel: 'HIGH', + includeThoughts: true, + }); + expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'medium' }).config['thinkingConfig']).toEqual({ + thinkingLevel: 'MEDIUM', + includeThoughts: true, }); expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'low' }).config['thinkingConfig']).toEqual({ thinkingLevel: 'LOW', + includeThoughts: true, }); - // Gemini has no universal disable (a Pro model rejects budget 0) — `off` degrades to the lowest tier. + // Gemini has no universal disable (a Pro model rejects budget 0) — `off` degrades to the lowest tier and does + // NOT force thought output on (minimal thinking). expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'off' }).config['thinkingConfig']).toEqual({ thinkingLevel: 'MINIMAL', }); expect('thinkingConfig' in buildGeminiRequest(REQ).config).toBe(false); // unset ⇒ omitted (provider default) }); + it('deep-merges the tier onto a caller providerOptions.thinkingConfig — sibling keys survive (ADR-0066)', () => { + // A caller who enabled thought output + a budget must not lose them when effort is also set: the canonical + // thinkingLevel wins on its one key, includeThoughts:false is respected, and thinkingBudget survives. + const built = buildGeminiRequest({ + ...REQ, + reasoningEffort: 'high', + providerOptions: { thinkingConfig: { includeThoughts: false, thinkingBudget: 2048 } }, + }); + expect(built.config['thinkingConfig']).toEqual({ + thinkingLevel: 'HIGH', // canonical wins on this key + includeThoughts: false, // the caller's explicit choice is NOT overridden + thinkingBudget: 2048, // a non-colliding sibling survives + }); + }); + it('round-trips tool_call → functionCall and tool_result → functionResponse by name', () => { const request = buildGeminiRequest({ model: 'gemini-2.5-flash', diff --git a/packages/llm/src/adapters/gemini.ts b/packages/llm/src/adapters/gemini.ts index 844ba56c..23439dc9 100644 --- a/packages/llm/src/adapters/gemini.ts +++ b/packages/llm/src/adapters/gemini.ts @@ -582,9 +582,25 @@ export function buildGeminiRequest(req: LlmRequest): GeminiRequest { config['maxOutputTokens'] = req.maxTokens; } if (req.reasoningEffort !== undefined) { - // ADR-0066: Gemini's tier-native thinking control (thinkingLevel). Set on thinkingConfig; canonical config wins - // over a providerOptions thinkingConfig on the shallow merge below (the normalized field wins on collision). - config['thinkingConfig'] = { thinkingLevel: GEMINI_THINKING_LEVEL[req.reasoningEffort] }; + // ADR-0066: Gemini's tier-native thinking control (thinkingLevel). DEEP-merge onto a caller's + // providerOptions.thinkingConfig so the canonical thinkingLevel wins on THAT key while the caller's sibling + // knobs (includeThoughts / thinkingBudget) survive — a shallow replace (config wins on the top-level merge + // below) would silently drop them, so turning effort up could paradoxically SILENCE the reasoning output the + // caller enabled while still billing thought tokens. (Mirrors the Anthropic adapter preserving output_config.format.) + const poThinking: Record = + req.providerOptions !== undefined && isRecord(req.providerOptions['thinkingConfig']) + ? req.providerOptions['thinkingConfig'] + : {}; + config['thinkingConfig'] = { + ...poThinking, + thinkingLevel: GEMINI_THINKING_LEVEL[req.reasoningEffort], + // Surface the reasoning the tier bills for: `includeThoughts` is the ONLY switch that returns thought parts + // (the adapter's reasoning stream depends on it). Default it on for a thinking tier so raising effort actually + // shows more reasoning — but never override a caller's explicit choice, and never force it for 'off' (minimal). + ...(req.reasoningEffort !== 'off' && poThinking['includeThoughts'] === undefined + ? { includeThoughts: true } + : {}), + }; } if (req.outputModalities !== undefined && req.outputModalities.some((m) => m !== 'text')) { // Lower the node's non-text output_modalities to Gemini `responseModalities` (inline media-out, diff --git a/packages/llm/src/fallback-chain.test.ts b/packages/llm/src/fallback-chain.test.ts index 97479442..214c8e58 100644 --- a/packages/llm/src/fallback-chain.test.ts +++ b/packages/llm/src/fallback-chain.test.ts @@ -4,6 +4,7 @@ import { CostTracker } from './cost-tracker.js'; import { FallbackChain, stripReasoningParts, + withEntryModel, withFallback, type AttemptRecord, type FallbackChainOptions, @@ -554,6 +555,41 @@ describe('stripReasoningParts', () => { }); }); +describe('withEntryModel (ADR-0066 §4 — per-fallback-entry reasoning gate)', () => { + it('STRIPS a reasoning-effort tier for a fallback entry model that does not reason', () => { + const req: LlmRequest = { + model: 'o1', + messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }], + reasoningEffort: 'high', + }; + // Failover to a non-reasoning model (gpt-4o) must NOT carry the tier — the provider would 400-reject it, and a + // fatal unsupported-parameter error would abort the whole remaining chain. + const out = withEntryModel(req, 'gpt-4o'); + expect(out.model).toBe('gpt-4o'); + expect('reasoningEffort' in out).toBe(false); + expect(req.reasoningEffort).toBe('high'); // input untouched + }); + + it('KEEPS the tier for a fallback entry model that DOES reason', () => { + const req: LlmRequest = { + model: 'claude-opus-4-8', + messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }], + reasoningEffort: 'max', + }; + const out = withEntryModel(req, 'gpt-5.5'); // a reasoning model ⇒ the tier rides + expect(out.model).toBe('gpt-5.5'); + expect(out.reasoningEffort).toBe('max'); + }); + + it('is a plain model swap when no tier is set', () => { + const req: LlmRequest = { + model: 'gpt-4o', + messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }], + }; + expect(withEntryModel(req, 'claude-sonnet-4-6')).toEqual({ ...req, model: 'claude-sonnet-4-6' }); + }); +}); + // --- auth nuance: no blind retry + optional one-shot refresh ---------------------------------- describe('FallbackChain.generate — auth handling', () => { diff --git a/packages/llm/src/fallback-chain.ts b/packages/llm/src/fallback-chain.ts index 3b0bece5..637a72b1 100644 --- a/packages/llm/src/fallback-chain.ts +++ b/packages/llm/src/fallback-chain.ts @@ -13,6 +13,7 @@ import type { Usage, } from './types.js'; import { requestSupportReason } from './capabilities.js'; +import { modelSupportsReasoning } from './pricing.js'; export type { BackoffStrategy }; @@ -206,6 +207,23 @@ export function stripReasoningParts(req: LlmRequest): LlmRequest { return { ...req, messages }; } +/** + * Point a request at a chain entry's model AND strip a reasoning-effort tier the entry model does NOT support + * ([ADR-0066](../../../docs/decisions/0066-normalized-reasoning-effort-control.md) §4). A failover to a + * non-reasoning model must not carry the primary's tier: the provider would reject the unsupported parameter, and a + * `400` on an unsupported param is fatal + non-retryable — so the whole remaining chain would abort rather than the + * failover rescuing the turn. The per-model capability is the SAME {@link modelSupportsReasoning} the host projects + * to the engine gate, so the primary is gated at the engine and each fallback entry is re-gated here. Exported for + * a focused unit test (like {@link stripReasoningParts}). + */ +export function withEntryModel(req: LlmRequest, model: string): LlmRequest { + const next = { ...req, model }; + if (next.reasoningEffort !== undefined && !modelSupportsReasoning(model)) { + delete next.reasoningEffort; + } + return next; +} + /** The backoff delay before the `retryIndex`-th retry of an entry (0 = before the 2nd attempt). */ function backoffDelayMs( strategy: BackoffStrategy, @@ -794,7 +812,7 @@ class ChainRun { * pollute `#lastProvider` (which would wrongly strip reasoning for a later same-provider entry). */ previewRequest(entry: FallbackPlanEntry): LlmRequest { - return { ...this.#req, model: entry.model }; + return withEntryModel(this.#req, entry.model); } /** @@ -810,7 +828,7 @@ class ChainRun { this.#req = stripReasoningParts(this.#req); } this.#lastProvider = providerId; - return { ...this.#req, model: entry.model }; + return withEntryModel(this.#req, entry.model); } /** Allocate the next 1-based attempt record skeleton for this entry. */ diff --git a/packages/llm/src/model-catalog.test.ts b/packages/llm/src/model-catalog.test.ts index 2b0c56dc..4b1a5129 100644 --- a/packages/llm/src/model-catalog.test.ts +++ b/packages/llm/src/model-catalog.test.ts @@ -52,12 +52,21 @@ describe('mergeModelCatalog (ADR-0064 §6)', () => { expect(byId(entries, 'deepseek-v4-flash')?.supportsReasoning).toBe(false); }); - it('a LIVE-only model (no registry tier) is never reasoning-capable — capability is a shipped-registry fact', () => { + it('a LIVE-only model gates via the §4 id heuristic — a known reasoning family ON, an ambiguous id OFF (ADR-0066)', () => { const entries = mergeModelCatalog({ - live: liveMap([['openai', [{ id: 'gpt-6-preview', displayName: 'GPT-6 preview' }]]]), + live: liveMap([ + [ + 'openai', + [ + { id: 'o5-mini', displayName: 'o5 mini' }, // a future o-series id (whole family reasons) ⇒ ON + { id: 'gpt-4o-2026', displayName: 'GPT-4o' }, // not a reasoning family ⇒ OFF (over-match would 400) + ], + ], + ]), now: BEFORE_DEEPSEEK_DEPRECATION, }); - expect(byId(entries, 'gpt-6-preview')?.supportsReasoning).toBe(false); // not inferred from a discovery listing + expect(byId(entries, 'o5-mini')?.supportsReasoning).toBe(true); // heuristic covers a new reasoning-family member + expect(byId(entries, 'gpt-4o-2026')?.supportsReasoning).toBe(false); // conservative — no false positive }); it('availability: a static model NOT in a CONNECTED provider live list is dimmed, one present is available', () => { diff --git a/packages/llm/src/model-catalog.ts b/packages/llm/src/model-catalog.ts index 79984998..aa8f7309 100644 --- a/packages/llm/src/model-catalog.ts +++ b/packages/llm/src/model-catalog.ts @@ -1,4 +1,4 @@ -import { MODEL_PRICING, type ModelPricing } from './pricing.js'; +import { MODEL_PRICING, modelSupportsReasoning, type ModelPricing } from './pricing.js'; import type { ModelListing, ProviderId } from './types.js'; /** @@ -189,10 +189,11 @@ export function mergeModelCatalog(input: MergeModelCatalogInput): ModelCatalogEn ...(unavailableReason !== undefined ? { unavailableReason } : {}), deprecated, ...(deprecatedAt !== undefined ? { deprecatedAt } : {}), - // Reasoning capability rides the STATIC registry tier only (never live/user) — the same source as - // `modelSupportsReasoning`, so the picker's effort sub-step lights up exactly for the models the engine gate - // will actually honor (ADR-0066). - supportsReasoning: t.registry?.reasoning === true, + // Reasoning capability via the SAME authority as the engine gate (ADR-0066 §4): the registry flag for a known + // id (authoritative — true or false), else the conservative id heuristic for a live-discovered id. So the + // picker's effort sub-step lights up exactly for the models the engine will actually honor — including a newly + // released reasoning family member absent from the registry. + supportsReasoning: modelSupportsReasoning(modelId), }); } diff --git a/packages/llm/src/pricing.test.ts b/packages/llm/src/pricing.test.ts index 4bd903b3..c35a4982 100644 --- a/packages/llm/src/pricing.test.ts +++ b/packages/llm/src/pricing.test.ts @@ -21,10 +21,29 @@ describe('modelSupportsReasoning (ADR-0066)', () => { // DeepSeek reasons (v4 thinking) but its adapter mapping is deferred, so the capability stays OFF (the effort // is not controllable there yet — the picker must not offer it). expect(modelSupportsReasoning('deepseek-v4-flash')).toBe(false); - // Unknown / custom base-URL model ⇒ the SAFE default (never send the tier to a model that would reject it). + // Unknown / custom base-URL model NOT matching a reasoning family ⇒ the SAFE default. expect(modelSupportsReasoning('some-custom-base-url-model-xyz')).toBe(false); expect(modelSupportsReasoning('')).toBe(false); }); + + it('the §4 id heuristic gates a NON-registry model in a known reasoning family (conservative)', () => { + // A live-discovered id absent from MODEL_PRICING but in a family whose whole set reasons is gated ON… + expect(modelSupportsReasoning('o5-mini')).toBe(true); // a future o-series id + expect(modelSupportsReasoning('gpt-5.9-turbo')).toBe(true); // a future reasoning gpt-5 id + expect(modelSupportsReasoning('claude-opus-5')).toBe(true); // a future Opus + expect(modelSupportsReasoning('gemini-3.0-flash-thinking')).toBe(true); // an explicit "thinking" id + // …while an AMBIGUOUS / non-reasoning id stays OFF (over-matching would earn a provider 400). + expect(modelSupportsReasoning('gpt-4o')).toBe(false); // not a reasoning family + expect(modelSupportsReasoning('gpt-5-chat-latest')).toBe(false); // the non-reasoning gpt-5 conversational variant + expect(modelSupportsReasoning('claude-sonnet-9')).toBe(false); // base Sonnet is version-dependent — registry only + expect(modelSupportsReasoning('gemini-2.0-flash')).toBe(false); // Gemini by version is not heuristic-matched + }); + + it('the registry is AUTHORITATIVE for a canonical id — a false/absent flag is not overridden by the heuristic', () => { + // deepseek-v4-flash is a canonical id whose registry flag is not set; the id heuristic never runs for a canonical + // id, so it stays false (the deferred-adapter decision) even though it is a v4 "thinking" model. + expect(modelSupportsReasoning('deepseek-v4-flash')).toBe(false); + }); }); describe('contextWindowForModel (ADR-0062 §7)', () => { diff --git a/packages/llm/src/pricing.ts b/packages/llm/src/pricing.ts index 4d87d48f..8baf19e4 100644 --- a/packages/llm/src/pricing.ts +++ b/packages/llm/src/pricing.ts @@ -247,17 +247,41 @@ export const KNOWN_MODEL_IDS: readonly CanonicalModelId[] = Object.keys(MODEL_PRICING).filter(isCanonicalModelId); /** - * Whether a model supports a reasoning-effort control ([ADR-0066](../../../docs/decisions/0066-normalized-reasoning-effort-control.md)) - * — the static per-model capability the host projects to the engine's `resolveReasoning` gate (and the `/models` - * picker's effort selector). `false` for an unknown/custom (non-catalog) model — the SAFE default, so the tier is - * never sent to a model that would reject it. A pure host-side helper, like {@link contextWindowForModel}. + * A CONSERVATIVE model-id reasoning heuristic ([ADR-0066](../../../docs/decisions/0066-normalized-reasoning-effort-control.md) §4) + * — the second arm of {@link modelSupportsReasoning}, applied ONLY to an id absent from the static registry (a + * live-discovered model whose list endpoint omits a reasoning flag). Each arm is a family/pattern where the WHOLE + * matched set reasons, so a new member of a known reasoning family (e.g. a next o-series id) gates correctly before + * the registry is updated. Deliberately **narrow**: it does NOT prefix-match ambiguous families whose lineup mixes + * reasoning and non-reasoning members (base Claude Sonnet, Gemini by version, DeepSeek — reasoning deferred), because + * OVER-matching would send the tier to a non-reasoning model and earn a provider rejection — strictly worse than the + * safe under-match (no effort UX until the registry adds the model, the same maintenance shape as pricing). The + * `-chat` exclusion keeps OpenAI's non-reasoning `gpt-5-chat` conversational variant out. + */ +export function reasoningModelIdHeuristic(model: string): boolean { + const m = model.toLowerCase(); + if (/^o\d/.test(m)) return true; // OpenAI o-series (o1 / o3 / o4 / o5+) — the entire family reasons + if (/^gpt-5/.test(m) && !m.includes('chat')) return true; // the reasoning gpt-5 line (gpt-5-chat is non-reasoning) + if (m.startsWith('claude-opus')) return true; // Claude Opus reasons (extended thinking) + if (m.includes('thinking')) return true; // an explicit "thinking" model id (e.g. a Gemini thinking variant) + return false; +} + +/** + * Whether a model supports a reasoning-effort control ([ADR-0066](../../../docs/decisions/0066-normalized-reasoning-effort-control.md) + * §4) — the per-model capability the host projects to the engine's `resolveReasoning` gate (and the `/models` + * picker's effort selector). The STATIC registry is authoritative for a canonical id (its `reasoning` flag, `true` + * OR `false` — so an explicit non-reasoning member always wins); a NON-registry id (a live-discovered model) falls + * back to the conservative {@link reasoningModelIdHeuristic}. A pure host-side helper, like {@link contextWindowForModel}. */ export function modelSupportsReasoning(model: string): boolean { - if (!isCanonicalModelId(model)) return false; - // Widen the literal-union entry to `ModelPricing` so `.reasoning` (absent on the non-reasoning members) reads as - // `boolean | undefined` — each entry `satisfies ModelPricing`, so this is assignment, not a cast. - const entry: ModelPricing = MODEL_PRICING[model]; - return entry.reasoning === true; + if (isCanonicalModelId(model)) { + // Widen the literal-union entry to `ModelPricing` so `.reasoning` (absent on the non-reasoning members) reads as + // `boolean | undefined` — each entry `satisfies ModelPricing`, so this is assignment, not a cast. The registry + // is authoritative for a known id, so a false/absent flag is NOT overridden by the id heuristic. + const entry: ModelPricing = MODEL_PRICING[model]; + return entry.reasoning === true; + } + return reasoningModelIdHeuristic(model); } /** From 07e9bc41662d6fa372c9bcf35931b69048b6b2aa Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 19:59:29 +0300 Subject: [PATCH 58/68] =?UTF-8?q?test(core,cli,llm):=202.5.E-E5=20P3=20?= =?UTF-8?q?=E2=80=94=20reasoning-effort=20regression=20coverage=20(ADR-006?= =?UTF-8?q?6)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Close the test-coverage gaps the Step E review flagged, so every ADR-stated reasoning-effort invariant is locked by an assertion: - core/agent-runner: the workflow-path gate (resolveGenKnobs) — capable⇒sent, not-capable⇒withheld, no-resolver⇒withheld (safe default), no-authored⇒none even when capable. (Previously only the AgentSession mirror was covered.) - cli/resolve: [chat].reasoning_effort precedence project > workspace, per-field fall-through, and NO global-layer fallback (that is default_model's alone); reasoningEffort added to the empty-defaults assertion. - cli/default-agent + agent-source: the config default is baked onto the DEFAULT agent and OMITTED when absent; an authored --agent's reasoning_effort is NOT overridden by the config default (the §5 non-leak invariant). - cli/session-host: createChatModeControl seeds the footer effort from the agent (gated by capability — a non-reasoning model shows none) and onSetEffort pushes a no-reseat session override that updates both the session and the footer. - llm/openai + anthropic: assert all FIVE tiers (add the untested low + medium — medium is the picker default) so a valid-but-wrong within-domain map is caught. Verified (already-correct, no new test): reasoning/thinking tokens fold into outputTokens so cost reflects effort; one-shot `agent run --json` uses the same gated engine path; Anthropic adaptive thinking avoids the budget conflict. Refs: ADR-0066 Co-Authored-By: Claude --- apps/cli/src/chat/agent-source.test.ts | 24 +++++++++++++ apps/cli/src/chat/default-agent.test.ts | 6 ++++ apps/cli/src/chat/session-host.test.ts | 26 ++++++++++++++ apps/cli/src/config/resolve.test.ts | 14 ++++++++ packages/core/src/engine/agent-runner.test.ts | 34 +++++++++++++++++++ packages/llm/src/adapters/anthropic.test.ts | 6 ++++ packages/llm/src/adapters/openai.test.ts | 5 +++ 7 files changed, 115 insertions(+) diff --git a/apps/cli/src/chat/agent-source.test.ts b/apps/cli/src/chat/agent-source.test.ts index 7b75f576..2ea828ac 100644 --- a/apps/cli/src/chat/agent-source.test.ts +++ b/apps/cli/src/chat/agent-source.test.ts @@ -45,6 +45,30 @@ describe('resolveChatAgent', () => { expect(agent.model).toBe(DEFAULT_CHAT_MODEL); }); + it('bakes opts.reasoningEffort onto the DEFAULT agent (ADR-0066)', () => { + const agent = resolveChatAgent(undefined, { + cwd: dir, + projectConfigDir: undefined, + defaultModel: 'claude-opus-4-8', + reasoningEffort: 'high', + }); + expect(agent.id).toBe(DEFAULT_CHAT_AGENT_ID); + expect(agent.reasoning_effort).toBe('high'); + }); + + it('does NOT leak [chat].reasoning_effort into an explicit --agent — the authored tier wins (ADR-0066 §5)', () => { + const path = join(dir, 'coder.agent.yaml'); + writeFileSync(path, `${AGENT_YAML}\nreasoning_effort: off`); // the authored agent pins 'off' + const agent = resolveChatAgent(path, { + cwd: dir, + projectConfigDir: undefined, + defaultModel: undefined, + reasoningEffort: 'high', // a config default MUST NOT override the authored agent's own tier + }); + expect(agent.id).toBe('coder'); + expect(agent.reasoning_effort).toBe('off'); // the authored 'off' survives — config did not leak in + }); + it('resolves an explicit --agent path through the strict core parseAgent', () => { const path = join(dir, 'coder.agent.yaml'); writeFileSync(path, AGENT_YAML); diff --git a/apps/cli/src/chat/default-agent.test.ts b/apps/cli/src/chat/default-agent.test.ts index 4b1826ce..e3fad0d5 100644 --- a/apps/cli/src/chat/default-agent.test.ts +++ b/apps/cli/src/chat/default-agent.test.ts @@ -56,6 +56,12 @@ describe('buildDefaultChatAgent', () => { expect(buildDefaultChatAgent('gemini-2.5-pro').provider).toBe('gemini'); }); + it('bakes the [chat].reasoning_effort default onto the agent, and OMITS it when absent (ADR-0066)', () => { + expect(buildDefaultChatAgent('claude-opus-4-8', 'high').reasoning_effort).toBe('high'); + // Absent ⇒ the key is not present (never an explicit `undefined` under exactOptionalPropertyTypes). + expect('reasoning_effort' in buildDefaultChatAgent('claude-opus-4-8')).toBe(false); + }); + it('throws a clean exit-2 CliError when the provider cannot be inferred', () => { let caught: unknown; try { diff --git a/apps/cli/src/chat/session-host.test.ts b/apps/cli/src/chat/session-host.test.ts index 6f7ea91c..6e16d8a6 100644 --- a/apps/cli/src/chat/session-host.test.ts +++ b/apps/cli/src/chat/session-host.test.ts @@ -934,6 +934,32 @@ describe('buildChatSession + 2.5.A tool-host wiring (ADR-0055)', () => { expect(existsSync(join(workspace, 'x.txt'))).toBe(false); }); + it('createChatModeControl seeds the store effort from the agent + onSetEffort pushes a no-reseat override (ADR-0066)', async () => { + const built = await build({ + chat: { ...EMPTY_CHAT, defaultModel: 'claude-opus-4-8', reasoningEffort: 'medium' }, + }); + const store = createChatStore(false); + const control = createChatModeControl(built, store); + // Seeded from the agent's effective tier — opus is reasoning-capable, so the footer shows it. + expect(store.getSnapshot().reasoningEffort).toBe('medium'); + expect(built.session.reasoningEffort).toBe('medium'); + // onSetEffort pushes the SESSION override (no reseat) + updates the footer — effective next turn. + control.onSetEffort('max'); + expect(built.session.reasoningEffort).toBe('max'); + expect(store.getSnapshot().reasoningEffort).toBe('max'); + }); + + it('createChatModeControl surfaces NO effort on a non-reasoning model — the footer stays clear (ADR-0066)', async () => { + const built = await build({ + chat: { ...EMPTY_CHAT, defaultModel: 'gpt-4o', reasoningEffort: 'high' }, + }); + const store = createChatStore(false); + createChatModeControl(built, store); + // The config default is baked onto the agent, but gpt-4o has no reasoning tier — the footer shows nothing + // (the tier is gated off at send anyway), so a user is never shown an inert effort. + expect(store.getSnapshot().reasoningEffort).toBeUndefined(); + }); + it('createChatModeControl ask: denies an EGRESS-class dispatch too (http_request), not just fs_write', async () => { // The confirm floor rejects EVERY governed class; prove the egress class end-to-end (governedAction maps // http_request → 'egress', a distinct ToolActionClass) — the deny happens BEFORE dispatch, so the egress diff --git a/apps/cli/src/config/resolve.test.ts b/apps/cli/src/config/resolve.test.ts index e7d1f1cb..96f0a993 100644 --- a/apps/cli/src/config/resolve.test.ts +++ b/apps/cli/src/config/resolve.test.ts @@ -14,6 +14,7 @@ describe('resolveConfig', () => { mediaGcGraceMs: undefined, chat: { defaultModel: undefined, + reasoningEffort: undefined, fsScope: undefined, maxTurns: undefined, maxMessages: undefined, @@ -73,6 +74,19 @@ describe('resolveConfig', () => { expect(resolved.onExceed).toBe('warn'); // only on project }); + it('resolves [chat].reasoning_effort (ADR-0066) last-writer-wins project > workspace, no global fallback', () => { + const workspace: ProjectConfig = { chat: { reasoning_effort: 'low' } }; + const project: ProjectConfig = { chat: { reasoning_effort: 'high' } }; + expect(resolveConfig({ workspace, project }).chat.reasoningEffort).toBe('high'); // project wins + // A project present but omitting it falls through to workspace (per-field, like the sibling keys). + expect(resolveConfig({ workspace, project: { chat: { max_turns: 5 } } }).chat.reasoningEffort).toBe('low'); + // Absent everywhere ⇒ undefined (no global-layer fallback — that extra fallback is default_model's alone). + expect(resolveConfig({}).chat.reasoningEffort).toBeUndefined(); + expect( + resolveConfig({ global: { preferences: { default_model: 'g' } } }).chat.reasoningEffort, + ).toBeUndefined(); + }); + it('resolves [chat].auto_compact + compact_threshold (ADR-0062) last-writer-wins, per field', () => { const workspace: ProjectConfig = { chat: { auto_compact: false, compact_threshold: 0.7 } }; const project: ProjectConfig = { chat: { compact_threshold: 0.9 } }; diff --git a/packages/core/src/engine/agent-runner.test.ts b/packages/core/src/engine/agent-runner.test.ts index dd9f24bf..501cee68 100644 --- a/packages/core/src/engine/agent-runner.test.ts +++ b/packages/core/src/engine/agent-runner.test.ts @@ -986,3 +986,37 @@ describe('generativeUnits — authored media volume (ADR-0045 §5)', () => { ); }); }); + +describe('createAgentNodeExecutor — reasoning-effort gate (ADR-0066, the workflow path)', () => { + const reasoningAgent: Agent = { ...AGENT, reasoning_effort: 'high' }; + async function capturedReq( + overrides: Partial, + agent: Agent, + ): Promise { + const cap = reqCapturingProvider(); + const exec = createAgentNodeExecutor(deps(cap.provider, overrides)); + const { ctx } = ctxFor(vertexFor({ kind: 'agent', node: agentNode(), resolvedAgent: agent })); + await exec.execute(ctx); + return cap.req(); + } + + it('SENDS the authored tier when the model is reasoning-capable', async () => { + const req = await capturedReq({ resolveReasoning: () => true }, reasoningAgent); + expect(req?.reasoningEffort).toBe('high'); + }); + + it('WITHHOLDS the tier when the model is NOT reasoning-capable (a non-reasoning model would reject it)', async () => { + const req = await capturedReq({ resolveReasoning: () => false }, reasoningAgent); + expect(req?.reasoningEffort).toBeUndefined(); + }); + + it('WITHHOLDS the tier when NO resolver is wired (the safe default)', async () => { + const req = await capturedReq({}, reasoningAgent); + expect(req?.reasoningEffort).toBeUndefined(); + }); + + it('sends nothing when the agent authored NO tier — even on a capable model', async () => { + const req = await capturedReq({ resolveReasoning: () => true }, AGENT); + expect(req?.reasoningEffort).toBeUndefined(); + }); +}); diff --git a/packages/llm/src/adapters/anthropic.test.ts b/packages/llm/src/adapters/anthropic.test.ts index 38b3fcec..4a49bb7a 100644 --- a/packages/llm/src/adapters/anthropic.test.ts +++ b/packages/llm/src/adapters/anthropic.test.ts @@ -187,6 +187,12 @@ describe('AnthropicAdapter', () => { maxTokens: 1024, messages: [{ role: 'user' as const, content: [{ type: 'text' as const, text: 'go' }] }], }; + // All five tiers, so a valid-but-wrong within-domain swap on any row is caught (medium is the picker default). + await adapter.generate({ ...base, reasoningEffort: 'low' }, 'k'); + expect(sent['thinking']).toEqual({ type: 'adaptive' }); + expect(sent['output_config']).toEqual({ effort: 'low' }); + await adapter.generate({ ...base, reasoningEffort: 'medium' }, 'k'); + expect(sent['output_config']).toEqual({ effort: 'medium' }); await adapter.generate({ ...base, reasoningEffort: 'high' }, 'k'); expect(sent['thinking']).toEqual({ type: 'adaptive' }); expect(sent['output_config']).toEqual({ effort: 'high' }); diff --git a/packages/llm/src/adapters/openai.test.ts b/packages/llm/src/adapters/openai.test.ts index b89cd2a9..4655fa1b 100644 --- a/packages/llm/src/adapters/openai.test.ts +++ b/packages/llm/src/adapters/openai.test.ts @@ -243,6 +243,11 @@ describe('OpenAI-compatible adapter', () => { model: 'gpt-5.5', messages: [{ role: 'user' as const, content: [{ type: 'text' as const, text: 'go' }] }], }; + // All FIVE tiers, so a valid-but-wrong within-domain swap on any row is caught (medium is the picker default). + await oai.generate({ ...base, reasoningEffort: 'low' }, 'k'); + expect(sent['reasoning_effort']).toBe('low'); + await oai.generate({ ...base, reasoningEffort: 'medium' }, 'k'); + expect(sent['reasoning_effort']).toBe('medium'); await oai.generate({ ...base, reasoningEffort: 'high' }, 'k'); expect(sent['reasoning_effort']).toBe('high'); await oai.generate({ ...base, reasoningEffort: 'max' }, 'k'); From df27e01b00cadf50e2f5d6debe3e73f52509fcd9 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 20:01:11 +0300 Subject: [PATCH 59/68] =?UTF-8?q?docs:=202.5.E-E5=20P0=20=E2=80=94=20recon?= =?UTF-8?q?cile=20/models=20+=20config=20docs=20with=20the=20per-turn=20ef?= =?UTF-8?q?fort=20override=20(ADR-0066)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit After the P0 fix, the effort sub-step is a per-turn SESSION override (no reseat) for a same-model pick, not "bound onto the reseated agent." Update the docs the E5 commit wrote: - commands.md: add /effort to the in-REPL command list; the effort sub-step is a no-reseat setter (same model) / carried onto a reseat (different model); the active tier shows in the footer. - config-spec.md: /effort + the picker set a per-turn override without editing config. - roadmap: reflect the setter (not reseat) + the id heuristic + the review that caught the reseat mis-design. Refs: ADR-0066 Co-Authored-By: Claude --- docs/reference/cli/commands.md | 2 +- docs/reference/contracts/config-spec.md | 2 +- docs/roadmap/current.md | 7 +++++-- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index ce8009e1..ed307a1e 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -152,7 +152,7 @@ The **command manifest** is the one source the **shell** command surfaces derive ### In-REPL slash commands -The interactive `/` palette + slash commands inside the **Home and chat** are a SEPARATE, **curated** surface ([ADR-0056](../../decisions/0056-cli-in-app-slash-command-system-and-manifest.md) amendment, 2.5.C) — the runtime registry is `apps/cli/src/commands/repl-commands.ts` (`REPL_COMMANDS`), the single source for the palette, the `/help` list, and the unknown-slash hint. It surfaces only the commands that make sense in a live REPL — lifecycle (`/exit`, `/cancel`, `/export`, `/clear`), info/discovery (`/help`, `/workflows`, `/cost`, `/doctor`), and — in a chat — `/mode ` to switch the chat mode (2.5.E, [chat-session.md](chat-session.md)) plus the ADR-0062 context commands (`/compact`, `/trim`). `/models` opens an in-tree model picker over the merged live/static catalog; its **action depends on the surface**. Inside a **live chat** (standalone `relavium chat` or an in-Home chat) it triggers a **live reseat** ([ADR-0059](../../decisions/0059-cli-mid-session-model-reseat.md)) — rebinding the session to the picked model (dropping the old fallback chain), carrying the text-only transcript + cumulative cost/turns under the SAME `sessionId`. At the **bare Home** (no active chat) it instead writes the **next** session's default model ([ADR-0064](../../decisions/0064-live-model-catalog.md) §10, via the [ADR-0063](../../decisions/0063-cli-config-write-contract.md) config writer). On a **reasoning-capable** model *during a reseat*, a second **effort sub-step** ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md)) picks the reasoning-effort tier (`off`/`low`/`medium`/`high`/`max`) bound onto the reseated agent (`Esc` backs out to the model list, `Ctrl-C` cancels); the bare-Home default-write stays single-step (its effort default is the `[chat].reasoning_effort` config key). Under `--json`/plain (non-TTY) the live reseat is unavailable — one machine stream stays one session lifecycle ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)) — so a typed `/models` there falls through to an actionable "interactive terminal" hint. The picker renders the ADR-0064 first-class UX: per-model pricing (an unpriced model shows a "cost cap will not apply" hint), a dimmed non-selectable "unavailable on your key" row, a `deprecated` flag, a loading spinner, a per-provider partial-failure banner, and a "last updated" freshness badge; opening it over an empty/stale cache renders immediately and kicks a background refresh (the long-lived Home is where that is sound), `Ctrl+R` forces a live refresh, and `Esc` cancels. The heavy, session-starting shell commands above (`run`, `chat`, `provider`, …) are **never** in-REPL slashes — they stay shell-only (`relavium …`). A bare `/` at an **empty** prompt opens the filterable palette (the footer hint-bar surfaces `/ for commands` exactly there, 2.5.C S6); an unknown slash — or an undeclared argument on a known command (`/exit now`) — prints a sanitized, secret-free hint. A command may declare flags (`/doctor --deep`) or a single positional value (`/mode plan`); the palette runs the bare form, so a flag/value is opt-in by typing it. There is no separate `/shortcuts` command — the palette renders its own nav hints (`↑/↓ · Enter · Esc`) and the footer surfaces `/`, so keys stay discoverable in context. +The interactive `/` palette + slash commands inside the **Home and chat** are a SEPARATE, **curated** surface ([ADR-0056](../../decisions/0056-cli-in-app-slash-command-system-and-manifest.md) amendment, 2.5.C) — the runtime registry is `apps/cli/src/commands/repl-commands.ts` (`REPL_COMMANDS`), the single source for the palette, the `/help` list, and the unknown-slash hint. It surfaces only the commands that make sense in a live REPL — lifecycle (`/exit`, `/cancel`, `/export`, `/clear`), info/discovery (`/help`, `/workflows`, `/cost`, `/doctor`), and — in a chat — `/mode ` to switch the chat mode (2.5.E, [chat-session.md](chat-session.md)), `/effort ` to set the reasoning-effort tier ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md); a per-turn session override, no reseat), plus the ADR-0062 context commands (`/compact`, `/trim`). `/models` opens an in-tree model picker over the merged live/static catalog; its **action depends on the surface**. Inside a **live chat** (standalone `relavium chat` or an in-Home chat) it triggers a **live reseat** ([ADR-0059](../../decisions/0059-cli-mid-session-model-reseat.md)) — rebinding the session to the picked model (dropping the old fallback chain), carrying the text-only transcript + cumulative cost/turns under the SAME `sessionId`. At the **bare Home** (no active chat) it instead writes the **next** session's default model ([ADR-0064](../../decisions/0064-live-model-catalog.md) §10, via the [ADR-0063](../../decisions/0063-cli-config-write-contract.md) config writer). On a **reasoning-capable** model, a second **effort sub-step** ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md)) picks the reasoning-effort tier (`off`/`low`/`medium`/`high`/`max`; `Esc` backs out to the model list, `Ctrl-C` cancels): a **same-model** pick applies the tier as a **per-turn session override — no reseat, no teardown** (effort changes neither provider, pricing, nor the plan, §5), while a **different-model** pick carries it onto the reseat. The active tier rides the footer (parity with the mode), and `/effort ` sets it directly without opening the picker. The bare-Home default-write stays single-step (its effort default is the `[chat].reasoning_effort` config key). Under `--json`/plain (non-TTY) the live reseat is unavailable — one machine stream stays one session lifecycle ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)) — so a typed `/models` there falls through to an actionable "interactive terminal" hint. The picker renders the ADR-0064 first-class UX: per-model pricing (an unpriced model shows a "cost cap will not apply" hint), a dimmed non-selectable "unavailable on your key" row, a `deprecated` flag, a loading spinner, a per-provider partial-failure banner, and a "last updated" freshness badge; opening it over an empty/stale cache renders immediately and kicks a background refresh (the long-lived Home is where that is sound), `Ctrl+R` forces a live refresh, and `Esc` cancels. The heavy, session-starting shell commands above (`run`, `chat`, `provider`, …) are **never** in-REPL slashes — they stay shell-only (`relavium …`). A bare `/` at an **empty** prompt opens the filterable palette (the footer hint-bar surfaces `/ for commands` exactly there, 2.5.C S6); an unknown slash — or an undeclared argument on a known command (`/exit now`) — prints a sanitized, secret-free hint. A command may declare flags (`/doctor --deep`) or a single positional value (`/mode plan`); the palette runs the bare form, so a flag/value is opt-in by typing it. There is no separate `/shortcuts` command — the palette renders its own nav hints (`↑/↓ · Enter · Esc`) and the footer surfaces `/`, so keys stay discoverable in context. ### `relavium run` diff --git a/docs/reference/contracts/config-spec.md b/docs/reference/contracts/config-spec.md index f18676b8..72c102c4 100644 --- a/docs/reference/contracts/config-spec.md +++ b/docs/reference/contracts/config-spec.md @@ -155,7 +155,7 @@ allowed_command_globs = [] # opt-in glob form of the !-shell allowlist ( > > `max_messages` (revived in 2.5.F) is the bound `/trim` enforces (keep the last N messages, no LLM call) and the deterministic fallback if a summarization fails. `auto_compact` + `compact_threshold` ([ADR-0062](../../decisions/0062-context-compaction-and-cli-history-commands.md)) drive **automatic** model-summarised compaction: after a turn completes, if its **real** input tokens exceed `compact_threshold` (default `0.8`, a fraction in (0, 1]) × the serving model's context window, the session compacts before the next turn. `auto_compact` absent ⇒ enabled; a model with no known context window (a custom base-URL id) skips auto-compaction, but manual `/compact` still works. The summarization spend is accounted to the session budget and surfaced, never silent. > -> `reasoning_effort` ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md)) is the normalized reasoning-effort tier — `off | low | medium | high | max` — baked onto the **built-in default chat agent** only (an explicit `--agent` owns its own `reasoning_effort` in its YAML). It resolves per-field project → workspace like the other `[chat]` keys (no global-layer fallback — that extra fallback is `default_model`'s alone). Each adapter maps the tier to its provider's **native** control; a model with no controllable reasoning tier ignores it (the engine gates on the model's capability). Absent ⇒ no reasoning control (the provider default). Interactively, the `/models` picker's **effort sub-step** binds a tier onto a live reseat without editing config. +> `reasoning_effort` ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md)) is the normalized reasoning-effort tier — `off | low | medium | high | max` — baked onto the **built-in default chat agent** only (an explicit `--agent` owns its own `reasoning_effort` in its YAML). It resolves per-field project → workspace like the other `[chat]` keys (no global-layer fallback — that extra fallback is `default_model`'s alone). Each adapter maps the tier to its provider's **native** control; a model with no controllable reasoning tier ignores it (the engine gates on the model's capability). Absent ⇒ no reasoning control (the provider default). Interactively, the `/effort` command and the `/models` picker's **effort sub-step** set the tier as a **per-turn session override** (no reseat) without editing config; the active tier shows in the footer. > > The `[chat]` block resolves **per field** (each key independently, last-writer-wins project → workspace) — a project that sets only `max_turns` still inherits `default_model`/`max_messages` from the workspace layer. (Contrast `[defaults].media_cost_estimate`, which resolves **whole-object**: the highest layer present replaces the table outright.) **`default_model` has one extra fallback**: absent at both `[chat]` layers, it falls through to the global **`[preferences].default_model`** ([ADR-0063](../../decisions/0063-cli-config-write-contract.md) §1) — the write target of `/models` and the wizard — so a user's "preferred model everywhere" governs chat too, exactly as it already governs a workflow's `[defaults].model`. Full precedence: `[chat].default_model` (project → workspace) → `[preferences].default_model` (global). No other `[chat]` field reads the global layer. The `!`-shell allowlist is the **one exception**: `allowed_commands` (exact) + `allowed_command_globs` (globs) are a **coupled unit**, so a project that sets **either** array owns the **whole** allowlist and does **not** inherit the other array from the workspace. Otherwise a project narrowing `allowed_commands` would silently keep the workspace's broader globs — lock to `git status`, yet still allow `git push` via an inherited `git *`. Only when a project sets **neither** allowlist array do both fall through to the workspace; a present array otherwise REPLACES (never merges) the lower layer's. This is what guarantees a narrower project can never inherit a broader workspace entry. > diff --git a/docs/roadmap/current.md b/docs/roadmap/current.md index 2e6f1233..59f73ef4 100644 --- a/docs/roadmap/current.md +++ b/docs/roadmap/current.md @@ -200,8 +200,11 @@ the onboarding-wizard live key-validation + retry UX, wizard-dynamic provider do now Accepted), and **normalized reasoning-effort control** ([ADR-0066](../decisions/0066-normalized-reasoning-effort-control.md), now Accepted): a provider-agnostic effort tier (`off`/`low`/`medium`/`high`/`max`) authored in agent YAML or the `[chat].reasoning_effort` config default, each adapter mapping it to its provider's **native** tier, gated per-model -by a host-injected capability resolver, and chosen live via the `/models` picker's **effort sub-step** on a -reasoning-capable model during a reseat. +by a host-injected capability resolver (plus a conservative id heuristic for a live-discovered model), and changed +live — via the `/effort` command or the `/models` picker's **effort sub-step** — as a **per-turn session override** +(no reseat, §5), with the active tier shown in the footer. The comprehensive multi-agent review after the first +implementation caught that the picker had (wrongly) routed the effort through a full model reseat; the P0 fix +rebuilt it as the ADR-mandated session-level setter. Carry-over hardening is tracked in [deferred-tasks.md](deferred-tasks.md) — Phase 2 picks items up as it first touches each file. Notable inheritances: 1.AH's host-wiring half From 929f11beaf5a49c8ce6db8171ad959c88484478a Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 20:20:18 +0300 Subject: [PATCH 60/68] =?UTF-8?q?docs(cli):=202.5.E-E5=20=E2=80=94=20re-re?= =?UTF-8?q?view=20comment=20fixes=20(ADR-0066)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The final adversarial re-review (7 fix-area lenses) confirmed all 15 original findings resolved and surfaced only two LOW comment/clarity issues: - chat.ts onSetEffort: the comment wrongly claimed the session effort override "survives a reseat" — it does not (a reseat rebuilds a fresh session; a model change carries its own picked effort via the ReseatTarget). Corrected. - home-controller.ts acceptModel: a reviewer flagged that an effort pick "could fall through to a config-default write when reseatChat is absent." Verified a FALSE POSITIVE — the in-chat /models that opens the effort sub-step is itself gated on reseatChat (the palette + typed intercepts), so an active-session pick never runs without it. Added a comment documenting that invariant so the double guard reads as intentional, not redundant. No behavior change. Toolchain green. Refs: ADR-0066 Co-Authored-By: Claude --- apps/cli/src/commands/chat.ts | 5 +++-- apps/cli/src/render/tui/home-controller.ts | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/apps/cli/src/commands/chat.ts b/apps/cli/src/commands/chat.ts index 6746e4bf..2861954d 100644 --- a/apps/cli/src/commands/chat.ts +++ b/apps/cli/src/commands/chat.ts @@ -706,8 +706,9 @@ export function createChatModeControl( applyChatMode(modeEnv, mode); }, // ADR-0066 §5: push the SESSION override (no reseat) + update the footer. The tier is gated per-model at send, - // so the footer reflects it only on a reasoning-capable model — but the override is still stored so a later - // reseat to a capable model would honor it. + // so the footer reflects it only on a reasoning-capable model. The override lives on THIS session instance only + // (it does not survive a reseat — a model change rebuilds the session, carrying its own picked effort via the + // ReseatTarget); so on a non-reasoning model the tier is stored but inert until the very next same-model turn. onSetEffort: (effort) => { built.session.setReasoningEffort(effort); store.setReasoningEffort(capable ? effort : undefined); diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index c46ffec9..65ebb932 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -661,6 +661,10 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { reasoningEffort?: ReasoningEffort, ): void => { const active = state.session; + // A LIVE in-Home chat owns the pick (the bare-Home config write is the `else` below). The effort setter path + // does not itself need `reseatChat`, but it is only REACHED with `reseatChat` wired: the in-chat `/models` that + // opens the effort sub-step is gated on `reseatChat` (see the palette/typed intercepts), so an active-session + // pick never runs without it. Guarding on both here therefore never diverts an effort pick to the config write. if (active !== undefined && deps.reseatChat !== undefined) { if (modelId === active.store.getSnapshot().state.model) { if (reasoningEffort === undefined) { From e83755e047ae34cd57f1fb90d0f78689af7dd351 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 22:39:46 +0300 Subject: [PATCH 61/68] =?UTF-8?q?refactor,fix,docs:=20address=20PR=20#66?= =?UTF-8?q?=20review=20=E2=80=94=20CI=20format:check=20+=20Sonar=20finding?= =?UTF-8?q?s?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Resolve the CI format:check hard-fail and the SonarCloud quality-gate findings on the Phase-2.5.G integration PR. Verified each finding against current code; fixed the still-valid ones minimally, preserving behavior; skipped the rest. CI blocker — Prettier: `prettier --write .` across the repo (format:check now passes). One knock-on: Prettier split provider.ts's long UNSAFE_URL_CHARS regex onto its own line, detaching an `eslint-disable-next-line no-control-regex` — switched to a block `eslint-disable`/`enable` that survives the wrap. Functional (db): ModelCatalogStore.upsert now honors the `number | null` media- cost contract — an explicit `null` CLEARS a stored rate while an OMITTED field still preserves it (was `??`, which treated a clearing null as an omission). Added a regression test (null clears, sibling omission preserves). Cognitive-complexity refactors (behavior-preserving helper extraction, tests green): mergeModelCatalog 38→~10 (buildTiers/buildEntry/resolveAvailability/ pricingSourceOf); model-picker foldModelPhaseKey (acceptVisibleModel/foldFilterKey); chat-ink routeModelPickerKey (acceptModelPick/blockedHint); home-controller acceptModel (applyLiveSessionPick/applyEffortOnlyUpdate/writeNextSessionDefault/ defaultWriteNotice); gemini buildGeminiRequest (buildThinkingConfig); chat.ts runReplLoop (resolveSwapRebuild) + driveOneSession (buildInteractiveMentionReader/ finalizeReseatOutcome); wizard validateWithRetry (buildRetryPrompt); adapters/ shared boundedListModels (resolveListModelsError); validated-fetch (inputToUrl, no-`void`, a defensive 15-min egress-signal ceiling). Nested-ternary / mechanical smells: model-picker-view (unavailableParts/rowColorFor); Number.NaN; String#startsWith; Math.max; a default param; a ChatStopReason type alias for the repeated `'exit'|'clear'|'reseat'` union. Layering: hoisted EFFORT_TIER_HINT out of render/tui into @relavium/shared (beside REASONING_EFFORTS) so commands-layer code no longer imports the TUI. Docs: home.md wizard writes `[preferences].default_model` (not `[chat]`); ADR-0011/0024 MD028 (contiguous amendment blockquotes); budget-governor test comment $9/MTok×10k = 9_000_000µ¢. Skipped with reason: the four Drizzle "out of sync" findings (db:generate reports no changes — artifacts ARE in sync); the provider.ts optional-chain (a `?.===null` rewrite is semantically wrong — undefined ≠ null under `===`); openai.ts String.raw (a 4-char escape reads worse as String.raw); the "parameterize N tests" + "prefer specific assertion" nits (the repo favors explicit, individually-named cases for clarity + precise failure messages). Co-Authored-By: Claude --- apps/cli/src/chat/session-host.test.ts | 10 +- apps/cli/src/commands/chat.ts | 147 ++++++++++----- apps/cli/src/commands/dispatch.test.ts | 4 +- apps/cli/src/commands/dispatch.ts | 15 +- apps/cli/src/commands/models-pricing.test.ts | 17 +- apps/cli/src/commands/models-pricing.ts | 7 +- apps/cli/src/commands/provider.test.ts | 27 ++- apps/cli/src/commands/provider.ts | 15 +- apps/cli/src/commands/specs.ts | 9 +- apps/cli/src/config/resolve.test.ts | 4 +- apps/cli/src/config/write.ts | 14 +- apps/cli/src/engine/media-wiring.test.ts | 2 +- apps/cli/src/engine/model-catalog-port.ts | 4 +- .../cli/src/engine/model-catalog-view.test.ts | 18 +- apps/cli/src/engine/model-catalog-view.ts | 5 +- apps/cli/src/engine/providers.test.ts | 16 +- apps/cli/src/engine/providers.ts | 18 +- apps/cli/src/engine/validated-fetch.test.ts | 38 +++- apps/cli/src/engine/validated-fetch.ts | 42 ++++- apps/cli/src/home/drive-home.tsx | 25 ++- apps/cli/src/onboarding/wizard.test.ts | 115 +++++++++--- apps/cli/src/onboarding/wizard.ts | 68 ++++--- apps/cli/src/render/tui/chat-ink.tsx | 83 ++++---- apps/cli/src/render/tui/chat-store.ts | 9 +- .../src/render/tui/home-controller.test.ts | 39 ++-- apps/cli/src/render/tui/home-controller.ts | 177 +++++++++++------- apps/cli/src/render/tui/model-picker-view.tsx | 45 +++-- apps/cli/src/render/tui/model-picker.test.ts | 46 +++-- apps/cli/src/render/tui/model-picker.ts | 163 ++++++++-------- .../0011-internal-llm-abstraction.md | 4 +- ...24-agent-first-entry-point-agentsession.md | 2 +- docs/reference/cli/home.md | 2 +- packages/core/src/engine/agent-runner.ts | 6 +- .../core/src/engine/agent-session.test.ts | 12 +- .../core/src/engine/budget-governor.test.ts | 2 +- packages/db/src/model-catalog-store.test.ts | 24 +++ packages/db/src/model-catalog-store.ts | 17 +- packages/db/src/provider-store.test.ts | 12 +- packages/llm/src/adapters/gemini.test.ts | 36 ++-- packages/llm/src/adapters/gemini.ts | 46 +++-- packages/llm/src/adapters/openai.test.ts | 9 +- packages/llm/src/adapters/openai.ts | 5 +- packages/llm/src/adapters/shared.ts | 45 +++-- packages/llm/src/cost-tracker.test.ts | 6 +- packages/llm/src/fallback-chain.test.ts | 5 +- packages/llm/src/model-catalog.ts | 164 +++++++++------- packages/llm/src/pricing.ts | 2 +- packages/shared/src/constants.ts | 13 ++ 48 files changed, 1071 insertions(+), 523 deletions(-) diff --git a/apps/cli/src/chat/session-host.test.ts b/apps/cli/src/chat/session-host.test.ts index 6e16d8a6..bc7e07a0 100644 --- a/apps/cli/src/chat/session-host.test.ts +++ b/apps/cli/src/chat/session-host.test.ts @@ -752,14 +752,20 @@ describe('swapAgentModel (ADR-0059 model-switch rule)', () => { }); it('DROPS a prior effort when none is passed (a non-reasoning target can’t carry a stale tier)', () => { - const withEffort = { ...buildDefaultChatAgent('claude-opus-4-8'), reasoning_effort: 'max' as const }; + const withEffort = { + ...buildDefaultChatAgent('claude-opus-4-8'), + reasoning_effort: 'max' as const, + }; const next = swapAgentModel(withEffort, 'deepseek-chat', 'deepseek'); // no reasoningEffort arg expect('reasoning_effort' in next).toBe(false); // dropped, not carried onto the new model expect(withEffort.reasoning_effort).toBe('max'); // the input is untouched (a fresh copy) }); it('OVERWRITES a prior effort with the newly-picked tier', () => { - const withEffort = { ...buildDefaultChatAgent('claude-opus-4-8'), reasoning_effort: 'low' as const }; + const withEffort = { + ...buildDefaultChatAgent('claude-opus-4-8'), + reasoning_effort: 'low' as const, + }; const next = swapAgentModel(withEffort, 'claude-sonnet-4-6', 'anthropic', 'off'); expect(next.reasoning_effort).toBe('off'); }); diff --git a/apps/cli/src/commands/chat.ts b/apps/cli/src/commands/chat.ts index 2861954d..bbea722c 100644 --- a/apps/cli/src/commands/chat.ts +++ b/apps/cli/src/commands/chat.ts @@ -9,7 +9,12 @@ import { type UserCommandOutcome, } from '@relavium/core'; import { modelSupportsReasoning, type ProviderId } from '@relavium/llm'; -import { REASONING_EFFORTS, type AgentSessionRecord, type ReasoningEffort } from '@relavium/shared'; +import { + EFFORT_TIER_HINT, + REASONING_EFFORTS, + type AgentSessionRecord, + type ReasoningEffort, +} from '@relavium/shared'; import { exportSession } from '../chat/export.js'; import { formatDoctorReport, runDoctorChecks, type DoctorProbes } from '../chat/doctor.js'; import { assembleDoctorProbes } from '../chat/doctor-host.js'; @@ -68,7 +73,6 @@ import { stripTerminalControls, } from '../render/tui/chat-projection.js'; import { createChatStore, type ChatStoreController } from '../render/tui/chat-store.js'; -import { EFFORT_TIER_HINT } from '../render/tui/model-picker.js'; import { createMentionReader, type MentionReader } from '../render/tui/mention.js'; import { createMcpSecretResolver, type McpSecretResolver } from '../secrets/mcp-secret.js'; @@ -121,6 +125,15 @@ export interface ReseatTarget { readonly reasoningEffort?: ReasoningEffort; } +/** + * WHY a chat driver's input loop ended (ADR-0062 §7 · [ADR-0059](../../../../docs/decisions/0059-cli-mid-session-model-reseat.md)): + * `'exit'` (`/exit`, `/cancel`, or an input EOF), `'clear'` (`/clear`, TTY-interactive only — swap in a FRESH + * session), or `'reseat'` (a `/models` model switch, TTY-interactive only — swap in a NEW-model session carrying the + * transcript). Both interactive gates keep this at `'exit'` under `--json`/plain (one machine stream stays one + * session lifecycle, ADR-0049). + */ +export type ChatStopReason = 'exit' | 'clear' | 'reseat'; + /** * The catalog port the ink `/models` reseat picker (ADR-0059) reads — the SHARED load/refresh trio * ({@link ModelCatalogPort}) plus the session's currently-bound model (the picker's `✓` "you are here" marker). @@ -163,7 +176,7 @@ export interface ChatDriveContext { * to swap in a NEW-model session carrying the transcript. Both interactive gates keep `stopReason()` at `'exit'` * under `--json` / plain non-TTY, so those drivers only ever return `'exit'`. */ - readonly stopReason: () => 'exit' | 'clear' | 'reseat'; + readonly stopReason: () => ChatStopReason; /** * Switch the bound model mid-session (ADR-0059) — the ink model-picker overlay calls this on accept. It signals a * host-side reseat (a new instance bound to `target`), so like `/clear` it sets the stop state; the driver then @@ -250,7 +263,7 @@ export interface ChatDriveContext { * present iff `kind === 'reseat'` — {@link driveOneSession} attaches it from the line handler's captured target. */ export interface ChatDriveOutcome { - readonly kind: 'exit' | 'clear' | 'reseat'; + readonly kind: ChatStopReason; readonly target?: ReseatTarget; } export type ChatDriver = (ctx: ChatDriveContext) => Promise; @@ -727,7 +740,7 @@ export interface ChatLineHandler extends ChatModeControl { /** WHY the loop stopped (ADR-0062 §7 · ADR-0059) — `'reseat'` after a `/models` switch, `'clear'` after a `/clear`, * else `'exit'`. The standalone re-drive loop swaps in a new-model / fresh session accordingly; the Home reads it * to swap-in-place vs. return to the bare Home. */ - readonly stopReason: () => 'exit' | 'clear' | 'reseat'; + readonly stopReason: () => ChatStopReason; /** Request a mid-session model switch (ADR-0059) — sets the stop state + captures the target for the reseat. */ readonly onReseat: (target: ReseatTarget) => void; /** The captured reseat target once {@link onReseat} fired (else `undefined`) — {@link driveOneSession} reads it. */ @@ -948,7 +961,9 @@ export function createChatLineHandler( } const tier = REASONING_EFFORTS.find((e) => e === requested); if (tier === undefined) { - emitOutput(`/effort: unknown tier '${requested.replace(/[^\x20-\x7e]/g, '?').slice(0, 16)}'`); + emitOutput( + `/effort: unknown tier '${requested.replace(/[^\x20-\x7e]/g, '?').slice(0, 16)}'`, + ); return; } modeControl.onSetEffort(tier); @@ -1078,7 +1093,10 @@ export function createChatLineHandler( // Priority: a reseat is a swap-to-new-model, a clear is a swap-to-fresh, else the REPL ends. A `/models` reseat // and a `/clear` are mutually exclusive in one settle (each sets `stop`), but order the check so an explicit // reseat is never mis-read as a clear. - stopReason: () => (reseatRequested !== undefined ? 'reseat' : clearRequested ? 'clear' : 'exit'), + stopReason: (): ChatStopReason => { + if (reseatRequested !== undefined) return 'reseat'; + return clearRequested ? 'clear' : 'exit'; + }, onReseat: (target) => { reseatRequested = target; stop = true; @@ -1149,7 +1167,10 @@ async function buildFreshChatWiring(deps: FreshChatWiringDeps, intro: string): P now: deps.now, uuid: deps.uuid, // ADR-0059 attribution — resolved over the SAME shared db (a `/clear` rebuild re-reads it fresh). - resolveModelCatalogId: makeCatalogIdResolver(deps.opened.db, { uuid: deps.uuid, now: deps.now }), + resolveModelCatalogId: makeCatalogIdResolver(deps.opened.db, { + uuid: deps.uuid, + now: deps.now, + }), }); } catch (err) { // Acquire-then-guard: the fresh MCP children are already spawned — reclaim them before the failure propagates @@ -1172,7 +1193,13 @@ async function buildFreshChatWiring(deps: FreshChatWiringDeps, intro: string): P doctorProbes, startSession: () => built.session.start(), intro, - modelPicker: buildChatModelsPort(deps.opened, deps.providers, built.agent.model, deps.now, deps.uuid), + modelPicker: buildChatModelsPort( + deps.opened, + deps.providers, + built.agent.model, + deps.now, + deps.uuid, + ), ...(deps.chat.maxMessages === undefined ? {} : { chatMaxMessages: deps.chat.maxMessages }), }; } @@ -1353,7 +1380,13 @@ async function buildReseatWiring( startSession: () => {}, intro: modelSwitchNotice(target.modelId, resumed.resumeState.turnCount), // The picker's `boundModel` is now the SWITCHED model — a further reseat marks it as the ✓ "you are here". - modelPicker: buildChatModelsPort(deps.opened, deps.providers, resumed.agent.model, deps.now, deps.uuid), + modelPicker: buildChatModelsPort( + deps.opened, + deps.providers, + resumed.agent.model, + deps.now, + deps.uuid, + ), ...(deps.chat.maxMessages === undefined ? {} : { chatMaxMessages: deps.chat.maxMessages }), }; } @@ -1403,6 +1436,39 @@ function createReseatRebuild(params: { * connections down — but NOT the shared db (the loop owns it across swaps). Returns the driver's outcome so the loop * can decide between ending and re-driving over a fresh session (`/clear`). */ +/** + * The `@`-mention completion reader for an INTERACTIVE driver (2.5.D, ADR-0061): a READ-ONLY fs jail at the + * session's fs-scope tier + workspace, so `@`-completion browses + injects through the identical confidentiality + * floor + listing-gate. `undefined` on a plain/`--json` driver (a leading `@` is a literal there) or when the fs + * arm is unavailable. READ-ONLY by construction — the mention path never writes. Pure (no I/O). + */ +function buildInteractiveMentionReader( + deps: ChatReplDeps, + built: ReplWiring['built'], +): MentionReader | undefined { + if (!chatIsInteractive(deps.io, deps.global)) return undefined; + const fsArm = assembleToolEnv({ + profile: 'chat-read-only', + fsScopeTier: built.context.fsScopeTier, + workspaceDir: built.context.workingDir, + }).host.fs; + return fsArm === undefined ? undefined : createMentionReader(fsArm); +} + +/** + * Attach the captured reseat target to a `'reseat'` outcome (ADR-0059) — done in the ONE place holding the line + * handler, so every driver stays target-agnostic. A missing target (never expected: `onReseat` always captures one) + * degrades to a plain end rather than a broken rebuild loop. + */ +function finalizeReseatOutcome( + outcome: ChatDriveOutcome, + reseatTarget: () => ReseatTarget | undefined, +): ChatDriveOutcome { + if (outcome.kind !== 'reseat') return outcome; + const target = reseatTarget(); + return target === undefined ? { kind: 'exit' } : { kind: 'reseat', target }; +} + async function driveOneSession(wiring: ReplWiring, deps: ChatReplDeps): Promise { const { built, store, persister, startSession, intro } = wiring; const { @@ -1422,19 +1488,8 @@ async function driveOneSession(wiring: ReplWiring, deps: ChatReplDeps): Promise< const reseatEnabled = chatIsInteractive(deps.io, deps.global); // The `@`-mention completion reader (2.5.D, ADR-0061): a READ-ONLY fs jail at the SAME fs-scope tier + workspace - // as the session's tools, so `@`-completion browses + injects through the identical confidentiality floor + - // listing-gate (a `.ssh`/`.env` entry is never listed nor read). READ-ONLY by construction — the mention path - // never writes. TTY-only: an interactive driver wires the completion; a plain/`--json` driver treats a leading - // `@` as a literal, so it needs no reader. Building it is pure (no I/O). - const mentionReader = ((): MentionReader | undefined => { - if (!chatIsInteractive(deps.io, deps.global)) return undefined; - const fsArm = assembleToolEnv({ - profile: 'chat-read-only', - fsScopeTier: built.context.fsScopeTier, - workspaceDir: built.context.workingDir, - }).host.fs; - return fsArm === undefined ? undefined : createMentionReader(fsArm); - })(); + // as the session's tools (a `.ssh`/`.env` entry is never listed nor read). TTY-only; see the helper. + const mentionReader = buildInteractiveMentionReader(deps, built); // The `!`-shell runner (2.5.D step 5, ADR-0061) — a thin wrapper over the session's `runUserCommand`. TTY-only. const runShellCommand = chatIsInteractive(deps.io, deps.global) @@ -1469,18 +1524,14 @@ async function driveOneSession(wiring: ReplWiring, deps: ChatReplDeps): Promise< // The reseat trigger (onReseat) + its picker are wired together, interactive-only: the ink overlay reads the // catalog through `modelPicker` and calls `onReseat` on accept. A plain/`--json` driver gets neither. ...(reseatEnabled ? { onReseat } : {}), - ...(reseatEnabled && wiring.modelPicker !== undefined ? { modelPicker: wiring.modelPicker } : {}), + ...(reseatEnabled && wiring.modelPicker !== undefined + ? { modelPicker: wiring.modelPicker } + : {}), ...(mentionReader === undefined ? {} : { mentionReader }), ...(runShellCommand === undefined ? {} : { runShellCommand }), }); - // A `/models` reseat: the driver returns `{ kind: 'reseat' }` (from `stopReason()`); attach the captured target - // HERE — the one place holding the line handler — so every driver stays target-agnostic. A missing target - // (never expected: `onReseat` always captures one) degrades to a plain end rather than a broken rebuild loop. - if (outcome.kind === 'reseat') { - const target = reseatTarget(); - return target === undefined ? { kind: 'exit' } : { kind: 'reseat', target }; - } - return outcome; + // A `/models` reseat attaches the captured target here (the one place holding the line handler); see the helper. + return finalizeReseatOutcome(outcome, reseatTarget); } finally { cancelOnce(); // emit the terminal even on /exit, /clear, or EOF (idempotent); flips the row to 'ended' // Attempt EVERY teardown step (a reject in one must not skip the next) and never let a cleanup fault mask the @@ -1499,6 +1550,26 @@ async function driveOneSession(wiring: ReplWiring, deps: ChatReplDeps): Promise< * closed exactly ONCE in the outer finally; each session's own teardown (terminal + persister + MCP) is owned by * `driveOneSession`. `/exit`, `/cancel`, and an input EOF all end the session with **exit code 4**. */ +/** + * The rebuild closure for a swap outcome (ADR-0062 §7 `/clear` → a FRESH session; ADR-0059 `reseat` → a NEW-model + * session carrying the transcript), or `undefined` to END the REPL (an `'exit'`, or a swap whose builder is not + * wired — both TTY-only, so a non-TTY outcome is always `'exit'`). The reseat target is captured into a const so + * the closure keeps its narrowed (non-undefined) type — no unsafe non-null assertion. + */ +function resolveSwapRebuild( + outcome: ChatDriveOutcome, + oldSessionId: string, + rebuild: ((oldSessionId: string) => Promise) | undefined, + reseatRebuild: ((oldSessionId: string, target: ReseatTarget) => Promise) | undefined, +): (() => Promise) | undefined { + if (outcome.kind === 'clear' && rebuild !== undefined) return () => rebuild(oldSessionId); + if (outcome.kind === 'reseat' && reseatRebuild !== undefined && outcome.target !== undefined) { + const target = outcome.target; + return () => reseatRebuild(oldSessionId, target); + } + return undefined; +} + async function runReplLoop( wiring: ReplWiring, deps: ChatReplDeps, @@ -1512,17 +1583,9 @@ async function runReplLoop( for (;;) { const outcome = await driveOneSession(current, deps); // The old session is ALREADY torn down (driveOneSession's finally fired its terminal → the row is 'ended' + - // resumable). Resolve the rebuild for this swap kind (both /clear + reseat are TTY-only; a non-TTY outcome is - // always 'exit'), or leave `next` unset to END the REPL. The target is captured into a const so the closure - // keeps its narrowed (non-undefined) type — no unsafe non-null assertion. + // resumable). Resolve the rebuild for this swap kind, or `undefined` to END the REPL (see resolveSwapRebuild). const oldSessionId = current.built.sessionId; - let next: (() => Promise) | undefined; - if (outcome.kind === 'clear' && rebuild !== undefined) { - next = () => rebuild(oldSessionId); - } else if (outcome.kind === 'reseat' && reseatRebuild !== undefined && outcome.target !== undefined) { - const target = outcome.target; - next = () => reseatRebuild(oldSessionId, target); - } + const next = resolveSwapRebuild(outcome, oldSessionId, rebuild, reseatRebuild); if (next === undefined) break; // Build the swap session over the same db and re-drive; a build failure is surfaced actionably (the prior // conversation is still resumable) and ends the REPL rather than looping on a broken build. diff --git a/apps/cli/src/commands/dispatch.test.ts b/apps/cli/src/commands/dispatch.test.ts index 88b41f1a..d07da4a6 100644 --- a/apps/cli/src/commands/dispatch.test.ts +++ b/apps/cli/src/commands/dispatch.test.ts @@ -168,9 +168,7 @@ describe('build*Args (argv → typed core args)', () => { it('provider.add: name + optional baseUrl + optional pricingUrl (each omitted when absent)', () => { expect( - buildProviderAddArgs( - input(['anthropic'], { baseUrl: 'https://x', pricingUrl: 'https://p' }), - ), + buildProviderAddArgs(input(['anthropic'], { baseUrl: 'https://x', pricingUrl: 'https://p' })), ).toEqual({ action: 'add', name: 'anthropic', diff --git a/apps/cli/src/commands/dispatch.ts b/apps/cli/src/commands/dispatch.ts index f47e4cc7..80bfaf62 100644 --- a/apps/cli/src/commands/dispatch.ts +++ b/apps/cli/src/commands/dispatch.ts @@ -207,7 +207,10 @@ function parseUsdPerMtok(raw: string, flag: string): number { const trimmed = raw.trim(); const value = Number(trimmed); if (trimmed === '' || !Number.isFinite(value)) { - throw new CliError('invalid_invocation', `${flag} must be a finite number of USD per million tokens.`); + throw new CliError( + 'invalid_invocation', + `${flag} must be a finite number of USD per million tokens.`, + ); } return value; } @@ -262,7 +265,10 @@ function storeAwareResolver( }); const { db, close } = openLocalDb(homeDir); try { - const providerStore = createProviderStore(db, { uuid: () => randomUUID(), now: () => Date.now() }); + const providerStore = createProviderStore(db, { + uuid: () => randomUUID(), + now: () => Date.now(), + }); return createProviderResolver(ctx.io.env, keychain, { providerStore }); } finally { close(); @@ -465,7 +471,10 @@ const executeModelsRefresh: CommandExecutor = (_input, ctx) => */ const executeModelsPricing: CommandExecutor = (input, ctx) => { const args = buildModelsPricingArgs(input); // a bad/absent flag is an invocation fault before any db work - const { homeDir } = loadResolvedConfig({ cwd: ctx.global.cwd, configPath: ctx.global.configPath }); + const { homeDir } = loadResolvedConfig({ + cwd: ctx.global.cwd, + configPath: ctx.global.configPath, + }); const { db, close } = openLocalDb(homeDir); try { const storeDeps = { uuid: () => randomUUID(), now: () => Date.now() }; diff --git a/apps/cli/src/commands/models-pricing.test.ts b/apps/cli/src/commands/models-pricing.test.ts index b4000636..e3d708de 100644 --- a/apps/cli/src/commands/models-pricing.test.ts +++ b/apps/cli/src/commands/models-pricing.test.ts @@ -40,7 +40,11 @@ describe('modelsPricingCommand (2.5.G S10)', () => { now: () => 1_700_000_000_000, }; providers = createProviderStore(client.db, deps); - providers.upsert({ name: 'openai', displayName: 'OpenAI', baseUrl: 'https://api.openai.com/v1' }); + providers.upsert({ + name: 'openai', + displayName: 'OpenAI', + baseUrl: 'https://api.openai.com/v1', + }); catalog = createModelCatalogStore(client.db, deps); }); @@ -48,7 +52,10 @@ describe('modelsPricingCommand (2.5.G S10)', () => { client.sqlite.close(); }); - function run(args: ModelsPricingCommandArgs, json = false): { code: number; out: string; err: string } { + function run( + args: ModelsPricingCommandArgs, + json = false, + ): { code: number; out: string; err: string } { const { io, out, err } = captureIo(); const code = modelsPricingCommand(args, { io, @@ -146,7 +153,11 @@ describe('modelsPricingCommand (2.5.G S10)', () => { it('REJECTS pricing a model id already user-priced under a DIFFERENT provider (the overlay keys by id)', () => { // Register a second provider + price the SAME model id under it, then try to price it under openai. - providers.upsert({ name: 'deepseek', displayName: 'DeepSeek', baseUrl: 'https://api.deepseek.com' }); + providers.upsert({ + name: 'deepseek', + displayName: 'DeepSeek', + baseUrl: 'https://api.deepseek.com', + }); const deepseekId = providers.list().find((p) => p.name === 'deepseek')?.id ?? ''; catalog.upsert({ providerId: deepseekId, diff --git a/apps/cli/src/commands/models-pricing.ts b/apps/cli/src/commands/models-pricing.ts index d233cae3..016ff18c 100644 --- a/apps/cli/src/commands/models-pricing.ts +++ b/apps/cli/src/commands/models-pricing.ts @@ -105,7 +105,8 @@ export function modelsPricingCommand( (m) => m.source === 'user' && m.modelId === args.model && m.providerId !== providerRow.id, ); if (dup !== undefined) { - const otherProvider = deps.providers.list().find((p) => p.id === dup.providerId)?.name ?? 'another provider'; + const otherProvider = + deps.providers.list().find((p) => p.id === dup.providerId)?.name ?? 'another provider'; throw new CliError( 'invalid_invocation', `'${args.model}' is already user-priced under '${otherProvider}'. The cost cap keys by model id, so a second provider's price can't be distinguished — remove that price (re-price under '${otherProvider}') or use a distinct model id.`, @@ -148,9 +149,7 @@ export function modelsPricingCommand( } const cachedNote = - args.cachedInputUsdPerMtok === undefined - ? '' - : `, cached $${args.cachedInputUsdPerMtok}/Mtok`; + args.cachedInputUsdPerMtok === undefined ? '' : `, cached $${args.cachedInputUsdPerMtok}/Mtok`; // Strip any terminal-control byte from the (user-typed) model id before echo — parity with `renderModelList`'s // FIX 2. `ModelListingSchema` only requires min(1), so an id can carry a control byte; the JSON path is safe on // its own (JSON.stringify escapes them). The provider is a validated (kebab) ProviderId, and the prices are diff --git a/apps/cli/src/commands/provider.test.ts b/apps/cli/src/commands/provider.test.ts index 45eee32c..6bd603b7 100644 --- a/apps/cli/src/commands/provider.test.ts +++ b/apps/cli/src/commands/provider.test.ts @@ -71,7 +71,13 @@ describe('relavium provider commands (2.C)', () => { Promise.resolve({} as Awaited>), ), readSecret: () => Promise.resolve(RAW_KEY), - global: { json: false, color: false, cwd: process.cwd(), configPath: undefined, verbosity: 'normal' }, + global: { + json: false, + color: false, + cwd: process.cwd(), + configPath: undefined, + verbosity: 'normal', + }, ...over, }); }); @@ -127,7 +133,9 @@ describe('relavium provider commands (2.C)', () => { }); it('list --verify reports "no key" for a provider with no resolvable key — and never probes it (no hang)', async () => { - const stub = stubResolver(() => Promise.resolve({} as Awaited>)); + const stub = stubResolver(() => + Promise.resolve({} as Awaited>), + ); const noKeyResolver: ProviderResolver = { resolveProvider: stub.resolveProvider, keyFor: () => { @@ -177,7 +185,12 @@ describe('relavium provider commands (2.C)', () => { const listIo = captureIo(); await runProviderCommand({ action: 'list' }, deps({ io: listIo.io, global: jsonGlobal })); const [rec] = parseNdjson(listIo.out()); - expect(rec).toMatchObject({ name: 'openai', keySet: false, verified: null, verifyDetail: null }); + expect(rec).toMatchObject({ + name: 'openai', + keySet: false, + verified: null, + verifyDetail: null, + }); }); const jsonGlobal = { @@ -206,7 +219,9 @@ describe('relavium provider commands (2.C)', () => { }); it('list --json --verify distinguishes keyless (verified:null, verifyDetail:"no key") from not-probed', async () => { - const stub = stubResolver(() => Promise.resolve({} as Awaited>)); + const stub = stubResolver(() => + Promise.resolve({} as Awaited>), + ); const noKeyResolver: ProviderResolver = { resolveProvider: stub.resolveProvider, keyFor: () => { @@ -241,7 +256,9 @@ describe('relavium provider commands (2.C)', () => { it('list --verify PROPAGATES an unexpected keyFor fault (not "no key") — a locked/faulted keychain is loud', async () => { // A non-`invalid_invocation` error (e.g. a native keychain-binding fault) must NOT be mislabeled "no key" for // every provider; it propagates as the command's fault. verifyProvider re-throws it → Promise.all rejects. - const stub = stubResolver(() => Promise.resolve({} as Awaited>)); + const stub = stubResolver(() => + Promise.resolve({} as Awaited>), + ); const faulted: ProviderResolver = { resolveProvider: stub.resolveProvider, keyFor: () => { diff --git a/apps/cli/src/commands/provider.ts b/apps/cli/src/commands/provider.ts index b3d7a966..ab8ca07b 100644 --- a/apps/cli/src/commands/provider.ts +++ b/apps/cli/src/commands/provider.ts @@ -133,8 +133,10 @@ const BIDI_ZERO_WIDTH = /[\u200b-\u200f\u2060\u2066-\u2069\u202a-\u202e\ufeff]/g /** C0/C1 control bytes + the {@link BIDI_ZERO_WIDTH} spoof family, as a NON-global tester (safe for `.test`) \u2014 none * is valid in a base URL, so `requireHttpsUrl` REJECTS a raw containing any at `add` time, making the stored value * inherently terminal-safe on every surface (list / `--json` / the add confirmation), not just after a render-strip. */ -// eslint-disable-next-line no-control-regex -- intentionally matches C0/C1 control bytes to reject them from a URL -const UNSAFE_URL_CHARS = /[\u0000-\u001f\u007f-\u009f\u200b-\u200f\u2060\u2066-\u2069\u202a-\u202e\ufeff]/u; +/* eslint-disable no-control-regex -- intentionally matches C0/C1 control bytes to reject them from a URL */ +const UNSAFE_URL_CHARS = + /[\u0000-\u001f\u007f-\u009f\u200b-\u200f\u2060\u2066-\u2069\u202a-\u202e\ufeff]/u; +/* eslint-enable no-control-regex */ /** Neutralize a provider-supplied string for inline echo: strip ANSI/C0/C1 control bytes ({@link stripTerminalControls}) * AND the {@link BIDI_ZERO_WIDTH} spoof family, then squeeze whitespace so one row stays one line. Used for a stored @@ -161,7 +163,9 @@ async function providerList(args: ProviderCommandArgs, deps: ProviderCommandDeps const outcomes = new Map(); if (args.verify) { const probed = await Promise.all( - providers.map(async (p): Promise<[string, VerifyOutcome]> => [p.name, await verifyProvider(p, deps)]), + providers.map( + async (p): Promise<[string, VerifyOutcome]> => [p.name, await verifyProvider(p, deps)], + ), ); for (const [name, outcome] of probed) outcomes.set(name, outcome); } @@ -368,7 +372,10 @@ function requireHttpsPricingUrl(raw: string): string { throw new CliError('invalid_invocation', '--pricing-url must be HTTPS.'); } if (urlHasCredentials(raw)) { - throw new CliError('invalid_invocation', '--pricing-url must not embed credentials (user:pass@…).'); + throw new CliError( + 'invalid_invocation', + '--pricing-url must not embed credentials (user:pass@…).', + ); } return url.href; } diff --git a/apps/cli/src/commands/specs.ts b/apps/cli/src/commands/specs.ts index 0f8bd871..e020efe7 100644 --- a/apps/cli/src/commands/specs.ts +++ b/apps/cli/src/commands/specs.ts @@ -458,7 +458,9 @@ function registerModels(program: Command, ctx?: CommandContext): void { .description("Re-fetch each connected provider's live model list into the local cache."); const pricing = models .command('pricing ') - .description('Set a user price for a model the registry does not know (custom / new provider models).') + .description( + 'Set a user price for a model the registry does not know (custom / new provider models).', + ) .requiredOption('--provider ', 'the provider that serves the model (must be registered)') .requiredOption('--input ', 'input (prompt) price, USD per million tokens') .requiredOption('--output ', 'output (completion) price, USD per million tokens') @@ -528,7 +530,10 @@ function registerProvider(program: Command, ctx?: CommandContext): void { .command('add ') .description('Register a provider.') .option('--base-url ', 'override the provider base URL') - .option('--pricing-url ', 'override the pricing reference page (where you find model prices)'); + .option( + '--pricing-url ', + 'override the pricing reference page (where you find model prices)', + ); const setKey = provider .command('set-key ') .description('Store a provider API key in the OS keychain (the key is read from stdin).'); diff --git a/apps/cli/src/config/resolve.test.ts b/apps/cli/src/config/resolve.test.ts index 96f0a993..34ec1cfc 100644 --- a/apps/cli/src/config/resolve.test.ts +++ b/apps/cli/src/config/resolve.test.ts @@ -79,7 +79,9 @@ describe('resolveConfig', () => { const project: ProjectConfig = { chat: { reasoning_effort: 'high' } }; expect(resolveConfig({ workspace, project }).chat.reasoningEffort).toBe('high'); // project wins // A project present but omitting it falls through to workspace (per-field, like the sibling keys). - expect(resolveConfig({ workspace, project: { chat: { max_turns: 5 } } }).chat.reasoningEffort).toBe('low'); + expect( + resolveConfig({ workspace, project: { chat: { max_turns: 5 } } }).chat.reasoningEffort, + ).toBe('low'); // Absent everywhere ⇒ undefined (no global-layer fallback — that extra fallback is default_model's alone). expect(resolveConfig({}).chat.reasoningEffort).toBeUndefined(); expect( diff --git a/apps/cli/src/config/write.ts b/apps/cli/src/config/write.ts index bc6ede2e..4664e7e8 100644 --- a/apps/cli/src/config/write.ts +++ b/apps/cli/src/config/write.ts @@ -1,5 +1,13 @@ import { randomUUID } from 'node:crypto'; -import { closeSync, fchmodSync, fsyncSync, openSync, renameSync, unlinkSync, writeFileSync } from 'node:fs'; +import { + closeSync, + fchmodSync, + fsyncSync, + openSync, + renameSync, + unlinkSync, + writeFileSync, +} from 'node:fs'; import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; @@ -136,7 +144,9 @@ function verifyRoundTrips(text: string, target: string): void { /** A write-attributed {@link ConfigError} whose detail is the loader's **value-free** field-path reason. */ function configWriteError(target: string, error: ZodError): ConfigError { - return new ConfigError(target, `could not be written — ${formatZodError(error)}.`, { cause: error }); + return new ConfigError(target, `could not be written — ${formatZodError(error)}.`, { + cause: error, + }); } /** diff --git a/apps/cli/src/engine/media-wiring.test.ts b/apps/cli/src/engine/media-wiring.test.ts index fe7085f3..0516209e 100644 --- a/apps/cli/src/engine/media-wiring.test.ts +++ b/apps/cli/src/engine/media-wiring.test.ts @@ -34,7 +34,7 @@ const EMPTY_CONFIG: ResolvedConfig = { onExceed: undefined, allowedCommands: undefined, allowedCommandGlobs: undefined, - reasoningEffort: undefined, + reasoningEffort: undefined, }, variables: {}, mcpServers: [], diff --git a/apps/cli/src/engine/model-catalog-port.ts b/apps/cli/src/engine/model-catalog-port.ts index 6e9679ac..829d4e84 100644 --- a/apps/cli/src/engine/model-catalog-port.ts +++ b/apps/cli/src/engine/model-catalog-port.ts @@ -52,7 +52,9 @@ export function createModelCatalogPort(params: { // Rebuild the UUID→slug map on every load (NOT memoized): a refresh may register a provider's FK row, and the // next load must resolve its live rows' provider — not drop them (mirrors the Home's load). const slugByUuid = new Map(providerStore.list().map((p) => [p.id, p.name] as const)); - keyedProviders ??= new Set(KNOWN_PROVIDER_IDS.filter((id) => providerHasKey(params.providers, id))); + keyedProviders ??= new Set( + KNOWN_PROVIDER_IDS.filter((id) => providerHasKey(params.providers, id)), + ); return buildMergedCatalog({ rows: catalogStore.listAll(), providerSlug: (uuid) => slugByUuid.get(uuid) ?? uuid, diff --git a/apps/cli/src/engine/model-catalog-view.test.ts b/apps/cli/src/engine/model-catalog-view.test.ts index 6bd30754..da6806e7 100644 --- a/apps/cli/src/engine/model-catalog-view.test.ts +++ b/apps/cli/src/engine/model-catalog-view.test.ts @@ -77,8 +77,18 @@ describe('buildMergedCatalog', () => { it('reports refreshedAt as the newest lastRefreshedAt across the live rows only', () => { const view = buildMergedCatalog({ rows: [ - row({ modelId: MODEL_PRESENT, providerId: 'p-anthropic', source: 'live', lastRefreshedAt: 100 }), - row({ modelId: MODEL_ABSENT, providerId: 'p-anthropic', source: 'live', lastRefreshedAt: 300 }), + row({ + modelId: MODEL_PRESENT, + providerId: 'p-anthropic', + source: 'live', + lastRefreshedAt: 100, + }), + row({ + modelId: MODEL_ABSENT, + providerId: 'p-anthropic', + source: 'live', + lastRefreshedAt: 300, + }), // a non-live row's stamp (if any) must NOT count toward freshness row({ modelId: 'x', providerId: 'p-anthropic', source: 'user', lastRefreshedAt: 999 }), ], @@ -103,7 +113,9 @@ describe('buildMergedCatalog', () => { // An unmapped uuid resolves to itself ('rogue'), which is not a ProviderId ⇒ the row is skipped, so anthropic // has NO live data and its statics stay available (the rogue row cannot dim an unrelated provider). const view = buildMergedCatalog({ - rows: [row({ modelId: MODEL_PRESENT, providerId: 'rogue', source: 'live', lastRefreshedAt: 50 })], + rows: [ + row({ modelId: MODEL_PRESENT, providerId: 'rogue', source: 'live', lastRefreshedAt: 50 }), + ], providerSlug: slugResolver({}), now: 0, }); diff --git a/apps/cli/src/engine/model-catalog-view.ts b/apps/cli/src/engine/model-catalog-view.ts index 206d47fc..8634c95b 100644 --- a/apps/cli/src/engine/model-catalog-view.ts +++ b/apps/cli/src/engine/model-catalog-view.ts @@ -141,7 +141,10 @@ export function buildMergedCatalog(input: BuildMergedCatalogInput): MergedCatalo if (!isProviderId(slug)) continue; // an unmapped UUID / a non-enum (custom) provider — not this step's concern // Freshness reflects only VALID (enum-provider) live rows — a dropped rogue row must not skew the badge. if (row.lastRefreshedAt !== undefined) { - refreshedAt = refreshedAt === undefined ? row.lastRefreshedAt : Math.max(refreshedAt, row.lastRefreshedAt); + refreshedAt = + refreshedAt === undefined + ? row.lastRefreshedAt + : Math.max(refreshedAt, row.lastRefreshedAt); } const list = live.get(slug) ?? []; list.push(rowToListing(row)); diff --git a/apps/cli/src/engine/providers.test.ts b/apps/cli/src/engine/providers.test.ts index 27d118fe..f8e9f4c6 100644 --- a/apps/cli/src/engine/providers.test.ts +++ b/apps/cli/src/engine/providers.test.ts @@ -312,10 +312,13 @@ describe('createProviderResolver custom endpoints (2.5.G S9 / ADR-0065 §3–4)' const fetch: FetchLike = (input) => { urls.push(typeof input === 'string' ? input : input instanceof URL ? input.href : input.url); return Promise.resolve( - new Response(JSON.stringify({ object: 'list', data: [{ id: 'gpt-5.4-mini', object: 'model' }] }), { - status: 200, - headers: { 'content-type': 'application/json' }, - }), + new Response( + JSON.stringify({ object: 'list', data: [{ id: 'gpt-5.4-mini', object: 'model' }] }), + { + status: 200, + headers: { 'content-type': 'application/json' }, + }, + ), ); }; return { fetch, urls }; @@ -348,7 +351,10 @@ describe('createProviderResolver custom endpoints (2.5.G S9 / ADR-0065 §3–4)' const { fetch } = recordingFetch(); const resolver = createProviderResolver({}, undefined, { providerStore: { - list: () => [row('anthropic', 'https://custom.example'), row('gemini', 'https://custom.example')], + list: () => [ + row('anthropic', 'https://custom.example'), + row('gemini', 'https://custom.example'), + ], }, validatedFetch: fetch, }); diff --git a/apps/cli/src/engine/providers.ts b/apps/cli/src/engine/providers.ts index 8c91214b..571df113 100644 --- a/apps/cli/src/engine/providers.ts +++ b/apps/cli/src/engine/providers.ts @@ -174,7 +174,11 @@ export type ValidationReason = 'ok' | 'auth' | 'network' | 'other'; */ export type ProviderKeyValidation = | { readonly ok: true; readonly detail: string; readonly reason: 'ok' } - | { readonly ok: false; readonly detail: string; readonly reason: Exclude }; + | { + readonly ok: false; + readonly detail: string; + readonly reason: Exclude; + }; /** * Validate a provider key with a minimal live request (`maxTokens: 1` 'ping'). Returns a RESULT so the caller @@ -206,7 +210,11 @@ export async function validateProviderKey( const timeout = new Promise((resolve) => { timer = setTimeout(() => { controller.abort(); - resolve({ ok: false, detail: `key test failed — timeout (${timeoutMs}ms)`, reason: 'network' }); + resolve({ + ok: false, + detail: `key test failed — timeout (${timeoutMs}ms)`, + reason: 'network', + }); }, timeoutMs); }); const probe = (async (): Promise => { @@ -376,7 +384,11 @@ function applyCustomEndpoints( if (id !== 'openai' && id !== 'deepseek') continue; // custom base_url is openai-compatible only this round (§3) validatedFetch ??= options.validatedFetch ?? createValidatedFetch(); // built lazily, once, only when needed try { - adapters[id] = createCustomOpenAiProvider({ providerId: id, baseURL: row.baseUrl, fetch: validatedFetch }); + adapters[id] = createCustomOpenAiProvider({ + providerId: id, + baseURL: row.baseUrl, + fetch: validatedFetch, + }); } catch (err) { // A bad stored base_url (non-HTTPS / private / creds) — refuse the custom endpoint, keep the default adapter. if (!(err instanceof InvalidBaseUrlError)) throw err; diff --git a/apps/cli/src/engine/validated-fetch.test.ts b/apps/cli/src/engine/validated-fetch.test.ts index abde0ce8..6ab741ea 100644 --- a/apps/cli/src/engine/validated-fetch.test.ts +++ b/apps/cli/src/engine/validated-fetch.test.ts @@ -48,7 +48,9 @@ function fakeDeps(opts: { describe('createValidatedFetch', () => { it('routes a request through connectValidated and maps the response (status + headers + body)', async () => { - const fetch = createValidatedFetch(fakeDeps({ ips: ['1.2.3.4'], status: 200, chunks: ['{"models":[]}'] })); + const fetch = createValidatedFetch( + fakeDeps({ ips: ['1.2.3.4'], status: 200, chunks: ['{"models":[]}'] }), + ); const res = await fetch('https://api.example.com/v1/models'); expect(res.status).toBe(200); expect(res.headers.get('content-type')).toBe('application/json'); @@ -57,7 +59,9 @@ describe('createValidatedFetch', () => { it('BLOCKS a host that resolves to a private/loopback address (SSRF) — the fetch rejects', async () => { const fetch = createValidatedFetch(fakeDeps({ ips: ['10.0.0.1'] })); // a private IP - await expect(fetch('https://sneaky-rebind.example.com/v1/models')).rejects.toBeInstanceOf(SafeEgressError); + await expect(fetch('https://sneaky-rebind.example.com/v1/models')).rejects.toBeInstanceOf( + SafeEgressError, + ); }); it('rejects a non-HTTPS url (the shared HTTPS policy), never connecting', async () => { @@ -69,7 +73,9 @@ describe('createValidatedFetch', () => { it('pins the connection to the validated IP and passes method + headers + body through', async () => { let captured: HopRequest | undefined; - const fetch = createValidatedFetch(fakeDeps({ ips: ['203.0.113.7'], onConnect: (req) => (captured = req) })); + const fetch = createValidatedFetch( + fakeDeps({ ips: ['203.0.113.7'], onConnect: (req) => (captured = req) }), + ); await fetch('https://api.example.com/v1/chat/completions', { method: 'POST', headers: { authorization: 'Bearer sk-secret', 'content-type': 'application/json' }, @@ -86,7 +92,10 @@ describe('createValidatedFetch', () => { const fetch = createValidatedFetch( fakeDeps({ ips: ['1.2.3.4'], chunks: ['data: a\n\n', 'data: b\n\n', 'data: [DONE]\n\n'] }), ); - const res = await fetch('https://api.example.com/v1/chat/completions', { method: 'POST', body: '{}' }); + const res = await fetch('https://api.example.com/v1/chat/completions', { + method: 'POST', + body: '{}', + }); const reader = res.body?.getReader(); if (reader === undefined) throw new Error('no response body stream'); const received: string[] = []; @@ -101,7 +110,9 @@ describe('createValidatedFetch', () => { it('disposes the socket when the response stream is cancelled early', async () => { const dispose = vi.fn(); - const fetch = createValidatedFetch(fakeDeps({ ips: ['1.2.3.4'], chunks: ['a', 'b', 'c'], dispose })); + const fetch = createValidatedFetch( + fakeDeps({ ips: ['1.2.3.4'], chunks: ['a', 'b', 'c'], dispose }), + ); const res = await fetch('https://api.example.com/v1/models'); await res.body?.cancel(); // stop reading early expect(dispose).toHaveBeenCalled(); @@ -140,7 +151,10 @@ describe('createValidatedFetch', () => { const fetch = createValidatedFetch( fakeDeps({ ips: ['1.2.3.4'], body: failingBody(['data: a\n\n']), dispose }), ); - const res = await fetch('https://api.example.com/v1/chat/completions', { method: 'POST', body: '{}' }); + const res = await fetch('https://api.example.com/v1/chat/completions', { + method: 'POST', + body: '{}', + }); const reader = res.body?.getReader(); if (reader === undefined) throw new Error('no body stream'); let thrown: unknown; @@ -160,7 +174,9 @@ describe('createValidatedFetch', () => { it('accepts a Request-object input (not just a url + init)', async () => { let captured: HopRequest | undefined; - const fetch = createValidatedFetch(fakeDeps({ ips: ['203.0.113.9'], onConnect: (req) => (captured = req) })); + const fetch = createValidatedFetch( + fakeDeps({ ips: ['203.0.113.9'], onConnect: (req) => (captured = req) }), + ); await fetch( new Request('https://api.example.com/v1/chat/completions', { method: 'POST', @@ -176,7 +192,9 @@ describe('createValidatedFetch', () => { it('rejects an out-of-range HTTP status (a hostile 999) as a typed SafeEgressError, never a raw RangeError', async () => { const dispose = vi.fn(); const fetch = createValidatedFetch(fakeDeps({ ips: ['1.2.3.4'], status: 999, dispose })); - await expect(fetch('https://api.example.com/v1/models')).rejects.toBeInstanceOf(SafeEgressError); + await expect(fetch('https://api.example.com/v1/models')).rejects.toBeInstanceOf( + SafeEgressError, + ); expect(dispose).toHaveBeenCalled(); }); @@ -208,7 +226,9 @@ describe('createValidatedFetch', () => { it('strips accept-encoding from the request (this fetch does not auto-decompress the streamed body)', async () => { let captured: HopRequest | undefined; - const fetch = createValidatedFetch(fakeDeps({ ips: ['1.2.3.4'], onConnect: (req) => (captured = req) })); + const fetch = createValidatedFetch( + fakeDeps({ ips: ['1.2.3.4'], onConnect: (req) => (captured = req) }), + ); await fetch('https://api.example.com/v1/models', { headers: { 'accept-encoding': 'gzip, br', authorization: 'Bearer sk-x' }, }); diff --git a/apps/cli/src/engine/validated-fetch.ts b/apps/cli/src/engine/validated-fetch.ts index 87213607..1bf55aea 100644 --- a/apps/cli/src/engine/validated-fetch.ts +++ b/apps/cli/src/engine/validated-fetch.ts @@ -19,9 +19,11 @@ import { * **Streaming-safe:** `connectValidated` returns a LIVE `AsyncIterable` body (it never buffers — `readBounded` is * a separate helper), which this wraps in a **backpressure-aware** `ReadableStream`, so an SSE completion streams * chunk-by-chunk. It deliberately does NOT wrap the call in `withEgressTimeout` — that would abort a long-lived - * stream at a fixed deadline; the caller's `AbortSignal` (the OpenAI SDK's own timeout, `boundedListModels`'s 15s, - * `validateProviderKey`'s 10s) is threaded straight through and tears the socket down on connect AND during - * streaming. Every escaping error is normalized to a reason-only `SafeEgressError` (never the url/IP/host/key) — + * stream at a fixed working deadline; the caller's `AbortSignal` (the OpenAI SDK's own timeout, `boundedListModels`'s + * 15s, `validateProviderKey`'s 10s) drives cancellation and tears the socket down on connect AND during streaming. + * A caller signal is only composed (`AbortSignal.any`) with a very generous, no-op-in-practice ceiling + * ({@link DEFAULT_EGRESS_CEILING_MS}) that backstops a signal-less / never-firing caller against an infinite hang — + * it sits far above every real deadline, so it never clips normal streaming. Every escaping error is normalized to a reason-only `SafeEgressError` (never the url/IP/host/key) — * `connectValidated`'s own throws already are, and a raw resolver/socket fault (a DNS error carries the non-secret * hostname) is re-wrapped here. The `Authorization` key rides the request headers to the endpoint (as the API * requires) but is never logged. The `EgressDeps` (DNS + connect) are injectable so the SSRF policy is @@ -77,13 +79,20 @@ const EGRESS_METHODS = ['GET', 'POST', 'PUT', 'DELETE'] as const satisfies reado const isEgressMethod = (method: string): method is EgressMethod => (EGRESS_METHODS as readonly string[]).includes(method); +/** Resolve the request URL string from a `Request`, a `URL`, or a raw string input (no nested ternary). */ +function inputToUrl(input: string | URL | Request): string { + if (input instanceof Request) return input.url; + if (typeof input === 'string') return input; + return input.href; +} + /** Resolve the url / method / headers / body / signal from either a `Request` or a `(url, init)` pair. */ async function normalizeRequest( input: string | URL | Request, init: RequestInit | undefined, ): Promise { const isRequest = input instanceof Request; - const url = isRequest ? input.url : typeof input === 'string' ? input : input.href; + const url = inputToUrl(input); const rawMethod = (init?.method ?? (isRequest ? input.method : 'GET')).toUpperCase(); if (!isEgressMethod(rawMethod)) { // The OpenAI SDK uses only GET (models.list) + POST (completions); refuse anything else loudly rather than @@ -92,10 +101,25 @@ async function normalizeRequest( } const headers = headersToRecord(init?.headers ?? (isRequest ? input.headers : undefined)); const body = await bodyToString(init?.body ?? (isRequest ? input.body : undefined)); - const signal = init?.signal ?? (isRequest ? input.signal : undefined) ?? new AbortController().signal; + const signal = composeEgressSignal(init?.signal ?? (isRequest ? input.signal : undefined)); return { url, method: rawMethod, headers, body, signal }; } +/** A very generous backstop deadline (NOT a working timeout). A signal-less — or a never-firing — caller would + * otherwise ride an unowned `AbortController` signal that never aborts, so the request could hang forever. This + * ceiling sits far ABOVE every real caller deadline (the OpenAI SDK's own request timeout, `boundedListModels`'s + * 15s, `validateProviderKey`'s 10s) and above any normal streamed completion, so a real caller's (shorter) signal + * always wins in practice — it never clips normal long-lived streaming, it only backstops a pathological hang. */ +const DEFAULT_EGRESS_CEILING_MS = 15 * 60_000; // 15 minutes — a backstop, not a working deadline + +/** Compose the caller's `AbortSignal` (if any) with the generous {@link DEFAULT_EGRESS_CEILING_MS} backstop. + * `AbortSignal.timeout`'s timer is unref'd, so the ceiling never keeps the process alive after a fast call, and + * a caller-provided signal still drives cancellation on connect AND during streaming (it fires long before this). */ +function composeEgressSignal(callerSignal: AbortSignal | undefined): AbortSignal { + const ceiling = AbortSignal.timeout(DEFAULT_EGRESS_CEILING_MS); + return callerSignal === undefined ? ceiling : AbortSignal.any([callerSignal, ceiling]); +} + /** Flatten a `RequestInit['headers']` (Headers | record | pairs) to a plain record; connectValidated re-sanitizes it. * `new Headers(...)` normalizes every `HeadersInit` form (and lower-cases the keys) in one step. Drops * `accept-encoding`: this fetch does NOT auto-decompress the streamed body (unlike a platform `fetch`), so it must @@ -177,7 +201,13 @@ function hopBodyToStream(hop: HopResponse): ReadableStream { }, cancel() { dispose(); - void iterator.return?.(undefined); + // Best-effort iterator cleanup — a `return()` rejection must never escape `cancel()` (and must not float). + const returned = iterator.return?.(undefined); + if (returned !== undefined) { + returned.catch(() => { + // ignore — cancel is best-effort and must not throw + }); + } }, }); } diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index 283a904e..9bf62125 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -248,11 +248,18 @@ export async function driveHome(deps: HomeDeps): Promise { }); // createChatLineHandler owns the mode control (ADR-0057): it applies the initial `ask` mode → the // fail-closed approval regime — BEFORE the session opens, so the full-capability host is never live without it. - const { processLine, cancelOnce, shouldStop, stopReason, onAbort, onModeChange, onSetEffort } = - createChatLineHandler( - { built, opened, store, persister, doctorProbes: chatDoctorProbes }, - deps, - ); + const { + processLine, + cancelOnce, + shouldStop, + stopReason, + onAbort, + onModeChange, + onSetEffort, + } = createChatLineHandler( + { built, opened, store, persister, doctorProbes: chatDoctorProbes }, + deps, + ); // Subscribe the view store BEFORE opening the session so the synchronous session:started is observed. unsubscribe = built.handle.subscribe((event) => store.apply(event)); frame = setInterval(() => store.tick(), FRAME_MS); @@ -345,7 +352,10 @@ export async function driveHome(deps: HomeDeps): Promise { // the just-torn-down session's transcript from the SHARED db and RESUME it under a model-swapped agent (dropping // the original fallback_chain), carrying the text-only transcript + cumulative cost/turns; a NEW instance, // honoring ADR-0024's one-model-per-lifetime rule. The controller drives the tear-down / swap (mirroring clearChat). - const reseatChat = async (sessionId: string, target: ReseatTarget): Promise => { + const reseatChat = async ( + sessionId: string, + target: ReseatTarget, + ): Promise => { const loaded = opened.store.loadFull(sessionId); if (loaded === undefined || loaded.session.agentSnapshot === undefined) { throw new CliError( @@ -412,7 +422,8 @@ export async function driveHome(deps: HomeDeps): Promise { io: deps.io, // Reuse the SAME config-write target as the `/models` port (honors `--config`) so the wizard's starter // model + a later `/models` pick + the started session all agree on one file (2.5.G S7/S8). - writeDefaultModel: (modelId) => writeGlobalDefaultModel(modelId, homeDir, deps.global.configPath), + writeDefaultModel: (modelId) => + writeGlobalDefaultModel(modelId, homeDir, deps.global.configPath), ...(deps.onboardingPrompter === undefined ? {} : { prompter: deps.onboardingPrompter }), }); } diff --git a/apps/cli/src/onboarding/wizard.test.ts b/apps/cli/src/onboarding/wizard.test.ts index 3b9c4ef8..a22fa830 100644 --- a/apps/cli/src/onboarding/wizard.test.ts +++ b/apps/cli/src/onboarding/wizard.test.ts @@ -11,11 +11,7 @@ import { type KeychainStore, } from '../secrets/keychain.js'; import type { ProviderKeyValidation } from '../engine/providers.js'; -import { - isProviderKeyless, - runOnboardingWizard, - type ClackOnboardingDeps, -} from './wizard.js'; +import { isProviderKeyless, runOnboardingWizard, type ClackOnboardingDeps } from './wizard.js'; const CANCEL = Symbol('clack-cancel'); @@ -57,7 +53,9 @@ function queuedPrompter( } /** A scripted live-validation port draining a queue of outcomes (defaulting to ok when exhausted). */ -function validateSeq(results: ProviderKeyValidation[]): (id: string, key: string) => Promise { +function validateSeq( + results: ProviderKeyValidation[], +): (id: string, key: string) => Promise { return () => Promise.resolve(results.shift() ?? { ok: true, detail: 'ok', reason: 'ok' }); } @@ -89,7 +87,9 @@ function scriptedPrompter(script: { provider?: string | symbol; key?: string | s } /** An in-memory keychain; `throwOnSet` makes `set` raise `KeychainUnavailableError` (the locked-keychain case). */ -function memKeychain(opts: { throwOnSet?: boolean } = {}): KeychainStore & { store: Map } { +function memKeychain( + opts: { throwOnSet?: boolean } = {}, +): KeychainStore & { store: Map } { const store = new Map(); return { store, @@ -121,7 +121,13 @@ const io: CliIo = { describe('isProviderKeyless', () => { it('is TRUE when every provider key resolution throws (no keychain + no env)', () => { - expect(isProviderKeyless({ keyFor: () => { throw new Error('no key'); } })).toBe(true); + expect( + isProviderKeyless({ + keyFor: () => { + throw new Error('no key'); + }, + }), + ).toBe(true); }); it('is FALSE as soon as one provider resolves a key (keychain or env)', () => { let calls = 0; @@ -148,7 +154,7 @@ describe('runOnboardingWizard', () => { const store = (): ReturnType => createProviderStore(client.db, { uuid: () => 'id-fixed', now: () => 1 }); - it('stores the pasted key + registers the provider row + sets the chosen provider\'s starter model (secret-free)', async () => { + it("stores the pasted key + registers the provider row + sets the chosen provider's starter model (secret-free)", async () => { const keychain = memKeychain(); const s = store(); const writeDefaultModel = vi.fn(); @@ -156,7 +162,14 @@ describe('runOnboardingWizard', () => { provider: 'anthropic', key: ' sk-ant-supersecret-1234 ', // incidental paste whitespace — must be trimmed before storing }); - await runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io, writeDefaultModel }); + await runOnboardingWizard({ + prompter, + store: s, + keychain, + resolver: stubResolver, + io, + writeDefaultModel, + }); // The TRIMMED key landed in the keychain under the provider account (no stray whitespace persisted)... expect(keychain.store.get(keychainAccount('anthropic'))).toBe('sk-ant-supersecret-1234'); @@ -177,7 +190,14 @@ describe('runOnboardingWizard', () => { const s = store(); const writeDefaultModel = vi.fn(); const { prompter } = scriptedPrompter({ provider: 'openai', key: 'sk-openai-xyz' }); - await runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io, writeDefaultModel }); + await runOnboardingWizard({ + prompter, + store: s, + keychain, + resolver: stubResolver, + io, + writeDefaultModel, + }); // NOT the anthropic default — the openai starter, so a first chat doesn't try (and fail) an anthropic key. expect(writeDefaultModel).toHaveBeenCalledWith(KNOWN_PROVIDERS.openai.testModel); expect(KNOWN_PROVIDERS.openai.testModel).not.toBe(KNOWN_PROVIDERS.anthropic.testModel); @@ -188,7 +208,14 @@ describe('runOnboardingWizard', () => { const s = store(); const writeDefaultModel = vi.fn(); const { prompter, password, notes } = scriptedPrompter({ provider: CANCEL }); - await runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io, writeDefaultModel }); + await runOnboardingWizard({ + prompter, + store: s, + keychain, + resolver: stubResolver, + io, + writeDefaultModel, + }); expect(keychain.store.size).toBe(0); // no key captured or stored expect(password).not.toHaveBeenCalled(); // never even prompted for a key expect(writeDefaultModel).not.toHaveBeenCalled(); // no default model written on cancel @@ -200,7 +227,14 @@ describe('runOnboardingWizard', () => { const s = store(); const writeDefaultModel = vi.fn(); const { prompter, notes } = scriptedPrompter({ provider: 'openai', key: CANCEL }); - await runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io, writeDefaultModel }); + await runOnboardingWizard({ + prompter, + store: s, + keychain, + resolver: stubResolver, + io, + writeDefaultModel, + }); expect(keychain.store.size).toBe(0); expect(s.get('openai')).toBeUndefined(); // no row registered on cancel expect(writeDefaultModel).not.toHaveBeenCalled(); @@ -217,7 +251,14 @@ describe('runOnboardingWizard', () => { }); // Must resolve (not throw) — a keychain-unavailable first run degrades gracefully. await expect( - runOnboardingWizard({ prompter, store: s, keychain, resolver: stubResolver, io, writeDefaultModel }), + runOnboardingWizard({ + prompter, + store: s, + keychain, + resolver: stubResolver, + io, + writeDefaultModel, + }), ).resolves.toBeUndefined(); expect(keychain.store.size).toBe(0); // nothing persisted to the keychain... @@ -242,9 +283,19 @@ describe('runOnboardingWizard', () => { }, }; const writeDefaultModel = vi.fn(); - const { prompter, notes, outros } = scriptedPrompter({ provider: 'deepseek', key: 'sk-ds-1234' }); + const { prompter, notes, outros } = scriptedPrompter({ + provider: 'deepseek', + key: 'sk-ds-1234', + }); await expect( - runOnboardingWizard({ prompter, store: brokenStore, keychain, resolver: stubResolver, io, writeDefaultModel }), + runOnboardingWizard({ + prompter, + store: brokenStore, + keychain, + resolver: stubResolver, + io, + writeDefaultModel, + }), ).resolves.toBeUndefined(); const all = [...notes, ...outros].join('\n'); @@ -285,11 +336,14 @@ describe('runOnboardingWizard', () => { expect(all).not.toContain('sk-good-second'); }); - it('a bad key (auth) → CONTINUE anyway: stores the FIRST key, note says couldn\'t be verified (secret-free)', async () => { + it("a bad key (auth) → CONTINUE anyway: stores the FIRST key, note says couldn't be verified (secret-free)", async () => { const keychain = memKeychain(); const s = store(); const writeDefaultModel = vi.fn(); - const { prompter, notes, selectCalls } = queuedPrompter(['openai', 'continue'], ['sk-unverified-key']); + const { prompter, notes, selectCalls } = queuedPrompter( + ['openai', 'continue'], + ['sk-unverified-key'], + ); await runOnboardingWizard({ prompter, store: s, @@ -299,7 +353,11 @@ describe('runOnboardingWizard', () => { writeDefaultModel, // A PLANTED secret in the detail must never reach the select MESSAGE (the wizard surfaces res.detail there). validate: validateSeq([ - { ok: false, detail: 'key test failed — LEAKED-sk-unverified-key rejected', reason: 'auth' }, + { + ok: false, + detail: 'key test failed — LEAKED-sk-unverified-key rejected', + reason: 'auth', + }, ]), }); expect(keychain.store.get(keychainAccount('openai'))).toBe('sk-unverified-key'); // consciously accepted @@ -314,7 +372,7 @@ describe('runOnboardingWizard', () => { expect(retrySelect?.message).toContain('LEAKED-sk-unverified-key'); // detail is surfaced verbatim (already redacted at the seam) }); - it('a NETWORK failure → default Continue (offline first-run isn\'t blocked): stores the key', async () => { + it("a NETWORK failure → default Continue (offline first-run isn't blocked): stores the key", async () => { const keychain = memKeychain(); const s = store(); const writeDefaultModel = vi.fn(); @@ -326,7 +384,9 @@ describe('runOnboardingWizard', () => { resolver: stubResolver, io, writeDefaultModel, - validate: validateSeq([{ ok: false, detail: 'key test failed — timeout (10000ms)', reason: 'network' }]), + validate: validateSeq([ + { ok: false, detail: 'key test failed — timeout (10000ms)', reason: 'network' }, + ]), }); expect(keychain.store.get(keychainAccount('gemini'))).toBe('sk-offline-key'); expect(writeDefaultModel).toHaveBeenCalledWith(KNOWN_PROVIDERS.gemini.testModel); @@ -346,7 +406,9 @@ describe('runOnboardingWizard', () => { resolver: stubResolver, io, writeDefaultModel, - validate: validateSeq([{ ok: false, detail: 'key test failed — invalid_api_key', reason: 'auth' }]), + validate: validateSeq([ + { ok: false, detail: 'key test failed — invalid_api_key', reason: 'auth' }, + ]), }); expect(keychain.store.size).toBe(0); expect(writeDefaultModel).not.toHaveBeenCalled(); @@ -365,7 +427,9 @@ describe('runOnboardingWizard', () => { resolver: stubResolver, io, writeDefaultModel, - validate: validateSeq([{ ok: false, detail: 'key test failed — invalid_api_key', reason: 'auth' }]), + validate: validateSeq([ + { ok: false, detail: 'key test failed — invalid_api_key', reason: 'auth' }, + ]), }); expect(keychain.store.size).toBe(0); expect(writeDefaultModel).not.toHaveBeenCalled(); @@ -374,7 +438,10 @@ describe('runOnboardingWizard', () => { it('threads keyToStore across MULTIPLE retries (retry → retry → good stores the THIRD key)', async () => { const keychain = memKeychain(); - const { prompter } = queuedPrompter(['openai', 'retry', 'retry'], ['sk-bad1', 'sk-bad2', 'sk-good3']); + const { prompter } = queuedPrompter( + ['openai', 'retry', 'retry'], + ['sk-bad1', 'sk-bad2', 'sk-good3'], + ); await runOnboardingWizard({ prompter, store: store(), diff --git a/apps/cli/src/onboarding/wizard.ts b/apps/cli/src/onboarding/wizard.ts index cfd1bba3..e0aca63c 100644 --- a/apps/cli/src/onboarding/wizard.ts +++ b/apps/cli/src/onboarding/wizard.ts @@ -118,9 +118,7 @@ export interface OnboardingDeps { * bare Home offers the wizard. A run with EITHER a keychain key or an env key is NOT key-less (no wizard) — so a * working env-key user is never nagged, and the env fallback IS the resolver's built-in key import. */ -export function isProviderKeyless( - resolver: Pick, -): boolean { +export function isProviderKeyless(resolver: Pick): boolean { return !KNOWN_PROVIDER_IDS.some((id) => providerHasKey(resolver, id)); } @@ -188,7 +186,13 @@ export async function runOnboardingWizard(deps: OnboardingDeps): Promise { readSecret: () => Promise.resolve(keyToStore), // `global` is read only by `provider list --json`; `set-key` never touches it — a throwaway (the wizard is // always interactive, never `--json`). - global: { json: false, color: false, cwd: process.cwd(), configPath: undefined, verbosity: 'normal' }, + global: { + json: false, + color: false, + cwd: process.cwd(), + configPath: undefined, + verbosity: 'normal', + }, }, ); } catch (err) { @@ -285,31 +289,9 @@ async function validateWithRetry( } if (res.ok) return { keyToStore, verified: true }; - // The order + the pre-highlighted (bare-Enter) option depend on the CAUSE: a TRANSIENT (`network` — offline / - // timeout / rate-limit / overloaded) failure defaults to "save it anyway" (don't block an offline first-run); - // a non-transient failure (a rejected key, a billing/account issue, or an unexpected fault) defaults to - // "re-enter". `res.detail` (already key-redacted, and it names the specific reason, e.g. `invalid_api_key`) is - // the neutral evidence — we do NOT assert "the key is bad" for the non-network case, since a 402/400 may mean - // billing or a stale test model rather than a wrong key. const isTransient = res.reason === 'network'; const displayName = KNOWN_PROVIDERS[provider].displayName; - const choice = await p.select({ - message: isTransient - ? `Couldn't reach ${displayName} to verify — you may be offline or it's busy (${res.detail}).` - : `Couldn't verify your ${displayName} key (${res.detail}).`, - options: isTransient - ? [ - { value: 'continue', label: 'Save it anyway', hint: 'verify later with /doctor' }, - { value: 'retry', label: 'Enter a different key' }, - { value: 'skip', label: 'Skip setup' }, - ] - : [ - { value: 'retry', label: 'Enter a new key' }, - { value: 'continue', label: 'Save it anyway', hint: 'fix it later with /doctor' }, - { value: 'skip', label: 'Skip setup' }, - ], - initialValue: isTransient ? 'continue' : 'retry', - }); + const choice = await p.select(buildRetryPrompt(isTransient, displayName, res.detail)); if (p.isCancel(choice) || choice === 'skip') { skip(p); return null; @@ -331,6 +313,38 @@ async function validateWithRetry( } } +/** + * Build the failed-key retry prompt (the `p.select` payload). The order + the pre-highlighted (bare-Enter) option + * depend on the CAUSE: a TRANSIENT (`network` — offline / timeout / rate-limit / overloaded) failure defaults to + * "save it anyway" (don't block an offline first-run); a non-transient failure (a rejected key, a billing/account + * issue, or an unexpected fault) defaults to "re-enter". `detail` (already key-redacted, and it names the specific + * reason, e.g. `invalid_api_key`) is the neutral evidence — we do NOT assert "the key is bad" for the non-network + * case, since a 402/400 may mean billing or a stale test model rather than a wrong key. + */ +function buildRetryPrompt( + isTransient: boolean, + displayName: string, + detail: string, +): Parameters[0] { + return { + message: isTransient + ? `Couldn't reach ${displayName} to verify — you may be offline or it's busy (${detail}).` + : `Couldn't verify your ${displayName} key (${detail}).`, + options: isTransient + ? [ + { value: 'continue', label: 'Save it anyway', hint: 'verify later with /doctor' }, + { value: 'retry', label: 'Enter a different key' }, + { value: 'skip', label: 'Skip setup' }, + ] + : [ + { value: 'retry', label: 'Enter a new key' }, + { value: 'continue', label: 'Save it anyway', hint: 'fix it later with /doctor' }, + { value: 'skip', label: 'Skip setup' }, + ], + initialValue: isTransient ? 'continue' : 'retry', + }; +} + /** The cancel/skip exit: a friendly pointer to the manual path, then hand off to the Home. */ function skip(p: ClackOnboardingDeps): void { p.note( diff --git a/apps/cli/src/render/tui/chat-ink.tsx b/apps/cli/src/render/tui/chat-ink.tsx index 9ac9c2a7..5249621b 100644 --- a/apps/cli/src/render/tui/chat-ink.tsx +++ b/apps/cli/src/render/tui/chat-ink.tsx @@ -17,6 +17,7 @@ import { partialFailureBanner, type ModelPickerKey, type ModelPickerState, + type ModelPickerStep, } from './model-picker.js'; import { ModelPickerView } from './model-picker-view.js'; import { EXIT_CODES } from '../../process/exit-codes.js'; @@ -202,7 +203,8 @@ interface ChatViewProps { * sequence cannot corrupt the terminal or inject ANSI/OSC. */ export function ChatView(props: Readonly): ReactElement { - const { state, tick, color, editor, running, mode, reasoningEffort, approval, paletteOpen } = props; + const { state, tick, color, editor, running, mode, reasoningEffort, approval, paletteOpen } = + props; const attachments = props.attachments ?? []; // When the palette is open it renders its own query line + hint below, so suppress the idle prompt + footer to // avoid two competing prompts (the palette owns the input focus until it closes). @@ -536,7 +538,8 @@ export function ChatApp(props: Readonly): ReactElement { const port = props.modelPicker; const open = modelPickerRef.current; if (epoch !== pickerEpochRef.current || open === undefined || port === undefined) return; - const failed = report?.providers.filter((p) => p.status === 'failed').map((p) => p.provider) ?? []; + const failed = + report?.providers.filter((p) => p.status === 'failed').map((p) => p.provider) ?? []; let view: ReturnType; try { view = port.load(); // a DB read — never crash the REPL (parity with the Home) @@ -563,7 +566,8 @@ export function ChatApp(props: Readonly): ReactElement { // refresh()/refreshIfStale() never reject (per-provider isolation), but stay defensive: drop the spinner // only when this is still the same open picker generation. const cur = modelPickerRef.current; - if (epoch === pickerEpochRef.current && cur !== undefined) applyModelPicker({ ...cur, loading: false }); + if (epoch === pickerEpochRef.current && cur !== undefined) + applyModelPicker({ ...cur, loading: false }); }, ); }; @@ -599,9 +603,43 @@ export function ChatApp(props: Readonly): ReactElement { }); runPickerRefresh(() => port.refreshIfStale()); }; - // The open picker owns every key (mirrors routeMentionKey). On accept → a LIVE reseat (onReseat sets the stop - // state) then end the driver loop (onExit) so runReplLoop rebuilds the session on the new model. A DIMMED model is - // non-selectable (a transient hint, ADR-0064 §6); any nav/filter keystroke clears that hint. + // A DIMMED (unavailable) model's actionable hint (ADR-0064 §6) — a keyless provider names the remedy, else the + // pre-existing "not on your key" note. + const blockedHint = (step: Extract): string => + step.reason === 'no-key' + ? `${step.displayName}: no key for ${step.provider} — run \`relavium provider set-key ${step.provider}\`` + : `${step.displayName} is not available on your key — pick another`; + + // Act on an accepted pick. A SAME-model pick is an effort-only change (ADR-0066 §5) — a per-turn SESSION override + // (no reseat, no teardown/approval-wipe/MCP-reconnect/context-loss); a re-pick of the same tier (or a non-reasoning + // model with no tier) is a gentle no-op. A DIFFERENT-model pick is a live reseat (ADR-0059) carrying the chosen + // effort, then ends the driver loop so runReplLoop rebuilds on the new model. + const acceptModelPick = ( + step: Extract, + open: ModelPickerState, + ): void => { + if (step.modelId === open.currentDefault) { + if (step.reasoningEffort === undefined || step.reasoningEffort === open.currentEffort) { + const at = step.reasoningEffort === undefined ? '' : ` at effort ${step.reasoningEffort}`; + props.store.note(`Already on ${step.displayName}${at}.`); + } else { + props.onSetEffort?.(step.reasoningEffort); + props.store.note( + `Reasoning effort set to ${step.reasoningEffort} — applies to your next message.`, + ); + } + return; + } + props.onReseat?.({ + modelId: step.modelId, + provider: step.provider, + ...(step.reasoningEffort === undefined ? {} : { reasoningEffort: step.reasoningEffort }), + }); + props.onExit(); + }; + + // The open picker owns every key (mirrors routeMentionKey). Route the fold's step; the accept/blocked cases + // delegate to the helpers above. const routeModelPickerKey = (char: string, key: ModelPickerKey): void => { const open = modelPickerRef.current; if (open === undefined) return; @@ -610,38 +648,13 @@ export function ChatApp(props: Readonly): ReactElement { case 'close': applyModelPicker(undefined); return; - case 'accept': { + case 'accept': applyModelPicker(undefined); - if (step.modelId === open.currentDefault) { - // SAME model: this is an effort-only change (ADR-0066 §5) — a per-turn SESSION override, NOT a reseat. No - // teardown, no ADR-0057 approval-cache wipe, no MCP reconnect, no text-only context loss. A re-pick of the - // same tier (or a non-reasoning model with no tier) is a gentle no-op. - if (step.reasoningEffort === undefined || step.reasoningEffort === open.currentEffort) { - const at = step.reasoningEffort === undefined ? '' : ` at effort ${step.reasoningEffort}`; - props.store.note(`Already on ${step.displayName}${at}.`); - } else { - props.onSetEffort?.(step.reasoningEffort); - props.store.note(`Reasoning effort set to ${step.reasoningEffort} — applies to your next message.`); - } - return; - } - // DIFFERENT model: a live reseat (ADR-0059), carrying the chosen effort onto the new binding. - props.onReseat?.({ - modelId: step.modelId, - provider: step.provider, - ...(step.reasoningEffort === undefined ? {} : { reasoningEffort: step.reasoningEffort }), - }); - props.onExit(); // the reseat set the stop state; end the loop so runReplLoop swaps in the new-model session + acceptModelPick(step, open); return; - } - case 'blocked': { - const hint = - step.reason === 'no-key' - ? `${step.displayName}: no key for ${step.provider} — run \`relavium provider set-key ${step.provider}\`` - : `${step.displayName} is not available on your key — pick another`; - applyModelPicker({ ...open, hint }); + case 'blocked': + applyModelPicker({ ...open, hint: blockedHint(step) }); return; - } case 'refresh': runPickerRefresh(() => props.modelPicker?.refresh() ?? Promise.resolve(undefined)); return; diff --git a/apps/cli/src/render/tui/chat-store.ts b/apps/cli/src/render/tui/chat-store.ts index b475de14..23f89f48 100644 --- a/apps/cli/src/render/tui/chat-store.ts +++ b/apps/cli/src/render/tui/chat-store.ts @@ -126,7 +126,14 @@ export function createChatStore(color: boolean, seed?: SessionViewSeed): ChatSto let settleApproval: ((answer: ApprovalAnswer) => void) | undefined; let tickCount = 0; let dirty = false; - let snapshot: ChatStoreSnapshot = { state, mode, reasoningEffort, approval, tick: tickCount, color }; + let snapshot: ChatStoreSnapshot = { + state, + mode, + reasoningEffort, + approval, + tick: tickCount, + color, + }; const flush = (): void => { snapshot = { state, mode, reasoningEffort, approval, tick: tickCount, color }; diff --git a/apps/cli/src/render/tui/home-controller.test.ts b/apps/cli/src/render/tui/home-controller.test.ts index 02fbb808..52dffb90 100644 --- a/apps/cli/src/render/tui/home-controller.test.ts +++ b/apps/cli/src/render/tui/home-controller.test.ts @@ -1614,16 +1614,18 @@ function pickerEntry( * `[chat].default_model` shadowing the global write (the effective default stays the override). `writeThrows` * simulates a config-write fault. */ -function makeModelsPort(opts: { - entries?: readonly ModelCatalogEntry[]; - refreshedAt?: number; - overrideDefault?: string; - writeThrows?: boolean; - readFaults?: boolean; // currentDefault always returns undefined (a config re-read fault after a good write) - loadThrows?: boolean; // load() throws (a DB read fault) - refreshIfStale?: () => Promise>>; - refresh?: () => Promise>>; -} = {}): { +function makeModelsPort( + opts: { + entries?: readonly ModelCatalogEntry[]; + refreshedAt?: number; + overrideDefault?: string; + writeThrows?: boolean; + readFaults?: boolean; // currentDefault always returns undefined (a config re-read fault after a good write) + loadThrows?: boolean; // load() throws (a DB read fault) + refreshIfStale?: () => Promise>>; + refresh?: () => Promise>>; + } = {}, +): { port: HomeModelsPort; load: ReturnType; refreshIfStale: ReturnType; @@ -1648,7 +1650,8 @@ function makeModelsPort(opts: { refresh, // The EFFECTIVE default: a read fault ⇒ undefined; else the override wins (shadows the global write), else the // last written id. - currentDefault: () => (opts.readFaults === true ? undefined : (opts.overrideDefault ?? written)), + currentDefault: () => + opts.readFaults === true ? undefined : (opts.overrideDefault ?? written), writeDefault, }; return { port, load, refreshIfStale, refresh, writeDefault }; @@ -1812,7 +1815,9 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => it('Ctrl+R runs an unbounded refresh; a per-provider failure surfaces a secret-free banner', async () => { const { port, refresh } = makeModelsPort({ refresh: () => - Promise.resolve({ providers: [{ provider: 'openai', status: 'failed', error: 'redacted' }] }), + Promise.resolve({ + providers: [{ provider: 'openai', status: 'failed', error: 'redacted' }], + }), }); const c = openPicker(port); await flush(); @@ -1825,7 +1830,7 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => expect(picker?.banner).not.toContain('redacted'); }); - it('a reopened picker is NOT clobbered by a prior open\'s slow refresh (the epoch guard)', async () => { + it("a reopened picker is NOT clobbered by a prior open's slow refresh (the epoch guard)", async () => { // Open #1 kicks a SLOW refreshIfStale; open #2 (after a close) gets a fast one. When the slow first refresh // finally resolves with a partial failure, it must NOT stamp the SECOND picker's banner (a different generation). let resolveSlow: (report: RefreshReport) => void = () => undefined; @@ -1935,7 +1940,9 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => const sessionA = makeSession({ sessionId: 'sess-A', store: boundStore, onSetEffort }); const reseatChat = vi.fn(() => Promise.resolve(makeSession().session)); const { port } = makeModelsPort({ - entries: [pickerEntry({ modelId: 'claude-opus-4-8', provider: 'anthropic', supportsReasoning: true })], + entries: [ + pickerEntry({ modelId: 'claude-opus-4-8', provider: 'anthropic', supportsReasoning: true }), + ], }); const c = createHomeController({ doctorProbes: STUB_DOCTOR_PROBES, @@ -1973,7 +1980,9 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => const sessionA = makeSession({ sessionId: 'sess-A', store: boundStore, onSetEffort }); const reseatChat = vi.fn(() => Promise.resolve(makeSession().session)); const { port } = makeModelsPort({ - entries: [pickerEntry({ modelId: 'claude-opus-4-8', provider: 'anthropic', supportsReasoning: true })], + entries: [ + pickerEntry({ modelId: 'claude-opus-4-8', provider: 'anthropic', supportsReasoning: true }), + ], }); const c = createHomeController({ doctorProbes: STUB_DOCTOR_PROBES, diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index 65ebb932..4e64b5d9 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -8,11 +8,7 @@ import { type ReplCommandContext, } from '../../commands/repl-commands.js'; import type { RefreshReport } from '../../engine/model-refresh.js'; -import { - foldModelPickerKey, - partialFailureBanner, - type ModelPickerState, -} from './model-picker.js'; +import { foldModelPickerKey, partialFailureBanner, type ModelPickerState } from './model-picker.js'; import type { ReseatTarget } from '../../commands/chat.js'; import { nextMode, type ChatMode } from '../../chat/chat-mode.js'; import { clearedNotice, modelSwitchNotice } from '../../chat/repl-info.js'; @@ -179,7 +175,10 @@ export interface HomeControllerState { */ export interface HomeModelsPort { /** The merged catalog (all providers) + the newest live-refresh stamp (the freshness badge). Sync read + merge. */ - load: () => { readonly entries: readonly ModelCatalogEntry[]; readonly refreshedAt: number | undefined }; + load: () => { + readonly entries: readonly ModelCatalogEntry[]; + readonly refreshedAt: number | undefined; + }; /** TTL-bounded background refresh (ADR-0064 §5c) — refreshes empty/stale providers; `undefined` when none were. */ refreshIfStale: () => Promise; /** Unbounded, user-initiated refresh (Ctrl+R) — every connected provider, per-provider-isolated. Never rejects. */ @@ -450,7 +449,9 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { .catch(() => undefined); cancelFired = false; // the reseated session starts with a clean cancel latch (parity with clearChat) pasting = false; // a lost paste-end marker must not leak the latch into the reseated chat - next.store.notice(modelSwitchNotice(target.modelId, next.store.getSnapshot().state.turnCount)); + next.store.notice( + modelSwitchNotice(target.modelId, next.store.getSnapshot().state.turnCount), + ); set({ session: next, mode: 'chat', // STAY in chat — the model switched underneath, the conversation continues @@ -572,7 +573,8 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { const applyRefreshResult = (epoch: number, report: RefreshReport | undefined): void => { const open = state.modelPicker; if (epoch !== pickerEpoch || open === undefined || deps.models === undefined) return; // stale/closed — drop it - const failed = report?.providers.filter((p) => p.status === 'failed').map((p) => p.provider) ?? []; + const failed = + report?.providers.filter((p) => p.status === 'failed').map((p) => p.provider) ?? []; let view: ReturnType; try { view = deps.models.load(); // a DB read — never crash the REPL (parity with runDoctor / acceptModel's guard) @@ -603,7 +605,8 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { // refresh()/refreshIfStale() never reject (per-provider isolation), but stay defensive: drop the spinner // only when this is still the same open picker generation. const cur = state.modelPicker; - if (epoch === pickerEpoch && cur !== undefined) set({ modelPicker: { ...cur, loading: false } }); + if (epoch === pickerEpoch && cur !== undefined) + set({ modelPicker: { ...cur, loading: false } }); }, ); }; @@ -651,59 +654,25 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { // the long-lived process the S5 background constraint requires. An empty/stale cache repopulates as it resolves. runPickerRefresh(() => port.refreshIfStale()); }; - // Accept the chosen model. TWO surface-specific actions off the ONE picker (ADR-0059/ADR-0063): - // - a LIVE in-Home chat ⇒ RESEAT it onto the picked model (mirrors the standalone `relavium chat` /models reseat); - // - the BARE Home ⇒ persist the chosen model as the NEXT session's default (the pre-existing behavior below). - const acceptModel = ( + // The bare-Home config-write notice (ADR-0063): an HONEST re-read of the EFFECTIVE default (project → workspace → + // global) — success only when the chosen model actually became effective; a higher-layer override says so; and a + // post-write `undefined` read can only be a re-read fault (the global was just written valid), reported distinctly. + const defaultWriteNotice = ( + effective: string | undefined, modelId: string, displayName: string, - provider: ReseatTarget['provider'], - reasoningEffort?: ReasoningEffort, - ): void => { - const active = state.session; - // A LIVE in-Home chat owns the pick (the bare-Home config write is the `else` below). The effort setter path - // does not itself need `reseatChat`, but it is only REACHED with `reseatChat` wired: the in-chat `/models` that - // opens the effort sub-step is gated on `reseatChat` (see the palette/typed intercepts), so an active-session - // pick never runs without it. Guarding on both here therefore never diverts an effort pick to the config write. - if (active !== undefined && deps.reseatChat !== undefined) { - if (modelId === active.store.getSnapshot().state.model) { - if (reasoningEffort === undefined) { - // MODEL-phase no-op (a non-reasoning same-model re-pick — no effort sub-step): keep the picker OPEN with a - // hint so the user can pick a different model. The model-phase view renders the hint (unchanged ADR-0059). - const open = state.modelPicker; - set( - open === undefined - ? { modelPicker: undefined } - : { modelPicker: { ...open, hint: `Already on ${displayName} — pick a different model or Esc.` } }, - ); - return; - } - // EFFORT-phase accept (a reasoning model): an effort-only change (ADR-0066 §5) — a per-turn SESSION override - // via the setter, NOT a reseat (no teardown, no ADR-0057 approval-cache wipe, no MCP reconnect, no context - // loss). Close + note: the effort sub-list renders no hint, so a no-op must give visible store feedback. - const current = active.store.getSnapshot().reasoningEffort; - if (reasoningEffort !== current) { - active.onSetEffort?.(reasoningEffort); - active.store.note(`Reasoning effort set to ${reasoningEffort} — applies to your next message.`); - } else { - active.store.note(`Already on ${displayName} at effort ${reasoningEffort}.`); - } - set({ modelPicker: undefined }); - return; - } - // DIFFERENT model: a live reseat (ADR-0059), carrying the chosen effort onto the new binding. - reseatChat(active, { - modelId, - provider, - ...(reasoningEffort === undefined ? {} : { reasoningEffort }), - }); - return; + ): string => { + if (effective === modelId) + return `Default model set to ${displayName} — applies to your next chat session.`; + if (effective === undefined) { + return `Saved ${displayName} as your global default, but your config could not be re-read to confirm it.`; } - // ---- The bare-Home next-session-default write (ADR-0063) --------------------------------------------------- - // `writeGlobalDefaultModel` writes only the GLOBAL `[preferences].default_model` while the EFFECTIVE default - // resolves project → workspace → global (ADR-0063 §1), so the notice is HONEST: success only when the - // freshly-resolved effective default actually became the chosen model, else a project/workspace override says so. - // A write fault keeps the picker open with a secret-free `hint` rather than crashing the Home. + return `Saved ${displayName} as your global default, but a project or workspace setting overrides it here.`; + }; + + // The BARE-Home next-session-default write (ADR-0063). `writeGlobalDefaultModel` writes only the GLOBAL + // `[preferences].default_model`; a write fault keeps the picker open with a secret-free hint rather than crashing. + const writeNextSessionDefault = (modelId: string, displayName: string): void => { const port = deps.models; if (port === undefined) return; try { @@ -715,18 +684,84 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { } return; } - // Report HONESTLY by re-reading the effective default (project → workspace → global, ADR-0063 §1): success only - // if the chosen model actually became effective; a genuine higher-layer override says so; and — since the write - // just succeeded (so the global is valid) — an `undefined` effective read can only be a re-read fault, reported - // distinctly rather than mislabeled as an override. - const effective = port.currentDefault(); - const notice = - effective === modelId - ? `Default model set to ${displayName} — applies to your next chat session.` - : effective === undefined - ? `Saved ${displayName} as your global default, but your config could not be re-read to confirm it.` - : `Saved ${displayName} as your global default, but a project or workspace setting overrides it here.`; - set({ modelPicker: undefined, notice }); + set({ + modelPicker: undefined, + notice: defaultWriteNotice(port.currentDefault(), modelId, displayName), + }); + }; + + // EFFORT-phase accept on the live session (ADR-0066 §5): a per-turn SESSION override via the setter — NOT a reseat + // (no teardown/approval-wipe/MCP-reconnect/context-loss). Close + note (the effort sub-list renders no hint, so a + // no-op must give visible store feedback). + const applyEffortOnlyUpdate = ( + active: HomeChatSession, + displayName: string, + reasoningEffort: ReasoningEffort, + ): void => { + if (reasoningEffort !== active.store.getSnapshot().reasoningEffort) { + active.onSetEffort?.(reasoningEffort); + active.store.note( + `Reasoning effort set to ${reasoningEffort} — applies to your next message.`, + ); + } else { + active.store.note(`Already on ${displayName} at effort ${reasoningEffort}.`); + } + set({ modelPicker: undefined }); + }; + + // A pick on a LIVE in-Home chat: a SAME-model model-phase no-op keeps the picker OPEN with a hint (the user can + // pick another); a SAME-model effort pick is the per-turn setter; a DIFFERENT model is a live reseat (ADR-0059) + // carrying the chosen effort. + const applyLiveSessionPick = ( + active: HomeChatSession, + modelId: string, + displayName: string, + provider: ReseatTarget['provider'], + reasoningEffort: ReasoningEffort | undefined, + ): void => { + if (modelId !== active.store.getSnapshot().state.model) { + reseatChat(active, { + modelId, + provider, + ...(reasoningEffort === undefined ? {} : { reasoningEffort }), + }); + return; + } + if (reasoningEffort === undefined) { + // MODEL-phase no-op (a non-reasoning same-model re-pick — no effort sub-step): keep the picker OPEN with a hint + // so the user can pick a different model (the model-phase view renders the hint, unchanged ADR-0059). + const open = state.modelPicker; + set( + open === undefined + ? { modelPicker: undefined } + : { + modelPicker: { + ...open, + hint: `Already on ${displayName} — pick a different model or Esc.`, + }, + }, + ); + return; + } + applyEffortOnlyUpdate(active, displayName, reasoningEffort); + }; + + // Accept the chosen model. TWO surface-specific actions off the ONE picker (ADR-0059/ADR-0063): a LIVE in-Home chat + // RESEATs / updates effort (the effort setter path is only REACHED with `reseatChat` wired — the in-chat `/models` + // that opens the effort sub-step is gated on it — so guarding on both never diverts an effort pick to the config + // write); the BARE Home persists the chosen model as the NEXT session's default. + const acceptModel = ( + modelId: string, + displayName: string, + provider: ReseatTarget['provider'], + reasoningEffort?: ReasoningEffort, + ): void => { + const active = state.session; + if (active !== undefined && deps.reseatChat !== undefined) { + applyLiveSessionPick(active, modelId, displayName, provider, reasoningEffort); + return; + } + writeNextSessionDefault(modelId, displayName); }; // The open `/models` picker owns every key (2.5.G S7) — parity with routeMentionKey. Returns whether the key was // consumed. A DIMMED (unavailable-on-your-key) model is non-selectable (ADR §6): accepting one shows a transient diff --git a/apps/cli/src/render/tui/model-picker-view.tsx b/apps/cli/src/render/tui/model-picker-view.tsx index 0706bf97..8623ada6 100644 --- a/apps/cli/src/render/tui/model-picker-view.tsx +++ b/apps/cli/src/render/tui/model-picker-view.tsx @@ -1,11 +1,11 @@ import { Box, Text } from 'ink'; import type { ReactElement, ReactNode } from 'react'; -import { REASONING_EFFORTS } from '@relavium/shared'; +import type { ModelCatalogEntry } from '@relavium/llm'; +import { EFFORT_TIER_HINT, REASONING_EFFORTS } from '@relavium/shared'; import { sanitizeInline } from './chat-projection.js'; import { - EFFORT_TIER_HINT, formatContextWindow, formatModelPrice, formatRefreshedBadge, @@ -44,6 +44,26 @@ export function modelWindow(count: number, selected: number): { start: number; e return { start, end: start + MODEL_WINDOW }; } +/** The "unavailable" reason chip(s) for a row (2.5.G key-awareness): a keyless provider names the remedy; a keyed + * provider whose live list omits the model shows the pre-existing "not on your key". Empty when available. */ +function unavailableParts(entry: ModelCatalogEntry): string[] { + if (entry.available) return []; + if (entry.unavailableReason === 'no-key') return [`no key for ${entry.provider}`]; + return ['unavailable on your key']; +} + +/** The row color: selected → cyan (the highlight wins for visibility); else an unavailable/deprecated row is + * dimmed; else the default (no color props). */ +function rowColorFor( + entry: ModelCatalogEntry, + isSelected: boolean, + color: boolean, +): ReturnType | ReturnType { + if (isSelected) return colorProps(color, 'cyan'); + if (!entry.available || entry.deprecated) return dimProps(color); + return {}; +} + /** * The `'effort'` sub-list (ADR-0066) — the reasoning-effort tiers for the model chosen in the `'model'` phase. A * fixed five-row list (no scroll window needed): each tier + its one-line hint, the highlighted row in cyan, a `✓` @@ -53,7 +73,8 @@ export function modelWindow(count: number, selected: number): { start: number; e function EffortSubList(props: Readonly<{ state: ModelPickerState; color: boolean }>): ReactElement { const { state, color } = props; const selected = Math.max(0, Math.min(state.effortSelected, REASONING_EFFORTS.length - 1)); - const forModel = state.pending === undefined ? '' : ` · ${sanitizeInline(state.pending.displayName)}`; + const forModel = + state.pending === undefined ? '' : ` · ${sanitizeInline(state.pending.displayName)}`; return ( @@ -85,7 +106,8 @@ export function ModelPickerView(props: Readonly): ReactEle const visible = visibleModels(state); // Clamp the highlight for display — a refresh can shrink the list under a `selected` past the new end until the // next keystroke re-clamps (foldModelPickerKey clamps on move). - const selected = visible.length === 0 ? 0 : Math.max(0, Math.min(state.selected, visible.length - 1)); + const selected = + visible.length === 0 ? 0 : Math.max(0, Math.min(state.selected, visible.length - 1)); const { start, end } = modelWindow(visible.length, selected); const windowed = visible.slice(start, end); const badge = `${formatRefreshedBadge(state.refreshedAt, nowMs)}${state.loading ? ' · refreshing…' : ''}`; @@ -124,21 +146,10 @@ export function ModelPickerView(props: Readonly): ReactEle entry.provider, ...(ctx.length > 0 ? [ctx] : []), formatModelPrice(entry), - // Distinguish the two unavailable reasons (2.5.G key-awareness): a keyless provider names the remedy, - // vs the pre-existing "not on your key" (a keyed provider whose live list omits the model). - ...(entry.available - ? [] - : entry.unavailableReason === 'no-key' - ? [`no key for ${entry.provider}`] - : ['unavailable on your key']), + ...unavailableParts(entry), ...(entry.deprecated ? ['deprecated'] : []), ]; - // Selected → cyan (highlight wins for visibility); else an unavailable/deprecated row is dimmed; else default. - const rowColor = isSelected - ? colorProps(color, 'cyan') - : !entry.available || entry.deprecated - ? dimProps(color) - : {}; + const rowColor = rowColorFor(entry, isSelected, color); return ( {`${isSelected ? '›' : ' '} ${isDefault ? '✓' : ' '} ${parts.join(' · ')}`} diff --git a/apps/cli/src/render/tui/model-picker.test.ts b/apps/cli/src/render/tui/model-picker.test.ts index dfac060b..bb974b47 100644 --- a/apps/cli/src/render/tui/model-picker.test.ts +++ b/apps/cli/src/render/tui/model-picker.test.ts @@ -12,7 +12,9 @@ import { } from './model-picker.js'; /** A merged catalog entry with sensible defaults; override what a case cares about. */ -function entry(partial: Partial & Pick): ModelCatalogEntry { +function entry( + partial: Partial & Pick, +): ModelCatalogEntry { return { provider: 'anthropic', displayName: partial.modelId, @@ -56,7 +58,10 @@ describe('foldModelPickerKey', () => { expect(typed).toEqual({ kind: 'state', state: state({ filter: 'r', selected: 0 }) }); // While a refresh is in flight, Ctrl+R is a no-op (stays open, unchanged) — no racing double-refresh. const loading = state({ loading: true }); - expect(foldModelPickerKey('r', { ctrl: true }, loading)).toEqual({ kind: 'state', state: loading }); + expect(foldModelPickerKey('r', { ctrl: true }, loading)).toEqual({ + kind: 'state', + state: loading, + }); }); it('arrows move the selection, clamped to the visible list', () => { @@ -89,7 +94,9 @@ describe('foldModelPickerKey', () => { it('Enter on a DIMMED (unavailable) model is BLOCKED, never an accept — carrying the provider (ADR §6)', () => { const s = state({ - entries: [entry({ modelId: 'x', displayName: 'Model X', available: false, provider: 'openai' })], + entries: [ + entry({ modelId: 'x', displayName: 'Model X', available: false, provider: 'openai' }), + ], selected: 0, }); expect(foldModelPickerKey('', { return: true }, s)).toEqual({ @@ -129,7 +136,11 @@ describe('foldModelPickerKey', () => { it('a printable char extends the filter and resets the selection; backspace trims it', () => { const typed = foldModelPickerKey('b', {}, state({ selected: 2 })); expect(typed).toEqual({ kind: 'state', state: state({ filter: 'b', selected: 0 }) }); - const trimmed = foldModelPickerKey('', { backspace: true }, state({ filter: 'ab', selected: 1 })); + const trimmed = foldModelPickerKey( + '', + { backspace: true }, + state({ filter: 'ab', selected: 1 }), + ); expect(trimmed).toEqual({ kind: 'state', state: state({ filter: 'a', selected: 0 }) }); // Backspace on an EMPTY filter is inert (Esc cancels; backspace never closes) — stays open, unchanged. expect(foldModelPickerKey('', { backspace: true }, state({ filter: '' }))).toEqual({ @@ -193,7 +204,11 @@ describe('foldModelPickerKey — the ADR-0066 effort sub-step', () => { }); it('opens the effort sub-list on the session BOUND effort when one is set', () => { - const step = foldModelPickerKey('', { return: true }, reasoningState({ currentEffort: 'high' })); + const step = foldModelPickerKey( + '', + { return: true }, + reasoningState({ currentEffort: 'high' }), + ); if (step.kind !== 'state') throw new Error('expected a state step'); expect(step.state.effortSelected).toBe(3); // index of 'high' }); @@ -217,10 +232,12 @@ describe('foldModelPickerKey — the ADR-0066 effort sub-step', () => { state: effortPhase({ effortSelected: 1 }), }); // Clamp at the ends: max (index 4) Down stays 4; off (index 0) Up stays 0. - expect(foldModelPickerKey('', { downArrow: true }, effortPhase({ effortSelected: 4 }))).toEqual({ - kind: 'state', - state: effortPhase({ effortSelected: 4 }), - }); + expect(foldModelPickerKey('', { downArrow: true }, effortPhase({ effortSelected: 4 }))).toEqual( + { + kind: 'state', + state: effortPhase({ effortSelected: 4 }), + }, + ); expect(foldModelPickerKey('', { upArrow: true }, effortPhase({ effortSelected: 0 }))).toEqual({ kind: 'state', state: effortPhase({ effortSelected: 0 }), @@ -247,7 +264,10 @@ describe('foldModelPickerKey — the ADR-0066 effort sub-step', () => { it('Esc in the effort phase BACKS OUT to the model list (clears pending) — it does NOT cancel the picker', () => { const step = foldModelPickerKey('', { escape: true }, effortPhase()); - expect(step).toEqual({ kind: 'state', state: effortPhase({ phase: 'model', pending: undefined }) }); + expect(step).toEqual({ + kind: 'state', + state: effortPhase({ phase: 'model', pending: undefined }), + }); }); it('Ctrl-C in the effort phase is the HARD cancel (closes the whole picker)', () => { @@ -335,9 +355,11 @@ describe('display formatters', () => { it('summarizes partial refresh failures into a secret-free banner (or undefined when clean)', () => { expect(partialFailureBanner([])).toBeUndefined(); - expect(partialFailureBanner(['openai'])).toBe('couldn\'t refresh openai — showing last-known models'); + expect(partialFailureBanner(['openai'])).toBe( + "couldn't refresh openai — showing last-known models", + ); expect(partialFailureBanner(['openai', 'gemini'])).toBe( - 'couldn\'t refresh openai, gemini — showing last-known models', + "couldn't refresh openai, gemini — showing last-known models", ); }); }); diff --git a/apps/cli/src/render/tui/model-picker.ts b/apps/cli/src/render/tui/model-picker.ts index bbfaec05..21338f01 100644 --- a/apps/cli/src/render/tui/model-picker.ts +++ b/apps/cli/src/render/tui/model-picker.ts @@ -55,11 +55,13 @@ export interface ModelPickerState { */ readonly effortStep: boolean; /** The model chosen in `'model'` phase, awaiting an effort pick — carried so `'effort'`'s accept emits the pair. */ - readonly pending: { - readonly modelId: string; - readonly displayName: string; - readonly provider: ProviderId; - } | undefined; + readonly pending: + | { + readonly modelId: string; + readonly displayName: string; + readonly provider: ProviderId; + } + | undefined; /** The highlighted index into {@link REASONING_EFFORTS} while in `'effort'` phase. */ readonly effortSelected: number; /** The session's currently-bound effort (the `✓` in the effort sub-list + the initial highlight); `undefined` ⇒ @@ -130,10 +132,16 @@ function foldArrow( visibleCount: number, ): ModelPickerStep | undefined { if (key.upArrow === true) { - return { kind: 'state', state: { ...state, selected: clampSelection(state.selected - 1, visibleCount) } }; + return { + kind: 'state', + state: { ...state, selected: clampSelection(state.selected - 1, visibleCount) }, + }; } if (key.downArrow === true) { - return { kind: 'state', state: { ...state, selected: clampSelection(state.selected + 1, visibleCount) } }; + return { + kind: 'state', + state: { ...state, selected: clampSelection(state.selected + 1, visibleCount) }, + }; } return undefined; } @@ -163,6 +171,73 @@ export function foldModelPickerKey( * backspace trims the filter; a single printable code point extends the filter (a multi-char paste blob is dropped); * every other key stays open. */ +/** + * `Enter` in the model phase: act on the highlighted model. An empty (over-filtered) list is a gentle close; a + * DIMMED (unavailable) model yields `blocked` (non-selectable, ADR §6); a reasoning-capable model on a reseat + * surface (`effortStep`) advances to the `'effort'` sub-step opened on the bound effort (else a neutral middle + * tier); otherwise it accepts the model. + */ +function acceptVisibleModel( + state: ModelPickerState, + visible: readonly ModelCatalogEntry[], +): ModelPickerStep { + const chosen = visible[clampSelection(state.selected, visible.length)]; + if (chosen === undefined) return { kind: 'close' }; // an empty list — Enter is a gentle cancel + if (!chosen.available) { + return { + kind: 'blocked', + displayName: chosen.displayName, + provider: chosen.provider, + ...(chosen.unavailableReason !== undefined ? { reason: chosen.unavailableReason } : {}), + }; + } + if (state.effortStep && chosen.supportsReasoning) { + return { + kind: 'state', + state: { + ...state, + phase: 'effort', + pending: { + modelId: chosen.modelId, + displayName: chosen.displayName, + provider: chosen.provider, + }, + effortSelected: initialEffortIndex(state.currentEffort), + hint: undefined, + }, + }; + } + return { + kind: 'accept', + modelId: chosen.modelId, + displayName: chosen.displayName, + provider: chosen.provider, + }; +} + +/** + * Filter editing in the model phase: backspace trims a whole CODE POINT (so an astral char is removed whole — the + * other submodes' `dropLastCodePoint` discipline); a single printable code point extends the filter (a multi-char + * paste blob is dropped, matching the other submodes); any other key is inert. + */ +function foldFilterKey( + char: string, + key: ModelPickerKey, + state: ModelPickerState, +): ModelPickerStep { + if (key.backspace === true || key.delete === true) { + if (state.filter.length === 0) return { kind: 'state', state }; // nothing to trim (Esc cancels; backspace is inert) + return { + kind: 'state', + state: { ...state, filter: dropLastCodePoint(state.filter), selected: 0 }, + }; + } + if ([...char].length === 1 && key.ctrl !== true && key.meta !== true) { + return { kind: 'state', state: { ...state, filter: state.filter + char, selected: 0 } }; + } + return { kind: 'state', state }; +} + function foldModelPhaseKey( char: string, key: ModelPickerKey, @@ -178,53 +253,8 @@ function foldModelPhaseKey( const visible = visibleModels(state); const arrow = foldArrow(key, state, visible.length); if (arrow !== undefined) return arrow; - if (key.return === true) { - const chosen = visible[clampSelection(state.selected, visible.length)]; - if (chosen === undefined) return { kind: 'close' }; // an empty list — Enter is a gentle cancel - if (!chosen.available) { - return { - kind: 'blocked', - displayName: chosen.displayName, - provider: chosen.provider, - ...(chosen.unavailableReason !== undefined ? { reason: chosen.unavailableReason } : {}), - }; - } - // A reasoning-capable model on a reseat surface advances to the effort sub-step (ADR-0066) instead of accepting - // immediately; the sub-list opens on the session's bound effort (else a neutral middle tier). - if (state.effortStep && chosen.supportsReasoning) { - return { - kind: 'state', - state: { - ...state, - phase: 'effort', - pending: { - modelId: chosen.modelId, - displayName: chosen.displayName, - provider: chosen.provider, - }, - effortSelected: initialEffortIndex(state.currentEffort), - hint: undefined, - }, - }; - } - return { - kind: 'accept', - modelId: chosen.modelId, - displayName: chosen.displayName, - provider: chosen.provider, - }; - } - if (key.backspace === true || key.delete === true) { - if (state.filter.length === 0) return { kind: 'state', state }; // nothing to trim (Esc cancels; backspace is inert) - // Trim by whole CODE POINT so backspacing an astral char removes it whole (no lone surrogate) — same discipline - // as the other submodes' `dropLastCodePoint`. - return { kind: 'state', state: { ...state, filter: dropLastCodePoint(state.filter), selected: 0 } }; - } - // A single printable code point extends the filter (a multi-char paste blob is dropped); any other key stays open. - if ([...char].length === 1 && key.ctrl !== true && key.meta !== true) { - return { kind: 'state', state: { ...state, filter: state.filter + char, selected: 0 } }; - } - return { kind: 'state', state }; + if (key.return === true) return acceptVisibleModel(state, visible); + return foldFilterKey(char, key, state); } /** @@ -251,7 +281,8 @@ function foldEffortPhaseKey( } if (key.return === true) { const pending = state.pending; - const effort = REASONING_EFFORTS[clampSelection(state.effortSelected, REASONING_EFFORTS.length)]; + const effort = + REASONING_EFFORTS[clampSelection(state.effortSelected, REASONING_EFFORTS.length)]; // Defensive: a missing pending model (never expected — set on the transition) or an out-of-range tier backs out // to the model list rather than emitting a malformed accept. if (pending === undefined || effort === undefined) { @@ -269,24 +300,10 @@ function foldEffortPhaseKey( } /** The effort sub-list's opening highlight: the session's bound effort, else a neutral middle tier (`'medium'`). */ -function initialEffortIndex(currentEffort: ReasoningEffort | undefined): number { - const target = currentEffort ?? 'medium'; - const index = REASONING_EFFORTS.indexOf(target); - return index < 0 ? 0 : index; +function initialEffortIndex(currentEffort: ReasoningEffort = 'medium'): number { + return Math.max(0, REASONING_EFFORTS.indexOf(currentEffort)); } -/** - * The one-line hint shown beside each reasoning-effort tier in the effort sub-list (ADR-0066). Display-only, so the - * picker explains what each tier trades off (latency/cost vs depth) without the user consulting the docs. - */ -export const EFFORT_TIER_HINT: Record = { - off: 'no reasoning — fastest, lowest cost', - low: 'brief reasoning', - medium: 'balanced reasoning', - high: 'deep reasoning', - max: 'maximum reasoning — slowest, highest cost', -}; - /* -------------------------------------------------------------------------------------------------- * * Pure display formatters (unit-tested; the ink view is not render-tested, per the repo convention). * -------------------------------------------------------------------------------------------------- */ @@ -342,9 +359,7 @@ export function formatRefreshedBadge(refreshedAt: number | undefined, now: numbe * kept last-known rows (ADR-0064 §8: drift/failure is visible, non-fatal). The per-provider `error` strings are * already seam-redacted; this joins only the provider ids, never an error body. */ -export function partialFailureBanner( - failedProviders: readonly string[], -): string | undefined { +export function partialFailureBanner(failedProviders: readonly string[]): string | undefined { if (failedProviders.length === 0) return undefined; const list = failedProviders.join(', '); return `couldn't refresh ${list} — showing last-known models`; diff --git a/docs/decisions/0011-internal-llm-abstraction.md b/docs/decisions/0011-internal-llm-abstraction.md index 5316ea34..adf80b47 100644 --- a/docs/decisions/0011-internal-llm-abstraction.md +++ b/docs/decisions/0011-internal-llm-abstraction.md @@ -27,9 +27,9 @@ The seam is the immovable contract; the adapter implementation behind it is deli > *resolved key* on Node-style hosts (CLI, VS Code host, Phase-2 Bun API) and a key > *reference* on the desktop, where Rust performs the egress via `llm_stream`. The seam's > **types and contract are unchanged**; only the per-host transport wiring is refined. - +> > **Amended 2026-07-05 by [ADR-0064](0064-live-model-catalog.md) and [ADR-0065](0065-provider-economics-and-extensibility.md)** (append-only — this body is unchanged). The seam's method **set** grows again (its **shape** stays frozen): ADR-0064 adds an **optional `listModels?`** capability (returning a Relavium-typed `ModelListing[]` — the vendor `models.list()` is mapped inside the adapter, no vendor type crosses) plus a provider **`kind`** protocol abstraction (`anthropic` | `openai-compatible` | `gemini`) that derives the adapter, list-models endpoint, auth, and response-mapper **per protocol** — formalizing the DeepSeek-via-OpenAI-compatible precedent noted in the Decision above. ADR-0065 makes the host `resolveProvider` build the adapter from the **stored provider row** (`kind` + a custom `base_url`) rather than only the static default registry, and injects a **pricing overlay** into the cost path so a user-priced model prices instead of throwing. The provider-**id** `z.enum` stays **closed** — a truly-open custom-provider registry is named as a future supersede of this ADR, not taken here. - +> > Amended 2026-06-05: the same `LLMProvider` seam is reused **unchanged** by the agent-first > `AgentSession` entry point ([ADR-0024](0024-agent-first-entry-point-agentsession.md)) — chat-mode > agents call providers through the identical contract, so no vendor SDK type crosses the seam for diff --git a/docs/decisions/0024-agent-first-entry-point-agentsession.md b/docs/decisions/0024-agent-first-entry-point-agentsession.md index 2bec9fe7..1430fba4 100644 --- a/docs/decisions/0024-agent-first-entry-point-agentsession.md +++ b/docs/decisions/0024-agent-first-entry-point-agentsession.md @@ -9,7 +9,7 @@ > [ADR-0050](0050-cli-history-db-at-rest-posture.md)). Only the **desktop** surface uses a SQLCipher-encrypted > store. The Context below originally said "encrypted" without that surface distinction; read it as > surface-specific. - +> > Amended 2026-07-06: the "one agent + one model bound for the session lifetime" rule is **refined** (not > reversed) by [ADR-0059](0059-cli-mid-session-model-reseat.md) — a mid-session `/models` **model switch** is a > host-side **reseat** (a new `AgentSession.resume` instance bound to the new model, carrying the text-only diff --git a/docs/reference/cli/home.md b/docs/reference/cli/home.md index 75ff4255..4c7a0827 100644 --- a/docs/reference/cli/home.md +++ b/docs/reference/cli/home.md @@ -27,7 +27,7 @@ stdoutIsTty && stdinIsTty && !json && !isCiEnv(env) When a bare `relavium` opens the Home on a **truly key-less** run — **no** known provider has a resolvable key (the resolver finds neither an OS-keychain key **nor** a `RELAVIUM__API_KEY` env var) — a `@clack/prompts` **onboarding wizard** runs **before** the ink Home mounts (clack + ink both take the terminal's raw mode, so the wizard fully settles first). It is already behind the TTY gate above, so it never runs piped / `--json` / in CI. A run with **either** a keychain key or an env key is not key-less — no wizard, so a working user (including an env-key user) is never prompted. -The flow: **pick a provider → paste a hidden (masked) API key → store it in the OS keychain** — riding the same tested `provider set-key` path (keychain write + the provider row + the keychain-ref, secret-free by construction; the key is captured via clack's masked `password`, whitespace-trimmed, never echoed, never written to disk, never logged beyond its last-4 hint). On a successful store the wizard also sets **`[chat].default_model` to the chosen provider's cheap/fast starter model** (`KNOWN_PROVIDERS[provider].testModel`, via the same config-write target as `/models`) — so the very next chat binds a model **whose key was just stored**, not the built-in `claude-sonnet-4-6` (which would error for a user who picked a non-Anthropic provider). Richer **model selection** is not part of the wizard — the user upgrades via the `/models` picker. Three fallbacks keep a first run unblocked: +The flow: **pick a provider → paste a hidden (masked) API key → store it in the OS keychain** — riding the same tested `provider set-key` path (keychain write + the provider row + the keychain-ref, secret-free by construction; the key is captured via clack's masked `password`, whitespace-trimmed, never echoed, never written to disk, never logged beyond its last-4 hint). On a successful store the wizard also sets **`[preferences].default_model` to the chosen provider's cheap/fast starter model** (`KNOWN_PROVIDERS[provider].testModel`, via the same global config-write target as `/models` — `[chat].default_model` is only the read-time fallback that resolves down to it, ADR-0063 §1) — so the very next chat binds a model **whose key was just stored**, not the built-in `claude-sonnet-4-6` (which would error for a user who picked a non-Anthropic provider). Richer **model selection** is not part of the wizard — the user upgrades via the `/models` picker. Three fallbacks keep a first run unblocked: - **Keychain-write failure** (a locked keychain / no Secret Service / a headless box) → the wizard **never** persists the key to disk; it prints the `RELAVIUM__API_KEY` env-var to set instead (the resolver imports an env key at call time), then hands off to the Home. - **Any other store fault** (e.g. a db write failure *after* the keychain write succeeded) → a **generic** "setup could not be completed" note (never mislabeled a keychain failure, and the raw error is never rendered — the key may well be in the keychain, so the copy must not claim otherwise); the underlying issue resurfaces at the Home, which reads the same store. diff --git a/packages/core/src/engine/agent-runner.ts b/packages/core/src/engine/agent-runner.ts index 5540b32a..35ff9f8d 100644 --- a/packages/core/src/engine/agent-runner.ts +++ b/packages/core/src/engine/agent-runner.ts @@ -806,7 +806,11 @@ function resolveGenKnobs( // reasoning-capable (the shared {@link gateReasoningEffort} rule — a non-reasoning model would reject the field). // The per-fallback-entry re-gate lives in the chain (a non-reasoning fallback entry strips the tier), so a failover // to a different-capability model never carries an unsupported field. - const reasoningEffort = gateReasoningEffort(agent.reasoning_effort, agent.model, deps.resolveReasoning); + const reasoningEffort = gateReasoningEffort( + agent.reasoning_effort, + agent.model, + deps.resolveReasoning, + ); return { ...(temperature === undefined ? {} : { temperature }), ...(maxTokens === undefined ? {} : { maxTokens }), diff --git a/packages/core/src/engine/agent-session.test.ts b/packages/core/src/engine/agent-session.test.ts index c910d07a..93b7643d 100644 --- a/packages/core/src/engine/agent-session.test.ts +++ b/packages/core/src/engine/agent-session.test.ts @@ -794,8 +794,10 @@ describe('AgentSession — reseat-less modes + mid-turn abort (ADR-0057 Step 2)' // Reasoning-capable ⇒ the tier reaches the request. const on = capturing(); const onSession = session( - harness([textTurn('ok')], { resolveProvider: () => on.provider, resolveReasoning: () => true }) - .deps, + harness([textTurn('ok')], { + resolveProvider: () => on.provider, + resolveReasoning: () => true, + }).deps, reader, ); onSession.start(); @@ -836,7 +838,8 @@ describe('AgentSession — reseat-less modes + mid-turn abort (ADR-0057 Step 2)' }, }; const s = session( - harness([textTurn('ok')], { resolveProvider: () => provider, resolveReasoning: () => true }).deps, + harness([textTurn('ok')], { resolveProvider: () => provider, resolveReasoning: () => true }) + .deps, reader, ); s.start(); @@ -874,7 +877,8 @@ describe('AgentSession — reseat-less modes + mid-turn abort (ADR-0057 Step 2)' }, }; const s = session( - harness([textTurn('ok')], { resolveProvider: () => provider, resolveReasoning: () => false }).deps, + harness([textTurn('ok')], { resolveProvider: () => provider, resolveReasoning: () => false }) + .deps, reader, ); s.start(); diff --git a/packages/core/src/engine/budget-governor.test.ts b/packages/core/src/engine/budget-governor.test.ts index 12551a4a..1e8833bf 100644 --- a/packages/core/src/engine/budget-governor.test.ts +++ b/packages/core/src/engine/budget-governor.test.ts @@ -149,7 +149,7 @@ describe('BudgetGovernor', () => { }); describe('user-pricing overlay (2.5.G S10, ADR-0065 §2 — closes the cost-cap gap)', () => { - // A user price for a model the static registry does not know — output $9/MTok so 10_000 tok ⇒ 90_000µ¢. + // A user price for a model the static registry does not know — output $9/MTok so 10_000 tok ⇒ 9_000_000µ¢. const OVERLAY: PricingOverlay = new Map([ [ 'acme-custom-1', diff --git a/packages/db/src/model-catalog-store.test.ts b/packages/db/src/model-catalog-store.test.ts index 67b3277e..9633d679 100644 --- a/packages/db/src/model-catalog-store.test.ts +++ b/packages/db/src/model-catalog-store.test.ts @@ -997,6 +997,30 @@ describe('createModelCatalogStore (2.5.G / ADR-0064 — live-discovery cache)', expect(reListing?.outputCostPerMtokMicrocents).toBe(950); // updated }); + it('upsert() distinguishes an OMITTED media cost (preserve) from an explicit null (CLEAR) — the `number | null` contract', () => { + store.upsert({ + providerId, + modelId: 'media-cost-clear', + mediaImageCostMicrocents: 1_900_000, + mediaAudioCostMicrocents: 2_000_000, + }); + // OMITTED ⇒ the existing rate is preserved (the never-clobber invariant). + store.upsert({ providerId, modelId: 'media-cost-clear', source: 'user' }); + const preserved = store.getByModelId('media-cost-clear'); + expect(preserved?.mediaImageCostMicrocents).toBe(1_900_000); + expect(preserved?.mediaAudioCostMicrocents).toBe(2_000_000); + // Explicit `null` ⇒ the stored rate is CLEARED (not treated as an omission by `??`), while a still-omitted + // sibling is preserved. + store.upsert({ + providerId, + modelId: 'media-cost-clear', + mediaImageCostMicrocents: null, + }); + const cleared = store.getByModelId('media-cost-clear'); + expect(cleared?.mediaImageCostMicrocents).toBeNull(); // an explicit null cleared it + expect(cleared?.mediaAudioCostMicrocents).toBe(2_000_000); // the omitted sibling survived + }); + it('a pricing-only upsert PRESERVES the display name + limits of a SOFT-DEACTIVATED row (S10 re-price)', () => { // A live refresh discovers the model with a real name + context, then a later refresh drops it → soft-deactivated // (isActive=false, source='live', deletedAt=null). The active-only `listByProvider` the command reads can no diff --git a/packages/db/src/model-catalog-store.ts b/packages/db/src/model-catalog-store.ts index a7e49450..83e3ee0b 100644 --- a/packages/db/src/model-catalog-store.ts +++ b/packages/db/src/model-catalog-store.ts @@ -381,12 +381,23 @@ export function createModelCatalogStore(db: Db, deps: ModelCatalogStoreDeps): Mo input.capabilities !== undefined ? JSON.stringify(input.capabilities) : (existing?.capabilities ?? JSON.stringify({})), + // The media cost fields are `number | null` (unlike the NOT-NULL text-token prices below): OMITTED + // (`undefined`) preserves the existing row's value (the "never clobber" invariant), an explicit `null` + // CLEARS it, and a number writes it — so a `!== undefined` check (not `??`, which would treat a clearing + // `null` as an omission and preserve the old rate). A true INSERT (`existing` undefined) still defaults to + // `null`. mediaImageCostMicrocents: - input.mediaImageCostMicrocents ?? existing?.mediaImageCostMicrocents ?? null, + input.mediaImageCostMicrocents !== undefined + ? input.mediaImageCostMicrocents + : (existing?.mediaImageCostMicrocents ?? null), mediaAudioCostMicrocents: - input.mediaAudioCostMicrocents ?? existing?.mediaAudioCostMicrocents ?? null, + input.mediaAudioCostMicrocents !== undefined + ? input.mediaAudioCostMicrocents + : (existing?.mediaAudioCostMicrocents ?? null), mediaVideoCostMicrocents: - input.mediaVideoCostMicrocents ?? existing?.mediaVideoCostMicrocents ?? null, + input.mediaVideoCostMicrocents !== undefined + ? input.mediaVideoCostMicrocents + : (existing?.mediaVideoCostMicrocents ?? null), // USER text-token pricing (2.5.G S10) — write the supplied prices, else PRESERVE the existing row's (an // update that omits them must not zero a hand-entered price), else the NOT-NULL default `0`. inputCostPerMtokMicrocents: diff --git a/packages/db/src/provider-store.test.ts b/packages/db/src/provider-store.test.ts index 57eb61fb..05811293 100644 --- a/packages/db/src/provider-store.test.ts +++ b/packages/db/src/provider-store.test.ts @@ -95,14 +95,22 @@ describe('createProviderStore', () => { }); it('reads an absent kind / pricingReferenceUrl as undefined (a plain add omits them)', () => { - store.upsert({ name: 'anthropic', displayName: 'Anthropic', baseUrl: 'https://api.anthropic.com' }); + store.upsert({ + name: 'anthropic', + displayName: 'Anthropic', + baseUrl: 'https://api.anthropic.com', + }); const got = store.get('anthropic'); expect(got?.kind).toBeUndefined(); expect(got?.pricingReferenceUrl).toBeUndefined(); }); it('coerces a FOREIGN stored kind to undefined at the read boundary (fail-closed, no DB CHECK)', () => { - const rec = store.upsert({ name: 'openai', displayName: 'OpenAI', baseUrl: 'https://api.openai.com/v1' }); + const rec = store.upsert({ + name: 'openai', + displayName: 'OpenAI', + baseUrl: 'https://api.openai.com/v1', + }); // Simulate a tampered/foreign value written outside the typed setter (no DB CHECK on the ALTER-ADD column). client.db.run(sql`update llm_providers set kind = 'rogue-protocol' where id = ${rec.id}`); expect(store.get('openai')?.kind).toBeUndefined(); // a non-PROVIDER_KINDS value is never trusted diff --git a/packages/llm/src/adapters/gemini.test.ts b/packages/llm/src/adapters/gemini.test.ts index d54817bd..48c9ceda 100644 --- a/packages/llm/src/adapters/gemini.test.ts +++ b/packages/llm/src/adapters/gemini.test.ts @@ -480,28 +480,38 @@ describe('Gemini adapter — request building (buildGeminiRequest)', () => { it('maps the reasoning-effort tier to thinkingConfig.thinkingLevel + includeThoughts on a thinking tier (ADR-0066)', () => { // A non-off tier also sets includeThoughts:true so raising effort SURFACES the reasoning it bills for (the only // switch that returns Gemini thought parts). All five tiers assert thinkingLevel; medium is the picker default. - expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'high' }).config['thinkingConfig']).toEqual({ + expect( + buildGeminiRequest({ ...REQ, reasoningEffort: 'high' }).config['thinkingConfig'], + ).toEqual({ thinkingLevel: 'HIGH', includeThoughts: true, }); // Gemini tops out at HIGH — `max` coarsens to it (no separate xhigh/max tier). - expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'max' }).config['thinkingConfig']).toEqual({ - thinkingLevel: 'HIGH', - includeThoughts: true, - }); - expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'medium' }).config['thinkingConfig']).toEqual({ + expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'max' }).config['thinkingConfig']).toEqual( + { + thinkingLevel: 'HIGH', + includeThoughts: true, + }, + ); + expect( + buildGeminiRequest({ ...REQ, reasoningEffort: 'medium' }).config['thinkingConfig'], + ).toEqual({ thinkingLevel: 'MEDIUM', includeThoughts: true, }); - expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'low' }).config['thinkingConfig']).toEqual({ - thinkingLevel: 'LOW', - includeThoughts: true, - }); + expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'low' }).config['thinkingConfig']).toEqual( + { + thinkingLevel: 'LOW', + includeThoughts: true, + }, + ); // Gemini has no universal disable (a Pro model rejects budget 0) — `off` degrades to the lowest tier and does // NOT force thought output on (minimal thinking). - expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'off' }).config['thinkingConfig']).toEqual({ - thinkingLevel: 'MINIMAL', - }); + expect(buildGeminiRequest({ ...REQ, reasoningEffort: 'off' }).config['thinkingConfig']).toEqual( + { + thinkingLevel: 'MINIMAL', + }, + ); expect('thinkingConfig' in buildGeminiRequest(REQ).config).toBe(false); // unset ⇒ omitted (provider default) }); diff --git a/packages/llm/src/adapters/gemini.ts b/packages/llm/src/adapters/gemini.ts index 23439dc9..4424818c 100644 --- a/packages/llm/src/adapters/gemini.ts +++ b/packages/llm/src/adapters/gemini.ts @@ -558,6 +558,31 @@ const GEMINI_RESPONSE_MODALITY: Record = { video: 'VIDEO', }; +/** + * The Gemini `thinkingConfig` for a reasoning-effort tier (ADR-0066): DEEP-merge onto a caller's + * `providerOptions.thinkingConfig` so the canonical `thinkingLevel` wins on THAT key while the caller's sibling + * knobs (`includeThoughts` / `thinkingBudget`) survive — a shallow replace would silently drop them, so turning + * effort up could paradoxically SILENCE the reasoning output the caller enabled while still billing thought tokens + * (mirrors the Anthropic adapter preserving `output_config.format`). A non-off tier defaults `includeThoughts: true` + * — the ONLY switch that returns thought parts (the adapter's reasoning stream depends on it) — so raising effort + * actually shows more reasoning, but never overriding a caller's explicit choice, and never forced for `'off'`. + */ +function buildThinkingConfig( + reasoningEffort: ReasoningEffort, + providerOptions: LlmRequest['providerOptions'], +): Record { + const poThinking: Record = + providerOptions !== undefined && isRecord(providerOptions['thinkingConfig']) + ? providerOptions['thinkingConfig'] + : {}; + const surfaceThoughts = reasoningEffort !== 'off' && poThinking['includeThoughts'] === undefined; + return { + ...poThinking, + thinkingLevel: GEMINI_THINKING_LEVEL[reasoningEffort], + ...(surfaceThoughts ? { includeThoughts: true } : {}), + }; +} + /** Lower a canonical request into the Gemini request shape (system → `systemInstruction`, etc.). */ export function buildGeminiRequest(req: LlmRequest): GeminiRequest { const config: Record = {}; @@ -582,25 +607,8 @@ export function buildGeminiRequest(req: LlmRequest): GeminiRequest { config['maxOutputTokens'] = req.maxTokens; } if (req.reasoningEffort !== undefined) { - // ADR-0066: Gemini's tier-native thinking control (thinkingLevel). DEEP-merge onto a caller's - // providerOptions.thinkingConfig so the canonical thinkingLevel wins on THAT key while the caller's sibling - // knobs (includeThoughts / thinkingBudget) survive — a shallow replace (config wins on the top-level merge - // below) would silently drop them, so turning effort up could paradoxically SILENCE the reasoning output the - // caller enabled while still billing thought tokens. (Mirrors the Anthropic adapter preserving output_config.format.) - const poThinking: Record = - req.providerOptions !== undefined && isRecord(req.providerOptions['thinkingConfig']) - ? req.providerOptions['thinkingConfig'] - : {}; - config['thinkingConfig'] = { - ...poThinking, - thinkingLevel: GEMINI_THINKING_LEVEL[req.reasoningEffort], - // Surface the reasoning the tier bills for: `includeThoughts` is the ONLY switch that returns thought parts - // (the adapter's reasoning stream depends on it). Default it on for a thinking tier so raising effort actually - // shows more reasoning — but never override a caller's explicit choice, and never force it for 'off' (minimal). - ...(req.reasoningEffort !== 'off' && poThinking['includeThoughts'] === undefined - ? { includeThoughts: true } - : {}), - }; + // ADR-0066: Gemini's tier-native thinking control — see {@link buildThinkingConfig} for the deep-merge rationale. + config['thinkingConfig'] = buildThinkingConfig(req.reasoningEffort, req.providerOptions); } if (req.outputModalities !== undefined && req.outputModalities.some((m) => m !== 'text')) { // Lower the node's non-text output_modalities to Gemini `responseModalities` (inline media-out, diff --git a/packages/llm/src/adapters/openai.test.ts b/packages/llm/src/adapters/openai.test.ts index 4655fa1b..7a0f9e00 100644 --- a/packages/llm/src/adapters/openai.test.ts +++ b/packages/llm/src/adapters/openai.test.ts @@ -875,12 +875,17 @@ describe('openaiErrorToLlmError — classification', () => { ).toMatchObject({ kind: 'unknown', retryable: false }); }); - it('EXACT-redacts the resolved key from an echoed error body (a custom endpoint\'s opaque key) (2.5.G S9)', () => { + it("EXACT-redacts the resolved key from an echoed error body (a custom endpoint's opaque key) (2.5.G S9)", () => { // A custom OpenAI-compatible endpoint's key has no `sk-`/`Bearer` shape, so the shape-based scrubSecrets can't // match it — a hostile/misconfigured proxy that echoes the received credential in its error body would leak the // real key into history.db / --json / the TUI unless the resolved key is exact-redacted (CLAUDE.md #6). const key = ['opaque', 'proxy', 'CREDENTIAL', '4f2a9'].join('-'); // no vendor key shape - const echoed = new APIError(401, undefined, `rejected token '${key}' for this endpoint`, undefined); + const echoed = new APIError( + 401, + undefined, + `rejected token '${key}' for this endpoint`, + undefined, + ); // WITHOUT the key (the listModels path redacts separately) the opaque token would pass through... expect(openaiErrorToLlmError(echoed, 'openai').message).toContain(key); // ...WITH the resolved key threaded (generate/stream/media), it is exact-redacted before it can escape. diff --git a/packages/llm/src/adapters/openai.ts b/packages/llm/src/adapters/openai.ts index c7cb3edb..2716b90d 100644 --- a/packages/llm/src/adapters/openai.ts +++ b/packages/llm/src/adapters/openai.ts @@ -652,7 +652,10 @@ function toOpenAiTool(toolDef: ToolDef, provider: ProviderId): OpenAI.ChatComple /** ADR-0066: the normalized reasoning-effort tier → OpenAI's native `reasoning_effort` values. `off`→'none', * `max`→'xhigh' (its highest); low/medium/high are 1:1. A SUBSET of the SDK's `ReasoningEffort` union, so the * assignment to `body.reasoning_effort` needs no cast. */ -const OPENAI_REASONING_EFFORT: Record = { +const OPENAI_REASONING_EFFORT: Record< + ReasoningEffort, + 'none' | 'low' | 'medium' | 'high' | 'xhigh' +> = { off: 'none', low: 'low', medium: 'medium', diff --git a/packages/llm/src/adapters/shared.ts b/packages/llm/src/adapters/shared.ts index ed652c91..d5ed85c7 100644 --- a/packages/llm/src/adapters/shared.ts +++ b/packages/llm/src/adapters/shared.ts @@ -254,6 +254,33 @@ export function assertListModelsShape( } } +/** + * Resolve the base {@link LlmError} for a failed {@link boundedListModels} race, BEFORE the final redact + + * `cause`-strip re-wrap. A timeout wins first; then a pre-classified error (the §8 drift throw, or any + * adapter-side `LlmProviderError`) passes THROUGH with its own `kind` — never re-run through `classify` + * (which would flatten a `bad_request` drift throw to `unknown`); otherwise the adapter classifier runs. + */ +function resolveListModelsError(params: { + readonly provider: ProviderId; + readonly err: unknown; + readonly timedOut: boolean; + readonly timeoutMs: number; + readonly classify: (err: unknown) => LlmError; +}): LlmError { + const { provider, err, timedOut, timeoutMs, classify } = params; + if (timedOut) { + return makeLlmError({ + provider, + kind: 'timeout', + message: `model list timed out after ${String(timeoutMs)}ms`, + }); + } + if (err instanceof LlmProviderError) { + return err.llmError; + } + return classify(err); +} + /** * Run a `listModels` collect bounded + abortable + secret-free (ADR-0064 §3). An internal * `AbortController` is threaded to the SDK (so a caller `signal` OR the hard timeout actually cancels the @@ -298,20 +325,10 @@ export async function boundedListModels(params: { try { return await Promise.race([collecting, timeout]); } catch (err) { - // A pre-classified error (the §8 drift throw, or any adapter-side `LlmProviderError`) passes THROUGH - // with its own `kind` — never re-run through `classify` (which would flatten a `bad_request` drift - // throw to `unknown`). It is still redacted + `cause`-stripped by the re-wrap below. - const base = timedOut - ? makeLlmError({ - provider, - kind: 'timeout', - message: `model list timed out after ${String(timeoutMs)}ms`, - }) - : err instanceof LlmProviderError - ? err.llmError - : classify(err); - // Re-wrap through makeLlmError so scrubSecrets runs again AND redactKey strips the resolved key; never - // pass `cause` (it could carry the key or the raw vendor payload — ADR-0064 §3). + // Resolve the base error (timeout wins; a pre-classified `LlmProviderError` passes through with its own + // `kind`), then re-wrap through makeLlmError so scrubSecrets runs again AND redactKey strips the resolved + // key; never pass `cause` (it could carry the key or the raw vendor payload — ADR-0064 §3). + const base = resolveListModelsError({ provider, err, timedOut, timeoutMs, classify }); throw new LlmProviderError( makeLlmError({ provider, diff --git a/packages/llm/src/cost-tracker.test.ts b/packages/llm/src/cost-tracker.test.ts index f3199b82..8ca02cfb 100644 --- a/packages/llm/src/cost-tracker.test.ts +++ b/packages/llm/src/cost-tracker.test.ts @@ -251,9 +251,9 @@ describe('user-pricing overlay (2.5.G S10, ADR-0065 §2)', () => { it('CostTracker WITHOUT an overlay throws on the same unknown model (degrades loudly, never a silent 0)', () => { const tracker = new CostTracker(); - expect(() => tracker.record('acme-custom-1', { inputTokens: 1000, outputTokens: 500 })).toThrowError( - UnknownModelError, - ); + expect(() => + tracker.record('acme-custom-1', { inputTokens: 1000, outputTokens: 500 }), + ).toThrowError(UnknownModelError); }); }); diff --git a/packages/llm/src/fallback-chain.test.ts b/packages/llm/src/fallback-chain.test.ts index 214c8e58..9b42d476 100644 --- a/packages/llm/src/fallback-chain.test.ts +++ b/packages/llm/src/fallback-chain.test.ts @@ -586,7 +586,10 @@ describe('withEntryModel (ADR-0066 §4 — per-fallback-entry reasoning gate)', model: 'gpt-4o', messages: [{ role: 'user', content: [{ type: 'text', text: 'hi' }] }], }; - expect(withEntryModel(req, 'claude-sonnet-4-6')).toEqual({ ...req, model: 'claude-sonnet-4-6' }); + expect(withEntryModel(req, 'claude-sonnet-4-6')).toEqual({ + ...req, + model: 'claude-sonnet-4-6', + }); }); }); diff --git a/packages/llm/src/model-catalog.ts b/packages/llm/src/model-catalog.ts index aa8f7309..9baa47de 100644 --- a/packages/llm/src/model-catalog.ts +++ b/packages/llm/src/model-catalog.ts @@ -98,34 +98,55 @@ interface Tiers { /** The earlier of two optional ISO dates (their "union" for deprecation), skipping any that fails to parse. */ function earlierIsoDate(a: string | undefined, b: string | undefined): string | undefined { - const pa = a === undefined ? NaN : Date.parse(a); - const pb = b === undefined ? NaN : Date.parse(b); + const pa = a === undefined ? Number.NaN : Date.parse(a); + const pb = b === undefined ? Number.NaN : Date.parse(b); if (Number.isNaN(pa)) return Number.isNaN(pb) ? undefined : b; if (Number.isNaN(pb)) return a; return pa <= pb ? a : b; } +/** The pricing provenance for a merged entry: the registry wins, then the user tier, else none (ADR-0064 §6). */ +function pricingSourceOf(t: Tiers): PricingSource { + if (t.registry) return 'registry'; + if (t.user) return 'user'; + return 'none'; +} + /** - * Reconcile live discovery ⋈ the static registry ⋈ the user tier into one deterministically-ordered catalog - * (ADR-0064 §6). Pure: no I/O, no `Date.now()` (the caller passes `now`). Per-field precedence — - * availability ← live (else static presence); price ← registry ?? user (never live); context/output ← live ?? - * static ?? user; deprecation ← the earliest of the static, live, and user dates; priceKnown ← a static or - * user price exists. + * Availability + its reason (2.5.G key-awareness). Key gate FIRST: a provider absent from `keyedProviders` has no + * resolvable key, so its model is genuinely uncallable → unavailable with an actionable `'no-key'` reason, + * regardless of live/static presence. A KEYED provider keeps the pre-existing rule: live-list membership when it + * has live data (a static model absent from the list is `'not-on-key'`-dimmed), else static presence (the ADR-0064 + * §6 "never everything unavailable" safe default — PRESERVED, but now only for a KEYED provider). `keyedProviders` + * ABSENT ⇒ not key-gated (every provider treated as keyed): the `available` BOOLEAN is unchanged from pre-change; + * the only new output is the additive `'not-on-key'` reason on a live-omitted static model — informational. */ -export function mergeModelCatalog(input: MergeModelCatalogInput): ModelCatalogEntry[] { - const live = input.live ?? new Map(); - const userPricing = input.userPricing ?? new Map(); - const tiers = new Map(); +function resolveAvailability( + t: Tiers, + live: ReadonlyMap, + keyedProviders: ReadonlySet | undefined, +): { available: boolean; unavailableReason?: 'no-key' | 'not-on-key' } { + const providerKeyed = keyedProviders === undefined || keyedProviders.has(t.provider); + if (!providerKeyed) return { available: false, unavailableReason: 'no-key' }; + if (live.has(t.provider)) { + return t.live !== undefined + ? { available: true } + : { available: false, unavailableReason: 'not-on-key' }; + } + return { available: true }; +} - // Registry tier — every static model. +/** Build the tier map (registry ⋈ live ⋈ user). A listing/price whose id COLLIDES with a model already anchored to + * a DIFFERENT provider is IGNORED, so a mis-keyed or custom-endpoint rogue id can never corrupt an unrelated entry + * (model ids are globally unique in practice). */ +function buildTiers( + live: ReadonlyMap, + userPricing: ReadonlyMap, +): Map { + const tiers = new Map(); for (const [id, registry] of Object.entries(MODEL_PRICING) as [string, ModelPricing][]) { tiers.set(id, { provider: registry.provider, registry }); } - // Live tier — per provider present in the map. The map key is authoritative for a live-only id's provider. - // A listing whose id COLLIDES with a model already anchored to a DIFFERENT provider is IGNORED, so a live - // list fetched under one provider key can never overwrite another provider's static context/output/name/ - // deprecation (model ids are globally unique in practice; this guard keeps a mis-keyed or custom-endpoint - // rogue id from corrupting an unrelated entry). for (const [provider, listings] of live) { for (const listing of listings) { const prev = tiers.get(listing.id); @@ -133,68 +154,69 @@ export function mergeModelCatalog(input: MergeModelCatalogInput): ModelCatalogEn tiers.set(listing.id, { ...prev, provider: prev?.provider ?? provider, live: listing }); } } - // User tier — fills an unknown id; a known id keeps its registry provider. Same cross-provider guard. for (const [id, pricing] of userPricing) { const prev = tiers.get(id); if (prev !== undefined && prev.provider !== pricing.provider) continue; // cross-provider id collision — drop tiers.set(id, { ...prev, provider: prev?.provider ?? pricing.provider, user: pricing }); } + return tiers; +} +/** Reconcile one tier-set into a catalog entry (ADR-0064 §6 per-field precedence). */ +function buildEntry( + modelId: string, + t: Tiers, + input: MergeModelCatalogInput, + live: ReadonlyMap, +): ModelCatalogEntry { + const pricing = t.registry ?? t.user; // registry wins for a known id; user fills an unknown one. + const pricingSource = pricingSourceOf(t); + const contextWindowTokens = + t.live?.contextWindowTokens ?? t.registry?.contextWindowTokens ?? t.user?.contextWindowTokens; + const maxOutputTokens = + t.live?.maxOutputTokens ?? t.registry?.maxOutputTokens ?? t.user?.maxOutputTokens; + const { available, unavailableReason } = resolveAvailability(t, live, input.keyedProviders); + const deprecatedAt = earlierIsoDate( + earlierIsoDate(t.registry?.deprecatedAt, t.live?.deprecatedAt), + t.user?.deprecatedAt, + ); + const parsedDeprecation = deprecatedAt === undefined ? Number.NaN : Date.parse(deprecatedAt); + const deprecated = !Number.isNaN(parsedDeprecation) && parsedDeprecation <= input.now; + return { + modelId, + provider: t.provider, + displayName: t.registry?.displayName ?? t.live?.displayName ?? t.user?.displayName ?? modelId, + ...(contextWindowTokens !== undefined ? { contextWindowTokens } : {}), + ...(maxOutputTokens !== undefined ? { maxOutputTokens } : {}), + ...(pricing !== undefined ? { pricing } : {}), + pricingSource, + priceKnown: pricingSource !== 'none', + available, + ...(unavailableReason !== undefined ? { unavailableReason } : {}), + deprecated, + ...(deprecatedAt !== undefined ? { deprecatedAt } : {}), + // Reasoning capability via the SAME authority as the engine gate (ADR-0066 §4): the registry flag for a known + // id (authoritative — true or false), else the conservative id heuristic for a live-discovered id. So the + // picker's effort sub-step lights up exactly for the models the engine will actually honor — including a newly + // released reasoning family member absent from the registry. + supportsReasoning: modelSupportsReasoning(modelId), + }; +} + +/** + * Reconcile live discovery ⋈ the static registry ⋈ the user tier into one deterministically-ordered catalog + * (ADR-0064 §6). Pure: no I/O, no `Date.now()` (the caller passes `now`). Per-field precedence — + * availability ← live (else static presence); price ← registry ?? user (never live); context/output ← live ?? + * static ?? user; deprecation ← the earliest of the static, live, and user dates; priceKnown ← a static or + * user price exists. + */ +export function mergeModelCatalog(input: MergeModelCatalogInput): ModelCatalogEntry[] { + const live = input.live ?? new Map(); + const userPricing = input.userPricing ?? new Map(); + const tiers = buildTiers(live, userPricing); const entries: ModelCatalogEntry[] = []; for (const [modelId, t] of tiers) { - const pricing = t.registry ?? t.user; // registry wins for a known id; user fills an unknown one. - const pricingSource: PricingSource = t.registry ? 'registry' : t.user ? 'user' : 'none'; - const contextWindowTokens = - t.live?.contextWindowTokens ?? t.registry?.contextWindowTokens ?? t.user?.contextWindowTokens; - const maxOutputTokens = - t.live?.maxOutputTokens ?? t.registry?.maxOutputTokens ?? t.user?.maxOutputTokens; - // Availability (2.5.G key-awareness). Key gate FIRST: a provider absent from `keyedProviders` has no - // resolvable key, so its model is genuinely uncallable → unavailable with an actionable `'no-key'` reason, - // regardless of live/static presence. A KEYED provider keeps the pre-existing rule: live-list membership when - // it has live data (a static model absent from the list is `'not-on-key'`-dimmed), else static presence (the - // ADR-0064 §6 "never everything unavailable" safe default — PRESERVED, but now only for a KEYED provider). - // `keyedProviders` ABSENT ⇒ not key-gated (every provider treated as keyed): the `available` BOOLEAN is - // unchanged from pre-change; the only new output is the additive-optional `unavailableReason` (`'not-on-key'` - // on a live-omitted static model) — informational, and the sole live-data-passing caller passes keyedProviders. - const providerKeyed = - input.keyedProviders === undefined || input.keyedProviders.has(t.provider); - let available: boolean; - let unavailableReason: 'no-key' | 'not-on-key' | undefined; - if (!providerKeyed) { - available = false; - unavailableReason = 'no-key'; - } else if (live.has(t.provider)) { - available = t.live !== undefined; - if (!available) unavailableReason = 'not-on-key'; - } else { - available = true; - } - const deprecatedAt = earlierIsoDate( - earlierIsoDate(t.registry?.deprecatedAt, t.live?.deprecatedAt), - t.user?.deprecatedAt, - ); - const parsedDeprecation = deprecatedAt === undefined ? NaN : Date.parse(deprecatedAt); - const deprecated = !Number.isNaN(parsedDeprecation) && parsedDeprecation <= input.now; - - entries.push({ - modelId, - provider: t.provider, - displayName: t.registry?.displayName ?? t.live?.displayName ?? t.user?.displayName ?? modelId, - ...(contextWindowTokens !== undefined ? { contextWindowTokens } : {}), - ...(maxOutputTokens !== undefined ? { maxOutputTokens } : {}), - ...(pricing !== undefined ? { pricing } : {}), - pricingSource, - priceKnown: pricingSource !== 'none', - available, - ...(unavailableReason !== undefined ? { unavailableReason } : {}), - deprecated, - ...(deprecatedAt !== undefined ? { deprecatedAt } : {}), - // Reasoning capability via the SAME authority as the engine gate (ADR-0066 §4): the registry flag for a known - // id (authoritative — true or false), else the conservative id heuristic for a live-discovered id. So the - // picker's effort sub-step lights up exactly for the models the engine will actually honor — including a newly - // released reasoning family member absent from the registry. - supportsReasoning: modelSupportsReasoning(modelId), - }); + entries.push(buildEntry(modelId, t, input, live)); } // Order (maintainer, 2.5.G): AVAILABLE (selectable) models FIRST, then the dimmed/unavailable ones — each group diff --git a/packages/llm/src/pricing.ts b/packages/llm/src/pricing.ts index 8baf19e4..52c0343e 100644 --- a/packages/llm/src/pricing.ts +++ b/packages/llm/src/pricing.ts @@ -260,7 +260,7 @@ export const KNOWN_MODEL_IDS: readonly CanonicalModelId[] = export function reasoningModelIdHeuristic(model: string): boolean { const m = model.toLowerCase(); if (/^o\d/.test(m)) return true; // OpenAI o-series (o1 / o3 / o4 / o5+) — the entire family reasons - if (/^gpt-5/.test(m) && !m.includes('chat')) return true; // the reasoning gpt-5 line (gpt-5-chat is non-reasoning) + if (m.startsWith('gpt-5') && !m.includes('chat')) return true; // the reasoning gpt-5 line (gpt-5-chat is non-reasoning) if (m.startsWith('claude-opus')) return true; // Claude Opus reasons (extended thinking) if (m.includes('thinking')) return true; // an explicit "thinking" model id (e.g. a Gemini thinking variant) return false; diff --git a/packages/shared/src/constants.ts b/packages/shared/src/constants.ts index 7b09c1f6..c2af875a 100644 --- a/packages/shared/src/constants.ts +++ b/packages/shared/src/constants.ts @@ -280,6 +280,19 @@ export type LlmProviderId = (typeof LLM_PROVIDERS)[number]; export const REASONING_EFFORTS = ['off', 'low', 'medium', 'high', 'max'] as const; export type ReasoningEffort = (typeof REASONING_EFFORTS)[number]; +/** + * The one-line hint for each reasoning-effort tier — display-only text (latency/cost vs depth), so a surface (the + * CLI `/models` effort sub-list, the `/effort` command) can explain each tier without the user consulting the docs. + * Owned here beside {@link REASONING_EFFORTS} so it is a neutral, surface-agnostic home (no `render/tui` dependency). + */ +export const EFFORT_TIER_HINT: Record = { + off: 'no reasoning — fastest, lowest cost', + low: 'brief reasoning', + medium: 'balanced reasoning', + high: 'deep reasoning', + max: 'maximum reasoning — slowest, highest cost', +}; + /** * The three provider **protocol kinds** (the `kind` abstraction, [ADR-0064] §2) — a closed vocabulary * that derives, **once per protocol rather than per provider**, the adapter factory, the list-models From cf8a83d84c0c5c38497b70e040d581f9450f7826 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Mon, 6 Jul 2026 23:01:46 +0300 Subject: [PATCH 62/68] =?UTF-8?q?fix(cli):=20PR=20#66=20review=20round=202?= =?UTF-8?q?=20=E2=80=94=20functional=20edge-cases=20+=20provider=20optiona?= =?UTF-8?q?l-chain?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified each new finding against current code; fixed the still-valid ones minimally with tests, skipped the rest with reasons. Functional: - models-pricing: a re-price that OMITS `--cached` no longer ZEROES a previously stored cached price — the command now passes `undefined` (not `0`) so the store omits the column and PRESERVES the existing rate; the `--json` field stays `0` when omitted (unchanged contract). Regression test added. - model-picker foldFilterKey: a single CONTROL character (Tab / ESC / NUL / DEL, `\p{Cc}`) is now ignored rather than appended as invisible filter text. Test added. - chat-ink acceptModelPick: a DIFFERENT-model pick only calls `onExit()` when `onReseat` is actually wired — otherwise the pick is dropped rather than closing the chat WITHOUT applying the switch (defensive; production always wires both). - validated-fetch cancel(): a SYNCHRONOUS throw from `iterator.return()` is now caught too (was only guarding an async rejection), so cleanup never escapes. - home-controller: the save-failure hint no longer hard-codes `~/.relavium/config.toml` (misleading under a `--config` override) — a generic "check your config file" message. - provider.ts statusColumn: the missing-outcome / `verified: null` guard is now an optional chain (`outcome?.verified == null`) — the one deliberate `== null`. Skipped with reason: openai.ts String.raw (a `String.raw` for the regex REPLACEMENT `\$&` is more error-prone than the well-understood `'\\$&'` idiom in a security-adjacent escapeRegExp — correctness over a style nit); the "parameterize N tests" findings (the repo deliberately uses explicit, individually-named cases for clarity + precise failure attribution); the "prefer specific assertion" Minors (cosmetic — the current assertions are correct with adequate failure messages). Co-Authored-By: Claude --- apps/cli/src/commands/models-pricing.test.ts | 8 ++++++++ apps/cli/src/commands/models-pricing.ts | 13 +++++++++---- apps/cli/src/commands/provider.ts | 4 +++- apps/cli/src/engine/validated-fetch.ts | 17 +++++++++++------ apps/cli/src/render/tui/chat-ink.tsx | 8 ++++++-- apps/cli/src/render/tui/home-controller.ts | 9 ++++++++- apps/cli/src/render/tui/model-picker.test.ts | 12 ++++++++++++ apps/cli/src/render/tui/model-picker.ts | 4 +++- 8 files changed, 60 insertions(+), 15 deletions(-) diff --git a/apps/cli/src/commands/models-pricing.test.ts b/apps/cli/src/commands/models-pricing.test.ts index e3d708de..fdcd46c2 100644 --- a/apps/cli/src/commands/models-pricing.test.ts +++ b/apps/cli/src/commands/models-pricing.test.ts @@ -97,6 +97,14 @@ describe('modelsPricingCommand (2.5.G S10)', () => { expect(listing?.cachedInputCostPerMtokMicrocents).toBe(3_000_000); // $0.03 × 1e8 }); + it('a re-price that OMITS --cached PRESERVES the previously-stored cached price (not zeroed)', () => { + run({ ...baseArgs, cachedInputUsdPerMtok: 0.03 }); // first: set a cached price + run({ ...baseArgs, inputUsdPerMtok: 5 }); // re-price WITHOUT --cached (change input only) + const listing = catalog.listAll().find((m) => m.modelId === 'acme-custom-1'); + expect(listing?.cachedInputCostPerMtokMicrocents).toBe(3_000_000); // preserved — NOT overwritten with 0 + expect(listing?.inputCostPerMtokMicrocents).toBe(500_000_000); // the input re-price DID take ($5 × 1e8) + }); + it('--json emits one key-free record with the stored micro-cents', () => { const { code, out } = run(baseArgs, true); expect(code).toBe(EXIT_CODES.success); diff --git a/apps/cli/src/commands/models-pricing.ts b/apps/cli/src/commands/models-pricing.ts index 016ff18c..3f29033a 100644 --- a/apps/cli/src/commands/models-pricing.ts +++ b/apps/cli/src/commands/models-pricing.ts @@ -115,22 +115,25 @@ export function modelsPricingCommand( // Convert + bounds-validate BEFORE the write (a bad `--cached` must not leave a partially-applied row). const inputCostPerMtokMicrocents = usdToMicrocents(args.inputUsdPerMtok, '--input'); const outputCostPerMtokMicrocents = usdToMicrocents(args.outputUsdPerMtok, '--output'); + // OMITTED `--cached` ⇒ `undefined` (not `0`): so the upsert can OMIT the column and the store PRESERVES an + // existing cached price on a re-price, rather than the `??`-passing `0` silently zeroing a hand-entered rate. const cachedInputCostPerMtokMicrocents = args.cachedInputUsdPerMtok === undefined - ? 0 + ? undefined : usdToMicrocents(args.cachedInputUsdPerMtok, '--cached'); // A pricing-ONLY upsert: omit display name + limits (and every media/capability column) so the store PRESERVES // whatever an existing row carries — including a soft-deactivated live row the active-only reader cannot see, so // a re-price never zeroes a discovered name/context. A brand-new user-priced model defaults display → the id and - // limits → the `0` "unknown" sentinel (in the store), so no read is needed here. + // limits → the `0` "unknown" sentinel (in the store), so no read is needed here. `cachedInput…` is omitted when + // `--cached` was not passed (so the store preserves the existing cached rate — see the local above). deps.catalog.upsert({ providerId: providerRow.id, modelId: args.model, source: 'user', inputCostPerMtokMicrocents, outputCostPerMtokMicrocents, - cachedInputCostPerMtokMicrocents, + ...(cachedInputCostPerMtokMicrocents === undefined ? {} : { cachedInputCostPerMtokMicrocents }), }); if (deps.global.json) { @@ -142,7 +145,9 @@ export function modelsPricingCommand( source: 'user', inputCostPerMtokMicrocents, outputCostPerMtokMicrocents, - cachedInputCostPerMtokMicrocents, + // The `--json` field stays present as `0` when `--cached` was omitted (unchanged contract) even though the + // store now PRESERVES the existing cached rate rather than writing this `0` (see the upsert above). + cachedInputCostPerMtokMicrocents: cachedInputCostPerMtokMicrocents ?? 0, }, ]); return EXIT_CODES.success; diff --git a/apps/cli/src/commands/provider.ts b/apps/cli/src/commands/provider.ts index ab8ca07b..273624e9 100644 --- a/apps/cli/src/commands/provider.ts +++ b/apps/cli/src/commands/provider.ts @@ -213,7 +213,9 @@ function statusColumn( outcome: VerifyOutcome | undefined, ): string { if (!verify) return record.apiKeyKeychainRef !== undefined ? 'key set' : 'no key'; - if (outcome === undefined || outcome.verified === null) return 'no key'; + // `== null` (the one deliberate loose check) folds a missing outcome AND a `verified: null` (no key / not + // verifiable) into the single "no key" column — via an optional chain, so `outcome` need not be re-tested. + if (outcome?.verified == null) return 'no key'; // `outcome.detail` is already cleaned + bounded at the source (verifyProvider → cleanDetail). return outcome.verified ? 'verified' : `failed — ${outcome.detail ?? 'verification failed'}`; } diff --git a/apps/cli/src/engine/validated-fetch.ts b/apps/cli/src/engine/validated-fetch.ts index 1bf55aea..f684b3fa 100644 --- a/apps/cli/src/engine/validated-fetch.ts +++ b/apps/cli/src/engine/validated-fetch.ts @@ -201,12 +201,17 @@ function hopBodyToStream(hop: HopResponse): ReadableStream { }, cancel() { dispose(); - // Best-effort iterator cleanup — a `return()` rejection must never escape `cancel()` (and must not float). - const returned = iterator.return?.(undefined); - if (returned !== undefined) { - returned.catch(() => { - // ignore — cancel is best-effort and must not throw - }); + // Best-effort iterator cleanup — a `return()` fault must never escape `cancel()` nor leave a floating promise: + // catch a SYNCHRONOUS throw, and attach a no-op `.catch` to an async rejection. + try { + const returned = iterator.return?.(undefined); + if (returned !== undefined) { + returned.catch(() => { + // ignore — cancel is best-effort and must not throw + }); + } + } catch { + // ignore — a synchronous return() throw is best-effort cleanup, never propagated } }, }); diff --git a/apps/cli/src/render/tui/chat-ink.tsx b/apps/cli/src/render/tui/chat-ink.tsx index 5249621b..65eec497 100644 --- a/apps/cli/src/render/tui/chat-ink.tsx +++ b/apps/cli/src/render/tui/chat-ink.tsx @@ -630,12 +630,16 @@ export function ChatApp(props: Readonly): ReactElement { } return; } - props.onReseat?.({ + // DIFFERENT model: a live reseat — only when `onReseat` is actually wired. Without it (a driver/test with the + // picker but no reseat support), do NOT call `onExit()` — that would close the chat WITHOUT applying the switch; + // the pick is simply dropped (defensive — production always wires `onReseat` alongside the picker). + if (props.onReseat === undefined) return; + props.onReseat({ modelId: step.modelId, provider: step.provider, ...(step.reasoningEffort === undefined ? {} : { reasoningEffort: step.reasoningEffort }), }); - props.onExit(); + props.onExit(); // the reseat set the stop state; end the loop so runReplLoop swaps in the new-model session }; // The open picker owns every key (mirrors routeMentionKey). Route the fold's step; the accept/blocked cases diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index 4e64b5d9..fcffd6a7 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -678,9 +678,16 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { try { port.writeDefault(modelId); } catch { + // A generic save-failure hint — the actual write target may be a `--config` override, not the canonical + // `~/.relavium/config.toml`, so don't name a path the user may not be using. const open = state.modelPicker; if (open !== undefined) { - set({ modelPicker: { ...open, hint: 'could not save — check ~/.relavium/config.toml' } }); + set({ + modelPicker: { + ...open, + hint: 'could not save the default model — check your config file.', + }, + }); } return; } diff --git a/apps/cli/src/render/tui/model-picker.test.ts b/apps/cli/src/render/tui/model-picker.test.ts index bb974b47..5ae618e2 100644 --- a/apps/cli/src/render/tui/model-picker.test.ts +++ b/apps/cli/src/render/tui/model-picker.test.ts @@ -153,6 +153,18 @@ describe('foldModelPickerKey', () => { const blob = foldModelPickerKey('pasted text', {}, state()); expect(blob).toEqual({ kind: 'state', state: state() }); // unchanged }); + + it('ignores a single CONTROL character (Tab / ESC / NUL / DEL) \u2014 never inserted as invisible filter text', () => { + for (const control of ['\t', '\x1b', '\x00', '\x7f']) { + const s = state({ filter: 'ab' }); + expect(foldModelPickerKey(control, {}, s)).toEqual({ kind: 'state', state: s }); // filter unchanged + } + // A regular printable char still extends it (the guard rejects only control chars). + expect(foldModelPickerKey('x', {}, state({ filter: 'ab' }))).toEqual({ + kind: 'state', + state: state({ filter: 'abx', selected: 0 }), + }); + }); }); describe('foldModelPickerKey — the ADR-0066 effort sub-step', () => { diff --git a/apps/cli/src/render/tui/model-picker.ts b/apps/cli/src/render/tui/model-picker.ts index 21338f01..1dcd6be2 100644 --- a/apps/cli/src/render/tui/model-picker.ts +++ b/apps/cli/src/render/tui/model-picker.ts @@ -232,7 +232,9 @@ function foldFilterKey( state: { ...state, filter: dropLastCodePoint(state.filter), selected: 0 }, }; } - if ([...char].length === 1 && key.ctrl !== true && key.meta !== true) { + // A single PRINTABLE code point extends the filter — a multi-char paste blob is dropped (matching the other + // submodes), and a control character (`\p{Cc}` — e.g. Tab) is ignored rather than inserted as invisible filter text. + if ([...char].length === 1 && key.ctrl !== true && key.meta !== true && !/\p{Cc}/u.test(char)) { return { kind: 'state', state: { ...state, filter: state.filter + char, selected: 0 } }; } return { kind: 'state', state }; From b6fde2fcb110d13d4522a98b4a618eba528bdf65 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 7 Jul 2026 08:31:28 +0300 Subject: [PATCH 63/68] =?UTF-8?q?test,style:=20PR=20#66=20review=20round?= =?UTF-8?q?=203=20=E2=80=94=20clear=20the=20remaining=20Sonar=20findings?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last recurring Sonar findings, now all cleared (a clean gate): Test parameterization (Major) — converted each group of near-identical tests to one `it.each` with descriptive `$label` per-case names, so clarity + failure attribution are preserved (not merged into an anonymous loop): - agent-source: the 3 idSuffix-discovery tests (.agent.yaml / .relavium.yaml / .yaml). - chat: the 3 read-only/rejected-slash-command "reported on stderr, session continues" tests (/bogus, /cost, /exit now). - gate: the 6 damaged-store → clean exit-2 tests (only the corrupting SQL differs). Specific assertions (Minor): `.toHaveLength()` for the model-catalog / model-catalog-view / wizard length checks; `expect(typeof x).toBe('function')` for the providers listModels check (a `.toBeDefined()` on the method reference would trip `@typescript-eslint/unbound-method`, so `typeof` keeps it specific AND lint-clean). openai.ts (Minor): `String.raw`\$&`` for the escapeRegExp replacement (drops the doubled backslash of `'\\$&'`; byte-identical value). Co-Authored-By: Claude --- apps/cli/src/chat/agent-source.test.ts | 36 ++----- apps/cli/src/commands/chat.test.ts | 46 ++++----- apps/cli/src/commands/gate.test.ts | 98 ++++++++----------- .../cli/src/engine/model-catalog-view.test.ts | 2 +- apps/cli/src/engine/providers.test.ts | 3 +- apps/cli/src/onboarding/wizard.test.ts | 2 +- packages/llm/src/adapters/openai.ts | 3 +- packages/llm/src/model-catalog.test.ts | 2 +- 8 files changed, 81 insertions(+), 111 deletions(-) diff --git a/apps/cli/src/chat/agent-source.test.ts b/apps/cli/src/chat/agent-source.test.ts index 2ea828ac..d3972d7b 100644 --- a/apps/cli/src/chat/agent-source.test.ts +++ b/apps/cli/src/chat/agent-source.test.ts @@ -81,36 +81,16 @@ describe('resolveChatAgent', () => { expect(agent.provider).toBe('anthropic'); }); - it('discovers a bare --agent id under /agents/ (.agent.yaml suffix)', () => { + // A bare `--agent id` is discovered under `/agents/` by trying the idSuffix chain in order: + // `.agent.yaml` → `.relavium.yaml` → `.yaml`. One case per suffix, each proving that suffix ALONE resolves. + it.each([ + { label: '.agent.yaml', filename: 'coder.agent.yaml' }, + { label: 'the .relavium.yaml fallback (no .agent.yaml)', filename: 'coder.relavium.yaml' }, + { label: 'the bare .yaml fallback (last in the chain)', filename: 'coder.yaml' }, + ])('discovers a bare --agent id under agents/ via the $label suffix', ({ filename }) => { const projectConfigDir = join(dir, '.relavium'); mkdirSync(join(projectConfigDir, 'agents'), { recursive: true }); - writeFileSync(join(projectConfigDir, 'agents', 'coder.agent.yaml'), AGENT_YAML); - const agent = resolveChatAgent('coder', { - cwd: dir, - projectConfigDir, - defaultModel: undefined, - }); - expect(agent.id).toBe('coder'); - }); - - it('discovers a bare --agent id via the fallback suffixes (.relavium.yaml when no .agent.yaml)', () => { - const projectConfigDir = join(dir, '.relavium'); - mkdirSync(join(projectConfigDir, 'agents'), { recursive: true }); - // Only the .relavium.yaml exists — the resolver must try it after .agent.yaml misses. - writeFileSync(join(projectConfigDir, 'agents', 'coder.relavium.yaml'), AGENT_YAML); - const agent = resolveChatAgent('coder', { - cwd: dir, - projectConfigDir, - defaultModel: undefined, - }); - expect(agent.id).toBe('coder'); - }); - - it('discovers a bare --agent id via the bare .yaml suffix (last in the fallback chain)', () => { - const projectConfigDir = join(dir, '.relavium'); - mkdirSync(join(projectConfigDir, 'agents'), { recursive: true }); - // Neither .agent.yaml nor .relavium.yaml exists — only coder.yaml, the third/last idSuffix. - writeFileSync(join(projectConfigDir, 'agents', 'coder.yaml'), AGENT_YAML); + writeFileSync(join(projectConfigDir, 'agents', filename), AGENT_YAML); const agent = resolveChatAgent('coder', { cwd: dir, projectConfigDir, diff --git a/apps/cli/src/commands/chat.test.ts b/apps/cli/src/commands/chat.test.ts index 66cd796d..1fc3c942 100644 --- a/apps/cli/src/commands/chat.test.ts +++ b/apps/cli/src/commands/chat.test.ts @@ -243,21 +243,29 @@ describe('chatCommand', () => { expect(store.loadFull(sessionId)?.session.status).toBe('ended'); }); - it('reports an unknown slash command on stderr without ending the session', async () => { - const { d, err, store, sessionId } = deps(['/bogus', 'hello', '/exit'], [textTurn('hi')]); - await chatCommand({ agent: undefined }, d); - expect(err()).toContain("unknown command '/bogus'"); - // the session continued after the bad command — the 'hello' turn persisted. - expect(store.loadFull(sessionId)?.messages).toHaveLength(2); - }); - - it('/cost reports the session spend (no cost ⇒ $0.0000) on the non-TTY path, without ending the session', async () => { - const { d, err, store, sessionId } = deps(['/cost', 'hello', '/exit'], [textTurn('hi')]); - await chatCommand({ agent: undefined }, d); - expect(err()).toContain('Session cost: $0.0000'); - // /cost is read-only: the session continued — the 'hello' turn persisted (user + assistant = 2). - expect(store.loadFull(sessionId)?.messages).toHaveLength(2); - }); + // A read-only or rejected slash command is reported on stderr and the session CONTINUES (the following user turn + // still persists → 2 messages). One case per command; only that the session continued is asserted, not the content. + it.each([ + { label: 'an unknown command', command: '/bogus', expectedErr: "unknown command '/bogus'" }, + { + label: '/cost (read-only, $0 spend)', + command: '/cost', + expectedErr: 'Session cost: $0.0000', + }, + { + label: 'an undeclared slash argument', + command: '/exit now', + expectedErr: "/exit: unknown argument 'now'", + }, + ])( + '$label is reported on stderr without ending the session', + async ({ command, expectedErr }) => { + const { d, err, store, sessionId } = deps([command, 'hello', '/exit'], [textTurn('hi')]); + await chatCommand({ agent: undefined }, d); + expect(err()).toContain(expectedErr); + expect(store.loadFull(sessionId)?.messages).toHaveLength(2); // read-only: the session continued + }, + ); // A fake fast-tier-passing probe set; `--deep` adds one ok provider + one warn MCP check (deterministic, no I/O). const fakeDoctorProbes: DoctorProbes = { @@ -296,14 +304,6 @@ describe('chatCommand', () => { expect(out).toContain('⚠ MCP servers: none configured'); // the deep MCP probe ran }); - it('rejects an undeclared slash argument (a zero-arg command takes no tokens)', async () => { - const { d, err, store, sessionId } = deps(['/exit now', 'hi', '/exit'], [textTurn('hi')]); - await chatCommand({ agent: undefined }, d); - expect(err()).toContain("/exit: unknown argument 'now'"); - // '/exit now' was rejected (not run as exit), so the session continued and the 'hi' turn persisted. - expect(store.loadFull(sessionId)?.messages).toHaveLength(2); - }); - it('rejects an unknown flag on an arg-taking command (/doctor --bogus)', async () => { const { d, err } = deps(['/doctor --bogus', '/exit'], [textTurn('hi')], { doctorProbes: fakeDoctorProbes, diff --git a/apps/cli/src/commands/gate.test.ts b/apps/cli/src/commands/gate.test.ts index f989f9b7..5c1af198 100644 --- a/apps/cli/src/commands/gate.test.ts +++ b/apps/cli/src/commands/gate.test.ts @@ -429,62 +429,50 @@ workflow: expect(code).toBe(EXIT_CODES.success); // the resume completed; the GC rejection was swallowed at the call site }); - it('surfaces a corrupt stored inputs blob as a clean exit-2 fault (no silent empty-inputs resume)', async () => { + // Every DAMAGED-store shape resumes to the SAME clean exit-2 (`invalid_invocation`) — never a silent + // empty-inputs resume nor a raw escaping error. One case per corruption; only the corrupting statement differs. + it.each([ + { + label: 'a corrupt stored inputs blob (non-JSON)', + corrupt: (runId: string) => + client.sqlite + .prepare('UPDATE runs SET input_json = ? WHERE id = ?') + .run('{not json', runId), + }, + { + label: 'a corrupt persisted event log (non-JSON payload)', + corrupt: (runId: string) => + client.sqlite + .prepare('UPDATE run_events SET payload_json = ? WHERE run_id = ? AND seq = 0') + .run('{not json', runId), + }, + { + label: 'a valid-JSON-but-non-object inputs blob (array)', + corrupt: (runId: string) => + client.sqlite.prepare('UPDATE runs SET input_json = ? WHERE id = ?').run('[]', runId), + }, + { + label: 'a corrupt workflow snapshot (bad JSON)', + corrupt: (runId: string) => + client.sqlite + .prepare('UPDATE runs SET workflow_definition_snapshot = ? WHERE id = ?') + .run('{not json', runId), + }, + { + label: 'a schema-invalid workflow snapshot', + corrupt: (runId: string) => + client.sqlite + .prepare('UPDATE runs SET workflow_definition_snapshot = ? WHERE id = ?') + .run('{"workflow":{}}', runId), + }, + { + label: 'a snapshot but no event log (no resumable state)', + corrupt: (runId: string) => + client.sqlite.prepare('DELETE FROM run_events WHERE run_id = ?').run(runId), + }, + ])('surfaces $label as a clean exit-2 fault', async ({ corrupt }) => { const { runId } = await setupPausedRun(); - // Corrupt the persisted input_json to a non-JSON blob (simulating a damaged store row). - client.sqlite.prepare('UPDATE runs SET input_json = ? WHERE id = ?').run('{not json', runId); - const { io } = captureIo(); - await expect(gateCommand({ runId, approve: true }, deps(io))).rejects.toMatchObject({ - code: 'invalid_invocation', - }); - }); - - it('surfaces a corrupt persisted event log as a clean exit-2 fault (no raw escaping error)', async () => { - const { runId } = await setupPausedRun(); - // Corrupt a run_events payload so the checkpoint reconstruction (loadRunEvents → JSON.parse) throws. - client.sqlite - .prepare('UPDATE run_events SET payload_json = ? WHERE run_id = ? AND seq = 0') - .run('{not json', runId); - const { io } = captureIo(); - await expect(gateCommand({ runId, approve: true }, deps(io))).rejects.toMatchObject({ - code: 'invalid_invocation', - }); - }); - - it('surfaces a valid-JSON-but-non-object inputs blob (array) as exit-2', async () => { - const { runId } = await setupPausedRun(); - client.sqlite.prepare('UPDATE runs SET input_json = ? WHERE id = ?').run('[]', runId); - const { io } = captureIo(); - await expect(gateCommand({ runId, approve: true }, deps(io))).rejects.toMatchObject({ - code: 'invalid_invocation', - }); - }); - - it('surfaces a corrupt workflow snapshot (bad JSON) as exit-2', async () => { - const { runId } = await setupPausedRun(); - client.sqlite - .prepare('UPDATE runs SET workflow_definition_snapshot = ? WHERE id = ?') - .run('{not json', runId); - const { io } = captureIo(); - await expect(gateCommand({ runId, approve: true }, deps(io))).rejects.toMatchObject({ - code: 'invalid_invocation', - }); - }); - - it('surfaces a schema-invalid workflow snapshot as exit-2', async () => { - const { runId } = await setupPausedRun(); - client.sqlite - .prepare('UPDATE runs SET workflow_definition_snapshot = ? WHERE id = ?') - .run('{"workflow":{}}', runId); - const { io } = captureIo(); - await expect(gateCommand({ runId, approve: true }, deps(io))).rejects.toMatchObject({ - code: 'invalid_invocation', - }); - }); - - it('surfaces a run with a snapshot but no event log as exit-2 (no resumable state)', async () => { - const { runId } = await setupPausedRun(); - client.sqlite.prepare('DELETE FROM run_events WHERE run_id = ?').run(runId); + corrupt(runId); const { io } = captureIo(); await expect(gateCommand({ runId, approve: true }, deps(io))).rejects.toMatchObject({ code: 'invalid_invocation', diff --git a/apps/cli/src/engine/model-catalog-view.test.ts b/apps/cli/src/engine/model-catalog-view.test.ts index da6806e7..55c5e91e 100644 --- a/apps/cli/src/engine/model-catalog-view.test.ts +++ b/apps/cli/src/engine/model-catalog-view.test.ts @@ -40,7 +40,7 @@ function slugResolver(map: Record): (uuid: string) => string { describe('buildMergedCatalog', () => { it('seeds the full static registry even with no live rows (never an empty picker)', () => { const view = buildMergedCatalog({ rows: [], providerSlug: slugResolver({}), now: 0 }); - expect(view.entries.length).toBe(Object.keys(MODEL_PRICING).length); + expect(view.entries).toHaveLength(Object.keys(MODEL_PRICING).length); expect(view.refreshedAt).toBeUndefined(); // With NO live data for any provider, every static model falls back to static presence (ADR-0064 §6). expect(view.entries.every((e) => e.available)).toBe(true); diff --git a/apps/cli/src/engine/providers.test.ts b/apps/cli/src/engine/providers.test.ts index f8e9f4c6..51f374cd 100644 --- a/apps/cli/src/engine/providers.test.ts +++ b/apps/cli/src/engine/providers.test.ts @@ -331,7 +331,8 @@ describe('createProviderResolver custom endpoints (2.5.G S9 / ADR-0065 §3–4)' validatedFetch: fetch, }); const openai = resolver.resolveProvider('openai'); - expect(openai?.listModels !== undefined).toBe(true); // the openai-compatible adapter carries the live-list capability + // `typeof` (not a bare `expect(openai?.listModels)`) keeps the specific assertion without an unbound-method ref. + expect(typeof openai?.listModels).toBe('function'); // the openai-compatible adapter carries the live-list capability // The listModels egress hits the CUSTOM endpoint (the dead-base_url bug is fixed) — via the validated fetch. await openai?.listModels?.('sk-test')?.catch(() => undefined); // the ROUTING is the assertion, not the parse result expect(urls.some((url) => url.startsWith('https://my-proxy.example/v1'))).toBe(true); diff --git a/apps/cli/src/onboarding/wizard.test.ts b/apps/cli/src/onboarding/wizard.test.ts index a22fa830..921242a5 100644 --- a/apps/cli/src/onboarding/wizard.test.ts +++ b/apps/cli/src/onboarding/wizard.test.ts @@ -493,6 +493,6 @@ describe('runOnboardingWizard', () => { validate: () => Promise.reject(new Error('probe boom')), }), ).rejects.toThrow('probe boom'); // propagates (not swallowed) to the Home's cleanup - expect(stops.length).toBe(1); // the finally stopped the spinner despite the rejection + expect(stops).toHaveLength(1); // the finally stopped the spinner despite the rejection }); }); diff --git a/packages/llm/src/adapters/openai.ts b/packages/llm/src/adapters/openai.ts index 2716b90d..ddb94b6d 100644 --- a/packages/llm/src/adapters/openai.ts +++ b/packages/llm/src/adapters/openai.ts @@ -433,7 +433,8 @@ const OPENAI_DENY_SUBSTRINGS = [ * (`dall-e`'s `-` is literal outside a character class), but this keeps the boundary match safe if one is * ever added. */ function escapeRegExp(text: string): string { - return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); + // `String.raw` avoids the doubled backslash of `'\\$&'` — the replacement is a literal `\` + the `$&` match ref. + return text.replace(/[.*+?^${}()|[\]\\]/g, String.raw`\$&`); } /** True when a deny `token` occurs on a `-`/`_` segment boundary in `lower` — a word-boundary match, so diff --git a/packages/llm/src/model-catalog.test.ts b/packages/llm/src/model-catalog.test.ts index 4b1a5129..38bfb277 100644 --- a/packages/llm/src/model-catalog.test.ts +++ b/packages/llm/src/model-catalog.test.ts @@ -30,7 +30,7 @@ const userPricing = (provider: ProviderId): ModelPricing => ({ describe('mergeModelCatalog (ADR-0064 §6)', () => { it('with no live/user data, surfaces every static model as registry-priced and available (static presence)', () => { const entries = mergeModelCatalog({ now: BEFORE_DEEPSEEK_DEPRECATION }); - expect(entries.length).toBe(Object.keys(MODEL_PRICING).length); + expect(entries).toHaveLength(Object.keys(MODEL_PRICING).length); const opus = byId(entries, 'claude-opus-4-8'); expect(opus).toMatchObject({ provider: 'anthropic', From 5c940771e7ab0f8504fdd0ca96ee983df235a30a Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 7 Jul 2026 09:08:07 +0300 Subject: [PATCH 64/68] =?UTF-8?q?fix(cli),docs:=20PR=20#66=20review=20roun?= =?UTF-8?q?d=204=20=E2=80=94=20skipped-unsupported=20models=20message=20+?= =?UTF-8?q?=20MD028=20+=20dedup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Verified each finding against current code; fixed the still-valid ones, skipped the false-positives with reasons. Functional: - models list (first-run, empty catalog): a CONNECTED provider that is `skipped-unsupported` (a key IS set, but the adapter has no `listModels`) now gets a status-aware "does not support listing models" line instead of the misleading "add a provider key" default — same intent as the existing `failed` branch (`skipped-no-key` still falls through to the add-a-key guidance). Test added. Docs (MD028): - ADR-0059: a genuine bare blank line between the two amendment blockquotes → the standard `>` empty-line separator (matches the fix already in ADR-0011/0024). Tests (dedup nitpick): provider.test.ts — hoisted the thrice-duplicated `jsonGlobal` into one shared describe-level constant. Skipped with reason: - gate.test.ts "duplicate dbDeps" — FALSE POSITIVE: the three `dbDeps` are in separate `it`-block scopes (typecheck is clean; no redeclaration). - Drizzle 0007/0008 SQL + snapshots "out of sync" — FALSE POSITIVE: `pnpm --filter @relavium/db db:generate` reports "No schema changes, nothing to migrate"; the artifacts ARE in sync and git is clean after generate. - ADR-0011/0024 MD028 re-flag — already fixed with the documented markdownlint fix (a `>` empty line "combines the blockquote"); markdownlint is not a CI gate here. - persister.test.ts seedCatalogModel dedup (nitpick) — consolidating would hoist a counter-based local helper into a shared test-support module and reconcile the two call sites' differing uuid/field needs; churn beyond "minimal" for two small, self-documenting inline seeds. Co-Authored-By: Claude --- apps/cli/src/commands/models.test.ts | 19 ++++++++++++ apps/cli/src/commands/models.ts | 16 ++++++++-- apps/cli/src/commands/provider.test.ts | 31 ++++++------------- .../0059-cli-mid-session-model-reseat.md | 2 +- 4 files changed, 42 insertions(+), 26 deletions(-) diff --git a/apps/cli/src/commands/models.test.ts b/apps/cli/src/commands/models.test.ts index bd7f435a..b8c2c796 100644 --- a/apps/cli/src/commands/models.test.ts +++ b/apps/cli/src/commands/models.test.ts @@ -299,4 +299,23 @@ describe('modelsCommand — refresh', () => { const code = await modelsCommand({ refresh: true }, deps(io, stubCatalog(rowsRef), refresh)); expect(code).toBe(EXIT_CODES.success); }); + + it('reports a SKIPPED-UNSUPPORTED first-run refresh (not "add a key") when the catalog is still empty (FIX 6)', async () => { + // On the LIST path (first-run auto-refresh), a CONNECTED provider with no `listModels` leaves the catalog empty + // for a reason that is NOT "no key" — so the plain add-a-key guidance would mislead (a key IS set). + const { io, out } = captureIo(); + const rowsRef: { value: ModelCatalogListing[] } = { value: [] }; + const refresh = stubRefresh({ + providers: [ + { provider: 'anthropic', status: 'skipped-unsupported' }, + { provider: 'openai', status: 'skipped-no-key' }, + ], + }); + + const code = await modelsCommand({ refresh: false }, deps(io, stubCatalog(rowsRef), refresh)); + expect(code).toBe(EXIT_CODES.success); + const text = out(); + expect(text).toContain('does not support listing models'); // the status-aware line for anthropic + expect(text).not.toContain('No models cached'); // NOT the misleading add-a-key default + }); }); diff --git a/apps/cli/src/commands/models.ts b/apps/cli/src/commands/models.ts index 05a4acab..32e961e5 100644 --- a/apps/cli/src/commands/models.ts +++ b/apps/cli/src/commands/models.ts @@ -102,9 +102,11 @@ async function runList(deps: ModelsCommandDeps): Promise { ); return EXIT_CODES.success; } - // Human mode only (FIX 4): when the catalog is STILL empty after a first-run refresh because a CONNECTED - // provider's refresh FAILED (not merely because no key was set), print a failure-aware line — the plain - // "add a key" guidance would be misleading (a key IS set; the fetch failed). + // Human mode only (FIX 4/6): when the catalog is STILL empty after a first-run refresh because a CONNECTED + // provider either FAILED its refresh OR does not support listing (`skipped-unsupported`) — NOT merely because no + // key was set — print a status-aware line. The plain `renderModelList` "add a provider key" guidance would + // mislead in both cases (a key IS set). `skipped-no-key` is genuinely keyless, so it falls through to that + // default guidance as before. if (listings.length === 0 && firstRunReport !== undefined) { const failed = firstRunReport.providers.filter((p) => p.status === 'failed'); if (failed.length > 0) { @@ -114,6 +116,14 @@ async function runList(deps: ModelsCommandDeps): Promise { ); return EXIT_CODES.success; } + const unsupported = firstRunReport.providers.filter((p) => p.status === 'skipped-unsupported'); + if (unsupported.length > 0) { + const names = unsupported.map((p) => p.provider).join(', '); + deps.io.writeOut( + `No models to list for ${names} — the provider does not support listing models (a key is set, so this is not a missing-key issue).\n`, + ); + return EXIT_CODES.success; + } } renderModelList(deps.io, listings, deps.providerSlug); return EXIT_CODES.success; diff --git a/apps/cli/src/commands/provider.test.ts b/apps/cli/src/commands/provider.test.ts index 6bd603b7..8c4f619a 100644 --- a/apps/cli/src/commands/provider.test.ts +++ b/apps/cli/src/commands/provider.test.ts @@ -54,6 +54,15 @@ describe('relavium provider commands (2.C)', () => { let deps: (over: Partial) => ProviderCommandDeps; let io: ReturnType; + // The `--json` global options every `provider list --json` test reuses (a stateless constant). + const jsonGlobal = { + json: true, + color: false, + cwd: process.cwd(), + configPath: undefined, + verbosity: 'normal' as const, + }; + beforeEach(() => { client = createClient(':memory:'); runMigrations(client.db); @@ -150,13 +159,6 @@ describe('relavium provider commands (2.C)', () => { }); it('list --json emits one key-free NDJSON record per provider with the verify state (2.5.G S11)', async () => { - const jsonGlobal = { - json: true, - color: false, - cwd: process.cwd(), - configPath: undefined, - verbosity: 'normal' as const, - }; // Register two providers WITH keys via a non-json setup (their confirmations don't pollute the json capture). const setup = deps({}); await runProviderCommand({ action: 'set-key', name: 'anthropic' }, setup); @@ -174,13 +176,6 @@ describe('relavium provider commands (2.C)', () => { }); it('list --json without --verify leaves verified/verifyDetail null (no probe)', async () => { - const jsonGlobal = { - json: true, - color: false, - cwd: process.cwd(), - configPath: undefined, - verbosity: 'normal' as const, - }; await runProviderCommand({ action: 'add', name: 'openai' }, deps({})); const listIo = captureIo(); await runProviderCommand({ action: 'list' }, deps({ io: listIo.io, global: jsonGlobal })); @@ -193,14 +188,6 @@ describe('relavium provider commands (2.C)', () => { }); }); - const jsonGlobal = { - json: true, - color: false, - cwd: process.cwd(), - configPath: undefined, - verbosity: 'normal' as const, - }; - it('list --json --verify records a probe FAILURE as { verified: false, verifyDetail: } (no key)', async () => { await runProviderCommand({ action: 'add', name: 'openai' }, deps({})); const listIo = captureIo(); diff --git a/docs/decisions/0059-cli-mid-session-model-reseat.md b/docs/decisions/0059-cli-mid-session-model-reseat.md index 2dc88b2a..e34058cf 100644 --- a/docs/decisions/0059-cli-mid-session-model-reseat.md +++ b/docs/decisions/0059-cli-mid-session-model-reseat.md @@ -8,7 +8,7 @@ > (the in-chat `/models` reseat requested with the Phase-2.5 CLI-consolidation model work), pulling the ADR forward > from 2.6.C. The reseat reuses the `/clear` host-swap machinery ([ADR-0062](0062-context-compaction-and-cli-history-commands.md) §7) > and the `chat-resume` transcript path — zero engine change — exactly as designed below. - +> > **Note (2026-07-06): the FK reality behind the per-message `modelId`.** The Decision below says "only the CLI > persister wiring is missing"; on implementation that proved under-specified. `session_messages.model_id` / > `agent_sessions.model_id` are **foreign keys to `model_catalog.id` (a UUID row PK), not the raw model string** — and From 2ab41854e74dc6477a60e73dbf9a64163606a93f Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 7 Jul 2026 09:55:56 +0300 Subject: [PATCH 65/68] feat(llm): DeepSeek reasoning-effort via the thinking param (ADR-0066) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DeepSeek was the one deferred provider in the reasoning-effort work — the OpenAI-compatible adapter had no pinned SDK to verify the control against. Its create-chat-completion API is now doc-verified (api-docs.deepseek.com): v4 takes a `thinking` object — `type: 'enabled'|'disabled'` plus a graded `reasoning_effort: 'high'|'max'` — richer than the ADR's assumed on/off. - adapter: map the normalized tier onto DeepSeek's native `thinking` object (off→disabled, low/medium/high→enabled+high, max→enabled+max) via a typed OpenAiCompatibleBody extension; no vendor type crosses the seam. - pricing: `reasoning: true` for deepseek-v4-flash/-pro; the §4 id heuristic covers deepseek-v[4-9]; the legacy non-thinking deepseek-chat stays off. - tests: adapter thinking-object mapping + registry/heuristic reasoning flags. - ADR-0066: append-only note refining §2/§4 to the doc-verified shape. The reasoning OUTPUT (reasoning_content) + token accounting were already mapped, so the effort-in → reasoning-out loop is now end-to-end for DeepSeek. Refs: ADR-0066 Co-Authored-By: Claude --- ...066-normalized-reasoning-effort-control.md | 2 + packages/llm/src/adapters/openai.test.ts | 52 ++++++++++++++++--- packages/llm/src/adapters/openai.ts | 47 +++++++++++++---- packages/llm/src/model-catalog.test.ts | 7 +-- packages/llm/src/pricing.test.ts | 25 ++++++--- packages/llm/src/pricing.ts | 20 ++++--- 6 files changed, 117 insertions(+), 36 deletions(-) diff --git a/docs/decisions/0066-normalized-reasoning-effort-control.md b/docs/decisions/0066-normalized-reasoning-effort-control.md index 193a08d9..9ffeeca3 100644 --- a/docs/decisions/0066-normalized-reasoning-effort-control.md +++ b/docs/decisions/0066-normalized-reasoning-effort-control.md @@ -5,6 +5,8 @@ - **Related**: [ADR-0011](0011-internal-llm-abstraction.md) (the `LLMProvider` seam this amends — a new `LlmRequest` field), [ADR-0030](0030-llm-seam-shape-amendment-reasoning-response-format-provider-executed.md) (added the reasoning **output** channel; this ADR adds the reasoning **input** control), [ADR-0039](0039-same-provider-reasoning-replay.md) (same-provider signed-reasoning replay — the output whose *production* this input governs), [ADR-0064](0064-live-model-catalog.md) (the `model_catalog.capabilities` JSON this populates per model), [ADR-0059](0059-cli-mid-session-model-reseat.md) (the `/models` picker that surfaces the effort selector), [llm-provider-seam.md](../reference/shared-core/llm-provider-seam.md) (the seam's one canonical home), [agent-yaml-spec.md](../reference/contracts/agent-yaml-spec.md) + [config-spec.md](../reference/contracts/config-spec.md) (the authoring surfaces) > Accepted 2026-07-06 (the 2.5.G "reasoning-effort" workstream, maintainer question #1). Two independent ADR reviews caught that the original draft's "OpenAI = tier, Anthropic/Gemini = token budget" split was **stale**: all four currently-targeted providers are **tier-native**, and the shipped adapters already merge `{...providerOptions, ...body}` (canonical wins). The Decision below is rewritten around that — a strictly *simpler* design (one tier → each provider's native tier), with the token-budget derivation demoted to a documented legacy fallback. +> +> **Note (2026-07-07): DeepSeek wired + its native shape refined (§2/§4).** DeepSeek was initially DEFERRED (the OpenAI-compatible adapter had no pinned SDK to verify the param against). Its create-chat-completion API was then verified against the official docs (api-docs.deepseek.com): v4 takes a **`thinking` object** — `type: 'enabled' | 'disabled'` + `reasoning_effort: 'high' | 'max'` — richer than §2's assumed "on/off only". The adapter now maps `off` → `{ type: 'disabled' }`, `low`/`medium`/`high` → `{ type: 'enabled', reasoning_effort: 'high' }` (DeepSeek's minimum thinking level), and `max` → `{ type: 'enabled', reasoning_effort: 'max' }` (an honest coarsening onto v4's two graded levels); `deepseek-v4-flash`/`-pro` are `reasoning: true`, the §4 heuristic covers `deepseek-v[4-9]`, and the legacy non-thinking `deepseek-chat` alias stays uncontrollable. So the §2 DeepSeek row's "on/off, no graded tiers" — and the matching on/off phrasing in Consequences → Negative — are both superseded by the doc-verified `thinking` shape here (off / high / max); the reasoning **output** (`reasoning_content`) + token accounting were already mapped. ## Context diff --git a/packages/llm/src/adapters/openai.test.ts b/packages/llm/src/adapters/openai.test.ts index 7a0f9e00..72bed473 100644 --- a/packages/llm/src/adapters/openai.test.ts +++ b/packages/llm/src/adapters/openai.test.ts @@ -1,7 +1,7 @@ import { APIConnectionError, APIConnectionTimeoutError, APIError, APIUserAbortError } from 'openai'; import { describe, expect, it } from 'vitest'; -import type { AbortSignalLike } from '@relavium/shared'; +import type { AbortSignalLike, ReasoningEffort } from '@relavium/shared'; import { InvalidBaseUrlError, UnsupportedCapabilityError } from '../errors.js'; import { LlmProviderError } from '../llm-error.js'; @@ -231,7 +231,7 @@ describe('OpenAI-compatible adapter', () => { }); }); - it('maps the reasoning-effort tier to OpenAI reasoning_effort (max→xhigh, off→none, unset omitted); DeepSeek is not mapped (ADR-0066)', async () => { + it('maps the reasoning-effort tier per provider: OpenAI reasoning_effort (max→xhigh, off→none) + DeepSeek thinking (ADR-0066)', async () => { let sent: Record = {}; const oai = createOpenAiAdapter({ fetch: (_input, init) => { @@ -257,7 +257,9 @@ describe('OpenAI-compatible adapter', () => { await oai.generate({ ...base }, 'k'); // unset ⇒ omitted (provider default, unchanged behavior) expect('reasoning_effort' in sent).toBe(false); - // DeepSeek (the other id this shared adapter serves) controls thinking differently — reasoning_effort is NOT sent. + // DeepSeek (the other id this shared adapter serves) controls thinking via a `thinking` OBJECT, not the OpenAI + // `reasoning_effort` key (ADR-0066): off→disabled; DeepSeek has only two graded levels, so low/medium/high→high + // and max→max; unset ⇒ omitted. let dsSent: Record = {}; const ds = createOpenAiAdapter({ providerId: 'deepseek', @@ -266,11 +268,45 @@ describe('OpenAI-compatible adapter', () => { return Promise.resolve(okResponse()); }, }); - await ds.generate( - { model: 'deepseek-v4-flash', messages: base.messages, reasoningEffort: 'high' }, - 'k', - ); - expect('reasoning_effort' in dsSent).toBe(false); + const dsReq = (effort?: ReasoningEffort): Parameters[0] => ({ + model: 'deepseek-v4-flash', + messages: base.messages, + ...(effort === undefined ? {} : { reasoningEffort: effort }), + }); + await ds.generate(dsReq('high'), 'k'); + expect('reasoning_effort' in dsSent).toBe(false); // never the OpenAI key + expect(dsSent['thinking']).toEqual({ type: 'enabled', reasoning_effort: 'high' }); + await ds.generate(dsReq('off'), 'k'); + expect(dsSent['thinking']).toEqual({ type: 'disabled' }); // off DISABLES thinking + await ds.generate(dsReq('max'), 'k'); + expect(dsSent['thinking']).toEqual({ type: 'enabled', reasoning_effort: 'max' }); // top graded level + await ds.generate(dsReq('low'), 'k'); + expect(dsSent['thinking']).toEqual({ type: 'enabled', reasoning_effort: 'high' }); // coarsened to high + await ds.generate(dsReq(), 'k'); // unset ⇒ omitted (provider default) + expect('thinking' in dsSent).toBe(false); + + // The streaming path spreads the SAME buildCommonBody, so the `thinking` control must reach the wire there too. + let dsStreamSent: Record = {}; + const dsStream = createOpenAiAdapter({ + providerId: 'deepseek', + fetch: (_input, init) => { + dsStreamSent = parseJsonBody(init); + return Promise.resolve( + sse([ + { + id: 's', + object: 'chat.completion.chunk', + created: 0, + model: 'deepseek-v4-flash', + choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: 'stop' }], + }, + ]), + ); + }, + maxRetries: 0, + }); + await collect(dsStream.stream(dsReq('max'), 'k')); + expect(dsStreamSent['thinking']).toEqual({ type: 'enabled', reasoning_effort: 'max' }); }); it('round-trips inline audio-out: lowers output_modalities → modalities+audio and parses the response (1.AG/ADR-0046)', async () => { diff --git a/packages/llm/src/adapters/openai.ts b/packages/llm/src/adapters/openai.ts index ddb94b6d..f9fe06cb 100644 --- a/packages/llm/src/adapters/openai.ts +++ b/packages/llm/src/adapters/openai.ts @@ -664,6 +664,24 @@ const OPENAI_REASONING_EFFORT: Record< max: 'xhigh', }; +/** DeepSeek's native reasoning control (a Relavium-local shape — NOT a vendor SDK type — so nothing crosses the + * seam): the create-chat-completion `thinking` object (verified 2026-07-07, api-docs.deepseek.com). */ +interface DeepSeekThinking { + readonly type: 'enabled' | 'disabled'; + readonly reasoning_effort?: 'high' | 'max'; +} + +/** ADR-0066: the normalized tier → DeepSeek's `thinking` object. `off` DISABLES thinking; DeepSeek's thinking has + * only two graded levels (`high`/`max`), so `low`/`medium`/`high` coarsen to `high` and `max` → `max` (an honest, + * documented coarsening onto v4's actual capability). */ +const DEEPSEEK_THINKING: Record = { + off: { type: 'disabled' }, + low: { type: 'enabled', reasoning_effort: 'high' }, + medium: { type: 'enabled', reasoning_effort: 'high' }, + high: { type: 'enabled', reasoning_effort: 'high' }, + max: { type: 'enabled', reasoning_effort: 'max' }, +}; + function toOpenAiToolChoice(choice: ToolChoice): OpenAI.ChatCompletionToolChoiceOption { if (choice === 'auto') { return 'auto'; @@ -686,11 +704,14 @@ function toJsonSchemaName(name: string | undefined): string { return sanitized.length > 0 ? sanitized : 'response'; } +/** The shared request body: the OpenAI non-streaming params PLUS DeepSeek's `thinking` extension (a Relavium-local + * optional field the OpenAI-compatible endpoint accepts; assignable to the SDK create() param as an extra key). */ +type OpenAiCompatibleBody = Omit & { + thinking?: DeepSeekThinking; +}; + /** The shared request body (everything except the `stream` discriminant each method sets). */ -function buildCommonBody( - req: LlmRequest, - provider: ProviderId, -): Omit { +function buildCommonBody(req: LlmRequest, provider: ProviderId): OpenAiCompatibleBody { const messages: OpenAI.ChatCompletionMessageParam[] = []; if (req.system !== undefined) { messages.push({ role: 'system', content: req.system }); @@ -698,7 +719,7 @@ function buildCommonBody( for (const message of req.messages) { messages.push(...toOpenAiMessages(message, provider)); } - const body: Omit = { + const body: OpenAiCompatibleBody = { model: req.model, messages, }; @@ -717,12 +738,16 @@ function buildCommonBody( if (req.maxTokens !== undefined) { body.max_tokens = req.maxTokens; } - // ADR-0066: map the normalized reasoning-effort tier to OpenAI's native `reasoning_effort` (also a tier). ONLY for - // the `openai` provider — DeepSeek (the other id this shared adapter serves) controls thinking differently (its - // own follow-up), so its effort is not sent here. The host gates this to reasoning-capable models (a non-reasoning - // model would reject it), and `body` is spread LAST below so this mapped field wins over any providerOptions echo. - if (provider === 'openai' && req.reasoningEffort !== undefined) { - body.reasoning_effort = OPENAI_REASONING_EFFORT[req.reasoningEffort]; + // ADR-0066: map the normalized reasoning-effort tier to each provider's NATIVE control. OpenAI takes a + // `reasoning_effort` tier; DeepSeek (the other id this shared adapter serves) takes a `thinking` object + // (off→disabled, else enabled + high/max). The host gates this to reasoning-capable models (a non-reasoning model + // would reject it), and `body` is spread LAST below so the mapped field wins over any providerOptions echo. + if (req.reasoningEffort !== undefined) { + if (provider === 'openai') { + body.reasoning_effort = OPENAI_REASONING_EFFORT[req.reasoningEffort]; + } else if (provider === 'deepseek') { + body.thinking = DEEPSEEK_THINKING[req.reasoningEffort]; + } } if (req.stopSequences !== undefined) { body.stop = req.stopSequences; diff --git a/packages/llm/src/model-catalog.test.ts b/packages/llm/src/model-catalog.test.ts index 38bfb277..665c5284 100644 --- a/packages/llm/src/model-catalog.test.ts +++ b/packages/llm/src/model-catalog.test.ts @@ -46,10 +46,11 @@ describe('mergeModelCatalog (ADR-0064 §6)', () => { it('surfaces supportsReasoning from the STATIC registry tier (ADR-0066) — true for a reasoning model, false otherwise', () => { const entries = mergeModelCatalog({ now: BEFORE_DEEPSEEK_DEPRECATION }); - // A registry model tagged `reasoning: true` (flagship) exposes the effort-controllable capability… + // A registry model tagged `reasoning: true` exposes the effort-controllable capability (incl. DeepSeek v4)… expect(byId(entries, 'claude-opus-4-8')?.supportsReasoning).toBe(true); - // …a registry model NOT so tagged (DeepSeek is deferred, ADR-0066) is false — the picker skips its effort sub-step. - expect(byId(entries, 'deepseek-v4-flash')?.supportsReasoning).toBe(false); + expect(byId(entries, 'deepseek-v4-flash')?.supportsReasoning).toBe(true); + // …a registry model NOT so tagged (the legacy non-thinking `deepseek-chat`) is false — no effort sub-step. + expect(byId(entries, 'deepseek-chat')?.supportsReasoning).toBe(false); }); it('a LIVE-only model gates via the §4 id heuristic — a known reasoning family ON, an ambiguous id OFF (ADR-0066)', () => { diff --git a/packages/llm/src/pricing.test.ts b/packages/llm/src/pricing.test.ts index c35a4982..c56b43ac 100644 --- a/packages/llm/src/pricing.test.ts +++ b/packages/llm/src/pricing.test.ts @@ -14,13 +14,18 @@ import { * indicator + auto-compaction to "not applicable", never a crash). */ describe('modelSupportsReasoning (ADR-0066)', () => { - it('is true for a tagged reasoning model, false for DeepSeek (adapter deferred) + unknown/custom', () => { + it('is true for a tagged reasoning model (incl. DeepSeek v4), false for a non-reasoning + unknown/custom', () => { expect(modelSupportsReasoning('claude-opus-4-8')).toBe(true); expect(modelSupportsReasoning('gpt-5.5')).toBe(true); expect(modelSupportsReasoning('gemini-2.5-pro')).toBe(true); - // DeepSeek reasons (v4 thinking) but its adapter mapping is deferred, so the capability stays OFF (the effort - // is not controllable there yet — the picker must not offer it). - expect(modelSupportsReasoning('deepseek-v4-flash')).toBe(false); + // DeepSeek v4 exposes the controllable `thinking` param (ADR-0066) — the effort IS controllable, so ON. + expect(modelSupportsReasoning('deepseek-v4-flash')).toBe(true); + expect(modelSupportsReasoning('deepseek-v4-pro')).toBe(true); + // The legacy `deepseek-chat` (non-thinking alias) is NOT reasoning-controllable → OFF. + expect(modelSupportsReasoning('deepseek-chat')).toBe(false); + // `deepseek-reasoner` IS a thinking model, but it's the fixed always-on alias — effort is not adjustable on it + // (sending `thinking:{type:'disabled'}` for `off` would 400), so it stays OFF (no effort picker). ADR-0066 §2. + expect(modelSupportsReasoning('deepseek-reasoner')).toBe(false); // Unknown / custom base-URL model NOT matching a reasoning family ⇒ the SAFE default. expect(modelSupportsReasoning('some-custom-base-url-model-xyz')).toBe(false); expect(modelSupportsReasoning('')).toBe(false); @@ -31,18 +36,22 @@ describe('modelSupportsReasoning (ADR-0066)', () => { expect(modelSupportsReasoning('o5-mini')).toBe(true); // a future o-series id expect(modelSupportsReasoning('gpt-5.9-turbo')).toBe(true); // a future reasoning gpt-5 id expect(modelSupportsReasoning('claude-opus-5')).toBe(true); // a future Opus + expect(modelSupportsReasoning('deepseek-v4-turbo')).toBe(true); // a future non-registry DeepSeek v4 id + expect(modelSupportsReasoning('deepseek-v5-pro')).toBe(true); // a future DeepSeek v5 expect(modelSupportsReasoning('gemini-3.0-flash-thinking')).toBe(true); // an explicit "thinking" id // …while an AMBIGUOUS / non-reasoning id stays OFF (over-matching would earn a provider 400). expect(modelSupportsReasoning('gpt-4o')).toBe(false); // not a reasoning family expect(modelSupportsReasoning('gpt-5-chat-latest')).toBe(false); // the non-reasoning gpt-5 conversational variant + expect(modelSupportsReasoning('deepseek-v5-chat')).toBe(false); // a hypothetical future non-thinking DeepSeek `-chat` variant (mirrors the gpt-5-chat guard) expect(modelSupportsReasoning('claude-sonnet-9')).toBe(false); // base Sonnet is version-dependent — registry only expect(modelSupportsReasoning('gemini-2.0-flash')).toBe(false); // Gemini by version is not heuristic-matched }); - it('the registry is AUTHORITATIVE for a canonical id — a false/absent flag is not overridden by the heuristic', () => { - // deepseek-v4-flash is a canonical id whose registry flag is not set; the id heuristic never runs for a canonical - // id, so it stays false (the deferred-adapter decision) even though it is a v4 "thinking" model. - expect(modelSupportsReasoning('deepseek-v4-flash')).toBe(false); + it('the registry is AUTHORITATIVE for a canonical id — the flag (not the id heuristic) decides', () => { + // `deepseek-chat` is a canonical id (the legacy non-thinking alias) with NO `reasoning` flag → false, decided by + // the registry, not any id match. `deepseek-v4-flash`, also canonical, is true via its registry flag (above). + expect(modelSupportsReasoning('deepseek-chat')).toBe(false); + expect(modelSupportsReasoning('deepseek-v4-flash')).toBe(true); }); }); diff --git a/packages/llm/src/pricing.ts b/packages/llm/src/pricing.ts index 52c0343e..b4a460be 100644 --- a/packages/llm/src/pricing.ts +++ b/packages/llm/src/pricing.ts @@ -178,8 +178,10 @@ export const MODEL_PRICING = { // --- DeepSeek (verified 2026-07-03: api-docs.deepseek.com/quick_start/pricing; via the OpenAI-compatible // adapter) — the current ids are `deepseek-v4-flash` (default tier) and `deepseek-v4-pro` (premium tier). Each // serves BOTH non-thinking and thinking (default) modes on ONE id — the mode is a request param, not a - // separate model — so there is no per-mode row. The legacy `deepseek-chat` / `deepseek-reasoner` ids (the old - // non-thinking / thinking aliases of v4-flash) are kept below until they deprecate on 2026-07-24 15:59 UTC. + // separate model — so there is no per-mode row. Reasoning-effort IS controllable (ADR-0066): the create-chat- + // completion API takes a `thinking` object (`type: enabled|disabled` + `reasoning_effort: high|max`), mapped in + // openai.ts (`reasoning: true` below). The legacy `deepseek-chat` (non-thinking) / `deepseek-reasoner` (thinking) + // aliases are kept below until they deprecate on 2026-07-24 15:59 UTC and stay reasoning-uncontrollable. 'deepseek-v4-flash': { provider: 'deepseek', nativeId: 'deepseek-v4-flash', @@ -189,6 +191,7 @@ export const MODEL_PRICING = { inputPerMtokMicrocents: usd(0.14), outputPerMtokMicrocents: usd(0.28), cachedInputPerMtokMicrocents: usd(0.0028), // cache-hit input + reasoning: true, // ADR-0066: v4 exposes a controllable `thinking` param (off / high / max) }, 'deepseek-v4-pro': { provider: 'deepseek', @@ -199,6 +202,7 @@ export const MODEL_PRICING = { inputPerMtokMicrocents: usd(0.435), outputPerMtokMicrocents: usd(0.87), cachedInputPerMtokMicrocents: usd(0.003625), // cache-hit input + reasoning: true, // ADR-0066: v4 exposes a controllable `thinking` param (off / high / max) }, // Legacy aliases — deprecating 2026-07-24 15:59 UTC. Kept so an existing agent/config that still names them // keeps costing correctly until then; the pricing page no longer lists them, so these hold the last verified @@ -252,16 +256,20 @@ export const KNOWN_MODEL_IDS: readonly CanonicalModelId[] = * live-discovered model whose list endpoint omits a reasoning flag). Each arm is a family/pattern where the WHOLE * matched set reasons, so a new member of a known reasoning family (e.g. a next o-series id) gates correctly before * the registry is updated. Deliberately **narrow**: it does NOT prefix-match ambiguous families whose lineup mixes - * reasoning and non-reasoning members (base Claude Sonnet, Gemini by version, DeepSeek — reasoning deferred), because - * OVER-matching would send the tier to a non-reasoning model and earn a provider rejection — strictly worse than the - * safe under-match (no effort UX until the registry adds the model, the same maintenance shape as pricing). The - * `-chat` exclusion keeps OpenAI's non-reasoning `gpt-5-chat` conversational variant out. + * reasoning and non-reasoning members by *version* (base Claude Sonnet, Gemini by version), because OVER-matching + * would send the tier to a non-reasoning model and earn a provider rejection — strictly worse than the safe + * under-match (no effort UX until the registry adds the model, the same maintenance shape as pricing). DeepSeek IS + * matched, but only the `deepseek-v[4-9]` prefix (whose whole set serves the `thinking` param); its legacy + * `deepseek-chat`/`-reasoner` aliases are not `v`-prefixed, so they never match. The `-chat` exclusion keeps + * OpenAI's non-reasoning `gpt-5-chat` conversational variant — and any future `deepseek-v_-chat` non-thinking + * variant — out. */ export function reasoningModelIdHeuristic(model: string): boolean { const m = model.toLowerCase(); if (/^o\d/.test(m)) return true; // OpenAI o-series (o1 / o3 / o4 / o5+) — the entire family reasons if (m.startsWith('gpt-5') && !m.includes('chat')) return true; // the reasoning gpt-5 line (gpt-5-chat is non-reasoning) if (m.startsWith('claude-opus')) return true; // Claude Opus reasons (extended thinking) + if (/^deepseek-v[4-9]/.test(m) && !m.includes('chat')) return true; // DeepSeek v4+ serves the `thinking` param; a future `-chat` non-thinking variant stays out (ADR-0066 §4) if (m.includes('thinking')) return true; // an explicit "thinking" model id (e.g. a Gemini thinking variant) return false; } From 685c55cbea610befe420c5207292facf7026da88 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 7 Jul 2026 11:25:10 +0300 Subject: [PATCH 66/68] =?UTF-8?q?feat(cli):=20interactive=20/effort=20tier?= =?UTF-8?q?-selector=20overlay=20(ADR-0066=20=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bare `/effort` printed a grey informational notice — the tiers listed but not selectable; you had to know to type `/effort `. §6 anticipated "a future `/effort`"; this ships it as a first-class interactive overlay. - effort-picker.ts: a pure keyboard-owning fold (off/low/medium/high/max, opens on the bound tier, arrow+Enter to apply) — mirrors the model-picker fold; no catalog/filter/refresh, so no port + no async load. - effort-tier-list.tsx: the shared tier-list view, now used by BOTH the new overlay AND the `/models` effort sub-step (extracted from model-picker-view so the two entry points can never drift). - chat-ink.tsx + home-controller.ts/home-app.tsx: wire the overlay as a keyboard-owning submode in the standalone chat AND the in-Home chat, opened by a typed or palette `/effort`. Opens ONLY on a reasoning-capable bound model; a non-reasoning model falls through to the informational notice. Applying calls the §5 per-turn onSetEffort — no reseat. - tests: effort-picker fold units (9) + home-controller integration (open on a reasoning model + apply → setter/no-reseat; non-reasoning → dispatch). - ADR-0066: append-only §6 note (the overlay realized). Refs: ADR-0066 Co-Authored-By: Claude --- apps/cli/src/render/tui/chat-ink.tsx | 80 ++++++++++- apps/cli/src/render/tui/effort-picker.test.ts | 98 ++++++++++++++ apps/cli/src/render/tui/effort-picker.ts | 89 +++++++++++++ apps/cli/src/render/tui/effort-tier-list.tsx | 60 +++++++++ apps/cli/src/render/tui/home-app.tsx | 17 ++- .../src/render/tui/home-controller.test.ts | 125 +++++++++++++++++- apps/cli/src/render/tui/home-controller.ts | 84 +++++++++++- apps/cli/src/render/tui/model-picker-view.tsx | 39 ++---- ...066-normalized-reasoning-effort-control.md | 2 + 9 files changed, 560 insertions(+), 34 deletions(-) create mode 100644 apps/cli/src/render/tui/effort-picker.test.ts create mode 100644 apps/cli/src/render/tui/effort-picker.ts create mode 100644 apps/cli/src/render/tui/effort-tier-list.tsx diff --git a/apps/cli/src/render/tui/chat-ink.tsx b/apps/cli/src/render/tui/chat-ink.tsx index 65eec497..bd056b44 100644 --- a/apps/cli/src/render/tui/chat-ink.tsx +++ b/apps/cli/src/render/tui/chat-ink.tsx @@ -12,6 +12,13 @@ import { } from '../../commands/chat.js'; import { CHAT_PALETTE_COMMANDS } from '../../commands/repl-commands.js'; import type { RefreshReport } from '../../engine/model-refresh.js'; +import { + canControlEffort, + foldEffortPickerKey, + initialEffortPickerState, + type EffortPickerState, +} from './effort-picker.js'; +import { EffortTierList } from './effort-tier-list.js'; import { foldModelPickerKey, partialFailureBanner, @@ -378,6 +385,14 @@ export function ChatApp(props: Readonly): ReactElement { setModelPicker(next); }; const pickerEpochRef = useRef(0); + // The standalone `/effort` overlay (ADR-0066 §6) — a keyboard-owning submode like `/models`, ref-shadowed so a + // coalesced stdin chunk sees a just-applied open/close/accept. Opened only on a reasoning-capable bound model. + const [effortPicker, setEffortPicker] = useState(undefined); + const effortPickerRef = useRef(undefined); + const applyEffortPicker = (next: EffortPickerState | undefined): void => { + effortPickerRef.current = next; + setEffortPicker(next); + }; // A monotonic submit generation: bumped every time the compose buffer is submitted (cleared). An async mention // read captures it at accept time and DROPS its inject if a submit has since happened — so a slow read that // resolves after Enter can never splice the file into the (now-empty) buffer meant for the NEXT message. @@ -669,6 +684,41 @@ export function ChatApp(props: Readonly): ReactElement { } }; + // ---- The standalone `/effort` overlay (ADR-0066 §6) — interactive tier selection (no reseat) ---------------- + // Open on the LIVE bound model + the LIVE store tier (so it opens on the currently-bound effort). The caller + // (submit) gates this on a reasoning-capable model + a wired setter, so opening here is unconditional. + const openEffortPicker = (): void => { + const snap = props.store.getSnapshot(); + if (snap.state.model === undefined) return; // defensive — submit only opens on a bound reasoning-capable model + applyEffortPicker(initialEffortPickerState(snap.state.model, snap.reasoningEffort)); + }; + // The open effort overlay owns every key (mirrors routeModelPickerKey). Accept applies the tier via the per-turn + // setter (no reseat); a re-pick of the same tier is a gentle no-op with visible store feedback. + const routeEffortPickerKey = (char: string, key: ModelPickerKey): void => { + const open = effortPickerRef.current; + if (open === undefined) return; + const step = foldEffortPickerKey(char, key, open); + switch (step.kind) { + case 'close': + applyEffortPicker(undefined); + return; + case 'accept': + applyEffortPicker(undefined); + if (step.effort === open.current) { + props.store.note(`Already at reasoning effort ${step.effort}.`); + } else { + props.onSetEffort?.(step.effort); + props.store.note( + `Reasoning effort set to ${step.effort} — applies to your next message.`, + ); + } + return; + case 'state': + applyEffortPicker(step.state); + return; + } + }; + const submit = (message: string, display?: string): void => { // A typed `/models` opens the reseat picker overlay (ADR-0059) instead of sending — interactive only (the port // is wired). Covers a directly-typed `/models` AND a chat-palette selection (both route through `submit`). @@ -676,6 +726,17 @@ export function ChatApp(props: Readonly): ReactElement { openModelPicker(); return; } + // A typed (or palette-selected) bare `/effort` opens the interactive tier overlay (ADR-0066 §6) instead of the + // informational notice — but ONLY when the setter is wired AND the bound model is reasoning-capable; a + // non-reasoning model falls through to the dispatch, whose ctx handler prints the "no controllable tier" notice. + // `/effort ` (with an arg) is NOT intercepted (exact match) — it dispatches and sets the tier directly. + if ( + message.trim() === '/effort' && + canControlEffort(props.store.getSnapshot().state.model, props.onSetEffort !== undefined) + ) { + openEffortPicker(); + return; + } // Mark the submit in flight so input is gated + the spinner runs for the WHOLE operation (streaming AND any // after-turn auto-compaction, ADR-0062) — cleared in EVERY settle branch (success / reject / defensive catch). applySubmitBusy(true); @@ -718,6 +779,12 @@ export function ChatApp(props: Readonly): ReactElement { routeModelPickerKey(char, key); return; } + // The open `/effort` overlay owns every key (ADR-0066 §6) — mutually exclusive with the other submodes (it only + // opens at an idle prompt). Read the REF so a coalesced same-chunk key sees a just-applied open/close/accept. + if (effortPickerRef.current !== undefined) { + routeEffortPickerKey(char, key); + return; + } // The open `@`-mention completion owns every key (2.5.D step 4): Esc/Ctrl-C cancels + restores the literal // keystrokes; ↑/↓ select; Enter/Tab/'/' accept (a dir descends, a file injects); backspace trims the filter // then deletes the `@`; a printable extends the filter. Read the REF so a coalesced same-chunk key sees a @@ -948,7 +1015,8 @@ export function ChatApp(props: Readonly): ReactElement { palette !== undefined || search !== undefined || mention !== undefined || - modelPicker !== undefined + modelPicker !== undefined || + effortPicker !== undefined } /> {palette !== undefined && ( @@ -963,6 +1031,16 @@ export function ChatApp(props: Readonly): ReactElement { {modelPicker !== undefined && ( )} + {/* The standalone `/effort` overlay (ADR-0066 §6) — the shared tier list; `Esc` cancels (not a back-out). */} + {effortPicker !== undefined && ( + + )} ); } diff --git a/apps/cli/src/render/tui/effort-picker.test.ts b/apps/cli/src/render/tui/effort-picker.test.ts new file mode 100644 index 00000000..f0dd67cf --- /dev/null +++ b/apps/cli/src/render/tui/effort-picker.test.ts @@ -0,0 +1,98 @@ +import { REASONING_EFFORTS } from '@relavium/shared'; +import { describe, expect, it } from 'vitest'; + +import { + canControlEffort, + foldEffortPickerKey, + initialEffortPickerState, + type EffortPickerState, +} from './effort-picker.js'; + +function state(partial: Partial = {}): EffortPickerState { + return { selected: 0, current: undefined, model: 'deepseek-v4-flash', ...partial }; +} + +describe('canControlEffort', () => { + it('is true only when the setter is wired AND the bound model is reasoning-capable', () => { + expect(canControlEffort('claude-opus-4-8', true)).toBe(true); // registry reasoning model + setter + expect(canControlEffort('deepseek-v4-flash', true)).toBe(true); + }); + + it('is false for a non-reasoning model, an unbound model, or an unwired setter', () => { + expect(canControlEffort('gpt-4o', true)).toBe(false); // gpt-4o is not a reasoning model + expect(canControlEffort('deepseek-chat', true)).toBe(false); // the legacy non-thinking alias + expect(canControlEffort(undefined, true)).toBe(false); // no bound model yet (pre session:started) + expect(canControlEffort('claude-opus-4-8', false)).toBe(false); // setter not wired (a non-interactive driver) + }); +}); + +describe('initialEffortPickerState', () => { + it('opens on the bound effort when set', () => { + // 'high' is index 3 in [off, low, medium, high, max]. + expect(initialEffortPickerState('m', 'high').selected).toBe(REASONING_EFFORTS.indexOf('high')); + expect(initialEffortPickerState('m', 'off').selected).toBe(REASONING_EFFORTS.indexOf('off')); + }); + + it('opens on a neutral middle tier (medium) when no effort is bound', () => { + const s = initialEffortPickerState('m', undefined); + expect(REASONING_EFFORTS[s.selected]).toBe('medium'); + }); + + it('carries the model + current through', () => { + const s = initialEffortPickerState('deepseek-v4-pro', 'max'); + expect(s.model).toBe('deepseek-v4-pro'); + expect(s.current).toBe('max'); + }); +}); + +describe('foldEffortPickerKey', () => { + it('Esc and Ctrl-C close (no apply)', () => { + expect(foldEffortPickerKey('', { escape: true }, state()).kind).toBe('close'); + expect(foldEffortPickerKey('c', { ctrl: true }, state()).kind).toBe('close'); + }); + + it('↑/↓ move the highlight, clamped to the tier list', () => { + const down = foldEffortPickerKey('', { downArrow: true }, state({ selected: 0 })); + expect(down).toEqual({ kind: 'state', state: state({ selected: 1 }) }); + // Clamp at the top edge — Up on index 0 stays 0. + const up = foldEffortPickerKey('', { upArrow: true }, state({ selected: 0 })); + expect(up).toEqual({ kind: 'state', state: state({ selected: 0 }) }); + // Clamp at the bottom edge — Down on the last index stays. + const last = REASONING_EFFORTS.length - 1; + const downLast = foldEffortPickerKey('', { downArrow: true }, state({ selected: last })); + expect(downLast).toEqual({ kind: 'state', state: state({ selected: last }) }); + }); + + it('Enter accepts the highlighted tier', () => { + const step = foldEffortPickerKey( + '', + { return: true }, + state({ selected: REASONING_EFFORTS.indexOf('high') }), + ); + expect(step).toEqual({ kind: 'accept', effort: 'high' }); + }); + + it('Enter accepts each tier by index (all five reachable)', () => { + REASONING_EFFORTS.forEach((effort, index) => { + const step = foldEffortPickerKey('', { return: true }, state({ selected: index })); + expect(step).toEqual({ kind: 'accept', effort }); + }); + }); + + it('a clamped/out-of-range highlight accepts the last valid tier (never a malformed accept)', () => { + // Enter clamps the index, so a hand-built OOR index resolves to the last tier — the `undefined` guard branch is + // unreachable for the fixed non-empty list, but this pins that Enter never emits an out-of-range/undefined tier. + const step = foldEffortPickerKey('', { return: true }, state({ selected: 999 })); + expect(step).toEqual({ + kind: 'accept', + effort: REASONING_EFFORTS[REASONING_EFFORTS.length - 1], + }); + }); + + it('any other key is inert (returns the same state ref)', () => { + const s = state({ selected: 2 }); + const step = foldEffortPickerKey('x', {}, s); + expect(step).toEqual({ kind: 'state', state: s }); + if (step.kind === 'state') expect(step.state).toBe(s); // same ref — no needless re-render/hint-wipe + }); +}); diff --git a/apps/cli/src/render/tui/effort-picker.ts b/apps/cli/src/render/tui/effort-picker.ts new file mode 100644 index 00000000..a35072b0 --- /dev/null +++ b/apps/cli/src/render/tui/effort-picker.ts @@ -0,0 +1,89 @@ +import { modelSupportsReasoning } from '@relavium/llm'; +import { REASONING_EFFORTS, type ReasoningEffort } from '@relavium/shared'; + +import type { ModelPickerKey } from './model-picker.js'; + +/** + * The standalone `/effort` overlay ([ADR-0066](../../../../../docs/decisions/0066-normalized-reasoning-effort-control.md) §6) + * — a keyboard-owning submode (like the `/models` picker + the `/` palette) that lists the reasoning-effort tiers and, + * on Enter, pushes the chosen tier as the session's per-turn override via the surface's effort setter (NO reseat — + * effort changes neither provider, pricing, nor the plan). It is a FIXED five-row list: no catalog, filter, or + * refresh — so it needs no port and no async load, unlike the `/models` picker. + * + * Two surfaces route the SAME fold (standalone `relavium chat` + the in-Home live chat); the accept is UNIFORM (call + * `onSetEffort` + note), so — unlike the model picker's surface-divergent reseat-vs-default-write accept — no + * per-surface branching lives here. Offered ONLY when the bound model is reasoning-capable; a non-reasoning model + * never opens this overlay (the surface falls through to the `/effort` notice instead). The pure fold + state live + * here; the ink view is the shared {@link effort-tier-list.tsx} `EffortTierList`. + */ +export interface EffortPickerState { + /** The highlighted index into {@link REASONING_EFFORTS}. */ + readonly selected: number; + /** The session's currently-bound effort (the `✓` + the opening highlight); `undefined` ⇒ the provider default, so + * the list opens on a neutral middle tier. */ + readonly current: ReasoningEffort | undefined; + /** The bound model id — shown after the "Reasoning effort" header (sanitized at the display boundary). */ + readonly model: string; +} + +/** What a keystroke does to the open effort overlay. */ +export type EffortPickerStep = + | { readonly kind: 'close' } // Esc / Ctrl-C — cancel without acting + | { readonly kind: 'accept'; readonly effort: ReasoningEffort } // Enter — apply the highlighted tier + | { readonly kind: 'state'; readonly state: EffortPickerState }; + +/** + * Whether a surface should open the interactive `/effort` overlay: the per-turn effort setter must be wired AND the + * bound model reasoning-capable. A non-reasoning model (or no bound model yet / no setter) returns false, so the + * surface falls through to the informational `/effort` notice rather than opening a dead overlay. Shared by the + * standalone chat + the in-Home chat so the gate can never diverge between them. + */ +export function canControlEffort(model: string | undefined, setterWired: boolean): boolean { + return setterWired && model !== undefined && modelSupportsReasoning(model); +} + +/** Clamp an index to `0..count-1` (or 0 when the list is empty — never for the fixed non-empty tier list). */ +function clampSelection(index: number, count: number): number { + if (count <= 0) return 0; + return Math.max(0, Math.min(index, count - 1)); +} + +/** The overlay's opening state: the highlight starts on the bound effort, else a neutral middle tier (`'medium'`). */ +export function initialEffortPickerState( + model: string, + current: ReasoningEffort | undefined, +): EffortPickerState { + return { selected: Math.max(0, REASONING_EFFORTS.indexOf(current ?? 'medium')), current, model }; +} + +/** + * Fold one keystroke into the open effort overlay (the keyboard-owning contract, mirroring the model-picker fold). + * `Ctrl-C`/`Esc` cancel (nothing applied); `↑`/`↓` move over {@link REASONING_EFFORTS}; `Enter` accepts the + * highlighted tier. It is a fixed list with no filter/refresh, so every other key is inert (returns the same state). + */ +export function foldEffortPickerKey( + char: string, + key: ModelPickerKey, + state: EffortPickerState, +): EffortPickerStep { + if (key.escape === true || (key.ctrl === true && char === 'c')) return { kind: 'close' }; + if (key.upArrow === true) { + return { + kind: 'state', + state: { ...state, selected: clampSelection(state.selected - 1, REASONING_EFFORTS.length) }, + }; + } + if (key.downArrow === true) { + return { + kind: 'state', + state: { ...state, selected: clampSelection(state.selected + 1, REASONING_EFFORTS.length) }, + }; + } + if (key.return === true) { + const effort = REASONING_EFFORTS[clampSelection(state.selected, REASONING_EFFORTS.length)]; + // Defensive: an out-of-range highlight (never expected — the fold clamps every move) closes rather than emitting + // a malformed accept. + return effort === undefined ? { kind: 'close' } : { kind: 'accept', effort }; + } + return { kind: 'state', state }; +} diff --git a/apps/cli/src/render/tui/effort-tier-list.tsx b/apps/cli/src/render/tui/effort-tier-list.tsx new file mode 100644 index 00000000..07293348 --- /dev/null +++ b/apps/cli/src/render/tui/effort-tier-list.tsx @@ -0,0 +1,60 @@ +import { Box, Text } from 'ink'; +import type { ReactElement } from 'react'; + +import { EFFORT_TIER_HINT, REASONING_EFFORTS, type ReasoningEffort } from '@relavium/shared'; + +import { sanitizeInline } from './chat-projection.js'; +import { colorProps, dimProps } from './projection.js'; + +/** + * The shared reasoning-effort tier list (ADR-0066) — the fixed five-row picker body used by BOTH the `/models` + * effort sub-step ({@link model-picker-view.tsx}'s `EffortSubList`, reached after choosing a reasoning model on a + * reseat surface) and the standalone `/effort` overlay ({@link effort-picker.ts}). One canonical presentation so the + * two entry points can never drift: each tier + its one-line hint, the highlighted row in cyan, and a `✓` on the + * session's currently-bound effort. The header suffix (a provider-controlled model name) is sanitized at this display + * boundary, exactly as the model rows are — a crafted name can neither forge a row nor inject a terminal escape. + * + * PURE: it owns no `useInput`; the surface routes keys to the fold and re-renders this from the resulting state. + */ +export interface EffortTierListProps { + /** The highlighted tier index (already clamped by the caller's fold, but re-clamped here for display safety). */ + readonly selected: number; + /** The session's currently-bound effort — the `✓` marker; `undefined` ⇒ no tier bound (the provider default). */ + readonly current: ReasoningEffort | undefined; + /** A trailing context label after the "Reasoning effort" header (e.g. the bound model's name) — sanitized here; + * `undefined` ⇒ no suffix. */ + readonly labelSuffix?: string | undefined; + /** The nav-hint footer — differs by entry point ("Esc back" in the model sub-step vs "Esc cancel" standalone). */ + readonly footer: string; + readonly color: boolean; +} + +export function EffortTierList(props: Readonly): ReactElement { + const { selected, current, labelSuffix, footer, color } = props; + // Re-clamp for display: a caller could pass an out-of-range index (a shrunk source, a stale render) — never index + // past the fixed five-row list. + const highlighted = Math.max(0, Math.min(selected, REASONING_EFFORTS.length - 1)); + const suffix = labelSuffix === undefined || labelSuffix.length === 0 ? '' : ` · ${labelSuffix}`; + return ( + + + Reasoning effort + {sanitizeInline(suffix)} + + {REASONING_EFFORTS.map((effort, index) => { + const isSelected = index === highlighted; + const isCurrent = effort === current; + const rowColor = isSelected ? colorProps(color, 'cyan') : {}; + return ( + + {`${isSelected ? '›' : ' '} ${isCurrent ? '✓' : ' '} ${effort} · `} + {EFFORT_TIER_HINT[effort]} + + ); + })} + + {footer} + + + ); +} diff --git a/apps/cli/src/render/tui/home-app.tsx b/apps/cli/src/render/tui/home-app.tsx index ffda7c81..57baa4e9 100644 --- a/apps/cli/src/render/tui/home-app.tsx +++ b/apps/cli/src/render/tui/home-app.tsx @@ -12,6 +12,8 @@ import { HomeView } from './home-view.js'; import type { ReverseSearchState } from './input-history.js'; import type { MentionState } from './mention.js'; import { MentionView } from './mention-view.js'; +import type { EffortPickerState } from './effort-picker.js'; +import { EffortTierList } from './effort-tier-list.js'; import type { ModelPickerState } from './model-picker.js'; import { ModelPickerView } from './model-picker-view.js'; import { PaletteView } from './palette-view.js'; @@ -49,6 +51,7 @@ function ChatRegion( search: ReverseSearchState | undefined; mention: MentionState | undefined; modelPicker: ModelPickerState | undefined; + effortPicker: EffortPickerState | undefined; nowMs: number; shellBusy: boolean; submitBusy: boolean; @@ -78,7 +81,8 @@ function ChatRegion( props.palette !== undefined || props.search !== undefined || props.mention !== undefined || - props.modelPicker !== undefined + props.modelPicker !== undefined || + props.effortPicker !== undefined } /> {props.palette !== undefined && ( @@ -92,6 +96,16 @@ function ChatRegion( {props.modelPicker !== undefined && ( )} + {/* The standalone `/effort` overlay in a live in-Home chat (ADR-0066 §6) — the shared tier list. */} + {props.effortPicker !== undefined && ( + + )} ); } @@ -115,6 +129,7 @@ export function RootApp(props: Readonly): ReactElement { search={state.search} mention={state.mention} modelPicker={state.modelPicker} + effortPicker={state.effortPicker} nowMs={props.nowMs()} shellBusy={state.shellBusy} submitBusy={state.submitBusy} diff --git a/apps/cli/src/render/tui/home-controller.test.ts b/apps/cli/src/render/tui/home-controller.test.ts index 52dffb90..75a62869 100644 --- a/apps/cli/src/render/tui/home-controller.test.ts +++ b/apps/cli/src/render/tui/home-controller.test.ts @@ -12,7 +12,7 @@ import { } from './home-controller.js'; import type { UserCommandOutcome } from '@relavium/core'; import type { ModelCatalogEntry } from '@relavium/llm'; -import type { ReasoningEffort } from '@relavium/shared'; +import { REASONING_EFFORTS, type ReasoningEffort } from '@relavium/shared'; import type { RefreshReport } from '../../engine/model-refresh.js'; @@ -2041,4 +2041,127 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => expect(sessionA.teardown).not.toHaveBeenCalled(); // the live session is untouched expect(c.getSnapshot().modelPicker?.hint).toContain('Already on'); // the picker stays open with a hint }); + + it('in-Home chat: /effort opens the interactive tier overlay; applying pushes the setter — NO reseat (ADR-0066 §6)', async () => { + // The standalone `/effort` overlay (distinct from the `/models` effort sub-step): a reasoning-capable live chat + // opens a fixed tier list on the bound effort; picking a new tier pushes the per-turn session override, never a + // reseat. Reached via the `/` palette (typing `/` opens it), matching how a user runs it. + const boundStore = createChatStore(false, { model: 'claude-opus-4-8' }); + boundStore.setReasoningEffort('low'); // the session's current tier — drives the overlay's ✓/opening highlight + const onSetEffort = vi.fn(); + const sessionA = makeSession({ sessionId: 'sess-A', store: boundStore, onSetEffort }); + const reseatChat = vi.fn(() => Promise.resolve(makeSession().session)); + const c = createHomeController({ + doctorProbes: STUB_DOCTOR_PROBES, + startChat: () => Promise.resolve(sessionA.session), + reseatChat, + homeStore, + onExit: vi.fn(), + onError: vi.fn(), + }); + type(c, 'hi'); + c.handleKey('', ENTER); + await flush(); + c.handleKey('/', {}); // open the `/` palette + type(c, 'effort'); // filter CHAT_PALETTE_COMMANDS → [/effort] + c.handleKey('', ENTER); // run /effort → the interactive overlay opens (NOT the model picker) + expect(c.getSnapshot().effortPicker?.current).toBe('low'); + expect(c.getSnapshot().effortPicker?.selected).toBe(REASONING_EFFORTS.indexOf('low')); + expect(c.getSnapshot().modelPicker).toBeUndefined(); + c.handleKey('', { downArrow: true }); // low → medium + c.handleKey('', { downArrow: true }); // medium → high + c.handleKey('', ENTER); // apply 'high' + + expect(onSetEffort).toHaveBeenCalledWith('high'); // the per-turn SESSION override + expect(reseatChat).not.toHaveBeenCalled(); // an effort change is NOT a reseat (ADR-0066 §5) + expect(sessionA.teardown).not.toHaveBeenCalled(); // the live session is untouched + expect(c.getSnapshot().effortPicker).toBeUndefined(); // the overlay closed after applying + }); + + it('in-Home chat: /effort on a NON-reasoning model does NOT open the overlay — it dispatches to the notice (ADR-0066 §6)', async () => { + // A non-reasoning bound model has no controllable tier, so `/effort` must fall through to the slash dispatch + // (the ctx handler prints "no controllable tier"), never opening a dead overlay. + const boundStore = createChatStore(false, { model: 'gpt-4o' }); // gpt-4o is not a reasoning model + const onSetEffort = vi.fn(); + const made = makeSession({ store: boundStore, onSetEffort }); + const c = createHomeController({ + doctorProbes: STUB_DOCTOR_PROBES, + startChat: () => Promise.resolve(made.session), + homeStore, + onExit: vi.fn(), + onError: vi.fn(), + }); + type(c, 'hi'); + c.handleKey('', ENTER); + await flush(); + c.handleKey('/', {}); + type(c, 'effort'); + c.handleKey('', ENTER); + await flush(); + + expect(c.getSnapshot().effortPicker).toBeUndefined(); // no overlay for a non-reasoning model + expect(made.lines).toContain('/effort'); // it dispatched (the ctx handler surfaces the "no tier" notice) + expect(onSetEffort).not.toHaveBeenCalled(); + }); + + it('in-Home chat: a typed (pasted) /effort line opens the overlay via applySubmitAction (ADR-0066 §6)', async () => { + // Typing `/` at an empty prompt opens the palette, so a LITERAL `/effort` line reaches applySubmitAction only via + // paste (bracketed paste appends verbatim). This covers the typed-intercept branch, distinct from the palette one. + const boundStore = createChatStore(false, { model: 'claude-opus-4-8' }); + boundStore.setReasoningEffort('medium'); + const onSetEffort = vi.fn(); + const sessionA = makeSession({ store: boundStore, onSetEffort }); + const c = createHomeController({ + doctorProbes: STUB_DOCTOR_PROBES, + startChat: () => Promise.resolve(sessionA.session), + homeStore, + onExit: vi.fn(), + onError: vi.fn(), + }); + type(c, 'hi'); + c.handleKey('', ENTER); + await flush(); + c.handleKey(PASTE_START, {}); + c.handleKey('/effort', {}); // pasted literally — the palette does NOT open (a paste bypasses the `/` intercept) + c.handleKey(PASTE_END, {}); + expect(c.getSnapshot().input.text).toBe('/effort'); + c.handleKey('', ENTER); // submit the literal line → applySubmitAction → the typed intercept opens the overlay + + expect(c.getSnapshot().effortPicker?.current).toBe('medium'); // opened on the bound tier + expect(c.getSnapshot().input.text).toBe(''); // the buffer was cleared + expect(sessionA.lines).not.toContain('/effort'); // it did NOT dispatch — it opened the overlay + }); + + it('in-Home chat: Esc closes the /effort overlay without applying; same-tier re-pick is a no-op (ADR-0066 §6)', async () => { + const boundStore = createChatStore(false, { model: 'claude-opus-4-8' }); + boundStore.setReasoningEffort('high'); // the overlay opens highlighted on 'high' + const onSetEffort = vi.fn(); + const sessionA = makeSession({ store: boundStore, onSetEffort }); + const c = createHomeController({ + doctorProbes: STUB_DOCTOR_PROBES, + startChat: () => Promise.resolve(sessionA.session), + homeStore, + onExit: vi.fn(), + onError: vi.fn(), + }); + type(c, 'hi'); + c.handleKey('', ENTER); + await flush(); + // Open via the palette, then Esc — closes the overlay without a setter call. + c.handleKey('/', {}); + type(c, 'effort'); + c.handleKey('', ENTER); + expect(c.getSnapshot().effortPicker).toBeDefined(); + c.handleKey('', { escape: true }); + expect(c.getSnapshot().effortPicker).toBeUndefined(); // Esc closed it + expect(onSetEffort).not.toHaveBeenCalled(); // …applying nothing + + // Re-open and Enter on the ALREADY-bound 'high' → a gentle no-op (no setter call), overlay closed. + c.handleKey('/', {}); + type(c, 'effort'); + c.handleKey('', ENTER); + c.handleKey('', ENTER); // Enter on the opening highlight (the bound 'high') + expect(onSetEffort).not.toHaveBeenCalled(); // same tier ⇒ no setter call + expect(c.getSnapshot().effortPicker).toBeUndefined(); // closed + }); }); diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index fcffd6a7..3e2161d3 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -8,6 +8,12 @@ import { type ReplCommandContext, } from '../../commands/repl-commands.js'; import type { RefreshReport } from '../../engine/model-refresh.js'; +import { + canControlEffort, + foldEffortPickerKey, + initialEffortPickerState, + type EffortPickerState, +} from './effort-picker.js'; import { foldModelPickerKey, partialFailureBanner, type ModelPickerState } from './model-picker.js'; import type { ReseatTarget } from '../../commands/chat.js'; import { nextMode, type ChatMode } from '../../chat/chat-mode.js'; @@ -165,6 +171,10 @@ export interface HomeControllerState { * action); a keyboard-owning overlay like the palette. Opened from the Home palette's `/models`; on selection it * writes the next session's default (ADR-0063), never rebinding the live session. Mutually exclusive with the palette. */ readonly modelPicker: ModelPickerState | undefined; + /** The open standalone `/effort` overlay (ADR-0066 §6) — `undefined` ⇒ closed. CHAT-scoped (a live in-Home chat + * with the effort setter wired + a reasoning-capable model); a keyboard-owning overlay like the mention/search + * submodes. On accept it pushes the per-turn session override via `active.onSetEffort` (no reseat). */ + readonly effortPicker: EffortPickerState | undefined; } /** @@ -244,6 +254,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { historyEntries: [], notice: undefined, modelPicker: undefined, + effortPicker: undefined, }; // Per-session command history for the in-Home chat (2.5.D step 3) — accumulates submitted lines across the Home // process; Up/Down recall, Ctrl+R reverse-searches. Not persisted (a chat-resume starts fresh). @@ -330,6 +341,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { palette: undefined, // a palette left open when /exit ran must not leak into the returned Home search: undefined, // ditto a reverse-search submode mention: undefined, // ditto an `@`-completion submode + effortPicker: undefined, // ditto the `/effort` overlay (chat-scoped — closed on return to the bare Home) modelPicker: undefined, // ditto the `/models` picker (Home-only, so never open here — reset for hygiene) shellBusy: false, // a `!`-command in flight when the chat ended must not leave the returned Home gated submitBusy: false, // ditto a submit/compaction in flight — the returned Home must not be left gated @@ -391,6 +403,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { search: undefined, mention: undefined, modelPicker: undefined, + effortPicker: undefined, shellBusy: false, submitBusy: false, // the swap is done — un-gate the fresh chat shellCommand: undefined, @@ -463,6 +476,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { search: undefined, mention: undefined, modelPicker: undefined, + effortPicker: undefined, shellBusy: false, submitBusy: false, // the swap is done — un-gate the reseated chat shellCommand: undefined, @@ -534,6 +548,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { search: undefined, mention: undefined, modelPicker: undefined, + effortPicker: undefined, historyEntries: history.entries, }); // Track the in-flight build so a signal (or a mid-build exit) during `loading` can reclaim its just-spawned @@ -808,6 +823,46 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { return true; }; + // ---- The in-Home `/effort` overlay (ADR-0066 §6) — interactive tier selection (no reseat) ------------------- + // Whether `/effort` should open the overlay: a live chat with the setter wired AND a reasoning-capable bound model. + // A non-reasoning model returns false, so the surface falls through to the `/effort` notice (parity with ChatApp). + const canOpenEffortPicker = (active: HomeChatSession): boolean => + canControlEffort(active.store.getSnapshot().state.model, active.onSetEffort !== undefined); + // Open on the LIVE bound model + the LIVE store tier (so it opens on the currently-bound effort). Callers gate on + // `canOpenEffortPicker`, so the model is present here (the guard is defensive). + const openEffortPicker = (active: HomeChatSession): void => { + const snap = active.store.getSnapshot(); + if (snap.state.model === undefined) return; + set({ effortPicker: initialEffortPickerState(snap.state.model, snap.reasoningEffort) }); + }; + // The open effort overlay owns every key (mirrors routeModelPickerKey). Accept applies the tier via the session's + // per-turn setter (no reseat); a re-pick of the same tier is a gentle no-op with visible store feedback. + const routeEffortPickerKey = (active: HomeChatSession, input: string, key: ChatKey): boolean => { + const open = state.effortPicker; + if (open === undefined) return false; + const step = foldEffortPickerKey(input, key, open); + switch (step.kind) { + case 'close': + set({ effortPicker: undefined }); + break; + case 'accept': + set({ effortPicker: undefined }); + if (step.effort === open.current) { + active.store.note(`Already at reasoning effort ${step.effort}.`); + } else { + active.onSetEffort?.(step.effort); + active.store.note( + `Reasoning effort set to ${step.effort} — applies to your next message.`, + ); + } + break; + case 'state': + set({ effortPicker: step.state }); + break; + } + return true; + }; + // Drive the open `/` palette (2.5.C S3b): fold the keystroke, then apply — keep open with new state, run the // highlighted command by submitting its slash line through the SAME chat dispatch, or close. Ctrl-C closes it // (a gentle escape back to the prompt — never trapping the user). @@ -885,6 +940,10 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { // chat: `/models` opens the reseat picker (ADR-0059) — parity with the typed-`/models` intercept, so the // palette route + the typed route behave identically (never the "interactive terminal" dispatch hint). openModelPicker(); + } else if (step.command.name === 'effort' && canOpenEffortPicker(active)) { + // chat: bare `/effort` on a reasoning-capable model opens the interactive overlay (ADR-0066 §6) — parity + // with the typed-`/effort` intercept; a non-reasoning model falls through to the notice below. + openEffortPicker(active); } else { sendChatLine(active, `/${step.command.name}`); // chat: reuse the S3a slash dispatch (createChatLineHandler) } @@ -1143,6 +1202,15 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { openModelPicker(); return; } + // Bare `/effort` on a reasoning-capable model opens the interactive tier overlay (ADR-0066 §6) instead of the + // informational notice — parity with the standalone ChatApp. A non-reasoning model falls through to the slash + // dispatch (the ctx handler's "no controllable tier" notice). `/effort ` (with an arg) is not intercepted. + if (trimmed === '/effort' && canOpenEffortPicker(active)) { + history = recordHistory(history, line); + set({ input: emptyEditor(), historyEntries: history.entries }); + openEffortPicker(active); + return; + } if (trimmed.startsWith('/') || state.attachments.length === 0) { // a slash command, or a plain message with no attachments — the simple path history = recordHistory(history, line); @@ -1312,7 +1380,8 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { state.palette === undefined && state.search === undefined && state.mention === undefined && - state.modelPicker === undefined; // a paste while the picker owns the keyboard must not leak into the buffer + state.modelPicker === undefined && // a paste while the picker owns the keyboard must not leak into the buffer + state.effortPicker === undefined; // ditto the `/effort` overlay const pasted = input.replace(/\r\n?/g, '\n'); if (pasted.length > 0 && editable) { // Match the typed-edit path: appending clears any stale `/doctor` report + invalidates an in-flight run. @@ -1328,12 +1397,21 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { handlePaletteKey(input, key); return; } - // The `/models` picker (2.5.G S7) likewise owns every key while open — a Home-only overlay, mutually exclusive - // with the palette (the palette closes before running `/models`), so this can be reached only in `mode: 'home'`. + // The `/models` picker (2.5.G S7) likewise owns every key while open — mutually exclusive with the palette + // (the palette closes before running `/models`). Opened from the bare-Home palette (next-session default, + // ADR-0063) OR a live in-Home chat (a typed/palette `/models` → the reseat picker, ADR-0059), so it is routed + // before the mode branches rather than assuming a single mode. if (state.modelPicker !== undefined) { routeModelPickerKey(input, key); return; } + // The `/effort` overlay (ADR-0066 §6) likewise owns every key while open — chat-scoped, so it always has a + // live session; the guard keeps `routeEffortPickerKey`'s `active` non-null (a stale overlay with no session, + // never expected, falls through instead of driving a torn-down session). + if (state.effortPicker !== undefined && state.session !== undefined) { + routeEffortPickerKey(state.session, input, key); + return; + } if (state.mode === 'chat' && state.session !== undefined) { handleChatKey(state.session, input, key); return; diff --git a/apps/cli/src/render/tui/model-picker-view.tsx b/apps/cli/src/render/tui/model-picker-view.tsx index 8623ada6..91f17f80 100644 --- a/apps/cli/src/render/tui/model-picker-view.tsx +++ b/apps/cli/src/render/tui/model-picker-view.tsx @@ -2,9 +2,9 @@ import { Box, Text } from 'ink'; import type { ReactElement, ReactNode } from 'react'; import type { ModelCatalogEntry } from '@relavium/llm'; -import { EFFORT_TIER_HINT, REASONING_EFFORTS } from '@relavium/shared'; import { sanitizeInline } from './chat-projection.js'; +import { EffortTierList } from './effort-tier-list.js'; import { formatContextWindow, formatModelPrice, @@ -65,37 +65,20 @@ function rowColorFor( } /** - * The `'effort'` sub-list (ADR-0066) — the reasoning-effort tiers for the model chosen in the `'model'` phase. A - * fixed five-row list (no scroll window needed): each tier + its one-line hint, the highlighted row in cyan, a `✓` - * on the session's currently-bound effort. The `pending` model's name (a provider-controlled string) is sanitized - * at this display boundary, exactly as the model rows are. + * The `'effort'` sub-list (ADR-0066) — the reasoning-effort tiers for the model chosen in the `'model'` phase. + * Delegates to the shared {@link EffortTierList} (one canonical presentation, also used by the standalone `/effort` + * overlay); here `Esc` backs OUT to the model list (hence the "Esc back" footer), not a cancel. */ function EffortSubList(props: Readonly<{ state: ModelPickerState; color: boolean }>): ReactElement { const { state, color } = props; - const selected = Math.max(0, Math.min(state.effortSelected, REASONING_EFFORTS.length - 1)); - const forModel = - state.pending === undefined ? '' : ` · ${sanitizeInline(state.pending.displayName)}`; return ( - - - Reasoning effort - {forModel} - - {REASONING_EFFORTS.map((effort, index) => { - const isSelected = index === selected; - const isCurrent = effort === state.currentEffort; - const rowColor = isSelected ? colorProps(color, 'cyan') : {}; - return ( - - {`${isSelected ? '›' : ' '} ${isCurrent ? '✓' : ' '} ${effort} · `} - {EFFORT_TIER_HINT[effort]} - - ); - })} - - ↑/↓ select · Enter apply · Esc back - - + ); } diff --git a/docs/decisions/0066-normalized-reasoning-effort-control.md b/docs/decisions/0066-normalized-reasoning-effort-control.md index 9ffeeca3..ed6d60bc 100644 --- a/docs/decisions/0066-normalized-reasoning-effort-control.md +++ b/docs/decisions/0066-normalized-reasoning-effort-control.md @@ -7,6 +7,8 @@ > Accepted 2026-07-06 (the 2.5.G "reasoning-effort" workstream, maintainer question #1). Two independent ADR reviews caught that the original draft's "OpenAI = tier, Anthropic/Gemini = token budget" split was **stale**: all four currently-targeted providers are **tier-native**, and the shipped adapters already merge `{...providerOptions, ...body}` (canonical wins). The Decision below is rewritten around that — a strictly *simpler* design (one tier → each provider's native tier), with the token-budget derivation demoted to a documented legacy fallback. > > **Note (2026-07-07): DeepSeek wired + its native shape refined (§2/§4).** DeepSeek was initially DEFERRED (the OpenAI-compatible adapter had no pinned SDK to verify the param against). Its create-chat-completion API was then verified against the official docs (api-docs.deepseek.com): v4 takes a **`thinking` object** — `type: 'enabled' | 'disabled'` + `reasoning_effort: 'high' | 'max'` — richer than §2's assumed "on/off only". The adapter now maps `off` → `{ type: 'disabled' }`, `low`/`medium`/`high` → `{ type: 'enabled', reasoning_effort: 'high' }` (DeepSeek's minimum thinking level), and `max` → `{ type: 'enabled', reasoning_effort: 'max' }` (an honest coarsening onto v4's two graded levels); `deepseek-v4-flash`/`-pro` are `reasoning: true`, the §4 heuristic covers `deepseek-v[4-9]`, and the legacy non-thinking `deepseek-chat` alias stays uncontrollable. So the §2 DeepSeek row's "on/off, no graded tiers" — and the matching on/off phrasing in Consequences → Negative — are both superseded by the doc-verified `thinking` shape here (off / high / max); the reasoning **output** (`reasoning_content`) + token accounting were already mapped. +> +> **Note (2026-07-07): interactive `/effort` overlay realized (§6).** §6 anticipated "a future `/effort`" for standalone effort changes; it is now shipped as a first-class **interactive tier-selector overlay** (not just a typed `/effort ` — that still works). Bare `/effort` (typed or selected from the `/` palette) opens a keyboard-owning overlay — a fixed off/low/medium/high/max list, opening on the bound tier, arrow+Enter to apply — in BOTH `relavium chat` and the in-Home live chat, sharing one pure fold ([effort-picker.ts](../../apps/cli/src/render/tui/effort-picker.ts)) and one view ([effort-tier-list.tsx](../../apps/cli/src/render/tui/effort-tier-list.tsx), also used by the `/models` effort sub-step). Applying calls the §5 per-turn `onSetEffort` — **no reseat**. It opens ONLY on a reasoning-capable bound model; a non-reasoning model falls through to the informational notice (` has no controllable reasoning tier`). This realizes §6's "standalone effort changes ride the same session override" — from a promise to a shipped surface. (§6's separate point, that the bare-Home picker writes only the config default and not a live session's effort, is unchanged here — a follow-up gives that picker its own effort sub-step.) ## Context From a58f63518b4302388c7bcc53c0c36ca9e257ded2 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 7 Jul 2026 12:06:32 +0300 Subject: [PATCH 67/68] =?UTF-8?q?feat(cli,shared):=20bare-Home=20/models?= =?UTF-8?q?=20effort=20sub-step=20writes=20the=20effort=20default=20(ADR-0?= =?UTF-8?q?066=20=C2=A76)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Symptom 1: picking a model in the bare Home offered no effort selection. By design the Home /models write persisted only the model (ADR-0063) — but [chat].reasoning_effort had no global home to write to. This adds one. - shared: a new global `[preferences].reasoning_effort` key (the effort counterpart of `default_model`) on GlobalConfigSchema. - config: `resolveChat` falls back to it below the [chat] layers — the exact precedence `default_model` already has; the writer generalizes to `writeGlobalPreferences({ defaultModel?, reasoningEffort? })` (writes one or both in ONE atomic, schema-round-tripped write; a partial write leaves the other key unchanged), with `writeGlobalDefaultModel` now a thin wrapper. - Home picker: a reasoning-capable model advances to the effort sub-step (on both surfaces now); the bare-Home accept writes model + effort defaults, the live chat still sets the per-turn override (no reseat). The sub-list opens on the config effort default (a new `HomeModelsPort.currentEffort`). - tests: resolve global-fallback, writeGlobalPreferences (effort-only / combined / partial-merge / invalid-tier), the bare-Home reasoning-model sub-step, and the updated single-phase (non-reasoning) + drive-home paths. - docs: ADR-0063 + ADR-0066 §6 append-only notes + config-spec.md (the new key, its resolution precedence, and the two-phase Home write). Refs: ADR-0066, ADR-0063 Co-Authored-By: Claude --- apps/cli/src/config/resolve.test.ts | 9 ++- apps/cli/src/config/resolve.ts | 15 +++-- apps/cli/src/config/write.test.ts | 48 +++++++++++++- apps/cli/src/config/write.ts | 60 ++++++++++++----- apps/cli/src/home/drive-home.test.ts | 21 +++++- apps/cli/src/home/drive-home.tsx | 43 ++++++++---- .../src/render/tui/home-controller.test.ts | 66 ++++++++++++++++++- apps/cli/src/render/tui/home-controller.ts | 60 +++++++++++------ apps/cli/src/render/tui/model-picker.ts | 9 +-- .../0063-cli-config-write-contract.md | 2 + ...066-normalized-reasoning-effort-control.md | 2 + docs/reference/cli/commands.md | 2 +- docs/reference/contracts/config-spec.md | 14 ++-- packages/shared/src/config.ts | 4 ++ 14 files changed, 283 insertions(+), 72 deletions(-) diff --git a/apps/cli/src/config/resolve.test.ts b/apps/cli/src/config/resolve.test.ts index 34ec1cfc..60ff3732 100644 --- a/apps/cli/src/config/resolve.test.ts +++ b/apps/cli/src/config/resolve.test.ts @@ -74,7 +74,7 @@ describe('resolveConfig', () => { expect(resolved.onExceed).toBe('warn'); // only on project }); - it('resolves [chat].reasoning_effort (ADR-0066) last-writer-wins project > workspace, no global fallback', () => { + it('resolves [chat].reasoning_effort (ADR-0066) last-writer-wins project > workspace > global [preferences]', () => { const workspace: ProjectConfig = { chat: { reasoning_effort: 'low' } }; const project: ProjectConfig = { chat: { reasoning_effort: 'high' } }; expect(resolveConfig({ workspace, project }).chat.reasoningEffort).toBe('high'); // project wins @@ -82,7 +82,12 @@ describe('resolveConfig', () => { expect( resolveConfig({ workspace, project: { chat: { max_turns: 5 } } }).chat.reasoningEffort, ).toBe('low'); - // Absent everywhere ⇒ undefined (no global-layer fallback — that extra fallback is default_model's alone). + // ADR-0066 §6: like `default_model`, it now ALSO falls back to the GLOBAL `[preferences].reasoning_effort` + // (the `/models` picker effort sub-step's write target) — BELOW any project/workspace `[chat]` override. + const global = { preferences: { reasoning_effort: 'max' as const } }; + expect(resolveConfig({ global }).chat.reasoningEffort).toBe('max'); // global fallback applies… + expect(resolveConfig({ global, workspace }).chat.reasoningEffort).toBe('low'); // …but a workspace/project wins + // Absent at EVERY layer ⇒ undefined (the provider default; no reasoning control sent). expect(resolveConfig({}).chat.reasoningEffort).toBeUndefined(); expect( resolveConfig({ global: { preferences: { default_model: 'g' } } }).chat.reasoningEffort, diff --git a/apps/cli/src/config/resolve.ts b/apps/cli/src/config/resolve.ts index 255deffd..ad8dedce 100644 --- a/apps/cli/src/config/resolve.ts +++ b/apps/cli/src/config/resolve.ts @@ -110,12 +110,12 @@ function resolveGraceMs( /** * Resolve the `[chat]` block (last-writer-wins: project → workspace). Most fields are project/workspace-scoped - * only; `default_model` additionally falls back to the GLOBAL `[preferences].default_model` - * ([ADR-0063](../../../../docs/decisions/0063-cli-config-write-contract.md)) — the write target of `/models` and - * the onboarding wizard — so a user's "preferred model everywhere" applies to chat when no project/workspace - * `[chat].default_model` overrides it, mirroring how the workflow default (`resolveConfig.defaultModel`) already - * reads `[preferences].default_model`. Absent at every layer ⇒ all-`undefined` fields, so the chat host falls - * back to its engine defaults (e.g. `maxTurns` ⇒ `SessionDeps`'s built-in 50). + * only; `default_model` AND `reasoning_effort` additionally fall back to their GLOBAL `[preferences]` counterparts + * ([ADR-0063](../../../../docs/decisions/0063-cli-config-write-contract.md) · ADR-0066 §6) — the write targets of + * the `/models` picker (model + its effort sub-step) and the onboarding wizard — so a user's "preferred model / + * effort everywhere" applies to chat when no project/workspace `[chat]` override wins, mirroring how the workflow + * default (`resolveConfig.defaultModel`) already reads `[preferences].default_model`. Absent at every layer ⇒ + * all-`undefined` fields, so the chat host falls back to its engine defaults (e.g. `maxTurns` ⇒ `SessionDeps`'s 50). */ function resolveChat( project: ProjectConfig | undefined, @@ -143,7 +143,8 @@ function resolveChat( onExceed: p?.on_exceed ?? w?.on_exceed, allowedCommands: projectSetsAllowlist ? p?.allowed_commands : w?.allowed_commands, allowedCommandGlobs: projectSetsAllowlist ? p?.allowed_command_globs : w?.allowed_command_globs, - reasoningEffort: p?.reasoning_effort ?? w?.reasoning_effort, + reasoningEffort: + p?.reasoning_effort ?? w?.reasoning_effort ?? global?.preferences?.reasoning_effort, }; } diff --git a/apps/cli/src/config/write.test.ts b/apps/cli/src/config/write.test.ts index b0c63204..e59688c1 100644 --- a/apps/cli/src/config/write.test.ts +++ b/apps/cli/src/config/write.test.ts @@ -16,7 +16,12 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest'; import { ConfigError } from './errors.js'; import { loadConfigFile } from './load.js'; -import { globalConfigPath, writeFileAtomic, writeGlobalDefaultModel } from './write.js'; +import { + globalConfigPath, + writeFileAtomic, + writeGlobalDefaultModel, + writeGlobalPreferences, +} from './write.js'; /** Read the global config back through the SAME validating loader the rest of the CLI uses. */ function readBack(home: string): GlobalConfig | undefined { @@ -225,6 +230,47 @@ describe('writeGlobalDefaultModel', () => { }); }); +describe('writeGlobalPreferences (ADR-0066 §6 — the /models effort sub-step write)', () => { + let home: string; + beforeEach(() => { + home = mkdtempSync(join(tmpdir(), 'relavium-write-')); + }); + afterEach(() => { + rmSync(home, { recursive: true, force: true }); + }); + + it('writes [preferences].reasoning_effort alone (round-trip)', () => { + writeGlobalPreferences({ reasoningEffort: 'high' }, home); + expect(readBack(home)).toEqual({ preferences: { reasoning_effort: 'high' } }); + }); + + it('writes default_model AND reasoning_effort together in one atomic write', () => { + writeGlobalPreferences({ defaultModel: 'deepseek-v4-flash', reasoningEffort: 'max' }, home); + expect(readBack(home)).toEqual({ + preferences: { default_model: 'deepseek-v4-flash', reasoning_effort: 'max' }, + }); + }); + + it('a model-only write PRESERVES an existing reasoning_effort (partial merge — an absent field is unchanged)', () => { + // Seed a prior effort default, then write only the model — the effort must survive (the ADR-0066 §6 partial-merge + // guarantee, so setting a model in the picker never silently clears the user's effort preference). + writeGlobalPreferences({ reasoningEffort: 'low' }, home); + writeGlobalPreferences({ defaultModel: 'gpt-4o' }, home); + expect(readBack(home)).toEqual({ + preferences: { reasoning_effort: 'low', default_model: 'gpt-4o' }, + }); + }); + + it('an invalid tier is rejected by the schema round-trip (value-free ConfigError), file untouched', () => { + // The typed setter only accepts a `ReasoningEffort`, but a cast-through would fail the strict re-validation + // rather than reach disk — pin that the schema is the guard (ADR-0063 §3), not the caller's discipline. + expect(() => writeGlobalPreferences({ reasoningEffort: 'ludicrous' as never }, home)).toThrow( + ConfigError, + ); + expect(existsSync(globalConfigPath(home))).toBe(false); // nothing written on a rejected value + }); +}); + describe('globalConfigPath', () => { it('is ~/.relavium/config.toml under the given home', () => { const home = '/some/home'; diff --git a/apps/cli/src/config/write.ts b/apps/cli/src/config/write.ts index 4664e7e8..89199ad8 100644 --- a/apps/cli/src/config/write.ts +++ b/apps/cli/src/config/write.ts @@ -11,7 +11,7 @@ import { import { homedir } from 'node:os'; import { dirname, join } from 'node:path'; -import { GlobalConfigSchema, type GlobalConfig } from '@relavium/shared'; +import { GlobalConfigSchema, type GlobalConfig, type ReasoningEffort } from '@relavium/shared'; import { parse as parseToml, stringify as stringifyToml } from 'smol-toml'; import type { ZodError } from 'zod'; @@ -21,14 +21,15 @@ import { ensureGlobalConfigDir, globalConfigDir } from './paths.js'; /** * The **first on-disk config WRITER** ([ADR-0063](../../../../docs/decisions/0063-cli-config-write-contract.md)) — - * the sibling of the read-only [load.ts](./load.ts). It persists a user's chosen chat default model to the - * **global** `~/.relavium/config.toml` `[preferences].default_model`, the write target for `/models` and the - * onboarding wizard. Every later writer inherits this primitive, so its guarantees are enforced by construction, - * not by convention. + * the sibling of the read-only [load.ts](./load.ts). It persists a user's chosen chat defaults to the **global** + * `~/.relavium/config.toml` `[preferences]` block — `default_model` and (ADR-0066 §6) `reasoning_effort` — the + * write targets for the `/models` picker (model + its effort sub-step) and the onboarding wizard. Every later writer + * inherits this primitive, so its guarantees are enforced by construction, not by convention. * * Four load-bearing guarantees (the ADR-0063 contract; a security-reviewed surface): - * 1. **Secret-free by construction.** The surface is a **typed setter** ({@link writeGlobalDefaultModel}) — never - * a generic `writeKey(k, v)` — so it can only ever set `default_model` (a non-secret). There is no API-key + * 1. **Secret-free by construction.** The surface is a **typed setter** ({@link writeGlobalPreferences}, of which + * {@link writeGlobalDefaultModel} is a thin single-key wrapper) — never a generic `writeKey(k, v)` — so it can + * only ever set the two non-secret `[preferences]` keys `default_model` / `reasoning_effort`. There is no API-key * field in the schema to write to (keys live only in the OS keychain, ADR-0006). And a schema-validation * failure on the write path is reported through the **same value-free formatter** the loader uses * ({@link formatZodError}) — never a raw `ZodError`, whose `.message` embeds the received value for several @@ -58,11 +59,20 @@ import { ensureGlobalConfigDir, globalConfigDir } from './paths.js'; * file). Accepted for a single-user, rarely-written preference store; a future multi-writer path would need a lock. */ +/** The typed set of global `[preferences]` a write MAY touch (ADR-0063 · ADR-0066 §6). Only non-secret keys — + * the typed setter can never reach an API-key field (there is none in the schema; keys live in the keychain). A + * field absent from the object is left UNCHANGED (a partial merge), so a model-only write never clears the effort. */ +export interface GlobalPreferenceWrite { + readonly defaultModel?: string; + readonly reasoningEffort?: ReasoningEffort; +} + /** - * Set the global `[preferences].default_model`, preserving every other config key. Reads + validates the existing - * config (an absent file ⇒ a fresh `{}`; a **malformed/invalid** existing config throws a {@link ConfigError} - * rather than clobbering a file the user must fix), merges only `default_model`, re-validates, verifies the - * serialized text round-trips, and writes atomically. `home` is injectable for tests. + * Set one or more global `[preferences]` keys (`default_model` and/or `reasoning_effort`), preserving every other + * config key. Reads + validates the existing config (an absent file ⇒ a fresh `{}`; a **malformed/invalid** existing + * config throws a {@link ConfigError} rather than clobbering a file the user must fix), merges only the PROVIDED + * keys (an absent field is left unchanged), re-validates, verifies the serialized text round-trips, and writes + * atomically. `home` is injectable for tests. * * `targetPath` (the CLI `--config` override) writes to that **exact** file — the SAME file `loadResolvedConfig` * treats as "global" under `--config` — so a `/models` write, the picker's re-read, and the started chat session @@ -71,11 +81,14 @@ import { ensureGlobalConfigDir, globalConfigDir } from './paths.js'; * same-filesystem rename and is `0600` by construction, the schema round-trip runs, and the typed setter stays * secret-incapable (config holds no secrets, so an arbitrary `--config` dir's own mode is immaterial at rest). */ -export function writeGlobalDefaultModel( - model: string, +export function writeGlobalPreferences( + prefs: GlobalPreferenceWrite, home: string = homedir(), targetPath?: string, ): void { + // An all-absent write is a no-op: touch nothing rather than emit an empty `[preferences]` table where none + // existed. Unreachable from the current callers (each passes ≥1 key), but keeps the typed setter footgun-free. + if (prefs.defaultModel === undefined && prefs.reasoningEffort === undefined) return; let target: string; let dir: string; if (targetPath === undefined) { @@ -98,11 +111,16 @@ export function writeGlobalDefaultModel( } // Read the EXISTING config through the same validating loader (so we merge onto known-good data and preserve - // update_channel / mcp_servers / preferences.theme). An absent file is a fresh object; an invalid one throws. + // update_channel / mcp_servers / preferences.theme + the OTHER preference). An absent file is a fresh object; an + // invalid one throws. Only the provided keys override — a model-only write keeps a prior `reasoning_effort`. const existing = loadConfigFile(target, GlobalConfigSchema) ?? {}; const merged: GlobalConfig = { ...existing, - preferences: { ...existing.preferences, default_model: model }, + preferences: { + ...existing.preferences, + ...(prefs.defaultModel === undefined ? {} : { default_model: prefs.defaultModel }), + ...(prefs.reasoningEffort === undefined ? {} : { reasoning_effort: prefs.reasoningEffort }), + }, }; // Re-validate the whole object so the emission is provably schema-valid (ADR-0033) — a `.strict()` round-trip @@ -117,6 +135,16 @@ export function writeGlobalDefaultModel( writeFileAtomic(dir, target, text); } +/** Set the global `[preferences].default_model` (the common single-key write) — a thin wrapper over + * {@link writeGlobalPreferences}. Kept as the named setter the onboarding wizard + the non-effort callers use. */ +export function writeGlobalDefaultModel( + model: string, + home: string = homedir(), + targetPath?: string, +): void { + writeGlobalPreferences({ defaultModel: model }, home, targetPath); +} + /** Schema-validate the object to write, mapping a failure to the value-free {@link ConfigError} path. */ function validateForWrite(value: GlobalConfig, target: string): GlobalConfig { const result = GlobalConfigSchema.safeParse(value); @@ -159,7 +187,7 @@ function configWriteError(target: string, error: ZodError): ConfigError { * Exported ONLY as a test seam — for the fault-injection test that drives the catch path with a real failing * rename (a `target` that is an existing directory ⇒ `EISDIR`), verifying the temp is cleaned up and the fd is * not leaked. It is NOT part of the writer's public contract: it is a generic "replace this path with this text" - * primitive with no schema, so the secret-incapability guarantee is a property of {@link writeGlobalDefaultModel} + * primitive with no schema, so the secret-incapability guarantee is a property of {@link writeGlobalPreferences} * (the typed setter), never of this helper. Do not call it to write arbitrary content. */ export function writeFileAtomic(dir: string, target: string, text: string): void { diff --git a/apps/cli/src/home/drive-home.test.ts b/apps/cli/src/home/drive-home.test.ts index 4e95a82d..76342159 100644 --- a/apps/cli/src/home/drive-home.test.ts +++ b/apps/cli/src/home/drive-home.test.ts @@ -4,6 +4,7 @@ import { join } from 'node:path'; import { Readable } from 'node:stream'; import { createClient, createSessionStore, runMigrations, type DbClient } from '@relavium/db'; +import { REASONING_EFFORTS, type ReasoningEffort } from '@relavium/shared'; import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { buildChatSession } from '../chat/session-host.js'; @@ -232,11 +233,13 @@ describe('driveHome (2.5.B / ADR-0054)', () => { const configFile = join(cwd, 'home-config.toml'); // a real, writable global config (the write + re-read target) let captured: RootAppProps | undefined; const builtDefaults: Array = []; + const builtEfforts: Array = []; const { deps } = makeDeps((p) => (captured = p), { global: { ...global, configPath: configFile }, - // Record the default the session is built with, then delegate to the real builder (so the chat still works). + // Record the model + effort the session is built with, then delegate to the real builder (so the chat works). buildSession: (args) => { builtDefaults.push(args.chat.defaultModel); + builtEfforts.push(args.chat.reasoningEffort); return buildChatSession(args); }, }); @@ -256,7 +259,16 @@ describe('driveHome (2.5.B / ADR-0054)', () => { const chosen = picker.entries[picker.selected]?.modelId; // the highlighted model (filter empty ⇒ entries[0]) if (chosen === undefined) throw new Error('no selected model'); - props.controller.handleKey('', ENTER); // accept ⇒ writeGlobalDefaultModel(chosen) to configFile + props.controller.handleKey('', ENTER); // accept the model + // ADR-0066 §6: a reasoning-capable pick advances to the bare-Home effort sub-step (writes model + effort); a + // non-reasoning one writes + closes directly. Accept the (default) effort too when the sub-step opened, and + // capture the tier it will write (the opening highlight) so the next-chat re-read can be asserted. + let writtenEffort: ReasoningEffort | undefined; + const effortStep = props.controller.getSnapshot().modelPicker; + if (effortStep?.phase === 'effort') { + writtenEffort = REASONING_EFFORTS[effortStep.effortSelected]; // the tier the opening highlight accepts + props.controller.handleKey('', ENTER); // accept the default effort ⇒ writeGlobalPreferences(chosen, effort) + } expect(props.controller.getSnapshot().modelPicker).toBeUndefined(); // picker closed on accept expect(props.controller.getSnapshot().notice).toContain('next chat session'); // honest success (write took effect) @@ -266,6 +278,11 @@ describe('driveHome (2.5.B / ADR-0054)', () => { await flush(); expect(props.controller.getSnapshot().mode).toBe('chat'); expect(builtDefaults.at(-1)).toBe(chosen); // the regression assertion — a stale snapshot would NOT equal `chosen` + // ADR-0066 §6 regression: the effort write (not only the model) must ALSO flow to the next chat in the SAME Home + // process — a stale `config.chat.reasoningEffort` snapshot would silently drop the just-written effort default. + if (writtenEffort !== undefined) { + expect(builtEfforts.at(-1)).toBe(writtenEffort); + } props.controller.handleKey('c', CTRL_C); // chat Ctrl-C ⇒ /cancel ⇒ back to Home await flush(); diff --git a/apps/cli/src/home/drive-home.tsx b/apps/cli/src/home/drive-home.tsx index 9bf62125..ddc669cd 100644 --- a/apps/cli/src/home/drive-home.tsx +++ b/apps/cli/src/home/drive-home.tsx @@ -1,7 +1,7 @@ import { randomUUID } from 'node:crypto'; import { createProviderStore, createRunHistoryReader } from '@relavium/db'; -import type { AgentSessionRecord } from '@relavium/shared'; +import type { AgentSessionRecord, ReasoningEffort } from '@relavium/shared'; import { render } from 'ink'; import { createElement } from 'react'; @@ -20,7 +20,7 @@ import { type SessionPersister, } from '../chat/persister.js'; import { loadResolvedConfig } from '../config/load.js'; -import { writeGlobalDefaultModel } from '../config/write.js'; +import { writeGlobalDefaultModel, writeGlobalPreferences } from '../config/write.js'; import { createModelCatalogPort } from '../engine/model-catalog-port.js'; import { readUserPricingOverlay } from '../engine/pricing-overlay.js'; import { assembleToolEnv } from '../engine/tool-host/assemble.js'; @@ -177,26 +177,42 @@ export async function driveHome(deps: HomeDeps): Promise { // `config` snapshot) so it reflects a same-session `/models` write AND a project/workspace override AND an edit // from another terminal; a config edited to malformed mid-session degrades to `undefined` rather than crashing // the picker. Cheap (a few small file reads) and only called on a picker open / accept, never per keystroke. - const readEffectiveDefault = (): string | undefined => { + // The EFFECTIVE `[chat]` block (project → workspace → global) — the shared read LOGIC (not a cached instance) + // behind the model + effort readers, so a bare-Home picker open does up to two cheap reads (currentDefault + + // currentEffort), never per-keystroke. A mid-session malformed config degrades to `undefined`, never a crash. + const readEffectiveChat = (): + | ReturnType['config']['chat'] + | undefined => { try { return loadResolvedConfig({ cwd: deps.global.cwd, home: homeDir, ...(deps.global.configPath === undefined ? {} : { configPath: deps.global.configPath }), - }).config.chat.defaultModel; + }).config.chat; } catch { return undefined; // a mid-session malformed config must not crash the picker } }; + const readEffectiveDefault = (): string | undefined => readEffectiveChat()?.defaultModel; + const readEffectiveEffort = (): ReasoningEffort | undefined => + readEffectiveChat()?.reasoningEffort; // The `/models` catalog port (ADR-0064 §10) — the SHARED load/refresh + key-aware merge trio (the SAME one the // chat reseat picker uses, ADR-0059), over the ONE open db + the store-aware resolver. The Home layers its own - // accept action on top: `currentDefault` (the ✓ marker) + `writeDefault` (the next-session default, ADR-0063 §1). + // accept action on top: `currentDefault`/`currentEffort` (the ✓ markers) + `writeDefault` (the next-session + // default model + its effort tier, ADR-0063 §1 · ADR-0066 §6). const models: HomeModelsPort = { ...createModelCatalogPort({ db: opened.db, providers, now, uuid }), currentDefault: readEffectiveDefault, + currentEffort: readEffectiveEffort, // Write to the SAME file the picker re-reads + the started session resolves (honors `--config`), so a `/models` - // write is never a silent no-op to a different file (2.5.G S7). - writeDefault: (modelId) => writeGlobalDefaultModel(modelId, homeDir, deps.global.configPath), + // write is never a silent no-op to a different file (2.5.G S7). The effort rides the SAME atomic write; an + // absent `reasoningEffort` (a non-reasoning model) leaves any prior effort default unchanged. + writeDefault: (modelId, reasoningEffort) => + writeGlobalPreferences( + { defaultModel: modelId, ...(reasoningEffort === undefined ? {} : { reasoningEffort }) }, + homeDir, + deps.global.configPath, + ), }; // The SHARED HomeChatSession wiring over a built (FRESH or RESUMED) session + its view store — the persister, @@ -327,10 +343,15 @@ export async function driveHome(deps: HomeDeps): Promise { // on a read fault). Static `MODEL_PRICING` still wins. const resolvePrice = readUserPricingOverlay(opened.db); const built: BuiltChatSession = await (deps.buildSession ?? buildChatSession)({ - // Re-read the EFFECTIVE default model FRESH per chat (not the load-once `config` snapshot) so a same-session - // `/models` write takes effect on the very next chat started in this long-lived Home (2.5.G S7). A read - // fault degrades to the startup value. Other `[chat]` settings keep the startup snapshot. - chat: { ...config.chat, defaultModel: readEffectiveDefault() ?? config.chat.defaultModel }, + // Re-read the EFFECTIVE default model AND reasoning-effort FRESH per chat (not the load-once `config` + // snapshot) so a same-session `/models` write — the model (2.5.G S7) AND its effort sub-step (ADR-0066 §6) — + // takes effect on the very next chat started in this long-lived Home. A read fault degrades to the startup + // value. The other `[chat]` settings keep the startup snapshot. + chat: { + ...config.chat, + defaultModel: readEffectiveDefault() ?? config.chat.defaultModel, + reasoningEffort: readEffectiveEffort() ?? config.chat.reasoningEffort, + }, agentRef: undefined, // the built-in default agent (zero-config first run) cwd: deps.global.cwd, projectConfigDir, diff --git a/apps/cli/src/render/tui/home-controller.test.ts b/apps/cli/src/render/tui/home-controller.test.ts index 75a62869..bc7940ed 100644 --- a/apps/cli/src/render/tui/home-controller.test.ts +++ b/apps/cli/src/render/tui/home-controller.test.ts @@ -1619,6 +1619,7 @@ function makeModelsPort( entries?: readonly ModelCatalogEntry[]; refreshedAt?: number; overrideDefault?: string; + currentEffort?: ReasoningEffort; // the effective effort default (the bare-Home effort sub-list's ✓/highlight) writeThrows?: boolean; readFaults?: boolean; // currentDefault always returns undefined (a config re-read fault after a good write) loadThrows?: boolean; // load() throws (a DB read fault) @@ -1634,15 +1635,17 @@ function makeModelsPort( } { const entries = opts.entries ?? [pickerEntry({ modelId: 'a' }), pickerEntry({ modelId: 'b' })]; let written: string | undefined; + let writtenEffort: ReasoningEffort | undefined; const load = vi.fn(() => { if (opts.loadThrows === true) throw new Error('catalog read failed'); return { entries, refreshedAt: opts.refreshedAt }; }); const refreshIfStale = vi.fn(opts.refreshIfStale ?? (() => Promise.resolve(undefined))); const refresh = vi.fn(opts.refresh ?? (() => Promise.resolve({ providers: [] }))); - const writeDefault = vi.fn((modelId: string) => { + const writeDefault = vi.fn((modelId: string, reasoningEffort?: ReasoningEffort) => { if (opts.writeThrows === true) throw new Error('config write failed'); written = modelId; + if (reasoningEffort !== undefined) writtenEffort = reasoningEffort; }); const port: HomeModelsPort = { load, @@ -1652,6 +1655,9 @@ function makeModelsPort( // last written id. currentDefault: () => opts.readFaults === true ? undefined : (opts.overrideDefault ?? written), + // The EFFECTIVE effort default: an explicit `currentEffort` opt wins (a pre-existing config default), else the + // last written effort (so a re-opened picker shows the ✓ on it). + currentEffort: () => opts.currentEffort ?? writtenEffort, writeDefault, }; return { port, load, refreshIfStale, refresh, writeDefault }; @@ -1703,12 +1709,66 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => const c = openPicker(port); await flush(); c.handleKey('', ENTER); // accept the selected (available) model - expect(writeDefault).toHaveBeenCalledWith('claude-x'); + expect(writeDefault).toHaveBeenCalledWith('claude-x', undefined); // non-reasoning ⇒ model only, no effort expect(c.getSnapshot().modelPicker).toBeUndefined(); // the picker closed expect(c.getSnapshot().notice).toContain('Claude X'); // the confirmation names the model expect(c.getSnapshot().notice).toContain('next chat session'); // it is a NEXT-session action, not a live reseat }); + it('bare Home: a REASONING model offers the effort sub-step, writing BOTH model + effort defaults (ADR-0066 §6)', async () => { + // ADR-0066 §6: in the bare Home a reasoning model advances to the effort sub-step (opened on the config effort + // default), and accepting writes model + effort together — so the user sets both future-session defaults at once. + const { port, writeDefault } = makeModelsPort({ + entries: [ + pickerEntry({ + modelId: 'deepseek-v4-flash', + displayName: 'DeepSeek V4 Flash', + supportsReasoning: true, + }), + ], + currentEffort: 'low', // the existing effort default — the sub-list opens highlighted on it + }); + const c = openPicker(port); + await flush(); + c.handleKey('', ENTER); // model phase: Enter on the reasoning model ⇒ advance to the effort sub-step + expect(c.getSnapshot().modelPicker?.phase).toBe('effort'); + expect(c.getSnapshot().modelPicker?.currentEffort).toBe('low'); // opened on the config effort default + c.handleKey('', { downArrow: true }); // low → medium + c.handleKey('', { downArrow: true }); // medium → high + c.handleKey('', ENTER); // accept the model + 'high' + + expect(writeDefault).toHaveBeenCalledWith('deepseek-v4-flash', 'high'); // BOTH written (one atomic call) + expect(c.getSnapshot().modelPicker).toBeUndefined(); // closed + expect(c.getSnapshot().notice).toContain('DeepSeek V4 Flash'); + expect(c.getSnapshot().notice).toContain('effort high'); // the notice names the written effort + expect(c.getSnapshot().notice).toContain('next chat session'); + }); + + it('bare Home: a reasoning model with NO prior effort default opens on medium; immediate Enter writes medium (ADR-0066 §6)', async () => { + // No `currentEffort` opt ⇒ the sub-list opens on the neutral 'medium' (initialEffortIndex(undefined)). There is + // no "model-only, skip effort" path for a reasoning model in the bare Home (Esc only backs to the model list), so + // accepting immediately writes the neutral default — pin that intended behavior (ADR-0066 §6). + const { port, writeDefault } = makeModelsPort({ + entries: [ + pickerEntry({ + modelId: 'deepseek-v4-flash', + displayName: 'DeepSeek V4 Flash', + supportsReasoning: true, + }), + ], + // no currentEffort → port.currentEffort() is undefined + }); + const c = openPicker(port); + await flush(); + c.handleKey('', ENTER); // advance to the effort sub-step + expect(c.getSnapshot().modelPicker?.phase).toBe('effort'); + expect(c.getSnapshot().modelPicker?.currentEffort).toBeUndefined(); // no config effort default + c.handleKey('', ENTER); // immediate Enter on the opening highlight (the neutral 'medium') + + expect(writeDefault).toHaveBeenCalledWith('deepseek-v4-flash', 'medium'); // the neutral default is written + expect(c.getSnapshot().notice).toContain('effort medium'); + }); + it('an honest notice when a project/workspace setting overrides the global write (no false success)', async () => { // The write lands on the global file, but the effective default stays the project/workspace override, so the // notice must NOT claim "applies to your next chat session" — it says the override still wins here. @@ -1719,7 +1779,7 @@ describe('the /models picker in the bare Home (2.5.G S7 / ADR-0064 §10)', () => const c = openPicker(port); await flush(); c.handleKey('', ENTER); - expect(writeDefault).toHaveBeenCalledWith('claude-x'); + expect(writeDefault).toHaveBeenCalledWith('claude-x', undefined); // non-reasoning ⇒ model only, no effort expect(c.getSnapshot().notice).toContain('overrides it here'); expect(c.getSnapshot().notice).not.toContain('next chat session'); // no false claim of effect }); diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index 3e2161d3..f41055f0 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -195,8 +195,13 @@ export interface HomeModelsPort { refresh: () => Promise; /** The current `[preferences].default_model` (the picker's `✓` marker), or `undefined` when none is set. */ currentDefault: () => string | undefined; - /** Persist the chosen model as the next session's default (writeGlobalDefaultModel). Throws `ConfigError` on a bad write. */ - writeDefault: (modelId: string) => void; + /** The current resolved default reasoning-effort tier (ADR-0066 §6) — the `✓`/opening highlight of the bare-Home + * effort sub-step; `undefined` ⇒ none set (the sub-list opens on a neutral middle tier). */ + currentEffort: () => ReasoningEffort | undefined; + /** Persist the chosen model as the next session's default, and (ADR-0066 §6) — when the effort sub-step ran for a + * reasoning model — its effort tier too, in ONE atomic write (writeGlobalPreferences). An absent `reasoningEffort` + * leaves any prior effort default unchanged. Throws `ConfigError` on a bad write. */ + writeDefault: (modelId: string, reasoningEffort?: ReasoningEffort) => void; } export interface HomeControllerDeps { @@ -643,10 +648,11 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { // bare Home it is the effective next-session default. The accept action mirrors this (reseat vs default-write). const active = state.session; const activeModel = active?.store.getSnapshot().state.model; - // The effort sub-step (ADR-0066) is offered ONLY in a LIVE in-Home chat (a session with the effort setter wired) - // — the bare-Home default-write persists only the model (ADR-0063), so it stays single-phase. `currentEffort` - // reads the LIVE store tier, so after a no-reseat `/effort` change the sub-list opens on it. - const effortStep = active?.onSetEffort !== undefined; + // The effort sub-step (ADR-0066) is offered for a reasoning model on BOTH surfaces: a LIVE in-Home chat (the + // setter is wired → the pick is a per-turn session override) AND the bare Home (ADR-0066 §6 → the pick writes the + // NEXT session's effort default alongside the model). `currentEffort` opens the sub-list on the right tier: the + // LIVE store tier in a chat (so a prior no-reseat `/effort` change is reflected), else the config effort default. + const effortStep = active !== undefined ? active.onSetEffort !== undefined : true; set({ notice: undefined, // opening the picker clears any stale /doctor report behind it modelPicker: { @@ -662,7 +668,11 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { effortStep, pending: undefined, effortSelected: 0, - currentEffort: effortStep ? active?.store.getSnapshot().reasoningEffort : undefined, + currentEffort: !effortStep + ? undefined + : active !== undefined + ? active.store.getSnapshot().reasoningEffort + : port.currentEffort(), }, }); // Render the cache immediately (above), then kick a TTL-bounded background refresh (ADR-0064 §5c) — the Home is @@ -676,22 +686,31 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { effective: string | undefined, modelId: string, displayName: string, + reasoningEffort: ReasoningEffort | undefined, ): string => { + // The effort (ADR-0066 §6) is written to the SAME layer atomically, so it shares the model's effectiveness — the + // suffix just names what was set (a reasoning model went through the effort sub-step; a non-reasoning one did not). + const effort = reasoningEffort === undefined ? '' : ` at effort ${reasoningEffort}`; if (effective === modelId) - return `Default model set to ${displayName} — applies to your next chat session.`; + return `Default model set to ${displayName}${effort} — applies to your next chat session.`; if (effective === undefined) { - return `Saved ${displayName} as your global default, but your config could not be re-read to confirm it.`; + return `Saved ${displayName}${effort} as your global default, but your config could not be re-read to confirm it.`; } - return `Saved ${displayName} as your global default, but a project or workspace setting overrides it here.`; + return `Saved ${displayName}${effort} as your global default, but a project or workspace setting overrides it here.`; }; - // The BARE-Home next-session-default write (ADR-0063). `writeGlobalDefaultModel` writes only the GLOBAL - // `[preferences].default_model`; a write fault keeps the picker open with a secret-free hint rather than crashing. - const writeNextSessionDefault = (modelId: string, displayName: string): void => { + // The BARE-Home next-session-default write (ADR-0063 · ADR-0066 §6). Persists the GLOBAL `[preferences].default_model` + // and — when the effort sub-step ran for a reasoning model — `[preferences].reasoning_effort` too, in ONE atomic + // write; a write fault keeps the picker open with a secret-free hint rather than crashing. + const writeNextSessionDefault = ( + modelId: string, + displayName: string, + reasoningEffort?: ReasoningEffort, + ): void => { const port = deps.models; if (port === undefined) return; try { - port.writeDefault(modelId); + port.writeDefault(modelId, reasoningEffort); } catch { // A generic save-failure hint — the actual write target may be a `--config` override, not the canonical // `~/.relavium/config.toml`, so don't name a path the user may not be using. @@ -708,7 +727,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { } set({ modelPicker: undefined, - notice: defaultWriteNotice(port.currentDefault(), modelId, displayName), + notice: defaultWriteNotice(port.currentDefault(), modelId, displayName, reasoningEffort), }); }; @@ -768,10 +787,11 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { applyEffortOnlyUpdate(active, displayName, reasoningEffort); }; - // Accept the chosen model. TWO surface-specific actions off the ONE picker (ADR-0059/ADR-0063): a LIVE in-Home chat - // RESEATs / updates effort (the effort setter path is only REACHED with `reseatChat` wired — the in-chat `/models` - // that opens the effort sub-step is gated on it — so guarding on both never diverts an effort pick to the config - // write); the BARE Home persists the chosen model as the NEXT session's default. + // Accept the chosen model. TWO surface-specific actions off the ONE picker (ADR-0059/ADR-0063 · ADR-0066 §6): a LIVE + // in-Home chat RESEATs / updates effort (the effort setter path is only REACHED with `reseatChat` wired — the + // in-chat `/models` that opens the effort sub-step is gated on it — so guarding on both never diverts an effort pick + // to the config write); the BARE Home persists the chosen model AND (when the effort sub-step ran) its effort tier + // as the NEXT session's defaults. const acceptModel = ( modelId: string, displayName: string, @@ -783,7 +803,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { applyLiveSessionPick(active, modelId, displayName, provider, reasoningEffort); return; } - writeNextSessionDefault(modelId, displayName); + writeNextSessionDefault(modelId, displayName, reasoningEffort); }; // The open `/models` picker owns every key (2.5.G S7) — parity with routeMentionKey. Returns whether the key was // consumed. A DIMMED (unavailable-on-your-key) model is non-selectable (ADR §6): accepting one shows a transient diff --git a/apps/cli/src/render/tui/model-picker.ts b/apps/cli/src/render/tui/model-picker.ts index 1dcd6be2..5406ee01 100644 --- a/apps/cli/src/render/tui/model-picker.ts +++ b/apps/cli/src/render/tui/model-picker.ts @@ -42,16 +42,17 @@ export interface ModelPickerState { readonly hint: string | undefined; /** * The picker's TWO-PHASE step ([ADR-0066](../../../../../docs/decisions/0066-normalized-reasoning-effort-control.md)): - * `'model'` is the catalog list (the default); accepting a reasoning-capable model on a reseat surface + * `'model'` is the catalog list (the default); accepting a reasoning-capable model on an effort-offering surface * ({@link effortStep}) advances to `'effort'` — a fixed sub-list of the reasoning-effort tiers for the chosen * model. The two surfaces route the SAME fold, so the phase transition lives here, not in either host. */ readonly phase: 'model' | 'effort'; /** * Whether this picker offers the reasoning-effort sub-step. `true` for a LIVE reseat surface (standalone - * `relavium chat` + the in-Home live chat, where the effort binds onto the reseated agent, ADR-0059); `false` - * for the bare-Home next-session-default write (which persists only the model, ADR-0063 — the effort default is - * the `[chat].reasoning_effort` config key, not a per-write pick), so a non-reseat surface stays single-phase. + * `relavium chat` + the in-Home live chat, where the effort binds onto the reseated agent as a per-turn override, + * ADR-0059/§5) AND for the bare Home (ADR-0066 §6 — where the accept instead WRITES the model + effort tier as the + * next-session config defaults via `writeGlobalPreferences`); `false` only where no effort control applies. A + * reasoning-capable model advances to the sub-step; a non-reasoning one stays single-phase (accepts the model alone). */ readonly effortStep: boolean; /** The model chosen in `'model'` phase, awaiting an effort pick — carried so `'effort'`'s accept emits the pair. */ diff --git a/docs/decisions/0063-cli-config-write-contract.md b/docs/decisions/0063-cli-config-write-contract.md index b79778e6..d1b5fca4 100644 --- a/docs/decisions/0063-cli-config-write-contract.md +++ b/docs/decisions/0063-cli-config-write-contract.md @@ -4,6 +4,8 @@ - **Date**: 2026-07-05 - **Related**: [ADR-0048](0048-toml-config-parser.md) (`smol-toml`, confined to the CLI config loader — this ADR extends its use to the *writer*, same package boundary, no new dependency; **append-only top-note added there**) · [ADR-0033](0033-strict-config-files-amends-0023.md) (strict config — the written file must re-parse under `.strict()`) · [ADR-0023](0023-strict-authored-yaml-validation.md) · [ADR-0024](0024-agent-first-entry-point-agentsession.md) (the `[chat]` block + one-model-per-session) · [ADR-0006](0006-os-keychain-for-api-keys.md) (secrets live only in the keychain — never config) · [ADR-0049](0049-cli-machine-output-contract.md) (the write is a TTY-interactive action) · [ADR-0064](0064-live-model-catalog.md) + [ADR-0065](0065-provider-economics-and-extensibility.md) (the `/models` picker + onboarding wizard that drive this write). The config keys + resolution order are the canonical [config-spec.md](../reference/contracts/config-spec.md); the schemas live in [config.ts](../../packages/shared/src/config.ts). +> **Note (2026-07-07): the typed setter gained a second key — `[preferences].reasoning_effort` (ADR-0066 §6).** The writer generalized from `writeGlobalDefaultModel` to `writeGlobalPreferences({ defaultModel?, reasoningEffort? })` (the former is now a thin wrapper). It writes ONE or BOTH of the two non-secret `[preferences]` keys in a single atomic, schema-round-tripped write; a field absent from the call is a **partial merge** (left unchanged), so a model-only pick never clears a prior effort default. The typed-setter secret-incapability guarantee is unchanged — both keys are non-secret enums/strings, and the strict `GlobalConfigSchema` still gates the emission. The `/models` picker's effort sub-step (bare Home) is the write's new driver, and `resolveChat` now reads `[preferences].reasoning_effort` as the global fallback below `[chat].reasoning_effort` — the exact shape §1 established for `default_model`. + ## Context The CLI config layer is **read-only**. [load.ts](../../apps/cli/src/config/load.ts) imports `smol-toml` diff --git a/docs/decisions/0066-normalized-reasoning-effort-control.md b/docs/decisions/0066-normalized-reasoning-effort-control.md index ed6d60bc..dab0bda6 100644 --- a/docs/decisions/0066-normalized-reasoning-effort-control.md +++ b/docs/decisions/0066-normalized-reasoning-effort-control.md @@ -9,6 +9,8 @@ > **Note (2026-07-07): DeepSeek wired + its native shape refined (§2/§4).** DeepSeek was initially DEFERRED (the OpenAI-compatible adapter had no pinned SDK to verify the param against). Its create-chat-completion API was then verified against the official docs (api-docs.deepseek.com): v4 takes a **`thinking` object** — `type: 'enabled' | 'disabled'` + `reasoning_effort: 'high' | 'max'` — richer than §2's assumed "on/off only". The adapter now maps `off` → `{ type: 'disabled' }`, `low`/`medium`/`high` → `{ type: 'enabled', reasoning_effort: 'high' }` (DeepSeek's minimum thinking level), and `max` → `{ type: 'enabled', reasoning_effort: 'max' }` (an honest coarsening onto v4's two graded levels); `deepseek-v4-flash`/`-pro` are `reasoning: true`, the §4 heuristic covers `deepseek-v[4-9]`, and the legacy non-thinking `deepseek-chat` alias stays uncontrollable. So the §2 DeepSeek row's "on/off, no graded tiers" — and the matching on/off phrasing in Consequences → Negative — are both superseded by the doc-verified `thinking` shape here (off / high / max); the reasoning **output** (`reasoning_content`) + token accounting were already mapped. > > **Note (2026-07-07): interactive `/effort` overlay realized (§6).** §6 anticipated "a future `/effort`" for standalone effort changes; it is now shipped as a first-class **interactive tier-selector overlay** (not just a typed `/effort ` — that still works). Bare `/effort` (typed or selected from the `/` palette) opens a keyboard-owning overlay — a fixed off/low/medium/high/max list, opening on the bound tier, arrow+Enter to apply — in BOTH `relavium chat` and the in-Home live chat, sharing one pure fold ([effort-picker.ts](../../apps/cli/src/render/tui/effort-picker.ts)) and one view ([effort-tier-list.tsx](../../apps/cli/src/render/tui/effort-tier-list.tsx), also used by the `/models` effort sub-step). Applying calls the §5 per-turn `onSetEffort` — **no reseat**. It opens ONLY on a reasoning-capable bound model; a non-reasoning model falls through to the informational notice (` has no controllable reasoning tier`). This realizes §6's "standalone effort changes ride the same session override" — from a promise to a shipped surface. (§6's separate point, that the bare-Home picker writes only the config default and not a live session's effort, is unchanged here — a follow-up gives that picker its own effort sub-step.) +> +> **Note (2026-07-07): the bare-Home `/models` picker gained an effort sub-step (§6, superseding "config default only").** §6 said the bare-Home picker "writes the config default, not a live session's effort" and stays single-phase. It is now **two-phase for a reasoning model** on BOTH surfaces: picking a reasoning-capable model advances to the effort sub-step (opened on the current default tier), and accepting writes the model AND its effort tier as the NEXT session's defaults. This required a **new global config key**, `[preferences].reasoning_effort` (the effort counterpart of `default_model`), added to `GlobalConfigSchema`; the CLI config writer generalized to `writeGlobalPreferences({ defaultModel?, reasoningEffort? })` (ADR-0063 §note), writing both in one atomic, schema-round-tripped write (a partial write leaves the other key unchanged); and `resolveChat` now falls back to `[preferences].reasoning_effort` below the `[chat]` layers — the exact precedence `default_model` already has. So a user can set BOTH their default model and default effort from the Home without starting a chat. A live in-Home chat's `/models` effort sub-step still sets the per-turn override (no reseat, §5) rather than writing config — the accept action is surface-specific, off the one picker. ## Context diff --git a/docs/reference/cli/commands.md b/docs/reference/cli/commands.md index ed307a1e..6f4ce39b 100644 --- a/docs/reference/cli/commands.md +++ b/docs/reference/cli/commands.md @@ -152,7 +152,7 @@ The **command manifest** is the one source the **shell** command surfaces derive ### In-REPL slash commands -The interactive `/` palette + slash commands inside the **Home and chat** are a SEPARATE, **curated** surface ([ADR-0056](../../decisions/0056-cli-in-app-slash-command-system-and-manifest.md) amendment, 2.5.C) — the runtime registry is `apps/cli/src/commands/repl-commands.ts` (`REPL_COMMANDS`), the single source for the palette, the `/help` list, and the unknown-slash hint. It surfaces only the commands that make sense in a live REPL — lifecycle (`/exit`, `/cancel`, `/export`, `/clear`), info/discovery (`/help`, `/workflows`, `/cost`, `/doctor`), and — in a chat — `/mode ` to switch the chat mode (2.5.E, [chat-session.md](chat-session.md)), `/effort ` to set the reasoning-effort tier ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md); a per-turn session override, no reseat), plus the ADR-0062 context commands (`/compact`, `/trim`). `/models` opens an in-tree model picker over the merged live/static catalog; its **action depends on the surface**. Inside a **live chat** (standalone `relavium chat` or an in-Home chat) it triggers a **live reseat** ([ADR-0059](../../decisions/0059-cli-mid-session-model-reseat.md)) — rebinding the session to the picked model (dropping the old fallback chain), carrying the text-only transcript + cumulative cost/turns under the SAME `sessionId`. At the **bare Home** (no active chat) it instead writes the **next** session's default model ([ADR-0064](../../decisions/0064-live-model-catalog.md) §10, via the [ADR-0063](../../decisions/0063-cli-config-write-contract.md) config writer). On a **reasoning-capable** model, a second **effort sub-step** ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md)) picks the reasoning-effort tier (`off`/`low`/`medium`/`high`/`max`; `Esc` backs out to the model list, `Ctrl-C` cancels): a **same-model** pick applies the tier as a **per-turn session override — no reseat, no teardown** (effort changes neither provider, pricing, nor the plan, §5), while a **different-model** pick carries it onto the reseat. The active tier rides the footer (parity with the mode), and `/effort ` sets it directly without opening the picker. The bare-Home default-write stays single-step (its effort default is the `[chat].reasoning_effort` config key). Under `--json`/plain (non-TTY) the live reseat is unavailable — one machine stream stays one session lifecycle ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)) — so a typed `/models` there falls through to an actionable "interactive terminal" hint. The picker renders the ADR-0064 first-class UX: per-model pricing (an unpriced model shows a "cost cap will not apply" hint), a dimmed non-selectable "unavailable on your key" row, a `deprecated` flag, a loading spinner, a per-provider partial-failure banner, and a "last updated" freshness badge; opening it over an empty/stale cache renders immediately and kicks a background refresh (the long-lived Home is where that is sound), `Ctrl+R` forces a live refresh, and `Esc` cancels. The heavy, session-starting shell commands above (`run`, `chat`, `provider`, …) are **never** in-REPL slashes — they stay shell-only (`relavium …`). A bare `/` at an **empty** prompt opens the filterable palette (the footer hint-bar surfaces `/ for commands` exactly there, 2.5.C S6); an unknown slash — or an undeclared argument on a known command (`/exit now`) — prints a sanitized, secret-free hint. A command may declare flags (`/doctor --deep`) or a single positional value (`/mode plan`); the palette runs the bare form, so a flag/value is opt-in by typing it. There is no separate `/shortcuts` command — the palette renders its own nav hints (`↑/↓ · Enter · Esc`) and the footer surfaces `/`, so keys stay discoverable in context. +The interactive `/` palette + slash commands inside the **Home and chat** are a SEPARATE, **curated** surface ([ADR-0056](../../decisions/0056-cli-in-app-slash-command-system-and-manifest.md) amendment, 2.5.C) — the runtime registry is `apps/cli/src/commands/repl-commands.ts` (`REPL_COMMANDS`), the single source for the palette, the `/help` list, and the unknown-slash hint. It surfaces only the commands that make sense in a live REPL — lifecycle (`/exit`, `/cancel`, `/export`, `/clear`), info/discovery (`/help`, `/workflows`, `/cost`, `/doctor`), and — in a chat — `/mode ` to switch the chat mode (2.5.E, [chat-session.md](chat-session.md)), `/effort ` to set the reasoning-effort tier ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md); a per-turn session override, no reseat), plus the ADR-0062 context commands (`/compact`, `/trim`). `/models` opens an in-tree model picker over the merged live/static catalog; its **action depends on the surface**. Inside a **live chat** (standalone `relavium chat` or an in-Home chat) it triggers a **live reseat** ([ADR-0059](../../decisions/0059-cli-mid-session-model-reseat.md)) — rebinding the session to the picked model (dropping the old fallback chain), carrying the text-only transcript + cumulative cost/turns under the SAME `sessionId`. At the **bare Home** (no active chat) it instead writes the **next** session's default model ([ADR-0064](../../decisions/0064-live-model-catalog.md) §10, via the [ADR-0063](../../decisions/0063-cli-config-write-contract.md) config writer). On a **reasoning-capable** model, a second **effort sub-step** ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md)) picks the reasoning-effort tier (`off`/`low`/`medium`/`high`/`max`; `Esc` backs out to the model list, `Ctrl-C` cancels): a **same-model** pick applies the tier as a **per-turn session override — no reseat, no teardown** (effort changes neither provider, pricing, nor the plan, §5), while a **different-model** pick carries it onto the reseat. The active tier rides the footer (parity with the mode), and `/effort ` sets it directly without opening the picker. At the **bare Home** the effort sub-step likewise appears for a reasoning-capable model, but its accept **writes** the model + effort tier as the next session's config defaults (`[preferences].default_model` + `[preferences].reasoning_effort`, [ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md) §6, via the same [ADR-0063](../../decisions/0063-cli-config-write-contract.md) writer) — not a live override. Under `--json`/plain (non-TTY) the live reseat is unavailable — one machine stream stays one session lifecycle ([ADR-0049](../../decisions/0049-cli-machine-output-contract.md)) — so a typed `/models` there falls through to an actionable "interactive terminal" hint. The picker renders the ADR-0064 first-class UX: per-model pricing (an unpriced model shows a "cost cap will not apply" hint), a dimmed non-selectable "unavailable on your key" row, a `deprecated` flag, a loading spinner, a per-provider partial-failure banner, and a "last updated" freshness badge; opening it over an empty/stale cache renders immediately and kicks a background refresh (the long-lived Home is where that is sound), `Ctrl+R` forces a live refresh, and `Esc` cancels. The heavy, session-starting shell commands above (`run`, `chat`, `provider`, …) are **never** in-REPL slashes — they stay shell-only (`relavium …`). A bare `/` at an **empty** prompt opens the filterable palette (the footer hint-bar surfaces `/ for commands` exactly there, 2.5.C S6); an unknown slash — or an undeclared argument on a known command (`/exit now`) — prints a sanitized, secret-free hint. A command may declare flags (`/doctor --deep`) or a single positional value (`/mode plan`); the palette runs the bare form, so a flag/value is opt-in by typing it. There is no separate `/shortcuts` command — the palette renders its own nav hints (`↑/↓ · Enter · Esc`) and the footer surfaces `/`, so keys stay discoverable in context. ### `relavium run` diff --git a/docs/reference/contracts/config-spec.md b/docs/reference/contracts/config-spec.md index 72c102c4..52983461 100644 --- a/docs/reference/contracts/config-spec.md +++ b/docs/reference/contracts/config-spec.md @@ -66,6 +66,7 @@ update_channel = "stable" # stable | beta [preferences] default_model = "claude-sonnet-4-6" +reasoning_effort = "medium" # ADR-0066 §6: the GLOBAL default reasoning-effort tier — off | low | medium | high | max; the fallback BELOW any [chat].reasoning_effort. Written by the /models picker's effort sub-step. Absent ⇒ no reasoning control (the provider default). theme = "dark" [[mcp_servers]] # repeatable — an agent references one by name via `ref:` (ADR-0052 §5) @@ -90,9 +91,12 @@ stdio-only fields (`command`/`args`/`env`) are rejected on a network registratio > **Writing the global config** ([ADR-0063](../../decisions/0063-cli-config-write-contract.md)). Config is > almost entirely **read-only** (hand-edited, git-committed). The one write path is the CLI persisting a chosen -> default model: `/models` and the 2.5.G onboarding wizard set **`[preferences].default_model`** — and *only* -> that key — through a **typed setter** (never a generic key/value writer), so a secret can never be written by -> construction (there is no `api_key` field in the schema; keys live only in the OS keychain, [ADR-0006](../../decisions/0006-os-keychain-for-api-keys.md)). +> default: `/models` and the 2.5.G onboarding wizard set **`[preferences].default_model`**, and (ADR-0066 §6) the +> `/models` picker's **effort sub-step** additionally sets **`[preferences].reasoning_effort`** for a reasoning +> model — **only** those two `[preferences]` keys, and only through a **typed setter** (never a generic key/value +> writer), so a secret can never be written by construction (there is no `api_key` field in the schema; keys live +> only in the OS keychain, [ADR-0006](../../decisions/0006-os-keychain-for-api-keys.md)). A partial write leaves +> the other key unchanged (a model-only pick never clears a prior effort default). > The write is **atomic + durable** (a `0600` temp file in the `0700` `~/.relavium/`, `fsync` the file, `rename` > over the target, then `fsync` the parent directory so the rename itself survives a crash) and it both > re-validates the merged object against the strict `GlobalConfigSchema` AND re-parses the emitted TOML back @@ -155,9 +159,9 @@ allowed_command_globs = [] # opt-in glob form of the !-shell allowlist ( > > `max_messages` (revived in 2.5.F) is the bound `/trim` enforces (keep the last N messages, no LLM call) and the deterministic fallback if a summarization fails. `auto_compact` + `compact_threshold` ([ADR-0062](../../decisions/0062-context-compaction-and-cli-history-commands.md)) drive **automatic** model-summarised compaction: after a turn completes, if its **real** input tokens exceed `compact_threshold` (default `0.8`, a fraction in (0, 1]) × the serving model's context window, the session compacts before the next turn. `auto_compact` absent ⇒ enabled; a model with no known context window (a custom base-URL id) skips auto-compaction, but manual `/compact` still works. The summarization spend is accounted to the session budget and surfaced, never silent. > -> `reasoning_effort` ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md)) is the normalized reasoning-effort tier — `off | low | medium | high | max` — baked onto the **built-in default chat agent** only (an explicit `--agent` owns its own `reasoning_effort` in its YAML). It resolves per-field project → workspace like the other `[chat]` keys (no global-layer fallback — that extra fallback is `default_model`'s alone). Each adapter maps the tier to its provider's **native** control; a model with no controllable reasoning tier ignores it (the engine gates on the model's capability). Absent ⇒ no reasoning control (the provider default). Interactively, the `/effort` command and the `/models` picker's **effort sub-step** set the tier as a **per-turn session override** (no reseat) without editing config; the active tier shows in the footer. +> `reasoning_effort` ([ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md)) is the normalized reasoning-effort tier — `off | low | medium | high | max` — baked onto the **built-in default chat agent** only (an explicit `--agent` owns its own `reasoning_effort` in its YAML). It resolves per-field project → workspace, and (ADR-0066 §6) — like `default_model` — additionally falls back to the global **`[preferences].reasoning_effort`** below both `[chat]` layers. Each adapter maps the tier to its provider's **native** control; a model with no controllable reasoning tier ignores it (the engine gates on the model's capability). Absent ⇒ no reasoning control (the provider default). Interactively, the `/effort` command and a **live chat**'s `/models` effort sub-step set the tier as a **per-turn session override** (no reseat) without editing config; the **bare-Home** `/models` effort sub-step instead writes the `[preferences].reasoning_effort` default for the next session. The active tier shows in the footer. > -> The `[chat]` block resolves **per field** (each key independently, last-writer-wins project → workspace) — a project that sets only `max_turns` still inherits `default_model`/`max_messages` from the workspace layer. (Contrast `[defaults].media_cost_estimate`, which resolves **whole-object**: the highest layer present replaces the table outright.) **`default_model` has one extra fallback**: absent at both `[chat]` layers, it falls through to the global **`[preferences].default_model`** ([ADR-0063](../../decisions/0063-cli-config-write-contract.md) §1) — the write target of `/models` and the wizard — so a user's "preferred model everywhere" governs chat too, exactly as it already governs a workflow's `[defaults].model`. Full precedence: `[chat].default_model` (project → workspace) → `[preferences].default_model` (global). No other `[chat]` field reads the global layer. The `!`-shell allowlist is the **one exception**: `allowed_commands` (exact) + `allowed_command_globs` (globs) are a **coupled unit**, so a project that sets **either** array owns the **whole** allowlist and does **not** inherit the other array from the workspace. Otherwise a project narrowing `allowed_commands` would silently keep the workspace's broader globs — lock to `git status`, yet still allow `git push` via an inherited `git *`. Only when a project sets **neither** allowlist array do both fall through to the workspace; a present array otherwise REPLACES (never merges) the lower layer's. This is what guarantees a narrower project can never inherit a broader workspace entry. +> The `[chat]` block resolves **per field** (each key independently, last-writer-wins project → workspace) — a project that sets only `max_turns` still inherits `default_model`/`max_messages` from the workspace layer. (Contrast `[defaults].media_cost_estimate`, which resolves **whole-object**: the highest layer present replaces the table outright.) **`default_model` and `reasoning_effort` each have one extra fallback**: absent at both `[chat]` layers, each falls through to its global **`[preferences]`** counterpart (`default_model` per [ADR-0063](../../decisions/0063-cli-config-write-contract.md) §1; `reasoning_effort` per [ADR-0066](../../decisions/0066-normalized-reasoning-effort-control.md) §6) — the write targets of `/models` (model + its effort sub-step) and the wizard — so a user's "preferred model / effort everywhere" governs chat too, exactly as `[preferences].default_model` already governs a workflow's `[defaults].model`. Full precedence for each: `[chat].` (project → workspace) → `[preferences].` (global). No OTHER `[chat]` field reads the global layer. The `!`-shell allowlist is the **one exception**: `allowed_commands` (exact) + `allowed_command_globs` (globs) are a **coupled unit**, so a project that sets **either** array owns the **whole** allowlist and does **not** inherit the other array from the workspace. Otherwise a project narrowing `allowed_commands` would silently keep the workspace's broader globs — lock to `git status`, yet still allow `git push` via an inherited `git *`. Only when a project sets **neither** allowlist array do both fall through to the workspace; a present array otherwise REPLACES (never merges) the lower layer's. This is what guarantees a narrower project can never inherit a broader workspace entry. > > `allowed_commands` / `allowed_command_globs` gate the **`!`-shell escape** (2.5.D, [ADR-0061](../../decisions/0061-cli-input-layer-file-injection-and-shell-escape.md)) — a chat user typing `!command` runs it through the **one** `run_command` boundary (they map to the engine's camelCase `allowedCommands` / `allowedCommandGlobs`, the SAME allowlist a workflow `run_command` uses). `allowed_commands` is **exact full-command-string** match (`git status`, `ls -la` — `git` never authorizes `git push --force`); `allowed_command_globs` is the opt-in, riskier pattern form. **Both default to EMPTY ⇒ `!`-shell is disabled** — the `empty ⇒ disabled` symmetry [security-review.md](../../standards/security-review.md) pins, with **no chat-specific relaxation** (there is no curated default: `run_command` has no argument/file confidentiality floor, so even a "read-only" default set — `cat`, `grep` — would reopen `!cat .env` → provider). `!`-shell is first-class via a first-class **opt-in** (the user lists commands, or the 2.5.G onboarding offers a reviewed seed), and a non-allowlisted `!cmd` gets an **actionable, secret-free deny hint** naming the exact line to add. `enforcePolicy(allowedCommands)` runs **before** the mode-aware `confirmAction`, so even `auto` mode never runs a command absent from the allowlist. Editing chat `allowed_commands` is a [security-review.md](../../standards/security-review.md) trigger. diff --git a/packages/shared/src/config.ts b/packages/shared/src/config.ts index 3b59b37f..1eb76195 100644 --- a/packages/shared/src/config.ts +++ b/packages/shared/src/config.ts @@ -137,6 +137,10 @@ export const GlobalConfigSchema = z .object({ default_model: z.string().optional(), theme: z.string().optional(), + // The GLOBAL default reasoning-effort tier (ADR-0066 §6) — the effort counterpart of `default_model`, the + // write target of the `/models` picker's effort sub-step. Resolved BELOW project/workspace + // `[chat].reasoning_effort` (config-spec.md), so a project override still wins; absent ⇒ the provider default. + reasoning_effort: z.enum(REASONING_EFFORTS).optional(), }) .strict() .optional(), From 6b3e32a62e895698e72f2fd71fa6aa77b7e3e717 Mon Sep 17 00:00:00 2001 From: Cemil ILIK Date: Tue, 7 Jul 2026 12:38:51 +0300 Subject: [PATCH 68/68] =?UTF-8?q?refactor(cli):=20resolve=20two=20Sonar=20?= =?UTF-8?q?smells=20in=20home-controller=20(ADR-0066=20=C2=A76=20follow-up?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two SonarQube findings on the Step-2 additions: - L673 (nested ternary): the `/models` picker's `currentEffort` was a nested ternary inside the object literal — extracted to a `let` + single ternary in an `if`, an independent statement. - L1371 (cognitive complexity 17 > 15): the `/effort` overlay routing branch pushed `handleKey` over the budget — extracted the whole open-overlay + mode dispatch tail (palette → /models → /effort → chat → Home) into a `dispatchKey` helper, leaving handleKey with just the bracketed-paste latch + the call. Pure refactor — no behavior change (88 home-controller tests green, lint + typecheck clean). Co-Authored-By: Claude --- apps/cli/src/render/tui/home-controller.ts | 68 ++++++++++++---------- 1 file changed, 38 insertions(+), 30 deletions(-) diff --git a/apps/cli/src/render/tui/home-controller.ts b/apps/cli/src/render/tui/home-controller.ts index f41055f0..793dd546 100644 --- a/apps/cli/src/render/tui/home-controller.ts +++ b/apps/cli/src/render/tui/home-controller.ts @@ -653,6 +653,13 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { // NEXT session's effort default alongside the model). `currentEffort` opens the sub-list on the right tier: the // LIVE store tier in a chat (so a prior no-reseat `/effort` change is reflected), else the config effort default. const effortStep = active !== undefined ? active.onSetEffort !== undefined : true; + // The effort sub-list's opening tier: absent when the sub-step isn't offered; else the LIVE store tier in a chat + // (reflecting a prior no-reseat /effort change), else the config effort default in the bare Home. + let currentEffort: ReasoningEffort | undefined; + if (effortStep) { + currentEffort = + active !== undefined ? active.store.getSnapshot().reasoningEffort : port.currentEffort(); + } set({ notice: undefined, // opening the picker clears any stale /doctor report behind it modelPicker: { @@ -668,11 +675,7 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { effortStep, pending: undefined, effortSelected: 0, - currentEffort: !effortStep - ? undefined - : active !== undefined - ? active.store.getSnapshot().reasoningEffort - : port.currentEffort(), + currentEffort, }, }); // Render the cache immediately (above), then kick a TTL-bounded background refresh (ADR-0064 §5c) — the Home is @@ -1360,6 +1363,34 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { } }; + // The open-overlay + mode key router — extracted from `handleKey` so its bracketed-paste handling stays within the + // cognitive-complexity budget. Precedence (each open overlay owns EVERY key while open, and only one opens at a + // time): the `/` palette → the `/models` picker → the `/effort` overlay → the live chat → else the bare Home. + const dispatchKey = (input: string, key: HomeKey & ChatKey & PaletteKey): void => { + // The `/` palette (when open) owns every key — before the mode dispatch, so it overlays Home/chat input. + if (state.palette !== undefined) { + handlePaletteKey(input, key); + return; + } + // The `/models` picker (2.5.G S7) — opened from the bare-Home palette (next-session default, ADR-0063) OR a live + // in-Home chat (a typed/palette `/models` → the reseat picker, ADR-0059), so it is routed before the mode branches. + if (state.modelPicker !== undefined) { + routeModelPickerKey(input, key); + return; + } + // The `/effort` overlay (ADR-0066 §6) — chat-scoped, so it always has a live session; the guard keeps + // `routeEffortPickerKey`'s `active` non-null (a stale overlay with no session falls through, never expected). + if (state.effortPicker !== undefined && state.session !== undefined) { + routeEffortPickerKey(state.session, input, key); + return; + } + if (state.mode === 'chat' && state.session !== undefined) { + handleChatKey(state.session, input, key); + return; + } + handleHomeKey(input, key); + }; + return { subscribe(listener) { listeners.add(listener); @@ -1412,31 +1443,8 @@ export function createHomeController(deps: HomeControllerDeps): HomeController { } pasting = false; } - // The `/` palette (when open) owns every key — before the mode dispatch, so it overlays Home/chat input. - if (state.palette !== undefined) { - handlePaletteKey(input, key); - return; - } - // The `/models` picker (2.5.G S7) likewise owns every key while open — mutually exclusive with the palette - // (the palette closes before running `/models`). Opened from the bare-Home palette (next-session default, - // ADR-0063) OR a live in-Home chat (a typed/palette `/models` → the reseat picker, ADR-0059), so it is routed - // before the mode branches rather than assuming a single mode. - if (state.modelPicker !== undefined) { - routeModelPickerKey(input, key); - return; - } - // The `/effort` overlay (ADR-0066 §6) likewise owns every key while open — chat-scoped, so it always has a - // live session; the guard keeps `routeEffortPickerKey`'s `active` non-null (a stale overlay with no session, - // never expected, falls through instead of driving a torn-down session). - if (state.effortPicker !== undefined && state.session !== undefined) { - routeEffortPickerKey(state.session, input, key); - return; - } - if (state.mode === 'chat' && state.session !== undefined) { - handleChatKey(state.session, input, key); - return; - } - handleHomeKey(input, key); + // Past the paste latch: the open-overlay + mode router owns the rest (palette → /models → /effort → chat → Home). + dispatchKey(input, key); }, async teardownActive() { exiting = true; // terminating: a deferred endChat/clearChat skips the (about-to-close) db; an in-flight build reclaims itself