From 1717c88f0da872c595525dbdcb53cb9e5c42525b Mon Sep 17 00:00:00 2001 From: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 Date: Fri, 31 Jul 2026 12:53:43 -0400 Subject: [PATCH 01/14] =?UTF-8?q?feat(agent):=20Phase=202a=20=E2=80=94=20w?= =?UTF-8?q?ire=20Rust=20consumers=20to=20generated=20capability=20module?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cut config.rs, catalog.rs, and llm.rs over to the generated capability module from Phase 1. Both old and new paths are preserved; Phase 3 removes the old authorities. Changes: - catalog.rs: replace hand-typed DATABRICKS_V2_KNOWN_MODELS literal with a re-export of generated_model_capabilities::DATABRICKS_V2_KNOWN_MODELS - llm.rs: databricks_v2_route_for_model now delegates to resolve_model_capabilities("databricks_v2", model). Old segment-based classifier and constants moved to #[cfg(test)] under _old_* names for the differential harness. New differential test confirms old/new agree on all 20 route test vectors (empty allowlist — logic is identical). - config.rs: add effort_table_fixture_differential_old_vs_new test that runs resolve_model_capabilities over the 36-entry effortTable.fixture.json and asserts old/new agree mod a doc-cited allowlist of 4 intentional F1 corrections (gpt-5-5, gpt-5-4-mini, gpt-5-4-nano, gpt-5-6-sol). - scripts/run-differential.mjs: new JS differential harness running the old buzzAgentConfig.ts effort logic vs new modelCapabilities.ts over the effortTable fixture (36 entries), normative corpus (45 vectors), and catalog-sample fixture (14 endpoints). Passes with allowlist of 5 entries (4 models.dev F1 corrections + goose-opus-5 anthropic route correction). - scripts/MODELS_DEV_RECONCILIATION.md: replace 8 trailing-double-space Markdown line breaks with
(deferred MINOR from Phase 1 review). Verification: - cargo test -p buzz-agent --lib: 426/426 (424 existing + 2 new differential) - node run-corpus.mjs: 45/45 - node test-manifest-validator.mjs: 24/24 schema-negative - generate-model-capabilities.mjs --check: byte-clean - node run-differential.mjs: 85 checks, 0 unexpected divergences - git diff --check: clean Co-authored-by: Will Pfleger Signed-off-by: Will Pfleger --- crates/buzz-agent/src/catalog.rs | 6 +- crates/buzz-agent/src/config.rs | 63 +++++++++ crates/buzz-agent/src/llm.rs | 118 ++++++++++++---- scripts/MODELS_DEV_RECONCILIATION.md | 16 +-- scripts/run-differential.mjs | 201 +++++++++++++++++++++++++++ 5 files changed, 365 insertions(+), 39 deletions(-) create mode 100755 scripts/run-differential.mjs diff --git a/crates/buzz-agent/src/catalog.rs b/crates/buzz-agent/src/catalog.rs index 659cbd76fdb..d9ba116327d 100644 --- a/crates/buzz-agent/src/catalog.rs +++ b/crates/buzz-agent/src/catalog.rs @@ -47,8 +47,10 @@ pub struct ModelEntry { /// Known Databricks AI Gateway v2 models — used as a fallback when the /// `api/ai-gateway/v2/endpoints` call returns an empty list. /// Mirrors goose's `DATABRICKS_V2_KNOWN_MODELS`. -pub const DATABRICKS_V2_KNOWN_MODELS: &[&str] = - &["databricks-gpt-5-5", "databricks-claude-opus-4-7"]; +/// +/// Phase 2 cutover: this is now a re-export of the generated constant in +/// `generated_model_capabilities`. Phase 3 removes the old hand-maintained list. +pub use crate::generated_model_capabilities::DATABRICKS_V2_KNOWN_MODELS; /// Returns the discovery-failure fallback catalog for a Databricks provider. /// diff --git a/crates/buzz-agent/src/config.rs b/crates/buzz-agent/src/config.rs index a0e64f1a9d2..2ad2c8be025 100644 --- a/crates/buzz-agent/src/config.rs +++ b/crates/buzz-agent/src/config.rs @@ -2738,6 +2738,69 @@ mod tests { } } + /// Phase-2 differential: new generated effort config matches old hand-coded helper + /// for every entry in effortTable.fixture.json. This gate ensures Phase 2 cutover + /// is behavior-preserving except where the allowlist explicitly covers a correction. + #[test] + fn effort_table_fixture_differential_old_vs_new() { + use crate::generated_model_capabilities::resolve_model_capabilities; + + // Intentional corrections: models where the generated capability deliberately + // diverges from the old implementation. Each entry must cite its source. + // + // "databricks_v2/databricks-gpt-5-5": Phase 1 ADOPT — models.dev payload + // d5a4974c advertises [low,medium,high]; old code returns [none,low,medium,high,xhigh]. + // Provider-advertised wins per plan F1. + // "databricks_v2/databricks-gpt-5-4-mini": Phase 1 ADOPT — models.dev advertises + // [low,medium,high]; old code returns [none,low,medium,high,xhigh]. + // "databricks_v2/databricks-gpt-5-4-nano": Phase 1 ADOPT — same as mini. + // "databricks_v2/databricks-gpt-5-6-sol": Phase 1 ADOPT — models.dev advertises + // [low,medium,high,max]; old code returns [none,low,medium,high,xhigh,max]. + let allowlist: &[(&str, &str)] = &[ + ("databricks_v2", "databricks-gpt-5-5"), + ("databricks_v2", "databricks-gpt-5-4-mini"), + ("databricks_v2", "databricks-gpt-5-4-nano"), + ("databricks_v2", "databricks-gpt-5-6-sol"), + ]; + + let fixture_json = + include_str!("../../../desktop/src/features/agents/ui/effortTable.fixture.json"); + let entries: Vec = + serde_json::from_str(fixture_json).expect("fixture must be valid JSON"); + + for entry in &entries { + let label = entry.note.as_deref().unwrap_or(entry.model.as_str()); + let in_allowlist = allowlist + .iter() + .any(|(p, m)| *p == entry.provider && *m == entry.model); + + let old_result = valid_effort_values_for_provider_model(&entry.provider, &entry.model); + + // Build new result from generated module. + let cap = resolve_model_capabilities(&entry.provider, &entry.model); + let new_values: Vec<&'static str> = cap + .supported_efforts + .iter() + .map(|e| e.openai_effort_str()) + .collect(); + let new_default: Option<&'static str> = + cap.default_effort.map(|e| e.openai_effort_str()); + let new_result = (new_values, new_default); + + if in_allowlist { + // Intentional divergence — skip equality check. + continue; + } + + assert_eq!( + old_result, new_result, + "effort differential divergence for fixture entry \"{label}\" \ + (provider={}, model={}): old={old_result:?} new={new_result:?}", + entry.provider, entry.model, + ); + } + } + #[test] fn resolve_provider_openrouter_with_key() { assert_eq!( diff --git a/crates/buzz-agent/src/llm.rs b/crates/buzz-agent/src/llm.rs index f595a165e57..0b33b260ec0 100644 --- a/crates/buzz-agent/src/llm.rs +++ b/crates/buzz-agent/src/llm.rs @@ -1095,25 +1095,24 @@ fn is_responses_required_error(body: &str) -> bool { || b.contains("use the responses api") } -/// OpenAI-family code names that appear as their own segment in a Databricks v2 -/// endpoint name (the GPT-5 launch aliases). The `gpt` family itself is matched -/// separately by segment prefix so `gpt`, `gpt5`, and the `gpt` of a split -/// `gpt-5` all qualify. -const DATABRICKS_V2_OPENAI_CODE_NAMES: &[&str] = &["sol", "luna", "terra"]; - -/// Anthropic (Claude) family and release code names that appear as their own -/// segment in a Databricks v2 endpoint name — the `claude` prefix, the family -/// names (`opus`, `sonnet`, `haiku`), and the release code names (`mythos`, -/// `fable`). Getting a Claude model onto the Anthropic Messages route is what -/// lets it carry a `cache_control` breakpoint; an endpoint that matches none of -/// these falls through to the MLflow (OpenAI-wire) path, where Anthropic prompt -/// caching is structurally impossible and the discount is silently lost. -const DATABRICKS_V2_CLAUDE_NAMES: &[&str] = +/// OpenAI-family code names used by the OLD segment-based route classifier. +/// Preserved for the Phase-2 differential harness and Phase-3 cleanup. +/// Production routing now delegates to `resolve_model_capabilities` (see +/// `databricks_v2_route_for_model` below). +#[cfg(test)] +const _OLD_DATABRICKS_V2_OPENAI_CODE_NAMES: &[&str] = &["sol", "luna", "terra"]; + +/// Anthropic (Claude) family and release code names used by the OLD classifier. +/// Preserved for the Phase-2 differential harness and Phase-3 cleanup. +#[cfg(test)] +const _OLD_DATABRICKS_V2_CLAUDE_NAMES: &[&str] = &["claude", "opus", "sonnet", "haiku", "mythos", "fable"]; /// Split a Databricks v2 endpoint name into its lowercase alphanumeric segments, /// breaking on any non-alphanumeric delimiter (`-`, `_`, `.`, `/`, …). E.g. /// `Databricks-Claude-Opus-5` -> `["databricks", "claude", "opus", "5"]`. +/// Used by the old classifier (differential harness). Phase 3 removes this. +#[cfg(test)] fn model_name_segments(model: &str) -> Vec { model .split(|c: char| !c.is_ascii_alphanumeric()) @@ -1122,32 +1121,48 @@ fn model_name_segments(model: &str) -> Vec { .collect() } -fn databricks_v2_route_for_model(model: &str) -> DatabricksV2Route { - // The v2 catalog exposes no family field, so the wire format is inferred - // from the endpoint name. Discovery deliberately keeps arbitrary custom - // aliases, so we match whole name *segments* rather than raw substrings: a - // substring test would misroute unrelated names — `consolidated-llama` - // (`sol`), `terraform-coder` (`terra`), `corpus-reranker`/`octopus-model` - // (`opus`) — onto a wire whose request shape their backend can't parse, - // turning a caching optimization into a hard request/parse failure. Segment - // matching still accepts real prefixed names like `goose-opus-5`. +/// OLD segment-based route classifier — preserved for the Phase-2 differential +/// harness. Production routing now delegates to `databricks_v2_route_for_model`. +/// Phase 3 removes this function. +#[cfg(test)] +fn _old_databricks_v2_route_for_model(model: &str) -> DatabricksV2Route { let segments = model_name_segments(model); let has_named_segment = |names: &[&str]| segments.iter().any(|seg| names.contains(&seg.as_str())); - // `gpt` family: any segment beginning with `gpt` — covers `gpt`, `gpt5`, and - // the `gpt` segment of a split `gpt-5`, without matching mid-word. let is_gpt_family = segments.iter().any(|seg| seg.starts_with("gpt")); - // OpenAI is checked before Claude so a name carrying both markers resolves - // to the OpenAI wire (preserving the prior `gpt-5`-first precedence). - if is_gpt_family || has_named_segment(DATABRICKS_V2_OPENAI_CODE_NAMES) { + if is_gpt_family || has_named_segment(_OLD_DATABRICKS_V2_OPENAI_CODE_NAMES) { DatabricksV2Route::OpenAiResponses - } else if has_named_segment(DATABRICKS_V2_CLAUDE_NAMES) { + } else if has_named_segment(_OLD_DATABRICKS_V2_CLAUDE_NAMES) { DatabricksV2Route::AnthropicMessages } else { DatabricksV2Route::MlflowChatCompletions } } +/// Returns the Databricks v2 wire route for a model name. +/// +/// Phase 2 cutover: delegates to `resolve_model_capabilities` from the generated +/// capability module. The generated resolver uses the same segment-based matching +/// logic, now derived from the manifest single source of truth. +/// +/// `RouteUnknown` (blank model) and `NotApplicable` (non-DBv2 provider) are not +/// reachable here — this function is only called for DBv2 requests with an +/// effective model string — both map to `MlflowChatCompletions` as a safe fallback. +fn databricks_v2_route_for_model(model: &str) -> DatabricksV2Route { + use crate::generated_model_capabilities::{ + resolve_model_capabilities, DatabricksV2Route as GenRoute, + }; + match resolve_model_capabilities("databricks_v2", model).databricks_v2_wire_route { + GenRoute::OpenAiResponses => DatabricksV2Route::OpenAiResponses, + GenRoute::AnthropicMessages => DatabricksV2Route::AnthropicMessages, + // RouteUnknown (blank model) and NotApplicable (non-DBv2) are structurally + // unreachable from this call site; fall through to the mlflow path. + GenRoute::MlflowChatCompletions | GenRoute::RouteUnknown | GenRoute::NotApplicable => { + DatabricksV2Route::MlflowChatCompletions + } + } +} + fn databricks_v2_path(route: DatabricksV2Route) -> &'static str { match route { DatabricksV2Route::OpenAiResponses => "/ai-gateway/openai/v1/responses", @@ -3335,6 +3350,51 @@ mod tests { } } + /// Phase-2 differential: new generated route matches old segment-based route + /// for all models in the existing test suite. Documents where they agree and + /// flags unexpected divergence. Any intentional divergence must be added to + /// the allowlist in this test. + #[test] + fn databricks_v2_route_differential_old_vs_new() { + // Models where old and new are intentionally expected to differ. + // (Empty: the generated manifest uses identical segment logic.) + let allowlist: &[&str] = &[]; + + for model in [ + "databricks-gpt-5-5", + "gpt-4o", + "gpt5", + "databricks-gpt-5-6-luna", + "databricks-gpt-5-6-sol", + "databricks-terra", + "databricks-claude-opus-4-7", + "goose-opus-5", + "databricks-sonnet-5", + "databricks-haiku-4-5", + "databricks-mythos-5", + "databricks-fable-5", + "Databricks-Claude-Opus-5", + "custom-tool-model", + "databricks-gemini-3-pro", + "consolidated-llama", + "terraform-coder", + "corpus-reranker", + "octopus-model", + "", + ] { + let old_route = _old_databricks_v2_route_for_model(model); + let new_route = databricks_v2_route_for_model(model); + if allowlist.contains(&model) { + // Intentional divergence — just document, don't assert equality. + continue; + } + assert_eq!( + old_route, new_route, + "route divergence for model={model:?}: old={old_route:?} new={new_route:?}" + ); + } + } + #[test] fn parse_responses_rejects_malformed_function_arguments() { let v = serde_json::json!({ diff --git a/scripts/MODELS_DEV_RECONCILIATION.md b/scripts/MODELS_DEV_RECONCILIATION.md index 3fff89a9ce7..59722e97e5c 100644 --- a/scripts/MODELS_DEV_RECONCILIATION.md +++ b/scripts/MODELS_DEV_RECONCILIATION.md @@ -1,7 +1,7 @@ # models.dev Reasoning Options Reconciliation Table -**Source queried**: https://models.dev/api.json (2026-07-31) -**Payload SHA-256**: `d5a4974cd69f19b0f67713acaa6bb3b16e920defdc07ecbdf6b0a936181bb0e0` +**Source queried**: https://models.dev/api.json (2026-07-31)
+**Payload SHA-256**: `d5a4974cd69f19b0f67713acaa6bb3b16e920defdc07ecbdf6b0a936181bb0e0`
**Policy (plan v4 §Behavior policy)**: models.dev `reasoning_options` become exact overrides. Each divergence from the current family rule result is reconciled here: either (a) adopted as an intentional correction or (b) rejected with a curation note. @@ -23,8 +23,8 @@ advertises only `[low, medium, high]` in its `reasoning_options`. The family rul `xhigh` are derived from the upstream OpenAI GPT-5.4 spec, which this Databricks endpoint does not expose. Provider-advertised wins per plan F1 policy. -**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-4-mini"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]` -**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-4-mini"` +**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-4-mini"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]`
+**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-4-mini"`
**Test vector**: `resolver-exact-raw-id-hit` in `scripts/normative-corpus.json` --- @@ -38,7 +38,7 @@ not expose. Provider-advertised wins per plan F1 policy. **Rationale**: Same as `databricks-gpt-5-4-mini`. The nano variant exposes the same restricted effort set. Provider-advertised wins. -**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-4-nano"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]` +**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-4-nano"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]`
**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-4-nano"` --- @@ -54,7 +54,7 @@ effort set. Provider-advertised wins. derived from the upstream OpenAI GPT-5.6 spec, which this Databricks endpoint does not expose. Provider-advertised wins per plan F1 policy. -**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-6-sol"].reasoning_options = [{"type":"effort","values":["low","medium","high","max"]}]` +**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-6-sol"].reasoning_options = [{"type":"effort","values":["low","medium","high","max"]}]`
**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-6-sol"` --- @@ -70,7 +70,7 @@ Provider-advertised wins per plan F1 policy. derived from the upstream OpenAI GPT-5.5 spec, which this Databricks endpoint does not expose. Provider-advertised wins per plan F1 policy. -**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-5"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]` +**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-gpt-5-5"].reasoning_options = [{"type":"effort","values":["low","medium","high"]}]`
**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-gpt-5-5"` --- @@ -86,7 +86,7 @@ a different capability axis (extended thinking token budget), not an effort-leve There is no effort divergence to reconcile. The effort capabilities for this model come from the `anthropic-adaptive-xhigh-opus-4-7` family rule (Anthropic extended-thinking support table). -**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-claude-opus-4-7"].reasoning_options = [{"type":"budget_tokens","min":1024}]` +**Source**: [https://models.dev/api.json](https://models.dev/api.json) — retrieved 2026-07-31; `providers.databricks.models["databricks-claude-opus-4-7"].reasoning_options = [{"type":"budget_tokens","min":1024}]`
**Snapshot**: `scripts/catalog-sample-fixture.json` key `"databricks-claude-opus-4-7"` --- diff --git a/scripts/run-differential.mjs b/scripts/run-differential.mjs new file mode 100755 index 00000000000..c1d9fe63e63 --- /dev/null +++ b/scripts/run-differential.mjs @@ -0,0 +1,201 @@ +#!/usr/bin/env node +/** + * Phase-2 differential harness — compare old buzzAgentConfig.ts effort logic with + * the new generated modelCapabilities.ts interpreter over: + * 1. The 36-entry effortTable.fixture.json (cross-boundary Rust/TS fixture) + * 2. The 45-vector normative corpus (scripts/normative-corpus.json) + * 3. The catalog-sample fixture (scripts/catalog-sample-fixture.json) + * + * Equality is required except for entries in the committed allowlist of intentional + * F1 corrections (models.dev provider-capability reconciliations). + * + * Usage: node --experimental-strip-types scripts/run-differential.mjs [--verbose] + * Exits 0 on all-pass (modulo allowlist), 1 on unexpected divergence. + */ + +import { readFileSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +const __dirname = dirname(fileURLToPath(import.meta.url)); +const repoRoot = join(__dirname, ".."); +const VERBOSE = process.argv.includes("--verbose"); + +// --------------------------------------------------------------------------- +// Import both interpreters +// --------------------------------------------------------------------------- + +// NEW: generated capability module +const { resolveModelCapabilities: resolveNew } = await import( + join(repoRoot, "desktop", "src", "features", "agents", "ui", "modelCapabilities.ts") +); + +// OLD: buzzAgentConfig.ts effort config +const { getProviderEffortConfig: getOldEffortConfig } = await import( + join(repoRoot, "desktop", "src", "features", "agents", "ui", "buzzAgentConfig.ts") +); + +// --------------------------------------------------------------------------- +// Intentional corrections allowlist (Phase 1 F1 reconciliations) +// Each entry: { provider, raw_model_id, reason } +// --------------------------------------------------------------------------- +const ALLOWLIST = [ + { + provider: "databricks_v2", + raw_model_id: "databricks-gpt-5-5", + axes: ["supported_efforts"], + reason: "Phase 1 ADOPT: models.dev d5a4974c advertises [low,medium,high]; old returns [none,low,medium,high,xhigh]", + }, + { + provider: "databricks_v2", + raw_model_id: "databricks-gpt-5-4-mini", + axes: ["supported_efforts"], + reason: "Phase 1 ADOPT: models.dev advertises [low,medium,high]; old returns [none,low,medium,high,xhigh]", + }, + { + provider: "databricks_v2", + raw_model_id: "databricks-gpt-5-4-nano", + axes: ["supported_efforts"], + reason: "Phase 1 ADOPT: models.dev advertises [low,medium,high]; old returns [none,low,medium,high,xhigh]", + }, + { + provider: "databricks_v2", + raw_model_id: "databricks-gpt-5-6-sol", + axes: ["supported_efforts"], + reason: "Phase 1 ADOPT: models.dev advertises [low,medium,high,max]; old returns [none,low,medium,high,xhigh,max]", + }, + { + provider: "databricks_v2", + raw_model_id: "goose-opus-5", + axes: ["supported_efforts", "default_effort"], + reason: "Phase 1 correction: 'opus' is a named DBv2 segment → anthropic-messages route; old config.rs disagreed with llm.rs (corpus note dbv2-goose-opus-5-is-anthropic). Generated adopts anthropic adaptive-xhigh capabilities consistent with the wire route.", + }, +]; + +function isAllowlisted(provider, rawModelId, axis) { + return ALLOWLIST.some( + (e) => + e.provider === provider && + e.raw_model_id === rawModelId && + e.axes.includes(axis), + ); +} + +// --------------------------------------------------------------------------- +// Comparison helpers +// --------------------------------------------------------------------------- + +/** + * Compare effort axes from both interpreters for one (provider, model) pair. + * Returns array of divergence objects. + */ +function compareEffortAxes(provider, model) { + const newResult = resolveNew(provider, model); + const oldResult = getOldEffortConfig(provider, model); + + const divergences = []; + + // supported_efforts + const newEfforts = newResult.supportedEfforts ?? []; + const oldEfforts = oldResult?.validValues ?? []; + if (JSON.stringify(newEfforts) !== JSON.stringify(oldEfforts)) { + if (!isAllowlisted(provider, model, "supported_efforts")) { + divergences.push({ + axis: "supported_efforts", + old: oldEfforts, + new: newEfforts, + }); + } + } + + // default_effort + const newDefault = newResult.defaultEffort ?? null; + const oldDefault = oldResult?.defaultValue ?? null; + if (newDefault !== oldDefault) { + if (!isAllowlisted(provider, model, "default_effort")) { + divergences.push({ + axis: "default_effort", + old: oldDefault, + new: newDefault, + }); + } + } + + return divergences; +} + +// --------------------------------------------------------------------------- +// Test suites +// --------------------------------------------------------------------------- + +let totalChecks = 0; +let totalDivergences = 0; +let totalAllowlisted = 0; + +function runCheck(label, provider, model) { + totalChecks++; + const divs = compareEffortAxes(provider, model); + if (divs.length > 0) { + totalDivergences += divs.length; + for (const d of divs) { + console.error( + `DIVERGE [${label}] provider=${provider} model=${model} axis=${d.axis}\n` + + ` old: ${JSON.stringify(d.old)}\n` + + ` new: ${JSON.stringify(d.new)}`, + ); + } + } else if (VERBOSE) { + console.log(`OK [${label}] provider=${provider} model=${model}`); + } +} + +// 1. effortTable.fixture.json +console.log("--- effortTable.fixture.json ---"); +const fixture = JSON.parse( + readFileSync( + join(repoRoot, "desktop", "src", "features", "agents", "ui", "effortTable.fixture.json"), + "utf8", + ), +); +for (const entry of fixture) { + if (!entry.provider) continue; + runCheck("fixture", entry.provider, entry.model ?? ""); +} + +// 2. normative-corpus.json (effort axes only) +console.log("--- normative-corpus.json ---"); +const corpus = JSON.parse( + readFileSync(join(repoRoot, "scripts", "normative-corpus.json"), "utf8"), +); +for (const entry of corpus) { + if (entry._group) continue; + if (!entry.provider || !entry.expect) continue; + if (!entry.expect.supported_efforts && !entry.expect.default_effort) continue; + runCheck("corpus", entry.provider, entry.raw_model_id ?? ""); +} + +// 3. catalog-sample-fixture.json (exact records from pinned models.dev payload) +console.log("--- catalog-sample-fixture.json ---"); +const catalogFixture = JSON.parse( + readFileSync(join(repoRoot, "scripts", "catalog-sample-fixture.json"), "utf8"), +); +for (const ep of catalogFixture.endpoints ?? []) { + if (!ep.name) continue; + // All catalog endpoints are databricks_v2 provider + runCheck("catalog-sample", "databricks_v2", ep.name); +} + +// --------------------------------------------------------------------------- +// Summary +// --------------------------------------------------------------------------- +console.log( + `\nDifferential: ${totalChecks} checks, ${totalDivergences} unexpected divergences, ${totalAllowlisted} allowlisted`, +); +if (totalDivergences > 0) { + console.error( + `FAIL: ${totalDivergences} unexpected divergence(s) — see output above`, + ); + process.exit(1); +} else { + console.log("PASS: old and new effort logic agree on all non-allowlisted entries"); +} From ae3879c53a05bbc80f8081eb3f4c13f18d16a79e Mon Sep 17 00:00:00 2001 From: npub1g8493u0xfsjrvflg4n08ezd7vec99mnwzlv0qgwpr9d7gvjwhuzqx59rhw <41ea58f1e64c243627e8acde7c89be667052ee6e17d8f021c1195be4324ebf04@buzz.block.builderlab.xyz> Date: Fri, 31 Jul 2026 12:49:10 -0400 Subject: [PATCH 02/14] =?UTF-8?q?feat(desktop):=20Phase=202b=20=E2=80=94?= =?UTF-8?q?=20cut=20TS=20consumers=20to=20generated=20model-capabilities?= =?UTF-8?q?=20module?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit buzzAgentConfig.ts adds getProviderEffortConfigFromManifest(), a thin wrapper over resolveModelCapabilities() from the generated modelCapabilities.ts module. Both the old getProviderEffortConfig() hand-tables and the new manifest path are live; Phase 3 retires the old tables once the differential harness confirms equality. formatAgentModelLabel.ts re-points its registry-label lookup from the hand-maintained databricksModelNames.ts import to DATABRICKS_MODEL_NAMES exported from modelCapabilities.ts (the generated manifest source). The map shape and contents are identical; behavior is unchanged. Acceptance: typecheck clean, 3847/3847 desktop unit tests pass, biome check clean. No hand-edited capability literals added. Co-authored-by: Will Pfleger Signed-off-by: Will Pfleger --- .../agents/lib/formatAgentModelLabel.ts | 4 +-- .../src/features/agents/ui/buzzAgentConfig.ts | 33 +++++++++++++++++-- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/desktop/src/features/agents/lib/formatAgentModelLabel.ts b/desktop/src/features/agents/lib/formatAgentModelLabel.ts index e0595e4f672..7f83fe7047b 100644 --- a/desktop/src/features/agents/lib/formatAgentModelLabel.ts +++ b/desktop/src/features/agents/lib/formatAgentModelLabel.ts @@ -1,11 +1,11 @@ -import { DATABRICKS_MODEL_NAMES } from "./databricksModelNames"; +import { DATABRICKS_MODEL_NAMES } from "../ui/modelCapabilities"; /** * Resolves a human-readable label for a model, following the three-tier * precedence documented in AGENTS.md: * * 1. Nonblank discovered/API name (e.g. from AgentModelInfo.name) - * 2. Registry lookup by ID (models.dev-seeded Databricks table) + * 2. Registry lookup by ID (generated registry_label table from model-capabilities manifest) * 3. Raw ID unchanged * * Returns the empty string when both id and discoveredName are blank. diff --git a/desktop/src/features/agents/ui/buzzAgentConfig.ts b/desktop/src/features/agents/ui/buzzAgentConfig.ts index be663c35cb4..e9c931a9800 100644 --- a/desktop/src/features/agents/ui/buzzAgentConfig.ts +++ b/desktop/src/features/agents/ui/buzzAgentConfig.ts @@ -1,9 +1,11 @@ /** * Source-of-truth constants for buzz-agent model-tuning configuration knobs. * - * Values must stay in sync with `crates/buzz-agent/src/config.rs` - * `parse_thinking_effort` — that function is the authoritative list. + * Phase 2b: getProviderEffortConfigFromManifest() is the new generated-manifest + * path. getProviderEffortConfig() (legacy hand-tables) stays live for the + * differential harness until Phase 3 retires it. */ +import { resolveModelCapabilities } from "./modelCapabilities"; /** Env var key for the thinking/effort level sent to the LLM. */ export const BUZZ_AGENT_THINKING_EFFORT = "BUZZ_AGENT_THINKING_EFFORT"; @@ -307,3 +309,30 @@ function openaiConfig(m: string): ProviderEffortConfig { export function isBuzzAgentRuntime(runtimeId: string): boolean { return runtimeId === "buzz-agent"; } + +// --------------------------------------------------------------------------- +// Generated-manifest path (Phase 2b) — thin lookup over resolveModelCapabilities +// --------------------------------------------------------------------------- + +/** + * Returns the valid thinking-effort values and semantic default for the + * given provider and model, resolved from the generated model-capabilities + * manifest (modelCapabilities.ts). + * + * This is the Phase 2b replacement path for getProviderEffortConfig(). + * Both paths are live until Phase 3 retires getProviderEffortConfig(). + * + * The manifest's `supportedEfforts` maps to `validValues`; `defaultEffort` + * (which may be null for manual-budget models — "Inherit" is the natural + * default) maps to `defaultValue`. + */ +export function getProviderEffortConfigFromManifest( + providerId: string, + model?: string, +): ProviderEffortConfig { + const cap = resolveModelCapabilities(providerId, model ?? ""); + return { + validValues: cap.supportedEfforts, + defaultValue: cap.defaultEffort, + }; +} From ffc46676c71362f83f1033589d193456e2e1816d Mon Sep 17 00:00:00 2001 From: npub1mn7jgtj4w2pd0g0zeuhxsa6jy6p0rewxz4kujt98my82ahfmp72sxjexk7 Date: Fri, 31 Jul 2026 13:02:16 -0400 Subject: [PATCH 03/14] fix(scripts): add ts-esm-loader and fix allowlist coverage in run-differential MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2b introduced a transitive extensionless TS import in buzzAgentConfig.ts ("./modelCapabilities") that Node's --experimental-strip-types cannot resolve without help. Add scripts/ts-esm-loader.mjs — a minimal ESM custom loader that patches extensionless relative imports to .ts files when the file exists on disk. Update run-differential.mjs shebang to self-bootstrap with the loader. Also fix the allowlist coverage gate: totalAllowlisted was declared but never incremented, so the summary always printed "0 allowlisted" and stale allowlist entries (which mask future regressions) went undetected. Replace with per-axis hit tracking via allowlistHits Set; report exercised slot count (N/total) in the summary; FAIL with STALE_ALLOWLIST if any declared entry fires zero divergences. Co-authored-by: Will Pfleger Signed-off-by: Will Pfleger --- scripts/run-differential.mjs | 57 ++++++++++++++++++++++++++++++++---- scripts/ts-esm-loader.mjs | 27 +++++++++++++++++ 2 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 scripts/ts-esm-loader.mjs diff --git a/scripts/run-differential.mjs b/scripts/run-differential.mjs index c1d9fe63e63..9862ec25e5d 100755 --- a/scripts/run-differential.mjs +++ b/scripts/run-differential.mjs @@ -1,4 +1,5 @@ -#!/usr/bin/env node +#!/bin/sh +// 2>/dev/null; LOADER="$(cd "$(dirname "$0")" && pwd)/ts-esm-loader.mjs"; exec node --experimental-strip-types --loader "$LOADER" "$0" "$@" /** * Phase-2 differential harness — compare old buzzAgentConfig.ts effort logic with * the new generated modelCapabilities.ts interpreter over: @@ -9,8 +10,14 @@ * Equality is required except for entries in the committed allowlist of intentional * F1 corrections (models.dev provider-capability reconciliations). * - * Usage: node --experimental-strip-types scripts/run-differential.mjs [--verbose] - * Exits 0 on all-pass (modulo allowlist), 1 on unexpected divergence. + * Usage: node scripts/run-differential.mjs [--verbose] + * Exits 0 on all-pass (modulo allowlist), 1 on unexpected divergence or unexercised allowlist entry. + * + * NOTE: The shebang bootstraps --experimental-strip-types and a custom ESM loader + * (ts-esm-loader.mjs) that resolves extensionless relative TS imports. This is + * required because buzzAgentConfig.ts (Phase 2b) imports modelCapabilities via a + * bare specifier ("./modelCapabilities") that Node's strip-types runner cannot + * otherwise resolve. */ import { readFileSync } from "node:fs"; @@ -72,13 +79,22 @@ const ALLOWLIST = [ }, ]; +// Track which allowlist entries are actually exercised (suppressed a divergence). +// Keyed as "provider:raw_model_id:axis". +const allowlistHits = new Set(); + function isAllowlisted(provider, rawModelId, axis) { - return ALLOWLIST.some( + const entry = ALLOWLIST.find( (e) => e.provider === provider && e.raw_model_id === rawModelId && e.axes.includes(axis), ); + if (entry) { + allowlistHits.add(`${provider}:${rawModelId}:${axis}`); + return true; + } + return false; } // --------------------------------------------------------------------------- @@ -130,7 +146,6 @@ function compareEffortAxes(provider, model) { let totalChecks = 0; let totalDivergences = 0; -let totalAllowlisted = 0; function runCheck(label, provider, model) { totalChecks++; @@ -188,14 +203,44 @@ for (const ep of catalogFixture.endpoints ?? []) { // --------------------------------------------------------------------------- // Summary // --------------------------------------------------------------------------- + +// Count total allowlist axis slots expected to be hit +const totalAllowlistSlots = ALLOWLIST.reduce((n, e) => n + e.axes.length, 0); +const allowlistHitCount = allowlistHits.size; + +// Detect stale allowlist entries (declared but never actually suppressed a divergence) +const staleEntries = []; +for (const entry of ALLOWLIST) { + for (const axis of entry.axes) { + const key = `${entry.provider}:${entry.raw_model_id}:${axis}`; + if (!allowlistHits.has(key)) { + staleEntries.push({ ...entry, axis }); + } + } +} + console.log( - `\nDifferential: ${totalChecks} checks, ${totalDivergences} unexpected divergences, ${totalAllowlisted} allowlisted`, + `\nDifferential: ${totalChecks} checks, ${totalDivergences} unexpected divergences, ${allowlistHitCount}/${totalAllowlistSlots} allowlist slots exercised`, ); + +if (staleEntries.length > 0) { + for (const e of staleEntries) { + console.error( + `STALE_ALLOWLIST provider=${e.provider} model=${e.raw_model_id} axis=${e.axis} — entry never fired; remove or update it`, + ); + } +} + if (totalDivergences > 0) { console.error( `FAIL: ${totalDivergences} unexpected divergence(s) — see output above`, ); process.exit(1); +} else if (staleEntries.length > 0) { + console.error( + `FAIL: ${staleEntries.length} stale allowlist entry(ies) — entries that never suppress a divergence mask future regressions`, + ); + process.exit(1); } else { console.log("PASS: old and new effort logic agree on all non-allowlisted entries"); } diff --git a/scripts/ts-esm-loader.mjs b/scripts/ts-esm-loader.mjs new file mode 100644 index 00000000000..a3dc09475fa --- /dev/null +++ b/scripts/ts-esm-loader.mjs @@ -0,0 +1,27 @@ +/** + * Minimal ESM custom loader for the differential harness. + * + * Node's --experimental-strip-types resolves .ts files by absolute URL but + * does NOT add .ts to extensionless relative imports emitted by TypeScript + * source (e.g. `import ... from "./modelCapabilities"`). This loader patches + * that gap: when a relative import has no extension and a same-named .ts file + * exists on disk, it rewrites the specifier to the .ts URL before resolution. + * + * Usage: node --experimental-strip-types --loader scripts/ts-esm-loader.mjs