From 14708e33f4c116869164b0cb9f94ff2ccf5bf5e8 Mon Sep 17 00:00:00 2001 From: codeitlikemiley Date: Mon, 24 Aug 2026 11:52:16 +0800 Subject: [PATCH] M12.5 groundwork: record what the classifier said, not only what the router used MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Task 7 of the run brief asked me to wire `ShadowClassifier` behind a flag, "zero production call sites". Two things turned out otherwise, and the second is the reason this commit exists. **The seam already exists.** `GatewayBuilder::classifier()` takes an `Arc`, and `ShadowClassifier` *is* a `Classifier` that returns the incumbent's answer by construction. Wiring it is one line. The blocker is not a missing flag — it is that there is no candidate classifier to shadow, and there will not be until M19.3. A flag guarding a comparison against nothing would be ceremony. **But the stated goal — accumulate routing evidence before a model exists — had a real gap, upstream of shadow mode.** `Gateway::resolve` read: let (task, _confidence, _trusted) = classify_or_default(..) Both discarded. So `route_decisions` recorded the class the router *used* and nothing about how it was arrived at: a row saying `chat` means either "the heuristic was certain" or "it guessed something at 0.2 and docs/12's confidence gate fell back to chat". Those are opposite facts about the same field, and the difference is exactly the label quality a learned router (M19.3) would train on. Every request was discarding it, unrecoverably. Migration 0011 adds `confidence real` and `trusted boolean`, nullable and with no DEFAULT — rows written before this genuinely have no value, and 0.0 would read as "the classifier was certain of nothing" rather than "nobody asked". A gap is visibly a gap. Both columns are content-free like the rest of the table (docs/20 T5): a float and a bool say nothing about what the user typed. Also fixed, because I was editing the line it sits on: the comment there claimed "A declared class wins; otherwise classify". It does not — `CallMeta.task` is never read, so a caller declaring `Code` gets whatever the heuristic guesses. docs/12 says "Callers that know, say", so the spec and the code disagree and the spec is not the one that is wrong. Latent (no ingress populates the field), and honouring it changes routing, so it is a question in RUN-REPORT.md and its own commit — not a silent behaviour change smuggled in here. The false comment is replaced by an accurate one rather than left to mislead. The round-trip test uses `routes::insert` rather than `PgRouteAudit::record`, because the trait impl detaches the write onto a background task so inference never waits on the database. Reading straight after `record` races the spawn — it did, once, with `RowNotFound`. What is under test is the columns, not the detachment. Verified: integration lane 149/149 twice; unbinding the two columns turns the new test red and nothing else. fmt, clippy -D warnings, 1114 workspace tests, schemas, ts-sdk, sbom, deny. Claude-Session: https://claude.ai/code/session_017kFpYDqvz6sKGSkM4YKaRf --- crates/panday-gateway/src/gateway.rs | 39 ++++++++++++++-- .../migrations/0011_route_confidence.sql | 16 +++++++ crates/panday-platform/src/pg.rs | 4 ++ crates/panday-platform/src/routes.rs | 7 ++- .../panday-platform/tests/route_audit_pg.rs | 44 +++++++++++++++++++ 5 files changed, 104 insertions(+), 6 deletions(-) create mode 100644 crates/panday-platform/migrations/0011_route_confidence.sql diff --git a/crates/panday-gateway/src/gateway.rs b/crates/panday-gateway/src/gateway.rs index 3b5d467..8d05046 100644 --- a/crates/panday-gateway/src/gateway.rs +++ b/crates/panday-gateway/src/gateway.rs @@ -134,6 +134,20 @@ pub struct RouteRecord { /// Legs actually attempted. `1` is the healthy case; more is failover, and a rule whose /// attempts climb is a rule pointing at a sick provider. pub attempts: u32, + /// How sure the classifier was, 0..1. + /// + /// Recorded because `task` alone cannot be read back: a row saying `chat` means either "the + /// heuristic was certain" or "it guessed something at 0.2 and the gate fell back to `chat`", + /// and those are opposite facts about the same field. Until now both were discarded at the + /// call site (`let (task, _confidence, _trusted) = ...`), so every request threw away the one + /// signal a learned router would be trained on (M19.3). + /// + /// Content-free, like every other column here (docs/20 T5): a float and a bool say nothing + /// about what the user typed. + pub confidence: f32, + /// Whether the classifier's guess cleared docs/12's confidence gate. `false` means `task` is + /// the fallback, not the guess — which is exactly the distinction a training label needs. + pub trusted: bool, } /// Where routing decisions go. Postgres implements this (`panday_platform::routes`). @@ -291,10 +305,18 @@ impl Gateway { /// here" is a routing outcome worth *recording*, not just an error to return: a rule whose pool /// names models this deployment has no adapter for is invisible otherwise (M12.2). pub fn resolve(&self, req: &ChatRequest) -> Result { - // A declared class wins; otherwise classify, and fall back to `Chat` - // when the guess is not trusted (docs/12: confidence "gates whether we - // trust it"). - let (task, _confidence, _trusted) = classify_or_default( + // Classify, and fall back to `Chat` when the guess is not trusted (docs/12: confidence + // "gates whether we trust it"). + // + // This comment used to open "A declared class wins; otherwise classify", which the code + // does not do: `CallMeta.task` is never read here, so a caller that declared `Code` gets + // whatever the heuristic guesses. docs/12 says "Callers that know, say", so the spec and + // the code disagree and the spec is not the one that is wrong. Latent today — no ingress + // populates `CallMeta.task` — but it is a protocol field that reads like a control and + // controls nothing, which is the shape of defect this session has already found twice. + // Left as a behaviour question rather than changed here, because honouring it changes + // routing and belongs in its own commit. + let (task, confidence, trusted) = classify_or_default( self.classifier.as_ref(), req, panday_types::model::TaskClass::Chat, @@ -354,6 +376,8 @@ impl Gateway { Ok(Resolution { decision, task, + confidence, + trusted, usable, skipped, }) @@ -389,6 +413,11 @@ pub struct Resolution { pub decision: panday_router::RouteDecision, /// The class the router keyed on — declared, or the classifier's guess. pub task: panday_types::model::TaskClass, + /// The classifier's confidence, and whether it cleared docs/12's gate. Always produced — + /// `classify_or_default` classifies every request — and carried here so the audit can record + /// what the classifier *said*, not only what the router used. See `RouteRecord::confidence`. + pub confidence: f32, + pub trusted: bool, /// Targets with an adapter behind them, in failover order. pub usable: Vec, /// Targets that were dropped, each with the reason. The message a caller sees when `usable` is @@ -573,6 +602,8 @@ impl Gateway { .map(|m| m.0.clone()) .collect(), chosen: None, + confidence: resolution.confidence, + trusted: resolution.trusted, attempts: 0, }; diff --git a/crates/panday-platform/migrations/0011_route_confidence.sql b/crates/panday-platform/migrations/0011_route_confidence.sql new file mode 100644 index 0000000..75a1429 --- /dev/null +++ b/crates/panday-platform/migrations/0011_route_confidence.sql @@ -0,0 +1,16 @@ +-- M12.5 groundwork: what the classifier said, not only what the router used. +-- +-- `task` alone cannot be read back. A row saying `chat` means either "the heuristic was certain" +-- or "it guessed something at 0.2 and docs/12's confidence gate fell back to chat" — opposite +-- facts about the same field. Both were discarded at the call site +-- (`let (task, _confidence, _trusted) = ...`), so every request threw away the one signal a +-- learned router (M19.3) would be trained on, and no amount of later analysis could recover it. +-- +-- tenant-scoping: inherited. These are columns on `route_decisions`, which is already scoped by +-- `account_id` and indexed on it; nothing here changes what a query must filter by. +-- +-- Nullable, because rows written before this migration genuinely have no value. A DEFAULT would +-- put a number on history that was never measured, which is worse than a gap: a gap is visibly a +-- gap, and 0.0 reads as "the classifier was certain of nothing" rather than "nobody asked". +ALTER TABLE route_decisions ADD COLUMN IF NOT EXISTS confidence real; +ALTER TABLE route_decisions ADD COLUMN IF NOT EXISTS trusted boolean; diff --git a/crates/panday-platform/src/pg.rs b/crates/panday-platform/src/pg.rs index 0c186e8..9407371 100644 --- a/crates/panday-platform/src/pg.rs +++ b/crates/panday-platform/src/pg.rs @@ -126,6 +126,10 @@ pub const EMBEDDED_MIGRATIONS: &[(&str, &str)] = &[ "0010_exact_cache.sql", include_str!("../migrations/0010_exact_cache.sql"), ), + ( + "0011_route_confidence.sql", + include_str!("../migrations/0011_route_confidence.sql"), + ), ]; /// Apply the compiled-in migrations. What a deployed service calls. diff --git a/crates/panday-platform/src/routes.rs b/crates/panday-platform/src/routes.rs index 68b6017..2066b36 100644 --- a/crates/panday-platform/src/routes.rs +++ b/crates/panday-platform/src/routes.rs @@ -45,8 +45,9 @@ impl RouteAudit for PgRouteAudit { pub async fn insert(pool: &PgPool, record: &RouteRecord) -> Result<(), PgError> { sqlx::query( "INSERT INTO route_decisions - (request_id, account_id, requested, task, matched_rule, pool, chain, chosen, attempts) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + (request_id, account_id, requested, task, matched_rule, pool, chain, chosen, attempts, + confidence, trusted) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT (request_id) DO NOTHING", ) .bind(record.request.0) @@ -58,6 +59,8 @@ pub async fn insert(pool: &PgPool, record: &RouteRecord) -> Result<(), PgError> .bind(serde_json::json!(record.chain)) .bind(record.chosen.as_deref()) .bind(record.attempts as i32) + .bind(record.confidence) + .bind(record.trusted) .execute(pool) .await .map_err(|e| PgError::Query(e.to_string()))?; diff --git a/crates/panday-platform/tests/route_audit_pg.rs b/crates/panday-platform/tests/route_audit_pg.rs index 54b0039..72012d3 100644 --- a/crates/panday-platform/tests/route_audit_pg.rs +++ b/crates/panday-platform/tests/route_audit_pg.rs @@ -29,6 +29,8 @@ fn record(account: uuid::Uuid, rule: &str, chosen: Option<&str>, attempts: u32) ], chosen: chosen.map(str::to_string), attempts, + confidence: 0.82, + trusted: true, } } @@ -138,3 +140,45 @@ async fn the_gateway_sink_writes_without_making_the_caller_wait() { } panic!("the audit row never landed"); } + +#[tokio::test] +#[ignore = "needs the integration lane (deploy/integration-compose.yml)"] +async fn what_the_classifier_said_survives_the_round_trip() { + // The point of 0011. `task` alone conflates "the heuristic was certain" with "it guessed + // something at 0.2 and the gate fell back", and a learned router (M19.3) is trained on exactly + // that distinction. Both were discarded before this, so the evidence a future model needs was + // being thrown away one request at a time. + let pool = database().await; + let account = pg::create_account(&pool, &format!("acme-{}", uuid::Uuid::now_v7())) + .await + .expect("account"); + + let mut untrusted = record(account, "rules[0]", Some("anthropic/claude-sonnet-4-5"), 1); + untrusted.confidence = 0.21; + untrusted.trusted = false; + let id = untrusted.request.0; + + // `routes::insert`, not `PgRouteAudit::record`: the trait impl detaches the write onto a + // background task so inference never waits on the database, so a test that read straight after + // it would race the spawn — and did, once, with `RowNotFound`. What is under test here is the + // column round trip, not the detachment. + routes::insert(&pool, &untrusted).await.expect("insert"); + + let (confidence, trusted): (Option, Option) = + sqlx::query_as("SELECT confidence, trusted FROM route_decisions WHERE request_id = $1") + .bind(id) + .fetch_one(&pool) + .await + .expect("the row"); + + assert_eq!( + trusted, + Some(false), + "the gate fired and the row must say so" + ); + let confidence = confidence.expect("a confidence was recorded"); + assert!( + (confidence - 0.21).abs() < 1e-6, + "confidence must survive as written, not rounded to a bucket: {confidence}" + ); +}