diff --git a/crates/panday-gateway/src/gateway.rs b/crates/panday-gateway/src/gateway.rs index 3b5d467..8d05046 100644 --- a/crates/panday-gateway/src/gateway.rs +++ b/crates/panday-gateway/src/gateway.rs @@ -134,6 +134,20 @@ pub struct RouteRecord { /// Legs actually attempted. `1` is the healthy case; more is failover, and a rule whose /// attempts climb is a rule pointing at a sick provider. pub attempts: u32, + /// How sure the classifier was, 0..1. + /// + /// Recorded because `task` alone cannot be read back: a row saying `chat` means either "the + /// heuristic was certain" or "it guessed something at 0.2 and the gate fell back to `chat`", + /// and those are opposite facts about the same field. Until now both were discarded at the + /// call site (`let (task, _confidence, _trusted) = ...`), so every request threw away the one + /// signal a learned router would be trained on (M19.3). + /// + /// Content-free, like every other column here (docs/20 T5): a float and a bool say nothing + /// about what the user typed. + pub confidence: f32, + /// Whether the classifier's guess cleared docs/12's confidence gate. `false` means `task` is + /// the fallback, not the guess — which is exactly the distinction a training label needs. + pub trusted: bool, } /// Where routing decisions go. Postgres implements this (`panday_platform::routes`). @@ -291,10 +305,18 @@ impl Gateway { /// here" is a routing outcome worth *recording*, not just an error to return: a rule whose pool /// names models this deployment has no adapter for is invisible otherwise (M12.2). pub fn resolve(&self, req: &ChatRequest) -> Result { - // A declared class wins; otherwise classify, and fall back to `Chat` - // when the guess is not trusted (docs/12: confidence "gates whether we - // trust it"). - let (task, _confidence, _trusted) = classify_or_default( + // Classify, and fall back to `Chat` when the guess is not trusted (docs/12: confidence + // "gates whether we trust it"). + // + // This comment used to open "A declared class wins; otherwise classify", which the code + // does not do: `CallMeta.task` is never read here, so a caller that declared `Code` gets + // whatever the heuristic guesses. docs/12 says "Callers that know, say", so the spec and + // the code disagree and the spec is not the one that is wrong. Latent today — no ingress + // populates `CallMeta.task` — but it is a protocol field that reads like a control and + // controls nothing, which is the shape of defect this session has already found twice. + // Left as a behaviour question rather than changed here, because honouring it changes + // routing and belongs in its own commit. + let (task, confidence, trusted) = classify_or_default( self.classifier.as_ref(), req, panday_types::model::TaskClass::Chat, @@ -354,6 +376,8 @@ impl Gateway { Ok(Resolution { decision, task, + confidence, + trusted, usable, skipped, }) @@ -389,6 +413,11 @@ pub struct Resolution { pub decision: panday_router::RouteDecision, /// The class the router keyed on — declared, or the classifier's guess. pub task: panday_types::model::TaskClass, + /// The classifier's confidence, and whether it cleared docs/12's gate. Always produced — + /// `classify_or_default` classifies every request — and carried here so the audit can record + /// what the classifier *said*, not only what the router used. See `RouteRecord::confidence`. + pub confidence: f32, + pub trusted: bool, /// Targets with an adapter behind them, in failover order. pub usable: Vec, /// Targets that were dropped, each with the reason. The message a caller sees when `usable` is @@ -573,6 +602,8 @@ impl Gateway { .map(|m| m.0.clone()) .collect(), chosen: None, + confidence: resolution.confidence, + trusted: resolution.trusted, attempts: 0, }; diff --git a/crates/panday-platform/migrations/0011_route_confidence.sql b/crates/panday-platform/migrations/0011_route_confidence.sql new file mode 100644 index 0000000..75a1429 --- /dev/null +++ b/crates/panday-platform/migrations/0011_route_confidence.sql @@ -0,0 +1,16 @@ +-- M12.5 groundwork: what the classifier said, not only what the router used. +-- +-- `task` alone cannot be read back. A row saying `chat` means either "the heuristic was certain" +-- or "it guessed something at 0.2 and docs/12's confidence gate fell back to chat" — opposite +-- facts about the same field. Both were discarded at the call site +-- (`let (task, _confidence, _trusted) = ...`), so every request threw away the one signal a +-- learned router (M19.3) would be trained on, and no amount of later analysis could recover it. +-- +-- tenant-scoping: inherited. These are columns on `route_decisions`, which is already scoped by +-- `account_id` and indexed on it; nothing here changes what a query must filter by. +-- +-- Nullable, because rows written before this migration genuinely have no value. A DEFAULT would +-- put a number on history that was never measured, which is worse than a gap: a gap is visibly a +-- gap, and 0.0 reads as "the classifier was certain of nothing" rather than "nobody asked". +ALTER TABLE route_decisions ADD COLUMN IF NOT EXISTS confidence real; +ALTER TABLE route_decisions ADD COLUMN IF NOT EXISTS trusted boolean; diff --git a/crates/panday-platform/src/pg.rs b/crates/panday-platform/src/pg.rs index 0c186e8..9407371 100644 --- a/crates/panday-platform/src/pg.rs +++ b/crates/panday-platform/src/pg.rs @@ -126,6 +126,10 @@ pub const EMBEDDED_MIGRATIONS: &[(&str, &str)] = &[ "0010_exact_cache.sql", include_str!("../migrations/0010_exact_cache.sql"), ), + ( + "0011_route_confidence.sql", + include_str!("../migrations/0011_route_confidence.sql"), + ), ]; /// Apply the compiled-in migrations. What a deployed service calls. diff --git a/crates/panday-platform/src/routes.rs b/crates/panday-platform/src/routes.rs index 68b6017..2066b36 100644 --- a/crates/panday-platform/src/routes.rs +++ b/crates/panday-platform/src/routes.rs @@ -45,8 +45,9 @@ impl RouteAudit for PgRouteAudit { pub async fn insert(pool: &PgPool, record: &RouteRecord) -> Result<(), PgError> { sqlx::query( "INSERT INTO route_decisions - (request_id, account_id, requested, task, matched_rule, pool, chain, chosen, attempts) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + (request_id, account_id, requested, task, matched_rule, pool, chain, chosen, attempts, + confidence, trusted) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) ON CONFLICT (request_id) DO NOTHING", ) .bind(record.request.0) @@ -58,6 +59,8 @@ pub async fn insert(pool: &PgPool, record: &RouteRecord) -> Result<(), PgError> .bind(serde_json::json!(record.chain)) .bind(record.chosen.as_deref()) .bind(record.attempts as i32) + .bind(record.confidence) + .bind(record.trusted) .execute(pool) .await .map_err(|e| PgError::Query(e.to_string()))?; diff --git a/crates/panday-platform/tests/route_audit_pg.rs b/crates/panday-platform/tests/route_audit_pg.rs index 54b0039..72012d3 100644 --- a/crates/panday-platform/tests/route_audit_pg.rs +++ b/crates/panday-platform/tests/route_audit_pg.rs @@ -29,6 +29,8 @@ fn record(account: uuid::Uuid, rule: &str, chosen: Option<&str>, attempts: u32) ], chosen: chosen.map(str::to_string), attempts, + confidence: 0.82, + trusted: true, } } @@ -138,3 +140,45 @@ async fn the_gateway_sink_writes_without_making_the_caller_wait() { } panic!("the audit row never landed"); } + +#[tokio::test] +#[ignore = "needs the integration lane (deploy/integration-compose.yml)"] +async fn what_the_classifier_said_survives_the_round_trip() { + // The point of 0011. `task` alone conflates "the heuristic was certain" with "it guessed + // something at 0.2 and the gate fell back", and a learned router (M19.3) is trained on exactly + // that distinction. Both were discarded before this, so the evidence a future model needs was + // being thrown away one request at a time. + let pool = database().await; + let account = pg::create_account(&pool, &format!("acme-{}", uuid::Uuid::now_v7())) + .await + .expect("account"); + + let mut untrusted = record(account, "rules[0]", Some("anthropic/claude-sonnet-4-5"), 1); + untrusted.confidence = 0.21; + untrusted.trusted = false; + let id = untrusted.request.0; + + // `routes::insert`, not `PgRouteAudit::record`: the trait impl detaches the write onto a + // background task so inference never waits on the database, so a test that read straight after + // it would race the spawn — and did, once, with `RowNotFound`. What is under test here is the + // column round trip, not the detachment. + routes::insert(&pool, &untrusted).await.expect("insert"); + + let (confidence, trusted): (Option, Option) = + sqlx::query_as("SELECT confidence, trusted FROM route_decisions WHERE request_id = $1") + .bind(id) + .fetch_one(&pool) + .await + .expect("the row"); + + assert_eq!( + trusted, + Some(false), + "the gate fired and the row must say so" + ); + let confidence = confidence.expect("a confidence was recorded"); + assert!( + (confidence - 0.21).abs() < 1e-6, + "confidence must survive as written, not rounded to a bucket: {confidence}" + ); +}