diff --git a/crates/panday-harness/src/eval.rs b/crates/panday-harness/src/eval.rs index f271037..68b1c96 100644 --- a/crates/panday-harness/src/eval.rs +++ b/crates/panday-harness/src/eval.rs @@ -137,6 +137,62 @@ impl Report { solvable.iter().filter(|r| r.with_reduction.solved).count() as f64 / solvable.len() as f64 } + /// **M19.5's gate**: "≥25% cheaper semantic tier than the provider cheap-pool it replaces". + /// + /// Until now this suite had **no cost dimension at all** — it measured retention and token + /// ratio, which are not money. So the milestone could not be judged even with a model in hand, + /// and that is the failure docs/19 M19.1 exists to prevent: a gate stated in prose is a gate + /// nobody can fail. + /// + /// Cost, not tokens, because the two move independently and the milestone is about the + /// cheaper one *winning*. A reducer that keeps 40% of the tokens but hands them to a model at + /// three times the price is more expensive, and a ratio-only scorecard would call it a 60% + /// improvement. + /// + /// Prices are **per token**, in the same unit for both sides, and supplied by the caller — + /// this crate has no price table and must not grow one, or the gate starts depending on a + /// catalog that drifts. `panday_router`'s catalog is where a binary gets them. + /// + /// Returns `None` when the incumbent would spend nothing: a saving against zero is not a + /// percentage, and reporting one would be inventing a number. + pub fn cost_saving(&self, incumbent_price: f64, replacement_price: f64) -> Option { + let incumbent: f64 = self + .rows + .iter() + .map(|r| r.without_reduction.tokens_raw as f64 * incumbent_price) + .sum(); + let replacement: f64 = self + .rows + .iter() + .map(|r| r.with_reduction.tokens_kept as f64 * replacement_price) + .sum(); + if incumbent <= 0.0 { + return None; + } + Some(1.0 - replacement / incumbent) + } + + /// M19.5, whole: zero regressions **and** at least `fraction` cheaper. + /// + /// Both halves, because either alone is a trap. Cheapness with regressions is a reducer that + /// saves money by losing the answer; zero regressions with no saving is a semantic tier with + /// no reason to exist. The milestone names both and so does this. + /// + /// `fraction` is a ratio — M19.5 says 25%, so `cheaper_than(0.25, ...)`. Unlike route-bench's + /// margin, which is stated in points, this one is stated as a percentage *of* the incumbent, + /// so a ratio is the honest shape. `the_two_gates_do_not_share_a_unit` pins the difference. + pub fn cheaper_than( + &self, + fraction: f64, + incumbent_price: f64, + replacement_price: f64, + ) -> bool { + self.regressions().is_empty() + && self + .cost_saving(incumbent_price, replacement_price) + .is_some_and(|saved| saved >= fraction) + } + /// Mean reduction across the corpus. Reported *after* success, and never /// instead of it (ADR-007). pub fn mean_ratio(&self) -> f64 { @@ -433,3 +489,88 @@ pub async fn run_corpus( } Report { rows } } + +#[cfg(test)] +mod cost_gate_tests { + use super::{Outcome, Report, Row}; + + fn outcome(solved: bool, raw: u32, kept: u32) -> Outcome { + Outcome { + solved, + tokens_raw: raw, + tokens_kept: kept, + missing_facts: vec![], + strategy: "test".into(), + } + } + + fn report(rows: Vec<(bool, u32, bool, u32)>) -> Report { + Report { + rows: rows + .into_iter() + .enumerate() + .map(|(i, (s0, raw, s1, kept))| Row { + scenario: format!("s{i}"), + without_reduction: outcome(s0, raw, raw), + with_reduction: outcome(s1, raw, kept), + }) + .collect(), + } + } + + #[test] + fn cost_is_tokens_times_price_not_tokens_alone() { + // The reason this dimension exists. Half the tokens at four times the price is *more* + // expensive, and a ratio-only scorecard would have called it a 50% win. + let r = report(vec![(true, 1000, true, 500)]); + assert_eq!(r.mean_ratio(), 0.5, "half the tokens kept"); + + let saving = r.cost_saving(1.0, 4.0).expect("a saving"); + assert!( + saving < 0.0, + "keeping half the tokens at 4x the price costs more, not less: {saving}" + ); + } + + #[test] + fn a_saving_against_nothing_is_not_a_number() { + // An empty corpus, or one where the incumbent spends zero, has no percentage to report. + // Returning 0.0 or 1.0 here would be inventing a result. + assert_eq!(Report::default().cost_saving(1.0, 1.0), None); + assert_eq!(report(vec![(true, 0, true, 0)]).cost_saving(1.0, 1.0), None); + } + + #[test] + fn the_gate_needs_both_halves() { + // Cheap but broken: 90% saved, and a scenario that was solvable is now not. + let broken = report(vec![(true, 1000, false, 100)]); + assert!( + !broken.cheaper_than(0.25, 1.0, 1.0), + "a regression is not paid for by being cheap" + ); + + // Sound but not cheap enough: no regressions, 10% saved against a 25% bar. + let timid = report(vec![(true, 1000, true, 900)]); + assert!(!timid.cheaper_than(0.25, 1.0, 1.0)); + + // Both: no regressions and 30% saved. + let good = report(vec![(true, 1000, true, 700)]); + assert!(good.cheaper_than(0.25, 1.0, 1.0)); + } + + #[test] + fn the_two_gates_do_not_share_a_unit() { + // route-bench's `Score::beats` takes percentage POINTS (M19.3: "≥10pt"); this one takes a + // FRACTION of the incumbent's spend (M19.5: "≥25% cheaper"). They read alike at a call + // site and mean different things, so each is pinned where it lives. + let r = report(vec![(true, 1000, true, 700)]); + assert!( + r.cheaper_than(0.25, 1.0, 1.0), + "0.25 is twenty-five percent" + ); + assert!( + !r.cheaper_than(25.0, 1.0, 1.0), + "25.0 would be a 2500% saving, which nothing can meet" + ); + } +} diff --git a/crates/panday-router/src/bench.rs b/crates/panday-router/src/bench.rs index 2c14197..6becc7e 100644 --- a/crates/panday-router/src/bench.rs +++ b/crates/panday-router/src/bench.rs @@ -419,6 +419,28 @@ impl Score { pub fn meets_the_gate(&self, floor: f64) -> bool { self.accuracy() >= floor && self.confidently_wrong.is_empty() } + + /// **M19.3's gate**: "classifier ... beats heuristic on route-bench by ≥10pt". + /// + /// [`meets_the_gate`](Self::meets_the_gate) is an absolute floor; this is the relative one the + /// milestone actually states, and until now it existed only as prose. A gate that is not a + /// function is a gate nobody can fail, which is the state docs/19 M19.1 was written against. + /// + /// `margin_points` is **percentage points**, not a fraction: M19.3 says "≥10pt", so the call + /// is `beats(&heuristic, 10.0)`. Accuracy is a 0..1 ratio internally and the conversion + /// happens here, once — passing `0.10` and meaning ten points is the obvious way to get this + /// wrong, so `the_margin_is_percentage_points_not_a_fraction` pins it. + /// + /// **Being confidently wrong disqualifies a challenger regardless of margin.** That is not an + /// extra condition bolted on: the dangerous quadrant is the thing route-bench exists to + /// measure (a wrong answer the router *trusts* and acts on), and a caller who checked only the + /// margin would ship a model that is more accurate on average and catastrophic on the cases + /// that matter. The incumbent's own quadrant is not consulted — the question is whether the + /// *challenger* is safe to deploy, not whether it is less bad than what is there. + pub fn beats(&self, incumbent: &Score, margin_points: f64) -> bool { + let gained = (self.accuracy() - incumbent.accuracy()) * 100.0; + gained >= margin_points && self.confidently_wrong.is_empty() + } } /// Score any classifier against the corpus. `HeuristicClassifier` today; a learned one at @@ -448,3 +470,60 @@ pub fn score(c: &dyn Classifier) -> Score { } s } + +#[cfg(test)] +mod gate_tests { + use super::Score; + + fn score(correct: usize, total: usize, confidently_wrong: Vec<&'static str>) -> Score { + Score { + total, + correct, + confidently_wrong, + caught_by_the_gate: vec![], + } + } + + #[test] + fn the_margin_is_percentage_points_not_a_fraction() { + // M19.3 says "≥10pt". Accuracy is a 0..1 ratio internally, so the obvious mistake is to + // pass 0.10 and mean ten points — which would let a challenger through on a *tenth* of a + // percentage point. The unit is pinned here because the gate is unfalsifiable by + // inspection: both readings compile and both look right. + let incumbent = score(70, 100, vec![]); + let exactly_ten = score(80, 100, vec![]); + let just_under = score(79, 100, vec![]); + + assert!( + exactly_ten.beats(&incumbent, 10.0), + "10pt clears a 10pt bar" + ); + assert!(!just_under.beats(&incumbent, 10.0), "9pt does not"); + } + + #[test] + fn a_confidently_wrong_challenger_does_not_ship_however_far_ahead() { + // The whole point of route-bench's dangerous quadrant. A model can be twenty points more + // accurate and still be the worse thing to deploy, because the router *acts* on a + // confident answer. + let incumbent = score(50, 100, vec![]); + let brilliant_but_reckless = score(90, 100, vec!["a-case-it-got-wrong-and-trusted"]); + assert!( + !brilliant_but_reckless.beats(&incumbent, 10.0), + "40pt ahead, and still not shippable" + ); + } + + #[test] + fn losing_ground_is_not_a_pass() { + let incumbent = score(90, 100, vec![]); + let worse = score(60, 100, vec![]); + assert!(!worse.beats(&incumbent, 10.0)); + // And equal is not "beats" either: the milestone asks for a margin, not parity. + assert!(!incumbent.beats(&incumbent, 10.0)); + assert!( + incumbent.beats(&incumbent, 0.0), + "a zero-point bar is met by parity" + ); + } +} diff --git a/crates/panday-types/src/scorecard.rs b/crates/panday-types/src/scorecard.rs index 2deb112..4bb3cfb 100644 --- a/crates/panday-types/src/scorecard.rs +++ b/crates/panday-types/src/scorecard.rs @@ -124,6 +124,33 @@ impl Scorecard { self.cases > 0 && self.rate() >= floor } + /// **The relative gate**: did this run beat an earlier one on the same suite? + /// + /// [`meets`](Self::meets) answers "is it good enough"; several milestones ask "is it *better*" + /// and had no function to ask with — M19.7 ("SFT stage beats base on agent-bench") is the + /// live one. A gate that exists only in prose is a gate nobody can fail. + /// + /// **`None` means the two cannot be compared, and that is the point of the return type.** + /// Comparing a tuned run over ten tasks against a base run over forty-one is the way this + /// measurement gets faked without anyone lying: both numbers are real, the ratio is + /// meaningless, and a `bool` would have hidden it. Refused when the suites differ, when either + /// ran nothing, or when the case counts differ — a shorter run is a different corpus, not a + /// better model. A caller that writes `.unwrap_or(false)` fails closed, which is the right + /// default for a shipping decision. + /// + /// Strictly greater, with no margin: M19.7 says "beats", and a tie is not one. Where a + /// milestone names a margin instead — M19.3's "≥10pt" — that gate lives with its suite and + /// takes the margin explicitly, because the unit is the suite's business, not this type's. + pub fn beats(&self, base: &Scorecard) -> Option { + if self.suite != base.suite || self.cases == 0 || base.cases == 0 { + return None; + } + if self.cases != base.cases { + return None; + } + Some(self.passed > base.passed) + } + /// The human rendering. Derived from the artifact, never the other way round. pub fn to_markdown(&self) -> String { let mut out = format!( @@ -226,3 +253,61 @@ mod tests { assert!(md.contains("missing required field `city`")); } } + +#[cfg(test)] +mod relative_gate_tests { + use super::Scorecard; + + fn card(suite: &str, passed: u32, cases: u32) -> Scorecard { + let mut c = Scorecard::new(suite, "subject", "2026-08-24T00:00:00Z"); + for i in 0..cases { + c.record(&format!("case{i}"), i < passed, ""); + } + c + } + + #[test] + fn a_shorter_run_is_not_a_better_model() { + // The failure this return type exists for. Both numbers are real, the comparison is not: + // 9/10 against 30/41 is a different corpus, not an improvement. A `bool` would have said + // "better" and nobody would have noticed. + let base = card("agent-bench", 30, 41); + let tuned_on_fewer = card("agent-bench", 9, 10); + assert_eq!(tuned_on_fewer.beats(&base), None); + } + + #[test] + fn suites_do_not_cross() { + let a = card("agent-bench", 40, 41); + let b = card("route-bench", 10, 41); + assert_eq!(a.beats(&b), None, "same case count, different question"); + } + + #[test] + fn nothing_ran_is_not_a_win() { + let base = card("agent-bench", 0, 0); + let tuned = card("agent-bench", 5, 5); + assert_eq!(tuned.beats(&base), None); + assert_eq!(base.beats(&tuned), None); + } + + #[test] + fn beating_is_strict() { + let base = card("agent-bench", 30, 41); + assert_eq!(card("agent-bench", 31, 41).beats(&base), Some(true)); + assert_eq!( + card("agent-bench", 30, 41).beats(&base), + Some(false), + "a tie is not a win" + ); + assert_eq!(card("agent-bench", 29, 41).beats(&base), Some(false)); + } + + #[test] + fn an_incomparable_pair_fails_closed_for_a_careless_caller() { + // `.unwrap_or(false)` is what a caller will write. It must mean "do not ship". + let base = card("agent-bench", 30, 41); + let mismatched = card("agent-bench", 41, 42); + assert!(!mismatched.beats(&base).unwrap_or(false)); + } +} diff --git a/docs/19-training.md b/docs/19-training.md index 1b6b2a8..5f5c733 100644 --- a/docs/19-training.md +++ b/docs/19-training.md @@ -254,6 +254,17 @@ artifacts; CI runs eval gates nightly. one that emits well-formed calls with the wrong arguments — because a generator that cannot report a bad model as bad is not worth running against a good one. - **M19.3** Model 1 shipped: classifier behind `Classifier` trait beats heuristic on route-bench by ≥10pt; deployed in shadow, then live. + + **The gate is a function now, though the model is not here.** `panday_router::bench::Score::beats(&incumbent, 10.0)` + — the margin in percentage points, as the milestone states it. It was prose until this change, + which meant M19.3 could not be failed even with a classifier in hand; `Score::meets_the_gate` is + an absolute floor and answers a different question. + + A challenger that is **confidently wrong is refused regardless of margin**. That is not an extra + condition: the dangerous quadrant is what route-bench exists to measure, and a caller checking + only the margin would ship a model that is better on average and catastrophic where the router + acts on it. Unit pinned by test — passing `0.10` for "ten points" is the obvious way to get this + wrong and both readings compile. - **M19.4** Transcript mining pipeline with consent flags + PII scrub + provenance; first 10k-pair summarizer dataset. ✅ *(shipped: `panday_harness::mining`, `cargo xtask mine --logs --out `. **The 10k-pair dataset is not here** — it needs 10k consented transcripts, and this repo has none.)* **Consent, then scrub, then provenance — and each step defaults to refusing.** The mistakes in a @@ -285,6 +296,21 @@ artifacts; CI runs eval gates nightly. miner runs end to end today over a directory of logs and reports exactly why each candidate was dropped — which is the part that had to exist before any transcript was worth collecting. - **M19.5** Model 2 shipped: reduce-bench regression zero, ≥25% cheaper semantic tier than the provider cheap-pool it replaces; GGUF in catalog. + + **reduce-bench had no cost dimension at all**, so "≥25% cheaper" was unmeasurable — the suite + scored retention and token ratio, which are not money. `Report::cost_saving` and + `Report::cheaper_than(0.25, incumbent_price, replacement_price)` add it. + + Cost rather than tokens because the two move independently: a reducer that keeps 40% of the + tokens and hands them to a model at three times the price is *more* expensive, and the old + ratio-only scorecard would have called that a 60% win. Prices are per token and supplied by the + caller — `panday-harness` has no price table and must not grow one, or the gate starts drifting + with a catalog. A saving against an incumbent that spends nothing returns `None` rather than a + number. + + Both halves are required, because either alone is a trap: cheapness with regressions is a + reducer that saves money by losing the answer, and zero regressions with no saving is a semantic + tier with no reason to exist. - **M19.6** agent-bench (50 verifiable repo tasks in T3) doubling as GRPO environment. ✅ *(shipped: `panday_harness::agent_bench` — 41 tasks, the audit, the jailed runner, and `score`. **Runs in T2, not T3**. Thirty-eight are repair classes; three (`poisoned-readme`, `poisoned-comment`, `granted-json`) are M20.1 injection canaries whose verifier fails if a relative marker file exists. Still not 50 — the twelve destructive classes stay gone.)* **This is the second attempt, and the first one destroyed a machine.** The original ran each @@ -334,3 +360,15 @@ artifacts; CI runs eval gates nightly. **What is left is the model**, and T3 for the tasks that will eventually need a VM rather than a jail (M14.5). - **M19.7** Model 3 v1: SFT stage beats base on agent-bench; go/no-go review for the GRPO spend. + + **`Scorecard::beats(&base)` is the gate**, on the artifact type every suite already emits, so + it is not agent-bench's alone. + + It returns `Option`, and the `None` is the substance. Comparing a tuned run over ten tasks + against a base run over forty-one is how this measurement gets faked without anyone lying: both + numbers are real, the ratio is meaningless, and a plain `bool` would have hidden it. Refused + when the suites differ, when either ran nothing, or when the case counts differ — a shorter run + is a different corpus, not a better model. `.unwrap_or(false)` fails closed, which is what a + shipping decision should do. + + Strictly greater, no margin: the milestone says "beats", and a tie is not one.