M19.3/M19.5/M19.7: the three training gates become functions

crates/panday-harness/src/eval.rs

@@ -137,6 +137,62 @@ impl Report {
  solvable.iter().filter(|r| r.with_reduction.solved).count() as f64 / solvable.len() as f64
  }
 
+ /// **M19.5's gate**: "≥25% cheaper semantic tier than the provider cheap-pool it replaces".
+ ///
+ /// Until now this suite had **no cost dimension at all** — it measured retention and token
+ /// ratio, which are not money. So the milestone could not be judged even with a model in hand,
+ /// and that is the failure docs/19 M19.1 exists to prevent: a gate stated in prose is a gate
+ /// nobody can fail.
+ ///
+ /// Cost, not tokens, because the two move independently and the milestone is about the
+ /// cheaper one *winning*. A reducer that keeps 40% of the tokens but hands them to a model at
+ /// three times the price is more expensive, and a ratio-only scorecard would call it a 60%
+ /// improvement.
+ ///
+ /// Prices are **per token**, in the same unit for both sides, and supplied by the caller —
+ /// this crate has no price table and must not grow one, or the gate starts depending on a
+ /// catalog that drifts. `panday_router`'s catalog is where a binary gets them.
+ ///
+ /// Returns `None` when the incumbent would spend nothing: a saving against zero is not a
+ /// percentage, and reporting one would be inventing a number.
+ pub fn cost_saving(&self, incumbent_price: f64, replacement_price: f64) -> Option<f64> {
+ let incumbent: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.without_reduction.tokens_raw as f64 * incumbent_price)
+ .sum();
+ let replacement: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.with_reduction.tokens_kept as f64 * replacement_price)
+ .sum();
+ if incumbent <= 0.0 {
+ return None;
+ }
+ Some(1.0 - replacement / incumbent)
+ }
+
+ /// M19.5, whole: zero regressions **and** at least `fraction` cheaper.
+ ///
+ /// Both halves, because either alone is a trap. Cheapness with regressions is a reducer that
+ /// saves money by losing the answer; zero regressions with no saving is a semantic tier with
+ /// no reason to exist. The milestone names both and so does this.
+ ///
+ /// `fraction` is a ratio — M19.5 says 25%, so `cheaper_than(0.25, ...)`. Unlike route-bench's
+ /// margin, which is stated in points, this one is stated as a percentage *of* the incumbent,
+ /// so a ratio is the honest shape. `the_two_gates_do_not_share_a_unit` pins the difference.
+ pub fn cheaper_than(
+ &self,
+ fraction: f64,
+ incumbent_price: f64,
+ replacement_price: f64,
+ ) -> bool {
+ self.regressions().is_empty()
+ && self
+ .cost_saving(incumbent_price, replacement_price)
+ .is_some_and(|saved| saved >= fraction)
+ }
+
  /// Mean reduction across the corpus. Reported *after* success, and never
  /// instead of it (ADR-007).
  pub fn mean_ratio(&self) -> f64 {
@@ -433,3 +489,88 @@ pub async fn run_corpus(
  }
  Report { rows }
 }
+
+#[cfg(test)]
+mod cost_gate_tests {
+ use super::{Outcome, Report, Row};
+
+ fn outcome(solved: bool, raw: u32, kept: u32) -> Outcome {
+ Outcome {
+ solved,
+ tokens_raw: raw,
+ tokens_kept: kept,
+ missing_facts: vec![],
+ strategy: "test".into(),
+ }
+ }
+
+ fn report(rows: Vec<(bool, u32, bool, u32)>) -> Report {
+ Report {
+ rows: rows
+ .into_iter()
+ .enumerate()
+ .map(|(i, (s0, raw, s1, kept))| Row {
+ scenario: format!("s{i}"),
+ without_reduction: outcome(s0, raw, raw),
+ with_reduction: outcome(s1, raw, kept),
+ })
+ .collect(),
+ }
+ }
+
+ #[test]
+ fn cost_is_tokens_times_price_not_tokens_alone() {
+ // The reason this dimension exists. Half the tokens at four times the price is *more*
+ // expensive, and a ratio-only scorecard would have called it a 50% win.
+ let r = report(vec![(true, 1000, true, 500)]);
+ assert_eq!(r.mean_ratio(), 0.5, "half the tokens kept");
+
+ let saving = r.cost_saving(1.0, 4.0).expect("a saving");
+ assert!(
+ saving < 0.0,
+ "keeping half the tokens at 4x the price costs more, not less: {saving}"
+ );
+ }
+
+ #[test]
+ fn a_saving_against_nothing_is_not_a_number() {
+ // An empty corpus, or one where the incumbent spends zero, has no percentage to report.
+ // Returning 0.0 or 1.0 here would be inventing a result.
+ assert_eq!(Report::default().cost_saving(1.0, 1.0), None);
+ assert_eq!(report(vec![(true, 0, true, 0)]).cost_saving(1.0, 1.0), None);
+ }
+
+ #[test]
+ fn the_gate_needs_both_halves() {
+ // Cheap but broken: 90% saved, and a scenario that was solvable is now not.
+ let broken = report(vec![(true, 1000, false, 100)]);
+ assert!(
+ !broken.cheaper_than(0.25, 1.0, 1.0),
+ "a regression is not paid for by being cheap"
+ );
+
+ // Sound but not cheap enough: no regressions, 10% saved against a 25% bar.
+ let timid = report(vec![(true, 1000, true, 900)]);
+ assert!(!timid.cheaper_than(0.25, 1.0, 1.0));
+
+ // Both: no regressions and 30% saved.
+ let good = report(vec![(true, 1000, true, 700)]);
+ assert!(good.cheaper_than(0.25, 1.0, 1.0));
+ }
+
+ #[test]
+ fn the_two_gates_do_not_share_a_unit() {
+ // route-bench's `Score::beats` takes percentage POINTS (M19.3: "≥10pt"); this one takes a
+ // FRACTION of the incumbent's spend (M19.5: "≥25% cheaper"). They read alike at a call
+ // site and mean different things, so each is pinned where it lives.
+ let r = report(vec![(true, 1000, true, 700)]);
+ assert!(
+ r.cheaper_than(0.25, 1.0, 1.0),
+ "0.25 is twenty-five percent"
+ );
+ assert!(
+ !r.cheaper_than(25.0, 1.0, 1.0),
+ "25.0 would be a 2500% saving, which nothing can meet"
+ );
+ }
+}

crates/panday-router/src/bench.rs

@@ -419,6 +419,28 @@ impl Score {
  pub fn meets_the_gate(&self, floor: f64) -> bool {
  self.accuracy() >= floor && self.confidently_wrong.is_empty()
  }
+
+ /// **M19.3's gate**: "classifier ... beats heuristic on route-bench by ≥10pt".
+ ///
+ /// [`meets_the_gate`](Self::meets_the_gate) is an absolute floor; this is the relative one the
+ /// milestone actually states, and until now it existed only as prose. A gate that is not a
+ /// function is a gate nobody can fail, which is the state docs/19 M19.1 was written against.
+ ///
+ /// `margin_points` is **percentage points**, not a fraction: M19.3 says "≥10pt", so the call
+ /// is `beats(&heuristic, 10.0)`. Accuracy is a 0..1 ratio internally and the conversion
+ /// happens here, once — passing `0.10` and meaning ten points is the obvious way to get this
+ /// wrong, so `the_margin_is_percentage_points_not_a_fraction` pins it.
+ ///
+ /// **Being confidently wrong disqualifies a challenger regardless of margin.** That is not an
+ /// extra condition bolted on: the dangerous quadrant is the thing route-bench exists to
+ /// measure (a wrong answer the router *trusts* and acts on), and a caller who checked only the
+ /// margin would ship a model that is more accurate on average and catastrophic on the cases
+ /// that matter. The incumbent's own quadrant is not consulted — the question is whether the
+ /// *challenger* is safe to deploy, not whether it is less bad than what is there.
+ pub fn beats(&self, incumbent: &Score, margin_points: f64) -> bool {
+ let gained = (self.accuracy() - incumbent.accuracy()) * 100.0;
+ gained >= margin_points && self.confidently_wrong.is_empty()
+ }
 }
 
 /// Score any classifier against the corpus. `HeuristicClassifier` today; a learned one at
@@ -448,3 +470,60 @@ pub fn score(c: &dyn Classifier) -> Score {
  }
  s
 }
+
+#[cfg(test)]
+mod gate_tests {
+ use super::Score;
+
+ fn score(correct: usize, total: usize, confidently_wrong: Vec<&'static str>) -> Score {
+ Score {
+ total,
+ correct,
+ confidently_wrong,
+ caught_by_the_gate: vec![],
+ }
+ }
+
+ #[test]
+ fn the_margin_is_percentage_points_not_a_fraction() {
+ // M19.3 says "≥10pt". Accuracy is a 0..1 ratio internally, so the obvious mistake is to
+ // pass 0.10 and mean ten points — which would let a challenger through on a *tenth* of a
+ // percentage point. The unit is pinned here because the gate is unfalsifiable by
+ // inspection: both readings compile and both look right.
+ let incumbent = score(70, 100, vec![]);
+ let exactly_ten = score(80, 100, vec![]);
+ let just_under = score(79, 100, vec![]);
+
+ assert!(
+ exactly_ten.beats(&incumbent, 10.0),
+ "10pt clears a 10pt bar"
+ );
+ assert!(!just_under.beats(&incumbent, 10.0), "9pt does not");
+ }
+
+ #[test]
+ fn a_confidently_wrong_challenger_does_not_ship_however_far_ahead() {
+ // The whole point of route-bench's dangerous quadrant. A model can be twenty points more
+ // accurate and still be the worse thing to deploy, because the router *acts* on a
+ // confident answer.
+ let incumbent = score(50, 100, vec![]);
+ let brilliant_but_reckless = score(90, 100, vec!["a-case-it-got-wrong-and-trusted"]);
+ assert!(
+ !brilliant_but_reckless.beats(&incumbent, 10.0),
+ "40pt ahead, and still not shippable"
+ );
+ }
+
+ #[test]
+ fn losing_ground_is_not_a_pass() {
+ let incumbent = score(90, 100, vec![]);
+ let worse = score(60, 100, vec![]);
+ assert!(!worse.beats(&incumbent, 10.0));
+ // And equal is not "beats" either: the milestone asks for a margin, not parity.
+ assert!(!incumbent.beats(&incumbent, 10.0));
+ assert!(
+ incumbent.beats(&incumbent, 0.0),
+ "a zero-point bar is met by parity"
+ );
+ }
+}

crates/panday-types/src/scorecard.rs

@@ -124,6 +124,33 @@ impl Scorecard {
  self.cases > 0 && self.rate() >= floor
  }
 
+ /// **The relative gate**: did this run beat an earlier one on the same suite?
+ ///
+ /// [`meets`](Self::meets) answers "is it good enough"; several milestones ask "is it *better*"
+ /// and had no function to ask with — M19.7 ("SFT stage beats base on agent-bench") is the
+ /// live one. A gate that exists only in prose is a gate nobody can fail.
+ ///
+ /// **`None` means the two cannot be compared, and that is the point of the return type.**
+ /// Comparing a tuned run over ten tasks against a base run over forty-one is the way this
+ /// measurement gets faked without anyone lying: both numbers are real, the ratio is
+ /// meaningless, and a `bool` would have hidden it. Refused when the suites differ, when either
+ /// ran nothing, or when the case counts differ — a shorter run is a different corpus, not a
+ /// better model. A caller that writes `.unwrap_or(false)` fails closed, which is the right
+ /// default for a shipping decision.
+ ///
+ /// Strictly greater, with no margin: M19.7 says "beats", and a tie is not one. Where a
+ /// milestone names a margin instead — M19.3's "≥10pt" — that gate lives with its suite and
+ /// takes the margin explicitly, because the unit is the suite's business, not this type's.
+ pub fn beats(&self, base: &Scorecard) -> Option<bool> {
+ if self.suite != base.suite || self.cases == 0 || base.cases == 0 {
+ return None;
+ }
+ if self.cases != base.cases {
+ return None;
+ }
+ Some(self.passed > base.passed)
+ }
+
  /// The human rendering. Derived from the artifact, never the other way round.
  pub fn to_markdown(&self) -> String {
  let mut out = format!(
@@ -226,3 +253,61 @@ mod tests {
  assert!(md.contains("missing required field `city`"));
  }
 }
+
+#[cfg(test)]
+mod relative_gate_tests {
+ use super::Scorecard;
+
+ fn card(suite: &str, passed: u32, cases: u32) -> Scorecard {
+ let mut c = Scorecard::new(suite, "subject", "2026-08-24T00:00:00Z");
+ for i in 0..cases {
+ c.record(&format!("case{i}"), i < passed, "");
+ }
+ c
+ }
+
+ #[test]
+ fn a_shorter_run_is_not_a_better_model() {
+ // The failure this return type exists for. Both numbers are real, the comparison is not:
+ // 9/10 against 30/41 is a different corpus, not an improvement. A `bool` would have said
+ // "better" and nobody would have noticed.
+ let base = card("agent-bench", 30, 41);
+ let tuned_on_fewer = card("agent-bench", 9, 10);
+ assert_eq!(tuned_on_fewer.beats(&base), None);
+ }
+
+ #[test]
+ fn suites_do_not_cross() {
+ let a = card("agent-bench", 40, 41);
+ let b = card("route-bench", 10, 41);
+ assert_eq!(a.beats(&b), None, "same case count, different question");
+ }
+
+ #[test]
+ fn nothing_ran_is_not_a_win() {
+ let base = card("agent-bench", 0, 0);
+ let tuned = card("agent-bench", 5, 5);
+ assert_eq!(tuned.beats(&base), None);
+ assert_eq!(base.beats(&tuned), None);
+ }
+
+ #[test]
+ fn beating_is_strict() {
+ let base = card("agent-bench", 30, 41);
+ assert_eq!(card("agent-bench", 31, 41).beats(&base), Some(true));
+ assert_eq!(
+ card("agent-bench", 30, 41).beats(&base),
+ Some(false),
+ "a tie is not a win"
+ );
+ assert_eq!(card("agent-bench", 29, 41).beats(&base), Some(false));
+ }
+
+ #[test]
+ fn an_incomparable_pair_fails_closed_for_a_careless_caller() {
+ // `.unwrap_or(false)` is what a caller will write. It must mean "do not ship".
+ let base = card("agent-bench", 30, 41);
+ let mismatched = card("agent-bench", 41, 42);
+ assert!(!mismatched.beats(&base).unwrap_or(false));
+ }
+}

docs/19-training.md

@@ -254,6 +254,17 @@ artifacts; CI runs eval gates nightly.
  one that emits well-formed calls with the wrong arguments — because a generator that cannot
  report a bad model as bad is not worth running against a good one.
 - **M19.3** Model 1 shipped: classifier behind `Classifier` trait beats heuristic on route-bench by ≥10pt; deployed in shadow, then live.
+
+ **The gate is a function now, though the model is not here.** `panday_router::bench::Score::beats(&incumbent, 10.0)`
+ — the margin in percentage points, as the milestone states it. It was prose until this change,
+ which meant M19.3 could not be failed even with a classifier in hand; `Score::meets_the_gate` is
+ an absolute floor and answers a different question.
+
+ A challenger that is **confidently wrong is refused regardless of margin**. That is not an extra
+ condition: the dangerous quadrant is what route-bench exists to measure, and a caller checking
+ only the margin would ship a model that is better on average and catastrophic where the router
+ acts on it. Unit pinned by test — passing `0.10` for "ten points" is the obvious way to get this
+ wrong and both readings compile.
 - **M19.4** Transcript mining pipeline with consent flags + PII scrub + provenance; first 10k-pair summarizer dataset. ✅ *(shipped: `panday_harness::mining`, `cargo xtask mine --logs <dir> --out <file>`. **The 10k-pair dataset is not here** — it needs 10k consented transcripts, and this repo has none.)*
 
  **Consent, then scrub, then provenance — and each step defaults to refusing.** The mistakes in a
@@ -285,6 +296,21 @@ artifacts; CI runs eval gates nightly.
  miner runs end to end today over a directory of logs and reports exactly why each candidate was
  dropped — which is the part that had to exist before any transcript was worth collecting.
 - **M19.5** Model 2 shipped: reduce-bench regression zero, ≥25% cheaper semantic tier than the provider cheap-pool it replaces; GGUF in catalog.
+
+ **reduce-bench had no cost dimension at all**, so "≥25% cheaper" was unmeasurable — the suite
+ scored retention and token ratio, which are not money. `Report::cost_saving` and
+ `Report::cheaper_than(0.25, incumbent_price, replacement_price)` add it.
+
+ Cost rather than tokens because the two move independently: a reducer that keeps 40% of the
+ tokens and hands them to a model at three times the price is *more* expensive, and the old
+ ratio-only scorecard would have called that a 60% win. Prices are per token and supplied by the
+ caller — `panday-harness` has no price table and must not grow one, or the gate starts drifting
+ with a catalog. A saving against an incumbent that spends nothing returns `None` rather than a
+ number.
+
+ Both halves are required, because either alone is a trap: cheapness with regressions is a
+ reducer that saves money by losing the answer, and zero regressions with no saving is a semantic
+ tier with no reason to exist.
 - **M19.6** agent-bench (50 verifiable repo tasks in T3) doubling as GRPO environment. ✅ *(shipped: `panday_harness::agent_bench` — 41 tasks, the audit, the jailed runner, and `score`. **Runs in T2, not T3**. Thirty-eight are repair classes; three (`poisoned-readme`, `poisoned-comment`, `granted-json`) are M20.1 injection canaries whose verifier fails if a relative marker file exists. Still not 50 — the twelve destructive classes stay gone.)*
 
  **This is the second attempt, and the first one destroyed a machine.** The original ran each
@@ -334,3 +360,15 @@ artifacts; CI runs eval gates nightly.
  **What is left is the model**, and T3 for the tasks that will eventually need a VM rather than a
  jail (M14.5).
 - **M19.7** Model 3 v1: SFT stage beats base on agent-bench; go/no-go review for the GRPO spend.
+
+ **`Scorecard::beats(&base)` is the gate**, on the artifact type every suite already emits, so
+ it is not agent-bench's alone.
+
+ It returns `Option<bool>`, and the `None` is the substance. Comparing a tuned run over ten tasks
+ against a base run over forty-one is how this measurement gets faked without anyone lying: both
+ numbers are real, the ratio is meaningless, and a plain `bool` would have hidden it. Refused
+ when the suites differ, when either ran nothing, or when the case counts differ — a shorter run
+ is a different corpus, not a better model. `.unwrap_or(false)` fails closed, which is what a
+ shipping decision should do.
+
+ Strictly greater, no margin: the milestone says "beats", and a tie is not one.

crates/panday-harness/src/eval.rs

@@ -137,6 +137,62 @@ impl Report {
  solvable.iter().filter(|r| r.with_reduction.solved).count() as f64 / solvable.len() as f64
  }
 
+ /// **M19.5's gate**: "≥25% cheaper semantic tier than the provider cheap-pool it replaces".
+ ///
+ /// Until now this suite had **no cost dimension at all** — it measured retention and token
+ /// ratio, which are not money. So the milestone could not be judged even with a model in hand,
+ /// and that is the failure docs/19 M19.1 exists to prevent: a gate stated in prose is a gate
+ /// nobody can fail.
+ ///
+ /// Cost, not tokens, because the two move independently and the milestone is about the
+ /// cheaper one *winning*. A reducer that keeps 40% of the tokens but hands them to a model at
+ /// three times the price is more expensive, and a ratio-only scorecard would call it a 60%
+ /// improvement.
+ ///
+ /// Prices are **per token**, in the same unit for both sides, and supplied by the caller —
+ /// this crate has no price table and must not grow one, or the gate starts depending on a
+ /// catalog that drifts. `panday_router`'s catalog is where a binary gets them.
+ ///
+ /// Returns `None` when the incumbent would spend nothing: a saving against zero is not a
+ /// percentage, and reporting one would be inventing a number.
+ pub fn cost_saving(&self, incumbent_price: f64, replacement_price: f64) -> Option<f64> {
+ let incumbent: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.without_reduction.tokens_raw as f64 * incumbent_price)
+ .sum();
+ let replacement: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.with_reduction.tokens_kept as f64 * replacement_price)
+ .sum();
+ if incumbent <= 0.0 {
+ return None;
+ }
+ Some(1.0 - replacement / incumbent)
+ }
+
+ /// M19.5, whole: zero regressions **and** at least `fraction` cheaper.
+ ///
+ /// Both halves, because either alone is a trap. Cheapness with regressions is a reducer that
+ /// saves money by losing the answer; zero regressions with no saving is a semantic tier with
+ /// no reason to exist. The milestone names both and so does this.
+ ///
+ /// `fraction` is a ratio — M19.5 says 25%, so `cheaper_than(0.25, ...)`. Unlike route-bench's
+ /// margin, which is stated in points, this one is stated as a percentage *of* the incumbent,
+ /// so a ratio is the honest shape. `the_two_gates_do_not_share_a_unit` pins the difference.
+ pub fn cheaper_than(
+ &self,
+ fraction: f64,
+ incumbent_price: f64,
+ replacement_price: f64,
+ ) -> bool {
+ self.regressions().is_empty()
+ && self
+ .cost_saving(incumbent_price, replacement_price)
+ .is_some_and(|saved| saved >= fraction)
+ }
+
  /// Mean reduction across the corpus. Reported *after* success, and never
  /// instead of it (ADR-007).
  pub fn mean_ratio(&self) -> f64 {
@@ -433,3 +489,88 @@ pub async fn run_corpus(
  }
  Report { rows }
 }
+
+#[cfg(test)]
+mod cost_gate_tests {
+ use super::{Outcome, Report, Row};
+
+ fn outcome(solved: bool, raw: u32, kept: u32) -> Outcome {
+ Outcome {
+ solved,
+ tokens_raw: raw,
+ tokens_kept: kept,
+ missing_facts: vec![],
+ strategy: "test".into(),
+ }
+ }
+
+ fn report(rows: Vec<(bool, u32, bool, u32)>) -> Report {
+ Report {
+ rows: rows
+ .into_iter()
+ .enumerate()
+ .map(|(i, (s0, raw, s1, kept))| Row {
+ scenario: format!("s{i}"),
+ without_reduction: outcome(s0, raw, raw),
+ with_reduction: outcome(s1, raw, kept),
+ })
+ .collect(),
+ }
+ }
+
+ #[test]
+ fn cost_is_tokens_times_price_not_tokens_alone() {
+ // The reason this dimension exists. Half the tokens at four times the price is *more*
+ // expensive, and a ratio-only scorecard would have called it a 50% win.
+ let r = report(vec![(true, 1000, true, 500)]);
+ assert_eq!(r.mean_ratio(), 0.5, "half the tokens kept");
+
+ let saving = r.cost_saving(1.0, 4.0).expect("a saving");
+ assert!(
+ saving < 0.0,
+ "keeping half the tokens at 4x the price costs more, not less: {saving}"
+ );
+ }
+
+ #[test]
+ fn a_saving_against_nothing_is_not_a_number() {
+ // An empty corpus, or one where the incumbent spends zero, has no percentage to report.
+ // Returning 0.0 or 1.0 here would be inventing a result.
+ assert_eq!(Report::default().cost_saving(1.0, 1.0), None);
+ assert_eq!(report(vec![(true, 0, true, 0)]).cost_saving(1.0, 1.0), None);
+ }
+
+ #[test]
+ fn the_gate_needs_both_halves() {
+ // Cheap but broken: 90% saved, and a scenario that was solvable is now not.
+ let broken = report(vec![(true, 1000, false, 100)]);
+ assert!(
+ !broken.cheaper_than(0.25, 1.0, 1.0),
+ "a regression is not paid for by being cheap"
+ );
+
+ // Sound but not cheap enough: no regressions, 10% saved against a 25% bar.
+ let timid = report(vec![(true, 1000, true, 900)]);
+ assert!(!timid.cheaper_than(0.25, 1.0, 1.0));
+
+ // Both: no regressions and 30% saved.
+ let good = report(vec![(true, 1000, true, 700)]);
+ assert!(good.cheaper_than(0.25, 1.0, 1.0));
+ }
+
+ #[test]
+ fn the_two_gates_do_not_share_a_unit() {
+ // route-bench's `Score::beats` takes percentage POINTS (M19.3: "≥10pt"); this one takes a
+ // FRACTION of the incumbent's spend (M19.5: "≥25% cheaper"). They read alike at a call
+ // site and mean different things, so each is pinned where it lives.
+ let r = report(vec![(true, 1000, true, 700)]);
+ assert!(
+ r.cheaper_than(0.25, 1.0, 1.0),
+ "0.25 is twenty-five percent"
+ );
+ assert!(
+ !r.cheaper_than(25.0, 1.0, 1.0),
+ "25.0 would be a 2500% saving, which nothing can meet"
+ );
+ }
+}

crates/panday-router/src/bench.rs

@@ -419,6 +419,28 @@ impl Score {
  pub fn meets_the_gate(&self, floor: f64) -> bool {
  self.accuracy() >= floor && self.confidently_wrong.is_empty()
  }
+
+ /// **M19.3's gate**: "classifier ... beats heuristic on route-bench by ≥10pt".
+ ///
+ /// [`meets_the_gate`](Self::meets_the_gate) is an absolute floor; this is the relative one the
+ /// milestone actually states, and until now it existed only as prose. A gate that is not a
+ /// function is a gate nobody can fail, which is the state docs/19 M19.1 was written against.
+ ///
+ /// `margin_points` is **percentage points**, not a fraction: M19.3 says "≥10pt", so the call
+ /// is `beats(&heuristic, 10.0)`. Accuracy is a 0..1 ratio internally and the conversion
+ /// happens here, once — passing `0.10` and meaning ten points is the obvious way to get this
+ /// wrong, so `the_margin_is_percentage_points_not_a_fraction` pins it.
+ ///
+ /// **Being confidently wrong disqualifies a challenger regardless of margin.** That is not an
+ /// extra condition bolted on: the dangerous quadrant is the thing route-bench exists to
+ /// measure (a wrong answer the router *trusts* and acts on), and a caller who checked only the
+ /// margin would ship a model that is more accurate on average and catastrophic on the cases
+ /// that matter. The incumbent's own quadrant is not consulted — the question is whether the
+ /// *challenger* is safe to deploy, not whether it is less bad than what is there.
+ pub fn beats(&self, incumbent: &Score, margin_points: f64) -> bool {
+ let gained = (self.accuracy() - incumbent.accuracy()) * 100.0;
+ gained >= margin_points && self.confidently_wrong.is_empty()
+ }
 }
 
 /// Score any classifier against the corpus. `HeuristicClassifier` today; a learned one at
@@ -448,3 +470,60 @@ pub fn score(c: &dyn Classifier) -> Score {
  }
  s
 }
+
+#[cfg(test)]
+mod gate_tests {
+ use super::Score;
+
+ fn score(correct: usize, total: usize, confidently_wrong: Vec<&'static str>) -> Score {
+ Score {
+ total,
+ correct,
+ confidently_wrong,
+ caught_by_the_gate: vec![],
+ }
+ }
+
+ #[test]
+ fn the_margin_is_percentage_points_not_a_fraction() {
+ // M19.3 says "≥10pt". Accuracy is a 0..1 ratio internally, so the obvious mistake is to
+ // pass 0.10 and mean ten points — which would let a challenger through on a *tenth* of a
+ // percentage point. The unit is pinned here because the gate is unfalsifiable by
+ // inspection: both readings compile and both look right.
+ let incumbent = score(70, 100, vec![]);
+ let exactly_ten = score(80, 100, vec![]);
+ let just_under = score(79, 100, vec![]);
+
+ assert!(
+ exactly_ten.beats(&incumbent, 10.0),
+ "10pt clears a 10pt bar"
+ );
+ assert!(!just_under.beats(&incumbent, 10.0), "9pt does not");
+ }
+
+ #[test]
+ fn a_confidently_wrong_challenger_does_not_ship_however_far_ahead() {
+ // The whole point of route-bench's dangerous quadrant. A model can be twenty points more
+ // accurate and still be the worse thing to deploy, because the router *acts* on a
+ // confident answer.
+ let incumbent = score(50, 100, vec![]);
+ let brilliant_but_reckless = score(90, 100, vec!["a-case-it-got-wrong-and-trusted"]);
+ assert!(
+ !brilliant_but_reckless.beats(&incumbent, 10.0),
+ "40pt ahead, and still not shippable"
+ );
+ }
+
+ #[test]
+ fn losing_ground_is_not_a_pass() {
+ let incumbent = score(90, 100, vec![]);
+ let worse = score(60, 100, vec![]);
+ assert!(!worse.beats(&incumbent, 10.0));
+ // And equal is not "beats" either: the milestone asks for a margin, not parity.
+ assert!(!incumbent.beats(&incumbent, 10.0));
+ assert!(
+ incumbent.beats(&incumbent, 0.0),
+ "a zero-point bar is met by parity"
+ );
+ }
+}

crates/panday-types/src/scorecard.rs

@@ -124,6 +124,33 @@ impl Scorecard {
  self.cases > 0 && self.rate() >= floor
  }
 
+ /// **The relative gate**: did this run beat an earlier one on the same suite?
+ ///
+ /// [`meets`](Self::meets) answers "is it good enough"; several milestones ask "is it *better*"
+ /// and had no function to ask with — M19.7 ("SFT stage beats base on agent-bench") is the
+ /// live one. A gate that exists only in prose is a gate nobody can fail.
+ ///
+ /// **`None` means the two cannot be compared, and that is the point of the return type.**
+ /// Comparing a tuned run over ten tasks against a base run over forty-one is the way this
+ /// measurement gets faked without anyone lying: both numbers are real, the ratio is
+ /// meaningless, and a `bool` would have hidden it. Refused when the suites differ, when either
+ /// ran nothing, or when the case counts differ — a shorter run is a different corpus, not a
+ /// better model. A caller that writes `.unwrap_or(false)` fails closed, which is the right
+ /// default for a shipping decision.
+ ///
+ /// Strictly greater, with no margin: M19.7 says "beats", and a tie is not one. Where a
+ /// milestone names a margin instead — M19.3's "≥10pt" — that gate lives with its suite and
+ /// takes the margin explicitly, because the unit is the suite's business, not this type's.
+ pub fn beats(&self, base: &Scorecard) -> Option<bool> {
+ if self.suite != base.suite || self.cases == 0 || base.cases == 0 {
+ return None;
+ }
+ if self.cases != base.cases {
+ return None;
+ }
+ Some(self.passed > base.passed)
+ }
+
  /// The human rendering. Derived from the artifact, never the other way round.
  pub fn to_markdown(&self) -> String {
  let mut out = format!(
@@ -226,3 +253,61 @@ mod tests {
  assert!(md.contains("missing required field `city`"));
  }
 }
+
+#[cfg(test)]
+mod relative_gate_tests {
+ use super::Scorecard;
+
+ fn card(suite: &str, passed: u32, cases: u32) -> Scorecard {
+ let mut c = Scorecard::new(suite, "subject", "2026-08-24T00:00:00Z");
+ for i in 0..cases {
+ c.record(&format!("case{i}"), i < passed, "");
+ }
+ c
+ }
+
+ #[test]
+ fn a_shorter_run_is_not_a_better_model() {
+ // The failure this return type exists for. Both numbers are real, the comparison is not:
+ // 9/10 against 30/41 is a different corpus, not an improvement. A `bool` would have said
+ // "better" and nobody would have noticed.
+ let base = card("agent-bench", 30, 41);
+ let tuned_on_fewer = card("agent-bench", 9, 10);
+ assert_eq!(tuned_on_fewer.beats(&base), None);
+ }
+
+ #[test]
+ fn suites_do_not_cross() {
+ let a = card("agent-bench", 40, 41);
+ let b = card("route-bench", 10, 41);
+ assert_eq!(a.beats(&b), None, "same case count, different question");
+ }
+
+ #[test]
+ fn nothing_ran_is_not_a_win() {
+ let base = card("agent-bench", 0, 0);
+ let tuned = card("agent-bench", 5, 5);
+ assert_eq!(tuned.beats(&base), None);
+ assert_eq!(base.beats(&tuned), None);
+ }
+
+ #[test]
+ fn beating_is_strict() {
+ let base = card("agent-bench", 30, 41);
+ assert_eq!(card("agent-bench", 31, 41).beats(&base), Some(true));
+ assert_eq!(
+ card("agent-bench", 30, 41).beats(&base),
+ Some(false),
+ "a tie is not a win"
+ );
+ assert_eq!(card("agent-bench", 29, 41).beats(&base), Some(false));
+ }
+
+ #[test]
+ fn an_incomparable_pair_fails_closed_for_a_careless_caller() {
+ // `.unwrap_or(false)` is what a caller will write. It must mean "do not ship".
+ let base = card("agent-bench", 30, 41);
+ let mismatched = card("agent-bench", 41, 42);
+ assert!(!mismatched.beats(&base).unwrap_or(false));
+ }
+}

docs/19-training.md

@@ -254,6 +254,17 @@ artifacts; CI runs eval gates nightly.
  one that emits well-formed calls with the wrong arguments — because a generator that cannot
  report a bad model as bad is not worth running against a good one.
 - **M19.3** Model 1 shipped: classifier behind `Classifier` trait beats heuristic on route-bench by ≥10pt; deployed in shadow, then live.
+
+ **The gate is a function now, though the model is not here.** `panday_router::bench::Score::beats(&incumbent, 10.0)`
+ — the margin in percentage points, as the milestone states it. It was prose until this change,
+ which meant M19.3 could not be failed even with a classifier in hand; `Score::meets_the_gate` is
+ an absolute floor and answers a different question.
+
+ A challenger that is **confidently wrong is refused regardless of margin**. That is not an extra
+ condition: the dangerous quadrant is what route-bench exists to measure, and a caller checking
+ only the margin would ship a model that is better on average and catastrophic where the router
+ acts on it. Unit pinned by test — passing `0.10` for "ten points" is the obvious way to get this
+ wrong and both readings compile.
 - **M19.4** Transcript mining pipeline with consent flags + PII scrub + provenance; first 10k-pair summarizer dataset. ✅ *(shipped: `panday_harness::mining`, `cargo xtask mine --logs <dir> --out <file>`. **The 10k-pair dataset is not here** — it needs 10k consented transcripts, and this repo has none.)*
 
  **Consent, then scrub, then provenance — and each step defaults to refusing.** The mistakes in a
@@ -285,6 +296,21 @@ artifacts; CI runs eval gates nightly.
  miner runs end to end today over a directory of logs and reports exactly why each candidate was
  dropped — which is the part that had to exist before any transcript was worth collecting.
 - **M19.5** Model 2 shipped: reduce-bench regression zero, ≥25% cheaper semantic tier than the provider cheap-pool it replaces; GGUF in catalog.
+
+ **reduce-bench had no cost dimension at all**, so "≥25% cheaper" was unmeasurable — the suite
+ scored retention and token ratio, which are not money. `Report::cost_saving` and
+ `Report::cheaper_than(0.25, incumbent_price, replacement_price)` add it.
+
+ Cost rather than tokens because the two move independently: a reducer that keeps 40% of the
+ tokens and hands them to a model at three times the price is *more* expensive, and the old
+ ratio-only scorecard would have called that a 60% win. Prices are per token and supplied by the
+ caller — `panday-harness` has no price table and must not grow one, or the gate starts drifting
+ with a catalog. A saving against an incumbent that spends nothing returns `None` rather than a
+ number.
+
+ Both halves are required, because either alone is a trap: cheapness with regressions is a
+ reducer that saves money by losing the answer, and zero regressions with no saving is a semantic
+ tier with no reason to exist.
 - **M19.6** agent-bench (50 verifiable repo tasks in T3) doubling as GRPO environment. ✅ *(shipped: `panday_harness::agent_bench` — 41 tasks, the audit, the jailed runner, and `score`. **Runs in T2, not T3**. Thirty-eight are repair classes; three (`poisoned-readme`, `poisoned-comment`, `granted-json`) are M20.1 injection canaries whose verifier fails if a relative marker file exists. Still not 50 — the twelve destructive classes stay gone.)*
 
  **This is the second attempt, and the first one destroyed a machine.** The original ran each
@@ -334,3 +360,15 @@ artifacts; CI runs eval gates nightly.
  **What is left is the model**, and T3 for the tasks that will eventually need a VM rather than a
  jail (M14.5).
 - **M19.7** Model 3 v1: SFT stage beats base on agent-bench; go/no-go review for the GRPO spend.
+
+ **`Scorecard::beats(&base)` is the gate**, on the artifact type every suite already emits, so
+ it is not agent-bench's alone.
+
+ It returns `Option<bool>`, and the `None` is the substance. Comparing a tuned run over ten tasks
+ against a base run over forty-one is how this measurement gets faked without anyone lying: both
+ numbers are real, the ratio is meaningless, and a plain `bool` would have hidden it. Refused
+ when the suites differ, when either ran nothing, or when the case counts differ — a shorter run
+ is a different corpus, not a better model. `.unwrap_or(false)` fails closed, which is what a
+ shipping decision should do.
+
+ Strictly greater, no margin: the milestone says "beats", and a tie is not one.

crates/panday-harness/src/eval.rs

@@ -137,6 +137,62 @@ impl Report {
  solvable.iter().filter(|r| r.with_reduction.solved).count() as f64 / solvable.len() as f64
  }
 
+ /// **M19.5's gate**: "≥25% cheaper semantic tier than the provider cheap-pool it replaces".
+ ///
+ /// Until now this suite had **no cost dimension at all** — it measured retention and token
+ /// ratio, which are not money. So the milestone could not be judged even with a model in hand,
+ /// and that is the failure docs/19 M19.1 exists to prevent: a gate stated in prose is a gate
+ /// nobody can fail.
+ ///
+ /// Cost, not tokens, because the two move independently and the milestone is about the
+ /// cheaper one *winning*. A reducer that keeps 40% of the tokens but hands them to a model at
+ /// three times the price is more expensive, and a ratio-only scorecard would call it a 60%
+ /// improvement.
+ ///
+ /// Prices are **per token**, in the same unit for both sides, and supplied by the caller —
+ /// this crate has no price table and must not grow one, or the gate starts depending on a
+ /// catalog that drifts. `panday_router`'s catalog is where a binary gets them.
+ ///
+ /// Returns `None` when the incumbent would spend nothing: a saving against zero is not a
+ /// percentage, and reporting one would be inventing a number.
+ pub fn cost_saving(&self, incumbent_price: f64, replacement_price: f64) -> Option<f64> {
+ let incumbent: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.without_reduction.tokens_raw as f64 * incumbent_price)
+ .sum();
+ let replacement: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.with_reduction.tokens_kept as f64 * replacement_price)
+ .sum();
+ if incumbent <= 0.0 {
+ return None;
+ }
+ Some(1.0 - replacement / incumbent)
+ }
+
+ /// M19.5, whole: zero regressions **and** at least `fraction` cheaper.
+ ///
+ /// Both halves, because either alone is a trap. Cheapness with regressions is a reducer that
+ /// saves money by losing the answer; zero regressions with no saving is a semantic tier with
+ /// no reason to exist. The milestone names both and so does this.
+ ///
+ /// `fraction` is a ratio — M19.5 says 25%, so `cheaper_than(0.25, ...)`. Unlike route-bench's
+ /// margin, which is stated in points, this one is stated as a percentage *of* the incumbent,
+ /// so a ratio is the honest shape. `the_two_gates_do_not_share_a_unit` pins the difference.
+ pub fn cheaper_than(
+ &self,
+ fraction: f64,
+ incumbent_price: f64,
+ replacement_price: f64,
+ ) -> bool {
+ self.regressions().is_empty()
+ && self
+ .cost_saving(incumbent_price, replacement_price)
+ .is_some_and(|saved| saved >= fraction)
+ }
+
  /// Mean reduction across the corpus. Reported *after* success, and never
  /// instead of it (ADR-007).
  pub fn mean_ratio(&self) -> f64 {
@@ -433,3 +489,88 @@ pub async fn run_corpus(
  }
  Report { rows }
 }
+
+#[cfg(test)]
+mod cost_gate_tests {
+ use super::{Outcome, Report, Row};
+
+ fn outcome(solved: bool, raw: u32, kept: u32) -> Outcome {
+ Outcome {
+ solved,
+ tokens_raw: raw,
+ tokens_kept: kept,
+ missing_facts: vec![],
+ strategy: "test".into(),
+ }
+ }
+
+ fn report(rows: Vec<(bool, u32, bool, u32)>) -> Report {
+ Report {
+ rows: rows
+ .into_iter()
+ .enumerate()
+ .map(|(i, (s0, raw, s1, kept))| Row {
+ scenario: format!("s{i}"),
+ without_reduction: outcome(s0, raw, raw),
+ with_reduction: outcome(s1, raw, kept),
+ })
+ .collect(),
+ }
+ }
+
+ #[test]
+ fn cost_is_tokens_times_price_not_tokens_alone() {
+ // The reason this dimension exists. Half the tokens at four times the price is *more*
+ // expensive, and a ratio-only scorecard would have called it a 50% win.
+ let r = report(vec![(true, 1000, true, 500)]);
+ assert_eq!(r.mean_ratio(), 0.5, "half the tokens kept");
+
+ let saving = r.cost_saving(1.0, 4.0).expect("a saving");
+ assert!(
+ saving < 0.0,
+ "keeping half the tokens at 4x the price costs more, not less: {saving}"
+ );
+ }
+
+ #[test]
+ fn a_saving_against_nothing_is_not_a_number() {
+ // An empty corpus, or one where the incumbent spends zero, has no percentage to report.
+ // Returning 0.0 or 1.0 here would be inventing a result.
+ assert_eq!(Report::default().cost_saving(1.0, 1.0), None);
+ assert_eq!(report(vec![(true, 0, true, 0)]).cost_saving(1.0, 1.0), None);
+ }
+
+ #[test]
+ fn the_gate_needs_both_halves() {
+ // Cheap but broken: 90% saved, and a scenario that was solvable is now not.
+ let broken = report(vec![(true, 1000, false, 100)]);
+ assert!(
+ !broken.cheaper_than(0.25, 1.0, 1.0),
+ "a regression is not paid for by being cheap"
+ );
+
+ // Sound but not cheap enough: no regressions, 10% saved against a 25% bar.
+ let timid = report(vec![(true, 1000, true, 900)]);
+ assert!(!timid.cheaper_than(0.25, 1.0, 1.0));
+
+ // Both: no regressions and 30% saved.
+ let good = report(vec![(true, 1000, true, 700)]);
+ assert!(good.cheaper_than(0.25, 1.0, 1.0));
+ }
+
+ #[test]
+ fn the_two_gates_do_not_share_a_unit() {
+ // route-bench's `Score::beats` takes percentage POINTS (M19.3: "≥10pt"); this one takes a
+ // FRACTION of the incumbent's spend (M19.5: "≥25% cheaper"). They read alike at a call
+ // site and mean different things, so each is pinned where it lives.
+ let r = report(vec![(true, 1000, true, 700)]);
+ assert!(
+ r.cheaper_than(0.25, 1.0, 1.0),
+ "0.25 is twenty-five percent"
+ );
+ assert!(
+ !r.cheaper_than(25.0, 1.0, 1.0),
+ "25.0 would be a 2500% saving, which nothing can meet"
+ );
+ }
+}

crates/panday-router/src/bench.rs

@@ -419,6 +419,28 @@ impl Score {
  pub fn meets_the_gate(&self, floor: f64) -> bool {
  self.accuracy() >= floor && self.confidently_wrong.is_empty()
  }
+
+ /// **M19.3's gate**: "classifier ... beats heuristic on route-bench by ≥10pt".
+ ///
+ /// [`meets_the_gate`](Self::meets_the_gate) is an absolute floor; this is the relative one the
+ /// milestone actually states, and until now it existed only as prose. A gate that is not a
+ /// function is a gate nobody can fail, which is the state docs/19 M19.1 was written against.
+ ///
+ /// `margin_points` is **percentage points**, not a fraction: M19.3 says "≥10pt", so the call
+ /// is `beats(&heuristic, 10.0)`. Accuracy is a 0..1 ratio internally and the conversion
+ /// happens here, once — passing `0.10` and meaning ten points is the obvious way to get this
+ /// wrong, so `the_margin_is_percentage_points_not_a_fraction` pins it.
+ ///
+ /// **Being confidently wrong disqualifies a challenger regardless of margin.** That is not an
+ /// extra condition bolted on: the dangerous quadrant is the thing route-bench exists to
+ /// measure (a wrong answer the router *trusts* and acts on), and a caller who checked only the
+ /// margin would ship a model that is more accurate on average and catastrophic on the cases
+ /// that matter. The incumbent's own quadrant is not consulted — the question is whether the
+ /// *challenger* is safe to deploy, not whether it is less bad than what is there.
+ pub fn beats(&self, incumbent: &Score, margin_points: f64) -> bool {
+ let gained = (self.accuracy() - incumbent.accuracy()) * 100.0;
+ gained >= margin_points && self.confidently_wrong.is_empty()
+ }
 }
 
 /// Score any classifier against the corpus. `HeuristicClassifier` today; a learned one at
@@ -448,3 +470,60 @@ pub fn score(c: &dyn Classifier) -> Score {
  }
  s
 }
+
+#[cfg(test)]
+mod gate_tests {
+ use super::Score;
+
+ fn score(correct: usize, total: usize, confidently_wrong: Vec<&'static str>) -> Score {
+ Score {
+ total,
+ correct,
+ confidently_wrong,
+ caught_by_the_gate: vec![],
+ }
+ }
+
+ #[test]
+ fn the_margin_is_percentage_points_not_a_fraction() {
+ // M19.3 says "≥10pt". Accuracy is a 0..1 ratio internally, so the obvious mistake is to
+ // pass 0.10 and mean ten points — which would let a challenger through on a *tenth* of a
+ // percentage point. The unit is pinned here because the gate is unfalsifiable by
+ // inspection: both readings compile and both look right.
+ let incumbent = score(70, 100, vec![]);
+ let exactly_ten = score(80, 100, vec![]);
+ let just_under = score(79, 100, vec![]);
+
+ assert!(
+ exactly_ten.beats(&incumbent, 10.0),
+ "10pt clears a 10pt bar"
+ );
+ assert!(!just_under.beats(&incumbent, 10.0), "9pt does not");
+ }
+
+ #[test]
+ fn a_confidently_wrong_challenger_does_not_ship_however_far_ahead() {
+ // The whole point of route-bench's dangerous quadrant. A model can be twenty points more
+ // accurate and still be the worse thing to deploy, because the router *acts* on a
+ // confident answer.
+ let incumbent = score(50, 100, vec![]);
+ let brilliant_but_reckless = score(90, 100, vec!["a-case-it-got-wrong-and-trusted"]);
+ assert!(
+ !brilliant_but_reckless.beats(&incumbent, 10.0),
+ "40pt ahead, and still not shippable"
+ );
+ }
+
+ #[test]
+ fn losing_ground_is_not_a_pass() {
+ let incumbent = score(90, 100, vec![]);
+ let worse = score(60, 100, vec![]);
+ assert!(!worse.beats(&incumbent, 10.0));
+ // And equal is not "beats" either: the milestone asks for a margin, not parity.
+ assert!(!incumbent.beats(&incumbent, 10.0));
+ assert!(
+ incumbent.beats(&incumbent, 0.0),
+ "a zero-point bar is met by parity"
+ );
+ }
+}

crates/panday-types/src/scorecard.rs

@@ -124,6 +124,33 @@ impl Scorecard {
  self.cases > 0 && self.rate() >= floor
  }
 
+ /// **The relative gate**: did this run beat an earlier one on the same suite?
+ ///
+ /// [`meets`](Self::meets) answers "is it good enough"; several milestones ask "is it *better*"
+ /// and had no function to ask with — M19.7 ("SFT stage beats base on agent-bench") is the
+ /// live one. A gate that exists only in prose is a gate nobody can fail.
+ ///
+ /// **`None` means the two cannot be compared, and that is the point of the return type.**
+ /// Comparing a tuned run over ten tasks against a base run over forty-one is the way this
+ /// measurement gets faked without anyone lying: both numbers are real, the ratio is
+ /// meaningless, and a `bool` would have hidden it. Refused when the suites differ, when either
+ /// ran nothing, or when the case counts differ — a shorter run is a different corpus, not a
+ /// better model. A caller that writes `.unwrap_or(false)` fails closed, which is the right
+ /// default for a shipping decision.
+ ///
+ /// Strictly greater, with no margin: M19.7 says "beats", and a tie is not one. Where a
+ /// milestone names a margin instead — M19.3's "≥10pt" — that gate lives with its suite and
+ /// takes the margin explicitly, because the unit is the suite's business, not this type's.
+ pub fn beats(&self, base: &Scorecard) -> Option<bool> {
+ if self.suite != base.suite || self.cases == 0 || base.cases == 0 {
+ return None;
+ }
+ if self.cases != base.cases {
+ return None;
+ }
+ Some(self.passed > base.passed)
+ }
+
  /// The human rendering. Derived from the artifact, never the other way round.
  pub fn to_markdown(&self) -> String {
  let mut out = format!(
@@ -226,3 +253,61 @@ mod tests {
  assert!(md.contains("missing required field `city`"));
  }
 }
+
+#[cfg(test)]
+mod relative_gate_tests {
+ use super::Scorecard;
+
+ fn card(suite: &str, passed: u32, cases: u32) -> Scorecard {
+ let mut c = Scorecard::new(suite, "subject", "2026-08-24T00:00:00Z");
+ for i in 0..cases {
+ c.record(&format!("case{i}"), i < passed, "");
+ }
+ c
+ }
+
+ #[test]
+ fn a_shorter_run_is_not_a_better_model() {
+ // The failure this return type exists for. Both numbers are real, the comparison is not:
+ // 9/10 against 30/41 is a different corpus, not an improvement. A `bool` would have said
+ // "better" and nobody would have noticed.
+ let base = card("agent-bench", 30, 41);
+ let tuned_on_fewer = card("agent-bench", 9, 10);
+ assert_eq!(tuned_on_fewer.beats(&base), None);
+ }
+
+ #[test]
+ fn suites_do_not_cross() {
+ let a = card("agent-bench", 40, 41);
+ let b = card("route-bench", 10, 41);
+ assert_eq!(a.beats(&b), None, "same case count, different question");
+ }
+
+ #[test]
+ fn nothing_ran_is_not_a_win() {
+ let base = card("agent-bench", 0, 0);
+ let tuned = card("agent-bench", 5, 5);
+ assert_eq!(tuned.beats(&base), None);
+ assert_eq!(base.beats(&tuned), None);
+ }
+
+ #[test]
+ fn beating_is_strict() {
+ let base = card("agent-bench", 30, 41);
+ assert_eq!(card("agent-bench", 31, 41).beats(&base), Some(true));
+ assert_eq!(
+ card("agent-bench", 30, 41).beats(&base),
+ Some(false),
+ "a tie is not a win"
+ );
+ assert_eq!(card("agent-bench", 29, 41).beats(&base), Some(false));
+ }
+
+ #[test]
+ fn an_incomparable_pair_fails_closed_for_a_careless_caller() {
+ // `.unwrap_or(false)` is what a caller will write. It must mean "do not ship".
+ let base = card("agent-bench", 30, 41);
+ let mismatched = card("agent-bench", 41, 42);
+ assert!(!mismatched.beats(&base).unwrap_or(false));
+ }
+}

docs/19-training.md

@@ -254,6 +254,17 @@ artifacts; CI runs eval gates nightly.
  one that emits well-formed calls with the wrong arguments — because a generator that cannot
  report a bad model as bad is not worth running against a good one.
 - **M19.3** Model 1 shipped: classifier behind `Classifier` trait beats heuristic on route-bench by ≥10pt; deployed in shadow, then live.
+
+ **The gate is a function now, though the model is not here.** `panday_router::bench::Score::beats(&incumbent, 10.0)`
+ — the margin in percentage points, as the milestone states it. It was prose until this change,
+ which meant M19.3 could not be failed even with a classifier in hand; `Score::meets_the_gate` is
+ an absolute floor and answers a different question.
+
+ A challenger that is **confidently wrong is refused regardless of margin**. That is not an extra
+ condition: the dangerous quadrant is what route-bench exists to measure, and a caller checking
+ only the margin would ship a model that is better on average and catastrophic where the router
+ acts on it. Unit pinned by test — passing `0.10` for "ten points" is the obvious way to get this
+ wrong and both readings compile.
 - **M19.4** Transcript mining pipeline with consent flags + PII scrub + provenance; first 10k-pair summarizer dataset. ✅ *(shipped: `panday_harness::mining`, `cargo xtask mine --logs <dir> --out <file>`. **The 10k-pair dataset is not here** — it needs 10k consented transcripts, and this repo has none.)*
 
  **Consent, then scrub, then provenance — and each step defaults to refusing.** The mistakes in a
@@ -285,6 +296,21 @@ artifacts; CI runs eval gates nightly.
  miner runs end to end today over a directory of logs and reports exactly why each candidate was
  dropped — which is the part that had to exist before any transcript was worth collecting.
 - **M19.5** Model 2 shipped: reduce-bench regression zero, ≥25% cheaper semantic tier than the provider cheap-pool it replaces; GGUF in catalog.
+
+ **reduce-bench had no cost dimension at all**, so "≥25% cheaper" was unmeasurable — the suite
+ scored retention and token ratio, which are not money. `Report::cost_saving` and
+ `Report::cheaper_than(0.25, incumbent_price, replacement_price)` add it.
+
+ Cost rather than tokens because the two move independently: a reducer that keeps 40% of the
+ tokens and hands them to a model at three times the price is *more* expensive, and the old
+ ratio-only scorecard would have called that a 60% win. Prices are per token and supplied by the
+ caller — `panday-harness` has no price table and must not grow one, or the gate starts drifting
+ with a catalog. A saving against an incumbent that spends nothing returns `None` rather than a
+ number.
+
+ Both halves are required, because either alone is a trap: cheapness with regressions is a
+ reducer that saves money by losing the answer, and zero regressions with no saving is a semantic
+ tier with no reason to exist.
 - **M19.6** agent-bench (50 verifiable repo tasks in T3) doubling as GRPO environment. ✅ *(shipped: `panday_harness::agent_bench` — 41 tasks, the audit, the jailed runner, and `score`. **Runs in T2, not T3**. Thirty-eight are repair classes; three (`poisoned-readme`, `poisoned-comment`, `granted-json`) are M20.1 injection canaries whose verifier fails if a relative marker file exists. Still not 50 — the twelve destructive classes stay gone.)*
 
  **This is the second attempt, and the first one destroyed a machine.** The original ran each
@@ -334,3 +360,15 @@ artifacts; CI runs eval gates nightly.
  **What is left is the model**, and T3 for the tasks that will eventually need a VM rather than a
  jail (M14.5).
 - **M19.7** Model 3 v1: SFT stage beats base on agent-bench; go/no-go review for the GRPO spend.
+
+ **`Scorecard::beats(&base)` is the gate**, on the artifact type every suite already emits, so
+ it is not agent-bench's alone.
+
+ It returns `Option<bool>`, and the `None` is the substance. Comparing a tuned run over ten tasks
+ against a base run over forty-one is how this measurement gets faked without anyone lying: both
+ numbers are real, the ratio is meaningless, and a plain `bool` would have hidden it. Refused
+ when the suites differ, when either ran nothing, or when the case counts differ — a shorter run
+ is a different corpus, not a better model. `.unwrap_or(false)` fails closed, which is what a
+ shipping decision should do.
+
+ Strictly greater, no margin: the milestone says "beats", and a tie is not one.

crates/panday-harness/src/eval.rs

@@ -137,6 +137,62 @@ impl Report {
  solvable.iter().filter(|r| r.with_reduction.solved).count() as f64 / solvable.len() as f64
  }
 
+ /// **M19.5's gate**: "≥25% cheaper semantic tier than the provider cheap-pool it replaces".
+ ///
+ /// Until now this suite had **no cost dimension at all** — it measured retention and token
+ /// ratio, which are not money. So the milestone could not be judged even with a model in hand,
+ /// and that is the failure docs/19 M19.1 exists to prevent: a gate stated in prose is a gate
+ /// nobody can fail.
+ ///
+ /// Cost, not tokens, because the two move independently and the milestone is about the
+ /// cheaper one *winning*. A reducer that keeps 40% of the tokens but hands them to a model at
+ /// three times the price is more expensive, and a ratio-only scorecard would call it a 60%
+ /// improvement.
+ ///
+ /// Prices are **per token**, in the same unit for both sides, and supplied by the caller —
+ /// this crate has no price table and must not grow one, or the gate starts depending on a
+ /// catalog that drifts. `panday_router`'s catalog is where a binary gets them.
+ ///
+ /// Returns `None` when the incumbent would spend nothing: a saving against zero is not a
+ /// percentage, and reporting one would be inventing a number.
+ pub fn cost_saving(&self, incumbent_price: f64, replacement_price: f64) -> Option<f64> {
+ let incumbent: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.without_reduction.tokens_raw as f64 * incumbent_price)
+ .sum();
+ let replacement: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.with_reduction.tokens_kept as f64 * replacement_price)
+ .sum();
+ if incumbent <= 0.0 {
+ return None;
+ }
+ Some(1.0 - replacement / incumbent)
+ }
+
+ /// M19.5, whole: zero regressions **and** at least `fraction` cheaper.
+ ///
+ /// Both halves, because either alone is a trap. Cheapness with regressions is a reducer that
+ /// saves money by losing the answer; zero regressions with no saving is a semantic tier with
+ /// no reason to exist. The milestone names both and so does this.
+ ///
+ /// `fraction` is a ratio — M19.5 says 25%, so `cheaper_than(0.25, ...)`. Unlike route-bench's
+ /// margin, which is stated in points, this one is stated as a percentage *of* the incumbent,
+ /// so a ratio is the honest shape. `the_two_gates_do_not_share_a_unit` pins the difference.
+ pub fn cheaper_than(
+ &self,
+ fraction: f64,
+ incumbent_price: f64,
+ replacement_price: f64,
+ ) -> bool {
+ self.regressions().is_empty()
+ && self
+ .cost_saving(incumbent_price, replacement_price)
+ .is_some_and(|saved| saved >= fraction)
+ }
+
  /// Mean reduction across the corpus. Reported *after* success, and never
  /// instead of it (ADR-007).
  pub fn mean_ratio(&self) -> f64 {
@@ -433,3 +489,88 @@ pub async fn run_corpus(
  }
  Report { rows }
 }
+
+#[cfg(test)]
+mod cost_gate_tests {
+ use super::{Outcome, Report, Row};
+
+ fn outcome(solved: bool, raw: u32, kept: u32) -> Outcome {
+ Outcome {
+ solved,
+ tokens_raw: raw,
+ tokens_kept: kept,
+ missing_facts: vec![],
+ strategy: "test".into(),
+ }
+ }
+
+ fn report(rows: Vec<(bool, u32, bool, u32)>) -> Report {
+ Report {
+ rows: rows
+ .into_iter()
+ .enumerate()
+ .map(|(i, (s0, raw, s1, kept))| Row {
+ scenario: format!("s{i}"),
+ without_reduction: outcome(s0, raw, raw),
+ with_reduction: outcome(s1, raw, kept),
+ })
+ .collect(),
+ }
+ }
+
+ #[test]
+ fn cost_is_tokens_times_price_not_tokens_alone() {
+ // The reason this dimension exists. Half the tokens at four times the price is *more*
+ // expensive, and a ratio-only scorecard would have called it a 50% win.
+ let r = report(vec![(true, 1000, true, 500)]);
+ assert_eq!(r.mean_ratio(), 0.5, "half the tokens kept");
+
+ let saving = r.cost_saving(1.0, 4.0).expect("a saving");
+ assert!(
+ saving < 0.0,
+ "keeping half the tokens at 4x the price costs more, not less: {saving}"
+ );
+ }
+
+ #[test]
+ fn a_saving_against_nothing_is_not_a_number() {
+ // An empty corpus, or one where the incumbent spends zero, has no percentage to report.
+ // Returning 0.0 or 1.0 here would be inventing a result.
+ assert_eq!(Report::default().cost_saving(1.0, 1.0), None);
+ assert_eq!(report(vec![(true, 0, true, 0)]).cost_saving(1.0, 1.0), None);
+ }
+
+ #[test]
+ fn the_gate_needs_both_halves() {
+ // Cheap but broken: 90% saved, and a scenario that was solvable is now not.
+ let broken = report(vec![(true, 1000, false, 100)]);
+ assert!(
+ !broken.cheaper_than(0.25, 1.0, 1.0),
+ "a regression is not paid for by being cheap"
+ );
+
+ // Sound but not cheap enough: no regressions, 10% saved against a 25% bar.
+ let timid = report(vec![(true, 1000, true, 900)]);
+ assert!(!timid.cheaper_than(0.25, 1.0, 1.0));
+
+ // Both: no regressions and 30% saved.
+ let good = report(vec![(true, 1000, true, 700)]);
+ assert!(good.cheaper_than(0.25, 1.0, 1.0));
+ }
+
+ #[test]
+ fn the_two_gates_do_not_share_a_unit() {
+ // route-bench's `Score::beats` takes percentage POINTS (M19.3: "≥10pt"); this one takes a
+ // FRACTION of the incumbent's spend (M19.5: "≥25% cheaper"). They read alike at a call
+ // site and mean different things, so each is pinned where it lives.
+ let r = report(vec![(true, 1000, true, 700)]);
+ assert!(
+ r.cheaper_than(0.25, 1.0, 1.0),
+ "0.25 is twenty-five percent"
+ );
+ assert!(
+ !r.cheaper_than(25.0, 1.0, 1.0),
+ "25.0 would be a 2500% saving, which nothing can meet"
+ );
+ }
+}

crates/panday-router/src/bench.rs

@@ -419,6 +419,28 @@ impl Score {
  pub fn meets_the_gate(&self, floor: f64) -> bool {
  self.accuracy() >= floor && self.confidently_wrong.is_empty()
  }
+
+ /// **M19.3's gate**: "classifier ... beats heuristic on route-bench by ≥10pt".
+ ///
+ /// [`meets_the_gate`](Self::meets_the_gate) is an absolute floor; this is the relative one the
+ /// milestone actually states, and until now it existed only as prose. A gate that is not a
+ /// function is a gate nobody can fail, which is the state docs/19 M19.1 was written against.
+ ///
+ /// `margin_points` is **percentage points**, not a fraction: M19.3 says "≥10pt", so the call
+ /// is `beats(&heuristic, 10.0)`. Accuracy is a 0..1 ratio internally and the conversion
+ /// happens here, once — passing `0.10` and meaning ten points is the obvious way to get this
+ /// wrong, so `the_margin_is_percentage_points_not_a_fraction` pins it.
+ ///
+ /// **Being confidently wrong disqualifies a challenger regardless of margin.** That is not an
+ /// extra condition bolted on: the dangerous quadrant is the thing route-bench exists to
+ /// measure (a wrong answer the router *trusts* and acts on), and a caller who checked only the
+ /// margin would ship a model that is more accurate on average and catastrophic on the cases
+ /// that matter. The incumbent's own quadrant is not consulted — the question is whether the
+ /// *challenger* is safe to deploy, not whether it is less bad than what is there.
+ pub fn beats(&self, incumbent: &Score, margin_points: f64) -> bool {
+ let gained = (self.accuracy() - incumbent.accuracy()) * 100.0;
+ gained >= margin_points && self.confidently_wrong.is_empty()
+ }
 }
 
 /// Score any classifier against the corpus. `HeuristicClassifier` today; a learned one at
@@ -448,3 +470,60 @@ pub fn score(c: &dyn Classifier) -> Score {
  }
  s
 }
+
+#[cfg(test)]
+mod gate_tests {
+ use super::Score;
+
+ fn score(correct: usize, total: usize, confidently_wrong: Vec<&'static str>) -> Score {
+ Score {
+ total,
+ correct,
+ confidently_wrong,
+ caught_by_the_gate: vec![],
+ }
+ }
+
+ #[test]
+ fn the_margin_is_percentage_points_not_a_fraction() {
+ // M19.3 says "≥10pt". Accuracy is a 0..1 ratio internally, so the obvious mistake is to
+ // pass 0.10 and mean ten points — which would let a challenger through on a *tenth* of a
+ // percentage point. The unit is pinned here because the gate is unfalsifiable by
+ // inspection: both readings compile and both look right.
+ let incumbent = score(70, 100, vec![]);
+ let exactly_ten = score(80, 100, vec![]);
+ let just_under = score(79, 100, vec![]);
+
+ assert!(
+ exactly_ten.beats(&incumbent, 10.0),
+ "10pt clears a 10pt bar"
+ );
+ assert!(!just_under.beats(&incumbent, 10.0), "9pt does not");
+ }
+
+ #[test]
+ fn a_confidently_wrong_challenger_does_not_ship_however_far_ahead() {
+ // The whole point of route-bench's dangerous quadrant. A model can be twenty points more
+ // accurate and still be the worse thing to deploy, because the router *acts* on a
+ // confident answer.
+ let incumbent = score(50, 100, vec![]);
+ let brilliant_but_reckless = score(90, 100, vec!["a-case-it-got-wrong-and-trusted"]);
+ assert!(
+ !brilliant_but_reckless.beats(&incumbent, 10.0),
+ "40pt ahead, and still not shippable"
+ );
+ }
+
+ #[test]
+ fn losing_ground_is_not_a_pass() {
+ let incumbent = score(90, 100, vec![]);
+ let worse = score(60, 100, vec![]);
+ assert!(!worse.beats(&incumbent, 10.0));
+ // And equal is not "beats" either: the milestone asks for a margin, not parity.
+ assert!(!incumbent.beats(&incumbent, 10.0));
+ assert!(
+ incumbent.beats(&incumbent, 0.0),
+ "a zero-point bar is met by parity"
+ );
+ }
+}

crates/panday-types/src/scorecard.rs

@@ -124,6 +124,33 @@ impl Scorecard {
  self.cases > 0 && self.rate() >= floor
  }
 
+ /// **The relative gate**: did this run beat an earlier one on the same suite?
+ ///
+ /// [`meets`](Self::meets) answers "is it good enough"; several milestones ask "is it *better*"
+ /// and had no function to ask with — M19.7 ("SFT stage beats base on agent-bench") is the
+ /// live one. A gate that exists only in prose is a gate nobody can fail.
+ ///
+ /// **`None` means the two cannot be compared, and that is the point of the return type.**
+ /// Comparing a tuned run over ten tasks against a base run over forty-one is the way this
+ /// measurement gets faked without anyone lying: both numbers are real, the ratio is
+ /// meaningless, and a `bool` would have hidden it. Refused when the suites differ, when either
+ /// ran nothing, or when the case counts differ — a shorter run is a different corpus, not a
+ /// better model. A caller that writes `.unwrap_or(false)` fails closed, which is the right
+ /// default for a shipping decision.
+ ///
+ /// Strictly greater, with no margin: M19.7 says "beats", and a tie is not one. Where a
+ /// milestone names a margin instead — M19.3's "≥10pt" — that gate lives with its suite and
+ /// takes the margin explicitly, because the unit is the suite's business, not this type's.
+ pub fn beats(&self, base: &Scorecard) -> Option<bool> {
+ if self.suite != base.suite || self.cases == 0 || base.cases == 0 {
+ return None;
+ }
+ if self.cases != base.cases {
+ return None;
+ }
+ Some(self.passed > base.passed)
+ }
+
  /// The human rendering. Derived from the artifact, never the other way round.
  pub fn to_markdown(&self) -> String {
  let mut out = format!(
@@ -226,3 +253,61 @@ mod tests {
  assert!(md.contains("missing required field `city`"));
  }
 }
+
+#[cfg(test)]
+mod relative_gate_tests {
+ use super::Scorecard;
+
+ fn card(suite: &str, passed: u32, cases: u32) -> Scorecard {
+ let mut c = Scorecard::new(suite, "subject", "2026-08-24T00:00:00Z");
+ for i in 0..cases {
+ c.record(&format!("case{i}"), i < passed, "");
+ }
+ c
+ }
+
+ #[test]
+ fn a_shorter_run_is_not_a_better_model() {
+ // The failure this return type exists for. Both numbers are real, the comparison is not:
+ // 9/10 against 30/41 is a different corpus, not an improvement. A `bool` would have said
+ // "better" and nobody would have noticed.
+ let base = card("agent-bench", 30, 41);
+ let tuned_on_fewer = card("agent-bench", 9, 10);
+ assert_eq!(tuned_on_fewer.beats(&base), None);
+ }
+
+ #[test]
+ fn suites_do_not_cross() {
+ let a = card("agent-bench", 40, 41);
+ let b = card("route-bench", 10, 41);
+ assert_eq!(a.beats(&b), None, "same case count, different question");
+ }
+
+ #[test]
+ fn nothing_ran_is_not_a_win() {
+ let base = card("agent-bench", 0, 0);
+ let tuned = card("agent-bench", 5, 5);
+ assert_eq!(tuned.beats(&base), None);
+ assert_eq!(base.beats(&tuned), None);
+ }
+
+ #[test]
+ fn beating_is_strict() {
+ let base = card("agent-bench", 30, 41);
+ assert_eq!(card("agent-bench", 31, 41).beats(&base), Some(true));
+ assert_eq!(
+ card("agent-bench", 30, 41).beats(&base),
+ Some(false),
+ "a tie is not a win"
+ );
+ assert_eq!(card("agent-bench", 29, 41).beats(&base), Some(false));
+ }
+
+ #[test]
+ fn an_incomparable_pair_fails_closed_for_a_careless_caller() {
+ // `.unwrap_or(false)` is what a caller will write. It must mean "do not ship".
+ let base = card("agent-bench", 30, 41);
+ let mismatched = card("agent-bench", 41, 42);
+ assert!(!mismatched.beats(&base).unwrap_or(false));
+ }
+}

docs/19-training.md

@@ -254,6 +254,17 @@ artifacts; CI runs eval gates nightly.
  one that emits well-formed calls with the wrong arguments — because a generator that cannot
  report a bad model as bad is not worth running against a good one.
 - **M19.3** Model 1 shipped: classifier behind `Classifier` trait beats heuristic on route-bench by ≥10pt; deployed in shadow, then live.
+
+ **The gate is a function now, though the model is not here.** `panday_router::bench::Score::beats(&incumbent, 10.0)`
+ — the margin in percentage points, as the milestone states it. It was prose until this change,
+ which meant M19.3 could not be failed even with a classifier in hand; `Score::meets_the_gate` is
+ an absolute floor and answers a different question.
+
+ A challenger that is **confidently wrong is refused regardless of margin**. That is not an extra
+ condition: the dangerous quadrant is what route-bench exists to measure, and a caller checking
+ only the margin would ship a model that is better on average and catastrophic where the router
+ acts on it. Unit pinned by test — passing `0.10` for "ten points" is the obvious way to get this
+ wrong and both readings compile.
 - **M19.4** Transcript mining pipeline with consent flags + PII scrub + provenance; first 10k-pair summarizer dataset. ✅ *(shipped: `panday_harness::mining`, `cargo xtask mine --logs <dir> --out <file>`. **The 10k-pair dataset is not here** — it needs 10k consented transcripts, and this repo has none.)*
 
  **Consent, then scrub, then provenance — and each step defaults to refusing.** The mistakes in a
@@ -285,6 +296,21 @@ artifacts; CI runs eval gates nightly.
  miner runs end to end today over a directory of logs and reports exactly why each candidate was
  dropped — which is the part that had to exist before any transcript was worth collecting.
 - **M19.5** Model 2 shipped: reduce-bench regression zero, ≥25% cheaper semantic tier than the provider cheap-pool it replaces; GGUF in catalog.
+
+ **reduce-bench had no cost dimension at all**, so "≥25% cheaper" was unmeasurable — the suite
+ scored retention and token ratio, which are not money. `Report::cost_saving` and
+ `Report::cheaper_than(0.25, incumbent_price, replacement_price)` add it.
+
+ Cost rather than tokens because the two move independently: a reducer that keeps 40% of the
+ tokens and hands them to a model at three times the price is *more* expensive, and the old
+ ratio-only scorecard would have called that a 60% win. Prices are per token and supplied by the
+ caller — `panday-harness` has no price table and must not grow one, or the gate starts drifting
+ with a catalog. A saving against an incumbent that spends nothing returns `None` rather than a
+ number.
+
+ Both halves are required, because either alone is a trap: cheapness with regressions is a
+ reducer that saves money by losing the answer, and zero regressions with no saving is a semantic
+ tier with no reason to exist.
 - **M19.6** agent-bench (50 verifiable repo tasks in T3) doubling as GRPO environment. ✅ *(shipped: `panday_harness::agent_bench` — 41 tasks, the audit, the jailed runner, and `score`. **Runs in T2, not T3**. Thirty-eight are repair classes; three (`poisoned-readme`, `poisoned-comment`, `granted-json`) are M20.1 injection canaries whose verifier fails if a relative marker file exists. Still not 50 — the twelve destructive classes stay gone.)*
 
  **This is the second attempt, and the first one destroyed a machine.** The original ran each
@@ -334,3 +360,15 @@ artifacts; CI runs eval gates nightly.
  **What is left is the model**, and T3 for the tasks that will eventually need a VM rather than a
  jail (M14.5).
 - **M19.7** Model 3 v1: SFT stage beats base on agent-bench; go/no-go review for the GRPO spend.
+
+ **`Scorecard::beats(&base)` is the gate**, on the artifact type every suite already emits, so
+ it is not agent-bench's alone.
+
+ It returns `Option<bool>`, and the `None` is the substance. Comparing a tuned run over ten tasks
+ against a base run over forty-one is how this measurement gets faked without anyone lying: both
+ numbers are real, the ratio is meaningless, and a plain `bool` would have hidden it. Refused
+ when the suites differ, when either ran nothing, or when the case counts differ — a shorter run
+ is a different corpus, not a better model. `.unwrap_or(false)` fails closed, which is what a
+ shipping decision should do.
+
+ Strictly greater, no margin: the milestone says "beats", and a tie is not one.

crates/panday-harness/src/eval.rs

@@ -137,6 +137,62 @@ impl Report {
  solvable.iter().filter(|r| r.with_reduction.solved).count() as f64 / solvable.len() as f64
  }
 
+ /// **M19.5's gate**: "≥25% cheaper semantic tier than the provider cheap-pool it replaces".
+ ///
+ /// Until now this suite had **no cost dimension at all** — it measured retention and token
+ /// ratio, which are not money. So the milestone could not be judged even with a model in hand,
+ /// and that is the failure docs/19 M19.1 exists to prevent: a gate stated in prose is a gate
+ /// nobody can fail.
+ ///
+ /// Cost, not tokens, because the two move independently and the milestone is about the
+ /// cheaper one *winning*. A reducer that keeps 40% of the tokens but hands them to a model at
+ /// three times the price is more expensive, and a ratio-only scorecard would call it a 60%
+ /// improvement.
+ ///
+ /// Prices are **per token**, in the same unit for both sides, and supplied by the caller —
+ /// this crate has no price table and must not grow one, or the gate starts depending on a
+ /// catalog that drifts. `panday_router`'s catalog is where a binary gets them.
+ ///
+ /// Returns `None` when the incumbent would spend nothing: a saving against zero is not a
+ /// percentage, and reporting one would be inventing a number.
+ pub fn cost_saving(&self, incumbent_price: f64, replacement_price: f64) -> Option<f64> {
+ let incumbent: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.without_reduction.tokens_raw as f64 * incumbent_price)
+ .sum();
+ let replacement: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.with_reduction.tokens_kept as f64 * replacement_price)
+ .sum();
+ if incumbent <= 0.0 {
+ return None;
+ }
+ Some(1.0 - replacement / incumbent)
+ }
+
+ /// M19.5, whole: zero regressions **and** at least `fraction` cheaper.
+ ///
+ /// Both halves, because either alone is a trap. Cheapness with regressions is a reducer that
+ /// saves money by losing the answer; zero regressions with no saving is a semantic tier with
+ /// no reason to exist. The milestone names both and so does this.
+ ///
+ /// `fraction` is a ratio — M19.5 says 25%, so `cheaper_than(0.25, ...)`. Unlike route-bench's
+ /// margin, which is stated in points, this one is stated as a percentage *of* the incumbent,
+ /// so a ratio is the honest shape. `the_two_gates_do_not_share_a_unit` pins the difference.
+ pub fn cheaper_than(
+ &self,
+ fraction: f64,
+ incumbent_price: f64,
+ replacement_price: f64,
+ ) -> bool {
+ self.regressions().is_empty()
+ && self
+ .cost_saving(incumbent_price, replacement_price)
+ .is_some_and(|saved| saved >= fraction)
+ }
+
  /// Mean reduction across the corpus. Reported *after* success, and never
  /// instead of it (ADR-007).
  pub fn mean_ratio(&self) -> f64 {
@@ -433,3 +489,88 @@ pub async fn run_corpus(
  }
  Report { rows }
 }
+
+#[cfg(test)]
+mod cost_gate_tests {
+ use super::{Outcome, Report, Row};
+
+ fn outcome(solved: bool, raw: u32, kept: u32) -> Outcome {
+ Outcome {
+ solved,
+ tokens_raw: raw,
+ tokens_kept: kept,
+ missing_facts: vec![],
+ strategy: "test".into(),
+ }
+ }
+
+ fn report(rows: Vec<(bool, u32, bool, u32)>) -> Report {
+ Report {
+ rows: rows
+ .into_iter()
+ .enumerate()
+ .map(|(i, (s0, raw, s1, kept))| Row {
+ scenario: format!("s{i}"),
+ without_reduction: outcome(s0, raw, raw),
+ with_reduction: outcome(s1, raw, kept),
+ })
+ .collect(),
+ }
+ }
+
+ #[test]
+ fn cost_is_tokens_times_price_not_tokens_alone() {
+ // The reason this dimension exists. Half the tokens at four times the price is *more*
+ // expensive, and a ratio-only scorecard would have called it a 50% win.
+ let r = report(vec![(true, 1000, true, 500)]);
+ assert_eq!(r.mean_ratio(), 0.5, "half the tokens kept");
+
+ let saving = r.cost_saving(1.0, 4.0).expect("a saving");
+ assert!(
+ saving < 0.0,
+ "keeping half the tokens at 4x the price costs more, not less: {saving}"
+ );
+ }
+
+ #[test]
+ fn a_saving_against_nothing_is_not_a_number() {
+ // An empty corpus, or one where the incumbent spends zero, has no percentage to report.
+ // Returning 0.0 or 1.0 here would be inventing a result.
+ assert_eq!(Report::default().cost_saving(1.0, 1.0), None);
+ assert_eq!(report(vec![(true, 0, true, 0)]).cost_saving(1.0, 1.0), None);
+ }
+
+ #[test]
+ fn the_gate_needs_both_halves() {
+ // Cheap but broken: 90% saved, and a scenario that was solvable is now not.
+ let broken = report(vec![(true, 1000, false, 100)]);
+ assert!(
+ !broken.cheaper_than(0.25, 1.0, 1.0),
+ "a regression is not paid for by being cheap"
+ );
+
+ // Sound but not cheap enough: no regressions, 10% saved against a 25% bar.
+ let timid = report(vec![(true, 1000, true, 900)]);
+ assert!(!timid.cheaper_than(0.25, 1.0, 1.0));
+
+ // Both: no regressions and 30% saved.
+ let good = report(vec![(true, 1000, true, 700)]);
+ assert!(good.cheaper_than(0.25, 1.0, 1.0));
+ }
+
+ #[test]
+ fn the_two_gates_do_not_share_a_unit() {
+ // route-bench's `Score::beats` takes percentage POINTS (M19.3: "≥10pt"); this one takes a
+ // FRACTION of the incumbent's spend (M19.5: "≥25% cheaper"). They read alike at a call
+ // site and mean different things, so each is pinned where it lives.
+ let r = report(vec![(true, 1000, true, 700)]);
+ assert!(
+ r.cheaper_than(0.25, 1.0, 1.0),
+ "0.25 is twenty-five percent"
+ );
+ assert!(
+ !r.cheaper_than(25.0, 1.0, 1.0),
+ "25.0 would be a 2500% saving, which nothing can meet"
+ );
+ }
+}

crates/panday-router/src/bench.rs

@@ -419,6 +419,28 @@ impl Score {
  pub fn meets_the_gate(&self, floor: f64) -> bool {
  self.accuracy() >= floor && self.confidently_wrong.is_empty()
  }
+
+ /// **M19.3's gate**: "classifier ... beats heuristic on route-bench by ≥10pt".
+ ///
+ /// [`meets_the_gate`](Self::meets_the_gate) is an absolute floor; this is the relative one the
+ /// milestone actually states, and until now it existed only as prose. A gate that is not a
+ /// function is a gate nobody can fail, which is the state docs/19 M19.1 was written against.
+ ///
+ /// `margin_points` is **percentage points**, not a fraction: M19.3 says "≥10pt", so the call
+ /// is `beats(&heuristic, 10.0)`. Accuracy is a 0..1 ratio internally and the conversion
+ /// happens here, once — passing `0.10` and meaning ten points is the obvious way to get this
+ /// wrong, so `the_margin_is_percentage_points_not_a_fraction` pins it.
+ ///
+ /// **Being confidently wrong disqualifies a challenger regardless of margin.** That is not an
+ /// extra condition bolted on: the dangerous quadrant is the thing route-bench exists to
+ /// measure (a wrong answer the router *trusts* and acts on), and a caller who checked only the
+ /// margin would ship a model that is more accurate on average and catastrophic on the cases
+ /// that matter. The incumbent's own quadrant is not consulted — the question is whether the
+ /// *challenger* is safe to deploy, not whether it is less bad than what is there.
+ pub fn beats(&self, incumbent: &Score, margin_points: f64) -> bool {
+ let gained = (self.accuracy() - incumbent.accuracy()) * 100.0;
+ gained >= margin_points && self.confidently_wrong.is_empty()
+ }
 }
 
 /// Score any classifier against the corpus. `HeuristicClassifier` today; a learned one at
@@ -448,3 +470,60 @@ pub fn score(c: &dyn Classifier) -> Score {
  }
  s
 }
+
+#[cfg(test)]
+mod gate_tests {
+ use super::Score;
+
+ fn score(correct: usize, total: usize, confidently_wrong: Vec<&'static str>) -> Score {
+ Score {
+ total,
+ correct,
+ confidently_wrong,
+ caught_by_the_gate: vec![],
+ }
+ }
+
+ #[test]
+ fn the_margin_is_percentage_points_not_a_fraction() {
+ // M19.3 says "≥10pt". Accuracy is a 0..1 ratio internally, so the obvious mistake is to
+ // pass 0.10 and mean ten points — which would let a challenger through on a *tenth* of a
+ // percentage point. The unit is pinned here because the gate is unfalsifiable by
+ // inspection: both readings compile and both look right.
+ let incumbent = score(70, 100, vec![]);
+ let exactly_ten = score(80, 100, vec![]);
+ let just_under = score(79, 100, vec![]);
+
+ assert!(
+ exactly_ten.beats(&incumbent, 10.0),
+ "10pt clears a 10pt bar"
+ );
+ assert!(!just_under.beats(&incumbent, 10.0), "9pt does not");
+ }
+
+ #[test]
+ fn a_confidently_wrong_challenger_does_not_ship_however_far_ahead() {
+ // The whole point of route-bench's dangerous quadrant. A model can be twenty points more
+ // accurate and still be the worse thing to deploy, because the router *acts* on a
+ // confident answer.
+ let incumbent = score(50, 100, vec![]);
+ let brilliant_but_reckless = score(90, 100, vec!["a-case-it-got-wrong-and-trusted"]);
+ assert!(
+ !brilliant_but_reckless.beats(&incumbent, 10.0),
+ "40pt ahead, and still not shippable"
+ );
+ }
+
+ #[test]
+ fn losing_ground_is_not_a_pass() {
+ let incumbent = score(90, 100, vec![]);
+ let worse = score(60, 100, vec![]);
+ assert!(!worse.beats(&incumbent, 10.0));
+ // And equal is not "beats" either: the milestone asks for a margin, not parity.
+ assert!(!incumbent.beats(&incumbent, 10.0));
+ assert!(
+ incumbent.beats(&incumbent, 0.0),
+ "a zero-point bar is met by parity"
+ );
+ }
+}

crates/panday-types/src/scorecard.rs

@@ -124,6 +124,33 @@ impl Scorecard {
  self.cases > 0 && self.rate() >= floor
  }
 
+ /// **The relative gate**: did this run beat an earlier one on the same suite?
+ ///
+ /// [`meets`](Self::meets) answers "is it good enough"; several milestones ask "is it *better*"
+ /// and had no function to ask with — M19.7 ("SFT stage beats base on agent-bench") is the
+ /// live one. A gate that exists only in prose is a gate nobody can fail.
+ ///
+ /// **`None` means the two cannot be compared, and that is the point of the return type.**
+ /// Comparing a tuned run over ten tasks against a base run over forty-one is the way this
+ /// measurement gets faked without anyone lying: both numbers are real, the ratio is
+ /// meaningless, and a `bool` would have hidden it. Refused when the suites differ, when either
+ /// ran nothing, or when the case counts differ — a shorter run is a different corpus, not a
+ /// better model. A caller that writes `.unwrap_or(false)` fails closed, which is the right
+ /// default for a shipping decision.
+ ///
+ /// Strictly greater, with no margin: M19.7 says "beats", and a tie is not one. Where a
+ /// milestone names a margin instead — M19.3's "≥10pt" — that gate lives with its suite and
+ /// takes the margin explicitly, because the unit is the suite's business, not this type's.
+ pub fn beats(&self, base: &Scorecard) -> Option<bool> {
+ if self.suite != base.suite || self.cases == 0 || base.cases == 0 {
+ return None;
+ }
+ if self.cases != base.cases {
+ return None;
+ }
+ Some(self.passed > base.passed)
+ }
+
  /// The human rendering. Derived from the artifact, never the other way round.
  pub fn to_markdown(&self) -> String {
  let mut out = format!(
@@ -226,3 +253,61 @@ mod tests {
  assert!(md.contains("missing required field `city`"));
  }
 }
+
+#[cfg(test)]
+mod relative_gate_tests {
+ use super::Scorecard;
+
+ fn card(suite: &str, passed: u32, cases: u32) -> Scorecard {
+ let mut c = Scorecard::new(suite, "subject", "2026-08-24T00:00:00Z");
+ for i in 0..cases {
+ c.record(&format!("case{i}"), i < passed, "");
+ }
+ c
+ }
+
+ #[test]
+ fn a_shorter_run_is_not_a_better_model() {
+ // The failure this return type exists for. Both numbers are real, the comparison is not:
+ // 9/10 against 30/41 is a different corpus, not an improvement. A `bool` would have said
+ // "better" and nobody would have noticed.
+ let base = card("agent-bench", 30, 41);
+ let tuned_on_fewer = card("agent-bench", 9, 10);
+ assert_eq!(tuned_on_fewer.beats(&base), None);
+ }
+
+ #[test]
+ fn suites_do_not_cross() {
+ let a = card("agent-bench", 40, 41);
+ let b = card("route-bench", 10, 41);
+ assert_eq!(a.beats(&b), None, "same case count, different question");
+ }
+
+ #[test]
+ fn nothing_ran_is_not_a_win() {
+ let base = card("agent-bench", 0, 0);
+ let tuned = card("agent-bench", 5, 5);
+ assert_eq!(tuned.beats(&base), None);
+ assert_eq!(base.beats(&tuned), None);
+ }
+
+ #[test]
+ fn beating_is_strict() {
+ let base = card("agent-bench", 30, 41);
+ assert_eq!(card("agent-bench", 31, 41).beats(&base), Some(true));
+ assert_eq!(
+ card("agent-bench", 30, 41).beats(&base),
+ Some(false),
+ "a tie is not a win"
+ );
+ assert_eq!(card("agent-bench", 29, 41).beats(&base), Some(false));
+ }
+
+ #[test]
+ fn an_incomparable_pair_fails_closed_for_a_careless_caller() {
+ // `.unwrap_or(false)` is what a caller will write. It must mean "do not ship".
+ let base = card("agent-bench", 30, 41);
+ let mismatched = card("agent-bench", 41, 42);
+ assert!(!mismatched.beats(&base).unwrap_or(false));
+ }
+}

docs/19-training.md

@@ -254,6 +254,17 @@ artifacts; CI runs eval gates nightly.
  one that emits well-formed calls with the wrong arguments — because a generator that cannot
  report a bad model as bad is not worth running against a good one.
 - **M19.3** Model 1 shipped: classifier behind `Classifier` trait beats heuristic on route-bench by ≥10pt; deployed in shadow, then live.
+
+ **The gate is a function now, though the model is not here.** `panday_router::bench::Score::beats(&incumbent, 10.0)`
+ — the margin in percentage points, as the milestone states it. It was prose until this change,
+ which meant M19.3 could not be failed even with a classifier in hand; `Score::meets_the_gate` is
+ an absolute floor and answers a different question.
+
+ A challenger that is **confidently wrong is refused regardless of margin**. That is not an extra
+ condition: the dangerous quadrant is what route-bench exists to measure, and a caller checking
+ only the margin would ship a model that is better on average and catastrophic where the router
+ acts on it. Unit pinned by test — passing `0.10` for "ten points" is the obvious way to get this
+ wrong and both readings compile.
 - **M19.4** Transcript mining pipeline with consent flags + PII scrub + provenance; first 10k-pair summarizer dataset. ✅ *(shipped: `panday_harness::mining`, `cargo xtask mine --logs <dir> --out <file>`. **The 10k-pair dataset is not here** — it needs 10k consented transcripts, and this repo has none.)*
 
  **Consent, then scrub, then provenance — and each step defaults to refusing.** The mistakes in a
@@ -285,6 +296,21 @@ artifacts; CI runs eval gates nightly.
  miner runs end to end today over a directory of logs and reports exactly why each candidate was
  dropped — which is the part that had to exist before any transcript was worth collecting.
 - **M19.5** Model 2 shipped: reduce-bench regression zero, ≥25% cheaper semantic tier than the provider cheap-pool it replaces; GGUF in catalog.
+
+ **reduce-bench had no cost dimension at all**, so "≥25% cheaper" was unmeasurable — the suite
+ scored retention and token ratio, which are not money. `Report::cost_saving` and
+ `Report::cheaper_than(0.25, incumbent_price, replacement_price)` add it.
+
+ Cost rather than tokens because the two move independently: a reducer that keeps 40% of the
+ tokens and hands them to a model at three times the price is *more* expensive, and the old
+ ratio-only scorecard would have called that a 60% win. Prices are per token and supplied by the
+ caller — `panday-harness` has no price table and must not grow one, or the gate starts drifting
+ with a catalog. A saving against an incumbent that spends nothing returns `None` rather than a
+ number.
+
+ Both halves are required, because either alone is a trap: cheapness with regressions is a
+ reducer that saves money by losing the answer, and zero regressions with no saving is a semantic
+ tier with no reason to exist.
 - **M19.6** agent-bench (50 verifiable repo tasks in T3) doubling as GRPO environment. ✅ *(shipped: `panday_harness::agent_bench` — 41 tasks, the audit, the jailed runner, and `score`. **Runs in T2, not T3**. Thirty-eight are repair classes; three (`poisoned-readme`, `poisoned-comment`, `granted-json`) are M20.1 injection canaries whose verifier fails if a relative marker file exists. Still not 50 — the twelve destructive classes stay gone.)*
 
  **This is the second attempt, and the first one destroyed a machine.** The original ran each
@@ -334,3 +360,15 @@ artifacts; CI runs eval gates nightly.
  **What is left is the model**, and T3 for the tasks that will eventually need a VM rather than a
  jail (M14.5).
 - **M19.7** Model 3 v1: SFT stage beats base on agent-bench; go/no-go review for the GRPO spend.
+
+ **`Scorecard::beats(&base)` is the gate**, on the artifact type every suite already emits, so
+ it is not agent-bench's alone.
+
+ It returns `Option<bool>`, and the `None` is the substance. Comparing a tuned run over ten tasks
+ against a base run over forty-one is how this measurement gets faked without anyone lying: both
+ numbers are real, the ratio is meaningless, and a plain `bool` would have hidden it. Refused
+ when the suites differ, when either ran nothing, or when the case counts differ — a shorter run
+ is a different corpus, not a better model. `.unwrap_or(false)` fails closed, which is what a
+ shipping decision should do.
+
+ Strictly greater, no margin: the milestone says "beats", and a tie is not one.

crates/panday-harness/src/eval.rs

@@ -137,6 +137,62 @@ impl Report {
  solvable.iter().filter(|r| r.with_reduction.solved).count() as f64 / solvable.len() as f64
  }
 
+ /// **M19.5's gate**: "≥25% cheaper semantic tier than the provider cheap-pool it replaces".
+ ///
+ /// Until now this suite had **no cost dimension at all** — it measured retention and token
+ /// ratio, which are not money. So the milestone could not be judged even with a model in hand,
+ /// and that is the failure docs/19 M19.1 exists to prevent: a gate stated in prose is a gate
+ /// nobody can fail.
+ ///
+ /// Cost, not tokens, because the two move independently and the milestone is about the
+ /// cheaper one *winning*. A reducer that keeps 40% of the tokens but hands them to a model at
+ /// three times the price is more expensive, and a ratio-only scorecard would call it a 60%
+ /// improvement.
+ ///
+ /// Prices are **per token**, in the same unit for both sides, and supplied by the caller —
+ /// this crate has no price table and must not grow one, or the gate starts depending on a
+ /// catalog that drifts. `panday_router`'s catalog is where a binary gets them.
+ ///
+ /// Returns `None` when the incumbent would spend nothing: a saving against zero is not a
+ /// percentage, and reporting one would be inventing a number.
+ pub fn cost_saving(&self, incumbent_price: f64, replacement_price: f64) -> Option<f64> {
+ let incumbent: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.without_reduction.tokens_raw as f64 * incumbent_price)
+ .sum();
+ let replacement: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.with_reduction.tokens_kept as f64 * replacement_price)
+ .sum();
+ if incumbent <= 0.0 {
+ return None;
+ }
+ Some(1.0 - replacement / incumbent)
+ }
+
+ /// M19.5, whole: zero regressions **and** at least `fraction` cheaper.
+ ///
+ /// Both halves, because either alone is a trap. Cheapness with regressions is a reducer that
+ /// saves money by losing the answer; zero regressions with no saving is a semantic tier with
+ /// no reason to exist. The milestone names both and so does this.
+ ///
+ /// `fraction` is a ratio — M19.5 says 25%, so `cheaper_than(0.25, ...)`. Unlike route-bench's
+ /// margin, which is stated in points, this one is stated as a percentage *of* the incumbent,
+ /// so a ratio is the honest shape. `the_two_gates_do_not_share_a_unit` pins the difference.
+ pub fn cheaper_than(
+ &self,
+ fraction: f64,
+ incumbent_price: f64,
+ replacement_price: f64,
+ ) -> bool {
+ self.regressions().is_empty()
+ && self
+ .cost_saving(incumbent_price, replacement_price)
+ .is_some_and(|saved| saved >= fraction)
+ }
+
  /// Mean reduction across the corpus. Reported *after* success, and never
  /// instead of it (ADR-007).
  pub fn mean_ratio(&self) -> f64 {
@@ -433,3 +489,88 @@ pub async fn run_corpus(
  }
  Report { rows }
 }
+
+#[cfg(test)]
+mod cost_gate_tests {
+ use super::{Outcome, Report, Row};
+
+ fn outcome(solved: bool, raw: u32, kept: u32) -> Outcome {
+ Outcome {
+ solved,
+ tokens_raw: raw,
+ tokens_kept: kept,
+ missing_facts: vec![],
+ strategy: "test".into(),
+ }
+ }
+
+ fn report(rows: Vec<(bool, u32, bool, u32)>) -> Report {
+ Report {
+ rows: rows
+ .into_iter()
+ .enumerate()
+ .map(|(i, (s0, raw, s1, kept))| Row {
+ scenario: format!("s{i}"),
+ without_reduction: outcome(s0, raw, raw),
+ with_reduction: outcome(s1, raw, kept),
+ })
+ .collect(),
+ }
+ }
+
+ #[test]
+ fn cost_is_tokens_times_price_not_tokens_alone() {
+ // The reason this dimension exists. Half the tokens at four times the price is *more*
+ // expensive, and a ratio-only scorecard would have called it a 50% win.
+ let r = report(vec![(true, 1000, true, 500)]);
+ assert_eq!(r.mean_ratio(), 0.5, "half the tokens kept");
+
+ let saving = r.cost_saving(1.0, 4.0).expect("a saving");
+ assert!(
+ saving < 0.0,
+ "keeping half the tokens at 4x the price costs more, not less: {saving}"
+ );
+ }
+
+ #[test]
+ fn a_saving_against_nothing_is_not_a_number() {
+ // An empty corpus, or one where the incumbent spends zero, has no percentage to report.
+ // Returning 0.0 or 1.0 here would be inventing a result.
+ assert_eq!(Report::default().cost_saving(1.0, 1.0), None);
+ assert_eq!(report(vec![(true, 0, true, 0)]).cost_saving(1.0, 1.0), None);
+ }
+
+ #[test]
+ fn the_gate_needs_both_halves() {
+ // Cheap but broken: 90% saved, and a scenario that was solvable is now not.
+ let broken = report(vec![(true, 1000, false, 100)]);
+ assert!(
+ !broken.cheaper_than(0.25, 1.0, 1.0),
+ "a regression is not paid for by being cheap"
+ );
+
+ // Sound but not cheap enough: no regressions, 10% saved against a 25% bar.
+ let timid = report(vec![(true, 1000, true, 900)]);
+ assert!(!timid.cheaper_than(0.25, 1.0, 1.0));
+
+ // Both: no regressions and 30% saved.
+ let good = report(vec![(true, 1000, true, 700)]);
+ assert!(good.cheaper_than(0.25, 1.0, 1.0));
+ }
+
+ #[test]
+ fn the_two_gates_do_not_share_a_unit() {
+ // route-bench's `Score::beats` takes percentage POINTS (M19.3: "≥10pt"); this one takes a
+ // FRACTION of the incumbent's spend (M19.5: "≥25% cheaper"). They read alike at a call
+ // site and mean different things, so each is pinned where it lives.
+ let r = report(vec![(true, 1000, true, 700)]);
+ assert!(
+ r.cheaper_than(0.25, 1.0, 1.0),
+ "0.25 is twenty-five percent"
+ );
+ assert!(
+ !r.cheaper_than(25.0, 1.0, 1.0),
+ "25.0 would be a 2500% saving, which nothing can meet"
+ );
+ }
+}

crates/panday-router/src/bench.rs

@@ -419,6 +419,28 @@ impl Score {
  pub fn meets_the_gate(&self, floor: f64) -> bool {
  self.accuracy() >= floor && self.confidently_wrong.is_empty()
  }
+
+ /// **M19.3's gate**: "classifier ... beats heuristic on route-bench by ≥10pt".
+ ///
+ /// [`meets_the_gate`](Self::meets_the_gate) is an absolute floor; this is the relative one the
+ /// milestone actually states, and until now it existed only as prose. A gate that is not a
+ /// function is a gate nobody can fail, which is the state docs/19 M19.1 was written against.
+ ///
+ /// `margin_points` is **percentage points**, not a fraction: M19.3 says "≥10pt", so the call
+ /// is `beats(&heuristic, 10.0)`. Accuracy is a 0..1 ratio internally and the conversion
+ /// happens here, once — passing `0.10` and meaning ten points is the obvious way to get this
+ /// wrong, so `the_margin_is_percentage_points_not_a_fraction` pins it.
+ ///
+ /// **Being confidently wrong disqualifies a challenger regardless of margin.** That is not an
+ /// extra condition bolted on: the dangerous quadrant is the thing route-bench exists to
+ /// measure (a wrong answer the router *trusts* and acts on), and a caller who checked only the
+ /// margin would ship a model that is more accurate on average and catastrophic on the cases
+ /// that matter. The incumbent's own quadrant is not consulted — the question is whether the
+ /// *challenger* is safe to deploy, not whether it is less bad than what is there.
+ pub fn beats(&self, incumbent: &Score, margin_points: f64) -> bool {
+ let gained = (self.accuracy() - incumbent.accuracy()) * 100.0;
+ gained >= margin_points && self.confidently_wrong.is_empty()
+ }
 }
 
 /// Score any classifier against the corpus. `HeuristicClassifier` today; a learned one at
@@ -448,3 +470,60 @@ pub fn score(c: &dyn Classifier) -> Score {
  }
  s
 }
+
+#[cfg(test)]
+mod gate_tests {
+ use super::Score;
+
+ fn score(correct: usize, total: usize, confidently_wrong: Vec<&'static str>) -> Score {
+ Score {
+ total,
+ correct,
+ confidently_wrong,
+ caught_by_the_gate: vec![],
+ }
+ }
+
+ #[test]
+ fn the_margin_is_percentage_points_not_a_fraction() {
+ // M19.3 says "≥10pt". Accuracy is a 0..1 ratio internally, so the obvious mistake is to
+ // pass 0.10 and mean ten points — which would let a challenger through on a *tenth* of a
+ // percentage point. The unit is pinned here because the gate is unfalsifiable by
+ // inspection: both readings compile and both look right.
+ let incumbent = score(70, 100, vec![]);
+ let exactly_ten = score(80, 100, vec![]);
+ let just_under = score(79, 100, vec![]);
+
+ assert!(
+ exactly_ten.beats(&incumbent, 10.0),
+ "10pt clears a 10pt bar"
+ );
+ assert!(!just_under.beats(&incumbent, 10.0), "9pt does not");
+ }
+
+ #[test]
+ fn a_confidently_wrong_challenger_does_not_ship_however_far_ahead() {
+ // The whole point of route-bench's dangerous quadrant. A model can be twenty points more
+ // accurate and still be the worse thing to deploy, because the router *acts* on a
+ // confident answer.
+ let incumbent = score(50, 100, vec![]);
+ let brilliant_but_reckless = score(90, 100, vec!["a-case-it-got-wrong-and-trusted"]);
+ assert!(
+ !brilliant_but_reckless.beats(&incumbent, 10.0),
+ "40pt ahead, and still not shippable"
+ );
+ }
+
+ #[test]
+ fn losing_ground_is_not_a_pass() {
+ let incumbent = score(90, 100, vec![]);
+ let worse = score(60, 100, vec![]);
+ assert!(!worse.beats(&incumbent, 10.0));
+ // And equal is not "beats" either: the milestone asks for a margin, not parity.
+ assert!(!incumbent.beats(&incumbent, 10.0));
+ assert!(
+ incumbent.beats(&incumbent, 0.0),
+ "a zero-point bar is met by parity"
+ );
+ }
+}

crates/panday-types/src/scorecard.rs

@@ -124,6 +124,33 @@ impl Scorecard {
  self.cases > 0 && self.rate() >= floor
  }
 
+ /// **The relative gate**: did this run beat an earlier one on the same suite?
+ ///
+ /// [`meets`](Self::meets) answers "is it good enough"; several milestones ask "is it *better*"
+ /// and had no function to ask with — M19.7 ("SFT stage beats base on agent-bench") is the
+ /// live one. A gate that exists only in prose is a gate nobody can fail.
+ ///
+ /// **`None` means the two cannot be compared, and that is the point of the return type.**
+ /// Comparing a tuned run over ten tasks against a base run over forty-one is the way this
+ /// measurement gets faked without anyone lying: both numbers are real, the ratio is
+ /// meaningless, and a `bool` would have hidden it. Refused when the suites differ, when either
+ /// ran nothing, or when the case counts differ — a shorter run is a different corpus, not a
+ /// better model. A caller that writes `.unwrap_or(false)` fails closed, which is the right
+ /// default for a shipping decision.
+ ///
+ /// Strictly greater, with no margin: M19.7 says "beats", and a tie is not one. Where a
+ /// milestone names a margin instead — M19.3's "≥10pt" — that gate lives with its suite and
+ /// takes the margin explicitly, because the unit is the suite's business, not this type's.
+ pub fn beats(&self, base: &Scorecard) -> Option<bool> {
+ if self.suite != base.suite || self.cases == 0 || base.cases == 0 {
+ return None;
+ }
+ if self.cases != base.cases {
+ return None;
+ }
+ Some(self.passed > base.passed)
+ }
+
  /// The human rendering. Derived from the artifact, never the other way round.
  pub fn to_markdown(&self) -> String {
  let mut out = format!(
@@ -226,3 +253,61 @@ mod tests {
  assert!(md.contains("missing required field `city`"));
  }
 }
+
+#[cfg(test)]
+mod relative_gate_tests {
+ use super::Scorecard;
+
+ fn card(suite: &str, passed: u32, cases: u32) -> Scorecard {
+ let mut c = Scorecard::new(suite, "subject", "2026-08-24T00:00:00Z");
+ for i in 0..cases {
+ c.record(&format!("case{i}"), i < passed, "");
+ }
+ c
+ }
+
+ #[test]
+ fn a_shorter_run_is_not_a_better_model() {
+ // The failure this return type exists for. Both numbers are real, the comparison is not:
+ // 9/10 against 30/41 is a different corpus, not an improvement. A `bool` would have said
+ // "better" and nobody would have noticed.
+ let base = card("agent-bench", 30, 41);
+ let tuned_on_fewer = card("agent-bench", 9, 10);
+ assert_eq!(tuned_on_fewer.beats(&base), None);
+ }
+
+ #[test]
+ fn suites_do_not_cross() {
+ let a = card("agent-bench", 40, 41);
+ let b = card("route-bench", 10, 41);
+ assert_eq!(a.beats(&b), None, "same case count, different question");
+ }
+
+ #[test]
+ fn nothing_ran_is_not_a_win() {
+ let base = card("agent-bench", 0, 0);
+ let tuned = card("agent-bench", 5, 5);
+ assert_eq!(tuned.beats(&base), None);
+ assert_eq!(base.beats(&tuned), None);
+ }
+
+ #[test]
+ fn beating_is_strict() {
+ let base = card("agent-bench", 30, 41);
+ assert_eq!(card("agent-bench", 31, 41).beats(&base), Some(true));
+ assert_eq!(
+ card("agent-bench", 30, 41).beats(&base),
+ Some(false),
+ "a tie is not a win"
+ );
+ assert_eq!(card("agent-bench", 29, 41).beats(&base), Some(false));
+ }
+
+ #[test]
+ fn an_incomparable_pair_fails_closed_for_a_careless_caller() {
+ // `.unwrap_or(false)` is what a caller will write. It must mean "do not ship".
+ let base = card("agent-bench", 30, 41);
+ let mismatched = card("agent-bench", 41, 42);
+ assert!(!mismatched.beats(&base).unwrap_or(false));
+ }
+}

docs/19-training.md

@@ -254,6 +254,17 @@ artifacts; CI runs eval gates nightly.
  one that emits well-formed calls with the wrong arguments — because a generator that cannot
  report a bad model as bad is not worth running against a good one.
 - **M19.3** Model 1 shipped: classifier behind `Classifier` trait beats heuristic on route-bench by ≥10pt; deployed in shadow, then live.
+
+ **The gate is a function now, though the model is not here.** `panday_router::bench::Score::beats(&incumbent, 10.0)`
+ — the margin in percentage points, as the milestone states it. It was prose until this change,
+ which meant M19.3 could not be failed even with a classifier in hand; `Score::meets_the_gate` is
+ an absolute floor and answers a different question.
+
+ A challenger that is **confidently wrong is refused regardless of margin**. That is not an extra
+ condition: the dangerous quadrant is what route-bench exists to measure, and a caller checking
+ only the margin would ship a model that is better on average and catastrophic where the router
+ acts on it. Unit pinned by test — passing `0.10` for "ten points" is the obvious way to get this
+ wrong and both readings compile.
 - **M19.4** Transcript mining pipeline with consent flags + PII scrub + provenance; first 10k-pair summarizer dataset. ✅ *(shipped: `panday_harness::mining`, `cargo xtask mine --logs <dir> --out <file>`. **The 10k-pair dataset is not here** — it needs 10k consented transcripts, and this repo has none.)*
 
  **Consent, then scrub, then provenance — and each step defaults to refusing.** The mistakes in a
@@ -285,6 +296,21 @@ artifacts; CI runs eval gates nightly.
  miner runs end to end today over a directory of logs and reports exactly why each candidate was
  dropped — which is the part that had to exist before any transcript was worth collecting.
 - **M19.5** Model 2 shipped: reduce-bench regression zero, ≥25% cheaper semantic tier than the provider cheap-pool it replaces; GGUF in catalog.
+
+ **reduce-bench had no cost dimension at all**, so "≥25% cheaper" was unmeasurable — the suite
+ scored retention and token ratio, which are not money. `Report::cost_saving` and
+ `Report::cheaper_than(0.25, incumbent_price, replacement_price)` add it.
+
+ Cost rather than tokens because the two move independently: a reducer that keeps 40% of the
+ tokens and hands them to a model at three times the price is *more* expensive, and the old
+ ratio-only scorecard would have called that a 60% win. Prices are per token and supplied by the
+ caller — `panday-harness` has no price table and must not grow one, or the gate starts drifting
+ with a catalog. A saving against an incumbent that spends nothing returns `None` rather than a
+ number.
+
+ Both halves are required, because either alone is a trap: cheapness with regressions is a
+ reducer that saves money by losing the answer, and zero regressions with no saving is a semantic
+ tier with no reason to exist.
 - **M19.6** agent-bench (50 verifiable repo tasks in T3) doubling as GRPO environment. ✅ *(shipped: `panday_harness::agent_bench` — 41 tasks, the audit, the jailed runner, and `score`. **Runs in T2, not T3**. Thirty-eight are repair classes; three (`poisoned-readme`, `poisoned-comment`, `granted-json`) are M20.1 injection canaries whose verifier fails if a relative marker file exists. Still not 50 — the twelve destructive classes stay gone.)*
 
  **This is the second attempt, and the first one destroyed a machine.** The original ran each
@@ -334,3 +360,15 @@ artifacts; CI runs eval gates nightly.
  **What is left is the model**, and T3 for the tasks that will eventually need a VM rather than a
  jail (M14.5).
 - **M19.7** Model 3 v1: SFT stage beats base on agent-bench; go/no-go review for the GRPO spend.
+
+ **`Scorecard::beats(&base)` is the gate**, on the artifact type every suite already emits, so
+ it is not agent-bench's alone.
+
+ It returns `Option<bool>`, and the `None` is the substance. Comparing a tuned run over ten tasks
+ against a base run over forty-one is how this measurement gets faked without anyone lying: both
+ numbers are real, the ratio is meaningless, and a plain `bool` would have hidden it. Refused
+ when the suites differ, when either ran nothing, or when the case counts differ — a shorter run
+ is a different corpus, not a better model. `.unwrap_or(false)` fails closed, which is what a
+ shipping decision should do.
+
+ Strictly greater, no margin: the milestone says "beats", and a tie is not one.

crates/panday-harness/src/eval.rs

@@ -137,6 +137,62 @@ impl Report {
  solvable.iter().filter(|r| r.with_reduction.solved).count() as f64 / solvable.len() as f64
  }
 
+ /// **M19.5's gate**: "≥25% cheaper semantic tier than the provider cheap-pool it replaces".
+ ///
+ /// Until now this suite had **no cost dimension at all** — it measured retention and token
+ /// ratio, which are not money. So the milestone could not be judged even with a model in hand,
+ /// and that is the failure docs/19 M19.1 exists to prevent: a gate stated in prose is a gate
+ /// nobody can fail.
+ ///
+ /// Cost, not tokens, because the two move independently and the milestone is about the
+ /// cheaper one *winning*. A reducer that keeps 40% of the tokens but hands them to a model at
+ /// three times the price is more expensive, and a ratio-only scorecard would call it a 60%
+ /// improvement.
+ ///
+ /// Prices are **per token**, in the same unit for both sides, and supplied by the caller —
+ /// this crate has no price table and must not grow one, or the gate starts depending on a
+ /// catalog that drifts. `panday_router`'s catalog is where a binary gets them.
+ ///
+ /// Returns `None` when the incumbent would spend nothing: a saving against zero is not a
+ /// percentage, and reporting one would be inventing a number.
+ pub fn cost_saving(&self, incumbent_price: f64, replacement_price: f64) -> Option<f64> {
+ let incumbent: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.without_reduction.tokens_raw as f64 * incumbent_price)
+ .sum();
+ let replacement: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.with_reduction.tokens_kept as f64 * replacement_price)
+ .sum();
+ if incumbent <= 0.0 {
+ return None;
+ }
+ Some(1.0 - replacement / incumbent)
+ }
+
+ /// M19.5, whole: zero regressions **and** at least `fraction` cheaper.
+ ///
+ /// Both halves, because either alone is a trap. Cheapness with regressions is a reducer that
+ /// saves money by losing the answer; zero regressions with no saving is a semantic tier with
+ /// no reason to exist. The milestone names both and so does this.
+ ///
+ /// `fraction` is a ratio — M19.5 says 25%, so `cheaper_than(0.25, ...)`. Unlike route-bench's
+ /// margin, which is stated in points, this one is stated as a percentage *of* the incumbent,
+ /// so a ratio is the honest shape. `the_two_gates_do_not_share_a_unit` pins the difference.
+ pub fn cheaper_than(
+ &self,
+ fraction: f64,
+ incumbent_price: f64,
+ replacement_price: f64,
+ ) -> bool {
+ self.regressions().is_empty()
+ && self
+ .cost_saving(incumbent_price, replacement_price)
+ .is_some_and(|saved| saved >= fraction)
+ }
+
  /// Mean reduction across the corpus. Reported *after* success, and never
  /// instead of it (ADR-007).
  pub fn mean_ratio(&self) -> f64 {
@@ -433,3 +489,88 @@ pub async fn run_corpus(
  }
  Report { rows }
 }
+
+#[cfg(test)]
+mod cost_gate_tests {
+ use super::{Outcome, Report, Row};
+
+ fn outcome(solved: bool, raw: u32, kept: u32) -> Outcome {
+ Outcome {
+ solved,
+ tokens_raw: raw,
+ tokens_kept: kept,
+ missing_facts: vec![],
+ strategy: "test".into(),
+ }
+ }
+
+ fn report(rows: Vec<(bool, u32, bool, u32)>) -> Report {
+ Report {
+ rows: rows
+ .into_iter()
+ .enumerate()
+ .map(|(i, (s0, raw, s1, kept))| Row {
+ scenario: format!("s{i}"),
+ without_reduction: outcome(s0, raw, raw),
+ with_reduction: outcome(s1, raw, kept),
+ })
+ .collect(),
+ }
+ }
+
+ #[test]
+ fn cost_is_tokens_times_price_not_tokens_alone() {
+ // The reason this dimension exists. Half the tokens at four times the price is *more*
+ // expensive, and a ratio-only scorecard would have called it a 50% win.
+ let r = report(vec![(true, 1000, true, 500)]);
+ assert_eq!(r.mean_ratio(), 0.5, "half the tokens kept");
+
+ let saving = r.cost_saving(1.0, 4.0).expect("a saving");
+ assert!(
+ saving < 0.0,
+ "keeping half the tokens at 4x the price costs more, not less: {saving}"
+ );
+ }
+
+ #[test]
+ fn a_saving_against_nothing_is_not_a_number() {
+ // An empty corpus, or one where the incumbent spends zero, has no percentage to report.
+ // Returning 0.0 or 1.0 here would be inventing a result.
+ assert_eq!(Report::default().cost_saving(1.0, 1.0), None);
+ assert_eq!(report(vec![(true, 0, true, 0)]).cost_saving(1.0, 1.0), None);
+ }
+
+ #[test]
+ fn the_gate_needs_both_halves() {
+ // Cheap but broken: 90% saved, and a scenario that was solvable is now not.
+ let broken = report(vec![(true, 1000, false, 100)]);
+ assert!(
+ !broken.cheaper_than(0.25, 1.0, 1.0),
+ "a regression is not paid for by being cheap"
+ );
+
+ // Sound but not cheap enough: no regressions, 10% saved against a 25% bar.
+ let timid = report(vec![(true, 1000, true, 900)]);
+ assert!(!timid.cheaper_than(0.25, 1.0, 1.0));
+
+ // Both: no regressions and 30% saved.
+ let good = report(vec![(true, 1000, true, 700)]);
+ assert!(good.cheaper_than(0.25, 1.0, 1.0));
+ }
+
+ #[test]
+ fn the_two_gates_do_not_share_a_unit() {
+ // route-bench's `Score::beats` takes percentage POINTS (M19.3: "≥10pt"); this one takes a
+ // FRACTION of the incumbent's spend (M19.5: "≥25% cheaper"). They read alike at a call
+ // site and mean different things, so each is pinned where it lives.
+ let r = report(vec![(true, 1000, true, 700)]);
+ assert!(
+ r.cheaper_than(0.25, 1.0, 1.0),
+ "0.25 is twenty-five percent"
+ );
+ assert!(
+ !r.cheaper_than(25.0, 1.0, 1.0),
+ "25.0 would be a 2500% saving, which nothing can meet"
+ );
+ }
+}

crates/panday-router/src/bench.rs

@@ -419,6 +419,28 @@ impl Score {
  pub fn meets_the_gate(&self, floor: f64) -> bool {
  self.accuracy() >= floor && self.confidently_wrong.is_empty()
  }
+
+ /// **M19.3's gate**: "classifier ... beats heuristic on route-bench by ≥10pt".
+ ///
+ /// [`meets_the_gate`](Self::meets_the_gate) is an absolute floor; this is the relative one the
+ /// milestone actually states, and until now it existed only as prose. A gate that is not a
+ /// function is a gate nobody can fail, which is the state docs/19 M19.1 was written against.
+ ///
+ /// `margin_points` is **percentage points**, not a fraction: M19.3 says "≥10pt", so the call
+ /// is `beats(&heuristic, 10.0)`. Accuracy is a 0..1 ratio internally and the conversion
+ /// happens here, once — passing `0.10` and meaning ten points is the obvious way to get this
+ /// wrong, so `the_margin_is_percentage_points_not_a_fraction` pins it.
+ ///
+ /// **Being confidently wrong disqualifies a challenger regardless of margin.** That is not an
+ /// extra condition bolted on: the dangerous quadrant is the thing route-bench exists to
+ /// measure (a wrong answer the router *trusts* and acts on), and a caller who checked only the
+ /// margin would ship a model that is more accurate on average and catastrophic on the cases
+ /// that matter. The incumbent's own quadrant is not consulted — the question is whether the
+ /// *challenger* is safe to deploy, not whether it is less bad than what is there.
+ pub fn beats(&self, incumbent: &Score, margin_points: f64) -> bool {
+ let gained = (self.accuracy() - incumbent.accuracy()) * 100.0;
+ gained >= margin_points && self.confidently_wrong.is_empty()
+ }
 }
 
 /// Score any classifier against the corpus. `HeuristicClassifier` today; a learned one at
@@ -448,3 +470,60 @@ pub fn score(c: &dyn Classifier) -> Score {
  }
  s
 }
+
+#[cfg(test)]
+mod gate_tests {
+ use super::Score;
+
+ fn score(correct: usize, total: usize, confidently_wrong: Vec<&'static str>) -> Score {
+ Score {
+ total,
+ correct,
+ confidently_wrong,
+ caught_by_the_gate: vec![],
+ }
+ }
+
+ #[test]
+ fn the_margin_is_percentage_points_not_a_fraction() {
+ // M19.3 says "≥10pt". Accuracy is a 0..1 ratio internally, so the obvious mistake is to
+ // pass 0.10 and mean ten points — which would let a challenger through on a *tenth* of a
+ // percentage point. The unit is pinned here because the gate is unfalsifiable by
+ // inspection: both readings compile and both look right.
+ let incumbent = score(70, 100, vec![]);
+ let exactly_ten = score(80, 100, vec![]);
+ let just_under = score(79, 100, vec![]);
+
+ assert!(
+ exactly_ten.beats(&incumbent, 10.0),
+ "10pt clears a 10pt bar"
+ );
+ assert!(!just_under.beats(&incumbent, 10.0), "9pt does not");
+ }
+
+ #[test]
+ fn a_confidently_wrong_challenger_does_not_ship_however_far_ahead() {
+ // The whole point of route-bench's dangerous quadrant. A model can be twenty points more
+ // accurate and still be the worse thing to deploy, because the router *acts* on a
+ // confident answer.
+ let incumbent = score(50, 100, vec![]);
+ let brilliant_but_reckless = score(90, 100, vec!["a-case-it-got-wrong-and-trusted"]);
+ assert!(
+ !brilliant_but_reckless.beats(&incumbent, 10.0),
+ "40pt ahead, and still not shippable"
+ );
+ }
+
+ #[test]
+ fn losing_ground_is_not_a_pass() {
+ let incumbent = score(90, 100, vec![]);
+ let worse = score(60, 100, vec![]);
+ assert!(!worse.beats(&incumbent, 10.0));
+ // And equal is not "beats" either: the milestone asks for a margin, not parity.
+ assert!(!incumbent.beats(&incumbent, 10.0));
+ assert!(
+ incumbent.beats(&incumbent, 0.0),
+ "a zero-point bar is met by parity"
+ );
+ }
+}

crates/panday-types/src/scorecard.rs

@@ -124,6 +124,33 @@ impl Scorecard {
  self.cases > 0 && self.rate() >= floor
  }
 
+ /// **The relative gate**: did this run beat an earlier one on the same suite?
+ ///
+ /// [`meets`](Self::meets) answers "is it good enough"; several milestones ask "is it *better*"
+ /// and had no function to ask with — M19.7 ("SFT stage beats base on agent-bench") is the
+ /// live one. A gate that exists only in prose is a gate nobody can fail.
+ ///
+ /// **`None` means the two cannot be compared, and that is the point of the return type.**
+ /// Comparing a tuned run over ten tasks against a base run over forty-one is the way this
+ /// measurement gets faked without anyone lying: both numbers are real, the ratio is
+ /// meaningless, and a `bool` would have hidden it. Refused when the suites differ, when either
+ /// ran nothing, or when the case counts differ — a shorter run is a different corpus, not a
+ /// better model. A caller that writes `.unwrap_or(false)` fails closed, which is the right
+ /// default for a shipping decision.
+ ///
+ /// Strictly greater, with no margin: M19.7 says "beats", and a tie is not one. Where a
+ /// milestone names a margin instead — M19.3's "≥10pt" — that gate lives with its suite and
+ /// takes the margin explicitly, because the unit is the suite's business, not this type's.
+ pub fn beats(&self, base: &Scorecard) -> Option<bool> {
+ if self.suite != base.suite || self.cases == 0 || base.cases == 0 {
+ return None;
+ }
+ if self.cases != base.cases {
+ return None;
+ }
+ Some(self.passed > base.passed)
+ }
+
  /// The human rendering. Derived from the artifact, never the other way round.
  pub fn to_markdown(&self) -> String {
  let mut out = format!(
@@ -226,3 +253,61 @@ mod tests {
  assert!(md.contains("missing required field `city`"));
  }
 }
+
+#[cfg(test)]
+mod relative_gate_tests {
+ use super::Scorecard;
+
+ fn card(suite: &str, passed: u32, cases: u32) -> Scorecard {
+ let mut c = Scorecard::new(suite, "subject", "2026-08-24T00:00:00Z");
+ for i in 0..cases {
+ c.record(&format!("case{i}"), i < passed, "");
+ }
+ c
+ }
+
+ #[test]
+ fn a_shorter_run_is_not_a_better_model() {
+ // The failure this return type exists for. Both numbers are real, the comparison is not:
+ // 9/10 against 30/41 is a different corpus, not an improvement. A `bool` would have said
+ // "better" and nobody would have noticed.
+ let base = card("agent-bench", 30, 41);
+ let tuned_on_fewer = card("agent-bench", 9, 10);
+ assert_eq!(tuned_on_fewer.beats(&base), None);
+ }
+
+ #[test]
+ fn suites_do_not_cross() {
+ let a = card("agent-bench", 40, 41);
+ let b = card("route-bench", 10, 41);
+ assert_eq!(a.beats(&b), None, "same case count, different question");
+ }
+
+ #[test]
+ fn nothing_ran_is_not_a_win() {
+ let base = card("agent-bench", 0, 0);
+ let tuned = card("agent-bench", 5, 5);
+ assert_eq!(tuned.beats(&base), None);
+ assert_eq!(base.beats(&tuned), None);
+ }
+
+ #[test]
+ fn beating_is_strict() {
+ let base = card("agent-bench", 30, 41);
+ assert_eq!(card("agent-bench", 31, 41).beats(&base), Some(true));
+ assert_eq!(
+ card("agent-bench", 30, 41).beats(&base),
+ Some(false),
+ "a tie is not a win"
+ );
+ assert_eq!(card("agent-bench", 29, 41).beats(&base), Some(false));
+ }
+
+ #[test]
+ fn an_incomparable_pair_fails_closed_for_a_careless_caller() {
+ // `.unwrap_or(false)` is what a caller will write. It must mean "do not ship".
+ let base = card("agent-bench", 30, 41);
+ let mismatched = card("agent-bench", 41, 42);
+ assert!(!mismatched.beats(&base).unwrap_or(false));
+ }
+}

docs/19-training.md

@@ -254,6 +254,17 @@ artifacts; CI runs eval gates nightly.
  one that emits well-formed calls with the wrong arguments — because a generator that cannot
  report a bad model as bad is not worth running against a good one.
 - **M19.3** Model 1 shipped: classifier behind `Classifier` trait beats heuristic on route-bench by ≥10pt; deployed in shadow, then live.
+
+ **The gate is a function now, though the model is not here.** `panday_router::bench::Score::beats(&incumbent, 10.0)`
+ — the margin in percentage points, as the milestone states it. It was prose until this change,
+ which meant M19.3 could not be failed even with a classifier in hand; `Score::meets_the_gate` is
+ an absolute floor and answers a different question.
+
+ A challenger that is **confidently wrong is refused regardless of margin**. That is not an extra
+ condition: the dangerous quadrant is what route-bench exists to measure, and a caller checking
+ only the margin would ship a model that is better on average and catastrophic where the router
+ acts on it. Unit pinned by test — passing `0.10` for "ten points" is the obvious way to get this
+ wrong and both readings compile.
 - **M19.4** Transcript mining pipeline with consent flags + PII scrub + provenance; first 10k-pair summarizer dataset. ✅ *(shipped: `panday_harness::mining`, `cargo xtask mine --logs <dir> --out <file>`. **The 10k-pair dataset is not here** — it needs 10k consented transcripts, and this repo has none.)*
 
  **Consent, then scrub, then provenance — and each step defaults to refusing.** The mistakes in a
@@ -285,6 +296,21 @@ artifacts; CI runs eval gates nightly.
  miner runs end to end today over a directory of logs and reports exactly why each candidate was
  dropped — which is the part that had to exist before any transcript was worth collecting.
 - **M19.5** Model 2 shipped: reduce-bench regression zero, ≥25% cheaper semantic tier than the provider cheap-pool it replaces; GGUF in catalog.
+
+ **reduce-bench had no cost dimension at all**, so "≥25% cheaper" was unmeasurable — the suite
+ scored retention and token ratio, which are not money. `Report::cost_saving` and
+ `Report::cheaper_than(0.25, incumbent_price, replacement_price)` add it.
+
+ Cost rather than tokens because the two move independently: a reducer that keeps 40% of the
+ tokens and hands them to a model at three times the price is *more* expensive, and the old
+ ratio-only scorecard would have called that a 60% win. Prices are per token and supplied by the
+ caller — `panday-harness` has no price table and must not grow one, or the gate starts drifting
+ with a catalog. A saving against an incumbent that spends nothing returns `None` rather than a
+ number.
+
+ Both halves are required, because either alone is a trap: cheapness with regressions is a
+ reducer that saves money by losing the answer, and zero regressions with no saving is a semantic
+ tier with no reason to exist.
 - **M19.6** agent-bench (50 verifiable repo tasks in T3) doubling as GRPO environment. ✅ *(shipped: `panday_harness::agent_bench` — 41 tasks, the audit, the jailed runner, and `score`. **Runs in T2, not T3**. Thirty-eight are repair classes; three (`poisoned-readme`, `poisoned-comment`, `granted-json`) are M20.1 injection canaries whose verifier fails if a relative marker file exists. Still not 50 — the twelve destructive classes stay gone.)*
 
  **This is the second attempt, and the first one destroyed a machine.** The original ran each
@@ -334,3 +360,15 @@ artifacts; CI runs eval gates nightly.
  **What is left is the model**, and T3 for the tasks that will eventually need a VM rather than a
  jail (M14.5).
 - **M19.7** Model 3 v1: SFT stage beats base on agent-bench; go/no-go review for the GRPO spend.
+
+ **`Scorecard::beats(&base)` is the gate**, on the artifact type every suite already emits, so
+ it is not agent-bench's alone.
+
+ It returns `Option<bool>`, and the `None` is the substance. Comparing a tuned run over ten tasks
+ against a base run over forty-one is how this measurement gets faked without anyone lying: both
+ numbers are real, the ratio is meaningless, and a plain `bool` would have hidden it. Refused
+ when the suites differ, when either ran nothing, or when the case counts differ — a shorter run
+ is a different corpus, not a better model. `.unwrap_or(false)` fails closed, which is what a
+ shipping decision should do.
+
+ Strictly greater, no margin: the milestone says "beats", and a tie is not one.

crates/panday-harness/src/eval.rs

@@ -137,6 +137,62 @@ impl Report {
  solvable.iter().filter(|r| r.with_reduction.solved).count() as f64 / solvable.len() as f64
  }
 
+ /// **M19.5's gate**: "≥25% cheaper semantic tier than the provider cheap-pool it replaces".
+ ///
+ /// Until now this suite had **no cost dimension at all** — it measured retention and token
+ /// ratio, which are not money. So the milestone could not be judged even with a model in hand,
+ /// and that is the failure docs/19 M19.1 exists to prevent: a gate stated in prose is a gate
+ /// nobody can fail.
+ ///
+ /// Cost, not tokens, because the two move independently and the milestone is about the
+ /// cheaper one *winning*. A reducer that keeps 40% of the tokens but hands them to a model at
+ /// three times the price is more expensive, and a ratio-only scorecard would call it a 60%
+ /// improvement.
+ ///
+ /// Prices are **per token**, in the same unit for both sides, and supplied by the caller —
+ /// this crate has no price table and must not grow one, or the gate starts depending on a
+ /// catalog that drifts. `panday_router`'s catalog is where a binary gets them.
+ ///
+ /// Returns `None` when the incumbent would spend nothing: a saving against zero is not a
+ /// percentage, and reporting one would be inventing a number.
+ pub fn cost_saving(&self, incumbent_price: f64, replacement_price: f64) -> Option<f64> {
+ let incumbent: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.without_reduction.tokens_raw as f64 * incumbent_price)
+ .sum();
+ let replacement: f64 = self
+ .rows
+ .iter()
+ .map(|r| r.with_reduction.tokens_kept as f64 * replacement_price)
+ .sum();
+ if incumbent <= 0.0 {
+ return None;
+ }
+ Some(1.0 - replacement / incumbent)
+ }
+
+ /// M19.5, whole: zero regressions **and** at least `fraction` cheaper.
+ ///
+ /// Both halves, because either alone is a trap. Cheapness with regressions is a reducer that
+ /// saves money by losing the answer; zero regressions with no saving is a semantic tier with
+ /// no reason to exist. The milestone names both and so does this.
+ ///
+ /// `fraction` is a ratio — M19.5 says 25%, so `cheaper_than(0.25, ...)`. Unlike route-bench's
+ /// margin, which is stated in points, this one is stated as a percentage *of* the incumbent,
+ /// so a ratio is the honest shape. `the_two_gates_do_not_share_a_unit` pins the difference.
+ pub fn cheaper_than(
+ &self,
+ fraction: f64,
+ incumbent_price: f64,
+ replacement_price: f64,
+ ) -> bool {
+ self.regressions().is_empty()
+ && self
+ .cost_saving(incumbent_price, replacement_price)
+ .is_some_and(|saved| saved >= fraction)
+ }
+
  /// Mean reduction across the corpus. Reported *after* success, and never
  /// instead of it (ADR-007).
  pub fn mean_ratio(&self) -> f64 {
@@ -433,3 +489,88 @@ pub async fn run_corpus(
  }
  Report { rows }
 }
+
+#[cfg(test)]
+mod cost_gate_tests {
+ use super::{Outcome, Report, Row};
+
+ fn outcome(solved: bool, raw: u32, kept: u32) -> Outcome {
+ Outcome {
+ solved,
+ tokens_raw: raw,
+ tokens_kept: kept,
+ missing_facts: vec![],
+ strategy: "test".into(),
+ }
+ }
+
+ fn report(rows: Vec<(bool, u32, bool, u32)>) -> Report {
+ Report {
+ rows: rows
+ .into_iter()
+ .enumerate()
+ .map(|(i, (s0, raw, s1, kept))| Row {
+ scenario: format!("s{i}"),
+ without_reduction: outcome(s0, raw, raw),
+ with_reduction: outcome(s1, raw, kept),
+ })
+ .collect(),
+ }
+ }
+
+ #[test]
+ fn cost_is_tokens_times_price_not_tokens_alone() {
+ // The reason this dimension exists. Half the tokens at four times the price is *more*
+ // expensive, and a ratio-only scorecard would have called it a 50% win.
+ let r = report(vec![(true, 1000, true, 500)]);
+ assert_eq!(r.mean_ratio(), 0.5, "half the tokens kept");
+
+ let saving = r.cost_saving(1.0, 4.0).expect("a saving");
+ assert!(
+ saving < 0.0,
+ "keeping half the tokens at 4x the price costs more, not less: {saving}"
+ );
+ }
+
+ #[test]
+ fn a_saving_against_nothing_is_not_a_number() {
+ // An empty corpus, or one where the incumbent spends zero, has no percentage to report.
+ // Returning 0.0 or 1.0 here would be inventing a result.
+ assert_eq!(Report::default().cost_saving(1.0, 1.0), None);
+ assert_eq!(report(vec![(true, 0, true, 0)]).cost_saving(1.0, 1.0), None);
+ }
+
+ #[test]
+ fn the_gate_needs_both_halves() {
+ // Cheap but broken: 90% saved, and a scenario that was solvable is now not.
+ let broken = report(vec![(true, 1000, false, 100)]);
+ assert!(
+ !broken.cheaper_than(0.25, 1.0, 1.0),
+ "a regression is not paid for by being cheap"
+ );
+
+ // Sound but not cheap enough: no regressions, 10% saved against a 25% bar.
+ let timid = report(vec![(true, 1000, true, 900)]);
+ assert!(!timid.cheaper_than(0.25, 1.0, 1.0));
+
+ // Both: no regressions and 30% saved.
+ let good = report(vec![(true, 1000, true, 700)]);
+ assert!(good.cheaper_than(0.25, 1.0, 1.0));
+ }
+
+ #[test]
+ fn the_two_gates_do_not_share_a_unit() {
+ // route-bench's `Score::beats` takes percentage POINTS (M19.3: "≥10pt"); this one takes a
+ // FRACTION of the incumbent's spend (M19.5: "≥25% cheaper"). They read alike at a call
+ // site and mean different things, so each is pinned where it lives.
+ let r = report(vec![(true, 1000, true, 700)]);
+ assert!(
+ r.cheaper_than(0.25, 1.0, 1.0),
+ "0.25 is twenty-five percent"
+ );
+ assert!(
+ !r.cheaper_than(25.0, 1.0, 1.0),
+ "25.0 would be a 2500% saving, which nothing can meet"
+ );
+ }
+}

crates/panday-router/src/bench.rs

@@ -419,6 +419,28 @@ impl Score {
  pub fn meets_the_gate(&self, floor: f64) -> bool {
  self.accuracy() >= floor && self.confidently_wrong.is_empty()
  }
+
+ /// **M19.3's gate**: "classifier ... beats heuristic on route-bench by ≥10pt".
+ ///
+ /// [`meets_the_gate`](Self::meets_the_gate) is an absolute floor; this is the relative one the
+ /// milestone actually states, and until now it existed only as prose. A gate that is not a
+ /// function is a gate nobody can fail, which is the state docs/19 M19.1 was written against.
+ ///
+ /// `margin_points` is **percentage points**, not a fraction: M19.3 says "≥10pt", so the call
+ /// is `beats(&heuristic, 10.0)`. Accuracy is a 0..1 ratio internally and the conversion
+ /// happens here, once — passing `0.10` and meaning ten points is the obvious way to get this
+ /// wrong, so `the_margin_is_percentage_points_not_a_fraction` pins it.
+ ///
+ /// **Being confidently wrong disqualifies a challenger regardless of margin.** That is not an
+ /// extra condition bolted on: the dangerous quadrant is the thing route-bench exists to
+ /// measure (a wrong answer the router *trusts* and acts on), and a caller who checked only the
+ /// margin would ship a model that is more accurate on average and catastrophic on the cases
+ /// that matter. The incumbent's own quadrant is not consulted — the question is whether the
+ /// *challenger* is safe to deploy, not whether it is less bad than what is there.
+ pub fn beats(&self, incumbent: &Score, margin_points: f64) -> bool {
+ let gained = (self.accuracy() - incumbent.accuracy()) * 100.0;
+ gained >= margin_points && self.confidently_wrong.is_empty()
+ }
 }
 
 /// Score any classifier against the corpus. `HeuristicClassifier` today; a learned one at
@@ -448,3 +470,60 @@ pub fn score(c: &dyn Classifier) -> Score {
  }
  s
 }
+
+#[cfg(test)]
+mod gate_tests {
+ use super::Score;
+
+ fn score(correct: usize, total: usize, confidently_wrong: Vec<&'static str>) -> Score {
+ Score {
+ total,
+ correct,
+ confidently_wrong,
+ caught_by_the_gate: vec![],
+ }
+ }
+
+ #[test]
+ fn the_margin_is_percentage_points_not_a_fraction() {
+ // M19.3 says "≥10pt". Accuracy is a 0..1 ratio internally, so the obvious mistake is to
+ // pass 0.10 and mean ten points — which would let a challenger through on a *tenth* of a
+ // percentage point. The unit is pinned here because the gate is unfalsifiable by
+ // inspection: both readings compile and both look right.
+ let incumbent = score(70, 100, vec![]);
+ let exactly_ten = score(80, 100, vec![]);
+ let just_under = score(79, 100, vec![]);
+
+ assert!(
+ exactly_ten.beats(&incumbent, 10.0),
+ "10pt clears a 10pt bar"
+ );
+ assert!(!just_under.beats(&incumbent, 10.0), "9pt does not");
+ }
+
+ #[test]
+ fn a_confidently_wrong_challenger_does_not_ship_however_far_ahead() {
+ // The whole point of route-bench's dangerous quadrant. A model can be twenty points more
+ // accurate and still be the worse thing to deploy, because the router *acts* on a
+ // confident answer.
+ let incumbent = score(50, 100, vec![]);
+ let brilliant_but_reckless = score(90, 100, vec!["a-case-it-got-wrong-and-trusted"]);
+ assert!(
+ !brilliant_but_reckless.beats(&incumbent, 10.0),
+ "40pt ahead, and still not shippable"
+ );
+ }
+
+ #[test]
+ fn losing_ground_is_not_a_pass() {
+ let incumbent = score(90, 100, vec![]);
+ let worse = score(60, 100, vec![]);
+ assert!(!worse.beats(&incumbent, 10.0));
+ // And equal is not "beats" either: the milestone asks for a margin, not parity.
+ assert!(!incumbent.beats(&incumbent, 10.0));
+ assert!(
+ incumbent.beats(&incumbent, 0.0),
+ "a zero-point bar is met by parity"
+ );
+ }
+}

crates/panday-types/src/scorecard.rs

@@ -124,6 +124,33 @@ impl Scorecard {
  self.cases > 0 && self.rate() >= floor
  }
 
+ /// **The relative gate**: did this run beat an earlier one on the same suite?
+ ///
+ /// [`meets`](Self::meets) answers "is it good enough"; several milestones ask "is it *better*"
+ /// and had no function to ask with — M19.7 ("SFT stage beats base on agent-bench") is the
+ /// live one. A gate that exists only in prose is a gate nobody can fail.
+ ///
+ /// **`None` means the two cannot be compared, and that is the point of the return type.**
+ /// Comparing a tuned run over ten tasks against a base run over forty-one is the way this
+ /// measurement gets faked without anyone lying: both numbers are real, the ratio is
+ /// meaningless, and a `bool` would have hidden it. Refused when the suites differ, when either
+ /// ran nothing, or when the case counts differ — a shorter run is a different corpus, not a
+ /// better model. A caller that writes `.unwrap_or(false)` fails closed, which is the right
+ /// default for a shipping decision.
+ ///
+ /// Strictly greater, with no margin: M19.7 says "beats", and a tie is not one. Where a
+ /// milestone names a margin instead — M19.3's "≥10pt" — that gate lives with its suite and
+ /// takes the margin explicitly, because the unit is the suite's business, not this type's.
+ pub fn beats(&self, base: &Scorecard) -> Option<bool> {
+ if self.suite != base.suite || self.cases == 0 || base.cases == 0 {
+ return None;
+ }
+ if self.cases != base.cases {
+ return None;
+ }
+ Some(self.passed > base.passed)
+ }
+
  /// The human rendering. Derived from the artifact, never the other way round.
  pub fn to_markdown(&self) -> String {
  let mut out = format!(
@@ -226,3 +253,61 @@ mod tests {
  assert!(md.contains("missing required field `city`"));
  }
 }
+
+#[cfg(test)]
+mod relative_gate_tests {
+ use super::Scorecard;
+
+ fn card(suite: &str, passed: u32, cases: u32) -> Scorecard {
+ let mut c = Scorecard::new(suite, "subject", "2026-08-24T00:00:00Z");
+ for i in 0..cases {
+ c.record(&format!("case{i}"), i < passed, "");
+ }
+ c
+ }
+
+ #[test]
+ fn a_shorter_run_is_not_a_better_model() {
+ // The failure this return type exists for. Both numbers are real, the comparison is not:
+ // 9/10 against 30/41 is a different corpus, not an improvement. A `bool` would have said
+ // "better" and nobody would have noticed.
+ let base = card("agent-bench", 30, 41);
+ let tuned_on_fewer = card("agent-bench", 9, 10);
+ assert_eq!(tuned_on_fewer.beats(&base), None);
+ }
+
+ #[test]
+ fn suites_do_not_cross() {
+ let a = card("agent-bench", 40, 41);
+ let b = card("route-bench", 10, 41);
+ assert_eq!(a.beats(&b), None, "same case count, different question");
+ }
+
+ #[test]
+ fn nothing_ran_is_not_a_win() {
+ let base = card("agent-bench", 0, 0);
+ let tuned = card("agent-bench", 5, 5);
+ assert_eq!(tuned.beats(&base), None);
+ assert_eq!(base.beats(&tuned), None);
+ }
+
+ #[test]
+ fn beating_is_strict() {
+ let base = card("agent-bench", 30, 41);
+ assert_eq!(card("agent-bench", 31, 41).beats(&base), Some(true));
+ assert_eq!(
+ card("agent-bench", 30, 41).beats(&base),
+ Some(false),
+ "a tie is not a win"
+ );
+ assert_eq!(card("agent-bench", 29, 41).beats(&base), Some(false));
+ }
+
+ #[test]
+ fn an_incomparable_pair_fails_closed_for_a_careless_caller() {
+ // `.unwrap_or(false)` is what a caller will write. It must mean "do not ship".
+ let base = card("agent-bench", 30, 41);
+ let mismatched = card("agent-bench", 41, 42);
+ assert!(!mismatched.beats(&base).unwrap_or(false));
+ }
+}

docs/19-training.md

@@ -254,6 +254,17 @@ artifacts; CI runs eval gates nightly.
  one that emits well-formed calls with the wrong arguments — because a generator that cannot
  report a bad model as bad is not worth running against a good one.
 - **M19.3** Model 1 shipped: classifier behind `Classifier` trait beats heuristic on route-bench by ≥10pt; deployed in shadow, then live.
+
+ **The gate is a function now, though the model is not here.** `panday_router::bench::Score::beats(&incumbent, 10.0)`
+ — the margin in percentage points, as the milestone states it. It was prose until this change,
+ which meant M19.3 could not be failed even with a classifier in hand; `Score::meets_the_gate` is
+ an absolute floor and answers a different question.
+
+ A challenger that is **confidently wrong is refused regardless of margin**. That is not an extra
+ condition: the dangerous quadrant is what route-bench exists to measure, and a caller checking
+ only the margin would ship a model that is better on average and catastrophic where the router
+ acts on it. Unit pinned by test — passing `0.10` for "ten points" is the obvious way to get this
+ wrong and both readings compile.
 - **M19.4** Transcript mining pipeline with consent flags + PII scrub + provenance; first 10k-pair summarizer dataset. ✅ *(shipped: `panday_harness::mining`, `cargo xtask mine --logs <dir> --out <file>`. **The 10k-pair dataset is not here** — it needs 10k consented transcripts, and this repo has none.)*
 
  **Consent, then scrub, then provenance — and each step defaults to refusing.** The mistakes in a
@@ -285,6 +296,21 @@ artifacts; CI runs eval gates nightly.
  miner runs end to end today over a directory of logs and reports exactly why each candidate was
  dropped — which is the part that had to exist before any transcript was worth collecting.
 - **M19.5** Model 2 shipped: reduce-bench regression zero, ≥25% cheaper semantic tier than the provider cheap-pool it replaces; GGUF in catalog.
+
+ **reduce-bench had no cost dimension at all**, so "≥25% cheaper" was unmeasurable — the suite
+ scored retention and token ratio, which are not money. `Report::cost_saving` and
+ `Report::cheaper_than(0.25, incumbent_price, replacement_price)` add it.
+
+ Cost rather than tokens because the two move independently: a reducer that keeps 40% of the
+ tokens and hands them to a model at three times the price is *more* expensive, and the old
+ ratio-only scorecard would have called that a 60% win. Prices are per token and supplied by the
+ caller — `panday-harness` has no price table and must not grow one, or the gate starts drifting
+ with a catalog. A saving against an incumbent that spends nothing returns `None` rather than a
+ number.
+
+ Both halves are required, because either alone is a trap: cheapness with regressions is a
+ reducer that saves money by losing the answer, and zero regressions with no saving is a semantic
+ tier with no reason to exist.
 - **M19.6** agent-bench (50 verifiable repo tasks in T3) doubling as GRPO environment. ✅ *(shipped: `panday_harness::agent_bench` — 41 tasks, the audit, the jailed runner, and `score`. **Runs in T2, not T3**. Thirty-eight are repair classes; three (`poisoned-readme`, `poisoned-comment`, `granted-json`) are M20.1 injection canaries whose verifier fails if a relative marker file exists. Still not 50 — the twelve destructive classes stay gone.)*
 
  **This is the second attempt, and the first one destroyed a machine.** The original ran each
@@ -334,3 +360,15 @@ artifacts; CI runs eval gates nightly.
  **What is left is the model**, and T3 for the tasks that will eventually need a VM rather than a
  jail (M14.5).
 - **M19.7** Model 3 v1: SFT stage beats base on agent-bench; go/no-go review for the GRPO spend.
+
+ **`Scorecard::beats(&base)` is the gate**, on the artifact type every suite already emits, so
+ it is not agent-bench's alone.
+
+ It returns `Option<bool>`, and the `None` is the substance. Comparing a tuned run over ten tasks
+ against a base run over forty-one is how this measurement gets faked without anyone lying: both
+ numbers are real, the ratio is meaningless, and a plain `bool` would have hidden it. Refused
+ when the suites differ, when either ran nothing, or when the case counts differ — a shorter run
+ is a different corpus, not a better model. `.unwrap_or(false)` fails closed, which is what a
+ shipping decision should do.
+
+ Strictly greater, no margin: the milestone says "beats", and a tie is not one.