feat(integrity): per-bench expected_n + sample-health badge

benchmarks/aggregator-head-lag.yml

@@ -77,6 +77,13 @@ findings:
 
 source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/aggregator-head-lag
 
+# Methodology declares 5760 expected samples per (provider, chain,
+# region) per day at the 15 s cadence. The sample_size query sums
+# across the 3 chains and 3 regions, so the healthy floor is
+# 5760 × 9 ≈ 51840. Set conservative to absorb the 60 s scrape gaps
+# Prometheus introduces between probe and ingest.
+expected_n: 50000
+
 prometheus:
  window: 24h
 

benchmarks/bridge-fee.yml

@@ -91,6 +91,12 @@ faq:
 
 source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/bridge-monitor
 
+# sample_size queries filter to amount_usd="300", so the healthy floor
+# per bridge is 4 routes × 12 sweeps/hour × 24h ≈ 1152. Use 1000 as a
+# conservative round number; bridges that don't support all 4 routes
+# will land in the "low" band rather than insufficient.
+expected_n: 1000
+
 prometheus:
  url: https://prometheus-production-9ffe.up.railway.app
  window: 24h

benchmarks/bridge-quote-latency.yml

@@ -75,6 +75,12 @@ findings:
 
 source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/bridge-monitor
 
+# 4 routes × 3 notional sizes swept every 5 min × 24h ≈ 3456 expected
+# quote attempts per bridge per day. Conservative 3000 to absorb the
+# routes that a given bridge does not support (those return early as
+# unsupported and are excluded from the histogram count).
+expected_n: 3000
+
 prometheus:
  url: https://prometheus-production-9ffe.up.railway.app
  window: 24h

benchmarks/oracle-deviation.yml

@@ -98,6 +98,15 @@ faq:
 
 source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/oracle-deviation
 
+# Healthy sample_size for this bench is the count of source series per
+# pair (4 for the 4-source pairs, 3 for XRP/ADA/DOGE where Chainlink is
+# deprecated). The sample_size query counts distinct sources, not 24h
+# data points, so the displayed n on the leaderboard tops out at 4. The
+# threshold here is set to 4 so the 4-source pairs render as healthy and
+# the 3-source pairs read as 75 percent of expected, still healthy by
+# the 50 percent gate.
+expected_n: 4
+
 prometheus:
  window: 24h
 

benchmarks/perp-funding.yml

@@ -96,6 +96,14 @@ faq:
 
 source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/hyperliquid-frontends
 
+# Healthy sample_size per venue is 3 (one perp_funding_rate_hourly_bps
+# series per asset across BTC, ETH and SOL). The sample_size query
+# counts distinct asset series per venue, so the n reported on the
+# leaderboard is the asset coverage count, not the 24h tick count. 3 is
+# full coverage; a venue dropping to 2 (one asset feed dead) reads as
+# 67 percent and earns the low-sample badge.
+expected_n: 3
+
 prometheus:
  window: 24h
  expected_freshness_seconds: 3600

src/app/api/citable/route.ts

@@ -48,16 +48,28 @@ export async function GET(req: Request) {
  }
  const data = benches.map((b) => {
  const top = leader(b);
+ // Insufficient aggregate: explicitly null the value + leader so
+ // downstream LLM agents and journalists do not quote a number drawn
+ // from an undersized field. The headline sentence is rewritten to
+ // "insufficient data" by headlineSentence above.
+ const insufficient = b.dataConfidence === "insufficient";
  return {
  slug: b.slug,
  title: b.title,
  category: b.category,
  metric: b.metric,
  unit: b.unit,
  status: b.status,
- value: fieldValue(b),
- leader: top ? { name: top.name, slug: top.slug, value: top.value } : null,
+ value: insufficient ? null : fieldValue(b),
+ leader:
+ insufficient
+ ? null
+ : top
+ ? { name: top.name, slug: top.slug, value: top.value }
+ : null,
  sampleSize: b.sampleSize,
+ expectedN: b.expectedN,
+ dataConfidence: b.dataConfidence,
  asOf: b.lastRunAt,
  headline: headlineSentence(b),
  url: `${SITE.url}/benchmarks/${b.slug}`,

src/app/api/stat/[slug]/route.ts

@@ -44,6 +44,7 @@ export async function GET(
  }
 
  const top = leader(b);
+ const insufficient = b.dataConfidence === "insufficient";
  const payload = {
  slug: b.slug,
  title: b.title,
@@ -53,20 +54,33 @@ export async function GET(
  unit: b.unit,
  status: b.status,
  higherIsBetter: b.higherIsBetter,
- value: fieldValue(b),
- leader: top,
+ // Aggregate is "insufficient" (median per-provider sample health
+ // below 10 percent of expected_n): refuse to publish a value or
+ // leader; the headline is rewritten by headlineSentence so the
+ // agent / journalist reads "insufficient data" instead of quoting
+ // a number drawn from undersized samples.
+ value: insufficient ? null : fieldValue(b),
+ leader: insufficient ? null : top,
  rankings: b.results
  .filter((r) => r.ms.p50 > 0)
+ // Drop "insufficient" rows from the machine-readable ranking too:
+ // a row that the page hides from the leaderboard must not surface
+ // here either.
+ .filter((r) => r.dataConfidence !== "insufficient")
  .sort((a, c) => (b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50))
  .map((r) => ({
  name: r.name,
  slug: r.slug,
  ms: r.ms,
  successRate: r.successRate,
  sampleSize: r.sampleSize,
+ sampleHealth: r.sampleHealth,
+ dataConfidence: r.dataConfidence,
  })),
  sparkline: sparklineFor(b, top?.slug),
  sampleSize: b.sampleSize,
+ expectedN: b.expectedN,
+ dataConfidence: b.dataConfidence,
  asOf: b.lastRunAt,
  headline: headlineSentence(b),
  quote: citationQuote(b, SITE.url),

src/components/ledger-table.tsx

@@ -142,6 +142,12 @@ export function LedgerTable({
  // lost — only the noisy ledger rows are pruned.
  const sortedAll = [...results]
  .filter((r) => {
+ // Sample-health gate. Rows tagged "insufficient" by the load path
+ // (sampleSize < 0.1 × expectedN) drop out of the ranking entirely
+ // so the leaderboard cannot assert a position from a wildly
+ // undersized field. "low" rows stay; the row renderer shows them
+ // with a soft pill instead.
+ if (r.dataConfidence === "insufficient") return false;
  if (activePanel) {
  const v = activePanel.values[r.slug];
  return v != null && Number.isFinite(v) && v !== 0;
@@ -429,6 +435,20 @@ function Row({
  </span>
  </Hint>
  )}
+ {!isOffline && r.dataConfidence === "low" && (
+ <Hint
+ label={
+ r.sampleHealth != null
+ ? `Sample count is below half of the expected per provider for this bench (${Math.round(r.sampleHealth * 100)}% of expected). The ranking still includes this provider; confidence in the headline number is lower than for healthy rows.`
+ : "Sample count is below half of the expected per provider for this bench. The ranking still includes this provider; confidence in the headline number is lower than for healthy rows."
+ }
+ >
+ <span className="inline-flex items-center gap-1 shrink-0 font-sans text-[10px] uppercase tracking-[0.14em] text-ink-muted">
+ <span className="inline-block w-1.5 h-1.5 rounded-full bg-[var(--color-warn,#c08a3c)]" aria-hidden />
+ Low sample
+ </span>
+ </Hint>
+ )}
  {r.type && !isOffline && (
  <span className="hidden md:inline-flex">
  <ProviderTypeBadge type={r.type} />

src/lib/citation.ts

@@ -4,16 +4,32 @@
  * everyone (LLMs, journalists, ourselves) sees.
  */
 
-import type { Benchmark } from "@/types/benchmark";
+import type { Benchmark, ProviderResult } from "@/types/benchmark";
 import { liveResults } from "@/lib/provider-filters";
 import { fmtUnit } from "@/lib/format";
 
+/** Provider set used to derive the headline figures. Drops rows whose
+ * per-provider sample-health is "insufficient" (set on the load path
+ * when the bench declares expected_n and the row falls below the 10
+ * percent of expected floor). Those rows can still render in some
+ * surfaces with a soft tag, but they must not contribute to the leader
+ * claim shipped to AI agents and journalists via the citable APIs. */
+function citationCandidates(b: Benchmark): ProviderResult[] {
+ const live = liveResults(b.results);
+ if (!b.expectedN) return live;
+ return live.filter((r) => r.dataConfidence !== "insufficient");
+}
+
 /** Median value of the benchmark (the field shown in the headline). */
 export function fieldValue(b: Benchmark): number | null {
  if (b.status !== "live") return null;
- const live = liveResults(b.results);
- if (live.length === 0) return null;
- const sorted = [...live].sort((a, c) =>
+ // Bench-wide aggregate is insufficient: refuse to publish a value
+ // (downstream LLM tools and SERP snippets would otherwise quote a
+ // number drawn from a wildly undersized field).
+ if (b.dataConfidence === "insufficient") return null;
+ const candidates = citationCandidates(b);
+ if (candidates.length === 0) return null;
+ const sorted = [...candidates].sort((a, c) =>
  b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50
  );
  return sorted[0].ms.p50;
@@ -22,9 +38,10 @@ export function fieldValue(b: Benchmark): number | null {
 /** Who is currently #1 on this benchmark, if any. */
 export function leader(b: Benchmark): { name: string; slug: string; value: number } | null {
  if (b.status !== "live") return null;
- const live = liveResults(b.results);
- if (live.length === 0) return null;
- const sorted = [...live].sort((a, c) =>
+ if (b.dataConfidence === "insufficient") return null;
+ const candidates = citationCandidates(b);
+ if (candidates.length === 0) return null;
+ const sorted = [...candidates].sort((a, c) =>
  b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50
  );
  return { name: sorted[0].name, slug: sorted[0].slug, value: sorted[0].ms.p50 };
@@ -41,6 +58,9 @@ function windowSuffix(unit: string): string {
 
 /** Short factual sentence ready to paste into an article. Templated, no LLM. */
 export function headlineSentence(b: Benchmark): string {
+ if (b.dataConfidence === "insufficient") {
+ return `${b.title}. Insufficient data to assert a leader.`;
+ }
  const top = leader(b);
  if (!top) return `${b.title}. Awaiting first run.`;
  const value = fmtUnit(top.value, b.unit);

src/lib/materialize/editorial.ts

@@ -37,6 +37,7 @@ export function buildEditorial(
  source: spec.source,
  dimensions: spec.dimensions,
  ledgerColumns: spec.ledger_columns,
+ expectedN: spec.expected_n,
  };
 }
 

src/lib/materialize/load.ts

@@ -20,10 +20,14 @@
 import { SpecSchema, type Spec } from "@/lib/spec-schema";
 import { renderBenchmarkText } from "@/lib/bench-template";
 import { liveResults as liveProviderResults } from "@/lib/provider-filters";
+import {
+ aggregateConfidence,
+ classifyHealth,
+} from "@/lib/sample-health";
 import {
  SECONDS_PER_DAY,
  SECONDS_PER_HOUR,
  SECONDS_PER_MINUTE,
 } from "@/lib/time-constants";
 import {
  activeFilterLabels,
@@ -109,6 +113,23 @@
  // "unknown" everywhere else in the code.
  for (const r of live.results) r.availability = "live";
 
+ // Per-provider sample-health classification. When the spec declares
+ // expected_n, every live provider gets `dataConfidence` (healthy /
+ // low / insufficient) + `sampleHealth` (raw ratio) so the renderer
+ // can badge undersized rows and the citable APIs can refuse to
+ // assert a winner from a degraded field. Specs without expected_n
+ // leave the fields undefined and the renderer behaves as before.
+ const expectedN = spec.expected_n;
+ if (expectedN) {
+ for (const r of live.results) {
+ const h = classifyHealth(r.sampleSize, expectedN);
+ if (h) {
+ r.dataConfidence = h.confidence;
+ r.sampleHealth = h.ratio;
+ }
+ }
+ }
+
  // Augment with spec-declared providers that didn't return data this
  // cycle, but only on the *unfiltered* view. When the reader has
  // applied a dimension filter (e.g. chain=bnb on rpc-capabilities)
@@ -238,6 +259,13 @@
  // surfaces fall back to the coarser bestPerChain path.
  const cellRanks = !isFiltered ? await tryLoadCellRanks(spec) : undefined;
 
+ // Bench-wide sample-health aggregate. Median of the per-provider
+ // ratios, classified into the same healthy/low/insufficient bands.
+ // Drives the citable APIs and the headline sentence: an "insufficient"
+ // aggregate makes /api/citable + /api/stat report value=null + leader=null
+ // rather than asserting a winner from a degraded field.
+ const agg = aggregateConfidence(live.results, spec.expected_n);
+
  // Resolve {{p50:slug}} / {{best_name}} / {{count}} etc. placeholders
  // against the freshly loaded numbers so editorial text (findings,
  // seo_intro, faq) never drifts from the displayed data.
@@ -248,6 +276,7 @@
  worstPerChain,
  providersPerChain,
  cellRanks,
+ dataConfidence: agg?.confidence,
  });
  // Persistence is the caller's concern (site: KV snapshot write,
  // worker: store publish). Only the unfiltered "All" view of a live