diff --git a/benchmarks/aggregator-head-lag.yml b/benchmarks/aggregator-head-lag.yml index 1ea556cd..b358fc79 100644 --- a/benchmarks/aggregator-head-lag.yml +++ b/benchmarks/aggregator-head-lag.yml @@ -77,6 +77,13 @@ findings: source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/aggregator-head-lag +# Methodology declares 5760 expected samples per (provider, chain, +# region) per day at the 15 s cadence. The sample_size query sums +# across the 3 chains and 3 regions, so the healthy floor is +# 5760 × 9 ≈ 51840. Set conservative to absorb the 60 s scrape gaps +# Prometheus introduces between probe and ingest. +expected_n: 50000 + prometheus: window: 24h diff --git a/benchmarks/bridge-fee.yml b/benchmarks/bridge-fee.yml index c50e2600..dae95381 100644 --- a/benchmarks/bridge-fee.yml +++ b/benchmarks/bridge-fee.yml @@ -91,6 +91,12 @@ faq: source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/bridge-monitor +# sample_size queries filter to amount_usd="300", so the healthy floor +# per bridge is 4 routes × 12 sweeps/hour × 24h ≈ 1152. Use 1000 as a +# conservative round number; bridges that don't support all 4 routes +# will land in the "low" band rather than insufficient. +expected_n: 1000 + prometheus: url: https://prometheus-production-9ffe.up.railway.app window: 24h diff --git a/benchmarks/bridge-quote-latency.yml b/benchmarks/bridge-quote-latency.yml index 7fea30c9..16d0c9fb 100644 --- a/benchmarks/bridge-quote-latency.yml +++ b/benchmarks/bridge-quote-latency.yml @@ -75,6 +75,12 @@ findings: source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/bridge-monitor +# 4 routes × 3 notional sizes swept every 5 min × 24h ≈ 3456 expected +# quote attempts per bridge per day. Conservative 3000 to absorb the +# routes that a given bridge does not support (those return early as +# unsupported and are excluded from the histogram count). +expected_n: 3000 + prometheus: url: https://prometheus-production-9ffe.up.railway.app window: 24h diff --git a/benchmarks/oracle-deviation.yml b/benchmarks/oracle-deviation.yml index 4bf578e1..5e3febf0 100644 --- a/benchmarks/oracle-deviation.yml +++ b/benchmarks/oracle-deviation.yml @@ -98,6 +98,15 @@ faq: source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/oracle-deviation +# Healthy sample_size for this bench is the count of source series per +# pair (4 for the 4-source pairs, 3 for XRP/ADA/DOGE where Chainlink is +# deprecated). The sample_size query counts distinct sources, not 24h +# data points, so the displayed n on the leaderboard tops out at 4. The +# threshold here is set to 4 so the 4-source pairs render as healthy and +# the 3-source pairs read as 75 percent of expected, still healthy by +# the 50 percent gate. +expected_n: 4 + prometheus: window: 24h diff --git a/benchmarks/perp-funding.yml b/benchmarks/perp-funding.yml index 704692d5..f4063a3d 100644 --- a/benchmarks/perp-funding.yml +++ b/benchmarks/perp-funding.yml @@ -96,6 +96,14 @@ faq: source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/hyperliquid-frontends +# Healthy sample_size per venue is 3 (one perp_funding_rate_hourly_bps +# series per asset across BTC, ETH and SOL). The sample_size query +# counts distinct asset series per venue, so the n reported on the +# leaderboard is the asset coverage count, not the 24h tick count. 3 is +# full coverage; a venue dropping to 2 (one asset feed dead) reads as +# 67 percent and earns the low-sample badge. +expected_n: 3 + prometheus: window: 24h expected_freshness_seconds: 3600 diff --git a/src/app/api/citable/route.ts b/src/app/api/citable/route.ts index 5d453509..bb31ab41 100644 --- a/src/app/api/citable/route.ts +++ b/src/app/api/citable/route.ts @@ -48,6 +48,11 @@ export async function GET(req: Request) { } const data = benches.map((b) => { const top = leader(b); + // Insufficient aggregate: explicitly null the value + leader so + // downstream LLM agents and journalists do not quote a number drawn + // from an undersized field. The headline sentence is rewritten to + // "insufficient data" by headlineSentence above. + const insufficient = b.dataConfidence === "insufficient"; return { slug: b.slug, title: b.title, @@ -55,9 +60,16 @@ export async function GET(req: Request) { metric: b.metric, unit: b.unit, status: b.status, - value: fieldValue(b), - leader: top ? { name: top.name, slug: top.slug, value: top.value } : null, + value: insufficient ? null : fieldValue(b), + leader: + insufficient + ? null + : top + ? { name: top.name, slug: top.slug, value: top.value } + : null, sampleSize: b.sampleSize, + expectedN: b.expectedN, + dataConfidence: b.dataConfidence, asOf: b.lastRunAt, headline: headlineSentence(b), url: `${SITE.url}/benchmarks/${b.slug}`, diff --git a/src/app/api/stat/[slug]/route.ts b/src/app/api/stat/[slug]/route.ts index ec496bf5..b53c5767 100644 --- a/src/app/api/stat/[slug]/route.ts +++ b/src/app/api/stat/[slug]/route.ts @@ -44,6 +44,7 @@ export async function GET( } const top = leader(b); + const insufficient = b.dataConfidence === "insufficient"; const payload = { slug: b.slug, title: b.title, @@ -53,10 +54,19 @@ export async function GET( unit: b.unit, status: b.status, higherIsBetter: b.higherIsBetter, - value: fieldValue(b), - leader: top, + // Aggregate is "insufficient" (median per-provider sample health + // below 10 percent of expected_n): refuse to publish a value or + // leader; the headline is rewritten by headlineSentence so the + // agent / journalist reads "insufficient data" instead of quoting + // a number drawn from undersized samples. + value: insufficient ? null : fieldValue(b), + leader: insufficient ? null : top, rankings: b.results .filter((r) => r.ms.p50 > 0) + // Drop "insufficient" rows from the machine-readable ranking too: + // a row that the page hides from the leaderboard must not surface + // here either. + .filter((r) => r.dataConfidence !== "insufficient") .sort((a, c) => (b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50)) .map((r) => ({ name: r.name, @@ -64,9 +74,13 @@ export async function GET( ms: r.ms, successRate: r.successRate, sampleSize: r.sampleSize, + sampleHealth: r.sampleHealth, + dataConfidence: r.dataConfidence, })), sparkline: sparklineFor(b, top?.slug), sampleSize: b.sampleSize, + expectedN: b.expectedN, + dataConfidence: b.dataConfidence, asOf: b.lastRunAt, headline: headlineSentence(b), quote: citationQuote(b, SITE.url), diff --git a/src/components/ledger-table.tsx b/src/components/ledger-table.tsx index 22bbeedf..a5ea3dec 100644 --- a/src/components/ledger-table.tsx +++ b/src/components/ledger-table.tsx @@ -142,6 +142,12 @@ export function LedgerTable({ // lost — only the noisy ledger rows are pruned. const sortedAll = [...results] .filter((r) => { + // Sample-health gate. Rows tagged "insufficient" by the load path + // (sampleSize < 0.1 × expectedN) drop out of the ranking entirely + // so the leaderboard cannot assert a position from a wildly + // undersized field. "low" rows stay; the row renderer shows them + // with a soft pill instead. + if (r.dataConfidence === "insufficient") return false; if (activePanel) { const v = activePanel.values[r.slug]; return v != null && Number.isFinite(v) && v !== 0; @@ -429,6 +435,20 @@ function Row({ )} + {!isOffline && r.dataConfidence === "low" && ( + + + + Low sample + + + )} {r.type && !isOffline && ( diff --git a/src/lib/citation.ts b/src/lib/citation.ts index c744e788..b137deb4 100644 --- a/src/lib/citation.ts +++ b/src/lib/citation.ts @@ -4,16 +4,32 @@ * everyone (LLMs, journalists, ourselves) sees. */ -import type { Benchmark } from "@/types/benchmark"; +import type { Benchmark, ProviderResult } from "@/types/benchmark"; import { liveResults } from "@/lib/provider-filters"; import { fmtUnit } from "@/lib/format"; +/** Provider set used to derive the headline figures. Drops rows whose + * per-provider sample-health is "insufficient" (set on the load path + * when the bench declares expected_n and the row falls below the 10 + * percent of expected floor). Those rows can still render in some + * surfaces with a soft tag, but they must not contribute to the leader + * claim shipped to AI agents and journalists via the citable APIs. */ +function citationCandidates(b: Benchmark): ProviderResult[] { + const live = liveResults(b.results); + if (!b.expectedN) return live; + return live.filter((r) => r.dataConfidence !== "insufficient"); +} + /** Median value of the benchmark (the field shown in the headline). */ export function fieldValue(b: Benchmark): number | null { if (b.status !== "live") return null; - const live = liveResults(b.results); - if (live.length === 0) return null; - const sorted = [...live].sort((a, c) => + // Bench-wide aggregate is insufficient: refuse to publish a value + // (downstream LLM tools and SERP snippets would otherwise quote a + // number drawn from a wildly undersized field). + if (b.dataConfidence === "insufficient") return null; + const candidates = citationCandidates(b); + if (candidates.length === 0) return null; + const sorted = [...candidates].sort((a, c) => b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50 ); return sorted[0].ms.p50; @@ -22,9 +38,10 @@ export function fieldValue(b: Benchmark): number | null { /** Who is currently #1 on this benchmark, if any. */ export function leader(b: Benchmark): { name: string; slug: string; value: number } | null { if (b.status !== "live") return null; - const live = liveResults(b.results); - if (live.length === 0) return null; - const sorted = [...live].sort((a, c) => + if (b.dataConfidence === "insufficient") return null; + const candidates = citationCandidates(b); + if (candidates.length === 0) return null; + const sorted = [...candidates].sort((a, c) => b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50 ); return { name: sorted[0].name, slug: sorted[0].slug, value: sorted[0].ms.p50 }; @@ -41,6 +58,9 @@ function windowSuffix(unit: string): string { /** Short factual sentence ready to paste into an article. Templated, no LLM. */ export function headlineSentence(b: Benchmark): string { + if (b.dataConfidence === "insufficient") { + return `${b.title}. Insufficient data to assert a leader.`; + } const top = leader(b); if (!top) return `${b.title}. Awaiting first run.`; const value = fmtUnit(top.value, b.unit); diff --git a/src/lib/materialize/editorial.ts b/src/lib/materialize/editorial.ts index 23049e98..42f28cb3 100644 --- a/src/lib/materialize/editorial.ts +++ b/src/lib/materialize/editorial.ts @@ -37,6 +37,7 @@ export function buildEditorial( source: spec.source, dimensions: spec.dimensions, ledgerColumns: spec.ledger_columns, + expectedN: spec.expected_n, }; } diff --git a/src/lib/materialize/load.ts b/src/lib/materialize/load.ts index d8873027..bbc52f6c 100644 --- a/src/lib/materialize/load.ts +++ b/src/lib/materialize/load.ts @@ -20,6 +20,10 @@ import { getPrometheus } from "@/lib/prometheus"; import { SpecSchema, type Spec } from "@/lib/spec-schema"; import { renderBenchmarkText } from "@/lib/bench-template"; import { liveResults as liveProviderResults } from "@/lib/provider-filters"; +import { + aggregateConfidence, + classifyHealth, +} from "@/lib/sample-health"; import { SECONDS_PER_DAY, SECONDS_PER_HOUR, @@ -109,6 +113,23 @@ export async function specToBenchmark( // "unknown" everywhere else in the code. for (const r of live.results) r.availability = "live"; + // Per-provider sample-health classification. When the spec declares + // expected_n, every live provider gets `dataConfidence` (healthy / + // low / insufficient) + `sampleHealth` (raw ratio) so the renderer + // can badge undersized rows and the citable APIs can refuse to + // assert a winner from a degraded field. Specs without expected_n + // leave the fields undefined and the renderer behaves as before. + const expectedN = spec.expected_n; + if (expectedN) { + for (const r of live.results) { + const h = classifyHealth(r.sampleSize, expectedN); + if (h) { + r.dataConfidence = h.confidence; + r.sampleHealth = h.ratio; + } + } + } + // Augment with spec-declared providers that didn't return data this // cycle, but only on the *unfiltered* view. When the reader has // applied a dimension filter (e.g. chain=bnb on rpc-capabilities) @@ -238,6 +259,13 @@ export async function specToBenchmark( // surfaces fall back to the coarser bestPerChain path. const cellRanks = !isFiltered ? await tryLoadCellRanks(spec) : undefined; + // Bench-wide sample-health aggregate. Median of the per-provider + // ratios, classified into the same healthy/low/insufficient bands. + // Drives the citable APIs and the headline sentence: an "insufficient" + // aggregate makes /api/citable + /api/stat report value=null + leader=null + // rather than asserting a winner from a degraded field. + const agg = aggregateConfidence(live.results, spec.expected_n); + // Resolve {{p50:slug}} / {{best_name}} / {{count}} etc. placeholders // against the freshly loaded numbers so editorial text (findings, // seo_intro, faq) never drifts from the displayed data. @@ -248,6 +276,7 @@ export async function specToBenchmark( worstPerChain, providersPerChain, cellRanks, + dataConfidence: agg?.confidence, }); // Persistence is the caller's concern (site: KV snapshot write, // worker: store publish). Only the unfiltered "All" view of a live diff --git a/src/lib/sample-health.test.ts b/src/lib/sample-health.test.ts new file mode 100644 index 00000000..cf372ef6 --- /dev/null +++ b/src/lib/sample-health.test.ts @@ -0,0 +1,128 @@ +import { describe, expect, test } from "bun:test"; +import { + aggregateConfidence, + classifyHealth, + formatHealthPct, + HEALTHY_THRESHOLD, + LOW_THRESHOLD, +} from "./sample-health"; +import type { ProviderResult } from "@/types/benchmark"; + +function row( + slug: string, + sampleSize: number | undefined, + availability: ProviderResult["availability"] = "live", +): ProviderResult { + return { + name: slug, + slug, + ms: { p50: 1, p90: 1, p99: 1, mean: 1 }, + successRate: 100, + sampleSize, + availability, + }; +} + +describe("classifyHealth", () => { + test("returns undefined when expectedN is missing", () => { + expect(classifyHealth(100, undefined)).toBeUndefined(); + expect(classifyHealth(100, 0)).toBeUndefined(); + }); + + test("returns undefined when sampleSize is missing", () => { + expect(classifyHealth(undefined, 1000)).toBeUndefined(); + expect(classifyHealth(NaN, 1000)).toBeUndefined(); + }); + + test("classifies healthy at and above 0.5", () => { + expect(classifyHealth(500, 1000)?.confidence).toBe("healthy"); + expect(classifyHealth(1000, 1000)?.confidence).toBe("healthy"); + expect(classifyHealth(10_000, 1000)?.confidence).toBe("healthy"); + }); + + test("classifies low between 0.1 and 0.5", () => { + expect(classifyHealth(100, 1000)?.confidence).toBe("low"); + expect(classifyHealth(499, 1000)?.confidence).toBe("low"); + }); + + test("classifies insufficient below 0.1", () => { + expect(classifyHealth(0, 1000)?.confidence).toBe("insufficient"); + expect(classifyHealth(99, 1000)?.confidence).toBe("insufficient"); + }); + + test("matches the documented thresholds", () => { + expect(HEALTHY_THRESHOLD).toBe(0.5); + expect(LOW_THRESHOLD).toBe(0.1); + }); + + test("handles low-cadence per-bench expected_n (perp-funding case)", () => { + // perp-funding declares expected_n: 3 (BTC, ETH, SOL series per venue). + expect(classifyHealth(3, 3)?.confidence).toBe("healthy"); + expect(classifyHealth(2, 3)?.confidence).toBe("healthy"); // 0.67 + expect(classifyHealth(1, 3)?.confidence).toBe("low"); // 0.33 + expect(classifyHealth(0, 3)?.confidence).toBe("insufficient"); + }); + + test("handles 4-source oracle-deviation case", () => { + expect(classifyHealth(4, 4)?.confidence).toBe("healthy"); + expect(classifyHealth(3, 4)?.confidence).toBe("healthy"); // 0.75 (XRP/ADA/DOGE) + expect(classifyHealth(1, 4)?.confidence).toBe("low"); // 0.25 + }); +}); + +describe("aggregateConfidence", () => { + test("returns undefined when expectedN is missing", () => { + const results = [row("a", 1000)]; + expect(aggregateConfidence(results, undefined)).toBeUndefined(); + }); + + test("ignores unavailable providers", () => { + const results = [ + row("a", 1000, "unavailable"), + row("b", 1000, "live"), + ]; + expect(aggregateConfidence(results, 1000)?.confidence).toBe("healthy"); + }); + + test("returns undefined when no live provider carries a sample count", () => { + const results = [row("a", undefined), row("b", undefined)]; + expect(aggregateConfidence(results, 1000)).toBeUndefined(); + }); + + test("uses the median ratio so one healthy provider does not mask a bad field", () => { + const results = [ + row("a", 10), // 0.01 insufficient + row("b", 50), // 0.05 insufficient + row("c", 10_000), // healthy + ]; + expect(aggregateConfidence(results, 1000)?.confidence).toBe( + "insufficient", + ); + }); + + test("a healthy median wins regardless of outliers", () => { + const results = [ + row("a", 0), // insufficient + row("b", 1000), // healthy + row("c", 5000), // healthy + ]; + expect(aggregateConfidence(results, 1000)?.confidence).toBe("healthy"); + }); +}); + +describe("formatHealthPct", () => { + test("formats ratios as integer percent", () => { + expect(formatHealthPct(0.75)).toBe("75%"); + expect(formatHealthPct(0.5)).toBe("50%"); + expect(formatHealthPct(0.09)).toBe("9%"); + }); + + test("clamps very large ratios for display", () => { + expect(formatHealthPct(50)).toBe("999%"); + }); + + test("returns n/a for missing ratios", () => { + expect(formatHealthPct(undefined)).toBe("n/a"); + expect(formatHealthPct(NaN)).toBe("n/a"); + }); +}); diff --git a/src/lib/sample-health.ts b/src/lib/sample-health.ts new file mode 100644 index 00000000..9dc750a0 --- /dev/null +++ b/src/lib/sample-health.ts @@ -0,0 +1,92 @@ +/** + * Sample-health classification. + * + * Bench pages used to rank providers from sample counts spanning 3 to + * tens of thousands without surfacing the confidence level, while the + * methodology page advertised "n >= 1000 per provider". On low cadence + * benches like perp-funding (3 series per venue by design) the methodology + * line read as broken, an E-E-A-T penalty waiting to happen. + * + * A flat threshold would mis-classify legitimately low-cadence benches. + * Per-bench `expected_n` declared in the YAML lets each bench publish + * what "full coverage" looks like, and this module computes + * health = sampleSize / expectedN per provider: + * + * - healthy health >= 0.5 + * - low 0.1 <= health < 0.5 + * - insufficient health < 0.1 + * + * Renderers gate UI accordingly: hide insufficient rows from rankings, + * show low rows with a "Low sample" pill, healthy rows render normally. + * + * Citation surfaces (/api/citable, /api/stat, headline sentences) read + * the bench-level aggregate so a benchmark with mostly-insufficient + * providers does not assert a winner in machine-readable feeds. + */ + +import type { ProviderResult } from "@/types/benchmark"; + +/** Inclusive ratio at and above which a provider is treated as healthy. */ +export const HEALTHY_THRESHOLD = 0.5; +/** Inclusive ratio at and above which a provider is treated as low. */ +export const LOW_THRESHOLD = 0.1; + +export type DataConfidence = "healthy" | "low" | "insufficient"; + +/** Classify a single provider's sample health against the bench-declared + * expected sample count. Returns undefined when the bench did not declare + * expected_n (legacy display behavior, no badge) or when the provider's + * sample count is missing. */ +export function classifyHealth( + sampleSize: number | undefined, + expectedN: number | undefined, +): { confidence: DataConfidence; ratio: number } | undefined { + if (!expectedN || expectedN <= 0) return undefined; + if (sampleSize == null || !Number.isFinite(sampleSize)) return undefined; + const ratio = sampleSize / expectedN; + let confidence: DataConfidence; + if (ratio < LOW_THRESHOLD) confidence = "insufficient"; + else if (ratio < HEALTHY_THRESHOLD) confidence = "low"; + else confidence = "healthy"; + return { confidence, ratio }; +} + +/** + * Aggregate confidence across a provider set, used at the bench level + * (citable JSON, headline sentence, structured data). The aggregate is + * the classification of the median per-provider ratio so a single + * low-cadence outlier does not poison the verdict for an otherwise + * healthy field. Returns undefined when no provider carries a + * computable health (no expectedN, or no provider returned data). + */ +export function aggregateConfidence( + results: ProviderResult[], + expectedN: number | undefined, +): { confidence: DataConfidence; ratio: number } | undefined { + if (!expectedN || expectedN <= 0) return undefined; + const ratios: number[] = []; + for (const r of results) { + if (r.availability === "unavailable") continue; + const h = classifyHealth(r.sampleSize, expectedN); + if (h) ratios.push(h.ratio); + } + if (ratios.length === 0) return undefined; + const sorted = [...ratios].sort((a, b) => a - b); + const mid = sorted.length >> 1; + const median = + sorted.length % 2 === 0 + ? (sorted[mid - 1] + sorted[mid]) / 2 + : sorted[mid]; + let confidence: DataConfidence; + if (median < LOW_THRESHOLD) confidence = "insufficient"; + else if (median < HEALTHY_THRESHOLD) confidence = "low"; + else confidence = "healthy"; + return { confidence, ratio: median }; +} + +/** Human-readable percentage of expected, clamped to a sensible display + * range for tooltips. Returns "n/a" when health is undefined. */ +export function formatHealthPct(ratio: number | undefined): string { + if (ratio == null || !Number.isFinite(ratio)) return "n/a"; + return `${Math.round(Math.min(ratio, 9.99) * 100)}%`; +} diff --git a/src/lib/snapshot.ts b/src/lib/snapshot.ts index 76945505..910818a6 100644 --- a/src/lib/snapshot.ts +++ b/src/lib/snapshot.ts @@ -78,6 +78,8 @@ const ProviderResultSchema = z.object({ slots: z.object({ p50: z.number(), p99: z.number() }).optional(), successRate: z.number(), sampleSize: z.number().optional(), + dataConfidence: z.enum(["healthy", "low", "insufficient"]).optional(), + sampleHealth: z.number().optional(), secondary: z.object({ label: z.string(), value: z.string() }).optional(), availability: z.enum(["live", "unavailable"]).optional(), meta: StalenessMetaSchema.optional(), @@ -169,7 +171,7 @@ const KEY_PREFIX = "ocb:snap:v1:"; * new field don't try to deserialize old-shape values. The Zod schema * below would also reject those, but the version prefix lets us * invalidate without writing strict-mode parsers. */ -const SCHEMA_VERSION = 3 as const; +const SCHEMA_VERSION = 4 as const; // Minimal runtime payload. Editorial metadata isn't snapshotted because // it lives in YAML and is rebuilt from the spec on every read. @@ -178,6 +180,8 @@ const SnapshotSchema = z.object({ savedAt: z.number().int().positive(), lastRunAt: z.string(), sampleSize: z.number(), + expectedN: z.number().optional(), + dataConfidence: z.enum(["healthy", "low", "insufficient"]).optional(), results: z.array(ProviderResultSchema), extras: ResultExtrasSchema, bestPerChain: z.record(z.string(), ProviderResultSchema).optional(), @@ -191,6 +195,8 @@ export type SnapshotPayload = { results: ProviderResult[]; extras: ResultExtras; sampleSize: number; + expectedN?: number; + dataConfidence?: "healthy" | "low" | "insufficient"; lastRunAt: string; bestPerChain?: Record; worstPerChain?: Record; @@ -379,6 +385,8 @@ async function readSnapshotWithAge( results: parsed.data.results, extras: parsed.data.extras, sampleSize: parsed.data.sampleSize, + expectedN: parsed.data.expectedN, + dataConfidence: parsed.data.dataConfidence, lastRunAt: parsed.data.lastRunAt, bestPerChain: parsed.data.bestPerChain, worstPerChain: parsed.data.worstPerChain, @@ -407,6 +415,8 @@ export function snapshotFromBenchmark(b: Benchmark): SnapshotPayload { results: b.results, extras: b.extras, sampleSize: b.sampleSize, + expectedN: b.expectedN, + dataConfidence: b.dataConfidence, lastRunAt: b.lastRunAt, // Persist per-chain leader stash so the snapshot-recovery path // (loadBenchmarkUnfilteredCached's KV fallback in spec.ts) can diff --git a/src/lib/spec-schema.ts b/src/lib/spec-schema.ts index a8be16a9..5c9d7244 100644 --- a/src/lib/spec-schema.ts +++ b/src/lib/spec-schema.ts @@ -255,6 +255,25 @@ export const SpecSchema = z findings: z.array(seoText(1, 500)).max(40).default([]), source: z.url(), + /** + * Expected sample count per provider over the bench's measurement + * window. Drives the bench page's sample-health badge: low cadence + * benches like perp-funding legitimately publish 3 samples per 24h, + * so a flat n threshold would falsely flag them as broken. By + * declaring expected_n in the YAML, the page computes + * health = sampleSize / expected_n per provider and tags rows + * below 50 percent as "low sample" or hides rankings below 10 + * percent as "insufficient". Roughly equals + * cadence_per_minute * window_minutes * routes_per_provider. + * Optional. Benches without expected_n behave as before (no badge). + */ + expected_n: z + .number() + .int() + .positive() + .optional() + .describe("Expected sample count per provider over the bench's measurement window. Used to badge undersized samples on the page. Roughly cadence_per_minute * window_minutes * routes_per_provider."), + /* Data source. OpenChainBench is a federation: every contributor * declares the Prometheus their harness publishes to. Schema-time * isPublicHttpsUrl + runtime DNS-resolve guard in the Prom client diff --git a/src/lib/spec.ts b/src/lib/spec.ts index f5f078a9..cd8b0e16 100644 --- a/src/lib/spec.ts +++ b/src/lib/spec.ts @@ -91,6 +91,13 @@ function overlayEditorial(stored: Benchmark, spec: Spec): Benchmark { // and the bench page filters without waiting on the materialise // worker to rewrite the snapshot. dimensions: spec.dimensions ?? stored.dimensions, + // expected_n is a YAML editorial declaration too: drives the + // sample-health badge logic on the page + the citable APIs. A + // freshly added/edited value must take effect immediately, before + // the worker re-publishes the snapshot, otherwise a bench keeps + // ranking 3-sample providers as healthy through the materialise + // lag. + expectedN: spec.expected_n ?? stored.expectedN, }; // Resolve `{{p50:slug}}`, `{{name:slug}}`, `{{best_name}}` etc. in the // overlaid editorial text. Without this, a YAML edit that ships AHEAD @@ -189,7 +196,11 @@ const loadBenchmarkUnfilteredCached = unstable_cache( // and metadata-coverage stayed invisible until the next cold cache // window. Bumping the key forces every read to regenerate against // the post-overlay shape immediately on deploy. - ["bench-unfiltered-v10"], + // v11: added per-provider dataConfidence + sampleHealth + bench-wide + // expectedN + dataConfidence aggregate. Cached v10 entries lack + // these fields, so the sample-health badge would not render on + // existing benches until the cache aged out. + ["bench-unfiltered-v11"], { revalidate: 60, tags: ["benchmarks"] }, ); @@ -297,7 +308,8 @@ const loadAllBenchmarksCached = unstable_cache( // all-draft so unstable_cache no longer caches the bad set, but any // already-stored v12 snapshot in Upstash KV would still serve for up // to 60s after deploy. Bumping the key sidesteps that window. - ["all-benchmarks-v13"], + // v14: bumped with bench-unfiltered-v11 (sample-health badges). + ["all-benchmarks-v14"], { revalidate: 60, tags: ["benchmarks"] }, ); export const loadAllBenchmarks = cache(loadAllBenchmarksCached); @@ -373,7 +385,8 @@ const loadBenchmarkFiltered = unstable_cache( // v5: bumped with bench-unfiltered-v8 (sec unit). // v6: bumped with bench-unfiltered-v9 (bp unit). // v7: bumped with bench-unfiltered-v10 (dimensions overlay). - ["bench-filters-v7"], + // v8: bumped with bench-unfiltered-v11 (sample-health badges). + ["bench-filters-v8"], { revalidate: 60, tags: ["benchmarks"] } ); diff --git a/src/types/benchmark.ts b/src/types/benchmark.ts index bcffa8ca..a5ca4d3c 100644 --- a/src/types/benchmark.ts +++ b/src/types/benchmark.ts @@ -43,6 +43,26 @@ export type ProviderResult = { successRate: number; /** Per-provider sample count over the run window. */ sampleSize?: number; + /** + * Sample-health classification derived from sampleSize / expectedN. + * + * - "healthy" : sampleSize >= 0.5 × expectedN. Render normally. + * - "low" : 0.1 × expectedN <= sampleSize < 0.5 × expectedN. + * Row stays in the ranking with a "Low sample" + * pill and a tooltip explaining the gap. + * - "insufficient": sampleSize < 0.1 × expectedN. Row drops out of + * the sorted ranking; aggregate citations should + * read "insufficient data" rather than assert a + * winner. + * + * Absent on benches whose spec does not declare `expected_n`, in + * which case no badge is rendered (legacy display behavior). */ + dataConfidence?: "healthy" | "low" | "insufficient"; + /** Sample-health ratio, 0..1+. Same source as `dataConfidence` but + * exposed as the raw fraction for downstream consumers (citable / + * stat APIs, dashboards, tooltips). Absent when the spec declares + * no expected_n or when sampleSize itself is missing. */ + sampleHealth?: number; secondary?: { label: string; value: string }; /** Defaults to "live" when the provider returns numbers; the spec * loader sets "unavailable" when prom has no data for the p50 / p90 / @@ -156,6 +176,15 @@ export type Benchmark = { * published-but-awaiting-data bench remains visible. */ editorialStatus: "live" | "draft"; sampleSize: number; + /** Spec-declared expected sample count per provider over the bench's + * window. Drives the per-provider sample-health badge logic. Absent + * when the spec author chose not to declare it (low cadence benches + * whose healthy n cannot be computed deterministically). */ + expectedN?: number; + /** Aggregate sample-health for the bench. Derived from the median of + * the per-provider healths; "insufficient" silences the leader + * assertion at every citable surface. Absent when expectedN is. */ + dataConfidence?: "healthy" | "low" | "insufficient"; abstract: string; metric: string; unit: "ms" | "s" | "sec" | "pct" | "bps" | "bp" | "count" | "slots" | "usd";