From 75f4b4b8f1ce178660f43fd991acc2ffc57b0fc4 Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Wed, 10 Jun 2026 22:03:48 +0200 Subject: [PATCH 1/2] feat(ledger): per-bench ledger column relabeling (ledger_columns) --- benchmarks/hyperliquid-frontends.yml | 25 ++++++ src/components/benchmark-body.tsx | 4 +- src/components/ledger-table.tsx | 122 ++++++++++++++++++++++----- src/lib/format.ts | 6 +- src/lib/spec-schema.ts | 52 ++++++++++++ src/types/benchmark.ts | 17 ++++ 6 files changed, 200 insertions(+), 26 deletions(-) diff --git a/benchmarks/hyperliquid-frontends.yml b/benchmarks/hyperliquid-frontends.yml index facacd0d..0a6a6a30 100644 --- a/benchmarks/hyperliquid-frontends.yml +++ b/benchmarks/hyperliquid-frontends.yml @@ -928,3 +928,28 @@ metric_panels: metric: hl_frontend_last_fill_age_seconds_v2 unit: s description: "Seconds since this frontend's most recent attributed fill. An outage signal." + - id: revenue_7d + label: Revenue 7d + metric: hl_frontend_fees_usd_7d_v2 + unit: usd + higher_is_better: true + description: "USD builder fees collected over the rolling 7 day window. Smooths promo cycles and one off campaigns; use for ranking stability." + - id: revenue_30d + label: Revenue 30d + metric: hl_frontend_fees_usd_30d_v2 + unit: usd + higher_is_better: true + description: "USD builder fees collected over the rolling 30 day window." + +# Honest column labels for the main table. The p50/p90/p99/mean slots are +# repurposed on this bench (USD revenue, no percentile semantics), so the +# ledger renders these labeled columns instead of latency headers, with +# unique users surfaced inline from the users panel. Matches the column +# set of the reference builder-code dashboards (volume, users, $/user, +# revenue) so readers can cross check without clicking through tabs. +ledger_columns: + - { label: "Revenue (24h)", slot: p50 } + - { label: "Volume (24h)", slot: p90 } + - { label: "Users (24h)", panel: users } + - { label: "$ / user (24h)", slot: p99 } + - { label: "Rev / day (7d avg)", slot: mean } diff --git a/src/components/benchmark-body.tsx b/src/components/benchmark-body.tsx index 761a87fd..ca5b92ef 100644 --- a/src/components/benchmark-body.tsx +++ b/src/components/benchmark-body.tsx @@ -492,7 +492,9 @@ export function BenchmarkBody({ ? "Product ledger" : activePanel ? `Product ledger · sorted by ${activePanel.label}` - : "Product ledger · sorted by p50"} + : viewBenchmark.ledgerColumns?.length + ? `Product ledger · sorted by ${viewBenchmark.ledgerColumns[0].label}` + : "Product ledger · sorted by p50"}

diff --git a/src/components/ledger-table.tsx b/src/components/ledger-table.tsx index a6821a3c..e040d663 100644 --- a/src/components/ledger-table.tsx +++ b/src/components/ledger-table.tsx @@ -3,7 +3,12 @@ import { useMemo } from "react"; import Link from "next/link"; -import type { Benchmark, MetricPanel, ProviderResult } from "@/types/benchmark"; +import type { + Benchmark, + LedgerColumn, + MetricPanel, + ProviderResult, +} from "@/types/benchmark"; import { ChainCoverageChip } from "@/components/chain-coverage-chip"; import { Hint } from "@/components/hint"; import { Sparkline } from "@/components/sparkline"; @@ -41,12 +46,35 @@ export function LedgerTable({ benchmark, activePanel, topN }: Props) { const pickValue = (r: ProviderResult): number => activePanel ? (activePanel.values[r.slug] ?? 0) : r.ms.p50; const panelActive = !!activePanel; - const secondary = results[0]?.secondary?.label; + // Custom column mode: benches that repurpose the p50/p90/p99/mean slots + // (USD revenue leaderboards) declare ledger_columns in their YAML so + // every column carries an honest label + unit, and panel-backed columns + // (e.g. unique users) surface inline instead of behind a tab click. + // Disabled while a panel tab is active — the panel sort already owns + // the table and the aggregate columns are dashed out. + const customCols = !panelActive ? benchmark.ledgerColumns : undefined; + const panelById = useMemo( + () => new Map((benchmark.metricPanels ?? []).map((p) => [p.id, p])), + [benchmark.metricPanels], + ); + const secondary = customCols ? undefined : results[0]?.secondary?.label; + + // Resolve one custom column's value for a row. Slot columns read the + // repurposed headline slots; panel columns read the panel's values map + // (null when the provider returned no data for that metric this cycle). + const colValue = (r: ProviderResult, col: LedgerColumn): number | null => { + if (col.slot) return r.ms[col.slot]; + const v = panelById.get(col.panel ?? "")?.values[r.slug]; + return v != null && Number.isFinite(v) ? v : null; + }; + const colUnit = (col: LedgerColumn): string => + col.unit ?? + (col.panel ? (panelById.get(col.panel)?.unit ?? unit) : unit); // Detected from the first provider's results — if ANY provider declares // slot_p50/slot_p99 in its YAML queries, every row gets the column (with // "-" for providers that don't declare it). Used by Solana-native benches // where slot_delta is the canonical metric and ms is wall-clock derived. - const hasSlots = results.some((r) => r.slots != null); + const hasSlots = !customCols && results.some((r) => r.slots != null); // Drop unscored providers (availability=unavailable AND p50=0). They // stay in the underlying spec so /products/ pages still resolve // and SEO coverage holds, but they're noise in a "ranked by performance" @@ -110,12 +138,14 @@ export function LedgerTable({ benchmark, activePanel, topN }: Props) { Product - Latency aggregates + {customCols ? benchmark.metric : "Latency aggregates"} + + + {customCols ? customCols[0].label : "p50"} - p50 Reliability @@ -138,10 +168,23 @@ export function LedgerTable({ benchmark, activePanel, topN }: Props) { № Name - p50 - p90 - p99 - Mean + {customCols ? ( + customCols.map((c, idx) => ( + + {c.label} + + )) + ) : ( + <> + p50 + p90 + p99 + Mean + + )} Δ field Success 24h @@ -150,7 +193,11 @@ export function LedgerTable({ benchmark, activePanel, topN }: Props) { @@ -168,6 +215,10 @@ export function LedgerTable({ benchmark, activePanel, topN }: Props) { panelActive={panelActive} hasSecondary={!!secondary} hasSlots={hasSlots} + customCells={customCols?.map((c) => ({ + v: colValue(r, c), + unit: colUnit(c), + }))} series={ activePanel ? (activePanel.seriesByProvider?.[r.slug] ?? []) @@ -195,6 +246,7 @@ function Row({ panelActive, hasSecondary, hasSlots, + customCells, series, sparkMin, sparkMax, @@ -210,6 +262,9 @@ function Row({ panelActive: boolean; hasSecondary: boolean; hasSlots: boolean; + /** Custom-column mode (benchmark.ledgerColumns): one pre-resolved + * {value, unit} per declared column, replacing p50/p90/p99/Mean. */ + customCells?: { v: number | null; unit: string }[]; series: number[]; sparkMin: number; sparkMax: number; @@ -294,14 +349,18 @@ function Row({ {isOffline ? ( Awaiting next successful scrape ) : ( <> - {/* p50 with inline data bar */} + {/* Headline column with inline data bar */} - {fmtUnit(value, unit)} + {customCells + ? customCells[0].v != null + ? fmtUnit(customCells[0].v, customCells[0].unit) + : "-" + : fmtUnit(value, unit)} - - {panelActive ? "—" : fmtUnit(r.ms.p90, unit)} - - - {panelActive ? "—" : fmtUnit(r.ms.p99, unit)} - - - {panelActive ? "—" : fmtUnit(r.ms.mean, unit)} - + {customCells ? ( + customCells.slice(1).map((c, idx) => ( + + {c.v != null ? fmtUnit(c.v, c.unit) : "-"} + + )) + ) : ( + <> + + {panelActive ? "—" : fmtUnit(r.ms.p90, unit)} + + + {panelActive ? "—" : fmtUnit(r.ms.p99, unit)} + + + {panelActive ? "—" : fmtUnit(r.ms.mean, unit)} + + + )} {fieldValue > 0 ? `${deltaSign}${Math.abs(deltaPct).toFixed(0)}%` : "-"} diff --git a/src/lib/format.ts b/src/lib/format.ts index 38674b6c..7ba401a7 100644 --- a/src/lib/format.ts +++ b/src/lib/format.ts @@ -50,7 +50,7 @@ export function fmtUnit(value: number, unit: string) { if (abs < 0.001) return `$${value.toFixed(6)}`; if (abs < 0.01) return `$${value.toFixed(5)}`; if (abs < 1) return `$${value.toFixed(4)}`; - if (abs < 1000) return `$${value.toLocaleString(undefined, { maximumFractionDigits: 2 })}`; + if (abs < 1000) return `$${value.toLocaleString("en-US", { maximumFractionDigits: 2 })}`; return `$${formatCompactCount(value)}`; } if (value >= 1000) return `${(value / 1000).toFixed(2)} s`; @@ -102,7 +102,9 @@ function formatCompactCount(value: number): string { if (abs >= 1e9) return `${(value / 1e9).toFixed(2)}B`; if (abs >= 1e6) return `${(value / 1e6).toFixed(2)}M`; if (abs >= 1e4) return `${(value / 1e3).toFixed(1)}K`; - return value.toLocaleString(); + // Locale pinned: rendered both server-side and client-side ("use client" + // ledger). Browser-default locale produced "5 560,409" on fr-FR machines. + return value.toLocaleString("en-US", { maximumFractionDigits: 2 }); } /** Smart-precision percent formatter. picks decimals based on magnitude diff --git a/src/lib/spec-schema.ts b/src/lib/spec-schema.ts index 20aa3a11..b61d7e85 100644 --- a/src/lib/spec-schema.ts +++ b/src/lib/spec-schema.ts @@ -364,6 +364,34 @@ export const SpecSchema = z ) .max(8) .optional(), + + /** + * Optional relabeling of the ledger's aggregate columns. For benches + * whose unit has no percentile semantics (USD revenue leaderboards), + * the p50/p90/p99/mean slots are repurposed; declaring ledger_columns + * renders each column with an honest label and unit instead of the + * default latency headers. `slot` reads the provider's headline slot, + * `panel` reads the values of a metric_panels entry by id. The first + * column is the headline (sort key, data bar, mobile column). + */ + ledger_columns: z + .array( + z + .object({ + label: z.string().min(1).max(28), + slot: z.enum(["p50", "p90", "p99", "mean"]).optional(), + panel: z.string().min(1).max(40).optional(), + unit: z + .enum(["ms", "s", "pct", "bps", "count", "slots", "usd"]) + .optional(), + }) + .refine((c) => (c.slot != null) !== (c.panel != null), { + message: "ledger column must set exactly one of slot or panel", + }), + ) + .min(1) + .max(6) + .optional(), }) .strict() .superRefine((spec, ctx) => { @@ -382,6 +410,30 @@ export const SpecSchema = z "Benches declaring dimensions.region must provide rank_matrix_query so badge claims are scoped per region", }); } + + // A ledger column referencing a panel id that doesn't exist would + // silently render "-" for every provider. Refuse at validate time. + const panelIds = new Set((spec.metric_panels ?? []).map((p) => p.id)); + for (const [i, col] of (spec.ledger_columns ?? []).entries()) { + if (col.panel && !panelIds.has(col.panel)) { + ctx.addIssue({ + code: "custom", + path: ["ledger_columns", i, "panel"], + message: `Unknown metric_panels id "${col.panel}"`, + }); + } + } + // The ledger sorts, bars and badges off p50. The first displayed + // column must be that same number or the table reads as mis-sorted. + const firstCol = spec.ledger_columns?.[0]; + if (firstCol && firstCol.slot !== "p50") { + ctx.addIssue({ + code: "custom", + path: ["ledger_columns", 0, "slot"], + message: + "First ledger column must be slot p50 (the headline the table sorts by)", + }); + } }); export type Spec = z.infer; diff --git a/src/types/benchmark.ts b/src/types/benchmark.ts index 07cb8754..f5894060 100644 --- a/src/types/benchmark.ts +++ b/src/types/benchmark.ts @@ -198,4 +198,21 @@ export type Benchmark = { * below the main table. Populated by the spec loader from * `metric_panels` in the YAML. */ metricPanels?: MetricPanel[]; + /** Optional per-bench relabeling of the ledger's aggregate columns. + * Declared by benches whose unit has no percentile semantics (the + * p50/p90/p99/mean slots are repurposed, e.g. USD revenue leaderboards) + * so the table headers describe what each slot actually holds. The + * first column is the headline (sort key, data bar, mobile column). */ + ledgerColumns?: LedgerColumn[]; +}; + +export type LedgerColumn = { + label: string; + /** Reads the provider's headline slot value. */ + slot?: "p50" | "p90" | "p99" | "mean"; + /** Reads the per-provider values of a metric_panels entry by id. */ + panel?: string; + /** Display unit override; defaults to the panel's unit (panel columns) + * or the bench unit (slot columns). */ + unit?: "ms" | "s" | "pct" | "bps" | "count" | "slots" | "usd"; }; From 998498e84c5d409e3b0745f470ebddc8e256690a Mon Sep 17 00:00:00 2001 From: Florent Tapponnier Date: Wed, 10 Jun 2026 22:03:48 +0200 Subject: [PATCH 2/2] fix(data): harden cache layer per multi-agent review - prom client: global 8-slot concurrency cap, kills the cold-start burst that browned out Prom - spec: retry-once on null percentiles; quorum floored at 3 so a decimated field can still go live; cellRanks chain/region matching case-insensitive; marginals restricted to providers covering every cell of the collapsed dimension (Simpson's bias) - snapshot: ratchet compares live coverage (was dead code against the padded results array); KV read failure skips the write; after() keeps the lambda alive until the write lands - badge: escape scope label in SVG attrs, cache-control on the last bare 404 - products: collapsed badge claims require full declared cell coverage - also carries the bench cache key bumps for ledgerColumns --- src/app/api/badge/[slug]/[provider]/route.ts | 11 +- src/app/products/[slug]/page.tsx | 18 ++- src/lib/prometheus.ts | 30 +++++ src/lib/snapshot.ts | 50 ++++++-- src/lib/spec.ts | 127 +++++++++++++++---- 5 files changed, 193 insertions(+), 43 deletions(-) diff --git a/src/app/api/badge/[slug]/[provider]/route.ts b/src/app/api/badge/[slug]/[provider]/route.ts index 94657249..5f29beed 100644 --- a/src/app/api/badge/[slug]/[provider]/route.ts +++ b/src/app/api/badge/[slug]/[provider]/route.ts @@ -234,7 +234,12 @@ export async function GET( .join(" · "); } else { r = rankOf(b.results, provider, b.higherIsBetter); - if (!r) return new NextResponse("not found", { status: 404 }); + if (!r) { + return new NextResponse("not found", { + status: 404, + headers: { "cache-control": "public, s-maxage=60" }, + }); + } // Hint the aggregate scope when the bench declares dimensions so // embedders can read it. Benches without dimensions get no scope // label (it would be noise). @@ -281,7 +286,9 @@ export async function GET( // Scope marker: small caps tspan appended to the figure line, so the // badge height stays constant whether or not a scope is present. - const scopeAriaSuffix = scopeLabel ? ` (${scopeLabel})` : ""; + // Escaped at construction: scopeLabel comes from YAML dimension labels + // and lands in XML attribute/text contexts below. + const scopeAriaSuffix = scopeLabel ? ` (${escapeXml(scopeLabel)})` : ""; const scopeTspan = scopeLabel ? `${escapeXml(scopeLabel.toUpperCase())}` : ""; diff --git a/src/app/products/[slug]/page.tsx b/src/app/products/[slug]/page.tsx index 009c86c5..e9977e01 100644 --- a/src/app/products/[slug]/page.tsx +++ b/src/app/products/[slug]/page.tsx @@ -131,7 +131,16 @@ export default async function ProviderPage({ (k) => cellRanks[k][0]?.slug.toLowerCase() === me, ), ); - if (wonKeys.size === finestKeys.length) { + // Collapsed claims require FULL declared coverage, not just the + // cells that happen to have data this cycle. Without this, a + // degraded matrix (one surviving cell) would mint an unscoped + // global "#1" from a single win. + const expectedCells = + Math.max(chainDims.length, 1) * regionDims.length; + if ( + wonKeys.size === finestKeys.length && + finestKeys.length === expectedCells + ) { badgeCards.push({ key: benchSlug, title, benchSlug }); continue; } @@ -141,7 +150,9 @@ export default async function ProviderPage({ const covered = new Set(); for (const c of chainDims) { const row = finestKeys.filter((k) => chainOf(k) === c.value); - if (row.length === 0 || !row.every((k) => wonKeys.has(k))) continue; + // Row collapse only when every DECLARED region reported a cell + // for this chain and the provider won them all. + if (row.length !== regionDims.length || !row.every((k) => wonKeys.has(k))) continue; badgeCards.push({ key: `${benchSlug}-${c.value}`, title, @@ -152,7 +163,8 @@ export default async function ProviderPage({ } for (const r of regionDims) { const col = finestKeys.filter((k) => regionOf(k) === r.value); - if (col.length === 0 || !col.every((k) => wonKeys.has(k))) continue; + const expectedCols = Math.max(chainDims.length, 1); + if (col.length !== expectedCols || !col.every((k) => wonKeys.has(k))) continue; if (col.every((k) => covered.has(k))) continue; badgeCards.push({ key: `${benchSlug}-r-${r.value}`, diff --git a/src/lib/prometheus.ts b/src/lib/prometheus.ts index 92afbd02..6d38cc41 100644 --- a/src/lib/prometheus.ts +++ b/src/lib/prometheus.ts @@ -163,6 +163,13 @@ export class Prometheus { // / metadata / ULA / CGNAT. await assertPublicHost(url); + // Global concurrency cap. At cold start / build every bench loads at + // once, which used to fire hundreds of 24h-window quantile queries + // within seconds and brown out the single Prom instance (providers + // timing out → partial leaderboards). Queueing here keeps the burst + // at a level Prom absorbs; total wall time barely moves because Prom + // was serializing on CPU anyway. + await acquireQuerySlot(); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), DEFAULT_TIMEOUT_MS); try { @@ -187,10 +194,33 @@ export class Prometheus { return json.data; } finally { clearTimeout(timeout); + releaseQuerySlot(); } } } +/** Module-level semaphore for fetchEnvelope. 8 concurrent queries keeps + * a Railway-sized Prom responsive while ~30 benches load in parallel. */ +const MAX_CONCURRENT_QUERIES = 8; +let activeQueries = 0; +const queryWaiters: (() => void)[] = []; + +function acquireQuerySlot(): Promise { + if (activeQueries < MAX_CONCURRENT_QUERIES) { + activeQueries++; + return Promise.resolve(); + } + return new Promise((resolve) => queryWaiters.push(resolve)); +} + +function releaseQuerySlot(): void { + const next = queryWaiters.shift(); + // Hand the slot directly to the next waiter (activeQueries unchanged) + // or free it when the queue is empty. + if (next) next(); + else activeQueries--; +} + /** PromQL built-in functions and keywords we should skip when scanning * for the first raw metric name in a query string. Updated from the * Prometheus 2.49 function reference - any identifier here is guaranteed diff --git a/src/lib/snapshot.ts b/src/lib/snapshot.ts index 4c8c4a95..57d6de8d 100644 --- a/src/lib/snapshot.ts +++ b/src/lib/snapshot.ts @@ -32,6 +32,7 @@ * outage. */ +import { after } from "next/server"; import { z } from "zod"; import type { Benchmark, @@ -123,17 +124,30 @@ const SNAPSHOT_GUARD_WINDOW_MS = 2 * 60 * 60 * 1000; */ export function writeSnapshot(slug: string, payload: SnapshotPayload): void { if (!isConfigured()) return; - void (async () => { + const work = (async () => { try { const existing = await readSnapshotWithAge(slug); + // KV unreachable (as opposed to "no snapshot yet"): skip the write + // entirely. A degraded render slipping past the guards precisely + // when the infra is under stress is the scenario the guards exist + // for, and a healthy cycle lands 60s later anyway. + if (existing === "error") { + console.warn(`snapshot.write skipped for ${slug}: KV read failed`); + return; + } const merged: SnapshotPayload = { ...payload }; if (existing) { - if ( - existing.ageMs < SNAPSHOT_GUARD_WINDOW_MS && - existing.payload.results.length > payload.results.length - ) { + // Compare actual live coverage, not array length: the rendered + // bench pads `results` with an "unavailable" row for every + // declared provider, so lengths are always equal by construction. + const liveCount = (rs: ProviderResult[]) => + rs.filter((r) => r.availability !== "unavailable" && r.ms.p50 > 0) + .length; + const newLive = liveCount(payload.results); + const oldLive = liveCount(existing.payload.results); + if (existing.ageMs < SNAPSHOT_GUARD_WINDOW_MS && oldLive > newLive) { console.warn( - `snapshot.write skipped for ${slug}: render has ${payload.results.length} providers, snapshot has ${existing.payload.results.length}`, + `snapshot.write skipped for ${slug}: render has ${newLive} live providers, snapshot has ${oldLive}`, ); return; } @@ -162,6 +176,15 @@ export function writeSnapshot(slug: string, payload: SnapshotPayload): void { ); } })(); + // Vercel can freeze the lambda as soon as the response is sent; an + // unawaited promise then silently never completes. after() keeps the + // function alive until the write lands. Falls back to fire-and-forget + // outside a request scope (build-time prerender, tests). + try { + after(work); + } catch { + void work; + } } /** @@ -179,14 +202,17 @@ export async function readSnapshot( slug: string, ): Promise { const hit = await readSnapshotWithAge(slug); - return hit ? hit.payload : null; + return hit && hit !== "error" ? hit.payload : null; } -/** Same as readSnapshot but exposes the snapshot's age, used by the - * write-path degradation guards. */ +/** Same as readSnapshot but exposes the snapshot's age. Used by the + * write-path degradation guards, which need to distinguish "no usable + * snapshot exists" (null → write proceeds) from "KV is unreachable" + * ("error" → write is skipped so a degraded render can't slip past the + * guards while the infra is down). */ async function readSnapshotWithAge( slug: string, -): Promise<{ payload: SnapshotPayload; ageMs: number } | null> { +): Promise<{ payload: SnapshotPayload; ageMs: number } | null | "error"> { if (!isConfigured()) return null; try { const res = await fetch( @@ -202,7 +228,7 @@ async function readSnapshotWithAge( ); if (!res.ok) { console.warn(`[DRAFT-TRACE] kv_http slug=${slug} status=${res.status}`); - return null; + return "error"; } const env = (await res.json()) as { result?: string | null }; if (!env.result) { @@ -253,7 +279,7 @@ async function readSnapshotWithAge( ? "kv_timeout" : "kv_neterr"; console.warn(`[DRAFT-TRACE] ${tag} slug=${slug} err=${msg}`); - return null; + return "error"; } } diff --git a/src/lib/spec.ts b/src/lib/spec.ts index 9db75df2..96d5b9b1 100644 --- a/src/lib/spec.ts +++ b/src/lib/spec.ts @@ -105,7 +105,8 @@ const loadBenchmarkUnfilteredCached = unstable_cache( // v6: added cellRanks (exact chain × region rankings from // rank_matrix_query). Cached objects from v5 deploys lack the field, // which made region-scoped badge URLs 404 after the deploy. - ["bench-unfiltered-v6"], + // v7: added ledgerColumns (per-bench ledger column relabeling). + ["bench-unfiltered-v7"], { revalidate: 60, tags: ["benchmarks"] }, ); @@ -167,7 +168,8 @@ const loadAllBenchmarksCached = unstable_cache( // outer cache can keep serving v5-era benchmarks (no providersPerChain) // even after the inner cache is fresh. // v8: bumped with bench-unfiltered-v6 (cellRanks) for the same reason. - ["all-benchmarks-v8"], + // v9: bumped with bench-unfiltered-v7 (ledgerColumns). + ["all-benchmarks-v9"], { revalidate: 60, tags: ["benchmarks"] }, ); export const loadAllBenchmarks = cache(loadAllBenchmarksCached); @@ -210,7 +212,8 @@ const loadBenchmarkFiltered = unstable_cache( } return bench; }, - ["bench-filters-v3"], + // v4: bumped with bench-unfiltered-v7 (ledgerColumns). + ["bench-filters-v4"], { revalidate: 60, tags: ["benchmarks"] } ); @@ -314,6 +317,7 @@ function buildEditorial( findings: spec.findings, source: spec.source, dimensions: spec.dimensions, + ledgerColumns: spec.ledger_columns, }; } @@ -530,11 +534,18 @@ async function tryLoadCellRanks( const slugByLower = new Map( spec.providers.map((p) => [p.slug.toLowerCase(), p.slug] as const), ); - const chainValues = new Set( - (spec.dimensions?.chain ?? []).map((c) => c.value).filter((v) => v !== "all"), + // Canonical dimension value by lowercase, so a harness emitting + // `chain="Base"` still maps onto the declared `base` value instead + // of silently dropping the cell. + const chainByLower = new Map( + (spec.dimensions?.chain ?? []) + .filter((c) => c.value !== "all") + .map((c) => [c.value.toLowerCase(), c.value] as const), ); - const regionValues = new Set( - (spec.dimensions?.region ?? []).map((r) => r.value).filter((v) => v !== "all"), + const regionByLower = new Map( + (spec.dimensions?.region ?? []) + .filter((r) => r.value !== "all") + .map((r) => [r.value.toLowerCase(), r.value] as const), ); // key → provider slug → samples (averaged if the grouping left @@ -543,10 +554,16 @@ async function tryLoadCellRanks( for (const sample of res.result) { const slug = slugByLower.get((sample.metric.provider ?? "").toLowerCase()); if (!slug) continue; - const chain = chainValues.size > 0 ? sample.metric.chain : undefined; - const region = regionValues.size > 0 ? sample.metric.region : undefined; - if (chainValues.size > 0 && (!chain || !chainValues.has(chain))) continue; - if (regionValues.size > 0 && (!region || !regionValues.has(region))) continue; + const chain = + chainByLower.size > 0 + ? chainByLower.get((sample.metric.chain ?? "").toLowerCase()) + : undefined; + const region = + regionByLower.size > 0 + ? regionByLower.get((sample.metric.region ?? "").toLowerCase()) + : undefined; + if (chainByLower.size > 0 && !chain) continue; + if (regionByLower.size > 0 && !region) continue; const v = Number(sample.value[1]); if (!Number.isFinite(v) || v <= 0) continue; const key = `${chain ?? "all"}|${region ?? "all"}`; @@ -571,22 +588,55 @@ async function tryLoadCellRanks( for (const [key, cell] of acc) out[key] = sortCell(cell); // Marginals, only when both dimensions exist in the finest cells. - if (chainValues.size > 0 && regionValues.size > 0) { - const marginal = new Map>(); - for (const [key, cell] of acc) { + // A provider only enters a marginal if it covers EVERY cell of the + // collapsed dimension that exists for that row/column. Without this, + // a provider measured only from its fastest region wins the + // `|all` average by omission (Simpson's bias), and the badge + // for "leads chain X" disagrees with the per-cell wins that earned it. + if (chainByLower.size > 0 && regionByLower.size > 0) { + const regionsOfChain = new Map>(); + const chainsOfRegion = new Map>(); + for (const key of acc.keys()) { const [chain, region] = key.split("|"); - for (const [slug, vals] of cell) { - const v = mean(vals); - for (const mKey of [`${chain}|all`, `all|${region}`]) { - const mCell = marginal.get(mKey) ?? new Map(); - const mVals = mCell.get(slug) ?? []; - mVals.push(v); - mCell.set(slug, mVals); - marginal.set(mKey, mCell); + (regionsOfChain.get(chain) ?? regionsOfChain.set(chain, new Set()).get(chain)!).add(region); + (chainsOfRegion.get(region) ?? chainsOfRegion.set(region, new Set()).get(region)!).add(chain); + } + const marginalFor = ( + groups: Map>, + keyOf: (group: string, member: string) => string, + mKeyOf: (group: string) => string, + ) => { + for (const [group, members] of groups) { + const cell = new Map(); + // Providers present in every member cell of the group. + let eligible: Set | undefined; + for (const member of members) { + const slugs = new Set(acc.get(keyOf(group, member))?.keys() ?? []); + eligible = eligible + ? new Set([...eligible].filter((s) => slugs.has(s))) + : slugs; + } + for (const slug of eligible ?? []) { + const vals: number[] = []; + for (const member of members) { + const v = acc.get(keyOf(group, member))?.get(slug); + if (v) vals.push(mean(v)); + } + if (vals.length > 0) cell.set(slug, [mean(vals)]); } + if (cell.size > 0) out[mKeyOf(group)] = sortCell(cell); } - } - for (const [key, cell] of marginal) out[key] = sortCell(cell); + }; + marginalFor( + regionsOfChain, + (chain, region) => `${chain}|${region}`, + (chain) => `${chain}|all`, + ); + marginalFor( + chainsOfRegion, + (region, chain) => `${chain}|${region}`, + (region) => `all|${region}`, + ); } return out; } catch (e) { @@ -681,10 +731,12 @@ async function tryLoadLive( const q = p.queries; if (!q) return null; - const [p50, p90, p99, mean, success, sampleSize, slotP50, slotP99] = await Promise.all([ + let [p50, p90, p99] = await Promise.all([ q.p50 ? prom.scalar(q.p50) : Promise.resolve(null), q.p90 ? prom.scalar(q.p90) : Promise.resolve(null), q.p99 ? prom.scalar(q.p99) : Promise.resolve(null), + ]); + const [mean, success, sampleSize, slotP50, slotP99] = await Promise.all([ q.mean ? prom.scalar(q.mean) : Promise.resolve(null), q.success ? prom.scalar(q.success) : Promise.resolve(null), q.sample_size ? prom.scalar(q.sample_size) : Promise.resolve(null), @@ -692,6 +744,23 @@ async function tryLoadLive( q.slot_p99 ? prom.scalar(q.slot_p99) : Promise.resolve(null), ]); + // One retry on the load-bearing percentiles. A null here is either + // "provider has no data" (retry returns null again, harmless) or a + // transient Prom timeout under burst load (retry usually lands now + // that the concurrency cap has drained the burst). This is the + // difference between a provider flickering off the leaderboard for + // 60s and a stable board. + if ((p50 == null || p90 == null || p99 == null) && q.p50 && q.p90 && q.p99) { + const [r50, r90, r99] = await Promise.all([ + p50 == null ? prom.scalar(q.p50) : Promise.resolve(p50), + p90 == null ? prom.scalar(q.p90) : Promise.resolve(p90), + p99 == null ? prom.scalar(q.p99) : Promise.resolve(p99), + ]); + p50 = r50; + p90 = r90; + p99 = r99; + } + // If a provider has no data for the current filter (e.g. Jupiter on // BNB Chain when Jupiter is Solana-only), skip it instead of failing // the whole benchmark. The page still renders with the providers @@ -798,7 +867,13 @@ async function tryLoadLive( // legitimately has fewer providers than the spec declares. if (!isFiltered) { const declared = spec.providers.filter((p) => p.queries).length; - const quorum = Math.ceil(declared / 2); + // Floor at 3 so a bench whose field is legitimately decimated (most + // declared providers dead for days) can still go live with its + // survivors instead of drafting forever once the KV snapshot ages + // out. 3 is still enough to make a leaderboard meaningful, and the + // brownout scenario this guards against (1-2 stragglers passing + // while the rest time out) stays caught. + const quorum = Math.min(Math.ceil(declared / 2), 3); if (liveResults.length < quorum) { console.warn( `bench quorum fail: ${spec.slug} live=${liveResults.length}/${declared} → keeping previous render`,