diff --git a/benchmarks/evm-quote-latency.yml b/benchmarks/evm-quote-latency.yml index ddafbf19..0e0b39e2 100644 --- a/benchmarks/evm-quote-latency.yml +++ b/benchmarks/evm-quote-latency.yml @@ -78,6 +78,10 @@ dimensions: - { value: eu-west, label: EU West } - { value: sgp, label: Singapore } +# Exact per-cell (chain x region) rankings for scoped badges. One query, +# one sample per (provider, chain, region) cell. +rank_matrix_query: histogram_quantile(0.50, sum by (provider, chain, region, le) (rate(evm_swap_quote_latency_ms_bucket[24h]))) + providers: - slug: mobula name: Mobula diff --git a/benchmarks/oracle-deviation.yml b/benchmarks/oracle-deviation.yml index 2fce93e0..8c9391f7 100644 --- a/benchmarks/oracle-deviation.yml +++ b/benchmarks/oracle-deviation.yml @@ -69,7 +69,7 @@ methodology: - "Chainlink round age. The on-chain `updatedAt` timestamp from `latestRoundData()` is published separately as `ocb_oracle_last_round_age_seconds{source=\"chainlink\", pair}`. Chainlink updates only on deviation (typically 0.25-0.5% for blue chips) or heartbeat (~1 h), so a 30-minute age on a quiet ETH minute is normal; a 2-hour age on SOL during a volatile minute is the actual signal." - "USDT ≈ USD assumption. Binance only quotes USDT pairs (BTCUSDT, etc.). We treat USDT as ≈ USD ± 10 bps drift, which is acceptable for a bench whose alert floor is ≥ 10 bps. A real USDT depeg would surface as Binance drifting from the other three sources for *every* pair simultaneously, exactly what we want this bench to flag, not hide." - "MATIC → POL migration. Polygon migrated MATIC → POL 1:1 on Sep 4 2024. The Chainlink mainnet feed contract is still named MATIC/USD but on-chain `description()` confirms it tracks the POL token; Pyth renamed the feed to POL/USD; Coinbase delisted MATIC-USD and only lists POL-USD; Binance kept MATICUSDT as a frozen historical pair AND lists POLUSDT. We point Binance at POLUSDT so all four sources track the same underlying asset. Bench label is kept as `pair=\"MATIC/USD\"` for query continuity." - - "Time-aligned deviation (canonical headline). For every pair of sources (a, b) we anchor on the more recent of their two SourceTSs (Chainlink's on-chain `updatedAt` for Chainlink, fetch time for the continuously-updating sources) and look up the older source's price in a 30-minute per-source rolling history at the anchor moment. The result is published as `ocb_oracle_deviation_at_oracle_ts_pct` and feeds the leaderboard headline `ocb_oracle_max_deviation_pct`. This eliminates the artifact where Chainlink's heartbeat lag inflates 'deviation' by the market's drift between its updates — a researcher grading oracle quality wants to compare Chainlink's price against the market at Chainlink's own updatedAt, not at the harness's fetch instant. The legacy fetch-time gauge (`ocb_oracle_deviation_pct`, aliased as `ocb_oracle_deviation_at_fetch_ts_pct`) is preserved unchanged for backward compatibility. Alignment misses (no history sample within ±10s of the anchor) are counted as `ocb_oracle_alignment_miss_total`. Methodology recommendation from Coinpaprika data team review; matches the convention published by Chaos Labs and Risk DAO oracle risk reports." + - "Time-aligned deviation (canonical headline). For every source pair we anchor on the more recent SourceTS (Chainlink's on-chain `updatedAt`, fetch time for continuous sources) and look up the older source's price in a 30-minute rolling history at that moment. Published as `ocb_oracle_deviation_at_oracle_ts_pct`, feeding `ocb_oracle_max_deviation_pct`. Removes Chainlink heartbeat-lag inflating deviation; alignment misses count as `ocb_oracle_alignment_miss_total`." - "Excluded by design. Redstone (push-pull, no continuous gauge to scrape without integration contract), Uniswap V3 TWAP (per-pool integration + derivation of same CEX prints), DIA (smaller footprint than the four kept), and aggregator-of-aggregators (CoinGecko, CMC, DefiLlama), re-aggregating already-aggregated data adds latency and hides per-source disagreement." findings: diff --git a/benchmarks/rpc-capabilities.yml b/benchmarks/rpc-capabilities.yml index d791c114..ea579bac 100644 --- a/benchmarks/rpc-capabilities.yml +++ b/benchmarks/rpc-capabilities.yml @@ -124,6 +124,12 @@ dimensions: - { value: eu-west, label: EU-West } - { value: sgp, label: Singapore } +# One instant query feeding the exact per-cell (chain x region) rankings +# behind scoped badges and leadership claims. Cross-region averages hide +# region-restricted leaders (dRPC wins every chain from sgp only), so +# badge scoping reads these cells instead of the aggregate ranks. +rank_matrix_query: avg by (provider, chain, region) (quantile_over_time(0.50, rpc_latency_milliseconds[24h])) + # Real metrics emitted by the rpc-capabilities harness: # rpc_latency_milliseconds{provider, chain} gauge # rpc_latency_milliseconds_histogram{provider, chain} histogram diff --git a/benchmarks/solana-dex-quote-latency.yml b/benchmarks/solana-dex-quote-latency.yml index 248612ec..5a1dfb26 100644 --- a/benchmarks/solana-dex-quote-latency.yml +++ b/benchmarks/solana-dex-quote-latency.yml @@ -76,6 +76,10 @@ dimensions: - { value: eu-west, label: EU-West } - { value: sgp, label: Singapore } +# Exact per-region rankings for scoped badges. One query, one sample per +# (provider, region) cell. +rank_matrix_query: histogram_quantile(0.50, sum by (provider, region, le) (rate(solana_quote_latency_ms_bucket[24h]))) + faq: - q: "Which Solana DEX has the fastest quote API right now?" a: "{{best_name}} currently returns quotes the fastest at {{best_p50}} (p50, 24 h) across {{count}} measured providers. The leaderboard re-sorts every 60 seconds against fresh samples and rotates the target token each tick so no provider can serve from edge cache. The answer reflects 24 hours of measured latency across us-east, eu-west and sgp, not a marketing-page claim." diff --git a/benchmarks/stablecoin-peg.yml b/benchmarks/stablecoin-peg.yml index 89dd6e18..58a074ac 100644 --- a/benchmarks/stablecoin-peg.yml +++ b/benchmarks/stablecoin-peg.yml @@ -58,12 +58,12 @@ methodology: - "Sources per stable, USDC: Binance USDC/USDT (USDT-anchored, secondary), Kraken USDCUSD, Bitstamp usdcusd. USDT: Kraken USDTUSD, Bitstamp usdtusd. FDUSD: Binance FDUSDUSDT (USDT-anchored, secondary). USDe: Binance USDEUSDT (USDT-anchored, secondary). DAI: Curve 3pool `get_dy` forward (USDC → DAI) and reverse (DAI → USDC)." - "Cadence: 5 seconds for CEX REST tickers, 12 seconds for Curve `get_dy` on-chain (matches Ethereum block time). Per-venue samples are grouped into 60-second buckets for the liquidity-weighted median aggregation." - "Aggregation: per-minute liquidity-weighted median across USD-quoted venues (`Quote = QuoteUSD`). Weight is 24 h USD volume estimate. The aggregated price drives `peg_deviation_bps` (per-minute median, retained for backward compatibility) and `peg_deviation_bps_histogram`." - - "Primary headline: `peg_deviation_worst_bps` — the per-minute MAX |price − $1.00| across every venue sample in the 60-second bucket, in basis points. The leaderboard ranks stables on `quantile_over_time(0.99, peg_deviation_worst_bps[24h])`. Switching from per-minute median to per-minute max surfaces the sub-minute depeg wicks (e.g. a 5-second print to $0.92 on one venue) that the median would smooth into invisibility — matching the OHLC convention used by every TradFi reference rate. The companion gauges `peg_minute_min_bps`, `peg_minute_open_bps` and `peg_minute_close_bps` expose the full bar shape so consumers can reconstruct the within-minute distribution." + - "Primary headline: `peg_deviation_worst_bps`, the per-minute MAX |price - $1.00| across all venue samples in the 60-second bucket, in basis points. The leaderboard ranks stables on `quantile_over_time(0.99, peg_deviation_worst_bps[24h])`. Per-minute max surfaces sub-minute depeg wicks that a median would smooth away, matching the OHLC convention of TradFi reference rates. Gauges `peg_minute_min_bps`, `peg_minute_open_bps`, `peg_minute_close_bps` expose the bar shape." - "Cross-venue gap (OCB-only number): `max(price across USD-quoted venues), min(price)`, per minute, in basis points. Surfaces the windows where Coinbase quotes $1.0003 while Kraken quotes $0.9978. Stored as `peg_cross_venue_gap_bps`." - "USDT-anchored secondary metric: Binance USDC/USDT, FDUSD/USDT and USDE/USDT are exposed on `peg_deviation_usdt_anchored_bps{venue}` separately so the USD-anchored primary leaderboard is not contaminated by USDT's own peg deviation." - "Time outside band: total seconds in the trailing 24 h during which the per-minute aggregated price fell outside [0.995, 1.005] (±50 bps). Split into `peg_time_below_peg_24h_seconds` (< 0.995) and `peg_time_above_peg_24h_seconds` (> 1.005) because Circle redemption only clears above-peg, so the direction tells you which failure mode is active." - "Depeg event flag: binary `peg_depeg_event_flag` set to 1 when the per-minute aggregated price has been outside [0.97, 1.03] for ≥5 consecutive minutes; cleared after 30 minutes back inside. Conservative so it does not flap during normal stress events." - - "Outlier rule (multi-venue consensus): a single sample more than 2% off peg is kept only when at least one OTHER venue has also been outside the same band in the same direction within the last 30 seconds. Single-venue glitches (one CEX returns a stale or fat-finger print while every other venue is at $1.00) are dropped as `dropped_isolated`. Multi-venue confirmation (Kraken AND Bitstamp both at $0.87 = real depeg) is kept as `kept_corroborated` so the percentile metric surfaces the event. Replaces the previous flat 20% drop / 10% cap which would have clipped USDC at $0.87 during the March 2023 SVB depeg to $0.90, erasing the event in the percentile metric. Sanity floor: anything more than 50% off peg is treated as a parser bug and dropped regardless. Methodology recommendation from Coinpaprika data team review." + - "Outlier rule (multi-venue consensus): a sample more than 2% off peg is kept only when at least one other venue was outside the same band in the same direction within the last 30 seconds. Isolated single-venue glitches are dropped as `dropped_isolated`; corroborated moves (Kraken AND Bitstamp both at $0.87) are kept as `kept_corroborated`, so a real depeg like USDC during SVB in March 2023 survives into the percentile metric. Anything more than 50% off peg is treated as a parser bug and dropped." - "Excluded by design: aggregator-only prices (CoinGecko, Coinmarketcap, DefiLlama) because they are themselves liquidity-weighted medians of the venues we already poll directly. Algo-stables that have already failed (UST, USDR) are out of scope; the bench tracks live, currently-redeemable stables." findings: diff --git a/src/app/about/page.tsx b/src/app/about/page.tsx index 07bda322..202c7bd5 100644 --- a/src/app/about/page.tsx +++ b/src/app/about/page.tsx @@ -52,7 +52,29 @@ export default function AboutPage() { 13 live benchmarks. ~150 (provider × chain) probe pairs. Every metric is queryable on the public Prometheus and reproducible from the harness source.

- + +

+ OpenChainBench is funded by Mobula because we needed honest infrastructure benchmark data internally and saw no neutral source existed. The site is open-sourced because the industry needs it. +

+

+ Mobula competes in five live benchmarks: aggregator head lag, bridge fee, bridge quote latency, network coverage and metadata coverage. Mobula currently leads aggregator head lag and bridge quote latency. This is a real conflict of interest and we want it visible above the fold, not buried. +

+

+ Three safeguards stand against the conflict: +

+ +

+ We do not run a paid tier. We do not sell ranking slots. We do not take provider sponsorship in exchange for inclusion. There is no token. If you spot any deviation from this policy, file a private security advisory and we will treat it as the integrity incident it would be. +

+

+ Hosting and infrastructure costs are paid by Mobula. The site runs on Vercel; harnesses run on Railway. We are open to grant funding that preserves editorial independence, but we will not accept funding from any party we benchmark. +

+ +

Every benchmark is a YAML spec plus a harness. The spec describes what to measure, which providers, which Prometheus queries hold the numbers; the harness runs continuously on Railway and exposes those metrics. A single shared Prometheus scrapes every harness; the site queries Prometheus directly and re-renders every minute. Every provider is rendered with equal visual weight. readers do their own ranking.

@@ -62,7 +84,7 @@ export default function AboutPage() { walks through the steps. New providers, new metrics, new chains. all welcome via pull request.

- +
  • ·Read the{" "}live benchmarks{" "}.
  • ·Reproduce any number. the{" "}methodology{" "}page tells you how.
    • @@ -70,7 +92,7 @@ export default function AboutPage() {
  • ·Discuss ideas in{" "}GitHub Discussions{" "}or follow{" "}@OpenChainBench.
- +

See a number you can't reproduce? File a{" "} diff --git a/src/app/api/badge/[slug]/[provider]/route.ts b/src/app/api/badge/[slug]/[provider]/route.ts index d54fbfa7..8d1ffedc 100644 --- a/src/app/api/badge/[slug]/[provider]/route.ts +++ b/src/app/api/badge/[slug]/[provider]/route.ts @@ -1,24 +1,27 @@ /** * Embeddable SVG badge. One badge per (benchmark, provider). * - * Endpoint shape: /api/badge//?chain= + * Endpoint shape: + * /api/badge//?chain=®ion= * * Returns an SVG showing the provider's current rank + headline figure * on that bench. Cache-Control is short so the figure refreshes within * a few minutes of a new run. * - * The optional `?chain=` query param scopes the rank computation to a - * single chain (e.g. `?chain=solana`). When present, the badge: - * - computes rank within the providers that have a measurement on - * that chain (read from `benchmark.bestPerChain`). - * - prints the chain label as a subscript on the SVG so embedders - * don't mistakenly broadcast a chain-restricted "#1" as a global - * finish (e.g. a Solana-only provider being aggregate #1 mechanically - * on a cross-chain bench). + * The optional `?chain=` / `?region=` query params scope the rank + * computation. When the bench carries exact per-cell rankings (from its + * `rank_matrix_query`), the rank comes from the matching cell, including + * the combined scope `?chain=bnb®ion=sgp`, and the scope labels are + * printed as a subscript on the SVG so embedders don't broadcast a + * chain- or region-restricted "#1" as a global finish (e.g. dRPC leading + * only from Singapore reading as the worldwide chain leader). * - * When NO `chain` is provided, the badge falls back to the unfiltered - * aggregate AND adds an "all chains" textual hint to the SVG so the - * scope of the rank is visible at a glance. + * `?chain=` without cell data falls back to the legacy bestPerChain + * approximation. `?region=` requires cell data (404 otherwise). + * + * When NO scope is provided, the badge falls back to the unfiltered + * aggregate AND adds an "all chains" / "all regions" textual hint to the + * SVG so the scope of the rank is visible at a glance. */ import { type NextRequest, NextResponse } from "next/server"; @@ -33,11 +36,12 @@ export const revalidate = 300; type Params = { slug: string; provider: string }; -const H = 36; +const H = 44; // Width is fixed but generous so most benchmark titles fit without // truncation. Anything over ~32 chars gets ellipsis. const W = 360; -const LEFT_W = 78; +// Text column starts right of the spinning logo sphere. +const TEXT_X = 48; const TITLE_MAX = 32; function rankOf( @@ -102,6 +106,27 @@ function rankOfChain( }; } +/** + * Exact scoped rank from the bench's per-cell rankings (populated by + * `rank_matrix_query` in the spec). Cell keys are `|` with + * "all" standing in for an unscoped side (derived marginals included). + * Returns null when the bench has no cell data, the cell is empty this + * cycle, or the provider isn't ranked in it. + */ +function rankOfCell( + b: Benchmark, + providerSlug: string, + chain: string | null, + region: string | null, +): { rank: number; total: number; value: number } | null { + const cell = b.cellRanks?.[`${chain ?? "all"}|${region ?? "all"}`]; + if (!cell || cell.length === 0) return null; + const lower = providerSlug.toLowerCase(); + const idx = cell.findIndex((e) => e.slug.toLowerCase() === lower); + if (idx === -1) return null; + return { rank: idx + 1, total: cell.length, value: cell[idx].p50 }; +} + function valueSuffix(unit: string): string { if (unit === "count") return "(24h)"; if (unit === "pct" || unit === "bps") return "(24h avg)"; @@ -118,6 +143,11 @@ function chainLabel(b: Benchmark, chain: string): string { return b.dimensions?.chain?.find((c) => c.value === chain)?.label ?? chain; } +/** Returns the human label for a region value from the bench's spec. */ +function regionLabel(b: Benchmark, region: string): string { + return b.dimensions?.region?.find((r) => r.value === region)?.label ?? region; +} + export async function GET( req: NextRequest, { params }: { params: Promise }, @@ -140,44 +170,69 @@ export async function GET( }); } - // Chain scoping. Query param is normalized to lowercase and validated - // against the bench's declared chain dimensions; an unknown chain is - // treated as a 400 rather than silently falling back to "all", so an - // embedder who mistypes can fix it instead of shipping a misleading - // unfiltered figure under a chain badge. + // Scope params are normalized to lowercase and validated against the + // bench's declared dimensions; an unknown value is treated as a 400 + // rather than silently falling back to "all", so an embedder who + // mistypes can fix it instead of shipping a misleading unfiltered + // figure under a scoped badge. The canonical dimension value (not the + // raw param) feeds the cell lookup. const url = new URL(req.url); - const chainParam = url.searchParams.get("chain")?.toLowerCase().trim() || null; - if (chainParam) { - const known = b.dimensions?.chain?.some( - (c) => c.value.toLowerCase() === chainParam, - ); - if (!known) { - return new NextResponse("unknown chain", { - status: 400, - headers: { "cache-control": "public, s-maxage=60" }, - }); - } + const rawChain = url.searchParams.get("chain")?.toLowerCase().trim() || null; + const rawRegion = url.searchParams.get("region")?.toLowerCase().trim() || null; + const chainParam = rawChain + ? (b.dimensions?.chain?.find((c) => c.value.toLowerCase() === rawChain) + ?.value ?? null) + : null; + const regionParam = rawRegion + ? (b.dimensions?.region?.find((d) => d.value.toLowerCase() === rawRegion) + ?.value ?? null) + : null; + if ((rawChain && !chainParam) || (rawRegion && !regionParam)) { + return new NextResponse(rawChain && !chainParam ? "unknown chain" : "unknown region", { + status: 400, + headers: { "cache-control": "public, s-maxage=60" }, + }); } let r: { rank: number; total: number; value: number } | null; let scopeLabel: string; - if (chainParam) { - const scoped = rankOfChain(b, provider, chainParam); - if (!scoped) { + if (chainParam || regionParam) { + const cell = rankOfCell(b, provider, chainParam, regionParam); + if (cell) { + r = cell; + } else if (chainParam && !regionParam) { + // Legacy approximation for chain-dimensioned benches without a + // rank_matrix_query in their spec. + const scoped = rankOfChain(b, provider, chainParam); + if (!scoped) { + return new NextResponse("not found", { status: 404 }); + } + r = { rank: scoped.rank, total: scoped.total, value: scoped.value }; + } else { return new NextResponse("not found", { status: 404 }); } - r = { rank: scoped.rank, total: scoped.total, value: scoped.value }; - scopeLabel = chainLabel(b, chainParam); + scopeLabel = [ + chainParam ? chainLabel(b, chainParam) : null, + regionParam ? regionLabel(b, regionParam) : null, + ] + .filter(Boolean) + .join(" · "); } else { r = rankOf(b.results, provider, b.higherIsBetter); if (!r) return new NextResponse("not found", { status: 404 }); - // Add an "all chains" hint when the bench declares chain dimensions so - // embedders can read the scope. Benches without chain dimensions get - // no scope label (it would be noise). - scopeLabel = + // Hint the aggregate scope when the bench declares dimensions so + // embedders can read it. Benches without dimensions get no scope + // label (it would be noise). + scopeLabel = [ (b.dimensions?.chain?.filter((c) => c.value !== "all").length ?? 0) > 0 ? "all chains" - : ""; + : null, + (b.dimensions?.region?.filter((d) => d.value !== "all").length ?? 0) > 0 + ? "all regions" + : null, + ] + .filter(Boolean) + .join(" · "); } // Colour signals rank. green for #1, dark ink for everyone else. @@ -187,32 +242,55 @@ export async function GET( const suffix = valueSuffix(b.unit); const title = truncate(b.title, TITLE_MAX); - // Provider initials in the bottom-left corner. mirrors the brand - // chip the site uses internally. - const ocbBrand = "OCB"; - - // Scope marker: rendered as a small subscript next to the rank when - // present. Keeps the badge layout stable when absent (most benches). + // Scope marker: small caps tspan appended to the figure line, so the + // badge height stays constant whether or not a scope is present. const scopeAriaSuffix = scopeLabel ? ` (${scopeLabel})` : ""; - const scopeSvg = scopeLabel - ? `${escapeXml(scopeLabel.toUpperCase())}` + const scopeTspan = scopeLabel + ? `${escapeXml(scopeLabel.toUpperCase())}` : ""; - // Bump the SVG canvas height when a scope label is rendered so the - // subscript doesn't clip outside the box on stricter image renderers. - const svgH = scopeLabel ? H + 8 : H; + // The spinning mark: the dark-mode brand sphere (near-black skin, + // light C-ring + grey corner markers, same geometry as site-logo.tsx) + // doing a continuous 360 around its vertical axis. The spin is a CSS + // coin-flip (scaleX 1 -> 0 -> 1); while scaleX is 0 the plain dark + // sphere shows, which is exactly what the back of the masthead sphere + // looks like. CSS animations inside SVG run in embeds, and the + // reduced-motion media query freezes it for users who opted out. + // Static renderers without CSS support just show the un-rotated mark. const svg = ` - + OpenChainBench. ${escapeXml(b.title)}. ${rankLabel}${scopeAriaSuffix}, ${value} ${suffix} - - - + + + + + + + + + + + + + + + + + + + + + + + - ${ocbBrand} - ${rankLabel} - ${escapeXml(title)} - ${value} ${suffix} - ${scopeSvg} + ${rankLabel}${escapeXml(title)} + ${value} ${suffix}${scopeTspan} + OCB `; diff --git a/src/app/benchmarks/[slug]/[chain]/page.tsx b/src/app/benchmarks/[slug]/[chain]/page.tsx new file mode 100644 index 00000000..35dc5faa --- /dev/null +++ b/src/app/benchmarks/[slug]/[chain]/page.tsx @@ -0,0 +1,374 @@ +import type { Metadata } from "next"; +import { notFound } from "next/navigation"; +import Link from "next/link"; +import { ArrowLeft, ArrowUpRight } from "lucide-react"; +import { getBenchmark, getBenchmarks } from "@/data/benchmarks"; +import { liveResults } from "@/lib/provider-filters"; +import { fmtUnit } from "@/lib/format"; +import { capDescription } from "@/lib/seo-text"; +import { SITE } from "@/data/site"; +import { safeJsonLd } from "@/lib/jsonld"; +import { CATEGORY_COLOR } from "@/lib/category-colors"; +import type { Benchmark, ProviderResult } from "@/types/benchmark"; + +// Dedicated per-chain landing pages. Only chains that have a hand-written +// `per_chain_explainer` entry in the bench YAML get a route here - that +// unique editorial body is what keeps these pages from being doorway +// duplicates of the parent bench. The parent's `?chain=` query filter +// stays a client-side UI affordance; THIS route is the indexable, +// self-canonical document targeting long-tail queries like "ethereum +// finality time". +export const revalidate = 60; + +type Params = { slug: string; chain: string }; + +export async function generateStaticParams() { + const benchmarks = await getBenchmarks(); + return benchmarks.flatMap((b) => { + const resultSlugs = new Set(b.results.map((r) => r.slug)); + return (b.perChainExplainer ?? []) + .filter((e) => resultSlugs.has(e.slug)) + .map((e) => ({ slug: b.slug, chain: e.slug })); + }); +} + +type ChainPageData = { + benchmark: Benchmark; + explainer: { slug: string; h2: string; body: string }; + result: ProviderResult; + sorted: ProviderResult[]; + rank: number; +}; + +/** The spec loader's renderTemplate only resolves placeholders for + * providers that are "live" this cycle. When a chain's data is in a + * transient gap, raw `{{p50:ethereum}}` tokens would leak into the H1 + * copy and the meta description. Resolve leftovers against the full + * results array (the row always exists - generateStaticParams gates on + * it), so the page degrades to the last scraped value instead of + * shipping template syntax to the SERP. */ +function resolveLeftoverPlaceholders(text: string, b: Benchmark): string { + return text.replace( + /\{\{\s*(p50|p90|p99|mean|name):([a-z0-9-]+)\s*\}\}/gi, + (whole, keyword: string, slug: string) => { + const row = b.results.find( + (r) => r.slug.toLowerCase() === slug.toLowerCase(), + ); + const k = keyword.toLowerCase(); + if (k === "name") return row ? row.name : whole; + const raw = row?.ms[k as "p50" | "p90" | "p99" | "mean"]; + // Transient data gap: degrade to neutral copy ("... is measured + // live (p50, 24h)") rather than shipping template syntax. + if (!raw || raw <= 0) return "measured live"; + return fmtUnit(raw, b.unit); + }, + ); +} + +async function loadChainPage( + slug: string, + chain: string, +): Promise { + const benchmark = await getBenchmark(slug); + if (!benchmark) return null; + const found = (benchmark.perChainExplainer ?? []).find( + (e) => e.slug === chain, + ); + if (!found) return null; + const result = benchmark.results.find((r) => r.slug === chain); + if (!result) return null; + const explainer = { + ...found, + h2: resolveLeftoverPlaceholders(found.h2, benchmark), + body: resolveLeftoverPlaceholders(found.body, benchmark), + }; + const live = liveResults(benchmark.results); + const sorted = [...live].sort((a, b) => + benchmark.higherIsBetter ? b.ms.p50 - a.ms.p50 : a.ms.p50 - b.ms.p50, + ); + const rank = sorted.findIndex((r) => r.slug === chain) + 1; + return { benchmark, explainer, result, sorted, rank }; +} + +/** Meta descriptions must not leak inline markdown from the YAML body + * (backticks around RPC method names, bold, links). */ +function stripInlineMarkdown(text: string): string { + return text + .replace(/`([^`]*)`/g, "$1") + .replace(/\*\*([^*]*)\*\*/g, "$1") + .replace(/\*([^*]*)\*/g, "$1") + .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") + .replace(/\s+/g, " ") + .trim(); +} + +function asOfDate(lastRunAt: string | undefined): string { + const d = lastRunAt ? new Date(lastRunAt) : new Date(); + return d.toLocaleDateString("en-US", { + year: "numeric", + month: "long", + day: "numeric", + }); +} + +export async function generateMetadata({ + params, +}: { + params: Promise; +}): Promise { + const { slug, chain } = await params; + const data = await loadChainPage(slug, chain); + if (!data) return {}; + const { benchmark, explainer, result } = data; + const hasData = result.ms.p50 > 0; + const title = hasData + ? `${explainer.h2}: ${fmtUnit(result.ms.p50, benchmark.unit)} p50 live` + : `${explainer.h2}: live benchmark`; + const description = capDescription( + stripInlineMarkdown(explainer.body), + 158, + ); + const canonical = `${SITE.url}/benchmarks/${benchmark.slug}/${chain}`; + const ogImage = `${SITE.url}/api/og/${benchmark.slug}`; + return { + title, + description, + alternates: { canonical }, + openGraph: { + title, + description, + type: "article", + url: canonical, + images: [ogImage], + }, + twitter: { + card: "summary_large_image", + title, + description, + images: [ogImage], + }, + }; +} + +export default async function BenchmarkChainPage({ + params, +}: { + params: Promise; +}) { + const { slug, chain } = await params; + const data = await loadChainPage(slug, chain); + if (!data) notFound(); + const { benchmark, explainer, result, sorted, rank } = data; + + const benchmarkUrl = `${SITE.url}/benchmarks/${benchmark.slug}`; + const pageUrl = `${benchmarkUrl}/${chain}`; + const catColor = CATEGORY_COLOR[benchmark.category]; + const p50 = fmtUnit(result.ms.p50, benchmark.unit); + const p90 = fmtUnit(result.ms.p90, benchmark.unit); + const p99 = fmtUnit(result.ms.p99, benchmark.unit); + const explainerSlugs = new Set( + (benchmark.perChainExplainer ?? []).map((e) => e.slug), + ); + + // Dated, citable key-facts sentence. LLM crawlers and journalists quote + // stats that carry an explicit date + source; featured snippets prefer + // the same shape. + const hasData = result.ms.p50 > 0; + const subject = `${result.name} ${benchmark.metric.toLowerCase()}`; + const keyFacts = hasData + ? `As of ${asOfDate(benchmark.lastRunAt)}, ${subject} is ${p50} at the median (p50, 24h window), with ${p90} at p90 and ${p99} at p99.` + + (rank > 0 + ? ` ${result.name} ranks #${rank} of ${sorted.length} chains measured on this benchmark.` + : "") + : `${subject} is measured continuously on this benchmark. Live numbers will appear here as soon as the harness reports fresh samples.`; + + const jsonLd = { + "@context": "https://schema.org", + "@graph": [ + { + "@type": "TechArticle", + "@id": `${pageUrl}#article`, + headline: explainer.h2, + description: capDescription(stripInlineMarkdown(explainer.body), 158), + url: pageUrl, + mainEntityOfPage: pageUrl, + articleBody: `${keyFacts} ${stripInlineMarkdown(explainer.body)}`, + image: `${SITE.url}/api/og/${benchmark.slug}`, + dateModified: benchmark.lastRunAt, + author: { "@id": `${SITE.url}/#org` }, + publisher: { "@id": `${SITE.url}/#org` }, + about: { "@id": `${benchmarkUrl}#dataset` }, + isPartOf: { "@id": `${benchmarkUrl}#article` }, + }, + { + "@type": "BreadcrumbList", + itemListElement: [ + { "@type": "ListItem", position: 1, name: "Home", item: SITE.url }, + { + "@type": "ListItem", + position: 2, + name: "Benchmarks", + item: `${SITE.url}/benchmarks`, + }, + { + "@type": "ListItem", + position: 3, + name: benchmark.title, + item: benchmarkUrl, + }, + { + "@type": "ListItem", + position: 4, + name: result.name, + item: pageUrl, + }, + ], + }, + ], + }; + + return ( +
+ + +

    +
  1. + + Home + +
    2. +
  2. /
    3. +
  3. + + Benchmarks + +
    4. +
  4. /
    5. +
  5. + + {benchmark.title} + +
    6. +
  6. /
    7. +
  7. + {result.name} +
    8. +
+ + + + + Full benchmark: {benchmark.title} + + +
+ + {benchmark.category} + + {result.name} +
+ +

+ {explainer.h2} +

+ + {/* Dated key-facts line. Server-rendered first so the citable, + date-stamped stat lands in the first ~100 words of the document. */} +

+ {keyFacts} +

+ +
+ {explainer.body.split(/\n\n+/).map((para, i) => ( +

{para.trim()}

+ ))} +
+ + {/* Cross-chain context table. Every sibling with its own explainer + links to its dedicated page (internal mesh); chains without one + deep-link to their anchor on the parent bench. */} + {sorted.length > 1 && ( +
+

+ How {result.name} compares +

+

+ Live p50 over the last 24 hours across every chain on this + benchmark, ranked{" "} + {benchmark.higherIsBetter ? "highest" : "lowest"} first. +

+
    + {sorted.map((r, i) => { + const isCurrent = r.slug === chain; + const href = explainerSlugs.has(r.slug) + ? `/benchmarks/${benchmark.slug}/${r.slug}` + : `/benchmarks/${benchmark.slug}#${r.slug}`; + const row = ( + + + + #{i + 1} + + + {r.name} + + + + {fmtUnit(r.ms.p50, benchmark.unit)} + + + ); + return ( +
  1. + {isCurrent ? ( + row + ) : ( + + {row} + + )} +
    2. + ); + })} +
+
+ )} + +

+ Methodology, charts and the full ledger live on the{" "} + + {benchmark.title} + {" "} + page. Raw data:{" "} + + JSON endpoint + + + . +

+ + ); +} diff --git a/src/app/products/[slug]/page.tsx b/src/app/products/[slug]/page.tsx index bdbbe46f..009c86c5 100644 --- a/src/app/products/[slug]/page.tsx +++ b/src/app/products/[slug]/page.tsx @@ -81,6 +81,130 @@ export default async function ProviderPage({ return a.benchmark.title.localeCompare(b.benchmark.title); }); + // Embeddable badge cards. Scope rules, most exact source first: + // + // 1. Benches with a `rank_matrix_query` AND region dimensions use the + // exact per-cell rankings (chain × region). The per-chain ranks the + // legacy path relies on are cross-region averages: a provider that + // only wins from Singapore (dRPC) still reads as the chain leader + // because one fast region drags the mean down. Cells fix that: + // - leads EVERY cell → one unscoped global badge. + // - leads a full chain row (all regions) → one "on " badge. + // - leads a full region column (all chains) → one "from " + // badge (skipped when its cells are already claimed by rows). + // - leftover isolated cells → "on from " badges. + // 2. Benches with chain dimensions but no cell data keep the per-chain + // logic: global badge only for a true cross-chain leader, otherwise + // one badge per chain led. + // 3. Benches without dimensions: one global badge per aggregate #1. + type BadgeCard = { + key: string; + title: string; + chain?: { value: string; label: string }; + region?: { value: string; label: string }; + benchSlug: string; + }; + const badgeCards: BadgeCard[] = []; + for (const a of sorted) { + const chainDims = (a.benchmark.chainDimensions ?? []).filter( + (c) => c.value !== "all", + ); + const regionDims = (a.benchmark.regionDimensions ?? []).filter( + (r) => r.value !== "all", + ); + const cellRanks = a.benchmark.cellRanks; + const me = a.result.slug.toLowerCase(); + const benchSlug = a.benchmark.slug; + const title = a.benchmark.title; + + let handledByCells = false; + if (cellRanks && regionDims.length > 0) { + const finestKeys = Object.keys(cellRanks).filter((k) => { + const [c, r] = k.split("|"); + const chainOk = chainDims.length > 0 ? c !== "all" : c === "all"; + return chainOk && r !== "all"; + }); + if (finestKeys.length > 0) { + handledByCells = true; + const wonKeys = new Set( + finestKeys.filter( + (k) => cellRanks[k][0]?.slug.toLowerCase() === me, + ), + ); + if (wonKeys.size === finestKeys.length) { + badgeCards.push({ key: benchSlug, title, benchSlug }); + continue; + } + if (wonKeys.size === 0) continue; + const chainOf = (k: string) => k.split("|")[0]; + const regionOf = (k: string) => k.split("|")[1]; + const covered = new Set(); + for (const c of chainDims) { + const row = finestKeys.filter((k) => chainOf(k) === c.value); + if (row.length === 0 || !row.every((k) => wonKeys.has(k))) continue; + badgeCards.push({ + key: `${benchSlug}-${c.value}`, + title, + chain: c, + benchSlug, + }); + for (const k of row) covered.add(k); + } + for (const r of regionDims) { + const col = finestKeys.filter((k) => regionOf(k) === r.value); + if (col.length === 0 || !col.every((k) => wonKeys.has(k))) continue; + if (col.every((k) => covered.has(k))) continue; + badgeCards.push({ + key: `${benchSlug}-r-${r.value}`, + title, + region: r, + benchSlug, + }); + for (const k of col) covered.add(k); + } + for (const k of wonKeys) { + if (covered.has(k)) continue; + const chain = chainDims.find((c) => c.value === chainOf(k)); + const region = regionDims.find((r) => r.value === regionOf(k)); + if (!region) continue; + badgeCards.push({ + key: `${benchSlug}-${chainOf(k)}-${regionOf(k)}`, + title, + ...(chain ? { chain } : {}), + region, + benchSlug, + }); + } + continue; + } + } + if (handledByCells) continue; + + const perChain = a.rankPerChain ?? {}; + const wonChains = chainDims.filter((c) => perChain[c.value]?.rank === 1); + const isGlobalNumberOne = a.rank === 1; + if (chainDims.length === 0) { + if (isGlobalNumberOne) { + badgeCards.push({ key: benchSlug, title, benchSlug }); + } + continue; + } + const leadsAllChains = + chainDims.length > 0 && wonChains.length === chainDims.length; + if (isGlobalNumberOne && leadsAllChains) { + badgeCards.push({ key: benchSlug, title, benchSlug }); + continue; + } + for (const c of wonChains) { + badgeCards.push({ + key: `${benchSlug}-${c.value}`, + title, + chain: c, + benchSlug, + }); + } + } + const url = `${SITE.url}/products/${p.slug}`; const sameAs: string[] = []; if (reg?.url) sameAs.push(reg.url); @@ -392,7 +516,7 @@ export default async function ProviderPage({ - {p.wins > 0 && ( + {badgeCards.length > 0 && (

Embeddable badges @@ -403,7 +527,7 @@ export default async function ProviderPage({ stays accurate without redeploying.

    - {sorted.filter((a) => a.rank === 1).map((a) => { + {badgeCards.map((card) => { // Absolute URL is the one shipped to embedders (it has to // work from any third-party origin), but the in-page preview // uses a relative path so it loads under the current @@ -411,25 +535,56 @@ export default async function ProviderPage({ // shows the browser's broken-image glyph on every non-prod // origin (staging Preview URLs, Vercel branch previews, etc.) // because the CSP refuses the cross-origin fetch. - const badgePath = `/api/badge/${a.benchmark.slug}/${p.slug}`; + const scopeParams = new URLSearchParams(); + if (card.chain) scopeParams.set("chain", card.chain.value); + if (card.region) scopeParams.set("region", card.region.value); + const qs = scopeParams.size > 0 ? `?${scopeParams.toString()}` : ""; + const badgePath = `/api/badge/${card.benchSlug}/${p.slug}${qs}`; const badgeUrl = `${SITE.url}${badgePath}`; - const targetUrl = `${SITE.url}/benchmarks/${a.benchmark.slug}`; - const html = `Ranked #1 on OpenChainBench: ${a.benchmark.title}`; + const targetUrl = `${SITE.url}/benchmarks/${card.benchSlug}${qs}`; + const scopeSuffix = `${card.chain ? ` on ${card.chain.label}` : ""}${card.region ? ` from ${card.region.label}` : ""}`; + const scopeLabels = [card.chain?.label, card.region?.label] + .filter(Boolean) + .join(" · "); + const cardTitle = scopeLabels + ? `${card.title} — ${scopeLabels}` + : card.title; + const altText = `Ranked #1 on OpenChainBench: ${card.title}${scopeSuffix}`; + const html = `${altText}`; + const markdown = `[![${altText}](${badgeUrl})](${targetUrl})`; + // Pre-baked X intent. Providers click → tweet draft opens + // with the ranking claim, the bench URL and the OCB handle + // already filled in. Removes the friction of writing the + // post themselves and gives us the canonical anchor text + // back as a tagged tweet on every share. + const tweetText = `Independently benchmarked #1 on ${card.title}${scopeSuffix} by @OpenChainBench.\n\nReproducible methodology, live data:`; + const tweetIntent = `https://x.com/intent/tweet?text=${encodeURIComponent(tweetText)}&url=${encodeURIComponent(targetUrl)}`; return ( -
  • +

  • - {a.benchmark.title} + {cardTitle}

    {/* eslint-disable-next-line @next/next/no-img-element */} {`Ranked
    +
    Copy HTML @@ -438,6 +593,14 @@ export default async function ProviderPage({ {html}
    +
    + + Copy Markdown + + 
    +{markdown}
+
    +
    • ); })} diff --git a/src/app/sitemap.ts b/src/app/sitemap.ts index a3c155ee..fa188316 100644 --- a/src/app/sitemap.ts +++ b/src/app/sitemap.ts @@ -94,16 +94,15 @@ export default async function sitemap(): Promise { { url: `${SITE.url}/press`, lastModified: pageMtime("press/page.tsx"), changeFrequency: "monthly", priority: 0.4 }, ]; - // Bench routes. The hub URL (no query string) is the canonical entry - // and ranks highest. Per-chain variants (`?chain=X`) are emitted as - // secondary URLs so Google's crawler discovers the chain-honest - // metadata / OG card pairs for each filter. We skip the "all" sentinel - // (which maps to the canonical hub) and any chain dimension whose - // value would collide with the hub after URL-encoding. Each variant - // shares the parent bench's `lastModified` because the chain filter - // doesn't change the underlying scrape cadence — they all refresh as - // a single Prom poll. Priority is dropped one tier on variants so - // Search Console reads the hub as the head of the cluster. + // Bench routes. The hub URL is the canonical entry and ranks highest. + // `?chain=X` query variants are deliberately NOT emitted: those URLs + // declare a canonical pointing at the unfiltered hub, so listing them + // told Google to index pages that self-identify as duplicates (GSC + // filed them under "Duplicate, Google chose different canonical"). + // The indexable per-chain surface is the dedicated route + // `/benchmarks//`, generated only for chains that carry a + // hand-written `per_chain_explainer` entry (unique editorial content, + // self-canonical). See src/app/benchmarks/[slug]/[chain]/page.tsx. const benchmarkRoutes: MetadataRoute.Sitemap = benchmarks.flatMap((b) => { const last = b.lastRunAt ? new Date(b.lastRunAt) : BUILD_TIME; const entries: MetadataRoute.Sitemap = [ @@ -114,15 +113,14 @@ export default async function sitemap(): Promise { priority: 0.95, }, ]; - const chains = (b.dimensions?.chain ?? []).filter( - (c) => c.value && c.value.toLowerCase() !== "all", - ); - for (const c of chains) { + const resultSlugs = new Set(b.results.map((r) => r.slug)); + for (const e of b.perChainExplainer ?? []) { + if (!resultSlugs.has(e.slug)) continue; entries.push({ - url: `${SITE.url}/benchmarks/${b.slug}?chain=${encodeURIComponent(c.value)}`, + url: `${SITE.url}/benchmarks/${b.slug}/${e.slug}`, lastModified: last, changeFrequency: "hourly", - priority: 0.8, + priority: 0.85, }); } return entries; diff --git a/src/components/chain-headings-summary.tsx b/src/components/chain-headings-summary.tsx index 9c348ca0..75f7c6c2 100644 --- a/src/components/chain-headings-summary.tsx +++ b/src/components/chain-headings-summary.tsx @@ -1,3 +1,4 @@ +import Link from "next/link"; import type { Benchmark } from "@/types/benchmark"; import { liveResults } from "@/lib/provider-filters"; import { fmtUnit } from "@/lib/format"; @@ -69,7 +70,19 @@ export function ChainHeadingsSummary({ benchmark }: { benchmark: Benchmark }) { return (

    - {heading} + {/* Chains with an explainer have a dedicated landing page + (/benchmarks//); the heading links there so + crawlers discover the per-chain documents from the hub. */} + {explainer ? ( + + {heading} + + ) : ( + heading + )}

    diff --git a/src/lib/providers.ts b/src/lib/providers.ts index 461a27b3..e528c2c2 100644 --- a/src/lib/providers.ts +++ b/src/lib/providers.ts @@ -106,6 +106,15 @@ export type ProviderAppearance = { * can compute per-chain rank for this provider without a full bench * re-fetch. */ bestPerChain?: Record; + /** Region dimension values from the spec, when present. Mirrors + * `benchmark.dimensions.region`. */ + regionDimensions?: { value: string; label: string }[]; + /** Exact per-cell rankings from the bench's `rank_matrix_query` + * (key = `|`, "all" for an undeclared dimension or a + * derived marginal). When present this is the authoritative source + * for scoped leadership claims — per-chain ranks built from + * cross-region averages hide region-restricted leaders. */ + cellRanks?: Benchmark["cellRanks"]; }; result: ProviderResult; rank: number; @@ -279,6 +288,8 @@ export const getProviders = cache(async (): Promise => { lastRunAt: b.lastRunAt, chainDimensions: b.dimensions?.chain, bestPerChain: benchBestPerChain, + regionDimensions: b.dimensions?.region, + cellRanks: b.cellRanks, }, result: r, rank: isRanked ? (idx as number) + 1 : 0, diff --git a/src/lib/snapshot.ts b/src/lib/snapshot.ts index 5f5b4746..b30ef125 100644 --- a/src/lib/snapshot.ts +++ b/src/lib/snapshot.ts @@ -33,7 +33,12 @@ */ import { z } from "zod"; -import type { Benchmark, ProviderResult, ResultExtras } from "@/types/benchmark"; +import type { + Benchmark, + CellRankEntry, + ProviderResult, + ResultExtras, +} from "@/types/benchmark"; /** Refuse snapshots older than this on read. 24 h matches the bench * query window — a value older than that isn't meaningful as the @@ -60,6 +65,7 @@ const SnapshotSchema = z.object({ bestPerChain: z.record(z.string(), z.any()).optional(), worstPerChain: z.record(z.string(), z.any()).optional(), providersPerChain: z.record(z.string(), z.array(z.string())).optional(), + cellRanks: z.record(z.string(), z.any()).optional(), }); export type SnapshotPayload = { @@ -70,6 +76,7 @@ export type SnapshotPayload = { bestPerChain?: Record; worstPerChain?: Record; providersPerChain?: Record; + cellRanks?: Record; }; function isConfigured(): boolean { @@ -183,6 +190,9 @@ export async function readSnapshot( providersPerChain: parsed.data.providersPerChain as | Record | undefined, + cellRanks: parsed.data.cellRanks as + | Record + | undefined, }; } catch (err) { const msg = err instanceof Error ? err.message : String(err); @@ -215,5 +225,6 @@ export function snapshotFromBenchmark(b: Benchmark): SnapshotPayload { worstPerChain: b.worstPerChain, providersPerChain: (b as { providersPerChain?: Record }) .providersPerChain, + cellRanks: b.cellRanks, }; } diff --git a/src/lib/spec-schema.ts b/src/lib/spec-schema.ts index 611247f5..20aa3a11 100644 --- a/src/lib/spec-schema.ts +++ b/src/lib/spec-schema.ts @@ -318,6 +318,22 @@ export const SpecSchema = z }) .optional(), + /* Optional single PromQL returning one instant sample per + * (provider[, chain][, region]) — e.g. + * avg by (provider, chain, region) (quantile_over_time(0.50, m[24h])) + * Powers exact per-cell rankings (badge scoping, leadership claims) + * in ONE Prom roundtrip instead of a chains × regions query fan-out. + * Label values must match provider slugs / dimension values. */ + rank_matrix_query: z + .string() + .min(1) + .max(2000) + .refine( + (q) => q.includes("provider"), + "rank_matrix_query must group by the provider label" + ) + .optional(), + providers: z.array(provider).min(1), /** @@ -349,7 +365,24 @@ export const SpecSchema = z .max(8) .optional(), }) - .strict(); + .strict() + .superRefine((spec, ctx) => { + // A region-dimensioned bench without exact cell rankings would fall + // back to cross-region-average badge logic, which is precisely the + // bias the matrix exists to fix (a provider winning from one region + // reads as the global leader). Refuse the spec instead. + const realRegions = (spec.dimensions?.region ?? []).filter( + (r) => r.value !== "all", + ); + if (realRegions.length > 0 && !spec.rank_matrix_query) { + ctx.addIssue({ + code: "custom", + path: ["rank_matrix_query"], + message: + "Benches declaring dimensions.region must provide rank_matrix_query so badge claims are scoped per region", + }); + } + }); export type Spec = z.infer; export type SpecProvider = z.infer; diff --git a/src/lib/spec.ts b/src/lib/spec.ts index b6c6160c..535c21a5 100644 --- a/src/lib/spec.ts +++ b/src/lib/spec.ts @@ -14,7 +14,12 @@ import path from "node:path"; import { cache } from "react"; import { unstable_cache } from "next/cache"; import yaml from "js-yaml"; -import type { Benchmark, MetricPanel, ProviderResult } from "@/types/benchmark"; +import type { + Benchmark, + CellRankEntry, + MetricPanel, + ProviderResult, +} from "@/types/benchmark"; import { Prometheus } from "@/lib/prometheus"; import { SpecSchema, type Spec } from "@/lib/spec-schema"; import { renderBenchmarkText } from "@/lib/bench-template"; @@ -455,6 +460,11 @@ async function specToBenchmark( if (Object.keys(providers).length > 0) providersPerChain = providers; } + // Exact per-cell rankings (chain × region) from the spec's single + // grouped matrix query. Failures are tolerated: badge/product + // surfaces fall back to the coarser bestPerChain path. + const cellRanks = !isFiltered ? await tryLoadCellRanks(spec) : undefined; + // Resolve {{p50:slug}} / {{best_name}} / {{count}} etc. placeholders // against the freshly loaded numbers so editorial text (findings, // seo_intro, faq) never drifts from the displayed data. @@ -464,6 +474,7 @@ async function specToBenchmark( bestPerChain, worstPerChain, providersPerChain, + cellRanks, }); // Persist a snapshot of the runtime data so a future cold start // during a Prom blackout can still render this bench. Only the @@ -486,6 +497,102 @@ function activeFilterLabels(opts: BenchmarkFilters): Record { return out; } +/** + * Run the spec's `rank_matrix_query` (one instant vector with a sample per + * (provider[, chain][, region])) and fold it into full per-cell rankings. + * + * Output keys are `|` with "all" standing in for an + * undeclared dimension. When BOTH dimensions are declared, marginal cells + * (`|all`, `all|`) are derived by averaging a provider's + * finest-cell values over the collapsed dimension — same semantics as the + * bench page's unscoped `avg(...)` headline queries. + * + * Samples whose provider label doesn't match a spec provider slug, or + * whose chain/region label isn't a declared dimension value, are dropped: + * the matrix is unfiltered PromQL, so stray series (retired providers, + * staging labels) must not leak into rankings. + */ +async function tryLoadCellRanks( + spec: Spec, +): Promise | undefined> { + if (!spec.rank_matrix_query) return undefined; + const url = spec.prometheus?.url ?? process.env.PROMETHEUS_URL; + if (!url) return undefined; + try { + const prom = new Prometheus(url); + const res = await prom.query(spec.rank_matrix_query); + if (res.resultType !== "vector") return undefined; + + const slugByLower = new Map( + spec.providers.map((p) => [p.slug.toLowerCase(), p.slug] as const), + ); + const chainValues = new Set( + (spec.dimensions?.chain ?? []).map((c) => c.value).filter((v) => v !== "all"), + ); + const regionValues = new Set( + (spec.dimensions?.region ?? []).map((r) => r.value).filter((v) => v !== "all"), + ); + + // key → provider slug → samples (averaged if the grouping left + // residual label splits, e.g. multiple replicas per region). + const acc = new Map>(); + for (const sample of res.result) { + const slug = slugByLower.get((sample.metric.provider ?? "").toLowerCase()); + if (!slug) continue; + const chain = chainValues.size > 0 ? sample.metric.chain : undefined; + const region = regionValues.size > 0 ? sample.metric.region : undefined; + if (chainValues.size > 0 && (!chain || !chainValues.has(chain))) continue; + if (regionValues.size > 0 && (!region || !regionValues.has(region))) continue; + const v = Number(sample.value[1]); + if (!Number.isFinite(v) || v <= 0) continue; + const key = `${chain ?? "all"}|${region ?? "all"}`; + const cell = acc.get(key) ?? new Map(); + const vals = cell.get(slug) ?? []; + vals.push(v); + cell.set(slug, vals); + acc.set(key, cell); + } + if (acc.size === 0) return undefined; + + const mean = (vals: number[]) => + vals.reduce((a, b) => a + b, 0) / vals.length; + const sortCell = (cell: Map): CellRankEntry[] => + [...cell.entries()] + .map(([slug, vals]) => ({ slug, p50: mean(vals) })) + .sort((a, b) => + spec.higher_is_better ? b.p50 - a.p50 : a.p50 - b.p50, + ); + + const out: Record = {}; + for (const [key, cell] of acc) out[key] = sortCell(cell); + + // Marginals, only when both dimensions exist in the finest cells. + if (chainValues.size > 0 && regionValues.size > 0) { + const marginal = new Map>(); + for (const [key, cell] of acc) { + const [chain, region] = key.split("|"); + for (const [slug, vals] of cell) { + const v = mean(vals); + for (const mKey of [`${chain}|all`, `all|${region}`]) { + const mCell = marginal.get(mKey) ?? new Map(); + const mVals = mCell.get(slug) ?? []; + mVals.push(v); + mCell.set(slug, mVals); + marginal.set(mKey, mCell); + } + } + } + for (const [key, cell] of marginal) out[key] = sortCell(cell); + } + return out; + } catch (e) { + console.warn( + `cellRanks skip: ${spec.slug} matrix query failed: ${e instanceof Error ? e.message : String(e)}`, + ); + return undefined; + } +} + /** Inject every active `