diff --git a/.github/workflows/hf-publish.yml b/.github/workflows/hf-publish.yml new file mode 100644 index 00000000..09bd50b3 --- /dev/null +++ b/.github/workflows/hf-publish.yml @@ -0,0 +1,85 @@ +name: HF dataset publish + +# Daily snapshot of the live citable feed pushed to +# https://huggingface.co/datasets/OpenChainBench/benchmarks. +# +# Trigger: +# - 00:00 UTC every day (after the daily Prom roll-up is settled). +# - workflow_dispatch with an optional `dry_run` flag so we can +# validate changes without touching HF. +# +# Required secrets: +# HF_TOKEN write-scoped token for the dataset repo. +# SLACK_WEBHOOK_URL optional, incoming-webhook URL for ops alerts. +# KAGGLE_USERNAME optional, owner of the Kaggle mirror dataset. +# KAGGLE_KEY optional, API key paired with KAGGLE_USERNAME. +# +# When both KAGGLE_USERNAME and KAGGLE_KEY are set, the publisher mirrors +# the snapshot to kaggle.com/datasets/openchainbench/benchmarks right +# after the HF push. A Kaggle-side failure never aborts the run. + +on: + schedule: + - cron: "0 0 * * *" + workflow_dispatch: + inputs: + dry_run: + description: "Stage parquet locally without pushing to HF" + type: boolean + default: false + +concurrency: + group: hf-publish + cancel-in-progress: false + +jobs: + publish: + runs-on: ubuntu-latest + timeout-minutes: 15 + permissions: + contents: read + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + KAGGLE_USERNAME: ${{ secrets.KAGGLE_USERNAME }} + KAGGLE_KEY: ${{ secrets.KAGGLE_KEY }} + OCB_API: https://openchainbench.com + HF_REPO_ID: OpenChainBench/benchmarks + KAGGLE_DATASET_ID: openchainbench/benchmarks + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + cache: pip + cache-dependency-path: scripts/hf_publisher/requirements.txt + + - name: Install deps + run: pip install -r scripts/hf_publisher/requirements.txt + + - name: Run publisher tests (offline) + # Schema/quorum regressions caught here never reach the dataset. + run: | + cd scripts/hf_publisher + python -m unittest test_publish.py -v + + - name: Publish snapshot + run: | + cd scripts/hf_publisher + if [ "${{ github.event_name }}" = "workflow_dispatch" ] && [ "${{ inputs.dry_run }}" = "true" ]; then + python publish.py --dry-run + else + python publish.py + fi + + - name: Summary + if: always() + run: | + { + echo "## HF publish ${{ job.status }}" + echo "" + echo "**Repo:** https://huggingface.co/datasets/${HF_REPO_ID}" + echo "**Mode:** ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run == true && 'dry-run' || 'live' }}" + echo "**Time:** $(date -u +%FT%TZ)" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/hf-space-deploy.yml b/.github/workflows/hf-space-deploy.yml new file mode 100644 index 00000000..7d9a2bfb --- /dev/null +++ b/.github/workflows/hf-space-deploy.yml @@ -0,0 +1,82 @@ +name: HF Space deploy + +# Pushes scripts/hf_space/ to the HF Space at +# https://huggingface.co/spaces/OpenChainBench/leaderboard. +# +# Trigger: +# - push to main that touches scripts/hf_space/** +# - workflow_dispatch for manual redeploys +# +# Required secrets: +# HF_TOKEN write-scoped token for the Space repo (same token as the +# dataset publisher). + +on: + push: + branches: [main] + paths: + - "scripts/hf_space/**" + - ".github/workflows/hf-space-deploy.yml" + workflow_dispatch: + +concurrency: + group: hf-space-deploy + cancel-in-progress: false + +jobs: + deploy: + runs-on: ubuntu-latest + timeout-minutes: 10 + permissions: + contents: read + env: + HF_TOKEN: ${{ secrets.HF_TOKEN }} + HF_SPACE_REPO_ID: OpenChainBench/leaderboard + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install huggingface_hub + run: pip install "huggingface_hub>=0.26" + + - name: Upload to HF Space + env: + GIT_SHA: ${{ github.sha }} + run: | + python - <<'PY' + import os + from huggingface_hub import HfApi + + token = os.environ["HF_TOKEN"] + repo_id = os.environ["HF_SPACE_REPO_ID"] + sha = os.environ.get("GIT_SHA", "manual") + api = HfApi(token=token) + api.create_repo( + repo_id=repo_id, + repo_type="space", + space_sdk="gradio", + exist_ok=True, + private=False, + ) + info = api.upload_folder( + folder_path="scripts/hf_space", + repo_id=repo_id, + repo_type="space", + commit_message=f"deploy from {sha}", + ) + print("uploaded:", getattr(info, "oid", "ok")) + PY + + - name: Summary + if: always() + run: | + { + echo "## HF Space deploy ${{ job.status }}" + echo "" + echo "**Space:** https://huggingface.co/spaces/${HF_SPACE_REPO_ID}" + echo "**Commit:** ${{ github.sha }}" + echo "**Time:** $(date -u +%FT%TZ)" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/hf-space-keepalive.yml b/.github/workflows/hf-space-keepalive.yml new file mode 100644 index 00000000..ef0edbb7 --- /dev/null +++ b/.github/workflows/hf-space-keepalive.yml @@ -0,0 +1,34 @@ +name: HF Space keepalive + +# Free HF Spaces fall asleep after a stretch of inactivity, and the +# first visitor after a sleep waits 30 to 60 seconds for the container +# to boot. That looks broken from a tweet. A light curl every 4 hours +# keeps the Space warm without burning meaningful GH Actions minutes +# (~6 runs per day, a few seconds each). + +on: + schedule: + - cron: "0 */4 * * *" + workflow_dispatch: + +concurrency: + group: hf-space-keepalive + cancel-in-progress: false + +jobs: + ping: + runs-on: ubuntu-latest + timeout-minutes: 2 + steps: + - name: Wake leaderboard Space + run: | + code=$(curl -sLm 60 -o /dev/null -w "%{http_code}" \ + "https://openchainbench-leaderboard.hf.space") + echo "leaderboard: $code" + # 200 is a hot hit. 503 means HF is still booting the + # container, which is exactly the case we are warming for, so + # don't fail the workflow on it. + case "$code" in + 200|302|503) exit 0 ;; + *) exit 1 ;; + esac diff --git a/.github/workflows/prod-deploy.yml b/.github/workflows/prod-deploy.yml index 2d443b7d..9c7c6cb1 100644 --- a/.github/workflows/prod-deploy.yml +++ b/.github/workflows/prod-deploy.yml @@ -68,6 +68,23 @@ jobs: | xargs -P 4 -I{} curl -s -o /dev/null -w "%{http_code} %{time_total}s {}\n" --max-time 120 {} \ || echo "warm-up failed (non-blocking)" + # Warm /api/citable explicitly: the aggregator cache is independent + # from per-bench caches, and the bench-page warm-up only populates + # the latter. Without this step, /api/citable's first hit after + # deploy runs 26 parallel Prom queries on a cold function instance, + # which has historically produced mostly-placeholder snapshots that + # cached for 60s and shipped status=insufficient to LLM agents. + - name: Warm citable aggregate + run: | + base="https://openchainbench.com" + # /api/mcp is the docs path under /mcp, the live JSON-RPC + # endpoint is /api/mcp/mcp (Streamable HTTP). Warm the real + # endpoint so the first agent request after deploy is hot. + for ep in /api/citable /api/llm-context /api/mcp/mcp; do + code=$(curl -s -o /dev/null -w "%{http_code}" --max-time 120 "$base$ep" || echo "000") + echo "$code $base$ep" + done + - name: Summary run: | { diff --git a/.gitignore b/.gitignore index 32b1bcfe..b40a263f 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,9 @@ harnesses/*/script harnesses/*/monitor harnesses/*/cmd/script/script harnesses/*/cmd/monitor/monitor + +# Python venvs for the HF publisher and Space (created locally for dry-runs) +scripts/hf_publisher/.venv/ +scripts/hf_space/.venv/ +**/__pycache__/ +*.pyc diff --git a/alternatives/alchemy.yml b/alternatives/alchemy.yml index 97fd309c..68991814 100644 --- a/alternatives/alchemy.yml +++ b/alternatives/alchemy.yml @@ -5,7 +5,7 @@ description: Multi-chain node + enhanced API provider benchmark: aggregator-head-lag intro: | - Alchemy serves RPC endpoints and enhanced data APIs across EVM chains and Solana. For real-time feeds, the metric that matters most is freshness. how long between a transaction landing on chain and the API reflecting it? Below is the live head-lag for each major onchain data provider, measured against a canonical-tip archive node and refreshed every minute. Lower is faster. + Alchemy is an EVM-first node and API provider whose surface is shaped by its enhanced endpoints (`alchemy_getAssetTransfers`, NFT API, webhooks, getTokenBalances, the simulation and debug suite, the account-abstraction bundler) rather than by a raw RPC tuned for edge latency. Pricing is metered in compute units, which charges more for an enhanced call than for a plain `eth_call`, so the ceiling shows up first on teams streaming high-cardinality events into a backend rather than on dapp front-ends issuing single reads. Solana coverage exists but sits on a stack built around Ethereum and its L2s. Teams that need a live view of trades on Base, BNB Chain or Solana usually leave Alchemy at the point where the enhanced API list stops mapping to what the product actually reads from chain. seo_title: Alchemy alternatives. live head-lag benchmark across data providers seo_description: Compare Alchemy alternatives on real-time data freshness. Live head-lag against a canonical archive node, refreshed every minute and published openly. diff --git a/alternatives/bitquery.yml b/alternatives/bitquery.yml index 32564fbb..e3280c40 100644 --- a/alternatives/bitquery.yml +++ b/alternatives/bitquery.yml @@ -5,7 +5,7 @@ description: GraphQL API for blockchain data, multi-chain benchmark: network-coverage intro: | - Bitquery is a GraphQL data API covering many chains. If you're sizing alternatives, the first axis is breadth. how many networks does each provider officially support? Below is the live count from each major onchain data provider, scraped from their public supported-networks endpoint and refreshed every six hours. + Bitquery is a GraphQL surface on top of indexed onchain data, with the schema as the actual product: queries declare exactly the fields needed across transfers, trades, DEX events and account state, and the indexer fans those out across chains. Pricing is point-based, with each field, filter and join consuming from a monthly point budget rather than from a flat request quota, which means a single dashboard with many panels can hit the ceiling well before request volume looks high. Chain coverage is broad but uneven in depth: some networks expose full DEX trade indexing, others only basic transfers. The team usually compared next is the one whose chain list either covers a long-tail network Bitquery does not, or covers the same network with a flatter pricing model that does not penalise wide GraphQL fragments on a per-field basis. seo_title: Bitquery alternatives. live network coverage benchmark seo_description: Compare Bitquery alternatives on the number of blockchains each major onchain data provider officially supports. Live data, refreshed every six hours. diff --git a/alternatives/blocknative.yml b/alternatives/blocknative.yml deleted file mode 100644 index 2686aab2..00000000 --- a/alternatives/blocknative.yml +++ /dev/null @@ -1,13 +0,0 @@ -slug: blocknative -target_product: Blocknative -target_url: https://blocknative.com -description: Probability-of-inclusion gas oracle with EIP-1559 tiered predictions -benchmark: gas-estimation - -intro: | - Blocknative publishes a probability-of-inclusion gas oracle that emits EIP-1559 tiered predictions (70, 80, 90, 95, 99 percent confidence) for Ethereum and Polygon, served from a no-key endpoint `?chainid=`. The architecture is deliberate: an inclusion-confidence oracle over-predicts to guarantee landing, while a percentile tracker built on `eth_feeHistory` hugs the realized number by construction. Both choices have implications for a wallet or swap router picking the value to suggest to a user, which is what this benchmark surfaces. The harness polls each oracle at its tier-tolerant cadence (Blocknative every 12 seconds per chain), buffers the predicted priority fee with the predicted block height, and when that block is mined pulls the full block via `eth_getBlockByNumber(.., true)` on the chain's PublicNode RPC. Realized percentiles are computed from actual `maxPriorityFeePerGas` values across every included transaction, and the absolute error per (oracle, tier, chain) lands on both a gauge and a histogram. The table ranks on the p99 gap over 24 hours (the worst 1 percent of blocks) because at current fee levels the typical-minute gaps are fractions of a micro-gwei apart, while gas spikes are where wrong predictions actually cost money. A covered-rate column shows the share of time each prediction sat at or above the realized p50, the inclusion-side risk an absolute gap cannot show. Coverage is Ethereum and Polygon. Avalanche C-Chain was dropped because its auto-tuning fee market drives priority to zero across all oracles; BNB and OP-Stack L2s are excluded for structural reasons. The cohort competing with Blocknative includes PublicNode `eth_feeHistory`, Owlracle (multi-oracle aggregator on a 60-second free quota) and Etherscan v2's free tier. - -seo_title: Blocknative alternatives. Live gas oracle accuracy benchmark -seo_description: Compare Blocknative against PublicNode feeHistory, Owlracle and Etherscan on gwei gap between predicted and realized priority fee, ranked on p99 over 24h. - -status: live diff --git a/alternatives/coingecko.yml b/alternatives/coingecko.yml index afe7e614..6fc8c981 100644 --- a/alternatives/coingecko.yml +++ b/alternatives/coingecko.yml @@ -5,7 +5,7 @@ description: Crypto price + market data API, multi-chain benchmark: network-coverage intro: | - CoinGecko offers a broad price + market data REST API used across thousands of crypto apps. The first axis people compare alternatives on is breadth. how many blockchains does each provider officially support? Below is the live count from each major onchain data API, scraped directly from their public supported-networks endpoint and refreshed every six hours. + CoinGecko is a coin-centric price and market data REST API built around a curated asset list (the `/coins` universe) rather than around on-chain events. Tokens are added through a listing process, prices land on the API after that process completes, and the free tier is famously throttled (around 30 requests per minute on the demo key) which pushes any real product onto a paid plan within days of integration. The on-chain DEX side ships as a separate `/onchain` namespace inherited from GeckoTerminal, with its own quotas. Teams usually leave CoinGecko at one of two points: when the launch cadence of the tokens they care about outpaces the listing pipeline, or when the per-minute cap on the free tier forces a migration before the product is monetised enough to justify the enterprise tier. seo_title: CoinGecko alternatives. live network coverage benchmark seo_description: Compare CoinGecko alternatives on the number of blockchains each major onchain data API officially supports. Live data, refreshed every six hours. diff --git a/alternatives/etherscan.yml b/alternatives/etherscan.yml index 2d075c7f..5baa7b6d 100644 --- a/alternatives/etherscan.yml +++ b/alternatives/etherscan.yml @@ -5,9 +5,9 @@ description: Block explorer and gas tracker covering Ethereum and Polygon benchmark: gas-estimation intro: | - Etherscan is the most-visited block explorer on the web and publishes a gas tracker exposing `SafeGasPrice`, `ProposeGasPrice` and `FastGasPrice` via the v2 API on Ethereum and Polygon (free tier). The tiering was originally designed for the pre-EIP-1559 single-price world and is now mapped onto a priority-fee comparison alongside dedicated oracles. The question every wallet, swap router and bridge UI faces before sending a transaction is which gas oracle actually matches what the next block will charge on the chain their product runs on, rather than what a marketing page claims. This benchmark polls each oracle at its tier-tolerant cadence (Etherscan every 15 seconds with a global rate-gate enforcing at least 6 seconds between any two Etherscan requests across chains, because the no-key limit is 1 req per 5 s per IP shared), buffers the prediction with the predicted block height, and when that block is mined pulls the full block via `eth_getBlockByNumber(.., true)` on the chain's PublicNode RPC. Realized percentiles are computed from actual `maxPriorityFeePerGas` values, and the absolute error per (oracle, tier, chain) lands on both a gauge and a histogram. The ranking metric is the p99 gap over 24 hours, the worst 1 percent of blocks, because typical-minute gaps sit within fractions of a micro-gwei (economically indistinguishable noise) while gas spikes are where a wrong prediction either overpays or misses the block. A covered-rate column shows the share of time each prediction sat at or above the realized p50, the inclusion-side risk an absolute gap cannot show. The cohort competing with Etherscan includes Blocknative (probability-of-inclusion model), PublicNode `eth_feeHistory` (thin wrapper over EIP-1559 reward percentiles) and Owlracle (multi-oracle aggregator). + Etherscan is the most-visited block explorer on the web and publishes a gas tracker exposing `SafeGasPrice`, `ProposeGasPrice` and `FastGasPrice` via the v2 API on Ethereum and Polygon (free tier). The tiering was originally designed for the pre-EIP-1559 single-price world and is now mapped onto a priority-fee comparison alongside dedicated oracles. The question every wallet, swap router and bridge UI faces before sending a transaction is which gas oracle actually matches what the next block will charge on the chain their product runs on, rather than what a marketing page claims. This benchmark polls each oracle at its tier-tolerant cadence (Etherscan every 15 seconds with a global rate-gate enforcing at least 6 seconds between any two Etherscan requests across chains, because the no-key limit is 1 req per 5 s per IP shared), buffers the prediction with the predicted block height, and when that block is mined pulls the full block via `eth_getBlockByNumber(.., true)` on the chain's PublicNode RPC. Realized percentiles are computed from actual `maxPriorityFeePerGas` values, and the absolute error per (oracle, tier, chain) lands on both a gauge and a histogram. The ranking metric is the p99 gap over 24 hours, the worst 1 percent of blocks, because typical-minute gaps sit within fractions of a micro-gwei (economically indistinguishable noise) while gas spikes are where a wrong prediction either overpays or misses the block. A covered-rate column shows the share of time each prediction sat at or above the realized p50, the inclusion-side risk an absolute gap cannot show. The cohort competing with Etherscan includes PublicNode `eth_feeHistory` (thin wrapper over EIP-1559 reward percentiles) and Owlracle (multi-oracle aggregator). seo_title: Etherscan alternatives. Live gas oracle accuracy benchmark -seo_description: Compare Etherscan against Blocknative, PublicNode feeHistory and Owlracle on gwei gap between predicted and realized priority fee, ranked on p99 over 24h. +seo_description: Compare Etherscan against PublicNode feeHistory and Owlracle on gwei gap between predicted and realized priority fee, ranked on p99 over 24h. status: live diff --git a/alternatives/helius.yml b/alternatives/helius.yml deleted file mode 100644 index e5d9963b..00000000 --- a/alternatives/helius.yml +++ /dev/null @@ -1,13 +0,0 @@ -slug: helius -target_product: Helius -target_url: https://helius.dev -description: Solana RPC provider with a transaction landing service (Sender) -benchmark: solana-tx-landing-latency - -intro: | - Helius runs a Solana RPC provider with a dedicated transaction landing service called Sender, available in both default mode (with Jito fan-out) and `swqos_only=true` mode (own-path only, no Jito leg). The only question that matters to a Solana trader picking a landing service is how many slots a signed mainnet transaction takes to reach the confirmed state on chain, because Solana confirmation is a slot-level event and a 1-slot difference is roughly 400 ms (enough for a MEV bot to front-run a competitor). This benchmark probes five landing services from a us-east Railway region, once per hour, by submitting an identical signed mainnet transaction to each in parallel. The payload is a compute-budget instruction pair (50k CU limit, 50k micro-lamport/CU price), a 1-lamport self-transfer, the per-service tip transfer at a pre-registered floor and an OCB-prefixed memo for forensic traceability. The headline metric is slot delta (`land_slot` minus `submit_slot`) captured from the `signatureSubscribe` WebSocket notification's `context.slot` field at commitment level confirmed. Wall-clock milliseconds is published alongside as a derived approximation but the canonical RTT-independent number is the slot delta. Helius is probed in `swqos_only=true` mode against `ewr-sender.helius-rpc.com/fast` to isolate the own-path from the Jito leg, with a 10,000-lamport tip floor. Other services in the V0-Lean cohort: Jito (control baseline), Astralane Iris (tip-refund), Nozomi by Temporal Labs (1M-lamport hard floor) and the Mobula multi-RPC fan-out aggregator. p50 and p99 slot delta are computed over a rolling 7-day window (168 samples per service per cell). - -seo_title: Helius alternatives. Live Solana tx landing slot delta benchmark -seo_description: Compare Helius Sender against Jito, Astralane, Nozomi and Mobula on slot delta from submit to confirmed. Signed mainnet probes every hour from us-east, 7-day window. - -status: live diff --git a/alternatives/jito.yml b/alternatives/jito.yml deleted file mode 100644 index 08e00369..00000000 --- a/alternatives/jito.yml +++ /dev/null @@ -1,13 +0,0 @@ -slug: jito -target_product: Jito -target_url: https://jito.network -description: Solana block engine with atomic bundles and tip auction -benchmark: solana-tx-landing-latency - -intro: | - Jito operates a Solana block engine with atomic bundles and a tip auction, live since 2022, and acts as the baseline that every premium landing service must beat: Helius default mode, Astralane and Nozomi all internally route some flow through Jito, so a same-slot result against the Jito control on a given cycle means the suspect service is essentially using Jito as its inclusion path. The only question that matters when picking a landing service is how many slots a signed mainnet transaction takes to reach the confirmed state on chain, because Solana confirmation is a slot-level event and a 1-slot difference is roughly 400 ms (enough for a MEV bot to front-run a competitor). This benchmark probes five services from a us-east Railway region, once per hour, by submitting an identical signed mainnet transaction in parallel. The payload is a compute-budget instruction pair, a 1-lamport self-transfer, the per-service tip transfer at a pre-registered floor and an OCB-prefixed memo. Jito is probed at `ny.mainnet.block-engine.jito.wtf/api/v1/transactions` with a 10,000-lamport tip floor. The headline metric is slot delta (`land_slot` minus `submit_slot`) read from the `signatureSubscribe` notification's `context.slot` at commitment confirmed; wall-clock ms is published alongside as a derived approximation. p50 and p99 slot delta are computed over a rolling 7-day window (168 samples per service per cell). The cohort includes Helius Sender in `swqos_only` mode (isolated own-path), Astralane Iris (tip-refund mechanism), Nozomi by Temporal Labs (1M-lamport hard floor) and the Mobula multi-RPC fan-out aggregator. Each probe carries the same blockhash and a comparable tip so the slot delta column reads as a direct routing comparison rather than a tip-auction wallclock race. - -seo_title: Jito alternatives. Live Solana tx landing slot delta benchmark -seo_description: Compare Jito against Helius Sender, Astralane, Nozomi and Mobula on slot delta from submit to confirmed. Signed mainnet probes every hour from us-east, 7-day window. - -status: live diff --git a/alternatives/lifi.yml b/alternatives/lifi.yml index 9e04eaa2..997f3475 100644 --- a/alternatives/lifi.yml +++ b/alternatives/lifi.yml @@ -5,7 +5,7 @@ description: Cross-chain bridge + DEX aggregator benchmark: bridge-quote-latency intro: | - Li.Fi is a cross-chain aggregator that quotes routes across many bridges and DEXes. If you're sizing alternatives, quote latency is one of the most user-felt axes. how long does the aggregator take to return a usable route for a given pair? Below is the live measurement for the four major bridge aggregators on identical routes (Solana ↔ Base ↔ Arbitrum), across notional sizes ($5/$50/$300). Numbers refresh every five minutes. + Li.Fi is a cross-chain aggregator that stitches together third-party bridges (Stargate, Across, Connext, Hop and others) and DEX aggregators behind a single `/quote` endpoint. The quote response carries a full execution plan with calldata, approval steps and gas estimates, which is convenient for a wallet integration but means the latency reflects fan-out across many underlying providers plus the time to pick a winner. Each upstream has its own SLA, and a slow leg in the cohort drags the quote even when the chosen route is fast. The trade-off is breadth for tail-latency: Li.Fi covers a long tail of bridges that intent-routed competitors do not, but pays for that breadth on the p95 of its `/quote` call. Teams move off when the latency of the fastest route they actually use exceeds what a narrower router can return for the same pair. seo_title: Li.Fi alternatives. Debridge, Mobula & Relay live quote latency benchmark seo_description: Compare Li.Fi alternatives on cross-chain quote latency. Debridge, Mobula and Relay measured on identical routes, refreshed every five minutes. diff --git a/alternatives/pump-portal.yml b/alternatives/pump-portal.yml index fa18f631..f30f60a5 100644 --- a/alternatives/pump-portal.yml +++ b/alternatives/pump-portal.yml @@ -5,7 +5,7 @@ description: Real-time pump.fun token data API on Solana benchmark: aggregator-head-lag intro: | - Pump Portal is a real-time API serving pump.fun-launched tokens on Solana. websocket trade events, new pool detection, swap data. Looking for an alternative? Below is how the three major onchain data providers compare on the metric that matters most for this kind of feed: time between an on-chain event and its delivery on a WebSocket. Numbers are live, refreshed every minute, measured against a canonical-tip archive node. Lower is faster. + Pump Portal is a single-purpose WebSocket on Solana, scoped to pump.fun and the bonding-curve launchpad ecosystem around it (new token creations, migration to Raydium, per-mint trade events). The surface is intentionally narrow: no support for other chains, no general DEX coverage outside the pump.fun graduates and no historical query layer beyond the live stream. The free tier exists, but high-volume readers route through a paid trading endpoint that adds a per-trade priority fee on top. The provider sits close to the launchpad it tracks rather than near a canonical Solana archive, so the bottleneck is usually the upstream indexer rather than the WebSocket itself. Teams that outgrow the pump.fun-only scope (cross-DEX routing, non-Solana chains, OHLCV history) tend to drop it the moment a second venue or chain enters the product. seo_title: Pump Portal alternatives. Codex, GeckoTerminal & Mobula live latency benchmark seo_description: Looking for an alternative to Pump Portal? Compare Codex, GeckoTerminal and Mobula on real-time blockchain data latency, measured continuously and published openly. diff --git a/alternatives/quicknode.yml b/alternatives/quicknode.yml index b9d3ce8b..17ce396e 100644 --- a/alternatives/quicknode.yml +++ b/alternatives/quicknode.yml @@ -5,7 +5,7 @@ description: Multi-chain RPC + data infrastructure benchmark: aggregator-head-lag intro: | - QuickNode runs RPC endpoints, streams and data APIs across EVM chains, Solana and more. For any pipeline that consumes the chain in real time, the practical question is how quickly the provider sees a fresh block. Below is the live head-lag for each major onchain data provider, measured against a canonical-tip archive node and refreshed every minute. Lower is faster. + QuickNode runs a multi-chain RPC business plus a Marketplace of paid add-ons (token API, NFT API, streams, Yellowstone for Solana) that bolt onto the base node subscription. The pricing model layers per-add-on fees on top of the per-method credit metering, so the bill scales with the breadth of features turned on rather than just request volume; teams running a single workload often discover that two add-ons cost more than the base plan. Chain coverage is broad across EVM and Solana, with regional endpoints in major cloud zones, but the streams product (filter-based push) lives behind a separate quota from the RPC. Real-time price and swap pipelines often end up offloading the head-of-chain stream to a dedicated provider so the QuickNode bill stays on the dapp-read side of the workload. seo_title: QuickNode alternatives. live head-lag benchmark across data providers seo_description: Compare QuickNode alternatives on real-time data freshness. Live head-lag against a canonical archive node, refreshed every minute and published openly. diff --git a/alternatives/relay.yml b/alternatives/relay.yml index 73ebf4b4..f5cf12b5 100644 --- a/alternatives/relay.yml +++ b/alternatives/relay.yml @@ -5,7 +5,7 @@ description: Cross-chain bridge with intent-based routing benchmark: bridge-quote-latency intro: | - Relay is a cross-chain bridge that routes via solver intents. If you're sizing alternatives, quote latency is one of the most felt UX axes. how long does the bridge take to return a usable quote for a given route? Below is the live measurement for the four major bridges on identical routes (Solana ↔ Base ↔ Arbitrum), with multiple notional sizes ($5/$50/$300). Numbers refresh every five minutes. + Relay is an intent-based bridge: the user signs an order, a solver fronts the destination-chain funds and the cross-chain settlement happens off the user's critical path. The architecture compresses the user-felt latency because the quote does not have to scan many upstream venues; the solver inventory itself is the route. That same architecture exposes a different failure mode, which is solver depth: routes the solvers do not maintain inventory on either fail or fall back to a slower path, so coverage on long-tail pairs is narrower than a fan-out aggregator's. Fees are quoted inclusive of the solver spread rather than as a separate gas-plus-bridge breakdown. Teams that switch away usually do so for a chain pair the Relay solvers do not actively support, where a route that exists at all beats a fast quote that does not. seo_title: Relay alternatives. Debridge, Li.Fi & Mobula live quote latency benchmark seo_description: Compare Relay alternatives on cross-chain bridge quote latency. Debridge, Li.Fi and Mobula measured on identical routes, refreshed every five minutes. diff --git a/alternatives/the-graph.yml b/alternatives/the-graph.yml index d40abf0f..c1fbeb56 100644 --- a/alternatives/the-graph.yml +++ b/alternatives/the-graph.yml @@ -5,7 +5,7 @@ description: Decentralised indexing protocol with subgraph queries benchmark: network-coverage intro: | - The Graph powers subgraph queries used by many dApps for indexed onchain data. If you are evaluating alternatives, the first dimension worth checking is the breadth of supported networks. how many chains does each major data provider officially cover? Below is the live count from each provider's public supported-networks endpoint, refreshed every six hours. + The Graph is a decentralised indexing protocol where each dataset (a subgraph) is written, deployed and queried independently rather than read off a pre-built schema. Queries are GQL against indexer nodes, billed in GRT, with a hosted gateway in front. The model means data shape is owned by whoever wrote the subgraph: a missing field, a stale index or a deprecated mapping is on the publisher, not on a central API team. Chain coverage tracks what indexers have chosen to support, which leans heavily EVM and lags on newer L2s and non-EVM networks until someone publishes a subgraph for them. The two reasons teams move off are the cost of running a private subgraph at production load, and the operational burden of debugging an indexer regression on a chain the public hosted service does not cover. seo_title: The Graph alternatives. live network coverage benchmark seo_description: Compare The Graph alternatives on the number of blockchains each major onchain data provider officially supports. Live data, refreshed every six hours. diff --git a/answers/which-blockchain-has-cheapest-transaction-fees.yml b/answers/which-blockchain-has-cheapest-transaction-fees.yml index aa6aedeb..4248ab53 100644 --- a/answers/which-blockchain-has-cheapest-transaction-fees.yml +++ b/answers/which-blockchain-has-cheapest-transaction-fees.yml @@ -6,7 +6,7 @@ short_answer: | benchmark: network-fees intro: | - Wallet UX, micropayments, on-chain agents and gaming all break the same way: a five dollar transfer fee makes the use case impossible. This page answers one question with live data. Which chain is actually the cheapest to send a native transfer on right now, measured in dollars, not in gwei or lamports or stroops or sun. The OpenChainBench network-fees harness queries every chain's own fee market directly (eth_feeHistory for the EVM family, getRecentPrioritizationFees on Solana, koios epoch params on Cardano, fee_stats on Stellar, get_fee_estimate on Monero, Bag-of-Cells emulation on TON, getChainParameters on TRON, suix_getReferenceGasPrice on Sui), converts the result to the chain's smallest native unit, then multiplies by the live USD price of the native token. The output is the dollar amount a wallet user actually pays today. No marketing claim, no protocol-deterministic baseline that ignores priority bidding, no MATIC at peak gas reframed as the median. + Wallet UX, micropayments, on-chain agents and gaming all break the same way: a five dollar transfer fee makes the use case impossible. This page answers one question with live data. Which chain is actually the cheapest to send a native transfer on right now, measured in dollars, not in gwei or lamports or stroops or sun. The OpenChainBench network-fees harness queries every chain's own fee market directly (eth_feeHistory for the EVM family, getRecentPrioritizationFees on Solana, koios epoch params on Cardano, fee_stats on Stellar, get_fee_estimate on Monero, Bag-of-Cells emulation on Gram (formerly TON), getChainParameters on TRON, suix_getReferenceGasPrice on Sui), converts the result to the chain's smallest native unit, then multiplies by the live USD price of the native token. The output is the dollar amount a wallet user actually pays today. No marketing claim, no protocol-deterministic baseline that ignores priority bidding, no MATIC at peak gas reframed as the median. methodology: | Each chain is queried on its own fee surface every 30 seconds, never normalised to a synthetic gas estimate. The harness multiplies the chain's published fee parameter by a canonical transfer size for that chain (21000 gas for EVM native transfers, 5000 lamports base plus typical priority on Solana, 225 vBytes for a SegWit Litecoin transfer, 1500 bytes for a one-input two-output Monero RingCT transfer, 250 bytes for ADA, etc.), then converts to USD using Mobula's live price feed. The leaderboard ranks by 24h p50, so a single congestion spike does not move the headline number. Chains where the fee model is protocol-deterministic (Cardano, Stellar, TRON) are shown at the published rate parameter, which moves only when the parameter itself moves through on-chain governance. @@ -14,7 +14,7 @@ methodology: | limitations: - "Native transfers are the cheapest possible transaction on each chain. Smart-contract calls (token transfer, swap, NFT mint) cost more, often by an order of magnitude on the EVM family because of storage writes." - "L2 figures cover L2 execution cost only. The L1 data-posting component (Ethereum blob calldata, EigenDA for Mantle) is not yet included, so the actual user-visible cost during expensive blob windows is higher than this leaderboard shows." - - "TON publishes a conservative observed value because TON has no clean fee-estimate RPC; the real fee can fluctuate by workchain and shard." + - "Gram (formerly TON) publishes a conservative observed value because the Gram chain has no clean fee-estimate RPC; the real fee can fluctuate by workchain and shard." - "Native currency price volatility moves the USD figure independently of any change in the chain's fee market. A 20 percent ETH move shifts every Ethereum-denominated L1 and L2 transfer fee by the same percent." faq: diff --git a/answers/which-gas-oracle-is-the-most-accurate.yml b/answers/which-gas-oracle-is-the-most-accurate.yml index ee6e9b3f..3de2013e 100644 --- a/answers/which-gas-oracle-is-the-most-accurate.yml +++ b/answers/which-gas-oracle-is-the-most-accurate.yml @@ -1,18 +1,18 @@ slug: which-gas-oracle-is-the-most-accurate question: "Which Ethereum gas oracle is the most accurate in 2026?" short_answer: | - {{best_name}} currently posts the tightest priority fee prediction at {{best_p50}} (gwei, p99 absolute gap, 24h) on Ethereum mainnet across Blocknative, PublicNode feeHistory, Owlracle and Etherscan, ranked on the worst 1% of blocks where gas spikes actually cost money. + {{best_name}} currently posts the tightest priority fee prediction at {{best_p50}} (gwei, p99 absolute gap, 24h) on Ethereum mainnet across PublicNode feeHistory, Owlracle and Etherscan, ranked on the worst 1% of blocks where gas spikes actually cost money. benchmark: gas-estimation intro: | - Every wallet, swap router and bridge UI hits a gas oracle before posting a transaction, and the gap between the predicted priority fee and what the next mined block actually charges is the difference between a transaction landing in 12 seconds and a transaction sitting in the mempool through three gas spikes. Marketing pages quote "fast / standard / slow" tiers without ever publishing the gap between prediction and reality. This page answers one question with live data. Which oracle predicts the next block's realized priority fee most closely, measured per block, on Ethereum mainnet and Polygon PoS. The OpenChainBench gas-estimation harness polls Blocknative, PublicNode `eth_feeHistory`, Owlracle and Etherscan v2 at their tier tolerant cadences, buffers each prediction with its predicted block height, then pulls the full block via `eth_getBlockByNumber` when it lands and computes the realized priority percentile from the actual `maxPriorityFeePerGas` values across every included transaction. The headline ranks on the p99 of the absolute gap over 24 h, the worst 1% of blocks, because typical minute gaps are micro gwei noise while volatile minutes are where a wrong prediction either overpays or misses the block. + Every wallet, swap router and bridge UI hits a gas oracle before posting a transaction, and the gap between the predicted priority fee and what the next mined block actually charges is the difference between a transaction landing in 12 seconds and a transaction sitting in the mempool through three gas spikes. Marketing pages quote "fast / standard / slow" tiers without ever publishing the gap between prediction and reality. This page answers one question with live data. Which oracle predicts the next block's realized priority fee most closely, measured per block, on Ethereum mainnet and Polygon PoS. The OpenChainBench gas-estimation harness polls PublicNode `eth_feeHistory`, Owlracle and Etherscan v2 at their tier tolerant cadences, buffers each prediction with its predicted block height, then pulls the full block via `eth_getBlockByNumber` when it lands and computes the realized priority percentile from the actual `maxPriorityFeePerGas` values across every included transaction. The headline ranks on the p99 of the absolute gap over 24 h, the worst 1% of blocks, because typical minute gaps are micro gwei noise while volatile minutes are where a wrong prediction either overpays or misses the block. methodology: | - Each oracle is polled per chain at its tier tolerant cadence: Blocknative and PublicNode feeHistory every 12 s, Owlracle every 60 s (free quota ceiling), Etherscan every 15 s with a global rate gate enforcing 6 s between any two Etherscan requests across chains (the no key limit is 1 request per 5 s per IP, shared). Each oracle's named tiers (fast, standard, slow, safe, propose) are mapped onto a unified p25, p50, p75, p90, p99 scheme and the leaderboard ranks the p50 tier (the standard speed wallets use by default). Predicted priority fees are buffered with the predicted block height; when that block is mined, the harness pulls the full block via `eth_getBlockByNumber(.., true)` on the chain's PublicNode RPC, computes the realized percentile from actual `maxPriorityFeePerGas` values, and records the absolute error per (oracle, tier, chain) as both a gauge and a histogram. The headline metric is `quantile_over_time(0.99, gas_error_priority_gwei{tier="p50"}[24h])`. A covered rate column (share of time the prediction sat at or above the realized p50) surfaces the inclusion side risk an absolute gap cannot show. BNB Chain is excluded (not EIP-1559), Avalanche C-Chain is excluded (auto tuning fee market drives priority fee toward zero), L2 OP Stack chains are excluded (priority fee near zero because the sequencer is centralised, the relevant cost is L1 data fee). + Each oracle is polled per chain at its tier tolerant cadence: PublicNode feeHistory every 12 s, Owlracle every 60 s (free quota ceiling), Etherscan every 15 s with a global rate gate enforcing 6 s between any two Etherscan requests across chains (the no key limit is 1 request per 5 s per IP, shared). Each oracle's named tiers (fast, standard, slow, safe, propose) are mapped onto a unified p25, p50, p75, p90, p99 scheme and the leaderboard ranks the p50 tier (the standard speed wallets use by default). Predicted priority fees are buffered with the predicted block height; when that block is mined, the harness pulls the full block via `eth_getBlockByNumber(.., true)` on the chain's PublicNode RPC, computes the realized percentile from actual `maxPriorityFeePerGas` values, and records the absolute error per (oracle, tier, chain) as both a gauge and a histogram. The headline metric is `quantile_over_time(0.99, gas_error_priority_gwei{tier="p50"}[24h])`. A covered rate column (share of time the prediction sat at or above the realized p50) surfaces the inclusion side risk an absolute gap cannot show. BNB Chain is excluded (not EIP-1559), Avalanche C-Chain is excluded (auto tuning fee market drives priority fee toward zero), L2 OP Stack chains are excluded (priority fee near zero because the sequencer is centralised, the relevant cost is L1 data fee). limitations: - - "Lower gap is not the same as best oracle. Inclusion confidence oracles (Blocknative, Etherscan) over predict by design to guarantee inclusion; percentile trackers (PublicNode feeHistory, Owlracle) hug the realized number by construction. The leaderboard ranks on p99 to surface tail behaviour and pairs it with the covered column so over prediction (overpay, harmless) and under prediction (transaction misses its block) do not collapse into the same score." + - "Lower gap is not the same as best oracle. Inclusion confidence oracles (Etherscan) over predict by design to guarantee inclusion; percentile trackers (PublicNode feeHistory, Owlracle) hug the realized number by construction. The leaderboard ranks on p99 to surface tail behaviour and pairs it with the covered column so over prediction (overpay, harmless) and under prediction (transaction misses its block) do not collapse into the same score." - "Ethereum mainnet and Polygon PoS are the only ranked chains. BNB Chain is not EIP-1559 (gasPrice only). Avalanche C-Chain auto tunes its fee market toward zero priority fee, which collapses the error metric to noise. L2 OP Stack chains (Optimism, Base, Arbitrum) have priority fee near zero because the sequencer is centralised; the meaningful cost there is the L1 data fee, a different prediction problem." - "The p50 tier is the ranked tier (standard wallet default). Per tier breakdowns (p25, p75, p90, p99) are emitted by the harness and visible in raw Prometheus, but the bench UI does not yet surface them as a tab because the spec schema is restricted to chain and region dimensions." - "Realized percentile is noisy on empty or low transaction count blocks. The harness flags those via `gas_realized_tx_count{chain}` so a low traffic block does not silently inflate the error histogram." @@ -20,11 +20,11 @@ limitations: faq: - q: "Which gas oracle is the most accurate right now?" - a: "{{best_name}} currently leads at {{best_p50}} gwei (p99 absolute gap, 24h) on the active chain tab across 4 measured oracles. The leaderboard refreshes every minute against fresh Prometheus samples; per chain leaders can differ from the cross chain headline so the chain tabs at the top of the bench page expose each ranking separately. Read the gap alongside the covered rate column for the full picture, inclusion confidence oracles trade a wider gap for higher coverage." + a: "{{best_name}} currently leads at {{best_p50}} gwei (p99 absolute gap, 24h) on the active chain tab across 3 measured oracles. The leaderboard refreshes every minute against fresh Prometheus samples; per chain leaders can differ from the cross chain headline so the chain tabs at the top of the bench page expose each ranking separately. Read the gap alongside the covered rate column for the full picture, inclusion confidence oracles trade a wider gap for higher coverage." - q: "Why rank on p99 instead of p50?" a: "At current fee levels the typical minute p50 gaps are fractions of a micro gwei apart across oracles. On a 100k gas transaction a 0.001 gwei error is about a thousandth of a cent, so a p50 ranking orders economically indistinguishable noise. The p99 captures the volatile minutes (mempool spikes, NFT mints, MEV bursts) where predictions actually diverge and a wrong number either overpays or misses the block. The typical p50 and p90 gaps remain visible as secondary columns." - - q: "How does Blocknative compare to Etherscan or PublicNode feeHistory?" - a: "Blocknative publishes a probability of inclusion model across 70, 80, 90, 95 and 99 percent confidence tiers; the parser maps those onto the unified p25 to p99 scheme. Etherscan's `ProposeGasPrice` was designed for the pre EIP-1559 single price world and re uses tier names that map onto the same scheme. PublicNode feeHistory is a thin wrapper over the EIP-1559 reward percentile spec, so it tracks the realized number closely by construction. The three answer different design questions: inclusion confidence, retail recognition, and protocol grounded reading." + - q: "How does Etherscan compare to PublicNode feeHistory or Owlracle?" + a: "Etherscan's `ProposeGasPrice` was designed for the pre EIP-1559 single price world and re uses tier names mapped onto the unified p25 to p99 scheme; it over predicts by design to guarantee inclusion. PublicNode feeHistory is a thin wrapper over the EIP-1559 reward percentile spec, so it tracks the realized number closely by construction. Owlracle aggregates several upstream oracles into a single recommendation, polled every 60 s on the free quota. The three answer different design questions: retail recognition with inclusion bias, protocol grounded reading, and aggregated consensus." - q: "What does the covered rate column mean?" a: "The share of time over 24 h that the oracle's posted p50 tier prediction was at or above the realized p50 priority fee of the latest mined block. Absolute gap treats over prediction and under prediction the same, but they are not symmetric for users: over predicting means slightly overpaying, under predicting means a transaction paying exactly the predicted fee would have ranked below the block's median and risks waiting. Read gap and covered together; low p99 gap plus high covered is the sweet spot." - q: "Why is Avalanche C-Chain excluded?" @@ -32,7 +32,7 @@ faq: - q: "Which oracle should I integrate for my wallet?" a: "Read the p99 gap (does it blow out during spikes?) together with the covered rate (does it err on the side of inclusion or under bid?) for the chain your product runs on. {{best_name}} currently leads the active tab at {{best_p50}} gwei but the right choice depends on whether your UX tolerates over pay (then prefer high covered) or optimises for tail behaviour (then prefer low p99 gap), and whether the oracle's free quota fits your call volume. The bench gives you the live numbers, it cannot tell you which trade off your product wants." - q: "How often is the leaderboard refreshed?" - a: "Per oracle polling runs continuously (12 s for Blocknative and PublicNode feeHistory, 15 s for Etherscan with the 6 s global gate, 60 s for Owlracle). The page reads a rolling 24 h p50, p90 and p99 every minute. A single misprediction during a gas spike cannot move the headline because the p99 window absorbs it across thousands of blocks." + a: "Per oracle polling runs continuously (12 s for PublicNode feeHistory, 15 s for Etherscan with the 6 s global gate, 60 s for Owlracle). The page reads a rolling 24 h p50, p90 and p99 every minute. A single misprediction during a gas spike cannot move the headline because the p99 window absorbs it across thousands of blocks." related: - which-blockchain-has-cheapest-transaction-fees @@ -40,5 +40,5 @@ related: - which-evm-aggregator-has-the-fastest-quote seo_title: "Which Ethereum gas oracle is the most accurate in 2026?" -seo_description: "{{best_name}} leads at {{best_p50}} gwei (p99 absolute gap, 24h) across Blocknative, PublicNode feeHistory, Owlracle and Etherscan on Ethereum and Polygon, measured per block by OpenChainBench." +seo_description: "{{best_name}} leads at {{best_p50}} gwei (p99 absolute gap, 24h) across PublicNode feeHistory, Owlracle and Etherscan on Ethereum and Polygon, measured per block by OpenChainBench." status: live diff --git a/answers/which-l1-has-the-fastest-finality.yml b/answers/which-l1-has-the-fastest-finality.yml index 85504587..b8c4fcfe 100644 --- a/answers/which-l1-has-the-fastest-finality.yml +++ b/answers/which-l1-has-the-fastest-finality.yml @@ -6,16 +6,16 @@ short_answer: | benchmark: l1-finality intro: | - Finality is the point at which a confirmed transaction cannot be reversed without breaking the protocol's security assumptions. It is the right number to look at for cross-chain settlement, exchange withdrawal thresholds, bridge unlock conditions and any system that needs to know "is this real now or can it still be undone." This page answers one question with live measurement. Which L1 chain actually finalizes the fastest in wall-clock seconds, not in slot counts or in marketing claims. OpenChainBench measures every L1 against its own native finality definition: deterministic for BFT chains (Ethereum Casper FFG, Stellar SCP, Hedera Hashgraph, SUI Mysticeti, TON BAG) and convention-based for probabilistic chains (Bitcoin and similar by confirmation depth). The leaderboard ranks by 24h p50 wall-clock seconds. + Finality is the point at which a confirmed transaction cannot be reversed without breaking the protocol's security assumptions. It is the right number to look at for cross-chain settlement, exchange withdrawal thresholds, bridge unlock conditions and any system that needs to know "is this real now or can it still be undone." This page answers one question with live measurement. Which L1 chain actually finalizes the fastest in wall-clock seconds, not in slot counts or in marketing claims. OpenChainBench measures every L1 against its own native finality definition: deterministic for BFT chains (Ethereum Casper FFG, Stellar SCP, Hedera Hashgraph, SUI Mysticeti, Gram BAG) and convention-based for probabilistic chains (Bitcoin and similar by confirmation depth). The leaderboard ranks by 24h p50 wall-clock seconds. methodology: | - Two methods are used, picked per chain. For chains whose finality is much longer than our 10 second poll interval (Ethereum, Solana, TRON, Litecoin, Monero) the harness compares `latest` and `finalized` block timestamps from the chain RPCs and takes the delta. For chains whose finality is faster than the poll interval (BNB, Avalanche, SUI, TON, Stellar) the harness maintains a persistent WebSocket or SSE subscription, recording wall-clock time T1 when block N is first seen as `latest` and T2 when it becomes `finalized`, with millisecond precision. The WS path is the only honest way to measure sub-poll-cadence finality; comparing two pointers at a single instant collapses to zero when finalization catches up to the head. Probabilistic chains (Litecoin, Monero) are measured against their conventional confirmation depth, not against a uniform depth across all chains. + Two methods are used, picked per chain. For chains whose finality is much longer than our 10 second poll interval (Ethereum, Solana, TRON, Litecoin, Monero) the harness compares `latest` and `finalized` block timestamps from the chain RPCs and takes the delta. For chains whose finality is faster than the poll interval (BNB, Avalanche, SUI, Gram, Stellar) the harness maintains a persistent WebSocket or SSE subscription, recording wall-clock time T1 when block N is first seen as `latest` and T2 when it becomes `finalized`, with millisecond precision. The WS path is the only honest way to measure sub-poll-cadence finality; comparing two pointers at a single instant collapses to zero when finalization catches up to the head. Probabilistic chains (Litecoin, Monero) are measured against their conventional confirmation depth, not against a uniform depth across all chains. limitations: - "Hedera's documented 3 to 5 second Hashgraph aBFT finality is not on the live leaderboard. Hedera mirror nodes only expose already-final blocks, so wall-clock measurement is impossible from public endpoints; re-enables once Block Nodes (HIP-1056) leave private preview." - "Probabilistic chains (Litecoin, Monero) settle on a confirmation-depth convention. The reported value is the time to that depth, which is the exchange-style settlement threshold, not a strict protocol-level finality." - "XRP is excluded by design: ledger_current has no close-time field, so wall-clock measurement requires a WS subscription to the ledger stream which is not yet implemented." - - "Sub-second finalities on TON and SUI are real because both chains expose millisecond-precision timestamps. Sub-second finalities claimed by chains without millisecond timestamps would be a measurement artifact, not a protocol property." + - "Sub-second finalities on Gram (formerly TON) and SUI are real because both chains expose millisecond-precision timestamps. Sub-second finalities claimed by chains without millisecond timestamps would be a measurement artifact, not a protocol property." faq: - q: "What is blockchain finality?" @@ -24,8 +24,8 @@ faq: a: "No. Solana exposes two commitments. Processed is optimistic and lands sub-second, typically under 500 milliseconds. Finalized requires 32 confirmed slots and clocks closer to 13 seconds in practice. The headline value here is finalized, the stricter guarantee. Sub-second Solana finality is real but only at the processed commitment level." - q: "Why is Ethereum finality 12.8 minutes?" a: "Ethereum uses Casper FFG which finalizes a checkpoint two epochs after it is justified. Each epoch is 32 slots of 12 seconds, giving 12.8 minutes as the documented target. Reorgs of unfinalized blocks remain possible inside the two-epoch window, but a finalized block is treated as irreversible by every Ethereum client." - - q: "How is TON able to finalize so fast?" - a: "TON's BAG consensus pushes masterchain finality under one second by separating state across a masterchain and many workchains. The number reported here is the masterchain commit, which is the canonical reference for cross-chain settlement. Workchain blocks are also finalized but only after the masterchain commit references them." + - q: "How is Gram (formerly TON) able to finalize so fast?" + a: "Gram's BAG consensus pushes masterchain finality under one second by separating state across a masterchain and many workchains. The number reported here is the masterchain commit, which is the canonical reference for cross-chain settlement. Workchain blocks are also finalized but only after the masterchain commit references them." - q: "Can I trust a sub-second L1 finality for cross-chain settlement?" a: "It depends on what you are settling. For bridges and exchanges with their own deeper confirmation thresholds, sub-second native finality means you reach those internal thresholds faster, not that you can skip them. For protocols that consume finality directly (e.g. LayerZero finality oracle, IBC client), the native value is what matters and the sub-second chains genuinely settle in under a second." diff --git a/answers/which-solana-rpc-lands-the-most-transactions.yml b/answers/which-solana-rpc-lands-the-most-transactions.yml deleted file mode 100644 index 6921b2f1..00000000 --- a/answers/which-solana-rpc-lands-the-most-transactions.yml +++ /dev/null @@ -1,38 +0,0 @@ -slug: which-solana-rpc-lands-the-most-transactions -question: "Which Solana RPC provider lands the most transactions in 2026?" -short_answer: | - {{best_name}} currently leads Solana transaction landing latency at {{best_p50}} (p50, 24h), the lowest slot-delta between transaction submission and confirmation across the measured RPC field. - -benchmark: solana-tx-landing-latency - -intro: | - Solana trading bots, MEV searchers and on-chain settlement all live or die on the same metric: how reliably and quickly does the RPC endpoint actually land the transaction on a leader's block. Marketing pages publish landing-rate numbers; almost none publish methodology or a live, neutral comparison. This page answers the question that wallet integrations, agent infrastructure and trading desks ask before pasting a URL into production. Which RPC provider is actually landing transactions the fastest right now, measured in slot delta between submission and confirmation, with a probe that runs continuously from multiple regions against the same canonical leader schedule. - -methodology: | - The harness submits a self-signed compute-unit-cheap transaction every few seconds through each RPC provider's submission endpoint, then watches a canonical archive node for the resulting confirmation. The landing latency is the wall-clock slot delta between submission and confirmation, expressed in milliseconds at Solana's 400 ms slot interval. The p50 over 24h is the headline metric; p99 captures the worst 1 percent of cases, where a provider's regional infrastructure or leader proximity surfaces clearly. Probes run from US-East, EU-West and Singapore against the same canonical archive node so any geographic asymmetry shows up as a per-region split, not as a noise floor on the aggregate. - -limitations: - - "Slot delta is not the same as fee. A provider can land transactions fastest while charging a per-transaction priority fee through Jito or a similar bundler; cost-per-landed-transaction is a composite metric the leaderboard does not currently surface." - - "Self-signed test transactions do not exercise the full priority-fee mempool. A real production transaction with a high priority fee and CU budget lands faster than the probes shown here, and the relative ordering can shift when paying for inclusion." - - "Provider landing performance shifts with Solana validator leader schedule. A provider with relayers physically close to today's leader can outperform on this window and lose its lead next epoch when the schedule rotates." - - "This is not a stake-weighted measurement. The harness measures wall-clock landing time at the canonical archive node level, not the share of stake reached at each submission." - -faq: - - q: "What does landing latency actually measure?" - a: "Wall-clock milliseconds between the moment a probe submits a self-signed transaction to a Solana RPC and the moment a canonical archive node sees the same transaction in a confirmed block. Lower is faster. The number is the time from your code calling send to the network treating the transaction as included." - - q: "Why is this different from Solana block time?" - a: "Solana block time is the chain's slot interval, fixed at 400 milliseconds. Landing latency is the time your transaction takes to reach the leader plus the leader's time to include it plus the propagation back to a canonical observer. The chain produces a slot every 400 milliseconds whether or not your transaction lands in it; the question this page answers is which provider's path gets you into the next available slot most consistently." - - q: "Does Jito's bundler beat raw RPC landing?" - a: "On the measured probes, Jito bundling is treated as a provider option, not as a separate metric. When the harness submits through a Jito-aware provider with bundle inclusion enabled, the path includes the bundler. The leaderboard surfaces both Jito and non-Jito providers in the same field so the relative cost of bundling is visible." - - q: "What regions are the probes from?" - a: "US-East, EU-West and Singapore. Cross-region probes catch providers whose landing performance is asymmetric across geography (an RPC fast from EU but slow from APAC is common). The leaderboard reports the cross-region p50; the per-region breakdown is on the bench page." - - q: "Why not measure with my own real workload?" - a: "Real workloads are the ground truth, but they are not comparable across providers because they carry different priority fees, different program calls, and run from different infrastructure. The harness controls for those variables to publish a fair cross-provider comparison; for your specific workload, run the same harness yourself (it is open source) and compare." - -related: - - which-blockchain-has-cheapest-transaction-fees - - which-l1-has-the-fastest-finality - -seo_title: "Which Solana RPC provider lands the most transactions in 2026?" -seo_description: "{{best_name}} leads Solana transaction landing at {{best_p50}} slot delta (p50, 24h) measured live by OpenChainBench. Methodology, regional probes and limitations on this page." -status: live diff --git a/benchmarks/bridge-fee.yml b/benchmarks/bridge-fee.yml index 50a928e9..c50e2600 100644 --- a/benchmarks/bridge-fee.yml +++ b/benchmarks/bridge-fee.yml @@ -3,14 +3,26 @@ slug: bridge-fee number: "003" title: Cheapest cross-chain bridge for USDC at $300 notional -seo_title: "Cheapest cross-chain bridge 2026: deBridge, LI.FI, Mobula, Relay" -seo_description: "Cheapest cross-chain bridge for USDC at $300 notional. Total cost (fees, slippage, dest gas) across Solana, Base, Arbitrum. deBridge, LI.FI, Mobula, Relay." +seo_title: "Cheapest cross-chain bridge 2026: deBridge, LI.FI, Mobula, Relay, Near Intents" +seo_description: "Cheapest cross-chain bridge for USDC at $300 notional. Total cost (fees, slippage, dest gas) across Solana, Base, Arbitrum. deBridge, LI.FI, Mobula, Relay, Near Intents." subtitle: Total cost as a percent of notional, fees plus slippage plus destination gas combined, sampled at $300 USDC across Solana, Base and Arbitrum corridors. category: Bridges status: live metric: Effective fee unit: pct +# Per-destination breakdown. Same rationale as bridge-quote-latency: +# bridges quote different cost structures per corridor (gas on the +# destination dominates short hops, solver spread dominates long +# auction routes), so an aggregate cross-corridor cost percent +# obscures more than it reveals. +dimensions: + chain: + - { value: Base, label: Sol to Base } + - { value: Arbitrum, label: Base to Arb } + - { value: Solana, label: Arb to Sol } + - { value: HyperCore, label: Arb to HyperCore } + seo_intro: | This benchmark measures the cheapest cross-chain bridge live, refreshed every five minutes across the major USDC corridors. The headline figure @@ -106,7 +118,7 @@ providers: p90: quantile_over_time(0.90, bridge_cost_percent{bridge="mobula", amount_usd="300"}[24h]) p99: quantile_over_time(0.99, bridge_cost_percent{bridge="mobula", amount_usd="300"}[24h]) mean: avg_over_time(bridge_cost_percent{bridge="mobula", amount_usd="300"}[24h]) - success: avg_over_time(bridge_quote_success{bridge="mobula", amount_usd="300"}[24h]) + success: avg(avg_over_time(bridge_quote_success{bridge="mobula", amount_usd="300"}[24h])) sample_size: sum(count_over_time(bridge_cost_percent{bridge="mobula", amount_usd="300"}[24h])) series: avg_over_time(bridge_cost_percent{bridge="mobula", amount_usd="300"}[1h]) @@ -120,7 +132,7 @@ providers: p90: quantile_over_time(0.90, bridge_cost_percent{bridge="relay", amount_usd="300"}[24h]) p99: quantile_over_time(0.99, bridge_cost_percent{bridge="relay", amount_usd="300"}[24h]) mean: avg_over_time(bridge_cost_percent{bridge="relay", amount_usd="300"}[24h]) - success: avg_over_time(bridge_quote_success{bridge="relay", amount_usd="300"}[24h]) + success: avg(avg_over_time(bridge_quote_success{bridge="relay", amount_usd="300"}[24h])) sample_size: sum(count_over_time(bridge_cost_percent{bridge="relay", amount_usd="300"}[24h])) series: avg_over_time(bridge_cost_percent{bridge="relay", amount_usd="300"}[1h]) @@ -134,7 +146,7 @@ providers: p90: quantile_over_time(0.90, bridge_cost_percent{bridge="lifi", amount_usd="300"}[24h]) p99: quantile_over_time(0.99, bridge_cost_percent{bridge="lifi", amount_usd="300"}[24h]) mean: avg_over_time(bridge_cost_percent{bridge="lifi", amount_usd="300"}[24h]) - success: avg_over_time(bridge_quote_success{bridge="lifi", amount_usd="300"}[24h]) + success: avg(avg_over_time(bridge_quote_success{bridge="lifi", amount_usd="300"}[24h])) sample_size: sum(count_over_time(bridge_cost_percent{bridge="lifi", amount_usd="300"}[24h])) series: avg_over_time(bridge_cost_percent{bridge="lifi", amount_usd="300"}[1h]) @@ -148,6 +160,20 @@ providers: p90: quantile_over_time(0.90, bridge_cost_percent{bridge="debridge", amount_usd="300"}[24h]) p99: quantile_over_time(0.99, bridge_cost_percent{bridge="debridge", amount_usd="300"}[24h]) mean: avg_over_time(bridge_cost_percent{bridge="debridge", amount_usd="300"}[24h]) - success: avg_over_time(bridge_quote_success{bridge="debridge", amount_usd="300"}[24h]) + success: avg(avg_over_time(bridge_quote_success{bridge="debridge", amount_usd="300"}[24h])) sample_size: sum(count_over_time(bridge_cost_percent{bridge="debridge", amount_usd="300"}[24h])) series: avg_over_time(bridge_cost_percent{bridge="debridge", amount_usd="300"}[1h]) + + - slug: near-intents + name: Near Intents + tag: Intent layer (NEAR) + formula: "Median over 24h of total cost percent (solver spread + bridge fee) on a $300 USDC quote returned by Near Intents 1Click API, sampled every 5 minutes from eu-west." + type: intent + queries: + p50: quantile_over_time(0.50, bridge_cost_percent{bridge="near-intents", amount_usd="300"}[24h]) + p90: quantile_over_time(0.90, bridge_cost_percent{bridge="near-intents", amount_usd="300"}[24h]) + p99: quantile_over_time(0.99, bridge_cost_percent{bridge="near-intents", amount_usd="300"}[24h]) + mean: avg_over_time(bridge_cost_percent{bridge="near-intents", amount_usd="300"}[24h]) + success: avg(avg_over_time(bridge_quote_success{bridge="near-intents", amount_usd="300"}[24h])) + sample_size: sum(count_over_time(bridge_cost_percent{bridge="near-intents", amount_usd="300"}[24h])) + series: avg_over_time(bridge_cost_percent{bridge="near-intents", amount_usd="300"}[1h]) diff --git a/benchmarks/bridge-quote-latency.yml b/benchmarks/bridge-quote-latency.yml index 3f1f4a55..7fea30c9 100644 --- a/benchmarks/bridge-quote-latency.yml +++ b/benchmarks/bridge-quote-latency.yml @@ -3,14 +3,28 @@ slug: bridge-quote-latency number: "002" title: Fastest cross-chain bridge quote API, live ms ranking -seo_title: "Fastest bridge quote API 2026: Mobula, deBridge, Relay, LI.FI" -seo_description: "{{best_name}} leads fastest bridge quote API at {{best_p50}} (p50, 24h). Mobula, deBridge, Relay, LI.FI on identical USDC routes, refreshed every 5 minutes." -subtitle: Time to receive a usable cross-chain quote, in milliseconds. Identical route and identical notional, measured every five minutes across Mobula, deBridge, Relay and LI.FI. +seo_title: "Fastest bridge quote API 2026: Mobula, deBridge, Relay, LI.FI, Near Intents" +seo_description: "{{best_name}} leads fastest bridge quote API at {{best_p50}} (p50, 24h). Mobula, deBridge, Relay, LI.FI, Near Intents on identical USDC routes, refreshed every 5 minutes." +subtitle: Time to receive a usable cross-chain quote, in milliseconds. Identical route and identical notional, measured every five minutes across Mobula, deBridge, Relay, LI.FI and Near Intents. category: Bridges status: live metric: Quote latency unit: ms +# Per-destination breakdown. The bridge-monitor harness emits the chain +# label set to route.ToChain (capitalized). Each bridge has wildly +# different per-corridor latency profiles (Near Intents in particular is +# bimodal: solver-cached corridors return in ~30ms while uncached ones +# wait the full 3s solver auction window) so aggregating across all +# destinations produces meaningless cross-corridor averages. The "All" +# tab keeps the aggregate view; the per-chain tabs reveal the truth. +dimensions: + chain: + - { value: Base, label: Sol to Base } + - { value: Arbitrum, label: Base to Arb } + - { value: Solana, label: Arb to Sol } + - { value: HyperCore, label: Arb to HyperCore } + seo_intro: | This benchmark measures how fast each cross-chain bridge quote API returns a usable response, the response-time half of the bridge story @@ -97,7 +111,7 @@ providers: p90: histogram_quantile(0.90, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="mobula"}[24h]))) p99: histogram_quantile(0.99, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="mobula"}[24h]))) mean: sum(rate(bridge_quote_latency_ms_sum{bridge="mobula"}[24h])) / sum(rate(bridge_quote_latency_ms_count{bridge="mobula"}[24h])) - success: avg_over_time(bridge_quote_success{bridge="mobula"}[24h]) + success: avg(avg_over_time(bridge_quote_success{bridge="mobula"}[24h])) sample_size: sum(increase(bridge_quote_latency_ms_count{bridge="mobula"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="mobula"}[1h]))) @@ -110,7 +124,7 @@ providers: p90: histogram_quantile(0.90, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="relay"}[24h]))) p99: histogram_quantile(0.99, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="relay"}[24h]))) mean: sum(rate(bridge_quote_latency_ms_sum{bridge="relay"}[24h])) / sum(rate(bridge_quote_latency_ms_count{bridge="relay"}[24h])) - success: avg_over_time(bridge_quote_success{bridge="relay"}[24h]) + success: avg(avg_over_time(bridge_quote_success{bridge="relay"}[24h])) sample_size: sum(increase(bridge_quote_latency_ms_count{bridge="relay"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="relay"}[1h]))) @@ -123,7 +137,7 @@ providers: p90: histogram_quantile(0.90, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="lifi"}[24h]))) p99: histogram_quantile(0.99, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="lifi"}[24h]))) mean: sum(rate(bridge_quote_latency_ms_sum{bridge="lifi"}[24h])) / sum(rate(bridge_quote_latency_ms_count{bridge="lifi"}[24h])) - success: avg_over_time(bridge_quote_success{bridge="lifi"}[24h]) + success: avg(avg_over_time(bridge_quote_success{bridge="lifi"}[24h])) sample_size: sum(increase(bridge_quote_latency_ms_count{bridge="lifi"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="lifi"}[1h]))) @@ -136,6 +150,19 @@ providers: p90: histogram_quantile(0.90, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="debridge"}[24h]))) p99: histogram_quantile(0.99, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="debridge"}[24h]))) mean: sum(rate(bridge_quote_latency_ms_sum{bridge="debridge"}[24h])) / sum(rate(bridge_quote_latency_ms_count{bridge="debridge"}[24h])) - success: avg_over_time(bridge_quote_success{bridge="debridge"}[24h]) + success: avg(avg_over_time(bridge_quote_success{bridge="debridge"}[24h])) sample_size: sum(increase(bridge_quote_latency_ms_count{bridge="debridge"}[24h])) series: histogram_quantile(0.50, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="debridge"}[1h]))) + + - slug: near-intents + name: Near Intents + tag: Intent layer (NEAR) + formula: "Median wall-clock ms over 24h for Near Intents 1Click API to return a usable USDC quote via the solver auction bus, sampled every 5 minutes across the supported routes and 3 notionals from eu-west." + queries: + p50: histogram_quantile(0.50, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="near-intents"}[24h]))) + p90: histogram_quantile(0.90, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="near-intents"}[24h]))) + p99: histogram_quantile(0.99, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="near-intents"}[24h]))) + mean: sum(rate(bridge_quote_latency_ms_sum{bridge="near-intents"}[24h])) / sum(rate(bridge_quote_latency_ms_count{bridge="near-intents"}[24h])) + success: avg(avg_over_time(bridge_quote_success{bridge="near-intents"}[24h])) + sample_size: sum(increase(bridge_quote_latency_ms_count{bridge="near-intents"}[24h])) + series: histogram_quantile(0.50, sum by (le) (rate(bridge_quote_latency_ms_bucket{bridge="near-intents"}[1h]))) diff --git a/benchmarks/gas-estimation.yml b/benchmarks/gas-estimation.yml index 64164e3c..1f28abe7 100644 --- a/benchmarks/gas-estimation.yml +++ b/benchmarks/gas-estimation.yml @@ -3,7 +3,7 @@ slug: gas-estimation number: "013" title: Most accurate gas oracle, live gap vs realized priority fee -seo_title: "Most accurate gas oracle 2026: Blocknative, Etherscan, Owlracle" +seo_title: "Most accurate gas oracle 2026: Etherscan, Owlracle, PublicNode" seo_description: "Most accurate gas oracle ranked live on Ethereum and Polygon. Gwei gap between predicted and realized priority fee in the next mined block. Ranked on the p99 gap over 24h." subtitle: Absolute gap in gwei between each oracle's predicted priority-fee tier and the realized percentile in the next mined block, measured per chain. Ranked on the p99 gap, the worst 1% of blocks, because that is where gas spikes hurt integrations. @@ -11,7 +11,7 @@ per_chain_explainer: - slug: ethereum h2: "Most accurate Ethereum gas oracle" body: | - {{best_name:chain:ethereum}} currently leads the Ethereum gas-oracle field at {{best_p50:chain:ethereum}} (gwei, p99 absolute gap, 24h) across 4 oracles. Ethereum's EIP-1559 base fee adjusts by up to 12.5 percent per block based on prior gas usage, so percentile trackers (PublicNode `eth_feeHistory`) hug the realized priority fee by construction while inclusion-confidence oracles (Blocknative, Etherscan) deliberately over-predict. Realized priority percentiles are computed per next-mined block from actual `maxPriorityFeePerGas` values. + {{best_name:chain:ethereum}} currently leads the Ethereum gas-oracle field at {{best_p50:chain:ethereum}} (gwei, p99 absolute gap, 24h) across 3 oracles. Ethereum's EIP-1559 base fee adjusts by up to 12.5 percent per block based on prior gas usage, so percentile trackers (PublicNode `eth_feeHistory`) hug the realized priority fee by construction while inclusion-confidence oracles (Etherscan) deliberately over-predict. Realized priority percentiles are computed per next-mined block from actual `maxPriorityFeePerGas` values. - slug: polygon h2: "Most accurate Polygon gas oracle" body: | @@ -24,7 +24,7 @@ unit: count higher_is_better: false disclaimer: | - Lower gap is NOT the same as "best oracle". Inclusion-confidence oracles (Blocknative, Etherscan) over-predict by design to guarantee inclusion; percentile trackers (PublicNode, Owlracle) hug the realized number by construction. The table therefore ranks on the p99 gap (typical p50 gaps are micro-gwei noise) and pairs it with the covered column: share of time the prediction sat at or above the realized p50, the inclusion-side risk a gap alone cannot show. + Lower gap is NOT the same as "best oracle". Inclusion-confidence oracles (Etherscan) over-predict by design to guarantee inclusion; percentile trackers (PublicNode, Owlracle) hug the realized number by construction. The table therefore ranks on the p99 gap (typical p50 gaps are micro-gwei noise) and pairs it with the covered column: share of time the prediction sat at or above the realized p50, the inclusion-side risk a gap alone cannot show. seo_intro: | This benchmark answers the question wallets, swap routers and @@ -42,8 +42,8 @@ seo_intro: | actually diverge. A covered-rate column shows the share of time each prediction sat at or above the realized p50 (the inclusion-side risk an absolute gap cannot show). Coverage. - Ethereum mainnet (Blocknative + PublicNode feeHistory + Owlracle + - Etherscan v2) and Polygon (same four). Use the chain tab above to + Ethereum mainnet (PublicNode feeHistory + Owlracle + Etherscan v2) + and Polygon (same three). Use the chain tab above to slice the leaderboard. Avalanche C-Chain was dropped because its auto-tuning fee market drives the priority fee to ~0 by design, which collapses prediction-error to ~0 across all oracles and makes @@ -68,7 +68,7 @@ abstract: | and records the absolute error per (oracle, tier, chain) as both a gauge and a histogram. p50 / p90 / p99 are computed via Prometheus `quantile_over_time` over the 24 h window. Per-chain - coverage: Ethereum and Polygon both have all four oracles + coverage: Ethereum and Polygon both have all three oracles (Etherscan v2 free tier covers chainid 1 and 137). The Etherscan call goes through a global rate-gate (≥6s between any two Etherscan requests across chains) because the no-key limit @@ -82,9 +82,9 @@ abstract: | methodology: - "Chains: Ethereum mainnet (chainid=1), Polygon PoS (chainid=137). Both are EIP-1559 with proper dynamic base fee, so priority-fee prediction is apples-to-apples comparable. BNB Chain excluded (not EIP-1559). Avalanche C-Chain excluded (auto-tuning drives priority fee to ~0, prediction-error collapses to ~0)." - - "Oracles per chain. Ethereum + Polygon: Blocknative, PublicNode feeHistory, Owlracle, Etherscan v2 (free tier)." - - "Per-oracle endpoints. Blocknative `?chainid=` (no-key); PublicNode `eth_feeHistory` on each chain's RPC; Owlracle `/v4/{eth|poly}/gas` (slug per chain); Etherscan v2 `?chainid=&module=gastracker&action=gasoracle`." - - "Cadences. Blocknative & PublicNode feeHistory every 12s per chain; Owlracle every 60s per chain (free quota ceiling); Etherscan every 15s per chain with a global rate-gate enforcing ≥6s between any two Etherscan requests across chains (no-key limit is 1 req/5s per IP, shared)." + - "Oracles per chain. Ethereum + Polygon: PublicNode feeHistory, Owlracle, Etherscan v2 (free tier)." + - "Per-oracle endpoints. PublicNode `eth_feeHistory` on each chain's RPC; Owlracle `/v4/{eth|poly}/gas` (slug per chain); Etherscan v2 `?chainid=&module=gastracker&action=gasoracle`." + - "Cadences. PublicNode feeHistory every 12s per chain; Owlracle every 60s per chain (free quota ceiling); Etherscan every 15s per chain with a global rate-gate enforcing ≥6s between any two Etherscan requests across chains (no-key limit is 1 req/5s per IP, shared)." - "Tier normalization. Each oracle's named tiers (fast / standard / slow / safe / propose) are mapped onto a unified p25 / p50 / p75 / p90 / p99 scheme. The leaderboard ranks the p50 tier (the standard speed wallets use by default)." - "Realized priority fee. For each pending block, the harness pulls the full block via `eth_getBlockByNumber(.., true)` on the chain's PublicNode RPC and computes the percentile from actual `maxPriorityFeePerGas` values across every included transaction. Empty or low-tx blocks (`gas_realized_tx_count` near zero) are surfaced separately because the realized percentile is noisy when tx count is low." - "Realized base fee. Recorded for completeness as `gas_realized_base_gwei{chain}` and per-oracle baseFee error as `gas_error_base_gwei{oracle, chain}`. Not the ranking signal since every EIP-1559 oracle inherits this from `eth_feeHistory` and converges." @@ -96,7 +96,6 @@ methodology: findings: - "{{best_name}} currently leads at {{best_p50}} (absolute error in gwei, p99, 24 h) on the active chain tab, across {{count}} measured oracles. The ranking uses the p99 gap, the worst 1% of blocks, because typical-minute gaps sit within fractions of a micro-gwei of each other (economically indistinguishable noise) while gas spikes are where a wrong prediction actually costs money." - - "{{name:blocknative}} shows a p99 gap of {{p50:blocknative}}. Blocknative publishes a probability-of-inclusion model per tier (70 / 80 / 90 / 95 / 99% confidence) which we map onto the unified p25 / p50 / p75 / p90 / p99 scheme; the same parser works for every chain via the `?chainid=` query parameter." - "{{name:publicnode-feehistory}} shows a p99 gap of {{p50:publicnode-feehistory}}. PublicNode's `eth_feeHistory` predictor is a thin wrapper over the EIP-1559 spec's reward percentiles, so it tends to agree closely with the realized percentile by construction. the gap is mostly the difference between the rolling lookback window and the next-block reality." - "{{name:owlracle}} shows a p99 gap of {{p50:owlracle}}. Owlracle aggregates several upstream oracles into a recommendation, which compresses tail risk but adds polling latency at the 60s cadence required by the free quota." - "{{name:etherscan}} shows a p99 gap of {{p50:etherscan}}. The most-visited gas tracker on the web; its tiering (`SafeGasPrice` / `ProposeGasPrice` / `FastGasPrice`) was originally designed for the pre-EIP-1559 single-price world. Audited on Ethereum + Polygon, both supported by Etherscan v2's free tier." @@ -108,7 +107,7 @@ faq: - q: "Why rank on the p99 gap instead of the typical p50 gap?" a: "Because at current fee levels the p50 gaps are fractions of a micro-gwei apart across oracles, on a 100k-gas transaction a 0.001 gwei error is about a thousandth of a cent, so a p50 ranking orders economically indistinguishable noise. The p99 gap captures the volatile minutes (mempool spikes, NFT mints, MEV bursts) where predictions genuinely diverge and a wrong number either overpays or misses the block. The typical p50 and p90 gaps remain visible as secondary columns." - q: "What is the covered rate column?" - a: "The share of time over 24h that the oracle's posted p50-tier prediction was at or above the realized p50 priority fee of the latest mined block. Absolute gap treats over-prediction and under-prediction the same, but they are not symmetric for users: over-predicting means slightly overpaying, under-predicting means a transaction paying exactly the predicted fee would have ranked below the block's median and risks waiting. Inclusion-confidence oracles (Blocknative, Etherscan) should score high here by design; percentile trackers sit near 50% by construction. Read gap and covered together: low p99 gap + high covered is the sweet spot." + a: "The share of time over 24h that the oracle's posted p50-tier prediction was at or above the realized p50 priority fee of the latest mined block. Absolute gap treats over-prediction and under-prediction the same, but they are not symmetric for users: over-predicting means slightly overpaying, under-predicting means a transaction paying exactly the predicted fee would have ranked below the block's median and risks waiting. Inclusion-confidence oracles (Etherscan) should score high here by design; percentile trackers sit near 50% by construction. Read gap and covered together: low p99 gap + high covered is the sweet spot." - q: "Why these specific chains (Ethereum, Polygon)?" a: "Both are EIP-1559 with a proper dynamic base fee, which makes 'priority-fee prediction error' an apples-to-apples comparable metric across them. Avalanche C-Chain was dropped because its auto-tuning fee market collapses the priority fee to ~0 by design, which makes the prediction-error metric uniformly ~0 across all oracles and non-informative. BNB Chain is excluded because it isn't EIP-1559 (effective fee = gasPrice only, priority is structural noise). Optimism / Base / Arbitrum and other L2 OP Stack rollups are excluded because their priority fee is ~0 (centralised sequencer, no MEV) and the relevant cost is L1 data fee, a different metric that belongs in a separate bench. Solana uses lamports per CU with Jito MEV, different fee model entirely." - q: "Which oracle should I pick for my wallet?" @@ -176,19 +175,6 @@ metric_panels: higher_is_better: true providers: - - slug: blocknative - name: Blocknative - tag: Probability-of-inclusion model, EIP-1559 tiered - formula: "p99 over 24h of |Blocknative's predicted p50-tier priority fee − realized p50 priority fee| in gwei, computed per next-mined block on the active chain." - queries: - p50: quantile_over_time(0.99, gas_error_priority_gwei{oracle="blocknative", tier="p50"}[24h]) - p90: quantile_over_time(0.90, gas_error_priority_gwei{oracle="blocknative", tier="p50"}[24h]) - p99: quantile_over_time(0.50, gas_error_priority_gwei{oracle="blocknative", tier="p50"}[24h]) - mean: avg_over_time(gas_error_priority_gwei{oracle="blocknative", tier="p50"}[24h]) - success: sum(rate(gas_oracle_call_total{oracle="blocknative", result="ok"}[24h])) / sum(rate(gas_oracle_call_total{oracle="blocknative"}[24h])) - sample_size: sum(increase(gas_oracle_call_total{oracle="blocknative"}[24h])) - series: gas_error_priority_gwei{oracle="blocknative", tier="p50"} - - slug: publicnode-feehistory name: PublicNode tag: Thin wrapper over EIP-1559 reward percentiles diff --git a/benchmarks/l1-finality.md b/benchmarks/l1-finality.md index 949b0925..57fe41f1 100644 --- a/benchmarks/l1-finality.md +++ b/benchmarks/l1-finality.md @@ -20,7 +20,7 @@ This works for: **Ethereum, Solana, TRON, Stellar, Hedera, SUI, Litecoin, Monero For chains where finality is faster than our poll interval, comparing two pointers at one instant doesn't measure finalization time - it measures the gap-at-instant, which collapses to zero when finalization catches up to head. The honest path is to subscribe to a push stream, record `T1 = time.Now()` when block N is first observed, and `T2 = time.Now()` when N becomes finalized. `lag = T2 − T1`, with millisecond precision, independent of chain timestamp resolution. -This works for: **BNB, Avalanche, TON**. +This works for: **BNB, Avalanche, Gram**. ## Per-chain methodology @@ -34,7 +34,7 @@ This works for: **BNB, Avalanche, TON**. | **Stellar** | HTTP poll | Horizon `/ledgers?order=desc&limit=2` | 1 ledger back (SCP-final) | Circle = 1, deterministic SCP | | **Hedera** | HTTP poll | Mirror `/api/v1/blocks?order=desc&limit=2` | 1 block back. Timestamps parsed at ns precision | Hashgraph aBFT deterministic | | **SUI** | HTTP poll | `sui_getLatestCheckpointSequenceNumber` + `sui_getCheckpoint` | 1 checkpoint back | Circle USDC = 1, Mysticeti finalizes in 1 | -| **TON** | SSE wall-clock | `tonapi.io/v2/sse/blocks?workchain=-1` (masterchain only) | Time between consecutive masterchain blocks | TON docs: a tx is final once included in a masterchain block, so block_N is final when block_N+1 commits | +| **Gram** | SSE wall-clock | `tonapi.io/v2/sse/blocks?workchain=-1` (masterchain only) | Time between consecutive masterchain blocks | Gram (formerly TON) docs: a tx is final once included in a masterchain block, so block_N is final when block_N+1 commits | | **Litecoin** | HTTP poll (probabilistic) | blockchair `/stats.best_block_height` and `/dashboards/block/{height}.block.time` | 12 confirmations | Coinbase deposit standard, post-April-2026 13-block MWEB reorg | | **Monero** | HTTP poll (probabilistic) | monero-rpc `get_info` + `get_block_header_by_height` (with cakewallet/sethforprivacy/monerujo failover) | 10 confirmations | Wallet protocol unlock period | | **Cardano** | HTTP poll (probabilistic) | koios `/tip` + `/blocks?block_height=eq.` | 15 confirmations | Above Coinbase 10 / Kraken 15. Far below the academic k=2160 (~12 h) | @@ -56,7 +56,7 @@ This works for: **BNB, Avalanche, TON**. | Hedera | High | Hashgraph aBFT + ns-precision timestamps | | SUI | High | 1 checkpoint = Circle USDC standard | | Stellar | High | SCP deterministic, Circle = 1 | -| TON | High (after SSE refactor) | tonapi `workchain=-1` SSE stream, ms-precise | +| Gram | High (after SSE refactor) | tonapi `workchain=-1` SSE stream, ms-precise | | Cardano | Medium | 15-conf compromise between Coinbase 10 and Kraken 15. Academic k=2160 is theoretical; no actor uses it | | Litecoin | Medium | 12-conf post-April-2026 reorg; standard is evolving | | TRON | Medium | CEX confirmation counts vary 19 to 30; we use the 19-block protocol minimum | @@ -66,7 +66,7 @@ This works for: **BNB, Avalanche, TON**. Rather than copy the docs (which are optimistic targets), we anchored each chain's depth to what production actors (Coinbase, Circle, Kraken, Binance, Fireblocks) actually require before crediting a deposit or treating a transfer as irreversible. That's the practical-settlement standard people put real money behind, and it's the closest thing to a ground truth for "real-world finality." -For chains where no canonical CEX number exists (TON, Monero), we fall back to protocol-level minimums plus a small safety margin and flag them as "Lower" / inferred in the audit. +For chains where no canonical CEX number exists (Gram, Monero), we fall back to protocol-level minimums plus a small safety margin and flag them as "Lower" / inferred in the audit. ## Metrics emitted @@ -81,7 +81,7 @@ l1_finality_last_refresh_timestamp_seconds{chain} l1_finality_fetch_errors_total{chain, error_type} l1_finality_health{chain} # 1 if last sample succeeded -# Wall-clock-measured chains (BNB, Avalanche, TON) +# Wall-clock-measured chains (BNB, Avalanche, Gram) l1_finality_wallclock_lag_milliseconds{chain} # ms-precise gauge l1_finality_wallclock_lag_milliseconds_histogram # histogram for tail latency l1_finality_wallclock_health{chain} # 1 if WS/SSE connected diff --git a/benchmarks/l1-finality.yml b/benchmarks/l1-finality.yml index b4ab1580..9f3dc0b1 100644 --- a/benchmarks/l1-finality.yml +++ b/benchmarks/l1-finality.yml @@ -3,11 +3,11 @@ slug: l1-finality number: "006" title: Fastest L1 blockchain finality, live across 11 chains -seo_title: "Fastest L1 finality 2026: TON, SUI, Stellar, Solana, Ethereum" -seo_description: "Fastest L1 blockchain finality, measured live for 11 chains. TON, SUI and Hedera in seconds, Solana ~13 s, Ethereum ~12.8 min. Live percentiles over 24h, open methodology." -subtitle: Wall-clock seconds from latest block to the finalized block on Ethereum, Solana, TON, SUI, Stellar and 5 more chains, refreshed every 10 seconds. +seo_title: "Fastest L1 finality 2026: Gram, SUI, Stellar, Solana, Ethereum" +seo_description: "Fastest L1 blockchain finality, measured live for 11 chains. Gram (formerly TON), SUI and Hedera in seconds, Solana ~13 s, Ethereum ~12.8 min. Live percentiles over 24h, open methodology." +subtitle: Wall-clock seconds from latest block to the finalized block on Ethereum, Solana, Gram, SUI, Stellar and 5 more chains, refreshed every 10 seconds. seo_intro: | - This page measures L1 finality time live for every major Layer-1 blockchain, with p50 / p90 / p99 refreshed every 10 seconds. Stellar finality time is ~5 seconds, the close interval the Stellar Consensus Protocol locks in via federated Byzantine agreement. Solana finality time goes from sub-second on the processed commitment to ~12.8 s on finalized after 32 confirmed slots. Ethereum finality time is ~12.8 minutes, the 2-epoch Casper FFG window. Hedera finality time clears in 3-5 seconds via Hashgraph aBFT. SUI finality time and TON finality time both sit under one second via Mysticeti DAG-BFT and BAG consensus. BNB and Avalanche finality time land near two seconds through fast-finality forks. Probabilistic chains (Litecoin, Monero) settle on a confirmation-depth convention measured here in minutes. + This page measures L1 finality time live for every major Layer-1 blockchain, with p50 / p90 / p99 refreshed every 10 seconds. Stellar finality time is ~5 seconds, the close interval the Stellar Consensus Protocol locks in via federated Byzantine agreement. Solana finality time goes from sub-second on the processed commitment to ~12.8 s on finalized after 32 confirmed slots. Ethereum finality time is ~12.8 minutes, the 2-epoch Casper FFG window. Hedera finality time clears in 3-5 seconds via Hashgraph aBFT. SUI finality time and Gram finality time (formerly TON) both sit under one second via Mysticeti DAG-BFT and BAG consensus. BNB and Avalanche finality time land near two seconds through fast-finality forks. Probabilistic chains (Litecoin, Monero) settle on a confirmation-depth convention measured here in minutes. faq: - q: "What is blockchain finality?" @@ -20,8 +20,8 @@ faq: a: "Solana exposes two commitments. Processed is optimistic and lands sub-second, typically under 500 ms. Finalized requires 32 confirmed slots and clocks {{p50:solana}} (24h average). The leaderboard value is finalized, the stricter guarantee. Sub-second Solana finality is real but only at the processed commitment level, not finalized." - q: "What is SUI finality time?" a: "SUI clocks {{p50:sui}} (p50, 24h) on this benchmark via the Mysticeti DAG-BFT consensus protocol. The chain exposes millisecond-precision timestamps so the measurement is genuinely sub-second. Mysticeti's two-vote commitment pattern reaches deterministic finality without the multi-block confirmation depth used by classical Byzantine fault tolerant chains." - - q: "What is TON finality time?" - a: "TON's BAG consensus pushes masterchain finality to {{p50:ton}} (p50, 24h), one of the lowest deterministic finalities measured on this leaderboard. TON's design splits state across a masterchain and many workchains. The figure here is the masterchain commit, the canonical reference for cross-chain settlement." + - q: "What is Gram finality time?" + a: "Gram's BAG consensus (the network formerly branded TON, with native token renamed Toncoin → Gram in June 2026) pushes masterchain finality to {{p50:gram}} (p50, 24h), one of the lowest deterministic finalities measured on this leaderboard. Gram's design splits state across a masterchain and many workchains. The figure here is the masterchain commit, the canonical reference for cross-chain settlement." - q: "What is Stellar finality time?" a: "Stellar uses the Stellar Consensus Protocol, a federated Byzantine agreement that reaches deterministic finality at every ledger close, roughly every 5 seconds, no probabilistic confirmation needed. p50 sits at {{p50:stellar}} (24h). The benchmark records wall-clock time between a new ledger appearing on the Horizon stream and its SCP-final commit." - q: "What is Hedera finality time?" @@ -29,9 +29,9 @@ faq: - q: "What is BNB Chain finality time?" a: "BNB Smart Chain finalizes via the BEP-126 fast-finality fork, dropping the confirmation depth that legacy probabilistic chains require. p50 sits at {{p50:bnb}} (24h), measured via persistent WebSocket subscription that records T1 when block N first appears as latest and T2 when it crosses the finalized threshold." - q: "Which blockchain has the fastest finality time?" - a: "Sub-second BFT chains lead. TON and SUI both clock under one second on this live benchmark. BNB and Avalanche sit around one to two seconds via their fast-finality forks. Solana finalized lands around 12.8 s, Ethereum at 12.8 min, and probabilistic chains (Litecoin, Monero) trail at 15 to 30 minutes by confirmation-depth convention." + a: "Sub-second BFT chains lead. Gram and SUI both clock under one second on this live benchmark. BNB and Avalanche sit around one to two seconds via their fast-finality forks. Solana finalized lands around 12.8 s, Ethereum at 12.8 min, and probabilistic chains (Litecoin, Monero) trail at 15 to 30 minutes by confirmation-depth convention." - q: "How is L1 finality time measured on this page?" - a: "Two methods, picked per chain. RPC pollers compare latest vs finalized block timestamps every 10 seconds, used for Ethereum, Solana, TRON, Stellar, SUI, TON, Litecoin, Monero. WebSocket subscribers record wall-clock time T1 when a block first appears on the head stream and T2 when it crosses the finality threshold, giving millisecond-precision lag for sub-poll chains (BNB, Avalanche)." + a: "Two methods, picked per chain. RPC pollers compare latest vs finalized block timestamps every 10 seconds, used for Ethereum, Solana, TRON, Stellar, SUI, Gram, Litecoin, Monero. WebSocket subscribers record wall-clock time T1 when a block first appears on the head stream and T2 when it crosses the finality threshold, giving millisecond-precision lag for sub-poll chains (BNB, Avalanche)." per_chain_explainer: - slug: ethereum @@ -54,10 +54,10 @@ per_chain_explainer: h2: "SUI finality time" body: | SUI clocks {{p50:sui}} (p50, 24h) on this benchmark via the Mysticeti DAG-BFT consensus protocol. The chain exposes millisecond-precision timestamps, so the measurement is genuinely sub-second. Mysticeti's two-vote commitment pattern reaches deterministic finality without the multi-block confirmation depth used by classical Byzantine fault tolerant chains. Measured via `sui_getLatestCheckpointSequenceNumber` minus a 5-checkpoint lookback. - - slug: ton - h2: "TON finality time" + - slug: gram + h2: "Gram finality time" body: | - TON's BAG consensus pushes masterchain finality to {{p50:ton}} (p50, 24h), one of the lowest deterministic finalities measured on this leaderboard. TON's design splits state across a masterchain and many workchains; the figure reported here is the masterchain commit, the canonical reference for cross-chain settlement. Measured via the tonapi.io `/blockchain/masterchain-head` endpoint with a 3-seqno lookback. + Gram's BAG consensus (the chain formerly branded TON, native token renamed Toncoin → Gram in June 2026) pushes masterchain finality to {{p50:gram}} (p50, 24h), one of the lowest deterministic finalities measured on this leaderboard. Gram's design splits state across a masterchain and many workchains; the figure reported here is the masterchain commit, the canonical reference for cross-chain settlement. Measured via the tonapi.io `/blockchain/masterchain-head` endpoint with a 3-seqno lookback. - slug: bnb h2: "BNB Chain finality time" body: | @@ -91,7 +91,7 @@ abstract: | chains with finality much longer than our 10 s poll interval we read `latest.timestamp, finalized.timestamp` from the chain RPCs (Ethereum, Solana, TRON, Litecoin, Monero). For chains whose finality - is faster than the poll interval (BNB, Avalanche, SUI, TON, Stellar) + is faster than the poll interval (BNB, Avalanche, SUI, Gram, Stellar) we maintain a persistent WebSocket / SSE subscription, recording wall-clock time T1 when block N is first seen as `latest` and T2 when it becomes `finalized`. lag = T2 − T1 with millisecond @@ -111,7 +111,7 @@ methodology: - "TRON: `/wallet/getnowblock` (head) minus `/walletsolidity/getnowblock` (solidity-confirmed)." - "Stellar (SSE wall-clock): Horizon `/ledgers?cursor=now&order=asc` event-stream records T1 on first sight of ledger N and T2 on the next ledger close (SCP-final at every close)." - "SUI: `sui_getLatestCheckpointSequenceNumber` minus 5 checkpoints back." - - "TON: tonapi.io `/blockchain/masterchain-head` minus 3 seqno." + - "Gram: tonapi.io `/blockchain/masterchain-head` minus 3 seqno." - "Litecoin (probabilistic): blockchair `/stats.best_block_height` minus 6 confirmations via `/dashboards/block/{height}.block.time`." - "Monero (probabilistic): monero-rpc `get_info` minus 10 confirmations via `get_block_header_by_height`." - "Hedera (planned, currently disabled): mirror-node `/api/v1/blocks` only exposes already-final blocks, so true wall-clock lag is impossible from public endpoints. Re-enables once Block Nodes (HIP-1056) leave private preview." @@ -187,18 +187,22 @@ providers: sample_size: increase(l1_finality_wallclock_samples_total{chain="sui"}[24h]) series: l1_finality_wallclock_lag_milliseconds{chain="sui"} - - slug: ton - name: TON + - slug: gram + name: Gram tag: Masterchain commit, polled via tonapi.io - formula: "Median wall-clock milliseconds for a TON masterchain block to commit under BAG consensus, polled via tonapi.io with a 3-seqno lookback, p50 over 24h." + formula: "Median wall-clock milliseconds for a Gram (formerly TON) masterchain block to commit under BAG consensus, polled via tonapi.io with a 3-seqno lookback, p50 over 24h." + # Prom selectors straddle ton|gram during the harness relabel window. + # The harness on Railway still emits chain="ton"; once it redeploys + # with chain="gram", the regex keeps matching both. Drop the ton + # branch after ~7 days of clean gram-labelled data. queries: - p50: quantile_over_time(0.50, l1_finality_wallclock_lag_milliseconds{chain="ton"}[24h]) - p90: quantile_over_time(0.90, l1_finality_wallclock_lag_milliseconds{chain="ton"}[24h]) - p99: quantile_over_time(0.99, l1_finality_wallclock_lag_milliseconds{chain="ton"}[24h]) - mean: avg_over_time(l1_finality_wallclock_lag_milliseconds{chain="ton"}[24h]) - success: avg_over_time(l1_finality_wallclock_health{chain="ton"}[24h]) - sample_size: increase(l1_finality_wallclock_samples_total{chain="ton"}[24h]) - series: l1_finality_wallclock_lag_milliseconds{chain="ton"} + p50: quantile_over_time(0.50, l1_finality_wallclock_lag_milliseconds{chain=~"ton|gram"}[24h]) + p90: quantile_over_time(0.90, l1_finality_wallclock_lag_milliseconds{chain=~"ton|gram"}[24h]) + p99: quantile_over_time(0.99, l1_finality_wallclock_lag_milliseconds{chain=~"ton|gram"}[24h]) + mean: avg_over_time(l1_finality_wallclock_lag_milliseconds{chain=~"ton|gram"}[24h]) + success: avg_over_time(l1_finality_wallclock_health{chain=~"ton|gram"}[24h]) + sample_size: increase(l1_finality_wallclock_samples_total{chain=~"ton|gram"}[24h]) + series: l1_finality_wallclock_lag_milliseconds{chain=~"ton|gram"} - slug: stellar name: Stellar diff --git a/benchmarks/l2-block-time.yml b/benchmarks/l2-block-time.yml index 5dab47b9..a37a08c9 100644 --- a/benchmarks/l2-block-time.yml +++ b/benchmarks/l2-block-time.yml @@ -106,10 +106,10 @@ providers: tag: Nitro fast-finality, sub-second sequencer formula: "Median wall-clock milliseconds between consecutive `newHeads` events from the Arbitrum One sequencer WebSocket, capturing Nitro's ~250ms cadence, p50 over 24h." queries: - p50: quantile_over_time(0.50, l2_block_time_milliseconds{chain="arbitrum"}[24h]) - p90: quantile_over_time(0.90, l2_block_time_milliseconds{chain="arbitrum"}[24h]) - p99: quantile_over_time(0.99, l2_block_time_milliseconds{chain="arbitrum"}[24h]) - mean: avg_over_time(l2_block_time_milliseconds{chain="arbitrum"}[24h]) + p50: histogram_quantile(0.50, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="arbitrum"}[24h])) by (le)) + p90: histogram_quantile(0.90, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="arbitrum"}[24h])) by (le)) + p99: histogram_quantile(0.99, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="arbitrum"}[24h])) by (le)) + mean: sum(rate(l2_block_time_milliseconds_histogram_sum{chain="arbitrum"}[24h])) / sum(rate(l2_block_time_milliseconds_histogram_count{chain="arbitrum"}[24h])) success: avg_over_time(l2_block_time_health{chain="arbitrum"}[24h]) sample_size: increase(l2_block_time_samples_total{chain="arbitrum"}[24h]) series: l2_block_time_milliseconds{chain="arbitrum"} @@ -119,10 +119,10 @@ providers: tag: OP Stack default, 2 s sequencer formula: "Median wall-clock milliseconds between consecutive `newHeads` events on the Optimism sequencer WebSocket, matching the OP Stack 2s default cadence, p50 over 24h." queries: - p50: quantile_over_time(0.50, l2_block_time_milliseconds{chain="optimism"}[24h]) - p90: quantile_over_time(0.90, l2_block_time_milliseconds{chain="optimism"}[24h]) - p99: quantile_over_time(0.99, l2_block_time_milliseconds{chain="optimism"}[24h]) - mean: avg_over_time(l2_block_time_milliseconds{chain="optimism"}[24h]) + p50: histogram_quantile(0.50, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="optimism"}[24h])) by (le)) + p90: histogram_quantile(0.90, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="optimism"}[24h])) by (le)) + p99: histogram_quantile(0.99, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="optimism"}[24h])) by (le)) + mean: sum(rate(l2_block_time_milliseconds_histogram_sum{chain="optimism"}[24h])) / sum(rate(l2_block_time_milliseconds_histogram_count{chain="optimism"}[24h])) success: avg_over_time(l2_block_time_health{chain="optimism"}[24h]) sample_size: increase(l2_block_time_samples_total{chain="optimism"}[24h]) series: l2_block_time_milliseconds{chain="optimism"} @@ -132,10 +132,10 @@ providers: tag: OP Stack, 2 s sequencer formula: "Median wall-clock milliseconds between consecutive `newHeads` events on the Base sequencer WebSocket, tracking the OP Stack 2s interval, p50 over 24h." queries: - p50: quantile_over_time(0.50, l2_block_time_milliseconds{chain="base"}[24h]) - p90: quantile_over_time(0.90, l2_block_time_milliseconds{chain="base"}[24h]) - p99: quantile_over_time(0.99, l2_block_time_milliseconds{chain="base"}[24h]) - mean: avg_over_time(l2_block_time_milliseconds{chain="base"}[24h]) + p50: histogram_quantile(0.50, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="base"}[24h])) by (le)) + p90: histogram_quantile(0.90, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="base"}[24h])) by (le)) + p99: histogram_quantile(0.99, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="base"}[24h])) by (le)) + mean: sum(rate(l2_block_time_milliseconds_histogram_sum{chain="base"}[24h])) / sum(rate(l2_block_time_milliseconds_histogram_count{chain="base"}[24h])) success: avg_over_time(l2_block_time_health{chain="base"}[24h]) sample_size: increase(l2_block_time_samples_total{chain="base"}[24h]) series: l2_block_time_milliseconds{chain="base"} @@ -145,10 +145,10 @@ providers: tag: zk-rollup, batched producer formula: "Median wall-clock milliseconds between consecutive `newHeads` events on the zkSync Era WebSocket, capturing batched zk-rollup cadence (within-burst vs idle gap), p50 over 24h." queries: - p50: quantile_over_time(0.50, l2_block_time_milliseconds{chain="zksync"}[24h]) - p90: quantile_over_time(0.90, l2_block_time_milliseconds{chain="zksync"}[24h]) - p99: quantile_over_time(0.99, l2_block_time_milliseconds{chain="zksync"}[24h]) - mean: avg_over_time(l2_block_time_milliseconds{chain="zksync"}[24h]) + p50: histogram_quantile(0.50, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="zksync"}[24h])) by (le)) + p90: histogram_quantile(0.90, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="zksync"}[24h])) by (le)) + p99: histogram_quantile(0.99, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="zksync"}[24h])) by (le)) + mean: sum(rate(l2_block_time_milliseconds_histogram_sum{chain="zksync"}[24h])) / sum(rate(l2_block_time_milliseconds_histogram_count{chain="zksync"}[24h])) success: avg_over_time(l2_block_time_health{chain="zksync"}[24h]) sample_size: increase(l2_block_time_samples_total{chain="zksync"}[24h]) series: l2_block_time_milliseconds{chain="zksync"} @@ -158,10 +158,10 @@ providers: tag: zk-rollup, prover-bound cadence formula: "Median wall-clock milliseconds between consecutive `newHeads` events on the Linea sequencer WebSocket, governed by prover-bound batching, p50 over 24h." queries: - p50: quantile_over_time(0.50, l2_block_time_milliseconds{chain="linea"}[24h]) - p90: quantile_over_time(0.90, l2_block_time_milliseconds{chain="linea"}[24h]) - p99: quantile_over_time(0.99, l2_block_time_milliseconds{chain="linea"}[24h]) - mean: avg_over_time(l2_block_time_milliseconds{chain="linea"}[24h]) + p50: histogram_quantile(0.50, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="linea"}[24h])) by (le)) + p90: histogram_quantile(0.90, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="linea"}[24h])) by (le)) + p99: histogram_quantile(0.99, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="linea"}[24h])) by (le)) + mean: sum(rate(l2_block_time_milliseconds_histogram_sum{chain="linea"}[24h])) / sum(rate(l2_block_time_milliseconds_histogram_count{chain="linea"}[24h])) success: avg_over_time(l2_block_time_health{chain="linea"}[24h]) sample_size: increase(l2_block_time_samples_total{chain="linea"}[24h]) series: l2_block_time_milliseconds{chain="linea"} @@ -171,10 +171,10 @@ providers: tag: zkEVM rollup, prover-bound cadence formula: "Median wall-clock milliseconds between consecutive `newHeads` events on the Scroll zkEVM sequencer WebSocket, with prover-bound batching driving the cadence, p50 over 24h." queries: - p50: quantile_over_time(0.50, l2_block_time_milliseconds{chain="scroll"}[24h]) - p90: quantile_over_time(0.90, l2_block_time_milliseconds{chain="scroll"}[24h]) - p99: quantile_over_time(0.99, l2_block_time_milliseconds{chain="scroll"}[24h]) - mean: avg_over_time(l2_block_time_milliseconds{chain="scroll"}[24h]) + p50: histogram_quantile(0.50, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="scroll"}[24h])) by (le)) + p90: histogram_quantile(0.90, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="scroll"}[24h])) by (le)) + p99: histogram_quantile(0.99, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="scroll"}[24h])) by (le)) + mean: sum(rate(l2_block_time_milliseconds_histogram_sum{chain="scroll"}[24h])) / sum(rate(l2_block_time_milliseconds_histogram_count{chain="scroll"}[24h])) success: avg_over_time(l2_block_time_health{chain="scroll"}[24h]) sample_size: increase(l2_block_time_samples_total{chain="scroll"}[24h]) series: l2_block_time_milliseconds{chain="scroll"} @@ -184,10 +184,10 @@ providers: tag: OP Stack fork, 2 s sequencer + native yield formula: "Median wall-clock milliseconds between consecutive `newHeads` events on the Blast sequencer WebSocket, tracking the OP Stack fork's 2s interval, p50 over 24h." queries: - p50: quantile_over_time(0.50, l2_block_time_milliseconds{chain="blast"}[24h]) - p90: quantile_over_time(0.90, l2_block_time_milliseconds{chain="blast"}[24h]) - p99: quantile_over_time(0.99, l2_block_time_milliseconds{chain="blast"}[24h]) - mean: avg_over_time(l2_block_time_milliseconds{chain="blast"}[24h]) + p50: histogram_quantile(0.50, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="blast"}[24h])) by (le)) + p90: histogram_quantile(0.90, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="blast"}[24h])) by (le)) + p99: histogram_quantile(0.99, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="blast"}[24h])) by (le)) + mean: sum(rate(l2_block_time_milliseconds_histogram_sum{chain="blast"}[24h])) / sum(rate(l2_block_time_milliseconds_histogram_count{chain="blast"}[24h])) success: avg_over_time(l2_block_time_health{chain="blast"}[24h]) sample_size: increase(l2_block_time_samples_total{chain="blast"}[24h]) series: l2_block_time_milliseconds{chain="blast"} @@ -197,10 +197,10 @@ providers: tag: OP Stack fork, modular DA, 2 s sequencer formula: "Median wall-clock milliseconds between consecutive `newHeads` events on the Mantle sequencer WebSocket, tracking the OP Stack fork's 2s interval, p50 over 24h." queries: - p50: quantile_over_time(0.50, l2_block_time_milliseconds{chain="mantle"}[24h]) - p90: quantile_over_time(0.90, l2_block_time_milliseconds{chain="mantle"}[24h]) - p99: quantile_over_time(0.99, l2_block_time_milliseconds{chain="mantle"}[24h]) - mean: avg_over_time(l2_block_time_milliseconds{chain="mantle"}[24h]) + p50: histogram_quantile(0.50, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="mantle"}[24h])) by (le)) + p90: histogram_quantile(0.90, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="mantle"}[24h])) by (le)) + p99: histogram_quantile(0.99, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="mantle"}[24h])) by (le)) + mean: sum(rate(l2_block_time_milliseconds_histogram_sum{chain="mantle"}[24h])) / sum(rate(l2_block_time_milliseconds_histogram_count{chain="mantle"}[24h])) success: avg_over_time(l2_block_time_health{chain="mantle"}[24h]) sample_size: increase(l2_block_time_samples_total{chain="mantle"}[24h]) series: l2_block_time_milliseconds{chain="mantle"} @@ -210,10 +210,10 @@ providers: tag: Based rollup, L1-sequenced by Ethereum validators formula: "Median wall-clock milliseconds between consecutive `newHeads` events on Taiko's based-rollup WebSocket, with Ethereum L1 validators driving sequencing, p50 over 24h." queries: - p50: quantile_over_time(0.50, l2_block_time_milliseconds{chain="taiko"}[24h]) - p90: quantile_over_time(0.90, l2_block_time_milliseconds{chain="taiko"}[24h]) - p99: quantile_over_time(0.99, l2_block_time_milliseconds{chain="taiko"}[24h]) - mean: avg_over_time(l2_block_time_milliseconds{chain="taiko"}[24h]) + p50: histogram_quantile(0.50, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="taiko"}[24h])) by (le)) + p90: histogram_quantile(0.90, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="taiko"}[24h])) by (le)) + p99: histogram_quantile(0.99, sum(rate(l2_block_time_milliseconds_histogram_bucket{chain="taiko"}[24h])) by (le)) + mean: sum(rate(l2_block_time_milliseconds_histogram_sum{chain="taiko"}[24h])) / sum(rate(l2_block_time_milliseconds_histogram_count{chain="taiko"}[24h])) success: avg_over_time(l2_block_time_health{chain="taiko"}[24h]) sample_size: increase(l2_block_time_samples_total{chain="taiko"}[24h]) series: l2_block_time_milliseconds{chain="taiko"} diff --git a/benchmarks/network-fees.yml b/benchmarks/network-fees.yml index 406fb3fc..368b822a 100644 --- a/benchmarks/network-fees.yml +++ b/benchmarks/network-fees.yml @@ -8,17 +8,17 @@ seo_description: "Cheapest blockchain transaction fee in USD ranked live across subtitle: "Live USD cost of one native token transaction on 20 Layer 1 and Layer 2 chains, refreshed every 30 seconds." seo_intro: | - This page answers one question. How much does it cost in dollars to send one transaction on each major blockchain right now. We track 20 chains in parallel and refresh the number every 30 seconds. The eleven Layer 1 chains are Ethereum, Solana, BNB Chain, Avalanche, TRON, Cardano, Sui, TON, Stellar, Litecoin and Monero. The nine Layer 2 rollups are Arbitrum, Optimism, Base, zkSync Era, Linea, Scroll, Blast, Mantle and Taiko. For every chain we query its own fee market directly (eth_feeHistory for the EVM family, getRecentPrioritizationFees on Solana, koios epoch params on Cardano, fee_stats on Stellar, get_fee_estimate on Monero, the mempool oracle on Litecoin, getChainParameters on TRON, suix_getReferenceGasPrice on Sui), convert the result to the smallest native unit, then multiply by the live USD price of the chain's native token from Mobula. The output is the actual dollar amount a wallet user pays today. No gwei to lamport conversion, no marketing claim. Compare Ethereum gas now versus Solana fee in USD, see whether Arbitrum is still cheaper than Base today, find out which Layer 1 has the lowest transaction cost this minute. + This page answers one question. How much does it cost in dollars to send one transaction on each major blockchain right now. We track 20 chains in parallel and refresh the number every 30 seconds. The eleven Layer 1 chains are Ethereum, Solana, BNB Chain, Avalanche, TRON, Cardano, Sui, Gram (formerly TON), Stellar, Litecoin and Monero. The nine Layer 2 rollups are Arbitrum, Optimism, Base, zkSync Era, Linea, Scroll, Blast, Mantle and Taiko. For every chain we query its own fee market directly (eth_feeHistory for the EVM family, getRecentPrioritizationFees on Solana, koios epoch params on Cardano, fee_stats on Stellar, get_fee_estimate on Monero, the mempool oracle on Litecoin, getChainParameters on TRON, suix_getReferenceGasPrice on Sui), convert the result to the smallest native unit, then multiply by the live USD price of the chain's native token from Mobula. The output is the actual dollar amount a wallet user pays today. No gwei to lamport conversion, no marketing claim. Compare Ethereum gas now versus Solana fee in USD, see whether Arbitrum is still cheaper than Base today, find out which Layer 1 has the lowest transaction cost this minute. faq: - q: "What does this benchmark measure?" a: "The USD cost of one native token transaction on each of the 20 tracked chains, refreshed every 30 seconds. A native transaction is the simplest action on a chain. Send ETH on Ethereum, SOL on Solana, ADA on Cardano, XLM on Stellar, and so on. We do not yet measure ERC 20 transfers, DEX swaps or smart contract deployments. Those will ship as companion metrics in a later phase." - q: "Which chains are tracked?" - a: "Eleven Layer 1 chains on the L1 tab. Ethereum, Solana, BNB Chain, Avalanche, TRON, Cardano, Sui, TON, Stellar, Litecoin and Monero. Nine Layer 2 rollups on the L2 tab. Arbitrum, Optimism, Base, zkSync Era, Linea, Scroll, Blast, Mantle and Taiko. The list matches the L1 finality and L2 block time benches so you can read cost and speed side by side." + a: "Eleven Layer 1 chains on the L1 tab. Ethereum, Solana, BNB Chain, Avalanche, TRON, Cardano, Sui, Gram (formerly TON), Stellar, Litecoin and Monero. Nine Layer 2 rollups on the L2 tab. Arbitrum, Optimism, Base, zkSync Era, Linea, Scroll, Blast, Mantle and Taiko. The list matches the L1 finality and L2 block time benches so you can read cost and speed side by side." - q: "Why USD instead of gas price in gwei?" a: "Gas price in gwei on Ethereum cannot be compared to lamports per compute unit on Solana, stroops per operation on Stellar or sun per byte on TRON. The only honest cross chain unit is the dollar cost of a user facing action, computed at scrape time using a live USD price for each native token. Mobula's market API delivers the prices we multiply by." - q: "What do slow, standard and fast tiers mean?" - a: "Tiers exist on chains with a priority market where users can pay more for faster inclusion. Slow targets the 25th percentile of recent priority bids, standard the 50th, fast the 90th. Chains with deterministic or near deterministic fees (Cardano, Stellar, TON, TRON native transfer) emit a single tier because there is no priority market to bid into." + a: "Tiers exist on chains with a priority market where users can pay more for faster inclusion. Slow targets the 25th percentile of recent priority bids, standard the 50th, fast the 90th. Chains with deterministic or near deterministic fees (Cardano, Stellar, Gram, TRON native transfer) emit a single tier because there is no priority market to bid into." - q: "Why is the Solana fee so low?" a: "Solana charges 5000 lamports per signature as a hard base, plus an optional priority fee priced in micro lamports per compute unit. A simple SOL transfer uses around 200 compute units, so the priority component is typically dwarfed by the base. At current SOL prices the headline fee sits well under one cent on non congested blocks." - q: "Why is the Cardano fee always similar?" @@ -26,11 +26,11 @@ faq: - q: "Are the L2 numbers complete?" a: "Not yet. The figure for each Layer 2 reflects L2 execution cost only (the wallet visible gas price times 21000 gas times the ETH price). The L1 data posting fee (blob market for EIP 4844 rollups like Arbitrum, Optimism and Base after Dencun, calldata for the rest) is a separate component that varies block to block and is currently excluded. A blended total cost figure will ship in a later phase. For now use the L1 view for true wallet cost comparison, and read the L2 view as the execution component only." - q: "Which Layer 1 chain has the cheapest transaction fee right now?" - a: "Open the page. The leaderboard refreshes every 30 seconds and is sorted by cost. As a general pattern, Stellar, Avalanche, Litecoin, Solana, BNB Chain and TON cluster below one cent, Ethereum and Cardano around three to five cents, and TRON and Monero in the ten cent range. Sui sits in the low one cent range. Exact ordering depends on congestion and native token price at the moment of read." + a: "Open the page. The leaderboard refreshes every 30 seconds and is sorted by cost. As a general pattern, Stellar, Avalanche, Litecoin, Solana, BNB Chain and Gram cluster below one cent, Ethereum and Cardano around three to five cents, and TRON and Monero in the ten cent range. Sui sits in the low one cent range. Exact ordering depends on congestion and native token price at the moment of read." - q: "How often does the page refresh?" a: "Every 30 seconds. The harness re queries each chain's fee oracle and Mobula's price API on the same cadence, so headline values are at most 30 seconds stale plus chain RPC latency (typically under one second)." - q: "Why are some chains showing one tier instead of three?" - a: "Cardano fees are protocol deterministic. Stellar's base fee is 100 stroops per operation network wide. TON's typical fee is a conservative observed value because TON has no clean fee estimate RPC. TRON native transfers consume bandwidth at the published rate per byte. None of these chains expose a priority market a user can bid into for a TRX, ADA, XLM or TON transfer, so emitting a single tier is more honest than fabricating three identical values." + a: "Cardano fees are protocol deterministic. Stellar's base fee is 100 stroops per operation network wide. Gram's typical fee is a conservative observed value because the Gram chain has no clean fee estimate RPC. TRON native transfers consume bandwidth at the published rate per byte. None of these chains expose a priority market a user can bid into for a TRX, ADA, XLM or GRAM transfer, so emitting a single tier is more honest than fabricating three identical values." - q: "Can I cite a value from this page?" a: "Yes. Every number is a Prometheus query over a 24h window. The query string is shown in the row's hover tooltip. The harness source is open at the link in the source field below. Cite the value and the timestamp at the top of the page." @@ -63,10 +63,10 @@ per_chain_explainer: h2: "Sui transaction fee" body: | Sui native transfer fee is {{p50:sui}} (p50, 24h). Sui uses a reference-gas-price model where validators agree on a per-epoch gas price via DPoS auction; a `Coin::transfer` consumes around 76000 computation units, so the per-transfer cost stays in the sub-cent range even on busy epochs. Sui has separate computation and storage fees, with the latter rebated when objects are deleted. Computed via `suix_getReferenceGasPrice` times 76000 gas times SUI price. - - slug: ton - h2: "TON transaction fee" + - slug: gram + h2: "Gram transaction fee" body: | - TON native transfer fee is {{p50:ton}} (p50, 24h). TON has no clean fee-estimate RPC because its fee model uses a Bag-of-Cells emulation that accounts for storage, gas, forward and import fees separately per workchain and shard; a typical Wallet v4 transfer settles around 0.005 TON. We publish that conservative observed value times live TON price as a single deterministic tier. + Gram (formerly TON) native transfer fee is {{p50:gram}} (p50, 24h). The Gram chain has no clean fee-estimate RPC because its fee model uses a Bag-of-Cells emulation that accounts for storage, gas, forward and import fees separately per workchain and shard; a typical Wallet v4 transfer settles around 0.005 GRAM. We publish that conservative observed value times live GRAM price as a single deterministic tier. - slug: stellar h2: "Stellar transaction fee" body: | @@ -145,7 +145,7 @@ methodology: - "Cardano. koios epoch_params.min_fee_a and min_fee_b, times 250 bytes for a typical native transfer. Deterministic by protocol, refreshes only when on chain parameters change." - "Stellar. horizon fee_stats.last_ledger_base_fee times 1 operation. Single tier." - "Sui. suix_getReferenceGasPrice times 76000 gas (typical observed for a Coin::transfer call). Single standard tier." - - "TON. Hardcoded 0.005 TON, the typical observed wallet v4 transfer. TON's fee model uses Bag of Cells emulation and has no clean fee estimate RPC." + - "Gram (formerly TON). Hardcoded 0.005 GRAM, the typical observed wallet v4 transfer. The Gram fee model uses Bag of Cells emulation and has no clean fee estimate RPC." - "Litecoin. litecoinspace.org /api/v1/fees/recommended (hour, half hour and fastest fees in litoshi per vByte) times 225 vBytes for a typical 1 input 1 output P2WPKH transfer." - "Monero. monero rpc get_fee_estimate.fees[0..2] times 1500 bytes for a typical 1 input 2 output RingCT transaction." - "USD prices. api.mobula.io/api/1/market/multi-data polled every 30 seconds for all 20 native tokens in one call." @@ -265,18 +265,21 @@ providers: success: avg_over_time(tx_fee_health{chain="sui"}[24h]) series: tx_fee_native_transfer_usd{chain="sui",tier="std"} - - slug: ton - name: TON + - slug: gram + name: Gram layer: l1 - tag: Hardcoded 0.005 TON typical wallet v4 transfer - formula: "Conservative typical observed cost of a TON wallet transfer × TON USD price. TON has no fee estimate RPC; this is the observed median." + tag: Hardcoded 0.005 GRAM typical wallet v4 transfer + formula: "Conservative typical observed cost of a Gram (formerly TON) wallet transfer × GRAM USD price. Gram has no fee estimate RPC; this is the observed median." + # Straddle the ton/gram chain label during the harness relabel + # window. The harness Railway service currently emits chain="ton"; + # the regex keeps matching once it redeploys with chain="gram". queries: - p50: quantile_over_time(0.50, tx_fee_native_transfer_usd{chain="ton",tier="single"}[24h]) - p90: quantile_over_time(0.90, tx_fee_native_transfer_usd{chain="ton",tier="single"}[24h]) - p99: quantile_over_time(0.99, tx_fee_native_transfer_usd{chain="ton",tier="single"}[24h]) - mean: avg_over_time(tx_fee_native_transfer_usd{chain="ton",tier="single"}[24h]) - success: avg_over_time(tx_fee_health{chain="ton"}[24h]) - series: tx_fee_native_transfer_usd{chain="ton",tier="single"} + p50: quantile_over_time(0.50, tx_fee_native_transfer_usd{chain=~"ton|gram",tier="single"}[24h]) + p90: quantile_over_time(0.90, tx_fee_native_transfer_usd{chain=~"ton|gram",tier="single"}[24h]) + p99: quantile_over_time(0.99, tx_fee_native_transfer_usd{chain=~"ton|gram",tier="single"}[24h]) + mean: avg_over_time(tx_fee_native_transfer_usd{chain=~"ton|gram",tier="single"}[24h]) + success: avg_over_time(tx_fee_health{chain=~"ton|gram"}[24h]) + series: tx_fee_native_transfer_usd{chain=~"ton|gram",tier="single"} - slug: stellar name: Stellar diff --git a/benchmarks/perp-fees.yml b/benchmarks/perp-fees.yml index 3e4ccc57..775cbfbb 100644 --- a/benchmarks/perp-fees.yml +++ b/benchmarks/perp-fees.yml @@ -51,6 +51,20 @@ methodology: - "All-in formula: `all_in_bps = taker_fee_bps + spread_bps`. Both components emitted as separate metrics for transparency." - "Failures (5xx, timeouts, rate limits) leave the previous gauge in place and increment a per-venue `fetch_errors_total` counter. The page falls back to the last successful sample." +per_chain_explainer: + - slug: ETH + h2: "Cheapest perp DEX for ETH" + body: | + The cheapest perp DEX for ETH on this benchmark sits at {{best_p50:chain:ETH}} all-in (p50, 24h) for a $1000 ETH long 10x. ETH-PERP is the deepest pair on every venue measured (Hyperliquid, Lighter, dYdX v4, GMX v2, gains.trade), which means the rack-rate taker fee dominates the all-in number rather than the spread. Lighter quotes 0 bps taker, Hyperliquid 4.5 bps, dYdX 5 bps tier-0, GMX 4 or 6 bps on the impact branch, gains.trade reads the openFeeP slot live from the on-chain Gains v8 fees contract. Spread plus impact at $1000 notional is added on top via an orderbook walk. + - slug: BTC + h2: "Cheapest perp DEX for BTC" + body: | + The cheapest perp DEX for BTC on this benchmark sits at {{best_p50:chain:BTC}} all-in (p50, 24h) for a $1000 BTC long 10x. BTC-PERP carries the highest open interest across every venue in the cohort, so books are typically tighter than ETH on the same notional and the spread component compresses toward the rack rate. GMX v2 BTC uses the same positionFeeFactor schedule as ETH (4 bps positive impact, 6 bps negative); the bench reports the conservative negative branch. The leaderboard reorders intra-day with funding skew and orderbook depth. + - slug: SOL + h2: "Cheapest perp DEX for SOL" + body: | + The cheapest perp DEX for SOL on this benchmark sits at {{best_p50:chain:SOL}} all-in (p50, 24h) for a $1000 SOL long 10x. SOL-PERP books are thinner than ETH or BTC on every venue, so the spread plus impact component carries more weight in the all-in figure and a 5 bps taker venue can land above a 0 bps taker venue once the orderbook is walked. Lighter, Hyperliquid and dYdX v4 quote SOL-PERP natively; GMX v2 trades SOL as a synthetic on its Arbitrum deployment. + findings: - "{{best_name}} currently leads the leaderboard at {{best_p50}} all-in (24 h average) across {{count}} measured perp venues. The number includes taker fee plus the spread crossed at $1000 notional, not the rack-rate taker fee alone." - "Lighter charges zero taker fees on ETH-PERP, confirmed live via their public API. {{name:lighter}} clocks {{p50:lighter}} all-in (24 h average), so the headline figure is essentially the half-spread plus orderbook impact crossed at $1000." diff --git a/benchmarks/perp-funding-stability.yml b/benchmarks/perp-funding-stability.yml new file mode 100644 index 00000000..90445883 --- /dev/null +++ b/benchmarks/perp-funding-stability.yml @@ -0,0 +1,152 @@ +# OpenChainBench. Bench № 043 + +slug: perp-funding-stability +number: "043" +title: Perp DEX funding stability, 7-day stddev of ETH funding ranked +seo_title: "Perp DEX funding stability 2026: ETH funding stddev across 7 venues ranked" +seo_description: "Live 7-day standard deviation of ETH perpetual futures funding rate in bps across 7 major venues (Hyperliquid, Binance, Bybit, OKX, dYdX v4, Paradex, Aster). Lower means more stable funding for carry traders." +subtitle: 7-day standard deviation of ETH funding rate, in basis points per 24h, across the perp-funding cohort. Lower means funding stays in a tight band, the carry trader's preferred signal. +category: Trading +status: live +metric: 7d ETH funding stddev +unit: bps +higher_is_better: false + +disclaimer: | + Stability is not directionality. A venue with persistently positive but tight-banded funding ranks better here than one that flips between positive and negative every settlement, even if the second venue offers better average carry. Read alongside perp-funding for direction and average level. + +seo_intro: | + This benchmark ranks perp venues by how tightly their ETH funding + rate clusters over the trailing 7 days. Average funding tells you + which side gets paid; stability tells you how reliably. A venue + whose 24h hold cost wanders 15 bps in a week is harder to carry + trade than one that holds within 3 bps even if both average the + same number. The bench reads the same normalized series as + perp-funding (per-venue hold cost in bps per 24h) and exposes + stddev_over_time of that series on a 7 day window for ETH. + Cohort is the 7 venues with documented interval semantics: + Hyperliquid, Binance, Bybit, OKX, dYdX v4, Paradex, Aster. Lower + is better. + +abstract: | + The bench reuses the normalized 24h ETH funding series from + perp-funding (bench № 036). For each venue it publishes the 7 day + rolling standard deviation of the ETH hold-24h gauge, in bps. Sign + is preserved on the underlying series; the ranking metric is the + magnitude of the swing, not its direction. Lower means the venue's + funding stayed in a tight band, the cleanest signal for funding-rate + arbitrage and basis trades. Cadence and freshness inherit from the + upstream funding scrape (60 second poll, 30 second Prom scrape). + +methodology: + - "Source. Same upstream as perp-funding: each venue's public funding endpoint polled every 60 seconds, normalized to bps per hour and per 24h hold, sign preserved." + - "Metric. stddev_over_time(perp_funding_hold_24h_bps{asset=\"ETH\"}[7d]) per venue. The 7d window smooths intraday noise but reacts to regime shifts within a week." + - "Sign handling. The underlying series is signed (positive means longs pay). Stability measures the swing, so the ranking uses the unsigned stddev. Average direction and level are the perp-funding bench." + - "Cohort. 7 venues from the perp-funding bench: Hyperliquid (onchain), Binance, Bybit, OKX, dYdX v4 (Cosmos appchain), Paradex (Starknet L2), Aster (BNB Chain). All publish documented funding intervals." + - "Cadence. The upstream funding harness polls every 60 seconds; Prometheus scrapes every 30 seconds. The 7 day stddev refreshes on every scrape." + - "Failures. A venue whose upstream funding feed errors keeps its last gauge value; the leaderboard tags the row stale once the success gauge drops to 0." + - "Reproducibility. The harness emits perp_funding_hold_24h_bps with labels venue and asset; the bench query is the stddev over 7d on that series with asset=ETH pinned." + +findings: + - "{{best_name}} currently has the tightest ETH funding band at {{best_p50}} stddev (7d)." + - "{{name:hyperliquid}} sits at {{p50:hyperliquid}} stddev (7d) on ETH. Hourly settlement means rapid reaction to skew but tight clustering once the book is calm." + - "{{name:binance}} sits at {{p50:binance}} stddev (7d). Deepest book in the field anchors the rate." + - "Outliers on this bench are the venues to watch for funding arbitrage: a wider band means larger spreads to harvest against the cohort median." + +source: https://github.com/ChainBench/OpenChainBench/tree/main/benchmarks/perp-funding-stability.yml + +prometheus: + window: 7d + expected_freshness_seconds: 3600 + +faq: + - q: "What does funding stability measure?" + a: "It is the 7 day standard deviation of each venue's ETH funding rate, expressed in basis points per 24 hour hold. The metric measures how tightly funding clusters around its own 7 day mean, ignoring whether the rate sits positive or negative. A low value means traders carrying a position on that venue paid (or received) a predictable number every settlement; a high value means the rate swung a lot over the week." + - q: "Why is lower better?" + a: "Carry trades, basis trades and funding-rate arbitrage all assume the rate behaves predictably over the holding window. A venue with a noisy funding curve forces wider stop-losses and reduces the achievable Sharpe even when the average rate looks attractive. Stability is the second-order property that turns an average rate into a trade you can size." + - q: "How does this differ from perp-funding (bench 036)?" + a: "perp-funding ranks venues by the current normalized 24h funding cost on ETH; it answers 'who is cheapest to hold right now'. perp-funding-stability ranks venues by the 7 day standard deviation of that same series; it answers 'whose rate is most predictable'. The two are complementary: a cheap-but-noisy venue is still risky to carry; a stable-but-expensive venue is honest about its cost." + - q: "Why ETH only?" + a: "ETH is the deepest contract on every venue in the cohort, so the funding series has the strongest signal-to-noise ratio for cross-venue comparison. BTC and SOL are also published by the upstream harness as gauges; future revisions of this bench may expand the dimension." + - q: "How fresh are the numbers?" + a: "The upstream funding gauge is scraped every 30 seconds. The stddev refreshes on every Prom evaluation, so the leaderboard reflects the most recent 7 day window minus at most 30 seconds of lag." + +dimensions: {} + +providers: + - slug: hyperliquid + name: Hyperliquid + tag: Onchain order book perps, funding settled hourly + formula: "stddev_over_time of perp_funding_hold_24h_bps{venue=\"hyperliquid\",asset=\"ETH\"} over a 7d window." + queries: + p50: stddev_over_time(perp_funding_hold_24h_bps{venue="hyperliquid",asset="ETH"}[7d]) + mean: stddev_over_time(perp_funding_hold_24h_bps{venue="hyperliquid",asset="ETH"}[7d]) + success: clamp_max(count_over_time(perp_funding_hold_24h_bps{venue="hyperliquid",asset="ETH"}[1h]), 1) + sample_size: count_over_time(perp_funding_hold_24h_bps{venue="hyperliquid",asset="ETH"}[24h]) + series: stddev_over_time(perp_funding_hold_24h_bps{venue="hyperliquid",asset="ETH"}[7d]) + + - slug: binance + name: Binance + tag: Largest CEX perp book, 8h funding (4h on some pairs) + formula: "stddev_over_time of perp_funding_hold_24h_bps{venue=\"binance\",asset=\"ETH\"} over a 7d window." + queries: + p50: stddev_over_time(perp_funding_hold_24h_bps{venue="binance",asset="ETH"}[7d]) + mean: stddev_over_time(perp_funding_hold_24h_bps{venue="binance",asset="ETH"}[7d]) + success: clamp_max(count_over_time(perp_funding_hold_24h_bps{venue="binance",asset="ETH"}[1h]), 1) + sample_size: count_over_time(perp_funding_hold_24h_bps{venue="binance",asset="ETH"}[24h]) + series: stddev_over_time(perp_funding_hold_24h_bps{venue="binance",asset="ETH"}[7d]) + + - slug: bybit + name: Bybit + tag: USDT linear perps, funding interval per instrument + formula: "stddev_over_time of perp_funding_hold_24h_bps{venue=\"bybit\",asset=\"ETH\"} over a 7d window." + queries: + p50: stddev_over_time(perp_funding_hold_24h_bps{venue="bybit",asset="ETH"}[7d]) + mean: stddev_over_time(perp_funding_hold_24h_bps{venue="bybit",asset="ETH"}[7d]) + success: clamp_max(count_over_time(perp_funding_hold_24h_bps{venue="bybit",asset="ETH"}[1h]), 1) + sample_size: count_over_time(perp_funding_hold_24h_bps{venue="bybit",asset="ETH"}[24h]) + series: stddev_over_time(perp_funding_hold_24h_bps{venue="bybit",asset="ETH"}[7d]) + + - slug: okx + name: OKX + tag: USDT swaps, interval derived from settlement timestamps + formula: "stddev_over_time of perp_funding_hold_24h_bps{venue=\"okx\",asset=\"ETH\"} over a 7d window." + queries: + p50: stddev_over_time(perp_funding_hold_24h_bps{venue="okx",asset="ETH"}[7d]) + mean: stddev_over_time(perp_funding_hold_24h_bps{venue="okx",asset="ETH"}[7d]) + success: clamp_max(count_over_time(perp_funding_hold_24h_bps{venue="okx",asset="ETH"}[1h]), 1) + sample_size: count_over_time(perp_funding_hold_24h_bps{venue="okx",asset="ETH"}[24h]) + series: stddev_over_time(perp_funding_hold_24h_bps{venue="okx",asset="ETH"}[7d]) + + - slug: dydx + name: dYdX v4 + tag: Appchain order book perps, funding settled hourly + formula: "stddev_over_time of perp_funding_hold_24h_bps{venue=\"dydx\",asset=\"ETH\"} over a 7d window." + queries: + p50: stddev_over_time(perp_funding_hold_24h_bps{venue="dydx",asset="ETH"}[7d]) + mean: stddev_over_time(perp_funding_hold_24h_bps{venue="dydx",asset="ETH"}[7d]) + success: clamp_max(count_over_time(perp_funding_hold_24h_bps{venue="dydx",asset="ETH"}[1h]), 1) + sample_size: count_over_time(perp_funding_hold_24h_bps{venue="dydx",asset="ETH"}[24h]) + series: stddev_over_time(perp_funding_hold_24h_bps{venue="dydx",asset="ETH"}[7d]) + + - slug: paradex + name: Paradex + tag: Starknet L2 perps, 8h funding accrued continuously + formula: "stddev_over_time of perp_funding_hold_24h_bps{venue=\"paradex\",asset=\"ETH\"} over a 7d window." + queries: + p50: stddev_over_time(perp_funding_hold_24h_bps{venue="paradex",asset="ETH"}[7d]) + mean: stddev_over_time(perp_funding_hold_24h_bps{venue="paradex",asset="ETH"}[7d]) + success: clamp_max(count_over_time(perp_funding_hold_24h_bps{venue="paradex",asset="ETH"}[1h]), 1) + sample_size: count_over_time(perp_funding_hold_24h_bps{venue="paradex",asset="ETH"}[24h]) + series: stddev_over_time(perp_funding_hold_24h_bps{venue="paradex",asset="ETH"}[7d]) + + - slug: aster + name: Aster + tag: BNB Chain perps DEX, Binance compatible API, 8h funding + formula: "stddev_over_time of perp_funding_hold_24h_bps{venue=\"aster\",asset=\"ETH\"} over a 7d window." + queries: + p50: stddev_over_time(perp_funding_hold_24h_bps{venue="aster",asset="ETH"}[7d]) + mean: stddev_over_time(perp_funding_hold_24h_bps{venue="aster",asset="ETH"}[7d]) + success: clamp_max(count_over_time(perp_funding_hold_24h_bps{venue="aster",asset="ETH"}[1h]), 1) + sample_size: count_over_time(perp_funding_hold_24h_bps{venue="aster",asset="ETH"}[24h]) + series: stddev_over_time(perp_funding_hold_24h_bps{venue="aster",asset="ETH"}[7d]) diff --git a/benchmarks/perp-open-interest.yml b/benchmarks/perp-open-interest.yml new file mode 100644 index 00000000..95d29f40 --- /dev/null +++ b/benchmarks/perp-open-interest.yml @@ -0,0 +1,119 @@ +# OpenChainBench. Bench № 042 + +slug: perp-open-interest +number: "042" +title: Perp DEX open interest, live USD notional ranked +seo_title: "Perp DEX open interest 2026: Hyperliquid, Lighter, GMX, gains.trade ranked" +seo_description: "Live perpetual futures open interest in USD across Hyperliquid, Lighter, GMX v2 and gains.trade. Public API sourced, refreshed every 5 minutes." +subtitle: Live aggregate open interest in USD across major decentralized perpetual futures venues. Higher means more notional sits open. Read live from each venue's public API. +category: Trading +status: live +metric: Open interest +unit: usd +higher_is_better: true + +seo_intro: | + This benchmark ranks the major onchain perp venues by aggregate open + interest, in USD. Open interest is the canonical depth-of-market + number for perpetual futures: it tells you how much notional sits + open right now, on every market, on a given venue. We poll each + venue's public API every 5 minutes, sum open interest across all + listed perps, and publish one gauge per venue. Hyperliquid, Lighter, + GMX v2 and gains.trade are the cohort. Headline number is the 24h + average of the live OI gauge so a single noisy print does not move + the ranking. + +abstract: | + The bench polls four perp DEX venues every 5 minutes for live open + interest and exposes a USD-aggregated gauge per venue. Sources: + Hyperliquid info metaAndAssetCtxs (openInterest per asset, summed in + USD), Lighter orderBookDetails (open interest per market), GMX + Subsquid synthetics-arbitrum (openInterest by market), gains.trade + onchain reads aggregated by indexer. Each value carries a freshness + timestamp and a health gauge so a stale or errored venue does not + poison the leaderboard. Higher is better. + +methodology: + - "Cadence: every 5 minutes per venue in parallel, 10 second timeout per request." + - "Hyperliquid: info metaAndAssetCtxs, openInterest summed across all assets and priced in USD using the venue's own mark price." + - "Lighter: orderBookDetails per market, open interest summed across markets in USD." + - "GMX v2: synthetics-arbitrum subgraph, openInterestUSD summed across markets." + - "gains.trade: onchain indexer over Gains v8 contracts on Base, open interest summed across pairs in USD." + - "Headline. avg_over_time of the live OI gauge over the last 24 hours, so a one-print spike does not move the ranking. The Series tab plots the raw gauge." + - "Failures. A venue that errors or times out keeps its last gauge value and its perp_venue_health gauge drops to 0; the leaderboard tags it as stale." + +findings: + - "{{best_name}} currently leads the cohort at {{best_p50}} of open interest (24h average)." + - "{{name:hyperliquid}} sits at {{p50:hyperliquid}} of open interest. HyperBFT orderbook is the deepest decentralized perp venue." + - "{{name:lighter}} clocks {{p50:lighter}} of open interest. Fully onchain orderbook on zkSync." + - "{{name:gmx}} sits at {{p50:gmx}} of open interest. Pool-based execution on Arbitrum and Avalanche." + - "{{name:gains}} sits at {{p50:gains}} of open interest. Synthetic perps on Base." + +source: https://github.com/ChainBench/OpenChainBench/tree/main/benchmarks/perp-open-interest.yml + +prometheus: + window: 24h + expected_freshness_seconds: 1800 + +faq: + - q: "What does this benchmark measure?" + a: "The live aggregate open interest of major perp DEX venues in USD. Headline is the 24h average of the live OI gauge so single prints do not skew the ranking." + - q: "How is open interest different from volume?" + a: "Volume counts every trade that crosses the book during a window. Open interest counts notional that currently sits open, regardless of when it was opened. A high-volume venue with high turnover can carry lower OI than a slower venue with sticky positions." + - q: "Why these four venues?" + a: "Hyperliquid, Lighter, GMX v2 and gains.trade are the cohort whose public APIs expose open interest cleanly enough to compare without backfilling. Other venues join as their endpoints document the same series." + - q: "Is OI a good proxy for venue health?" + a: "It is one of the cleanest. A venue can spike volume with wash trading or incentive programs, but real OI is harder to fake because it ties up margin. Compared against perp-volume-share, persistent OI dominance signals real position-holding flow rather than turnover." + +providers: + - slug: hyperliquid + name: Hyperliquid + tag: HyperBFT L1 perp DEX + formula: "Open interest from Hyperliquid info metaAndAssetCtxs, summed across all assets and priced in USD using the venue's mark price; headline is the 24h time average." + queries: + p50: avg_over_time(perp_venue_oi_usd{venue="hyperliquid"}[24h]) + p90: quantile_over_time(0.90, perp_venue_oi_usd{venue="hyperliquid"}[24h]) + p99: quantile_over_time(0.99, perp_venue_oi_usd{venue="hyperliquid"}[24h]) + mean: avg_over_time(perp_venue_oi_usd{venue="hyperliquid"}[24h]) + success: avg_over_time(perp_venue_health{venue="hyperliquid"}[24h]) + sample_size: count_over_time(perp_venue_oi_usd{venue="hyperliquid"}[24h]) + series: perp_venue_oi_usd{venue="hyperliquid"} + + - slug: lighter + name: Lighter + tag: zk-rollup, zero taker fee + formula: "Open interest from Lighter orderBookDetails per market, summed across markets in USD; headline is the 24h time average." + queries: + p50: avg_over_time(perp_venue_oi_usd{venue="lighter"}[24h]) + p90: quantile_over_time(0.90, perp_venue_oi_usd{venue="lighter"}[24h]) + p99: quantile_over_time(0.99, perp_venue_oi_usd{venue="lighter"}[24h]) + mean: avg_over_time(perp_venue_oi_usd{venue="lighter"}[24h]) + success: avg_over_time(perp_venue_health{venue="lighter"}[24h]) + sample_size: count_over_time(perp_venue_oi_usd{venue="lighter"}[24h]) + series: perp_venue_oi_usd{venue="lighter"} + + - slug: gmx + name: GMX v2 + tag: Synthetics on Arbitrum, oracle-priced + formula: "openInterestUSD from the GMX synthetics-arbitrum subgraph summed across markets; headline is the 24h time average." + queries: + p50: avg_over_time(perp_venue_oi_usd{venue="gmx-v2"}[24h]) + p90: quantile_over_time(0.90, perp_venue_oi_usd{venue="gmx-v2"}[24h]) + p99: quantile_over_time(0.99, perp_venue_oi_usd{venue="gmx-v2"}[24h]) + mean: avg_over_time(perp_venue_oi_usd{venue="gmx-v2"}[24h]) + success: avg_over_time(perp_venue_health{venue="gmx-v2"}[24h]) + sample_size: count_over_time(perp_venue_oi_usd{venue="gmx-v2"}[24h]) + series: perp_venue_oi_usd{venue="gmx-v2"} + + - slug: gains + name: gains.trade + tag: Synthetic perps on Base + formula: "Open interest across Gains v8 pairs on Base, read onchain and summed in USD; headline is the 24h time average." + queries: + p50: avg_over_time(perp_venue_oi_usd{venue="gains"}[24h]) + p90: quantile_over_time(0.90, perp_venue_oi_usd{venue="gains"}[24h]) + p99: quantile_over_time(0.99, perp_venue_oi_usd{venue="gains"}[24h]) + mean: avg_over_time(perp_venue_oi_usd{venue="gains"}[24h]) + success: avg_over_time(perp_venue_health{venue="gains"}[24h]) + sample_size: count_over_time(perp_venue_oi_usd{venue="gains"}[24h]) + series: perp_venue_oi_usd{venue="gains"} diff --git a/benchmarks/perp-volume-share.yml b/benchmarks/perp-volume-share.yml new file mode 100644 index 00000000..ab70075f --- /dev/null +++ b/benchmarks/perp-volume-share.yml @@ -0,0 +1,120 @@ +# OpenChainBench. Bench № 041 + +slug: perp-volume-share +number: "041" +title: Perp DEX volume share, live 30-day rolling notional ranked +seo_title: "Perp DEX volume 2026: Hyperliquid, Lighter, GMX, gains.trade ranked by 30d notional" +seo_description: "Live 30-day rolling perp DEX volume across Hyperliquid, Lighter, GMX v2 and gains.trade in USD notional. Public API sourced, refreshed every 5 minutes." +subtitle: 30-day rolling perpetual futures notional volume in USD across major decentralized venues. Higher means more flow. Read live from each venue's public API. +category: Trading +status: live +metric: 30d perp volume +unit: usd +higher_is_better: true + +seo_intro: | + This benchmark ranks the major onchain perp venues by 30-day rolling + notional volume, in USD. Volume is the cleanest single-number proxy + for where perpetual futures flow actually lives today. We poll each + venue's public API every 5 minutes, sum the trailing 30 days of taker + notional in USD, and publish a single gauge per venue. Hyperliquid, + Lighter, GMX v2 and gains.trade are the cohort. Headline number is + the 24h average of the 30d rolling sum, so a single noisy print does + not move the ranking. The companion benches price the cost side + (perp-fees for opening, perp-funding for holding); this one prices + flow. + +abstract: | + The bench polls four perp DEX venues every 5 minutes for cumulative + taker notional and exposes a rolling 30 day USD sum per venue. + Sources: Hyperliquid info dayNtlVlm (per-asset day notional, summed + and rolled into 30d), Lighter public stats endpoint (24h volume per + market, rolled), GMX Subsquid synthetics-arbitrum (positionVolume by + day), gains.trade onchain reads aggregated by indexer. Each value + carries a freshness timestamp and a health gauge so a stale or + errored venue does not poison the leaderboard. Higher is better. + +methodology: + - "Cadence: every 5 minutes per venue in parallel, 10 second timeout per request." + - "Hyperliquid: info dayNtlVlm summed across all assets, rolled into a 30 day window via Prometheus sum_over_time on the daily gauge." + - "Lighter: public stats endpoint, 24h volume per market, summed across markets and rolled into 30 days the same way." + - "GMX v2: synthetics-arbitrum subgraph, positionVolume aggregated per day across all markets, rolled into 30 days." + - "gains.trade: onchain indexer over Gains v8 contracts on Base, taker notional summed per day and rolled." + - "Headline. avg_over_time of the rolling 30 day sum over the last 24 hours, so a one-print spike does not move the ranking. The Series tab plots the raw rolling sum." + - "Failures. A venue that errors or times out keeps its last gauge value and its perp_venue_health gauge drops to 0; the leaderboard tags it as stale." + +findings: + - "{{best_name}} currently leads the cohort at {{best_p50}} of 30d notional (24h average)." + - "{{name:hyperliquid}} sits at {{p50:hyperliquid}} of 30d notional, the deepest decentralized perp book in the field." + - "{{name:lighter}} clocks {{p50:lighter}} of 30d notional. Zero taker fee plus a fully onchain orderbook on zkSync." + - "{{name:gmx}} sits at {{p50:gmx}} of 30d notional. Pool-based execution on Arbitrum and Avalanche." + - "{{name:gains}} sits at {{p50:gains}} of 30d notional. Synthetic perps on Base with onchain fee reads." + +source: https://github.com/ChainBench/OpenChainBench/tree/main/benchmarks/perp-volume-share.yml + +prometheus: + window: 24h + expected_freshness_seconds: 1800 + +faq: + - q: "What does this benchmark measure?" + a: "The 30 day rolling notional volume of major perp DEX venues in USD. Headline is the 24h average of the rolling 30d sum so single prints do not skew the ranking." + - q: "Why 30 day rolling instead of daily?" + a: "Daily perp volume is too noisy for ranking, a single thin Sunday can flip positions. The 30 day window smooths the cycle and matches the cadence at which serious flow makers compare venues." + - q: "Why these four venues?" + a: "Hyperliquid, Lighter, GMX v2 and gains.trade are the cohort whose public APIs expose taker notional cleanly enough to compare without backfilling. Other venues join as their endpoints document the same series." + - q: "Is this the same as DeFiLlama volume?" + a: "Methodology is similar in spirit but each venue is queried directly here, not via an aggregator. Numbers should sit close to DeFiLlama on most days, divergences usually mean a downstream feed lagged." + +providers: + - slug: hyperliquid + name: Hyperliquid + tag: HyperBFT L1 perp DEX + formula: "Sum across all assets of Hyperliquid info dayNtlVlm, rolled into a 30 day window, headline is the 24h average." + queries: + p50: avg_over_time(perp_venue_volume_30d_usd{venue="hyperliquid"}[24h]) + p90: quantile_over_time(0.90, perp_venue_volume_30d_usd{venue="hyperliquid"}[24h]) + p99: quantile_over_time(0.99, perp_venue_volume_30d_usd{venue="hyperliquid"}[24h]) + mean: avg_over_time(perp_venue_volume_30d_usd{venue="hyperliquid"}[24h]) + success: avg_over_time(perp_venue_health{venue="hyperliquid"}[24h]) + sample_size: count_over_time(perp_venue_volume_30d_usd{venue="hyperliquid"}[24h]) + series: perp_venue_volume_30d_usd{venue="hyperliquid"} + + - slug: lighter + name: Lighter + tag: zk-rollup, zero taker fee + formula: "Sum of 24h volume per market from the Lighter public stats endpoint, rolled into a 30 day window, headline is the 24h average." + queries: + p50: avg_over_time(perp_venue_volume_30d_usd{venue="lighter"}[24h]) + p90: quantile_over_time(0.90, perp_venue_volume_30d_usd{venue="lighter"}[24h]) + p99: quantile_over_time(0.99, perp_venue_volume_30d_usd{venue="lighter"}[24h]) + mean: avg_over_time(perp_venue_volume_30d_usd{venue="lighter"}[24h]) + success: avg_over_time(perp_venue_health{venue="lighter"}[24h]) + sample_size: count_over_time(perp_venue_volume_30d_usd{venue="lighter"}[24h]) + series: perp_venue_volume_30d_usd{venue="lighter"} + + - slug: gmx + name: GMX v2 + tag: Synthetics on Arbitrum, oracle-priced + formula: "positionVolume aggregated per day from the GMX synthetics-arbitrum subgraph, rolled into a 30 day window, headline is the 24h average." + queries: + p50: avg_over_time(perp_venue_volume_30d_usd{venue="gmx-v2"}[24h]) + p90: quantile_over_time(0.90, perp_venue_volume_30d_usd{venue="gmx-v2"}[24h]) + p99: quantile_over_time(0.99, perp_venue_volume_30d_usd{venue="gmx-v2"}[24h]) + mean: avg_over_time(perp_venue_volume_30d_usd{venue="gmx-v2"}[24h]) + success: avg_over_time(perp_venue_health{venue="gmx-v2"}[24h]) + sample_size: count_over_time(perp_venue_volume_30d_usd{venue="gmx-v2"}[24h]) + series: perp_venue_volume_30d_usd{venue="gmx-v2"} + + - slug: gains + name: gains.trade + tag: Synthetic perps on Base + formula: "Taker notional aggregated per day from the Gains v8 onchain indexer, rolled into a 30 day window, headline is the 24h average." + queries: + p50: avg_over_time(perp_venue_volume_30d_usd{venue="gains"}[24h]) + p90: quantile_over_time(0.90, perp_venue_volume_30d_usd{venue="gains"}[24h]) + p99: quantile_over_time(0.99, perp_venue_volume_30d_usd{venue="gains"}[24h]) + mean: avg_over_time(perp_venue_volume_30d_usd{venue="gains"}[24h]) + success: avg_over_time(perp_venue_health{venue="gains"}[24h]) + sample_size: count_over_time(perp_venue_volume_30d_usd{venue="gains"}[24h]) + series: perp_venue_volume_30d_usd{venue="gains"} diff --git a/benchmarks/pm-api-latency.yml b/benchmarks/pm-api-latency.yml index 714e78ac..18dd9db0 100644 --- a/benchmarks/pm-api-latency.yml +++ b/benchmarks/pm-api-latency.yml @@ -41,6 +41,11 @@ abstract: | this API up" is answered by direct measurement from three regions rather than by user reports. Samples that fail because our pinned market expired are classified probe_invalid and never count against the venue. + Third party data aggregators that resell venue prices (Mobula, Codex, + Predexon) are tracked separately in the pm-data-freshness bench and on + the data feeds tab of the prediction markets hub. This page ranks the + five venues' own APIs only, so the leaderboard answers one clean + question: which prediction market venue API is fastest right now. methodology: - "Price endpoint per venue. Polymarket: CLOB `/midpoint`. Kalshi: `/markets/{ticker}`. Limitless: `/markets/{slug}`. Manifold: `/v0/market/{id}`. Myriad: `/markets/{slug}`. This is the hot path of real integrations: one market, one quote, polled in a loop." @@ -51,6 +56,7 @@ methodology: - "Division of labor with the sibling benches: how each venue behaves as request rates climb, including throttle onset and 429 handling, is measured in pm-rate-limits (bench 037). How fresh third party data providers relay Polymarket data is measured in pm-data-freshness (bench 032). This bench ranks the venues' own APIs on latency and uptime at a polite request rate." - "Myriad's origin is a single region US East deployment behind Heroku, so its latency from eu-west and sgp is dominated by geography. Reported as measured, called out per region." - "All five venues are probed by the same open source harness as bench 037, from the same processes, so the two benches share one probe budget and one identifying User-Agent: `OpenChainBench/1.0 (+https://openchainbench.com/methodology; contact@mobula.io)`. We publish latency and uptime measurements only, never market data." + - "Aggregators that resell venue data (Mobula, Codex, Predexon) are deliberately excluded from this leaderboard. They are not venue APIs, they are relays on top of venue APIs, and ranking them next to the venues they relay would compare two different products on one axis. Their freshness lag against the Polymarket CLOB T0 stream is the relevant metric, measured in the pm-data-freshness bench. The prediction markets hub data feeds tab lists each aggregator with the venues it covers." - "Regions: us-east, eu-west, sgp (Railway). Histogram buckets 25ms to 10s." findings: @@ -59,6 +65,7 @@ findings: - "{{name:manifold}} would look faster than it is if cache hits counted: its whole API sits behind a 5 second CDN cache. With cache hits excluded its origin answers at {{p50:manifold}} p50, an honest number a trading bot polling fresh quotes will actually see." - "{{name:myriad}} serves every region from a single US East origin, so its cohort worst p50 of {{p50:myriad}} is mostly geography. Switch the region dimension to us-east to see the API itself rather than the speed of light." - "{{name:polymarket}} answers its midpoint endpoint at {{p50:polymarket}} p50. Because we probe it every 5 seconds from three regions, the uptime panel doubles as a Polymarket API status check that updates continuously instead of waiting for user reports." + - "For builders who consume venue data through a managed relay, the relevant question is freshness lag, not request latency. Mobula, Codex and Predexon are measured in the pm-data-freshness bench (T0 against the Polymarket CLOB stream). The prediction markets hub data feeds tab lists each aggregator with the venues it covers." disclaimer: "Uptime here means our probes succeeded from three specific regions at a polite request rate. A venue can be up for us and degraded for you, especially during regional network incidents, and a brief blip between probe cycles can go unrecorded. Treat the panel as a measured signal, not a guarantee." @@ -101,7 +108,7 @@ prometheus: window: 24h expected_freshness_seconds: 300 -rank_matrix_query: 1000 * label_replace(histogram_quantile(0.50, sum by (venue, region, le) (rate(pmapi_request_duration_seconds_bucket{conn="warm",class="price",cache!="hit"}[24h]))), "provider", "$1", "venue", "(.+)") +rank_matrix_query: 1000 * label_replace(histogram_quantile(0.50, sum by (venue, region, le) (rate(pmapi_request_duration_seconds_bucket{source="direct",conn="warm",class="price",cache!="hit"}[24h]))), "provider", "$1", "venue", "(.+)") dimensions: region: @@ -114,21 +121,21 @@ metric_panels: - id: uptime_24h label: Uptime 24h description: "Share of probe cycles that succeeded over the last 24 hours, averaged across the three probe regions. 100 percent means every 5 second probe of the venue's API came back healthy." - metric: 100 * avg(avg_over_time(pmapi_health{}[24h])) + metric: 100 * avg(avg_over_time(pmapi_health{source="direct"}[24h])) label_key: venue unit: pct higher_is_better: true - id: uptime_7d label: Uptime 7d description: "Same health gauge averaged over 7 days. Short outages that vanish from the 24h figure stay visible here for a week." - metric: 100 * avg(avg_over_time(pmapi_health{}[7d])) + metric: 100 * avg(avg_over_time(pmapi_health{source="direct"}[7d])) label_key: venue unit: pct higher_is_better: true - id: uptime_30d label: Uptime 30d description: "Same health gauge averaged over 30 days. Feeds the ledger's 30d window toggle so a one-hour outage that's already aged out of the 24h panel still shows up in the table over a month." - metric: 100 * avg(avg_over_time(pmapi_health{}[30d])) + metric: 100 * avg(avg_over_time(pmapi_health{source="direct"}[30d])) label_key: venue unit: pct higher_is_better: true @@ -136,7 +143,7 @@ metric_panels: - id: cold_connect_p50 label: Cold connect description: "TCP plus TLS handshake time on the once a minute cold probe with keep alives disabled. The startup cost a brand new client pays before its first request." - metric: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_connect_seconds_bucket{}[24h])) by (le)) + metric: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_connect_seconds_bucket{source="direct"}[24h])) by (le)) label_key: venue unit: ms higher_is_better: false @@ -155,62 +162,62 @@ providers: tag: CLOB midpoint endpoint behind Cloudflare, public WebSocket, deepest books formula: "p50 of warm, non CDN cached round trips against the CLOB midpoint endpoint for the pinned market, successful requests only, 24h window." queries: - p50: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="polymarket",class="price",conn="warm",cache!="hit"}[24h])) by (le)) - p90: 1000 * histogram_quantile(0.90, sum(rate(pmapi_request_duration_seconds_bucket{venue="polymarket",class="price",conn="warm",cache!="hit"}[24h])) by (le)) - p99: 1000 * histogram_quantile(0.99, sum(rate(pmapi_request_duration_seconds_bucket{venue="polymarket",class="price",conn="warm",cache!="hit"}[24h])) by (le)) - mean: 1000 * sum(rate(pmapi_request_duration_seconds_sum{venue="polymarket",class="price",conn="warm",cache!="hit"}[24h])) / sum(rate(pmapi_request_duration_seconds_count{venue="polymarket",class="price",conn="warm",cache!="hit"}[24h])) - success: clamp_max(sum(rate(pmapi_requests_total{venue="polymarket",class="price",conn="warm",outcome="ok"}[24h])) / sum(rate(pmapi_requests_total{venue="polymarket",class="price",conn="warm",outcome!="probe_invalid"}[24h])), 1) - sample_size: sum(increase(pmapi_requests_total{venue="polymarket",class="price",conn="warm",outcome!="probe_invalid"}[24h])) - series: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="polymarket",class="price",conn="warm",cache!="hit"}[1h])) by (le)) + p50: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="polymarket",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + p90: 1000 * histogram_quantile(0.90, sum(rate(pmapi_request_duration_seconds_bucket{venue="polymarket",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + p99: 1000 * histogram_quantile(0.99, sum(rate(pmapi_request_duration_seconds_bucket{venue="polymarket",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + mean: 1000 * sum(rate(pmapi_request_duration_seconds_sum{venue="polymarket",source="direct",class="price",conn="warm",cache!="hit"}[24h])) / sum(rate(pmapi_request_duration_seconds_count{venue="polymarket",source="direct",class="price",conn="warm",cache!="hit"}[24h])) + success: clamp_max(sum(rate(pmapi_requests_total{venue="polymarket",source="direct",class="price",conn="warm",outcome="ok"}[24h])) / sum(rate(pmapi_requests_total{venue="polymarket",source="direct",class="price",conn="warm",outcome!="probe_invalid"}[24h])), 1) + sample_size: sum(increase(pmapi_requests_total{venue="polymarket",source="direct",class="price",conn="warm",outcome!="probe_invalid"}[24h])) + series: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="polymarket",source="direct",class="price",conn="warm",cache!="hit"}[1h])) by (le)) - slug: kalshi name: Kalshi tag: Regulated US venue, single market endpoint goes to origin on every request formula: "p50 of warm, non CDN cached round trips against the single market endpoint for the pinned ticker, successful requests only, 24h window." queries: - p50: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="kalshi",class="price",conn="warm",cache!="hit"}[24h])) by (le)) - p90: 1000 * histogram_quantile(0.90, sum(rate(pmapi_request_duration_seconds_bucket{venue="kalshi",class="price",conn="warm",cache!="hit"}[24h])) by (le)) - p99: 1000 * histogram_quantile(0.99, sum(rate(pmapi_request_duration_seconds_bucket{venue="kalshi",class="price",conn="warm",cache!="hit"}[24h])) by (le)) - mean: 1000 * sum(rate(pmapi_request_duration_seconds_sum{venue="kalshi",class="price",conn="warm",cache!="hit"}[24h])) / sum(rate(pmapi_request_duration_seconds_count{venue="kalshi",class="price",conn="warm",cache!="hit"}[24h])) - success: clamp_max(sum(rate(pmapi_requests_total{venue="kalshi",class="price",conn="warm",outcome="ok"}[24h])) / sum(rate(pmapi_requests_total{venue="kalshi",class="price",conn="warm",outcome!="probe_invalid"}[24h])), 1) - sample_size: sum(increase(pmapi_requests_total{venue="kalshi",class="price",conn="warm",outcome!="probe_invalid"}[24h])) - series: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="kalshi",class="price",conn="warm",cache!="hit"}[1h])) by (le)) + p50: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="kalshi",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + p90: 1000 * histogram_quantile(0.90, sum(rate(pmapi_request_duration_seconds_bucket{venue="kalshi",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + p99: 1000 * histogram_quantile(0.99, sum(rate(pmapi_request_duration_seconds_bucket{venue="kalshi",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + mean: 1000 * sum(rate(pmapi_request_duration_seconds_sum{venue="kalshi",source="direct",class="price",conn="warm",cache!="hit"}[24h])) / sum(rate(pmapi_request_duration_seconds_count{venue="kalshi",source="direct",class="price",conn="warm",cache!="hit"}[24h])) + success: clamp_max(sum(rate(pmapi_requests_total{venue="kalshi",source="direct",class="price",conn="warm",outcome="ok"}[24h])) / sum(rate(pmapi_requests_total{venue="kalshi",source="direct",class="price",conn="warm",outcome!="probe_invalid"}[24h])), 1) + sample_size: sum(increase(pmapi_requests_total{venue="kalshi",source="direct",class="price",conn="warm",outcome!="probe_invalid"}[24h])) + series: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="kalshi",source="direct",class="price",conn="warm",cache!="hit"}[1h])) by (le)) - slug: limitless name: Limitless tag: CLOB venue, undocumented API, errors CDN cached for 4 hours formula: "p50 of warm, non CDN cached round trips against the single market endpoint, successful requests only, 24h window. Stale pin errors are probe_invalid and excluded." queries: - p50: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="limitless",class="price",conn="warm",cache!="hit"}[24h])) by (le)) - p90: 1000 * histogram_quantile(0.90, sum(rate(pmapi_request_duration_seconds_bucket{venue="limitless",class="price",conn="warm",cache!="hit"}[24h])) by (le)) - p99: 1000 * histogram_quantile(0.99, sum(rate(pmapi_request_duration_seconds_bucket{venue="limitless",class="price",conn="warm",cache!="hit"}[24h])) by (le)) - mean: 1000 * sum(rate(pmapi_request_duration_seconds_sum{venue="limitless",class="price",conn="warm",cache!="hit"}[24h])) / sum(rate(pmapi_request_duration_seconds_count{venue="limitless",class="price",conn="warm",cache!="hit"}[24h])) - success: clamp_max(sum(rate(pmapi_requests_total{venue="limitless",class="price",conn="warm",outcome="ok"}[24h])) / sum(rate(pmapi_requests_total{venue="limitless",class="price",conn="warm",outcome!="probe_invalid"}[24h])), 1) - sample_size: sum(increase(pmapi_requests_total{venue="limitless",class="price",conn="warm",outcome!="probe_invalid"}[24h])) - series: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="limitless",class="price",conn="warm",cache!="hit"}[1h])) by (le)) + p50: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="limitless",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + p90: 1000 * histogram_quantile(0.90, sum(rate(pmapi_request_duration_seconds_bucket{venue="limitless",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + p99: 1000 * histogram_quantile(0.99, sum(rate(pmapi_request_duration_seconds_bucket{venue="limitless",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + mean: 1000 * sum(rate(pmapi_request_duration_seconds_sum{venue="limitless",source="direct",class="price",conn="warm",cache!="hit"}[24h])) / sum(rate(pmapi_request_duration_seconds_count{venue="limitless",source="direct",class="price",conn="warm",cache!="hit"}[24h])) + success: clamp_max(sum(rate(pmapi_requests_total{venue="limitless",source="direct",class="price",conn="warm",outcome="ok"}[24h])) / sum(rate(pmapi_requests_total{venue="limitless",source="direct",class="price",conn="warm",outcome!="probe_invalid"}[24h])), 1) + sample_size: sum(increase(pmapi_requests_total{venue="limitless",source="direct",class="price",conn="warm",outcome!="probe_invalid"}[24h])) + series: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="limitless",source="direct",class="price",conn="warm",cache!="hit"}[1h])) by (le)) - slug: manifold name: Manifold tag: AMM, bot friendly, whole API behind a 5s CDN cache, hits excluded here formula: "p50 of warm round trips against the single market endpoint, origin responses only (CDN cache hits excluded), successful requests, 24h window." queries: - p50: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="manifold",class="price",conn="warm",cache!="hit"}[24h])) by (le)) - p90: 1000 * histogram_quantile(0.90, sum(rate(pmapi_request_duration_seconds_bucket{venue="manifold",class="price",conn="warm",cache!="hit"}[24h])) by (le)) - p99: 1000 * histogram_quantile(0.99, sum(rate(pmapi_request_duration_seconds_bucket{venue="manifold",class="price",conn="warm",cache!="hit"}[24h])) by (le)) - mean: 1000 * sum(rate(pmapi_request_duration_seconds_sum{venue="manifold",class="price",conn="warm",cache!="hit"}[24h])) / sum(rate(pmapi_request_duration_seconds_count{venue="manifold",class="price",conn="warm",cache!="hit"}[24h])) - success: clamp_max(sum(rate(pmapi_requests_total{venue="manifold",class="price",conn="warm",outcome="ok"}[24h])) / sum(rate(pmapi_requests_total{venue="manifold",class="price",conn="warm",outcome!="probe_invalid"}[24h])), 1) - sample_size: sum(increase(pmapi_requests_total{venue="manifold",class="price",conn="warm",outcome!="probe_invalid"}[24h])) - series: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="manifold",class="price",conn="warm",cache!="hit"}[1h])) by (le)) + p50: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="manifold",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + p90: 1000 * histogram_quantile(0.90, sum(rate(pmapi_request_duration_seconds_bucket{venue="manifold",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + p99: 1000 * histogram_quantile(0.99, sum(rate(pmapi_request_duration_seconds_bucket{venue="manifold",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + mean: 1000 * sum(rate(pmapi_request_duration_seconds_sum{venue="manifold",source="direct",class="price",conn="warm",cache!="hit"}[24h])) / sum(rate(pmapi_request_duration_seconds_count{venue="manifold",source="direct",class="price",conn="warm",cache!="hit"}[24h])) + success: clamp_max(sum(rate(pmapi_requests_total{venue="manifold",source="direct",class="price",conn="warm",outcome="ok"}[24h])) / sum(rate(pmapi_requests_total{venue="manifold",source="direct",class="price",conn="warm",outcome!="probe_invalid"}[24h])), 1) + sample_size: sum(increase(pmapi_requests_total{venue="manifold",source="direct",class="price",conn="warm",outcome!="probe_invalid"}[24h])) + series: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="manifold",source="direct",class="price",conn="warm",cache!="hit"}[1h])) by (le)) - slug: myriad name: Myriad tag: Single region US East Heroku origin, smallest API surface in the cohort formula: "p50 of warm, non CDN cached round trips against the single market endpoint, successful requests only, 24h window. Cross region latency reflects the single US East origin." queries: - p50: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="myriad",class="price",conn="warm",cache!="hit"}[24h])) by (le)) - p90: 1000 * histogram_quantile(0.90, sum(rate(pmapi_request_duration_seconds_bucket{venue="myriad",class="price",conn="warm",cache!="hit"}[24h])) by (le)) - p99: 1000 * histogram_quantile(0.99, sum(rate(pmapi_request_duration_seconds_bucket{venue="myriad",class="price",conn="warm",cache!="hit"}[24h])) by (le)) - mean: 1000 * sum(rate(pmapi_request_duration_seconds_sum{venue="myriad",class="price",conn="warm",cache!="hit"}[24h])) / sum(rate(pmapi_request_duration_seconds_count{venue="myriad",class="price",conn="warm",cache!="hit"}[24h])) - success: clamp_max(sum(rate(pmapi_requests_total{venue="myriad",class="price",conn="warm",outcome="ok"}[24h])) / sum(rate(pmapi_requests_total{venue="myriad",class="price",conn="warm",outcome!="probe_invalid"}[24h])), 1) - sample_size: sum(increase(pmapi_requests_total{venue="myriad",class="price",conn="warm",outcome!="probe_invalid"}[24h])) - series: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="myriad",class="price",conn="warm",cache!="hit"}[1h])) by (le)) + p50: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="myriad",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + p90: 1000 * histogram_quantile(0.90, sum(rate(pmapi_request_duration_seconds_bucket{venue="myriad",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + p99: 1000 * histogram_quantile(0.99, sum(rate(pmapi_request_duration_seconds_bucket{venue="myriad",source="direct",class="price",conn="warm",cache!="hit"}[24h])) by (le)) + mean: 1000 * sum(rate(pmapi_request_duration_seconds_sum{venue="myriad",source="direct",class="price",conn="warm",cache!="hit"}[24h])) / sum(rate(pmapi_request_duration_seconds_count{venue="myriad",source="direct",class="price",conn="warm",cache!="hit"}[24h])) + success: clamp_max(sum(rate(pmapi_requests_total{venue="myriad",source="direct",class="price",conn="warm",outcome="ok"}[24h])) / sum(rate(pmapi_requests_total{venue="myriad",source="direct",class="price",conn="warm",outcome!="probe_invalid"}[24h])), 1) + sample_size: sum(increase(pmapi_requests_total{venue="myriad",source="direct",class="price",conn="warm",outcome!="probe_invalid"}[24h])) + series: 1000 * histogram_quantile(0.50, sum(rate(pmapi_request_duration_seconds_bucket{venue="myriad",source="direct",class="price",conn="warm",cache!="hit"}[1h])) by (le)) diff --git a/benchmarks/pm-data-freshness.yml b/benchmarks/pm-data-freshness.yml index 15a89842..45338cd3 100644 --- a/benchmarks/pm-data-freshness.yml +++ b/benchmarks/pm-data-freshness.yml @@ -2,74 +2,81 @@ slug: pm-data-freshness number: "032" -title: Fastest Polymarket data API, live freshness across Mobula and Codex -seo_title: "Fastest Polymarket data API 2026: Mobula vs Codex freshness" -seo_description: "Fastest prediction market data API ranked live by Polymarket freshness. Milliseconds Mobula and Codex lag the Polymarket CLOB WebSocket on top markets." -subtitle: Per event delay between provider arrival and Polymarket gateway publish, measured every minute on the top markets by 24 hour volume. +title: Fastest prediction market data API, live freshness across venues +seo_title: "Fastest prediction market data API 2026: Polymarket + Kalshi freshness" +seo_description: "Fastest prediction market data API ranked live across Polymarket and Kalshi. Milliseconds Mobula and Codex lag the venue gateway publish on top markets." +subtitle: Per event delay between provider arrival and the venue gateway publish, measured every minute on the top markets by 24 hour volume across Polymarket and Kalshi. category: Aggregators status: live -metric: Freshness delta vs Polymarket +metric: Freshness delta vs venue unit: ms higher_is_better: false seo_intro: | Prediction markets generate the most time sensitive event stream in crypto. An election market settles in seconds, a sports book moves on every play. - Builders that integrate Polymarket through a data provider rather than - hitting the CLOB directly need to know how many milliseconds that - provider adds between when Polymarket itself publishes a trade and when - the provider relays the same trade to its WebSocket subscribers. This - benchmark measures exactly that. The harness subscribes to the same - basket of top volume Polymarket markets on Polymarket's own CLOB - WebSocket (the canonical source, T0), on Mobula's PM WebSocket, and on - Codex GraphQL subscriptions. Each trade is cross correlated by - condition id and price across the three streams, and the per provider - lag versus Polymarket's gateway publish time is recorded as a - Prometheus histogram. The leaderboard sorts by p50 freshness delta in - milliseconds, lower is better. + Builders that integrate Polymarket or Kalshi through a data provider + rather than hitting each venue directly need to know how many + milliseconds that provider adds between when the venue itself publishes + a trade and when the provider relays the same trade to its WebSocket + subscribers. This benchmark measures exactly that. The harness + subscribes to the same basket of top volume markets on each venue's own + canonical source (T0), on Mobula's PM WebSocket, and on Codex GraphQL + subscriptions. Each trade is cross correlated across the streams, and + the per provider lag versus the venue gateway publish time is recorded + as a Prometheus histogram. The leaderboard sorts by p50 freshness delta + in milliseconds, lower is better. Use the venue tab at the top of the + page to switch between Polymarket and Kalshi. abstract: | Three WebSocket subscribers ride the same rotating basket of ~20 top - volume Polymarket markets simultaneously. For every trade event - published on Polymarket's own CLOB WebSocket gateway (the canonical - T0), we record the moment it lands and the moment each provider - relays the same trade. The signature used to match a trade across - providers is the tuple (conditionId, priceUSD rounded to 3 decimals, - trade size, 5 second time bucket), which is robust against the minor - clock skew between gateways. Providers that fail to relay a trade - within 90 seconds are not counted toward their p50, only toward their - receive total, so a provider can look fresh on the leaderboard while - silently dropping events. The success rate column flags that. + volume markets simultaneously on each venue. For every trade event + published on the venue's own canonical WebSocket gateway (the T0), we + record the moment it lands and the moment each provider relays the + same trade. The signature used to match a trade across providers is a + tuple based on the market identifier, price and size of the fill, and + a small time bucket, which is robust against the minor clock skew + between gateways. Providers that fail to relay a trade within 90 + seconds are not counted toward their p50, only toward their receive + total, so a provider can look fresh on the leaderboard while silently + dropping events. The success rate column flags that. Coverage today: + Mobula and Codex on Polymarket, Codex only on Kalshi (Mobula does not + yet cover Kalshi venue data). methodology: - - "Polymarket CLOB WebSocket is the canonical T0. `wss://ws-subscriptions-clob.polymarket.com/ws/market` is public, no auth, sub 50ms gateway publish latency from EU West." - - "Mobula PM WebSocket. `wss://pm-api-prod-eu.mobula.io`. Auth via API key in the subscribe payload. Cloudflare on the gateway requires a browser User Agent on the upgrade request, default Go HTTP UA is silently filtered." - - "Codex GraphQL subscriptions. `wss://graph.codex.io/graphql` with the `graphql-transport-ws` subprotocol. Firehose `onPredictionTradesCreated` subscription filtered client side to the Polymarket protocol marketIds in our basket." - - "Basket: top 20 active Polymarket markets by 24h volume, refreshed every 5 minutes from `gamma-api.polymarket.com`. Each market contributes two clobTokenIds (Yes and No outcomes)." - - "Cross correlation key: (conditionId, priceUSD × 1000 rounded, sizeUSD × 1_000_000 rounded, floor(trade_time / 5s)). The 5 second bucket absorbs clock skew without merging unrelated trades." - - "Histogram buckets: 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000 ms. Polymarket's own arrivals always land in the smallest bucket because by construction we measure other providers against it." - - "Region: `eu-west` (Railway europe-west4). Polymarket gateway is geographically distributed; deltas reflect EU client to EU gateway latency." + - "Polymarket T0. `wss://ws-subscriptions-clob.polymarket.com/ws/market` is public, no auth, sub 50ms gateway publish latency from EU West. Cross correlation uses (conditionId, priceUSD rounded to 3 decimals, sizeUSD micros, 5s time bucket)." + - "Kalshi T0. `https://api.elections.kalshi.com/v1/social/trades` (REST), polled every 5s with cursor pagination. Every trade carries `create_date` at microsecond precision; that timestamp is the canonical T0. Poll cadence affects only correlation timing, not the freshness number. The official Kalshi WebSocket needs RSA PSS signed headers from a US KYC account and 403s from non US IPs, not viable for a public benchmark." + - "Mobula PM WebSocket. `wss://pm-api-prod-eu.mobula.io`. Auth via API key in the subscribe payload. Cloudflare on the gateway requires a browser User Agent on the upgrade request, default Go HTTP UA is silently filtered. Covers Polymarket today, Kalshi coverage is not yet shipped." + - "Codex GraphQL subscriptions. `wss://graph.codex.io/graphql` with the `graphql-transport-ws` subprotocol. Firehose `onPredictionTradesCreated` subscription filtered client side to the venue's marketIds. Covers both Polymarket and Kalshi." + - "Basket. Top 20 active markets by 24h volume per venue, refreshed every 5 minutes. Polymarket pulls from `gamma-api.polymarket.com`, Kalshi pulls from `api.elections.kalshi.com/trade-api/v2/markets`." + - "Histogram buckets: 10, 25, 50, 100, 250, 500, 1000, 2500, 5000, 10000, 30000, 60000 ms. The venue's own arrivals always land in the smallest bucket because by construction we measure other providers against it." + - "Region. `eu-west` (Railway europe-west4). Both venue gateways are geographically distributed; deltas reflect EU client to EU gateway latency." + - "Venue clock asymmetry. Polymarket CLOB trades carry no venue timestamp so T0 is when our harness receives the WS event; the Polymarket-to-harness RTT cancels inside the Codex delta. Kalshi's REST `create_date` is venue-side so the Codex Kalshi delta includes a Codex-to-harness RTT (50 to 100 ms) the Polymarket math cancels out. Kalshi rows are slightly stricter for the same provider, not more favorable; the gap is small versus the multi-second deltas the chart highlights." findings: - - "Polymarket's own CLOB WebSocket is by definition the freshest source on the leaderboard. The lag versus its own gateway publish time is on the order of the network round trip from the harness to the gateway, typically below 100 ms p50 from EU West." - - "{{name:mobula}} relays Polymarket trades with a p50 delta of {{p50:mobula}} versus Polymarket native gateway. Mobula's PM WebSocket runs on a dedicated edge service in EU West and forwards events without orderbook reconstruction, which is why the gap to native is small." - - "{{name:codex}} sits at p50 {{p50:codex}} because Codex indexes the chain event (Polygon block confirmation) rather than the off chain orderbook publish. The lag includes Polygon block time (~2s) plus ingestion." + - "Each venue's own gateway is by definition the freshest source on the leaderboard for that venue. The lag versus its own publish time is on the order of the network round trip from the harness to the gateway, typically below 100 ms p50 from EU West." + - "{{name:mobula}} relays Polymarket trades with a p50 delta of {{p50:mobula}} versus the venue native gateway on the Polymarket tab. Mobula's PM WebSocket runs on a dedicated edge service in EU West and forwards events without orderbook reconstruction, which is why the gap to native is small. Mobula does not yet cover Kalshi, so the Kalshi tab shows Codex alongside the Kalshi T0 row only." + - "{{name:codex}} sits at p50 {{p50:codex}} on the Polymarket tab because Codex indexes the chain event (Polygon block confirmation) rather than the off chain orderbook publish. The lag includes Polygon block time (~2s) plus ingestion. On Kalshi Codex ingests Kalshi's own WebSocket feed directly, so the delta is purely pipeline latency." - "The spread between providers reflects integration depth: native gateway vs edge cached relay vs chain indexed pipeline. None of these is wrong, they answer different questions. For live trading UIs the gateway path is the only viable one." faq: - - q: "Which Polymarket data API has the lowest latency right now?" - a: "{{best_name}} currently leads at {{best_p50}} (p50 over the last 24h) measured as time from Polymarket's own CLOB WebSocket publish to provider relay. The leaderboard re sorts every minute on fresh Prometheus samples, so the answer reflects the actual measured lag on the active market basket, not a marketing claim." + - q: "Which prediction market data API has the lowest latency right now?" + a: "{{best_name}} currently leads at {{best_p50}} (p50 over the last 24h) measured as time from the venue's own canonical WebSocket publish to provider relay. The leaderboard re sorts every minute on fresh Prometheus samples, so the answer reflects the actual measured lag on the active market basket, not a marketing claim. Switch the venue tab at the top to see Polymarket vs Kalshi independently." - q: "What does 'freshness delta' mean for a prediction market API?" - a: "We connect to Polymarket's own CLOB WebSocket and to the provider's WebSocket simultaneously, subscribe to the same markets, and for every trade event we record how many milliseconds the provider takes to relay the event after Polymarket itself publishes it. Lower is better. Polymarket's own gateway publish time is the canonical T0 because by construction nothing downstream can be faster than the source." + a: "We connect to the venue's own canonical WebSocket and to the provider's WebSocket simultaneously, subscribe to the same markets, and for every trade event we record how many milliseconds the provider takes to relay the event after the venue itself publishes it. Lower is better. The venue's own gateway publish time is the canonical T0 because by construction nothing downstream can be faster than the source." - q: "Is Mobula's PM WebSocket faster than Codex?" - a: "It depends on what each provider does under the hood. {{name:mobula}} is an edge cached relay of Polymarket's own gateway, so the p50 delta is roughly the network round trip between the two gateways plus a few ms of bookkeeping. {{name:codex}} ingests the on chain confirmation on Polygon, which adds the block time (~2 seconds) before any trade can be relayed. For a live UI building on Polymarket, the relay path wins on freshness. For on chain reconciliation or settlement workflows, the chain indexed path is what you actually want. They answer different questions." + a: "When the Polymarket tab is selected, it depends on what each provider does under the hood. {{name:mobula}} is an edge cached relay of Polymarket's own gateway, so the p50 delta is roughly the network round trip between the two gateways plus a few ms of bookkeeping. {{name:codex}} ingests the on chain confirmation on Polygon, which adds the block time (~2 seconds) before any trade can be relayed. For a live UI building on Polymarket, the relay path wins on freshness. For on chain reconciliation or settlement workflows, the chain indexed path is what you actually want. On the Kalshi tab the comparison does not apply yet because Mobula does not cover Kalshi." - q: "Why don't you include Polymarket REST polling on this benchmark?" - a: "Freshness is a WebSocket question. REST polling at 1s would have a floor freshness around 500ms (poll interval / 2) plus RTT, dominated by how often you poll. The Polymarket gateway WebSocket exists for exactly this reason, to avoid that floor. Adding REST as a row would make the leaderboard noisy without changing the conclusion: WebSocket beats polling by definition for real time data." + a: "When the Polymarket tab is selected, freshness is a WebSocket question. REST polling at 1s would have a floor freshness around 500ms (poll interval / 2) plus RTT, dominated by how often you poll. The Polymarket gateway WebSocket exists for exactly this reason, to avoid that floor. Adding REST as a row would make the leaderboard noisy without changing the conclusion: WebSocket beats polling by definition for real time data." + - q: "Why is only Codex shown on the Kalshi tab?" + a: "Because Mobula's PM API does not cover Kalshi venue data today. The Kalshi tab compares Codex against the Kalshi T0 source only. Mobula coverage of Kalshi is on the roadmap; when it ships the same provider will appear on both tabs and the comparison becomes apples to apples again." + - q: "What does the Kalshi T0 source measure?" + a: "Kalshi's own public WebSocket at `wss://external-api-ws.kalshi.com/trade-api/ws/v2`, channel `trade`. The harness reads the `msg.ts_ms` field of each fill as the canonical publish timestamp and uses it as T0 for every Kalshi side comparison. Just like Polymarket's CLOB WebSocket on the Polymarket tab, it is the source against which provider latency is measured." - q: "Are these numbers comparable to Kalshi or Limitless?" - a: "Not directly. This benchmark measures Polymarket as the underlying venue, since it's the largest and the one most providers proxy. Kalshi and Limitless are separate exchanges with separate data feeds, and providers that cover them often have a different ingestion path. We may add a Kalshi specific tab in a later phase. For now, treat the leaderboard as 'how fresh is your Polymarket data feed'." + a: "Kalshi is now its own tab on this page, click the Venue selector at the top to switch. Each venue uses its own canonical WebSocket as T0, so the absolute numbers are not directly comparable across tabs (different gateways, different network paths), but the relative provider ordering within each tab is honest. Limitless is not yet covered, we will add a tab if and when meaningful provider coverage exists for it." - q: "How does OpenChainBench measure freshness?" - a: "Three WebSocket clients run in parallel inside the harness, all subscribed to the same basket of top volume Polymarket markets. Every 5 minutes we refresh the basket from `gamma-api.polymarket.com`. For each trade event, we compute a signature `(conditionId, price rounded to 3 decimals, size in micros, 5 second time bucket)` and record the wall clock receive time on each provider. The freshness delta is `recv_provider - recv_polymarket` for the same signature. We export the histogram to Prometheus, the leaderboard reads the 24h p50." + a: "Three WebSocket clients run in parallel inside the harness, all subscribed to the same basket of top volume markets on the active venue. Every 5 minutes we refresh the basket from the venue's own markets API. For each trade event, we compute a signature based on market id, price, size and a 5 second time bucket, and record the wall clock receive time on each provider. The freshness delta is `recv_provider - recv_venue` for the same signature. We export the histogram to Prometheus, the leaderboard reads the 24h p50." source: https://github.com/MobulaFi/mobula-monorepo/tree/main/miniapps/pm-freshness-bench @@ -77,28 +84,56 @@ prometheus: window: 24h expected_freshness_seconds: 300 +# Venue selector. tabs at the top of the page. Server injects +# `venue="X"` into every PromQL query for the active tab. There is no +# "all" entry because cross venue averaging would mix two different +# canonical T0 gateways, which is not a meaningful comparison; the +# default landing tab is Polymarket (the first entry below). +dimensions: + venue: + - { value: polymarket, label: Polymarket } + - { value: kalshi, label: Kalshi } + providers: - slug: polymarket name: Polymarket tag: Native gateway WebSocket, no auth, sub 50ms publish latency - formula: "Polymarket gateway publish time is the canonical T0. The near zero row is the network round trip from the harness to the gateway, not a comparison against another source." + formula: "Polymarket gateway publish time is the canonical T0 on the Polymarket tab. The near zero row is the network round trip from the harness to the gateway, not a comparison against another source." queries: - # Polymarket is T0 by construction. Delta vs itself is zero. Use a - # sub millisecond floor (0.5 ms) so the row and the chart line are - # visible on a linear y axis next to the multi second Codex value. - # Pure vector(0) gets clipped to the axis baseline and disappears. - p50: vector(0.5) - p90: vector(0.5) - p99: vector(0.5) - mean: vector(0.5) + # Polymarket is T0 by construction on the Polymarket tab. Multiply + # the 0.5 ms floor by pm_health{provider="polymarket"} so the + # dimension framework's injected venue label gates the row: + # pm_health{provider="polymarket", venue="polymarket"} = 1 -> 0.5, + # pm_health{provider="polymarket", venue="kalshi"} = 0 -> 0, + # and the leaderboard hides empty rows. + p50: pm_health{provider="polymarket"} * 0.5 + p90: pm_health{provider="polymarket"} * 0.5 + p99: pm_health{provider="polymarket"} * 0.5 + mean: pm_health{provider="polymarket"} * 0.5 success: avg_over_time(pm_health{provider="polymarket"}[24h]) sample_size: sum(increase(pm_events_total{provider="polymarket"}[24h])) - series: vector(0.5) + series: pm_health{provider="polymarket"} * 0.5 + + - slug: kalshi + name: Kalshi + tag: Native venue REST, create_date is canonical T0 + formula: "Kalshi gateway publish time is the canonical T0 on the Kalshi tab. The near zero row is the network round trip from the harness to the Kalshi endpoint, not a comparison against another source." + queries: + # Symmetric to the Polymarket row, gated by pm_health{provider="kalshi"} + # so the venue label injected at runtime zeros out the row on the + # Polymarket tab. + p50: pm_health{provider="kalshi"} * 0.5 + p90: pm_health{provider="kalshi"} * 0.5 + p99: pm_health{provider="kalshi"} * 0.5 + mean: pm_health{provider="kalshi"} * 0.5 + success: avg_over_time(pm_health{provider="kalshi"}[24h]) + sample_size: sum(increase(pm_events_total{provider="kalshi"}[24h])) + series: pm_health{provider="kalshi"} * 0.5 - slug: mobula name: Mobula - tag: Edge cached Polymarket relay, browser UA required - formula: "Median ms lag versus Polymarket CLOB gateway, measured by cross correlating trades by (conditionId, price, size, 5s bucket) over the rotating basket of top 20 markets." + tag: Edge cached venue relay, browser UA required (Polymarket only today) + formula: "Median ms lag versus the venue gateway, measured by cross correlating trades over the rotating basket of top 20 markets. Polymarket coverage today, Kalshi pending." queries: p50: histogram_quantile(0.50, sum(rate(pm_freshness_delta_ms_bucket{provider="mobula",kind="trade"}[24h])) by (le)) p90: histogram_quantile(0.90, sum(rate(pm_freshness_delta_ms_bucket{provider="mobula",kind="trade"}[24h])) by (le)) @@ -110,8 +145,8 @@ providers: - slug: codex name: Codex - tag: On chain Polygon indexer, ~2s block time floor - formula: "Median ms lag versus Polymarket CLOB gateway. Codex indexes the on chain Polygon settlement of each trade, so the lag includes block time (~2s) plus ingestion." + tag: Chain indexer on Polymarket, native ingestion on Kalshi + formula: "Median ms lag versus the venue gateway. On Polymarket Codex indexes the on chain Polygon settlement of each trade (block time ~2s plus ingestion). On Kalshi Codex ingests Kalshi's own WebSocket directly so the lag is pure pipeline latency." queries: p50: histogram_quantile(0.50, sum(rate(pm_freshness_delta_ms_bucket{provider="codex",kind="trade"}[24h])) by (le)) p90: histogram_quantile(0.90, sum(rate(pm_freshness_delta_ms_bucket{provider="codex",kind="trade"}[24h])) by (le)) diff --git a/benchmarks/solana-tx-landing-latency.yml b/benchmarks/solana-tx-landing-latency.yml deleted file mode 100644 index 8e1858b8..00000000 --- a/benchmarks/solana-tx-landing-latency.yml +++ /dev/null @@ -1,221 +0,0 @@ -# OpenChainBench. Bench № 027 - -slug: solana-tx-landing-latency -number: "027" -title: Fastest Solana RPC for tx landing, live slot delta benchmark -seo_title: "Fastest Solana RPC 2026: Helius, Jito, Mobula slot delta ranked" -seo_description: "{{best_name}} leads fastest Solana RPC for tx landing at {{best_p50}} (p50, 7d). Helius, Jito, Astralane, Mobula, Nozomi probed hourly with signed mainnet txs." -subtitle: How fast each landing service gets a signed mainnet tx confirmed. Slot delta = number of Solana slots between submit and confirmed (1 slot is roughly 400 ms). Active probing every hour from us-east. -category: Trading -status: live -metric: p50 slot delta to confirmed (7-day window) -unit: slots -higher_is_better: false - -disclaimer: | - Six caveats. (1) us-east only, sgp / eu-west arrive in V2. (2) One pre-registered tip per service. (3) Synthetic payload (1-lamport + memo); real swaps may land differently. (4) Helius / Astralane / Nozomi fan out to Jito internally; Jito control probe runs each cycle. (5) Confirmation = `confirmed`. (6) Slot delta is canonical; ms is derived (≈ slot_delta × 400 ms + RTT). Pair with /benchmarks/solana-tx-landing. - -seo_intro: | - This benchmark answers the only question that matters to a - Solana trader picking a landing service. how many slots does - your signed mainnet transaction take to reach the confirmed - state on chain. Every hour from a us-east probe, the harness - submits an identical signed tx through each of 5 services in - parallel, captures the submit slot before send and the land - slot from the signatureSubscribe WebSocket notification at - commitment=confirmed, and increments per-service Prometheus - histograms. Headline numbers shown are p50 and p99 slot delta - over a rolling 7-day window. Wall-clock milliseconds are - published alongside for intuition (one Solana slot is ~400 ms, - so a p50 of 1 slot is ~400 ms wall-clock plus submission RTT) - but slot delta is the canonical, sponsor-proof on-chain - measurement. - Why slot delta is the right metric. Solana confirmation is a - slot-level event. when a slot reaches supermajority vote, every - transaction in it becomes confirmed simultaneously. Wall-clock - ms conflates HTTP submission time, our RPC's polling lag, and - network RTT to the public WebSocket - all of which are - measurement artifacts unrelated to the landing service's actual - routing quality. Slot delta is what the chain itself records. - Coverage. 5 services probed in V0-Lean. Jito Block Engine (the - control / baseline because Helius, Astralane, Nozomi all - internally route some flow through it). Helius Sender in - `swqos_only=true` mode (isolates the Helius own-path from the - Jito leg). Astralane Iris (tip-refund mechanism). Nozomi by - Temporal Labs (premium tier, hard 1M lamport tip floor). - 0slot.trade (premium tier). NextBlock, bloXroute and - SolanaVibeStation arrive in the next tier (V1) once the first - sponsors land. Companion bench. /benchmarks/solana-tx-landing - measures market share via on-chain tip-wallet attribution - - who carries the flow today, regardless of speed. - -abstract: | - We probe 5 Solana transaction landing services from a single - Railway us-east region, once per hour, by submitting an - identical signed mainnet transaction to each. The payload is - the minimal valid Solana tx, compute-budget instructions - (50k CU limit, 50k micro-lamport/CU price), a 1-lamport - self-transfer, the per-service tip transfer to the service's - documented tip wallet, and an OCB-prefixed memo for forensic - traceability. All five services are submitted in parallel - goroutines within a single cycle so they sample the same chain - congestion window. The headline measurement is slot delta, - land_slot minus submit_slot, captured from the - signatureSubscribe WebSocket notification's context.slot field - at commitment=confirmed. Wall-clock ms is reported alongside - but is a derived approximation, slot_delta × ~400 ms plus - submission RTT and goroutine startup variance. A 60 s no- - confirmation deadline classifies the probe as - dropped{reason=timeout}; structured RPC errors classify as - invalid; transport failures as network_error; HTTP 419 / 429 / - "rate limit" errors classify as rate_limited (a separate label - so quota issues don't bias the bench against the throttled - service). Cost. ~$159/mo at SOL=$86, 86 % of which goes to the - four ≥1M-lamport-floor services (Nozomi, 0slot, bloXroute, - NextBlock, only two of these in V0-Lean). Sponsor SOL credits - covering a service's own probes are explicitly allowed per the - sponsor-proof framework. Limitations. (a) Single us-east - region, sgp / eu-west arrive in V2 once sponsors fund - geographic-edge story. (b) 1-hour cadence, 168 probes per - service per 7-day window, enough for stable p50 / p99 over the - publication window, not enough for intra-hour resolution - (V0.5 / V1 upgrade if needed). (c) Fan-out, Helius probed in - `swqos_only` mode only in V0-Lean to keep wire shape simple; - dual-mode arrives in v1.0.1 methodology PR. - -methodology: - - "Source endpoints (us-east Railway, base64 JSON-RPC sendTransaction unless noted). Jito `ny.mainnet.block-engine.jito.wtf/api/v1/transactions`. Helius Sender `ewr-sender.helius-rpc.com/fast?swqos_only=true` (skipPreflight + maxRetries=0). Nozomi `http://edge.nozomi.temporal.xyz/api/sendBatch?c=` (binary `[u16_BE_len][tx_bytes]`, HTTP per Temporal Labs). Astralane `ny.gateway.astralane.io/iris?api-key=` (3-elem params, mevProtect). 0slot `ny.0slot.trade?api-key=`." - - "Probe payload. 5 instructions in this exact order: SetComputeUnitLimit(50,000) + SetComputeUnitPrice(50,000 micro-lamports/CU) + SystemProgram.Transfer(payer→payer, 1 lamport) + SystemProgram.Transfer(payer→service tip wallet, floor lamports) + Memo(`ocb---`). cycle_id is an 8-byte random hex shared across the five parallel probes of one cycle, so the on-chain memos correlate." - - "Tip floors (pre-registered, methodology PR + 14-day window to change). Jito 10,000 lamports. Helius Sender 10,000. Astralane 500,000 net of refunds. Nozomi 1,000,000. 0slot 1,000,000." - - "Submission flow. One getLatestBlockhash(processed) shared across all five probes. One getSlot(processed) as submit_slot. For each service we subscribe to the signature via signatureSubscribe on the public WS BEFORE submission (otherwise a fast confirm could fire before we listen). Probes then fire in parallel goroutines, sign, POST. We block on the signatureNotification at commitment=confirmed; context.slot is land_slot; slot_delta = land_slot - submit_slot." - - "Why slot delta is canonical. Solana confirmation is slot-level. when a slot reaches supermajority, every tx in it becomes confirmed simultaneously. The WS pushes notifications for all subscribed sigs in that slot at the same instant. So sub-400 ms wallclock diffs between services in the same slot are artifacts (goroutine startup, RTT), not routing quality. slot_delta is what the chain records, what to cite in audits." - - "Wall-clock ms is a derived approximation. ms ≈ slot_delta × ~400 ms + HTTP submission RTT + variance. We publish it for intuition because traders think in seconds, not slots, but it should not be the sole metric in a sponsor pitch or audit. If a service argues 'your ms numbers are biased by your RTT', the answer is the slot delta column, which is RTT-independent." - - "Drop classification. timeout = no confirmation within 60 s. invalid = RPC error, on-chain Err, or BlockhashNotFound. network_error = transport-level (timeout, DNS, EOF, connection refused). rate_limited = HTTP 419 / 429 / 'rate limit' / 'too many requests'. landing_rate is published as success / (success + timeout), rate_limited and network_error are excluded so quota / transport issues don't bias the bench against a throttled service." - - "Jito control probe. Helius (default), Astralane, Nozomi route a portion of flow through Jito internally, conflating own-path vs Jito-caught-it. Jito is in the V0-Lean set so its control fires in the same cycle. Same slot_delta as Jito = no measurable own-path value. Suspect ahead by 1+ slot = real routing advantage." - - "Reproducibility. The full harness source is at github.com/ChainBench/OpenChainBench/tree/main/harnesses/solana-tx-landing. Anyone with a funded Solana keypair (~1 SOL) can clone, set SOLANA_PROBE_KEYPAIR_BASE58, run the binary, and reproduce these metrics. The bench does not rely on any private or internal service for measurement, the only RPC dependency is the public `api.mainnet-beta.solana.com` HTTP + WebSocket endpoints." - - "Methodology v1.3 pre-registered at github.com/ChainBench/OpenChainBench/blob/main/docs/methodology/solana-tx-landing-active.md. Any change (tip floor, probe payload, cadence, region, metric definitions) ships as a public PR with a 14-day comment window. Major version bumps run a 30-day shadow period publishing old and new metrics in parallel." - -findings: - - "{{best_name}} leads the V0-Lean probe set at p50 = {{best_p50}} slot delta over the rolling 7-day window. Lower = fewer Solana slots between submission and confirmation. The gap between fastest and slowest is the operational signal, every service claims '99 %+ landing rate' in marketing copy, but the chain doesn't lie about which slot included your tx. A 1-slot difference is ~400 ms, enough for a MEV bot to front-run a competitor." - - "{{name:jito}} is the baseline / control. Helius (default mode), Astralane, and Nozomi all internally fan out to Jito, so the Jito p50 is the floor any premium service must beat. Same slot_delta as Jito on a given cycle = the service is essentially using Jito as its inclusion path. {{name:jito}} sits at p50 = {{p50:jito}} slot delta." - - "{{name:helius-sender}} in `swqos_only` mode isolates Helius's own routing path from the Jito leg. p50 = {{p50:helius-sender}} slot delta. A v1.0.1 methodology update will publish Helius default mode (with Jito fan-out) side-by-side for direct comparison." - - "{{name:nozomi}} premium pricing (1M lamport hard floor, ~10 × Jito's competitive level) only makes economic sense if the slot_delta advantage is meaningful. p50 = {{p50:nozomi}} slot delta. The gap vs Jito quantifies whether the tip premium buys real slot priority." - - "{{worst_name}} trails at p50 = {{worst_p50}} slot delta. The worst slot delta in the V0-Lean set is not necessarily a bad service, it may be a service whose strength is in dimensions this bench doesn't measure (anti-MEV protection, durable nonce, fee-refund mechanics). Latency is one variable, not the whole product." - -faq: - - q: "Why is slot delta the headline metric instead of wall-clock latency?" - a: "Solana confirmation is a slot-level event. when a slot reaches supermajority vote (~2/3 of stake), every transaction in that slot becomes confirmed simultaneously. The WebSocket pushes notifications for all subscribed signatures in that slot at the same instant. So if 3 services delivered txs that all landed in the same slot, our wallclock measurement records the same time for all 3, the only differentiation is whether the next service's tx landed in slot N or N+1. slot_delta captures that directly. Wall-clock ms is derived (slot_delta × ~400 ms + RTT + variance) and conflates routing quality with measurement artifacts like HTTP submission speed and our public RPC's network latency. We publish wall-clock ms because traders think in seconds, but slot_delta is what you should cite in an audit or methodology dispute. It's RTT-independent and reads directly from the chain." - - q: "What does '1 slot' actually mean in time?" - a: "Solana slots are ~400 ms in practice (~625 ms target with leader skips and forks averaging it down). A p50 slot_delta of 1 means your tx typically lands in the slot immediately following your submission, ~400 ms after sendTransaction return. p50 of 2 means typically one slot later, ~800 ms. The gap between p50 = 1 and p50 = 2 is the operational signal, a service that consistently lands 1 slot earlier than its competitors is ~400 ms ahead, which is the difference between catching an arbitrage and missing it." - - q: "Why an active bench when /benchmarks/solana-tx-landing already exists?" - a: "/benchmarks/solana-tx-landing is observational, it watches the chain and counts who carries the flow. It cannot answer 'how fast does my tx land if I send it now', because it doesn't send anything. This bench (active probing) answers that, at the cost of running 24 / 7 with real SOL ($159 / month at the V0-Lean cadence). The two benches answer different product questions. Read both." - - q: "Why only 5 services, not the 8 you measure observationally?" - a: "NextBlock, bloXroute Trader, and SolanaVibeStation all require paid plans or sales-call onboarding before they issue an API key. We're shipping V0-Lean today with the 5 services that have a clear self-serve or contact-based path. The other 3 will be added as the bench scales. The observational bench at /benchmarks/solana-tx-landing already covers all 8 because it doesn't need API keys." - - q: "Why us-east only?" - a: "V0-Lean. us-east is the de-facto Solana baseline (Jito, NextBlock, bloXroute, Helius all anchor their best-connected POPs there) and is where most Solana bots deploy by default. Adding eu-west and sgp triples the bench cost and answers a different question ('does the ranking change by geography?'), which is a planned V2 scope expansion." - - q: "What's the probe payload?" - a: "Five instructions in this exact order, locked by methodology §3. (1) SetComputeUnitLimit(50,000). (2) SetComputeUnitPrice(50,000 micro-lamports/CU), together a 2,500-lamport priority fee. (3) SystemProgram.Transfer of 1 lamport from the prober keypair to itself, the minimal valid state-touching tx. (4) SystemProgram.Transfer to the service's documented tip wallet at the pre-registered floor. (5) Memo program write with the cycle ID, service name, and probe mode. Total weight: ~600 bytes, well under the 1,232-byte tx limit." - - q: "How is fan-out handled?" - a: "Helius (default mode), Astralane, and Nozomi route a portion of flow through Jito internally. The Jito control probe, Jito is part of the V0-Lean probe set, fires in the same cycle as the suspect services with the same blockhash and a comparable tip. The slot_delta column tells you immediately whether a suspect service is adding value beyond a Jito wrapper. Same slot_delta as Jito = same inclusion slot = Jito caught it. Suspect ahead by 1+ slot = real own-path routing. Helius is additionally probed in `?swqos_only=true` mode to fully isolate its own routing path." - - q: "Can a service detect and prioritise our probes?" - a: "Yes, in principle. The memo prefix `ocb-` is deterministic and the keypair is constant per region. Anti-fingerprinting (memo randomisation, sub-account rotation, tip jitter within the floor band) ships in v1.0.2 methodology PR. We disclose this risk openly; the trade-off is that announcing the bench publicly to providers gives them a chance to fix real performance issues before we publish, which is a good outcome. We do NOT accept private deals to alter the probe surface for any specific service." - - q: "Why is sample_size on the dashboard ~168 per service?" - a: "V0-Lean cadence = 1 probe per service per hour from 1 region. 168 = 24 hours × 7 days. The 7-day publication window is the trade-off between statistical resolution (sample size grows with window) and freshness (shorter window reflects current chain conditions). At ~168 samples per cell, p50 is stable to within ±5 % and p99 to within ±15 %. Lower confidence intervals are unlocked at V0.5 cadence (1 / 10 min, ~$760 / mo) and above." - - q: "How is the confirmation observed?" - a: "Via `signatureSubscribe` on the public mainnet WebSocket (`wss://api.mainnet-beta.solana.com`). The subscription is registered BEFORE submission so a fast-confirming tx cannot complete before we are listening (otherwise we'd miss the notification and timeout spuriously). The RPC pushes the notification at the instant the commitment level is reached, so observation resolution is RTT-bounded (~30-50 ms us-east → mainnet-beta) and slot_delta is read directly from the notification's context.slot field. HTTP polling at 200 ms is an automatic fallback if the WebSocket fails to connect for a given cycle." - -source: https://github.com/ChainBench/OpenChainBench/tree/main/harnesses/solana-tx-landing - -prometheus: - window: 7d - expected_freshness_seconds: 7200 - -# Real metrics emitted by the active prober in solana-tx-landing harness: -# solana_landing_probe_success_total{service, mode, region} counter -# solana_landing_probe_dropped_total{service, mode, region, reason} counter -# solana_landing_probe_latency_ms{service, mode, region} gauge (set every cycle) -# solana_landing_probe_latency_slots{service, mode, region} gauge (set every cycle) -# solana_landing_probe_latency_slots_histogram{service, mode, region} histogram (debug) -# solana_landing_probe_latency_ms_histogram{service, mode, region} histogram (debug) -# solana_landing_probe_keypair_balance_sol{region} gauge -# solana_landing_probe_cycle_total{region} counter -# solana_landing_probe_enabled{region} gauge -# -# Headline metric (canonical) = slot_p50 / slot_p99 read from the gauge. -# Wall-clock ms is published alongside via the standard p50/p90/p99 fields -# for reader intuition but is derived (slot_delta × ~400 ms + RTT + variance). -# Mode label is `swqos_only` for helius-sender, `default` for the rest. -# -# Why quantile_over_time(gauge) instead of histogram_quantile(histogram)? -# At V0-Lean cadence (1 probe / hour) we have ~168 samples per cell over 7d. -# Histogram buckets {100, 250, 500, 1000, 2000, 5000, 10000, 30000, 60000} ms -# have ~3 buckets in the 1-5s zone where probes actually land, so -# histogram_quantile collapses to bucket midpoints (1500, 3500 ms) and the -# series looks flat. quantile_over_time on the gauge takes the real sample -# at the 50th percentile, which is the accurate published number. - -providers: - - slug: jito - name: Jito - tag: Baseline + control probe; atomic bundles + tip auction since 2022 - formula: "50th percentile over 7d of slot delta (land_slot − submit_slot) for hourly signed mainnet probes submitted to Jito's `ny.mainnet.block-engine.jito.wtf` from us-east." - queries: - p50: quantile_over_time(0.5, solana_landing_probe_latency_slots{service="jito",region="us-east"}[7d]) - p90: quantile_over_time(0.9, solana_landing_probe_latency_slots{service="jito",region="us-east"}[7d]) - p99: quantile_over_time(0.99, solana_landing_probe_latency_slots{service="jito",region="us-east"}[7d]) - mean: avg_over_time(solana_landing_probe_latency_slots{service="jito",region="us-east"}[7d]) - success: sum(rate(solana_landing_probe_success_total{service="jito",region="us-east"}[7d])) / (sum(rate(solana_landing_probe_success_total{service="jito",region="us-east"}[7d])) + sum(rate(solana_landing_probe_dropped_total{service="jito",region="us-east",reason="timeout"}[7d]))) - sample_size: sum(increase(solana_landing_probe_success_total{service="jito",region="us-east"}[7d])) - series: solana_landing_probe_latency_slots{service="jito",region="us-east"} - - - slug: helius-sender - name: Helius - tag: Isolated Helius own-path (no Jito fan-out); anycast + 7 POPs - formula: "50th percentile over 7d of slot delta (land_slot − submit_slot) for hourly signed probes submitted to Helius Sender in `swqos_only=true` mode from us-east, isolating its own-path." - queries: - p50: quantile_over_time(0.5, solana_landing_probe_latency_slots{service="helius-sender",mode="swqos_only",region="us-east"}[7d]) - p90: quantile_over_time(0.9, solana_landing_probe_latency_slots{service="helius-sender",mode="swqos_only",region="us-east"}[7d]) - p99: quantile_over_time(0.99, solana_landing_probe_latency_slots{service="helius-sender",mode="swqos_only",region="us-east"}[7d]) - mean: avg_over_time(solana_landing_probe_latency_slots{service="helius-sender",mode="swqos_only",region="us-east"}[7d]) - success: sum(rate(solana_landing_probe_success_total{service="helius-sender",mode="swqos_only",region="us-east"}[7d])) / (sum(rate(solana_landing_probe_success_total{service="helius-sender",mode="swqos_only",region="us-east"}[7d])) + sum(rate(solana_landing_probe_dropped_total{service="helius-sender",mode="swqos_only",region="us-east",reason="timeout"}[7d]))) - sample_size: sum(increase(solana_landing_probe_success_total{service="helius-sender",mode="swqos_only",region="us-east"}[7d])) - series: solana_landing_probe_latency_slots{service="helius-sender",mode="swqos_only",region="us-east"} - - - slug: astralane - name: Astralane - tag: Tip-refund mechanism, sendBundle / sendIdeal modes, FRA + NY POPs - formula: "50th percentile over 7d of slot delta (land_slot − submit_slot) for hourly signed probes submitted with a 500k-lamport net tip to Astralane Iris's NY gateway from us-east." - queries: - p50: quantile_over_time(0.5, solana_landing_probe_latency_slots{service="astralane",region="us-east"}[7d]) - p90: quantile_over_time(0.9, solana_landing_probe_latency_slots{service="astralane",region="us-east"}[7d]) - p99: quantile_over_time(0.99, solana_landing_probe_latency_slots{service="astralane",region="us-east"}[7d]) - mean: avg_over_time(solana_landing_probe_latency_slots{service="astralane",region="us-east"}[7d]) - success: sum(rate(solana_landing_probe_success_total{service="astralane",region="us-east"}[7d])) / (sum(rate(solana_landing_probe_success_total{service="astralane",region="us-east"}[7d])) + sum(rate(solana_landing_probe_dropped_total{service="astralane",region="us-east",reason="timeout"}[7d]))) - sample_size: sum(increase(solana_landing_probe_success_total{service="astralane",region="us-east"}[7d])) - series: solana_landing_probe_latency_slots{service="astralane",region="us-east"} - - - slug: nozomi - name: Nozomi - tag: Temporal Labs, direct-to-leader, premium 1M-lamport hard floor - formula: "50th percentile over 7d of slot delta (land_slot − submit_slot) for hourly signed probes submitted with a 1M-lamport tip to Nozomi's `edge.nozomi.temporal.xyz` from us-east." - queries: - p50: quantile_over_time(0.5, solana_landing_probe_latency_slots{service="nozomi",region="us-east"}[7d]) - p90: quantile_over_time(0.9, solana_landing_probe_latency_slots{service="nozomi",region="us-east"}[7d]) - p99: quantile_over_time(0.99, solana_landing_probe_latency_slots{service="nozomi",region="us-east"}[7d]) - mean: avg_over_time(solana_landing_probe_latency_slots{service="nozomi",region="us-east"}[7d]) - success: sum(rate(solana_landing_probe_success_total{service="nozomi",region="us-east"}[7d])) / (sum(rate(solana_landing_probe_success_total{service="nozomi",region="us-east"}[7d])) + sum(rate(solana_landing_probe_dropped_total{service="nozomi",region="us-east",reason="timeout"}[7d]))) - sample_size: sum(increase(solana_landing_probe_success_total{service="nozomi",region="us-east"}[7d])) - series: solana_landing_probe_latency_slots{service="nozomi",region="us-east"} - - - slug: mobula - name: Mobula - tag: Multi-RPC fan-out aggregator (relays via Jito / Nozomi / zeroslot) - formula: "50th percentile over 7d of slot delta (land_slot − submit_slot) for hourly probes submitted via Mobula's `api.mobula.io/api/2/swap/send` multi-RPC fan-out from us-east, using a Jito tip wallet." - queries: - p50: quantile_over_time(0.5, solana_landing_probe_latency_slots{service="mobula",region="us-east"}[7d]) - p90: quantile_over_time(0.9, solana_landing_probe_latency_slots{service="mobula",region="us-east"}[7d]) - p99: quantile_over_time(0.99, solana_landing_probe_latency_slots{service="mobula",region="us-east"}[7d]) - mean: avg_over_time(solana_landing_probe_latency_slots{service="mobula",region="us-east"}[7d]) - success: sum(rate(solana_landing_probe_success_total{service="mobula",region="us-east"}[7d])) / (sum(rate(solana_landing_probe_success_total{service="mobula",region="us-east"}[7d])) + sum(rate(solana_landing_probe_dropped_total{service="mobula",region="us-east",reason="timeout"}[7d]))) - sample_size: sum(increase(solana_landing_probe_success_total{service="mobula",region="us-east"}[7d])) - series: solana_landing_probe_latency_slots{service="mobula",region="us-east"} diff --git a/benchmarks/wallet-labels-coverage.yml b/benchmarks/wallet-labels-coverage.yml index 0b949be7..cfef9907 100644 --- a/benchmarks/wallet-labels-coverage.yml +++ b/benchmarks/wallet-labels-coverage.yml @@ -40,7 +40,7 @@ seo_intro: | response is a meaningful name, generic fillers like "EOA", "Contract" or "Wallet" are excluded because they carry no entity signal. Coverage is also reported per chain so chain-specialists - (Helius on Solana, TonAPI on TON, StellarExpert on Stellar, XRPScan + (Helius on Solana, TonAPI on Gram, StellarExpert on Stellar, XRPScan on XRP, WalletExplorer on Bitcoin) are not unfairly penalised for chains they do not claim to cover. Universal providers (Mobula, Moralis, Blockscout, OLI) are scored on every chain they advertise. @@ -66,7 +66,7 @@ abstract: | averaged into one number. Coverage is also shown per chain. Some providers cover 90+ chains (Mobula), - others are chain-specialists (Helius for Solana, TonAPI for TON, StellarExpert + others are chain-specialists (Helius for Solana, TonAPI for Gram, StellarExpert for Stellar, XRPScan for XRP, WalletExplorer for Bitcoin). Showing per-chain rather than aggregating prevents specialists from being unfairly penalized for chains they don't claim to cover. @@ -81,12 +81,38 @@ methodology: - "Failures (timeouts, 5xx, auth errors) are counted as 'no label' and surfaced separately as `wallet_labels_fetch_errors_total`." - "Region: `eu-west` (single point)." +per_chain_explainer: + - slug: solana + h2: "Best wallet labeling API on Solana" + body: | + The Solana wallet labeling leaderboard is driven by chain-specialist curation rather than universal coverage. Helius wins by default on this chain because its label graph is built directly against native Solana programs (Jito stake pools, Jupiter routers, Pump.fun creators, Magic Eden marketplace IDs) rather than translated from an EVM-shaped schema. Mobula audits Solana as part of its universal coverage and clocks {{p50:mobula}} on the active Kind tab. Blockscout does not index Solana, so the per-chain leaderboard here is between Solana-native APIs. + - slug: gram + h2: "Best wallet labeling API on Gram" + body: | + Gram (formerly TON) wallet labeling is essentially a two-API conversation. TonAPI ships a curated entity directory covering core Gram contracts (jettons, Telegram bot wallets, DEX routers like STON.fi and DeDust). Mobula audits Gram as part of its universal coverage. Most EVM-centric providers (Moralis, Blockscout) do not index Gram at all, so the per-chain leaderboard here surfaces the genuine specialist gap. Per the bench's coverage definition, generic categorical fillers like `EOA` or `Wallet` do not count toward a hit; only entity-grade names do. + - slug: stellar + h2: "Best wallet labeling API on Stellar" + body: | + Stellar wallet labeling is dominated by StellarExpert, whose `/explorer/directory/{addr}` endpoint is the curated entity graph the rest of the Stellar ecosystem cites. The directory covers anchor issuers (Circle USDC issuer, AnchorUSD), SDF accounts, DEX market makers and SDF grant recipients. Mobula audits Stellar as part of its universal coverage. EVM-centric providers (Moralis, Blockscout) do not index Stellar, so the per-chain leaderboard reflects a specialist-heavy field rather than a universal one. + - slug: bitcoin + h2: "Best wallet labeling API on Bitcoin" + body: | + Bitcoin wallet labeling is the hardest case in the benchmark because the chain has no smart contract layer to derive names from, every entity has to come from a curated graph. WalletExplorer maintains the canonical public clustering of Bitcoin addresses (exchanges, mixers, pools, OFAC SDN), and most downstream Bitcoin labeling products cite it directly. Mobula audits Bitcoin as part of its universal coverage. The Kind = Contract tab is essentially empty on Bitcoin (no verified-source contracts), so the EOA tab is the only honest comparison. + - slug: ethereum + h2: "Best wallet labeling API on Ethereum" + body: | + Ethereum is the chain where the EOA vs Contract split matters most. On the Contract tab Blockscout saturates near 100% because every Uniswap V3 router, WETH9 and Aave V3 pool ships a verified-source constructor name that the explorer reads for free. On the EOA tab the comparison flips to curated entity graphs: Mobula, Moralis and OLI on Base EAS each maintain their own directory of Binance hot wallets, Safe multisigs, OFAC SDN addresses and public figures. Switch the Kind tab at the top of the page to read the side you actually need. + - slug: base + h2: "Best wallet labeling API on Base" + body: | + Base wallet labeling is shaped by Coinbase's OP Stack rollup pattern. Most EVM-centric providers (Mobula, Moralis, Blockscout, OLI via Base EAS) advertise Base coverage and audit cleanly on contracts (Uniswap V4, Aerodrome, the Coinbase Wallet Smart Wallet factory). The EOA tab is the harder job here because Base wallets are dominated by retail Smart Wallets created through the Coinbase Wallet factory, which collapses to a single contract type that providers must distinguish from one another via deployment-time call data. + findings: - "{{best_name}} currently leads coverage at {{best_p50}} (24 h) on the active tab, across {{count}} measured providers. The number is the share of curated anchor addresses for which the provider returns a non-generic entity name, audited every 30 minutes against ~180 publicly-known addresses split by kind (contract vs EOA)." - "The Kind toggle separates two distinct jobs. On the Contract tab any explorer that reads verified source code (Blockscout in particular) saturates near 100% because the contract's name is already in the bytecode metadata. On the EOA tab the score reflects how well the provider's curated entity graph covers plain wallets, the actually-hard job." - "{{name:mobula}} returns {{p50:mobula}}. Universal coverage providers score on every chain they advertise, so a single number summarizes how broad the underlying label graph actually is once chain-specialists are stripped out. Switch to the EOA tab to see the curated-entity comparison without contract names lifting every score." - "{{name:helius}} scores {{p50:helius}} on Solana. Chain-specialists tend to dominate their home chain because their label graph is curated against native protocols (Jito, Jupiter, Pump.fun, marketplace programs) rather than translated from an EVM-shaped schema." - - "{{name:moralis}} returns {{p50:moralis}}. EVM-centric providers usually trail on TON, Stellar, XRP and Bitcoin because their indexers and entity graphs were built for EVM patterns and ported chains afterwards." + - "{{name:moralis}} returns {{p50:moralis}}. EVM-centric providers usually trail on Gram, Stellar, XRP and Bitcoin because their indexers and entity graphs were built for EVM patterns and ported chains afterwards." - "{{name:blockscout}} sits at {{p50:blockscout}}. Explorer-derived labels rely on verified source code on the Contract tab and on public name tags on the EOA tab. Coverage on the Contract tab tracks how active each chain's Blockscout deployment is; on the EOA tab it tracks how curated each chain's `public_tags` table is, which is far less complete." - "{{worst_name}} trails at {{worst_p50}} on the active tab. The gap between leader and laggard is mostly which entities each provider's graph has been curated against: CEX hot wallets, OFAC SDN and Safe multisigs are easy hits; long-tail public figures and DEX routers separate the top tier." @@ -98,7 +124,7 @@ faq: - q: "What does 'wallet labeling' mean in a crypto API?" a: "A wallet labeling API takes an address and returns an entity name. The CEX it belongs to (`Binance hot wallet 14`), the protocol (`Uniswap V3 router`), the multisig owner (`Safe: foundation treasury`), the sanctioned status (`OFAC SDN`), or a public-figure tag (`Vitalik Buterin`). Generic categorical labels like `EOA`, `Contract`, `Wallet` are not considered a hit on this benchmark because they carry no entity signal." - q: "Is Mobula's labels API better than Moralis or Helius?" - a: "It depends on the chain and on the kind. {{name:mobula}} is a universal provider audited on every chain it advertises and currently returns {{p50:mobula}} on the active tab. {{name:helius}} is Solana-only and dominates that chain because its label graph is curated against native Solana programs. {{name:moralis}} is EVM-centric and trails on TON, Stellar, XRP and Bitcoin. The Kind = EOA tab is where curated entity graphs (Mobula, Helius, chain specialists) actually compete; the Kind = Contract tab favours explorers like Blockscout because they read verified source code for free." + a: "It depends on the chain and on the kind. {{name:mobula}} is a universal provider audited on every chain it advertises and currently returns {{p50:mobula}} on the active tab. {{name:helius}} is Solana-only and dominates that chain because its label graph is curated against native Solana programs. {{name:moralis}} is EVM-centric and trails on Gram, Stellar, XRP and Bitcoin. The Kind = EOA tab is where curated entity graphs (Mobula, Helius, chain specialists) actually compete; the Kind = Contract tab favours explorers like Blockscout because they read verified source code for free." - q: "What is the alternative to Arkham or Nansen for builders?" a: "Arkham and Nansen own the consumer visualization layer (browse-the-web-of-onchain-money). The builder side of the question, the API integrated under the hood by wallets, portfolio trackers and AML flows, lives elsewhere. Mobula, Helius, Moralis, Blockscout, OLI, TonAPI, StellarExpert, XRPScan and WalletExplorer are the labeling APIs benchmarked here. Pick the one whose coverage matches the chains your product touches and whose response shape fits your integration latency budget." - q: "How does OpenChainBench measure wallet label coverage?" @@ -137,7 +163,11 @@ dimensions: - { value: arbitrum, label: Arbitrum } - { value: polygon, label: Polygon } - { value: optimism, label: Optimism } - - { value: ton, label: TON } + # Dimension value kept as `ton` (the Prom label the wallet-labels + # harness emits). The dropdown shows "Gram" via label. Once the + # harness redeploys with chain="gram", flip value to `gram` and add + # the straddle to every selector in this file. + - { value: ton, label: Gram } - { value: stellar, label: Stellar } - { value: xrp, label: XRP } - { value: bitcoin, label: Bitcoin } @@ -218,8 +248,8 @@ providers: - slug: tonapi name: TonAPI - tag: TON specialist, native account graph, free tier - formula: "Share of TON anchor addresses for which TonAPI /v2/accounts/{addr} returns a non-generic entity name, success_total ÷ checks_total over 24h." + tag: Gram specialist, native account graph, free tier + formula: "Share of Gram anchor addresses for which TonAPI /v2/accounts/{addr} returns a non-generic entity name, success_total ÷ checks_total over 24h." queries: p50: 100 * sum(increase(wallet_labels_success_total{provider="tonapi"}[24h])) / sum(increase(wallet_labels_checks_total{provider="tonapi"}[24h])) p90: 100 * sum(increase(wallet_labels_success_total{provider="tonapi"}[24h])) / sum(increase(wallet_labels_checks_total{provider="tonapi"}[24h])) diff --git a/docs/methodology/solana-tx-landing-active.md b/docs/methodology/solana-tx-landing-active.md deleted file mode 100644 index c0cb98ce..00000000 --- a/docs/methodology/solana-tx-landing-active.md +++ /dev/null @@ -1,163 +0,0 @@ -# Methodology - Solana TX Landing (Active Probing) - -> **Pre-registered methodology.** Pinned commit before any sponsor contract is signed. Changes ship as public PRs with a 14-day comment window. Disputes go through public GitHub issues. -> -> **Version :** v1.0 - first commit 2026-05-21. Bench № 016 (Solana TX Landing). -> **Replaces / extends :** the observational tip-wallet attribution methodology that ships with the same bench page (kept as the "Market Share" tab). - ---- - -## 1. Question we answer - -For each Solana transaction landing service, **how long does it take for a transaction submitted via that service to be confirmed on mainnet, and what fraction never confirms within a usable window** - measured from a single fixed geographic origin, on a uniform synthetic payload, at a uniform cadence. - -The bench does **not** answer "which service is best for your trading bot" - that requires modeling your own payload size, tip elasticity, and venue. The bench answers "what is the typical, comparable, reproducible time-to-land per service today." - -## 2. Scope (V0-Lean launch) - -| Dimension | Value | -|---|---| -| Services probed | 5 - Jito Block Engine, Helius Sender, Astralane Iris, Nozomi (Temporal), 0slot.trade | -| Region | 1 - Railway us-east (Newark / NY area) | -| Cadence | 1 cycle per hour | -| Duration | continuous, 24 / 7 | -| Window for headline metrics | rolling 7-day weekly leaderboard | -| Confirmation level | `confirmed` (1+ block confirmation) | - -Services and regions are added through a public PR with a 14-day comment window. Any expansion is a PR + 14-day window - never a silent change. - -## 3. Probe payload (exact) - -Every probe is a single Solana transaction containing three instructions, in this order : - -1. `ComputeBudgetProgram.SetComputeUnitLimit(50 000)` - caps compute units. -2. `ComputeBudgetProgram.SetComputeUnitPrice(50 000 micro-lamports)` - priority fee per CU. -3. `SystemProgram.Transfer(from = prober keypair, to = prober keypair, lamports = 1)` - the payload itself, self-transfer of 1 lamport. Solana requires non-zero state-touching for a tx to be valid; self-transfer is the minimum honest payload. -4. `SystemProgram.Transfer(from = prober keypair, to = , lamports = )` - the tip required by the landing service. -5. `MemoProgram.Memo("ocb---")` where `cycle_id` is a per-cycle 8-byte random hex generated once and shared across all per-service probes in the cycle. This lets us correlate the 5 simultaneous probes on-chain. - -The exact tip amount per service is published as part of this methodology and frozen unless a methodology PR amends it : - -| Service | Tip lamports | Source / justification | -|---|---:|---| -| Jito Block Engine | 10 000 | "Competitive" floor per docs.jito.wtf - above 1 000 doc minimum, below 50th-percentile observed real-traffic tip | -| Helius Sender | 10 000 | Same with `?swqos_only=true` (isolates Helius own path from Jito fan-out) | -| Astralane Iris | 500 000 | Mid-range net of refunds per astralane.gitbook.io | -| Nozomi (Temporal) | 1 000 000 | Hard floor per use.temporal.xyz/nozomi/tipping-and-faq | -| 0slot.trade | 1 000 000 | Hard floor per 0slot.trade | - -We do **not** vary tip amount across cycles. A "tip elasticity" experiment is a separate, sponsored methodology PR. - -## 4. Submission flow (per service, per cycle) - -1. Fetch a recent blockhash via the public mainnet RPC (`api.mainnet-beta.solana.com`) with `commitment = "processed"`. The same blockhash is used for all services in the same cycle so they share a chain-state reference point. `processed` is preferred over `confirmed` because the resulting blockhash is fresher (~400 ms vs ~6 s); the marginal fork risk is acceptable since landing services dedup on signature, not blockhash. -2. Build the transaction described in §3 for that service (the tip-transfer differs per service). -3. Sign with the region's persistent keypair. The signature is known at this point, before any network call. -4. **Subscribe to the signature via `signatureSubscribe` on the public mainnet WebSocket** (`wss://api.mainnet-beta.solana.com`) at `commitment = "confirmed"`. The subscription is registered **before submission** so a fast-confirming tx cannot complete before we are listening (otherwise we would miss the notification and incorrectly timeout). -5. Capture `submit_slot = getSlot(commitment="processed")` and `submit_wallclock = time.Now()`. -6. POST the base64-encoded signed transaction to the service's documented submission endpoint (exact URLs published in the harness source) with `skipPreflight = true`, `maxRetries = 0`, `encoding = "base64"`. Per-service auth headers / query params are applied as documented. -7. Capture the returned signature (or fail-fast on RPC error). -8. Block on the `signatureNotification` push from the WebSocket. On notification, record `land_slot` from the notification context and `land_wallclock = time.Now()`. Classify as **landed**. Resolution is RTT-bounded (~30-50 ms us-east → mainnet-beta) since the RPC pushes the notification the instant the commitment level is reached, with no client polling cadence floor. -9. **Fallback:** if the WebSocket connection fails to establish at the start of the cycle (transient network issue, RPC overload), every probe in that cycle falls back to HTTP polling of `getSignatureStatuses` every 200 ms. This preserves bench continuity but adds a ~200 ms quantization penalty for the affected cycle. The fallback path is logged. -10. If 60 seconds elapse without a notification (or without a non-null `confirmationStatus` on the polling fallback), abandon the wait. Classify as **dropped** with reason `timeout`. -11. If the original submission returned a transport error (HTTP timeout, DNS, EOF, connection refused), classify as **dropped** with reason `network_error`. If the upstream returned HTTP 419 / 429 or a JSON-RPC error containing "rate limit" / "too many requests", classify as `rate_limited`. If the submission was rejected with a structured RPC error (`InstructionError`, `BlockhashNotFound`, etc.) or the on-chain status comes back with an `Err`, classify as `invalid`. - -The 5 services for a given cycle are submitted **in parallel** (Go goroutines) so they sample the same congestion window. The order of `submit_slot` reads is arbitrary but all reads happen within 200 ms. - -## 5. Metrics (Prometheus, exposed at `:2112/metrics`) - -``` -solana_landing_probe_success_total{service, mode, region} counter -solana_landing_probe_dropped_total{service, mode, region, reason} counter - # reason: timeout | invalid | network_error | rate_limited - -solana_landing_probe_latency_slots{service, mode, region} gauge (last observed slot delta) -solana_landing_probe_latency_slots_histogram{service, mode, region} histogram - # buckets: 1, 2, 3, 5, 10, 20, 50, 100 - -solana_landing_probe_latency_ms{service, mode, region} gauge (last observed wall-clock ms) -solana_landing_probe_latency_ms_histogram{service, mode, region} histogram - # buckets: 100, 250, 500, 1000, 2000, 5000, 10000, 30000, 60000 - -solana_landing_probe_keypair_balance_sol{region} gauge -solana_landing_probe_keypair_low_balance_total{region} counter -solana_landing_probe_cycle_total{region} counter -solana_landing_probe_last_cycle_timestamp_seconds{region} gauge -solana_landing_probe_enabled{region} gauge - # 1 when prober configured + running, 0 in pure observational mode -``` - -The `rate_limited` reason groups responses where the upstream service returns HTTP 419 / 429 or a JSON-RPC error containing "rate limit" / "too many requests". Reported separately from `invalid` because rate-limiting is a quota/operational state, not a landing-quality signal. - -**Label semantics :** - -- `service` ∈ {jito, helius-sender, astralane, nozomi, 0slot} -- `mode` is set on `service = helius-sender` only and takes values `swqos_only` or `dual` (other services: `mode = "default"`) -- `region` ∈ {us-east} (V0-Lean); will expand under §7 - -**Headline metrics** displayed publicly on the bench page : - -- `landing_rate = success_total / (success_total + dropped_total{reason="timeout"})` over the last 7 days -- `p50_latency_ms` and `p99_latency_ms` over the last 7 days, per service, per region -- `p50_slot_delta` and `p99_slot_delta` over the last 7 days, per service, per region - -## 6. Jito control probe - -Three services (Helius default mode, Astralane Iris, Nozomi) submit a portion of their flow through Jito internally. Without controlling for this, their measured landing time conflates "this service's own path" with "Jito's auction outcome via this service". - -The Jito control probe is the standard Jito-direct probe **fired in the same cycle as the suspect services**, with the same tip floor (10 000 lamports) and same blockhash. If a suspect service consistently lands at the same `submit_slot + Δ` as the Jito control, the suspect service is interpreted as a Jito routing wrapper for that cycle. We do not currently publish a derived "service-net-of-Jito" metric, but the raw data supports such derivation. - -For Helius we additionally probe in `?swqos_only=true` mode in every cycle (see §3) to publish a clean Helius-only metric series. - -## 7. Versioning rules - -1. **Any** change to §3 (payload), §4 (submission flow), §5 (metric definitions), or the per-service URLs ships as a public PR against this file with a 14-day comment window before merge. -2. Adding a new service requires a PR that updates §2 + §3 + §5 and is announced on the OpenChainBench blog with a 14-day window. Removing a service same. -3. Tip-amount changes (§3) require a PR with a written explanation of why the new amount better reflects "competitive" floor. -4. Region additions and cadence changes are PR + 14-day window. -5. Metric removal or label rename is **prohibited within a version**. Such changes require a major version bump (v1 → v2) with a 30-day shadow-run period publishing both old and new metric families in parallel. - -## 8. Reproducibility - -The full harness source is published at `mobula-api/miniapps/solana-tx-landing/` (Go). Anyone with a funded Solana keypair (~1 SOL) can clone, set the `SOLANA_PROBE_KEYPAIR_BASE58` env var, run the binary, and reproduce all metrics. The bench does not rely on any internal Mobula service for measurement. - -Anyone replicating the bench from a different geographic origin will see different absolute latencies (since service POPs vary by region) but should see the same relative ranking trends over a 7-day window. Discrepancies of more than 1 percentile rank between independent replications should be filed as GitHub issues. - -## 9. Statistical power statement - -At V0-Lean cadence (1 / h × 1 region × 5 services) a single service accumulates 168 probes per rolling 7-day window. With this sample size : - -- `p50_latency_ms` standard error ≈ ±5 % at typical service variance -- `p99_latency_ms` standard error ≈ ±15 % (broad; published with confidence interval) -- Difference in `landing_rate` between two services detectable at 5 pp gap, p < 0.05, after ~18 days of data - -For sub-weekly resolution, cadence must increase. Any such change ships as a public PR + 14-day comment window. - -## 10. Limitations (explicit) - -- **Single-region**: latency reflects what a us-east client sees. Services with non-us-east-anchored POPs may rank differently from sgp or eu-west. -- **Synthetic payload**: a 1-lamport self-transfer + memo is the smallest valid mainnet tx. Real trading payloads (Jupiter swap, Raydium add-liquidity) are heavier and may behave differently in tip elasticity. The bench does not extrapolate. -- **Fixed tip per service**: the bench measures landing performance *at one tip level*. A service that lands at 99 % for 1 M lamports may land at 50 % for 100 k lamports. The bench does not characterize the tip elasticity curve in V0-Lean. -- **Fan-out attribution**: §6 partially handles Helius / Astralane / Nozomi fan-out via the Jito control probe, but does not produce a derived "service-net" published metric in V0-Lean. -- **Confirmation level**: we use `confirmed` (1+ confirmation). A service that lands at `processed` but never reaches `confirmed` would be undercounted. We do not currently publish a `processed` series. -- **Mainnet incidents**: when Solana mainnet halts or congests beyond 60-second confirmation, all services rank identically high in `dropped{reason=timeout}`. The bench page must surface a "chain health overlay" so readers can distinguish service problems from chain problems. -- **Geographic biases**: services with no us-east POP (none in the V0-Lean scope) are penalized vs services with one. We disclose POP locations in the bench page methodology block. - -## 11. Sponsor independence - -Sponsorship contracts (when present) follow a fixed public template with these clauses : - -- Sponsors fund operations and receive newsletter visibility, case studies, integration support. Never leaderboard influence, advance results, or methodology changes. -- The non-suppression clause grants the sponsor a single remedy for unfavorable results : terminate the agreement and receive a pro-rata refund. Never edit, delay, or selectively publish. -- Methodology changes (this document) ship as PRs and are independent of sponsor contracts. - -## 12. Change log - -| Version | Date | Change | -|---|---|---| -| v1.0 | 2026-05-21 | Initial pre-registration. V0-Lean scope : 5 services × 1 region × 1 / h. | -| v1.1 | 2026-05-21 | Pre-launch reconciliation with implementation : metric names from `ocb_solana_landing_*` to `solana_landing_probe_*` (matches sibling OCB benches), `rate_limited` added as 4th drop reason, latency_ms histogram bucket list adjusted to include 250 ms and 30 s, blockhash commitment rationale clarified, memo format clarified (`ocb---` where cycle_id is itself the 8-byte random hex shared across all per-service probes in the cycle), `solana_landing_probe_enabled` gauge added so dashboards distinguish "prober disabled" from "prober stuck". | -| v1.2 | 2026-05-23 | `getSignatureStatuses` poll interval reduced from 1 s to 200 ms after the first 7 days of live data showed all services collapsing to identical 1.0 s p50. The 1 s poll was the measurement floor (Solana confirmed status arrives in ~400 ms-1 s), so 200 ms restores 5x the resolution. Bench page queries also switched from `histogram_quantile` on the latency_ms histogram to `quantile_over_time` on the latency_ms gauge, since the histogram bucket list only had ~3 buckets in the 1-5 s zone where probes actually land, collapsing p50 to bucket midpoints. | -| v1.3 | 2026-05-23 | Primary observation path switched from HTTP polling to `signatureSubscribe` over the public mainnet WebSocket (`wss://api.mainnet-beta.solana.com`). The RPC pushes the notification at the instant the commitment level is reached, so resolution is RTT-bounded (~30-50 ms us-east → mainnet-beta) rather than poll-cadence-bounded. Subscription is registered BEFORE submission to prevent missing fast confirmations. HTTP polling at 200 ms remains an automatic fallback if the WebSocket connect fails. Validated locally against mainnet-beta with end-to-end slot and signature subscribe tests before deploy. | -| v1.4 | 2026-05-23 | Nozomi endpoint switched from `https://ewr.nozomi.temporal.xyz/` (Newark, region-pinned) to `http://edge.nozomi.temporal.xyz/` (geo-routed DNS, HTTP). Reason: Railway us-east is Ashburn, the previous endpoint forced a cross-region hop adding ~30 ms RTT. The geo-routed DNS resolves to the POP closest to the caller. HTTP instead of HTTPS skips the TLS handshake on the hot path (Solana tx is already signed so plain-text body is not a confidentiality risk - the signature is public the moment the tx is on chain). Change requested by Jakob @ Temporal Labs on 2026-05-23 and applied to all probes from the same region. The same review pass is open for Jito / Helius / Astralane / 0slot - any service whose **publicly-documented** best-practice configuration differs from what we currently probe is invited to file an issue or PR. We do NOT accept private deals to alter the probe surface for any single service; we DO apply optimisations that the service publishes as their standard production recommendation. | -| v1.5 | 2026-05-24 | Two further Nozomi refinements per Jakob @ Temporal Labs follow-up. (a) Endpoint moved from `edge.nozomi.temporal.xyz` (JSON-RPC sendTransaction) to `http://edge.nozomi.temporal.xyz/api/sendBatch?c=` (binary `[u16_BE_len][tx_bytes]` framing, `Content-Type: application/octet-stream`). Per Jakob, this batch endpoint handles single-tx submissions and is the path most clients use. Implementation: single-tx wrap in the batch container. (b) Tip wallet switched from the saturated main wallet `TEMPaMeCRFAS9EKF53Jd6KpHxgL47uWLcpFArU1Fanq` to a non-saturated wallet `nEFs3jph8HJt7honu3k7XtGUufMnwAvSXmXcKSPxryP` recommended by Jakob; the main wallets receive heavy MEV-load traffic and clients typically rotate over less-busy alternates. The `TEMPaMe...` wallet remains in the Nozomi tip-wallet list as a fallback for the v1 anti-fingerprint randomisation feature. No change to tip floor (1 000 000 lamports). | diff --git a/harnesses/gas-estimation/README.md b/harnesses/gas-estimation/README.md index 968c3c6f..ae067e4f 100644 --- a/harnesses/gas-estimation/README.md +++ b/harnesses/gas-estimation/README.md @@ -17,12 +17,11 @@ For each gas oracle, capture the priority-fee + base-fee prediction at time T, w | Slug | Endpoint | Auth | Poll cadence | Tier mapping | |---|---|---|---|---| -| `blocknative` | `api.blocknative.com/gasprices/blockprices` | no key (free tier) | 12 s | confidence 70/80/90/95/99 → p25/p50/p75/p90/p99 | | `publicnode-feehistory` | `ethereum-rpc.publicnode.com` `eth_feeHistory` | no key | 12 s | reward[..][0/1/2] → p25/p50/p90 | | `owlracle` | `api.owlracle.info/v4/eth/gas` | no key | 60 s (free quota 100/h) | acceptance 0.35/0.6/0.9/1.0 → p25/p50/p90/p99 | | `etherscan` | `api.etherscan.io/v2/api?chainid=1&module=gastracker&action=gasoracle` | no key (throttled 1/5s) | 15 s | Safe/Propose/Fast − suggestBaseFee → p25/p50/p90 | -Two more oracles (Alchemy `eth_feeHistory`, Blocknative paid Gas Platform) require a free key signup — wired via env vars but disabled by default. See `config.go` for the `GAS_TOKEN_*` and `GAS_URL_*` overrides. +Alchemy `eth_feeHistory` is wired via env vars but disabled by default; it requires a free key signup. See `config.go` for the `GAS_URL_*` overrides. The verification agent confirmed 5 candidates DEAD in 2026: Etherchain (Cloudflare 403), ethgas.watch (deprecated), ethgasstation (defunct), gasstation.network (404), api.ethgas.org / eth.gasprice.network / gastracker.io (DNS gone). @@ -46,7 +45,7 @@ All effective priorities are sorted into a single series; p25/p50/p90 are simple ``` +-----------------+ +-----------------+ | oracle pollers | ---+ | realizer | -| (4 goroutines) | | | head every 12 s | +| (3 goroutines) | | | head every 12 s | +-----------------+ | | catch-up ≤5 blk | v +-----------------+ +---------------+ | @@ -110,8 +109,6 @@ go run ./cmd/script Optional env overrides: ```bash -# Bump Blocknative to a paid key (5 RPS, 100k/day) -GAS_TOKEN_BLOCKNATIVE=your-key-here \ # Use Alchemy feeHistory instead of (or alongside) publicnode GAS_URL_PUBLICNODE_FEEHISTORY=https://eth-mainnet.g.alchemy.com/v2/your-key \ go run ./cmd/script @@ -123,9 +120,9 @@ Standard OCB-miniapp shape — multi-stage Dockerfile, port 2112, internal-only ## Known limits -- **Ethereum mainnet only**. Owlracle covers bsc/poly/avax/arbitrum, feeHistory works on every chain, Blocknative free is mainnet-only, Etherscan v2 free is mainnet-only. Multi-chain bench is a v2 build that swaps the realized RPC + adapts the URL templating. -- **Blob fees not benchmarked**. Blocknative + feeHistory return `baseFeePerBlobGas` but the bench doesn't currently emit it. Add `fee_kind="blob"` axis in a v2 to surface blob predictions. -- **p75/p99 realized values are approximated** — we compute exact p25/p50/p90 from block txs but interpolate p75 = (p50+p90)/2 and p99 = p90. Oracles that emit these tiers (Blocknative, Owlracle) get a comparator, but the bench page should footnote that these are noisier than p25/p50/p90. +- **Ethereum mainnet only**. Owlracle covers bsc/poly/avax/arbitrum, feeHistory works on every chain, Etherscan v2 free is mainnet-only. Multi-chain bench is a v2 build that swaps the realized RPC + adapts the URL templating. +- **Blob fees not benchmarked**. feeHistory returns `baseFeePerBlobGas` but the bench doesn't currently emit it. Add `fee_kind="blob"` axis in a v2 to surface blob predictions. +- **p75/p99 realized values are approximated** — we compute exact p25/p50/p90 from block txs but interpolate p75 = (p50+p90)/2 and p99 = p90. Oracles that emit those tiers (Owlracle) get a comparator, but the bench page should footnote that these are noisier than p25/p50/p90. - **Etherscan no-key throttles at 1/5 s**. With our 15 s cadence we stay well within. If we ever go below 10 s, register a free key. - **Owlracle 100/h no-key quota**. At 60 s cadence we use 60 polls/h = 60% of quota. A free key bumps it to 1000/h. - **Startup window**: first few predictions during the first ~12 s aren't matched (realizer hasn't seen head yet). Owlracle skips buffering during this window; other oracles target an explicit block so are safe. diff --git a/harnesses/gas-estimation/cmd/script/config.go b/harnesses/gas-estimation/cmd/script/config.go index c7650b09..14013bdf 100644 --- a/harnesses/gas-estimation/cmd/script/config.go +++ b/harnesses/gas-estimation/cmd/script/config.go @@ -10,9 +10,9 @@ import ( // Tier is the unified percentile label every oracle's prediction is // mapped onto. We chose p25/p50/p90 because every oracle in the bench // exposes at least three buckets that roughly align with this scheme; -// finer buckets (p75/p99) exist on Blocknative and Owlracle and are -// added back as `tier="p75"`/`tier="p99"` when the oracle supplies -// them. The label is what the OCB site groups by, so the per-oracle +// finer buckets (p75/p99) exist on Owlracle and are added back as +// `tier="p75"`/`tier="p99"` when the oracle supplies them. The label +// is what the OCB site groups by, so the per-oracle // "fast/standard/slow" names never leak into the public metric. type Tier string @@ -28,26 +28,24 @@ const ( // Add an entry to src/data/provider-registry.ts on the OCB side when // onboarding a new oracle. const ( - OracleBlocknative Oracle = "blocknative" - OraclePublicNode Oracle = "publicnode-feehistory" - OracleOwlracle Oracle = "owlracle" - OracleEtherscan Oracle = "etherscan" + OraclePublicNode Oracle = "publicnode-feehistory" + OracleOwlracle Oracle = "owlracle" + OracleEtherscan Oracle = "etherscan" ) type Oracle string // Cadences pinned per oracle. Lifted directly from the verification -// agent's findings: Blocknative tolerates 12 s no-key; PublicNode -// feeHistory is fair-use at 5 req/s; Owlracle free tier is 100/hour, -// so 60 s is the safe ceiling; Etherscan no-key throttles to 1/5 s, -// so 15 s gives a comfortable margin. Cadences are per-chain too: -// running the same oracle against 3 chains at 12 s = 0.25 req/s -// per oracle, well inside every free-tier budget. +// agent's findings: PublicNode feeHistory is fair-use at 5 req/s; +// Owlracle free tier is 100/hour, so 60 s is the safe ceiling; +// Etherscan no-key throttles to 1/5 s, so 15 s gives a comfortable +// margin. Cadences are per-chain too: running the same oracle against +// 3 chains at 12 s = 0.25 req/s per oracle, well inside every +// free-tier budget. var pollIntervals = map[Oracle]time.Duration{ - OracleBlocknative: 12 * time.Second, - OraclePublicNode: 12 * time.Second, - OracleOwlracle: 60 * time.Second, - OracleEtherscan: 15 * time.Second, + OraclePublicNode: 12 * time.Second, + OracleOwlracle: 60 * time.Second, + OracleEtherscan: 15 * time.Second, } // Realized-block poll cadence per chain. Picked close to each chain's @@ -91,8 +89,8 @@ func chains() []Chain { RealizedRPC: envDefault("GAS_REALIZED_RPC_ETHEREUM", "https://ethereum-rpc.publicnode.com"), OwlracleSlug: "eth", BlockTimeSec: 12, - // Etherscan v2 free tier covers chainid=1. All four oracles work. - SupportedSet: []Oracle{OracleBlocknative, OraclePublicNode, OracleOwlracle, OracleEtherscan}, + // Etherscan v2 free tier covers chainid=1. All three oracles work. + SupportedSet: []Oracle{OraclePublicNode, OracleOwlracle, OracleEtherscan}, }, { Slug: "polygon", @@ -100,8 +98,8 @@ func chains() []Chain { RealizedRPC: envDefault("GAS_REALIZED_RPC_POLYGON", "https://polygon-bor-rpc.publicnode.com"), OwlracleSlug: "poly", BlockTimeSec: 2, - // Etherscan v2 free tier covers chainid=137 (verified). All four oracles work. - SupportedSet: []Oracle{OracleBlocknative, OraclePublicNode, OracleOwlracle, OracleEtherscan}, + // Etherscan v2 free tier covers chainid=137 (verified). All three oracles work. + SupportedSet: []Oracle{OraclePublicNode, OracleOwlracle, OracleEtherscan}, }, { Slug: "avalanche", @@ -110,8 +108,8 @@ func chains() []Chain { OwlracleSlug: "avax", BlockTimeSec: 2, // Etherscan v2 returns "Free API access is not supported for this chain" on chainid=43114 — paid plan required. - // Three oracles only (Blocknative + PublicNode feeHistory + Owlracle). - SupportedSet: []Oracle{OracleBlocknative, OraclePublicNode, OracleOwlracle}, + // Two oracles only (PublicNode feeHistory + Owlracle). + SupportedSet: []Oracle{OraclePublicNode, OracleOwlracle}, }, } } @@ -120,13 +118,11 @@ func chains() []Chain { // override without a rebuild. The URL field is the BASE — per-chain // rewriting happens in endpointForChain() below. type OracleEndpoint struct { - URL string - AuthHeader string + URL string } // endpointForChain builds the per-(oracle, chain) URL. // -// - Blocknative: same host, query param `?chainid=`. // - PublicNode feeHistory: per-chain RPC URL from the Chain struct. // - Owlracle: per-chain path slug from the Chain struct. // - Etherscan v2: same host, `?chainid=&module=gastracker&action=gasoracle`. @@ -139,14 +135,9 @@ func endpointForChain(o Oracle, c Chain) OracleEndpoint { strings.ToUpper(c.Slug), ) if override := envDefault(envKey, ""); override != "" { - return OracleEndpoint{URL: override, AuthHeader: oracleAuthHeader(o)} + return OracleEndpoint{URL: override} } switch o { - case OracleBlocknative: - return OracleEndpoint{ - URL: fmt.Sprintf("https://api.blocknative.com/gasprices/blockprices?chainid=%d", c.ChainID), - AuthHeader: envDefault("GAS_TOKEN_BLOCKNATIVE", ""), - } case OraclePublicNode: return OracleEndpoint{URL: c.RealizedRPC} case OracleOwlracle: @@ -161,15 +152,6 @@ func endpointForChain(o Oracle, c Chain) OracleEndpoint { return OracleEndpoint{} } -// oracleAuthHeader returns the optional Authorization header value -// for the given oracle. Currently only Blocknative supports a token. -func oracleAuthHeader(o Oracle) string { - if o == OracleBlocknative { - return envDefault("GAS_TOKEN_BLOCKNATIVE", "") - } - return "" -} - func envDefault(key, def string) string { if v := strings.TrimSpace(os.Getenv(key)); v != "" { return v diff --git a/harnesses/gas-estimation/cmd/script/main.go b/harnesses/gas-estimation/cmd/script/main.go index 0a7b69b2..d45fcec7 100644 --- a/harnesses/gas-estimation/cmd/script/main.go +++ b/harnesses/gas-estimation/cmd/script/main.go @@ -116,7 +116,7 @@ func runOraclePoller(ctx context.Context, o Oracle, ep OracleEndpoint, interval // Stagger startup so all pollers across chains don't fire at t=0. // Jitter folds in both the oracle name AND the chain slug so the - // 3 Blocknative pollers (one per chain) hit at different offsets. + // per-chain pollers for the same oracle hit at different offsets. time.Sleep(jitterFor(string(o) + ":" + chain.Slug)) tick() for { diff --git a/harnesses/gas-estimation/cmd/script/oracles.go b/harnesses/gas-estimation/cmd/script/oracles.go index a630cc58..e21578d4 100644 --- a/harnesses/gas-estimation/cmd/script/oracles.go +++ b/harnesses/gas-estimation/cmd/script/oracles.go @@ -22,10 +22,9 @@ const ( ) // pollResult is what every oracle client returns. TargetBlock is the -// block these predictions apply to (oracle-specific: Blocknative -// gives an explicit next-block number, feeHistory returns the -// projected baseFee for "the next block", Owlracle predicts the -// upcoming few blocks). +// block these predictions apply to (oracle-specific: feeHistory +// returns the projected baseFee for "the next block", Etherscan +// reports lastBlock+1, Owlracle predicts the upcoming few blocks). type pollResult struct { TargetBlock uint64 Predictions []Prediction @@ -42,8 +41,6 @@ type pollResult struct { // adapter. func pollOracle(ctx context.Context, o Oracle, ep OracleEndpoint) pollResult { switch o { - case OracleBlocknative: - return pollBlocknative(ctx, ep) case OraclePublicNode: return pollFeeHistory(ctx, ep) case OracleOwlracle: @@ -55,74 +52,6 @@ func pollOracle(ctx context.Context, o Oracle, ep OracleEndpoint) pollResult { } } -// ─── Blocknative ────────────────────────────────────────────────── - -type bnEstimatedPrice struct { - Confidence int `json:"confidence"` - Price float64 `json:"price"` - MaxPriorityFeePerGas float64 `json:"maxPriorityFeePerGas"` - MaxFeePerGas float64 `json:"maxFeePerGas"` -} - -type bnBlockPrice struct { - BlockNumber uint64 `json:"blockNumber"` - BaseFeePerGas float64 `json:"baseFeePerGas"` - BlobBaseFeePerGas float64 `json:"blobBaseFeePerGas"` - EstimatedTransactions int `json:"estimatedTransactionCount"` - EstimatedPrices []bnEstimatedPrice `json:"estimatedPrices"` -} - -type bnResp struct { - System string `json:"system"` - CurrentBlock uint64 `json:"currentBlockNumber"` - MsSinceLastBlock int `json:"msSinceLastBlock"` - BlockPrices []bnBlockPrice `json:"blockPrices"` -} - -func pollBlocknative(ctx context.Context, ep OracleEndpoint) pollResult { - req, _ := http.NewRequestWithContext(ctx, "GET", ep.URL, nil) - if ep.AuthHeader != "" { - req.Header.Set("Authorization", ep.AuthHeader) - } - body, status, err := httpDo(ctx, req) - if err != nil { - return pollResult{Err: err} - } - if status != 200 { - return pollResult{Err: fmt.Errorf("http %d", status)} - } - var r bnResp - if err := json.Unmarshal(body, &r); err != nil { - return pollResult{Err: fmt.Errorf("parse: %w", err)} - } - if len(r.BlockPrices) == 0 { - return pollResult{Err: fmt.Errorf("empty blockPrices")} - } - bp := r.BlockPrices[0] - // Confidence 70/80/90/95/99 → p25/p50/p75/p90/p99 per spec. - mapping := map[int]Tier{ - 70: TierP25, - 80: TierP50, - 90: TierP75, - 95: TierP90, - 99: TierP99, - } - out := pollResult{TargetBlock: bp.BlockNumber, BaseGwei: bp.BaseFeePerGas} - for _, e := range bp.EstimatedPrices { - tier, ok := mapping[e.Confidence] - if !ok { - continue - } - out.Predictions = append(out.Predictions, Prediction{ - Oracle: OracleBlocknative, - Tier: tier, - PriorityGwei: e.MaxPriorityFeePerGas, - BaseGwei: bp.BaseFeePerGas, - }) - } - return out -} - // ─── eth_feeHistory (PublicNode, Alchemy share this shape) ──────── type fhResult struct { diff --git a/harnesses/gas-estimation/cmd/script/realized.go b/harnesses/gas-estimation/cmd/script/realized.go index 97b5b8b7..3ba5b0eb 100644 --- a/harnesses/gas-estimation/cmd/script/realized.go +++ b/harnesses/gas-estimation/cmd/script/realized.go @@ -232,11 +232,11 @@ func processBlock(ctx context.Context, buf *Buffer, blockNum uint64, chain Chain } realized := map[Tier]float64{TierP25: p25, TierP50: p50, TierP90: p90} - // p75/p99 are emitted by Blocknative & Owlracle; we approximate - // realized p75 = (p50 + p90)/2 and p99 = p90 to give those - // tiers a comparator even though we don't compute them directly. - // Better than dropping the metric — but the realized side is - // noisy for tail tiers, so the bench page should footnote this. + // p75/p99 are emitted by Owlracle; we approximate realized p75 = + // (p50 + p90)/2 and p99 = p90 to give those tiers a comparator + // even though we don't compute them directly. Better than + // dropping the metric — but the realized side is noisy for tail + // tiers, so the bench page should footnote this. realized[TierP75] = (p50 + p90) / 2 realized[TierP99] = p90 diff --git a/harnesses/l1-finality/README.md b/harnesses/l1-finality/README.md index d26ae782..22a9a9a1 100644 --- a/harnesses/l1-finality/README.md +++ b/harnesses/l1-finality/README.md @@ -18,7 +18,7 @@ This works for: **Ethereum, Solana, TRON, Stellar, Hedera, SUI, Litecoin, Monero For chains where finality is faster than our poll interval, comparing two pointers at one instant doesn't measure finalization time — it measures the gap-at-instant, which collapses to zero when finalization catches up to head. The honest path is to subscribe to a push stream, record `T1 = time.Now()` when block N is first observed, and `T2 = time.Now()` when N becomes finalized. `lag = T2 − T1`, with millisecond precision, independent of chain timestamp resolution. -This works for: **BNB, Avalanche, TON**. +This works for: **BNB, Avalanche, Gram**. ## Per-chain methodology @@ -32,7 +32,7 @@ This works for: **BNB, Avalanche, TON**. | **Stellar** | HTTP poll | Horizon `/ledgers?order=desc&limit=2` | 1 ledger back (SCP-final) | Circle = 1, deterministic SCP | | **Hedera** | HTTP poll | Mirror `/api/v1/blocks?order=desc&limit=2` | 1 block back. Timestamps parsed at ns precision | Hashgraph aBFT deterministic | | **SUI** | HTTP poll | `sui_getLatestCheckpointSequenceNumber` + `sui_getCheckpoint` | 1 checkpoint back | Circle USDC = 1, Mysticeti finalizes in 1 | -| **TON** | SSE wall-clock | `tonapi.io/v2/sse/blocks?workchain=-1` (masterchain only) | Time between consecutive masterchain blocks | TON docs: a tx is final once included in a masterchain block, so block_N is final when block_N+1 commits | +| **Gram** | SSE wall-clock | `tonapi.io/v2/sse/blocks?workchain=-1` (masterchain only) | Time between consecutive masterchain blocks | Gram (formerly TON) docs: a tx is final once included in a masterchain block, so block_N is final when block_N+1 commits | | **Litecoin** | HTTP poll (probabilistic) | blockchair `/stats.best_block_height` and `/dashboards/block/{height}.block.time` | 12 confirmations | Coinbase deposit standard, post-April-2026 13-block MWEB reorg | | **Monero** | HTTP poll (probabilistic) | monero-rpc `get_info` + `get_block_header_by_height` (with cakewallet/sethforprivacy/monerujo failover) | 10 confirmations | Wallet protocol unlock period | | **Cardano** | HTTP poll (probabilistic) | koios `/tip` + `/blocks?block_height=eq.` | 15 confirmations | Above Coinbase 10 / Kraken 15. Far below the academic k=2160 (~12 h) | @@ -56,7 +56,7 @@ l1_finality_last_refresh_timestamp_seconds{chain} l1_finality_fetch_errors_total{chain, error_type} l1_finality_health{chain} # 1 if last sample succeeded -# Wall-clock-measured chains (BNB, Avalanche, TON) +# Wall-clock-measured chains (BNB, Avalanche, Gram) l1_finality_wallclock_lag_milliseconds{chain} # ms-precise gauge l1_finality_wallclock_lag_milliseconds_histogram # histogram for tail latency l1_finality_wallclock_health{chain} # 1 if WS/SSE connected @@ -74,7 +74,7 @@ l1_finality_wallclock_samples_total{chain} # cumulative count | Hedera | High | Hashgraph aBFT + ns-precision timestamps | | SUI | High | 1 checkpoint = Circle USDC standard | | Stellar | High | SCP deterministic, Circle = 1 | -| TON | High (after SSE refactor) | tonapi `workchain=-1` SSE stream, ms-precise | +| Gram | High (after SSE refactor) | tonapi `workchain=-1` SSE stream, ms-precise | | Cardano | Medium | 15-conf compromise between Coinbase 10 and Kraken 15. Academic k=2160 is theoretical; no actor uses it | | Litecoin | Medium | 12-conf post-April-2026 reorg; standard is evolving | | TRON | Medium | CEX confirmation counts vary 19–30; we use the 19-block protocol minimum | @@ -106,7 +106,7 @@ Deploy from this directory. No required env vars — public defaults work. Optio | `REFRESH_INTERVAL_SECONDS` | HTTP-poll cadence | Default 10, min 5. Doesn't affect WS-measured chains | | `LOGS_TOKEN` | Bearer token gating `/logs` | Optional | -The WS/SSE subscribers connect with hard-coded public endpoints (publicnode for BNB/Avalanche, tonapi.io for TON) — no auth required. +The WS/SSE subscribers connect with hard-coded public endpoints (publicnode for BNB/Avalanche, tonapi.io for Gram) — no auth required. ## Adding a chain @@ -118,7 +118,7 @@ The WS/SSE subscribers connect with hard-coded public endpoints (publicnode for ### Wall-clock-measured chain -1. Add a goroutine that maintains the WS/SSE subscription. See `evm_ws.go` (BNB/Avalanche) or `ton_ws.go` (TON) for the pattern. +1. Add a goroutine that maintains the WS/SSE subscription. See `evm_ws.go` (BNB/Avalanche) or `ton_ws.go` (Gram) for the pattern. The file is still named `ton_ws.go` after the chain's pre-rebrand identifier; rename safely once the harness is redeployed. 2. Record `firstSeen[height] = time.Now()` on each new-block event. 3. On finality observation (either an explicit finalized-tag advance or "next block referenced this one"), emit the lag via `wallClockLagGauge.WithLabelValues(slug).Set(lagMs)`. 4. Don't add the chain to the polled `Chains` config list — wall-clock chains run as separate goroutines. diff --git a/harnesses/l1-finality/cmd/script/config.go b/harnesses/l1-finality/cmd/script/config.go index 91270f66..e5b982ba 100644 --- a/harnesses/l1-finality/cmd/script/config.go +++ b/harnesses/l1-finality/cmd/script/config.go @@ -90,7 +90,7 @@ func loadConfig() *Config { // // SUI moved to high-frequency HTTP polling wall-clock — see // sui_ws.go. Same methodology mismatch as before fixed. - // TON removed from HTTP polling — same issue as BNB/Avalanche + // Gram (formerly TON) removed from HTTP polling — same issue as BNB/Avalanche // (masterchain blocks finalize in ~0.5s, polling 10s = wrong // methodology). Now measured via SSE wall-clock subscriber on // tonapi.io's /v2/sse/blocks stream. diff --git a/harnesses/l1-finality/cmd/script/ton.go b/harnesses/l1-finality/cmd/script/ton.go index d8f61a0a..7137eceb 100644 --- a/harnesses/l1-finality/cmd/script/ton.go +++ b/harnesses/l1-finality/cmd/script/ton.go @@ -10,7 +10,7 @@ import ( "time" ) -// TON: tonapi.io anonymous tier supports masterchain-head + blocks/{id} +// Gram (formerly TON): tonapi.io anonymous tier supports masterchain-head + blocks/{id} // reads. Toncenter free has 1 rps which we burst past with two block // fetches per cycle; tonapi is friendlier. Switching there. diff --git a/harnesses/l1-finality/cmd/script/ton_ws.go b/harnesses/l1-finality/cmd/script/ton_ws.go index 8b0f75be..d8a8f3a8 100644 --- a/harnesses/l1-finality/cmd/script/ton_ws.go +++ b/harnesses/l1-finality/cmd/script/ton_ws.go @@ -11,10 +11,12 @@ import ( "time" ) -// TON wall-clock finality measurement via tonapi.io SSE stream. +// Gram (formerly TON) wall-clock finality measurement via tonapi.io SSE stream. +// Function/type names and the TON_API_KEY env var keep their pre-rebrand +// identifiers so the deployed harness keeps booting without ops coordination. // // The tonapi `/v2/sse/blocks?workchain=-1` endpoint pushes one event per -// masterchain block (~0.4-0.7 s cadence). Per the TON payment-processor +// masterchain block (~0.4-0.7 s cadence). Per the Gram (formerly TON) payment-processor // docs, "a transaction is finalized once included in a masterchain // block" — so the wall-clock interval between block N and block N+1 is // the time the network needs to finalize block N. Recording the first @@ -38,7 +40,7 @@ type tonSSEMessage struct { FileHash string `json:"file_hash"` } -// StartTONWallClock launches a persistent SSE subscriber for TON +// StartTONWallClock launches a persistent SSE subscriber for Gram (formerly TON) // masterchain. Reconnects with exponential backoff on error. When the // SSE stream is unavailable (tonapi gated it behind auth in 2026-06 and // deprecated it in favor of webhooks), falls back to fast-polling the @@ -90,7 +92,7 @@ func pollTONWallClock(window time.Duration) { wallClockHealth.WithLabelValues("ton").Set(1) healthy = true } - // Cap at 2 s in poll mode: TON masterchain cadence is 0.4-0.7 s, + // Cap at 2 s in poll mode: Gram masterchain cadence is 0.4-0.7 s, // so a multi-second "lag" here is poll aliasing (429 backoff // stretching the cadence), not finality. Observed pre-cap: 10.6 s // garbage samples polluting the 24h histogram. diff --git a/harnesses/solana-tx-landing/README.md b/harnesses/solana-tx-landing/README.md index 1ea5e8f4..9c69b4e5 100644 --- a/harnesses/solana-tx-landing/README.md +++ b/harnesses/solana-tx-landing/README.md @@ -1,9 +1,10 @@ # solana-tx-landing harness -Source for two OpenChainBench benches that share a single binary: +Source for the OpenChainBench bench: - [`solana-tx-landing`](https://openchainbench.com/benchmarks/solana-tx-landing) — observational market-share view of Solana transaction landing services (Jito, Helius Sender, Nozomi, Astralane, 0slot, etc.) measured via on-chain tip-wallet attribution. -- [`solana-tx-landing-latency`](https://openchainbench.com/benchmarks/solana-tx-landing-latency) — active probing: a synthetic 1-lamport self-transfer submitted through each service, timing the slot delta to confirmation. See [`docs/methodology/solana-tx-landing-active.md`](../../docs/methodology/solana-tx-landing-active.md) for the pre-registered methodology. + +The binary also embeds an opt-in active prober (slot-delta latency), kept around for re-enablement but not currently wired to a public bench page on OCB. Exposes Prometheus metrics on `:2112/metrics` (OCB Railway convention). @@ -56,7 +57,7 @@ curl localhost:2112/metrics | grep solana_landing | `LOGS_TOKEN` | (unset) | Optional, gates `/logs?tail=N` | | `SLACK_WEBHOOK_URL` | (unset) | Optional, posts probe failures + low-balance alerts | -See [`docs/methodology/solana-tx-landing-active.md`](../../docs/methodology/solana-tx-landing-active.md) for the exact probe payload, tip floors per service, and statistical thresholds. +The probe payload, tip floors per service, and statistical thresholds are documented inline in `cmd/script/prober.go` and `cmd/script/senders.go`. ## Reproducibility diff --git a/harnesses/solana-tx-landing/cmd/script/active_metrics.go b/harnesses/solana-tx-landing/cmd/script/active_metrics.go index 2f16dce5..ddca8171 100644 --- a/harnesses/solana-tx-landing/cmd/script/active_metrics.go +++ b/harnesses/solana-tx-landing/cmd/script/active_metrics.go @@ -16,9 +16,8 @@ import ( // • landing_rate = success_total / (success_total + dropped{reason=timeout}) // • p50 / p99 derived in Prom from the *_histogram series // -// Methodology pinned at docs/methodology/solana-tx-landing-active.md -// (OpenChainBench repo). Any label / metric change is a methodology PR -// with the 14-day comment window. +// Label / metric shape is stable; any change should ship as a public PR +// with a 14-day comment window before redeploy. var ( // One increment per landed (confirmed) probe. Headline numerator of diff --git a/harnesses/solana-tx-landing/cmd/script/main.go b/harnesses/solana-tx-landing/cmd/script/main.go index 2e1fdd05..73893eec 100644 --- a/harnesses/solana-tx-landing/cmd/script/main.go +++ b/harnesses/solana-tx-landing/cmd/script/main.go @@ -53,7 +53,6 @@ func main() { go runSubscriber(ctx, wsURL) // Active prober (opt-in: requires SOLANA_PROBE_KEYPAIR_BASE58). - // Methodology: docs/methodology/solana-tx-landing-active.md go runProber(ctx) sig := make(chan os.Signal, 1) diff --git a/harnesses/solana-tx-landing/cmd/script/prober.go b/harnesses/solana-tx-landing/cmd/script/prober.go index 42e127db..e5bbe1f5 100644 --- a/harnesses/solana-tx-landing/cmd/script/prober.go +++ b/harnesses/solana-tx-landing/cmd/script/prober.go @@ -22,8 +22,7 @@ import ( // Active prober — submits a synthetic mainnet tx through each landing // service every cycle, then polls confirmation. Headline metrics = -// landing_rate + p50/p99 latency per service. Methodology pinned at -// OpenChainBench/docs/methodology/solana-tx-landing-active.md. +// landing_rate + p50/p99 latency per service. // // The prober is OPT-IN. It only runs when SOLANA_PROBE_KEYPAIR_BASE58 // is set; absence keeps the harness in pure observational mode. diff --git a/harnesses/wallet-labels/README.md b/harnesses/wallet-labels/README.md index 9efe49a2..3911b098 100644 --- a/harnesses/wallet-labels/README.md +++ b/harnesses/wallet-labels/README.md @@ -15,7 +15,7 @@ Emits Prometheus metrics consumed by openchainbench.com. | Helius | API key | Solana | | Blockscout | none | ethereum, base, optimism, polygon, gnosis | | OLI (Open Labels Initiative) | none | EVM via Base EAS | -| TonAPI | none | TON | +| TonAPI | none | Gram (formerly TON) | | StellarExpert | none | Stellar | | XRPScan | none | XRP | | WalletExplorer | none | Bitcoin | diff --git a/harnesses/wallet-labels/cmd/script/integration.go b/harnesses/wallet-labels/cmd/script/integration.go index 51b5b6e1..a948cabd 100644 --- a/harnesses/wallet-labels/cmd/script/integration.go +++ b/harnesses/wallet-labels/cmd/script/integration.go @@ -26,7 +26,7 @@ var integrationCases = []testCase{ {"bnb", "0xF977814e90dA44bFA03b6295A0616a897441aceC", "Binance Hot 8 (BSC)"}, // Solana {"solana", "5Q544fKrFoe6tsEbD7S8EmxGTJYAKtTVhAW5Q5pge4j1", "Raydium Authority"}, - // TON + // Gram (formerly TON) {"ton", "EQB3ncyBUTjZUA5EnFKR5_EnOMI9V1tTEAAPaiU71gc4TiUt", "STON.fi DEX"}, // Stellar {"stellar", "GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A", "Binance"}, diff --git a/next.config.ts b/next.config.ts index 6938a6f9..5fec030c 100644 --- a/next.config.ts +++ b/next.config.ts @@ -47,6 +47,17 @@ const nextConfig: NextConfig = { // pin it so a future Next minor that flips defaults can't silently // split bench rankings between the two surface URLs. trailingSlash: false, + // Inject a build-time timestamp so the sitemap can emit a stable + // per deploy instead of `new Date()` at request time. The + // sitemap runs on force-dynamic (to bypass Next's 2 MB Data Cache + // limit), which means `new Date()` at module init evaluates anew on + // every Google crawl. Result: every URL in the sitemap got a freshly + // updated lastmod each visit, Google flagged the signal as unreliable + // and stopped using it to prioritise recrawls. This baking pins the + // value at build time so it changes only when a new deploy ships. + env: { + NEXT_PUBLIC_BUILD_TIME: new Date().toISOString(), + }, turbopack: { root: __dirname, }, @@ -64,6 +75,24 @@ const nextConfig: NextConfig = { experimental: { optimizePackageImports: ["lucide-react"], }, + // YAML spec files live outside src and are loaded at runtime via + // fs.readFile(process.cwd() + "/benchmarks/..."). Next's default file + // tracer only follows static imports, so those YAML files were not + // making it into the Vercel build artifact and the prebuilt deployment + // kept serving a snapshot from whichever earlier build first cached + // them. outputFileTracingIncludes forces every YAML directory the + // spec loaders read at runtime to be packaged with the deployment so + // the page always reflects the dev HEAD of these specs. + outputFileTracingIncludes: { + "/**": [ + "./benchmarks/**/*.yml", + "./benchmarks/**/*.yaml", + "./answers/**/*.yml", + "./answers/**/*.yaml", + "./alternatives/**/*.yml", + "./alternatives/**/*.yaml", + ], + }, async headers() { return [ { @@ -102,7 +131,7 @@ const nextConfig: NextConfig = { "bnb", "avalanche", "sui", - "ton", + "gram", "stellar", "tron", "cardano", @@ -134,6 +163,26 @@ const nextConfig: NextConfig = { destination: "/benchmarks/rpc-capabilities", permanent: true, }, + // TON → Gram token rebrand (June 2026). Chain slug renamed + // ton → gram across CHAINS, PROVIDER_REGISTRY, per_chain_explainer + // and bench provider entries. Pin permanent 308 redirects from the + // old paths so inbound links + Google SERP entries land on the new + // canonical without losing rank signal. + { + source: "/chains/ton", + destination: "/chains/gram", + permanent: true, + }, + { + source: "/products/ton", + destination: "/chains/gram", + permanent: true, + }, + { + source: "/benchmarks/:slug/ton", + destination: "/benchmarks/:slug/gram", + permanent: true, + }, ...chainRedirects, ]; }, diff --git a/package.json b/package.json index 2807bfd1..a58efbb1 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,8 @@ "@modelcontextprotocol/sdk": "^1.26.0", "@react-three/fiber": "^9.6.1", "@vercel/analytics": "^2.0.1", + "cmdk": "^1.1.1", + "fuse.js": "^7.4.2", "ioredis": "^5.11.1", "js-yaml": "^4.1.1", "lucide-react": "^1.11.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index ec4c0fa2..0d7caede 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,12 @@ importers: '@vercel/analytics': specifier: ^2.0.1 version: 2.0.1(next@16.2.6(@babel/core@7.29.0)(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(react@19.2.4) + cmdk: + specifier: ^1.1.1 + version: 1.1.1(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + fuse.js: + specifier: ^7.4.2 + version: 7.4.2 ioredis: specifier: ^5.11.1 version: 5.11.1 @@ -653,6 +659,177 @@ packages: resolution: {integrity: sha512-nn5ozdjYQpUCZlWGuxcJY/KpxkWQs4DcbMCmKojjyrYDEAGy4Ce19NN4v5MduafTwJlbKc99UA8YhSVqq9yPZA==} engines: {node: '>=12.4.0'} + '@radix-ui/primitive@1.1.4': + resolution: {integrity: sha512-7AdCK9PQyiljKoBDbN8OuctCbd/esdwZPQ8RtOE3SsyQtUpiPb+ND75q0jEhC1m1ecBI0MFNeLJvwIh9iKHRcQ==} + + '@radix-ui/react-compose-refs@1.1.3': + resolution: {integrity: sha512-rYOP8OMnuuPMQF1uhPVlGNcCDlkokKqGFE3JcxFViIkAXP7EvFWUliJAstrapypaBLJNHbZL6jGhbVDGTwmVhA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-context@1.1.4': + resolution: {integrity: sha512-QwH4PO5urrbO+FaGd5Aglg+YJgWTyyuZ3g/6mKvsqraLkglDdckw9JafgL5McL5VEJ6EPNduPaT3ZE9BttDAqg==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-dialog@1.1.17': + resolution: {integrity: sha512-TDTYmpdq8dI2+Xgvgj9AJ8Ghqq+Eph/TRVEdaFQPDItIY+6QSkU7MJMeevw1568Yw/2Ijz8BTphPSP2XejKphw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-dismissable-layer@1.1.13': + resolution: {integrity: sha512-2v+zNAWWe0ySxgC0D0yeXMPQ23xZVgXZTerTz+JKlmdRj6gfTqmCcR29jb6d290DezXPGgruHWDX/vYUebtErg==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-focus-guards@1.1.4': + resolution: {integrity: sha512-cot/aB/mOm0IYVYTTmQcEEK1M48lZWi8FlYe5nDPQQ8NYZUlXEFgncJ9p2Kzer3RKSrY7cTTpEMLZKNo9QoP5Q==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-focus-scope@1.1.10': + resolution: {integrity: sha512-Fas/lXQqhVvqwAb64s5RFeHiHYElZ6SUQbZaNd6EkfhP/Al7wTIQ9WIR4QVX475tlu5yFCEdDcJH6/UwsZjMWw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-id@1.1.2': + resolution: {integrity: sha512-orBC88futVpqCmhX1p4cvquNHsELQ+w+vBJnuj3ftETI5bJb0bZn3Tqu3SWN2IOcPycTnMGnhwoermvISt72sA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-portal@1.1.12': + resolution: {integrity: sha512-m309havGzsjLHHaIX50G5PlvRs3xkgPCsGk/5PTvYm8D5q33yG0J7w/712PTOhid7NTaFETtnSXjngHQavvhVw==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-presence@1.1.6': + resolution: {integrity: sha512-zdTk4PlUO0E18HnZ3wYbW0KkJJxWCdiNYp6g6X1PtONFhxVkg01vliTJAmwIszU6mHiyBOoW9P0rAugl5/hULQ==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-primitive@2.1.6': + resolution: {integrity: sha512-wetd0QI77DbvrPpTAvH1SqOxsYF2wZe5TNxqwOd5Ty4XDpV3dpV0s8K/1MGMJBeY5o7lg8ub5VIt1Ub+yVen6g==} + peerDependencies: + '@types/react': '*' + '@types/react-dom': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + react-dom: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@types/react-dom': + optional: true + + '@radix-ui/react-slot@1.3.0': + resolution: {integrity: sha512-MojKku4U/miO8Av4Dkb+ctMAQx7JmY96LmtDQlAarCRtd7rN52QCSzBF+XAvr5S6coSVj9HEPBgHAHKEJVk/WA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-callback-ref@1.1.2': + resolution: {integrity: sha512-xCso9j1/u8sEgP1RNHjFrXJLApL8LiqOkI1R4ywuN00rxWdYg4oQXuwKLS3i0j5NWLromUD27/4nlxj2UFVvIw==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-controllable-state@1.2.3': + resolution: {integrity: sha512-PLzC90MS+ReootmjC597dvopoelpZ8Q61HJkDXZSExitIq7PL55vHNnesAHwguHK0aPfBnpdNzQtv1uliaqQrA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-effect-event@0.0.3': + resolution: {integrity: sha512-6c8ZqvPTWILEKnyVkP53EGRCcpnJiKTC21sS/6R1GF5xKyHJJWQEPfkqlcgUkdRQivd6tb23abUwe4ngWmY0JA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-escape-keydown@1.1.2': + resolution: {integrity: sha512-2uVLvLjgO7NZCWw01/FdqRwmA42J0BcjPMUCA+koFEOAb+zjqIP7SiFz/7zWPrKnVmSqr76Omq2ALyCuX4dhLw==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + '@radix-ui/react-use-layout-effect@1.1.2': + resolution: {integrity: sha512-jrBWOxZITuGcnjRCM2t2U5ZPkCLxD+Ym6DjfssS5haTj2iiak/DOb64JeN6OdLfLgptb6/e2kKR+ZuTrGoZTPA==} + peerDependencies: + '@types/react': '*' + react: ^16.8 || ^17.0 || ^18.0 || ^19.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + '@react-three/fiber@9.6.1': resolution: {integrity: sha512-zF0rsKcVYpcJwbFEnv2HkHX9cvOEgsfQo/X8lwmR2dn13S4qEQJXir9fxf5js2LQFoXqxOY7MDkOkYx2uZ4gSg==} peerDependencies: @@ -1077,6 +1254,10 @@ packages: argparse@2.0.1: resolution: {integrity: sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==} + aria-hidden@1.2.6: + resolution: {integrity: sha512-ik3ZgC9dY/lYVVM++OISsaYDeg1tb0VtP5uL3ouh1koGOaUMDPpbFIei4JkFimWUFPn90sbMNMXQAIVOlnYKJA==} + engines: {node: '>=10'} + aria-query@5.3.2: resolution: {integrity: sha512-COROpnaoap1E2F000S62r6A60uHZnmlvomhfyT2DlTcrY1OrBKn2UhH7qn5wTC9zMvD0AY7csdPSNwKP+7WiQw==} engines: {node: '>= 0.4'} @@ -1215,6 +1396,12 @@ packages: resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} engines: {node: '>=0.10.0'} + cmdk@1.1.1: + resolution: {integrity: sha512-Vsv7kFaXm+ptHDMZ7izaRsP70GgrW9NBNGswt9OZaVBLlE0SNpDq8eu/VGXyF9r7M0azK3Wy7OlYXsuyYLFzHg==} + peerDependencies: + react: ^18 || ^19 || ^19.0.0-rc + react-dom: ^18 || ^19 || ^19.0.0-rc + color-convert@2.0.1: resolution: {integrity: sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==} engines: {node: '>=7.0.0'} @@ -1318,6 +1505,9 @@ packages: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} + detect-node-es@1.1.0: + resolution: {integrity: sha512-ypdmJU/TbBby2Dxibuv7ZLW3Bs1QEmM7nHjEANfohJLvE0XVujisn1qPJcZxg+qDucsr+bP6fLD1rPS3AhJ7EQ==} + doctrine@2.1.0: resolution: {integrity: sha512-35mSku4ZXK0vfCuHEDAwt55dg2jNajHZ1odvF+8SSr82EsZY4QmXfuWso8oEd8zRhVObSN18aM0CjSdoBX7zIw==} engines: {node: '>=0.10.0'} @@ -1614,6 +1804,10 @@ packages: functions-have-names@1.2.3: resolution: {integrity: sha512-xckBUXyTIqT97tq2x2AMb+g163b5JFysYk0x4qxNFwbfQkmNZoiRHb6sPzI9/QV33WeuvVYBUIiD4NzNIyqaRQ==} + fuse.js@7.4.2: + resolution: {integrity: sha512-LVbzjD4WA6UP5B1UnP8wuaXJiLnqMdM/E4fiJXTJ5haJ5b/MBNsK29h2fm6swEoQaVQjvYFWKLE2RanyZIoRVQ==} + engines: {node: '>=10'} + generator-function@2.0.1: resolution: {integrity: sha512-SFdFmIJi+ybC0vjlHN0ZGVGHc3lgE0DxPAT0djjVg+kjOnSqclqmj0KQ7ykTOLP6YxoqOvuAODGdcHJn+43q3g==} engines: {node: '>= 0.4'} @@ -1630,6 +1824,10 @@ packages: resolution: {integrity: sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==} engines: {node: '>= 0.4'} + get-nonce@1.0.1: + resolution: {integrity: sha512-FJhYRoDaiatfEkUK8HKlicmu/3SGFD51q3itKDGoSTysQJBnfOcxU5GxnhE1E6soB76MbT0MBtnKJuXyAx+96Q==} + engines: {node: '>=6'} + get-proto@1.0.1: resolution: {integrity: sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==} engines: {node: '>= 0.4'} @@ -2266,6 +2464,36 @@ packages: react-is@16.13.1: resolution: {integrity: sha512-24e6ynE2H+OKt4kqsOvNd8kBpV65zoxbA4BVsEOB3ARVWQki/DHzaUoC5KuON/BiccDaCCTZBuOcfZs70kR8bQ==} + react-remove-scroll-bar@2.3.8: + resolution: {integrity: sha512-9r+yi9+mgU33AKcj6IbT9oRCO78WriSj6t/cF8DWBZJ9aOGPOTEDvdUDz1FwKim7QXWwmHqtdHnRJfhAxEG46Q==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 + peerDependenciesMeta: + '@types/react': + optional: true + + react-remove-scroll@2.7.2: + resolution: {integrity: sha512-Iqb9NjCCTt6Hf+vOdNIZGdTiH1QSqr27H/Ek9sv/a97gfueI/5h1s3yRi1nngzMUaOOToin5dI1dXKdXiF+u0Q==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + react-style-singleton@2.2.3: + resolution: {integrity: sha512-b6jSvxvVnyptAiLjbkWLE/lOnR4lfTtDAl+eUC7RZy+QQWc6wRzIV2CE6xBuMmDxc2qIihtDCZD5NPOFl7fRBQ==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + react-use-measure@2.1.7: resolution: {integrity: sha512-KrvcAo13I/60HpwGO5jpW7E9DfusKyLPLvuHlUyP5zqnmAPhNc6qTRjUQrdTADl0lpPpDVU2/Gg51UlOGHXbdg==} peerDependencies: @@ -2576,6 +2804,26 @@ packages: uri-js@4.4.1: resolution: {integrity: sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg==} + use-callback-ref@1.3.3: + resolution: {integrity: sha512-jQL3lRnocaFtu3V00JToYz/4QkNWswxijDaCVNZRiRTO3HQDLsdu1ZtmIUvV4yPp+rvWm5j0y0TG/S61cuijTg==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + + use-sidecar@1.1.3: + resolution: {integrity: sha512-Fedw0aZvkhynoPYlA5WXrMCAMm+nSWdZt6lzJQ7Ok8S6Q+VsHmHpRWndVRJ8Be0ZbkfPc5LRYH+5XrzXcEeLRQ==} + engines: {node: '>=10'} + peerDependencies: + '@types/react': '*' + react: ^16.8.0 || ^17.0.0 || ^18.0.0 || ^19.0.0 || ^19.0.0-rc + peerDependenciesMeta: + '@types/react': + optional: true + use-sync-external-store@1.6.0: resolution: {integrity: sha512-Pp6GSwGP/NrPIrxVFAIkOQeyw8lFenOHijQWkUTrDvrF4ALqylP2C/KCkeS9dpUM3KvYRQhna5vt7IL95+ZQ9w==} peerDependencies: @@ -3114,6 +3362,148 @@ snapshots: '@nolyfill/is-core-module@1.0.39': {} + '@radix-ui/primitive@1.1.4': {} + + '@radix-ui/react-compose-refs@1.1.3(@types/react@19.2.14)(react@19.2.4)': + dependencies: + react: 19.2.4 + optionalDependencies: + '@types/react': 19.2.14 + + '@radix-ui/react-context@1.1.4(@types/react@19.2.14)(react@19.2.4)': + dependencies: + react: 19.2.4 + optionalDependencies: + '@types/react': 19.2.14 + + '@radix-ui/react-dialog@1.1.17(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.4 + '@radix-ui/react-compose-refs': 1.1.3(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-context': 1.1.4(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-dismissable-layer': 1.1.13(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-focus-guards': 1.1.4(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-focus-scope': 1.1.10(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-id': 1.1.2(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-portal': 1.1.12(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-presence': 1.1.6(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-primitive': 2.1.6(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-slot': 1.3.0(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-use-controllable-state': 1.2.3(@types/react@19.2.14)(react@19.2.4) + aria-hidden: 1.2.6 + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + react-remove-scroll: 2.7.2(@types/react@19.2.14)(react@19.2.4) + optionalDependencies: + '@types/react': 19.2.14 + '@types/react-dom': 19.2.3(@types/react@19.2.14) + + '@radix-ui/react-dismissable-layer@1.1.13(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/primitive': 1.1.4 + '@radix-ui/react-compose-refs': 1.1.3(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-primitive': 2.1.6(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.2(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-use-escape-keydown': 1.1.2(@types/react@19.2.14)(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + optionalDependencies: + '@types/react': 19.2.14 + '@types/react-dom': 19.2.3(@types/react@19.2.14) + + '@radix-ui/react-focus-guards@1.1.4(@types/react@19.2.14)(react@19.2.4)': + dependencies: + react: 19.2.4 + optionalDependencies: + '@types/react': 19.2.14 + + '@radix-ui/react-focus-scope@1.1.10(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.3(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-primitive': 2.1.6(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-callback-ref': 1.1.2(@types/react@19.2.14)(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + optionalDependencies: + '@types/react': 19.2.14 + '@types/react-dom': 19.2.3(@types/react@19.2.14) + + '@radix-ui/react-id@1.1.2(@types/react@19.2.14)(react@19.2.4)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.2(@types/react@19.2.14)(react@19.2.4) + react: 19.2.4 + optionalDependencies: + '@types/react': 19.2.14 + + '@radix-ui/react-portal@1.1.12(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-primitive': 2.1.6(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-use-layout-effect': 1.1.2(@types/react@19.2.14)(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + optionalDependencies: + '@types/react': 19.2.14 + '@types/react-dom': 19.2.3(@types/react@19.2.14) + + '@radix-ui/react-presence@1.1.6(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.2(@types/react@19.2.14)(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + optionalDependencies: + '@types/react': 19.2.14 + '@types/react-dom': 19.2.3(@types/react@19.2.14) + + '@radix-ui/react-primitive@2.1.6(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)': + dependencies: + '@radix-ui/react-slot': 1.3.0(@types/react@19.2.14)(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + optionalDependencies: + '@types/react': 19.2.14 + '@types/react-dom': 19.2.3(@types/react@19.2.14) + + '@radix-ui/react-slot@1.3.0(@types/react@19.2.14)(react@19.2.4)': + dependencies: + '@radix-ui/react-compose-refs': 1.1.3(@types/react@19.2.14)(react@19.2.4) + react: 19.2.4 + optionalDependencies: + '@types/react': 19.2.14 + + '@radix-ui/react-use-callback-ref@1.1.2(@types/react@19.2.14)(react@19.2.4)': + dependencies: + react: 19.2.4 + optionalDependencies: + '@types/react': 19.2.14 + + '@radix-ui/react-use-controllable-state@1.2.3(@types/react@19.2.14)(react@19.2.4)': + dependencies: + '@radix-ui/react-use-effect-event': 0.0.3(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-use-layout-effect': 1.1.2(@types/react@19.2.14)(react@19.2.4) + react: 19.2.4 + optionalDependencies: + '@types/react': 19.2.14 + + '@radix-ui/react-use-effect-event@0.0.3(@types/react@19.2.14)(react@19.2.4)': + dependencies: + '@radix-ui/react-use-layout-effect': 1.1.2(@types/react@19.2.14)(react@19.2.4) + react: 19.2.4 + optionalDependencies: + '@types/react': 19.2.14 + + '@radix-ui/react-use-escape-keydown@1.1.2(@types/react@19.2.14)(react@19.2.4)': + dependencies: + '@radix-ui/react-use-callback-ref': 1.1.2(@types/react@19.2.14)(react@19.2.4) + react: 19.2.4 + optionalDependencies: + '@types/react': 19.2.14 + + '@radix-ui/react-use-layout-effect@1.1.2(@types/react@19.2.14)(react@19.2.4)': + dependencies: + react: 19.2.4 + optionalDependencies: + '@types/react': 19.2.14 + '@react-three/fiber@9.6.1(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4)(three@0.184.0)': dependencies: '@babel/runtime': 7.29.7 @@ -3473,6 +3863,10 @@ snapshots: argparse@2.0.1: {} + aria-hidden@1.2.6: + dependencies: + tslib: 2.8.1 + aria-query@5.3.2: {} array-buffer-byte-length@1.0.2: @@ -3642,6 +4036,18 @@ snapshots: cluster-key-slot@1.1.2: {} + cmdk@1.1.1(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4): + dependencies: + '@radix-ui/react-compose-refs': 1.1.3(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-dialog': 1.1.17(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + '@radix-ui/react-id': 1.1.2(@types/react@19.2.14)(react@19.2.4) + '@radix-ui/react-primitive': 2.1.6(@types/react-dom@19.2.3(@types/react@19.2.14))(@types/react@19.2.14)(react-dom@19.2.4(react@19.2.4))(react@19.2.4) + react: 19.2.4 + react-dom: 19.2.4(react@19.2.4) + transitivePeerDependencies: + - '@types/react' + - '@types/react-dom' + color-convert@2.0.1: dependencies: color-name: 1.1.4 @@ -3725,6 +4131,8 @@ snapshots: detect-libc@2.1.2: {} + detect-node-es@1.1.0: {} + doctrine@2.1.0: dependencies: esutils: 2.0.3 @@ -4216,6 +4624,8 @@ snapshots: functions-have-names@1.2.3: {} + fuse.js@7.4.2: {} + generator-function@2.0.1: {} generic-pool@3.9.0: {} @@ -4235,6 +4645,8 @@ snapshots: hasown: 2.0.3 math-intrinsics: 1.1.0 + get-nonce@1.0.1: {} + get-proto@1.0.1: dependencies: dunder-proto: 1.0.1 @@ -4828,6 +5240,33 @@ snapshots: react-is@16.13.1: {} + react-remove-scroll-bar@2.3.8(@types/react@19.2.14)(react@19.2.4): + dependencies: + react: 19.2.4 + react-style-singleton: 2.2.3(@types/react@19.2.14)(react@19.2.4) + tslib: 2.8.1 + optionalDependencies: + '@types/react': 19.2.14 + + react-remove-scroll@2.7.2(@types/react@19.2.14)(react@19.2.4): + dependencies: + react: 19.2.4 + react-remove-scroll-bar: 2.3.8(@types/react@19.2.14)(react@19.2.4) + react-style-singleton: 2.2.3(@types/react@19.2.14)(react@19.2.4) + tslib: 2.8.1 + use-callback-ref: 1.3.3(@types/react@19.2.14)(react@19.2.4) + use-sidecar: 1.1.3(@types/react@19.2.14)(react@19.2.4) + optionalDependencies: + '@types/react': 19.2.14 + + react-style-singleton@2.2.3(@types/react@19.2.14)(react@19.2.4): + dependencies: + get-nonce: 1.0.1 + react: 19.2.4 + tslib: 2.8.1 + optionalDependencies: + '@types/react': 19.2.14 + react-use-measure@2.1.7(react-dom@19.2.4(react@19.2.4))(react@19.2.4): dependencies: react: 19.2.4 @@ -5266,6 +5705,21 @@ snapshots: dependencies: punycode: 2.3.1 + use-callback-ref@1.3.3(@types/react@19.2.14)(react@19.2.4): + dependencies: + react: 19.2.4 + tslib: 2.8.1 + optionalDependencies: + '@types/react': 19.2.14 + + use-sidecar@1.1.3(@types/react@19.2.14)(react@19.2.4): + dependencies: + detect-node-es: 1.1.0 + react: 19.2.4 + tslib: 2.8.1 + optionalDependencies: + '@types/react': 19.2.14 + use-sync-external-store@1.6.0(react@19.2.4): dependencies: react: 19.2.4 diff --git a/public/logos/aevo.svg b/public/logos/aevo.svg new file mode 100644 index 00000000..09e01ae0 --- /dev/null +++ b/public/logos/aevo.svg @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/public/logos/aster.svg b/public/logos/aster.svg new file mode 100644 index 00000000..6559bc7b --- /dev/null +++ b/public/logos/aster.svg @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/public/logos/blocknative.svg b/public/logos/blocknative.svg deleted file mode 100644 index 815dc90b..00000000 --- a/public/logos/blocknative.svg +++ /dev/null @@ -1,38 +0,0 @@ - - - - - - - - - - - - - - - - - - diff --git a/public/logos/drift.svg b/public/logos/drift.svg new file mode 100644 index 00000000..959ac3c9 --- /dev/null +++ b/public/logos/drift.svg @@ -0,0 +1,5 @@ + + + + + diff --git a/public/logos/edgex.jpg b/public/logos/edgex.jpg new file mode 100644 index 00000000..0ecd0f19 Binary files /dev/null and b/public/logos/edgex.jpg differ diff --git a/public/logos/extended.svg b/public/logos/extended.svg new file mode 100644 index 00000000..9035442b --- /dev/null +++ b/public/logos/extended.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/logos/gains.svg b/public/logos/gains.svg new file mode 100644 index 00000000..0473bedb --- /dev/null +++ b/public/logos/gains.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/logos/grvt.jpg b/public/logos/grvt.jpg new file mode 100644 index 00000000..3ab8f1ab Binary files /dev/null and b/public/logos/grvt.jpg differ diff --git a/public/logos/hyperliquid.png b/public/logos/hyperliquid.png index 4a48daa3..8acd0ff7 100644 Binary files a/public/logos/hyperliquid.png and b/public/logos/hyperliquid.png differ diff --git a/public/logos/near-intents.svg b/public/logos/near-intents.svg new file mode 100644 index 00000000..0dab1130 --- /dev/null +++ b/public/logos/near-intents.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/logos/ostium.png b/public/logos/ostium.png new file mode 100644 index 00000000..ea2b4c81 Binary files /dev/null and b/public/logos/ostium.png differ diff --git a/public/logos/pacifica.svg b/public/logos/pacifica.svg new file mode 100644 index 00000000..8bc16b26 --- /dev/null +++ b/public/logos/pacifica.svg @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/public/logos/paradex.svg b/public/logos/paradex.svg new file mode 100644 index 00000000..4b4e0218 --- /dev/null +++ b/public/logos/paradex.svg @@ -0,0 +1,17 @@ + + + paradex-white + + + + + + + + + + + + + + \ No newline at end of file diff --git a/public/logos/predexon.svg b/public/logos/predexon.svg new file mode 100644 index 00000000..3a8be70d --- /dev/null +++ b/public/logos/predexon.svg @@ -0,0 +1,19 @@ + + + + + + + + + + Predexon + + + + + + + + + diff --git a/public/logos/ton.png b/public/logos/ton.png deleted file mode 100644 index 6f10e885..00000000 Binary files a/public/logos/ton.png and /dev/null differ diff --git a/public/logos/ton.svg b/public/logos/ton.svg new file mode 100644 index 00000000..6a79055b --- /dev/null +++ b/public/logos/ton.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/public/logos/variational.png b/public/logos/variational.png new file mode 100644 index 00000000..d473f3c8 Binary files /dev/null and b/public/logos/variational.png differ diff --git a/public/logos/vertex.png b/public/logos/vertex.png new file mode 100644 index 00000000..38d253d1 Binary files /dev/null and b/public/logos/vertex.png differ diff --git a/scripts/hf_publisher/README.md b/scripts/hf_publisher/README.md new file mode 100644 index 00000000..01e55dca --- /dev/null +++ b/scripts/hf_publisher/README.md @@ -0,0 +1,55 @@ +# HF dataset publisher + +Daily snapshot publisher for the public +[OpenChainBench/benchmarks](https://huggingface.co/datasets/OpenChainBench/benchmarks) +dataset on Hugging Face. + +## What it does + +- Fetches `https://openchainbench.com/api/citable` and + `/api/stat/` for every live bench. +- Refuses to publish if the source feed is degraded (quorum guard: half + the count below `live` status, or count below the floor). +- Projects the JSON into three Hive-partitioned Parquet tables: + - `headlines/` 1 row per (slug, day) + - `providers/` 1 row per (slug, provider, day) + - `timeseries/` 1 row per (slug, point, day) +- Stages a fixed set of static assets (README, CITATION.cff, LICENSE, + JSON schemas, example queries) and pushes the whole thing to HF. + +## Schema versioning + +`SCHEMA_VERSION` in `publish.py` is the source of truth. Bump it any +time a column is added. **Never** rename or remove columns: the dataset +is a long-lived public artifact and consumers will write queries +against the column names. + +## Local dry-run + +```bash +cd scripts/hf_publisher +pip install -r requirements.txt +python publish.py --dry-run --out /tmp/ocb-hf-test +ls /tmp/ocb-hf-test +``` + +## CI + +The `.github/workflows/hf-publish.yml` workflow runs the tests first, +then either `publish.py --dry-run` (manual dispatch with the flag) or +the real push (scheduled run or manual without the flag). + +Required GitHub secrets: +- `HF_TOKEN` write-scoped token on the dataset repo. +- `SLACK_WEBHOOK_URL` optional incoming-webhook URL for ops alerts. + +## Tests + +```bash +cd scripts/hf_publisher +python -m unittest test_publish.py -v +``` + +Tests cover the quorum guard, all three row builders, the partition +path layout, and the static-asset templating step. They never hit the +live API or HF Hub. diff --git a/scripts/hf_publisher/dataset_template/CITATION.cff b/scripts/hf_publisher/dataset_template/CITATION.cff new file mode 100644 index 00000000..b1514a15 --- /dev/null +++ b/scripts/hf_publisher/dataset_template/CITATION.cff @@ -0,0 +1,32 @@ +cff-version: 1.2.0 +title: OpenChainBench Crypto Infrastructure Benchmarks +abstract: >- + Daily snapshots of every public benchmark on openchainbench.com, + open, reproducible measurements of crypto infrastructure (RPCs, + oracles, bridges, data APIs, Polymarket adapters, Hyperliquid + builders). Released as Hive-partitioned Parquet under CC-BY-4.0. +authors: + - name: OpenChainBench Contributors + website: https://openchainbench.com +type: dataset +license: CC-BY-4.0 +repository-code: https://github.com/ChainBench/OpenChainBench +url: https://huggingface.co/datasets/OpenChainBench/benchmarks +date-released: "{{snapshot_date}}" +identifiers: + - type: doi + value: 10.5281/zenodo.20800311 + description: Concept DOI (always resolves to the latest version) + - type: doi + value: 10.5281/zenodo.20800312 + description: v1.0.1 version DOI +keywords: + - blockchain + - crypto + - benchmarks + - infrastructure + - latency + - oracles + - bridges + - polymarket + - hyperliquid diff --git a/scripts/hf_publisher/dataset_template/LICENSE b/scripts/hf_publisher/dataset_template/LICENSE new file mode 100644 index 00000000..eef3a499 --- /dev/null +++ b/scripts/hf_publisher/dataset_template/LICENSE @@ -0,0 +1,23 @@ +Creative Commons Attribution 4.0 International (CC BY 4.0) + +You are free to: + Share - copy and redistribute the material in any medium or format + Adapt - remix, transform, and build upon the material for any purpose, + even commercially. + +Under the following terms: + Attribution - You must give appropriate credit, provide a link to the + license, and indicate if changes were made. You may do + so in any reasonable manner, but not in any way that + suggests the licensor endorses you or your use. + +No additional restrictions - You may not apply legal terms or +technological measures that legally restrict others from doing anything +the license permits. + +Full license text: https://creativecommons.org/licenses/by/4.0/legalcode +Summary: https://creativecommons.org/licenses/by/4.0/ + +Suggested attribution: + OpenChainBench. (2026). OpenChainBench Benchmarks [Data set]. + Hugging Face. https://huggingface.co/datasets/OpenChainBench/benchmarks diff --git a/scripts/hf_publisher/dataset_template/README.md b/scripts/hf_publisher/dataset_template/README.md new file mode 100644 index 00000000..918b8e74 --- /dev/null +++ b/scripts/hf_publisher/dataset_template/README.md @@ -0,0 +1,618 @@ +--- +license: cc-by-4.0 +doi: 10.5281/zenodo.20800311 +language: + - en +language_creators: + - machine-generated +annotations_creators: + - machine-generated +multilinguality: + - monolingual +pretty_name: OpenChainBench Crypto Infrastructure Benchmarks +viewer: true +source_datasets: + - original +task_categories: + - time-series-forecasting + - tabular-regression + - other +task_ids: + - univariate-time-series-forecasting +tags: + - crypto + - blockchain + - benchmarks + - rpc + - oracles + - bridges + - polymarket + - infrastructure + - latency + - finance + - defi + - solana + - ethereum + - hyperliquid + - mev + - observability + - sla + - mlcroissant + - tabular + - timeseries + - monitoring +size_categories: + - 10K`. + +### Languages + +The dataset is monolingual: English (`en`). All text fields (title, +category, metric, unit) are English. Provider names are project +trademarks and preserve their original casing. + +## Dataset Structure + +### Data Instances + +Each config is a flat Parquet table partitioned by `snapshot_date`. +A row looks like (headlines): + +```json +{ + "snapshot_date": "2026-06-22", + "captured_at": "2026-06-22T14:37:00+00:00", + "slug": "bridge-quote-latency", + "title": "Bridge Quote Latency", + "category": "Bridges", + "metric": "Quote Latency", + "unit": "ms", + "status": "live", + "value": 412.0, + "higher_is_better": false, + "leader_name": "LI.FI", + "leader_slug": "lifi", + "leader_value": 412.0, + "bench_sample_size": 12480.0, + "as_of": "2026-06-22T14:30:00.000Z", + "citation_url": "https://openchainbench.com/benchmarks/bridge-quote-latency", + "stat_api_url": "https://openchainbench.com/api/stat/bridge-quote-latency", + "source_url": "https://github.com/ChainBench/OpenChainBench/blob/main/benchmarks/bridge-quote-latency.yml", + "license": "CC-BY-4.0", + "schema_version": 2 +} +``` + +### Data Fields + +Each config has its own schema. JSON Schema files live alongside this +README under `schemas/`. Tables below are the authoritative source for +column names and nullability. + +#### Data Fields, headlines + +One row per (slug, snapshot_date). The "who leads" feed. + +| Column | Type | Nullable | Description | Example | +|---|---|---|---|---| +| snapshot_date | string | no | Partition key, ISO date (UTC) of the capture | `2026-06-22` | +| captured_at | string | no | ISO 8601 timestamp of the capture | `2026-06-22T14:37:00+00:00` | +| slug | string | no | Benchmark slug, stable URL identifier | `bridge-quote-latency` | +| title | string | no | Human-readable benchmark title | `Bridge Quote Latency` | +| category | string | no | One of `RPCs`, `Bridges`, `Blockchains`, `Aggregators`, `Trading`, `Wallets`, `NFT APIs` | `Bridges` | +| metric | string | no | What is measured | `Quote Latency` | +| unit | string | no | One of `ms`, `s`, `sec`, `pct`, `bps`, `bp`, `count`, `slots`, `usd` | `ms` | +| status | string | no | One of `live`, `draft`, `insufficient` | `live` | +| value | float64 | yes | Headline value of the leader. Null when `status != live` | `412.0` | +| higher_is_better | bool | yes | Direction of the metric, sourced from `/api/stat`. Null when the per-slug fetch failed | `false` | +| leader_name | string | yes | Display name of the leading provider | `LI.FI` | +| leader_slug | string | yes | URL-safe slug of the leading provider | `lifi` | +| leader_value | float64 | yes | Leader's value in `unit` | `412.0` | +| bench_sample_size | float64 | yes | Aggregate sample count over the bench's run window | `12480.0` | +| as_of | string | yes | Source-side timestamp of the underlying measurement | `2026-06-22T14:30:00.000Z` | +| citation_url | string | no | Canonical citation URL for the benchmark | `https://openchainbench.com/benchmarks/bridge-quote-latency` | +| stat_api_url | string | no | Per-bench live JSON endpoint | `https://openchainbench.com/api/stat/bridge-quote-latency` | +| source_url | string | yes | URL of the bench YAML spec in this repo | `https://github.com/ChainBench/OpenChainBench/blob/main/benchmarks/bridge-quote-latency.yml` | +| license | string | no | Always `CC-BY-4.0` for the data | `CC-BY-4.0` | +| schema_version | int64 | no | Additive schema epoch, bumped on new columns | `2` | + +#### Data Fields, providers + +One row per (bench, provider, snapshot_date). Per-provider rankings. + +| Column | Type | Nullable | Description | Example | +|---|---|---|---|---| +| snapshot_date | string | no | Partition key | `2026-06-22` | +| captured_at | string | no | ISO 8601 capture timestamp | `2026-06-22T14:37:00+00:00` | +| bench_slug | string | no | Foreign key into `headlines.slug` | `bridge-quote-latency` | +| provider_name | string | no | Display name of the provider | `LI.FI` | +| provider_slug | string | no | URL-safe provider slug, stable across snapshots | `lifi` | +| provider_type | string | yes | Architectural category (e.g. `aggregator`, `node-rpc`, `oracle`) | `aggregator` | +| provider_layer | string | yes | Network layer when declared (`L1`, `L2`, etc.) | `L1` | +| provider_tag | string | yes | Free-form tag from the bench YAML | `premium` | +| p50 | float64 | yes | 50th percentile in the bench's `unit` | `412.0` | +| p90 | float64 | yes | 90th percentile | `780.0` | +| p99 | float64 | yes | 99th percentile | `1230.0` | +| mean | float64 | yes | Arithmetic mean | `465.3` | +| success_rate | float64 | yes | Fraction (0..1) or percent (0..100) per bench convention | `0.997` | +| provider_sample_size | float64 | yes | Per-provider sample count over the run window | `2080.0` | +| is_leader | bool | no | True for the provider whose slug matches `headlines.leader_slug` | `true` | +| schema_version | int64 | no | Schema epoch | `2` | + +#### Data Fields, timeseries + +One row per (bench, provider, window, point_index, snapshot_date). The +24h, 7d, and 30d trajectories sourced from `/api/series`. + +| Column | Type | Nullable | Description | Example | +|---|---|---|---|---| +| snapshot_date | string | no | Partition key | `2026-06-22` | +| captured_at | string | no | ISO 8601 capture timestamp | `2026-06-22T14:37:00+00:00` | +| bench_slug | string | no | Foreign key into `headlines.slug` | `bridge-quote-latency` | +| provider_slug | string | yes | Provider this point belongs to. Null only on legacy 24h fallback rows that predate per-provider series | `lifi` | +| point_index | int64 | no | Zero-based index inside the window | `42` | +| value | float64 | no | Value of the metric at this point, in the bench's `unit` | `423.7` | +| window | string | no | One of `24h`, `7d`, `30d` | `24h` | +| schema_version | int64 | no | Schema epoch | `2` | + +#### Data Fields, chain_leaders + +One row per (bench, chain, snapshot_date). Per-chain leader and worst +provider for benches whose spec declares a chain dimension. Currently +empty: see "Considerations for Using the Data" for the open task. + +| Column | Type | Nullable | Description | Example | +|---|---|---|---|---| +| snapshot_date | string | no | Partition key | `2026-06-22` | +| captured_at | string | no | ISO 8601 capture timestamp | `2026-06-22T14:37:00+00:00` | +| bench_slug | string | no | Foreign key into `headlines.slug` | `eth-rpc-head-lag` | +| chain | string | no | Chain slug from the bench spec | `ethereum` | +| leader_name | string | yes | Best provider on this chain | `Mobula` | +| leader_slug | string | yes | URL-safe leader slug | `mobula` | +| leader_value | float64 | yes | Leader's value on this chain | `87.2` | +| worst_name | string | yes | Worst provider on this chain | `LegacyRPC` | +| worst_slug | string | yes | URL-safe worst slug | `legacyrpc` | +| worst_value | float64 | yes | Worst provider's value | `1421.3` | +| schema_version | int64 | no | Schema epoch | `2` | + +### Data Splits + +Every config exposes a single `train` split. There is no held-out +evaluation split because the dataset is observational: it records +measurements as they happen and downstream users define their own +train / test cuts (typically by `snapshot_date`). + +## Dataset Creation + +### Curation Rationale + +The OpenChainBench site renders human-readable leaderboards but its +underlying JSON feeds are designed to be agent-friendly: every value +is paired with a methodology link, a license, and a sample size. This +dataset freezes those feeds daily so: + +- LLM agents and journalists can cite a deterministic snapshot. +- ML researchers can train models without depending on a live API + whose numbers move every minute. +- Operators can compare today's leader against arbitrary historical + baselines without rebuilding the harness. + +### Source Data + +#### Initial Data Collection and Normalization + +Raw measurements are collected by per-bench harnesses (open-sourced in +the [OpenChainBench GitHub repo](https://github.com/ChainBench/OpenChainBench) +or, for a few benches, in a private mobula-api repo where they exist +behind paid API keys). Harnesses publish Prometheus metrics that the +OCB Next.js app aggregates into a `Benchmark` object per bench. + +The dataset publisher reads from: + +- `https://openchainbench.com/api/citable` for the headline feed. +- `https://openchainbench.com/api/stat/` for per-bench detail + (provider rankings, sparkline, `higherIsBetter`). +- `https://openchainbench.com/api/series/?range=` for the + 24h / 7d / 30d per-provider trajectories. + +Each per-bench page documents its full methodology. The +`citation_url` column of `headlines` is the stable link to that page. + +#### Who are the source language producers? + +All text fields (titles, methodology copy, category labels) are +authored by OpenChainBench contributors in the YAML benchmark specs +under `benchmarks/` in the GitHub repo. The data values themselves are +machine-generated by the measurement harnesses. + +### Annotations + +The dataset has no human-applied annotation layer. Provider rankings +and leader flags are derived programmatically from the percentile +measurements according to each bench's `higher_is_better` direction. + +#### Annotation process + +The OCB Next.js layer computes the leader as the provider with the +best `p50` according to the bench's direction. `bestPerChain` and +`worstPerChain` (when populated) are computed with the same rule +scoped to chain-restricted samples. The `is_leader` boolean in the +`providers` table is a derived projection of `headlines.leader_slug`. + +#### Who are the annotators? + +There are no human annotators. Categorical fields like `category`, +`metric`, `unit`, `provider_type`, and `provider_layer` are authored +by the bench YAML maintainers and reviewed via the same PR process as +the harness code. + +### Personal and Sensitive Information + +The dataset contains no personal or sensitive information. Provider +identifiers refer to operational entities (companies, networks, public +APIs) and are publicly listed on the OpenChainBench site. + +## Considerations for Using the Data + +### Social Impact of Dataset + +Public, reproducible measurements of crypto infrastructure raise the +bar for operator transparency. Downstream consumers should not, however, +treat any single snapshot as definitive: providers' production +characteristics change with traffic, deployments, and incident +recovery. + +### Discussion of Biases + +- **Vantage bias**: latency benchmarks are scraped from a small set of + Prometheus harnesses located in specific cloud regions. The exact + vantage points and methodology are documented per bench at + `citation_url`. +- **Sample asymmetry**: providers that rate-limit our probes hard end + up with smaller `provider_sample_size` than providers that allow + generous quotas. This biases percentile estimates upward (fewer + samples surface tail latency less reliably). The `is_leader` flag is + derived purely from the p50 figure and may therefore reflect + measurement-side asymmetry, not just provider performance. +- **Aggregator coverage**: providers that wrap multiple upstream APIs + (aggregators, with `provider_type = "aggregator"`) compete on a + different surface than single-vendor providers and are not strictly + apples-to-apples comparable. The `provider_type` column is meant to + let consumers filter or stratify by this distinction. + +### Other Known Limitations + +- **Schema stability promise (additive only)**: new columns may be + added without warning. Consumer queries should select named columns + rather than `SELECT *`. Existing columns are never renamed or + removed within a `schema_version`. If a breaking change is + unavoidable, a parallel v3 / v4 folder ships alongside the v2 + partitions so old consumers keep working. +- **`chain_leaders` is currently empty**: the `bestPerChain` / + `worstPerChain` data exists inside the OCB aggregator but is not + exposed by `/api/citable` or `/api/stat` yet. The table is shipped + with its canonical schema so downstream pipelines can stabilize + against a real (zero-row) parquet today and start receiving rows + as soon as the API surfaces the field. Tracked in the publisher's + source code as a `TODO`. +- **24h legacy fallback**: when `/api/series` returns no payload for a + bench's 24h window, the publisher falls back to the aggregate + `sparkline` from `/api/stat`. Those fallback rows carry the leader's + `provider_slug` rather than a per-provider series; downstream users + who care about per-provider trajectories should filter on `window in + ('7d', '30d')` or join `provider_slug` against the `providers` + config. +- **Quorum guard**: if the source feed reports fewer than half its + benches as `live` on capture day, the publisher refuses to upload a + new partition. The previous good snapshot stays as truth that day. + +## Additional Information + +### Dataset Curators + +OpenChainBench Contributors. The publishing pipeline is open source +under the Apache 2.0 license at +[github.com/ChainBench/OpenChainBench](https://github.com/ChainBench/OpenChainBench) +(see `scripts/hf_publisher/`). + +### Licensing Information + +Data is released under +[Creative Commons Attribution 4.0](https://creativecommons.org/licenses/by/4.0/). +You may use it freely with attribution to OpenChainBench. + +The publishing scripts and benchmark YAML specs are released under +Apache 2.0. + +### Citation Information + +The dataset is archived on Zenodo with a permanent DOI. Use the concept +DOI when citing the dataset in general (it always resolves to the latest +version). Use the version DOI when citing a specific snapshot for +reproducibility. + +- Concept DOI: [10.5281/zenodo.20800311](https://doi.org/10.5281/zenodo.20800311) +- v1.0.1 version DOI: [10.5281/zenodo.20800312](https://doi.org/10.5281/zenodo.20800312) + +Suggested attribution string: + +> OpenChainBench. (2026). OpenChainBench Crypto Infrastructure Benchmarks +> [Data set]. Zenodo. https://doi.org/10.5281/zenodo.20800311 + +BibTeX: + +```bibtex +@dataset{openchainbench_2026, + author = {{OpenChainBench Contributors}}, + title = {OpenChainBench Crypto Infrastructure Benchmarks}, + year = {2026}, + publisher = {Zenodo}, + doi = {10.5281/zenodo.20800311}, + url = {https://doi.org/10.5281/zenodo.20800311}, + note = {Live mirror at https://huggingface.co/datasets/OpenChainBench/benchmarks} +} +``` + +A machine-readable `CITATION.cff` is also published at the root of +this dataset; GitHub, HF, and Zenodo all parse it. + +### Contributions + +Bug reports, schema requests, and new benchmark proposals go through +GitHub Issues at +[github.com/ChainBench/OpenChainBench/issues](https://github.com/ChainBench/OpenChainBench/issues). +Benchmarks are contributed as YAML files plus a Prometheus-emitting +harness; the contributor guide is in +[CONTRIBUTING.md](https://github.com/ChainBench/OpenChainBench/blob/main/CONTRIBUTING.md). + +## Quick start + +### Python (datasets) + +```python +from datasets import load_dataset +ds = load_dataset("OpenChainBench/benchmarks", "headlines", split="train") +print(ds.filter(lambda r: r["slug"] == "bridge-quote-latency")[0]) +``` + +### Polars (recommended for analytics) + +```python +import polars as pl +df = pl.scan_parquet( + "hf://datasets/OpenChainBench/benchmarks/headlines/**/*.parquet" +) +latest = ( + df.filter(pl.col("snapshot_date") == df.select(pl.col("snapshot_date").max()).collect().item()) + .select(["slug", "leader_name", "value", "unit"]) + .collect() +) +print(latest) +``` + +### DuckDB (one-liner) + +```sql +SELECT slug, leader_name, value, unit +FROM 'hf://datasets/OpenChainBench/benchmarks/headlines/**/*.parquet' +WHERE snapshot_date = ( + SELECT max(snapshot_date) + FROM 'hf://datasets/OpenChainBench/benchmarks/headlines/**/*.parquet' +); +``` + +More examples in `examples/`. + +## Update cadence + +Snapshots run daily at 00:00 UTC via a GitHub Action. If a run aborts +on a quorum check (the source feed has fewer than half its benches +live), no new partition is added that day. The previous good snapshot +stays as truth. + +## Provenance + +- Source code (publisher + benchmark YAML specs): https://github.com/ChainBench/OpenChainBench +- Live measurement APIs: https://openchainbench.com/api/citable, /api/stat, /api/series, /api/llm-context, /api/mcp/mcp +- Issues / questions: https://github.com/ChainBench/OpenChainBench/issues diff --git a/scripts/hf_publisher/dataset_template/dataset-metadata.json b/scripts/hf_publisher/dataset_template/dataset-metadata.json new file mode 100644 index 00000000..0e6648ca --- /dev/null +++ b/scripts/hf_publisher/dataset_template/dataset-metadata.json @@ -0,0 +1,33 @@ +{ + "id": "openchainbench/benchmarks", + "title": "OpenChainBench Crypto Infrastructure Benchmarks", + "subtitle": "Daily Parquet benchmarks of crypto infrastructure: RPCs, oracles, bridges", + "description": "Daily snapshots of every public benchmark on openchainbench.com, released as Hive-partitioned Parquet under CC-BY-4.0.\n\nOpenChainBench measures latency, cost, coverage and accuracy of crypto infrastructure (RPCs, oracles, bridges, data APIs, Polymarket adapters, Hyperliquid builders). Every snapshot mirrors the /api/citable, /api/stat/, and /api/series/ JSON feeds at the time of capture.\n\nLatest snapshot: {{snapshot_date}} (captured at {{captured_at}}, schema v{{schema_version}}).\n\nTables:\n headlines one row per (slug, date), the \"who leads\" feed used by LLM agents and journalists.\n providers one row per (slug, provider, date), full per-provider ranking with p50 / p90 / p99.\n timeseries one row per (slug, provider, window, point_index, date), 24h / 7d / 30d trajectories.\n chain_leaders one row per (slug, chain, date), per-chain best and worst provider.\n\nThis Kaggle dataset is a mirror of the canonical release at huggingface.co/datasets/OpenChainBench/benchmarks. Source code, schemas and the publishing pipeline are on GitHub at github.com/ChainBench/OpenChainBench (see scripts/hf_publisher/).\n\nData is released under Creative Commons Attribution 4.0. Cite as: OpenChainBench. (2026). OpenChainBench Crypto Infrastructure Benchmarks [Data set]. https://openchainbench.com.", + "isPrivate": false, + "licenses": [ + { + "name": "CC-BY-4.0" + } + ], + "keywords": [ + "crypto", + "blockchain", + "benchmarks", + "rpc", + "oracles", + "bridges", + "polymarket", + "infrastructure", + "latency", + "finance", + "defi", + "solana", + "ethereum", + "hyperliquid", + "tabular", + "time series", + "monitoring" + ], + "collaborators": [], + "data": [] +} diff --git a/scripts/hf_publisher/dataset_template/examples/01_pandas.py b/scripts/hf_publisher/dataset_template/examples/01_pandas.py new file mode 100644 index 00000000..0b641c57 --- /dev/null +++ b/scripts/hf_publisher/dataset_template/examples/01_pandas.py @@ -0,0 +1,25 @@ +""" +Load the OCB headlines feed with pandas via the Hugging Face datasets +library. Good when you want a familiar DataFrame and the dataset is +small enough to fit in memory (it is). +""" + +from datasets import load_dataset + +ds = load_dataset( + "OpenChainBench/benchmarks", + "headlines", + split="train", +) +df = ds.to_pandas() + +# Latest snapshot only +latest = df["snapshot_date"].max() +today = df[df["snapshot_date"] == latest] + +# Top 10 benchmarks by sample size today +print( + today.sort_values("sample_size", ascending=False)[ + ["slug", "leader_name", "value", "unit", "sample_size"] + ].head(10) +) diff --git a/scripts/hf_publisher/dataset_template/examples/02_polars.py b/scripts/hf_publisher/dataset_template/examples/02_polars.py new file mode 100644 index 00000000..43b60907 --- /dev/null +++ b/scripts/hf_publisher/dataset_template/examples/02_polars.py @@ -0,0 +1,23 @@ +""" +Stream the OCB providers feed with Polars directly from HF. Pushdown +predicate + projection means only the columns and partitions you ask +for ever hit the wire. Recommended for analytic workloads. +""" + +import polars as pl + +providers = pl.scan_parquet( + "hf://datasets/OpenChainBench/benchmarks/providers/**/*.parquet" +) + +# Trend of Mobula's p50 latency on bridge-quote-latency across all snapshots +trend = ( + providers.filter( + (pl.col("bench_slug") == "bridge-quote-latency") + & (pl.col("provider_slug") == "mobula") + ) + .select(["snapshot_date", "p50", "p90", "p99", "sample_size"]) + .sort("snapshot_date") + .collect() +) +print(trend) diff --git a/scripts/hf_publisher/dataset_template/examples/03_duckdb.sql b/scripts/hf_publisher/dataset_template/examples/03_duckdb.sql new file mode 100644 index 00000000..4787f420 --- /dev/null +++ b/scripts/hf_publisher/dataset_template/examples/03_duckdb.sql @@ -0,0 +1,32 @@ +-- DuckDB can read Parquet directly from Hugging Face over httpfs. +-- Install once: +-- INSTALL httpfs; LOAD httpfs; +-- Then run any query like the ones below. + +-- 1) Today's leader per benchmark, sorted by sample size +WITH latest AS ( + SELECT max(snapshot_date) AS d + FROM 'hf://datasets/OpenChainBench/benchmarks/headlines/**/*.parquet' +) +SELECT slug, leader_name, value, unit, sample_size +FROM 'hf://datasets/OpenChainBench/benchmarks/headlines/**/*.parquet' +WHERE snapshot_date = (SELECT d FROM latest) +ORDER BY sample_size DESC; + +-- 2) 7-day p50 trend for one bench / one provider +SELECT snapshot_date, p50, p90, p99, success_rate +FROM 'hf://datasets/OpenChainBench/benchmarks/providers/**/*.parquet' +WHERE bench_slug = 'bridge-quote-latency' + AND provider_slug = 'mobula' +ORDER BY snapshot_date DESC +LIMIT 7; + +-- 3) Sparkline for today, one bench +SELECT point_index, value +FROM 'hf://datasets/OpenChainBench/benchmarks/timeseries/**/*.parquet' +WHERE bench_slug = 'bridge-quote-latency' + AND snapshot_date = ( + SELECT max(snapshot_date) + FROM 'hf://datasets/OpenChainBench/benchmarks/timeseries/**/*.parquet' + ) +ORDER BY point_index; diff --git a/scripts/hf_publisher/dataset_template/schemas/chain_leaders.schema.json b/scripts/hf_publisher/dataset_template/schemas/chain_leaders.schema.json new file mode 100644 index 00000000..25552b16 --- /dev/null +++ b/scripts/hf_publisher/dataset_template/schemas/chain_leaders.schema.json @@ -0,0 +1,27 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://huggingface.co/datasets/OpenChainBench/benchmarks/blob/main/schemas/chain_leaders.schema.json", + "title": "OCB chain_leaders row", + "description": "One row per (bench, chain, snapshot_date). Per-chain leader and worst provider for benches whose spec declares a chain dimension. Empty until /api/citable or /api/stat exposes bestPerChain / worstPerChain.", + "type": "object", + "required": [ + "snapshot_date", + "captured_at", + "bench_slug", + "chain", + "schema_version" + ], + "properties": { + "snapshot_date": { "type": "string", "format": "date" }, + "captured_at": { "type": "string", "format": "date-time" }, + "bench_slug": { "type": "string" }, + "chain": { "type": "string" }, + "leader_name": { "type": ["string", "null"] }, + "leader_slug": { "type": ["string", "null"] }, + "leader_value": { "type": ["number", "null"] }, + "worst_name": { "type": ["string", "null"] }, + "worst_slug": { "type": ["string", "null"] }, + "worst_value": { "type": ["number", "null"] }, + "schema_version": { "type": "integer", "minimum": 1 } + } +} diff --git a/scripts/hf_publisher/dataset_template/schemas/headlines.schema.json b/scripts/hf_publisher/dataset_template/schemas/headlines.schema.json new file mode 100644 index 00000000..592cbf28 --- /dev/null +++ b/scripts/hf_publisher/dataset_template/schemas/headlines.schema.json @@ -0,0 +1,40 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://huggingface.co/datasets/OpenChainBench/benchmarks/blob/main/schemas/headlines.schema.json", + "title": "OCB headlines row", + "description": "One row per (slug, snapshot_date). Lightweight headline feed.", + "type": "object", + "required": [ + "snapshot_date", + "captured_at", + "slug", + "title", + "category", + "metric", + "unit", + "status", + "schema_version" + ], + "properties": { + "snapshot_date": { "type": "string", "format": "date" }, + "captured_at": { "type": "string", "format": "date-time" }, + "slug": { "type": "string" }, + "title": { "type": "string" }, + "category": { "type": "string" }, + "metric": { "type": "string" }, + "unit": { "type": "string" }, + "status": { "type": "string", "enum": ["live", "draft", "insufficient"] }, + "value": { "type": ["number", "null"] }, + "higher_is_better": { "type": ["boolean", "null"] }, + "leader_name": { "type": ["string", "null"] }, + "leader_slug": { "type": ["string", "null"] }, + "leader_value": { "type": ["number", "null"] }, + "bench_sample_size": { "type": ["number", "null"] }, + "as_of": { "type": ["string", "null"], "format": "date-time" }, + "citation_url": { "type": "string", "format": "uri" }, + "stat_api_url": { "type": "string", "format": "uri" }, + "source_url": { "type": ["string", "null"], "format": "uri" }, + "license": { "type": "string", "const": "CC-BY-4.0" }, + "schema_version": { "type": "integer", "minimum": 1 } + } +} diff --git a/scripts/hf_publisher/dataset_template/schemas/providers.schema.json b/scripts/hf_publisher/dataset_template/schemas/providers.schema.json new file mode 100644 index 00000000..ee07032c --- /dev/null +++ b/scripts/hf_publisher/dataset_template/schemas/providers.schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://huggingface.co/datasets/OpenChainBench/benchmarks/blob/main/schemas/providers.schema.json", + "title": "OCB providers row", + "description": "One row per (bench, provider, snapshot_date). Detailed ranking with percentiles plus provider type / layer / tag classification.", + "type": "object", + "required": [ + "snapshot_date", + "captured_at", + "bench_slug", + "provider_slug", + "schema_version" + ], + "properties": { + "snapshot_date": { "type": "string", "format": "date" }, + "captured_at": { "type": "string", "format": "date-time" }, + "bench_slug": { "type": "string" }, + "provider_name": { "type": "string" }, + "provider_slug": { "type": "string" }, + "provider_type": { "type": ["string", "null"] }, + "provider_layer": { "type": ["string", "null"] }, + "provider_tag": { "type": ["string", "null"] }, + "p50": { "type": ["number", "null"] }, + "p90": { "type": ["number", "null"] }, + "p99": { "type": ["number", "null"] }, + "mean": { "type": ["number", "null"] }, + "success_rate": { "type": ["number", "null"] }, + "provider_sample_size": { "type": ["number", "null"] }, + "is_leader": { "type": "boolean" }, + "schema_version": { "type": "integer", "minimum": 1 } + } +} diff --git a/scripts/hf_publisher/dataset_template/schemas/timeseries.schema.json b/scripts/hf_publisher/dataset_template/schemas/timeseries.schema.json new file mode 100644 index 00000000..90538d02 --- /dev/null +++ b/scripts/hf_publisher/dataset_template/schemas/timeseries.schema.json @@ -0,0 +1,26 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://huggingface.co/datasets/OpenChainBench/benchmarks/blob/main/schemas/timeseries.schema.json", + "title": "OCB timeseries row", + "description": "One row per (bench, provider, window, point_index, snapshot_date). 24h / 7d / 30d trajectories sourced from /api/series.", + "type": "object", + "required": [ + "snapshot_date", + "captured_at", + "bench_slug", + "point_index", + "value", + "window", + "schema_version" + ], + "properties": { + "snapshot_date": { "type": "string", "format": "date" }, + "captured_at": { "type": "string", "format": "date-time" }, + "bench_slug": { "type": "string" }, + "provider_slug": { "type": ["string", "null"] }, + "point_index": { "type": "integer", "minimum": 0 }, + "value": { "type": "number" }, + "window": { "type": "string", "enum": ["24h", "7d", "30d"] }, + "schema_version": { "type": "integer", "minimum": 1 } + } +} diff --git a/scripts/hf_publisher/publish.py b/scripts/hf_publisher/publish.py new file mode 100644 index 00000000..934a59c3 --- /dev/null +++ b/scripts/hf_publisher/publish.py @@ -0,0 +1,791 @@ +""" +Daily snapshot publisher for the Hugging Face dataset +`OpenChainBench/benchmarks`. + +Reads the live citable JSON API plus per-bench detail, projects it into +four Hive-partitioned Parquet tables (headlines, providers, timeseries, +chain_leaders) keyed by snapshot_date, and pushes the new partitions to +the HF dataset repo. + +Why multiple tables and not one wide table: + headlines 1 row per (slug, date). Light. The "who leads" feed used + by LLM agents and journalists. Cheap to scan. + providers 1 row per (slug, provider, date). Detailed per-provider + ranking with p50/p90/p99 plus type/layer/tag classification. + timeseries 1 row per (slug, point_index, window, date) holding the + 24h / 7d / 30d trajectories. Separated so consumers can + ignore it if they only want headlines. + chain_leaders 1 row per (slug, chain, date) holding per-chain leader + and worst provider, sourced from /api/stat's + bestPerChain / worstPerChain fields. Empty for benches + without a chain dimension (no chain-tagged Prom series). + +Quorum guard: refuses to publish if /api/citable returns fewer than half +its declared count as live. The previous good snapshot stays as truth +on HF instead of being overwritten by a degraded one. + +Schema versioning: each table embeds a `schema_version` int column. +Bump it when adding columns. Never remove columns. Never rename. The +HF dataset is a long-lived public artifact, downstream consumers will +write queries that assume column names are stable. + +Idempotency: same snapshot_date overwrites itself. Re-running the cron +for a given day is safe. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import subprocess +import sys +import urllib.error +import urllib.request +from dataclasses import dataclass +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +import pandas as pd +import pyarrow as pa +import pyarrow.parquet as pq + +logger = logging.getLogger("hf_publisher") + +# Bump together with any additive schema change in the row builders below. +# Never decrement. Never re-use a version for a breaking change. If a +# breaking change is unavoidable, ship a parallel `headlines_v2/` folder +# while keeping the v1 partitions readable for old consumers. +# +# v1 (2026-06-22): initial release. 3 tables (headlines, providers, +# timeseries). `sample_size` ambiguously named. +# v2 (2026-06-22): adds `higher_is_better` to headlines, renames +# `sample_size` to `bench_sample_size` (headlines) and +# `provider_sample_size` (providers), adds +# `provider_type`, `provider_layer`, `provider_tag` to +# providers, extends timeseries with 7d / 30d windows +# and a `provider_slug` column, and adds the +# `chain_leaders` table (empty until bestPerChain is +# exposed via the public API). +SCHEMA_VERSION = 2 + +# Minimum count of live benches in /api/citable to allow publishing. The +# bench registry sits around 26, a snapshot with <50% live is considered +# degraded and refused. Floor of 8 avoids tripping during early-stage +# dev where the registry is intentionally small. +QUORUM_MIN_LIVE = 8 +QUORUM_MIN_RATIO = 0.5 + +DEFAULT_API_BASE = "https://openchainbench.com" +DEFAULT_REPO_ID = "OpenChainBench/benchmarks" +USER_AGENT = "ocb-hf-publisher/2.0 (+https://openchainbench.com)" + +# Kaggle mirror config. The dataset URL on Kaggle is +# https://www.kaggle.com/datasets//. The owner is also the +# value of `KAGGLE_USERNAME` we authenticate with: the username has to +# be the dataset owner, otherwise the CLI returns 403 on create / version. +DEFAULT_KAGGLE_DATASET_ID = "openchainbench/benchmarks" +KAGGLE_METADATA_FILENAME = "dataset-metadata.json" + +# Time-series windows fetched per bench. `/api/series/?range=` +# returns one series per provider in the leaderboard at that range. We +# fold each provider's series into the parquet so consumers can compute +# trajectories without re-hitting the live API. +TIMESERIES_WINDOWS = ("24h", "7d", "30d") + + +class PublisherError(Exception): + """Raised when the snapshot is unfit to publish.""" + + +@dataclass(frozen=True) +class Snapshot: + date: str # ISO date, partition key value + captured_at: str # ISO timestamp UTC, embedded in every row + + +def fetch_json(url: str, timeout: float = 30.0) -> dict[str, Any]: + req = urllib.request.Request(url, headers={"User-Agent": USER_AGENT}) + try: + with urllib.request.urlopen(req, timeout=timeout) as resp: + payload = resp.read().decode("utf-8") + except urllib.error.HTTPError as e: + raise PublisherError(f"GET {url} returned HTTP {e.code}") from e + except urllib.error.URLError as e: + raise PublisherError(f"GET {url} failed: {e.reason}") from e + # socket-level read timeouts surface as bare TimeoutError on py3.10+ + # (subclass of OSError, not URLError). Catch explicitly so callers like + # fetch_series can swallow per-URL timeouts via PublisherError instead + # of aborting the whole snapshot. + except TimeoutError as e: + raise PublisherError(f"GET {url} timed out after {timeout}s") from e + except OSError as e: + raise PublisherError(f"GET {url} socket error: {e}") from e + try: + return json.loads(payload) + except json.JSONDecodeError as e: + raise PublisherError(f"GET {url} returned non-JSON body") from e + + +def validate_quorum(citable: dict[str, Any]) -> None: + count = int(citable.get("count") or 0) + benches = citable.get("benchmarks") or [] + live = sum(1 for b in benches if b.get("status") == "live") + if count < QUORUM_MIN_LIVE: + raise PublisherError( + f"degraded source: count={count} below minimum {QUORUM_MIN_LIVE}" + ) + if live < QUORUM_MIN_LIVE or live / max(count, 1) < QUORUM_MIN_RATIO: + raise PublisherError( + f"degraded source: only {live}/{count} live (<{QUORUM_MIN_RATIO:.0%})" + ) + logger.info("quorum ok: %d/%d live", live, count) + + +def build_headlines( + citable: dict[str, Any], + snap: Snapshot, + higher_is_better_by_slug: dict[str, bool] | None = None, +) -> pd.DataFrame: + """Build the headlines table. The `higher_is_better_by_slug` map is + sourced from the per-slug /api/stat fetches done by `run()`. /api/citable + does not surface this field today, so we backfill from the stat + payloads. Benches whose stat fetch failed get a null entry, which is + intentional since we cannot interpret their leader without it. + """ + higher_is_better_by_slug = higher_is_better_by_slug or {} + rows: list[dict[str, Any]] = [] + for b in citable.get("benchmarks", []): + leader = b.get("leader") or {} + slug = b.get("slug") + rows.append( + { + "snapshot_date": snap.date, + "captured_at": snap.captured_at, + "slug": slug, + "title": b.get("title"), + "category": b.get("category"), + "metric": b.get("metric"), + "unit": b.get("unit"), + "status": b.get("status"), + "value": _f(b.get("value")), + "higher_is_better": higher_is_better_by_slug.get(slug), + "leader_name": leader.get("name"), + "leader_slug": leader.get("slug"), + "leader_value": _f(leader.get("value")), + "bench_sample_size": _f(b.get("sampleSize")), + "as_of": b.get("asOf"), + "citation_url": b.get("url"), + "stat_api_url": b.get("api"), + "source_url": b.get("source"), + "license": b.get("license"), + "schema_version": SCHEMA_VERSION, + } + ) + return pd.DataFrame(rows) + + +def build_providers( + stats: Iterable[dict[str, Any]], + snap: Snapshot, +) -> pd.DataFrame: + rows: list[dict[str, Any]] = [] + for stat in stats: + slug = stat.get("slug") + leader_slug = (stat.get("leader") or {}).get("slug") + for r in stat.get("rankings") or []: + ms = r.get("ms") or {} + rows.append( + { + "snapshot_date": snap.date, + "captured_at": snap.captured_at, + "bench_slug": slug, + "provider_name": r.get("name"), + "provider_slug": r.get("slug"), + "provider_type": r.get("type"), + "provider_layer": r.get("layer"), + "provider_tag": r.get("tag"), + "p50": _f(ms.get("p50")), + "p90": _f(ms.get("p90")), + "p99": _f(ms.get("p99")), + "mean": _f(ms.get("mean")), + "success_rate": _f(r.get("successRate")), + "provider_sample_size": _f(r.get("sampleSize")), + "is_leader": r.get("slug") == leader_slug, + "schema_version": SCHEMA_VERSION, + } + ) + return pd.DataFrame(rows) + + +def build_timeseries( + stats: Iterable[dict[str, Any]], + series_by_slug: dict[str, dict[str, dict[str, Any]]], + snap: Snapshot, +) -> pd.DataFrame: + """Build the timeseries table from /api/series payloads. + + `series_by_slug[slug][window]` is the JSON payload from + /api/series/?range=. The 24h window also falls back + to the sparkline embedded in /api/stat when /api/series 404s, so + timeseries stays populated for benches whose series endpoint has no + data yet. The 7d / 30d windows have no such fallback. The bench + silently emits zero rows for them if /api/series said no_data. + """ + rows: list[dict[str, Any]] = [] + stats_by_slug = {s.get("slug"): s for s in stats if s.get("slug")} + + def _payload_has_data(payload: dict[str, Any] | None) -> bool: + if not payload: + return False + for prov in payload.get("providers") or []: + values = prov.get("values") or [] + if any(v is not None for v in values): + return True + return False + + for slug, by_window in series_by_slug.items(): + for window, payload in by_window.items(): + providers = payload.get("providers") or [] + for prov in providers: + provider_slug = prov.get("slug") + values = prov.get("values") or [] + for idx, value in enumerate(values): + if value is None: + continue + rows.append( + { + "snapshot_date": snap.date, + "captured_at": snap.captured_at, + "bench_slug": slug, + "provider_slug": provider_slug, + "point_index": idx, + "value": _f(value), + "window": window, + "schema_version": SCHEMA_VERSION, + } + ) + # Fallback for 24h: use the sparkline from /api/stat when + # /api/series returned nothing or only empty providers for the + # 24h range. Avoids losing historical 24h coverage on benches + # whose /api/series endpoint hasn't been wired up yet. + if not _payload_has_data(by_window.get("24h")): + stat = stats_by_slug.get(slug) or {} + spark = stat.get("sparkline") or [] + leader_slug = (stat.get("leader") or {}).get("slug") + for idx, value in enumerate(spark): + if value is None: + continue + rows.append( + { + "snapshot_date": snap.date, + "captured_at": snap.captured_at, + "bench_slug": slug, + "provider_slug": leader_slug, + "point_index": idx, + "value": _f(value), + "window": "24h", + "schema_version": SCHEMA_VERSION, + } + ) + return pd.DataFrame(rows) + + +def build_chain_leaders( + stats: Iterable[dict[str, Any]], + snap: Snapshot, +) -> pd.DataFrame: + """Build the chain_leaders table. One row per (bench, chain) when + the per-bench /api/stat response declares a `bestPerChain` (and + optional `worstPerChain`) map. Benches without a chain dimension + (no chain-tagged Prom labels) emit zero rows. Empty across the + whole field is valid and surfaces as a zero-row parquet, so the + schema, partitions, and downstream queries stay stable. + """ + rows: list[dict[str, Any]] = [] + columns = [ + "snapshot_date", + "captured_at", + "bench_slug", + "chain", + "leader_name", + "leader_slug", + "leader_value", + "worst_name", + "worst_slug", + "worst_value", + "schema_version", + ] + for stat in stats: + bench_slug = stat.get("slug") + best = stat.get("bestPerChain") or {} + worst = stat.get("worstPerChain") or {} + if not isinstance(best, dict): + continue + for chain, leader in best.items(): + if not isinstance(leader, dict): + continue + worst_row = worst.get(chain) if isinstance(worst, dict) else None + if not isinstance(worst_row, dict): + worst_row = {} + ms = leader.get("ms") or {} + worst_ms = worst_row.get("ms") or {} + rows.append( + { + "snapshot_date": snap.date, + "captured_at": snap.captured_at, + "bench_slug": bench_slug, + "chain": chain, + "leader_name": leader.get("name"), + "leader_slug": leader.get("slug"), + "leader_value": _f(ms.get("p50")), + "worst_name": worst_row.get("name"), + "worst_slug": worst_row.get("slug"), + "worst_value": _f(worst_ms.get("p50")), + "schema_version": SCHEMA_VERSION, + } + ) + return pd.DataFrame(rows, columns=columns) + + +def _f(v: Any) -> float | None: + if v is None: + return None + try: + return float(v) + except (TypeError, ValueError): + return None + + +def write_partition(df: pd.DataFrame, root: Path, table: str, snap: Snapshot) -> Path: + """Write a single Hive partition: //snapshot_date=/part-0.parquet. + + Snappy + ZSTD: Snappy is wider compatible (most readers default), + ZSTD compresses better. We use ZSTD because Polars/DuckDB/PyArrow + all read it natively now and the size delta matters at scale. + """ + target_dir = root / table / f"snapshot_date={snap.date}" + target_dir.mkdir(parents=True, exist_ok=True) + target = target_dir / "part-0.parquet" + table_ar = pa.Table.from_pandas(df, preserve_index=False) + pq.write_table(table_ar, target, compression="zstd") + logger.info("wrote %s rows=%d size=%dKB", target, len(df), target.stat().st_size // 1024) + return target + + +def post_slack(webhook: str | None, text: str) -> None: + if not webhook: + return + data = json.dumps({"text": text}).encode("utf-8") + req = urllib.request.Request( + webhook, + data=data, + headers={"Content-Type": "application/json", "User-Agent": USER_AGENT}, + ) + try: + with urllib.request.urlopen(req, timeout=10) as resp: + resp.read() + except Exception as e: + # Slack failure must not mask the underlying error + logger.warning("slack notify failed: %s", e) + + +def push_to_hf( + out_root: Path, + repo_id: str, + token: str, + commit_message: str, +) -> str: + """Push the staged dataset folder to HF Hub. Creates the repo if it + doesn't exist. Returns the commit hash.""" + from huggingface_hub import HfApi + + api = HfApi(token=token) + api.create_repo( + repo_id=repo_id, + repo_type="dataset", + exist_ok=True, + private=False, + ) + # upload_folder commits everything below `folder_path` keeping the + # relative paths. Hive partitions therefore land at the right place. + info = api.upload_folder( + folder_path=str(out_root), + repo_id=repo_id, + repo_type="dataset", + commit_message=commit_message, + ) + return info.oid if hasattr(info, "oid") else "unknown" + + +def build_kaggle_metadata( + template_root: Path, + snap: Snapshot, + dataset_id: str = DEFAULT_KAGGLE_DATASET_ID, +) -> dict[str, Any]: + """Load `dataset-metadata.json` from the template folder, substitute + placeholders (`{{snapshot_date}}`, `{{captured_at}}`, + `{{schema_version}}`), and override the `id` field with `dataset_id`. + + The shape returned matches what the Kaggle CLI expects: + https://github.com/Kaggle/kaggle-api/blob/main/docs/dataset-metadata.json + """ + src = template_root / KAGGLE_METADATA_FILENAME + if not src.is_file(): + raise PublisherError( + f"kaggle metadata template missing at {src}" + ) + text = src.read_text(encoding="utf-8") + text = ( + text.replace("{{snapshot_date}}", snap.date) + .replace("{{captured_at}}", snap.captured_at) + .replace("{{schema_version}}", str(SCHEMA_VERSION)) + ) + try: + meta = json.loads(text) + except json.JSONDecodeError as e: + raise PublisherError(f"kaggle metadata is not valid JSON: {e}") from e + + if not isinstance(meta, dict): + raise PublisherError("kaggle metadata must be a JSON object") + if "/" not in dataset_id: + raise PublisherError( + f"kaggle dataset id must be /, got {dataset_id!r}" + ) + meta["id"] = dataset_id + # Kaggle requires `licenses` to be a non-empty list of objects with a + # `name` field. Guard the template against accidental edits. + licenses = meta.get("licenses") or [] + if not isinstance(licenses, list) or not licenses: + raise PublisherError("kaggle metadata missing `licenses`") + # Kaggle enforces a 20-80 char subtitle. Surfacing the constraint here + # turns a remote API 400 into a local PublisherError with a precise + # location, which the next contributor can fix without hunting through + # the kaggle CLI output. + subtitle = meta.get("subtitle") or "" + if not isinstance(subtitle, str) or not 20 <= len(subtitle) <= 80: + raise PublisherError( + f"kaggle subtitle must be 20 to 80 chars, got {len(subtitle)}" + ) + return meta + + +def _run_kaggle(args: list[str], cwd: Path | None = None) -> subprocess.CompletedProcess[str]: + """Run the kaggle CLI and return the completed process. Capturing + stdout / stderr so we can branch on the error string for the first-push + case without spamming the GH Action log on the happy path.""" + return subprocess.run( + ["kaggle", *args], + cwd=str(cwd) if cwd else None, + capture_output=True, + text=True, + check=False, + ) + + +def push_to_kaggle( + out_root: Path, + template_root: Path, + snap: Snapshot, + dataset_id: str, + commit_message: str, +) -> str: + """Mirror the staged parquet folder to Kaggle. + + Writes a fresh `dataset-metadata.json` into `out_root`, then runs + `kaggle datasets version`. If the dataset does not exist yet on + Kaggle, the CLI returns a 404-ish error and we fall back to + `kaggle datasets create`. Returns the URL the dataset lands on. + + Errors here MUST never propagate: HF is the canonical sink, Kaggle is + a best-effort mirror. The caller wraps this in try/except. + """ + if not out_root.is_dir(): + raise PublisherError(f"kaggle staging dir missing: {out_root}") + + meta = build_kaggle_metadata(template_root, snap, dataset_id) + (out_root / KAGGLE_METADATA_FILENAME).write_text( + json.dumps(meta, indent=2) + "\n", + encoding="utf-8", + ) + + # `--dir-mode zip` packs the Hive partition folders into the upload + # archive so the layout is preserved on Kaggle's side. Without it the + # CLI would only upload files at the top level of out_root. + version = _run_kaggle( + [ + "datasets", + "version", + "-p", + str(out_root), + "-m", + commit_message, + "--dir-mode", + "zip", + ] + ) + if version.returncode == 0: + logger.info("kaggle: pushed new version of %s", dataset_id) + else: + # The CLI emits "Dataset not found" / 404 when the dataset has + # never been created. In that case we bootstrap with `create`. + # Any other failure is propagated. + stderr = (version.stderr or "") + (version.stdout or "") + looks_missing = ( + "404" in stderr + or "not found" in stderr.lower() + or "does not exist" in stderr.lower() + ) + if not looks_missing: + raise PublisherError( + f"kaggle version failed (code={version.returncode}): {stderr.strip()}" + ) + logger.info("kaggle: dataset %s missing, creating", dataset_id) + create = _run_kaggle( + [ + "datasets", + "create", + "-p", + str(out_root), + "-u", + "--dir-mode", + "zip", + ] + ) + if create.returncode != 0: + stderr = (create.stderr or "") + (create.stdout or "") + raise PublisherError( + f"kaggle create failed (code={create.returncode}): {stderr.strip()}" + ) + logger.info("kaggle: created %s", dataset_id) + + return f"https://www.kaggle.com/datasets/{dataset_id}" + + +def stage_static_assets(out_root: Path, template_root: Path, snap: Snapshot) -> None: + """Copy README.md + CITATION.cff + LICENSE + schemas + examples into + the upload folder. Templates may include `{{date}}` placeholders. + + `dataset-metadata.json` is deliberately skipped: it is a Kaggle-only + artifact and would just add noise to the HF repo if committed. It is + written into the staging folder later by `push_to_kaggle`. + """ + import shutil + + if not template_root.is_dir(): + return + for src in template_root.rglob("*"): + if not src.is_file(): + continue + rel = src.relative_to(template_root) + if rel.name == KAGGLE_METADATA_FILENAME: + continue + dst = out_root / rel + dst.parent.mkdir(parents=True, exist_ok=True) + if src.suffix in {".md", ".cff", ".json", ".py", ".sql"}: + text = src.read_text(encoding="utf-8") + text = ( + text.replace("{{snapshot_date}}", snap.date) + .replace("{{captured_at}}", snap.captured_at) + .replace("{{schema_version}}", str(SCHEMA_VERSION)) + ) + dst.write_text(text, encoding="utf-8") + else: + shutil.copy2(src, dst) + + +def fetch_series(api_base: str, slug: str) -> dict[str, dict[str, Any]]: + """Fetch /api/series/?range= for every supported window. + Returns a dict {window: payload}. Windows that 404, time out, or hit a + socket error are skipped silently so one slow bench cannot abort the + whole snapshot. The 30d window is the most likely to time out on a + cold cache.""" + out: dict[str, dict[str, Any]] = {} + for window in TIMESERIES_WINDOWS: + url = f"{api_base}/api/series/{slug}?range={window}" + try: + out[window] = fetch_json(url, timeout=60.0) + except PublisherError as e: + logger.info("skip series %s @ %s: %s", slug, window, e) + return out + + +def run( + api_base: str, + repo_id: str, + token: str | None, + out_root: Path, + template_root: Path, + dry_run: bool, + slack_webhook: str | None, + kaggle_dataset_id: str = DEFAULT_KAGGLE_DATASET_ID, + kaggle_username: str | None = None, + kaggle_key: str | None = None, +) -> None: + snap = Snapshot( + date=datetime.now(timezone.utc).strftime("%Y-%m-%d"), + captured_at=datetime.now(timezone.utc).isoformat(timespec="seconds"), + ) + logger.info("publishing snapshot %s", snap.date) + + citable = fetch_json(f"{api_base}/api/citable") + validate_quorum(citable) + + live_slugs = [ + b["slug"] + for b in citable.get("benchmarks", []) + if b.get("status") == "live" and b.get("slug") + ] + stats: list[dict[str, Any]] = [] + series_by_slug: dict[str, dict[str, dict[str, Any]]] = {} + for slug in live_slugs: + try: + stats.append(fetch_json(f"{api_base}/api/stat/{slug}")) + except PublisherError as e: + # One bad per-slug fetch shouldn't abort the run. Log and skip. + logger.warning("skip per-slug fetch for %s: %s", slug, e) + continue + series_by_slug[slug] = fetch_series(api_base, slug) + + # /api/citable does not surface higherIsBetter today, /api/stat does. + # We backfill from the per-slug payloads so headlines carries the + # field. Slugs whose stat fetch failed get a null entry. + higher_is_better_by_slug: dict[str, bool] = { + slug: bool(s.get("higherIsBetter")) + for s in stats + if (slug := s.get("slug")) and "higherIsBetter" in s + } + + headlines = build_headlines(citable, snap, higher_is_better_by_slug) + providers = build_providers(stats, snap) + timeseries = build_timeseries(stats, series_by_slug, snap) + chain_leaders = build_chain_leaders(stats, snap) + + if headlines.empty: + raise PublisherError("empty headlines table, refusing to publish") + + write_partition(headlines, out_root, "headlines", snap) + write_partition(providers, out_root, "providers", snap) + write_partition(timeseries, out_root, "timeseries", snap) + write_partition(chain_leaders, out_root, "chain_leaders", snap) + stage_static_assets(out_root, template_root, snap) + + if dry_run: + logger.info("dry-run: skipping HF push, files staged at %s", out_root) + if kaggle_username and kaggle_key: + logger.info( + "dry-run: skipping kaggle mirror (would push to %s)", + kaggle_dataset_id, + ) + else: + logger.info("kaggle skip: secrets not set") + return + + if not token: + raise PublisherError("HF_TOKEN missing in non-dry-run mode") + + commit_message = ( + f"snapshot {snap.date} " + f"(rows: h={len(headlines)} p={len(providers)} " + f"ts={len(timeseries)} cl={len(chain_leaders)})" + ) + oid = push_to_hf(out_root, repo_id, token, commit_message) + logger.info("pushed to HF: %s commit=%s", repo_id, oid) + post_slack( + slack_webhook, + f":white_check_mark: OCB HF snapshot {snap.date} published " + f"(h={len(headlines)}, p={len(providers)}, " + f"ts={len(timeseries)}, cl={len(chain_leaders)}) " + f"https://huggingface.co/datasets/{repo_id}/tree/main", + ) + + # Best-effort Kaggle mirror. HF is the canonical sink; a Kaggle + # failure must not abort the run nor mark the HF push as failed. + if not (kaggle_username and kaggle_key): + logger.info("kaggle skip: secrets not set") + return + + # The kaggle CLI reads `KAGGLE_USERNAME` / `KAGGLE_KEY` from the + # process env, so forwarding them in os.environ is sufficient. We + # only ensure they are visible to the subprocess. + os.environ["KAGGLE_USERNAME"] = kaggle_username + os.environ["KAGGLE_KEY"] = kaggle_key + try: + kaggle_url = push_to_kaggle( + out_root=out_root, + template_root=template_root, + snap=snap, + dataset_id=kaggle_dataset_id, + commit_message=commit_message, + ) + logger.info("kaggle mirror ok: %s", kaggle_url) + except Exception as e: + logger.warning("kaggle mirror failed: %s", e) + post_slack( + slack_webhook, + f":warning: OCB Kaggle mirror failed for snapshot {snap.date} " + f"(HF push succeeded): {e}", + ) + + +def main() -> int: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(message)s", + ) + p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--api-base", default=os.environ.get("OCB_API", DEFAULT_API_BASE)) + p.add_argument("--repo-id", default=os.environ.get("HF_REPO_ID", DEFAULT_REPO_ID)) + p.add_argument("--out", default=os.environ.get("OCB_OUT", "/tmp/ocb-hf-staging")) + p.add_argument( + "--kaggle-dataset-id", + default=os.environ.get("KAGGLE_DATASET_ID", DEFAULT_KAGGLE_DATASET_ID), + help="Kaggle dataset id in / form. The owner must match KAGGLE_USERNAME.", + ) + p.add_argument( + "--template", + default=str(Path(__file__).parent / "dataset_template"), + help="Static files copied into the dataset (README, CITATION, schemas, examples).", + ) + p.add_argument( + "--dry-run", + action="store_true", + help="Skip HF push. Use for local + CI checks.", + ) + args = p.parse_args() + + token = os.environ.get("HF_TOKEN") + slack = os.environ.get("SLACK_WEBHOOK_URL") + kaggle_username = os.environ.get("KAGGLE_USERNAME") + kaggle_key = os.environ.get("KAGGLE_KEY") + out_root = Path(args.out) + out_root.mkdir(parents=True, exist_ok=True) + + try: + run( + api_base=args.api_base, + repo_id=args.repo_id, + token=token, + out_root=out_root, + template_root=Path(args.template), + dry_run=args.dry_run, + slack_webhook=slack, + kaggle_dataset_id=args.kaggle_dataset_id, + kaggle_username=kaggle_username, + kaggle_key=kaggle_key, + ) + return 0 + except PublisherError as e: + logger.error("publisher aborted: %s", e) + post_slack(slack, f":x: OCB HF publisher aborted: {e}") + return 2 + except Exception as e: + logger.exception("publisher crashed: %s", e) + post_slack(slack, f":fire: OCB HF publisher crashed: {e}") + return 3 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/hf_publisher/requirements.txt b/scripts/hf_publisher/requirements.txt new file mode 100644 index 00000000..bf303280 --- /dev/null +++ b/scripts/hf_publisher/requirements.txt @@ -0,0 +1,4 @@ +huggingface_hub>=0.30,<1.0 +kaggle>=1.6,<2.0 +pandas>=2.2,<3.0 +pyarrow>=18.0,<22.0 diff --git a/scripts/hf_publisher/test_publish.py b/scripts/hf_publisher/test_publish.py new file mode 100644 index 00000000..f26ef804 --- /dev/null +++ b/scripts/hf_publisher/test_publish.py @@ -0,0 +1,452 @@ +""" +Offline tests for the HF publisher. + +These never hit the live API or HF Hub. They drive the row-builders + +quorum guard with hand-crafted payloads so a CI run can catch schema +regressions before they propagate to the public dataset. +""" + +from __future__ import annotations + +import json +import unittest +from pathlib import Path +from tempfile import TemporaryDirectory + +from publish import ( + DEFAULT_KAGGLE_DATASET_ID, + KAGGLE_METADATA_FILENAME, + PublisherError, + SCHEMA_VERSION, + Snapshot, + build_chain_leaders, + build_headlines, + build_kaggle_metadata, + build_providers, + build_timeseries, + stage_static_assets, + validate_quorum, + write_partition, +) + + +def _snap(date: str = "2026-06-22") -> Snapshot: + return Snapshot(date=date, captured_at=f"{date}T00:00:00+00:00") + + +def _citable_fixture(live_count: int = 20, total: int = 26) -> dict: + benches = [] + for i in range(total): + status = "live" if i < live_count else "insufficient" + benches.append( + { + "slug": f"bench-{i}", + "title": f"Bench {i}", + "category": "RPCs", + "metric": "Latency", + "unit": "ms", + "status": status, + "value": 100.5 + i if status == "live" else None, + "leader": ( + {"name": "Mobula", "slug": "mobula", "value": 100.5 + i} + if status == "live" + else None + ), + "sampleSize": 1000 + i, + "asOf": "2026-06-22T00:00:00.000Z", + "url": f"https://openchainbench.com/benchmarks/bench-{i}", + "api": f"https://openchainbench.com/api/stat/bench-{i}", + "source": "https://github.com/ChainBench/OpenChainBench/blob/main/benchmarks/bench-0.yml", + "license": "CC-BY-4.0", + } + ) + return {"count": total, "benchmarks": benches} + + +def _stat_fixture(slug: str = "bench-0", n_providers: int = 3, sparkline_len: int = 72) -> dict: + return { + "slug": slug, + "higherIsBetter": False, + "leader": {"name": "Mobula", "slug": "mobula"}, + "rankings": [ + { + "name": f"Provider {i}", + "slug": f"provider-{i}", + "type": "aggregator" if i == 0 else "node-rpc", + "layer": "L1" if i % 2 == 0 else "L2", + "tag": "premium" if i == 0 else None, + "ms": { + "p50": 100.0 + i, + "p90": 200.0 + i, + "p99": 500.0 + i, + "mean": 150.0 + i, + }, + "successRate": 99.0 - i * 0.1, + "sampleSize": 5000 - i * 100, + } + for i in range(n_providers) + ], + "sparkline": [100.0 + (j % 10) for j in range(sparkline_len)], + } + + +def _series_fixture( + slug: str = "bench-0", + windows: tuple[str, ...] = ("24h", "7d", "30d"), + n_providers: int = 2, + points_per_window: dict[str, int] | None = None, +) -> dict[str, dict]: + points_per_window = points_per_window or {"24h": 72, "7d": 84, "30d": 60} + out: dict[str, dict] = {} + for w in windows: + n = points_per_window.get(w, 72) + out[w] = { + "slug": slug, + "range": w, + "providers": [ + { + "slug": f"provider-{i}", + "name": f"Provider {i}", + "values": [float(j + i) for j in range(n)], + } + for i in range(n_providers) + ], + } + return out + + +class QuorumTests(unittest.TestCase): + def test_passes_at_full_live(self): + validate_quorum(_citable_fixture(live_count=26, total=26)) + + def test_passes_at_half(self): + validate_quorum(_citable_fixture(live_count=13, total=26)) + + def test_refuses_below_half(self): + with self.assertRaises(PublisherError): + validate_quorum(_citable_fixture(live_count=12, total=26)) + + def test_refuses_below_floor(self): + with self.assertRaises(PublisherError): + validate_quorum(_citable_fixture(live_count=7, total=20)) + + def test_refuses_empty(self): + with self.assertRaises(PublisherError): + validate_quorum({"count": 0, "benchmarks": []}) + + +class HeadlinesTests(unittest.TestCase): + def test_columns_stable(self): + df = build_headlines(_citable_fixture(), _snap()) + expected_cols = { + "snapshot_date", + "captured_at", + "slug", + "title", + "category", + "metric", + "unit", + "status", + "value", + "higher_is_better", + "leader_name", + "leader_slug", + "leader_value", + "bench_sample_size", + "as_of", + "citation_url", + "stat_api_url", + "source_url", + "license", + "schema_version", + } + self.assertEqual(set(df.columns), expected_cols) + + def test_schema_version_present(self): + df = build_headlines(_citable_fixture(), _snap()) + self.assertTrue((df["schema_version"] == SCHEMA_VERSION).all()) + + def test_insufficient_rows_have_null_value(self): + df = build_headlines(_citable_fixture(live_count=10, total=12), _snap()) + live = df[df["status"] == "live"] + insufficient = df[df["status"] == "insufficient"] + self.assertTrue(live["value"].notna().all()) + self.assertTrue(insufficient["value"].isna().all()) + + def test_higher_is_better_backfilled(self): + df = build_headlines( + _citable_fixture(live_count=2, total=2), + _snap(), + higher_is_better_by_slug={"bench-0": True, "bench-1": False}, + ) + by_slug = {r["slug"]: r["higher_is_better"] for _, r in df.iterrows()} + self.assertEqual(by_slug["bench-0"], True) + self.assertEqual(by_slug["bench-1"], False) + + def test_higher_is_better_null_when_missing(self): + df = build_headlines(_citable_fixture(live_count=1, total=1), _snap()) + self.assertIsNone(df.iloc[0]["higher_is_better"]) + + def test_bench_sample_size_renamed(self): + df = build_headlines(_citable_fixture(), _snap()) + self.assertIn("bench_sample_size", df.columns) + self.assertNotIn("sample_size", df.columns) + + +class ProvidersTests(unittest.TestCase): + def test_one_row_per_provider(self): + df = build_providers([_stat_fixture(n_providers=5)], _snap()) + self.assertEqual(len(df), 5) + + def test_leader_flag(self): + df = build_providers([_stat_fixture(n_providers=3)], _snap()) + # Fixture leader is "mobula" but no provider has that slug, so 0 leaders. + # Sanity: column exists and is boolean dtype. + self.assertIn("is_leader", df.columns) + + def test_leader_flag_matches(self): + stat = _stat_fixture(n_providers=3) + stat["leader"] = {"name": "Provider 0", "slug": "provider-0"} + df = build_providers([stat], _snap()) + self.assertEqual(df[df["is_leader"]]["provider_slug"].tolist(), ["provider-0"]) + + def test_provider_classification_columns(self): + df = build_providers([_stat_fixture(n_providers=3)], _snap()) + for col in ("provider_type", "provider_layer", "provider_tag"): + self.assertIn(col, df.columns) + # First provider has type=aggregator, tag=premium per fixture. + row0 = df[df["provider_slug"] == "provider-0"].iloc[0] + self.assertEqual(row0["provider_type"], "aggregator") + self.assertEqual(row0["provider_tag"], "premium") + + def test_provider_sample_size_renamed(self): + df = build_providers([_stat_fixture(n_providers=2)], _snap()) + self.assertIn("provider_sample_size", df.columns) + self.assertNotIn("sample_size", df.columns) + + +class TimeseriesTests(unittest.TestCase): + def test_one_row_per_point_per_provider_per_window(self): + series = {"bench-0": _series_fixture(n_providers=2)} + df = build_timeseries([_stat_fixture()], series, _snap()) + # 24h: 72 * 2, 7d: 84 * 2, 30d: 60 * 2 = 432 + self.assertEqual(len(df), (72 + 84 + 60) * 2) + + def test_windows_emitted(self): + series = {"bench-0": _series_fixture(n_providers=1)} + df = build_timeseries([_stat_fixture()], series, _snap()) + self.assertEqual(set(df["window"].unique()), {"24h", "7d", "30d"}) + + def test_skips_nulls(self): + series = { + "bench-0": { + "24h": { + "providers": [ + {"slug": "p", "values": [1.0, None, 3.0, None, 5.0]} + ] + } + } + } + df = build_timeseries([_stat_fixture()], series, _snap()) + self.assertEqual(len(df), 3) + + def test_fallback_to_sparkline_when_no_series(self): + # When /api/series returned no payloads, fall back to /api/stat sparkline + # for the 24h window (legacy behaviour). + df = build_timeseries([_stat_fixture(sparkline_len=72)], {"bench-0": {}}, _snap()) + only_24h = df[df["window"] == "24h"] + self.assertEqual(len(only_24h), 72) + + def test_provider_slug_column_present(self): + series = {"bench-0": _series_fixture(n_providers=1)} + df = build_timeseries([_stat_fixture()], series, _snap()) + self.assertIn("provider_slug", df.columns) + + +class ChainLeadersTests(unittest.TestCase): + EXPECTED_COLS = { + "snapshot_date", + "captured_at", + "bench_slug", + "chain", + "leader_name", + "leader_slug", + "leader_value", + "worst_name", + "worst_slug", + "worst_value", + "schema_version", + } + + def test_empty_when_no_chain_dimension(self): + # Stats with no bestPerChain produce zero rows but keep schema. + df = build_chain_leaders([_stat_fixture()], _snap()) + self.assertEqual(len(df), 0) + self.assertEqual(set(df.columns), self.EXPECTED_COLS) + + def test_populates_from_best_and_worst_per_chain(self): + stat = _stat_fixture(slug="bridge-quote-latency") + stat["bestPerChain"] = { + "ethereum": { + "name": "Mobula", + "slug": "mobula", + "ms": {"p50": 280.5, "p90": 600.0, "p99": 1100.0}, + }, + "solana": { + "name": "Codex", + "slug": "codex", + "ms": {"p50": 410.0, "p90": 800.0, "p99": 1500.0}, + }, + } + stat["worstPerChain"] = { + "ethereum": { + "name": "GeckoTerminal", + "slug": "geckoterminal", + "ms": {"p50": 980.0, "p90": 2000.0, "p99": 4000.0}, + }, + } + df = build_chain_leaders([stat], _snap()) + self.assertEqual(len(df), 2) + eth = df[df["chain"] == "ethereum"].iloc[0] + sol = df[df["chain"] == "solana"].iloc[0] + self.assertEqual(eth["leader_slug"], "mobula") + self.assertEqual(eth["worst_slug"], "geckoterminal") + self.assertEqual(eth["leader_value"], 280.5) + self.assertEqual(sol["leader_slug"], "codex") + # Solana has no worst entry: worst columns are null. + self.assertTrue(sol[["worst_name", "worst_slug", "worst_value"]].isna().all()) + + +class PartitioningTests(unittest.TestCase): + def test_write_partition_path_layout(self): + df = build_headlines(_citable_fixture(), _snap()) + with TemporaryDirectory() as tmp: + root = Path(tmp) + path = write_partition(df, root, "headlines", _snap("2026-06-22")) + self.assertTrue(path.exists()) + self.assertTrue( + path.as_posix().endswith( + "headlines/snapshot_date=2026-06-22/part-0.parquet" + ), + msg=path.as_posix(), + ) + + def test_template_substitution(self): + with TemporaryDirectory() as tmp: + tmpl = Path(tmp) / "tmpl" + tmpl.mkdir() + (tmpl / "README.md").write_text("date={{snapshot_date}} v={{schema_version}}") + out = Path(tmp) / "out" + out.mkdir() + stage_static_assets(out, tmpl, _snap("2026-06-22")) + self.assertEqual( + (out / "README.md").read_text(), + f"date=2026-06-22 v={SCHEMA_VERSION}", + ) + + +class SchemaVersionTests(unittest.TestCase): + def test_schema_version_is_v2(self): + # Guard: bumping the schema is intentional, never accidental. + self.assertEqual(SCHEMA_VERSION, 2) + + +KAGGLE_TEMPLATE = """{ + "id": "placeholder/will-be-overridden", + "title": "OCB", + "subtitle": "Daily benchmarks for snap {{snapshot_date}}", + "description": "captured {{captured_at}} schema v{{schema_version}}", + "isPrivate": false, + "licenses": [{"name": "CC-BY-4.0"}], + "keywords": ["crypto"] +} +""" + + +class KaggleMetadataTests(unittest.TestCase): + def _write_template(self, tmp: Path, body: str = KAGGLE_TEMPLATE) -> Path: + (tmp / KAGGLE_METADATA_FILENAME).write_text(body, encoding="utf-8") + return tmp + + def test_substitutes_placeholders_and_overrides_id(self): + with TemporaryDirectory() as raw: + tmpl = self._write_template(Path(raw)) + meta = build_kaggle_metadata( + tmpl, _snap("2026-06-22"), "alice/ocb-bench" + ) + self.assertEqual(meta["id"], "alice/ocb-bench") + self.assertEqual(meta["subtitle"], "Daily benchmarks for snap 2026-06-22") + self.assertIn("2026-06-22T00:00:00+00:00", meta["description"]) + self.assertIn(f"schema v{SCHEMA_VERSION}", meta["description"]) + + def test_default_id_is_openchainbench_benchmarks(self): + with TemporaryDirectory() as raw: + tmpl = self._write_template(Path(raw)) + meta = build_kaggle_metadata(tmpl, _snap()) + self.assertEqual(meta["id"], DEFAULT_KAGGLE_DATASET_ID) + self.assertEqual(DEFAULT_KAGGLE_DATASET_ID, "openchainbench/benchmarks") + + def test_rejects_id_without_slash(self): + with TemporaryDirectory() as raw: + tmpl = self._write_template(Path(raw)) + with self.assertRaises(PublisherError): + build_kaggle_metadata(tmpl, _snap(), "no-owner-here") + + def test_rejects_missing_template(self): + with TemporaryDirectory() as raw: + with self.assertRaises(PublisherError): + build_kaggle_metadata(Path(raw), _snap()) + + def test_rejects_template_without_licenses(self): + bad = '{"id": "x/y", "licenses": []}' + with TemporaryDirectory() as raw: + tmpl = self._write_template(Path(raw), body=bad) + with self.assertRaises(PublisherError): + build_kaggle_metadata(tmpl, _snap()) + + def test_rejects_subtitle_too_short(self): + bad = '{"id": "x/y", "title": "OCB", "subtitle": "too short", "licenses": [{"name": "CC-BY-4.0"}]}' + with TemporaryDirectory() as raw: + tmpl = self._write_template(Path(raw), body=bad) + with self.assertRaises(PublisherError): + build_kaggle_metadata(tmpl, _snap()) + + def test_rejects_subtitle_too_long(self): + long = "x" * 100 + bad = f'{{"id": "x/y", "title": "OCB", "subtitle": "{long}", "licenses": [{{"name": "CC-BY-4.0"}}]}}' + with TemporaryDirectory() as raw: + tmpl = self._write_template(Path(raw), body=bad) + with self.assertRaises(PublisherError): + build_kaggle_metadata(tmpl, _snap()) + + def test_stage_static_assets_skips_kaggle_metadata(self): + # The HF dataset must not carry dataset-metadata.json: it is a + # Kaggle-only artifact and gets written into the staging dir by + # push_to_kaggle, not by stage_static_assets. + with TemporaryDirectory() as raw: + tmpl = Path(raw) / "tmpl" + tmpl.mkdir() + (tmpl / KAGGLE_METADATA_FILENAME).write_text(KAGGLE_TEMPLATE) + (tmpl / "README.md").write_text("hello {{snapshot_date}}") + out = Path(raw) / "out" + out.mkdir() + stage_static_assets(out, tmpl, _snap("2026-06-22")) + self.assertTrue((out / "README.md").exists()) + self.assertFalse((out / KAGGLE_METADATA_FILENAME).exists()) + + def test_real_template_loads(self): + # The shipped dataset-metadata.json must parse and contain a + # valid / id once we override with the default. This + # catches accidental JSON syntax breakage in the template. + repo_template = Path(__file__).parent / "dataset_template" + meta = build_kaggle_metadata(repo_template, _snap()) + self.assertEqual(meta["id"], DEFAULT_KAGGLE_DATASET_ID) + self.assertIn("licenses", meta) + self.assertTrue(meta["licenses"]) + # snapshot_date / captured_at / schema_version must have been + # substituted: no raw `{{` placeholders should remain anywhere. + self.assertNotIn("{{", json.dumps(meta)) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/hf_space/README.md b/scripts/hf_space/README.md new file mode 100644 index 00000000..1af49296 --- /dev/null +++ b/scripts/hf_space/README.md @@ -0,0 +1,52 @@ +--- +title: OpenChainBench Leaderboard +emoji: 📊 +colorFrom: indigo +colorTo: blue +sdk: gradio +sdk_version: 4.44.1 +python_version: "3.11" +app_file: app.py +license: cc-by-4.0 +pinned: false +short_description: Live leaderboard for OpenChainBench benchmarks +--- + +# OpenChainBench leaderboard + +A small Gradio app that reads the daily parquet snapshot from the +[OpenChainBench/benchmarks](https://huggingface.co/datasets/OpenChainBench/benchmarks) +dataset and lets you browse it. Four tabs: today's leaderboard, per-chain +leaders, per-provider rankings, and an about page. + +The dataset itself is the source of truth. This Space is a viewer on top +of it. If you want raw access, query the parquet files directly with +`polars`, `duckdb`, `pandas`, or any tool that speaks parquet. + +```python +import polars as pl + +df = pl.scan_parquet( + "hf://datasets/OpenChainBench/benchmarks/headlines/**/*.parquet", + hive_partitioning=True, +) +print(df.collect().head()) +``` + +For the full website with methodology, per-bench detail pages, and the +historical view, head to [openchainbench.com](https://openchainbench.com). + +## Local dev + +```bash +pip install -r requirements.txt +python app.py +``` + +Open http://127.0.0.1:7860. + +## License + +Code in this Space is part of the OpenChainBench repo. The dataset is +released under CC-BY-4.0. Attribution: link back to openchainbench.com or +the dataset page. diff --git a/scripts/hf_space/app.py b/scripts/hf_space/app.py new file mode 100644 index 00000000..c60a4f59 --- /dev/null +++ b/scripts/hf_space/app.py @@ -0,0 +1,304 @@ +""" +Gradio Space for the OpenChainBench public dataset. + +Loads parquet partitions directly from the HF dataset at +hf://datasets/OpenChainBench/benchmarks via polars, surfaces a +sortable / filterable leaderboard, per-chain leaders, and per-provider +rankings. No local cache, no auth, no state. Each tab refresh re-reads +the latest snapshot from HF, which is cheap because polars only scans +the columns it needs. + +Run locally: + pip install -r requirements.txt + python app.py + +The HF Space picks up `app_file: app.py` from README.md frontmatter. +""" + +from __future__ import annotations + +import functools +import logging +from typing import Any + +import gradio as gr +import polars as pl + +logger = logging.getLogger("ocb_space") +logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s") + +DATASET_REPO = "OpenChainBench/benchmarks" +DATASET_URL = f"https://huggingface.co/datasets/{DATASET_REPO}" +SITE_URL = "https://openchainbench.com" +GITHUB_URL = "https://github.com/ChainBench/OpenChainBench" + +FOOTER = ( + f"Data sourced from {DATASET_URL} (CC-BY-4.0). Updated daily." +) + +# Hive partition layout:
/snapshot_date=YYYY-MM-DD/part-0.parquet. +# Globbing the partitions and reading only the most recent snapshot_date +# keeps the scan small even as the dataset accumulates history. +HF_BASE = f"hf://datasets/{DATASET_REPO}" + + +@functools.lru_cache(maxsize=1) +def latest_snapshot_date() -> str: + """Pick the most recent snapshot_date present in headlines. + + Scans the partition column only, no row data is materialized. Result + is cached for the lifetime of the process so every tab call reuses + the same date. + """ + lf = pl.scan_parquet(f"{HF_BASE}/headlines/**/*.parquet", hive_partitioning=True) + dates = lf.select("snapshot_date").unique().collect() + latest = dates["snapshot_date"].max() + if latest is None: + raise RuntimeError("no snapshots found in headlines/") + logger.info("latest snapshot: %s", latest) + return str(latest) + + +def _read_table(table: str, snapshot: str) -> pl.DataFrame: + lf = pl.scan_parquet( + f"{HF_BASE}/{table}/**/*.parquet", hive_partitioning=True + ).filter(pl.col("snapshot_date") == snapshot) + return lf.collect() + + +@functools.lru_cache(maxsize=1) +def headlines_df() -> pl.DataFrame: + return _read_table("headlines", latest_snapshot_date()) + + +@functools.lru_cache(maxsize=1) +def providers_df() -> pl.DataFrame: + return _read_table("providers", latest_snapshot_date()) + + +@functools.lru_cache(maxsize=1) +def chain_leaders_df() -> pl.DataFrame: + return _read_table("chain_leaders", latest_snapshot_date()) + + +def _categories() -> list[str]: + df = headlines_df() + if "category" not in df.columns: + return ["All"] + cats = sorted({c for c in df["category"].to_list() if c}) + return ["All", *cats] + + +def _bench_slugs() -> list[str]: + df = headlines_df() + return sorted({s for s in df["slug"].to_list() if s}) + + +def _bench_choices_for_chains() -> list[str]: + df = chain_leaders_df() + if df.is_empty(): + return ["All"] + return ["All", *sorted({s for s in df["bench_slug"].to_list() if s})] + + +def _chain_choices() -> list[str]: + df = chain_leaders_df() + if df.is_empty(): + return ["All"] + return ["All", *sorted({s for s in df["chain"].to_list() if s})] + + +def view_headlines(category: str) -> Any: + df = headlines_df() + if category and category != "All": + df = df.filter(pl.col("category") == category) + + # The detail URL pattern on openchainbench.com is /benchmarks/. + # We render the title as a markdown link so clicking opens the page + # in a new tab. + pdf = ( + df.select( + [ + pl.col("title").alias("Bench"), + pl.col("slug"), + pl.col("category").alias("Category"), + pl.col("metric").alias("Metric"), + pl.col("unit").alias("Unit"), + pl.col("leader_name").alias("Leader"), + pl.col("leader_value").alias("Leader value"), + pl.col("bench_sample_size").alias("Sample size"), + pl.col("as_of").alias("As of"), + ] + ) + .sort("Bench") + .to_pandas() + ) + pdf["Bench"] = pdf.apply( + lambda r: f"[{r['Bench']}]({SITE_URL}/benchmarks/{r['slug']})", axis=1 + ) + pdf = pdf.drop(columns=["slug"]) + return pdf + + +def view_chain_leaders(bench: str, chain: str) -> Any: + df = chain_leaders_df() + if df.is_empty(): + return df.to_pandas() + if bench and bench != "All": + df = df.filter(pl.col("bench_slug") == bench) + if chain and chain != "All": + df = df.filter(pl.col("chain") == chain) + return ( + df.select( + [ + pl.col("bench_slug").alias("Bench"), + pl.col("chain").alias("Chain"), + pl.col("leader_name").alias("Leader"), + pl.col("leader_value").alias("Leader value"), + pl.col("worst_name").alias("Worst"), + pl.col("worst_value").alias("Worst value"), + ] + ) + .sort(["Bench", "Chain"]) + .to_pandas() + ) + + +def view_providers(bench: str) -> Any: + df = providers_df() + if not bench: + return df.head(0).to_pandas() + df = df.filter(pl.col("bench_slug") == bench) + return ( + df.select( + [ + pl.col("provider_name").alias("Provider"), + pl.col("provider_type").alias("Type"), + pl.col("p50").alias("p50"), + pl.col("p90").alias("p90"), + pl.col("p99").alias("p99"), + pl.col("success_rate").alias("Success rate"), + pl.col("provider_sample_size").alias("Sample size"), + pl.col("is_leader").alias("Leader?"), + ] + ) + .sort("p50", nulls_last=True) + .to_pandas() + ) + + +ABOUT_MD = f""" +## OpenChainBench + +Public benchmarks for crypto infrastructure: RPCs, oracles, bridges, aggregators, +prediction markets, and more. The full leaderboard, methodology, and per-bench +detail live at [openchainbench.com]({SITE_URL}). + +This Space is a thin viewer over the daily parquet snapshot published to +[{DATASET_REPO}]({DATASET_URL}). Every tab reads directly from the dataset, so +the numbers you see here match the dataset exactly. + +### Links +- Website: [{SITE_URL}]({SITE_URL}) +- Dataset: [{DATASET_URL}]({DATASET_URL}) +- GitHub: [{GITHUB_URL}]({GITHUB_URL}) + +### License + +The dataset is released under **CC-BY-4.0**. Attribution required: link +back to {SITE_URL} or the dataset page. + +### Citation + +```bibtex +@misc{{openchainbench2026, + title = {{OpenChainBench: Public benchmarks for crypto infrastructure}}, + author = {{OpenChainBench contributors}}, + year = {{2026}}, + url = {{{DATASET_URL}}}, + note = {{CC-BY-4.0}} +}} +``` +""" + + +def build_app() -> gr.Blocks: + snapshot = latest_snapshot_date() + title = f"OpenChainBench leaderboard ({snapshot})" + + with gr.Blocks(title=title, theme=gr.themes.Soft()) as demo: + gr.Markdown(f"# {title}") + gr.Markdown( + "Sortable view of the daily snapshot. Click a bench title to open " + f"its page on {SITE_URL}." + ) + + with gr.Tabs(): + with gr.Tab("Today's leaderboard"): + cat = gr.Dropdown( + choices=_categories(), + value="All", + label="Category", + ) + table = gr.Dataframe( + value=view_headlines("All"), + interactive=False, + wrap=True, + datatype=["markdown", "str", "str", "str", "str", "number", "number", "str"], + ) + cat.change(view_headlines, inputs=cat, outputs=table) + + with gr.Tab("Per-chain leaders"): + with gr.Row(): + bench_dd = gr.Dropdown( + choices=_bench_choices_for_chains(), + value="All", + label="Bench", + ) + chain_dd = gr.Dropdown( + choices=_chain_choices(), + value="All", + label="Chain", + ) + chains_table = gr.Dataframe( + value=view_chain_leaders("All", "All"), + interactive=False, + wrap=True, + ) + bench_dd.change( + view_chain_leaders, + inputs=[bench_dd, chain_dd], + outputs=chains_table, + ) + chain_dd.change( + view_chain_leaders, + inputs=[bench_dd, chain_dd], + outputs=chains_table, + ) + + with gr.Tab("Provider rankings"): + slugs = _bench_slugs() + default_slug = slugs[0] if slugs else None + prov_dd = gr.Dropdown( + choices=slugs, + value=default_slug, + label="Bench slug", + ) + prov_table = gr.Dataframe( + value=view_providers(default_slug) if default_slug else None, + interactive=False, + wrap=True, + ) + prov_dd.change(view_providers, inputs=prov_dd, outputs=prov_table) + + with gr.Tab("About"): + gr.Markdown(ABOUT_MD) + + gr.Markdown(f"---\n{FOOTER}") + + return demo + + +if __name__ == "__main__": + app = build_app() + app.launch(server_name="0.0.0.0", server_port=7860) diff --git a/scripts/hf_space/requirements.txt b/scripts/hf_space/requirements.txt new file mode 100644 index 00000000..dfbe0feb --- /dev/null +++ b/scripts/hf_space/requirements.txt @@ -0,0 +1,10 @@ +gradio==4.44.1 +# gradio 4.44.1 leaves starlette/fastapi unpinned and recent versions +# break the Jinja2 template path with a TypeError at app boot. Pin to +# versions known to work with gradio 4.44.x. +starlette<0.42 +fastapi<0.116 +polars>=1.41.0 +pandas==2.2.3 +pyarrow==17.0.0 +huggingface_hub==0.26.2 diff --git a/src/app/about/page.tsx b/src/app/about/page.tsx index f1402786..bbc27c9f 100644 --- a/src/app/about/page.tsx +++ b/src/app/about/page.tsx @@ -47,7 +47,7 @@ export default function AboutPage() {
  • ·Bridges. quote latency and effective fee on $300 USDC corridors. Mobula, Relay, LiFi, Debridge.
  • ·Blockchains. L1 finality time across 11 chains. L2 sequencer block time across 9 L2s.
  • ·Trading venues. all-in opening cost on perp DEXes (Hyperliquid, Lighter, dYdX, GMX, gains). Stablecoin peg deviation on USDC, USDT, DAI + USDT-anchored swap-cost on FDUSD, USDe.
    • -
  • ·RPCs. fastest free no-key public RPC across 10 EVM chains × 15 providers (PublicNode, dRPC, 1RPC, Tenderly, Nodies, Lava, Merkle, MeowRPC, Flashbots, Cloudflare, foundation endpoints). Plus gas-oracle prediction error (Blocknative, Owlracle, Etherscan, PublicNode feeHistory) on Ethereum, Polygon, Avalanche.
    • +
  • ·RPCs. fastest free no-key public RPC across 10 EVM chains × 15 providers (PublicNode, dRPC, 1RPC, Tenderly, Nodies, Lava, Merkle, MeowRPC, Flashbots, Cloudflare, foundation endpoints). Plus gas-oracle prediction error (Owlracle, Etherscan, PublicNode feeHistory) on Ethereum, Polygon, Avalanche.

    • 13 live benchmarks. ~150 (provider × chain) probe pairs. Every metric is queryable on the public Prometheus and reproducible from the harness source. diff --git a/src/app/answers/[slug]/page.tsx b/src/app/answers/[slug]/page.tsx index 25a26b26..f33a21a7 100644 --- a/src/app/answers/[slug]/page.tsx +++ b/src/app/answers/[slug]/page.tsx @@ -43,10 +43,10 @@ export async function generateMetadata({ const url = `${SITE.url}/answers/${ans.slug}`; const title = ans.seo_title ?? ans.question; const descSource = ans.seo_description ?? ans.short_answer; - // Clean leftover tokens AFTER renderTemplate so a draft bench - // (e.g. solana-tx-landing-latency mid-soak) never leaks a literal - // `{{best_name}}` into the meta description, og:description or - // twitter:description, all of which feed the SERP and social previews. + // Clean leftover tokens AFTER renderTemplate so a draft bench never + // leaks a literal `{{best_name}}` into the meta description, + // og:description or twitter:description, all of which feed the SERP + // and social previews. const description = capDescription( cleanLeftoverTokens(renderTemplate(descSource, ans.bench)), 158, diff --git a/src/app/answers/page.tsx b/src/app/answers/page.tsx index 2b3e1451..653ce8de 100644 --- a/src/app/answers/page.tsx +++ b/src/app/answers/page.tsx @@ -28,10 +28,9 @@ export default async function AnswersHubPage() { // before the JSX touches the string. // // Tokens that renderTemplate can't resolve get a neutral fallback so - // a draft / awaiting-data bench (e.g. solana-tx-landing-latency mid-soak - // with every provider's p50 still at 0) never surfaces raw `{{best_name}}` - // to the SERP. Same pattern as resolveLeftoverPlaceholders on the - // per-chain bench page. + // a draft / awaiting-data bench (every provider's p50 still at 0) + // never surfaces raw `{{best_name}}` to the SERP. Same pattern as + // resolveLeftoverPlaceholders on the per-chain bench page. const rendered = await Promise.all( answers.map(async (a) => { const bench = await loadBenchmark(a.benchmark, { chain: a.chain }); diff --git a/src/app/api/badge/[slug]/[provider]/route.ts b/src/app/api/badge/[slug]/[provider]/route.ts index 5f29beed..08bd7f51 100644 --- a/src/app/api/badge/[slug]/[provider]/route.ts +++ b/src/app/api/badge/[slug]/[provider]/route.ts @@ -158,7 +158,7 @@ export async function GET( req: NextRequest, { params }: { params: Promise }, ) { - const rl = rateLimit(clientKey(req, "badge"), 120, 60); + const rl = rateLimit(clientKey(req, "badge"), 120, 60, req); if (!rl.ok) return tooManyRequests(rl.retryAfterSec); const { slug, provider } = await params; diff --git a/src/app/api/badge/[slug]/[provider]/snippet/route.ts b/src/app/api/badge/[slug]/[provider]/snippet/route.ts new file mode 100644 index 00000000..70625997 --- /dev/null +++ b/src/app/api/badge/[slug]/[provider]/snippet/route.ts @@ -0,0 +1,113 @@ +/** + * Copy-paste embed snippets for the per-(benchmark, provider) badge SVG. + * + * GET /api/badge///snippet?format=markdown|html|url|json + * + * Returns a ready-to-paste embed code so a provider can drop a live + * "Ranked #N on OpenChainBench" badge into their README, docs page, + * or marketing site without crafting the URL by hand. The badge SVG + * itself still lives at /api/badge//; this endpoint + * only wraps it. + * + * Why this exists separately from the SVG route: it is the one + * surface readers reach for when they want to BACKLINK us. Keeping + * snippet rendering out of the SVG path keeps the SVG cache hot + * (one cacheable shape per benchmark + provider) and avoids polluting + * the SVG content negotiation with a text/* branch. + * + * Optional query params forwarded to the badge URL so a provider can + * embed a scope-restricted badge (chain, region, kind). The site URL + * the badge links to also picks up the same scope where applicable, + * so a reader clicking through lands on the matching variant view. + */ + +import { type NextRequest, NextResponse } from "next/server"; +import { getBenchmark } from "@/data/benchmarks"; +import { clientKey, rateLimit, tooManyRequests } from "@/lib/rate-limit"; +import { PROVIDER_RE, SLUG_RE } from "@/lib/slug"; +import { SITE } from "@/data/site"; + +export const revalidate = 600; + +type Params = { slug: string; provider: string }; + +const FORMATS = ["markdown", "html", "url", "json"] as const; +type Format = (typeof FORMATS)[number]; + +function isFormat(v: string | null): v is Format { + return v != null && (FORMATS as readonly string[]).includes(v); +} + +export async function GET( + req: NextRequest, + { params }: { params: Promise }, +) { + const r = rateLimit(clientKey(req, "badge-snippet"), 120, 60, req); + if (!r.ok) return tooManyRequests(r.retryAfterSec); + + const { slug, provider } = await params; + if (!SLUG_RE.test(slug) || !PROVIDER_RE.test(provider)) { + return NextResponse.json({ error: "invalid_slug" }, { status: 400 }); + } + + const benchmark = await getBenchmark(slug); + if (!benchmark) { + return NextResponse.json({ error: "bench_not_found" }, { status: 404 }); + } + const result = benchmark.results.find((p) => p.slug === provider); + if (!result) { + return NextResponse.json({ error: "provider_not_found" }, { status: 404 }); + } + + const sp = req.nextUrl.searchParams; + const format: Format = isFormat(sp.get("format")) ? (sp.get("format") as Format) : "markdown"; + const chain = sp.get("chain")?.trim() || ""; + const region = sp.get("region")?.trim() || ""; + const kind = sp.get("kind")?.trim() || ""; + + const scopeQs = new URLSearchParams(); + if (chain) scopeQs.set("chain", chain); + if (region) scopeQs.set("region", region); + if (kind) scopeQs.set("kind", kind); + const scopeSuffix = scopeQs.toString(); + + const badgeUrl = + `${SITE.url}/api/badge/${slug}/${provider}` + + (scopeSuffix ? `?${scopeSuffix}` : ""); + const benchUrl = + `${SITE.url}/benchmarks/${slug}` + + (scopeSuffix ? `?${scopeSuffix}` : ""); + + const alt = `OpenChainBench ${benchmark.title} ranking for ${result.name}`; + + const snippets = { + markdown: `[![${alt}](${badgeUrl})](${benchUrl})`, + html: `${alt}`, + url: badgeUrl, + } as const; + + if (format === "json") { + return NextResponse.json( + { + benchmark: { slug, title: benchmark.title, url: benchUrl }, + provider: { slug: provider, name: result.name }, + badge_url: badgeUrl, + snippets, + scope: { chain: chain || null, region: region || null, kind: kind || null }, + license: "CC-BY-4.0", + }, + { + headers: { + "cache-control": "public, s-maxage=300, stale-while-revalidate=600", + }, + }, + ); + } + + return new NextResponse(snippets[format], { + headers: { + "content-type": "text/plain; charset=utf-8", + "cache-control": "public, s-maxage=300, stale-while-revalidate=600", + }, + }); +} diff --git a/src/app/api/bench/[slug]/oracle-pairs/route.ts b/src/app/api/bench/[slug]/oracle-pairs/route.ts index f5fb1c38..0b4c6552 100644 --- a/src/app/api/bench/[slug]/oracle-pairs/route.ts +++ b/src/app/api/bench/[slug]/oracle-pairs/route.ts @@ -42,7 +42,7 @@ export async function GET( req: Request, { params }: { params: Promise }, ) { - const r = rateLimit(clientKey(req, "oracle-pairs"), 60, 60); + const r = rateLimit(clientKey(req, "oracle-pairs"), 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); const { slug } = await params; diff --git a/src/app/api/bench/[slug]/variant/route.ts b/src/app/api/bench/[slug]/variant/route.ts index 7a9e9a08..37bcabe8 100644 --- a/src/app/api/bench/[slug]/variant/route.ts +++ b/src/app/api/bench/[slug]/variant/route.ts @@ -24,7 +24,7 @@ export async function GET( req: NextRequest, { params }: { params: Promise }, ) { - const rl = rateLimit(clientKey(req, "variant"), 120, 60); + const rl = rateLimit(clientKey(req, "variant"), 120, 60, req); if (!rl.ok) return tooManyRequests(rl.retryAfterSec); const { slug } = await params; diff --git a/src/app/api/builder/[slug]/daily-series/route.ts b/src/app/api/builder/[slug]/daily-series/route.ts index 3cce38a0..cdabb1b4 100644 --- a/src/app/api/builder/[slug]/daily-series/route.ts +++ b/src/app/api/builder/[slug]/daily-series/route.ts @@ -26,7 +26,7 @@ export async function GET( req: Request, { params }: { params: Promise }, ) { - const r = rateLimit(clientKey(req, "hl-daily-series"), 60, 60); + const r = rateLimit(clientKey(req, "hl-daily-series"), 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); const { slug } = await params; diff --git a/src/app/api/builder/[slug]/top-users/route.ts b/src/app/api/builder/[slug]/top-users/route.ts index f1569394..38ed6429 100644 --- a/src/app/api/builder/[slug]/top-users/route.ts +++ b/src/app/api/builder/[slug]/top-users/route.ts @@ -23,7 +23,7 @@ export async function GET( req: Request, { params }: { params: Promise }, ) { - const r = rateLimit(clientKey(req, "hl-top-users"), 60, 60); + const r = rateLimit(clientKey(req, "hl-top-users"), 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); const { slug } = await params; diff --git a/src/app/api/chain/[slug]/live-prices/route.ts b/src/app/api/chain/[slug]/live-prices/route.ts index c5de8430..bbce1c7b 100644 --- a/src/app/api/chain/[slug]/live-prices/route.ts +++ b/src/app/api/chain/[slug]/live-prices/route.ts @@ -77,7 +77,7 @@ export async function GET( req: Request, { params }: { params: Promise<{ slug: string }> }, ) { - const r = rateLimit(clientKey(req, "chain-kpis-live"), 120, 60); + const r = rateLimit(clientKey(req, "chain-kpis-live"), 120, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); const { slug } = await params; diff --git a/src/app/api/citable/route.ts b/src/app/api/citable/route.ts index 251fde3f..03d6b55c 100644 --- a/src/app/api/citable/route.ts +++ b/src/app/api/citable/route.ts @@ -1,37 +1,72 @@ import { NextResponse } from "next/server"; import { getBenchmarks } from "@/data/benchmarks"; import { SITE } from "@/data/site"; -import { fieldValue, leader, headlineSentence } from "@/lib/citation"; +import { AllBenchmarksDraftError } from "@/lib/spec"; +import { + fieldValue, + headlineSentence, + isInsufficient, + leader, +} from "@/lib/citation"; import { clientKey, rateLimit, tooManyRequests } from "@/lib/rate-limit"; export const runtime = "nodejs"; export const revalidate = 60; +/** Short 503 with a Retry-After hint, served when the aggregator has + * no live snapshot to surface (Prom blackout + cold KV). Beats serving + * an all-draft index that downstream LLM agents would treat as truth. */ +function unavailable(): NextResponse { + return NextResponse.json( + { error: "benchmarks_unavailable", retryAfterSec: 60 }, + { + status: 503, + headers: { + "cache-control": "no-store", + "retry-after": "60", + "access-control-allow-origin": "*", + }, + }, + ); +} + /** * Flat machine-readable index of every citable benchmark. Designed to be - * the **first** endpoint an AI agent or journalist crawls - gives them + * the **first** endpoint an AI agent or journalist crawls. Gives them * everything they need to decide whether to deep-link to a specific bench. * * License is intentionally surfaced per-row so downstream agents can * cite without needing to read the footer of every page. */ export async function GET(req: Request) { - const r = rateLimit(clientKey(req, "citable"), 60, 60); + const r = rateLimit(clientKey(req, "citable"), 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); - const benches = (await getBenchmarks()).filter((b) => b.editorialStatus === "live"); + let benches; + try { + benches = (await getBenchmarks()).filter( + (b) => b.editorialStatus === "live", + ); + } catch (err) { + if (err instanceof AllBenchmarksDraftError) return unavailable(); + throw err; + } const data = benches.map((b) => { - const top = leader(b); + const insufficient = isInsufficient(b); + const top = insufficient ? null : leader(b); + const status: "live" | "draft" | "insufficient" = insufficient + ? "insufficient" + : b.status; return { slug: b.slug, title: b.title, category: b.category, metric: b.metric, unit: b.unit, - status: b.status, - value: fieldValue(b), + status, + value: insufficient ? null : fieldValue(b), leader: top ? { name: top.name, slug: top.slug, value: top.value } : null, - sampleSize: b.sampleSize, + sampleSize: insufficient ? 0 : b.sampleSize, asOf: b.lastRunAt, headline: headlineSentence(b), url: `${SITE.url}/benchmarks/${b.slug}`, @@ -50,7 +85,7 @@ export async function GET(req: Request) { }, { headers: { - "cache-control": "public, s-maxage=60, stale-while-revalidate=300", + "cache-control": "public, s-maxage=300, stale-while-revalidate=900", "access-control-allow-origin": "*", }, }, diff --git a/src/app/api/cron/health-check/route.ts b/src/app/api/cron/health-check/route.ts index 03ad2274..fcd930cf 100644 --- a/src/app/api/cron/health-check/route.ts +++ b/src/app/api/cron/health-check/route.ts @@ -240,7 +240,7 @@ export async function GET(req: NextRequest) { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify({ - text: `🔴 OCB materialize worker heartbeat is stale (${ageTxt}) — pages are on the slow live fallback. Check the Railway service ocb-materialize-worker.`, + text: `🔴 OCB materialize worker heartbeat is stale (${ageTxt}). Pages are on the slow live fallback. Check the Railway service ocb-materialize-worker.`, }), }).catch((err) => console.error("slack heartbeat alert failed:", err)); } diff --git a/src/app/api/cron/snapshot-hl-cohort/route.ts b/src/app/api/cron/snapshot-hl-cohort/route.ts new file mode 100644 index 00000000..a51835bd --- /dev/null +++ b/src/app/api/cron/snapshot-hl-cohort/route.ts @@ -0,0 +1,111 @@ +import { timingSafeEqual } from "node:crypto"; +import { NextResponse, type NextRequest } from "next/server"; +import { + fetchHlCohortFresh, + fetchHlHip3CohortFresh, +} from "@/lib/hl-builder-stats"; +import { + cohortSnapshotConfigured, + writeCohortSnapshot, +} from "@/lib/cohort-snapshot"; + +export const runtime = "nodejs"; +export const dynamic = "force-dynamic"; + +/** + * Vercel cron: refresh the /hyperliquid hub's two cohort snapshots in + * Upstash. Single endpoint that updates both keys (hl-frontends, hl-hip3) + * so a single cron entry covers the whole hub. + * + * Runs every minute. Bypasses the snapshot-first readers (which would + * loop back to their own blob); calls the *Fresh helpers directly so the + * write reflects live Prom state. Token-gated by CRON_SECRET. When the + * Upstash creds are unset the route 200s with `{configured: false}` so + * an unprovisioned environment doesn't break the cron schedule. + * + * Per-cohort errors are isolated: a Prom miss on HIP-3 doesn't block a + * fresh frontends write, and vice versa. The response body lists the + * outcome of each key so the Vercel cron log shows partial recoveries. + */ + +function isAuthorized(req: NextRequest): boolean { + const secret = (process.env.CRON_SECRET ?? "").trim(); + const header = (req.headers.get("authorization") ?? "").trim(); + if (!secret) { + return process.env.NODE_ENV !== "production"; + } + const expected = Buffer.from(`Bearer ${secret}`); + const provided = Buffer.from(header); + if (provided.length !== expected.length) return false; + return timingSafeEqual(provided, expected); +} + +type KeyOutcome = + | { key: string; ok: true; asOf: number; rowCount: number } + | { key: string; ok: false; error: string }; + +async function refresh( + key: string, + fetcher: () => Promise, +): Promise { + let result: T | null; + try { + result = await fetcher(); + } catch (err) { + return { + key, + ok: false, + error: `fetch: ${err instanceof Error ? err.message : String(err)}`, + }; + } + if (!result) { + return { + key, + ok: false, + error: "fetch returned null (prom unreachable or empty)", + }; + } + try { + await writeCohortSnapshot(key, result); + } catch (err) { + return { + key, + ok: false, + error: `write: ${err instanceof Error ? err.message : String(err)}`, + }; + } + return { key, ok: true, asOf: result.asOf, rowCount: result.rows.length }; +} + +export async function GET(req: NextRequest) { + if (!isAuthorized(req)) { + return NextResponse.json({ error: "unauthorized" }, { status: 401 }); + } + if (!cohortSnapshotConfigured()) { + return NextResponse.json( + { + ok: true, + configured: false, + message: + "cohort snapshot store not configured (KV_REST_API_URL / UPSTASH_REDIS_REST_URL absent)", + }, + { status: 200 }, + ); + } + + const startedAt = Date.now(); + const [frontends, hip3] = await Promise.all([ + refresh("hl-frontends", fetchHlCohortFresh), + refresh("hl-hip3", fetchHlHip3CohortFresh), + ]); + + const okCount = (frontends.ok ? 1 : 0) + (hip3.ok ? 1 : 0); + return NextResponse.json( + { + ok: okCount > 0, + results: [frontends, hip3], + durationMs: Date.now() - startedAt, + }, + { status: okCount === 0 ? 502 : 200 }, + ); +} diff --git a/src/app/api/cron/snapshot-perp-cohort/route.ts b/src/app/api/cron/snapshot-perp-cohort/route.ts new file mode 100644 index 00000000..290ace1f --- /dev/null +++ b/src/app/api/cron/snapshot-perp-cohort/route.ts @@ -0,0 +1,111 @@ +import { timingSafeEqual } from "node:crypto"; +import { NextResponse, type NextRequest } from "next/server"; +import { fetchPerpCohortFresh } from "@/lib/perp-stats"; +import { + cohortSnapshotConfigured, + writeCohortSnapshot, +} from "@/lib/cohort-snapshot"; + +export const runtime = "nodejs"; +// No ISR. The cron's whole job is to refresh the cohort blob: a cached +// 200 from a previous run would silently skip the Prom call. +export const dynamic = "force-dynamic"; + +/** + * Vercel cron: refresh the /perps hub cohort snapshot in Upstash. + * + * Runs every minute (vercel.json crons block). Fetches the cohort straight + * from Prom (bypassing the snapshot-first reader in fetchPerpCohort so the + * cron never loops on its own blob) and SETs the result under + * ocb:cohort:perp-cohort:v1 with a 24 h safety-net TTL. + * + * Token-gated by CRON_SECRET (Bearer header). When Upstash creds are + * missing the route still 200s with `{configured: false}` so the cron + * schedule keeps working before the integration is provisioned. + */ + +function isAuthorized(req: NextRequest): boolean { + // Trim both sides. The vercel UI / `vercel env add` paste flow has + // historically appended a trailing newline that produced a constant + // 401 with no visible reason. + const secret = (process.env.CRON_SECRET ?? "").trim(); + const header = (req.headers.get("authorization") ?? "").trim(); + if (!secret) { + // Fail closed in prod, permissive in dev to keep local manual hits + // working without exporting a fake secret. + return process.env.NODE_ENV !== "production"; + } + const expected = Buffer.from(`Bearer ${secret}`); + const provided = Buffer.from(header); + if (provided.length !== expected.length) return false; + return timingSafeEqual(provided, expected); +} + +export async function GET(req: NextRequest) { + if (!isAuthorized(req)) { + return NextResponse.json({ error: "unauthorized" }, { status: 401 }); + } + if (!cohortSnapshotConfigured()) { + return NextResponse.json( + { + ok: true, + configured: false, + message: + "cohort snapshot store not configured (KV_REST_API_URL / UPSTASH_REDIS_REST_URL absent)", + }, + { status: 200 }, + ); + } + + const startedAt = Date.now(); + let result: Awaited>; + try { + result = await fetchPerpCohortFresh(); + } catch (err) { + return NextResponse.json( + { + ok: false, + stage: "fetch", + error: err instanceof Error ? err.message : String(err), + }, + { status: 502 }, + ); + } + + if (!result) { + // Prom unreachable or completely empty. Do NOT write null over the + // existing blob: the reader's stale-tolerance window would surface + // the null, but we'd rather the reader fall through to its own live + // path and keep the previous (still-valid-for-now) snapshot until + // either the cron or a request restores live data. + return NextResponse.json( + { + ok: false, + stage: "fetch", + error: "fetchPerpCohortFresh returned null (prom unreachable or empty)", + }, + { status: 502 }, + ); + } + + try { + await writeCohortSnapshot("perp-cohort", result); + } catch (err) { + return NextResponse.json( + { + ok: false, + stage: "write", + error: err instanceof Error ? err.message : String(err), + }, + { status: 502 }, + ); + } + + return NextResponse.json({ + ok: true, + asOf: result.asOf, + venueCount: result.venues.length, + trackedVenues: result.totals.trackedVenues, + durationMs: Date.now() - startedAt, + }); +} diff --git a/src/app/api/cron/warm-search-featured/route.ts b/src/app/api/cron/warm-search-featured/route.ts new file mode 100644 index 00000000..2c3309c2 --- /dev/null +++ b/src/app/api/cron/warm-search-featured/route.ts @@ -0,0 +1,90 @@ +import { timingSafeEqual } from "node:crypto"; +import { NextResponse, type NextRequest } from "next/server"; +import { + cohortSnapshotConfigured, + writeCohortSnapshot, +} from "@/lib/cohort-snapshot"; +import { buildFeaturedLeaders } from "@/lib/search-featured"; + +export const runtime = "nodejs"; +// No ISR — the cron's whole job is to refresh the blob. A cached 200 from +// a previous run would silently skip the rebuild. +export const dynamic = "force-dynamic"; + +/** + * Vercel cron: refresh the search dialog's "Live leaders" + "Trending" + * blob in Upstash. Runs every minute (vercel.json crons block). + * + * Why a dedicated blob: the search dialog used to fetch /api/citable on + * every open (the full ~30-bench citable index, ~50 KB, plus all the + * assembly cost server-side). Now the cron pre-computes the 12-card + * subset the dialog actually needs (~2 KB), the public endpoint becomes + * one KV GET, and the dialog opens with data already prefetched at page + * load. + * + * Same token gate + soft-no-op pattern as snapshot-perp-cohort. + */ + +function isAuthorized(req: NextRequest): boolean { + const secret = (process.env.CRON_SECRET ?? "").trim(); + const header = (req.headers.get("authorization") ?? "").trim(); + if (!secret) { + return process.env.NODE_ENV !== "production"; + } + const expected = Buffer.from(`Bearer ${secret}`); + const provided = Buffer.from(header); + if (provided.length !== expected.length) return false; + return timingSafeEqual(provided, expected); +} + +export async function GET(req: NextRequest) { + if (!isAuthorized(req)) { + return NextResponse.json({ error: "unauthorized" }, { status: 401 }); + } + if (!cohortSnapshotConfigured()) { + return NextResponse.json( + { + ok: true, + configured: false, + message: + "cohort snapshot store not configured (KV_REST_API_URL / UPSTASH_REDIS_REST_URL absent)", + }, + { status: 200 }, + ); + } + + const startedAt = Date.now(); + let blob; + try { + blob = await buildFeaturedLeaders(); + } catch (err) { + return NextResponse.json( + { + ok: false, + stage: "build", + error: err instanceof Error ? err.message : String(err), + }, + { status: 502 }, + ); + } + + try { + await writeCohortSnapshot("search-featured", blob); + } catch (err) { + return NextResponse.json( + { + ok: false, + stage: "write", + error: err instanceof Error ? err.message : String(err), + }, + { status: 502 }, + ); + } + + return NextResponse.json({ + ok: true, + featuredCount: blob.featured.length, + trendingCount: blob.trending.length, + durationMs: Date.now() - startedAt, + }); +} diff --git a/src/app/api/freshness/route.ts b/src/app/api/freshness/route.ts index ca2fded8..b9cd4c4f 100644 --- a/src/app/api/freshness/route.ts +++ b/src/app/api/freshness/route.ts @@ -67,11 +67,11 @@ const computeFreshness = unstable_cache( return { now: Date.now(), freshness }; }, ["freshness-v2"], - { revalidate: 2, tags: ["benchmarks", "freshness"] }, + { revalidate: 30, tags: ["benchmarks", "freshness"] }, ); export async function GET(req: Request) { - const r = rateLimit(clientKey(req, "freshness"), 120, 60); + const r = rateLimit(clientKey(req, "freshness"), 120, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); // Resolve the spec list outside the cached function so its slug list @@ -84,10 +84,12 @@ export async function GET(req: Request) { const data = await computeFreshness(sortedLiveSlugs); return Response.json(data, { headers: { - // 2 s s-maxage matches the unstable_cache window above. Short swr - // because anything beyond a few seconds produces a stale asOf that - // would defeat the point of the polling counter. - "cache-control": "public, s-maxage=2, stale-while-revalidate=4", + // 30s s-maxage matches the unstable_cache window above and the + // Prom scrape floor (~15s). LiveIndicator polls at the same + // cadence; client-side counter still ticks every 1s for UX. + // Egress reduction: previously s-maxage=2 produced near-100% MISS + // rate on Vercel edge, sending every poll to Railway prom-gateway. + "cache-control": "public, s-maxage=30, stale-while-revalidate=60, max-age=30", "access-control-allow-origin": "*", }, }); diff --git a/src/app/api/llm-context/route.ts b/src/app/api/llm-context/route.ts index d4414fb1..c6617f49 100644 --- a/src/app/api/llm-context/route.ts +++ b/src/app/api/llm-context/route.ts @@ -1,7 +1,13 @@ import { getBenchmarks } from "@/data/benchmarks"; import { SITE } from "@/data/site"; +import { AllBenchmarksDraftError } from "@/lib/spec"; import { fmtUnit } from "@/lib/format"; -import { fieldValue, headlineSentence, leader } from "@/lib/citation"; +import { + fieldValue, + headlineSentence, + isInsufficient, + leader, +} from "@/lib/citation"; import { clientKey, rateLimit, tooManyRequests } from "@/lib/rate-limit"; export const runtime = "nodejs"; @@ -20,13 +26,31 @@ export const revalidate = 60; * per-region breakdown) but covers all 8 benches in one round-trip. */ export async function GET(req: Request) { - const r = rateLimit(clientKey(req, "llm-context"), 30, 60); + const r = rateLimit(clientKey(req, "llm-context"), 30, 60, req); if (!r.ok) { const tooMany = tooManyRequests(r.retryAfterSec); return new Response(await tooMany.text(), { status: tooMany.status, headers: tooMany.headers }); } - const benches = (await getBenchmarks()).filter((b) => b.editorialStatus === "live"); + let benches; + try { + benches = (await getBenchmarks()).filter( + (b) => b.editorialStatus === "live", + ); + } catch (err) { + if (err instanceof AllBenchmarksDraftError) { + return new Response("benchmarks_unavailable\n", { + status: 503, + headers: { + "content-type": "text/plain; charset=utf-8", + "cache-control": "no-store", + "retry-after": "60", + "access-control-allow-origin": "*", + }, + }); + } + throw err; + } const now = new Date().toISOString(); const lines: string[] = []; @@ -48,11 +72,15 @@ export async function GET(req: Request) { lines.push(`- Metric: ${b.metric} (${b.unit})`); lines.push(`- Page: ${SITE.url}/benchmarks/${b.slug}`); lines.push(`- JSON: ${SITE.url}/api/stat/${b.slug}`); - lines.push(`- Status: ${b.status}`); + const insufficient = isInsufficient(b); + const reportedStatus: "live" | "draft" | "insufficient" = insufficient + ? "insufficient" + : b.status; + lines.push(`- Status: ${reportedStatus}`); const v = fieldValue(b); const lead = leader(b); - if (v != null && lead) { + if (!insufficient && v != null && lead) { lines.push(`- Headline: ${headlineSentence(b)}`); lines.push(""); lines.push(`**Rankings (p50, 24h):**`); @@ -70,6 +98,12 @@ export async function GET(req: Request) { )}, success ${r.successRate.toFixed(1)}%, sample ${r.sampleSize ?? "n/a"})`, ); } + } else if (insufficient) { + // Surface the same insufficient sentence the other citable surfaces + // emit, so an LLM that pastes this Markdown into context never sees + // a fabricated winner for a bench whose harness lacks data. + lines.push(`- Headline: ${headlineSentence(b)}`); + lines.push(`- Insufficient samples to rank providers yet.`); } else { lines.push(`- ${b.status === "draft" ? "Draft (no live data yet)" : "Awaiting samples"}.`); } @@ -93,7 +127,7 @@ export async function GET(req: Request) { status: 200, headers: { "content-type": "text/markdown; charset=utf-8", - "cache-control": "public, s-maxage=60, stale-while-revalidate=300", + "cache-control": "public, s-maxage=300, stale-while-revalidate=900", "access-control-allow-origin": "*", }, }); diff --git a/src/app/api/mcp/[transport]/route.ts b/src/app/api/mcp/[transport]/route.ts index 742010aa..809e4978 100644 --- a/src/app/api/mcp/[transport]/route.ts +++ b/src/app/api/mcp/[transport]/route.ts @@ -7,6 +7,7 @@ import { citationQuote, fieldValue, headlineSentence, + isInsufficient, leader, sparklineFor, } from "@/lib/citation"; @@ -208,15 +209,19 @@ const mcpHandler = createMcpHandler( async () => { const benches = (await getBenchmarks()).filter((b) => b.editorialStatus === "live"); const rows = benches.map((b) => { - const top = leader(b); + const insufficient = isInsufficient(b); + const top = insufficient ? null : leader(b); + const status: "live" | "draft" | "insufficient" = insufficient + ? "insufficient" + : b.status; return { slug: b.slug, title: b.title, category: b.category, metric: b.metric, unit: b.unit, - status: b.status, - value: fieldValue(b), + status, + value: insufficient ? null : fieldValue(b), leader: top, headline: headlineSentence(b), url: `${SITE.url}/benchmarks/${b.slug}`, @@ -279,25 +284,39 @@ const mcpHandler = createMcpHandler( isError: true, }; } - const top = leader(b); + const insufficient = isInsufficient(b); + const top = insufficient ? null : leader(b); + const status: "live" | "draft" | "insufficient" = insufficient + ? "insufficient" + : b.status; + const rankings = insufficient + ? b.results.map((r) => ({ + name: r.name, + slug: r.slug, + ms: { p50: null, p90: null, p99: null, mean: null }, + successRate: r.successRate, + })) + : b.results + .filter((r) => r.ms.p50 > 0) + .sort((a, c) => + b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, + ) + .map((r) => ({ + name: r.name, + slug: r.slug, + ms: r.ms, + successRate: r.successRate, + })); const payload = { slug: b.slug, title: b.title, metric: b.metric, unit: b.unit, - status: b.status, - value: fieldValue(b), + status, + value: insufficient ? null : fieldValue(b), leader: top, - rankings: b.results - .filter((r) => r.ms.p50 > 0) - .sort((a, c) => (b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50)) - .map((r) => ({ - name: r.name, - slug: r.slug, - ms: r.ms, - successRate: r.successRate, - })), - sparkline: sparklineFor(b, top?.slug), + rankings, + sparkline: insufficient ? [] : sparklineFor(b, top?.slug), headline: headlineSentence(b), quote: citationQuote(b, SITE.url), pageUrl: `${SITE.url}/benchmarks/${b.slug}`, @@ -458,10 +477,15 @@ const mcpHandler = createMcpHandler( ], }; } - const top = leader(b); - const ranked = b.results - .filter((r) => r.ms.p50 > 0) - .sort((a, c) => (b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50)); + const insufficient = isInsufficient(b); + const top = insufficient ? null : leader(b); + const ranked = insufficient + ? [] + : b.results + .filter((r) => r.ms.p50 > 0) + .sort((a, c) => + b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, + ); const md: string[] = []; md.push(`# ${b.title}`); @@ -501,21 +525,33 @@ const mcpHandler = createMcpHandler( // We attach both Markdown (default rendering) and JSON (structured // access) so clients can pick whichever matches their context. + const status: "live" | "draft" | "insufficient" = insufficient + ? "insufficient" + : b.status; const payload = { slug: b.slug, title: b.title, metric: b.metric, unit: b.unit, - value: fieldValue(b), + status, + value: insufficient ? null : fieldValue(b), leader: top, - rankings: ranked.map((r) => ({ - name: r.name, - slug: r.slug, - ms: r.ms, - successRate: r.successRate, - sampleSize: r.sampleSize, - })), - sparkline: sparklineFor(b, top?.slug), + rankings: insufficient + ? b.results.map((r) => ({ + name: r.name, + slug: r.slug, + ms: { p50: null, p90: null, p99: null, mean: null }, + successRate: r.successRate, + sampleSize: r.sampleSize ?? null, + })) + : ranked.map((r) => ({ + name: r.name, + slug: r.slug, + ms: r.ms, + successRate: r.successRate, + sampleSize: r.sampleSize, + })), + sparkline: insufficient ? [] : sparklineFor(b, top?.slug), headline: headlineSentence(b), quote: citationQuote(b, SITE.url), pageUrl: `${SITE.url}/benchmarks/${b.slug}`, @@ -549,7 +585,7 @@ const mcpHandler = createMcpHandler( * reject batches explicitly (see below). */ function rateLimited(req: Request): Response | null { const key = clientKey(req, "mcp"); - const r = rateLimit(key, 60, 60); + const r = rateLimit(key, 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); return null; } diff --git a/src/app/api/og/[slug]/route.tsx b/src/app/api/og/[slug]/route.tsx index a153fa32..d2449071 100644 --- a/src/app/api/og/[slug]/route.tsx +++ b/src/app/api/og/[slug]/route.tsx @@ -21,7 +21,7 @@ export async function GET( req: Request, { params }: { params: Promise<{ slug: string }> }, ) { - const r = rateLimit(clientKey(req, "og"), 60, 60); + const r = rateLimit(clientKey(req, "og"), 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); const { slug } = await params; diff --git a/src/app/api/search/featured/route.ts b/src/app/api/search/featured/route.ts new file mode 100644 index 00000000..eba33e04 --- /dev/null +++ b/src/app/api/search/featured/route.ts @@ -0,0 +1,65 @@ +import { NextResponse } from "next/server"; +import { readCohortSnapshot } from "@/lib/cohort-snapshot"; +import { + buildFeaturedLeaders, + type FeaturedLeadersBlob, +} from "@/lib/search-featured"; + +export const runtime = "nodejs"; +// 60 s revalidate so even a cold edge cache only needs one round-trip +// per minute per region. The cron rewrites the underlying KV blob on the +// same cadence; stale-while-revalidate keeps every other request hot. +export const revalidate = 60; + +/** + * Slim payload feeding the header search dialog's "Live leaders" + + * "Trending" sections. Reads through Upstash KV first (cron-warmed, + * ~5 ms response) and falls through to a live build only on cold KV + * (deploy minute zero, missing creds, or worker outage). + * + * The previous implementation called /api/citable which assembled the + * full ~30-bench citable index (~50 KB) on every dialog open. This + * route returns the 12-card subset (~2 KB) the dialog actually needs. + */ +export async function GET() { + const snap = await readCohortSnapshot("search-featured"); + if (snap?.data) { + return NextResponse.json( + { ok: true, source: "kv", ageMs: snap.ageMs, ...snap.data }, + { + headers: { + "cache-control": + "public, s-maxage=60, stale-while-revalidate=300", + "access-control-allow-origin": "*", + }, + }, + ); + } + + // Cold-fall-through: build live. Slower (one full benchmark assembly) + // but never blocks the dialog if the cron hasn't run yet or Upstash is + // unreachable. The cron will heal this on its next tick. + try { + const blob = await buildFeaturedLeaders(); + return NextResponse.json( + { ok: true, source: "live", ageMs: 0, ...blob }, + { + headers: { + "cache-control": + "public, s-maxage=60, stale-while-revalidate=300", + "access-control-allow-origin": "*", + }, + }, + ); + } catch (err) { + return NextResponse.json( + { + ok: false, + error: err instanceof Error ? err.message : String(err), + featured: [], + trending: [], + }, + { status: 503, headers: { "cache-control": "no-store" } }, + ); + } +} diff --git a/src/app/api/series/[slug]/route.ts b/src/app/api/series/[slug]/route.ts index 638c064d..689d6346 100644 --- a/src/app/api/series/[slug]/route.ts +++ b/src/app/api/series/[slug]/route.ts @@ -37,7 +37,7 @@ export async function GET( req: Request, { params }: { params: Promise<{ slug: string }> }, ) { - const r = rateLimit(clientKey(req, "series"), 60, 60); + const r = rateLimit(clientKey(req, "series"), 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); const { slug } = await params; diff --git a/src/app/api/stat/[slug]/route.ts b/src/app/api/stat/[slug]/route.ts index 0101ab9a..14d6c7d7 100644 --- a/src/app/api/stat/[slug]/route.ts +++ b/src/app/api/stat/[slug]/route.ts @@ -5,6 +5,7 @@ import { citationQuote, fieldValue, headlineSentence, + isInsufficient, leader, sparklineFor, } from "@/lib/citation"; @@ -19,12 +20,22 @@ export const revalidate = 60; * Single benchmark as a citable atomic unit. Designed to fit into one * agent tool call: ranked providers, sparkline, methodology link, * pre-formatted attribution string, and stable citation URL. + * + * Status field semantics: + * "live" - usable measurement, leader / value populated. + * "draft" - spec author has not published (editorialStatus draft). + * "insufficient" - editorially live but the harness has no usable + * sample yet (every provider p50 = 0, or runtime + * status flipped to draft mid-cycle). value, leader + * and rankings p50 are nulled so a consumer cannot + * accidentally cite a fabricated winner. The shape + * of the response is preserved. */ export async function GET( req: Request, { params }: { params: Promise<{ slug: string }> }, ) { - const r = rateLimit(clientKey(req, "stat"), 60, 60); + const r = rateLimit(clientKey(req, "stat"), 60, 60, req); if (!r.ok) return tooManyRequests(r.retryAfterSec); const { slug } = await params; @@ -42,7 +53,46 @@ export async function GET( ); } - const top = leader(b); + const insufficient = isInsufficient(b); + const top = insufficient ? null : leader(b); + const value = insufficient ? null : fieldValue(b); + // Status surfaced to consumers: "insufficient" wins over the raw + // runtime "live" flag when the predicate fires, so /api/stat stops + // claiming live data for a bench whose harness has nothing to show. + const status: "live" | "draft" | "insufficient" = insufficient + ? "insufficient" + : b.status; + + // Rankings: when insufficient we still return one entry per provider + // (shape preserved for any consumer that diff-tracks the provider set) + // but every p50 is nulled to drive home that no comparison is possible. + const rankings = insufficient + ? b.results.map((r) => ({ + name: r.name, + slug: r.slug, + type: r.type ?? null, + layer: r.layer ?? null, + tag: r.tag ?? null, + ms: { p50: null, p90: null, p99: null, mean: null }, + successRate: r.successRate, + sampleSize: r.sampleSize ?? null, + })) + : b.results + .filter((r) => r.ms.p50 > 0) + .sort((a, c) => + b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50, + ) + .map((r) => ({ + name: r.name, + slug: r.slug, + type: r.type ?? null, + layer: r.layer ?? null, + tag: r.tag ?? null, + ms: r.ms, + successRate: r.successRate, + sampleSize: r.sampleSize, + })); + const payload = { slug: b.slug, title: b.title, @@ -50,22 +100,13 @@ export async function GET( category: b.category, metric: b.metric, unit: b.unit, - status: b.status, + status, higherIsBetter: b.higherIsBetter, - value: fieldValue(b), + value, leader: top, - rankings: b.results - .filter((r) => r.ms.p50 > 0) - .sort((a, c) => (b.higherIsBetter ? c.ms.p50 - a.ms.p50 : a.ms.p50 - c.ms.p50)) - .map((r) => ({ - name: r.name, - slug: r.slug, - ms: r.ms, - successRate: r.successRate, - sampleSize: r.sampleSize, - })), - sparkline: sparklineFor(b, top?.slug), - sampleSize: b.sampleSize, + rankings, + sparkline: insufficient ? [] : sparklineFor(b, top?.slug), + sampleSize: insufficient ? 0 : b.sampleSize, asOf: b.lastRunAt, headline: headlineSentence(b), quote: citationQuote(b, SITE.url), @@ -74,6 +115,12 @@ export async function GET( source: b.source, methodology: b.methodology, license: "CC-BY-4.0", + // Per-chain leader / worst. Empty for benches without a chain + // dimension. Used by the HF publisher to populate chain_leaders. + // Insufficient benches null both since the underlying rankings + // are not citable. + bestPerChain: insufficient ? null : (b.bestPerChain ?? null), + worstPerChain: insufficient ? null : (b.worstPerChain ?? null), }; return NextResponse.json(payload, { diff --git a/src/app/benchmarks/[slug]/page.tsx b/src/app/benchmarks/[slug]/page.tsx index c165674d..5f515eea 100644 --- a/src/app/benchmarks/[slug]/page.tsx +++ b/src/app/benchmarks/[slug]/page.tsx @@ -3,25 +3,25 @@ import type { Metadata } from "next"; import { notFound } from "next/navigation"; import Link from "next/link"; import { ArrowLeft, ArrowUpRight, ChevronDown } from "lucide-react"; -import { getBenchmark, getBenchmarks } from "@/data/benchmarks"; +import { getBenchmark, getBenchmarksSafe } from "@/data/benchmarks"; import { Pill } from "@/components/pill"; import { BenchmarkBody } from "@/components/benchmark-body"; import { BenchmarkBodySkeleton } from "@/components/benchmark-body-skeleton"; import { OraclePairMatrix } from "@/components/oracle-pair-matrix"; import { Breadcrumb } from "@/components/breadcrumb"; import { ChainHeadingsSummary } from "@/components/chain-headings-summary"; -import { OraclePairMatrix } from "@/components/oracle-pair-matrix"; import { CitationBar } from "@/components/citation-bar"; import { LiveIndicator } from "@/components/live-indicator"; import { ShareSection } from "@/components/share-section"; import { ExportVideoSection } from "@/components/export-video-section"; import { ReportSection } from "@/components/report-section"; import { CATEGORY_COLOR } from "@/lib/category-colors"; -import { headlineSentence } from "@/lib/citation"; +import { headlineSentence, isInsufficient } from "@/lib/citation"; import { capDescription } from "@/lib/seo-text"; import { getBenchCreatedAt } from "@/lib/seo/bench-dates"; import { SITE } from "@/data/site"; import { buildBreadcrumbJsonLd, buildFaqPageJsonLd, safeJsonLd } from "@/lib/jsonld"; +import { buildBenchDatasetJsonLd } from "@/lib/dataset-jsonld"; import { renderTemplate } from "@/lib/bench-template"; import type { Benchmark } from "@/types/benchmark"; @@ -159,7 +159,7 @@ export default async function BenchmarkPage({ // /api/bench/[slug]/variant when a tab is flipped (per-variant // unstable_cache keeps that at one cheap Prom roundtrip per 60 s // across all users), and renders the aggregate while it loads. - const all = await getBenchmarks(); + const all = await getBenchmarksSafe(); // Seed ONLY the unfiltered key. Seeding the initially-selected // chain/region/kind combo with the aggregate made the client believe // it already had that variant, so it never fetched the real one: the @@ -173,6 +173,10 @@ export default async function BenchmarkPage({ const isDraft = benchmark.status === "draft"; const isAwaiting = isDraft && benchmark.editorialStatus === "live"; + // Insufficient: editorially live, runtime might say "live" too, but the + // shared predicate decided no provider has a usable p50. Drives the + // pill above the H1 and the headline degradation downstream. + const insufficient = isInsufficient(benchmark); // Cap the "more benchmarks" rail at 6 items so it doesn't turn into // an endless single-column scroll on mobile (with 18 benches the old // unlimited list rendered 17 full cards stacked). Prefer same-category @@ -188,43 +192,46 @@ export default async function BenchmarkPage({ const benchmarkUrl = `${SITE.url}/benchmarks/${benchmark.slug}`; const sentence = headlineSentence(benchmark); + // variableMeasured ships as an array so Google's Dataset validator + // reports each statistical aggregate individually rather than as one + // opaque string. Order matches what /api/stat/ returns per + // provider, so a crawler can map field names 1:1. + const variableMeasured = [ + benchmark.metric, + `${benchmark.metric}_p50`, + `${benchmark.metric}_p90`, + `${benchmark.metric}_p99`, + "sample_size", + ]; + const datasetNode = { + ...buildBenchDatasetJsonLd({ + slug: benchmark.slug, + name: benchmark.seoTitle ?? benchmark.title, + alternateName: benchmark.title, + // Google Rich Results validator caps description at ~1000 chars even + // though schema.org Dataset allows up to 5000. Keep it under 990 to + // avoid the "Invalid string length" warning that strips rich snippets. + description: capDescription(benchmark.abstract, 990), + url: benchmarkUrl, + variableMeasured, + category: benchmark.category, + datePublished: getBenchCreatedAt(benchmark.slug).toISOString(), + dateModified: benchmark.lastRunAt, + measurementTechnique: benchmark.methodology.join(" "), + }), + // Re-bind creator + publisher to the global @id reference so the bench + // Dataset resolves to the same Organization node emitted by layout.tsx + // when crawlers stitch the site graph back together. The helper sets + // an inline Organization for standalone consumption; the page-level + // override is the more accurate shape on a site that already declares + // the Organization globally. + creator: { "@id": `${SITE.url}/#org` }, + publisher: { "@id": `${SITE.url}/#org` }, + }; const jsonLd = { "@context": "https://schema.org", "@graph": [ - { - "@type": "Dataset", - "@id": `${benchmarkUrl}#dataset`, - name: benchmark.seoTitle ?? benchmark.title, - alternateName: benchmark.title, - // Google Rich Results validator caps description at ~1000 chars even - // though schema.org Dataset allows up to 5000. Keep it under 990 to - // avoid the "Invalid string length" warning that strips rich snippets. - description: capDescription(benchmark.abstract, 990), - url: benchmarkUrl, - identifier: benchmark.slug, - keywords: [ - benchmark.category, - benchmark.metric, - ...benchmark.results.map((r) => r.name), - "live benchmark", - "crypto infrastructure", - ].join(", "), - creator: { "@id": `${SITE.url}/#org` }, - publisher: { "@id": `${SITE.url}/#org` }, - isAccessibleForFree: true, - license: "https://creativecommons.org/licenses/by/4.0/", - datePublished: getBenchCreatedAt(benchmark.slug).toISOString(), - dateModified: benchmark.lastRunAt, - variableMeasured: benchmark.metric, - distribution: [ - { - "@type": "DataDownload", - encodingFormat: "application/json", - contentUrl: `${SITE.url}/api/stat/${benchmark.slug}`, - }, - ], - measurementTechnique: benchmark.methodology.join(" "), - }, + datasetNode, { "@type": "TechArticle", "@id": `${benchmarkUrl}#article`, @@ -322,7 +329,10 @@ export default async function BenchmarkPage({ {isAwaiting ? "awaiting samples" : "draft"} )} - {!isDraft && ( + {!isDraft && insufficient && ( + insufficient samples + )} + {!isDraft && !insufficient && ( diff --git a/src/app/benchmarks/[slug]/share-card/route.tsx b/src/app/benchmarks/[slug]/share-card/route.tsx index 02a21184..be8f7e7b 100644 --- a/src/app/benchmarks/[slug]/share-card/route.tsx +++ b/src/app/benchmarks/[slug]/share-card/route.tsx @@ -494,7 +494,7 @@ export async function GET( // benchmark loaders, each render is 50-200ms CPU. Without this an // attacker hitting random query-string variants would burn function // CPU even for unknown slugs. - const rl = rateLimit(clientKey(request, "share-card"), 60, 60); + const rl = rateLimit(clientKey(request, "share-card"), 60, 60, request); if (!rl.ok) return tooManyRequests(rl.retryAfterSec); const { slug } = await params; diff --git a/src/app/benchmarks/page.tsx b/src/app/benchmarks/page.tsx index 81817cc3..b65001c9 100644 --- a/src/app/benchmarks/page.tsx +++ b/src/app/benchmarks/page.tsx @@ -1,5 +1,5 @@ import type { Metadata } from "next"; -import { getBenchmarks } from "@/data/benchmarks"; +import { getBenchmarksSafe, toBenchmarkCardData } from "@/data/benchmarks"; import { BenchmarkGrid } from "@/components/benchmark-grid"; import { safeJsonLd } from "@/lib/jsonld"; @@ -28,7 +28,7 @@ export const metadata: Metadata = { }; export default async function BenchmarksPage() { - const benchmarks = await getBenchmarks(); + const benchmarks = await getBenchmarksSafe(); // ItemList + BreadcrumbList JSON-LD so search engines and LLMs see the // page as a structured registry (the data is already in the DOM but @@ -86,7 +86,7 @@ export default async function BenchmarksPage() { {DESCRIPTION}

    - + ); } diff --git a/src/app/compare/[slug]/loading.tsx b/src/app/compare/[slug]/loading.tsx deleted file mode 100644 index 2597da75..00000000 --- a/src/app/compare/[slug]/loading.tsx +++ /dev/null @@ -1,84 +0,0 @@ -/** - * Loading UI rendered by Next.js during navigation to /compare/[slug]. - * Picked up automatically when the route's async render is in flight, - * which is the visible window where ad-hoc (non-curated) pairs pay the - * full cold start cost: every loadBenchmark for every shared bench - * fans out chain + region variant fetches. Without this file the user - * sees a frozen current page while the browser waits on the route - * payload; with it the visitor gets instant feedback that the compare - * page is building. - */ -export default function ComparePairLoading() { - return ( -
    -
    - -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - -
    -

    - Loading live measurements -

    -
    - {Array.from({ length: 3 }).map((_, i) => ( -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    -
    - ))} -
    -

    - First hit on a brand new pair can take a few seconds while the - per chain and per region variants fan out. Subsequent visits - and other users land on the cached render. -

    -
    -
    - ); -} diff --git a/src/app/compare/[slug]/page.tsx b/src/app/compare/[slug]/page.tsx index 8b8609f4..1ef9c90e 100644 --- a/src/app/compare/[slug]/page.tsx +++ b/src/app/compare/[slug]/page.tsx @@ -122,16 +122,53 @@ function canonicalisationTarget(slug: string): string | null { return canonical === slug ? null : canonical; } +/** Lightweight precheck: does this pair have at least one shared bench + * after applying the whitelist + exclude rules? Pure set arithmetic on + * the already-loaded provider appearances. No Prom calls, no KV + * lookup, no fan out. + * + * Mirrors the candidate-slug computation inside `buildSharedBenches` + * so the two stay in lockstep. Called by `generateMetadata` so a pair + * whose providers both exist but share zero benches notFound()s + * before any HTML streams. */ +function hasSharedBenches( + pair: ComparePair, + aAppearances: Awaited>, + bAppearances: Awaited>, +): boolean { + if (!aAppearances || !bAppearances) return false; + const aSlugs = new Set(aAppearances.appearances.map((x) => x.benchmark.slug)); + const bSlugs = new Set(bAppearances.appearances.map((x) => x.benchmark.slug)); + const candidateSlugs = pair.benchmarks + ? pair.benchmarks.filter((s) => aSlugs.has(s) && bSlugs.has(s)) + : Array.from(aSlugs).filter((s) => bSlugs.has(s)); + const excluded = new Set(pair.excludeBenchmarks ?? []); + return candidateSlugs.some((s) => !excluded.has(s)); +} + export async function generateMetadata({ params, }: { params: Promise; }): Promise { const { slug } = await params; + // Run the same gating logic as the page render so non-canonical and + // invalid slugs short-circuit at the metadata phase. Combined with + // the loading.tsx removal in this hotfix, notFound() here cleanly + // produces a real 308 / 404 response from the route layer instead + // of a 200 wrapping a streamed loading skeleton. + const canonicalTarget = canonicalisationTarget(slug); + if (canonicalTarget) redirect(`/compare/${canonicalTarget}`); const pair = getComparePair(slug) ?? (await resolveAdHocPair(slug)); - if (!pair) return {}; + if (!pair) notFound(); const { a, b } = await loadPairProviders(pair); - if (!a || !b) return {}; + if (!a || !b) notFound(); + // Final SSR gate: an ad-hoc pair can have both providers resolved + // yet share zero benches (e.g. an RPC provider vs an oracle). + // Without this the page body's `shared.length === 0` check fires + // late and the response loses its chance to demote the status code. + // Cheap: only the appearance intersection, no Prom fan out. + if (!hasSharedBenches(pair, a, b)) notFound(); const url = `${SITE.url}/compare/${pair.slug}`; const title = `${a.name} vs ${b.name}: live OpenChainBench benchmark data`; @@ -987,7 +1024,7 @@ function ChainRegionMatrix({ ) : (     ); })} @@ -1010,7 +1047,7 @@ function ChainRegionMatrix({ ) : ( ); })} diff --git a/src/app/globals.css b/src/app/globals.css index b0c43984..a97dd466 100644 --- a/src/app/globals.css +++ b/src/app/globals.css @@ -219,7 +219,13 @@ table { /* Editorial section labels - sans-serif with wide tracking + uppercase * for a Bloomberg-style tech-editorial feel, instead of the developer-tool - * vibe that monospace gives off when used on body / UI labels. */ + * vibe that monospace gives off when used on body / UI labels. + * + * `.label-mono` remains for backward compat. The three sized variants below + * harmonise the three near-identical Tailwind class strings repeated ~133 + * times across `src/**/*.tsx` (text-[10px]/[11px]/xs + 0.16em/0.18em + the + * matching ink ramp colour). Centralising them here means a copy-edit to + * the tracking or color ramp lands in one place instead of every call site. */ .label-mono { font-family: var(--font-sans); font-size: 10px; @@ -228,6 +234,28 @@ table { letter-spacing: 0.18em; } +@layer components { + .label-mono-xs { + @apply text-[10px] uppercase tracking-[0.16em] text-ink-faint; + } + .label-mono-sm { + @apply text-[11px] uppercase tracking-[0.18em] text-ink-muted font-medium font-sans; + } + .label-mono-lg { + @apply text-xs uppercase tracking-[0.18em] text-ink-muted font-medium; + } + + /* Page-level H1 hero treatment. Matches the dominant inline class + * string shared by index / answers / compare / alternatives / + * chains landing pages (display + 3xl-4xl ramp + ink + tight + * leading). Centralised so a copy-edit to the page-title scale + * lands in one place instead of every list landing route. */ + .h1-hero { + font-family: var(--font-display); + @apply text-3xl sm:text-4xl text-ink leading-[1.05]; + } +} + /* Chart pop animations */ @keyframes chart-pop-rise { 0% { opacity: 0; transform: translate(0, calc(-50% + 4px)) scale(0.92); } @@ -302,6 +330,60 @@ textarea:focus-visible { border-radius: 4px; } +/* Search dialog open/close animations. tailwindcss-animate isn't + installed; these handcrafted keyframes give us the fade + slight + slide-down on open and the inverse on close. Linear-style: snappy, + no scale bounce. */ +@keyframes ocb-search-overlay-in { + from { opacity: 0; } + to { opacity: 1; } +} +@keyframes ocb-search-overlay-out { + from { opacity: 1; } + to { opacity: 0; } +} +@keyframes ocb-search-card-in { + from { opacity: 0; transform: translateY(-4px); } + to { opacity: 1; transform: translateY(0); } +} +@keyframes ocb-search-card-out { + from { opacity: 1; transform: translateY(0); } + to { opacity: 0; transform: translateY(-4px); } +} +.ocb-search-overlay-in { + animation: ocb-search-overlay-in 150ms ease-out; +} +.ocb-search-overlay-out { + animation: ocb-search-overlay-out 100ms ease-in forwards; +} +.ocb-search-card-in { + animation: ocb-search-card-in 150ms cubic-bezier(0.16, 0.84, 0.32, 1); +} +.ocb-search-card-out { + animation: ocb-search-card-out 100ms ease-in forwards; +} +@media (prefers-reduced-motion: reduce) { + .ocb-search-overlay-in, + .ocb-search-overlay-out, + .ocb-search-card-in, + .ocb-search-card-out { + animation: none !important; + } +} + +/* cmdk command palette input — opts out of the universal focus ring. + The dialog already provides visual focus context (modal backdrop + + centered card); a hard accent outline on the input itself reads as + a validation error rather than a focus state, which is the look the + designer flagged as "pas pro". The input still receives focus and + keyboard nav, just without the harsh outline. */ +[cmdk-input]:focus, +[cmdk-input]:focus-visible { + outline: none !important; + outline-offset: 0; + box-shadow: none !important; +} + /* Skip-link target. Hidden until keyboard focus brings it onto the page, then snaps to the top-left so a screen-reader / keyboard user can bypass the nav. */ diff --git a/src/app/layout.tsx b/src/app/layout.tsx index fdd7a364..be7690e2 100644 --- a/src/app/layout.tsx +++ b/src/app/layout.tsx @@ -4,6 +4,8 @@ import { Analytics } from "@vercel/analytics/next"; import "./globals.css"; import { SiteHeader } from "@/components/site-header"; import { SiteFooter } from "@/components/site-footer"; +import { SearchProvider } from "@/components/search/search-provider"; +import { buildSearchIndex } from "@/lib/search/buildIndex"; import { SITE } from "@/data/site"; import { safeJsonLd } from "@/lib/jsonld"; @@ -54,7 +56,7 @@ const IS_STAGING = !!process.env.VERCEL_ENV && process.env.VERCEL_ENV !== "production"; export const metadata: Metadata = { - metadataBase: new URL("https://openchainbench.com"), + metadataBase: new URL(SITE.url), title: { default: "OpenChainBench. Open benchmarks for crypto infrastructure", template: "%s · OpenChainBench", @@ -69,7 +71,7 @@ export const metadata: Metadata = { description: "Live benchmarks for crypto infrastructure: RPC latency, bridge fees, L2 finality and price feed accuracy.", type: "website", - url: "https://openchainbench.com", + url: SITE.url, siteName: "OpenChainBench", }, twitter: { @@ -140,9 +142,12 @@ const ORG_JSONLD = { ], }; -export default function RootLayout({ +export default async function RootLayout({ children, }: Readonly<{ children: React.ReactNode }>) { + // Index is built once per server runtime (memoised via React `cache`), + // shipped as JSON to the client provider. ~400 docs, ~25-40 KB raw. + const searchItems = await buildSearchIndex(); return ( + {/* Site-wide Organization + WebSite JSON-LD. Lives inside + (not body) so stricter parsers (some AI search crawlers, schema + validators) that only scan for structured data pick it + up. Google parses both head and body, so this is a pure + placement change with no schema-content delta. */} +
    - — + - - — + -