') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ', 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); })(); [ci] Benchmark comment: Best column + best/p75/p99 deltas (drop Avg/P10) by VaguelySerious · Pull Request #3005 · vercel/workflow · GitHub
Skip to content

[ci] Benchmark comment: Best column + best/p75/p99 deltas (drop Avg/P10) - #3005

Merged
VaguelySerious merged 3 commits into
mainfrom
peter/bench-best-column
Jul 20, 2026
Merged

[ci] Benchmark comment: Best column + best/p75/p99 deltas (drop Avg/P10)#3005
VaguelySerious merged 3 commits into
mainfrom
peter/bench-best-column

Conversation

@VaguelySerious

@VaguelySeriousVaguelySerious commented Jul 20, 2026•
edited
Loading

Copy link
Copy Markdown
Member

What

Reworks the Performance Benchmarks PR-comment table:

  • Drop the Avg column entirely.
  • Rename P10 → Best, reporting the fastest (warm-start) sample (min) instead of the 10th percentile — a cleaner warm-start floor for these bursty, cold-start-heavy runs.
  • vs-main deltas on Best, P75, P90, and P99 (every latency column).
  • Directional threshold marks on deltas: 🔻 when a delta is worse than +15% vs main, 💚 when better than −15%. Smaller moves show the percentage alone. (There is no green-triangle emoji, so 💚 is the improvement marker.)
  • Quieter target column: 🔴 only when a percentile is over its target; within-target cells are left unmarked (dropped the 🟢).

Example

MetricScenarioBest (ms)P75 (ms)P90 (ms)P99 (ms)Samples
TTFSstep1210 (+34%) 🔻1368 🔴 (+24%) 🔻1396 🔴 (±0%)1726 🔴 (-14%)30
SLstream latency40 (-67%) 💚48 (-68%) 💚55 (-66%) 💚120 (-60%) 💚30

How

  • packages/core/e2e/benchmark.test.ts — computeStats emits best (the fastest sample) in place of p10/min/max. avg is kept in the results JSON for reference but no longer rendered.
  • .github/scripts/render-benchmark-comment.mjs:
    • Best (ms) column; annotateWithBaseline records baseline best/p75/p90/p99 per row (via a BASELINE_FIELDS table; best falls back to a pre-rename baseline's min so existing main baselines still light up).
    • formatDelta appends 🔻/💚 past ±DELTA_MARK_THRESHOLD_PCT (15%).
    • formatCell marks 🔴 only when over target; within-target is unmarked.
    • Footer legend/notes updated.
  • .github/scripts/render-benchmark-comment.test.js — schema + assertions updated for the Best column, P90 deltas, the 🔻/💚 marks, and the quiet target column.

Notes

  • Methodology version stays 2 — the measurement window is unchanged, only which statistics are displayed/compared. Deltas remain comparable against existing main baselines (Best falls back to their stored min).
  • CI-scripts + e2e test only; no published package output, so no changeset (matches [ci] Run benchmarks in-deployment to avoid proxy overhead #2967).

Verification

  • node --test .github/scripts/render-benchmark-comment.test.js → 11/11 pass
  • pnpm --filter @workflow/core typecheck → clean; biome check → clean
  • Rendered sample comments locally covering 🔻 regressions, 💚 improvements, 🔴 target misses, and blank within-target cells (see the example above).

🤖 Generated with Claude Code

@VaguelySerious@claude
[ci] Benchmark comment: replace Avg/P10 with Best column; deltas on b…
d8cc358
…est/p75/p99
The PR-comment table dropped the Avg column and renamed P10 to Best,
now showing the fastest (warm-start) sample instead of the 10th
percentile. The vs-main deltas moved off Avg and onto Best, P75, and
P99 (P90 stays undecorated) so the comparison spans the warm floor,
the typical case, and the tail.
- benchmark.test.ts: computeStats emits `best` (fastest sample) in
place of `p10`/`min`/`max`; `avg` is retained in the JSON for
reference but no longer rendered. Docs/console updated.
- render-benchmark-comment.mjs: Best column; baseline annotation now
keyed to best/p75/p99 (falls back to a pre-rename baseline's `min`
for the Best delta); footer notes updated.
- render-benchmark-comment.test.js: schema + assertions updated (11/11).
CI-scripts + e2e test only; no published package output, so no
changeset (matches #2967).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
@changeset-bot

changeset-botBot commented Jul 20, 2026•
edited
Loading

Copy link
Copy Markdown

⚠️ No Changeset found

Latest commit: 8ca9fd3

Merging this PR will not cause a version bump for any packages. If these changes should not result in a new version, you're good to go. If these changes should result in a version bump, you need to add a changeset.

This PR includes no changesets

When changesets are added to this PR, you'll see the packages that this PR includes changesets for and the associated semver types

Click here to learn what changesets are, and how to add one.

Click here if you're a maintainer who wants to add a changeset to this PR

@vercel

vercelBot commented Jul 20, 2026•
edited
Loading

Copy link
Copy Markdown
Contributor

The latest updates on your projects. Learn more about Vercel for GitHub.

ProjectDeploymentActionsUpdated (UTC)
example-nextjs-workflow-turbopackReadyReadyPreview, CommentJul 20, 2026 8:20pm
example-nextjs-workflow-webpackReadyReadyPreview, CommentJul 20, 2026 8:20pm
example-workflowReadyReadyPreview, CommentJul 20, 2026 8:20pm
workbench-astro-workflowBuildingBuildingPreview, CommentJul 20, 2026 8:20pm
workbench-express-workflowReadyReadyPreview, CommentJul 20, 2026 8:20pm
workbench-fastify-workflowReadyReadyPreview, CommentJul 20, 2026 8:20pm
workbench-hono-workflowReadyReadyPreview, CommentJul 20, 2026 8:20pm
workbench-nestjs-workflowErrorErrorJul 20, 2026 8:20pm
workbench-nitro-workflowReadyReadyPreview, CommentJul 20, 2026 8:20pm
workbench-nuxt-workflowReadyReadyPreview, CommentJul 20, 2026 8:20pm
workbench-sveltekit-workflowReadyReadyPreview, CommentJul 20, 2026 8:20pm
workbench-tanstack-start-workflowReadyReadyPreview, CommentJul 20, 2026 8:20pm
workbench-vite-workflowReadyReadyPreview, CommentJul 20, 2026 8:20pm
workflow-docsReadyReadyPreview, Comment, Open in v0Jul 20, 2026 8:20pm
workflow-swc-playgroundReadyReadyPreview, CommentJul 20, 2026 8:20pm
workflow-tarballsReadyReadyPreview, CommentJul 20, 2026 8:20pm
workflow-webReadyReadyPreview, CommentJul 20, 2026 8:20pm

@github-actions

github-actionsBot commented Jul 20, 2026•
edited
Loading

Copy link
Copy Markdown
Contributor

🧪 E2E Test Results

❌ Some tests failed

Summary

PassedFailedSkippedTotal
❌ ▲ Vercel Production1398552301683
✅ 💻 Local Development148302001683
✅ 📦 Local Production161702191836
✅ 🐘 Local Postgres161702191836
✅ 🪟 Windows15300153
❌ 📋 Other88951771071
✅ vercel-multi-region270027
Total71846010458289

❌ Failed Tests

▲ Vercel Production (55 failed)

astro (5 failed):

  • error handling retry behavior regular Error retries until success
  • error handling retry behavior FatalError fails immediately without retries
  • error handling not registered StepNotRegisteredError fails the step but workflow can catch it
  • stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars) | wrun_41KY0KCXZ90GJ1DDCQY75RPQ5V | 🔍 observability
  • instanceMethodStepWorkflow - instance methods with "use step" directive | wrun_41KY0KG1QF0GKV6PT4MJ9NA1KB | 🔍 observability

example (5 failed):

  • error handling retry behavior regular Error retries until success
  • error handling retry behavior FatalError fails immediately without retries
  • error handling not registered StepNotRegisteredError fails the step but workflow can catch it
  • stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars) | wrun_41KY0KCXZ90GJ1DDCQY75RPQ5V | 🔍 observability
  • instanceMethodStepWorkflow - instance methods with "use step" directive | wrun_41KY0KG1QF0GKV6PT4MJ9NA1KB | 🔍 observability

express (5 failed):

  • error handling retry behavior regular Error retries until success
  • error handling retry behavior FatalError fails immediately without retries
  • error handling not registered StepNotRegisteredError fails the step but workflow can catch it
  • stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars) | wrun_41KY0KCXZ90GJ1DDCQY75RPQ5V | 🔍 observability
  • instanceMethodStepWorkflow - instance methods with "use step" directive | wrun_41KY0KG1QF0GKV6PT4MJ9NA1KB | 🔍 observability

fastify (5 failed):

  • error handling retry behavior regular Error retries until success
  • error handling retry behavior FatalError fails immediately without retries
  • error handling not registered StepNotRegisteredError fails the step but workflow can catch it
  • stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars) | wrun_41KY0KCXZ90GJ1DDCQY75RPQ5V | 🔍 observability
  • instanceMethodStepWorkflow - instance methods with "use step" directive | wrun_41KY0KG1QF0GKV6PT4MJ9NA1KB | 🔍 observability

hono (5 failed):

  • error handling retry behavior regular Error retries until success
  • error handling retry behavior FatalError fails immediately without retries
  • error handling not registered StepNotRegisteredError fails the step but workflow can catch it
  • stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars) | wrun_41KY0KCXZ90GJ1DDCQY75RPQ5V | 🔍 observability
  • instanceMethodStepWorkflow - instance methods with "use step" directive | wrun_41KY0KG1QF0GKV6PT4MJ9NA1KB | 🔍 observability

nextjs-turbopack (5 failed):

  • error handling retry behavior regular Error retries until success
  • error handling retry behavior FatalError fails immediately without retries
  • error handling not registered StepNotRegisteredError fails the step but workflow can catch it
  • stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars) | wrun_41KY0KCXZ90GJ1DDCQY75RPQ5V | 🔍 observability
  • instanceMethodStepWorkflow - instance methods with "use step" directive | wrun_41KY0KG1QF0GKV6PT4MJ9NA1KB | 🔍 observability

nextjs-webpack (5 failed):

  • error handling retry behavior regular Error retries until success
  • error handling retry behavior FatalError fails immediately without retries
  • error handling not registered StepNotRegisteredError fails the step but workflow can catch it
  • stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars) | wrun_41KY0KCXZ90GJ1DDCQY75RPQ5V | 🔍 observability
  • instanceMethodStepWorkflow - instance methods with "use step" directive | wrun_41KY0KG1QF0GKV6PT4MJ9NA1KB | 🔍 observability

nitro (5 failed):

  • error handling retry behavior regular Error retries until success
  • error handling retry behavior FatalError fails immediately without retries
  • error handling not registered StepNotRegisteredError fails the step but workflow can catch it
  • stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars) | wrun_41KY0KCXZ90GJ1DDCQY75RPQ5V | 🔍 observability
  • instanceMethodStepWorkflow - instance methods with "use step" directive | wrun_41KY0KG1QF0GKV6PT4MJ9NA1KB | 🔍 observability

nuxt (5 failed):

  • error handling retry behavior regular Error retries until success
  • error handling retry behavior FatalError fails immediately without retries
  • error handling not registered StepNotRegisteredError fails the step but workflow can catch it
  • stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars) | wrun_41KY0KCXZ90GJ1DDCQY75RPQ5V | 🔍 observability
  • instanceMethodStepWorkflow - instance methods with "use step" directive | wrun_41KY0KG1QF0GKV6PT4MJ9NA1KB | 🔍 observability

sveltekit (5 failed):

  • error handling retry behavior regular Error retries until success
  • error handling retry behavior FatalError fails immediately without retries
  • error handling not registered StepNotRegisteredError fails the step but workflow can catch it
  • stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars) | wrun_41KY0KCXZ90GJ1DDCQY75RPQ5V | 🔍 observability
  • instanceMethodStepWorkflow - instance methods with "use step" directive | wrun_41KY0KG1QF0GKV6PT4MJ9NA1KB | 🔍 observability

vite (5 failed):

  • error handling retry behavior regular Error retries until success
  • error handling retry behavior FatalError fails immediately without retries
  • error handling not registered StepNotRegisteredError fails the step but workflow can catch it
  • stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars) | wrun_41KY0KCXZ90GJ1DDCQY75RPQ5V | 🔍 observability
  • instanceMethodStepWorkflow - instance methods with "use step" directive | wrun_41KY0KG1QF0GKV6PT4MJ9NA1KB | 🔍 observability
📋 Other (5 failed)

e2e-vercel-prod-tanstack-start (5 failed):

  • error handling retry behavior regular Error retries until success
  • error handling retry behavior FatalError fails immediately without retries
  • error handling not registered StepNotRegisteredError fails the step but workflow can catch it
  • stepFunctionPassingWorkflow - step function references can be passed as arguments (without closure vars) | wrun_41KY0KCXZ90GJ1DDCQY75RPQ5V
  • instanceMethodStepWorkflow - instance methods with "use step" directive | wrun_41KY0KG1QF0GKV6PT4MJ9NA1KB

Details by Category

❌ ▲ Vercel Production
AppPassedFailedSkipped
❌ astro121527
❌ example121527
❌ express121527
❌ fastify121527
❌ hono121527
❌ nextjs-turbopack14553
❌ nextjs-webpack14553
❌ nitro121527
❌ nuxt121527
❌ sveltekit14058
❌ vite121527
✅ 💻 Local Development
AppPassedFailedSkipped
✅ astro-stable128025
✅ express-stable128025
✅ fastify-stable128025
✅ hono-stable128025
✅ nextjs-turbopack-canary134019
✅ nextjs-turbopack-stable15300
✅ nextjs-webpack-stable15300
✅ nitro-stable128025
✅ nuxt-stable128025
✅ sveltekit-stable14706
✅ vite-stable128025
✅ 📦 Local Production
AppPassedFailedSkipped
✅ astro-stable128025
✅ express-stable128025
✅ fastify-stable128025
✅ hono-stable128025
✅ nextjs-turbopack-canary134019
✅ nextjs-turbopack-stable15300
✅ nextjs-webpack-canary134019
✅ nextjs-webpack-stable15300
✅ nitro-stable128025
✅ nuxt-stable128025
✅ sveltekit-stable14706
✅ vite-stable128025
✅ 🐘 Local Postgres
AppPassedFailedSkipped
✅ astro-stable128025
✅ express-stable128025
✅ fastify-stable128025
✅ hono-stable128025
✅ nextjs-turbopack-canary134019
✅ nextjs-turbopack-stable15300
✅ nextjs-webpack-canary134019
✅ nextjs-webpack-stable15300
✅ nitro-stable128025
✅ nuxt-stable128025
✅ sveltekit-stable14706
✅ vite-stable128025
✅ 🪟 Windows
AppPassedFailedSkipped
✅ nextjs-turbopack15300
❌ 📋 Other
AppPassedFailedSkipped
✅ e2e-local-dev-nest-stable128025
✅ e2e-local-dev-tanstack-start-128025
✅ e2e-local-postgres-nest-stable128025
✅ e2e-local-postgres-tanstack-start-128025
✅ e2e-local-prod-nest-stable128025
✅ e2e-local-prod-tanstack-start-128025
❌ e2e-vercel-prod-tanstack-start121527
✅ vercel-multi-region
AppPassedFailedSkipped
✅ nextjs-turbopack2700

📋 View full workflow run

❌ Some E2E test jobs failed:

  • Vercel Prod: failure
  • Local Dev: failure
  • Local Prod: success
  • Local Postgres: success
  • Windows: success

Check the workflow run for details.

@github-actions

github-actionsBot commented Jul 20, 2026•
edited
Loading

Copy link
Copy Markdown
Contributor

📊 Workflow Benchmarks

commit 8ca9fd3 · Mon, 20 Jul 2026 20:31:57 GMT · run logs

Backend: vercel · app: nextjs-turbopack

MetricScenarioBest (ms)P75 (ms)P90 (ms)P99 (ms)Samples
TTFSstep308 (-65%) 💚1294 🔴 (+33%) 🔻1333 🔴 (+33%) 🔻1401 🔴 (-18%) 💚30
TTFSstream223 (-74%) 💚1254 🔴 (+30%) 🔻1272 🔴 (+27%) 🔻1317 🔴 (-1.8%)30
TTFShook + stream1417 (+35%) 🔻1562 🔴 (+32%) 🔻1625 🔴 (+34%) 🔻1956 🔴 (+45%) 🔻30
STSO1020 steps (1-20)184 (+8.2%)298 🔴 (-2.0%)334 🔴 (-14%)719 🔴 (+17%) 🔻19
STSO1020 steps (101-120)199 (-3.9%)260 🔴 (-11%)274 🔴 (-39%) 💚316 🔴 (-60%) 💚19
STSO1020 steps (1001-1020)461 (-6.9%)516 🔴 (-12%)594 🔴 (-1.8%)598 🔴 (-16%) 💚19
WO1020 steps389975 (-4.7%)389975 (-4.7%)389975 (-4.7%)389975 (-4.7%)1
SLstream latency85 (+10%)141 🔴 (+2.9%)165 🔴 (-11%)291 🔴 (-51%) 💚30
📜 Previous results (2)

a536d5f

Mon, 20 Jul 2026 19:25:29 GMT · run logs

vercel / nextjs-turbopack

MetricScenarioBest (ms)P75 (ms)P90 (ms)P99 (ms)Samples
TTFSstep1214 (+17%) 🔻1348 🔴 (+16%) 🔻1384 🔴 (+16%) 🔻1484 🔴 (+23%) 🔻30
TTFSstream1238 (+357%) 🔻1349 🔴 (+17%) 🔻1374 🔴 (+17%) 🔻1457 🔴 (+19%) 🔻30
TTFShook + stream1186 (-6.8%)1635 🔴 (+14%)1694 🔴 (+6.7%)1918 🔴 (+9.9%)30
STSO1020 steps (1-20)185 (-20%) 💚267 🔴 (-25%) 💚320 🔴 (-24%) 💚377 🔴 (-22%) 💚19
STSO1020 steps (101-120)200 (-1.5%)279 🔴 (-10%)361 🔴 (-0.8%)446 🔴 (+6.4%)19
STSO1020 steps (1001-1020)471 (-13%)544 🔴 (-19%) 💚642 🔴 (-23%) 💚702 🔴 (-16%) 💚19
WO1020 steps392944 (-17%) 💚392944 (-17%) 💚392944 (-17%) 💚392944 (-17%) 💚1
SLstream latency92 (-8.9%)183 🔴 (-1.1%)240 🔴 (+7.6%)283 🔴 (-23%) 💚30

d8cc358

Mon, 20 Jul 2026 15:59:29 GMT · run logs

vercel / nextjs-turbopack

MetricScenarioBest (ms)P75 (ms)P90 (ms)P99 (ms)Samples
TTFSstep1243 (+25%) 🔻1363 🔴 (+23%) 🔻1393 🔴1628 🔴 (-9.4%)30
TTFSstream1289 (+32%) 🔻1409 🔴 (+29%) 🔻1451 🔴1571 🔴 (+33%) 🔻30
TTFShook + stream1337 (+1.3%)1670 🔴 (+14%)1707 🔴1775 🔴 (+2.2%)30
STSO1020 steps (1-20)222 (+8.8%)316 🔴 (+8.6%)397 🔴407 🔴 (-7.7%)19
STSO1020 steps (101-120)238 (+3.9%)323 🔴 (+14%)385 🔴536 🔴 (+32%) 🔻19
STSO1020 steps (1001-1020)515 (-3.2%)563 🔴 (-6.0%)586 🔴592 🔴 (-5.0%)19
WO1020 steps427152 (-3.7%)427152 (-3.7%)427152427152 (-3.7%)1
SLstream latency124 (+5.1%)168 🔴 (±0%)182 🔴326 🔴 (+50%) 🔻30

Best/P75/P90/P99 deltas compare against the most recent benchmark run on main at the time of this run. 🔻 flags a delta worse than +15%, 💚 one better than −15%.

Metrics — TTFS: time to first step body (in-deployment start() → first step body, deployment clocks) · STSO: step-to-step overhead (gap between consecutive step bodies) · WO: workflow overhead (whole-run time outside step bodies, in-deployment anchored) · SL: stream latency (in-deployment write → read propagation, readAt - writtenAt)

Scenarios — step: one trivial no-op step, no stream; no hooks, so the run stays in turbo mode (in-process fast path) · stream: one streaming step; no hooks, so the run stays in turbo mode (in-process fast path) · hook + stream: registers a hook before one step, which exits turbo mode (dispatch path) · 1020 steps: 1020 trivial sequential steps; STSO is measured between consecutive steps in the given step ranges, and WO is the whole-run overhead outside step bodies · stream latency: parallel reader/writer steps on a dedicated stream; SL is the in-deployment write->read propagation (readAt - writtenAt)

🔴 marks a percentile over its target (within target is left unmarked). Targets (p75/p90/p99, ms) — TTFS 200/300/600 · SL 50/60/125 · STSO (1-20) 20/30/60 · STSO (101-120) 30/45/90 · STSO (1001-1020) 40/60/120

All metrics are measured from deployment-side timestamps only. Runs are triggered by an in-deployment route that stamps the anchor (clientStart) right before start(), so the CI runner’s request and its path through api.vercel.com sit outside every measured window. TTFS = in-deployment start() → first step body (turbo uses the in-process fast path, non-turbo the dispatch path), and includes the VQS dispatch hop plus any /flow cold start. STSO/WO are measured between step bodies on the deployment. SL is measured inside the workflow (parallel reader/writer steps), so it no longer includes the api.vercel.com read path.

Cold starts are kept in the numbers on purpose — they are part of real bursty-workload latency. The workbench deployment cold-starts the /flow invocation for a large fraction of runs, inflating P75+; the Best column shows the fastest (warm-start) sample for comparison.

@VaguelySerious
VaguelySerious marked this pull request as ready for review July 20, 2026 18:37
@VaguelySerious
VaguelySerious requested review from a team and ijjk as code ownersJuly 20, 2026 18:37
@VaguelySerious@claude
[ci] Benchmark comment: P90 delta + 🔻/💚 threshold marks; quiet target…
a536d5f
… column
- Add the vs-main delta to the P90 column (was undecorated); baseline
annotation now also records p90.
- Flag deltas beyond ±15% vs main directionally: 🔻 for a regression
(>+15%), 💚 for an improvement (<-15%). Smaller moves show the % alone.
- Target column is quieter: 🔴 only when a percentile is over target;
within-target cells are left unmarked (dropped the 🟢).
- Refactor annotateWithBaseline over a BASELINE_FIELDS table to keep
cyclomatic complexity under Biome's limit after adding p90.
render-benchmark-comment.test.js updated (11/11 pass).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
@VaguelySerious
VaguelySerious marked this pull request as draft July 20, 2026 19:11
@VaguelySerious
VaguelySerious marked this pull request as ready for review July 20, 2026 20:15
@VaguelySerious
VaguelySerious merged commit 0bc22c8 into mainJul 20, 2026
90 of 105 checks passed
@VaguelySerious
VaguelySerious deleted the peter/bench-best-column branch July 20, 2026 20:50
@github-actions

github-actionsBot commented Jul 20, 2026

Copy link
Copy Markdown
Contributor

No backport to stable for 0bc22c8 (AI decision).

All three changed files (.github/scripts/render-benchmark-comment.mjs, its test, and packages/core/e2e/benchmark.test.ts) do not exist on stable — verified via git ls-tree origin/stable, which shows the stable branch has an older benchmark setup (bench.bench.ts, aggregate-benchmarks.js) and its benchmarks.yml never references these scripts. The commit purely reworks main-only benchmark PR-comment infrastructure, so there is nothing on stable to apply it to.

To override, re-run the Backport to stable workflow manually via workflow_dispatch and paste this commit SHA into the ref input:

0bc22c8e9b50c84f5aafd38b833d2a20470843ee

pranaygp added a commit that referenced this pull request Jul 21, 2026
@pranaygp
Merge remote-tracking branch 'origin/main' into docs/competition
f7a526c
* origin/main: (21 commits)
docs(agents): note lint/format/typecheck are advisory, not blocking (#2886)
Retry transient connection timeouts (#3013)
fix(world-vercel): append caller User-Agent products instead of discarding them (#2998)
[ci] Enable NestJS e2e-vercel-prod and add to docs as "experimental" (#3011)
[ci] Benchmark comment: Best column + best/p75/p99 deltas (drop Avg/P10) (#3005)
docs: fall back to first child page for sidebar folders without an index (#3009)
[nest] Fix NestJS Vercel build output (#2988)
Avoid resolving run data for background steps (#2993)
chore(docs): update @vercel/geistdocs to 1.14.0 (#3002)
fix(docs): add version-switcher fallback redirects for pages missing in one version (#3003)
ci: update opencode to 1.18.4 and switch backport AI model to claude-fable-5 (#3006)
fix(core): batch stream writes via writeMulti (#2995)
perf(core): cache port discovery in step invocations for self-hosted worlds (#2996)
feat(web-shared): Alt+hover span measurement in the new trace viewer (#2985)
fix(world-postgres): throw EntityConflictError on duplicate run_created (#2983)
[ci] Run benchmarks in-deployment to avoid proxy overhead (#2967)
Enable additional perf optimizations when correctness guarantees are met (#2970)
perf(core): prepare replay payloads concurrently (#2980)
Fix dotted tsconfig alias workflow discovery (#2963)
Adjust helper position on trace viewer (#2968)
...
pranaygp added a commit that referenced this pull request Jul 21, 2026
@pranaygp
Merge remote-tracking branch 'origin/main' into pgp/returnvalue-stream
599cf72
* origin/main:
test: regression coverage for hook.resume() from isolated route bundles (o2flow beta.26 incident) (#3001)
docs(agents): note lint/format/typecheck are advisory, not blocking (#2886)
Retry transient connection timeouts (#3013)
fix(world-vercel): append caller User-Agent products instead of discarding them (#2998)
[ci] Enable NestJS e2e-vercel-prod and add to docs as "experimental" (#3011)
[ci] Benchmark comment: Best column + best/p75/p99 deltas (drop Avg/P10) (#3005)
docs: fall back to first child page for sidebar folders without an index (#3009)
[nest] Fix NestJS Vercel build output (#2988)
Avoid resolving run data for background steps (#2993)
chore(docs): update @vercel/geistdocs to 1.14.0 (#3002)
fix(docs): add version-switcher fallback redirects for pages missing in one version (#3003)
ci: update opencode to 1.18.4 and switch backport AI model to claude-fable-5 (#3006)
fix(core): batch stream writes via writeMulti (#2995)
pranaygp added a commit that referenced this pull request Jul 22, 2026
@pranaygp
Merge remote-tracking branch 'origin/main' into pgp/world-start-recovery
a73b788
* origin/main: (162 commits)
Implement `max_events` per run limit (#2986)
[core] Enforce maxRetries for steps that time out (#3035)
[world-vercel] Idempotent retry policy for stream close (5xx retriable) (#3038)
[world] Guard hook_received against a concurrent run termination (#2987)
docs: fix stale/incorrect v5 API reference details (#3017)
Default WORKFLOW_PRECONDITION_GUARD on (#2946)
docs: replace migration guides with a Comparisons section (#2676)
feat(core): add experimental Hook minimum retention (#2865)
test: regression coverage for hook.resume() from isolated route bundles (o2flow beta.26 incident) (#3001)
docs(agents): note lint/format/typecheck are advisory, not blocking (#2886)
Retry transient connection timeouts (#3013)
fix(world-vercel): append caller User-Agent products instead of discarding them (#2998)
[ci] Enable NestJS e2e-vercel-prod and add to docs as "experimental" (#3011)
[ci] Benchmark comment: Best column + best/p75/p99 deltas (drop Avg/P10) (#3005)
docs: fall back to first child page for sidebar folders without an index (#3009)
[nest] Fix NestJS Vercel build output (#2988)
Avoid resolving run data for background steps (#2993)
chore(docs): update @vercel/geistdocs to 1.14.0 (#3002)
fix(docs): add version-switcher fallback redirects for pages missing in one version (#3003)
ci: update opencode to 1.18.4 and switch backport AI model to claude-fable-5 (#3006)
...
# Conflicts:
#	docs/content/docs/v4/deploying/meta.json
#	docs/content/docs/v5/deploying/meta.json
#	packages/core/src/runtime.ts
#	packages/world-local/src/index.ts
#	packages/world-postgres/src/index.ts
#	packages/world/src/events.ts
#	packages/world/src/interfaces.ts
#	packages/world/src/recovery.ts
#	workbench/nest/src/main.ts
#	workbench/sveltekit/src/hooks.server.ts
Sign up for freeto join this conversation on GitHub. Already have an account? Sign in to comment

Reviewers

@karthikscale3karthikscale3karthikscale3 approved these changes
@ijjkijjkAwaiting requested review from ijjk

Assignees

No one assigned

Labels

None yet

Projects

None yet

Milestone

No milestone

Development

Successfully merging this pull request may close these issues.

2 participants

@VaguelySerious@karthikscale3