From cb2e7622e57e712ea155f1ad93adcf957384f483 Mon Sep 17 00:00:00 2001 From: BigSimmo <87357024+BigSimmo@users.noreply.github.com> Date: Sat, 8 Aug 2026 23:58:33 +0800 Subject: [PATCH 1/3] ci: harden Lighthouse budget execution --- .../setup-lighthouse-chromium/action.yml | 12 +- .github/workflows/ci.yml | 2 +- .github/workflows/live-web-vitals.yml | 27 ++--- docs/branch-review-ledger.md | 1 + docs/scripts-index.md | 2 +- docs/testing.md | 10 +- lighthouse-budget.json | 88 +++++++------- package.json | 2 +- scripts/ci-change-scope.mjs | 10 +- scripts/lighthouse-time-budget.mjs | 24 ++++ scripts/live-web-vitals-inputs.mjs | 111 ++++++++++++++++++ scripts/run-lighthouse-budget.mjs | 57 +++++++-- tests/check-lighthouse-budget.test.ts | 36 ++++++ tests/ci-cache-safety.test.ts | 24 ++++ tests/live-web-vitals-inputs.test.ts | 36 ++++++ 15 files changed, 360 insertions(+), 82 deletions(-) create mode 100644 scripts/lighthouse-time-budget.mjs create mode 100644 scripts/live-web-vitals-inputs.mjs create mode 100644 tests/live-web-vitals-inputs.test.ts diff --git a/.github/actions/setup-lighthouse-chromium/action.yml b/.github/actions/setup-lighthouse-chromium/action.yml index 4e131508eb..1d69d93d16 100644 --- a/.github/actions/setup-lighthouse-chromium/action.yml +++ b/.github/actions/setup-lighthouse-chromium/action.yml @@ -1,7 +1,8 @@ name: Setup pinned Chromium for Lighthouse description: > - Install Playwright's managed Chromium and export PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH, so every - Lighthouse job in this repository measures with the same browser build for a given commit. + Install Playwright's managed Chromium and export CHROME_PATH plus + PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH, so every Lighthouse job in this repository measures with the + same browser build for a given commit. # The Lighthouse budget used to drive whatever Chrome ships in the ubuntu-24.04 runner image # (matching live-web-vitals.yml, which measures a live domain and isn't graded against a @@ -46,4 +47,9 @@ runs: - name: Pin the Chromium executable path shell: bash - run: echo "PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=$(node -e "console.log(require('playwright').chromium.executablePath())")" >> "$GITHUB_ENV" + run: | + chromium_path="$(node -e "console.log(require('playwright').chromium.executablePath())")" + { + echo "CHROME_PATH=$chromium_path" + echo "PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=$chromium_path" + } >> "$GITHUB_ENV" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8cdc3ba63b..4b2eee2e79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -851,7 +851,7 @@ jobs: - name: Show the baseline diff and the browser it was measured on run: | git --no-pager diff --stat -- lighthouse-budget.json - node -e "const b=require('./lighthouse-budget.json');console.log([...new Set(Object.values(b.baseline??{}).map(r=>r.chromeVersion))].join('\n'))" + node -e "const b=require('./lighthouse-budget.json');const versions=[...new Set(Object.values(b.baseline??{}).map(r=>r.chromeVersion).filter(v=>typeof v==='string'&&v.length>0))];console.log(versions.join('\n'));if(versions.length!==1){console.error('Expected exactly one baseline Chrome version; found '+versions.length+'.');process.exit(1)}" - name: Upload the refreshed baseline if: always() diff --git a/.github/workflows/live-web-vitals.yml b/.github/workflows/live-web-vitals.yml index 8e897cec5f..c7f0c575c5 100644 --- a/.github/workflows/live-web-vitals.yml +++ b/.github/workflows/live-web-vitals.yml @@ -56,7 +56,7 @@ jobs: measure: name: Lighthouse against the live domain runs-on: ubuntu-24.04 - timeout-minutes: 30 + timeout-minutes: 45 env: # Repository variable overrides the default (e.g. a staging cutover), # matching live-domain-monitor.yml. @@ -99,26 +99,13 @@ jobs: echo "LIVE_DOMAIN_URL=$normalized" >> "$GITHUB_ENV" echo "origin -> $normalized" - - name: Reject a sample count too small to grade + - name: Validate the bounded target matrix run: | set -euo pipefail - # `samples` is a free-text dispatch input, so `samples=1` is accepted - # by the form and would produce one report per cell — a "median" of - # one, a zero-width range, and a straddle check that can never fire. - # The summariser refuses the same count, but refusing HERE means the - # operator finds out in seconds instead of after a full measurement - # pass against the live domain. - case "$SAMPLES" in - ''|*[!0-9]*) - echo "::error::samples must be a positive integer, got '$SAMPLES'" - exit 1 - ;; - esac - if [ "$SAMPLES" -lt 3 ]; then - echo "::error::samples=$SAMPLES cannot be graded — #017 needs at least 3 runs per cell so a median has dispersion behind it" - exit 1 - fi - echo "samples -> $SAMPLES per route/strategy" + # Dispatch inputs are free text. Validate route identity, collision-free + # artifact names, the minimum sample count, and the capped total before + # this workflow makes any request to the live origin. + node scripts/live-web-vitals-inputs.mjs "$LIVE_DOMAIN_URL" "$ROUTES" "$SAMPLES" - name: Confirm the target is reachable before spending a Lighthouse run run: | @@ -147,7 +134,7 @@ jobs: echo "::group::$strategy $route (sample $sample/$SAMPLES)" # One flaky route must not discard the whole run, so a failure is # a warning here; the summary step fails if NOTHING was produced. - npx --yes "lighthouse@$LIGHTHOUSE_VERSION" "$LIVE_DOMAIN_URL$route" \ + timeout --signal=TERM --kill-after=10s 80s npx --yes "lighthouse@$LIGHTHOUSE_VERSION" "$LIVE_DOMAIN_URL$route" \ --output=json --output-path="${out}.json" \ --preset="$([ "$strategy" = desktop ] && echo desktop || echo perf)" \ --only-categories=performance \ diff --git a/docs/branch-review-ledger.md b/docs/branch-review-ledger.md index 114f425ae7..aceb0130d6 100644 --- a/docs/branch-review-ledger.md +++ b/docs/branch-review-ledger.md @@ -752,3 +752,4 @@ Records before 2026-07-28 were written by hand and had drifted: 146 lines carrie | 2026-08-08 | claude/ds-tap-and-linkaction | 6916c80526603514d91bd29d224959dd420af59c | M5 LinkAction tone refusal plus re-measured corrections to outstanding-issues #270, #118 and #269 — final reviewed head, adds the tone?: never fix, its type-contract test and both regenerated manifests | PR #1720, superseding the 824c1b74a record. Codex found the Omit form still accepted tone through a spread; verified with a focused tsc probe before changing anything (Omit accepted the spread with no diagnostic, tone?: never rejected it with TS2345), because excess-property checking only fires on object literals. Fixed with tone?: never plus a type-level contract test that stops compiling if the prop widens back. CodeRabbit's future-dated finding fixed in ff307cc5b. CodeRabbit's ledger-scope finding does not apply: that row records a different ref and head and was accurate as written, but a superseding row for the final #1719 head was appended anyway since its scope grew after the review pass | tsc -p tsconfig.typecheck.json --noEmit exit 0 zero diagnostics; lint exit 0; check:design-system-contract exit 0 (676 production files, legacy shadow aliases 228 confirming the #262 re-measure, adoption 53 components 55 roots, design-sync 53 components and 7 guidelines); check-icon-scale.mjs --strict exit 0; vitest threads pool 3 files 164 tests passed; check:outstanding-issues pass; check:branch-review-ledger pass; prettier --check . pass whole-tree; main merged in with merge-tree proven clean first and an id-set proof over both merge parents showing 274 ids each side, none lost, none invented | | 2026-08-08 | claude/ds-close-276 (PR #1724) | 75c89993f3ea23b70a250f605b21437b4ea9aac8 | PR #1724 review-and-fix | fixed Codex P2 wrong #118 Lighthouse cause (150 overwrite vs 151 pin); dispositioned CodeRabbit #276 archive claim as false (issues:done move); merge-tree clean; required CI was green on prior tip 8ae8c48f; no Bugbot findings | check:outstanding-issues pass; prettier --check docs/outstanding-issues.md pass; no provider-backed checks | | 2026-08-08 | claude/ds-close-276 (PR #1724) | 4baa9a1b42fa05731a6f983b3e0d0ebbd37f5271 | PR #1724 review-and-fix | synced origin/main (#1725 conflict on outstanding-issues resolved by preferring main queue then re-applying #276 done + corrected #118 diagnosis); Codex P2 fixed; CodeRabbit #276 archive claim dispositioned false; merge-tree clean after sync | check:outstanding-issues pass; prettier --check docs/outstanding-issues.md pass; merge-tree clean vs origin/main; no provider-backed checks | +| 2026-08-08 | origin/main (PR #1722 follow-up) | eda8fe872de040e304621bce49535e1dfebb091e | Lighthouse testing thorough review | P1 fixed locally: stale Chrome 150 baseline, unbounded runner phases, and live input/process limits hardened; advisory policy retained | offline review; check:ci-scope PASS; test:ci-workflows 13 files 248 passed 11 skipped; check:github-actions PASS; verify:lighthouse dry-run PASS; no provider-backed checks run | diff --git a/docs/scripts-index.md b/docs/scripts-index.md index 411ad1216f..8253f0adaf 100644 --- a/docs/scripts-index.md +++ b/docs/scripts-index.md @@ -1,6 +1,6 @@ # Scripts index -Curated map of `scripts/` (219 files) and the `package.json` script surface (231 entries), +Curated map of `scripts/` (221 files) and the `package.json` script surface (231 entries), grouped by purpose. This is orientation, not an exhaustive per-file listing — the authoritative command list is `package.json`, and `npm run docs:check-scripts` verifies every `npm run ` referenced in docs resolves to a real script. `npm run docs:update` refreshes the exact counts above. diff --git a/docs/testing.md b/docs/testing.md index 67058524c0..4c507aa835 100644 --- a/docs/testing.md +++ b/docs/testing.md @@ -139,6 +139,10 @@ following the same shape as `check:bundle-budget`. written to `retries.txt` whether or not it recovered. A run that never started measured nothing about the diff; it is not a pass either, so if the retry also produces nothing the grader still fails closed. A cell that _did_ measure and produced bad numbers is never retried. +- The 45-minute advisory CI job reserves 10 minutes for the isolated build, 2 minutes for server + readiness, and 28 minutes for the complete measurement suite. Each Lighthouse process receives + the lesser of its 120-second cap and the suite time remaining. If time runs out, unmeasured cells + remain missing and the grader fails closed, while the artifact still uploads for diagnosis. ### Baseline browser pinning, and how to refresh @@ -193,7 +197,11 @@ immediate run. This is distinct from `.github/workflows/live-web-vitals.yml`, which measures the deployed origin for ledger #017 and is dispatch-only — by the time it runs, `main` has already auto-deployed. Both pin the same Lighthouse version, and `tests/check-lighthouse-budget.test.ts` fails if they drift apart. -Neither uses secrets or providers. +Neither uses secrets or providers. The live workflow validates canonical root-relative, collision-free +route paths before it contacts the origin; it requires at least three samples and caps the complete +matrix at 30 Lighthouse calls. Its 45-minute job gives each live child 80 seconds, sends `TERM`, then +allows a 10-second kill grace. A failed cell remains a warning so the summary can retain its existing +evidence-based verdict rather than disguising a public-network failure as a local gate result. ## Flake policy diff --git a/lighthouse-budget.json b/lighthouse-budget.json index f2781e9272..112dedb74e 100644 --- a/lighthouse-budget.json +++ b/lighthouse-budget.json @@ -20,75 +20,75 @@ }, "baseline": { "desktop-documents-search": { - "lcpMs": 790.8669, - "cls": 0.1192626548872241, - "tbtMs": 0, - "fcpMs": 390.7482, - "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/150.0.0.0 Safari/537.36" + "lcpMs": 887, + "cls": 0.119, + "tbtMs": 1, + "fcpMs": 404.55240000000003, + "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/151.0.0.0 Safari/537.36" }, "desktop-dsm": { - "lcpMs": 836.0241999999998, - "cls": 0.01342963896133897, - "tbtMs": 14.999999999999886, - "fcpMs": 381.7212, - "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/150.0.0.0 Safari/537.36" + "lcpMs": 936, + "cls": 0.013, + "tbtMs": 0, + "fcpMs": 399.6637, + "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/151.0.0.0 Safari/537.36" }, "desktop-forms": { - "lcpMs": 870.6212000000005, - "cls": 0.05606642664873546, + "lcpMs": 870, + "cls": 0.06, "tbtMs": 0, - "fcpMs": 383.5404, - "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/150.0.0.0 Safari/537.36" + "fcpMs": 393.5287, + "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/151.0.0.0 Safari/537.36" }, "desktop-root": { - "lcpMs": 812.1882499999992, - "cls": 0.006778250591016549, - "tbtMs": 16, - "fcpMs": 340.8753, - "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/150.0.0.0 Safari/537.36" + "lcpMs": 822, + "cls": 0.007, + "tbtMs": 3, + "fcpMs": 358.3345, + "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/151.0.0.0 Safari/537.36" }, "desktop-therapy-compass": { - "lcpMs": 895.1419000000001, + "lcpMs": 822, "cls": 0, "tbtMs": 0, - "fcpMs": 389.1834, - "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/150.0.0.0 Safari/537.36" + "fcpMs": 397.3599, + "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/151.0.0.0 Safari/537.36" }, "mobile-documents-search": { - "lcpMs": 2314.371, + "lcpMs": 2311, "cls": 0, - "tbtMs": 383.43999999999915, - "fcpMs": 2314.371, - "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/150.0.0.0 Safari/537.36" + "tbtMs": 399, + "fcpMs": 2329.335, + "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/151.0.0.0 Safari/537.36" }, "mobile-dsm": { - "lcpMs": 2339.109, + "lcpMs": 2319, "cls": 0, - "tbtMs": 352.6870000000008, - "fcpMs": 2339.109, - "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/150.0.0.0 Safari/537.36" + "tbtMs": 322, + "fcpMs": 2325.841, + "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/151.0.0.0 Safari/537.36" }, "mobile-forms": { - "lcpMs": 2280.29, - "cls": 0.08045469317717091, - "tbtMs": 326.0409999999997, - "fcpMs": 2280.29, - "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/150.0.0.0 Safari/537.36" + "lcpMs": 2304, + "cls": 0.08, + "tbtMs": 324, + "fcpMs": 2291.215, + "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/151.0.0.0 Safari/537.36" }, "mobile-root": { - "lcpMs": 3929.966, + "lcpMs": 3274, "cls": 0, - "tbtMs": 785.6220000000021, - "fcpMs": 3929.966, - "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/150.0.0.0 Safari/537.36" + "tbtMs": 763, + "fcpMs": 3274, + "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/151.0.0.0 Safari/537.36" }, "mobile-therapy-compass": { - "lcpMs": 2336.784, + "lcpMs": 2338, "cls": 0, - "tbtMs": 333.18399999999883, - "fcpMs": 2336.784, - "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/150.0.0.0 Safari/537.36" + "tbtMs": 302, + "fcpMs": 2327.357, + "chromeVersion": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/151.0.0.0 Safari/537.36" } }, - "updatedAt": "2026-08-08T05:38:39.988Z" + "updatedAt": "2026-08-07T13:42:17.043Z" } diff --git a/package.json b/package.json index fa1757c5d9..fbe6370d53 100644 --- a/package.json +++ b/package.json @@ -45,7 +45,7 @@ "test:coverage": "node scripts/run-vitest.mjs run --coverage", "test:coverage:node": "node scripts/run-vitest.mjs run --project=node --coverage", "test:coverage:ui": "node scripts/run-vitest.mjs run --project=jsdom --coverage", - "test:ci-workflows": "node scripts/run-vitest.mjs run tests/ci-cache-safety.test.ts tests/authenticated-live-workflow.test.ts tests/codex-autofix-workflow.test.ts tests/eval-canary-workflow.test.ts tests/container-ci-contract.test.ts tests/test-runner-safety.test.ts tests/installed-lock-parity.test.ts tests/railway-config.test.ts tests/ingestion-autopilot.test.ts tests/ingestion-autopilot-workflow.test.ts tests/check-lighthouse-budget.test.ts tests/offline-release-profile.test.ts", + "test:ci-workflows": "node scripts/run-vitest.mjs run tests/ci-cache-safety.test.ts tests/authenticated-live-workflow.test.ts tests/codex-autofix-workflow.test.ts tests/eval-canary-workflow.test.ts tests/container-ci-contract.test.ts tests/test-runner-safety.test.ts tests/installed-lock-parity.test.ts tests/railway-config.test.ts tests/ingestion-autopilot.test.ts tests/ingestion-autopilot-workflow.test.ts tests/check-lighthouse-budget.test.ts tests/live-web-vitals-inputs.test.ts tests/offline-release-profile.test.ts", "test:e2e": "node scripts/run-playwright.mjs", "test:e2e:all": "node scripts/run-playwright.mjs", "test:e2e:accessibility": "node scripts/run-playwright.mjs tests/ui-accessibility.spec.ts --project=chromium", diff --git a/scripts/ci-change-scope.mjs b/scripts/ci-change-scope.mjs index 5018332b21..7993c81e5f 100644 --- a/scripts/ci-change-scope.mjs +++ b/scripts/ci-change-scope.mjs @@ -205,7 +205,7 @@ const perfPatterns = [ // measuredRequestedPage from it — editing it changes the VERDICT, and before this // it matched no CI scope pattern at all. "lighthouse-budget.json", - /^scripts\/(run-lighthouse-budget|check-lighthouse-budget|summarise-web-vitals|lighthouse-measurement-outcome)\.mjs$/, + /^scripts\/(run-lighthouse-budget|check-lighthouse-budget|summarise-web-vitals|lighthouse-measurement-outcome|lighthouse-time-budget|child-process-result|test-environment)\.mjs$/, // Measuring and refreshing jobs MUST resolve the same Chromium. Editing this action // changes which browser grades the baseline, so it belongs in perf scope even when // no application source moved. @@ -908,6 +908,14 @@ function selfTest() { assertScope("perf-on-for-retry-outcome-module", ["scripts/lighthouse-measurement-outcome.mjs"], { perf_changed: true, }); + assertScope( + "perf-on-for-runner-dependencies", + ["scripts/lighthouse-time-budget.mjs", "scripts/child-process-result.mjs", "scripts/test-environment.mjs"], + { perf_changed: true }, + ); + // The Lighthouse script regex is intentionally anchored. A sibling script must + // not turn every scripts/ edit into an unnecessary full budget measurement. + assertScope("perf-off-for-unrelated-script", ["scripts/run-vitest.mjs"], { perf_changed: false }); assertScope("perf-on-for-chromium-pin-action", [".github/actions/setup-lighthouse-chromium/action.yml"], { workflow_changed: true, perf_changed: true, diff --git a/scripts/lighthouse-time-budget.mjs b/scripts/lighthouse-time-budget.mjs new file mode 100644 index 0000000000..1415c9bbec --- /dev/null +++ b/scripts/lighthouse-time-budget.mjs @@ -0,0 +1,24 @@ +/** + * Bounded-stage timing helpers for the local Lighthouse budget runner. + * + * The CI job allows 45 minutes. Its runner reserves 10 minutes for the Next + * build, 2 minutes for readiness, and 28 minutes for the complete measurement + * suite, leaving five minutes for grading, cleanup, and artifact upload. + */ +export const LIGHTHOUSE_BUILD_TIMEOUT_MS = 10 * 60_000; +export const LIGHTHOUSE_SERVER_READY_TIMEOUT_MS = 2 * 60_000; +export const LIGHTHOUSE_MEASUREMENT_SUITE_TIMEOUT_MS = 28 * 60_000; +export const LIGHTHOUSE_PROCESS_TIMEOUT_MS = 120_000; + +export function deadlineAfter(timeoutMs, now = Date.now()) { + return now + timeoutMs; +} + +export function remainingMs(deadline, now = Date.now()) { + return Math.max(0, deadline - now); +} + +/** The next Lighthouse child may use no more than the suite time still available. */ +export function processTimeoutMs(deadline, maximumMs, now = Date.now()) { + return Math.min(maximumMs, remainingMs(deadline, now)); +} diff --git a/scripts/live-web-vitals-inputs.mjs b/scripts/live-web-vitals-inputs.mjs new file mode 100644 index 0000000000..4d7fee2d9f --- /dev/null +++ b/scripts/live-web-vitals-inputs.mjs @@ -0,0 +1,111 @@ +#!/usr/bin/env node +/** + * Validate the dispatch-only live Web Vitals matrix before it can spend time on + * the public origin. Keeping this outside YAML makes the same rules unit-testable. + */ +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import { collidingRouteSlugs, WEB_VITALS_STRATEGIES, WEB_VITALS_MIN_SAMPLES } from "./summarise-web-vitals.mjs"; + +export const MAX_LIVE_LIGHTHOUSE_INVOCATIONS = 30; + +function parseOrigin(value) { + let origin; + try { + origin = new URL(String(value ?? "")); + } catch { + throw new Error(`LIVE_DOMAIN_URL must be an absolute HTTP(S) origin, got ${JSON.stringify(value)}.`); + } + + if (!["http:", "https:"].includes(origin.protocol) || origin.pathname !== "/" || origin.search || origin.hash) { + throw new Error( + `LIVE_DOMAIN_URL must be an absolute HTTP(S) origin without a path, query, or hash, got ${origin}.`, + ); + } + + return origin.origin; +} + +function parseRoutes(value, origin) { + const routes = String(value ?? "") + .split(",") + .map((route) => route.trim()); + + if (routes.length === 0 || routes.some((route) => !route)) { + throw new Error("routes must be a non-empty comma-separated list without blank entries."); + } + + const normalized = routes.map((route) => { + if ( + !route.startsWith("/") || + route.startsWith("//") || + route.includes("@") || + (route.length > 1 && route.endsWith("/")) + ) { + throw new Error(`route must be a canonical root-relative path, got ${JSON.stringify(route)}.`); + } + + const resolved = new URL(route, origin); + if ( + resolved.origin !== origin || + resolved.search || + resolved.hash || + resolved.pathname !== route || + route.includes("//") + ) { + throw new Error(`route must stay on ${origin} as a canonical root-relative path, got ${JSON.stringify(route)}.`); + } + + return route; + }); + + const duplicate = normalized.find((route, index) => normalized.indexOf(route) !== index); + if (duplicate) throw new Error(`routes contains duplicate path ${JSON.stringify(duplicate)}.`); + + const collisions = collidingRouteSlugs(normalized); + if (collisions.length > 0) { + throw new Error(`routes cannot share Lighthouse filename slugs: ${collisions.join(", ")}.`); + } + + return normalized; +} + +function parseSamples(value) { + const raw = String(value ?? "").trim(); + if (!/^[0-9]+$/.test(raw)) throw new Error(`samples must be a positive integer, got ${JSON.stringify(value)}.`); + + const samples = Number(raw); + if (!Number.isSafeInteger(samples) || samples < WEB_VITALS_MIN_SAMPLES) { + throw new Error(`samples must be at least ${WEB_VITALS_MIN_SAMPLES}, got ${raw}.`); + } + + return samples; +} + +export function parseLiveWebVitalsInputs({ origin, routes, samples }) { + const normalizedOrigin = parseOrigin(origin); + const normalizedRoutes = parseRoutes(routes, normalizedOrigin); + const normalizedSamples = parseSamples(samples); + const invocations = normalizedRoutes.length * WEB_VITALS_STRATEGIES.length * normalizedSamples; + + if (invocations > MAX_LIVE_LIGHTHOUSE_INVOCATIONS) { + throw new Error( + `requested ${invocations} Lighthouse runs; maximum is ${MAX_LIVE_LIGHTHOUSE_INVOCATIONS} ` + + "(routes × mobile/desktop × samples).", + ); + } + + return { origin: normalizedOrigin, routes: normalizedRoutes, samples: normalizedSamples, invocations }; +} + +if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1])) { + const result = parseLiveWebVitalsInputs({ + origin: process.argv[2], + routes: process.argv[3], + samples: process.argv[4], + }); + console.log( + `live Lighthouse matrix -> ${result.invocations} runs (${result.routes.length} routes × 2 strategies × ${result.samples} samples)`, + ); +} diff --git a/scripts/run-lighthouse-budget.mjs b/scripts/run-lighthouse-budget.mjs index 5eede6b2c6..c4098fd821 100644 --- a/scripts/run-lighthouse-budget.mjs +++ b/scripts/run-lighthouse-budget.mjs @@ -45,6 +45,15 @@ import { offlineTestEnvironment } from "./test-environment.mjs"; import { acquireHeavyRunLock } from "./test-run-lock.mjs"; import { loadBudget } from "./check-lighthouse-budget.mjs"; import { measurementFailureReason } from "./lighthouse-measurement-outcome.mjs"; +import { + deadlineAfter, + LIGHTHOUSE_BUILD_TIMEOUT_MS, + LIGHTHOUSE_MEASUREMENT_SUITE_TIMEOUT_MS, + LIGHTHOUSE_PROCESS_TIMEOUT_MS, + LIGHTHOUSE_SERVER_READY_TIMEOUT_MS, + processTimeoutMs, + remainingMs, +} from "./lighthouse-time-budget.mjs"; import { appName, circularProjectPortRange, @@ -131,9 +140,9 @@ async function findFreePort(startPort) { throw new Error("No free Lighthouse server port found in the configured project range."); } -function get(url) { +function get(url, timeoutMs) { return new Promise((resolve) => { - const request = http.get(url, { timeout: 5_000 }, (response) => { + const request = http.get(url, { timeout: timeoutMs }, (response) => { let body = ""; response.setEncoding("utf8"); response.on("data", (chunk) => { @@ -160,20 +169,26 @@ function isThisProject(body) { } } -async function waitForServer(baseUrl, server) { - for (let attempt = 0; attempt < 120; attempt += 1) { +async function waitForServer(baseUrl, server, timeoutMs) { + const deadline = deadlineAfter(timeoutMs); + while (remainingMs(deadline) > 0) { if (server.exitCode !== null || server.signalCode) { throw new Error("Lighthouse-owned Next server exited before it became ready."); } // Same identity check the rest of the repo's tooling uses, so this can never // attach to another project's server on a shared machine. - const body = await get(`${baseUrl}/api/local-project-id`); + const requestTimeout = Math.min(5_000, remainingMs(deadline)); + // Node treats a zero HTTP timeout as "disabled". The deadline can elapse + // between the loop condition and this request, so never pass zero through. + if (requestTimeout === 0) break; + const body = await get(`${baseUrl}/api/local-project-id`, requestTimeout); // A 200 with any body is not proof this is our app: another service could have // taken the port between the availability probe and Next binding it, and // Lighthouse would then measure the wrong application. Verify identity the way // scripts/playwright-base-url.ts does before accepting readiness. if (body && isThisProject(body)) return; - await new Promise((resolve) => setTimeout(resolve, 1_000)); + const delayMs = Math.min(1_000, remainingMs(deadline)); + if (delayMs > 0) await new Promise((resolve) => setTimeout(resolve, delayMs)); } throw new Error(`Timed out waiting for the Lighthouse-owned server at ${baseUrl}.`); } @@ -305,6 +320,7 @@ try { cwd: projectRoot, env: offlineEnv, stdio: "inherit", + timeout: LIGHTHOUSE_BUILD_TIMEOUT_MS, }); if (childProcessExitCode(build) !== 0) { throw new Error(`Lighthouse production build failed (${childProcessFailureSummary(build)}).`); @@ -318,13 +334,16 @@ try { stdio: ["ignore", "inherit", "inherit"], windowsHide: true, }); - await waitForServer(baseUrl, server); + await waitForServer(baseUrl, server, LIGHTHOUSE_SERVER_READY_TIMEOUT_MS); const chromePath = process.env.CHROME_PATH ?? process.env.PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH ?? ""; const failures = []; const retried = []; - const measure = (strategy, route, output) => + const suiteDeadline = deadlineAfter(LIGHTHOUSE_MEASUREMENT_SUITE_TIMEOUT_MS); + console.log(`Lighthouse measurement suite has ${LIGHTHOUSE_MEASUREMENT_SUITE_TIMEOUT_MS / 60_000} minutes.`); + + const measure = (strategy, route, output, timeout) => spawnSync( npxInvocation.command, [ @@ -344,6 +363,10 @@ try { cwd: projectRoot, env: { ...offlineEnv, ...(chromePath ? { CHROME_PATH: chromePath } : {}) }, stdio: "inherit", + // --max-wait-for-load bounds navigation only. A stuck Chrome or Lighthouse + // process must become one failed, retryable cell rather than consume the + // whole CI job and prevent its evidence artifact from uploading. + timeout, }, ); @@ -353,8 +376,14 @@ try { for (const route of routes) { const cell = `${strategy} ${route}`; const output = path.join(reportDirectory, `${strategy}-${slugFor(route)}.json`); + const firstAttemptTimeout = processTimeoutMs(suiteDeadline, LIGHTHOUSE_PROCESS_TIMEOUT_MS); + if (firstAttemptTimeout === 0) { + failures.push(cell); + console.log(`::warning::lighthouse ${cell} was not measured: the 28-minute suite deadline expired`); + continue; + } console.log(`Measuring ${cell}`); - let result = measure(strategy, route, output); + let result = measure(strategy, route, output, firstAttemptTimeout); let reason = measurementFailureReason(childProcessExitCode(result), readIfPresent(output)); if (reason) { @@ -365,12 +394,20 @@ try { // the grader's incompleteBudgetEvidence still fails closed regardless of // `enforce`. The retry is reported either way, so a chronically flaky route // cannot hide behind a green run. + const retryTimeout = processTimeoutMs(suiteDeadline, LIGHTHOUSE_PROCESS_TIMEOUT_MS); + if (retryTimeout === 0) { + failures.push(cell); + console.log( + `::warning::lighthouse ${cell} produced no measurement (${reason}); the suite deadline expired before retry`, + ); + continue; + } console.log(`::warning::lighthouse ${cell} produced no measurement (${reason}); retrying once`); // Never leave the first attempt's file behind: a runtimeError report would // otherwise be graded, or baked into a refreshed baseline, if the retry fails // before writing. rmSync(output, { force: true }); - result = measure(strategy, route, output); + result = measure(strategy, route, output, retryTimeout); const after = measurementFailureReason(childProcessExitCode(result), readIfPresent(output)); retried.push(`${cell} (${reason}${after ? ` -> still ${after}` : " -> recovered"})`); reason = after; diff --git a/tests/check-lighthouse-budget.test.ts b/tests/check-lighthouse-budget.test.ts index 50579a75d4..1d8490e7a1 100644 --- a/tests/check-lighthouse-budget.test.ts +++ b/tests/check-lighthouse-budget.test.ts @@ -15,6 +15,7 @@ import { renderBudgetTable, } from "../scripts/check-lighthouse-budget.mjs"; import { measurementFailureReason } from "../scripts/lighthouse-measurement-outcome.mjs"; +import { deadlineAfter, processTimeoutMs, remainingMs } from "../scripts/lighthouse-time-budget.mjs"; /** Kept in step with lighthouse-budget.json. */ const ROUTES = ["/", "/therapy-compass", "/documents/search", "/dsm", "/forms"]; @@ -372,6 +373,7 @@ describe("committed lighthouse-budget.json", () => { strategies: string[]; lighthouseVersion: string; enforce: boolean; + baseline: Record; }; it("measures the routes this suite grades", () => { @@ -393,6 +395,18 @@ describe("committed lighthouse-budget.json", () => { expect(committed.strategies).toEqual(["mobile", "desktop"]); }); + it("records a complete baseline from one named browser identity", () => { + const rows = Object.values(committed.baseline ?? {}); + const versions = rows + .map((row) => row.chromeVersion) + .filter((version): version is string => typeof version === "string" && version.length > 0); + + expect(rows).toHaveLength(committed.routes.length * committed.strategies.length); + expect(versions).toHaveLength(rows.length); + expect(new Set(versions).size).toBe(1); + expect(versions[0]).toContain("HeadlessChrome/"); + }); + it("measures every route without a query string", () => { // Budget routes stay query-free so a silent `?q=` addition cannot hide new // client-driven API traffic. This is a signal, not a complete proof that no API @@ -411,6 +425,28 @@ describe("committed lighthouse-budget.json", () => { expect(runner).not.toMatch(/spawnSync\(\s*"npx",/); expect(runner).not.toMatch(/spawnSync\(\s*"npx\.cmd",/); }); + + it("bounds each Lighthouse process independently of its navigation timeout", () => { + const runner = readFileSync(path.join(process.cwd(), "scripts", "run-lighthouse-budget.mjs"), "utf8"); + + expect(runner).toContain("timeout: LIGHTHOUSE_BUILD_TIMEOUT_MS"); + expect(runner).toContain("waitForServer(baseUrl, server, LIGHTHOUSE_SERVER_READY_TIMEOUT_MS)"); + expect(runner).toContain("if (requestTimeout === 0) break"); + expect(runner).toContain("deadlineAfter(LIGHTHOUSE_MEASUREMENT_SUITE_TIMEOUT_MS)"); + expect(runner).toContain("LIGHTHOUSE_PROCESS_TIMEOUT_MS"); + expect(runner).toContain("--max-wait-for-load=60000"); + }); +}); + +describe("Lighthouse time budget", () => { + it("uses a real deadline for server readiness and each process", () => { + const deadline = deadlineAfter(120_000, 1_000); + + expect(deadline).toBe(121_000); + expect(remainingMs(deadline, 61_000)).toBe(60_000); + expect(processTimeoutMs(deadline, 120_000, 61_000)).toBe(60_000); + expect(processTimeoutMs(deadline, 120_000, deadline)).toBe(0); + }); }); describe("measurementFailureReason", () => { diff --git a/tests/ci-cache-safety.test.ts b/tests/ci-cache-safety.test.ts index 444d39912d..a48ed24386 100644 --- a/tests/ci-cache-safety.test.ts +++ b/tests/ci-cache-safety.test.ts @@ -5,7 +5,15 @@ import { describe, expect, it } from "vitest"; const nodeSetup = readFileSync(new URL("../.github/actions/setup-node-cached/action.yml", import.meta.url), "utf8"); const uiSetup = readFileSync(new URL("../.github/actions/setup-ui-e2e/action.yml", import.meta.url), "utf8"); +const lighthouseChromiumSetup = readFileSync( + new URL("../.github/actions/setup-lighthouse-chromium/action.yml", import.meta.url), + "utf8", +); const workflow = readFileSync(new URL("../.github/workflows/ci.yml", import.meta.url), "utf8"); +const liveWebVitalsWorkflow = readFileSync( + new URL("../.github/workflows/live-web-vitals.yml", import.meta.url), + "utf8", +); const opsDigestWorkflow = readFileSync(new URL("../.github/workflows/ops-digest.yml", import.meta.url), "utf8"); describe("CI cache safety", () => { @@ -30,6 +38,22 @@ describe("CI cache safety", () => { expect(workflow).toMatch(/cache-hit.*?install-deps\n\s+npx playwright install/s); }); + it("rejects a refreshed Lighthouse baseline that has zero or mixed browser identities", () => { + expect(workflow).toContain("versions.length!==1"); + expect(workflow).toContain("Expected exactly one baseline Chrome version"); + }); + + it("exports the pinned browser through both Lighthouse environment contracts", () => { + expect(lighthouseChromiumSetup).toContain("CHROME_PATH=$chromium_path"); + expect(lighthouseChromiumSetup).toContain("PLAYWRIGHT_CHROMIUM_EXECUTABLE_PATH=$chromium_path"); + }); + + it("caps the dispatch-only live Lighthouse matrix and each child process", () => { + expect(liveWebVitalsWorkflow).toContain("timeout-minutes: 45"); + expect(liveWebVitalsWorkflow).toContain("node scripts/live-web-vitals-inputs.mjs"); + expect(liveWebVitalsWorkflow).toContain("timeout --signal=TERM --kill-after=10s 80s"); + }); + it("routes recognised workflow-only changes through focused contracts", () => { expect(workflow).toContain("static_heavy_changed: ${{ steps.scope.outputs.static_heavy_changed }}"); expect(workflow).toContain("workflow_changed: ${{ steps.scope.outputs.workflow_changed }}"); diff --git a/tests/live-web-vitals-inputs.test.ts b/tests/live-web-vitals-inputs.test.ts new file mode 100644 index 0000000000..e9c5985c45 --- /dev/null +++ b/tests/live-web-vitals-inputs.test.ts @@ -0,0 +1,36 @@ +import { describe, expect, it } from "vitest"; + +import { MAX_LIVE_LIGHTHOUSE_INVOCATIONS, parseLiveWebVitalsInputs } from "../scripts/live-web-vitals-inputs.mjs"; + +const origin = "https://psychiatry.tools"; + +describe("parseLiveWebVitalsInputs", () => { + it("normalizes a bounded, same-origin dispatch matrix", () => { + expect( + parseLiveWebVitalsInputs({ + origin: `${origin}/`, + routes: " /, /therapy-compass, /documents/search ", + samples: "3", + }), + ).toEqual({ origin, routes: ["/", "/therapy-compass", "/documents/search"], samples: 3, invocations: 18 }); + }); + + it.each([ + ["an absolute URL", "https://other.example/forms", "canonical root-relative"], + ["a protocol-relative URL", "//other.example/forms", "canonical root-relative"], + ["an at-sign path", "/@other", "canonical root-relative"], + ["a query string", "/forms?preview=1", "canonical root-relative"], + ["a trailing-slash redirect", "/forms/", "canonical root-relative"], + ["a duplicate route", "/forms,/forms", "duplicate path"], + ["a colliding artifact slug", "/a/b,/a-b", "filename slugs"], + ])("rejects %s", (_name, routes, message) => { + expect(() => parseLiveWebVitalsInputs({ origin, routes, samples: "3" })).toThrow(message); + }); + + it("requires enough samples without allowing an unbounded suite", () => { + expect(() => parseLiveWebVitalsInputs({ origin, routes: "/forms", samples: "2" })).toThrow("at least 3"); + expect(() => + parseLiveWebVitalsInputs({ origin, routes: "/,/therapy-compass,/documents/search,/dsm,/forms", samples: "4" }), + ).toThrow(`maximum is ${MAX_LIVE_LIGHTHOUSE_INVOCATIONS}`); + }); +}); From be77503241d99024606cd43148493ae9e97f1584 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 16:46:48 +0000 Subject: [PATCH 2/3] fix(ci): kill Lighthouse process trees and bound live vitals suite time - run-lighthouse-budget: spawn Lighthouse in an owned process group and stopOwnedProcessTree on per-cell timeout instead of spawnSync timeout (which only stopped the npx wrapper). - live-web-vitals: export normalized ROUTES/SAMPLES to GITHUB_ENV; cap the measurement loop with a 32-minute suite deadline inside the 45-minute job. - Update contract tests for spawn-based measurement and whitespace samples. --- .github/workflows/live-web-vitals.yml | 36 +++++++---- scripts/live-web-vitals-inputs.mjs | 21 +++++++ scripts/run-lighthouse-budget.mjs | 89 ++++++++++++++++++--------- tests/check-lighthouse-budget.test.ts | 5 +- tests/live-web-vitals-inputs.test.ts | 2 +- 5 files changed, 111 insertions(+), 42 deletions(-) diff --git a/.github/workflows/live-web-vitals.yml b/.github/workflows/live-web-vitals.yml index c7f0c575c5..a1a07924e7 100644 --- a/.github/workflows/live-web-vitals.yml +++ b/.github/workflows/live-web-vitals.yml @@ -122,26 +122,38 @@ jobs: set -euo pipefail mkdir -p web-vitals IFS=',' read -ra route_list <<< "$ROUTES" + suite_deadline=$((SECONDS + LIVE_WEB_VITALS_MEASUREMENT_SUITE_SECONDS)) + suite_expired=0 for strategy in mobile desktop; do + if [ "$suite_expired" -eq 1 ]; then break; fi for route in "${route_list[@]}"; do + if [ "$suite_expired" -eq 1 ]; then break; fi route="$(echo "$route" | xargs)" [ -n "$route" ] || continue # Filename-safe slug: "/" -> root, "/a/b" -> a-b slug="$(echo "$route" | sed 's|^/||; s|/|-|g')" [ -n "$slug" ] || slug="root" for sample in $(seq 1 "$SAMPLES"); do - out="web-vitals/${strategy}-${slug}-${sample}" - echo "::group::$strategy $route (sample $sample/$SAMPLES)" - # One flaky route must not discard the whole run, so a failure is - # a warning here; the summary step fails if NOTHING was produced. - timeout --signal=TERM --kill-after=10s 80s npx --yes "lighthouse@$LIGHTHOUSE_VERSION" "$LIVE_DOMAIN_URL$route" \ - --output=json --output-path="${out}.json" \ - --preset="$([ "$strategy" = desktop ] && echo desktop || echo perf)" \ - --only-categories=performance \ - --chrome-flags="--headless=new --no-sandbox --disable-dev-shm-usage" \ - --max-wait-for-load=60000 \ - --quiet || echo "::warning::lighthouse failed for $strategy $route sample $sample" - echo "::endgroup::" + remaining=$((suite_deadline - SECONDS)) + if [ "$remaining" -le 0 ]; then + echo "::warning::live web vitals measurement suite deadline expired; skipping remaining cells" + suite_expired=1 + break + fi + run_timeout="$LIVE_WEB_VITALS_PROCESS_TIMEOUT_SEC" + if [ "$remaining" -lt "$run_timeout" ]; then run_timeout="$remaining"; fi + out="web-vitals/${strategy}-${slug}-${sample}" + echo "::group::$strategy $route (sample $sample/$SAMPLES)" + # One flaky route must not discard the whole run, so a failure is + # a warning here; the summary step fails if NOTHING was produced. + timeout --signal=TERM --kill-after=10s "${run_timeout}s" npx --yes "lighthouse@$LIGHTHOUSE_VERSION" "$LIVE_DOMAIN_URL$route" \ + --output=json --output-path="${out}.json" \ + --preset="$([ "$strategy" = desktop ] && echo desktop || echo perf)" \ + --only-categories=performance \ + --chrome-flags="--headless=new --no-sandbox --disable-dev-shm-usage" \ + --max-wait-for-load=60000 \ + --quiet || echo "::warning::lighthouse failed for $strategy $route sample $sample" + echo "::endgroup::" done done done diff --git a/scripts/live-web-vitals-inputs.mjs b/scripts/live-web-vitals-inputs.mjs index 4d7fee2d9f..187c447dbc 100644 --- a/scripts/live-web-vitals-inputs.mjs +++ b/scripts/live-web-vitals-inputs.mjs @@ -3,12 +3,21 @@ * Validate the dispatch-only live Web Vitals matrix before it can spend time on * the public origin. Keeping this outside YAML makes the same rules unit-testable. */ +import { appendFileSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { collidingRouteSlugs, WEB_VITALS_STRATEGIES, WEB_VITALS_MIN_SAMPLES } from "./summarise-web-vitals.mjs"; export const MAX_LIVE_LIGHTHOUSE_INVOCATIONS = 30; +/** Per-run cap; matches the GNU timeout wrapper in live-web-vitals.yml. */ +export const LIVE_WEB_VITALS_PROCESS_TIMEOUT_SEC = 80; +/** + * The live job allows 45 minutes. Reserve ~13 minutes for checkout, validation, + * reachability, summarising, and artifact upload so measurement cannot consume + * the whole job before upload. + */ +export const LIVE_WEB_VITALS_MEASUREMENT_SUITE_TIMEOUT_MS = 32 * 60_000; function parseOrigin(value) { let origin; @@ -108,4 +117,16 @@ if (process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1 console.log( `live Lighthouse matrix -> ${result.invocations} runs (${result.routes.length} routes × 2 strategies × ${result.samples} samples)`, ); + if (process.env.GITHUB_ENV) { + appendFileSync(process.env.GITHUB_ENV, `ROUTES=${result.routes.join(",")}\n`); + appendFileSync(process.env.GITHUB_ENV, `SAMPLES=${result.samples}\n`); + appendFileSync( + process.env.GITHUB_ENV, + `LIVE_WEB_VITALS_PROCESS_TIMEOUT_SEC=${LIVE_WEB_VITALS_PROCESS_TIMEOUT_SEC}\n`, + ); + appendFileSync( + process.env.GITHUB_ENV, + `LIVE_WEB_VITALS_MEASUREMENT_SUITE_SECONDS=${LIVE_WEB_VITALS_MEASUREMENT_SUITE_TIMEOUT_MS / 1000}\n`, + ); + } } diff --git a/scripts/run-lighthouse-budget.mjs b/scripts/run-lighthouse-budget.mjs index c4098fd821..6bc98da207 100644 --- a/scripts/run-lighthouse-budget.mjs +++ b/scripts/run-lighthouse-budget.mjs @@ -197,6 +197,23 @@ let server = null; let released = false; let lock = null; +function stopOwnedProcessTree(child) { + if (!child?.pid || child.exitCode !== null) return; + if (process.platform === "win32") { + spawnSync("taskkill", ["/PID", String(child.pid), "/T", "/F"], { stdio: "ignore" }); + return; + } + try { + process.kill(-child.pid, "SIGTERM"); + } catch { + try { + child.kill("SIGTERM"); + } catch { + /* already gone */ + } + } +} + function cleanup() { if (released) return; released = true; @@ -343,32 +360,48 @@ try { const suiteDeadline = deadlineAfter(LIGHTHOUSE_MEASUREMENT_SUITE_TIMEOUT_MS); console.log(`Lighthouse measurement suite has ${LIGHTHOUSE_MEASUREMENT_SUITE_TIMEOUT_MS / 60_000} minutes.`); - const measure = (strategy, route, output, timeout) => - spawnSync( - npxInvocation.command, - [ - ...npxInvocation.prefixArgs, - "--yes", - `lighthouse@${LIGHTHOUSE_VERSION}`, - `${baseUrl}${route}`, - "--output=json", - `--output-path=${output}`, - `--preset=${strategy === "desktop" ? "desktop" : "perf"}`, - "--only-categories=performance", - "--chrome-flags=--headless=new --no-sandbox --disable-dev-shm-usage", - "--max-wait-for-load=60000", - "--quiet", - ], - { - cwd: projectRoot, - env: { ...offlineEnv, ...(chromePath ? { CHROME_PATH: chromePath } : {}) }, - stdio: "inherit", - // --max-wait-for-load bounds navigation only. A stuck Chrome or Lighthouse - // process must become one failed, retryable cell rather than consume the - // whole CI job and prevent its evidence artifact from uploading. - timeout, - }, - ); + const measure = (strategy, route, output, timeoutMs) => + new Promise((resolve) => { + const child = spawn( + npxInvocation.command, + [ + ...npxInvocation.prefixArgs, + "--yes", + `lighthouse@${LIGHTHOUSE_VERSION}`, + `${baseUrl}${route}`, + "--output=json", + `--output-path=${output}`, + `--preset=${strategy === "desktop" ? "desktop" : "perf"}`, + "--only-categories=performance", + "--chrome-flags=--headless=new --no-sandbox --disable-dev-shm-usage", + "--max-wait-for-load=60000", + "--quiet", + ], + { + cwd: projectRoot, + env: { ...offlineEnv, ...(chromePath ? { CHROME_PATH: chromePath } : {}) }, + stdio: "inherit", + // Own the npx/Lighthouse/Chrome tree so a per-cell timeout can SIGTERM + // the whole group. spawnSync's timeout only stops the npx wrapper. + detached: process.platform !== "win32", + }, + ); + + let settled = false; + const finish = (result) => { + if (settled) return; + settled = true; + clearTimeout(timer); + resolve(result); + }; + + child.on("error", (error) => finish({ status: 1, error })); + child.on("close", (code, signal) => finish({ status: code, signal })); + + const timer = setTimeout(() => { + stopOwnedProcessTree(child); + }, timeoutMs); + }); const readIfPresent = (file) => (existsSync(file) ? readFileSync(file, "utf8") : null); @@ -383,7 +416,7 @@ try { continue; } console.log(`Measuring ${cell}`); - let result = measure(strategy, route, output, firstAttemptTimeout); + let result = await measure(strategy, route, output, firstAttemptTimeout); let reason = measurementFailureReason(childProcessExitCode(result), readIfPresent(output)); if (reason) { @@ -407,7 +440,7 @@ try { // otherwise be graded, or baked into a refreshed baseline, if the retry fails // before writing. rmSync(output, { force: true }); - result = measure(strategy, route, output, retryTimeout); + result = await measure(strategy, route, output, retryTimeout); const after = measurementFailureReason(childProcessExitCode(result), readIfPresent(output)); retried.push(`${cell} (${reason}${after ? ` -> still ${after}` : " -> recovered"})`); reason = after; diff --git a/tests/check-lighthouse-budget.test.ts b/tests/check-lighthouse-budget.test.ts index 1d8490e7a1..e8c99ba2d8 100644 --- a/tests/check-lighthouse-budget.test.ts +++ b/tests/check-lighthouse-budget.test.ts @@ -420,10 +420,12 @@ describe("committed lighthouse-budget.json", () => { expect(runner).toContain('const npxCli = path.join(path.dirname(npmExecPath), "npx-cli.js")'); expect(runner).toContain("process.env.npm_node_execpath ?? process.execPath"); - expect(runner).toMatch(/spawnSync\(\s*npxInvocation\.command,/); + expect(runner).toMatch(/spawn\(\s*npxInvocation\.command,/); expect(runner).toContain("...npxInvocation.prefixArgs"); expect(runner).not.toMatch(/spawnSync\(\s*"npx",/); expect(runner).not.toMatch(/spawnSync\(\s*"npx\.cmd",/); + expect(runner).toContain("stopOwnedProcessTree(child)"); + expect(runner).toContain('detached: process.platform !== "win32"'); }); it("bounds each Lighthouse process independently of its navigation timeout", () => { @@ -435,6 +437,7 @@ describe("committed lighthouse-budget.json", () => { expect(runner).toContain("deadlineAfter(LIGHTHOUSE_MEASUREMENT_SUITE_TIMEOUT_MS)"); expect(runner).toContain("LIGHTHOUSE_PROCESS_TIMEOUT_MS"); expect(runner).toContain("--max-wait-for-load=60000"); + expect(runner).not.toMatch(/stdio:\s*"inherit",\s*\n\s*timeout,/); }); }); diff --git a/tests/live-web-vitals-inputs.test.ts b/tests/live-web-vitals-inputs.test.ts index e9c5985c45..7ddddf867a 100644 --- a/tests/live-web-vitals-inputs.test.ts +++ b/tests/live-web-vitals-inputs.test.ts @@ -10,7 +10,7 @@ describe("parseLiveWebVitalsInputs", () => { parseLiveWebVitalsInputs({ origin: `${origin}/`, routes: " /, /therapy-compass, /documents/search ", - samples: "3", + samples: " 3 ", }), ).toEqual({ origin, routes: ["/", "/therapy-compass", "/documents/search"], samples: 3, invocations: 18 }); }); From d4af6a6356de3327906936a237405e4e7ccba0cb Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Sat, 8 Aug 2026 17:01:07 +0000 Subject: [PATCH 3/3] fix(ci): align live-web-vitals timeout contract with suite deadline The workflow now uses a dynamic run_timeout under the suite deadline; update the focused CI contract test to match. --- tests/ci-cache-safety.test.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/ci-cache-safety.test.ts b/tests/ci-cache-safety.test.ts index a48ed24386..8ba2b0a504 100644 --- a/tests/ci-cache-safety.test.ts +++ b/tests/ci-cache-safety.test.ts @@ -51,7 +51,9 @@ describe("CI cache safety", () => { it("caps the dispatch-only live Lighthouse matrix and each child process", () => { expect(liveWebVitalsWorkflow).toContain("timeout-minutes: 45"); expect(liveWebVitalsWorkflow).toContain("node scripts/live-web-vitals-inputs.mjs"); - expect(liveWebVitalsWorkflow).toContain("timeout --signal=TERM --kill-after=10s 80s"); + expect(liveWebVitalsWorkflow).toContain('timeout --signal=TERM --kill-after=10s "${run_timeout}s"'); + expect(liveWebVitalsWorkflow).toContain("LIVE_WEB_VITALS_PROCESS_TIMEOUT_SEC"); + expect(liveWebVitalsWorkflow).toContain("LIVE_WEB_VITALS_MEASUREMENT_SUITE_SECONDS"); }); it("routes recognised workflow-only changes through focused contracts", () => {