diff --git a/.github/actions/setup-lighthouse-chromium/action.yml b/.github/actions/setup-lighthouse-chromium/action.yml index 1d69d93d16..5c92958293 100644 --- a/.github/actions/setup-lighthouse-chromium/action.yml +++ b/.github/actions/setup-lighthouse-chromium/action.yml @@ -35,15 +35,41 @@ runs: restore-keys: | playwright-chromium-${{ runner.os }}- + # The browser archive is cached, but apt libraries are runner-local and must be + # installed even on a browser-cache hit. `install-deps` shells out to `apt-get + # update`, which occasionally hangs indefinitely against the runner's default + # azure.archive.ubuntu.com mirror (no timeout of its own) and silently burns the + # whole job until something external cancels it. Rewrite the flaky azure mirror + # to archive.ubuntu.com, set bounded APT timeouts, and bound each attempt with + # retries so a stuck mirror fails fast and recovers instead of stalling the job. - name: Install Chromium shell: bash run: | - if [ "${{ steps.pw-cache.outputs.cache-hit }}" = "true" ]; then - npx playwright install-deps chromium - npx playwright install chromium - else - npx playwright install --with-deps chromium + if command -v sudo >/dev/null 2>&1; then + sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list 2>/dev/null || true + sudo mkdir -p /etc/apt/apt.conf.d + sudo tee /etc/apt/apt.conf.d/99flaky-mirror-resilience >/dev/null << 'EOF' + Acquire::Retries "3"; + Acquire::http::Timeout "30"; + Acquire::https::Timeout "30"; + EOF fi + install_chromium() { + if [ "${{ steps.pw-cache.outputs.cache-hit }}" = "true" ]; then + npx playwright install-deps chromium + npx playwright install chromium + else + npx playwright install --with-deps chromium + fi + } + for attempt in 1 2 3; do + if timeout 180 bash -c "$(declare -f install_chromium); install_chromium"; then + exit 0 + fi + echo "Chromium install attempt $attempt timed out or failed, retrying..." >&2 + sleep 5 + done + exit 1 - name: Pin the Chromium executable path shell: bash diff --git a/.github/actions/setup-ui-e2e/action.yml b/.github/actions/setup-ui-e2e/action.yml index 2faae6d790..94c785a613 100644 --- a/.github/actions/setup-ui-e2e/action.yml +++ b/.github/actions/setup-ui-e2e/action.yml @@ -26,11 +26,21 @@ runs: # installed even on a browser-cache hit. `install-deps` shells out to `apt-get # update`, which occasionally hangs indefinitely against the runner's default # azure.archive.ubuntu.com mirror (no timeout of its own) and silently burns the - # whole job until something external cancels it. Bound each attempt and retry so a - # stuck mirror fails fast instead of stalling the job for the better part of an hour. + # whole job until something external cancels it. Rewrite the flaky azure mirror + # to archive.ubuntu.com, set bounded APT timeouts, and bound each attempt with + # retries so a stuck mirror fails fast and recovers instead of stalling the job. - name: Install Chromium shell: bash run: | + if command -v sudo >/dev/null 2>&1; then + sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list 2>/dev/null || true + sudo mkdir -p /etc/apt/apt.conf.d + sudo tee /etc/apt/apt.conf.d/99flaky-mirror-resilience >/dev/null << 'EOF' + Acquire::Retries "3"; + Acquire::http::Timeout "30"; + Acquire::https::Timeout "30"; + EOF + fi install_chromium() { if [ "${{ steps.pw-cache.outputs.cache-hit }}" = "true" ]; then npx playwright install-deps chromium diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 861d6c5347..884849ba39 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1281,12 +1281,31 @@ jobs: - name: Install Playwright browsers run: | - if [ "${{ steps.pw-cache.outputs.cache-hit }}" = "true" ]; then - npx playwright install-deps - npx playwright install - else - npx playwright install --with-deps + if command -v sudo >/dev/null 2>&1; then + sudo sed -i 's/azure\.archive\.ubuntu\.com/archive.ubuntu.com/g' /etc/apt/sources.list /etc/apt/sources.list.d/*.sources /etc/apt/sources.list.d/*.list 2>/dev/null || true + sudo mkdir -p /etc/apt/apt.conf.d + sudo tee /etc/apt/apt.conf.d/99flaky-mirror-resilience >/dev/null << 'EOF' + Acquire::Retries "3"; + Acquire::http::Timeout "30"; + Acquire::https::Timeout "30"; + EOF fi + install_browsers() { + if [ "${{ steps.pw-cache.outputs.cache-hit }}" = "true" ]; then + npx playwright install-deps + npx playwright install + else + npx playwright install --with-deps + fi + } + for attempt in 1 2 3; do + if timeout 180 bash -c "$(declare -f install_browsers); install_browsers"; then + exit 0 + fi + echo "Playwright browsers install attempt $attempt timed out or failed, retrying..." >&2 + sleep 5 + done + exit 1 - name: Full browser UI matrix id: e2e-matrix diff --git a/tests/ci-cache-safety.test.ts b/tests/ci-cache-safety.test.ts index 1dbab6caee..0e84c64680 100644 --- a/tests/ci-cache-safety.test.ts +++ b/tests/ci-cache-safety.test.ts @@ -72,9 +72,22 @@ describe("CI cache safety", () => { it("installs Playwright system dependencies when browser caches hit", () => { expect(uiSetup).toMatch(/cache-hit.*?install-deps chromium.*?install chromium/s); + expect(lighthouseChromiumSetup).toMatch(/cache-hit.*?install-deps chromium.*?install chromium/s); expect(workflow).toMatch(/cache-hit.*?install-deps\n\s+npx playwright install/s); }); + it("hardens Playwright browser and dependency installation against flaky Ubuntu mirrors and apt hangs", () => { + expect(lighthouseChromiumSetup).toContain("azure\\.archive\\.ubuntu\\.com/archive.ubuntu.com"); + expect(lighthouseChromiumSetup).toContain("timeout 180"); + expect(lighthouseChromiumSetup).toContain("Acquire::Retries"); + expect(uiSetup).toContain("azure\\.archive\\.ubuntu\\.com/archive.ubuntu.com"); + expect(uiSetup).toContain("timeout 180"); + expect(uiSetup).toContain("Acquire::Retries"); + expect(workflow).toContain("azure\\.archive\\.ubuntu\\.com/archive.ubuntu.com"); + expect(workflow).toContain("timeout 180"); + expect(workflow).toContain("Acquire::Retries"); + }); + it("rejects a refreshed Lighthouse baseline that has zero or mixed browser identities", () => { expect(workflow).toContain("versions.length!==1"); expect(workflow).toContain("Expected exactly one baseline Chrome version");