Skip to content

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530)#3555

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
alangenfeld wants to merge 1 commit into from
Closed

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530) #3555

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/e2e-ci-retry.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
---
'@workflow/core': patch
---

Retry e2e tests once in CI and surface retried-then-passed tests as flaky in annotations and the PR results comment.
89 changes: 85 additions & 4 deletions .github/scripts/aggregate-e2e-results.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ function findResultFiles(dir) {
return findJsonFiles(dir, 'e2e-', [
'e2e-metadata-',
'e2e-failures-',
'e2e-flaky-',
'e2e-diagnostics-',
]);
}
Expand DownExpand Up@@ -146,6 +147,79 @@ function loadFailures(dir) {
return failures;
}

// Load flaky-test sidecar files (tests that passed only after a retry,
// written by github-reporter). Grouped per app; the same test flaking in
// several jobs for one app is collapsed into a single entry with an
// occurrence count so the section stays scannable.
function loadFlaky(dir) {
// Map of `${app}\u0000${testName}` -> { app, testName, retryCount, occurrences }
const flaky = new Map();
const files = findJsonFiles(dir, 'e2e-flaky-');

for (const file of files) {
const basename = path.basename(file, '.json');
const match = basename.match(/^e2e-flaky-(.+)-(?:vercel|local)$/);
const app = match ? match[1] : 'unknown';
try {
const entries = JSON.parse(fs.readFileSync(file, 'utf-8'));
for (const entry of entries) {
if (!entry.testName) continue;
const key = `${app}\u0000${entry.testName}`;
const existing = flaky.get(key);
if (existing) {
existing.occurrences++;
existing.retryCount = Math.max(
existing.retryCount,
entry.retryCount || 1
);
} else {
flaky.set(key, {
app,
testName: entry.testName,
retryCount: entry.retryCount || 1,
occurrences: 1,
});
}
}
} catch (_e) {
// Skip invalid files
}
}

return [...flaky.values()];
}

// Render the flaky-tests section shared by the PR comment and the per-job
// step summary. Retried-to-green tests would otherwise be invisible — the
// job is green — so this is the only place a recurring race stays visible.
function renderFlakySection(flakyTests) {
if (flakyTests.length === 0) return;

console.log('### ⚠️ Flaky E2E Tests (passed on retry)\n');
console.log(
'_These tests failed at least once and passed on a retry. A recurring entry here is a real race worth investigating._\n'
);

const sorted = [...flakyTests].sort(
(a, b) =>
b.occurrences - a.occurrences || a.testName.localeCompare(b.testName)
);
const collapse = sorted.length >= 10;
if (collapse) {
console.log('<details>');
console.log(`<summary>${sorted.length} flaky tests</summary>\n`);
}
for (const test of sorted) {
const jobs =
test.occurrences > 1 ? ` — flaked in ${test.occurrences} jobs` : '';
console.log(`- \`${test.testName}\` (${test.app})${jobs}`);
}
console.log('');
if (collapse) {
console.log('</details>\n');
}
}

// vitest's JSON reporter serializes only error stacks. For test timeouts the
// stack is the task-collection stack ("Error: STACK_TRACE_ERROR ..."), which
// carries no information about the failure. The github-reporter failures
Expand DownExpand Up@@ -355,7 +429,7 @@ function aggregateByCategory(files) {
}

// Render markdown summary for single job (step summary)
function renderSingleJobSummary(summary) {
function renderSingleJobSummary(summary, flakyTests = []) {
const total =
summary.totalPassed + summary.totalFailed + summary.totalSkipped;
const statusEmoji = summary.totalFailed > 0 ? '❌' : '✅';
Expand DownExpand Up@@ -394,6 +468,8 @@ function renderSingleJobSummary(summary) {
}
}

renderFlakySection(flakyTests);

// Results by file
if (summary.fileResults.length > 1) {
console.log('<details>');
Expand DownExpand Up@@ -438,7 +514,8 @@ function renderAggregatedSummary(
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
) {
const total =
overallSummary.totalPassed +
Expand DownExpand Up@@ -557,6 +634,8 @@ function renderAggregatedSummary(
}
}

renderFlakySection(flakyTests);

// Detailed breakdown by category
console.log('### Details by Category\n');

Expand DownExpand Up@@ -605,13 +684,15 @@ if (mode === 'aggregate') {
const metadata = loadMetadata(resultsDir);
const diagnostics = loadDiagnostics(resultsDir);
const failures = loadFailures(resultsDir);
const flakyTests = loadFlaky(resultsDir);
enrichFailedTestMessages(overallSummary.allFailedTests, failures);
renderAggregatedSummary(
categories,
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
);

// Exit with non-zero if any tests failed
Expand All@@ -621,7 +702,7 @@ if (mode === 'aggregate') {
} else {
const summary = aggregateResults(resultFiles);
enrichFailedTestMessages(summary.allFailedTests, loadFailures(resultsDir));
renderSingleJobSummary(summary);
renderSingleJobSummary(summary, loadFlaky(resultsDir));

// Exit with non-zero if any tests failed
if (summary.totalFailed > 0) {
Expand Down
17 changes: 13 additions & 4 deletions .github/workflows/tests.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -324,6 +324,7 @@ jobs:
e2e-vercel-prod-${{ matrix.app.name }}.json
e2e-metadata-${{ matrix.app.name }}-vercel.json
e2e-failures-${{ matrix.app.name }}-vercel.json
e2e-flaky-${{ matrix.app.name }}-vercel.json
e2e-diagnostics-${{ matrix.app.name }}-vercel.json
retention-days: 7
if-no-files-found: ignore
Expand DownExpand Up@@ -420,7 +421,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -498,7 +501,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -596,7 +601,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -678,7 +685,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-windows-nextjs-turbopack
path: e2e-windows-nextjs-turbopack.json
path: |
e2e-windows-nextjs-turbopack.json
e2e-flaky-nextjs-turbopack-local.json
retention-days: 7
if-no-files-found: ignore

Expand Down
5 changes: 4 additions & 1 deletion packages/core/e2e/event-log-race-repro.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -995,7 +995,10 @@ const testTimeoutMs =
config.stuckGraceMs +
60_000;

describe('event log race repro', () => {
// This harness's failures ARE the signal it exists to produce, and a single
// pass runs for the whole configured budget — never let the CI-wide e2e
// retry (vitest.config.ts) re-run it.
describe('event log race repro', { retry: 0 }, () => {
beforeAll(() => {
setupWorld(deploymentUrl);

Expand Down
64 changes: 58 additions & 6 deletions packages/core/e2e/github-reporter.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,10 @@
* Custom vitest reporter that emits GitHub Actions annotations for failed tests.
*
* When running in CI, failed e2e tests produce `::error` workflow commands that
* surface as annotations in the GitHub Actions UI and on PR file diffs.
* surface as annotations in the GitHub Actions UI and on PR file diffs. Tests
* that only passed after a retry (see `retry` in vitest.config.ts) produce
* `::warning` annotations and a `e2e-flaky-*.json` sidecar, so the retry that
* keeps a racy test from failing the job does not also hide the race.
*
* Also writes an enriched JSON sidecar file (`e2e-failures-*.json`) with
* per-test failure details including run IDs and dashboard links, which the
Expand DownExpand Up@@ -37,8 +40,16 @@ interface DiagnosticsEntry {
timestamp: string;
}

interface FlakyTestInfo {
testName: string;
fullName: string;
file: string;
retryCount: number;
}

export default class GithubAnnotationReporter implements Reporter {
private failedTests: FailedTestInfo[] = [];
private flakyTests: FlakyTestInfo[] = [];

onTestRunEnd(testModules: ReadonlyArray<TestModule>) {
for (const module of testModules) {
Expand All@@ -49,18 +60,36 @@ export default class GithubAnnotationReporter implements Reporter {
// Enrich failures with diagnostics sidecar data (run IDs, dashboard URLs)
this.enrichFromDiagnosticsSidecar();
this.writeFailuresSidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
if (this.flakyTests.length > 0) {
this.writeFlakySidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
}

private collectFailures(module: TestModule) {
for (const test of module.children.allTests()) {
const result = test.result();

if (result.state === 'passed') {
const retryCount = test.diagnostic()?.retryCount ?? 0;
if (retryCount > 0) {
this.flakyTests.push({
testName: test.name,
fullName: test.fullName,
file: module.moduleId,
retryCount,
});
}
continue;
}

if (result.state !== 'failed') continue;

const errors = result.errors || [];
Expand DownExpand Up@@ -127,6 +156,17 @@ export default class GithubAnnotationReporter implements Reporter {
* rather than the workflow source file (which may be a symlink).
*/
private emitAnnotations() {
for (const test of this.flakyTests) {
const title = `E2E flaky: ${test.testName}`;
const body = `Passed only after ${test.retryCount} retr${
test.retryCount === 1 ? 'y' : 'ies'
} — this test lost a race on its first attempt.`;
const relFile = path.relative(process.cwd(), test.file);
process.stdout.write(
`\n::warning file=${relFile},title=${title}::${body}\n`
);
}

for (const test of this.failedTests) {
const parts = [test.errorMessage.split('\n')[0].slice(0, 150)];
if (test.runId) parts.push(`Run: ${test.runId}`);
Expand DownExpand Up@@ -155,4 +195,16 @@ export default class GithubAnnotationReporter implements Reporter {

fs.writeFileSync(filePath, JSON.stringify(this.failedTests, null, 2));
}

private writeFlakySidecar() {
const appName = process.env.APP_NAME || 'unknown';
const isVercel = !!process.env.WORKFLOW_VERCEL_ENV;
const backend = isVercel ? 'vercel' : 'local';
const filePath = path.resolve(
process.cwd(),
`e2e-flaky-${appName}-${backend}.json`
);

fs.writeFileSync(filePath, JSON.stringify(this.flakyTests, null, 2));
}
}
9 changes: 9 additions & 0 deletions vitest.config.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,6 +3,15 @@ import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
testTimeout: 60_000,
// The e2e suites drive real deployments, so individual tests can lose
// timing races (queue delays, cold starts, watcher latency) that a
// second attempt absorbs. One CI retry keeps a single racy test from
// failing a 20+ minute matrix job; retried tests stay visible — the
// github-reporter annotates them and the PR comment lists them — so
// real races still get looked at. Harnesses where a failure is itself
// the signal (event-log-race-repro) pin `retry: 0` locally.
// Local runs keep retry at 0 so races reproduce while debugging.
retry: process.env.CI ? 1 : 0,
},
benchmark: {
include: ['**/*.bench.ts'],
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Add copy buttons to all \x3Cpre>\x3Ccode> blocks (function() { function addCopyButtons() { document.querySelectorAll('pre code').forEach(function(codeBlock) { if (codeBlock.parentElement.hasAttribute('data-copy-added')) return; codeBlock.parentElement.setAttribute('data-copy-added', 'true'); var btn = document.createElement('button'); btn.textContent = 'Copy'; btn.style.cssText = 'position:absolute;top:4px;right:4px;padding:2px 8px;font-size:11px;background:#4ecdc4;border:none;border-radius:4px;color:#1a1a2e;cursor:pointer;opacity:0.7;transition:opacity 0.2s;'; btn.onmouseover = function() { this.style.opacity = '1'; }; btn.onmouseout = function() { this.style.opacity = '0.7'; }; btn.onclick = function() { navigator.clipboard.writeText(codeBlock.textContent).then(function() { btn.textContent = 'Copied!'; setTimeout(function() { btn.textContent = 'Copy'; }, 1500); }); }; codeBlock.parentElement.style.position = 'relative'; codeBlock.parentElement.appendChild(btn); }); } addCopyButtons(); // Re-run on dynamic content var observer = new MutationObserver(addCopyButtons); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Add Copy Buttons to Code Blocks]', __e); } })(); (function(){ try { var __m = "github.com"; var __re = new RegExp('^' + "github\\.com" + ' [stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530) by alangenfeld · Pull Request #3555 · vercel/workflow · GitHub
Skip to content

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530)#3555

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
alangenfeld wants to merge 1 commit into from
Closed

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530) #3555

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/e2e-ci-retry.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
---
'@workflow/core': patch
---

Retry e2e tests once in CI and surface retried-then-passed tests as flaky in annotations and the PR results comment.
89 changes: 85 additions & 4 deletions .github/scripts/aggregate-e2e-results.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ function findResultFiles(dir) {
return findJsonFiles(dir, 'e2e-', [
'e2e-metadata-',
'e2e-failures-',
'e2e-flaky-',
'e2e-diagnostics-',
]);
}
Expand DownExpand Up@@ -146,6 +147,79 @@ function loadFailures(dir) {
return failures;
}

// Load flaky-test sidecar files (tests that passed only after a retry,
// written by github-reporter). Grouped per app; the same test flaking in
// several jobs for one app is collapsed into a single entry with an
// occurrence count so the section stays scannable.
function loadFlaky(dir) {
// Map of `${app}\u0000${testName}` -> { app, testName, retryCount, occurrences }
const flaky = new Map();
const files = findJsonFiles(dir, 'e2e-flaky-');

for (const file of files) {
const basename = path.basename(file, '.json');
const match = basename.match(/^e2e-flaky-(.+)-(?:vercel|local)$/);
const app = match ? match[1] : 'unknown';
try {
const entries = JSON.parse(fs.readFileSync(file, 'utf-8'));
for (const entry of entries) {
if (!entry.testName) continue;
const key = `${app}\u0000${entry.testName}`;
const existing = flaky.get(key);
if (existing) {
existing.occurrences++;
existing.retryCount = Math.max(
existing.retryCount,
entry.retryCount || 1
);
} else {
flaky.set(key, {
app,
testName: entry.testName,
retryCount: entry.retryCount || 1,
occurrences: 1,
});
}
}
} catch (_e) {
// Skip invalid files
}
}

return [...flaky.values()];
}

// Render the flaky-tests section shared by the PR comment and the per-job
// step summary. Retried-to-green tests would otherwise be invisible — the
// job is green — so this is the only place a recurring race stays visible.
function renderFlakySection(flakyTests) {
if (flakyTests.length === 0) return;

console.log('### ⚠️ Flaky E2E Tests (passed on retry)\n');
console.log(
'_These tests failed at least once and passed on a retry. A recurring entry here is a real race worth investigating._\n'
);

const sorted = [...flakyTests].sort(
(a, b) =>
b.occurrences - a.occurrences || a.testName.localeCompare(b.testName)
);
const collapse = sorted.length >= 10;
if (collapse) {
console.log('<details>');
console.log(`<summary>${sorted.length} flaky tests</summary>\n`);
}
for (const test of sorted) {
const jobs =
test.occurrences > 1 ? ` — flaked in ${test.occurrences} jobs` : '';
console.log(`- \`${test.testName}\` (${test.app})${jobs}`);
}
console.log('');
if (collapse) {
console.log('</details>\n');
}
}

// vitest's JSON reporter serializes only error stacks. For test timeouts the
// stack is the task-collection stack ("Error: STACK_TRACE_ERROR ..."), which
// carries no information about the failure. The github-reporter failures
Expand DownExpand Up@@ -355,7 +429,7 @@ function aggregateByCategory(files) {
}

// Render markdown summary for single job (step summary)
function renderSingleJobSummary(summary) {
function renderSingleJobSummary(summary, flakyTests = []) {
const total =
summary.totalPassed + summary.totalFailed + summary.totalSkipped;
const statusEmoji = summary.totalFailed > 0 ? '❌' : '✅';
Expand DownExpand Up@@ -394,6 +468,8 @@ function renderSingleJobSummary(summary) {
}
}

renderFlakySection(flakyTests);

// Results by file
if (summary.fileResults.length > 1) {
console.log('<details>');
Expand DownExpand Up@@ -438,7 +514,8 @@ function renderAggregatedSummary(
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
) {
const total =
overallSummary.totalPassed +
Expand DownExpand Up@@ -557,6 +634,8 @@ function renderAggregatedSummary(
}
}

renderFlakySection(flakyTests);

// Detailed breakdown by category
console.log('### Details by Category\n');

Expand DownExpand Up@@ -605,13 +684,15 @@ if (mode === 'aggregate') {
const metadata = loadMetadata(resultsDir);
const diagnostics = loadDiagnostics(resultsDir);
const failures = loadFailures(resultsDir);
const flakyTests = loadFlaky(resultsDir);
enrichFailedTestMessages(overallSummary.allFailedTests, failures);
renderAggregatedSummary(
categories,
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
);

// Exit with non-zero if any tests failed
Expand All@@ -621,7 +702,7 @@ if (mode === 'aggregate') {
} else {
const summary = aggregateResults(resultFiles);
enrichFailedTestMessages(summary.allFailedTests, loadFailures(resultsDir));
renderSingleJobSummary(summary);
renderSingleJobSummary(summary, loadFlaky(resultsDir));

// Exit with non-zero if any tests failed
if (summary.totalFailed > 0) {
Expand Down
17 changes: 13 additions & 4 deletions .github/workflows/tests.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -324,6 +324,7 @@ jobs:
e2e-vercel-prod-${{ matrix.app.name }}.json
e2e-metadata-${{ matrix.app.name }}-vercel.json
e2e-failures-${{ matrix.app.name }}-vercel.json
e2e-flaky-${{ matrix.app.name }}-vercel.json
e2e-diagnostics-${{ matrix.app.name }}-vercel.json
retention-days: 7
if-no-files-found: ignore
Expand DownExpand Up@@ -420,7 +421,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -498,7 +501,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -596,7 +601,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -678,7 +685,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-windows-nextjs-turbopack
path: e2e-windows-nextjs-turbopack.json
path: |
e2e-windows-nextjs-turbopack.json
e2e-flaky-nextjs-turbopack-local.json
retention-days: 7
if-no-files-found: ignore

Expand Down
5 changes: 4 additions & 1 deletion packages/core/e2e/event-log-race-repro.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -995,7 +995,10 @@ const testTimeoutMs =
config.stuckGraceMs +
60_000;

describe('event log race repro', () => {
// This harness's failures ARE the signal it exists to produce, and a single
// pass runs for the whole configured budget — never let the CI-wide e2e
// retry (vitest.config.ts) re-run it.
describe('event log race repro', { retry: 0 }, () => {
beforeAll(() => {
setupWorld(deploymentUrl);

Expand Down
64 changes: 58 additions & 6 deletions packages/core/e2e/github-reporter.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,10 @@
* Custom vitest reporter that emits GitHub Actions annotations for failed tests.
*
* When running in CI, failed e2e tests produce `::error` workflow commands that
* surface as annotations in the GitHub Actions UI and on PR file diffs.
* surface as annotations in the GitHub Actions UI and on PR file diffs. Tests
* that only passed after a retry (see `retry` in vitest.config.ts) produce
* `::warning` annotations and a `e2e-flaky-*.json` sidecar, so the retry that
* keeps a racy test from failing the job does not also hide the race.
*
* Also writes an enriched JSON sidecar file (`e2e-failures-*.json`) with
* per-test failure details including run IDs and dashboard links, which the
Expand DownExpand Up@@ -37,8 +40,16 @@ interface DiagnosticsEntry {
timestamp: string;
}

interface FlakyTestInfo {
testName: string;
fullName: string;
file: string;
retryCount: number;
}

export default class GithubAnnotationReporter implements Reporter {
private failedTests: FailedTestInfo[] = [];
private flakyTests: FlakyTestInfo[] = [];

onTestRunEnd(testModules: ReadonlyArray<TestModule>) {
for (const module of testModules) {
Expand All@@ -49,18 +60,36 @@ export default class GithubAnnotationReporter implements Reporter {
// Enrich failures with diagnostics sidecar data (run IDs, dashboard URLs)
this.enrichFromDiagnosticsSidecar();
this.writeFailuresSidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
if (this.flakyTests.length > 0) {
this.writeFlakySidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
}

private collectFailures(module: TestModule) {
for (const test of module.children.allTests()) {
const result = test.result();

if (result.state === 'passed') {
const retryCount = test.diagnostic()?.retryCount ?? 0;
if (retryCount > 0) {
this.flakyTests.push({
testName: test.name,
fullName: test.fullName,
file: module.moduleId,
retryCount,
});
}
continue;
}

if (result.state !== 'failed') continue;

const errors = result.errors || [];
Expand DownExpand Up@@ -127,6 +156,17 @@ export default class GithubAnnotationReporter implements Reporter {
* rather than the workflow source file (which may be a symlink).
*/
private emitAnnotations() {
for (const test of this.flakyTests) {
const title = `E2E flaky: ${test.testName}`;
const body = `Passed only after ${test.retryCount} retr${
test.retryCount === 1 ? 'y' : 'ies'
} — this test lost a race on its first attempt.`;
const relFile = path.relative(process.cwd(), test.file);
process.stdout.write(
`\n::warning file=${relFile},title=${title}::${body}\n`
);
}

for (const test of this.failedTests) {
const parts = [test.errorMessage.split('\n')[0].slice(0, 150)];
if (test.runId) parts.push(`Run: ${test.runId}`);
Expand DownExpand Up@@ -155,4 +195,16 @@ export default class GithubAnnotationReporter implements Reporter {

fs.writeFileSync(filePath, JSON.stringify(this.failedTests, null, 2));
}

private writeFlakySidecar() {
const appName = process.env.APP_NAME || 'unknown';
const isVercel = !!process.env.WORKFLOW_VERCEL_ENV;
const backend = isVercel ? 'vercel' : 'local';
const filePath = path.resolve(
process.cwd(),
`e2e-flaky-${appName}-${backend}.json`
);

fs.writeFileSync(filePath, JSON.stringify(this.flakyTests, null, 2));
}
}
9 changes: 9 additions & 0 deletions vitest.config.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,6 +3,15 @@ import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
testTimeout: 60_000,
// The e2e suites drive real deployments, so individual tests can lose
// timing races (queue delays, cold starts, watcher latency) that a
// second attempt absorbs. One CI retry keeps a single racy test from
// failing a 20+ minute matrix job; retried tests stay visible — the
// github-reporter annotates them and the PR comment lists them — so
// real races still get looked at. Harnesses where a failure is itself
// the signal (event-log-race-repro) pin `retry: 0` locally.
// Local runs keep retry at 0 so races reproduce while debugging.
retry: process.env.CI ? 1 : 0,
},
benchmark: {
include: ['**/*.bench.ts'],
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Force GitHub README to respect dark mode (function() { var style = document.createElement('style'); style.textContent = ' .markdown-body { color-scheme: dark light; } .markdown-body pre { background: #161b22 !important; } .markdown-body code { background: rgba(110, 118, 129, 0.4) !important; } .markdown-body table th, .markdown-body table td { border-color: #30363d !important; } .markdown-body img { background: #0d1117; } .markdown-body blockquote { border-left-color: #8b949e; } .markdown-body hr { border-color: #30363d; } '; document.head.appendChild(style); })(); } } catch(__e) { console.warn('[Userscript:GitHub Dark Mode README Fix]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' [stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530) by alangenfeld · Pull Request #3555 · vercel/workflow · GitHub
Skip to content

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530)#3555

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
alangenfeld wants to merge 1 commit into from
Closed

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530) #3555

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/e2e-ci-retry.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
---
'@workflow/core': patch
---

Retry e2e tests once in CI and surface retried-then-passed tests as flaky in annotations and the PR results comment.
89 changes: 85 additions & 4 deletions .github/scripts/aggregate-e2e-results.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ function findResultFiles(dir) {
return findJsonFiles(dir, 'e2e-', [
'e2e-metadata-',
'e2e-failures-',
'e2e-flaky-',
'e2e-diagnostics-',
]);
}
Expand DownExpand Up@@ -146,6 +147,79 @@ function loadFailures(dir) {
return failures;
}

// Load flaky-test sidecar files (tests that passed only after a retry,
// written by github-reporter). Grouped per app; the same test flaking in
// several jobs for one app is collapsed into a single entry with an
// occurrence count so the section stays scannable.
function loadFlaky(dir) {
// Map of `${app}\u0000${testName}` -> { app, testName, retryCount, occurrences }
const flaky = new Map();
const files = findJsonFiles(dir, 'e2e-flaky-');

for (const file of files) {
const basename = path.basename(file, '.json');
const match = basename.match(/^e2e-flaky-(.+)-(?:vercel|local)$/);
const app = match ? match[1] : 'unknown';
try {
const entries = JSON.parse(fs.readFileSync(file, 'utf-8'));
for (const entry of entries) {
if (!entry.testName) continue;
const key = `${app}\u0000${entry.testName}`;
const existing = flaky.get(key);
if (existing) {
existing.occurrences++;
existing.retryCount = Math.max(
existing.retryCount,
entry.retryCount || 1
);
} else {
flaky.set(key, {
app,
testName: entry.testName,
retryCount: entry.retryCount || 1,
occurrences: 1,
});
}
}
} catch (_e) {
// Skip invalid files
}
}

return [...flaky.values()];
}

// Render the flaky-tests section shared by the PR comment and the per-job
// step summary. Retried-to-green tests would otherwise be invisible — the
// job is green — so this is the only place a recurring race stays visible.
function renderFlakySection(flakyTests) {
if (flakyTests.length === 0) return;

console.log('### ⚠️ Flaky E2E Tests (passed on retry)\n');
console.log(
'_These tests failed at least once and passed on a retry. A recurring entry here is a real race worth investigating._\n'
);

const sorted = [...flakyTests].sort(
(a, b) =>
b.occurrences - a.occurrences || a.testName.localeCompare(b.testName)
);
const collapse = sorted.length >= 10;
if (collapse) {
console.log('<details>');
console.log(`<summary>${sorted.length} flaky tests</summary>\n`);
}
for (const test of sorted) {
const jobs =
test.occurrences > 1 ? ` — flaked in ${test.occurrences} jobs` : '';
console.log(`- \`${test.testName}\` (${test.app})${jobs}`);
}
console.log('');
if (collapse) {
console.log('</details>\n');
}
}

// vitest's JSON reporter serializes only error stacks. For test timeouts the
// stack is the task-collection stack ("Error: STACK_TRACE_ERROR ..."), which
// carries no information about the failure. The github-reporter failures
Expand DownExpand Up@@ -355,7 +429,7 @@ function aggregateByCategory(files) {
}

// Render markdown summary for single job (step summary)
function renderSingleJobSummary(summary) {
function renderSingleJobSummary(summary, flakyTests = []) {
const total =
summary.totalPassed + summary.totalFailed + summary.totalSkipped;
const statusEmoji = summary.totalFailed > 0 ? '❌' : '✅';
Expand DownExpand Up@@ -394,6 +468,8 @@ function renderSingleJobSummary(summary) {
}
}

renderFlakySection(flakyTests);

// Results by file
if (summary.fileResults.length > 1) {
console.log('<details>');
Expand DownExpand Up@@ -438,7 +514,8 @@ function renderAggregatedSummary(
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
) {
const total =
overallSummary.totalPassed +
Expand DownExpand Up@@ -557,6 +634,8 @@ function renderAggregatedSummary(
}
}

renderFlakySection(flakyTests);

// Detailed breakdown by category
console.log('### Details by Category\n');

Expand DownExpand Up@@ -605,13 +684,15 @@ if (mode === 'aggregate') {
const metadata = loadMetadata(resultsDir);
const diagnostics = loadDiagnostics(resultsDir);
const failures = loadFailures(resultsDir);
const flakyTests = loadFlaky(resultsDir);
enrichFailedTestMessages(overallSummary.allFailedTests, failures);
renderAggregatedSummary(
categories,
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
);

// Exit with non-zero if any tests failed
Expand All@@ -621,7 +702,7 @@ if (mode === 'aggregate') {
} else {
const summary = aggregateResults(resultFiles);
enrichFailedTestMessages(summary.allFailedTests, loadFailures(resultsDir));
renderSingleJobSummary(summary);
renderSingleJobSummary(summary, loadFlaky(resultsDir));

// Exit with non-zero if any tests failed
if (summary.totalFailed > 0) {
Expand Down
17 changes: 13 additions & 4 deletions .github/workflows/tests.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -324,6 +324,7 @@ jobs:
e2e-vercel-prod-${{ matrix.app.name }}.json
e2e-metadata-${{ matrix.app.name }}-vercel.json
e2e-failures-${{ matrix.app.name }}-vercel.json
e2e-flaky-${{ matrix.app.name }}-vercel.json
e2e-diagnostics-${{ matrix.app.name }}-vercel.json
retention-days: 7
if-no-files-found: ignore
Expand DownExpand Up@@ -420,7 +421,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -498,7 +501,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -596,7 +601,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -678,7 +685,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-windows-nextjs-turbopack
path: e2e-windows-nextjs-turbopack.json
path: |
e2e-windows-nextjs-turbopack.json
e2e-flaky-nextjs-turbopack-local.json
retention-days: 7
if-no-files-found: ignore

Expand Down
5 changes: 4 additions & 1 deletion packages/core/e2e/event-log-race-repro.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -995,7 +995,10 @@ const testTimeoutMs =
config.stuckGraceMs +
60_000;

describe('event log race repro', () => {
// This harness's failures ARE the signal it exists to produce, and a single
// pass runs for the whole configured budget — never let the CI-wide e2e
// retry (vitest.config.ts) re-run it.
describe('event log race repro', { retry: 0 }, () => {
beforeAll(() => {
setupWorld(deploymentUrl);

Expand Down
64 changes: 58 additions & 6 deletions packages/core/e2e/github-reporter.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,10 @@
* Custom vitest reporter that emits GitHub Actions annotations for failed tests.
*
* When running in CI, failed e2e tests produce `::error` workflow commands that
* surface as annotations in the GitHub Actions UI and on PR file diffs.
* surface as annotations in the GitHub Actions UI and on PR file diffs. Tests
* that only passed after a retry (see `retry` in vitest.config.ts) produce
* `::warning` annotations and a `e2e-flaky-*.json` sidecar, so the retry that
* keeps a racy test from failing the job does not also hide the race.
*
* Also writes an enriched JSON sidecar file (`e2e-failures-*.json`) with
* per-test failure details including run IDs and dashboard links, which the
Expand DownExpand Up@@ -37,8 +40,16 @@ interface DiagnosticsEntry {
timestamp: string;
}

interface FlakyTestInfo {
testName: string;
fullName: string;
file: string;
retryCount: number;
}

export default class GithubAnnotationReporter implements Reporter {
private failedTests: FailedTestInfo[] = [];
private flakyTests: FlakyTestInfo[] = [];

onTestRunEnd(testModules: ReadonlyArray<TestModule>) {
for (const module of testModules) {
Expand All@@ -49,18 +60,36 @@ export default class GithubAnnotationReporter implements Reporter {
// Enrich failures with diagnostics sidecar data (run IDs, dashboard URLs)
this.enrichFromDiagnosticsSidecar();
this.writeFailuresSidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
if (this.flakyTests.length > 0) {
this.writeFlakySidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
}

private collectFailures(module: TestModule) {
for (const test of module.children.allTests()) {
const result = test.result();

if (result.state === 'passed') {
const retryCount = test.diagnostic()?.retryCount ?? 0;
if (retryCount > 0) {
this.flakyTests.push({
testName: test.name,
fullName: test.fullName,
file: module.moduleId,
retryCount,
});
}
continue;
}

if (result.state !== 'failed') continue;

const errors = result.errors || [];
Expand DownExpand Up@@ -127,6 +156,17 @@ export default class GithubAnnotationReporter implements Reporter {
* rather than the workflow source file (which may be a symlink).
*/
private emitAnnotations() {
for (const test of this.flakyTests) {
const title = `E2E flaky: ${test.testName}`;
const body = `Passed only after ${test.retryCount} retr${
test.retryCount === 1 ? 'y' : 'ies'
} — this test lost a race on its first attempt.`;
const relFile = path.relative(process.cwd(), test.file);
process.stdout.write(
`\n::warning file=${relFile},title=${title}::${body}\n`
);
}

for (const test of this.failedTests) {
const parts = [test.errorMessage.split('\n')[0].slice(0, 150)];
if (test.runId) parts.push(`Run: ${test.runId}`);
Expand DownExpand Up@@ -155,4 +195,16 @@ export default class GithubAnnotationReporter implements Reporter {

fs.writeFileSync(filePath, JSON.stringify(this.failedTests, null, 2));
}

private writeFlakySidecar() {
const appName = process.env.APP_NAME || 'unknown';
const isVercel = !!process.env.WORKFLOW_VERCEL_ENV;
const backend = isVercel ? 'vercel' : 'local';
const filePath = path.resolve(
process.cwd(),
`e2e-flaky-${appName}-${backend}.json`
);

fs.writeFileSync(filePath, JSON.stringify(this.flakyTests, null, 2));
}
}
9 changes: 9 additions & 0 deletions vitest.config.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,6 +3,15 @@ import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
testTimeout: 60_000,
// The e2e suites drive real deployments, so individual tests can lose
// timing races (queue delays, cold starts, watcher latency) that a
// second attempt absorbs. One CI retry keeps a single racy test from
// failing a 20+ minute matrix job; retried tests stay visible — the
// github-reporter annotates them and the PR comment lists them — so
// real races still get looked at. Harnesses where a failure is itself
// the signal (event-log-race-repro) pin `retry: 0` locally.
// Local runs keep retry at 0 so races reproduce while debugging.
retry: process.env.CI ? 1 : 0,
},
benchmark: {
include: ['**/*.bench.ts'],
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Highlight search terms from Google/DuckDuckGo/Bing referrer (function() { var ref = document.referrer; var terms = []; if (ref.includes('google.com') || ref.includes('duckduckgo.com') || ref.includes('bing.com')) { var url = new URL(ref); var q = url.searchParams.get('q') || url.searchParams.get('p'); if (q) { terms = q.split(/\s+/).filter(function(t) { return t.length > 2; }); } } if (terms.length === 0) return; var style = document.createElement('style'); style.textContent = '.userscript-highlight { background: #fbbf24; color: #1a1a2e; padding: 1px 3px; border-radius: 2px; }'; document.head.appendChild(style); function highlight(node) { if (node.nodeType === 3) { // text node var text = node.textContent; var found = false; terms.forEach(function(term) { var regex = new RegExp('(' + term.replace(/[.*+?^${}()|[\]\\]/g, '\\') + ')', 'gi'); if (regex.test(text)) { found = true; var frag = document.createDocumentFragment(); var parts = text.split(regex); parts.forEach(function(part, i) { if (i % 2 === 0) { frag.appendChild(document.createTextNode(part)); } else { var span = document.createElement('span'); span.className = 'userscript-highlight'; span.textContent = part; frag.appendChild(span); } }); node.parentNode.replaceChild(frag, node); } }); } else if (node.nodeType === 1 && node.childNodes) { // element var skipTags = ['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEXTAREA', 'INPUT', 'SELECT']; if (!skipTags.includes(node.tagName)) { Array.from(node.childNodes).forEach(highlight); } } } highlight(document.body); // Re-highlight on dynamic content var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1 || node.nodeType === 3) highlight(node); }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Highlight Search Terms]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' [stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530) by alangenfeld · Pull Request #3555 · vercel/workflow · GitHub
Skip to content

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530)#3555

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
alangenfeld wants to merge 1 commit into from
Closed

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530) #3555

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/e2e-ci-retry.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
---
'@workflow/core': patch
---

Retry e2e tests once in CI and surface retried-then-passed tests as flaky in annotations and the PR results comment.
89 changes: 85 additions & 4 deletions .github/scripts/aggregate-e2e-results.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ function findResultFiles(dir) {
return findJsonFiles(dir, 'e2e-', [
'e2e-metadata-',
'e2e-failures-',
'e2e-flaky-',
'e2e-diagnostics-',
]);
}
Expand DownExpand Up@@ -146,6 +147,79 @@ function loadFailures(dir) {
return failures;
}

// Load flaky-test sidecar files (tests that passed only after a retry,
// written by github-reporter). Grouped per app; the same test flaking in
// several jobs for one app is collapsed into a single entry with an
// occurrence count so the section stays scannable.
function loadFlaky(dir) {
// Map of `${app}\u0000${testName}` -> { app, testName, retryCount, occurrences }
const flaky = new Map();
const files = findJsonFiles(dir, 'e2e-flaky-');

for (const file of files) {
const basename = path.basename(file, '.json');
const match = basename.match(/^e2e-flaky-(.+)-(?:vercel|local)$/);
const app = match ? match[1] : 'unknown';
try {
const entries = JSON.parse(fs.readFileSync(file, 'utf-8'));
for (const entry of entries) {
if (!entry.testName) continue;
const key = `${app}\u0000${entry.testName}`;
const existing = flaky.get(key);
if (existing) {
existing.occurrences++;
existing.retryCount = Math.max(
existing.retryCount,
entry.retryCount || 1
);
} else {
flaky.set(key, {
app,
testName: entry.testName,
retryCount: entry.retryCount || 1,
occurrences: 1,
});
}
}
} catch (_e) {
// Skip invalid files
}
}

return [...flaky.values()];
}

// Render the flaky-tests section shared by the PR comment and the per-job
// step summary. Retried-to-green tests would otherwise be invisible — the
// job is green — so this is the only place a recurring race stays visible.
function renderFlakySection(flakyTests) {
if (flakyTests.length === 0) return;

console.log('### ⚠️ Flaky E2E Tests (passed on retry)\n');
console.log(
'_These tests failed at least once and passed on a retry. A recurring entry here is a real race worth investigating._\n'
);

const sorted = [...flakyTests].sort(
(a, b) =>
b.occurrences - a.occurrences || a.testName.localeCompare(b.testName)
);
const collapse = sorted.length >= 10;
if (collapse) {
console.log('<details>');
console.log(`<summary>${sorted.length} flaky tests</summary>\n`);
}
for (const test of sorted) {
const jobs =
test.occurrences > 1 ? ` — flaked in ${test.occurrences} jobs` : '';
console.log(`- \`${test.testName}\` (${test.app})${jobs}`);
}
console.log('');
if (collapse) {
console.log('</details>\n');
}
}

// vitest's JSON reporter serializes only error stacks. For test timeouts the
// stack is the task-collection stack ("Error: STACK_TRACE_ERROR ..."), which
// carries no information about the failure. The github-reporter failures
Expand DownExpand Up@@ -355,7 +429,7 @@ function aggregateByCategory(files) {
}

// Render markdown summary for single job (step summary)
function renderSingleJobSummary(summary) {
function renderSingleJobSummary(summary, flakyTests = []) {
const total =
summary.totalPassed + summary.totalFailed + summary.totalSkipped;
const statusEmoji = summary.totalFailed > 0 ? '❌' : '✅';
Expand DownExpand Up@@ -394,6 +468,8 @@ function renderSingleJobSummary(summary) {
}
}

renderFlakySection(flakyTests);

// Results by file
if (summary.fileResults.length > 1) {
console.log('<details>');
Expand DownExpand Up@@ -438,7 +514,8 @@ function renderAggregatedSummary(
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
) {
const total =
overallSummary.totalPassed +
Expand DownExpand Up@@ -557,6 +634,8 @@ function renderAggregatedSummary(
}
}

renderFlakySection(flakyTests);

// Detailed breakdown by category
console.log('### Details by Category\n');

Expand DownExpand Up@@ -605,13 +684,15 @@ if (mode === 'aggregate') {
const metadata = loadMetadata(resultsDir);
const diagnostics = loadDiagnostics(resultsDir);
const failures = loadFailures(resultsDir);
const flakyTests = loadFlaky(resultsDir);
enrichFailedTestMessages(overallSummary.allFailedTests, failures);
renderAggregatedSummary(
categories,
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
);

// Exit with non-zero if any tests failed
Expand All@@ -621,7 +702,7 @@ if (mode === 'aggregate') {
} else {
const summary = aggregateResults(resultFiles);
enrichFailedTestMessages(summary.allFailedTests, loadFailures(resultsDir));
renderSingleJobSummary(summary);
renderSingleJobSummary(summary, loadFlaky(resultsDir));

// Exit with non-zero if any tests failed
if (summary.totalFailed > 0) {
Expand Down
17 changes: 13 additions & 4 deletions .github/workflows/tests.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -324,6 +324,7 @@ jobs:
e2e-vercel-prod-${{ matrix.app.name }}.json
e2e-metadata-${{ matrix.app.name }}-vercel.json
e2e-failures-${{ matrix.app.name }}-vercel.json
e2e-flaky-${{ matrix.app.name }}-vercel.json
e2e-diagnostics-${{ matrix.app.name }}-vercel.json
retention-days: 7
if-no-files-found: ignore
Expand DownExpand Up@@ -420,7 +421,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -498,7 +501,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -596,7 +601,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -678,7 +685,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-windows-nextjs-turbopack
path: e2e-windows-nextjs-turbopack.json
path: |
e2e-windows-nextjs-turbopack.json
e2e-flaky-nextjs-turbopack-local.json
retention-days: 7
if-no-files-found: ignore

Expand Down
5 changes: 4 additions & 1 deletion packages/core/e2e/event-log-race-repro.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -995,7 +995,10 @@ const testTimeoutMs =
config.stuckGraceMs +
60_000;

describe('event log race repro', () => {
// This harness's failures ARE the signal it exists to produce, and a single
// pass runs for the whole configured budget — never let the CI-wide e2e
// retry (vitest.config.ts) re-run it.
describe('event log race repro', { retry: 0 }, () => {
beforeAll(() => {
setupWorld(deploymentUrl);

Expand Down
64 changes: 58 additions & 6 deletions packages/core/e2e/github-reporter.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,10 @@
* Custom vitest reporter that emits GitHub Actions annotations for failed tests.
*
* When running in CI, failed e2e tests produce `::error` workflow commands that
* surface as annotations in the GitHub Actions UI and on PR file diffs.
* surface as annotations in the GitHub Actions UI and on PR file diffs. Tests
* that only passed after a retry (see `retry` in vitest.config.ts) produce
* `::warning` annotations and a `e2e-flaky-*.json` sidecar, so the retry that
* keeps a racy test from failing the job does not also hide the race.
*
* Also writes an enriched JSON sidecar file (`e2e-failures-*.json`) with
* per-test failure details including run IDs and dashboard links, which the
Expand DownExpand Up@@ -37,8 +40,16 @@ interface DiagnosticsEntry {
timestamp: string;
}

interface FlakyTestInfo {
testName: string;
fullName: string;
file: string;
retryCount: number;
}

export default class GithubAnnotationReporter implements Reporter {
private failedTests: FailedTestInfo[] = [];
private flakyTests: FlakyTestInfo[] = [];

onTestRunEnd(testModules: ReadonlyArray<TestModule>) {
for (const module of testModules) {
Expand All@@ -49,18 +60,36 @@ export default class GithubAnnotationReporter implements Reporter {
// Enrich failures with diagnostics sidecar data (run IDs, dashboard URLs)
this.enrichFromDiagnosticsSidecar();
this.writeFailuresSidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
if (this.flakyTests.length > 0) {
this.writeFlakySidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
}

private collectFailures(module: TestModule) {
for (const test of module.children.allTests()) {
const result = test.result();

if (result.state === 'passed') {
const retryCount = test.diagnostic()?.retryCount ?? 0;
if (retryCount > 0) {
this.flakyTests.push({
testName: test.name,
fullName: test.fullName,
file: module.moduleId,
retryCount,
});
}
continue;
}

if (result.state !== 'failed') continue;

const errors = result.errors || [];
Expand DownExpand Up@@ -127,6 +156,17 @@ export default class GithubAnnotationReporter implements Reporter {
* rather than the workflow source file (which may be a symlink).
*/
private emitAnnotations() {
for (const test of this.flakyTests) {
const title = `E2E flaky: ${test.testName}`;
const body = `Passed only after ${test.retryCount} retr${
test.retryCount === 1 ? 'y' : 'ies'
} — this test lost a race on its first attempt.`;
const relFile = path.relative(process.cwd(), test.file);
process.stdout.write(
`\n::warning file=${relFile},title=${title}::${body}\n`
);
}

for (const test of this.failedTests) {
const parts = [test.errorMessage.split('\n')[0].slice(0, 150)];
if (test.runId) parts.push(`Run: ${test.runId}`);
Expand DownExpand Up@@ -155,4 +195,16 @@ export default class GithubAnnotationReporter implements Reporter {

fs.writeFileSync(filePath, JSON.stringify(this.failedTests, null, 2));
}

private writeFlakySidecar() {
const appName = process.env.APP_NAME || 'unknown';
const isVercel = !!process.env.WORKFLOW_VERCEL_ENV;
const backend = isVercel ? 'vercel' : 'local';
const filePath = path.resolve(
process.cwd(),
`e2e-flaky-${appName}-${backend}.json`
);

fs.writeFileSync(filePath, JSON.stringify(this.flakyTests, null, 2));
}
}
9 changes: 9 additions & 0 deletions vitest.config.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,6 +3,15 @@ import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
testTimeout: 60_000,
// The e2e suites drive real deployments, so individual tests can lose
// timing races (queue delays, cold starts, watcher latency) that a
// second attempt absorbs. One CI retry keeps a single racy test from
// failing a 20+ minute matrix job; retried tests stay visible — the
// github-reporter annotates them and the PR comment lists them — so
// real races still get looked at. Harnesses where a failure is itself
// the signal (event-log-race-repro) pin `retry: 0` locally.
// Local runs keep retry at 0 so races reproduce while debugging.
retry: process.env.CI ? 1 : 0,
},
benchmark: {
include: ['**/*.bench.ts'],
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Strip utm_, fbclid, gclid, etc. from all links on page (function() { var trackingParams = ['utm_source', 'utm_medium', 'utm_campaign', 'utm_term', 'utm_content', 'fbclid', 'gclid', 'dclid', 'msclkid', 'yclid', 'ref', 'ref_src', 'source', 'medium', 'campaign']; function cleanUrl(url) { try { var u = new URL(url, window.location.origin); var changed = false; trackingParams.forEach(function(p) { if (u.searchParams.has(p)) { u.searchParams.delete(p); changed = true; } }); return changed ? u.toString() : url; } catch (e) { return url; } } function cleanLinks() { document.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } cleanLinks(); var observer = new MutationObserver(function(mutations) { mutations.forEach(function(m) { m.addedNodes.forEach(function(node) { if (node.nodeType === 1) { if (node.tagName === 'A') cleanLinks(); node.querySelectorAll('a[href]').forEach(function(a) { var clean = cleanUrl(a.href); if (clean !== a.href) a.href = clean; }); } }); }); }); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:Remove Tracking Parameters from Links]', __e); } })(); (function(){ try { var __m = "youtube.com"; var __re = new RegExp('^' + "youtube\\.com" + ' [stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530) by alangenfeld · Pull Request #3555 · vercel/workflow · GitHub
Skip to content

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530)#3555

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
alangenfeld wants to merge 1 commit into from
Closed

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530) #3555

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/e2e-ci-retry.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
---
'@workflow/core': patch
---

Retry e2e tests once in CI and surface retried-then-passed tests as flaky in annotations and the PR results comment.
89 changes: 85 additions & 4 deletions .github/scripts/aggregate-e2e-results.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ function findResultFiles(dir) {
return findJsonFiles(dir, 'e2e-', [
'e2e-metadata-',
'e2e-failures-',
'e2e-flaky-',
'e2e-diagnostics-',
]);
}
Expand DownExpand Up@@ -146,6 +147,79 @@ function loadFailures(dir) {
return failures;
}

// Load flaky-test sidecar files (tests that passed only after a retry,
// written by github-reporter). Grouped per app; the same test flaking in
// several jobs for one app is collapsed into a single entry with an
// occurrence count so the section stays scannable.
function loadFlaky(dir) {
// Map of `${app}\u0000${testName}` -> { app, testName, retryCount, occurrences }
const flaky = new Map();
const files = findJsonFiles(dir, 'e2e-flaky-');

for (const file of files) {
const basename = path.basename(file, '.json');
const match = basename.match(/^e2e-flaky-(.+)-(?:vercel|local)$/);
const app = match ? match[1] : 'unknown';
try {
const entries = JSON.parse(fs.readFileSync(file, 'utf-8'));
for (const entry of entries) {
if (!entry.testName) continue;
const key = `${app}\u0000${entry.testName}`;
const existing = flaky.get(key);
if (existing) {
existing.occurrences++;
existing.retryCount = Math.max(
existing.retryCount,
entry.retryCount || 1
);
} else {
flaky.set(key, {
app,
testName: entry.testName,
retryCount: entry.retryCount || 1,
occurrences: 1,
});
}
}
} catch (_e) {
// Skip invalid files
}
}

return [...flaky.values()];
}

// Render the flaky-tests section shared by the PR comment and the per-job
// step summary. Retried-to-green tests would otherwise be invisible — the
// job is green — so this is the only place a recurring race stays visible.
function renderFlakySection(flakyTests) {
if (flakyTests.length === 0) return;

console.log('### ⚠️ Flaky E2E Tests (passed on retry)\n');
console.log(
'_These tests failed at least once and passed on a retry. A recurring entry here is a real race worth investigating._\n'
);

const sorted = [...flakyTests].sort(
(a, b) =>
b.occurrences - a.occurrences || a.testName.localeCompare(b.testName)
);
const collapse = sorted.length >= 10;
if (collapse) {
console.log('<details>');
console.log(`<summary>${sorted.length} flaky tests</summary>\n`);
}
for (const test of sorted) {
const jobs =
test.occurrences > 1 ? ` — flaked in ${test.occurrences} jobs` : '';
console.log(`- \`${test.testName}\` (${test.app})${jobs}`);
}
console.log('');
if (collapse) {
console.log('</details>\n');
}
}

// vitest's JSON reporter serializes only error stacks. For test timeouts the
// stack is the task-collection stack ("Error: STACK_TRACE_ERROR ..."), which
// carries no information about the failure. The github-reporter failures
Expand DownExpand Up@@ -355,7 +429,7 @@ function aggregateByCategory(files) {
}

// Render markdown summary for single job (step summary)
function renderSingleJobSummary(summary) {
function renderSingleJobSummary(summary, flakyTests = []) {
const total =
summary.totalPassed + summary.totalFailed + summary.totalSkipped;
const statusEmoji = summary.totalFailed > 0 ? '❌' : '✅';
Expand DownExpand Up@@ -394,6 +468,8 @@ function renderSingleJobSummary(summary) {
}
}

renderFlakySection(flakyTests);

// Results by file
if (summary.fileResults.length > 1) {
console.log('<details>');
Expand DownExpand Up@@ -438,7 +514,8 @@ function renderAggregatedSummary(
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
) {
const total =
overallSummary.totalPassed +
Expand DownExpand Up@@ -557,6 +634,8 @@ function renderAggregatedSummary(
}
}

renderFlakySection(flakyTests);

// Detailed breakdown by category
console.log('### Details by Category\n');

Expand DownExpand Up@@ -605,13 +684,15 @@ if (mode === 'aggregate') {
const metadata = loadMetadata(resultsDir);
const diagnostics = loadDiagnostics(resultsDir);
const failures = loadFailures(resultsDir);
const flakyTests = loadFlaky(resultsDir);
enrichFailedTestMessages(overallSummary.allFailedTests, failures);
renderAggregatedSummary(
categories,
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
);

// Exit with non-zero if any tests failed
Expand All@@ -621,7 +702,7 @@ if (mode === 'aggregate') {
} else {
const summary = aggregateResults(resultFiles);
enrichFailedTestMessages(summary.allFailedTests, loadFailures(resultsDir));
renderSingleJobSummary(summary);
renderSingleJobSummary(summary, loadFlaky(resultsDir));

// Exit with non-zero if any tests failed
if (summary.totalFailed > 0) {
Expand Down
17 changes: 13 additions & 4 deletions .github/workflows/tests.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -324,6 +324,7 @@ jobs:
e2e-vercel-prod-${{ matrix.app.name }}.json
e2e-metadata-${{ matrix.app.name }}-vercel.json
e2e-failures-${{ matrix.app.name }}-vercel.json
e2e-flaky-${{ matrix.app.name }}-vercel.json
e2e-diagnostics-${{ matrix.app.name }}-vercel.json
retention-days: 7
if-no-files-found: ignore
Expand DownExpand Up@@ -420,7 +421,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -498,7 +501,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -596,7 +601,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -678,7 +685,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-windows-nextjs-turbopack
path: e2e-windows-nextjs-turbopack.json
path: |
e2e-windows-nextjs-turbopack.json
e2e-flaky-nextjs-turbopack-local.json
retention-days: 7
if-no-files-found: ignore

Expand Down
5 changes: 4 additions & 1 deletion packages/core/e2e/event-log-race-repro.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -995,7 +995,10 @@ const testTimeoutMs =
config.stuckGraceMs +
60_000;

describe('event log race repro', () => {
// This harness's failures ARE the signal it exists to produce, and a single
// pass runs for the whole configured budget — never let the CI-wide e2e
// retry (vitest.config.ts) re-run it.
describe('event log race repro', { retry: 0 }, () => {
beforeAll(() => {
setupWorld(deploymentUrl);

Expand Down
64 changes: 58 additions & 6 deletions packages/core/e2e/github-reporter.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,10 @@
* Custom vitest reporter that emits GitHub Actions annotations for failed tests.
*
* When running in CI, failed e2e tests produce `::error` workflow commands that
* surface as annotations in the GitHub Actions UI and on PR file diffs.
* surface as annotations in the GitHub Actions UI and on PR file diffs. Tests
* that only passed after a retry (see `retry` in vitest.config.ts) produce
* `::warning` annotations and a `e2e-flaky-*.json` sidecar, so the retry that
* keeps a racy test from failing the job does not also hide the race.
*
* Also writes an enriched JSON sidecar file (`e2e-failures-*.json`) with
* per-test failure details including run IDs and dashboard links, which the
Expand DownExpand Up@@ -37,8 +40,16 @@ interface DiagnosticsEntry {
timestamp: string;
}

interface FlakyTestInfo {
testName: string;
fullName: string;
file: string;
retryCount: number;
}

export default class GithubAnnotationReporter implements Reporter {
private failedTests: FailedTestInfo[] = [];
private flakyTests: FlakyTestInfo[] = [];

onTestRunEnd(testModules: ReadonlyArray<TestModule>) {
for (const module of testModules) {
Expand All@@ -49,18 +60,36 @@ export default class GithubAnnotationReporter implements Reporter {
// Enrich failures with diagnostics sidecar data (run IDs, dashboard URLs)
this.enrichFromDiagnosticsSidecar();
this.writeFailuresSidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
if (this.flakyTests.length > 0) {
this.writeFlakySidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
}

private collectFailures(module: TestModule) {
for (const test of module.children.allTests()) {
const result = test.result();

if (result.state === 'passed') {
const retryCount = test.diagnostic()?.retryCount ?? 0;
if (retryCount > 0) {
this.flakyTests.push({
testName: test.name,
fullName: test.fullName,
file: module.moduleId,
retryCount,
});
}
continue;
}

if (result.state !== 'failed') continue;

const errors = result.errors || [];
Expand DownExpand Up@@ -127,6 +156,17 @@ export default class GithubAnnotationReporter implements Reporter {
* rather than the workflow source file (which may be a symlink).
*/
private emitAnnotations() {
for (const test of this.flakyTests) {
const title = `E2E flaky: ${test.testName}`;
const body = `Passed only after ${test.retryCount} retr${
test.retryCount === 1 ? 'y' : 'ies'
} — this test lost a race on its first attempt.`;
const relFile = path.relative(process.cwd(), test.file);
process.stdout.write(
`\n::warning file=${relFile},title=${title}::${body}\n`
);
}

for (const test of this.failedTests) {
const parts = [test.errorMessage.split('\n')[0].slice(0, 150)];
if (test.runId) parts.push(`Run: ${test.runId}`);
Expand DownExpand Up@@ -155,4 +195,16 @@ export default class GithubAnnotationReporter implements Reporter {

fs.writeFileSync(filePath, JSON.stringify(this.failedTests, null, 2));
}

private writeFlakySidecar() {
const appName = process.env.APP_NAME || 'unknown';
const isVercel = !!process.env.WORKFLOW_VERCEL_ENV;
const backend = isVercel ? 'vercel' : 'local';
const filePath = path.resolve(
process.cwd(),
`e2e-flaky-${appName}-${backend}.json`
);

fs.writeFileSync(filePath, JSON.stringify(this.flakyTests, null, 2));
}
}
9 changes: 9 additions & 0 deletions vitest.config.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,6 +3,15 @@ import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
testTimeout: 60_000,
// The e2e suites drive real deployments, so individual tests can lose
// timing races (queue delays, cold starts, watcher latency) that a
// second attempt absorbs. One CI retry keeps a single racy test from
// failing a 20+ minute matrix job; retried tests stay visible — the
// github-reporter annotates them and the PR comment lists them — so
// real races still get looked at. Harnesses where a failure is itself
// the signal (event-log-race-repro) pin `retry: 0` locally.
// Local runs keep retry at 0 so races reproduce while debugging.
retry: process.env.CI ? 1 : 0,
},
benchmark: {
include: ['**/*.bench.ts'],
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Auto-enable theater mode on YouTube (function() { function tryTheater() { var btn = document.querySelector('button[aria-label="Theater mode"], ytd-player #player button[title="Theater mode"]'); if (btn && !btn.classList.contains('activated')) { btn.click(); } } // Try immediately tryTheater(); // Try after navigation (SPA) var lastUrl = location.href; setInterval(function() { if (location.href !== lastUrl) { lastUrl = location.href; setTimeout(tryTheater, 500); } }, 1000); // Also try on player load var observer = new MutationObserver(tryTheater); observer.observe(document.body, { childList: true, subtree: true }); })(); } } catch(__e) { console.warn('[Userscript:YouTube Theater Mode Default]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' [stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530) by alangenfeld · Pull Request #3555 · vercel/workflow · GitHub
Skip to content

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530)#3555

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
alangenfeld wants to merge 1 commit into from
Closed

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530) #3555

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/e2e-ci-retry.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
---
'@workflow/core': patch
---

Retry e2e tests once in CI and surface retried-then-passed tests as flaky in annotations and the PR results comment.
89 changes: 85 additions & 4 deletions .github/scripts/aggregate-e2e-results.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ function findResultFiles(dir) {
return findJsonFiles(dir, 'e2e-', [
'e2e-metadata-',
'e2e-failures-',
'e2e-flaky-',
'e2e-diagnostics-',
]);
}
Expand DownExpand Up@@ -146,6 +147,79 @@ function loadFailures(dir) {
return failures;
}

// Load flaky-test sidecar files (tests that passed only after a retry,
// written by github-reporter). Grouped per app; the same test flaking in
// several jobs for one app is collapsed into a single entry with an
// occurrence count so the section stays scannable.
function loadFlaky(dir) {
// Map of `${app}\u0000${testName}` -> { app, testName, retryCount, occurrences }
const flaky = new Map();
const files = findJsonFiles(dir, 'e2e-flaky-');

for (const file of files) {
const basename = path.basename(file, '.json');
const match = basename.match(/^e2e-flaky-(.+)-(?:vercel|local)$/);
const app = match ? match[1] : 'unknown';
try {
const entries = JSON.parse(fs.readFileSync(file, 'utf-8'));
for (const entry of entries) {
if (!entry.testName) continue;
const key = `${app}\u0000${entry.testName}`;
const existing = flaky.get(key);
if (existing) {
existing.occurrences++;
existing.retryCount = Math.max(
existing.retryCount,
entry.retryCount || 1
);
} else {
flaky.set(key, {
app,
testName: entry.testName,
retryCount: entry.retryCount || 1,
occurrences: 1,
});
}
}
} catch (_e) {
// Skip invalid files
}
}

return [...flaky.values()];
}

// Render the flaky-tests section shared by the PR comment and the per-job
// step summary. Retried-to-green tests would otherwise be invisible — the
// job is green — so this is the only place a recurring race stays visible.
function renderFlakySection(flakyTests) {
if (flakyTests.length === 0) return;

console.log('### ⚠️ Flaky E2E Tests (passed on retry)\n');
console.log(
'_These tests failed at least once and passed on a retry. A recurring entry here is a real race worth investigating._\n'
);

const sorted = [...flakyTests].sort(
(a, b) =>
b.occurrences - a.occurrences || a.testName.localeCompare(b.testName)
);
const collapse = sorted.length >= 10;
if (collapse) {
console.log('<details>');
console.log(`<summary>${sorted.length} flaky tests</summary>\n`);
}
for (const test of sorted) {
const jobs =
test.occurrences > 1 ? ` — flaked in ${test.occurrences} jobs` : '';
console.log(`- \`${test.testName}\` (${test.app})${jobs}`);
}
console.log('');
if (collapse) {
console.log('</details>\n');
}
}

// vitest's JSON reporter serializes only error stacks. For test timeouts the
// stack is the task-collection stack ("Error: STACK_TRACE_ERROR ..."), which
// carries no information about the failure. The github-reporter failures
Expand DownExpand Up@@ -355,7 +429,7 @@ function aggregateByCategory(files) {
}

// Render markdown summary for single job (step summary)
function renderSingleJobSummary(summary) {
function renderSingleJobSummary(summary, flakyTests = []) {
const total =
summary.totalPassed + summary.totalFailed + summary.totalSkipped;
const statusEmoji = summary.totalFailed > 0 ? '❌' : '✅';
Expand DownExpand Up@@ -394,6 +468,8 @@ function renderSingleJobSummary(summary) {
}
}

renderFlakySection(flakyTests);

// Results by file
if (summary.fileResults.length > 1) {
console.log('<details>');
Expand DownExpand Up@@ -438,7 +514,8 @@ function renderAggregatedSummary(
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
) {
const total =
overallSummary.totalPassed +
Expand DownExpand Up@@ -557,6 +634,8 @@ function renderAggregatedSummary(
}
}

renderFlakySection(flakyTests);

// Detailed breakdown by category
console.log('### Details by Category\n');

Expand DownExpand Up@@ -605,13 +684,15 @@ if (mode === 'aggregate') {
const metadata = loadMetadata(resultsDir);
const diagnostics = loadDiagnostics(resultsDir);
const failures = loadFailures(resultsDir);
const flakyTests = loadFlaky(resultsDir);
enrichFailedTestMessages(overallSummary.allFailedTests, failures);
renderAggregatedSummary(
categories,
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
);

// Exit with non-zero if any tests failed
Expand All@@ -621,7 +702,7 @@ if (mode === 'aggregate') {
} else {
const summary = aggregateResults(resultFiles);
enrichFailedTestMessages(summary.allFailedTests, loadFailures(resultsDir));
renderSingleJobSummary(summary);
renderSingleJobSummary(summary, loadFlaky(resultsDir));

// Exit with non-zero if any tests failed
if (summary.totalFailed > 0) {
Expand Down
17 changes: 13 additions & 4 deletions .github/workflows/tests.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -324,6 +324,7 @@ jobs:
e2e-vercel-prod-${{ matrix.app.name }}.json
e2e-metadata-${{ matrix.app.name }}-vercel.json
e2e-failures-${{ matrix.app.name }}-vercel.json
e2e-flaky-${{ matrix.app.name }}-vercel.json
e2e-diagnostics-${{ matrix.app.name }}-vercel.json
retention-days: 7
if-no-files-found: ignore
Expand DownExpand Up@@ -420,7 +421,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -498,7 +501,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -596,7 +601,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -678,7 +685,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-windows-nextjs-turbopack
path: e2e-windows-nextjs-turbopack.json
path: |
e2e-windows-nextjs-turbopack.json
e2e-flaky-nextjs-turbopack-local.json
retention-days: 7
if-no-files-found: ignore

Expand Down
5 changes: 4 additions & 1 deletion packages/core/e2e/event-log-race-repro.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -995,7 +995,10 @@ const testTimeoutMs =
config.stuckGraceMs +
60_000;

describe('event log race repro', () => {
// This harness's failures ARE the signal it exists to produce, and a single
// pass runs for the whole configured budget — never let the CI-wide e2e
// retry (vitest.config.ts) re-run it.
describe('event log race repro', { retry: 0 }, () => {
beforeAll(() => {
setupWorld(deploymentUrl);

Expand Down
64 changes: 58 additions & 6 deletions packages/core/e2e/github-reporter.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,10 @@
* Custom vitest reporter that emits GitHub Actions annotations for failed tests.
*
* When running in CI, failed e2e tests produce `::error` workflow commands that
* surface as annotations in the GitHub Actions UI and on PR file diffs.
* surface as annotations in the GitHub Actions UI and on PR file diffs. Tests
* that only passed after a retry (see `retry` in vitest.config.ts) produce
* `::warning` annotations and a `e2e-flaky-*.json` sidecar, so the retry that
* keeps a racy test from failing the job does not also hide the race.
*
* Also writes an enriched JSON sidecar file (`e2e-failures-*.json`) with
* per-test failure details including run IDs and dashboard links, which the
Expand DownExpand Up@@ -37,8 +40,16 @@ interface DiagnosticsEntry {
timestamp: string;
}

interface FlakyTestInfo {
testName: string;
fullName: string;
file: string;
retryCount: number;
}

export default class GithubAnnotationReporter implements Reporter {
private failedTests: FailedTestInfo[] = [];
private flakyTests: FlakyTestInfo[] = [];

onTestRunEnd(testModules: ReadonlyArray<TestModule>) {
for (const module of testModules) {
Expand All@@ -49,18 +60,36 @@ export default class GithubAnnotationReporter implements Reporter {
// Enrich failures with diagnostics sidecar data (run IDs, dashboard URLs)
this.enrichFromDiagnosticsSidecar();
this.writeFailuresSidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
if (this.flakyTests.length > 0) {
this.writeFlakySidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
}

private collectFailures(module: TestModule) {
for (const test of module.children.allTests()) {
const result = test.result();

if (result.state === 'passed') {
const retryCount = test.diagnostic()?.retryCount ?? 0;
if (retryCount > 0) {
this.flakyTests.push({
testName: test.name,
fullName: test.fullName,
file: module.moduleId,
retryCount,
});
}
continue;
}

if (result.state !== 'failed') continue;

const errors = result.errors || [];
Expand DownExpand Up@@ -127,6 +156,17 @@ export default class GithubAnnotationReporter implements Reporter {
* rather than the workflow source file (which may be a symlink).
*/
private emitAnnotations() {
for (const test of this.flakyTests) {
const title = `E2E flaky: ${test.testName}`;
const body = `Passed only after ${test.retryCount} retr${
test.retryCount === 1 ? 'y' : 'ies'
} — this test lost a race on its first attempt.`;
const relFile = path.relative(process.cwd(), test.file);
process.stdout.write(
`\n::warning file=${relFile},title=${title}::${body}\n`
);
}

for (const test of this.failedTests) {
const parts = [test.errorMessage.split('\n')[0].slice(0, 150)];
if (test.runId) parts.push(`Run: ${test.runId}`);
Expand DownExpand Up@@ -155,4 +195,16 @@ export default class GithubAnnotationReporter implements Reporter {

fs.writeFileSync(filePath, JSON.stringify(this.failedTests, null, 2));
}

private writeFlakySidecar() {
const appName = process.env.APP_NAME || 'unknown';
const isVercel = !!process.env.WORKFLOW_VERCEL_ENV;
const backend = isVercel ? 'vercel' : 'local';
const filePath = path.resolve(
process.cwd(),
`e2e-flaky-${appName}-${backend}.json`
);

fs.writeFileSync(filePath, JSON.stringify(this.flakyTests, null, 2));
}
}
9 changes: 9 additions & 0 deletions vitest.config.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,6 +3,15 @@ import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
testTimeout: 60_000,
// The e2e suites drive real deployments, so individual tests can lose
// timing races (queue delays, cold starts, watcher latency) that a
// second attempt absorbs. One CI retry keeps a single racy test from
// failing a 20+ minute matrix job; retried tests stay visible — the
// github-reporter annotates them and the PR comment lists them — so
// real races still get looked at. Harnesses where a failure is itself
// the signal (event-log-race-repro) pin `retry: 0` locally.
// Local runs keep retry at 0 so races reproduce while debugging.
retry: process.env.CI ? 1 : 0,
},
benchmark: {
include: ['**/*.bench.ts'],
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Remove or un-stick sticky/fixed headers that block content (function() { function unstick() { document.querySelectorAll('header, nav, [role="banner"], .header, .navbar, .sticky, .fixed-top, [style*="position: fixed"], [style*="position:sticky"]').forEach(function(el) { if (el.style.position === 'fixed' || el.style.position === 'sticky' || getComputedStyle(el).position === 'fixed' || getComputedStyle(el).position === 'sticky') { el.style.position = 'static'; el.style.top = 'auto'; el.style.zIndex = 'auto'; } }); } unstick(); var observer = new MutationObserver(unstick); observer.observe(document.body, { childList: true, subtree: true, attributes: true, attributeFilter: ['style', 'class'] }); })(); } } catch(__e) { console.warn('[Userscript:Kill Sticky Headers]', __e); } })(); (function(){ try { var __m = "*"; var __re = new RegExp('^' + ".*" + ' [stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530) by alangenfeld · Pull Request #3555 · vercel/workflow · GitHub
Skip to content

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530)#3555

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
alangenfeld wants to merge 1 commit into from
Closed

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530) #3555

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/e2e-ci-retry.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
---
'@workflow/core': patch
---

Retry e2e tests once in CI and surface retried-then-passed tests as flaky in annotations and the PR results comment.
89 changes: 85 additions & 4 deletions .github/scripts/aggregate-e2e-results.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ function findResultFiles(dir) {
return findJsonFiles(dir, 'e2e-', [
'e2e-metadata-',
'e2e-failures-',
'e2e-flaky-',
'e2e-diagnostics-',
]);
}
Expand DownExpand Up@@ -146,6 +147,79 @@ function loadFailures(dir) {
return failures;
}

// Load flaky-test sidecar files (tests that passed only after a retry,
// written by github-reporter). Grouped per app; the same test flaking in
// several jobs for one app is collapsed into a single entry with an
// occurrence count so the section stays scannable.
function loadFlaky(dir) {
// Map of `${app}\u0000${testName}` -> { app, testName, retryCount, occurrences }
const flaky = new Map();
const files = findJsonFiles(dir, 'e2e-flaky-');

for (const file of files) {
const basename = path.basename(file, '.json');
const match = basename.match(/^e2e-flaky-(.+)-(?:vercel|local)$/);
const app = match ? match[1] : 'unknown';
try {
const entries = JSON.parse(fs.readFileSync(file, 'utf-8'));
for (const entry of entries) {
if (!entry.testName) continue;
const key = `${app}\u0000${entry.testName}`;
const existing = flaky.get(key);
if (existing) {
existing.occurrences++;
existing.retryCount = Math.max(
existing.retryCount,
entry.retryCount || 1
);
} else {
flaky.set(key, {
app,
testName: entry.testName,
retryCount: entry.retryCount || 1,
occurrences: 1,
});
}
}
} catch (_e) {
// Skip invalid files
}
}

return [...flaky.values()];
}

// Render the flaky-tests section shared by the PR comment and the per-job
// step summary. Retried-to-green tests would otherwise be invisible — the
// job is green — so this is the only place a recurring race stays visible.
function renderFlakySection(flakyTests) {
if (flakyTests.length === 0) return;

console.log('### ⚠️ Flaky E2E Tests (passed on retry)\n');
console.log(
'_These tests failed at least once and passed on a retry. A recurring entry here is a real race worth investigating._\n'
);

const sorted = [...flakyTests].sort(
(a, b) =>
b.occurrences - a.occurrences || a.testName.localeCompare(b.testName)
);
const collapse = sorted.length >= 10;
if (collapse) {
console.log('<details>');
console.log(`<summary>${sorted.length} flaky tests</summary>\n`);
}
for (const test of sorted) {
const jobs =
test.occurrences > 1 ? ` — flaked in ${test.occurrences} jobs` : '';
console.log(`- \`${test.testName}\` (${test.app})${jobs}`);
}
console.log('');
if (collapse) {
console.log('</details>\n');
}
}

// vitest's JSON reporter serializes only error stacks. For test timeouts the
// stack is the task-collection stack ("Error: STACK_TRACE_ERROR ..."), which
// carries no information about the failure. The github-reporter failures
Expand DownExpand Up@@ -355,7 +429,7 @@ function aggregateByCategory(files) {
}

// Render markdown summary for single job (step summary)
function renderSingleJobSummary(summary) {
function renderSingleJobSummary(summary, flakyTests = []) {
const total =
summary.totalPassed + summary.totalFailed + summary.totalSkipped;
const statusEmoji = summary.totalFailed > 0 ? '❌' : '✅';
Expand DownExpand Up@@ -394,6 +468,8 @@ function renderSingleJobSummary(summary) {
}
}

renderFlakySection(flakyTests);

// Results by file
if (summary.fileResults.length > 1) {
console.log('<details>');
Expand DownExpand Up@@ -438,7 +514,8 @@ function renderAggregatedSummary(
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
) {
const total =
overallSummary.totalPassed +
Expand DownExpand Up@@ -557,6 +634,8 @@ function renderAggregatedSummary(
}
}

renderFlakySection(flakyTests);

// Detailed breakdown by category
console.log('### Details by Category\n');

Expand DownExpand Up@@ -605,13 +684,15 @@ if (mode === 'aggregate') {
const metadata = loadMetadata(resultsDir);
const diagnostics = loadDiagnostics(resultsDir);
const failures = loadFailures(resultsDir);
const flakyTests = loadFlaky(resultsDir);
enrichFailedTestMessages(overallSummary.allFailedTests, failures);
renderAggregatedSummary(
categories,
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
);

// Exit with non-zero if any tests failed
Expand All@@ -621,7 +702,7 @@ if (mode === 'aggregate') {
} else {
const summary = aggregateResults(resultFiles);
enrichFailedTestMessages(summary.allFailedTests, loadFailures(resultsDir));
renderSingleJobSummary(summary);
renderSingleJobSummary(summary, loadFlaky(resultsDir));

// Exit with non-zero if any tests failed
if (summary.totalFailed > 0) {
Expand Down
17 changes: 13 additions & 4 deletions .github/workflows/tests.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -324,6 +324,7 @@ jobs:
e2e-vercel-prod-${{ matrix.app.name }}.json
e2e-metadata-${{ matrix.app.name }}-vercel.json
e2e-failures-${{ matrix.app.name }}-vercel.json
e2e-flaky-${{ matrix.app.name }}-vercel.json
e2e-diagnostics-${{ matrix.app.name }}-vercel.json
retention-days: 7
if-no-files-found: ignore
Expand DownExpand Up@@ -420,7 +421,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -498,7 +501,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -596,7 +601,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -678,7 +685,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-windows-nextjs-turbopack
path: e2e-windows-nextjs-turbopack.json
path: |
e2e-windows-nextjs-turbopack.json
e2e-flaky-nextjs-turbopack-local.json
retention-days: 7
if-no-files-found: ignore

Expand Down
5 changes: 4 additions & 1 deletion packages/core/e2e/event-log-race-repro.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -995,7 +995,10 @@ const testTimeoutMs =
config.stuckGraceMs +
60_000;

describe('event log race repro', () => {
// This harness's failures ARE the signal it exists to produce, and a single
// pass runs for the whole configured budget — never let the CI-wide e2e
// retry (vitest.config.ts) re-run it.
describe('event log race repro', { retry: 0 }, () => {
beforeAll(() => {
setupWorld(deploymentUrl);

Expand Down
64 changes: 58 additions & 6 deletions packages/core/e2e/github-reporter.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,10 @@
* Custom vitest reporter that emits GitHub Actions annotations for failed tests.
*
* When running in CI, failed e2e tests produce `::error` workflow commands that
* surface as annotations in the GitHub Actions UI and on PR file diffs.
* surface as annotations in the GitHub Actions UI and on PR file diffs. Tests
* that only passed after a retry (see `retry` in vitest.config.ts) produce
* `::warning` annotations and a `e2e-flaky-*.json` sidecar, so the retry that
* keeps a racy test from failing the job does not also hide the race.
*
* Also writes an enriched JSON sidecar file (`e2e-failures-*.json`) with
* per-test failure details including run IDs and dashboard links, which the
Expand DownExpand Up@@ -37,8 +40,16 @@ interface DiagnosticsEntry {
timestamp: string;
}

interface FlakyTestInfo {
testName: string;
fullName: string;
file: string;
retryCount: number;
}

export default class GithubAnnotationReporter implements Reporter {
private failedTests: FailedTestInfo[] = [];
private flakyTests: FlakyTestInfo[] = [];

onTestRunEnd(testModules: ReadonlyArray<TestModule>) {
for (const module of testModules) {
Expand All@@ -49,18 +60,36 @@ export default class GithubAnnotationReporter implements Reporter {
// Enrich failures with diagnostics sidecar data (run IDs, dashboard URLs)
this.enrichFromDiagnosticsSidecar();
this.writeFailuresSidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
if (this.flakyTests.length > 0) {
this.writeFlakySidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
}

private collectFailures(module: TestModule) {
for (const test of module.children.allTests()) {
const result = test.result();

if (result.state === 'passed') {
const retryCount = test.diagnostic()?.retryCount ?? 0;
if (retryCount > 0) {
this.flakyTests.push({
testName: test.name,
fullName: test.fullName,
file: module.moduleId,
retryCount,
});
}
continue;
}

if (result.state !== 'failed') continue;

const errors = result.errors || [];
Expand DownExpand Up@@ -127,6 +156,17 @@ export default class GithubAnnotationReporter implements Reporter {
* rather than the workflow source file (which may be a symlink).
*/
private emitAnnotations() {
for (const test of this.flakyTests) {
const title = `E2E flaky: ${test.testName}`;
const body = `Passed only after ${test.retryCount} retr${
test.retryCount === 1 ? 'y' : 'ies'
} — this test lost a race on its first attempt.`;
const relFile = path.relative(process.cwd(), test.file);
process.stdout.write(
`\n::warning file=${relFile},title=${title}::${body}\n`
);
}

for (const test of this.failedTests) {
const parts = [test.errorMessage.split('\n')[0].slice(0, 150)];
if (test.runId) parts.push(`Run: ${test.runId}`);
Expand DownExpand Up@@ -155,4 +195,16 @@ export default class GithubAnnotationReporter implements Reporter {

fs.writeFileSync(filePath, JSON.stringify(this.failedTests, null, 2));
}

private writeFlakySidecar() {
const appName = process.env.APP_NAME || 'unknown';
const isVercel = !!process.env.WORKFLOW_VERCEL_ENV;
const backend = isVercel ? 'vercel' : 'local';
const filePath = path.resolve(
process.cwd(),
`e2e-flaky-${appName}-${backend}.json`
);

fs.writeFileSync(filePath, JSON.stringify(this.flakyTests, null, 2));
}
}
9 changes: 9 additions & 0 deletions vitest.config.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,6 +3,15 @@ import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
testTimeout: 60_000,
// The e2e suites drive real deployments, so individual tests can lose
// timing races (queue delays, cold starts, watcher latency) that a
// second attempt absorbs. One CI retry keeps a single racy test from
// failing a 20+ minute matrix job; retried tests stay visible — the
// github-reporter annotates them and the PR comment lists them — so
// real races still get looked at. Harnesses where a failure is itself
// the signal (event-log-race-repro) pin `retry: 0` locally.
// Local runs keep retry at 0 so races reproduce while debugging.
retry: process.env.CI ? 1 : 0,
},
benchmark: {
include: ['**/*.bench.ts'],
Expand Down
Loading
, 'i'); if (__m === '*' || __re.test(location.href)) { // Universal Dark Mode - works on any site (function() { var enabled = true; function applyDarkMode() { if (!enabled) return; // Create style element if it doesn't exist var style = document.getElementById('universal-dark-mode-style'); if (!style) { style = document.createElement('style'); style.id = 'universal-dark-mode-style'; document.head.appendChild(style); } // Dark mode CSS - inverts colors but preserves images/video style.textContent = ' /* Invert everything except media */ html { filter: invert(1) hue-rotate(180deg) !important; background: #1a1a2e !important; } /* Restore images, videos, iframes, canvas */ img, video, iframe, canvas, svg, picture, [style*="background-image"] { filter: invert(1) hue-rotate(180deg) !important; } /* Preserve specific elements that should not be inverted */ .no-dark-mode, .no-dark-mode *, [data-theme="light"], [data-theme="light"], .ace_editor, .ace_editor *, .CodeMirror, .CodeMirror *, .monaco-editor, .monaco-editor *, .markdown-body pre, .markdown-body pre *, .highlight, .highlight *, pre code, pre code * { filter: none !important; } /* Fix common UI elements */ .modal, .popup, .dropdown-menu, .tooltip, .popover { filter: invert(1) hue-rotate(180deg) !important; background: #2d2d44 !important; border-color: #444 !important; } /* Scrollbars */ ::-webkit-scrollbar { background: #1a1a2e !important; } ::-webkit-scrollbar-thumb { background: #444 !important; } ::-webkit-scrollbar-thumb:hover { background: #555 !important; } /* Selection */ ::selection { background: #4ecdc4 !important; color: #1a1a2e !important; } ::-moz-selection { background: #4ecdc4 !important; color: #1a1a2e !important; } '; } function removeDarkMode() { var style = document.getElementById('universal-dark-mode-style'); if (style) style.remove(); } // Toggle with Alt+Shift+D document.addEventListener('keydown', function(e) { if (e.altKey && e.shiftKey && e.key === 'D') { e.preventDefault(); enabled = !enabled; if (enabled) { applyDarkMode(); console.log('[Universal Dark Mode] Enabled'); } else { removeDarkMode(); console.log('[Universal Dark Mode] Disabled'); } } }); // Apply on load applyDarkMode(); // Re-apply on dynamic content var observer = new MutationObserver(function(mutations) { if (enabled && !document.getElementById('universal-dark-mode-style')) { applyDarkMode(); } }); observer.observe(document.head, { childList: true }); console.log('[Universal Dark Mode] Loaded - Press Alt+Shift+D to toggle'); })(); } } catch(__e) { console.warn('[Userscript:Universal Dark Mode]', __e); } })(); })(); [stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530) by alangenfeld · Pull Request #3555 · vercel/workflow · GitHub
Skip to content

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530)#3555

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
alangenfeld wants to merge 1 commit into from
Closed

[stable] Backport: Retry e2e tests once in CI, keeping retried tests visible (#3530) #3555

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/e2e-ci-retry.md
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
---
'@workflow/core': patch
---

Retry e2e tests once in CI and surface retried-then-passed tests as flaky in annotations and the PR results comment.
89 changes: 85 additions & 4 deletions .github/scripts/aggregate-e2e-results.js
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -53,6 +53,7 @@ function findResultFiles(dir) {
return findJsonFiles(dir, 'e2e-', [
'e2e-metadata-',
'e2e-failures-',
'e2e-flaky-',
'e2e-diagnostics-',
]);
}
Expand DownExpand Up@@ -146,6 +147,79 @@ function loadFailures(dir) {
return failures;
}

// Load flaky-test sidecar files (tests that passed only after a retry,
// written by github-reporter). Grouped per app; the same test flaking in
// several jobs for one app is collapsed into a single entry with an
// occurrence count so the section stays scannable.
function loadFlaky(dir) {
// Map of `${app}\u0000${testName}` -> { app, testName, retryCount, occurrences }
const flaky = new Map();
const files = findJsonFiles(dir, 'e2e-flaky-');

for (const file of files) {
const basename = path.basename(file, '.json');
const match = basename.match(/^e2e-flaky-(.+)-(?:vercel|local)$/);
const app = match ? match[1] : 'unknown';
try {
const entries = JSON.parse(fs.readFileSync(file, 'utf-8'));
for (const entry of entries) {
if (!entry.testName) continue;
const key = `${app}\u0000${entry.testName}`;
const existing = flaky.get(key);
if (existing) {
existing.occurrences++;
existing.retryCount = Math.max(
existing.retryCount,
entry.retryCount || 1
);
} else {
flaky.set(key, {
app,
testName: entry.testName,
retryCount: entry.retryCount || 1,
occurrences: 1,
});
}
}
} catch (_e) {
// Skip invalid files
}
}

return [...flaky.values()];
}

// Render the flaky-tests section shared by the PR comment and the per-job
// step summary. Retried-to-green tests would otherwise be invisible — the
// job is green — so this is the only place a recurring race stays visible.
function renderFlakySection(flakyTests) {
if (flakyTests.length === 0) return;

console.log('### ⚠️ Flaky E2E Tests (passed on retry)\n');
console.log(
'_These tests failed at least once and passed on a retry. A recurring entry here is a real race worth investigating._\n'
);

const sorted = [...flakyTests].sort(
(a, b) =>
b.occurrences - a.occurrences || a.testName.localeCompare(b.testName)
);
const collapse = sorted.length >= 10;
if (collapse) {
console.log('<details>');
console.log(`<summary>${sorted.length} flaky tests</summary>\n`);
}
for (const test of sorted) {
const jobs =
test.occurrences > 1 ? ` — flaked in ${test.occurrences} jobs` : '';
console.log(`- \`${test.testName}\` (${test.app})${jobs}`);
}
console.log('');
if (collapse) {
console.log('</details>\n');
}
}

// vitest's JSON reporter serializes only error stacks. For test timeouts the
// stack is the task-collection stack ("Error: STACK_TRACE_ERROR ..."), which
// carries no information about the failure. The github-reporter failures
Expand DownExpand Up@@ -355,7 +429,7 @@ function aggregateByCategory(files) {
}

// Render markdown summary for single job (step summary)
function renderSingleJobSummary(summary) {
function renderSingleJobSummary(summary, flakyTests = []) {
const total =
summary.totalPassed + summary.totalFailed + summary.totalSkipped;
const statusEmoji = summary.totalFailed > 0 ? '❌' : '✅';
Expand DownExpand Up@@ -394,6 +468,8 @@ function renderSingleJobSummary(summary) {
}
}

renderFlakySection(flakyTests);

// Results by file
if (summary.fileResults.length > 1) {
console.log('<details>');
Expand DownExpand Up@@ -438,7 +514,8 @@ function renderAggregatedSummary(
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
) {
const total =
overallSummary.totalPassed +
Expand DownExpand Up@@ -557,6 +634,8 @@ function renderAggregatedSummary(
}
}

renderFlakySection(flakyTests);

// Detailed breakdown by category
console.log('### Details by Category\n');

Expand DownExpand Up@@ -605,13 +684,15 @@ if (mode === 'aggregate') {
const metadata = loadMetadata(resultsDir);
const diagnostics = loadDiagnostics(resultsDir);
const failures = loadFailures(resultsDir);
const flakyTests = loadFlaky(resultsDir);
enrichFailedTestMessages(overallSummary.allFailedTests, failures);
renderAggregatedSummary(
categories,
overallSummary,
metadata,
diagnostics,
failures
failures,
flakyTests
);

// Exit with non-zero if any tests failed
Expand All@@ -621,7 +702,7 @@ if (mode === 'aggregate') {
} else {
const summary = aggregateResults(resultFiles);
enrichFailedTestMessages(summary.allFailedTests, loadFailures(resultsDir));
renderSingleJobSummary(summary);
renderSingleJobSummary(summary, loadFlaky(resultsDir));

// Exit with non-zero if any tests failed
if (summary.totalFailed > 0) {
Expand Down
17 changes: 13 additions & 4 deletions .github/workflows/tests.yml
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -324,6 +324,7 @@ jobs:
e2e-vercel-prod-${{ matrix.app.name }}.json
e2e-metadata-${{ matrix.app.name }}-vercel.json
e2e-failures-${{ matrix.app.name }}-vercel.json
e2e-flaky-${{ matrix.app.name }}-vercel.json
e2e-diagnostics-${{ matrix.app.name }}-vercel.json
retention-days: 7
if-no-files-found: ignore
Expand DownExpand Up@@ -420,7 +421,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-dev-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -498,7 +501,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-prod-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -596,7 +601,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}
path: e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
path: |
e2e-local-postgres-${{ matrix.app.name }}-${{ matrix.app.canary && 'canary' || 'stable' }}.json
e2e-flaky-${{ matrix.app.name }}-local.json
retention-days: 7
if-no-files-found: ignore

Expand DownExpand Up@@ -678,7 +685,9 @@ jobs:
uses: actions/upload-artifact@v4
with:
name: e2e-results-windows-nextjs-turbopack
path: e2e-windows-nextjs-turbopack.json
path: |
e2e-windows-nextjs-turbopack.json
e2e-flaky-nextjs-turbopack-local.json
retention-days: 7
if-no-files-found: ignore

Expand Down
5 changes: 4 additions & 1 deletion packages/core/e2e/event-log-race-repro.test.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -995,7 +995,10 @@ const testTimeoutMs =
config.stuckGraceMs +
60_000;

describe('event log race repro', () => {
// This harness's failures ARE the signal it exists to produce, and a single
// pass runs for the whole configured budget — never let the CI-wide e2e
// retry (vitest.config.ts) re-run it.
describe('event log race repro', { retry: 0 }, () => {
beforeAll(() => {
setupWorld(deploymentUrl);

Expand Down
64 changes: 58 additions & 6 deletions packages/core/e2e/github-reporter.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -2,7 +2,10 @@
* Custom vitest reporter that emits GitHub Actions annotations for failed tests.
*
* When running in CI, failed e2e tests produce `::error` workflow commands that
* surface as annotations in the GitHub Actions UI and on PR file diffs.
* surface as annotations in the GitHub Actions UI and on PR file diffs. Tests
* that only passed after a retry (see `retry` in vitest.config.ts) produce
* `::warning` annotations and a `e2e-flaky-*.json` sidecar, so the retry that
* keeps a racy test from failing the job does not also hide the race.
*
* Also writes an enriched JSON sidecar file (`e2e-failures-*.json`) with
* per-test failure details including run IDs and dashboard links, which the
Expand DownExpand Up@@ -37,8 +40,16 @@ interface DiagnosticsEntry {
timestamp: string;
}

interface FlakyTestInfo {
testName: string;
fullName: string;
file: string;
retryCount: number;
}

export default class GithubAnnotationReporter implements Reporter {
private failedTests: FailedTestInfo[] = [];
private flakyTests: FlakyTestInfo[] = [];

onTestRunEnd(testModules: ReadonlyArray<TestModule>) {
for (const module of testModules) {
Expand All@@ -49,18 +60,36 @@ export default class GithubAnnotationReporter implements Reporter {
// Enrich failures with diagnostics sidecar data (run IDs, dashboard URLs)
this.enrichFromDiagnosticsSidecar();
this.writeFailuresSidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
if (this.flakyTests.length > 0) {
this.writeFlakySidecar();
}

// Emit GitHub Actions annotations — this runs after vitest's own
// output is done, so ::error commands won't be mangled by ANSI codes.
if (process.env.CI) {
this.emitAnnotations();
}
}

private collectFailures(module: TestModule) {
for (const test of module.children.allTests()) {
const result = test.result();

if (result.state === 'passed') {
const retryCount = test.diagnostic()?.retryCount ?? 0;
if (retryCount > 0) {
this.flakyTests.push({
testName: test.name,
fullName: test.fullName,
file: module.moduleId,
retryCount,
});
}
continue;
}

if (result.state !== 'failed') continue;

const errors = result.errors || [];
Expand DownExpand Up@@ -127,6 +156,17 @@ export default class GithubAnnotationReporter implements Reporter {
* rather than the workflow source file (which may be a symlink).
*/
private emitAnnotations() {
for (const test of this.flakyTests) {
const title = `E2E flaky: ${test.testName}`;
const body = `Passed only after ${test.retryCount} retr${
test.retryCount === 1 ? 'y' : 'ies'
} — this test lost a race on its first attempt.`;
const relFile = path.relative(process.cwd(), test.file);
process.stdout.write(
`\n::warning file=${relFile},title=${title}::${body}\n`
);
}

for (const test of this.failedTests) {
const parts = [test.errorMessage.split('\n')[0].slice(0, 150)];
if (test.runId) parts.push(`Run: ${test.runId}`);
Expand DownExpand Up@@ -155,4 +195,16 @@ export default class GithubAnnotationReporter implements Reporter {

fs.writeFileSync(filePath, JSON.stringify(this.failedTests, null, 2));
}

private writeFlakySidecar() {
const appName = process.env.APP_NAME || 'unknown';
const isVercel = !!process.env.WORKFLOW_VERCEL_ENV;
const backend = isVercel ? 'vercel' : 'local';
const filePath = path.resolve(
process.cwd(),
`e2e-flaky-${appName}-${backend}.json`
);

fs.writeFileSync(filePath, JSON.stringify(this.flakyTests, null, 2));
}
}
9 changes: 9 additions & 0 deletions vitest.config.ts
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3,6 +3,15 @@ import { defineConfig } from 'vitest/config';
export default defineConfig({
test: {
testTimeout: 60_000,
// The e2e suites drive real deployments, so individual tests can lose
// timing races (queue delays, cold starts, watcher latency) that a
// second attempt absorbs. One CI retry keeps a single racy test from
// failing a 20+ minute matrix job; retried tests stay visible — the
// github-reporter annotates them and the PR comment lists them — so
// real races still get looked at. Harnesses where a failure is itself
// the signal (event-log-race-repro) pin `retry: 0` locally.
// Local runs keep retry at 0 so races reproduce while debugging.
retry: process.env.CI ? 1 : 0,
},
benchmark: {
include: ['**/*.bench.ts'],
Expand Down
Loading