test-runner: add option to rerun only failed tests

doc/api/cli.md

@@ -2647,6 +2647,20 @@ changes:
 The destination for the corresponding test reporter. See the documentation on
 [test reporters][] for more details.
 
+### `--test-rerun-failures`
+
+<!-- YAML
+added:
+ - REPLACEME
+-->
+
+A path to a file allowing the test runner to persist the state of the test
+suite between runs. The test runner will use this file to determine which tests
+have already succeeded or failed, allowing for re-running of failed tests
+without having to re-run the entire test suite. The test runner will create this
+file if it does not exist.
+See the documentation on [test reruns][] for more details.
+
 ### `--test-shard`
 
 <!-- YAML
@@ -3508,6 +3522,7 @@ one is included in the list below.
 * `--test-only`
 * `--test-reporter-destination`
 * `--test-reporter`
+* `--test-rerun-failures`
 * `--test-shard`
 * `--test-skip-pattern`
 * `--throw-deprecation`
@@ -4082,6 +4097,7 @@ node --stack-trace-limit=12 -p -e "Error.stackTraceLimit" # prints 12
 [snapshot testing]: test.md#snapshot-testing
 [syntax detection]: packages.md#syntax-detection
 [test reporters]: test.md#test-reporters
+[test reruns]: test.md#rerunning-failed-tests
 [test runner execution model]: test.md#test-runner-execution-model
 [timezone IDs]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
 [tracking issue for user-land snapshots]: https://github.com/nodejs/node/issues/44014

doc/api/test.md

@@ -153,6 +153,46 @@ test('skip() method with message', (t) => {
 });
 ```
 
+## Rerunning failed tests
+
+The test runner supports persisting the state of the run to a file, allowing
+the test runner to rerun failed tests without having to re-run the entire test suite.
+Use the [`--test-rerun-failures`][] command-line option to specify a file path where the
+state of the run is stored. if the state file does not exist, the test runner will
+create it.
+the state file is a JSON file that contains an array of run attempts.
+Each run attempt is an object mapping successful tests to the attempt they have passed in.
+The key identifying a test in this map is the test file path, with the line and column where the test is defined.
+in a case where a test defined in a specific location is run multiple times,
+for example within a function or a loop,
+a counter will be appended to the key, to disambiguate the test runs.
+note changing the order of test execution or the location of a test can lead the test runner
+to consider tests as passed on a previous attempt,
+meaning `--test-rerun-failures` should be used when tests run in a deterministic order.
+
+example of a state file:
+
+```json
+[
+ {
+ "test.js:10:5": { "passed_on_attempt": 0, "name": "test 1" },
+ },
+ {
+ "test.js:10:5": { "passed_on_attempt": 0, "name": "test 1" },
+ "test.js:20:5": { "passed_on_attempt": 1, "name": "test 2" }
+ }
+]
+```
+
+in this example, there are two run attempts, with two tests defined in `test.js`,
+the first test succeeded on the first attempt, and the second test succeeded on the second attempt.
+
+When the `--test-rerun-failures` option is used, the test runner will only run tests that have not yet passed.
+
+```bash
+node --test-rerun-failures /path/to/state/file
+```
+
 ## TODO tests
 
 Individual tests can be marked as flaky or incomplete by passing the `todo`
@@ -1342,6 +1382,9 @@ added:
  - v18.9.0
  - v16.19.0
 changes:
+ - version: REPLACEME
+ pr-url: https://github.com/nodejs/node/pull/59443
+ description: Added a rerunFailuresFilePath option.
  - version: v23.0.0
  pr-url: https://github.com/nodejs/node/pull/54705
  description: Added the `cwd` option.
@@ -1432,6 +1475,10 @@ changes:
  that specifies the index of the shard to run. This option is _required_.
  * `total` {number} is a positive integer that specifies the total number
  of shards to split the test files to. This option is _required_.
+ * `rerunFailuresFilePath` {string} A file path where the test runner will
+ store the state of the tests to allow rerunning only the failed tests on a next run.
+ see \[Rerunning failed tests]\[] for more information.
+ **Default:** `undefined`.
  * `coverage` {boolean} enable [code coverage][] collection.
  **Default:** `false`.
  * `coverageExcludeGlobs` {string|Array} Excludes specific files from code coverage
@@ -3220,6 +3267,8 @@ Emitted when a test is enqueued for execution.
  * `cause` {Error} The actual error thrown by the test.
  * `type` {string|undefined} The type of the test, used to denote whether
  this is a suite.
+ * `attempt` {number|undefined} The attempt number of the test run,
+ present only when using the [`--test-rerun-failures`][] flag.
  * `file` {string|undefined} The path of the test file,
  `undefined` if test was run through the REPL.
  * `line` {number|undefined} The line number where the test is defined, or
@@ -3244,6 +3293,10 @@ The corresponding execution ordered event is `'test:complete'`.
  * `duration_ms` {number} The duration of the test in milliseconds.
  * `type` {string|undefined} The type of the test, used to denote whether
  this is a suite.
+ * `attempt` {number|undefined} The attempt number of the test run,
+ present only when using the [`--test-rerun-failures`][] flag.
+ * `passed_on_attempt` {number|undefined} The attempt number the test passed on,
+ present only when using the [`--test-rerun-failures`][] flag.
  * `file` {string|undefined} The path of the test file,
  `undefined` if test was run through the REPL.
  * `line` {number|undefined} The line number where the test is defined, or
@@ -3947,6 +4000,7 @@ Can be used to abort test subtasks when the test has been aborted.
 [`--test-only`]: cli.md#--test-only
 [`--test-reporter-destination`]: cli.md#--test-reporter-destination
 [`--test-reporter`]: cli.md#--test-reporter
+[`--test-rerun-failures`]: cli.md#--test-rerun-failures
 [`--test-skip-pattern`]: cli.md#--test-skip-pattern
 [`--test-update-snapshots`]: cli.md#--test-update-snapshots
 [`--test`]: cli.md#--test

doc/node-config-schema.json

@@ -446,6 +446,9 @@
  }
  ]
  },
+ "test-rerun-failures": {
+ "type": "string"
+ },
  "test-shard": {
  "type": "string"
  },
@@ -698,6 +701,9 @@
  }
  ]
  },
+ "test-rerun-failures": {
+ "type": "string"
+ },
  "test-shard": {
  "type": "string"
  },

doc/node.1

@@ -493,6 +493,10 @@ A test reporter to use when running tests.
 .It Fl -test-reporter-destination
 The destination for the corresponding test reporter.
 .
+.It Fl -test-rerun-failures
+Configures the tests runner to persist the state of tests to allow
+rerunning only failed tests.
+.
 .It Fl -test-only
 Configures the test runner to only execute top level tests that have the `only`
 option set.

lib/internal/test_runner/harness.js

@@ -26,7 +26,10 @@ const {
  reporterScope,
  shouldColorizeTestFiles,
  setupGlobalSetupTeardownFunctions,
+ parsePreviousRuns,
 } = require('internal/test_runner/utils');
+const { PassThrough, compose } = require('stream');
+const { reportReruns } = require('internal/test_runner/reporter/rerun');
 const { queueMicrotask } = require('internal/process/task_queues');
 const { TIMEOUT_MAX } = require('internal/timers');
 const { clearInterval, setInterval } = require('timers');
@@ -69,6 +72,7 @@ function createTestTree(rootTestOptions, globalOptions) {
  shouldColorizeTestFiles: shouldColorizeTestFiles(globalOptions.destinations),
  teardown: null,
  snapshotManager: null,
+ previousRuns: null,
  isFilteringByName,
  isFilteringByOnly,
  async runBootstrap() {
@@ -203,6 +207,25 @@ function collectCoverage(rootTest, coverage) {
  return summary;
 }
 
+function setupFailureStateFile(rootTest, globalOptions) {
+ if (!globalOptions.rerunFailuresFilePath) {
+ return;
+ }
+ rootTest.harness.previousRuns = parsePreviousRuns(globalOptions.rerunFailuresFilePath);
+ if (rootTest.harness.previousRuns === null) {
+ rootTest.diagnostic(`Warning: The rerun failures file at ` +
+ `${globalOptions.rerunFailuresFilePath} is not a valid rerun file. ` +
+ 'The test runner will not be able to rerun failed tests.');
+ rootTest.harness.success = false;
+ process.exitCode = kGenericUserError;
+ return;
+ }
+ if (!process.env.NODE_TEST_CONTEXT) {
+ const reporter = reportReruns(rootTest.harness.previousRuns, globalOptions);
+ compose(rootTest.reporter, reporter).pipe(new PassThrough());
+ }
+}
+
 function setupProcessState(root, globalOptions) {
  const hook = createHook({
  __proto__: null,
@@ -230,6 +253,9 @@ function setupProcessState(root, globalOptions) {
  const rejectionHandler =
  createProcessEventHandler('unhandledRejection', root);
  const coverage = configureCoverage(root, globalOptions);
+
+ setupFailureStateFile(root, globalOptions);
+
  const exitHandler = async (kill) => {
  if (root.subtests.length === 0 && (root.hooks.before.length > 0 || root.hooks.after.length > 0)) {
  // Run global before/after hooks in case there are no tests

lib/internal/test_runner/reporter/rerun.js

@@ -0,0 +1,40 @@
+'use strict';
+
+const {
+ ArrayPrototypePush,
+ JSONStringify,
+} = primordials;
+const { relative } = require('path');
+const { writeFileSync } = require('fs');
+
+function reportReruns(previousRuns, globalOptions) {
+ return async function reporter(source) {
+ const obj = { __proto__: null };
+ const disambiguator = { __proto__: null };
+
+ for await (const { type, data } of source) {
+ if (type === 'test:pass') {
+ let identifier = `${relative(globalOptions.cwd, data.file)}:${data.line}:${data.column}`;
+ if (disambiguator[identifier] !== undefined) {
+ identifier += `:(${disambiguator[identifier]})`;
+ disambiguator[identifier] += 1;
+ } else {
+ disambiguator[identifier] = 1;
+ }
+ obj[identifier] = {
+ __proto__: null,
+ name: data.name,
+ passed_on_attempt: data.details.passed_on_attempt ?? data.details.attempt,
+ };
+ }
+ }
+
+ ArrayPrototypePush(previousRuns, obj);
+ writeFileSync(globalOptions.rerunFailuresFilePath, JSONStringify(previousRuns, null, 2), 'utf8');
+ };
+};
+
+module.exports = {
+ __proto__: null,
+ reportReruns,
+};

lib/internal/test_runner/runner.js

@@ -148,6 +148,7 @@ function getRunArgs(path, { forceExit,
  only,
  argv: suppliedArgs,
  execArgv,
+ rerunFailuresFilePath,
  root: { timeout },
  cwd }) {
  const processNodeOptions = getOptionsAsFlagsFromBinding();
@@ -170,6 +171,9 @@ function getRunArgs(path, { forceExit,
  if (timeout != null) {
  ArrayPrototypePush(runArgs, `--test-timeout=${timeout}`);
  }
+ if (rerunFailuresFilePath) {
+ ArrayPrototypePush(runArgs, `--test-rerun-failures=${rerunFailuresFilePath}`);
+ }
 
  ArrayPrototypePushApply(runArgs, execArgv);
 
@@ -588,6 +592,7 @@ function run(options = kEmptyObject) {
  execArgv = [],
  argv = [],
  cwd = process.cwd(),
+ rerunFailuresFilePath,
  } = options;
 
  if (files != null) {
@@ -620,6 +625,10 @@ function run(options = kEmptyObject) {
  );
  }
 
+ if (rerunFailuresFilePath) {
+ validatePath(rerunFailuresFilePath, 'options.rerunFailuresFilePath');
+ }
+
  if (shard != null) {
  validateObject(shard, 'options.shard');
  // Avoid re-evaluating the shard object in case it's a getter
@@ -702,6 +711,7 @@ function run(options = kEmptyObject) {
  coverage,
  coverageExcludeGlobs,
  coverageIncludeGlobs,
+ rerunFailuresFilePath,
  lineCoverage: lineCoverage,
  branchCoverage: branchCoverage,
  functionCoverage: functionCoverage,
@@ -735,6 +745,7 @@ function run(options = kEmptyObject) {
  isolation,
  argv,
  execArgv,
+ rerunFailuresFilePath,
  };
 
  if (isolation === 'process') {

lib/internal/test_runner/test.js

@@ -71,6 +71,7 @@ const {
 } = require('timers');
 const { TIMEOUT_MAX } = require('internal/timers');
 const { fileURLToPath } = require('internal/url');
+const { relative } = require('path');
 const { availableParallelism } = require('os');
 const { innerOk } = require('internal/assert/utils');
 const { bigint: hrtime } = process.hrtime;
@@ -290,6 +291,10 @@ class TestContext {
  return this.#test.passed;
  }
 
+ get attempt() {
+ return this.#test.attempt ?? 0;
+ }
+
  diagnostic(message) {
  this.#test.diagnostic(message);
  }
@@ -537,6 +542,7 @@ class Test extends AsyncResource {
  this.childNumber = 0;
  this.timeout = kDefaultTimeout;
  this.entryFile = entryFile;
+ this.testDisambiguator = new SafeMap();
  } else {
  const nesting = parent.parent === null ? parent.nesting :
  parent.nesting + 1;
@@ -646,6 +652,8 @@ class Test extends AsyncResource {
  this.endTime = null;
  this.passed = false;
  this.error = null;
+ this.attempt = undefined;
+ this.passedAttempt = undefined;
  this.message = typeof skip === 'string' ? skip :
  typeof todo === 'string' ? todo : null;
  this.activeSubtests = 0;
@@ -690,6 +698,23 @@ class Test extends AsyncResource {
  this.loc.file = fileURLToPath(this.loc.file);
  }
  }
+
+ if (this.loc != null && this.root.harness.previousRuns != null) {
+ let testIdentifier = `${relative(this.config.cwd, this.loc.file)}:${this.loc.line}:${this.loc.column}`;
+ const disambiguator = this.root.testDisambiguator.get(testIdentifier);
+ if (disambiguator !== undefined) {
+ testIdentifier += `:(${disambiguator})`;
+ this.root.testDisambiguator.set(testIdentifier, disambiguator + 1);
+ } else {
+ this.root.testDisambiguator.set(testIdentifier, 1);
+ }
+ this.attempt = this.root.harness.previousRuns.length;
+ const previousAttempt = this.root.harness.previousRuns[this.attempt - 1]?.[testIdentifier]?.passed_on_attempt;
+ if (previousAttempt != null) {
+ this.passedAttempt = previousAttempt;
+ this.fn = noop;
+ }
+ }
  }
 
  applyFilters() {
@@ -1329,6 +1354,12 @@ class Test extends AsyncResource {
  if (!this.passed) {
  details.error = this.error;
  }
+ if (this.attempt !== undefined) {
+ details.attempt = this.attempt;
+ }
+ if (this.passedAttempt !== undefined) {
+ details.passed_on_attempt = this.passedAttempt;
+ }
  return { __proto__: null, details, directive };
  }