test_runner: support test order randomization

PR-URL: #61747
Reviewed-By: Jacob Smith <jacob@frende.me>
Reviewed-By: Marco Ippolito <marcoippolito54@gmail.com>
Reviewed-By: Ethan Arrowood <ethan@arrowood.dev>
Reviewed-By: Benjamin Gruenbaum <benjamingr@gmail.com>
Reviewed-By: Aviv Keller <me@aviv.sh>
Reviewed-By: Chemi Atlow <chemi@atlow.co.il>
Reviewed-By: Moshe Atlow <moshe@atlow.co.il>

Author: pmarchini

doc/api/cli.md

@@ -2669,6 +2669,38 @@ changes:
 Configures the test runner to only execute top level tests that have the `only`
 option set. This flag is not necessary when test isolation is disabled.
 
+### `--test-random-seed`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Set the seed used to randomize test execution order. This applies to both test
+file execution order and queued tests within each file. Providing this flag
+enables randomization implicitly, even without `--test-randomize`.
+
+The value must be an integer between `0` and `4294967295`.
+
+This flag cannot be used with `--watch` or `--test-rerun-failures`.
+
+### `--test-randomize`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Randomize test execution order. This applies to both test file execution order
+and queued tests within each file. This can help detect tests that rely on
+shared state or execution order.
+
+The seed used for randomization is printed in the test summary and can be
+reused with `--test-random-seed`.
+
+For detailed behavior and examples, see
+[randomizing tests execution order][].
+
+This flag cannot be used with `--watch` or `--test-rerun-failures`.
+
 ### `--test-reporter`
 
 <!-- YAML
@@ -3582,6 +3614,8 @@ one is included in the list below.
 *`--test-isolation`
 *`--test-name-pattern`
 *`--test-only`
+*`--test-random-seed`
+*`--test-randomize`
 *`--test-reporter-destination`
 *`--test-reporter`
 *`--test-rerun-failures`
@@ -4162,6 +4196,7 @@ node --stack-trace-limit=12 -p -e "Error.stackTraceLimit" # prints 12
 [libuv threadpool documentation]: https://docs.libuv.org/en/latest/threadpool.html
 [module compile cache]: module.md#module-compile-cache
 [preloading asynchronous module customization hooks]: module.md#registration-of-asynchronous-customization-hooks
+[randomizing tests execution order]: test.md#randomizing-tests-execution-order
 [remote code execution]: https://www.owasp.org/index.php/Code_Injection
 [running tests from the command line]: test.md#running-tests-from-the-command-line
 [scavenge garbage collector]: https://v8.dev/blog/orinoco-parallel-scavenger

doc/api/test.md

@@ -626,6 +626,94 @@ prevent shell expansion, which can reduce portability across systems.
 node --test "**/*.test.js""**/*.spec.js"
 ```
 
+### Randomizing tests execution order
+
+<!-- YAML
+added: REPLACEME
+-->
+
+> Stability: 1.0 - Early development
+
+The test runner can randomize execution order to help detect
+order-dependent tests. When enabled, the runner randomizes both discovered
+test files and queued tests within each file. Use `--test-randomize` to
+enable this mode.
+
+```bash
+node --test --test-randomize
+```
+
+When randomization is enabled, the test runner prints the seed used for the run
+as a diagnostic message:
+
+```text
+Randomized test order seed: 12345
+```
+
+Use `--test-random-seed=<number>` to replay the same randomized order
+deterministically. Supplying `--test-random-seed` also enables randomization,
+so `--test-randomize` is optional when a seed is provided:
+
+```bash
+node --test --test-random-seed=12345
+```
+
+In most test files, randomization works automatically. One important exception
+is when subtests are awaited one by one. In that pattern, each subtest starts
+only after the previous one finishes, so the runner keeps declaration order
+instead of randomizing it.
+
+Example: this runs sequentially and is **not** randomized.
+
+```mjs
+importtestfrom'node:test';
+
+test('math', async (t) => {
+for (constnameof ['adds', 'subtracts', 'multiplies']) {
+// Sequentially awaiting each subtest preserves declaration order.
+awaitt.test(name, async () => {});
+ }
+});
+```
+
+```cjs
+consttest=require('node:test');
+
+test('math', async (t) => {
+for (constnameof ['adds', 'subtracts', 'multiplies']) {
+// Sequentially awaiting each subtest preserves declaration order.
+awaitt.test(name, async () => {});
+ }
+});
+```
+
+Using suite-style APIs such as `describe()`/`it()` or `suite()`/`test()`
+still allows randomization, because sibling tests are enqueued together.
+
+Example: this remains eligible for randomization.
+
+```mjs
+import { describe, it } from'node:test';
+
+describe('math', () => {
+it('adds', () => {});
+it('subtracts', () => {});
+it('multiplies', () => {});
+});
+```
+
+```cjs
+const { describe, it } =require('node:test');
+
+describe('math', () => {
+it('adds', () => {});
+it('subtracts', () => {});
+it('multiplies', () => {});
+});
+```
+
+`--test-randomize` and `--test-random-seed` are not supported with `--watch` mode.
+
 Matching files are executed as test files.
 More information on the test file execution can be found
 in the [test runner execution model][] section.
@@ -666,6 +754,10 @@ test runner functionality:
 *`--test-reporter` - Reporting is managed by the parent process
 *`--test-reporter-destination` - Output destinations are controlled by the parent
 *`--experimental-config-file` - Config file paths are managed by the parent
+*`--test-randomize` - Randomization is managed by the parent process and
+ propagated to child processes
+*`--test-random-seed` - Randomization seed is managed by the parent process and
+ propagated to child processes
 
 All other Node.js options from command line arguments, environment variables,
 and configuration files are inherited by the child processes.
@@ -1572,6 +1664,14 @@ changes:
  that specifies the index of the shard to run. This option is _required_.
 *`total` {number} is a positive integer that specifies the total number
  of shards to split the test files to. This option is _required_.
+*`randomize` {boolean} Randomize execution order for test files and queued tests.
+ This option is not supported with `watch: true`.
+**Default:**`false`.
+*`randomSeed` {number} Seed used when randomizing execution order. If this
+ option is set, runs can replay the same randomized order deterministically,
+ and setting this option also enables randomization. The value must be an
+ integer between `0` and `4294967295`.
+**Default:**`undefined`.
 *`rerunFailuresFilePath` {string} A file path where the test runner will
  store the state of the tests to allow rerunning only the failed tests on a next run.
  see \[Rerunning failed tests]\[] for more information.

doc/node-config-schema.json

@@ -524,6 +524,14 @@
 "type": "boolean",
 "description": "run tests with 'only' option set"
  },
+"test-random-seed": {
+"type": "number",
+"description": "seed used to randomize test execution order"
+ },
+"test-randomize": {
+"type": "boolean",
+"description": "run tests in a random order"
+ },
 "test-reporter": {
 "oneOf": [
  {
@@ -902,6 +910,14 @@
 "type": "boolean",
 "description": "run tests with 'only' option set"
  },
+"test-random-seed": {
+"type": "number",
+"description": "seed used to randomize test execution order"
+ },
+"test-randomize": {
+"type": "boolean",
+"description": "run tests in a random order"
+ },
 "test-reporter": {
 "oneOf": [
  {

doc/node.1

@@ -498,6 +498,22 @@ Configures the type of test isolation used in the test runner.
 A regular expression that configures the test runner to only execute tests
 whose name matches the provided pattern.
 .
+.ItFl-test-random-seed
+Set the seed used to randomize test execution order.
+This applies to both test file execution order and queued tests within each file.
+Providing this flag enables randomization implicitly, even without
+\fB--test-randomize\fR.
+The value must be an integer between 0 and 4294967295.
+This flag cannot be used with \fB--watch\fR or \fB--test-rerun-failures\fR.
+.
+.ItFl-test-randomize
+Randomize test execution order.
+This applies to both test file execution order and queued tests within each file.
+This can help detect tests that rely on shared state or execution order.
+The seed used for randomization is printed in the test summary and can be
+reused with \fB--test-random-seed\fR.
+This flag cannot be used with \fB--watch\fR or \fB--test-rerun-failures\fR.
+.
 .ItFl-test-reporter
 A test reporter to use when running tests.
 .

lib/internal/test_runner/runner.js

@@ -59,6 +59,7 @@ const {
  validateObject,
  validateOneOf,
  validateInteger,
+ validateUint32,
  validateString,
  validateStringArray,
 }=require('internal/validators');
@@ -84,6 +85,7 @@ const {
 const{ FastBuffer }=require('internal/buffer');
 
 const{
+ createRandomSeed,
  convertStringToRegExp,
  countCompletedTest,
  kDefaultPattern,
@@ -105,12 +107,14 @@ const kIsolatedProcessName = Symbol('kIsolatedProcessName');
 constkFilterArgs=[
 '--test',
 '--experimental-test-coverage',
+'--test-randomize',
 '--watch',
 '--experimental-default-config-file',
 ];
 constkFilterArgValues=[
 '--test-reporter',
 '--test-reporter-destination',
+'--test-random-seed',
 '--experimental-config-file',
 ];
 constkDiagnosticsFilterArgs=['tests','suites','pass','fail','cancelled','skipped','todo','duration_ms'];
@@ -168,6 +172,8 @@ function getRunArgs(path, { forceExit,
 argv: suppliedArgs,
  execArgv,
  rerunFailuresFilePath,
+ randomize,
+ randomSeed,
 root: { timeout },
  cwd }){
 constprocessNodeOptions=getOptionsAsFlagsFromBinding();
@@ -209,6 +215,12 @@ function getRunArgs(path, { forceExit,
 if(rerunFailuresFilePath){
 ArrayPrototypePush(runArgs,`--test-rerun-failures=${rerunFailuresFilePath}`);
 }
+if(randomize){
+ArrayPrototypePush(runArgs,'--test-randomize');
+}
+if(randomSeed!=null){
+ArrayPrototypePush(runArgs,`--test-random-seed=${randomSeed}`);
+}
 
 ArrayPrototypePushApply(runArgs,execArgv);
 
@@ -646,6 +658,8 @@ function run(options = kEmptyObject) {
  lineCoverage =0,
  branchCoverage =0,
  functionCoverage =0,
+randomize: suppliedRandomize,
+randomSeed: suppliedRandomSeed,
  execArgv =[],
  argv =[],
  cwd =process.cwd(),
@@ -674,6 +688,56 @@ function run(options = kEmptyObject) {
 if(globPatterns!=null){
 validateArray(globPatterns,'options.globPatterns');
 }
+if(suppliedRandomize!=null){
+validateBoolean(suppliedRandomize,'options.randomize');
+}
+if(suppliedRandomSeed!=null){
+validateUint32(suppliedRandomSeed,'options.randomSeed');
+}
+letrandomize=suppliedRandomize;
+letrandomSeed=suppliedRandomSeed;
+
+if(randomSeed!=null){
+randomize=true;
+}
+if(watch){
+if(randomSeed!=null){
+thrownewERR_INVALID_ARG_VALUE(
+'options.randomSeed',
+randomSeed,
+'is not supported with watch mode',
+);
+}
+if(randomize){
+thrownewERR_INVALID_ARG_VALUE(
+'options.randomize',
+randomize,
+'is not supported with watch mode',
+);
+}
+}
+if(rerunFailuresFilePath){
+validatePath(rerunFailuresFilePath,'options.rerunFailuresFilePath');
+// TODO(pmarchini): Support rerun-failures with randomization by
+// persisting the randomization seed in the rerun state file.
+if(randomSeed!=null){
+thrownewERR_INVALID_ARG_VALUE(
+'options.randomSeed',
+randomSeed,
+'is not supported with rerun failures mode',
+);
+}
+if(randomize){
+thrownewERR_INVALID_ARG_VALUE(
+'options.randomize',
+randomize,
+'is not supported with rerun failures mode',
+);
+}
+}
+if(randomize){
+randomSeed??=createRandomSeed();
+}
 
 validateString(cwd,'options.cwd');
 
@@ -683,10 +747,6 @@ function run(options = kEmptyObject) {
 );
 }
 
-if(rerunFailuresFilePath){
-validatePath(rerunFailuresFilePath,'options.rerunFailuresFilePath');
-}
-
 if(shard!=null){
 validateObject(shard,'options.shard');
 // Avoid re-evaluating the shard object in case it's a getter
@@ -783,11 +843,18 @@ function run(options = kEmptyObject) {
 functionCoverage: functionCoverage,
  cwd,
  globalSetupPath,
+ randomize,
+ randomSeed,
 };
+
 constroot=createTestTree(rootTestOptions,globalOptions);
 lettestFiles=files??createTestFileList(globPatterns,cwd);
 const{ isTestRunner }=globalOptions;
 
+if(randomize){
+root.diagnostic(`Randomized test order seed: ${randomSeed}`);
+}
+
 if(shard){
 testFiles=ArrayPrototypeFilter(testFiles,(_,index)=>index%shard.total===shard.index-1);
 }
@@ -833,6 +900,8 @@ function run(options = kEmptyObject) {
  rerunFailuresFilePath,
  env,
 workerIdPool: isolation==='process' ? workerIdPool : null,
+ randomize,
+ randomSeed,
 };
 
 if(isolation==='process'){