Add seeded test order randomization

docs/05-command-line.md

@@ -38,6 +38,9 @@ Options:
       --node-arguments     Additional Node.js arguments for launching worker
                            processes (specify as a single string)       [string]
   -s, --serial             Run tests serially                          [boolean]
+      --seed               Randomize test order using a reproducible seed
+                                                                        [number]
+      --shuffle            Randomize test order using a random seed     [boolean]
   -t, --tap                Generate TAP output                         [boolean]
   -T, --timeout            Set global timeout (milliseconds or human-readable,
                            e.g. 10s, 2m)                                [string]
@@ -146,6 +149,22 @@ test.only('boo will run but not exclusively', t => {
 });
 ```
 
+## Randomizing test order
+
+Use `--shuffle` to randomize the order in which test files and concurrent tests within each file are started. AVA reports the seed at the start of the run.
+
+```console
+npx ava --shuffle
+```
+
+Use `--seed` to reproduce a particular randomized order:
+
+```console
+npx ava --seed=12345
+```
+
+Serial tests still run in declaration order before concurrent tests.
+
 ## Running tests at specific line numbers
 
 [![Open in StackBlitz](https://developer.stackblitz.com/img/open_in_stackblitz.svg)](https://stackblitz.com/github/avajs/ava/tree/main/examples/specific-line-numbers?file=test.js&terminal=test&view=editor)

docs/06-configuration.md

@@ -52,6 +52,8 @@ Arguments passed to the CLI will always take precedence over the CLI options con
 - `failWithoutAssertions`: if `false`, does not fail a test if it doesn't run [assertions](./03-assertions.md)
 - `environmentVariables`: specifies environment variables to be made available to the tests. The environment variables defined here override the ones from `process.env`
 - `serial`: if `true`, prevents parallel execution of tests within a file
+- `seed`: randomizes test order using a reproducible non-negative integer seed
+- `shuffle`: if `true`, randomizes test order using a random seed
 - `tap`: if `true`, enables the [TAP reporter](./05-command-line.md#tap-reporter)
 - `verbose`: if `true`, enables verbose output (though there currently non-verbose output is not supported)
 - `snapshotDir`: specifies a fixed location for storing snapshot files. Use this if your snapshots are ending up in the wrong location

lib/api.js

@@ -17,6 +17,7 @@ import isCi from './is-ci.js';
 import {getApplicableLineNumbers} from './line-numbers.js';
 import {setCappedTimeout} from './now-and-timers.js';
 import {observeWorkerProcess} from './plugin-support/shared-workers.js';
+import {deriveSeed, shuffle} from './randomization.js';
 import RunStatus from './run-status.js';
 import scheduler from './scheduler.js';
 import serializeError from './serialize-error.js';
@@ -77,7 +78,7 @@ export default class Api extends Emittery {
 	constructor(options) {
 		super();
 
-		this.options = {match: [], ...options};
+		this.options = {match: [], randomSeed: null, ...options};
 		this.options.require = normalizeRequireOption(this.options.require);
 
 		this._cacheDir = null;
@@ -181,6 +182,10 @@ export default class Api extends Emittery {
 				// The files must be in the same order across all runs, so sort them.
 				const defaultComparator = (a, b) => a.localeCompare(b, [], {numeric: true});
 				selectedFiles = selectedFiles.toSorted(this.options.sortTestFiles ?? defaultComparator);
+				if (this.options.randomSeed !== null) {
+					selectedFiles = shuffle(selectedFiles, deriveSeed(this.options.randomSeed, 'files'));
+				}
+
 				selectedFiles = chunkd(selectedFiles, currentIndex, totalRuns);
 
 				const currentFileCount = selectedFiles.length;
@@ -192,6 +197,10 @@ export default class Api extends Emittery {
 					selectedFiles = selectedFiles.toSorted(this.options.sortTestFiles);
 				}
 
+				if (this.options.randomSeed !== null) {
+					selectedFiles = shuffle(selectedFiles, deriveSeed(this.options.randomSeed, 'files'));
+				}
+
 				runStatus = new RunStatus(selectedFiles.length, null, selectionInsights);
 			}
 
@@ -208,6 +217,7 @@ export default class Api extends Emittery {
 				matching: apiOptions.match.length > 0 || runtimeOptions.interactiveMatchPattern !== undefined,
 				previousFailures: runtimeOptions.countPreviousFailures?.() ?? 0,
 				firstRun: runtimeOptions.firstRun ?? true,
+				randomSeed: this.options.randomSeed,
 				status: runStatus,
 			});
 

lib/cli.js

@@ -19,6 +19,7 @@ import {splitPatternAndLineNumbers} from './line-numbers.js';
 import {loadConfig} from './load-config.js';
 import normalizeNodeArguments from './node-arguments.js';
 import pkg from './pkg.js';
+import {generateSeed, validateSeed} from './randomization.js';
 
 function exit(message) {
 	console.error(`\n  ${chalk.red(figures.cross)} ${message}`);
@@ -60,6 +61,16 @@ const FLAGS = {
 		description: 'Run tests serially',
 		type: 'boolean',
 	},
+	seed: {
+		coerce: coerceLastValue,
+		description: 'Randomize test order using a reproducible seed',
+		type: 'number',
+	},
+	shuffle: {
+		coerce: coerceLastValue,
+		description: 'Randomize test order using a random seed',
+		type: 'boolean',
+	},
 	tap: {
 		alias: 't',
 		coerce: coerceLastValue,
@@ -317,6 +328,17 @@ export default async function loadCli() { // eslint-disable-line complexity
 		exit('The --concurrency or -c flag must be provided with a non-negative integer.');
 	}
 
+	let randomSeed = null;
+	if (Object.hasOwn(combined, 'seed')) {
+		try {
+			randomSeed = validateSeed(combined.seed);
+		} catch (error) {
+			exit(error.message);
+		}
+	} else if (combined.shuffle === true) {
+		randomSeed = generateSeed();
+	}
+
 	if (Object.hasOwn(conf, 'sortTestFiles') && typeof conf.sortTestFiles !== 'function') {
 		exit('’sortTestFiles’ must be a comparator function.');
 	}
@@ -419,6 +441,7 @@ export default async function loadCli() { // eslint-disable-line complexity
 		match,
 		nodeArguments,
 		parallelRuns,
+		randomSeed,
 		sortTestFiles: conf.sortTestFiles,
 		projectDir,
 		providers,

lib/randomization.js

@@ -0,0 +1,45 @@
+const maximumSeed = 2_147_483_647;
+
+export const generateSeed = () => Math.floor(Math.random() * maximumSeed);
+
+export function validateSeed(value) {
+	const seed = Number(value);
+	if (!Number.isSafeInteger(seed) || seed < 0) {
+		throw new TypeError('The --seed flag must be provided with a non-negative integer.');
+	}
+
+	return seed;
+}
+
+export function deriveSeed(seed, salt) {
+	let hash = seed % maximumSeed;
+	for (const character of salt) {
+		hash = ((hash * 31) + character.codePointAt(0)) % maximumSeed;
+	}
+
+	return hash === 0 ? 1 : hash;
+}
+
+function createRandom(seed) {
+	let state = seed % maximumSeed;
+	if (state <= 0) {
+		state += maximumSeed - 1;
+	}
+
+	return () => {
+		state = (state * 16_807) % maximumSeed;
+		return (state - 1) / (maximumSeed - 1);
+	};
+}
+
+export function shuffle(items, seed) {
+	const random = createRandom(seed);
+	const shuffled = [...items];
+
+	for (let index = shuffled.length - 1; index > 0; index--) {
+		const swapIndex = Math.floor(random() * (index + 1));
+		[shuffled[index], shuffled[swapIndex]] = [shuffled[swapIndex], shuffled[index]];
+	}
+
+	return shuffled;
+}

lib/reporters/default.js

@@ -156,6 +156,10 @@ export default class Reporter {
 			this.lineWriter.write(chalk.gray.dim('\u2500'.repeat(this.lineWriter.columns)) + os.EOL);
 		}
 
+		if (plan.randomSeed !== null) {
+			this.lineWriter.writeLine(chalk.gray(`Random seed: ${plan.randomSeed}`));
+		}
+
 		this.lineWriter.writeLine();
 	}
 

lib/reporters/tap.js

@@ -72,6 +72,9 @@ export default class TapReporter {
 		plan.status.on('stateChange', ({data: evt}) => this.consumeStateChange(evt));
 
 		this.reportStream.write(supertap.start() + os.EOL);
+		if (plan.randomSeed !== null) {
+			this.reportStream.write(`# Random seed: ${plan.randomSeed}` + os.EOL);
+		}
 	}
 
 	endRun() {

lib/runner.js

@@ -7,6 +7,7 @@ import * as matcher from 'matcher';
 import ContextRef from './context-ref.js';
 import createChain from './create-chain.js';
 import parseTestArgs from './parse-test-args.js';
+import {deriveSeed, shuffle} from './randomization.js';
 import serializeError from './serialize-error.js';
 import {load as loadSnapshots, determineSnapshotDir} from './snapshot-manager.js';
 import Runnable from './test.js';
@@ -33,6 +34,7 @@ export default class Runner extends Emittery {
 		this.checkSelectedByLineNumbers = options.checkSelectedByLineNumbers;
 		this.matchPatterns = options.match ?? [];
 		this.projectDir = options.projectDir;
+		this.randomSeed = options.randomSeed ?? null;
 		this.recordNewSnapshots = options.recordNewSnapshots === true;
 		this.serial = options.serial === true;
 		this.snapshotDir = options.snapshotDir;
@@ -403,8 +405,16 @@ export default class Runner extends Emittery {
 		return alwaysOk && hooksOk && testOk;
 	}
 
+	randomizeConcurrentTests(tests) {
+		if (this.randomSeed === null || tests.length < 2) {
+			return tests;
+		}
+
+		return shuffle(tests, deriveSeed(this.randomSeed, this.file));
+	}
+
 	async start() {
-		const concurrentTests = [];
+		let concurrentTests = [];
 		const serialTests = [];
 		for (const task of this.tasks.serial) {
 			if (!task.metadata.selected || (this.runOnlyExclusive && !task.metadata.exclusive)) {
@@ -464,6 +474,8 @@ export default class Runner extends Emittery {
 			});
 		}
 
+		concurrentTests = this.randomizeConcurrentTests(concurrentTests);
+
 		await Promise.all(this.waitForReady);
 
 		if (concurrentTests.length === 0 && serialTests.length === 0) {

lib/worker/base.js

@@ -79,6 +79,7 @@ const run = async options => {
 		file: options.file,
 		match: options.match,
 		projectDir: options.projectDir,
+		randomSeed: options.randomSeed,
 		recordNewSnapshots: options.recordNewSnapshots,
 		serial: options.serial,
 		snapshotDir: options.snapshotDir,

test/randomization/fixtures/order/package.json

@@ -0,0 +1,8 @@
+{
+	"type": "module",
+	"ava": {
+		"files": [
+			"*.js"
+		]
+	}
+}

test/randomization/fixtures/order/test.js

@@ -0,0 +1,37 @@
+import fs from 'node:fs';
+
+import test from 'ava';
+
+function record(title) {
+	fs.appendFileSync(process.env.ORDER_FILE, `${title}\n`);
+}
+
+test.serial('serial one', t => {
+	record('serial one');
+	t.pass();
+});
+
+test.serial('serial two', t => {
+	record('serial two');
+	t.pass();
+});
+
+test('alpha', t => {
+	record('alpha');
+	t.pass();
+});
+
+test('bravo', t => {
+	record('bravo');
+	t.pass();
+});
+
+test('charlie', t => {
+	record('charlie');
+	t.pass();
+});
+
+test('delta', t => {
+	record('delta');
+	t.pass();
+});

test/randomization/test.js

@@ -0,0 +1,37 @@
+import fs from 'node:fs/promises';
+
+import test from '@ava/test';
+import {temporaryFile} from 'tempy';
+
+import {cleanOutput, cwd, fixture} from '../helpers/exec.js';
+
+async function runWithSeed(seed) {
+	const orderFile = temporaryFile();
+	const result = await fixture([`--seed=${seed}`, '--concurrency=1'], {
+		cwd: cwd('order'),
+		env: {ORDER_FILE: orderFile},
+	});
+	const contents = await fs.readFile(orderFile, 'utf8');
+
+	return {
+		order: contents.trim().split('\n'),
+		stdout: result.stdout,
+	};
+}
+
+test('seeded runs use a reproducible randomized order', async t => {
+	const first = await runWithSeed(1234);
+	const second = await runWithSeed(1234);
+	const third = await runWithSeed(9876);
+
+	t.deepEqual(first.order, second.order);
+	t.notDeepEqual(first.order, third.order);
+	t.deepEqual(first.order.slice(0, 2), ['serial one', 'serial two']);
+	t.true(first.stdout.includes('Random seed: 1234'));
+});
+
+test('bails when --seed is provided with invalid input', async t => {
+	const result = await t.throwsAsync(fixture(['--seed=-1'], {cwd: cwd('order')}));
+
+	t.is(cleanOutput(result.stderr), 'The --seed flag must be provided with a non-negative integer.');
+});