feat: add a seed value to test runs (#13400)

Author: jhwang98

CHANGELOG.md

@@ -2,8 +2,11 @@
 
 ### Features
 
+- `[@jest/cli, jest-config]` A seed for the test run will be randomly generated, or set by a CLI option ([#13400](https://github.com/facebook/jest/pull/13400))
+- `[@jest/cli, jest-config]` `--show-seed` will display the seed value in the report, and can be set via a CLI flag or through the config file ([#13400](https://github.com/facebook/jest/pull/13400))
 - `[jest-config]` Add `readInitialConfig` utility function ([#13356](https://github.com/facebook/jest/pull/13356))
 - `[jest-core]` Enable testResultsProcessor to be async ([#13343](https://github.com/facebook/jest/pull/13343))
+- `[@jest/environment, jest-environment-node, jest-environment-jsdom, jest-runtime]` Add `getSeed()` to the `jest` object ([#13400](https://github.com/facebook/jest/pull/13400))
 - `[expect, @jest/expect-utils]` Allow `isA` utility to take a type argument ([#13355](https://github.com/facebook/jest/pull/13355))
 - `[expect]` Expose `AsyncExpectationResult` and `SyncExpectationResult` types ([#13411](https://github.com/facebook/jest/pull/13411))
 

docs/CLI.md

@@ -350,6 +350,20 @@ The default regex matching works fine on small runs, but becomes slow if provide
 
 :::
 
+### `--seed=<num>`
+
+Sets a seed value that can be retrieved in a test file via [`jest.getSeed()`](JestObjectAPI.md#jestgetseed). The seed value must be between `-0x80000000` and `0x7fffffff` inclusive (`-2147483648` (`-(2 ** 31)`) and `2147483647` (`2 ** 31 - 1`) in decimal).
+
+```bash
+jest --seed=1324
+```
+
+:::tip
+
+If this option is not specified Jest will randomly generate the value. You can use the [`--showSeed`](#--showseed) flag to print the seed in the test report summary.
+
+:::
+
 ### `--selectProjects <project1> ... <projectN>`
 
 Run the tests of the specified projects. Jest uses the attribute `displayName` in the configuration to identify each project. If you use this option, you should provide a `displayName` to all your projects.
@@ -380,6 +394,12 @@ jest --shard=3/3
 
 Print your Jest config and then exits.
 
+### `--showSeed`
+
+Prints the seed value in the test report summary. See [`--seed=<num>`](#--seednum) for the details.
+
+Can also be set in configuration. See [`showSeed`](Configuration.md#showseed-boolean).
+
 ### `--silent`
 
 Prevent tests from printing messages through the console.

docs/Configuration.md

@@ -1614,6 +1614,12 @@ const config: Config = {
 export default config;
 ```
 
+### `showSeed` \[boolean]
+
+Default: `false`
+
+The equivalent of the [`--showSeed`](CLI.md#--showseed) flag to print the seed in the test report summary.
+
 ### `slowTestThreshold` \[number]
 
 Default: `5`

docs/JestObjectAPI.md

@@ -908,6 +908,16 @@ This function is not available when using legacy fake timers implementation.
 
 ## Misc
 
+### `jest.getSeed()`
+
+Every time Jest runs a seed value is randomly generated which you could use in a pseudorandom number generator or anywhere else.
+
+:::tip
+
+Use the [`--showSeed`](CLI.md#--showseed) flag to print the seed in the test report summary. To manually set the value of the seed use [`--seed=<num>`](CLI.md#--seednum) CLI argument.
+
+:::
+
 ### `jest.setTimeout(timeout)`
 
 Set the default timeout interval (in milliseconds) for all tests and before/after hooks in the test file. This only affects the test file from which this function is called. The default timeout interval is 5 seconds if this method is not called.

e2e/Utils.ts

@@ -157,6 +157,9 @@ export const copyDir = (src: string, dest: string) => {
   }
 };
 
+export const replaceSeed = (str: string) =>
+  str.replace(/Seed: {8}(-?\d+)/g, 'Seed:       <<REPLACED>>');
+
 export const replaceTime = (str: string) =>
   str
     .replace(/\d*\.?\d+ m?s\b/g, '<<REPLACED>>')
@@ -201,7 +204,7 @@ export const extractSummary = (stdout: string) => {
   const match = stdout
     .replace(/(?:\\[rn])+/g, '\n')
     .match(
-      /Test Suites:.*\nTests.*\nSnapshots.*\nTime.*(\nRan all test suites)*.*\n*$/gm,
+      /(Seed:.*\n)?Test Suites:.*\nTests.*\nSnapshots.*\nTime.*(\nRan all test suites)*.*\n*$/gm,
     );
   if (!match) {
     throw new Error(dedent`
@@ -254,7 +257,7 @@ export const extractSummaries = (
   stdout: string,
 ): Array<{rest: string; summary: string}> => {
   const regex =
-    /Test Suites:.*\nTests.*\nSnapshots.*\nTime.*(\nRan all test suites)*.*\n*$/gm;
+    /(Seed:.*\n)?Test Suites:.*\nTests.*\nSnapshots.*\nTime.*(\nRan all test suites)*.*\n*$/gm;
 
   let match = regex.exec(stdout);
   const matches: Array<RegExpExecArray> = [];

e2e/__tests__/__snapshots__/showConfig.test.ts.snap

@@ -125,6 +125,7 @@ exports[`--showConfig outputs config info and exits 1`] = `
     "projects": [],
     "rootDir": "<<REPLACED_ROOT_DIR>>",
     "runTestsByPath": false,
+    "seed": <<RANDOM_SEED>>,
     "skipFilter": false,
     "snapshotFormat": {
       "escapeString": false,

e2e/__tests__/jestObject.test.ts

@@ -0,0 +1,26 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import * as path from 'path';
+import runJest from '../runJest';
+
+const dir = path.resolve(__dirname, '../jest-object');
+
+test('passes with seed', () => {
+  const result = runJest(dir, ['get-seed.test.js', '--seed', '1234']);
+  expect(result.exitCode).toBe(0);
+});
+
+test('fails with wrong seed', () => {
+  const result = runJest(dir, ['get-seed.test.js', '--seed', '1111']);
+  expect(result.exitCode).toBe(1);
+});
+
+test('seed always exists', () => {
+  const result = runJest(dir, ['any-seed.test.js']);
+  expect(result.exitCode).toBe(0);
+});

e2e/__tests__/showConfig.test.ts

@@ -37,7 +37,8 @@ test('--showConfig outputs config info and exits', () => {
     .replace(/"version": "(.+)"/g, '"version": "[version]"')
     .replace(/"maxWorkers": (\d+)/g, '"maxWorkers": "[maxWorkers]"')
     .replace(/"\S*show-config-test/gm, '"<<REPLACED_ROOT_DIR>>')
-    .replace(/"\S*\/jest\/packages/gm, '"<<REPLACED_JEST_PACKAGES_DIR>>');
+    .replace(/"\S*\/jest\/packages/gm, '"<<REPLACED_JEST_PACKAGES_DIR>>')
+    .replace(/"seed": (-?\d+)/g, '"seed": <<RANDOM_SEED>>');
 
   expect(stdout).toMatchSnapshot();
 });

e2e/__tests__/showSeed.test.ts

@@ -0,0 +1,52 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import * as path from 'path';
+import {extractSummary, replaceSeed} from '../Utils';
+import runJest from '../runJest';
+
+const dir = path.resolve(__dirname, '../jest-object');
+
+const randomSeedValueRegExp = /Seed:\s+<<REPLACED>>/;
+const seedValueRegExp = /Seed:\s+1234/;
+
+test('--showSeed changes report to output seed', () => {
+  const {stderr} = runJest(dir, ['--showSeed', '--no-cache']);
+
+  const {summary} = extractSummary(stderr);
+
+  expect(replaceSeed(summary)).toMatch(randomSeedValueRegExp);
+});
+
+test('if --showSeed is not present the report will not show the seed', () => {
+  const {stderr} = runJest(dir, ['--seed', '1234']);
+
+  const {summary} = extractSummary(stderr);
+
+  expect(replaceSeed(summary)).not.toMatch(randomSeedValueRegExp);
+});
+
+test('if showSeed is present in the config the report will show the seed', () => {
+  const {stderr} = runJest(dir, [
+    '--seed',
+    '1234',
+    '--config',
+    'different-config.json',
+  ]);
+
+  const {summary} = extractSummary(stderr);
+
+  expect(summary).toMatch(seedValueRegExp);
+});
+
+test('--seed --showSeed will show the seed in the report', () => {
+  const {stderr} = runJest(dir, ['--showSeed', '--seed', '1234']);
+
+  const {summary} = extractSummary(stderr);
+
+  expect(summary).toMatch(seedValueRegExp);
+});

e2e/jest-object/__tests__/any-seed.test.js

@@ -0,0 +1,10 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+test('ensure seed exists', () => {
+  expect(jest.getSeed()).toEqual(expect.any(Number));
+});

e2e/jest-object/__tests__/get-seed.test.js

@@ -0,0 +1,10 @@
+/**
+ * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+test('getSeed', () => {
+  expect(jest.getSeed()).toBe(1234);
+});

e2e/jest-object/different-config.json

@@ -0,0 +1,4 @@
+{
+  "displayName": "Config from different-config.json file",
+  "showSeed": true
+}

e2e/jest-object/package.json

@@ -0,0 +1,5 @@
+{
+  "jest": {
+    "testEnvironment": "node"
+  }
+}

packages/jest-cli/src/args.ts

@@ -510,6 +510,11 @@ export const options: {[key: string]: Options} = {
       "Allows to use a custom runner instead of Jest's default test runner.",
     type: 'string',
   },
+  seed: {
+    description:
+      'Sets a seed value that can be retrieved in a tests file via `jest.getSeed()`. If this option is not specified Jest will randomly generate the value. The seed value must be between `-0x80000000` and `0x7fffffff` inclusive.',
+    type: 'number',
+  },
   selectProjects: {
     description:
       'Run the tests of the specified projects. ' +
@@ -541,6 +546,11 @@ export const options: {[key: string]: Options} = {
     description: 'Print your jest config and then exits.',
     type: 'boolean',
   },
+  showSeed: {
+    description:
+      'Prints the seed value in the test report summary. See `--seed` for how to set this value',
+    type: 'boolean',
+  },
   silent: {
     description: 'Prevent tests from printing messages through the console.',
     type: 'boolean',

packages/jest-config/src/ValidConfig.ts

@@ -137,6 +137,7 @@ const initialOptions: Config.InitialOptions = {
   sandboxInjectedGlobals: [],
   setupFiles: ['<rootDir>/setup.js'],
   setupFilesAfterEnv: ['<rootDir>/testSetupFile.js'],
+  showSeed: false,
   silent: true,
   skipFilter: false,
   skipNodeResolution: false,

packages/jest-config/src/__tests__/normalize.test.ts

@@ -114,14 +114,12 @@ it('keeps custom ids based on the rootDir', async () => {
 });
 
 it('minimal config is stable across runs', async () => {
-  const firstNormalization = await normalize(
-    {rootDir: '/root/path/foo'},
-    {} as Config.Argv,
-  );
-  const secondNormalization = await normalize(
-    {rootDir: '/root/path/foo'},
-    {} as Config.Argv,
-  );
+  const firstNormalization = await normalize({rootDir: '/root/path/foo'}, {
+    seed: 55555,
+  } as Config.Argv);
+  const secondNormalization = await normalize({rootDir: '/root/path/foo'}, {
+    seed: 55555,
+  } as Config.Argv);
 
   expect(firstNormalization).toEqual(secondNormalization);
   expect(JSON.stringify(firstNormalization)).toBe(
@@ -2113,3 +2111,56 @@ it('parses workerIdleMemoryLimit', async () => {
 
   expect(options.workerIdleMemoryLimit).toBe(47185920);
 });
+
+describe('seed', () => {
+  it('generates seed when not specified', async () => {
+    const {options} = await normalize({rootDir: '/root/'}, {} as Config.Argv);
+    expect(options.seed).toEqual(expect.any(Number));
+  });
+
+  it('uses seed specified', async () => {
+    const {options} = await normalize({rootDir: '/root/'}, {
+      seed: 4321,
+    } as Config.Argv);
+    expect(options.seed).toBe(4321);
+  });
+
+  it('throws if seed is too large or too small', async () => {
+    await expect(
+      normalize({rootDir: '/root/'}, {
+        seed: 2 ** 33,
+      } as Config.Argv),
+    ).rejects.toThrow(
+      'seed value must be between `-0x80000000` and `0x7fffffff` inclusive - is 8589934592',
+    );
+    await expect(
+      normalize({rootDir: '/root/'}, {
+        seed: -(2 ** 33),
+      } as Config.Argv),
+    ).rejects.toThrow(
+      'seed value must be between `-0x80000000` and `0x7fffffff` inclusive - is -8589934592',
+    );
+  });
+});
+
+describe('showSeed', () => {
+  test('showSeed is set when argv flag is set', async () => {
+    const {options} = await normalize({rootDir: '/root/'}, {
+      showSeed: true,
+    } as Config.Argv);
+    expect(options.showSeed).toBe(true);
+  });
+
+  test('showSeed is set when the config is set', async () => {
+    const {options} = await normalize(
+      {rootDir: '/root/', showSeed: true},
+      {} as Config.Argv,
+    );
+    expect(options.showSeed).toBe(true);
+  });
+
+  test('showSeed is false when neither is set', async () => {
+    const {options} = await normalize({rootDir: '/root/'}, {} as Config.Argv);
+    expect(options.showSeed).toBeFalsy();
+  });
+});

packages/jest-config/src/index.ts

@@ -115,7 +115,9 @@ const groupOptions = (
     reporters: options.reporters,
     rootDir: options.rootDir,
     runTestsByPath: options.runTestsByPath,
+    seed: options.seed,
     shard: options.shard,
+    showSeed: options.showSeed,
     silent: options.silent,
     skipFilter: options.skipFilter,
     snapshotFormat: options.snapshotFormat,

packages/jest-config/src/normalize.ts

@@ -918,6 +918,7 @@ export default async function normalize(
       case 'runTestsByPath':
       case 'sandboxInjectedGlobals':
       case 'silent':
+      case 'showSeed':
       case 'skipFilter':
       case 'skipNodeResolution':
       case 'slowTestThreshold':
@@ -1021,6 +1022,24 @@ export default async function normalize(
     newOptions.onlyChanged = newOptions.watch;
   }
 
+  newOptions.showSeed = newOptions.showSeed || argv.showSeed;
+
+  const upperBoundSeedValue = 2 ** 31;
+
+  // xoroshiro128plus is used in v8 and is used here (at time of writing)
+  newOptions.seed =
+    argv.seed ??
+    Math.floor((2 ** 32 - 1) * Math.random() - upperBoundSeedValue);
+  if (
+    newOptions.seed < -upperBoundSeedValue ||
+    newOptions.seed > upperBoundSeedValue - 1
+  ) {
+    throw new ValidationError(
+      'Validation Error',
+      `seed value must be between \`-0x80000000\` and \`0x7fffffff\` inclusive - is ${newOptions.seed}`,
+    );
+  }
+
   if (!newOptions.onlyChanged) {
     newOptions.onlyChanged = false;
   }

packages/jest-core/src/lib/__tests__/__snapshots__/logDebugMessages.test.ts.snap

@@ -95,6 +95,7 @@ exports[`prints the config object 1`] = `
     "reporters": [],
     "rootDir": "/test_root_dir/",
     "runTestsByPath": false,
+    "seed": 1234,
     "silent": false,
     "skipFilter": false,
     "snapshotFormat": {},