Deterministic snaphot files, declaration-ordered snapshot reports

lib/runner.js

@@ -31,6 +31,7 @@ class Runner extends Emittery {
 		this.boundCompareTestSnapshot = this.compareTestSnapshot.bind(this);
 		this.interrupted = false;
 		this.snapshots = null;
+		this.nextTaskIndex = 0;
 		this.tasks = {
 			after: [],
 			afterAlways: [],
@@ -76,6 +77,8 @@ class Runner extends Emittery {
 				});
 			}
 
+			metadata.taskIndex = this.nextTaskIndex++;
+
 			const {args, buildTitle, implementations, rawTitle} = parseTestArgs(testArgs);
 
 			if (this.checkSelectedByLineNumbers) {
@@ -285,7 +288,7 @@ class Runner extends Emittery {
 		return result;
 	}
 
-	async runHooks(tasks, contextRef, titleSuffix, testPassed) {
+	async runHooks(tasks, contextRef, {titleSuffix, testPassed, associatedTaskIndex} = {}) {
 		const hooks = tasks.map(task => new Runnable({
 			contextRef,
 			experiments: this.experiments,
@@ -295,7 +298,7 @@ class Runner extends Emittery {
 				t => task.implementation.apply(null, [t].concat(task.args)),
 			compareTestSnapshot: this.boundCompareTestSnapshot,
 			updateSnapshots: this.updateSnapshots,
-			metadata: task.metadata,
+			metadata: {...task.metadata, associatedTaskIndex},
 			powerAssert: this.powerAssert,
 			title: `${task.title}${titleSuffix || ''}`,
 			isHook: true,
@@ -326,7 +329,14 @@ class Runner extends Emittery {
 
 	async runTest(task, contextRef) {
 		const hookSuffix = ` for ${task.title}`;
-		let hooksOk = await this.runHooks(this.tasks.beforeEach, contextRef, hookSuffix);
+		let hooksOk = await this.runHooks(
+			this.tasks.beforeEach,
+			contextRef,
+			{
+				titleSuffix: hookSuffix,
+				associatedTaskIndex: task.metadata.taskIndex
+			}
+		);
 
 		let testOk = false;
 		if (hooksOk) {
@@ -358,7 +368,14 @@ class Runner extends Emittery {
 					logs: result.logs
 				});
 
-				hooksOk = await this.runHooks(this.tasks.afterEach, contextRef, hookSuffix, testOk);
+				hooksOk = await this.runHooks(
+					this.tasks.afterEach,
+					contextRef,
+					{
+						titleSuffix: hookSuffix,
+						testPassed: testOk,
+						associatedTaskIndex: task.metadata.taskIndex
+					});
 			} else {
 				this.emit('stateChange', {
 					type: 'test-failed',
@@ -372,7 +389,14 @@ class Runner extends Emittery {
 			}
 		}
 
-		const alwaysOk = await this.runHooks(this.tasks.afterEachAlways, contextRef, hookSuffix, testOk);
+		const alwaysOk = await this.runHooks(
+			this.tasks.afterEachAlways,
+			contextRef,
+			{
+				titleSuffix: hookSuffix,
+				testPassed: testOk,
+				associatedTaskIndex: task.metadata.taskIndex
+			});
 		return alwaysOk && hooksOk && testOk;
 	}
 

lib/snapshot-manager.js

@@ -104,13 +104,32 @@ function combineEntries(entries) {
 	const buffers = [];
 	let byteLength = 0;
 
-	const sortedKeys = [...entries.keys()].sort();
+	const sortedKeys = [...entries.keys()].sort((keyA, keyB) => {
+		const [a, b] = [entries.get(keyA), entries.get(keyB)];
+		const taskDifference = a.taskIndex - b.taskIndex;
+
+		if (taskDifference !== 0) {
+			return taskDifference;
+		}
+
+		const [assocA, assocB] = [a.associatedTaskIndex, b.associatedTaskIndex];
+		if (assocA !== undefined && assocB !== undefined) {
+			const assocDifference = assocA - assocB;
+
+			if (assocDifference !== 0) {
+				return assocDifference;
+			}
+		}
+
+		return a.snapIndex - b.snapIndex;
+	});
+
 	for (const key of sortedKeys) {
 		const keyBuffer = Buffer.from(`\n\n## ${key}\n\n`, 'utf8');
 		buffers.push(keyBuffer);
 		byteLength += keyBuffer.byteLength;
 
-		const formattedEntries = entries.get(key);
+		const formattedEntries = entries.get(key).buffers;
 		const last = formattedEntries[formattedEntries.length - 1];
 		for (const entry of formattedEntries) {
 			buffers.push(entry);
@@ -176,10 +195,11 @@ function encodeSnapshots(buffersByHash) {
 	byteOffset += 2;
 
 	const entries = [];
-	for (const pair of buffersByHash) {
-		const hash = pair[0];
-		const snapshotBuffers = pair[1];
-
+	// Maps can't have duplicate keys, so all items in [...buffersByHash.keys()]
+	// are unique, so sortedHashes should be deterministic.
+	const sortedHashes = [...buffersByHash.keys()].sort();
+	const sortedBuffersByHash = [...sortedHashes.map(hash => [hash, buffersByHash.get(hash)])];
+	for (const [hash, snapshotBuffers] of sortedBuffersByHash) {
 		buffers.push(Buffer.from(hash, 'hex'));
 		byteOffset += MD5_HASH_LENGTH;
 
@@ -332,6 +352,7 @@ class Manager {
 		const descriptor = concordance.describe(options.expected, concordanceOptions);
 		const snapshot = concordance.serialize(descriptor);
 		const entry = formatEntry(options.label, descriptor);
+		const {taskIndex, snapIndex, associatedTaskIndex} = options;
 
 		return () => { // Must be called in order!
 			this.hasChanges = true;
@@ -353,9 +374,9 @@ class Manager {
 			snapshots.push(snapshot);
 
 			if (this.reportEntries.has(options.belongsTo)) {
-				this.reportEntries.get(options.belongsTo).push(entry);
+				this.reportEntries.get(options.belongsTo).buffers.push(entry);
 			} else {
-				this.reportEntries.set(options.belongsTo, [entry]);
+				this.reportEntries.set(options.belongsTo, {buffers: [entry], taskIndex, snapIndex, associatedTaskIndex});
 			}
 		};
 	}

lib/test.js

@@ -230,7 +230,17 @@ class Test {
 			const index = id ? 0 : this.nextSnapshotIndex++;
 			const label = id ? '' : message || `Snapshot ${index + 1}`; // Human-readable labels start counting at 1.
 
-			const {record, ...result} = options.compareTestSnapshot({belongsTo, deferRecording, expected, index, label});
+			const {taskIndex, associatedTaskIndex} = this.metadata;
+			const {record, ...result} = options.compareTestSnapshot({
+				belongsTo,
+				deferRecording,
+				expected,
+				index,
+				label,
+				taskIndex,
+				snapIndex: this.snapshotCount,
+				associatedTaskIndex
+			});
 			if (record) {
 				this.deferredSnapshotRecordings.push(record);
 			}

test/snapshot-order/backwards-compatibility.js

@@ -0,0 +1,119 @@
+const test = require('@ava/test');
+const exec = require('../helpers/exec');
+const fs = require('fs');
+const path = require('path');
+const getSnapshotIds = require('./helpers/get-snapshot-ids');
+
+const unsortedSnapshotPath = path.join(__dirname, 'fixtures', 'backwards-compatibility', 'unsorted', 'test.js.snap');
+const unsortedReportPath = path.join(__dirname, 'fixtures', 'backwards-compatibility', 'unsorted', 'test.js.md');
+
+const unsortedSnapshot = fs.readFileSync(unsortedSnapshotPath);
+const unsortedReport = fs.readFileSync(unsortedReportPath);
+
+test('unsorted snapshots are unchanged when checking', async t => {
+	const options = {
+		cwd: exec.cwd('backwards-compatibility', 'check-only'),
+		env: {
+			AVA_FORCE_CI: 'not-ci'
+		}
+	};
+
+	// Schedule snapshot cleanup
+	t.teardown(() => {
+		fs.unlinkSync(snapshotPath);
+		fs.unlinkSync(reportPath);
+	});
+
+	const snapshotPath = path.join(options.cwd, 'test.js.snap');
+	const reportPath = path.join(options.cwd, 'test.js.md');
+
+	// Install a known-unsorted snapshot, report
+	fs.copyFileSync(unsortedSnapshotPath, snapshotPath);
+	fs.copyFileSync(unsortedReportPath, reportPath);
+
+	// Run test
+	await exec.fixture([], options);
+
+	// Assert snapshot, report are unchanged
+	const snapshot = fs.readFileSync(snapshotPath);
+	const report = fs.readFileSync(reportPath);
+
+	t.deepEqual(snapshot, unsortedSnapshot);
+	t.deepEqual(report, unsortedReport);
+});
+
+test('unsorted snapshots are changed when appending', async t => {
+	const options = {
+		cwd: exec.cwd('backwards-compatibility', 'append-only'),
+		env: {
+			AVA_FORCE_CI: 'not-ci'
+		}
+	};
+
+	const snapshotPath = path.join(options.cwd, 'test.js.snap');
+	const reportPath = path.join(options.cwd, 'test.js.md');
+
+	// Schedule snapshot cleanup
+	t.teardown(() => {
+		fs.unlinkSync(snapshotPath);
+		fs.unlinkSync(reportPath);
+	});
+
+	// Install a known-unsorted snapshot, report
+	fs.copyFileSync(unsortedSnapshotPath, snapshotPath);
+	fs.copyFileSync(unsortedReportPath, reportPath);
+
+	// Run test
+	await exec.fixture([], options);
+
+	// Assert snapshot, report are changed
+	const snapshot = fs.readFileSync(snapshotPath);
+	const report = fs.readFileSync(reportPath);
+
+	t.notDeepEqual(snapshot, unsortedSnapshot);
+	t.notDeepEqual(report, unsortedReport);
+
+	// Assert entries appended to report are sorted
+	const textReport = report.toString();
+	const ids = getSnapshotIds(textReport);
+
+	t.deepEqual(ids, [2, 1, 3, 4]);
+});
+
+test('unsorted snapshots are changed when updating', async t => {
+	const options = {
+		cwd: exec.cwd('backwards-compatibility', 'updating'),
+		env: {
+			AVA_FORCE_CI: 'not-ci'
+		}
+	};
+
+	// Schedule snapshot cleanup
+	t.teardown(() => {
+		fs.unlinkSync(snapshotPath);
+		fs.unlinkSync(reportPath);
+	});
+
+	const snapshotPath = path.join(options.cwd, 'test.js.snap');
+	const reportPath = path.join(options.cwd, 'test.js.md');
+
+	// Install a known-unsorted snapshot, report
+	fs.copyFileSync(unsortedSnapshotPath, snapshotPath);
+	fs.copyFileSync(unsortedReportPath, reportPath);
+
+	// Run test
+	await exec.fixture(['--update-snapshots'], options);
+
+	// Assert snapshot, report are changed
+	const snapshot = fs.readFileSync(snapshotPath);
+	const report = fs.readFileSync(reportPath);
+
+	t.notDeepEqual(snapshot, unsortedSnapshot);
+	t.notDeepEqual(report, unsortedReport);
+
+	// Assert report is sorted
+	const textReport = report.toString();
+	const ids = getSnapshotIds(textReport);
+
+	t.deepEqual(ids, [1, 2]);
+});

test/snapshot-order/fixtures/backwards-compatibility/append-only/package.json

@@ -0,0 +1 @@
+{}

test/snapshot-order/fixtures/backwards-compatibility/append-only/test.js

@@ -0,0 +1,26 @@
+const test = require('ava');
+
+const id = index => `index: ${index}`;
+
+let resolveDelay;
+const delay = new Promise(resolve => {
+	resolveDelay = resolve;
+});
+
+test('B - declared first, resolves second', async t => {
+	await delay;
+	t.snapshot(id(1));
+});
+
+test('A - declared second, resolves first', t => {
+	t.snapshot(id(2));
+	resolveDelay();
+});
+
+test('D - a new test, declared third', t => {
+	t.snapshot(id(3));
+});
+
+test('C - another new test, declared fourth', t => {
+	t.snapshot(id(4));
+});

test/snapshot-order/fixtures/backwards-compatibility/check-only/package.json

@@ -0,0 +1 @@
+{}

test/snapshot-order/fixtures/backwards-compatibility/check-only/test.js

@@ -0,0 +1,18 @@
+const test = require('ava');
+
+const id = index => `index: ${index}`;
+
+let resolveDelay;
+const delay = new Promise(resolve => {
+	resolveDelay = resolve;
+});
+
+test('B - declared first, resolves second', async t => {
+	await delay;
+	t.snapshot(id(1));
+});
+
+test('A - declared second, resolves first', t => {
+	t.snapshot(id(2));
+	resolveDelay();
+});

test/snapshot-order/fixtures/backwards-compatibility/unsorted/test.js.md

@@ -0,0 +1,17 @@
+# Snapshot report for `test.js`
+
+The actual snapshot is saved in `test.js.snap`.
+
+Generated by [AVA](https://avajs.dev).
+
+## A - declared second, resolves first
+
+> Snapshot 1
+
+    'index: 2'
+
+## B - declared first, resolves second
+
+> Snapshot 1
+
+    'index: 1'

test/snapshot-order/fixtures/backwards-compatibility/updating/package.json

@@ -0,0 +1 @@
+{}

test/snapshot-order/fixtures/backwards-compatibility/updating/test.js

@@ -0,0 +1,18 @@
+const test = require('ava');
+
+const id = index => `index: ${index}`;
+
+let resolveDelay;
+const delay = new Promise(resolve => {
+	resolveDelay = resolve;
+});
+
+test('B - declared first, resolves second', async t => {
+	await delay;
+	t.snapshot(id(1));
+});
+
+test('A - declared second, resolves first', t => {
+	t.snapshot(id(2));
+	resolveDelay();
+});

test/snapshot-order/fixtures/intertest-order/package.json

@@ -0,0 +1 @@
+{}