diff --git a/docs/06-configuration.md b/docs/06-configuration.md index 8c46647ff..8b08af336 100644 --- a/docs/06-configuration.md +++ b/docs/06-configuration.md @@ -164,7 +164,7 @@ AVA has a minimum depth of `3`. ## Experiments -From time to time, AVA will implement experimental features. These may change or be removed at any time, not just when there's a new major version. You can opt-in to such a feature by enabling it in the `nonSemVerExperiments` configuration. +From time to time, AVA will implement experimental features. These may change or be removed at any time, not just when there's a new major version. You can opt in to such a feature by enabling it in the `nonSemVerExperiments` configuration. `ava.config.js`: ```js @@ -175,6 +175,15 @@ export default { }; ``` -There are currently no such features available. +You can opt in to the new `t.try()` assertion by specifying `tryAssertion`: + +`ava.config.js`: +```js +export default { + nonSemVerExperiments: { + tryAssertion: true + } +}; +``` [CLI]: ./05-command-line.md diff --git a/index.d.ts b/index.d.ts index 5f1501fa0..35b34e983 100644 --- a/index.d.ts +++ b/index.d.ts @@ -25,6 +25,13 @@ export type ThrowsExpectation = { name?: string; }; +export type CommitDiscardOptions = { + /** + * Whether the logs should be included in those of the parent test. + */ + retainLogs?: boolean +} + /** Options that can be passed to the `t.snapshot()` assertion. */ export type SnapshotOptions = { /** If provided and not an empty string, used to select the snapshot to compare the `expected` value against. */ @@ -363,6 +370,7 @@ export interface ExecutionContext extends Assertions { log: LogFn; plan: PlanFn; timeout: TimeoutFn; + try: TryFn; } export interface LogFn { @@ -392,6 +400,69 @@ export interface TimeoutFn { (ms: number): void; } +export interface TryFn { + /** + * Requires opt-in. Attempt to run some assertions. The result must be explicitly committed or discarded or else + * the test will fail. A macro may be provided. The title may help distinguish attempts from + * one another. + */ + (title: string, fn: EitherMacro, ...args: Args): Promise; + + /** + * Requires opt-in. Attempt to run some assertions. The result must be explicitly committed or discarded or else + * the test will fail. A macro may be provided. The title may help distinguish attempts from + * one another. + */ + (title: string, fn: [EitherMacro, ...EitherMacro[]], ...args: Args): Promise; + + /** + * Requires opt-in. Attempt to run some assertions. The result must be explicitly committed or discarded or else + * the test will fail. A macro may be provided. + */ + (fn: EitherMacro, ...args: Args): Promise; + + /** + * Requires opt-in. Attempt to run some assertions. The result must be explicitly committed or discarded or else + * the test will fail. A macro may be provided. + */ + (fn: [EitherMacro, ...EitherMacro[]], ...args: Args): Promise; +} + +export interface AssertionError extends Error {} + +export interface TryResult { + /** + * Title of the attempt, helping you tell attempts aparts. + */ + title: string; + + /** + * Indicates whether all assertions passed, or at least one failed. + */ + passed: boolean; + + /** + * Errors raised for each failed assertion. + */ + errors: AssertionError[]; + + /** + * Logs created during the attempt using `t.log()`. Contains formatted values. + */ + logs: string[]; + + /** + * Commit the attempt. Counts as one assertion for the plan count. If the + * attempt failed, calling this will also cause your test to fail. + */ + commit(options?: CommitDiscardOptions): void; + + /** + * Discard the attempt. + */ + discard(options?: CommitDiscardOptions): void; +} + /** The `t` value passed to implementations for tests & hooks declared with the `.cb` modifier. */ export interface CbExecutionContext extends ExecutionContext { /** diff --git a/lib/load-config.js b/lib/load-config.js index ba4417a2a..17cf53dc0 100644 --- a/lib/load-config.js +++ b/lib/load-config.js @@ -6,7 +6,7 @@ const pkgConf = require('pkg-conf'); const NO_SUCH_FILE = Symbol('no ava.config.js file'); const MISSING_DEFAULT_EXPORT = Symbol('missing default export'); -const EXPERIMENTS = new Set([]); +const EXPERIMENTS = new Set(['tryAssertion']); function loadConfig({configFile, resolveFrom = process.cwd(), defaults = {}} = {}) { // eslint-disable-line complexity let packageConf = pkgConf.sync('ava', {cwd: resolveFrom}); diff --git a/lib/parse-test-args.js b/lib/parse-test-args.js new file mode 100644 index 000000000..5ea5f0aa4 --- /dev/null +++ b/lib/parse-test-args.js @@ -0,0 +1,15 @@ +'use strict'; +function parseTestArgs(args) { + const rawTitle = typeof args[0] === 'string' ? args.shift() : undefined; + const receivedImplementationArray = Array.isArray(args[0]); + const implementations = receivedImplementationArray ? args.shift() : args.splice(0, 1); + + const buildTitle = implementation => { + const title = implementation.title ? implementation.title(rawTitle, ...args) : rawTitle; + return {title, isSet: typeof title !== 'undefined', isValid: typeof title === 'string', isEmpty: !title}; + }; + + return {args, buildTitle, implementations, rawTitle, receivedImplementationArray}; +} + +module.exports = parseTestArgs; diff --git a/lib/runner.js b/lib/runner.js index 18af02f7a..cfb9cdcde 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -3,6 +3,7 @@ const Emittery = require('emittery'); const matcher = require('matcher'); const ContextRef = require('./context-ref'); const createChain = require('./create-chain'); +const parseTestArgs = require('./parse-test-args'); const snapshotManager = require('./snapshot-manager'); const serializeError = require('./serialize-error'); const Runnable = require('./test'); @@ -11,6 +12,7 @@ class Runner extends Emittery { constructor(options = {}) { super(); + this.experiments = options.experiments || {}; this.failFast = options.failFast === true; this.failWithoutAssertions = options.failWithoutAssertions !== false; this.file = options.file; @@ -39,12 +41,21 @@ class Runner extends Emittery { }; const uniqueTestTitles = new Set(); + this.registerUniqueTitle = title => { + if (uniqueTestTitles.has(title)) { + return false; + } + + uniqueTestTitles.add(title); + return true; + }; + let hasStarted = false; let scheduledStart = false; const meta = Object.freeze({ file: options.file }); - this.chain = createChain((metadata, args) => { // eslint-disable-line complexity + this.chain = createChain((metadata, testArgs) => { // eslint-disable-line complexity if (hasStarted) { throw new Error('All tests and hooks must be declared synchronously in your test file, and cannot be nested within other tests or hooks.'); } @@ -57,40 +68,33 @@ class Runner extends Emittery { }); } - const specifiedTitle = typeof args[0] === 'string' ? - args.shift() : - undefined; - const implementations = Array.isArray(args[0]) ? - args.shift() : - args.splice(0, 1); + const {args, buildTitle, implementations, rawTitle} = parseTestArgs(testArgs); if (metadata.todo) { if (implementations.length > 0) { throw new TypeError('`todo` tests are not allowed to have an implementation. Use `test.skip()` for tests with an implementation.'); } - if (specifiedTitle === undefined || specifiedTitle === '') { + if (!rawTitle) { // Either undefined or a string. throw new TypeError('`todo` tests require a title'); } - if (uniqueTestTitles.has(specifiedTitle)) { - throw new Error(`Duplicate test title: ${specifiedTitle}`); - } else { - uniqueTestTitles.add(specifiedTitle); + if (!this.registerUniqueTitle(rawTitle)) { + throw new Error(`Duplicate test title: ${rawTitle}`); } if (this.match.length > 0) { // --match selects TODO tests. - if (matcher([specifiedTitle], this.match).length === 1) { + if (matcher([rawTitle], this.match).length === 1) { metadata.exclusive = true; this.runOnlyExclusive = true; } } - this.tasks.todo.push({title: specifiedTitle, metadata}); + this.tasks.todo.push({title: rawTitle, metadata}); this.emit('stateChange', { type: 'declared-test', - title: specifiedTitle, + title: rawTitle, knownFailing: false, todo: true }); @@ -100,15 +104,13 @@ class Runner extends Emittery { } for (const implementation of implementations) { - let title = implementation.title ? - implementation.title(specifiedTitle, ...args) : - specifiedTitle; + let {title, isSet, isValid, isEmpty} = buildTitle(implementation); - if (title !== undefined && typeof title !== 'string') { + if (isSet && !isValid) { throw new TypeError('Test & hook titles must be strings'); } - if (title === undefined || title === '') { + if (isEmpty) { if (metadata.type === 'test') { throw new TypeError('Tests must have a title'); } else if (metadata.always) { @@ -118,12 +120,8 @@ class Runner extends Emittery { } } - if (metadata.type === 'test') { - if (uniqueTestTitles.has(title)) { - throw new Error(`Duplicate test title: ${title}`); - } else { - uniqueTestTitles.add(title); - } + if (metadata.type === 'test' && !this.registerUniqueTitle(title)) { + throw new Error(`Duplicate test title: ${title}`); } const task = { @@ -162,6 +160,7 @@ class Runner extends Emittery { todo: false, failing: false, callback: false, + inline: false, // Set for attempt metadata created by `t.try()` always: false }, meta); } @@ -269,6 +268,7 @@ class Runner extends Emittery { async runHooks(tasks, contextRef, titleSuffix) { const hooks = tasks.map(task => new Runnable({ contextRef, + experiments: this.experiments, failWithoutAssertions: false, fn: task.args.length === 0 ? task.implementation : @@ -309,6 +309,7 @@ class Runner extends Emittery { // Only run the test if all `beforeEach` hooks passed. const test = new Runnable({ contextRef, + experiments: this.experiments, failWithoutAssertions: this.failWithoutAssertions, fn: task.args.length === 0 ? task.implementation : @@ -316,7 +317,8 @@ class Runner extends Emittery { compareTestSnapshot: this.boundCompareTestSnapshot, updateSnapshots: this.updateSnapshots, metadata: task.metadata, - title: task.title + title: task.title, + registerUniqueTitle: this.registerUniqueTitle }); const result = await this.runSingle(test); diff --git a/lib/snapshot-manager.js b/lib/snapshot-manager.js index 545bd3651..2fed69aef 100644 --- a/lib/snapshot-manager.js +++ b/lib/snapshot-manager.js @@ -305,45 +305,64 @@ class Manager { compare(options) { const hash = md5Hex(options.belongsTo); const entries = this.snapshotsByHash.get(hash) || []; - if (options.index > entries.length) { - throw new RangeError(`Cannot record snapshot ${options.index} for ${JSON.stringify(options.belongsTo)}, exceeds expected index of ${entries.length}`); - } + const snapshotBuffer = entries[options.index]; - if (options.index === entries.length) { + if (!snapshotBuffer) { if (!this.recordNewSnapshots) { return {pass: false}; } + if (options.deferRecording) { + const record = this.deferRecord(hash, options); + return {pass: true, record}; + } + this.record(hash, options); return {pass: true}; } - const snapshotBuffer = entries[options.index]; const actual = concordance.deserialize(snapshotBuffer, concordanceOptions); - const expected = concordance.describe(options.expected, concordanceOptions); const pass = concordance.compareDescriptors(actual, expected); return {actual, expected, pass}; } - record(hash, options) { + deferRecord(hash, options) { const descriptor = concordance.describe(options.expected, concordanceOptions); - - this.hasChanges = true; const snapshot = concordance.serialize(descriptor); - if (this.snapshotsByHash.has(hash)) { - this.snapshotsByHash.get(hash).push(snapshot); - } else { - this.snapshotsByHash.set(hash, [snapshot]); - } - const entry = formatEntry(options.label, descriptor); - if (this.reportEntries.has(options.belongsTo)) { - this.reportEntries.get(options.belongsTo).push(entry); - } else { - this.reportEntries.set(options.belongsTo, [entry]); - } + + return () => { // Must be called in order! + this.hasChanges = true; + + let snapshots = this.snapshotsByHash.get(hash); + if (!snapshots) { + snapshots = []; + this.snapshotsByHash.set(hash, snapshots); + } + + if (options.index > snapshots.length) { + throw new RangeError(`Cannot record snapshot ${options.index} for ${JSON.stringify(options.belongsTo)}, exceeds expected index of ${snapshots.length}`); + } + + if (options.index < snapshots.length) { + throw new RangeError(`Cannot record snapshot ${options.index} for ${JSON.stringify(options.belongsTo)}, already exists`); + } + + snapshots.push(snapshot); + + if (this.reportEntries.has(options.belongsTo)) { + this.reportEntries.get(options.belongsTo).push(entry); + } else { + this.reportEntries.set(options.belongsTo, [entry]); + } + }; + } + + record(hash, options) { + const record = this.deferRecord(hash, options); + record(); } save() { diff --git a/lib/test.js b/lib/test.js index 10b277ec8..303e0d2e7 100644 --- a/lib/test.js +++ b/lib/test.js @@ -6,6 +6,7 @@ const isObservable = require('is-observable'); const plur = require('plur'); const assert = require('./assert'); const nowAndTimers = require('./now-and-timers'); +const parseTestArgs = require('./parse-test-args'); const concordanceOptions = require('./concordance-options').default; function formatErrorValue(label, error) { @@ -67,6 +68,95 @@ class ExecutionContext extends assert.Assertions { this.timeout = ms => { test.timeout(ms); }; + + this.try = async (...attemptArgs) => { + if (test.experiments.tryAssertion !== true) { + throw new Error('t.try() is currently an experiment. Opt in by setting `nonSemVerExperiments.tryAssertion` to `true`.'); + } + + const {args, buildTitle, implementations, receivedImplementationArray} = parseTestArgs(attemptArgs); + + if (implementations.length === 0) { + throw new TypeError('Expected an implementation.'); + } + + const attemptPromises = implementations.map(implementation => { + let {title, isSet, isValid, isEmpty} = buildTitle(implementation); + + if (!isSet || isEmpty) { + title = `${test.title} (attempt ${test.attemptCount + 1})`; + } else if (!isValid) { + throw new TypeError('`t.try()` titles must be strings'); // Throw synchronously! + } + + if (!test.registerUniqueTitle(title)) { + throw new Error(`Duplicate test title: ${title}`); + } + + return {implementation, title}; + }).map(async ({implementation, title}) => { + let committed = false; + let discarded = false; + + const {assertCount, deferredSnapshotRecordings, errors, logs, passed, snapshotCount, startingSnapshotCount} = await test.runAttempt(title, t => implementation(t, ...args)); + + return { + errors, + logs: [...logs], // Don't allow modification of logs. + passed, + title, + commit: ({retainLogs = true} = {}) => { + if (committed) { + return; + } + + if (discarded) { + test.saveFirstError(new Error('Can\'t commit a result that was previously discarded')); + return; + } + + committed = true; + test.finishAttempt({ + assertCount, + commit: true, + deferredSnapshotRecordings, + errors, + logs, + passed, + retainLogs, + snapshotCount, + startingSnapshotCount + }); + }, + discard: ({retainLogs = false} = {}) => { + if (committed) { + test.saveFirstError(new Error('Can\'t discard a result that was previously committed')); + return; + } + + if (discarded) { + return; + } + + discarded = true; + test.finishAttempt({ + assertCount: 0, + commit: false, + deferredSnapshotRecordings, + errors, + logs, + passed, + retainLogs, + snapshotCount, + startingSnapshotCount + }); + } + }; + }); + + const results = await Promise.all(attemptPromises); + return receivedImplementationArray ? results : results[0]; + }; } get end() { @@ -99,32 +189,74 @@ class ExecutionContext extends assert.Assertions { class Test { constructor(options) { this.contextRef = options.contextRef; + this.experiments = options.experiments || {}; this.failWithoutAssertions = options.failWithoutAssertions; this.fn = options.fn; this.metadata = options.metadata; this.title = options.title; + this.registerUniqueTitle = options.registerUniqueTitle; this.logs = []; - this.snapshotInvocationCount = 0; - this.compareWithSnapshot = assertionOptions => { - const belongsTo = assertionOptions.id || this.title; - const {expected} = assertionOptions; - const index = assertionOptions.id ? 0 : this.snapshotInvocationCount++; - const label = assertionOptions.id ? '' : assertionOptions.message || `Snapshot ${this.snapshotInvocationCount}`; - return options.compareTestSnapshot({belongsTo, expected, index, label}); + const {snapshotBelongsTo = this.title, nextSnapshotIndex = 0} = options; + this.snapshotBelongsTo = snapshotBelongsTo; + this.nextSnapshotIndex = nextSnapshotIndex; + this.snapshotCount = 0; + + const deferRecording = this.metadata.inline; + this.deferredSnapshotRecordings = []; + this.compareWithSnapshot = ({expected, id, message}) => { + this.snapshotCount++; + + // TODO: In a breaking change, reject non-undefined, falsy IDs and messages. + const belongsTo = id || snapshotBelongsTo; + const index = id ? 0 : this.nextSnapshotIndex++; + const label = id ? '' : message || `Snapshot ${index + 1}`; // Human-readable labels start counting at 1. + + const {record, ...result} = options.compareTestSnapshot({belongsTo, deferRecording, expected, index, label}); + if (record) { + this.deferredSnapshotRecordings.push(record); + } + + return result; }; this.skipSnapshot = () => { if (options.updateSnapshots) { this.addFailedAssertion(new Error('Snapshot assertions cannot be skipped when updating snapshots')); } else { - this.snapshotInvocationCount++; + this.nextSnapshotIndex++; + this.snapshotCount++; this.countPassedAssertion(); } }; + this.runAttempt = async (title, fn) => { + if (this.finishing) { + this.saveFirstError(new Error('Running a `t.try()`, but the test has already finished')); + } + + this.attemptCount++; + this.pendingAttemptCount++; + + const {contextRef, snapshotBelongsTo, nextSnapshotIndex, snapshotCount: startingSnapshotCount} = this; + const attempt = new Test({ + ...options, + fn, + metadata: {...options.metadata, callback: false, failing: false, inline: true}, + contextRef: contextRef.copy(), + snapshotBelongsTo, + nextSnapshotIndex, + title + }); + + const {deferredSnapshotRecordings, error, logs, passed, assertCount, snapshotCount} = await attempt.run(); + const errors = error ? [error] : []; + return {assertCount, deferredSnapshotRecordings, errors, logs, passed, snapshotCount, startingSnapshotCount}; + }; + this.assertCount = 0; this.assertError = undefined; + this.attemptCount = 0; this.calledEnd = false; this.duration = null; this.endCallbackFinisher = null; @@ -133,11 +265,12 @@ class Test { this.finishDueToTimeout = null; this.finishing = false; this.pendingAssertionCount = 0; + this.pendingAttemptCount = 0; this.pendingThrowsAssertion = null; this.planCount = null; this.startedAt = 0; - this.timeoutTimer = null; this.timeoutMs = 0; + this.timeoutTimer = null; } bindEndCallback() { @@ -147,7 +280,11 @@ class Test { }; } - throw new Error('`t.end()`` is not supported in this context. To use `t.end()` as a callback, you must use "callback mode" via `test.cb(testName, fn)`'); + if (this.metadata.inline) { + throw new Error('`t.end()` is not supported inside `t.try()`'); + } else { + throw new Error('`t.end()` is not supported in this context. To use `t.end()` as a callback, you must use "callback mode" via `test.cb(testName, fn)`'); + } } endCallback(error, savedError) { @@ -181,6 +318,10 @@ class Test { this.saveFirstError(new Error('Assertion passed, but test has already finished')); } + if (this.pendingAttemptCount > 0) { + this.saveFirstError(new Error('Assertion passed, but an attempt is pending. Use the attempt’s assertions instead')); + } + this.assertCount++; this.refreshTimeout(); } @@ -191,7 +332,11 @@ class Test { addPendingAssertion(promise) { if (this.finishing) { - this.saveFirstError(new Error('Assertion passed, but test has already finished')); + this.saveFirstError(new Error('Assertion started, but test has already finished')); + } + + if (this.pendingAttemptCount > 0) { + this.saveFirstError(new Error('Assertion started, but an attempt is pending. Use the attempt’s assertions instead')); } this.assertCount++; @@ -211,11 +356,53 @@ class Test { this.saveFirstError(new Error('Assertion failed, but test has already finished')); } + if (this.pendingAttemptCount > 0) { + this.saveFirstError(new Error('Assertion failed, but an attempt is pending. Use the attempt’s assertions instead')); + } + this.assertCount++; this.refreshTimeout(); this.saveFirstError(error); } + finishAttempt({commit, deferredSnapshotRecordings, errors, logs, passed, retainLogs, snapshotCount, startingSnapshotCount}) { + if (this.finishing) { + if (commit) { + this.saveFirstError(new Error('`t.try()` result was committed, but the test has already finished')); + } else { + this.saveFirstError(new Error('`t.try()` result was discarded, but the test has already finished')); + } + } + + if (commit) { + this.assertCount++; + + if (startingSnapshotCount === this.snapshotCount) { + this.snapshotCount += snapshotCount; + this.nextSnapshotIndex += snapshotCount; + for (const record of deferredSnapshotRecordings) { + record(); + } + } else { + this.saveFirstError(new Error('Cannot commit `t.try()` result. Do not run concurrent snapshot assertions when using `t.try()`')); + } + } + + this.pendingAttemptCount--; + + if (commit && !passed) { + this.saveFirstError(errors[0]); + } + + if (retainLogs) { + for (const log of logs) { + this.addLog(log); + } + } + + this.refreshTimeout(); + } + saveFirstError(error) { if (!this.assertError) { this.assertError = error; @@ -279,11 +466,27 @@ class Test { } verifyAssertions() { - if (!this.assertError) { - if (this.failWithoutAssertions && !this.calledEnd && this.planCount === null && this.assertCount === 0) { + if (this.assertError) { + return; + } + + if (this.pendingAttemptCount > 0) { + this.saveFirstError(new Error('Test finished, but not all attempts were committed or discarded')); + return; + } + + if (this.pendingAssertionCount > 0) { + this.saveFirstError(new Error('Test finished, but an assertion is still pending')); + return; + } + + if (this.failWithoutAssertions) { + if (this.planCount !== null) { + return; // `verifyPlan()` will report an error already. + } + + if (this.assertCount === 0 && !this.calledEnd) { this.saveFirstError(new Error('Test finished without running any assertions')); - } else if (this.pendingAssertionCount > 0) { - this.saveFirstError(new Error('Test finished, but an assertion is still pending')); } } } @@ -476,11 +679,14 @@ class Test { } return { + deferredSnapshotRecordings: this.deferredSnapshotRecordings, duration: this.duration, error, logs: this.logs, metadata: this.metadata, passed, + snapshotCount: this.snapshotCount, + assertCount: this.assertCount, title: this.title }; } diff --git a/lib/worker/subprocess.js b/lib/worker/subprocess.js index 90b829d6a..861f0ecfa 100644 --- a/lib/worker/subprocess.js +++ b/lib/worker/subprocess.js @@ -31,6 +31,7 @@ ipc.options.then(options => { } const runner = new Runner({ + experiments: options.experiments, failFast: options.failFast, failWithoutAssertions: options.failWithoutAssertions, file: options.file, diff --git a/test/fixture/try-snapshot.js.md b/test/fixture/try-snapshot.js.md new file mode 100644 index 000000000..5c5987f36 --- /dev/null +++ b/test/fixture/try-snapshot.js.md @@ -0,0 +1,43 @@ +# Snapshot report for `try-snapshot.js` + +The actual snapshot is saved in `try-snapshot.js.snap`. + +Generated by [AVA](https://ava.li). + +## concurrent + +> Snapshot 1 + + 'hello' + +> Snapshot 2 + + true + +> Snapshot 3 + + { + boo: 'far', + } + +## serial + +> Snapshot 1 + + 'hello' + +> Snapshot 2 + + true + +> Snapshot 3 + + { + boo: 'far', + } + +> Snapshot 4 + + { + foo: 'bar', + } diff --git a/test/fixture/try-snapshot.js.snap b/test/fixture/try-snapshot.js.snap new file mode 100644 index 000000000..b4919bad0 Binary files /dev/null and b/test/fixture/try-snapshot.js.snap differ diff --git a/test/helper/ava-test.js b/test/helper/ava-test.js new file mode 100644 index 000000000..36cbf67f6 --- /dev/null +++ b/test/helper/ava-test.js @@ -0,0 +1,57 @@ +const Test = require('../../lib/test'); +const ContextRef = require('../../lib/context-ref'); + +function withExperiments(experiments = {}) { + function ava(fn, contextRef) { + return new Test({ + contextRef: contextRef || new ContextRef(), + experiments, + failWithoutAssertions: true, + fn, + registerUniqueTitle: () => true, + metadata: {type: 'test', callback: false}, + title: 'test' + }); + } + + ava.failing = (fn, contextRef) => { + return new Test({ + contextRef: contextRef || new ContextRef(), + experiments, + failWithoutAssertions: true, + fn, + registerUniqueTitle: () => true, + metadata: {type: 'test', callback: false, failing: true}, + title: 'test.failing' + }); + }; + + ava.cb = (fn, contextRef) => { + return new Test({ + contextRef: contextRef || new ContextRef(), + experiments, + failWithoutAssertions: true, + fn, + registerUniqueTitle: () => true, + metadata: {type: 'test', callback: true}, + title: 'test.cb' + }); + }; + + ava.cb.failing = (fn, contextRef) => { + return new Test({ + contextRef: contextRef || new ContextRef(), + experiments, + failWithoutAssertions: true, + fn, + registerUniqueTitle: () => true, + metadata: {type: 'test', callback: true, failing: true}, + title: 'test.cb.failing' + }); + }; + + return ava; +} + +exports.ava = withExperiments(); +exports.withExperiments = withExperiments; diff --git a/test/test-try-commit.js b/test/test-try-commit.js new file mode 100644 index 000000000..759e74a9f --- /dev/null +++ b/test/test-try-commit.js @@ -0,0 +1,589 @@ +'use strict'; +require('../lib/chalk').set(); +require('../lib/worker/options').set({color: false}); + +const {test} = require('tap'); +const delay = require('delay'); +const ContextRef = require('../lib/context-ref'); +const {withExperiments} = require('./helper/ava-test'); + +const ava = withExperiments({tryAssertion: true}); + +test('try-commit works', async t => { + const instance = ava(async a => { + const res = await a.try(b => b.pass()); + t.true(res.passed); + res.commit(); + }); + + const result = await instance.run(); + + t.true(result.passed); + t.is(instance.assertCount, 1); +}); + +test('try-commit is bound', async t => { + const result = await ava(async a => { + const {try: tryFn} = a; + const res = await tryFn(b => b.pass()); + await res.commit(); + }).run(); + + t.true(result.passed); +}); + +test('try-commit discards failed attempt', async t => { + const result = await ava(async a => { + const res = await a.try(b => b.fail()); + await res.discard(); + await a.pass(); + }).run(); + + t.true(result.passed); +}); + +test('try-commit can discard produced result', async t => { + const result = await ava(async a => { + const res = await a.try(b => b.pass()); + res.discard(); + }).run(); + + t.false(result.passed); + t.ok(result.error); + t.match(result.error.message, /without running any assertions/); + t.is(result.error.name, 'Error'); +}); + +test('try-commit fails when not all assertions were committed/discarded', async t => { + const result = await ava(async a => { + a.pass(); + await a.try(b => b.pass()); + }).run(); + + t.false(result.passed); + t.ok(result.error); + t.match(result.error.message, /not all attempts were committed/); + t.is(result.error.name, 'Error'); +}); + +test('try-commit works with values', async t => { + const testValue1 = 123; + const testValue2 = 123; + + const result = await ava(async a => { + const res = await a.try((b, val1, val2) => { + b.is(val1, val2); + }, testValue1, testValue2); + t.true(res.passed); + res.commit(); + }).run(); + + t.true(result.passed); +}); + +test('try-commit is properly counted', async t => { + const instance = ava(async a => { + const res = await a.try(b => { + b.is(1, 1); + b.is(2, 2); + b.pass(); + }); + + t.true(res.passed); + t.is(instance.pendingAttemptCount, 1); + res.commit(); + t.is(instance.pendingAttemptCount, 0); + }); + + const result = await instance.run(); + + t.true(result.passed); + t.is(instance.assertCount, 1); +}); + +test('try-commit is properly counted multiple', async t => { + const instance = ava(async a => { + const [res1, res2, res3] = await Promise.all([ + a.try(b => b.pass()), + a.try(b => b.pass()), + a.try(b => b.pass()) + ]); + + t.is(instance.pendingAttemptCount, 3); + res1.commit(); + res2.discard(); + res3.commit(); + t.is(instance.pendingAttemptCount, 0); + }); + + const result = await instance.run(); + + t.true(result.passed); + t.is(instance.assertCount, 2); +}); + +test('try-commit goes as many levels', async t => { + t.plan(5); + const instance = ava(async a => { + t.ok(a.try); + const res1 = await a.try(async b => { + t.ok(b.try); + const res = await b.try(c => { + t.ok(c.try); + c.pass(); + }); + res.commit(); + }); + res1.commit(); + }); + + const result = await instance.run(); + + t.true(result.passed); + t.is(instance.assertCount, 1); +}); + +test('try-commit fails when not committed', async t => { + const result = await ava(async a => { + const res = await a.try(b => b.pass()); + t.true(res.passed); + }).run(); + + t.false(result.passed); + t.ok(result.error); + t.match(result.error.message, /not all attempts were committed/); + t.is(result.error.name, 'Error'); +}); + +test('try-commit fails when no assertions inside try', async t => { + const result = await ava(async a => { + const res = await a.try(() => {}); + t.false(res.passed); + t.ok(res.errors); + t.is(res.errors.length, 1); + const error = res.errors[0]; + t.match(error.message, /Test finished without running any assertions/); + t.is(error.name, 'Error'); + res.commit(); + }).run(); + + t.false(result.passed); +}); + +test('try-commit fails when no assertions inside multiple try', async t => { + const result = await ava(async a => { + const [res1, res2] = await Promise.all([ + a.try(b => b.pass()), + a.try(() => {}) + ]); + + res1.commit(); + t.true(res1.passed); + + t.false(res2.passed); + t.ok(res2.errors); + t.is(res2.errors.length, 1); + const error = res2.errors[0]; + t.match(error.message, /Test finished without running any assertions/); + t.is(error.name, 'Error'); + res2.commit(); + }).run(); + + t.false(result.passed); +}); + +test('test fails when try-commit committed to failed state', async t => { + const result = await ava(async a => { + const res = await a.try(b => b.fail()); + t.false(res.passed); + res.commit(); + }).run(); + + t.false(result.passed); +}); + +test('try-commit has proper titles, when going in depth and width', async t => { + t.plan(6); + await ava(async a => { + t.is(a.title, 'test'); + + await Promise.all([ + a.try(async b => { + t.is(b.title, 'test (attempt 1)'); + + await Promise.all([ + b.try(c => t.is(c.title, 'test (attempt 1) (attempt 1)')), + b.try(c => t.is(c.title, 'test (attempt 1) (attempt 2)')) + ]); + }), + a.try(b => t.is(b.title, 'test (attempt 2)')), + a.try(b => t.is(b.title, 'test (attempt 3)')) + ]); + }).run(); +}); + +test('try-commit does not fail when calling commit twice', async t => { + const result = await ava(async a => { + const res = await a.try(b => b.pass()); + res.commit(); + res.commit(); + }).run(); + + t.true(result.passed); + t.false(result.error); +}); + +test('try-commit does not fail when calling discard twice', async t => { + const result = await ava(async a => { + const res = await a.try(b => b.pass()); + res.discard(); + res.discard(); + }).run(); + + t.false(result.passed); + t.ok(result.error); + t.match(result.error.message, /Test finished without running any assertions/); + t.is(result.error.name, 'Error'); +}); + +test('try-commit allows planning inside the try', async t => { + const result = await ava(async a => { + const res = await a.try(b => { + b.plan(3); + + b.pass(); + b.pass(); + b.pass(); + }); + t.true(res.passed); + res.commit(); + }).run(); + + t.true(result.passed); +}); + +test('try-commit fails when plan is not reached inside the try', async t => { + const result = await ava(async a => { + const res = await a.try(b => { + b.plan(3); + + b.pass(); + b.pass(); + }); + t.false(res.passed); + res.commit(); + }).run(); + + t.false(result.passed); +}); + +test('plan within try-commit is not affected by assertions outside', async t => { + const result = await ava(async a => { + a.is(1, 1); + a.is(2, 2); + + const attempt = a.try(b => { + b.plan(3); + }); + + const res = await attempt; + t.false(res.passed); + res.commit(); + }).run(); + + t.false(result.passed); + t.ok(result.error); + t.match(result.error.message, /Planned for 3 assertions, but got 0/); +}); + +test('assertions within try-commit do not affect plan in the parent test', async t => { + const result = await ava(async a => { + a.plan(2); + + const res = await a.try(b => { + b.plan(3); + b.pass(); + b.pass(); + b.pass(); + }); + + t.true(res.passed); + res.commit(); + }).run(); + + t.false(result.passed); + t.ok(result.error); + t.match(result.error.message, /Planned for 2 assertions, but got 1/); +}); + +test('test expected to fail will pass with failing try-commit within the test', async t => { + const result = await ava.failing(async a => { + const res = await a.try(b => b.fail()); + t.false(res.passed); + t.ok(res.errors); + t.is(res.errors.length, 1); + const error = res.errors[0]; + t.match(error.message, /Test failed via `t\.fail\(\)`/); + t.is(error.name, 'AssertionError'); + res.commit(); + }).run(); + + t.true(result.passed); +}); + +test('try-commit works with callback test', async t => { + const result = await ava.cb(a => { + a + .try(b => b.pass()) + .then(res => { + res.commit(); + a.end(); + }); + }).run(); + + t.true(result.passed); +}); + +test('try-commit works with failing callback test', async t => { + const result = await ava.cb.failing(a => { + a + .try(b => b.fail()) + .then(res => { + t.false(res.passed); + t.ok(res.errors); + t.is(res.errors.length, 1); + const error = res.errors[0]; + t.match(error.message, /Test failed via `t\.fail\(\)`/); + t.is(error.name, 'AssertionError'); + res.commit(); + }) + .then(() => { + a.end(); + }); + }).run(); + + t.true(result.passed); +}); + +test('try-commit does not allow to use .end() in attempt when parent is callback test', async t => { + const result = await ava.cb(a => { + a + .try(b => { + b.pass(); + b.end(); + }) + .then(res => { + res.commit(); + a.end(); + }); + }).run(); + + t.false(result.passed); + t.ok(result.error); + t.match(result.error.message, /Error thrown in test/); + t.is(result.error.name, 'AssertionError'); + t.match(result.error.values[0].formatted, /t\.end.*not supported/); +}); + +test('try-commit does not allow to use .end() in attempt when parent is regular test', async t => { + const result = await ava(async a => { + const res = await a.try(b => { + b.pass(); + b.end(); + }); + + res.commit(); + }).run(); + + t.false(result.passed); + t.ok(result.error); + t.match(result.error.message, /Error thrown in test/); + t.is(result.error.name, 'AssertionError'); + t.match(result.error.values[0].formatted, /t\.end.*not supported/); +}); + +test('try-commit accepts macros', async t => { + const macro = b => { + t.is(b.title, ' Title'); + b.pass(); + }; + + macro.title = providedTitle => `${providedTitle ? providedTitle : ''} Title`; + + const result = await ava(async a => { + const res = await a.try(macro); + t.true(res.passed); + res.commit(); + }).run(); + + t.true(result.passed); +}); + +test('try-commit accepts multiple macros', async t => { + const macros = [b => b.pass(), b => b.fail()]; + const result = await ava(async a => { + const [res1, res2] = await a.try(macros); + t.true(res1.passed); + res1.commit(); + t.false(res2.passed); + res2.discard(); + }).run(); + + t.true(result.passed); +}); + +test('try-commit returns results in the same shape as when implementations are passed', async t => { + const result = await ava(async a => { + const [res1, res2, res3] = await Promise.all([ + a.try(b => b.pass()), + a.try([b => b.pass()]), + a.try([b => b.pass(), b => b.fail()]) + ]); + + t.match(res1, {passed: true}); + res1.commit(); + + t.is(res2.length, 1); + t.match(res2, [{passed: true}]); + res2[0].commit(); + + t.is(res3.length, 2); + t.match(res3, [{passed: true}, {passed: false}]); + res3[0].commit(); + res3[1].discard(); + }).run(); + + t.true(result.passed); +}); + +test('try-commit abides timeout', async t => { + const result1 = await ava(async a => { + a.timeout(10); + const result = await a.try(async b => { + b.pass(); + await delay(200); + }); + await result.commit(); + }).run(); + + t.is(result1.passed, false); + t.match(result1.error.message, /timeout/); +}); + +test('try-commit fails when it exceeds its own timeout', async t => { + const result = await ava(async a => { + a.timeout(200); + const result = await a.try(async b => { + b.timeout(50); + b.pass(); + await delay(100); + }); + + t.false(result.passed); + t.ok(result.errors); + t.is(result.errors.length, 1); + const error = result.errors[0]; + t.match(error.message, /Test timeout exceeded/); + t.is(error.name, 'Error'); + + result.discard(); + a.pass(); + }).run(); + + t.true(result.passed); +}); + +test('try-commit refreshes the timeout on commit/discard', async t => { + const result1 = await ava.cb(a => { + a.timeout(10); + a.plan(3); + setTimeout(() => a.try(b => b.pass()).then(result => result.commit()), 5); + setTimeout(() => a.try(b => b.pass()).then(result => result.commit()), 10); + setTimeout(() => a.try(b => b.pass()).then(result => result.commit()), 15); + setTimeout(() => a.end(), 20); + }).run(); + + t.is(result1.passed, true); +}); + +test('assertions within try-commit do not refresh the timeout', async t => { + const result = await ava(async a => { + a.timeout(15); + a.pass(); + + // Attempt by itself will refresh timeout, so it has to finish after + // timeout of the test in order to make sure that it does not refresh the + // timeout. However, if assert within attempt is called before test timeout + // expires and will refresh the timeout (which is faulty behavior), then + // the entire test will not fail by timeout. + const res = await a.try(async b => { + await delay(10); + b.is(1, 1); + await delay(10); + }); + res.commit(); + }).run(); + + t.false(result.passed); + t.ok(result.error); + t.match(result.error.message, /Test timeout exceeded/); + t.is(result.error.name, 'Error'); +}); + +test('try-commit inherits the test context', async t => { + const context = new ContextRef(); + const data = {foo: 'bar'}; + context.set(data); + const result = await ava(async a => { + const res = await a.try(b => { + b.pass(); + t.strictDeepEqual(b.context, data); + }); + await res.commit(); + }, context).run(); + + t.is(result.passed, true); +}); + +test('assigning context in try-commit does not affect parent', async t => { + const context = new ContextRef(); + const data = {foo: 'bar'}; + context.set(data); + const result = await ava(async a => { + t.strictDeepEqual(a.context, data); + const res = await a.try(b => { + b.pass(); + b.context = {bar: 'foo'}; + }); + res.commit(); + t.strictDeepEqual(a.context, data); + }, context).run(); + + t.is(result.passed, true); +}); + +test('do not run assertions outside of an active attempt', async t => { + const passing = await ava(async a => { + await a.try(() => {}); + a.pass(); + }).run(); + + t.false(passing.passed); + t.match(passing.error.message, /Assertion passed, but an attempt is pending. Use the attempt’s assertions instead/); + + const pending = await ava(async a => { + await a.try(() => {}); + await a.throwsAsync(Promise.reject(new Error(''))); + }).run(); + + t.false(pending.passed); + t.match(pending.error.message, /Assertion started, but an attempt is pending. Use the attempt’s assertions instead/); + + const failing = await ava(async a => { + await a.try(() => {}); + a.fail(); + }).run(); + + t.false(failing.passed); + t.match(failing.error.message, /Assertion failed, but an attempt is pending. Use the attempt’s assertions instead/); +}); diff --git a/test/test.js b/test/test.js index 61a766767..085105b11 100644 --- a/test/test.js +++ b/test/test.js @@ -9,63 +9,10 @@ const delay = require('delay'); const snapshotManager = require('../lib/snapshot-manager'); const Test = require('../lib/test'); const HelloMessage = require('./fixture/hello-message'); +const {ava} = require('./helper/ava-test'); const failingTestHint = 'Test was expected to fail, but succeeded, you should stop marking the test as failing'; -class ContextRef { - constructor() { - this.value = {}; - } - - get() { - return this.value; - } - - set(newValue) { - this.value = newValue; - } -} - -function ava(fn, contextRef) { - return new Test({ - contextRef: contextRef || new ContextRef(), - failWithoutAssertions: true, - fn, - metadata: {type: 'test', callback: false}, - title: 'test' - }); -} - -ava.failing = (fn, contextRef) => { - return new Test({ - contextRef: contextRef || new ContextRef(), - failWithoutAssertions: true, - fn, - metadata: {type: 'test', callback: false, failing: true}, - title: 'test.failing' - }); -}; - -ava.cb = (fn, contextRef) => { - return new Test({ - contextRef: contextRef || new ContextRef(), - failWithoutAssertions: true, - fn, - metadata: {type: 'test', callback: true}, - title: 'test.cb' - }); -}; - -ava.cb.failing = (fn, contextRef) => { - return new Test({ - contextRef: contextRef || new ContextRef(), - failWithoutAssertions: true, - fn, - metadata: {type: 'test', callback: true, failing: true}, - title: 'test.cb.failing' - }); -}; - test('run test', t => { return ava(a => { a.fail(); diff --git a/test/try-snapshot.js b/test/try-snapshot.js new file mode 100644 index 000000000..864533af1 --- /dev/null +++ b/test/try-snapshot.js @@ -0,0 +1,85 @@ +'use strict'; +require('../lib/chalk').set(); +require('../lib/worker/options').set({color: false}); + +const path = require('path'); +const {test} = require('tap'); +const snapshotManager = require('../lib/snapshot-manager'); +const Test = require('../lib/test'); +const ContextRef = require('../lib/context-ref'); + +function setup(title, manager, fn) { + return new Test({ + experiments: {tryAssertion: true}, + fn, + failWithoutAssertions: true, + metadata: {type: 'test', callback: false}, + contextRef: new ContextRef(), + registerUniqueTitle: () => true, + title, + compareTestSnapshot: options => manager.compare(options) + }); +} + +test(async t => { + // Set to `true` to update the snapshot, then run: + // "$(npm bin)"/tap --no-cov -R spec test/try-snapshot.js + // + // Ignore errors and make sure not to run tests with the `-b` (bail) option. + const updating = false; + + const projectDir = path.join(__dirname, 'fixture'); + const manager = snapshotManager.load({ + file: path.join(projectDir, 'try-snapshot.js'), + projectDir, + fixedLocation: null, + updating, + recordNewSnapshots: updating + }); + + await t.test('try-commit snapshots serially', async t => { + const ava = setup('serial', manager, async a => { + a.snapshot('hello'); + + const first = await a.try(t2 => { + t2.snapshot(true); + t2.snapshot({boo: 'far'}); + }); + first.commit(); + + const second = await a.try(t2 => { + t2.snapshot({foo: 'bar'}); + }); + second.commit(); + }); + + const result = await ava.run(); + t.true(result.passed); + }); + + await t.test('try-commit snapshots concurrently', async t => { + const ava = setup('concurrent', manager, async a => { + a.snapshot('hello'); + + const [first, second] = await Promise.all([ + a.try(t2 => { + t2.snapshot(true); + t2.snapshot({boo: 'far'}); + }), + a.try(t2 => { + t2.snapshot({foo: 'bar'}); + }) + ]); + first.commit(); + second.commit(); + }); + + const result = await ava.run(); + t.false(result.passed); + t.ok(result.error); + t.match(result.error.message, /not run concurrent snapshot assertions when using `t\.try\(\)`/); + t.is(result.error.name, 'Error'); + }); + + manager.save(); +}); diff --git a/test/ts-types/context.ts b/test/ts-types/context.ts index ff4962b94..c7f263cf4 100644 --- a/test/ts-types/context.ts +++ b/test/ts-types/context.ts @@ -15,3 +15,8 @@ test.beforeEach(t => { }); test('foo is bar', macro, 'bar'); + +anyTest('default context is unknown', t => { + // @ts-ignore + t.is(t.context.foo, 'bar') +}) diff --git a/test/ts-types/try-commit.ts b/test/ts-types/try-commit.ts new file mode 100644 index 000000000..8a4b9dd0d --- /dev/null +++ b/test/ts-types/try-commit.ts @@ -0,0 +1,86 @@ +import test, {ExecutionContext, Macro} from '../..'; + +{ + test('attempt', async t => { + const attempt = await t.try( + (u, a, b) => { + u.is(a.length, b); + }, + 'string', + 6 + ); + attempt.commit(); + }); + + test('attempt with title', async t => { + const attempt = await t.try( + 'attempt title', + (u, a, b) => { + u.is(a.length, b); + }, + 'string', + 6 + ); + attempt.commit(); + }); + + test('multiple attempts', async t => { + const attempts = [ + ...await t.try([tt => tt.pass(), tt => tt.pass()]), + ...await t.try('title', [tt => tt.pass(), tt => tt.pass()]), + ]; + for (const attempt of attempts) { + attempt.commit(); + } + }); +} + +{ + const lengthCheck = (t: ExecutionContext, a: string, b: number) => { + t.is(a.length, b); + }; + + test('attempt with helper', async t => { + const attempt = await t.try(lengthCheck, 'string', 6); + attempt.commit(); + }); + + test('attempt with title', async t => { + const attempt = await t.try(lengthCheck, 'string', 6); + attempt.commit(); + }); +} + +{ + test('all possible variants to pass to t.try', async t => { + // no params + t.try(tt => tt.pass()); + /* fails as expected */ // t.try([]); + t.try([tt => tt.pass()]); + t.try([tt => tt.pass(), tt => tt.fail()]); + + t.try('test', tt => tt.pass()); + /* fails as expected */ // t.try('test', []); + t.try('test', [tt => tt.pass()]); + t.try('test', [tt => tt.pass(), tt => tt.fail()]); + + // some params + t.try((tt, a, b) => tt.is(a.length, b), 'hello', 5); + /* fails as expected */ // t.try([], 'hello', 5); + t.try([(tt, a, b) => tt.is(a.length, b)], 'hello', 5); + t.try([(tt, a, b) => tt.is(a.length, b), (tt, a, b) => tt.is(a.slice(b), '')], 'hello', 5); + + t.try('test', (tt, a, b) => tt.is(a.length, b), 'hello', 5); + /* fails as expected */ // t.try('test', [], 'hello', 5); + t.try('test', [(tt, a, b) => tt.is(a.length, b)], 'hello', 5); + t.try('test', [(tt, a, b) => tt.is(a.length, b), (tt, a, b) => tt.is(a.slice(b), '')], 'hello', 5); + + // macro with title + const macro1: Macro<[string, number]> = (tt, a, b) => tt.is(a.length, b); + macro1.title = (title, a, b) => `${title ? `${title} `: ''}str: "${a}" with len: "${b}"`; + const macro2: Macro<[string, number]> = (tt, a, b) => tt.is(a.slice(b), ''); + + t.try([macro1, macro2], 'hello', 5); + t.try('title', [macro1, macro2], 'hello', 5); + }); +}