Implement experimental t.try() assertion

Author: qlonik

docs/06-configuration.md

@@ -164,7 +164,7 @@ AVA has a minimum depth of `3`.
 
 ## Experiments
 
-From time to time, AVA will implement experimental features. These may change or be removed at any time, not just when there's a new major version. You can opt-in to such a feature by enabling it in the `nonSemVerExperiments` configuration.
+From time to time, AVA will implement experimental features. These may change or be removed at any time, not just when there's a new major version. You can opt in to such a feature by enabling it in the `nonSemVerExperiments` configuration.
 
 `ava.config.js`:
 ```js
@@ -175,6 +175,15 @@ export default {
 };
 ```
 
-There are currently no such features available.
+You can opt in to the new `t.try()` assertion by specifying `tryAssertion`:
+
+`ava.config.js`:
+```js
+export default {
+	nonSemVerExperiments: {
+		tryAssertion: true
+	}
+};
+```
 
 [CLI]: ./05-command-line.md

index.d.ts

@@ -25,6 +25,13 @@ export type ThrowsExpectation = {
 	name?: string;
 };
 
+export type CommitDiscardOptions = {
+	/**
+	 * Whether the logs should be included in those of the parent test.
+	 */
+	retainLogs?: boolean
+}
+
 /** Options that can be passed to the `t.snapshot()` assertion. */
 export type SnapshotOptions = {
 	/** If provided and not an empty string, used to select the snapshot to compare the `expected` value against. */
@@ -363,6 +370,7 @@ export interface ExecutionContext<Context = unknown> extends Assertions {
 	log: LogFn;
 	plan: PlanFn;
 	timeout: TimeoutFn;
+	try: TryFn<Context>;
 }
 
 export interface LogFn {
@@ -392,6 +400,69 @@ export interface TimeoutFn {
 	(ms: number): void;
 }
 
+export interface TryFn<Context = unknown> {
+	/**
+	 * Requires opt-in. Attempt to run some assertions. The result must be explicitly committed or discarded or else
+	 * the test will fail. A macro may be provided. The title may help distinguish attempts from
+	 * one another.
+	 */
+	<Args extends any[]>(title: string, fn: EitherMacro<Args, Context>, ...args: Args): Promise<TryResult>;
+
+	/**
+	* Requires opt-in. Attempt to run some assertions. The result must be explicitly committed or discarded or else
+	 * the test will fail. A macro may be provided. The title may help distinguish attempts from
+	 * one another.
+	 */
+	<Args extends any[]>(title: string, fn: [EitherMacro<Args, Context>, ...EitherMacro<Args, Context>[]], ...args: Args): Promise<TryResult[]>;
+
+	/**
+	* Requires opt-in. Attempt to run some assertions. The result must be explicitly committed or discarded or else
+	* the test will fail. A macro may be provided.
+	*/
+	<Args extends any[]>(fn: EitherMacro<Args, Context>, ...args: Args): Promise<TryResult>;
+
+	/**
+	* Requires opt-in. Attempt to run some assertions. The result must be explicitly committed or discarded or else
+	* the test will fail. A macro may be provided.
+	*/
+	<Args extends any[]>(fn: [EitherMacro<Args, Context>, ...EitherMacro<Args, Context>[]], ...args: Args): Promise<TryResult[]>;
+}
+
+export interface AssertionError extends Error {}
+
+export interface TryResult {
+	/**
+	* Title of the attempt, helping you tell attempts aparts.
+	*/
+	title: string;
+
+	/**
+	* Indicates whether all assertions passed, or at least one failed.
+	*/
+	passed: boolean;
+
+	/**
+	* Errors raised for each failed assertion.
+	*/
+	errors: AssertionError[];
+
+	/**
+	 * Logs created during the attempt using `t.log()`. Contains formatted values.
+	 */
+	logs: string[];
+
+	/**
+	 * Commit the attempt. Counts as one assertion for the plan count. If the
+	 * attempt failed, calling this will also cause your test to fail.
+	 */
+	commit(options?: CommitDiscardOptions): void;
+
+	/**
+	 * Discard the attempt.
+	 */
+	discard(options?: CommitDiscardOptions): void;
+}
+
 /** The `t` value passed to implementations for tests & hooks declared with the `.cb` modifier. */
 export interface CbExecutionContext<Context = unknown> extends ExecutionContext<Context> {
 	/**

lib/load-config.js

@@ -6,7 +6,7 @@ const pkgConf = require('pkg-conf');
 
 const NO_SUCH_FILE = Symbol('no ava.config.js file');
 const MISSING_DEFAULT_EXPORT = Symbol('missing default export');
-const EXPERIMENTS = new Set([]);
+const EXPERIMENTS = new Set(['tryAssertion']);
 
 function loadConfig({configFile, resolveFrom = process.cwd(), defaults = {}} = {}) { // eslint-disable-line complexity
 	let packageConf = pkgConf.sync('ava', {cwd: resolveFrom});

lib/parse-test-args.js

@@ -0,0 +1,15 @@
+'use strict';
+function parseTestArgs(args) {
+	const rawTitle = typeof args[0] === 'string' ? args.shift() : undefined;
+	const receivedImplementationArray = Array.isArray(args[0]);
+	const implementations = receivedImplementationArray ? args.shift() : args.splice(0, 1);
+
+	const buildTitle = implementation => {
+		const title = implementation.title ? implementation.title(rawTitle, ...args) : rawTitle;
+		return {title, isSet: typeof title !== 'undefined', isValid: typeof title === 'string', isEmpty: !title};
+	};
+
+	return {args, buildTitle, implementations, rawTitle, receivedImplementationArray};
+}
+
+module.exports = parseTestArgs;

lib/runner.js

@@ -3,6 +3,7 @@ const Emittery = require('emittery');
 const matcher = require('matcher');
 const ContextRef = require('./context-ref');
 const createChain = require('./create-chain');
+const parseTestArgs = require('./parse-test-args');
 const snapshotManager = require('./snapshot-manager');
 const serializeError = require('./serialize-error');
 const Runnable = require('./test');
@@ -11,6 +12,7 @@ class Runner extends Emittery {
 	constructor(options = {}) {
 		super();
 
+		this.experiments = options.experiments || {};
 		this.failFast = options.failFast === true;
 		this.failWithoutAssertions = options.failWithoutAssertions !== false;
 		this.file = options.file;
@@ -39,12 +41,21 @@ class Runner extends Emittery {
 		};
 
 		const uniqueTestTitles = new Set();
+		this.registerUniqueTitle = title => {
+			if (uniqueTestTitles.has(title)) {
+				return false;
+			}
+
+			uniqueTestTitles.add(title);
+			return true;
+		};
+
 		let hasStarted = false;
 		let scheduledStart = false;
 		const meta = Object.freeze({
 			file: options.file
 		});
-		this.chain = createChain((metadata, args) => { // eslint-disable-line complexity
+		this.chain = createChain((metadata, testArgs) => { // eslint-disable-line complexity
 			if (hasStarted) {
 				throw new Error('All tests and hooks must be declared synchronously in your test file, and cannot be nested within other tests or hooks.');
 			}
@@ -57,40 +68,33 @@ class Runner extends Emittery {
 				});
 			}
 
-			const specifiedTitle = typeof args[0] === 'string' ?
-				args.shift() :
-				undefined;
-			const implementations = Array.isArray(args[0]) ?
-				args.shift() :
-				args.splice(0, 1);
+			const {args, buildTitle, implementations, rawTitle} = parseTestArgs(testArgs);
 
 			if (metadata.todo) {
 				if (implementations.length > 0) {
 					throw new TypeError('`todo` tests are not allowed to have an implementation. Use `test.skip()` for tests with an implementation.');
 				}
 
-				if (specifiedTitle === undefined || specifiedTitle === '') {
+				if (!rawTitle) { // Either undefined or a string.
 					throw new TypeError('`todo` tests require a title');
 				}
 
-				if (uniqueTestTitles.has(specifiedTitle)) {
-					throw new Error(`Duplicate test title: ${specifiedTitle}`);
-				} else {
-					uniqueTestTitles.add(specifiedTitle);
+				if (!this.registerUniqueTitle(rawTitle)) {
+					throw new Error(`Duplicate test title: ${rawTitle}`);
 				}
 
 				if (this.match.length > 0) {
 					// --match selects TODO tests.
-					if (matcher([specifiedTitle], this.match).length === 1) {
+					if (matcher([rawTitle], this.match).length === 1) {
 						metadata.exclusive = true;
 						this.runOnlyExclusive = true;
 					}
 				}
 
-				this.tasks.todo.push({title: specifiedTitle, metadata});
+				this.tasks.todo.push({title: rawTitle, metadata});
 				this.emit('stateChange', {
 					type: 'declared-test',
-					title: specifiedTitle,
+					title: rawTitle,
 					knownFailing: false,
 					todo: true
 				});
@@ -100,15 +104,13 @@ class Runner extends Emittery {
 				}
 
 				for (const implementation of implementations) {
-					let title = implementation.title ?
-						implementation.title(specifiedTitle, ...args) :
-						specifiedTitle;
+					let {title, isSet, isValid, isEmpty} = buildTitle(implementation);
 
-					if (title !== undefined && typeof title !== 'string') {
+					if (isSet && !isValid) {
 						throw new TypeError('Test & hook titles must be strings');
 					}
 
-					if (title === undefined || title === '') {
+					if (isEmpty) {
 						if (metadata.type === 'test') {
 							throw new TypeError('Tests must have a title');
 						} else if (metadata.always) {
@@ -118,12 +120,8 @@ class Runner extends Emittery {
 						}
 					}
 
-					if (metadata.type === 'test') {
-						if (uniqueTestTitles.has(title)) {
-							throw new Error(`Duplicate test title: ${title}`);
-						} else {
-							uniqueTestTitles.add(title);
-						}
+					if (metadata.type === 'test' && !this.registerUniqueTitle(title)) {
+						throw new Error(`Duplicate test title: ${title}`);
 					}
 
 					const task = {
@@ -162,6 +160,7 @@ class Runner extends Emittery {
 			todo: false,
 			failing: false,
 			callback: false,
+			inline: false, // Set for attempt metadata created by `t.try()`
 			always: false
 		}, meta);
 	}
@@ -269,6 +268,7 @@ class Runner extends Emittery {
 	async runHooks(tasks, contextRef, titleSuffix) {
 		const hooks = tasks.map(task => new Runnable({
 			contextRef,
+			experiments: this.experiments,
 			failWithoutAssertions: false,
 			fn: task.args.length === 0 ?
 				task.implementation :
@@ -309,14 +309,16 @@ class Runner extends Emittery {
 			// Only run the test if all `beforeEach` hooks passed.
 			const test = new Runnable({
 				contextRef,
+				experiments: this.experiments,
 				failWithoutAssertions: this.failWithoutAssertions,
 				fn: task.args.length === 0 ?
 					task.implementation :
 					t => task.implementation.apply(null, [t].concat(task.args)),
 				compareTestSnapshot: this.boundCompareTestSnapshot,
 				updateSnapshots: this.updateSnapshots,
 				metadata: task.metadata,
-				title: task.title
+				title: task.title,
+				registerUniqueTitle: this.registerUniqueTitle
 			});
 
 			const result = await this.runSingle(test);

lib/snapshot-manager.js

@@ -305,45 +305,64 @@ class Manager {
 	compare(options) {
 		const hash = md5Hex(options.belongsTo);
 		const entries = this.snapshotsByHash.get(hash) || [];
-		if (options.index > entries.length) {
-			throw new RangeError(`Cannot record snapshot ${options.index} for ${JSON.stringify(options.belongsTo)}, exceeds expected index of ${entries.length}`);
-		}
+		const snapshotBuffer = entries[options.index];
 
-		if (options.index === entries.length) {
+		if (!snapshotBuffer) {
 			if (!this.recordNewSnapshots) {
 				return {pass: false};
 			}
 
+			if (options.deferRecording) {
+				const record = this.deferRecord(hash, options);
+				return {pass: true, record};
+			}
+
 			this.record(hash, options);
 			return {pass: true};
 		}
 
-		const snapshotBuffer = entries[options.index];
 		const actual = concordance.deserialize(snapshotBuffer, concordanceOptions);
-
 		const expected = concordance.describe(options.expected, concordanceOptions);
 		const pass = concordance.compareDescriptors(actual, expected);
 
 		return {actual, expected, pass};
 	}
 
-	record(hash, options) {
+	deferRecord(hash, options) {
 		const descriptor = concordance.describe(options.expected, concordanceOptions);
-
-		this.hasChanges = true;
 		const snapshot = concordance.serialize(descriptor);
-		if (this.snapshotsByHash.has(hash)) {
-			this.snapshotsByHash.get(hash).push(snapshot);
-		} else {
-			this.snapshotsByHash.set(hash, [snapshot]);
-		}
-
 		const entry = formatEntry(options.label, descriptor);
-		if (this.reportEntries.has(options.belongsTo)) {
-			this.reportEntries.get(options.belongsTo).push(entry);
-		} else {
-			this.reportEntries.set(options.belongsTo, [entry]);
-		}
+
+		return () => { // Must be called in order!
+			this.hasChanges = true;
+
+			let snapshots = this.snapshotsByHash.get(hash);
+			if (!snapshots) {
+				snapshots = [];
+				this.snapshotsByHash.set(hash, snapshots);
+			}
+
+			if (options.index > snapshots.length) {
+				throw new RangeError(`Cannot record snapshot ${options.index} for ${JSON.stringify(options.belongsTo)}, exceeds expected index of ${snapshots.length}`);
+			}
+
+			if (options.index < snapshots.length) {
+				throw new RangeError(`Cannot record snapshot ${options.index} for ${JSON.stringify(options.belongsTo)}, already exists`);
+			}
+
+			snapshots.push(snapshot);
+
+			if (this.reportEntries.has(options.belongsTo)) {
+				this.reportEntries.get(options.belongsTo).push(entry);
+			} else {
+				this.reportEntries.set(options.belongsTo, [entry]);
+			}
+		};
+	}
+
+	record(hash, options) {
+		const record = this.deferRecord(hash, options);
+		record();
 	}
 
 	save() {