test_runner: support expecting a test-case to fail

Co-Authored-By: Alejandro Espa <[email protected]>

Author: JakobJingleheimer

lib/internal/test_runner/harness.js

@@ -377,7 +377,7 @@ function runInParentContext(Factory) {
 
     return run(name, options, fn, overrides);
   };
-  ArrayPrototypeForEach(['skip', 'todo', 'only'], (keyword) => {
+  ArrayPrototypeForEach(['fail', 'skip', 'todo', 'only'], (keyword) => {
     test[keyword] = (name, options, fn) => {
       const overrides = {
         __proto__: null,

lib/internal/test_runner/reporter/tap.js

@@ -33,12 +33,12 @@ async function * tapReporter(source) {
   for await (const { type, data } of source) {
     switch (type) {
       case 'test:fail': {
-        yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo);
+        yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.fail, data.skip, data.todo);
         const location = data.file ? `${data.file}:${data.line}:${data.column}` : null;
         yield reportDetails(data.nesting, data.details, location);
         break;
       } case 'test:pass':
-        yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo);
+        yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.expectFail, data.skip, data.todo);
         yield reportDetails(data.nesting, data.details, null);
         break;
       case 'test:plan':
@@ -65,18 +65,23 @@ async function * tapReporter(source) {
   }
 }
 
-function reportTest(nesting, testNumber, status, name, skip, todo) {
+function reportTest(nesting, testNumber, status, name, fail, skip, todo) {
   let line = `${indent(nesting)}${status} ${testNumber}`;
 
   if (name) {
     line += ` ${tapEscape(`- ${name}`)}`;
   }
-
+console.trace('fail:', fail)
   if (skip !== undefined) {
     line += ` # SKIP${typeof skip === 'string' && skip.length ? ` ${tapEscape(skip)}` : ''}`;
   } else if (todo !== undefined) {
     line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`;
   }
+  // `skip` should trump, and `todo` + `fail` is a mistake, but it will already fail, so nothing to
+  // do when this is after `todo`.
+  else if (fail !== undefined) {
+    line += ` # EXPECTED FAIL${typeof fail === 'string' && fail.length ? ` ${tapEscape(fail)}` : ''}`;
+  }
 
   line += '\n';
 

lib/internal/test_runner/test.js

@@ -351,6 +351,11 @@ class TestContext {
     this.#test.runOnlySubtests = !!value;
   }
 
+  fail(err) {
+    console.log('err:', err);
+    this.#test.fail(err);
+  }
+
   skip(message) {
     this.#test.skip(message);
   }
@@ -507,7 +512,7 @@ class Test extends AsyncResource {
     super('Test');
 
     let { fn, name, parent } = options;
-    const { concurrency, entryFile, loc, only, timeout, todo, skip, signal, plan } = options;
+    const { concurrency, entryFile, fail, loc, only, timeout, todo, skip, signal, plan } = options;
 
     if (typeof fn !== 'function') {
       fn = noop;
@@ -646,6 +651,7 @@ class Test extends AsyncResource {
     this.plan = null;
     this.expectedAssertions = plan;
     this.cancelled = false;
+    this.expectFail = fail !== undefined && fail !== false;
     this.skipped = skip !== undefined && skip !== false;
     this.isTodo = (todo !== undefined && todo !== false) || this.parent?.isTodo;
     this.startTime = null;
@@ -949,7 +955,12 @@ class Test extends AsyncResource {
       return;
     }
 
-    this.passed = false;
+    if (this.expectFail === true) {
+      this.passed = true;
+    } else {
+      this.passed = false;
+    }
+
     this.error = err;
   }
 

test/fixtures/test-runner/output/describe_it.js

@@ -5,6 +5,22 @@ const { describe, it, test } = require('node:test');
 const util = require('util');
 
 
+it.fail('sync expect to fail', () => {
+  throw new Error('should pass');
+});
+
+it('sync expect to fail', { fail: true }, () => {
+  throw new Error('should pass');
+});
+
+it.fail('sync expect to fail', async () => {
+  throw new Error('should pass');
+});
+
+it('sync expect to fail', { fail: true }, async () => {
+  throw new Error('should pass');
+});
+
 it.todo('sync pass todo', () => {
 
 });
@@ -16,13 +32,21 @@ it.todo('sync todo', () => {
   throw new Error('should not count as a failure');
 });
 
+it.todo('sync todo with expect fail', { fail: true }, () => {
+  throw new Error('should not count as an expected failure');
+});
+
 it('sync todo with message', { todo: 'this is a failing todo' }, () => {
   throw new Error('should not count as a failure');
 });
 
 it.skip('sync skip pass', () => {
 });
 
+it.skip('sync skip expect fail', { fail: true }, () => {
+  throw new Error('should not fail');
+});
+
 it('sync skip pass with message', { skip: 'this is skipped' }, () => {
 });