fix: Use custom reporter to flip expected failures

To verify that the new `fail` matcher works correctly, we need a way to
check if the tests failed as expected or not.

Typically one would use `test.failing` for this purpose, but that only
works if the test threw an exception (e.g. JestException). The `fail`
matcher does not do this so that it can still work inside `catch`
blocks.

Instead, we have to hook into the test reporting and mutate the test
results for our expected test failures prior to usage by other test
reporters.

This commit creates a custom test reporter which detects tests run in
the file `fail.test.js` (yes the name is hard-coded) and flips the
results of tests inside a `.fail` describe block (i.e. our expected
failures).
 - If the tests failed (expected) we flip the result to a pass.
 - If the tests passed (unexpected) we flip the result to a fail.

The custom reporter also handles the logic for updating the counts for
failing test suites so that later reporters reflect the actual test
results correctly.

Author: thehale

package.json

@@ -103,6 +103,10 @@
     "watchPlugins": [
       "jest-watch-typeahead/filename",
       "jest-watch-typeahead/testname"
+    ],
+    "reporters": [
+      "<rootDir>/test/reporters/ExceptionlessExpectedFailureReporter.js",
+      "default"
     ]
   },
   "babel": {

test/matchers/fail.test.js

@@ -1,15 +1,14 @@
 import * as matcher from 'src/matchers/fail';
 
 expect.extend(matcher);
-
 describe('.fail', () => {
-  xtest('fails without message', () => {
+  test('fails without message', () => {
     expect().fail(); // This should fail!
   });
-  xtest('fails with message', () => {
+  test('fails with message', () => {
     expect().fail('This should fail!');
   });
-  xtest('fails when invoked in a try/catch', () => {
+  test('fails when invoked in a try/catch', () => {
     try {
       expect().fail();
     } catch (error) {

test/reporters/ExceptionlessExpectedFailureReporter.js

@@ -0,0 +1,86 @@
+/**
+ * Flips the test results for fail.test.js > .fail > <test cases>
+ */
+class ExceptionlessExpectedFailureReporter {
+  constructor(globalConfig, reporterOptions, reporterContext) {
+    this._globalConfig = globalConfig;
+    this._options = reporterOptions;
+    this._context = reporterContext;
+  }
+  onTestCaseResult(test, testCaseResult) {
+    this._processTestCaseResult(testCaseResult);
+  }
+  onTestFileResult(test, testResult, results) {
+    if (this._suitePathEndsWith(testResult, 'fail.test.js')) {
+      this._processTestResults(results);
+    }
+  }
+  _suitePathEndsWith(testSuiteResult, value) {
+    return testSuiteResult.testFilePath.endsWith('fail.test.js');
+  }
+  _processTestResults(results) {
+    for (let testSuiteResult of results.testResults) {
+      if (this._suitePathEndsWith(testSuiteResult, 'fail.test.js')) {
+        let switchedToFailing = 0;
+        let switchedToPassing = 0;
+        for (let testCaseResult of testSuiteResult.testResults) {
+          const processResult = this._processTestCaseResult(testCaseResult);
+          if (processResult === 'switch-to-failing') switchedToFailing++;
+          if (processResult === 'switch-to-passing') switchedToPassing++;
+        }
+        const originalFailureCount = testSuiteResult.numFailingTests;
+        testSuiteResult.numFailingTests += switchedToFailing - switchedToPassing;
+        results.numFailedTests += switchedToFailing - switchedToPassing;
+        testSuiteResult.numPassingTests += switchedToPassing - switchedToFailing;
+        results.numPassedTests += switchedToPassing - switchedToFailing;
+        if (originalFailureCount === switchedToPassing) {
+          testSuiteResult.failureMessage = '';
+          results.numFailedTestSuites -= 1;
+          results.numPassedTestSuites += 1;
+          if (results.numFailedTestSuites === 0) results.success = true;
+          console.log('marking failing test suite as passing', testSuiteResult.testFilePath);
+        }
+      }
+    }
+  }
+
+  _processTestCaseResult(testCaseResult) {
+    if (this._hasDotFailAncestor(testCaseResult)) {
+      if (testCaseResult.status === 'failed') {
+        this._markPassing(testCaseResult);
+        return 'switch-to-passing';
+      } else if (testCaseResult.status === 'passed') {
+        this._markFailing(testCaseResult);
+        return 'switch-to-failing';
+      }
+    }
+    return 'unchanged';
+  }
+  _hasDotFailAncestor(result) {
+    return result.ancestorTitles.length > 0 && result.ancestorTitles[0] === '.fail';
+  }
+  _markPassing(result) {
+    result.status = 'passed';
+    result.failureDetails = [];
+    result.failureMessages = [];
+    result.numPassingAsserts = 1;
+  }
+  _markFailing(result) {
+    const message = `${result.fullName} was expected to fail, but did not.`;
+    result.status = 'failed';
+    result.failureDetails = [
+      {
+        matcherResult: {
+          pass: false,
+          message: message,
+        },
+        message: message,
+        stack: `${message}\n\tNo stack trace.\n\tThis is a placeholder message generated inside ExceptionlessExpectedFailureReporter`,
+      },
+    ];
+    result.failureMessages = [message];
+    result.numPassingAsserts = 0;
+  }
+}
+
+module.exports = ExceptionlessExpectedFailureReporter;