From d0474eebea2317a04af5232b5607f2a635611622 Mon Sep 17 00:00:00 2001 From: Joseph Hale Date: Mon, 9 Oct 2023 12:29:25 -0700 Subject: [PATCH 1/2] Allow `fail` to work in `try/catch` --- src/matchers/fail.js | 1 + test/matchers/__snapshots__/fail.test.js.snap | 5 ----- test/matchers/fail.test.js | 17 ++++++++++++----- 3 files changed, 13 insertions(+), 10 deletions(-) delete mode 100644 test/matchers/__snapshots__/fail.test.js.snap diff --git a/src/matchers/fail.js b/src/matchers/fail.js index 410f4146..d654c05a 100644 --- a/src/matchers/fail.js +++ b/src/matchers/fail.js @@ -1,4 +1,5 @@ export function fail(_, message) { + this.dontThrow(); return { pass: false, message: () => (message ? message : 'fails by .fail() assertion'), diff --git a/test/matchers/__snapshots__/fail.test.js.snap b/test/matchers/__snapshots__/fail.test.js.snap deleted file mode 100644 index 788a25ab..00000000 --- a/test/matchers/__snapshots__/fail.test.js.snap +++ /dev/null @@ -1,5 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`.fail fails with message 1`] = `"This shouldn't fail!"`; - -exports[`.fail fails without message 1`] = `"fails by .fail() assertion"`; diff --git a/test/matchers/fail.test.js b/test/matchers/fail.test.js index 7bfa1002..c8d0c1d9 100644 --- a/test/matchers/fail.test.js +++ b/test/matchers/fail.test.js @@ -3,11 +3,18 @@ import * as matcher from 'src/matchers/fail'; expect.extend(matcher); describe('.fail', () => { - test('fails without message', () => { - expect(() => expect().fail()).toThrowErrorMatchingSnapshot(); + xtest('fails without message', () => { + expect().fail(); // This should fail! }); - test('fails with message', () => { - expect(() => expect().fail("This shouldn't fail!")).toThrowErrorMatchingSnapshot(); + xtest('fails with message', () => { + expect().fail('This should fail!'); + }); + xtest('fails when invoked in a try/catch', () => { + try { + expect().fail(); + } catch (error) { + expect('this assertion').toBe('not checked'); + } }); }); @@ -16,6 +23,6 @@ describe('.not.fail', () => { expect().not.fail(); }); test('does not fail with message', () => { - expect().not.fail('this should fail!'); + expect().not.fail('this should not fail!'); }); }); From 01b1275d2507527b54f715811aee9b720caf5c69 Mon Sep 17 00:00:00 2001 From: Joseph Hale Date: Fri, 22 Dec 2023 18:59:07 -0700 Subject: [PATCH 2/2] fix: Use custom reporter to flip expected failures To verify that the new `fail` matcher works correctly, we need a way to check if the tests failed as expected or not. Typically one would use `test.failing` for this purpose, but that only works if the test threw an exception (e.g. JestException). The `fail` matcher does not do this so that it can still work inside `catch` blocks. Instead, we have to hook into the test reporting and mutate the test results for our expected test failures prior to usage by other test reporters. This commit creates a custom test reporter which detects tests run in the file `fail.test.js` (yes the name is hard-coded) and flips the results of tests inside a `.fail` describe block (i.e. our expected failures). - If the tests failed (expected) we flip the result to a pass. - If the tests passed (unexpected) we flip the result to a fail. The custom reporter also handles the logic for updating the counts for failing test suites so that later reporters reflect the actual test results correctly. --- package.json | 4 + test/matchers/fail.test.js | 7 +- .../ExceptionlessExpectedFailureReporter.js | 83 +++++++++++++++++++ 3 files changed, 90 insertions(+), 4 deletions(-) create mode 100644 test/reporters/ExceptionlessExpectedFailureReporter.js diff --git a/package.json b/package.json index 5c61a94c..ede9acc6 100644 --- a/package.json +++ b/package.json @@ -103,6 +103,10 @@ "watchPlugins": [ "jest-watch-typeahead/filename", "jest-watch-typeahead/testname" + ], + "reporters": [ + "/test/reporters/ExceptionlessExpectedFailureReporter.js", + "default" ] }, "babel": { diff --git a/test/matchers/fail.test.js b/test/matchers/fail.test.js index c8d0c1d9..9d88eb5e 100644 --- a/test/matchers/fail.test.js +++ b/test/matchers/fail.test.js @@ -1,15 +1,14 @@ import * as matcher from 'src/matchers/fail'; expect.extend(matcher); - describe('.fail', () => { - xtest('fails without message', () => { + test('fails without message', () => { expect().fail(); // This should fail! }); - xtest('fails with message', () => { + test('fails with message', () => { expect().fail('This should fail!'); }); - xtest('fails when invoked in a try/catch', () => { + test('fails when invoked in a try/catch', () => { try { expect().fail(); } catch (error) { diff --git a/test/reporters/ExceptionlessExpectedFailureReporter.js b/test/reporters/ExceptionlessExpectedFailureReporter.js new file mode 100644 index 00000000..20c31638 --- /dev/null +++ b/test/reporters/ExceptionlessExpectedFailureReporter.js @@ -0,0 +1,83 @@ +/** + * Flips the test results for fail.test.js > .fail > + */ +class ExceptionlessExpectedFailureReporter { + constructor(globalConfig, reporterOptions, reporterContext) { + this._globalConfig = globalConfig; + this._options = reporterOptions; + this._context = reporterContext; + } + onTestCaseResult(test, testCaseResult) { + this._processTestCaseResult(testCaseResult); + } + onTestFileResult(test, testResult, results) { + if (testResult.testFilePath.endsWith('fail.test.js')) { + this._processTestResults(results); + } + } + _processTestResults(results) { + for (let testSuiteResult of results.testResults) { + if (testSuiteResult.testFilePath.endsWith('fail.test.js')) { + let switchedToFailing = 0; + let switchedToPassing = 0; + for (let testCaseResult of testSuiteResult.testResults) { + const processResult = this._processTestCaseResult(testCaseResult); + if (processResult === 'switch-to-failing') switchedToFailing++; + if (processResult === 'switch-to-passing') switchedToPassing++; + } + const originalFailureCount = testSuiteResult.numFailingTests; + testSuiteResult.numFailingTests += switchedToFailing - switchedToPassing; + results.numFailedTests += switchedToFailing - switchedToPassing; + testSuiteResult.numPassingTests += switchedToPassing - switchedToFailing; + results.numPassedTests += switchedToPassing - switchedToFailing; + if (originalFailureCount === switchedToPassing) { + testSuiteResult.failureMessage = ''; + results.numFailedTestSuites -= 1; + results.numPassedTestSuites += 1; + if (results.numFailedTestSuites === 0) results.success = true; + console.log('marking failing test suite as passing', testSuiteResult.testFilePath); + } + } + } + } + + _processTestCaseResult(testCaseResult) { + if (this._hasDotFailAncestor(testCaseResult)) { + if (testCaseResult.status === 'failed') { + this._markPassing(testCaseResult); + return 'switch-to-passing'; + } else if (testCaseResult.status === 'passed') { + this._markFailing(testCaseResult); + return 'switch-to-failing'; + } + } + return 'unchanged'; + } + _hasDotFailAncestor(result) { + return result.ancestorTitles.length > 0 && result.ancestorTitles[0] === '.fail'; + } + _markPassing(result) { + result.status = 'passed'; + result.failureDetails = []; + result.failureMessages = []; + result.numPassingAsserts = 1; + } + _markFailing(result) { + const message = `${result.fullName} was expected to fail, but did not.`; + result.status = 'failed'; + result.failureDetails = [ + { + matcherResult: { + pass: false, + message: message, + }, + message: message, + stack: `${message}\n\tNo stack trace.\n\tThis is a placeholder message generated inside ExceptionlessExpectedFailureReporter`, + }, + ]; + result.failureMessages = [message]; + result.numPassingAsserts = 0; + } +} + +module.exports = ExceptionlessExpectedFailureReporter;