fix: report expected failures as passed

Author: jembezmamy

src/generate-report.ts

@@ -184,7 +184,10 @@ class GenerateCtrfReport implements Reporter {
   ): void {
     const test: CtrfTest = {
       name: testCase.title,
-      status: this.mapPlaywrightStatusToCtrf(testResult.status),
+      status:
+        testResult.status === testCase.expectedStatus
+          ? 'passed'
+          : this.mapPlaywrightStatusToCtrf(testResult.status),
       duration: testResult.duration,
     }
 
@@ -217,8 +220,9 @@ class GenerateCtrfReport implements Reporter {
         this.extractMetadata(testResult)?.name !== undefined ||
         this.extractMetadata(testResult)?.version !== undefined
       )
-        test.browser = `${this.extractMetadata(testResult)
-          ?.name} ${this.extractMetadata(testResult)?.version}`
+        test.browser = `${
+          this.extractMetadata(testResult)?.name
+        } ${this.extractMetadata(testResult)?.version}`
       test.attachments = this.filterValidAttachments(testResult.attachments)
       test.stdout = testResult.stdout.map((item) =>
         Buffer.isBuffer(item) ? item.toString() : String(item)

tests/dummy-suites/failed-test-suite.ts

@@ -31,9 +31,9 @@ export const createFailedTestSuite = (): Suite => {
     stderr: [],
   }
 
-  const testCase: TestCase = {
+  const failedTestCase: TestCase = {
     title: 'should validate the expected condition',
-    id: 'test-id-123',
+    id: 'test-id-1',
     annotations: [],
     expectedStatus: 'passed',
     timeout: 30000,
@@ -54,6 +54,26 @@ export const createFailedTestSuite = (): Suite => {
     retries: 0,
   }
 
+  const passedTestCase: TestCase = {
+    title: 'should fail as expected',
+    id: 'test-id-2',
+    annotations: [],
+    expectedStatus: 'failed',
+    timeout: 30000,
+    results: [testResult],
+    location: {
+      file: 'test-file.spec.ts',
+      line: 42,
+      column: 3,
+    },
+    parent: undefined as any, // Will be set later
+    outcome: () => 'expected',
+    ok: () => true,
+    titlePath: () => ['Failed Test Suite', 'should fail as expected'],
+    repeatEachIndex: 0,
+    retries: 0,
+  }
+
   const suite: Suite = {
     title: 'Failed Test Suite',
     titlePath: () => ['Failed Test Suite'],
@@ -78,12 +98,13 @@ export const createFailedTestSuite = (): Suite => {
       testMatch: [],
       snapshotDir: './snapshots',
     }),
-    allTests: () => [testCase],
-    tests: [testCase],
+    allTests: () => [failedTestCase, passedTestCase],
+    tests: [failedTestCase, passedTestCase],
     suites: [],
   }
 
-  testCase.parent = suite
+  failedTestCase.parent = suite
+  passedTestCase.parent = suite
 
   return suite
 }

tests/failed-tests.spec.ts

@@ -28,10 +28,17 @@ describe('Failed Tests', () => {
     const reportJsonContent = mockedFs.writeFileSync.mock.calls[0][1] as string
     const parsedReport: CtrfReport = JSON.parse(reportJsonContent)
 
-    expect(parsedReport.results.tests).toHaveLength(1)
+    expect(parsedReport.results.tests).toHaveLength(2)
+    expect(parsedReport.results.tests[0].status).toBe('failed')
+    expect(parsedReport.results.tests[0].rawStatus).toBe('failed')
     expect(parsedReport.results.tests[0].status).toBe('failed')
     expect(parsedReport.results.tests[0].message).toBe('test-error-message')
     expect(parsedReport.results.tests[0].trace).toBe('test-error-stack')
     expect(parsedReport.results.tests[0].snippet).toBe('test-error-snippet')
+    expect(parsedReport.results.tests[1].status).toBe('passed')
+    expect(parsedReport.results.tests[1].rawStatus).toBe('failed')
+    expect(parsedReport.results.tests[1].message).toBe('test-error-message')
+    expect(parsedReport.results.tests[1].trace).toBe('test-error-stack')
+    expect(parsedReport.results.tests[1].snippet).toBe('test-error-snippet')
   })
 })