Isolate from other tests (#80)

Author: davidjgoss

CHANGELOG.md

@@ -9,6 +9,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Support Node.js 24 ([#67](https://github.com/cucumber/cucumber-node/pull/67))
 
+### Fixed
+- Fully isolate Cucumber tests from other tests for accurate reporting ([#80](https://github.com/cucumber/cucumber-node/pull/80))
+
 ### Removed
 - Drop support for Node.js 23 ([#67](https://github.com/cucumber/cucumber-node/pull/67))
 

README.md

@@ -149,10 +149,11 @@ Some Cucumber formatters are included as Node.js test reporters:
 - JUnit `--test-reporter=@cucumber/node/reporters/junit --test-reporter-destination=./TEST-cucumber.xml`
 - Message `--test-reporter=@cucumber/node/reporters/message --test-reporter-destination=./messages.ndjson`
 
-There are some caveats that apply when using these reporters (but not otherwise):
+For now, avoid using the `spec` reporter at the same time as one of the above reporters - because we're abusing the `diagnostic` channel to send messages to the reporter, it makes the `spec` output very noisy - we recommend the `dot` reporter instead.
 
-- *Don't mix Cucumber tests with other tests* - if you have non-Cucumber tests to run with `node --test`, you should do that in a separate run.
-- *Don't use the `spec` reporter at the same time* - because we're abusing the `diagnostic` channel to send messages to the reporter, it makes the `spec` output very noisy - we recommend the `dot` reporter instead.
+## Mixing tests
+
+You can execute Cucumber tests and normal JavaScript tests in the same test run - cucumber-node won't interfere with the other tests. But the reporters mentioned above will only report on the Cucumber tests in your run.
 
 ## Limitations
 

package-lock.json

@@ -41,6 +41,7 @@
   },
   "devDependencies": {
     "@cucumber/compatibility-kit": "^18.0.3",
+    "@cucumber/query": "^13.5.0",
     "@eslint/compat": "^1.2.5",
     "@eslint/eslintrc": "^3.2.0",
     "@eslint/js": "^9.18.0",

package.json

@@ -1,6 +1,13 @@
+import path from 'node:path'
 import { TestEvent } from 'node:test/reporters'
 
-import { Envelope, TestRunStarted, TestStepFinished } from '@cucumber/messages'
+import {
+  Envelope,
+  TestRunStarted,
+  TestStepFinished,
+  TestStepResult,
+  TestStepResultStatus,
+} from '@cucumber/messages'
 
 import { makeId } from '../makeId.js'
 import { makeTimestamp } from '../makeTimestamp.js'
@@ -13,11 +20,11 @@ export async function* generateEnvelopes(
 ): AsyncGenerator<Envelope> {
   yield { meta }
 
+  const nodeFailOrPassEvents: Array<TestFail | TestPass> = []
+  const testStepFinishedMessages: Array<TestStepFinished> = []
+  const nonSuccessTestCaseStartedIds: Array<string> = []
   const testRunEnvelopes: Array<Envelope> = []
-  const testStepFinishedOrder: Array<TestStepFinished> = []
-  const testStepFinishEvents: Array<TestFail | TestPass> = []
 
-  let success: boolean = false
   const testRunStarted: TestRunStarted = {
     id: makeId(),
     timestamp: makeTimestamp(),
@@ -27,15 +34,12 @@ export async function* generateEnvelopes(
     switch (event.type) {
       case 'test:fail':
       case 'test:pass':
-        if (event.data.nesting === 2) {
-          testStepFinishEvents.push(event.data)
+        if (isFromHere(event.data) && event.data.nesting === 2) {
+          nodeFailOrPassEvents.push(event.data)
         }
         break
-      case 'test:summary':
-        success = event.data.success
-        break
       case 'test:diagnostic':
-        if (isEnvelope(event.data.message)) {
+        if (isFromHere(event.data) && isEnvelope(event.data.message)) {
           const envelope = fromPrefixed(event.data.message)
           for (const key of Object.keys(envelope) as ReadonlyArray<keyof Envelope>) {
             switch (key) {
@@ -52,10 +56,13 @@ export async function* generateEnvelopes(
               case 'testStepFinished':
                 {
                   const testStepFinished = envelope.testStepFinished!
-                  testStepFinishedOrder.push(testStepFinished)
+                  testStepFinishedMessages.push(testStepFinished)
                   testStepFinished.testStepResult = mapTestStepResult(
-                    testStepFinishEvents.at(testStepFinishedOrder.indexOf(testStepFinished))
+                    nodeFailOrPassEvents.at(testStepFinishedMessages.indexOf(testStepFinished))
                   )
+                  if (isNonSuccess(testStepFinished.testStepResult)) {
+                    nonSuccessTestCaseStartedIds.push(testStepFinished.testCaseStartedId)
+                  }
                   testRunEnvelopes.push(envelope)
                 }
                 break
@@ -74,7 +81,7 @@ export async function* generateEnvelopes(
   const testRunFinished = {
     testRunStartedId: testRunStarted.id,
     timestamp: makeTimestamp(),
-    success,
+    success: nonSuccessTestCaseStartedIds.length === 0,
   }
 
   yield { testRunStarted }
@@ -84,10 +91,22 @@ export async function* generateEnvelopes(
   yield { testRunFinished }
 }
 
+function isFromHere(testLocationInfo: TestLocationInfo) {
+  return testLocationInfo.file?.startsWith(path.join(import.meta.dirname, '..'))
+}
+
 function isEnvelope(data: string) {
   return data.startsWith(PROTOCOL_PREFIX)
 }
 
 function fromPrefixed(data: string): Envelope {
   return JSON.parse(data.substring(PROTOCOL_PREFIX.length))
 }
+
+function isNonSuccess(testStepResult: TestStepResult) {
+  return ![
+    TestStepResultStatus.UNKNOWN,
+    TestStepResultStatus.PASSED,
+    TestStepResultStatus.SKIPPED,
+  ].includes(testStepResult.status)
+}

src/reporters/generateEnvelopes.ts

@@ -42,23 +42,6 @@ const UNSUPPORTED = [
   'retry', // node:test doesnt support retries yet
 ]
 
-const OVERRIDES: Record<string, (actual: ReadonlyArray<Envelope>) => ReadonlyArray<Envelope>> = {
-  pending: (actual) => {
-    // node:test treats pending (aka todo) as a pass
-    return actual.map((envelope) => {
-      if (envelope.testRunFinished) {
-        return {
-          testRunFinished: {
-            ...envelope.testRunFinished,
-            success: false,
-          },
-        }
-      }
-      return envelope
-    })
-  },
-}
-
 describe('Cucumber Compatibility Kit', () => {
   const ndjsonPaths = globSync('node_modules/@cucumber/compatibility-kit/features/**/*.ndjson')
   for (const ndjsonPath of ndjsonPaths) {
@@ -82,10 +65,7 @@ describe('Cucumber Compatibility Kit', () => {
       const [actualOutput] = await harness.run('@cucumber/node/reporters/message')
       const expectedOutput = await readFile(ndjsonPath, { encoding: 'utf-8' })
 
-      let actualEnvelopes = parseEnvelopes(actualOutput)
-      if (OVERRIDES[name]) {
-        actualEnvelopes = OVERRIDES[name](actualEnvelopes)
-      }
+      const actualEnvelopes = parseEnvelopes(actualOutput)
       const expectedEnvelopes = parseEnvelopes(expectedOutput)
 
       // first assert on the order and type of messages

test/cck/cck.spec.ts

@@ -2,6 +2,7 @@ import { stripVTControlCharacters } from 'node:util'
 import { expect } from 'chai'
 import { makeTestHarness } from '../utils.js'
 import path from 'node:path'
+import { TestStepResultStatus } from '@cucumber/messages'
 
 describe('Reporters', () => {
   describe('spec', () => {
@@ -61,6 +62,47 @@ Given('a step', () => {})`
     })
   })
 
+  describe('message', () => {
+    it('only factors cucumber tests into results', async () => {
+      const harness = await makeTestHarness()
+      await harness.writeFile(
+        'features/first.feature',
+        `Feature:
+  Scenario:
+    Given a step
+`
+      )
+      await harness.writeFile(
+        'features/steps.js',
+        `import { Given } from '@cucumber/node'
+Given('a step', () => {})
+`
+      )
+      await harness.writeFile(
+        'example.test.mjs',
+        `import test from 'node:test'
+test('top level', (t) => {
+    test('next level', (t1) => {
+        t1.test('failing test', (t2) => {
+            t2.assert.strictEqual(1, 2)
+        })
+    })
+})
+`
+      )
+
+      const query = await harness.collectMessages()
+
+      expect(query.findAllTestCaseStarted().length).to.eq(1)
+      expect(
+        query
+          .findAllTestCaseStarted()
+          .map((testCaseStarted) => query.findMostSevereTestStepResultBy(testCaseStarted)?.status)
+      ).to.deep.eq([TestStepResultStatus.PASSED])
+      expect(query.findTestRunFinished()?.success).to.be.true
+    })
+  })
+
   describe('junit', () => {
     it('outputs a junit xml report', async () => {
       const harness = await makeTestHarness()

test/integration/reporters.spec.ts

@@ -2,6 +2,8 @@ import { copyFile, cp, mkdir, mkdtemp, symlink, writeFile } from 'node:fs/promis
 import path from 'node:path'
 import { tmpdir } from 'node:os'
 import { exec } from 'node:child_process'
+import { Query } from '@cucumber/query'
+import { Envelope } from '@cucumber/messages'
 
 class TestHarness {
   constructor(private readonly tempDir: string) {}
@@ -34,6 +36,7 @@ class TestHarness {
           `--test-reporter-destination=stdout`,
           ...extraArgs,
           `--test`,
+          `"*.test.mjs"`,
           `"features/**/*.feature"`,
           `"features/**/*.feature.md"`,
         ].join(' '),
@@ -46,6 +49,17 @@ class TestHarness {
       )
     })
   }
+
+  async collectMessages(): Promise<Query> {
+    const query = new Query()
+    const [output] = await this.run('@cucumber/node/reporters/message')
+    output
+      .trim()
+      .split('\n')
+      .map((line) => JSON.parse(line) as Envelope)
+      .forEach((envelope) => query.update(envelope))
+    return query
+  }
 }
 
 export async function makeTestHarness() {