Merge pull request #1 from cucumber/master

Merging from cucumber/cucmber

Author: eman2673

.eslintrc.yml

@@ -1,7 +1,6 @@
 extends:
   - standard-with-typescript
   - "plugin:prettier/recommended"
-  - "prettier/@typescript-eslint"
 parserOptions:
   project: './tsconfig.json'
 rules:

.github/workflows/build.yml

@@ -14,6 +14,8 @@ jobs:
           - ubuntu-latest
           - windows-latest
         node-version: [10.x, 12.x, 14.x, 15.x]
+      fail-fast: false
+
     steps:
     - uses: actions/checkout@v2
     - name: with Node.js ${{ matrix.node-version }} on ${{ matrix.os }}

CHANGELOG.md

@@ -22,6 +22,7 @@ Please see [CONTRIBUTING.md](https://github.com/cucumber/cucumber/blob/master/CO
 ### Fixed
 
 * Fix types for hook functions so they can return e.g. `'skipped'` ([#1542](https://github.com/cucumber/cucumber-js/pull/1542))
+* Display the response of the reports server when an error is returned before failing. ([#1608](https://github.com/cucumber/cucumber-js/pull/1608))
 
 ## [7.0.0] (2020-12-21)
 

cucumber.js

@@ -1,9 +1,7 @@
 const feature = [
   '--require-module ts-node/register',
   '--require features/**/*.ts',
-  `--format ${
-    process.env.CI || !process.stdout.isTTY ? 'progress' : 'progress-bar'
-  }`,
+  `--format progress-bar`,
   '--format rerun:@rerun.txt',
   '--format usage:usage.txt',
   '--format message:messages.ndjson',

docs/cli.md

@@ -153,6 +153,8 @@ abort the run on first failure (default: false)
 
 By default, cucumber-js runs the entire suite and reports all the failures. This flag allows a developer workflow where you work on one failure at a time. Combining this feature with rerun files allows you to work through all failures in an efficient manner.
 
+A note on using in conjunction with `--retry`: we consider a test case to have failed if it exhausts retries and still fails, but passed if it passes on a retry having failed previous attempts, so `--fail-fast` does still allow retries to happen.
+
 ## Retry failing tests
 
 Use `--retry <int>` to rerun tests that have been failing. This can be very helpful for flaky tests.
@@ -170,7 +172,7 @@ For instance, for ES6 support with [Babel](https://babeljs.io/) 7 add:
 --require-module @babel/register
 ```
 
-This will effectivally call `require('@babel/register')` prior to requiring any support files.
+This will effectively call `require('@babel/register')` prior to requiring any support files.
 
 ### Non JS files
 

docs/nodejs_example.md

@@ -4,9 +4,9 @@
 - Install Cucumber modules with [yarn](https://yarnpkg.com/en/) **or** [npm](https://www.npmjs.com/)
 
   ```
-  yarn add -D npm install @cucumber/cucumber
+  yarn add -D @cucumber/cucumber
 
-  npm i -D npm install @cucumber/cucumber
+  npm i -D @cucumber/cucumber
   ```
 
 * Add the following files

docs/support_files/timeouts.md

@@ -23,6 +23,8 @@ Given(/^a slow step$/, {timeout: 60 * 1000}, function() {
 });
 ```
 
+*Note that you should not call `setDefaultTimeout` from within a hook or step; it should be called globally.*
+
 ## Disable Timeouts
 
 **DO NOT USE THIS UNLESS ABSOLUTELY NECESSARY**

features/publish.feature

@@ -54,7 +54,7 @@ Feature: Publish reports
 
   @spawn
   Scenario: Report is published when CUCUMBER_PUBLISH_TOKEN is set
-    When I run cucumber-js with arguments `` and env `CUCUMBER_PUBLISH_TOKEN=keyboardcat`
+    When I run cucumber-js with arguments `` and env `CUCUMBER_PUBLISH_TOKEN=f318d9ec-5a3d-4727-adec-bd7b69e2edd3`
     Then it passes
     And the server should receive the following message types:
       | meta             |
@@ -69,7 +69,7 @@ Feature: Publish reports
       | testStepFinished |
       | testCaseFinished |
       | testRunFinished  |
-    And the server should receive an "Authorization" header with value "Bearer keyboardcat"
+    And the server should receive an "Authorization" header with value "Bearer f318d9ec-5a3d-4727-adec-bd7b69e2edd3"
 
   @spawn
   Scenario: a banner is displayed after publication
@@ -101,6 +101,20 @@ Feature: Publish reports
       │ module.exports = { default: '--publish-quiet' }                          │
       └──────────────────────────────────────────────────────────────────────────┘
       """
+
+  @spawn
+  Scenario: when results are not published due to an error raised by the server, the banner is displayed
+    When I run cucumber-js with env `CUCUMBER_PUBLISH_TOKEN=keyboardcat`
+    Then it fails
+    And the error output contains the text:
+      """
+      ┌─────────────────────┐
+      │ Error invalid token │
+      └─────────────────────┘
+
+      Unexpected http status 401 from GET http://localhost:9987
+      """
+
   @spawn
   Scenario: the publication banner is not shown when publication is done
     When I run cucumber-js with arguments `<args>` and env `<env>`
@@ -110,10 +124,10 @@ Feature: Publish reports
       """
 
   Examples:
-    | args      | env                           |
-    | --publish |                               |
-    |           | CUCUMBER_PUBLISH_ENABLED=true |
-    |           | CUCUMBER_PUBLISH_TOKEN=123456 |
+    | args      | env                                                         |
+    | --publish |                                                             |
+    |           | CUCUMBER_PUBLISH_ENABLED=true                               |
+    |           | CUCUMBER_PUBLISH_TOKEN=f318d9ec-5a3d-4727-adec-bd7b69e2edd3 |
 
   @spawn
   Scenario: the publication banner is not shown when publication is disabled

features/retry.feature

@@ -534,3 +534,60 @@ Feature: Retry flaky tests
       """
     When I run cucumber-js with `--retry 1`
     Then it passes
+
+  Rule: using retry in combination with fail-fast will exhaust retries before failing test run
+
+    Scenario: a flaky scenario that passes on the second attempt, set to fail fast
+      Given a file named "features/a.feature" with:
+      """
+      Feature:
+        Scenario: Flaky
+          Given a flaky step
+
+        Scenario: Passing
+          Given a passing step
+      """
+      Given a file named "features/step_definitions/cucumber_steps.js" with:
+      """
+      const {Given} = require('@cucumber/cucumber')
+
+      let willPass = false
+
+      Given(/^a flaky step$/, function() {
+        if (willPass) {
+          return
+        }
+        willPass = true
+        throw 'fail'
+      })
+
+      Given(/^a passing step$/, function() {})
+      """
+      When I run cucumber-js with `--retry 1 --fail-fast`
+      Then it passes
+      And scenario "Flaky" attempt 0 step "Given a flaky step" has status "failed"
+      And scenario "Flaky" attempt 1 step "Given a flaky step" has status "passed"
+      And scenario "Passing" step "Given a passing step" has status "passed"
+
+    Scenario: a scenario that fails every allotted attempt, set to fail fast
+      Given a file named "features/a.feature" with:
+      """
+      Feature:
+        Scenario: Failing
+          Given a failing step
+
+        Scenario: Passing
+          Given a passing step
+      """
+      Given a file named "features/step_definitions/cucumber_steps.js" with:
+      """
+      const {Given} = require('@cucumber/cucumber')
+
+      Given(/^a failing step$/, function() { throw 'fail' })
+      Given(/^a passing step$/, function() {})
+      """
+      When I run cucumber-js with `--retry 1 --fail-fast`
+      Then it fails
+      And scenario "Failing" attempt 0 step "Given a failing step" has status "failed"
+      And scenario "Failing" attempt 1 step "Given a failing step" has status "failed"
+      And scenario "Passing" step "Given a passing step" has status "skipped"

features/step_definitions/cli_steps.ts

@@ -37,6 +37,15 @@ When(
   }
 )
 
+When(
+  /^I run cucumber-js with env `(|.+)`$/,
+  { timeout: 10000 },
+  async function (this: World, envString: string) {
+    const env = this.parseEnvString(envString)
+    return await this.run(this.localExecutablePath, [], env)
+  }
+)
+
 When(
   /^I run cucumber-js with all formatters(?: and `(|.+)`)?$/,
   { timeout: 10000 },
@@ -67,10 +76,14 @@ When(
 Then(/^it passes$/, () => {}) // eslint-disable-line @typescript-eslint/no-empty-function
 
 Then(/^it fails$/, function (this: World) {
-  const actualCode = doesHaveValue(this.lastRun.error)
+  const actualCode: number = doesHaveValue(this.lastRun.error)
     ? this.lastRun.error.code
     : 0
-  expect(actualCode).not.to.eql(0)
+
+  expect(actualCode).not.to.eql(
+    0,
+    `Expected non-zero exit status, but got ${actualCode}`
+  )
   this.verifiedLastRunError = true
 })