diff --git a/cli/CHANGELOG.md b/cli/CHANGELOG.md index 6e4ab04ffe5..04c6c539384 100644 --- a/cli/CHANGELOG.md +++ b/cli/CHANGELOG.md @@ -1,4 +1,12 @@ +## 15.5.0 + +_Released 10/21/2025 (PENDING)_ + +**Features:** + +- When the `run` command requires successful negotiation with the Cypress Cloud API and the `--posix-exit-codes` flag is set, Cypress will now exit with code `112` when it cannot determine which spec to run next due to network conditions. These Cloud API negotiations are required when either `--record` or `--parallel` flags are set. Addresses [#32485](https://github.com/cypress-io/cypress/issues/32485). Addressed in [#32635](https://github.com/cypress-io/cypress/pull/32635). + ## 15.4.0 _Released 10/7/2025_ diff --git a/packages/data-context/schemas/schema.graphql b/packages/data-context/schemas/schema.graphql index 10f14f3ccf9..adf7cfa627d 100644 --- a/packages/data-context/schemas/schema.graphql +++ b/packages/data-context/schemas/schema.graphql @@ -1134,7 +1134,9 @@ enum ErrorTypeEnum { CLOUD_CANNOT_CONFIRM_ARTIFACTS CLOUD_CANNOT_CREATE_RUN_OR_INSTANCE CLOUD_CANNOT_PROCEED_IN_PARALLEL + CLOUD_CANNOT_PROCEED_IN_PARALLEL_NETWORK CLOUD_CANNOT_PROCEED_IN_SERIAL + CLOUD_CANNOT_PROCEED_IN_SERIAL_NETWORK CLOUD_CANNOT_UPLOAD_ARTIFACTS CLOUD_GRAPHQL_ERROR CLOUD_INVALID_RUN_REQUEST diff --git a/packages/errors/src/errors.ts b/packages/errors/src/errors.ts index 79aedf25a06..20ad76079fe 100644 --- a/packages/errors/src/errors.ts +++ b/packages/errors/src/errors.ts @@ -173,6 +173,21 @@ export const AllCypressErrors = { ciBuildId: '--ciBuildId', })}` }, + CLOUD_CANNOT_PROCEED_IN_PARALLEL_NETWORK: (arg1: { flags: any, response: Error }) => { + const message = normalizeNetworkErrorMessage(arg1.response) + + return errTemplate`\ + We encountered an unexpected error communicating with our servers. + + ${fmt.highlightSecondary(message)} + + Because you passed the ${fmt.flag(`--parallel`)} flag, this run cannot proceed since it requires a valid response from our servers. + + ${fmt.listFlags(arg1.flags, { + group: '--group', + ciBuildId: '--ciBuildId', + })}` + }, CLOUD_CANNOT_PROCEED_IN_SERIAL: (arg1: { flags: any, response: Error }) => { const message = normalizeNetworkErrorMessage(arg1.response) @@ -188,6 +203,21 @@ export const AllCypressErrors = { ciBuildId: '--ciBuildId', })}` }, + CLOUD_CANNOT_PROCEED_IN_SERIAL_NETWORK: (arg1: { flags: any, response: Error }) => { + const message = normalizeNetworkErrorMessage(arg1.response) + + return errTemplate`\ + We encountered an unexpected error communicating with our servers. + + ${fmt.highlightSecondary(message)} + + Because you passed the ${fmt.flag(`--record`)} flag, this run cannot proceed since it requires a valid response from our servers. + + ${fmt.listFlags(arg1.flags, { + group: '--group', + ciBuildId: '--ciBuildId', + })}` + }, CLOUD_UNKNOWN_INVALID_REQUEST: (arg1: { flags: any, response: Error }) => { const message = normalizeNetworkErrorMessage(arg1.response) diff --git a/packages/errors/test/__snapshots__/CLOUD_CANNOT_PROCEED_IN_PARALLEL_NETWORK.ansi b/packages/errors/test/__snapshots__/CLOUD_CANNOT_PROCEED_IN_PARALLEL_NETWORK.ansi new file mode 100644 index 00000000000..bfbe9a6a154 --- /dev/null +++ b/packages/errors/test/__snapshots__/CLOUD_CANNOT_PROCEED_IN_PARALLEL_NETWORK.ansi @@ -0,0 +1,8 @@ +We encountered an unexpected error communicating with our servers. + +Error: fail whale + +Because you passed the --parallel flag, this run cannot proceed since it requires a valid response from our servers. + +The --group flag you passed was: foo +The --ciBuildId flag you passed was: invalid \ No newline at end of file diff --git a/packages/errors/test/__snapshots__/CLOUD_CANNOT_PROCEED_IN_SERIAL_NETWORK.ansi b/packages/errors/test/__snapshots__/CLOUD_CANNOT_PROCEED_IN_SERIAL_NETWORK.ansi new file mode 100644 index 00000000000..302ecb07062 --- /dev/null +++ b/packages/errors/test/__snapshots__/CLOUD_CANNOT_PROCEED_IN_SERIAL_NETWORK.ansi @@ -0,0 +1,8 @@ +We encountered an unexpected error communicating with our servers. + +Error: fail whale + +Because you passed the --record flag, this run cannot proceed since it requires a valid response from our servers. + +The --group flag you passed was: foo +The --ciBuildId flag you passed was: invalid \ No newline at end of file diff --git a/packages/errors/test/visualSnapshotErrors.spec.ts b/packages/errors/test/visualSnapshotErrors.spec.ts index f736cfef42c..3814e874a3c 100644 --- a/packages/errors/test/visualSnapshotErrors.spec.ts +++ b/packages/errors/test/visualSnapshotErrors.spec.ts @@ -236,6 +236,17 @@ describe('visual error templates', () => { }], } }, + CLOUD_CANNOT_PROCEED_IN_PARALLEL_NETWORK: () => { + return { + default: [{ + flags: { + ciBuildId: 'invalid', + group: 'foo', + }, + response: makeErr(), + }], + } + }, CLOUD_CANNOT_PROCEED_IN_SERIAL: () => { return { default: [{ @@ -247,6 +258,17 @@ describe('visual error templates', () => { }], } }, + CLOUD_CANNOT_PROCEED_IN_SERIAL_NETWORK: () => { + return { + default: [{ + flags: { + ciBuildId: 'invalid', + group: 'foo', + }, + response: makeErr(), + }], + } + }, CLOUD_UNKNOWN_INVALID_REQUEST: () => { return { default: [{ diff --git a/packages/server/lib/cloud/api/index.ts b/packages/server/lib/cloud/api/index.ts index 16c775cd324..f28b095fd6c 100644 --- a/packages/server/lib/cloud/api/index.ts +++ b/packages/server/lib/cloud/api/index.ts @@ -43,6 +43,10 @@ const THIRTY_SECONDS = humanInterval('30 seconds') const SIXTY_SECONDS = humanInterval('60 seconds') const TWO_MINUTES = humanInterval('2 minutes') +function defaultTimeout () { + return process.env.CYPRESS_INTERNAL_API_TIMEOUT && !isNaN(Number(process.env.CYPRESS_INTERNAL_API_TIMEOUT)) ? Number(process.env.CYPRESS_INTERNAL_API_TIMEOUT) : SIXTY_SECONDS +} + function retryDelays (): number[] { return process.env.API_RETRY_INTERVALS ? process.env.API_RETRY_INTERVALS.split(',').map(_.toNumber) @@ -411,7 +415,7 @@ export default { url: recordRoutes.runs(), json: true, encrypt: preflightResult.encrypt, - timeout: options.timeout ?? SIXTY_SECONDS, + timeout: options.timeout ?? defaultTimeout(), headers: { 'x-route-version': '4', 'x-cypress-request-attempt': attemptIndex, @@ -485,7 +489,7 @@ export default { url: recordRoutes.instances(runId), json: true, encrypt: preflightResult.encrypt, - timeout: timeout ?? SIXTY_SECONDS, + timeout: timeout ?? defaultTimeout(), headers: { 'x-route-version': '5', 'x-cypress-run-id': runId, @@ -505,7 +509,7 @@ export default { url: recordRoutes.instanceTests(instanceId), json: true, encrypt: preflightResult.encrypt, - timeout: timeout ?? SIXTY_SECONDS, + timeout: timeout ?? defaultTimeout(), headers: { 'x-route-version': '1', 'x-cypress-run-id': runId, @@ -523,7 +527,7 @@ export default { return rp.put({ url: recordRoutes.instanceStdout(options.instanceId), json: true, - timeout: options.timeout ?? SIXTY_SECONDS, + timeout: options.timeout ?? defaultTimeout(), body: { stdout: options.stdout, }, @@ -545,7 +549,7 @@ export default { return rp.put({ url: recordRoutes.instanceArtifacts(options.instanceId), json: true, - timeout: options.timeout ?? SIXTY_SECONDS, + timeout: options.timeout ?? defaultTimeout(), body, headers: { 'x-route-version': '1', @@ -564,7 +568,7 @@ export default { url: recordRoutes.instanceResults(options.instanceId), json: true, encrypt: preflightResult.encrypt, - timeout: options.timeout ?? SIXTY_SECONDS, + timeout: options.timeout ?? defaultTimeout(), headers: { 'x-route-version': '1', 'x-cypress-run-id': options.runId, diff --git a/packages/server/lib/cypress.ts b/packages/server/lib/cypress.ts index a6d06a0b6aa..70c6dfa1b9e 100644 --- a/packages/server/lib/cypress.ts +++ b/packages/server/lib/cypress.ts @@ -16,7 +16,8 @@ import argsUtils from './util/args' import { telemetry } from '@packages/telemetry' import { getCtx, hasCtx } from '@packages/data-context' import { warning as errorsWarning } from './errors' - +import type { CypressError } from '@packages/errors' +import { toNumber } from 'lodash' const debug = Debug('cypress:server:cypress') type Mode = 'exit' | 'info' | 'interactive' | 'pkg' | 'record' | 'results' | 'run' | 'smokeTest' | 'version' | 'returnPkg' | 'exitWithCode' @@ -42,6 +43,8 @@ const exit = async (code = 0) => { debug('telemetry shutdown errored with: ', err) }) + debug('process.exit', code) + return process.exit(code) } @@ -66,18 +69,31 @@ const exit0 = () => { return exit(0) } -const exitErr = (err: any) => { +function isCypressError (err: unknown): err is CypressError { + return (err as CypressError).isCypressErr +} + +async function exitErr (err: unknown, posixExitCodes?: boolean) { // log errors to the console // and potentially raygun // and exit with 1 debug('exiting with err', err) - return require('./errors').logException(err) - .then(() => { - debug('calling exit 1') + await require('./errors').logException(err) - return exit(1) - }) + if (isCypressError(err)) { + if ( + posixExitCodes && ( + err.type === 'CLOUD_CANNOT_PROCEED_IN_PARALLEL_NETWORK' || + err.type === 'CLOUD_CANNOT_PROCEED_IN_SERIAL_NETWORK' + )) { + return exit(112) + } + } + + debug('calling exit 1') + + return exit(1) } export = { @@ -151,7 +167,7 @@ export = { debug('could not parse CLI arguments: %o', argv) // note - this is promise-returned call - return exitErr(argumentsError) + return exitErr(argumentsError, Boolean(options?.posixExitCodes)) } debug('from argv %o got options %o', argv, options) @@ -206,55 +222,51 @@ export = { }) }, - startInMode (mode: Mode, options: any) { + async startInMode (mode: Mode, options: any) { debug('starting in mode %s with options %o', mode, options) - switch (mode) { - case 'version': - return require('./modes/pkg')(options) - .get('version') - .then((version: any) => { - return console.log(version) // eslint-disable-line no-console - }).then(exit0) - .catch(exitErr) - - case 'info': - return require('./modes/info')(options) - .then(exit0) - .catch(exitErr) - - case 'smokeTest': - return this.runElectron(mode, options) - .then((pong: any) => { + if (mode === 'interactive') { + return this.runElectron(mode, options) + } + + try { + switch (mode) { + case 'version': { + const version = await require('./modes/pkg')(options).get('version') + + // eslint-disable-next-line no-console + console.log(version) + break + } + case 'info': { + await require('./modes/info')(options) + break + } + case 'smokeTest': { + const pong = await this.runElectron(mode, options) + if (!this.isCurrentlyRunningElectron()) { - return pong + return exit(pong) + } else if (pong !== options.ping) { + return exit(1) } - if (pong === options.ping) { - return 0 - } + break + } + case 'returnPkg': { + const pkg = await require('./modes/pkg')(options) + + // eslint-disable-next-line no-console + console.log(JSON.stringify(pkg)) + break + } + case 'exitWithCode': { + return exit(toNumber(options.exitWithCode)) + break + } + case 'run': { + const results = await this.runElectron(mode, options) - return 1 - }).then(exit) - .catch(exitErr) - - case 'returnPkg': - return require('./modes/pkg')(options) - .then((pkg: any) => { - return console.log(JSON.stringify(pkg)) // eslint-disable-line no-console - }).then(exit0) - .catch(exitErr) - - case 'exitWithCode': - return require('./modes/exit')(options) - .then(exit) - .catch(exitErr) - - case 'run': - // run headlessly and exit - // with num of totalFailed - return this.runElectron(mode, options) - .then((results: any) => { if (results.runs) { const isCanceled = results.runs.filter((run) => run.skippedSpec).length @@ -262,24 +274,27 @@ export = { // eslint-disable-next-line no-console console.log(require('chalk').magenta('\n Exiting with non-zero exit code because the run was canceled.')) - return 1 + return exit(1) } } + debug('results.totalFailed, posix?', results.totalFailed, options.posixExitCodes) + if (options.posixExitCodes) { - return results.totalFailed ? 1 : 0 + return exit(results.totalFailed ? 1 : 0) } - return results.totalFailed - }) - .then(exit) - .catch(exitErr) - - case 'interactive': - return this.runElectron(mode, options) - - default: - throw new Error(`Cannot start. Invalid mode: '${mode}'`) + return exit(results.totalFailed ?? 0) + } + default: { + throw new Error(`Cannot start. Invalid mode: '${mode}'`) + } + } + } catch (err) { + return exitErr(err, options.posixExitCodes) } + debug('end of startInMode, exit 0') + + return exit(0) }, } diff --git a/packages/server/lib/modes/exit.ts b/packages/server/lib/modes/exit.ts deleted file mode 100644 index 426ccb0bcc9..00000000000 --- a/packages/server/lib/modes/exit.ts +++ /dev/null @@ -1,8 +0,0 @@ -import { toNumber } from 'lodash' -import Promise from 'bluebird' - -export = (options) => { - return Promise.try(() => { - return toNumber(options.exitWithCode) - }) -} diff --git a/packages/server/lib/modes/record.ts b/packages/server/lib/modes/record.ts index afe990df268..1fda448c327 100644 --- a/packages/server/lib/modes/record.ts +++ b/packages/server/lib/modes/record.ts @@ -11,6 +11,8 @@ import { hideKeys } from '@packages/config' import { default as api } from '../cloud/api' import exception from '../cloud/exception' +import { getError, AllCypressErrorNames } from '@packages/errors' + import { get as getErrors, warning as errorsWarning, throwErr } from '../errors' import capture from '../capture' import { getResolvedRuntimeConfig } from '../config' @@ -385,7 +387,7 @@ const createRun = Promise.method((options: any = {}) => { link: billingLink(orgId), }) default: - return throwErr('CLOUD_UNKNOWN_INVALID_REQUEST', { + throw throwErr('CLOUD_UNKNOWN_INVALID_REQUEST', { response: err, flags: { group, @@ -480,8 +482,21 @@ const createRun = Promise.method((options: any = {}) => { }) } } - default: - throwCloudCannotProceed({ parallel, ciBuildId, group, err }) + case 500: + case 503: { + throw cloudCannotProceedErr({ parallel, ciBuildId, group, err }) + } + default: { + const errName: AllCypressErrorNames = parallel ? 'CLOUD_CANNOT_PROCEED_IN_PARALLEL_NETWORK' : 'CLOUD_CANNOT_PROCEED_IN_SERIAL_NETWORK' + + throw getError(errName, { + response: err, + flags: { + group, + ciBuildId, + }, + }) + } } }) }) diff --git a/packages/server/lib/modes/run.ts b/packages/server/lib/modes/run.ts index 30d3f95c685..aa5b68216b4 100644 --- a/packages/server/lib/modes/run.ts +++ b/packages/server/lib/modes/run.ts @@ -455,7 +455,7 @@ async function listenForProjectEnd (project: ProjectBase, exit: boolean): Promis res(results) }) }), - earlyExitTerminator.waitForEarlyExit(project, exit), + earlyExitTerminator.waitForEarlyExit(project), ]).then((results) => { if (exit === false) { console.log('not exiting due to options.exit being false') diff --git a/packages/server/lib/util/graceful_crash_handling.ts b/packages/server/lib/util/graceful_crash_handling.ts index 38d42718fc4..24a5515f455 100644 --- a/packages/server/lib/util/graceful_crash_handling.ts +++ b/packages/server/lib/util/graceful_crash_handling.ts @@ -79,7 +79,7 @@ export class EarlyExitTerminator { this.terminator = pDefer() } - waitForEarlyExit (project: ProjectBase, exit?: boolean) { + waitForEarlyExit (project: ProjectBase) { debug('waiting for early exit') project.on('test:before:run', ({ diff --git a/packages/server/test/integration/cypress_spec.js b/packages/server/test/integration/cypress_spec.js index faf49dda250..32e836481bb 100644 --- a/packages/server/test/integration/cypress_spec.js +++ b/packages/server/test/integration/cypress_spec.js @@ -468,6 +468,44 @@ describe('lib/cypress', () => { }) }) + it('exits with code 112 for cloud API failures when posix-exit-codes is enabled', function () { + // Mock cloud API to fail with a 504 error + sinon.stub(api, 'createRun').rejects({ + statusCode: 504, + message: 'Gateway timeout', + }) + + return cypress.start([ + `--run-project=${this.todosPath}`, + '--record', + '--key=test-key', + '--posix-exit-codes', + ]) + .then(() => { + this.expectExitWith(112) + }) + }) + + it('exits with code 1 for parallel cloud API failures when posix-exit-codes is enabled', function () { + // Mock cloud API to fail with a 500 error + sinon.stub(api, 'createRun').rejects({ + statusCode: 500, + message: 'Cloud service unavailable', + }) + + return cypress.start([ + `--run-project=${this.todosPath}`, + '--record', + '--key=test-key', + '--parallel', + '--ci-build-id=test-build-id', + '--posix-exit-codes', + ]) + .then(() => { + this.expectExitWith(112) + }) + }) + it('does not add project to the global cache', function () { return cache.getProjectRoots() .then((projects) => { diff --git a/system-tests/__snapshots__/record_spec.js b/system-tests/__snapshots__/record_spec.js index d145b8666b1..44a1a2204a7 100644 --- a/system-tests/__snapshots__/record_spec.js +++ b/system-tests/__snapshots__/record_spec.js @@ -1336,15 +1336,6 @@ The --ciBuildId flag you passed was: ciBuildId123 ` -exports['e2e record api interaction errors create run 402 - free plan exceeds monthly private tests errors and exits when on free plan and over recorded runs limit 1'] = ` -You've exceeded the limit of private test results under your free plan this month. The limit is 500 private test results. - -To continue recording tests this month you must upgrade your account. Please visit your billing to upgrade to another billing plan. - -https://on.cypress.io/dashboard/organizations/org-id-1234/billing - -` - exports['e2e record api interaction errors create run 402 - free plan exceeds monthly tests errors and exits when on free plan and over recorded tests limit 1'] = ` You've exceeded the limit of test results under your free plan this month. The limit is 500 test results. @@ -1962,93 +1953,6 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 -` - -exports['e2e record api interaction warnings create run warnings grace period - over private tests limit warns when over private test results 1'] = ` -You've exceeded the limit of private test results under your free plan this month. The limit is 500 private test results. - -Your plan is now in a grace period, which means your tests will still be recorded until 2999-12-31. Please upgrade your plan to continue recording tests on Cypress Cloud in the future. - -https://on.cypress.io/dashboard/organizations/org-id-1234/billing - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass.cy.js) │ - │ Searched: cypress/e2e/record_pass* │ - │ Params: Tag: false, Group: false, Parallel: false │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: record_pass.cy.js (1 of 1) - Estimated: X second(s) - - - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: false │ - │ Duration: X seconds │ - │ Estimated: X second(s) │ - │ Spec Ran: record_pass.cy.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - - - (Uploading Cloud Artifacts) - - - Video - Nothing to upload - - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Uploading Cloud Artifacts: . . . . . - - (Uploaded Cloud Artifacts) - - - Screenshot - Done Uploading 1 kB in Xm, Ys ZZ.ZZms 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - - - -─────────────────────────────────────────────────────────────────────────────────────────────────────── - - Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 - - ` exports['e2e record api interaction warnings create run warnings grace period - over tests limit warns when over test results 1'] = ` @@ -2312,93 +2216,6 @@ https://on.cypress.io/dashboard/organizations/org-id-1234/billing Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 -` - -exports['e2e record api interaction warnings create run warnings paid plan - over private tests limit warns when over private test results 1'] = ` -You've exceeded the limit of test results under your current billing plan this month. The limit is 500 private test results. - -To continue getting the full benefits of your current plan, please visit your billing to upgrade. - -https://on.cypress.io/dashboard/organizations/org-id-1234/billing - -==================================================================================================== - - (Run Starting) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Cypress: 1.2.3 │ - │ Browser: FooBrowser 88 │ - │ Specs: 1 found (record_pass.cy.js) │ - │ Searched: cypress/e2e/record_pass* │ - │ Params: Tag: false, Group: false, Parallel: false │ - │ Run URL: https://dashboard.cypress.io/projects/cjvoj7/runs/12 │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - -──────────────────────────────────────────────────────────────────────────────────────────────────── - - Running: record_pass.cy.js (1 of 1) - Estimated: X second(s) - - - record pass - ✓ passes - - is pending - - - 1 passing - 1 pending - - - (Results) - - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ Tests: 2 │ - │ Passing: 1 │ - │ Failing: 0 │ - │ Pending: 1 │ - │ Skipped: 0 │ - │ Screenshots: 1 │ - │ Video: false │ - │ Duration: X seconds │ - │ Estimated: X second(s) │ - │ Spec Ran: record_pass.cy.js │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - - - (Screenshots) - - - /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png (400x1022) - - - (Uploading Cloud Artifacts) - - - Video - Nothing to upload - - Screenshot - 1 kB /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - - Uploading Cloud Artifacts: . . . . . - - (Uploaded Cloud Artifacts) - - - Screenshot - Done Uploading 1 kB in Xm, Ys ZZ.ZZms 1/1 /XXX/XXX/XXX/cypress/screenshots/record_pass.cy.js/yay it passes.png - -==================================================================================================== - - (Run Finished) - - - Spec Tests Passing Failing Pending Skipped - ┌────────────────────────────────────────────────────────────────────────────────────────────────┐ - │ ✔ record_pass.cy.js XX:XX 2 1 - 1 - │ - └────────────────────────────────────────────────────────────────────────────────────────────────┘ - ✔ All specs passed! XX:XX 2 1 - 1 - - - -─────────────────────────────────────────────────────────────────────────────────────────────────────── - - Recorded Run: https://dashboard.cypress.io/projects/cjvoj7/runs/12 - - ` exports['e2e record api interaction warnings create run warnings paid plan - over tests limit warns when over test results 1'] = ` @@ -4491,3 +4308,27 @@ We will retry 3 more times in X second(s)... ` + +exports['e2e record network errors create run network errors uses CLOUD_CANNOT_PROCEED_IN_PARALLEL_NETWORK for parallel runs with network errors 1'] = ` +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 502 - "Bad Gateway" + +Because you passed the --parallel flag, this run cannot proceed since it requires a valid response from our servers. + +The --group flag you passed was: foo +The --ciBuildId flag you passed was: ciBuildId123 + +` + +exports['e2e record network errors create run network errors uses CLOUD_CANNOT_PROCEED_IN_SERIAL_NETWORK for serial runs with network errors 1'] = ` +We encountered an unexpected error communicating with our servers. + +StatusCodeError: 502 - "Bad Gateway" + +Because you passed the --record flag, this run cannot proceed since it requires a valid response from our servers. + +The --group flag you passed was: foo +The --ciBuildId flag you passed was: ciBuildId123 + +` diff --git a/system-tests/test/record_spec.js b/system-tests/test/record_spec.js index 5cf8cdd741d..0df5f6828e1 100644 --- a/system-tests/test/record_spec.js +++ b/system-tests/test/record_spec.js @@ -883,6 +883,19 @@ describe('e2e record', () => { }) }) + it('errors and exits with 1 when posix exit codes are enabled', function () { + process.env.DISABLE_API_RETRIES = 'true' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + record: true, + posixExitCodes: true, + expectedExitCode: 1, + }) + }) + it('when grouping without parallelization errors and exits', function () { process.env.DISABLE_API_RETRIES = 'true' @@ -905,6 +918,21 @@ describe('e2e record', () => { }) }) + it('when grouping without parallelization errors and exits with 1 when posix exit codes are enabled', function () { + process.env.DISABLE_API_RETRIES = 'true' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + group: 'foo', + record: true, + posixExitCodes: true, + ciBuildId: 'ciBuildId123', + expectedExitCode: 1, + }) + }) + it('does not proceed and exits with error when parallelizing', function () { process.env.DISABLE_API_RETRIES = 'true' @@ -966,6 +994,23 @@ describe('e2e record', () => { }) }) + it('does not proceed and exits with error with 1 when posix exit codes are enabled when parallelizing and creating instance', function () { + process.env.DISABLE_API_RETRIES = 'true' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + group: 'foo', + tag: 'nightly', + record: true, + parallel: true, + posixExitCodes: true, + ciBuildId: 'ciBuildId123', + expectedExitCode: 1, + }) + }) + it('without parallelization - does not proceed', async function () { process.env.DISABLE_API_RETRIES = 'true' @@ -986,6 +1031,19 @@ describe('e2e record', () => { ]) }) }) + + it('without parallelization - does not proceed with 1 when posix exit codes are enabled', async function () { + process.env.DISABLE_API_RETRIES = 'true' + + await systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'a_record.cy.js,b_record.cy.js', + record: true, + posixExitCodes: true, + expectedExitCode: 1, + }) + }) }) describe('update instance 500', () => { @@ -1036,6 +1094,23 @@ describe('e2e record', () => { ]) }) }) + + it('does not proceed and exits with error with 1 when posix exit codes are enabled when parallelizing and updating instance', function () { + process.env.DISABLE_API_RETRIES = 'true' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + group: 'foo', + tag: 'nightly', + record: true, + parallel: true, + ciBuildId: 'ciBuildId123', + expectedExitCode: 1, + posixExitCodes: true, + }) + }) }) describe('create run 422', () => { @@ -1074,6 +1149,27 @@ describe('e2e record', () => { ]) }) }) + + it('errors and exits with 1 when posix exit codes are enabled when group name is in use', function () { + process.env.CIRCLECI = '1' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + group: 'e2e-tests', + record: true, + posixExitCodes: true, + expectedExitCode: 1, + }) + .then(() => { + const urls = getRequestUrls() + + expect(urls).to.deep.eq([ + 'POST /runs', + ]) + }) + }) }) describe('create run 412', () => { @@ -1142,6 +1238,21 @@ describe('e2e record', () => { ]) }) }) + + it('errors and exits with 1 when posix exit codes are enabled when there is an unknown 422 response', function () { + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + group: 'e2e-tests', + tag: 'nightly', + record: true, + parallel: true, + ciBuildId: 'ciBuildId123', + expectedExitCode: 1, + posixExitCodes: true, + }) + }) }) describe('create run 402 - free plan exceeds monthly tests', () => { @@ -1170,6 +1281,17 @@ describe('e2e record', () => { expectedExitCode: 1, }) }) + + it('errors and exits with 1 when posix exit codes are enabled when on free plan and over recorded tests limit', function () { + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + record: true, + posixExitCodes: true, + expectedExitCode: 1, + }) + }) }) describe('create run 402 - parallel feature not available in plan', () => { @@ -1195,6 +1317,17 @@ describe('e2e record', () => { expectedExitCode: 1, }) }) + + it('errors and exits with 1 when posix exit codes are enabled when attempting parallel run when not available in plan', function () { + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + record: true, + posixExitCodes: true, + expectedExitCode: 1, + }) + }) }) describe('create run 402 - grouping feature not available in plan', () => { @@ -1219,6 +1352,17 @@ describe('e2e record', () => { expectedExitCode: 1, }) }) + + it('errors and exits when attempting parallel run when not available in plan with 1 when posix exit codes are enabled', function () { + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + record: true, + expectedExitCode: 1, + posixExitCodes: true, + }) + }) }) describe('create run 402 - unknown error', () => { @@ -1240,6 +1384,17 @@ describe('e2e record', () => { expectedExitCode: 1, }) }) + + it(`errors and exits when there's an unknown 402 error with 1 when posix exit codes are enabled`, function () { + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + record: true, + posixExitCodes: true, + expectedExitCode: 1, + }) + }) }) describe('create run 402 - auto cancel not available in plan', () => { @@ -1265,6 +1420,17 @@ describe('e2e record', () => { expectedExitCode: 1, }) }) + + it('errors and exits when auto cancel not available in plan', function () { + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + record: true, + expectedExitCode: 1, + posixExitCodes: true, + }) + }) }) describe('create instance', () => { @@ -1296,6 +1462,19 @@ describe('e2e record', () => { ]) }) }) + + it('errors and exits on createInstance error with 1 when posix exit codes are enabled', function () { + process.env.DISABLE_API_RETRIES = 'true' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'a_record_instantfail.cy.js', + record: true, + expectedExitCode: 1, + posixExitCodes: true, + }) + }) }) describe('postInstanceTests', () => { @@ -1354,6 +1533,21 @@ describe('e2e record', () => { }) }) + it('without parallelization errors and exits with 1 when posix exit codes are enabled', async function () { + process.env.DISABLE_API_RETRIES = 'true' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'a_record.cy.js,b_record.cy.js', + group: 'foo', + ciBuildId: 1, + expectedExitCode: 1, + record: true, + posixExitCodes: true, + }) + }) + it('with parallelization errors and exits', async function () { process.env.DISABLE_API_RETRIES = 'true' @@ -1378,6 +1572,22 @@ describe('e2e record', () => { ]) }) }) + + it('with parallelization errors and exits with 1 when posix exit codes are enabled', async function () { + process.env.DISABLE_API_RETRIES = 'true' + + await systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'a_record.cy.js,b_record.cy.js', + record: true, + group: 'foo', + ciBuildId: 'ciBuildId123', + expectedExitCode: 1, + parallel: true, + posixExitCodes: true, + }) + }) }) describe('postInstanceResults', () => { @@ -1413,6 +1623,19 @@ describe('e2e record', () => { ]) }) }) + + it('exits with code 1 when posix exit codes are enabled', function () { + process.env.DISABLE_API_RETRIES = 'true' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + record: true, + posixExitCodes: true, + expectedExitCode: 1, + }) + }) }) describe('update instance stdout', () => { @@ -1991,6 +2214,189 @@ describe('e2e record', () => { }) }) + describe('network errors', () => { + describe('create run network errors', () => { + const routes = createRoutes({ + postRun: { + res (req, res) { + return res.sendStatus(502) // Bad Gateway - should trigger default case + }, + }, + }) + + setupStubbedServer(routes) + + it('uses CLOUD_CANNOT_PROCEED_IN_PARALLEL_NETWORK for parallel runs with network errors', function () { + process.env.DISABLE_API_RETRIES = 'true' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + group: 'foo', + tag: 'nightly', + record: true, + parallel: true, + ciBuildId: 'ciBuildId123', + expectedExitCode: 1, + }) + .then(() => { + const urls = getRequestUrls() + + expect(urls).to.deep.eq([ + 'POST /runs', + ]) + }) + }) + + it('uses CLOUD_CANNOT_PROCEED_IN_SERIAL_NETWORK for serial runs with network errors', function () { + process.env.DISABLE_API_RETRIES = 'true' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + group: 'foo', + record: true, + ciBuildId: 'ciBuildId123', + expectedExitCode: 1, + }) + .then(() => { + const urls = getRequestUrls() + + expect(urls).to.deep.eq([ + 'POST /runs', + ]) + }) + }) + + it('exits with code 112 when posix exit codes are enabled for network errors in parallel mode', function () { + process.env.DISABLE_API_RETRIES = 'true' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + group: 'foo', + tag: 'nightly', + record: true, + parallel: true, + posixExitCodes: true, + ciBuildId: 'ciBuildId123', + expectedExitCode: 112, + }) + }) + + it('exits with code 112 when posix exit codes are enabled for network errors in serial mode', function () { + process.env.DISABLE_API_RETRIES = 'true' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + group: 'foo', + record: true, + posixExitCodes: true, + ciBuildId: 'ciBuildId123', + expectedExitCode: 112, + }) + }) + }) + + describe('create run connection timeout', () => { + const routes = createRoutes({ + postRun: { + res (req, res) { + // Simulate connection timeout by not responding + return new Promise(() => {}) // Never resolves + }, + }, + }) + + setupStubbedServer(routes) + + beforeEach(() => { + process.env.DISABLE_API_RETRIES = 'true' + process.env.CYPRESS_INTERNAL_API_TIMEOUT = '10' + }) + + afterEach(() => { + delete process.env.DISABLE_API_RETRIES + delete process.env.CYPRESS_INTERNAL_API_TIMEOUT + }) + + it('handles connection timeout errors in parallel mode', function () { + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + group: 'foo', + tag: 'nightly', + record: true, + parallel: true, + ciBuildId: 'ciBuildId123', + expectedExitCode: 1, + }) + }) + + it('handles connection timeout errors in serial mode', function () { + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + group: 'foo', + record: true, + ciBuildId: 'ciBuildId123', + expectedExitCode: 1, + }) + }) + }) + + describe('create run DNS resolution failure', () => { + const routes = createRoutes({ + postRun: { + res (req, res) { + return res.sendStatus(504) // Gateway Timeout - should trigger default case + }, + }, + }) + + setupStubbedServer(routes) + + it('handles DNS resolution failures in parallel mode', function () { + process.env.DISABLE_API_RETRIES = 'true' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + group: 'foo', + tag: 'nightly', + record: true, + parallel: true, + posixExitCodes: true, + ciBuildId: 'ciBuildId123', + expectedExitCode: 112, + }) + }) + + it('handles DNS resolution failures in serial mode', function () { + process.env.DISABLE_API_RETRIES = 'true' + + return systemTests.exec(this, { + key: 'f858a2bc-b469-4e48-be67-0876339ee7e1', + configFile: 'cypress-with-project-id.config.js', + spec: 'record_pass*', + group: 'foo', + record: true, + posixExitCodes: true, + ciBuildId: 'ciBuildId123', + expectedExitCode: 112, + }) + }) + }) + }) + describe('api interaction warnings', () => { describe('create run warnings', () => { describe('grace period - over tests limit', () => {