diff --git a/bin/codecept.js b/bin/codecept.js index 212f21639..4f890b59b 100755 --- a/bin/codecept.js +++ b/bin/codecept.js @@ -166,6 +166,8 @@ program .option('-p, --plugins ', 'enable plugins, comma-separated') .option('--shuffle', 'Shuffle the order in which test files run') .option('--shard ', 'run only a fraction of tests (e.g., --shard 1/4)') + .option('--save-failed-tests [path]', 'save failed tests to JSON file (default: failed-tests.json)') + .option('--failed-tests ', 'run only tests from failed tests JSON file') // mocha options .option('--colors', 'force enabling of colors') @@ -207,6 +209,8 @@ program .option('-p, --plugins ', 'enable plugins, comma-separated') .option('-O, --reporter-options ', 'reporter-specific options') .option('-R, --reporter ', 'specify the reporter to use') + .option('--save-failed-tests [path]', 'save failed tests to JSON file (default: failed-tests.json)') + .option('--failed-tests ', 'run only tests from failed tests JSON file') .action(errorHandler(require('../lib/command/run-workers'))) program diff --git a/docs/plugins.md b/docs/plugins.md index b3b20de8a..267d57549 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -440,6 +440,14 @@ I.see('new title', 'h1'); * `config` +## consolidateWorkerJsonResults + +Consolidates JSON reports from multiple workers into a single HTML report + +### Parameters + +* `config` + ## coverage Dumps code coverage from Playwright/Puppeteer after every test. @@ -651,6 +659,17 @@ const eachElement = codeceptjs.container.plugins('eachElement'); Returns **([Promise][9]\ | [undefined][10])** +## enhancedRetryFailedStep + +Enhanced retryFailedStep plugin that coordinates with other retry mechanisms + +This plugin provides step-level retries and coordinates with global retry settings +to avoid conflicts and provide predictable behavior. + +### Parameters + +* `config` + ## fakerTransform Use the `@faker-js/faker` package to generate fake data inside examples on your gherkin tests @@ -719,10 +738,10 @@ HTML Reporter Plugin for CodeceptJS Generates comprehensive HTML reports showing: -- Test statistics -- Feature/Scenario details -- Individual step results -- Test artifacts (screenshots, etc.) +* Test statistics +* Feature/Scenario details +* Individual step results +* Test artifacts (screenshots, etc.) ## Configuration @@ -749,7 +768,7 @@ Generates comprehensive HTML reports showing: ### Parameters -- `config` +* `config` ## pageInfo @@ -862,6 +881,14 @@ Scenario('scenario tite', { disableRetryFailedStep: true }, () => { * `config` +## safeJsonStringify + +Safely serialize data to JSON, handling circular references + +### Parameters + +* `data` + ## screenshotOnFail Creates screenshot on failure. Screenshot is saved into `output` directory. diff --git a/lib/codecept.js b/lib/codecept.js index 59d77cd34..ac95d44cc 100644 --- a/lib/codecept.js +++ b/lib/codecept.js @@ -192,6 +192,33 @@ class Codecept { } } + /** + * Filter tests to only include failed tests from a failed tests data + * + * @param {Array} failedTests - Array of failed test objects with uid, title, file, etc. + */ + filterByFailedTests(failedTests) { + if (!failedTests || failedTests.length === 0) { + this.testFiles = [] + return + } + + // Extract unique file paths from failed tests + const failedTestFiles = [...new Set(failedTests.map(test => test.file).filter(Boolean))] + + // Filter testFiles to only include files that contain failed tests + this.testFiles = this.testFiles.filter(file => { + const normalizedFile = fsPath.resolve(file) + return failedTestFiles.some(failedFile => { + const normalizedFailedFile = fsPath.resolve(failedFile) + return normalizedFile === normalizedFailedFile + }) + }) + + // Store failed test info for filtering during test execution + this.failedTestsFilter = failedTests + } + /** * Apply sharding to test files based on shard configuration * @@ -246,6 +273,18 @@ class Codecept { } mocha.files = mocha.files.filter(t => fsPath.basename(t, '.js') === test || t === test) } + + // Apply failed test filtering if specified + if (this.failedTestsFilter && this.failedTestsFilter.length > 0) { + // Use Mocha's grep functionality to filter tests by full title + const failedTestFullTitles = this.failedTestsFilter.map(t => t.fullTitle).filter(Boolean) + if (failedTestFullTitles.length > 0) { + // Create a regex pattern that matches any of the failed test full titles + const pattern = `^(${failedTestFullTitles.map(title => title.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')).join('|')})$` + mocha.grep(new RegExp(pattern)) + } + } + const done = () => { event.emit(event.all.result, container.result()) event.emit(event.all.after, this) @@ -262,6 +301,67 @@ class Codecept { }) } + /** + * Filter Mocha tests to only include failed tests + * + * @private + * @param {Mocha} mocha - Mocha instance + * @param {Array} failedTests - Array of failed test objects + */ + _filterMochaTestsByFailedTests(mocha, failedTests) { + const failedTestUids = new Set(failedTests.map(t => t.uid).filter(Boolean)) + const failedTestTitles = new Set(failedTests.map(t => t.title).filter(Boolean)) + const failedTestFullTitles = new Set(failedTests.map(t => t.fullTitle).filter(Boolean)) + + // Remove tests that are not in the failed tests list + for (const suite of mocha.suite.suites) { + suite.tests = suite.tests.filter(test => { + return failedTestUids.has(test.uid) || failedTestTitles.has(test.title) || failedTestFullTitles.has(test.fullTitle()) + }) + + // Remove nested suites' tests as well + this._filterNestedSuites(suite, failedTestUids, failedTestTitles, failedTestFullTitles) + } + + // Clean up empty suites + mocha.suite.suites = mocha.suite.suites.filter(suite => this._hasTests(suite)) + } + + /** + * Recursively filter nested suites + * + * @private + * @param {Suite} suite - Mocha suite + * @param {Set} failedTestUids - Set of failed test UIDs + * @param {Set} failedTestTitles - Set of failed test titles + * @param {Set} failedTestFullTitles - Set of failed test full titles + */ + _filterNestedSuites(suite, failedTestUids, failedTestTitles, failedTestFullTitles) { + for (const childSuite of suite.suites || []) { + childSuite.tests = childSuite.tests.filter(test => { + return failedTestUids.has(test.uid) || failedTestTitles.has(test.title) || failedTestFullTitles.has(test.fullTitle()) + }) + + this._filterNestedSuites(childSuite, failedTestUids, failedTestTitles, failedTestFullTitles) + } + + // Remove empty child suites + suite.suites = suite.suites.filter(childSuite => this._hasTests(childSuite)) + } + + /** + * Check if suite or any nested suite has tests + * + * @private + * @param {Suite} suite - Mocha suite + * @returns {boolean} + */ + _hasTests(suite) { + if (suite.tests && suite.tests.length > 0) return true + if (suite.suites && suite.suites.some(childSuite => this._hasTests(childSuite))) return true + return false + } + static version() { return JSON.parse(readFileSync(`${__dirname}/../package.json`, 'utf8')).version } diff --git a/lib/command/run-workers.js b/lib/command/run-workers.js index b5e3969fd..ac04411a1 100644 --- a/lib/command/run-workers.js +++ b/lib/command/run-workers.js @@ -4,6 +4,9 @@ const output = require('../output') const store = require('../store') const event = require('../event') const Workers = require('../workers') +const fs = require('fs') +const path = require('path') +const container = require('../container') module.exports = async function (workerCount, selectedRuns, options) { process.env.profile = options.profile @@ -27,11 +30,28 @@ module.exports = async function (workerCount, selectedRuns, options) { throw new Error(`Invalid --by strategy: ${by}. Valid options are: ${validStrategies.join(', ')}`) } delete options.parent + + // Handle failed tests loading + let failedTestsData = null + if (options.failedTests) { + const failedTestsFile = path.isAbsolute(options.failedTests) ? options.failedTests : path.resolve(options.failedTests) + + if (!fs.existsSync(failedTestsFile)) { + throw new Error(`Failed tests file not found: ${failedTestsFile}`) + } + + failedTestsData = JSON.parse(fs.readFileSync(failedTestsFile, 'utf8')) + if (!failedTestsData.tests || !Array.isArray(failedTestsData.tests)) { + throw new Error(`Invalid failed tests file format: ${failedTestsFile}`) + } + } + const config = { by, testConfig, options, selectedRuns, + failedTestsData, // Pass failed tests data to workers } const numberOfWorkers = parseInt(workerCount, 10) @@ -69,6 +89,16 @@ module.exports = async function (workerCount, selectedRuns, options) { } await workers.bootstrapAll() await workers.run() + + // Save failed tests if requested + if (options.saveFailedTests !== undefined) { + const result = container.result() + if (result.failedTests.length > 0) { + const fileName = typeof options.saveFailedTests === 'string' ? options.saveFailedTests : 'failed-tests.json' + result.saveFailedTests(fileName) + console.log(`Failed tests saved to: ${fileName}`) + } + } } catch (err) { output.error(err) process.exit(1) diff --git a/lib/command/run.js b/lib/command/run.js index e76257404..600935970 100644 --- a/lib/command/run.js +++ b/lib/command/run.js @@ -2,6 +2,9 @@ const { getConfig, printError, getTestRoot, createOutputDir } = require('./utils const Config = require('../config') const store = require('../store') const Codecept = require('../codecept') +const fs = require('fs') +const path = require('path') +const container = require('../container') module.exports = async function (test, options) { // registering options globally to use in config @@ -28,7 +31,26 @@ module.exports = async function (test, options) { try { codecept.init(testRoot) await codecept.bootstrap() - codecept.loadTests(test) + + // Handle failed tests file loading + if (options.failedTests) { + const failedTestsFile = path.isAbsolute(options.failedTests) ? options.failedTests : path.join(testRoot, options.failedTests) + + if (!fs.existsSync(failedTestsFile)) { + throw new Error(`Failed tests file not found: ${failedTestsFile}`) + } + + const failedTestsData = JSON.parse(fs.readFileSync(failedTestsFile, 'utf8')) + if (!failedTestsData.tests || !Array.isArray(failedTestsData.tests)) { + throw new Error(`Invalid failed tests file format: ${failedTestsFile}`) + } + + // Load all tests first, then filter to only failed ones + codecept.loadTests(test) + codecept.filterByFailedTests(failedTestsData.tests) + } else { + codecept.loadTests(test) + } if (options.verbose) { global.debugMode = true @@ -37,6 +59,16 @@ module.exports = async function (test, options) { } await codecept.run() + + // Save failed tests if requested + if (options.saveFailedTests !== undefined) { + const result = container.result() + if (result.failedTests.length > 0) { + const fileName = typeof options.saveFailedTests === 'string' ? options.saveFailedTests : 'failed-tests.json' + result.saveFailedTests(fileName) + console.log(`Failed tests saved to: ${fileName}`) + } + } } catch (err) { printError(err) process.exitCode = 1 diff --git a/lib/command/workers/runTests.js b/lib/command/workers/runTests.js index f2f8cacd9..407d081c6 100644 --- a/lib/command/workers/runTests.js +++ b/lib/command/workers/runTests.js @@ -20,7 +20,7 @@ const stderr = '' // Requiring of Codecept need to be after tty.getWindowSize is available. const Codecept = require(process.env.CODECEPT_CLASS_PATH || '../../codecept') -const { options, tests, testRoot, workerIndex, poolMode } = workerData +const { options, tests, testRoot, workerIndex, poolMode, failedTestsData } = workerData // hide worker output if (!options.debug && !options.verbose) @@ -38,6 +38,12 @@ const config = deepMerge(getConfig(options.config || testRoot), overrideConfigs) const codecept = new Codecept(config, options) codecept.init(testRoot) codecept.loadTests() + +// Apply failed tests filtering if provided +if (failedTestsData && failedTestsData.tests) { + codecept.filterByFailedTests(failedTestsData.tests) +} + const mocha = container.mocha() if (poolMode) { diff --git a/lib/result.js b/lib/result.js index 9e562d8fc..7bac28a9a 100644 --- a/lib/result.js +++ b/lib/result.js @@ -138,6 +138,28 @@ class Result { fs.writeFileSync(path.join(global.output_dir, fileName), JSON.stringify(this.simplify(), null, 2)) } + /** + * Save failed tests to JSON file + * + * @param {string} fileName + */ + saveFailedTests(fileName) { + if (!fileName) fileName = 'failed-tests.json' + const failedTests = this.failedTests.map(test => ({ + uid: test.uid, + title: test.title, + fullTitle: test.fullTitle ? test.fullTitle() : test.title, + file: test.file, + parent: test.parent ? { title: test.parent.title } : null, + })) + const failedTestsData = { + timestamp: new Date().toISOString(), + count: failedTests.length, + tests: failedTests, + } + fs.writeFileSync(path.join(global.output_dir, fileName), JSON.stringify(failedTestsData, null, 2)) + } + /** * Add stats to result * diff --git a/lib/workers.js b/lib/workers.js index 3ee853023..2114b117c 100644 --- a/lib/workers.js +++ b/lib/workers.js @@ -57,6 +57,7 @@ const createWorker = (workerObject, isPoolMode = false) => { testRoot: workerObject.testRoot, workerIndex: workerObject.workerIndex + 1, poolMode: isPoolMode, + failedTestsData: workerObject.failedTestsData, }, }) worker.on('error', err => output.error(`Worker Error: ${err.stack}`)) @@ -76,7 +77,7 @@ const simplifyObject = object => { }, {}) } -const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns) => { +const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns, failedTestsData) => { selectedRuns = options && options.all && config.multiple ? Object.keys(config.multiple) : selectedRuns if (selectedRuns === undefined || !selectedRuns.length || config.multiple === undefined) { return testGroups.map((tests, index) => { @@ -85,6 +86,9 @@ const createWorkerObjects = (testGroups, config, testRoot, options, selectedRuns workerObj.addTests(tests) workerObj.setTestRoot(testRoot) workerObj.addOptions(options) + if (failedTestsData) { + workerObj.addFailedTestsData(failedTestsData) + } return workerObj }) } @@ -221,6 +225,10 @@ class WorkerObject { ...opts, } } + + addFailedTestsData(failedTestsData) { + this.failedTestsData = failedTestsData + } } class Workers extends EventEmitter { @@ -244,13 +252,18 @@ class Workers extends EventEmitter { this.activeWorkers = new Map() this.maxWorkers = numberOfWorkers // Track original worker count for pool mode + // Handle failed tests filtering + if (config.failedTestsData) { + this.codecept.filterByFailedTests(config.failedTestsData.tests) + } + createOutputDir(config.testConfig) if (numberOfWorkers) this._initWorkers(numberOfWorkers, config) } _initWorkers(numberOfWorkers, config) { this.splitTestsByGroups(numberOfWorkers, config) - this.workers = createWorkerObjects(this.testGroups, this.codecept.config, config.testConfig, config.options, config.selectedRuns) + this.workers = createWorkerObjects(this.testGroups, this.codecept.config, config.testConfig, config.options, config.selectedRuns, config.failedTestsData) this.numberOfWorkers = this.workers.length } diff --git a/test/data/sandbox/failed-tests/codecept.conf.js b/test/data/sandbox/failed-tests/codecept.conf.js new file mode 100644 index 000000000..1a1c170b8 --- /dev/null +++ b/test/data/sandbox/failed-tests/codecept.conf.js @@ -0,0 +1,12 @@ +exports.config = { + tests: './*_test.js', + timeout: 10000, + output: './_output', + helpers: { + FileSystem: {}, + }, + include: {}, + bootstrap: false, + mocha: {}, + name: 'failed-tests', +}; diff --git a/test/data/sandbox/failed-tests/failed_test.js b/test/data/sandbox/failed-tests/failed_test.js new file mode 100644 index 000000000..f6762f422 --- /dev/null +++ b/test/data/sandbox/failed-tests/failed_test.js @@ -0,0 +1,19 @@ +Feature('Failed Tests') + +Scenario('should pass test', ({ I }) => { + // This test should pass +}) + +Scenario('should fail test 1', ({ I }) => { + // This test should fail + throw new Error('Test 1 failed') +}) + +Scenario('should fail test 2', ({ I }) => { + // This test should fail + throw new Error('Test 2 failed') +}) + +Scenario('should pass test 2', ({ I }) => { + // This test should pass +}) diff --git a/test/data/sandbox/failed-tests/failing_test.js b/test/data/sandbox/failed-tests/failing_test.js new file mode 100644 index 000000000..7c53d420f --- /dev/null +++ b/test/data/sandbox/failed-tests/failing_test.js @@ -0,0 +1,9 @@ +Feature('Test with failures') + +Scenario('failing test', ({ I }) => { + throw new Error('This test fails intentionally') +}) + +Scenario('passing test', ({ I }) => { + // This should pass +}) diff --git a/test/data/sandbox/failed-tests/my_test.js b/test/data/sandbox/failed-tests/my_test.js new file mode 100644 index 000000000..e73443ecb --- /dev/null +++ b/test/data/sandbox/failed-tests/my_test.js @@ -0,0 +1,17 @@ +Scenario('should pass test', ({ I }) => { + // This test should pass +}) + +Scenario('should fail test 1', ({ I }) => { + // This test should fail + throw new Error('Test 1 failed') +}) + +Scenario('should fail test 2', ({ I }) => { + // This test should fail + throw new Error('Test 2 failed') +}) + +Scenario('should pass test 2', ({ I }) => { + // This test should pass +}) diff --git a/test/runner/failed_tests_test.js b/test/runner/failed_tests_test.js new file mode 100644 index 000000000..b55e75422 --- /dev/null +++ b/test/runner/failed_tests_test.js @@ -0,0 +1,91 @@ +const fs = require('fs') +const path = require('path') +const exec = require('child_process').exec +const assert = require('assert') + +const runner = path.join(__dirname, '../../bin/codecept.js') +const codecept_dir = path.join(__dirname, '/../data/sandbox/failed-tests') + +describe('Failed Tests Feature', function () { + this.timeout(40000) + + afterEach(() => { + try { + fs.unlinkSync(`${codecept_dir}/_output/failed-tests.json`) + } catch (e) { + // continue regardless of error + } + }) + + it('should save failed tests to JSON file', done => { + exec(`${runner} run --config ${codecept_dir}/codecept.conf.js --save-failed-tests`, (err, stdout) => { + const failedTestsFile = `${codecept_dir}/_output/failed-tests.json` + + // Should have failed tests + assert(err, 'Expected tests to fail') + assert(stdout.match(/Failed tests saved to/), 'Expected failed tests message in stdout') + + // Check if failed tests file was created + assert(fs.existsSync(failedTestsFile), 'Expected failed tests file to be created') + + const failedTests = JSON.parse(fs.readFileSync(failedTestsFile, 'utf8')) + assert(failedTests.hasOwnProperty('timestamp'), 'Expected timestamp property') + assert(failedTests.hasOwnProperty('count'), 'Expected count property') + assert(failedTests.hasOwnProperty('tests'), 'Expected tests property') + assert(Array.isArray(failedTests.tests), 'Expected tests to be an array') + assert(failedTests.tests.length > 0, 'Expected at least one failed test') + + done() + }) + }) + + it('should run only failed tests from JSON file', done => { + // First create a simple failed tests file + const failedTestsFile = `${codecept_dir}/test-failed-tests.json` + const failedTestsData = { + timestamp: new Date().toISOString(), + count: 1, + tests: [ + { + uid: 'should fail test 1', + title: 'should fail test 1', + fullTitle: 'Failed Tests: should fail test 1', + file: `${codecept_dir}/failed_test.js`, + parent: { title: 'Failed Tests' }, + }, + ], + } + + fs.writeFileSync(failedTestsFile, JSON.stringify(failedTestsData, null, 2)) + + exec(`${runner} run --config ${codecept_dir}/codecept.conf.js --failed-tests ${failedTestsFile}`, (err, stdout) => { + // Should still fail but only run the specific failed test + assert(err, 'Expected test to fail') + assert(stdout.match(/should fail test 1/), 'Expected specific failed test to run') + assert(!stdout.match(/should pass test/), 'Should not run passing tests') + + // Clean up + fs.unlinkSync(failedTestsFile) + done() + }) + }) + + it('should work with run-workers command', done => { + exec(`${runner} run-workers 2 --config ${codecept_dir}/codecept.conf.js --save-failed-tests`, (err, stdout) => { + const failedTestsFile = `${codecept_dir}/_output/failed-tests.json` + + // Should have failed tests + assert(err, 'Expected tests to fail') + assert(stdout.match(/Failed tests saved to/), 'Expected failed tests message in stdout') + + // Check if failed tests file was created + assert(fs.existsSync(failedTestsFile), 'Expected failed tests file to be created') + + const failedTests = JSON.parse(fs.readFileSync(failedTestsFile, 'utf8')) + assert(Array.isArray(failedTests.tests), 'Expected tests to be an array') + assert(failedTests.tests.length > 0, 'Expected at least one failed test') + + done() + }) + }) +}) diff --git a/test/unit/helper/json_response_onResponse_test.js b/test/unit/helper/json_response_onResponse_test.js index 3fb35108f..2918ed3ef 100644 --- a/test/unit/helper/json_response_onResponse_test.js +++ b/test/unit/helper/json_response_onResponse_test.js @@ -10,13 +10,22 @@ let api_url = TestHelper.jsonServerUrl() describe('REST onResponse Hook Wrapper', () => { let rest + let isNetworkAvailable = false beforeEach(async () => { Container.helpers({}) try { await axios.get(`${api_url}`, { timeout: 1000 }) // Check if the server is reachable + isNetworkAvailable = true } catch (e) { - api_url = fallBackURL // Fallback to alternative endpoint + try { + await axios.get(fallBackURL, { timeout: 1000 }) // Check if fallback is reachable + api_url = fallBackURL // Fallback to alternative endpoint + isNetworkAvailable = true + } catch (fallbackError) { + isNetworkAvailable = false + return // Skip REST initialization if no network is available + } } rest = new REST({ @@ -33,27 +42,42 @@ describe('REST onResponse Hook Wrapper', () => { rest = null }) - it('should store response in this.response', async () => { - const response = await rest.sendGetRequest('/posts/1') - assert.ok(response, 'Expected response to be set on REST instance') - assert.equal(response.status, 200) + it('should store response in this.response', function() { + if (!isNetworkAvailable) { + return this.skip() + } + return rest.sendGetRequest('/posts/1').then(response => { + assert.ok(response, 'Expected response to be set on REST instance') + assert.equal(response.status, 200) + }) }) - it('should call onResponse function and preserve modifications', async () => { - const response = await rest.sendGetRequest('/posts/1') - assert.ok(response.customFlag, 'Expected original onResponse to run and modify response') + it('should call onResponse function and preserve modifications', function() { + if (!isNetworkAvailable) { + return this.skip() + } + return rest.sendGetRequest('/posts/1').then(response => { + assert.ok(response.customFlag, 'Expected original onResponse to run and modify response') + }) }) - it('should not fail if original onResponse is not set in the config', async () => { + it('should not fail if original onResponse is not set in the config', function() { + if (!isNetworkAvailable) { + return this.skip() + } const restNoHook = new REST({ endpoint: api_url }) restNoHook._before() - const response = await restNoHook.sendGetRequest('/posts/1') - assert.ok(response, 'Expected response to be returned') - assert.equal(response.status, 200) + return restNoHook.sendGetRequest('/posts/1').then(response => { + assert.ok(response, 'Expected response to be returned') + assert.equal(response.status, 200) + }) }) - it('should not throw if onResponse is not a function in the config', async () => { + it('should not throw if onResponse is not a function in the config', function() { + if (!isNetworkAvailable) { + return this.skip() + } const restInvalid = new REST({ endpoint: api_url, onResponse: undefined, @@ -61,8 +85,9 @@ describe('REST onResponse Hook Wrapper', () => { restInvalid._before() - const response = await restInvalid.sendGetRequest('/posts/1') - assert.ok(response) - assert.equal(response.status, 200) + return restInvalid.sendGetRequest('/posts/1').then(response => { + assert.ok(response) + assert.equal(response.status, 200) + }) }) })