diff --git a/lib/plugin/stepByStepReport.js b/lib/plugin/stepByStepReport.js index d2aae48aa..f246aa65d 100644 --- a/lib/plugin/stepByStepReport.js +++ b/lib/plugin/stepByStepReport.js @@ -40,6 +40,18 @@ const templates = {} * npx codeceptjs run --plugins stepByStepReport * ``` * + * Run tests with workers: + * + * ``` + * npx codeceptjs run-workers 2 --plugins stepByStepReport + * ``` + * + * Run tests with multiple configurations: + * + * ``` + * npx codeceptjs run-multiple --all --plugins stepByStepReport + * ``` + * * #### Configuration * * ```js @@ -60,6 +72,20 @@ const templates = {} * * `screenshotsForAllureReport`: If Allure plugin is enabled this plugin attaches each saved screenshot to allure report. Default: false. * * `disableScreenshotOnFail : Disables the capturing of screeshots after the failed step. Default: true. * + * #### Worker and Multiple Run Support + * + * When using `run-workers`, `run-multiple`, or combinations thereof, the plugin automatically + * detects all worker and run processes and creates a consolidated step-by-step report that + * includes screenshots from all processes while keeping them in their original directories. + * + * Screenshots remain in their respective process directories for traceability: + * - **run-workers**: Screenshots saved in `/output/worker1/`, `/output/worker2/`, etc. + * - **run-multiple**: Screenshots saved in `/output/config_name_hash/`, etc. + * - **Mixed scenarios**: Screenshots saved in `/output/config_name_hash/worker1/`, etc. + * + * The final consolidated report links to all screenshots while preserving their original locations + * and indicating which process or worker they came from. + * * @param {*} config */ @@ -87,8 +113,12 @@ module.exports = function (config) { const recordedTests = {} const pad = '0000' + const reportDir = config.output ? path.resolve(global.codecept_dir, config.output) : defaultConfig.output + // Ensure the report directory exists + mkdirp.sync(reportDir) + event.dispatcher.on(event.suite.before, suite => { stepNum = -1 }) @@ -137,11 +167,70 @@ module.exports = function (config) { event.dispatcher.on(event.workers.result, async () => { await recorder.add(() => { - const recordedTests = getRecordFoldersWithDetails(reportDir) + // For workers and run-multiple scenarios, we need to search across multiple directories + // to find all screenshot folders from different processes + const recordedTests = getRecordFoldersFromAllDirectories() generateRecordsHtml(recordedTests) }) }) + function getRecordFoldersFromAllDirectories() { + let results = {} + + // Determine the base output directory to search from + const baseOutputDir = config.output ? path.resolve(global.codecept_dir, config.output) : defaultConfig.output + + // Function to recursively search for record folders in a directory + function searchForRecordFolders(searchDir, basePath = '') { + try { + if (!fs.existsSync(searchDir)) return + + const items = fs.readdirSync(searchDir, { withFileTypes: true }) + + items.forEach(item => { + if (item.isDirectory()) { + const itemPath = path.join(searchDir, item.name) + const relativePath = basePath ? path.join(basePath, item.name) : item.name + + // If this is a record folder, process it + if (item.name.startsWith('record_')) { + const indexPath = path.join(itemPath, 'index.html') + + let name = '' + if (fs.existsSync(indexPath)) { + try { + const htmlContent = fs.readFileSync(indexPath, 'utf-8') + const $ = cheerio.load(htmlContent) + name = $('.navbar-brand').text().trim() + } catch (err) { + console.error(`Error reading index.html in ${itemPath}:`, err.message) + } + } + + // Include the relative path to show which process/worker this came from + const displayName = basePath ? `${name} (${basePath})` : name + results[displayName || 'Unknown'] = path.join(relativePath, 'index.html') + } else { + // Continue searching in subdirectories (worker folders, run-multiple folders) + searchForRecordFolders(itemPath, relativePath) + } + } + }) + } catch (err) { + console.error(`Error searching directory ${searchDir}:`, err.message) + } + } + + // Start the search from the base output directory + searchForRecordFolders(baseOutputDir) + + // Also check the current reportDir for backwards compatibility + const currentDirResults = getRecordFoldersWithDetails(reportDir) + Object.assign(results, currentDirResults) + + return results + } + function getRecordFoldersWithDetails(dirPath) { let results = {} @@ -186,9 +275,32 @@ module.exports = function (config) { records: links, }) - fs.writeFileSync(path.join(reportDir, 'records.html'), indexHTML) + // Determine where to write the main records.html file + // For worker/run-multiple scenarios, we want to write to the base output directory + let recordsHtmlDir = reportDir + + if (global.codecept_dir && (process.env.RUNS_WITH_WORKERS === 'true' || process.argv.some(arg => arg === '--child'))) { + // Extract base output directory by removing worker/run-specific segments + const baseOutputDir = config.output ? path.resolve(global.codecept_dir, config.output) : defaultConfig.output + let actualBaseDir = baseOutputDir + + // For workers: strip worker directory segment + if (process.env.RUNS_WITH_WORKERS === 'true') { + actualBaseDir = actualBaseDir.replace(/[/\\][^/\\]+$/, '') + } + + // For run-multiple: strip run directory segment + if (process.argv.some(arg => arg === '--child')) { + actualBaseDir = actualBaseDir.replace(/[/\\][^/\\]+$/, '') + } + + recordsHtmlDir = actualBaseDir + mkdirp.sync(recordsHtmlDir) + } + + fs.writeFileSync(path.join(recordsHtmlDir, 'records.html'), indexHTML) - output.print(`${figures.circleFilled} Step-by-step preview: ${colors.white.bold(`file://${reportDir}/records.html`)}`) + output.print(`${figures.circleFilled} Step-by-step preview: ${colors.white.bold(`file://${recordsHtmlDir}/records.html`)}`) } async function persistStep(step) { diff --git a/test/acceptance/stepByStepReport_regression_test.js b/test/acceptance/stepByStepReport_regression_test.js new file mode 100644 index 000000000..e70dc79d4 --- /dev/null +++ b/test/acceptance/stepByStepReport_regression_test.js @@ -0,0 +1,229 @@ +const path = require('path') +const fs = require('fs') +const { exec } = require('child_process') +const { expect } = require('expect') + +const runner = path.join(__dirname, '../../bin/codecept.js') +const codecept_dir = path.join(__dirname, '../data/sandbox') + +describe('stepByStepReport regression tests', function () { + this.timeout(120000) // Increased timeout for acceptance tests + + const outputDir = path.join(codecept_dir, 'output') + + beforeEach(() => { + // Clean up output directory before each test + if (fs.existsSync(outputDir)) { + fs.rmSync(outputDir, { recursive: true, force: true }) + } + }) + + afterEach(() => { + // Clean up output directory after each test + if (fs.existsSync(outputDir)) { + fs.rmSync(outputDir, { recursive: true, force: true }) + } + }) + + function createConfig(name, extraConfig = {}) { + const config = ` +// Override the standard acting helpers to include FakeDriver for testing +const Container = require('../../lib/container') +const originalHelpers = Container.STANDARD_ACTING_HELPERS +Object.defineProperty(Container, 'STANDARD_ACTING_HELPERS', { + get: () => [...originalHelpers, 'FakeDriver'] +}) + +exports.config = { + tests: './stepbystep_test.js', + timeout: 30000, + output: './output', + helpers: { + FakeDriver: { + require: '../fake_driver', + browser: 'dummy', + windowSize: '1024x768' + }, + }, + plugins: { + stepByStepReport: { + enabled: true, + deleteSuccessful: false, + ...${JSON.stringify(extraConfig)} + }, + }, + include: {}, + bootstrap: false, + mocha: {}, + name: '${name}', +} +` + const configPath = path.join(codecept_dir, `codecept.${name}.js`) + fs.writeFileSync(configPath, config) + return configPath + } + + function runCommand(command) { + return new Promise((resolve, reject) => { + exec(command, { cwd: codecept_dir }, (err, stdout, stderr) => { + resolve({ err, stdout, stderr }) + }) + }) + } + + it('should handle run-workers without consolidating screenshots', async function () { + const configPath = createConfig('workers-test') + + const command = `${runner} run-workers 2 --config ${configPath} --grep "@stepbystep"` + const result = await runCommand(command) + + console.log('STDOUT:', result.stdout) + console.log('STDERR:', result.stderr) + + // Key regression test: ensure no stepByStepReport consolidation directory + const consolidatedDir = path.join(outputDir, 'stepByStepReport') + expect(fs.existsSync(consolidatedDir)).toBe(false) + + console.log('✓ No stepByStepReport consolidation directory created for run-workers') + + // Verify basic functionality without requiring screenshots + expect(result.stdout).toContain('CodeceptJS') + + // Clean up + fs.unlinkSync(configPath) + }) + + it('should handle run-multiple without consolidating screenshots', async function () { + const multipleConfig = ` +// Override the standard acting helpers to include FakeDriver for testing +const Container = require('../../lib/container') +const originalHelpers = Container.STANDARD_ACTING_HELPERS +Object.defineProperty(Container, 'STANDARD_ACTING_HELPERS', { + get: () => [...originalHelpers, 'FakeDriver'] +}) + +exports.config = { + tests: './stepbystep_test.js', + timeout: 30000, + output: './output', + helpers: { + FakeDriver: { + require: '../fake_driver', + browser: 'dummy', + windowSize: '1024x768' + }, + }, + plugins: { + stepByStepReport: { + enabled: true, + deleteSuccessful: false, + }, + }, + include: {}, + bootstrap: false, + mocha: {}, + name: 'multiple-test', + multiple: { + basic: { + browsers: ['chrome'] + } + } +} +` + + const configPath = path.join(codecept_dir, 'codecept.multiple.js') + fs.writeFileSync(configPath, multipleConfig) + + const command = `${runner} run-multiple basic --config ${configPath} --grep "@stepbystep"` + const result = await runCommand(command) + + console.log('STDOUT:', result.stdout) + console.log('STDERR:', result.stderr) + + // Key regression test: ensure no stepByStepReport consolidation directory + const consolidatedDir = path.join(outputDir, 'stepByStepReport') + expect(fs.existsSync(consolidatedDir)).toBe(false) + + console.log('✓ No stepByStepReport consolidation directory created for run-multiple') + + // Verify that the command runs successfully with run-multiple + expect(result.stdout).toContain('CodeceptJS') + + // Clean up + fs.unlinkSync(configPath) + }) + + it('should handle regular run command with backward compatibility', async function () { + const configPath = createConfig('regular') + + const command = `${runner} run --config ${configPath} --grep "@stepbystep"` + const result = await runCommand(command) + + console.log('STDOUT:', result.stdout) + console.log('STDERR:', result.stderr) + + // Key regression test: ensure no stepByStepReport consolidation directory + const consolidatedDir = path.join(outputDir, 'stepByStepReport') + expect(fs.existsSync(consolidatedDir)).toBe(false) + + console.log('✓ No stepByStepReport consolidation directory created for regular run') + + // Verify backward compatibility - regular run should work + expect(result.stdout).toContain('CodeceptJS') + + // Clean up + fs.unlinkSync(configPath) + }) + + it('should handle custom output directories', async function () { + const configPath = createConfig('custom-output', { output: './output/custom' }) + + const command = `${runner} run-workers 2 --config ${configPath} --grep "@stepbystep"` + const result = await runCommand(command) + + console.log('STDOUT:', result.stdout) + console.log('STDERR:', result.stderr) + + // Check that no consolidation happens in any output directory + const mainConsolidatedDir = path.join(outputDir, 'stepByStepReport') + const customConsolidatedDir = path.join(outputDir, 'custom', 'stepByStepReport') + + expect(fs.existsSync(mainConsolidatedDir)).toBe(false) + expect(fs.existsSync(customConsolidatedDir)).toBe(false) + + console.log('✓ No stepByStepReport consolidation directory created with custom output') + + // Clean up + fs.unlinkSync(configPath) + }) + + it('should not crash with stepByStepReport plugin enabled', async function () { + // This test ensures the plugin initialization and basic operations work + // without causing crashes across different execution modes + const configPath = createConfig('no-crash-test') + + const commands = [`${runner} run --config ${configPath} --grep "@stepbystep"`, `${runner} run-workers 2 --config ${configPath} --grep "@stepbystep"`] + + for (const command of commands) { + const result = await runCommand(command) + + console.log(`Command: ${command}`) + console.log('STDOUT:', result.stdout) + console.log('STDERR:', result.stderr) + + // Ensure the plugin doesn't cause crashes + expect(result.stdout).toContain('CodeceptJS') + expect(result.stderr).not.toContain('Cannot read properties of undefined') + expect(result.stderr).not.toContain('TypeError') + + // Key regression test + const consolidatedDir = path.join(outputDir, 'stepByStepReport') + expect(fs.existsSync(consolidatedDir)).toBe(false) + } + + console.log('✓ Plugin works without crashes across execution modes') + + // Clean up + fs.unlinkSync(configPath) + }) +}) diff --git a/test/data/fake_driver.js b/test/data/fake_driver.js index 9f34d45cd..eede2aeb1 100644 --- a/test/data/fake_driver.js +++ b/test/data/fake_driver.js @@ -1,4 +1,6 @@ const Helper = require('../../lib/helper') +const fs = require('fs') +const path = require('path') class FakeDriver extends Helper { printBrowser() { @@ -8,6 +10,34 @@ class FakeDriver extends Helper { printWindowSize() { this.debug(this.config.windowSize) } + + wait(seconds) { + // Simple wait implementation + return new Promise(resolve => setTimeout(resolve, seconds * 1000)) + } + + see(text) { + // Always fail to trigger screenshot saving + throw new Error(`Expected to see "${text}" but this is a fake driver`) + } + + async saveScreenshot(fileName, fullPage) { + // Create a fake screenshot (1x1 PNG) for testing purposes + const fakePngBuffer = Buffer.from([ + 0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0x00, 0x00, 0x00, 0x0d, 0x49, 0x48, 0x44, 0x52, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x37, 0x6e, 0xf9, 0x24, 0x00, 0x00, + 0x00, 0x0a, 0x49, 0x44, 0x41, 0x54, 0x78, 0x9c, 0x62, 0x00, 0x00, 0x00, 0x02, 0x00, 0x01, 0xe5, 0x27, 0xde, 0xfc, 0x00, 0x00, 0x00, 0x00, 0x49, 0x45, 0x4e, 0x44, 0xae, 0x42, 0x60, 0x82, + ]) + + // Ensure directory exists + const dir = path.dirname(fileName) + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }) + } + + // Write the fake PNG file + fs.writeFileSync(fileName, fakePngBuffer) + this.debug(`Fake screenshot saved to: ${fileName}`) + } } module.exports = FakeDriver diff --git a/test/data/sandbox/Another_test_for_multiple_reports_@stepbystep_@failed.failed.png b/test/data/sandbox/Another_test_for_multiple_reports_@stepbystep_@failed.failed.png new file mode 100644 index 000000000..8a842651a Binary files /dev/null and b/test/data/sandbox/Another_test_for_multiple_reports_@stepbystep_@failed.failed.png differ diff --git a/test/data/sandbox/codecept.addt.js b/test/data/sandbox/codecept.addt.js index 78d01a5d3..48acf694b 100644 --- a/test/data/sandbox/codecept.addt.js +++ b/test/data/sandbox/codecept.addt.js @@ -2,10 +2,9 @@ exports.config = { tests: './*_test.addt.js', timeout: 10000, output: './output', - helpers: { - }, + helpers: {}, include: {}, bootstrap: false, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.async.bootstrapall.multiple.code.js b/test/data/sandbox/codecept.async.bootstrapall.multiple.code.js index 738a5ac14..406041f36 100644 --- a/test/data/sandbox/codecept.async.bootstrapall.multiple.code.js +++ b/test/data/sandbox/codecept.async.bootstrapall.multiple.code.js @@ -1,4 +1,4 @@ -const event = require('../../../lib/event'); +const event = require('../../../lib/event') exports.config = { tests: './*_test.js', @@ -17,12 +17,12 @@ exports.config = { }, }, bootstrapAll: async () => { - await Promise.resolve('inside Promise').then(res => console.log(`Results: ${res}`)); + await Promise.resolve('inside Promise').then(res => console.log(`Results: ${res}`)) event.dispatcher.on(event.multiple.before, () => { - console.log('"event.multiple.before" is called'); - }); + console.log('"event.multiple.before" is called') + }) }, teardownAll: async () => { - console.log('"teardownAll" is called.'); + console.log('"teardownAll" is called.') }, -}; +} diff --git a/test/data/sandbox/codecept.bdd.js b/test/data/sandbox/codecept.bdd.js index ac820c35e..cfb76dc4a 100644 --- a/test/data/sandbox/codecept.bdd.js +++ b/test/data/sandbox/codecept.bdd.js @@ -9,13 +9,10 @@ exports.config = { }, gherkin: { features: './features/*.feature', - steps: [ - './features/step_definitions/my_steps.js', - './features/step_definitions/my_other_steps.js', - ], + steps: ['./features/step_definitions/my_steps.js', './features/step_definitions/my_other_steps.js'], }, include: {}, bootstrap: false, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.beforetest.failure.js b/test/data/sandbox/codecept.beforetest.failure.js index 1f00629c8..747d06c87 100644 --- a/test/data/sandbox/codecept.beforetest.failure.js +++ b/test/data/sandbox/codecept.beforetest.failure.js @@ -9,4 +9,4 @@ exports.config = { bootstrap: false, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.bootstrapall.multiple.code.js b/test/data/sandbox/codecept.bootstrapall.multiple.code.js index 1c3c1a5ea..1d4170c35 100644 --- a/test/data/sandbox/codecept.bootstrapall.multiple.code.js +++ b/test/data/sandbox/codecept.bootstrapall.multiple.code.js @@ -16,4 +16,4 @@ exports.config = { }, bootstrapAll: () => console.log('"bootstrapAll" is called.'), teardownAll: () => console.log('"teardownAll" is called.'), -}; +} diff --git a/test/data/sandbox/codecept.bootstrapall.multiple.function.js b/test/data/sandbox/codecept.bootstrapall.multiple.function.js index 3da43e442..f4419cb48 100644 --- a/test/data/sandbox/codecept.bootstrapall.multiple.function.js +++ b/test/data/sandbox/codecept.bootstrapall.multiple.function.js @@ -16,4 +16,4 @@ exports.config = { }, bootstrapAll: './bootstrapall.function.js', teardownAll: './teardownall.function.js', -}; +} diff --git a/test/data/sandbox/codecept.bootstrapall.multiple.object.js b/test/data/sandbox/codecept.bootstrapall.multiple.object.js index 047cc339a..3b6dc9ebe 100644 --- a/test/data/sandbox/codecept.bootstrapall.multiple.object.js +++ b/test/data/sandbox/codecept.bootstrapall.multiple.object.js @@ -16,4 +16,4 @@ exports.config = { }, bootstrapAll: './bootstrapall.object.js', teardownAll: './teardownall.object.js', -}; +} diff --git a/test/data/sandbox/codecept.customLocator.js b/test/data/sandbox/codecept.customLocator.js index 9cdd1b3f7..42897ccbf 100644 --- a/test/data/sandbox/codecept.customLocator.js +++ b/test/data/sandbox/codecept.customLocator.js @@ -20,4 +20,4 @@ exports.config = { attribute: 'data-testid', }, }, -}; +} diff --git a/test/data/sandbox/codecept.customworker.js b/test/data/sandbox/codecept.customworker.js index fe4a19ae0..ee34e973b 100644 --- a/test/data/sandbox/codecept.customworker.js +++ b/test/data/sandbox/codecept.customworker.js @@ -13,14 +13,14 @@ exports.config = { }, include: {}, bootstrap: async () => { - process.stdout.write('bootstrap b1+'); + process.stdout.write('bootstrap b1+') return new Promise(done => { setTimeout(() => { - process.stdout.write('b2'); - done(); - }, 100); - }); + process.stdout.write('b2') + done() + }, 100) + }) }, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.ddt.js b/test/data/sandbox/codecept.ddt.js index 4ccdc0158..f1b12523c 100644 --- a/test/data/sandbox/codecept.ddt.js +++ b/test/data/sandbox/codecept.ddt.js @@ -2,10 +2,9 @@ exports.config = { tests: './*_test.ddt.js', timeout: 10000, output: './output', - helpers: { - }, + helpers: {}, include: {}, bootstrap: false, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.dummy.bdd.js b/test/data/sandbox/codecept.dummy.bdd.js index 652b8da7e..1ae37700d 100644 --- a/test/data/sandbox/codecept.dummy.bdd.js +++ b/test/data/sandbox/codecept.dummy.bdd.js @@ -9,13 +9,10 @@ exports.config = { }, gherkin: { features: './support/dummy.feature', - steps: [ - './features/step_definitions/my_steps.js', - './features/step_definitions/my_other_steps.js', - ], + steps: ['./features/step_definitions/my_steps.js', './features/step_definitions/my_other_steps.js'], }, include: {}, bootstrap: false, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.duplicate.bdd.js b/test/data/sandbox/codecept.duplicate.bdd.js index 808d00243..0dc1584bb 100644 --- a/test/data/sandbox/codecept.duplicate.bdd.js +++ b/test/data/sandbox/codecept.duplicate.bdd.js @@ -9,13 +9,10 @@ exports.config = { }, gherkin: { features: './support/duplicate.feature', - steps: [ - './features/step_definitions/my_steps.js', - './features/step_definitions/my_other_steps.js', - ], + steps: ['./features/step_definitions/my_steps.js', './features/step_definitions/my_other_steps.js'], }, include: {}, bootstrap: false, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.failed.js b/test/data/sandbox/codecept.failed.js index 23dda742c..5bebf52a0 100644 --- a/test/data/sandbox/codecept.failed.js +++ b/test/data/sandbox/codecept.failed.js @@ -9,4 +9,4 @@ exports.config = { bootstrap: false, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.flaky.js b/test/data/sandbox/codecept.flaky.js index 86ad4507c..f3ddaa9b2 100644 --- a/test/data/sandbox/codecept.flaky.js +++ b/test/data/sandbox/codecept.flaky.js @@ -2,10 +2,9 @@ exports.config = { tests: './*_test.flaky.js', timeout: 10000, output: './output', - helpers: { - }, + helpers: {}, include: {}, bootstrap: false, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.gddt.js b/test/data/sandbox/codecept.gddt.js index 770f6ee34..de6d1d7a2 100644 --- a/test/data/sandbox/codecept.gddt.js +++ b/test/data/sandbox/codecept.gddt.js @@ -2,10 +2,9 @@ exports.config = { tests: './*_test.gddt.js', timeout: 10000, output: './output', - helpers: { - }, + helpers: {}, include: {}, bootstrap: false, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.glob.js b/test/data/sandbox/codecept.glob.js index f8a467ff4..df13a4cd2 100644 --- a/test/data/sandbox/codecept.glob.js +++ b/test/data/sandbox/codecept.glob.js @@ -9,4 +9,4 @@ exports.config = { bootstrap: false, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.grep.2.js b/test/data/sandbox/codecept.grep.2.js index bac5f274e..f634c4ebb 100644 --- a/test/data/sandbox/codecept.grep.2.js +++ b/test/data/sandbox/codecept.grep.2.js @@ -13,4 +13,4 @@ exports.config = { bootstrap: false, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.grep.js b/test/data/sandbox/codecept.grep.js index 472696ee0..8da333279 100644 --- a/test/data/sandbox/codecept.grep.js +++ b/test/data/sandbox/codecept.grep.js @@ -10,4 +10,4 @@ exports.config = { bootstrap: false, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.multiple.features.js b/test/data/sandbox/codecept.multiple.features.js index c4ffcc9ff..daf70bf84 100644 --- a/test/data/sandbox/codecept.multiple.features.js +++ b/test/data/sandbox/codecept.multiple.features.js @@ -10,10 +10,7 @@ exports.config = { }, gherkin: { features: './features/*.feature', - steps: [ - './features/step_definitions/my_steps.js', - './features/step_definitions/my_other_steps.js', - ], + steps: ['./features/step_definitions/my_steps.js', './features/step_definitions/my_other_steps.js'], }, include: {}, bootstrap: false, @@ -23,4 +20,4 @@ exports.config = { chunks: 2, }, }, -}; +} diff --git a/test/data/sandbox/codecept.multiple.initFailure.js b/test/data/sandbox/codecept.multiple.initFailure.js index fe16aaa37..cb78ade84 100644 --- a/test/data/sandbox/codecept.multiple.initFailure.js +++ b/test/data/sandbox/codecept.multiple.initFailure.js @@ -10,14 +10,11 @@ exports.config = { multiple: { default: { - browsers: [ - 'chrome', - { browser: 'firefox' }, - ], + browsers: ['chrome', { browser: 'firefox' }], }, }, include: {}, bootstrap: false, mocha: {}, name: 'multiple-init-failure', -}; +} diff --git a/test/data/sandbox/codecept.non-test-events-worker.js b/test/data/sandbox/codecept.non-test-events-worker.js index af3bc81b4..d149b7380 100644 --- a/test/data/sandbox/codecept.non-test-events-worker.js +++ b/test/data/sandbox/codecept.non-test-events-worker.js @@ -9,13 +9,13 @@ exports.config = { }, }, include: {}, - bootstrap: (done) => { - process.stdout.write('bootstrap b1+'); + bootstrap: done => { + process.stdout.write('bootstrap b1+') setTimeout(() => { - process.stdout.write('b2'); - done(); - }, 1000); + process.stdout.write('b2') + done() + }, 1000) }, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.require.multiple.several.js b/test/data/sandbox/codecept.require.multiple.several.js index 5a68e6d49..dd1276068 100644 --- a/test/data/sandbox/codecept.require.multiple.several.js +++ b/test/data/sandbox/codecept.require.multiple.several.js @@ -12,10 +12,7 @@ exports.config = { require: ['requiredModule', 'requiredModule2'], multiple: { default: { - browsers: [ - 'chrome', - { browser: 'firefox' }, - ], + browsers: ['chrome', { browser: 'firefox' }], }, }, -}; +} diff --git a/test/data/sandbox/codecept.scenario-stale.js b/test/data/sandbox/codecept.scenario-stale.js index f07399b38..eda08fc09 100644 --- a/test/data/sandbox/codecept.scenario-stale.js +++ b/test/data/sandbox/codecept.scenario-stale.js @@ -7,4 +7,4 @@ exports.config = { bootstrap: false, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.testevents.js b/test/data/sandbox/codecept.testevents.js index 5907d0ce6..e92268629 100644 --- a/test/data/sandbox/codecept.testevents.js +++ b/test/data/sandbox/codecept.testevents.js @@ -1,7 +1,7 @@ -const eventHandlers = require('./eventHandlers'); -require('../fake_driver'); +const eventHandlers = require('./eventHandlers') +require('../fake_driver') -eventHandlers.setConsoleLogging(true); +eventHandlers.setConsoleLogging(true) module.exports.config = { tests: './*_test.testevents.js', @@ -16,4 +16,4 @@ module.exports.config = { bootstrap: false, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.workers-glob.conf.js b/test/data/sandbox/codecept.workers-glob.conf.js index 7bb7946b2..38b3d7d43 100644 --- a/test/data/sandbox/codecept.workers-glob.conf.js +++ b/test/data/sandbox/codecept.workers-glob.conf.js @@ -11,13 +11,13 @@ exports.config = { include: {}, bootstrap: async () => { return new Promise(done => { - process.stdout.write('bootstrap b1+'); + process.stdout.write('bootstrap b1+') setTimeout(() => { - process.stdout.write('b2'); - done(); - }, 1000); - }); + process.stdout.write('b2') + done() + }, 1000) + }) }, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.workers-incorrect-glob.conf.js b/test/data/sandbox/codecept.workers-incorrect-glob.conf.js index e87438f7a..823da6bcc 100644 --- a/test/data/sandbox/codecept.workers-incorrect-glob.conf.js +++ b/test/data/sandbox/codecept.workers-incorrect-glob.conf.js @@ -9,13 +9,13 @@ exports.config = { }, }, include: {}, - bootstrap: (done) => { - process.stdout.write('bootstrap b1+'); + bootstrap: done => { + process.stdout.write('bootstrap b1+') setTimeout(() => { - process.stdout.write('b2'); - done(); - }, 1000); + process.stdout.write('b2') + done() + }, 1000) }, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/codecept.workers.conf.js b/test/data/sandbox/codecept.workers.conf.js index 0439ccdaa..bb1719c39 100644 --- a/test/data/sandbox/codecept.workers.conf.js +++ b/test/data/sandbox/codecept.workers.conf.js @@ -11,13 +11,13 @@ exports.config = { include: {}, bootstrap: async () => { return new Promise(done => { - process.stdout.write('bootstrap b1+'); + process.stdout.write('bootstrap b1+') setTimeout(() => { - process.stdout.write('b2'); - done(); - }, 100); - }); + process.stdout.write('b2') + done() + }, 100) + }) }, mocha: {}, name: 'sandbox', -}; +} diff --git a/test/data/sandbox/stepbystep_test.js b/test/data/sandbox/stepbystep_test.js new file mode 100644 index 000000000..0b83c96fa --- /dev/null +++ b/test/data/sandbox/stepbystep_test.js @@ -0,0 +1,17 @@ +Feature('StepByStep Test') + +Scenario('Test with steps that should create screenshots @stepbystep', ({ I }) => { + I.printBrowser() + I.wait(0.1) // Small wait to create different steps + I.printWindowSize() + I.wait(0.1) + I.printBrowser() +}) + +Scenario('Another test for multiple reports @stepbystep', ({ I }) => { + I.printWindowSize() + I.wait(0.1) + I.printBrowser() + I.wait(0.1) + I.printWindowSize() +}) diff --git a/test/plugin/stepByStepReport_test.js b/test/plugin/stepByStepReport_test.js new file mode 100644 index 000000000..1f4140c05 --- /dev/null +++ b/test/plugin/stepByStepReport_test.js @@ -0,0 +1,194 @@ +const path = require('path') +const fs = require('fs') +const { exec } = require('child_process') +const { expect } = require('expect') + +const runner = path.join(__dirname, '../../bin/codecept.js') +const codecept_dir = path.join(__dirname, '../data/sandbox') + +describe('stepByStepReport plugin with different run commands', function () { + this.timeout(60000) + + const outputDir = path.join(codecept_dir, 'output') + + beforeEach(() => { + // Clean up output directory before each test + if (fs.existsSync(outputDir)) { + fs.rmSync(outputDir, { recursive: true, force: true }) + } + }) + + it('should keep screenshots in worker directories and create consolidated report for run-workers', function (done) { + const config = ` +exports.config = { + tests: './*_test.js', + timeout: 10000, + output: './output', + helpers: { + FakeDriver: { + require: '../fake_driver', + browser: 'dummy', + }, + }, + plugins: { + stepByStepReport: { + enabled: true, + deleteSuccessful: false, + }, + }, + include: {}, + bootstrap: false, + mocha: {}, + name: 'stepByStepTest', +} +` + + const configPath = path.join(codecept_dir, 'codecept.stepbystep.js') + fs.writeFileSync(configPath, config) + + const command = `${runner} run-workers 2 --config ${configPath} --grep "@stepbystep"` + + exec(command, (err, stdout, stderr) => { + console.log('STDOUT:', stdout) + console.log('STDERR:', stderr) + + // Screenshots should remain in worker directories, not consolidated + const worker1Dir = path.join(outputDir, 'worker1') + const worker2Dir = path.join(outputDir, 'worker2') + + // Check that worker directories exist (if tests ran) + if (fs.existsSync(outputDir)) { + const items = fs.readdirSync(outputDir) + console.log('Output directory contents:', items) + + // The consolidated records.html should be in the base output directory + const recordsHtml = path.join(outputDir, 'records.html') + + // If tests ran and created screenshots, we should see evidence of them + console.log('Records.html exists:', fs.existsSync(recordsHtml)) + + // Screenshots should NOT be in a consolidated stepByStepReport directory + const stepByStepDir = path.join(outputDir, 'stepByStepReport') + expect(fs.existsSync(stepByStepDir)).toBe(false) + } + + // Clean up + fs.unlinkSync(configPath) + + done() + }) + }) + + it('should keep screenshots in run-multiple directories and create consolidated report', function (done) { + const multipleConfig = ` +exports.config = { + tests: './*_test.js', + timeout: 10000, + output: './output', + helpers: { + FakeDriver: { + require: '../fake_driver', + browser: 'dummy', + }, + }, + plugins: { + stepByStepReport: { + enabled: true, + deleteSuccessful: false, + }, + }, + include: {}, + bootstrap: false, + mocha: {}, + name: 'stepByStepTest', + multiple: { + basic: { + browsers: ['chrome'] + }, + smoke: { + browsers: ['firefox'] + } + } +} +` + + const configPath = path.join(codecept_dir, 'codecept.multiple.js') + fs.writeFileSync(configPath, multipleConfig) + + const command = `${runner} run-multiple basic --config ${configPath} --grep "@stepbystep"` + + exec(command, (err, stdout, stderr) => { + console.log('STDOUT:', stdout) + console.log('STDERR:', stderr) + + if (fs.existsSync(outputDir)) { + const items = fs.readdirSync(outputDir) + console.log('Output directory contents:', items) + + // Screenshots should NOT be in a consolidated stepByStepReport directory + const stepByStepDir = path.join(outputDir, 'stepByStepReport') + expect(fs.existsSync(stepByStepDir)).toBe(false) + + // The consolidated records.html should be in the base output directory + const recordsHtml = path.join(outputDir, 'records.html') + console.log('Records.html exists:', fs.existsSync(recordsHtml)) + } + + // Clean up + fs.unlinkSync(configPath) + + done() + }) + }) + + it('should work with regular run command (backward compatibility)', function (done) { + const config = ` +exports.config = { + tests: './*_test.js', + timeout: 10000, + output: './output', + helpers: { + FakeDriver: { + require: '../fake_driver', + browser: 'dummy', + }, + }, + plugins: { + stepByStepReport: { + enabled: true, + deleteSuccessful: false, + }, + }, + include: {}, + bootstrap: false, + mocha: {}, + name: 'stepByStepTest', +} +` + + const configPath = path.join(codecept_dir, 'codecept.regular.js') + fs.writeFileSync(configPath, config) + + const command = `${runner} run --config ${configPath} --grep "@stepbystep"` + + exec(command, (err, stdout, stderr) => { + console.log('STDOUT:', stdout) + console.log('STDERR:', stderr) + + // For regular run, everything should work as before in the main output directory + if (fs.existsSync(outputDir)) { + const items = fs.readdirSync(outputDir) + console.log('Output directory contents:', items) + + // Should NOT create a consolidated stepByStepReport directory for regular runs + const stepByStepDir = path.join(outputDir, 'stepByStepReport') + expect(fs.existsSync(stepByStepDir)).toBe(false) + } + + // Clean up + fs.unlinkSync(configPath) + + done() + }) + }) +}) \ No newline at end of file diff --git a/test/unit/worker_test.js b/test/unit/worker_test.js index 811eeae87..e33f247e6 100644 --- a/test/unit/worker_test.js +++ b/test/unit/worker_test.js @@ -264,4 +264,78 @@ describe('Workers', function () { done() }) }) + + it('should handle stepByStep reporter directory resolution with workers', () => { + const path = require('path') + + // Mock the stepByStep directory resolution logic + function getStepByStepReportDir(config, isRunningWithWorkers, isRunMultipleChild, globalCodeceptDir) { + const needsConsolidation = isRunningWithWorkers || isRunMultipleChild + let reportDir + + if (needsConsolidation && globalCodeceptDir) { + const currentOutputDir = config.output ? path.resolve(globalCodeceptDir, config.output) : '/default-output' + + let baseOutputDir = currentOutputDir + + // For mixed scenario (run-multiple + workers), we need to strip both worker and run directory segments + // For run-workers only, strip worker directory segment + // For run-multiple only, strip run directory segment + if (isRunningWithWorkers) { + // Strip worker directory: /output/smoke_chrome_hash_1/worker1 -> /output/smoke_chrome_hash_1 or /output/worker1 -> /output + const workerDirPattern = /[/\\][^/\\]+$/ // Match the last directory segment (worker name) + baseOutputDir = baseOutputDir.replace(workerDirPattern, '') + } + + if (isRunMultipleChild) { + // Strip run directory: /output/smoke_chrome_hash_1 -> /output + const runDirPattern = /[/\\][^/\\]+$/ // Match the last directory segment (run name) + baseOutputDir = baseOutputDir.replace(runDirPattern, '') + } + + reportDir = path.join(baseOutputDir, 'stepByStepReport') + } else { + reportDir = config.output ? path.resolve(globalCodeceptDir, config.output) : '/default-output' + } + + return reportDir + } + + const globalCodeceptDir = '/tmp/test' + + // Test regular (non-worker) mode with default directory + const regularConfig = { output: './output' } + const regularDir = getStepByStepReportDir(regularConfig, false, false, globalCodeceptDir) + expect(regularDir).equal('/tmp/test/output') + + // Test regular (non-worker) mode with custom directory + const customConfig = { output: './custom-output' } + const customDir = getStepByStepReportDir(customConfig, false, false, globalCodeceptDir) + expect(customDir).equal('/tmp/test/custom-output') + + // Test run-workers mode with default directory + const workerConfig = { output: './output/worker1' } + const workerDir = getStepByStepReportDir(workerConfig, true, false, globalCodeceptDir) + expect(workerDir).equal('/tmp/test/output/stepByStepReport') + + // Test run-workers mode with custom directory + const workerCustomConfig = { output: './custom-output/worker2' } + const workerCustomDir = getStepByStepReportDir(workerCustomConfig, true, false, globalCodeceptDir) + expect(workerCustomDir).equal('/tmp/test/custom-output/stepByStepReport') + + // Test run-multiple mode with default directory + const multipleConfig = { output: './output/smoke_chrome_hash_1' } + const multipleDir = getStepByStepReportDir(multipleConfig, false, true, globalCodeceptDir) + expect(multipleDir).equal('/tmp/test/output/stepByStepReport') + + // Test run-multiple mode with custom directory + const multipleCustomConfig = { output: './custom-output/regression_firefox_hash_2' } + const multipleCustomDir = getStepByStepReportDir(multipleCustomConfig, false, true, globalCodeceptDir) + expect(multipleCustomDir).equal('/tmp/test/custom-output/stepByStepReport') + + // Test mixed run-multiple + workers mode + const mixedConfig = { output: './output/smoke_chrome_hash_1/worker1' } + const mixedDir = getStepByStepReportDir(mixedConfig, true, true, globalCodeceptDir) + expect(mixedDir).equal('/tmp/test/output/stepByStepReport') + }) })